-
diff --git a/doc/pub/Regression/html/._Regression-bs001.html b/doc/pub/Regression/html/._Regression-bs001.html index d905d84aa..9708d32ef 100644 --- a/doc/pub/Regression/html/._Regression-bs001.html +++ b/doc/pub/Regression/html/._Regression-bs001.html @@ -111,112 +111,112 @@ Automatically generated HTML file from DocOnce source ('The Boston housing data example', 2, None, '___sec27'), ('Housing data, the code', 2, None, '___sec28'), ('The singular value decomposition', 2, None, '___sec29'), - ('The Ising model', 2, None, '___sec30'), - ('Reformulating the problem to suit regression', - 2, - None, - '___sec31'), - ('Linear regression', 2, None, '___sec32'), - ('Singular Value decomposition', 2, None, '___sec33'), - ('Linear Regression Problems', 2, None, '___sec34'), - ('Fixing the singularity', 2, None, '___sec35'), - ('Basic math of the SVD', 2, None, '___sec36'), - ('The SVD, a Fantastic Algorithm', 2, None, '___sec37'), - ('Another Example', 2, None, '___sec38'), - ('Economy-size SVD', 2, None, '___sec39'), - ('Mathematical Properties', 2, None, '___sec40'), - ('Ridge and LASSO Regression', 2, None, '___sec41'), - ('More on Ridge Regression', 2, None, '___sec42'), - ('Interpreting the Ridge results', 2, None, '___sec43'), - ('More interpretations', 2, None, '___sec44'), - ('Where are we going?', 2, None, '___sec45'), - ('Resampling methods', 2, None, '___sec46'), + ('Linear Regression Problems', 2, None, '___sec30'), + ('Fixing the singularity', 2, None, '___sec31'), + ('Basic math of the SVD', 2, None, '___sec32'), + ('The SVD, a Fantastic Algorithm', 2, None, '___sec33'), + ('Another Example', 2, None, '___sec34'), + ('Economy-size SVD', 2, None, '___sec35'), + ('Mathematical Properties', 2, None, '___sec36'), + ('Ridge and LASSO Regression', 2, None, '___sec37'), + ('More on Ridge Regression', 2, None, '___sec38'), + ('Interpreting the Ridge results', 2, None, '___sec39'), + ('More interpretations', 2, None, '___sec40'), + ('Where are we going?', 2, None, '___sec41'), + ('Resampling methods', 2, None, '___sec42'), ('Resampling approaches can be computationally expensive', 2, None, - '___sec47'), - ('Why resampling methods ?', 2, None, '___sec48'), - ('Statistical analysis', 2, None, '___sec49'), - ('Statistics', 2, None, '___sec50'), - ('Statistics, moments', 2, None, '___sec51'), - ('Statistics, central moments', 2, None, '___sec52'), - ('Statistics, covariance', 2, None, '___sec53'), - ('Statistics, more covariance', 2, None, '___sec54'), - ('Covariance example', 2, None, '___sec55'), - ('Covariance in numpy', 2, None, '___sec56'), - ('Statistics, independent variables', 2, None, '___sec57'), - ('Statistics, more variance', 2, None, '___sec58'), - ('Statistics and stochastic processes', 2, None, '___sec59'), - ('Statistics and sample variables', 2, None, '___sec60'), + '___sec43'), + ('Why resampling methods ?', 2, None, '___sec44'), + ('Statistical analysis', 2, None, '___sec45'), + ('Statistics', 2, None, '___sec46'), + ('Statistics, moments', 2, None, '___sec47'), + ('Statistics, central moments', 2, None, '___sec48'), + ('Statistics, covariance', 2, None, '___sec49'), + ('Statistics, more covariance', 2, None, '___sec50'), + ('Covariance example', 2, None, '___sec51'), + ('Covariance in numpy', 2, None, '___sec52'), + ('Statistics, independent variables', 2, None, '___sec53'), + ('Statistics, more variance', 2, None, '___sec54'), + ('Statistics and stochastic processes', 2, None, '___sec55'), + ('Statistics and sample variables', 2, None, '___sec56'), ('Statistics, sample variance and covariance', 2, None, - '___sec61'), - ('Statistics, law of large numbers', 2, None, '___sec62'), - ('Statistics, more on sample error', 2, None, '___sec63'), - ('Statistics', 2, None, '___sec64'), - ('Statistics, central limit theorem', 2, None, '___sec65'), - ('Statistics, more technicalities', 2, None, '___sec66'), - ('Statistics', 2, None, '___sec67'), - ('Statistics and sample variance', 2, None, '___sec68'), - ('Statistics, uncorrelated results', 2, None, '___sec69'), - ('Statistics, computations', 2, None, '___sec70'), + '___sec57'), + ('Statistics, law of large numbers', 2, None, '___sec58'), + ('Statistics, more on sample error', 2, None, '___sec59'), + ('Statistics', 2, None, '___sec60'), + ('Statistics, central limit theorem', 2, None, '___sec61'), + ('Statistics, more technicalities', 2, None, '___sec62'), + ('Statistics', 2, None, '___sec63'), + ('Statistics and sample variance', 2, None, '___sec64'), + ('Statistics, uncorrelated results', 2, None, '___sec65'), + ('Statistics, computations', 2, None, '___sec66'), ('Statistics, more on computations of errors', 2, None, - '___sec71'), - ('Statistics, wrapping up 1', 2, None, '___sec72'), - ('Statistics, final expression', 2, None, '___sec73'), + '___sec67'), + ('Statistics, wrapping up 1', 2, None, '___sec68'), + ('Statistics, final expression', 2, None, '___sec69'), ('Statistics, effective number of correlations', 2, None, - '___sec74'), + '___sec70'), ('Linking the regression analysis with a statistical ' 'interpretation', 2, None, - '___sec75'), - ('Assumptions made', 2, None, '___sec76'), - ('Expectation value and variance', 2, None, '___sec77'), + '___sec71'), + ('Assumptions made', 2, None, '___sec72'), + ('Expectation value and variance', 2, None, '___sec73'), ('Expectation value and variance for $\\boldsymbol{\\beta}$', 2, None, - '___sec78'), - ('Cross-validation', 2, None, '___sec79'), - ('Computationally expensive', 2, None, '___sec80'), - ('Various steps in cross-validation', 2, None, '___sec81'), + '___sec74'), + ('Cross-validation', 2, None, '___sec75'), + ('Computationally expensive', 2, None, '___sec76'), + ('Various steps in cross-validation', 2, None, '___sec77'), ('How to set up the cross-validation for Ridge and/or Lasso', 2, None, - '___sec82'), + '___sec78'), ('Resampling methods: Jackknife and Bootstrap', 2, None, - '___sec83'), - ('Resampling methods: Jackknife', 2, None, '___sec84'), - ('Jackknife code example', 2, None, '___sec85'), - ('Resampling methods: Bootstrap', 2, None, '___sec86'), - ('Resampling methods: Bootstrap background', 2, None, '___sec87'), + '___sec79'), + ('Resampling methods: Jackknife', 2, None, '___sec80'), + ('Jackknife code example', 2, None, '___sec81'), + ('Resampling methods: Bootstrap', 2, None, '___sec82'), + ('Resampling methods: Bootstrap background', 2, None, '___sec83'), ('Resampling methods: More Bootstrap background', 2, None, - '___sec88'), - ('Resampling methods: Bootstrap approach', 2, None, '___sec89'), - ('Resampling methods: Bootstrap steps', 2, None, '___sec90'), - ('Code example for the Bootstrap method', 2, None, '___sec91'), + '___sec84'), + ('Resampling methods: Bootstrap approach', 2, None, '___sec85'), + ('Resampling methods: Bootstrap steps', 2, None, '___sec86'), + ('Code example for the Bootstrap method', 2, None, '___sec87'), ('Code Example for Cross-validation and $k$-fold ' 'Cross-validation', 2, None, - '___sec92'), - ('The bias-variance tradeoff', 2, None, '___sec93'), - ('Example code for Bias-Variance tradeoff', 2, None, '___sec94'), - ('Understanding what happens', 2, None, '___sec95'), - ('Summing up', 2, None, '___sec96'), + '___sec88'), + ('The bias-variance tradeoff', 2, None, '___sec89'), + ('Example code for Bias-Variance tradeoff', 2, None, '___sec90'), + ('Understanding what happens', 2, None, '___sec91'), + ('Summing up', 2, None, '___sec92'), ("Another Example rom Scikit-Learn's Repository", 2, None, - '___sec97'), + '___sec93'), + ('The Ising model', 2, None, '___sec94'), + ('Reformulating the problem to suit regression', + 2, + None, + '___sec95'), + ('Linear regression', 2, None, '___sec96'), + ('Singular Value decomposition', 2, None, '___sec97'), ('The one-dimensional Ising model', 2, None, '___sec98'), ('Ridge regression', 2, None, '___sec99'), ('LASSO regression', 2, None, '___sec100'), @@ -295,74 +295,74 @@ MathJax.Hub.Config({
This may @@ -417,9 +416,6 @@ inversion algorithm. Thereafter we dive into the math of the SVD. -
- -
diff --git a/doc/pub/Regression/html/._Regression-bs031.html b/doc/pub/Regression/html/._Regression-bs031.html index a2bf95958..0c1e209c9 100644 --- a/doc/pub/Regression/html/._Regression-bs031.html +++ b/doc/pub/Regression/html/._Regression-bs031.html @@ -111,112 +111,112 @@ Automatically generated HTML file from DocOnce source ('The Boston housing data example', 2, None, '___sec27'), ('Housing data, the code', 2, None, '___sec28'), ('The singular value decomposition', 2, None, '___sec29'), - ('The Ising model', 2, None, '___sec30'), - ('Reformulating the problem to suit regression', - 2, - None, - '___sec31'), - ('Linear regression', 2, None, '___sec32'), - ('Singular Value decomposition', 2, None, '___sec33'), - ('Linear Regression Problems', 2, None, '___sec34'), - ('Fixing the singularity', 2, None, '___sec35'), - ('Basic math of the SVD', 2, None, '___sec36'), - ('The SVD, a Fantastic Algorithm', 2, None, '___sec37'), - ('Another Example', 2, None, '___sec38'), - ('Economy-size SVD', 2, None, '___sec39'), - ('Mathematical Properties', 2, None, '___sec40'), - ('Ridge and LASSO Regression', 2, None, '___sec41'), - ('More on Ridge Regression', 2, None, '___sec42'), - ('Interpreting the Ridge results', 2, None, '___sec43'), - ('More interpretations', 2, None, '___sec44'), - ('Where are we going?', 2, None, '___sec45'), - ('Resampling methods', 2, None, '___sec46'), + ('Linear Regression Problems', 2, None, '___sec30'), + ('Fixing the singularity', 2, None, '___sec31'), + ('Basic math of the SVD', 2, None, '___sec32'), + ('The SVD, a Fantastic Algorithm', 2, None, '___sec33'), + ('Another Example', 2, None, '___sec34'), + ('Economy-size SVD', 2, None, '___sec35'), + ('Mathematical Properties', 2, None, '___sec36'), + ('Ridge and LASSO Regression', 2, None, '___sec37'), + ('More on Ridge Regression', 2, None, '___sec38'), + ('Interpreting the Ridge results', 2, None, '___sec39'), + ('More interpretations', 2, None, '___sec40'), + ('Where are we going?', 2, None, '___sec41'), + ('Resampling methods', 2, None, '___sec42'), ('Resampling approaches can be computationally expensive', 2, None, - '___sec47'), - ('Why resampling methods ?', 2, None, '___sec48'), - ('Statistical analysis', 2, None, '___sec49'), - ('Statistics', 2, None, '___sec50'), - ('Statistics, moments', 2, None, '___sec51'), - ('Statistics, central moments', 2, None, '___sec52'), - ('Statistics, covariance', 2, None, '___sec53'), - ('Statistics, more covariance', 2, None, '___sec54'), - ('Covariance example', 2, None, '___sec55'), - ('Covariance in numpy', 2, None, '___sec56'), - ('Statistics, independent variables', 2, None, '___sec57'), - ('Statistics, more variance', 2, None, '___sec58'), - ('Statistics and stochastic processes', 2, None, '___sec59'), - ('Statistics and sample variables', 2, None, '___sec60'), + '___sec43'), + ('Why resampling methods ?', 2, None, '___sec44'), + ('Statistical analysis', 2, None, '___sec45'), + ('Statistics', 2, None, '___sec46'), + ('Statistics, moments', 2, None, '___sec47'), + ('Statistics, central moments', 2, None, '___sec48'), + ('Statistics, covariance', 2, None, '___sec49'), + ('Statistics, more covariance', 2, None, '___sec50'), + ('Covariance example', 2, None, '___sec51'), + ('Covariance in numpy', 2, None, '___sec52'), + ('Statistics, independent variables', 2, None, '___sec53'), + ('Statistics, more variance', 2, None, '___sec54'), + ('Statistics and stochastic processes', 2, None, '___sec55'), + ('Statistics and sample variables', 2, None, '___sec56'), ('Statistics, sample variance and covariance', 2, None, - '___sec61'), - ('Statistics, law of large numbers', 2, None, '___sec62'), - ('Statistics, more on sample error', 2, None, '___sec63'), - ('Statistics', 2, None, '___sec64'), - ('Statistics, central limit theorem', 2, None, '___sec65'), - ('Statistics, more technicalities', 2, None, '___sec66'), - ('Statistics', 2, None, '___sec67'), - ('Statistics and sample variance', 2, None, '___sec68'), - ('Statistics, uncorrelated results', 2, None, '___sec69'), - ('Statistics, computations', 2, None, '___sec70'), + '___sec57'), + ('Statistics, law of large numbers', 2, None, '___sec58'), + ('Statistics, more on sample error', 2, None, '___sec59'), + ('Statistics', 2, None, '___sec60'), + ('Statistics, central limit theorem', 2, None, '___sec61'), + ('Statistics, more technicalities', 2, None, '___sec62'), + ('Statistics', 2, None, '___sec63'), + ('Statistics and sample variance', 2, None, '___sec64'), + ('Statistics, uncorrelated results', 2, None, '___sec65'), + ('Statistics, computations', 2, None, '___sec66'), ('Statistics, more on computations of errors', 2, None, - '___sec71'), - ('Statistics, wrapping up 1', 2, None, '___sec72'), - ('Statistics, final expression', 2, None, '___sec73'), + '___sec67'), + ('Statistics, wrapping up 1', 2, None, '___sec68'), + ('Statistics, final expression', 2, None, '___sec69'), ('Statistics, effective number of correlations', 2, None, - '___sec74'), + '___sec70'), ('Linking the regression analysis with a statistical ' 'interpretation', 2, None, - '___sec75'), - ('Assumptions made', 2, None, '___sec76'), - ('Expectation value and variance', 2, None, '___sec77'), + '___sec71'), + ('Assumptions made', 2, None, '___sec72'), + ('Expectation value and variance', 2, None, '___sec73'), ('Expectation value and variance for $\\boldsymbol{\\beta}$', 2, None, - '___sec78'), - ('Cross-validation', 2, None, '___sec79'), - ('Computationally expensive', 2, None, '___sec80'), - ('Various steps in cross-validation', 2, None, '___sec81'), + '___sec74'), + ('Cross-validation', 2, None, '___sec75'), + ('Computationally expensive', 2, None, '___sec76'), + ('Various steps in cross-validation', 2, None, '___sec77'), ('How to set up the cross-validation for Ridge and/or Lasso', 2, None, - '___sec82'), + '___sec78'), ('Resampling methods: Jackknife and Bootstrap', 2, None, - '___sec83'), - ('Resampling methods: Jackknife', 2, None, '___sec84'), - ('Jackknife code example', 2, None, '___sec85'), - ('Resampling methods: Bootstrap', 2, None, '___sec86'), - ('Resampling methods: Bootstrap background', 2, None, '___sec87'), + '___sec79'), + ('Resampling methods: Jackknife', 2, None, '___sec80'), + ('Jackknife code example', 2, None, '___sec81'), + ('Resampling methods: Bootstrap', 2, None, '___sec82'), + ('Resampling methods: Bootstrap background', 2, None, '___sec83'), ('Resampling methods: More Bootstrap background', 2, None, - '___sec88'), - ('Resampling methods: Bootstrap approach', 2, None, '___sec89'), - ('Resampling methods: Bootstrap steps', 2, None, '___sec90'), - ('Code example for the Bootstrap method', 2, None, '___sec91'), + '___sec84'), + ('Resampling methods: Bootstrap approach', 2, None, '___sec85'), + ('Resampling methods: Bootstrap steps', 2, None, '___sec86'), + ('Code example for the Bootstrap method', 2, None, '___sec87'), ('Code Example for Cross-validation and $k$-fold ' 'Cross-validation', 2, None, - '___sec92'), - ('The bias-variance tradeoff', 2, None, '___sec93'), - ('Example code for Bias-Variance tradeoff', 2, None, '___sec94'), - ('Understanding what happens', 2, None, '___sec95'), - ('Summing up', 2, None, '___sec96'), + '___sec88'), + ('The bias-variance tradeoff', 2, None, '___sec89'), + ('Example code for Bias-Variance tradeoff', 2, None, '___sec90'), + ('Understanding what happens', 2, None, '___sec91'), + ('Summing up', 2, None, '___sec92'), ("Another Example rom Scikit-Learn's Repository", 2, None, - '___sec97'), + '___sec93'), + ('The Ising model', 2, None, '___sec94'), + ('Reformulating the problem to suit regression', + 2, + None, + '___sec95'), + ('Linear regression', 2, None, '___sec96'), + ('Singular Value decomposition', 2, None, '___sec97'), ('The one-dimensional Ising model', 2, None, '___sec98'), ('Ridge regression', 2, None, '___sec99'), ('LASSO regression', 2, None, '___sec100'), @@ -295,74 +295,74 @@ MathJax.Hub.Config({
-The one-dimensional Ising model with nearest neighbor interaction, no -external field and a constant coupling constant \( J \) is given by - +One of the typical problems we encounter with linear regression, in particular +when the matrix \( \boldsymbol{X} \) (our so-called design matrix) is high-dimensional, +are problems with near singular or singular matrices. The column vectors of \( \boldsymbol{X} \) +may be linearly dependent, normally referred to as super-collinearity. +This means that the matrix may be rank deficient and it is basically impossible to +to model the data using linear regression. As an example, consider the matrix $$ -\begin{align} - H = -J \sum_{k}^L s_k s_{k + 1}, -\tag{1} -\end{align} +\begin{align*} +\mathbf{X} & = \left[ +\begin{array}{rrr} +1 & -1 & 2 +\\ +1 & 0 & 1 +\\ +1 & 2 & -1 +\\ +1 & 1 & 0 +\end{array} \right] +\end{align*} $$
-where \( s_i \in \{-1, 1\} \) and \( s_{N + 1} = s_1 \). The number of spins -in the system is determined by \( L \). For the one-dimensional system -there is no phase transition. +The columns of \( \boldsymbol{X} \) are linearly dependent. We see this easily since the +the first column is the row-wise sum of the other two columns. The rank (more correct, +the column rank) of a matrix is the dimension of the space spanned by the +column vectors. Hence, the rank of \( \mathbf{X} \) is equal to the number +of linearly independent columns. In this particular case the matrix has rank 2.
-We will look at a system of \( L = 40 \) spins with a coupling constant of -\( J = 1 \). To get enough training data we will generate 10000 states -with their respective energies. +Super-collinearity of an \( (n \times p) \)-dimensional design matrix \( \mathbf{X} \) implies +that the inverse of the matrix \( \boldsymbol{X}^T\boldsymbol{X} \) (the matrix we need to invert to solve the linear regression equations) is non-invertible. If we have a square matrix that does not have an inverse, we say this matrix singular. The example here demonstrates this +$$ +\begin{align*} +\boldsymbol{X} & = \left[ +\begin{array}{rr} +1 & -1 +\\ +1 & -1 +\end{array} \right]. +\end{align*} +$$ -
- - -
import numpy as np
-import matplotlib.pyplot as plt
-from mpl_toolkits.axes_grid1 import make_axes_locatable
-import seaborn as sns
-import scipy.linalg as scl
-from sklearn.model_selection import train_test_split
-import tqdm
-sns.set(color_codes=True)
-cmap_args=dict(vmin=-1., vmax=1., cmap='seismic')
-
-L = 40
-n = int(1e4)
-
-spins = np.random.choice([-1, 1], size=(n, L))
-J = 1.0
-
-energies = np.zeros(n)
-
-for i in range(n):
- energies[i] = - J * np.dot(spins[i], np.roll(spins[i], 1))
--Here we use ordinary least squares -regression to predict the energy for the nearest neighbor -one-dimensional Ising model on a ring, i.e., the endpoints wrap -around. We will use linear regression to fit a value for -the coupling constant to achieve this. +We see easily that \( \mbox{det}(\boldsymbol{X}) = x_{11} x_{22} - x_{12} x_{21} = 1 \times (-1) - 1 \times (-1) = 0 \). Hence, \( \mathbf{X} \) is singular and its inverse is undefined. +This is equivalent to saying that the matrix \( \boldsymbol{X} \) has at least an eigenvalue which is zero.
diff --git a/doc/pub/Regression/html/._Regression-bs032.html b/doc/pub/Regression/html/._Regression-bs032.html index 83c49de0e..3450418ad 100644 --- a/doc/pub/Regression/html/._Regression-bs032.html +++ b/doc/pub/Regression/html/._Regression-bs032.html @@ -111,112 +111,112 @@ Automatically generated HTML file from DocOnce source ('The Boston housing data example', 2, None, '___sec27'), ('Housing data, the code', 2, None, '___sec28'), ('The singular value decomposition', 2, None, '___sec29'), - ('The Ising model', 2, None, '___sec30'), - ('Reformulating the problem to suit regression', - 2, - None, - '___sec31'), - ('Linear regression', 2, None, '___sec32'), - ('Singular Value decomposition', 2, None, '___sec33'), - ('Linear Regression Problems', 2, None, '___sec34'), - ('Fixing the singularity', 2, None, '___sec35'), - ('Basic math of the SVD', 2, None, '___sec36'), - ('The SVD, a Fantastic Algorithm', 2, None, '___sec37'), - ('Another Example', 2, None, '___sec38'), - ('Economy-size SVD', 2, None, '___sec39'), - ('Mathematical Properties', 2, None, '___sec40'), - ('Ridge and LASSO Regression', 2, None, '___sec41'), - ('More on Ridge Regression', 2, None, '___sec42'), - ('Interpreting the Ridge results', 2, None, '___sec43'), - ('More interpretations', 2, None, '___sec44'), - ('Where are we going?', 2, None, '___sec45'), - ('Resampling methods', 2, None, '___sec46'), + ('Linear Regression Problems', 2, None, '___sec30'), + ('Fixing the singularity', 2, None, '___sec31'), + ('Basic math of the SVD', 2, None, '___sec32'), + ('The SVD, a Fantastic Algorithm', 2, None, '___sec33'), + ('Another Example', 2, None, '___sec34'), + ('Economy-size SVD', 2, None, '___sec35'), + ('Mathematical Properties', 2, None, '___sec36'), + ('Ridge and LASSO Regression', 2, None, '___sec37'), + ('More on Ridge Regression', 2, None, '___sec38'), + ('Interpreting the Ridge results', 2, None, '___sec39'), + ('More interpretations', 2, None, '___sec40'), + ('Where are we going?', 2, None, '___sec41'), + ('Resampling methods', 2, None, '___sec42'), ('Resampling approaches can be computationally expensive', 2, None, - '___sec47'), - ('Why resampling methods ?', 2, None, '___sec48'), - ('Statistical analysis', 2, None, '___sec49'), - ('Statistics', 2, None, '___sec50'), - ('Statistics, moments', 2, None, '___sec51'), - ('Statistics, central moments', 2, None, '___sec52'), - ('Statistics, covariance', 2, None, '___sec53'), - ('Statistics, more covariance', 2, None, '___sec54'), - ('Covariance example', 2, None, '___sec55'), - ('Covariance in numpy', 2, None, '___sec56'), - ('Statistics, independent variables', 2, None, '___sec57'), - ('Statistics, more variance', 2, None, '___sec58'), - ('Statistics and stochastic processes', 2, None, '___sec59'), - ('Statistics and sample variables', 2, None, '___sec60'), + '___sec43'), + ('Why resampling methods ?', 2, None, '___sec44'), + ('Statistical analysis', 2, None, '___sec45'), + ('Statistics', 2, None, '___sec46'), + ('Statistics, moments', 2, None, '___sec47'), + ('Statistics, central moments', 2, None, '___sec48'), + ('Statistics, covariance', 2, None, '___sec49'), + ('Statistics, more covariance', 2, None, '___sec50'), + ('Covariance example', 2, None, '___sec51'), + ('Covariance in numpy', 2, None, '___sec52'), + ('Statistics, independent variables', 2, None, '___sec53'), + ('Statistics, more variance', 2, None, '___sec54'), + ('Statistics and stochastic processes', 2, None, '___sec55'), + ('Statistics and sample variables', 2, None, '___sec56'), ('Statistics, sample variance and covariance', 2, None, - '___sec61'), - ('Statistics, law of large numbers', 2, None, '___sec62'), - ('Statistics, more on sample error', 2, None, '___sec63'), - ('Statistics', 2, None, '___sec64'), - ('Statistics, central limit theorem', 2, None, '___sec65'), - ('Statistics, more technicalities', 2, None, '___sec66'), - ('Statistics', 2, None, '___sec67'), - ('Statistics and sample variance', 2, None, '___sec68'), - ('Statistics, uncorrelated results', 2, None, '___sec69'), - ('Statistics, computations', 2, None, '___sec70'), + '___sec57'), + ('Statistics, law of large numbers', 2, None, '___sec58'), + ('Statistics, more on sample error', 2, None, '___sec59'), + ('Statistics', 2, None, '___sec60'), + ('Statistics, central limit theorem', 2, None, '___sec61'), + ('Statistics, more technicalities', 2, None, '___sec62'), + ('Statistics', 2, None, '___sec63'), + ('Statistics and sample variance', 2, None, '___sec64'), + ('Statistics, uncorrelated results', 2, None, '___sec65'), + ('Statistics, computations', 2, None, '___sec66'), ('Statistics, more on computations of errors', 2, None, - '___sec71'), - ('Statistics, wrapping up 1', 2, None, '___sec72'), - ('Statistics, final expression', 2, None, '___sec73'), + '___sec67'), + ('Statistics, wrapping up 1', 2, None, '___sec68'), + ('Statistics, final expression', 2, None, '___sec69'), ('Statistics, effective number of correlations', 2, None, - '___sec74'), + '___sec70'), ('Linking the regression analysis with a statistical ' 'interpretation', 2, None, - '___sec75'), - ('Assumptions made', 2, None, '___sec76'), - ('Expectation value and variance', 2, None, '___sec77'), + '___sec71'), + ('Assumptions made', 2, None, '___sec72'), + ('Expectation value and variance', 2, None, '___sec73'), ('Expectation value and variance for $\\boldsymbol{\\beta}$', 2, None, - '___sec78'), - ('Cross-validation', 2, None, '___sec79'), - ('Computationally expensive', 2, None, '___sec80'), - ('Various steps in cross-validation', 2, None, '___sec81'), + '___sec74'), + ('Cross-validation', 2, None, '___sec75'), + ('Computationally expensive', 2, None, '___sec76'), + ('Various steps in cross-validation', 2, None, '___sec77'), ('How to set up the cross-validation for Ridge and/or Lasso', 2, None, - '___sec82'), + '___sec78'), ('Resampling methods: Jackknife and Bootstrap', 2, None, - '___sec83'), - ('Resampling methods: Jackknife', 2, None, '___sec84'), - ('Jackknife code example', 2, None, '___sec85'), - ('Resampling methods: Bootstrap', 2, None, '___sec86'), - ('Resampling methods: Bootstrap background', 2, None, '___sec87'), + '___sec79'), + ('Resampling methods: Jackknife', 2, None, '___sec80'), + ('Jackknife code example', 2, None, '___sec81'), + ('Resampling methods: Bootstrap', 2, None, '___sec82'), + ('Resampling methods: Bootstrap background', 2, None, '___sec83'), ('Resampling methods: More Bootstrap background', 2, None, - '___sec88'), - ('Resampling methods: Bootstrap approach', 2, None, '___sec89'), - ('Resampling methods: Bootstrap steps', 2, None, '___sec90'), - ('Code example for the Bootstrap method', 2, None, '___sec91'), + '___sec84'), + ('Resampling methods: Bootstrap approach', 2, None, '___sec85'), + ('Resampling methods: Bootstrap steps', 2, None, '___sec86'), + ('Code example for the Bootstrap method', 2, None, '___sec87'), ('Code Example for Cross-validation and $k$-fold ' 'Cross-validation', 2, None, - '___sec92'), - ('The bias-variance tradeoff', 2, None, '___sec93'), - ('Example code for Bias-Variance tradeoff', 2, None, '___sec94'), - ('Understanding what happens', 2, None, '___sec95'), - ('Summing up', 2, None, '___sec96'), + '___sec88'), + ('The bias-variance tradeoff', 2, None, '___sec89'), + ('Example code for Bias-Variance tradeoff', 2, None, '___sec90'), + ('Understanding what happens', 2, None, '___sec91'), + ('Summing up', 2, None, '___sec92'), ("Another Example rom Scikit-Learn's Repository", 2, None, - '___sec97'), + '___sec93'), + ('The Ising model', 2, None, '___sec94'), + ('Reformulating the problem to suit regression', + 2, + None, + '___sec95'), + ('Linear regression', 2, None, '___sec96'), + ('Singular Value decomposition', 2, None, '___sec97'), ('The one-dimensional Ising model', 2, None, '___sec98'), ('Ridge regression', 2, None, '___sec99'), ('LASSO regression', 2, None, '___sec100'), @@ -295,74 +295,74 @@ MathJax.Hub.Config({
-A more general form for the one-dimensional Ising model is - +If our design matrix \( \boldsymbol{X} \) which enters the linear regression problem $$ \begin{align} - H = - \sum_j^L \sum_k^L s_j s_k J_{jk}. -\tag{2} +\boldsymbol{\beta} & = (\boldsymbol{X}^{T} \boldsymbol{X})^{-1} \boldsymbol{X}^{T} \boldsymbol{y}, +\tag{1} \end{align} $$ -
-Here we allow for interactions beyond the nearest neighbors and a state dependent -coupling constant. This latter expression can be formulated as -a matrix-product -$$ -\begin{align} - \boldsymbol{H} = \boldsymbol{X} J, -\tag{3} -\end{align} -$$ +has linearly dependent column vectors, we will not be able to compute the inverse +of \( \boldsymbol{X}^T\boldsymbol{X} \) and we cannot find the parameters (estimators) \( \beta_i \). +The estimators are only well-defined if \( (\boldsymbol{X}^{T}\boldsymbol{X})^{-1} \) exits. +This is more likely to happen when the matrix \( \boldsymbol{X} \) is high-dimensional. In this case it is likely to encounter a situation where +the regression parameters \( \beta_i \) cannot be estimated.
-where \( X_{jk} = s_j s_k \) and \( J \) is a matrix which consists of the -elements \( -J_{jk} \). This form of writing the energy fits perfectly -with the form utilized in linear regression, that is - +A cheap ad hoc approach is simply to add a small diagonal component to the matrix to invert, that is we change $$ -\begin{align} - \boldsymbol{y} = \boldsymbol{X}\boldsymbol{\beta} + \boldsymbol{\epsilon}, -\tag{4} -\end{align} +\boldsymbol{X}^{T} \boldsymbol{X} \rightarrow \boldsymbol{X}^{T} \boldsymbol{X}+\lambda \boldsymbol{I}, $$ -
-We split the data in training and test data as discussed in the previous example +where \( \boldsymbol{I} \) is the identity matrix. When we discuss Ridge regression this is actually what we end up evaluating. The parameter \( \lambda \) is called a hyperparameter. More about this later. -
- - -
X = np.zeros((n, L ** 2))
-for i in range(n):
- X[i] = np.outer(spins[i], spins[i]).ravel()
-y = energies
-X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)
-
diff --git a/doc/pub/Regression/html/._Regression-bs033.html b/doc/pub/Regression/html/._Regression-bs033.html index 64b27e207..f9723f764 100644 --- a/doc/pub/Regression/html/._Regression-bs033.html +++ b/doc/pub/Regression/html/._Regression-bs033.html @@ -111,112 +111,112 @@ Automatically generated HTML file from DocOnce source ('The Boston housing data example', 2, None, '___sec27'), ('Housing data, the code', 2, None, '___sec28'), ('The singular value decomposition', 2, None, '___sec29'), - ('The Ising model', 2, None, '___sec30'), - ('Reformulating the problem to suit regression', - 2, - None, - '___sec31'), - ('Linear regression', 2, None, '___sec32'), - ('Singular Value decomposition', 2, None, '___sec33'), - ('Linear Regression Problems', 2, None, '___sec34'), - ('Fixing the singularity', 2, None, '___sec35'), - ('Basic math of the SVD', 2, None, '___sec36'), - ('The SVD, a Fantastic Algorithm', 2, None, '___sec37'), - ('Another Example', 2, None, '___sec38'), - ('Economy-size SVD', 2, None, '___sec39'), - ('Mathematical Properties', 2, None, '___sec40'), - ('Ridge and LASSO Regression', 2, None, '___sec41'), - ('More on Ridge Regression', 2, None, '___sec42'), - ('Interpreting the Ridge results', 2, None, '___sec43'), - ('More interpretations', 2, None, '___sec44'), - ('Where are we going?', 2, None, '___sec45'), - ('Resampling methods', 2, None, '___sec46'), + ('Linear Regression Problems', 2, None, '___sec30'), + ('Fixing the singularity', 2, None, '___sec31'), + ('Basic math of the SVD', 2, None, '___sec32'), + ('The SVD, a Fantastic Algorithm', 2, None, '___sec33'), + ('Another Example', 2, None, '___sec34'), + ('Economy-size SVD', 2, None, '___sec35'), + ('Mathematical Properties', 2, None, '___sec36'), + ('Ridge and LASSO Regression', 2, None, '___sec37'), + ('More on Ridge Regression', 2, None, '___sec38'), + ('Interpreting the Ridge results', 2, None, '___sec39'), + ('More interpretations', 2, None, '___sec40'), + ('Where are we going?', 2, None, '___sec41'), + ('Resampling methods', 2, None, '___sec42'), ('Resampling approaches can be computationally expensive', 2, None, - '___sec47'), - ('Why resampling methods ?', 2, None, '___sec48'), - ('Statistical analysis', 2, None, '___sec49'), - ('Statistics', 2, None, '___sec50'), - ('Statistics, moments', 2, None, '___sec51'), - ('Statistics, central moments', 2, None, '___sec52'), - ('Statistics, covariance', 2, None, '___sec53'), - ('Statistics, more covariance', 2, None, '___sec54'), - ('Covariance example', 2, None, '___sec55'), - ('Covariance in numpy', 2, None, '___sec56'), - ('Statistics, independent variables', 2, None, '___sec57'), - ('Statistics, more variance', 2, None, '___sec58'), - ('Statistics and stochastic processes', 2, None, '___sec59'), - ('Statistics and sample variables', 2, None, '___sec60'), + '___sec43'), + ('Why resampling methods ?', 2, None, '___sec44'), + ('Statistical analysis', 2, None, '___sec45'), + ('Statistics', 2, None, '___sec46'), + ('Statistics, moments', 2, None, '___sec47'), + ('Statistics, central moments', 2, None, '___sec48'), + ('Statistics, covariance', 2, None, '___sec49'), + ('Statistics, more covariance', 2, None, '___sec50'), + ('Covariance example', 2, None, '___sec51'), + ('Covariance in numpy', 2, None, '___sec52'), + ('Statistics, independent variables', 2, None, '___sec53'), + ('Statistics, more variance', 2, None, '___sec54'), + ('Statistics and stochastic processes', 2, None, '___sec55'), + ('Statistics and sample variables', 2, None, '___sec56'), ('Statistics, sample variance and covariance', 2, None, - '___sec61'), - ('Statistics, law of large numbers', 2, None, '___sec62'), - ('Statistics, more on sample error', 2, None, '___sec63'), - ('Statistics', 2, None, '___sec64'), - ('Statistics, central limit theorem', 2, None, '___sec65'), - ('Statistics, more technicalities', 2, None, '___sec66'), - ('Statistics', 2, None, '___sec67'), - ('Statistics and sample variance', 2, None, '___sec68'), - ('Statistics, uncorrelated results', 2, None, '___sec69'), - ('Statistics, computations', 2, None, '___sec70'), + '___sec57'), + ('Statistics, law of large numbers', 2, None, '___sec58'), + ('Statistics, more on sample error', 2, None, '___sec59'), + ('Statistics', 2, None, '___sec60'), + ('Statistics, central limit theorem', 2, None, '___sec61'), + ('Statistics, more technicalities', 2, None, '___sec62'), + ('Statistics', 2, None, '___sec63'), + ('Statistics and sample variance', 2, None, '___sec64'), + ('Statistics, uncorrelated results', 2, None, '___sec65'), + ('Statistics, computations', 2, None, '___sec66'), ('Statistics, more on computations of errors', 2, None, - '___sec71'), - ('Statistics, wrapping up 1', 2, None, '___sec72'), - ('Statistics, final expression', 2, None, '___sec73'), + '___sec67'), + ('Statistics, wrapping up 1', 2, None, '___sec68'), + ('Statistics, final expression', 2, None, '___sec69'), ('Statistics, effective number of correlations', 2, None, - '___sec74'), + '___sec70'), ('Linking the regression analysis with a statistical ' 'interpretation', 2, None, - '___sec75'), - ('Assumptions made', 2, None, '___sec76'), - ('Expectation value and variance', 2, None, '___sec77'), + '___sec71'), + ('Assumptions made', 2, None, '___sec72'), + ('Expectation value and variance', 2, None, '___sec73'), ('Expectation value and variance for $\\boldsymbol{\\beta}$', 2, None, - '___sec78'), - ('Cross-validation', 2, None, '___sec79'), - ('Computationally expensive', 2, None, '___sec80'), - ('Various steps in cross-validation', 2, None, '___sec81'), + '___sec74'), + ('Cross-validation', 2, None, '___sec75'), + ('Computationally expensive', 2, None, '___sec76'), + ('Various steps in cross-validation', 2, None, '___sec77'), ('How to set up the cross-validation for Ridge and/or Lasso', 2, None, - '___sec82'), + '___sec78'), ('Resampling methods: Jackknife and Bootstrap', 2, None, - '___sec83'), - ('Resampling methods: Jackknife', 2, None, '___sec84'), - ('Jackknife code example', 2, None, '___sec85'), - ('Resampling methods: Bootstrap', 2, None, '___sec86'), - ('Resampling methods: Bootstrap background', 2, None, '___sec87'), + '___sec79'), + ('Resampling methods: Jackknife', 2, None, '___sec80'), + ('Jackknife code example', 2, None, '___sec81'), + ('Resampling methods: Bootstrap', 2, None, '___sec82'), + ('Resampling methods: Bootstrap background', 2, None, '___sec83'), ('Resampling methods: More Bootstrap background', 2, None, - '___sec88'), - ('Resampling methods: Bootstrap approach', 2, None, '___sec89'), - ('Resampling methods: Bootstrap steps', 2, None, '___sec90'), - ('Code example for the Bootstrap method', 2, None, '___sec91'), + '___sec84'), + ('Resampling methods: Bootstrap approach', 2, None, '___sec85'), + ('Resampling methods: Bootstrap steps', 2, None, '___sec86'), + ('Code example for the Bootstrap method', 2, None, '___sec87'), ('Code Example for Cross-validation and $k$-fold ' 'Cross-validation', 2, None, - '___sec92'), - ('The bias-variance tradeoff', 2, None, '___sec93'), - ('Example code for Bias-Variance tradeoff', 2, None, '___sec94'), - ('Understanding what happens', 2, None, '___sec95'), - ('Summing up', 2, None, '___sec96'), + '___sec88'), + ('The bias-variance tradeoff', 2, None, '___sec89'), + ('Example code for Bias-Variance tradeoff', 2, None, '___sec90'), + ('Understanding what happens', 2, None, '___sec91'), + ('Summing up', 2, None, '___sec92'), ("Another Example rom Scikit-Learn's Repository", 2, None, - '___sec97'), + '___sec93'), + ('The Ising model', 2, None, '___sec94'), + ('Reformulating the problem to suit regression', + 2, + None, + '___sec95'), + ('Linear regression', 2, None, '___sec96'), + ('Singular Value decomposition', 2, None, '___sec97'), ('The one-dimensional Ising model', 2, None, '___sec98'), ('Ridge regression', 2, None, '___sec99'), ('LASSO regression', 2, None, '___sec100'), @@ -295,74 +295,74 @@ MathJax.Hub.Config({
-In the ordinary least squares method we choose the cost function +From standard linear algebra we know that a square matrix \( \boldsymbol{X} \) can be diagonalized if and only it is +a so-called normal matrix, that is if \( \boldsymbol{X}\in {\mathbb{R}}^{n\times n} \) +we have \( \boldsymbol{X}\boldsymbol{X}^T=\boldsymbol{X}^T\boldsymbol{X} \) or if \( \boldsymbol{X}\in {\mathbb{C}}^{n\times n} \) we have \( \boldsymbol{X}\boldsymbol{X}^{\dagger}=\boldsymbol{X}^{\dagger}\boldsymbol{X} \). +The matrix has then a set of eigenpairs $$ -\begin{align} - C(\boldsymbol{X}, \boldsymbol{\beta})= \frac{1}{n}\left\{(\boldsymbol{X}\boldsymbol{\beta} - \boldsymbol{y})^T(\boldsymbol{X}\boldsymbol{\beta} - \boldsymbol{y})\right\}. -\tag{5} -\end{align} +(\lambda_1,\boldsymbol{u}_1),\dots, (\lambda_n,\boldsymbol{u}_n), $$ -
-We then find the extremal point of \( C \) by taking the derivative with respect to \( \boldsymbol{\beta} \) as discussed above. -This yields the expression for \( \boldsymbol{\beta} \) to be - +and the eigenvalues are given by the diagonal matrix $$ - \boldsymbol{\beta} = \frac{\boldsymbol{X}^T \boldsymbol{y}}{\boldsymbol{X}^T \boldsymbol{X}}, +\boldsymbol{\Sigma}=\mathrm{Diag}(\lambda_1, \dots,\lambda_n). $$ -
-which immediately imposes some requirements on \( \boldsymbol{X} \) as there must exist -an inverse of \( \boldsymbol{X}^T \boldsymbol{X} \). If the expression we are modeling contains an -intercept, i.e., a constant term, we must make sure that the -first column of \( \boldsymbol{X} \) consists of \( 1 \). We do this here +The matrix \( \boldsymbol{X} \) can be written in terms of an orthogonal/unitary transformation \( \boldsymbol{U} \) +$$ +\boldsymbol{X} = \boldsymbol{U}\boldsymbol{\Sigma}\boldsymbol{V}^T, +$$ + +with \( \boldsymbol{U}\boldsymbol{U}^T=\boldsymbol{I} \) or \( \boldsymbol{U}\boldsymbol{U}^{\dagger}=\boldsymbol{I} \).
+Not all square matrices are diagonalizable. A matrix like the one discussed above +$$ +\boldsymbol{X} = \begin{bmatrix} +1& -1 \\ +1& -1\\ +\end{bmatrix} +$$ - -
X_train_own = np.concatenate(
- (np.ones(len(X_train))[:, np.newaxis], X_train),
- axis=1
-)
-X_test_own = np.concatenate(
- (np.ones(len(X_test))[:, np.newaxis], X_test),
- axis=1
-)
-+is not diagonalizable, it is a so-called defective matrix. It is easy to see that the condition +\( \boldsymbol{X}\boldsymbol{X}^T=\boldsymbol{X}^T\boldsymbol{X} \) is not fulfilled. - -
def ols_inv(x: np.ndarray, y: np.ndarray) -> np.ndarray:
- return scl.inv(x.T @ x) @ (x.T @ y)
-beta = ols_inv(X_train_own, y_train)
-
diff --git a/doc/pub/Regression/html/._Regression-bs034.html b/doc/pub/Regression/html/._Regression-bs034.html index 45d40727a..6c36399ff 100644 --- a/doc/pub/Regression/html/._Regression-bs034.html +++ b/doc/pub/Regression/html/._Regression-bs034.html @@ -111,112 +111,112 @@ Automatically generated HTML file from DocOnce source ('The Boston housing data example', 2, None, '___sec27'), ('Housing data, the code', 2, None, '___sec28'), ('The singular value decomposition', 2, None, '___sec29'), - ('The Ising model', 2, None, '___sec30'), - ('Reformulating the problem to suit regression', - 2, - None, - '___sec31'), - ('Linear regression', 2, None, '___sec32'), - ('Singular Value decomposition', 2, None, '___sec33'), - ('Linear Regression Problems', 2, None, '___sec34'), - ('Fixing the singularity', 2, None, '___sec35'), - ('Basic math of the SVD', 2, None, '___sec36'), - ('The SVD, a Fantastic Algorithm', 2, None, '___sec37'), - ('Another Example', 2, None, '___sec38'), - ('Economy-size SVD', 2, None, '___sec39'), - ('Mathematical Properties', 2, None, '___sec40'), - ('Ridge and LASSO Regression', 2, None, '___sec41'), - ('More on Ridge Regression', 2, None, '___sec42'), - ('Interpreting the Ridge results', 2, None, '___sec43'), - ('More interpretations', 2, None, '___sec44'), - ('Where are we going?', 2, None, '___sec45'), - ('Resampling methods', 2, None, '___sec46'), + ('Linear Regression Problems', 2, None, '___sec30'), + ('Fixing the singularity', 2, None, '___sec31'), + ('Basic math of the SVD', 2, None, '___sec32'), + ('The SVD, a Fantastic Algorithm', 2, None, '___sec33'), + ('Another Example', 2, None, '___sec34'), + ('Economy-size SVD', 2, None, '___sec35'), + ('Mathematical Properties', 2, None, '___sec36'), + ('Ridge and LASSO Regression', 2, None, '___sec37'), + ('More on Ridge Regression', 2, None, '___sec38'), + ('Interpreting the Ridge results', 2, None, '___sec39'), + ('More interpretations', 2, None, '___sec40'), + ('Where are we going?', 2, None, '___sec41'), + ('Resampling methods', 2, None, '___sec42'), ('Resampling approaches can be computationally expensive', 2, None, - '___sec47'), - ('Why resampling methods ?', 2, None, '___sec48'), - ('Statistical analysis', 2, None, '___sec49'), - ('Statistics', 2, None, '___sec50'), - ('Statistics, moments', 2, None, '___sec51'), - ('Statistics, central moments', 2, None, '___sec52'), - ('Statistics, covariance', 2, None, '___sec53'), - ('Statistics, more covariance', 2, None, '___sec54'), - ('Covariance example', 2, None, '___sec55'), - ('Covariance in numpy', 2, None, '___sec56'), - ('Statistics, independent variables', 2, None, '___sec57'), - ('Statistics, more variance', 2, None, '___sec58'), - ('Statistics and stochastic processes', 2, None, '___sec59'), - ('Statistics and sample variables', 2, None, '___sec60'), + '___sec43'), + ('Why resampling methods ?', 2, None, '___sec44'), + ('Statistical analysis', 2, None, '___sec45'), + ('Statistics', 2, None, '___sec46'), + ('Statistics, moments', 2, None, '___sec47'), + ('Statistics, central moments', 2, None, '___sec48'), + ('Statistics, covariance', 2, None, '___sec49'), + ('Statistics, more covariance', 2, None, '___sec50'), + ('Covariance example', 2, None, '___sec51'), + ('Covariance in numpy', 2, None, '___sec52'), + ('Statistics, independent variables', 2, None, '___sec53'), + ('Statistics, more variance', 2, None, '___sec54'), + ('Statistics and stochastic processes', 2, None, '___sec55'), + ('Statistics and sample variables', 2, None, '___sec56'), ('Statistics, sample variance and covariance', 2, None, - '___sec61'), - ('Statistics, law of large numbers', 2, None, '___sec62'), - ('Statistics, more on sample error', 2, None, '___sec63'), - ('Statistics', 2, None, '___sec64'), - ('Statistics, central limit theorem', 2, None, '___sec65'), - ('Statistics, more technicalities', 2, None, '___sec66'), - ('Statistics', 2, None, '___sec67'), - ('Statistics and sample variance', 2, None, '___sec68'), - ('Statistics, uncorrelated results', 2, None, '___sec69'), - ('Statistics, computations', 2, None, '___sec70'), + '___sec57'), + ('Statistics, law of large numbers', 2, None, '___sec58'), + ('Statistics, more on sample error', 2, None, '___sec59'), + ('Statistics', 2, None, '___sec60'), + ('Statistics, central limit theorem', 2, None, '___sec61'), + ('Statistics, more technicalities', 2, None, '___sec62'), + ('Statistics', 2, None, '___sec63'), + ('Statistics and sample variance', 2, None, '___sec64'), + ('Statistics, uncorrelated results', 2, None, '___sec65'), + ('Statistics, computations', 2, None, '___sec66'), ('Statistics, more on computations of errors', 2, None, - '___sec71'), - ('Statistics, wrapping up 1', 2, None, '___sec72'), - ('Statistics, final expression', 2, None, '___sec73'), + '___sec67'), + ('Statistics, wrapping up 1', 2, None, '___sec68'), + ('Statistics, final expression', 2, None, '___sec69'), ('Statistics, effective number of correlations', 2, None, - '___sec74'), + '___sec70'), ('Linking the regression analysis with a statistical ' 'interpretation', 2, None, - '___sec75'), - ('Assumptions made', 2, None, '___sec76'), - ('Expectation value and variance', 2, None, '___sec77'), + '___sec71'), + ('Assumptions made', 2, None, '___sec72'), + ('Expectation value and variance', 2, None, '___sec73'), ('Expectation value and variance for $\\boldsymbol{\\beta}$', 2, None, - '___sec78'), - ('Cross-validation', 2, None, '___sec79'), - ('Computationally expensive', 2, None, '___sec80'), - ('Various steps in cross-validation', 2, None, '___sec81'), + '___sec74'), + ('Cross-validation', 2, None, '___sec75'), + ('Computationally expensive', 2, None, '___sec76'), + ('Various steps in cross-validation', 2, None, '___sec77'), ('How to set up the cross-validation for Ridge and/or Lasso', 2, None, - '___sec82'), + '___sec78'), ('Resampling methods: Jackknife and Bootstrap', 2, None, - '___sec83'), - ('Resampling methods: Jackknife', 2, None, '___sec84'), - ('Jackknife code example', 2, None, '___sec85'), - ('Resampling methods: Bootstrap', 2, None, '___sec86'), - ('Resampling methods: Bootstrap background', 2, None, '___sec87'), + '___sec79'), + ('Resampling methods: Jackknife', 2, None, '___sec80'), + ('Jackknife code example', 2, None, '___sec81'), + ('Resampling methods: Bootstrap', 2, None, '___sec82'), + ('Resampling methods: Bootstrap background', 2, None, '___sec83'), ('Resampling methods: More Bootstrap background', 2, None, - '___sec88'), - ('Resampling methods: Bootstrap approach', 2, None, '___sec89'), - ('Resampling methods: Bootstrap steps', 2, None, '___sec90'), - ('Code example for the Bootstrap method', 2, None, '___sec91'), + '___sec84'), + ('Resampling methods: Bootstrap approach', 2, None, '___sec85'), + ('Resampling methods: Bootstrap steps', 2, None, '___sec86'), + ('Code example for the Bootstrap method', 2, None, '___sec87'), ('Code Example for Cross-validation and $k$-fold ' 'Cross-validation', 2, None, - '___sec92'), - ('The bias-variance tradeoff', 2, None, '___sec93'), - ('Example code for Bias-Variance tradeoff', 2, None, '___sec94'), - ('Understanding what happens', 2, None, '___sec95'), - ('Summing up', 2, None, '___sec96'), + '___sec88'), + ('The bias-variance tradeoff', 2, None, '___sec89'), + ('Example code for Bias-Variance tradeoff', 2, None, '___sec90'), + ('Understanding what happens', 2, None, '___sec91'), + ('Summing up', 2, None, '___sec92'), ("Another Example rom Scikit-Learn's Repository", 2, None, - '___sec97'), + '___sec93'), + ('The Ising model', 2, None, '___sec94'), + ('Reformulating the problem to suit regression', + 2, + None, + '___sec95'), + ('Linear regression', 2, None, '___sec96'), + ('Singular Value decomposition', 2, None, '___sec97'), ('The one-dimensional Ising model', 2, None, '___sec98'), ('Ridge regression', 2, None, '___sec99'), ('LASSO regression', 2, None, '___sec100'), @@ -295,74 +295,74 @@ MathJax.Hub.Config({
-Doing the inversion directly turns out to be a bad idea since the matrix -\( \boldsymbol{X}^T\boldsymbol{X} \) is singular. An alternative approach is to use the singular -value decomposition. Using the definition of the Moore-Penrose -pseudoinverse we can write the equation for \( \boldsymbol{\beta} \) as +However, and this is the strength of the SVD algorithm, any general +matrix \( \boldsymbol{X} \) can be decomposed in terms of a diagonal matrix and +two orthogonal/unitary matrices. The Singular Value Decompostion +(SVD) theorem +states that a general \( m\times n \) matrix \( \boldsymbol{X} \) can be written in +terms of a diagonal matrix \( \boldsymbol{\Sigma} \) of dimensionality \( n\times n \) +and two orthognal matrices \( \boldsymbol{U} \) and \( \boldsymbol{V} \), where the first has +dimensionality \( m \times m \) and the last dimensionality \( n\times n \). +We have then -$$ - \boldsymbol{\beta} = \boldsymbol{X}^{+}\boldsymbol{y}, +$$ +\boldsymbol{X} = \boldsymbol{U}\boldsymbol{\Sigma}\boldsymbol{V}^T $$
-where the pseudoinverse of \( \boldsymbol{X} \) is given by +As an example, the above defective matrix can be decomposed as $$ - \boldsymbol{X}^{+} = \frac{\boldsymbol{X}^T}{\boldsymbol{X}^T\boldsymbol{X}}. +\boldsymbol{X} = \frac{1}{\sqrt{2}}\begin{bmatrix} 1& 1 \\ 1& -1\\ \end{bmatrix} \begin{bmatrix} 2& 0 \\ 0& 0\\ \end{bmatrix} \frac{1}{\sqrt{2}}\begin{bmatrix} 1& -1 \\ 1& 1\\ \end{bmatrix}=\boldsymbol{U}\boldsymbol{\Sigma}\boldsymbol{V}^T, $$
-Using singular value decomposition we can decompose the matrix \( \boldsymbol{X} = \boldsymbol{U}\boldsymbol{\Sigma} \boldsymbol{V}^T \), -where \( \boldsymbol{U} \) and \( \boldsymbol{V} \) are orthogonal(unitary) matrices and \( \boldsymbol{\Sigma} \) contains the singular values (more details below). -where \( X^{+} = V\Sigma^{+} U^T \). This reduces the equation for -\( \omega \) to -$$ -\begin{align} - \boldsymbol{\beta} = \boldsymbol{V}\boldsymbol{\Sigma}^{+} \boldsymbol{U}^T \boldsymbol{y}. -\tag{6} -\end{align} -$$ - -
-Note that solving this equation by actually doing the pseudoinverse -(which is what we will do) is not a good idea as this operation scales -as \( \mathcal{O}(n^3) \), where \( n \) is the number of elements in a -general matrix. Instead, doing \( QR \)-factorization and solving the -linear system as an equation would reduce this down to -\( \mathcal{O}(n^2) \) operations. - -
- - -
def ols_svd(x: np.ndarray, y: np.ndarray) -> np.ndarray:
- u, s, v = scl.svd(x)
- return v.T @ scl.pinv(scl.diagsvd(s, u.shape[0], v.shape[0])) @ u.T @ y
-- - -
beta = ols_svd(X_train_own,y_train)
--When extracting the \( J \)-matrix we need to make sure that we remove the intercept, as is done here - -
- - -
J = beta[1:].reshape(L, L)
--A way of looking at the coefficients in \( J \) is to plot the matrices as images. - -
- - -
fig = plt.figure(figsize=(20, 14))
-im = plt.imshow(J, **cmap_args)
-plt.title("OLS", fontsize=18)
-plt.xticks(fontsize=18)
-plt.yticks(fontsize=18)
-cb = fig.colorbar(im)
-cb.ax.set_yticklabels(cb.ax.get_yticklabels(), fontsize=18)
-plt.show()
--It is interesting to note that OLS -considers both \( J_{j, j + 1} = -0.5 \) and \( J_{j, j - 1} = -0.5 \) as -valid matrix elements for \( J \). -In our discussion below on hyperparameters and Ridge and Lasso regression we will see that -this problem can be removed, partly and only with Lasso regression. - -
-In this case our matrix inversion was actually possible. The obvious question now is what is the mathematics behind the SVD? +with eigenvalues \( \sigma_1=2 \) and \( \sigma_2=0 \). +The SVD exits always!
diff --git a/doc/pub/Regression/html/._Regression-bs035.html b/doc/pub/Regression/html/._Regression-bs035.html index 23a45041a..78c5abab6 100644 --- a/doc/pub/Regression/html/._Regression-bs035.html +++ b/doc/pub/Regression/html/._Regression-bs035.html @@ -111,112 +111,112 @@ Automatically generated HTML file from DocOnce source ('The Boston housing data example', 2, None, '___sec27'), ('Housing data, the code', 2, None, '___sec28'), ('The singular value decomposition', 2, None, '___sec29'), - ('The Ising model', 2, None, '___sec30'), - ('Reformulating the problem to suit regression', - 2, - None, - '___sec31'), - ('Linear regression', 2, None, '___sec32'), - ('Singular Value decomposition', 2, None, '___sec33'), - ('Linear Regression Problems', 2, None, '___sec34'), - ('Fixing the singularity', 2, None, '___sec35'), - ('Basic math of the SVD', 2, None, '___sec36'), - ('The SVD, a Fantastic Algorithm', 2, None, '___sec37'), - ('Another Example', 2, None, '___sec38'), - ('Economy-size SVD', 2, None, '___sec39'), - ('Mathematical Properties', 2, None, '___sec40'), - ('Ridge and LASSO Regression', 2, None, '___sec41'), - ('More on Ridge Regression', 2, None, '___sec42'), - ('Interpreting the Ridge results', 2, None, '___sec43'), - ('More interpretations', 2, None, '___sec44'), - ('Where are we going?', 2, None, '___sec45'), - ('Resampling methods', 2, None, '___sec46'), + ('Linear Regression Problems', 2, None, '___sec30'), + ('Fixing the singularity', 2, None, '___sec31'), + ('Basic math of the SVD', 2, None, '___sec32'), + ('The SVD, a Fantastic Algorithm', 2, None, '___sec33'), + ('Another Example', 2, None, '___sec34'), + ('Economy-size SVD', 2, None, '___sec35'), + ('Mathematical Properties', 2, None, '___sec36'), + ('Ridge and LASSO Regression', 2, None, '___sec37'), + ('More on Ridge Regression', 2, None, '___sec38'), + ('Interpreting the Ridge results', 2, None, '___sec39'), + ('More interpretations', 2, None, '___sec40'), + ('Where are we going?', 2, None, '___sec41'), + ('Resampling methods', 2, None, '___sec42'), ('Resampling approaches can be computationally expensive', 2, None, - '___sec47'), - ('Why resampling methods ?', 2, None, '___sec48'), - ('Statistical analysis', 2, None, '___sec49'), - ('Statistics', 2, None, '___sec50'), - ('Statistics, moments', 2, None, '___sec51'), - ('Statistics, central moments', 2, None, '___sec52'), - ('Statistics, covariance', 2, None, '___sec53'), - ('Statistics, more covariance', 2, None, '___sec54'), - ('Covariance example', 2, None, '___sec55'), - ('Covariance in numpy', 2, None, '___sec56'), - ('Statistics, independent variables', 2, None, '___sec57'), - ('Statistics, more variance', 2, None, '___sec58'), - ('Statistics and stochastic processes', 2, None, '___sec59'), - ('Statistics and sample variables', 2, None, '___sec60'), + '___sec43'), + ('Why resampling methods ?', 2, None, '___sec44'), + ('Statistical analysis', 2, None, '___sec45'), + ('Statistics', 2, None, '___sec46'), + ('Statistics, moments', 2, None, '___sec47'), + ('Statistics, central moments', 2, None, '___sec48'), + ('Statistics, covariance', 2, None, '___sec49'), + ('Statistics, more covariance', 2, None, '___sec50'), + ('Covariance example', 2, None, '___sec51'), + ('Covariance in numpy', 2, None, '___sec52'), + ('Statistics, independent variables', 2, None, '___sec53'), + ('Statistics, more variance', 2, None, '___sec54'), + ('Statistics and stochastic processes', 2, None, '___sec55'), + ('Statistics and sample variables', 2, None, '___sec56'), ('Statistics, sample variance and covariance', 2, None, - '___sec61'), - ('Statistics, law of large numbers', 2, None, '___sec62'), - ('Statistics, more on sample error', 2, None, '___sec63'), - ('Statistics', 2, None, '___sec64'), - ('Statistics, central limit theorem', 2, None, '___sec65'), - ('Statistics, more technicalities', 2, None, '___sec66'), - ('Statistics', 2, None, '___sec67'), - ('Statistics and sample variance', 2, None, '___sec68'), - ('Statistics, uncorrelated results', 2, None, '___sec69'), - ('Statistics, computations', 2, None, '___sec70'), + '___sec57'), + ('Statistics, law of large numbers', 2, None, '___sec58'), + ('Statistics, more on sample error', 2, None, '___sec59'), + ('Statistics', 2, None, '___sec60'), + ('Statistics, central limit theorem', 2, None, '___sec61'), + ('Statistics, more technicalities', 2, None, '___sec62'), + ('Statistics', 2, None, '___sec63'), + ('Statistics and sample variance', 2, None, '___sec64'), + ('Statistics, uncorrelated results', 2, None, '___sec65'), + ('Statistics, computations', 2, None, '___sec66'), ('Statistics, more on computations of errors', 2, None, - '___sec71'), - ('Statistics, wrapping up 1', 2, None, '___sec72'), - ('Statistics, final expression', 2, None, '___sec73'), + '___sec67'), + ('Statistics, wrapping up 1', 2, None, '___sec68'), + ('Statistics, final expression', 2, None, '___sec69'), ('Statistics, effective number of correlations', 2, None, - '___sec74'), + '___sec70'), ('Linking the regression analysis with a statistical ' 'interpretation', 2, None, - '___sec75'), - ('Assumptions made', 2, None, '___sec76'), - ('Expectation value and variance', 2, None, '___sec77'), + '___sec71'), + ('Assumptions made', 2, None, '___sec72'), + ('Expectation value and variance', 2, None, '___sec73'), ('Expectation value and variance for $\\boldsymbol{\\beta}$', 2, None, - '___sec78'), - ('Cross-validation', 2, None, '___sec79'), - ('Computationally expensive', 2, None, '___sec80'), - ('Various steps in cross-validation', 2, None, '___sec81'), + '___sec74'), + ('Cross-validation', 2, None, '___sec75'), + ('Computationally expensive', 2, None, '___sec76'), + ('Various steps in cross-validation', 2, None, '___sec77'), ('How to set up the cross-validation for Ridge and/or Lasso', 2, None, - '___sec82'), + '___sec78'), ('Resampling methods: Jackknife and Bootstrap', 2, None, - '___sec83'), - ('Resampling methods: Jackknife', 2, None, '___sec84'), - ('Jackknife code example', 2, None, '___sec85'), - ('Resampling methods: Bootstrap', 2, None, '___sec86'), - ('Resampling methods: Bootstrap background', 2, None, '___sec87'), + '___sec79'), + ('Resampling methods: Jackknife', 2, None, '___sec80'), + ('Jackknife code example', 2, None, '___sec81'), + ('Resampling methods: Bootstrap', 2, None, '___sec82'), + ('Resampling methods: Bootstrap background', 2, None, '___sec83'), ('Resampling methods: More Bootstrap background', 2, None, - '___sec88'), - ('Resampling methods: Bootstrap approach', 2, None, '___sec89'), - ('Resampling methods: Bootstrap steps', 2, None, '___sec90'), - ('Code example for the Bootstrap method', 2, None, '___sec91'), + '___sec84'), + ('Resampling methods: Bootstrap approach', 2, None, '___sec85'), + ('Resampling methods: Bootstrap steps', 2, None, '___sec86'), + ('Code example for the Bootstrap method', 2, None, '___sec87'), ('Code Example for Cross-validation and $k$-fold ' 'Cross-validation', 2, None, - '___sec92'), - ('The bias-variance tradeoff', 2, None, '___sec93'), - ('Example code for Bias-Variance tradeoff', 2, None, '___sec94'), - ('Understanding what happens', 2, None, '___sec95'), - ('Summing up', 2, None, '___sec96'), + '___sec88'), + ('The bias-variance tradeoff', 2, None, '___sec89'), + ('Example code for Bias-Variance tradeoff', 2, None, '___sec90'), + ('Understanding what happens', 2, None, '___sec91'), + ('Summing up', 2, None, '___sec92'), ("Another Example rom Scikit-Learn's Repository", 2, None, - '___sec97'), + '___sec93'), + ('The Ising model', 2, None, '___sec94'), + ('Reformulating the problem to suit regression', + 2, + None, + '___sec95'), + ('Linear regression', 2, None, '___sec96'), + ('Singular Value decomposition', 2, None, '___sec97'), ('The one-dimensional Ising model', 2, None, '___sec98'), ('Ridge regression', 2, None, '___sec99'), ('LASSO regression', 2, None, '___sec100'), @@ -295,74 +295,74 @@ MathJax.Hub.Config({
-One of the typical problems we encounter with linear regression, in particular -when the matrix \( \boldsymbol{X} \) (our so-called design matrix) is high-dimensional, -are problems with near singular or singular matrices. The column vectors of \( \boldsymbol{X} \) -may be linearly dependent, normally referred to as super-collinearity. -This means that the matrix may be rank deficient and it is basically impossible to -to model the data using linear regression. As an example, consider the matrix +Consider the following matrix which can be SVD decomposed as + $$ -\begin{align*} -\mathbf{X} & = \left[ -\begin{array}{rrr} -1 & -1 & 2 -\\ -1 & 0 & 1 -\\ -1 & 2 & -1 -\\ -1 & 1 & 0 -\end{array} \right] -\end{align*} +\boldsymbol{X} = \frac{1}{15}\begin{bmatrix} 14 & 2\\ 4 & 22\\ 16 & 13\end{bmatrix}=\frac{1}{3}\begin{bmatrix} 1& 2 & 2 \\ 2& -1 & 1\\ 2 & 1& -2\end{bmatrix} \begin{bmatrix} 2& 0 \\ 0& 1\\ 0 & 0\end{bmatrix}\frac{1}{5}\begin{bmatrix} 3& 4 \\ 4& -3\end{bmatrix}=\boldsymbol{U}\boldsymbol{\Sigma}\boldsymbol{V}^T. $$
-The columns of \( \boldsymbol{X} \) are linearly dependent. We see this easily since the -the first column is the row-wise sum of the other two columns. The rank (more correct, -the column rank) of a matrix is the dimension of the space spanned by the -column vectors. Hence, the rank of \( \mathbf{X} \) is equal to the number -of linearly independent columns. In this particular case the matrix has rank 2. +This is a \( 3\times 2 \) matrix which is decomposed in terms of a +\( 3\times 3 \) matrix \( \boldsymbol{U} \), and a \( 2\times 2 \) matrix \( \boldsymbol{V} \). It is easy to see +that \( \boldsymbol{U} \) and \( \boldsymbol{V} \) are orthogonal (how?).
-Super-collinearity of an \( (n \times p) \)-dimensional design matrix \( \mathbf{X} \) implies -that the inverse of the matrix \( \boldsymbol{X}^T\boldsymbol{x} \) (the matrix we need to invert to solve the linear regression equations) is non-invertible. If we have a square matrix that does not have an inverse, we say this matrix singular. The example here demonstrates this -$$ -\begin{align*} -\boldsymbol{X} & = \left[ -\begin{array}{rr} -1 & -1 -\\ -1 & -1 -\end{array} \right]. -\end{align*} -$$ +And the SVD +decomposition (singular values) gives eigenvalues +\( \sigma_i\geq\sigma_{i+1} \) for all \( i \) and for dimensions larger than \( i=2 \), the +eigenvalues (singular values) are zero. -We see easily that \( \mbox{det}(\boldsymbol{X}) = x_{11} x_{22} - x_{12} x_{21} = 1 \times (-1) - 1 \times (-1) = 0 \). Hence, \( \mathbf{X} \) is singular and its inverse is undefined. -This is equivalent to saying that the matrix \( \boldsymbol{X} \) has at least an eigenvalue which is zero. +
+In the general case, where our design matrix \( \boldsymbol{X} \) has dimension +\( n\times p \), the matrix is thus decomposed into an \( n\times n \) +orthogonal matrix \( \boldsymbol{U} \), a \( p\times p \) orthogonal matrix \( \boldsymbol{V} \) +and a diagonal matrix \( \boldsymbol{\Sigma} \) with \( r=\mathrm{min}(n,p) \) +singular values \( \sigma_i\geq 0 \) on the main diagonal and zeros filling +the rest of the matrix. There are at most \( p \) singular values +assuming that \( n > p \). In our regression examples for the nuclear +masses and the equation of state this is indeed the case, while for +the Ising model we have \( p > n \). These are often cases that lead to +near singular or singular matrices. + +
+The columns of \( \boldsymbol{U} \) are called the left singular vectors while the columns of \( \boldsymbol{V} \) are the right singular vectors.
diff --git a/doc/pub/Regression/html/._Regression-bs036.html b/doc/pub/Regression/html/._Regression-bs036.html index df2e0707a..73ca49000 100644 --- a/doc/pub/Regression/html/._Regression-bs036.html +++ b/doc/pub/Regression/html/._Regression-bs036.html @@ -111,112 +111,112 @@ Automatically generated HTML file from DocOnce source ('The Boston housing data example', 2, None, '___sec27'), ('Housing data, the code', 2, None, '___sec28'), ('The singular value decomposition', 2, None, '___sec29'), - ('The Ising model', 2, None, '___sec30'), - ('Reformulating the problem to suit regression', - 2, - None, - '___sec31'), - ('Linear regression', 2, None, '___sec32'), - ('Singular Value decomposition', 2, None, '___sec33'), - ('Linear Regression Problems', 2, None, '___sec34'), - ('Fixing the singularity', 2, None, '___sec35'), - ('Basic math of the SVD', 2, None, '___sec36'), - ('The SVD, a Fantastic Algorithm', 2, None, '___sec37'), - ('Another Example', 2, None, '___sec38'), - ('Economy-size SVD', 2, None, '___sec39'), - ('Mathematical Properties', 2, None, '___sec40'), - ('Ridge and LASSO Regression', 2, None, '___sec41'), - ('More on Ridge Regression', 2, None, '___sec42'), - ('Interpreting the Ridge results', 2, None, '___sec43'), - ('More interpretations', 2, None, '___sec44'), - ('Where are we going?', 2, None, '___sec45'), - ('Resampling methods', 2, None, '___sec46'), + ('Linear Regression Problems', 2, None, '___sec30'), + ('Fixing the singularity', 2, None, '___sec31'), + ('Basic math of the SVD', 2, None, '___sec32'), + ('The SVD, a Fantastic Algorithm', 2, None, '___sec33'), + ('Another Example', 2, None, '___sec34'), + ('Economy-size SVD', 2, None, '___sec35'), + ('Mathematical Properties', 2, None, '___sec36'), + ('Ridge and LASSO Regression', 2, None, '___sec37'), + ('More on Ridge Regression', 2, None, '___sec38'), + ('Interpreting the Ridge results', 2, None, '___sec39'), + ('More interpretations', 2, None, '___sec40'), + ('Where are we going?', 2, None, '___sec41'), + ('Resampling methods', 2, None, '___sec42'), ('Resampling approaches can be computationally expensive', 2, None, - '___sec47'), - ('Why resampling methods ?', 2, None, '___sec48'), - ('Statistical analysis', 2, None, '___sec49'), - ('Statistics', 2, None, '___sec50'), - ('Statistics, moments', 2, None, '___sec51'), - ('Statistics, central moments', 2, None, '___sec52'), - ('Statistics, covariance', 2, None, '___sec53'), - ('Statistics, more covariance', 2, None, '___sec54'), - ('Covariance example', 2, None, '___sec55'), - ('Covariance in numpy', 2, None, '___sec56'), - ('Statistics, independent variables', 2, None, '___sec57'), - ('Statistics, more variance', 2, None, '___sec58'), - ('Statistics and stochastic processes', 2, None, '___sec59'), - ('Statistics and sample variables', 2, None, '___sec60'), + '___sec43'), + ('Why resampling methods ?', 2, None, '___sec44'), + ('Statistical analysis', 2, None, '___sec45'), + ('Statistics', 2, None, '___sec46'), + ('Statistics, moments', 2, None, '___sec47'), + ('Statistics, central moments', 2, None, '___sec48'), + ('Statistics, covariance', 2, None, '___sec49'), + ('Statistics, more covariance', 2, None, '___sec50'), + ('Covariance example', 2, None, '___sec51'), + ('Covariance in numpy', 2, None, '___sec52'), + ('Statistics, independent variables', 2, None, '___sec53'), + ('Statistics, more variance', 2, None, '___sec54'), + ('Statistics and stochastic processes', 2, None, '___sec55'), + ('Statistics and sample variables', 2, None, '___sec56'), ('Statistics, sample variance and covariance', 2, None, - '___sec61'), - ('Statistics, law of large numbers', 2, None, '___sec62'), - ('Statistics, more on sample error', 2, None, '___sec63'), - ('Statistics', 2, None, '___sec64'), - ('Statistics, central limit theorem', 2, None, '___sec65'), - ('Statistics, more technicalities', 2, None, '___sec66'), - ('Statistics', 2, None, '___sec67'), - ('Statistics and sample variance', 2, None, '___sec68'), - ('Statistics, uncorrelated results', 2, None, '___sec69'), - ('Statistics, computations', 2, None, '___sec70'), + '___sec57'), + ('Statistics, law of large numbers', 2, None, '___sec58'), + ('Statistics, more on sample error', 2, None, '___sec59'), + ('Statistics', 2, None, '___sec60'), + ('Statistics, central limit theorem', 2, None, '___sec61'), + ('Statistics, more technicalities', 2, None, '___sec62'), + ('Statistics', 2, None, '___sec63'), + ('Statistics and sample variance', 2, None, '___sec64'), + ('Statistics, uncorrelated results', 2, None, '___sec65'), + ('Statistics, computations', 2, None, '___sec66'), ('Statistics, more on computations of errors', 2, None, - '___sec71'), - ('Statistics, wrapping up 1', 2, None, '___sec72'), - ('Statistics, final expression', 2, None, '___sec73'), + '___sec67'), + ('Statistics, wrapping up 1', 2, None, '___sec68'), + ('Statistics, final expression', 2, None, '___sec69'), ('Statistics, effective number of correlations', 2, None, - '___sec74'), + '___sec70'), ('Linking the regression analysis with a statistical ' 'interpretation', 2, None, - '___sec75'), - ('Assumptions made', 2, None, '___sec76'), - ('Expectation value and variance', 2, None, '___sec77'), + '___sec71'), + ('Assumptions made', 2, None, '___sec72'), + ('Expectation value and variance', 2, None, '___sec73'), ('Expectation value and variance for $\\boldsymbol{\\beta}$', 2, None, - '___sec78'), - ('Cross-validation', 2, None, '___sec79'), - ('Computationally expensive', 2, None, '___sec80'), - ('Various steps in cross-validation', 2, None, '___sec81'), + '___sec74'), + ('Cross-validation', 2, None, '___sec75'), + ('Computationally expensive', 2, None, '___sec76'), + ('Various steps in cross-validation', 2, None, '___sec77'), ('How to set up the cross-validation for Ridge and/or Lasso', 2, None, - '___sec82'), + '___sec78'), ('Resampling methods: Jackknife and Bootstrap', 2, None, - '___sec83'), - ('Resampling methods: Jackknife', 2, None, '___sec84'), - ('Jackknife code example', 2, None, '___sec85'), - ('Resampling methods: Bootstrap', 2, None, '___sec86'), - ('Resampling methods: Bootstrap background', 2, None, '___sec87'), + '___sec79'), + ('Resampling methods: Jackknife', 2, None, '___sec80'), + ('Jackknife code example', 2, None, '___sec81'), + ('Resampling methods: Bootstrap', 2, None, '___sec82'), + ('Resampling methods: Bootstrap background', 2, None, '___sec83'), ('Resampling methods: More Bootstrap background', 2, None, - '___sec88'), - ('Resampling methods: Bootstrap approach', 2, None, '___sec89'), - ('Resampling methods: Bootstrap steps', 2, None, '___sec90'), - ('Code example for the Bootstrap method', 2, None, '___sec91'), + '___sec84'), + ('Resampling methods: Bootstrap approach', 2, None, '___sec85'), + ('Resampling methods: Bootstrap steps', 2, None, '___sec86'), + ('Code example for the Bootstrap method', 2, None, '___sec87'), ('Code Example for Cross-validation and $k$-fold ' 'Cross-validation', 2, None, - '___sec92'), - ('The bias-variance tradeoff', 2, None, '___sec93'), - ('Example code for Bias-Variance tradeoff', 2, None, '___sec94'), - ('Understanding what happens', 2, None, '___sec95'), - ('Summing up', 2, None, '___sec96'), + '___sec88'), + ('The bias-variance tradeoff', 2, None, '___sec89'), + ('Example code for Bias-Variance tradeoff', 2, None, '___sec90'), + ('Understanding what happens', 2, None, '___sec91'), + ('Summing up', 2, None, '___sec92'), ("Another Example rom Scikit-Learn's Repository", 2, None, - '___sec97'), + '___sec93'), + ('The Ising model', 2, None, '___sec94'), + ('Reformulating the problem to suit regression', + 2, + None, + '___sec95'), + ('Linear regression', 2, None, '___sec96'), + ('Singular Value decomposition', 2, None, '___sec97'), ('The one-dimensional Ising model', 2, None, '___sec98'), ('Ridge regression', 2, None, '___sec99'), ('LASSO regression', 2, None, '___sec100'), @@ -295,74 +295,74 @@ MathJax.Hub.Config({
-If our design matrix \( \boldsymbol{X} \) which enters the linear regression problem -$$ -\begin{align} -\boldsymbol{\beta} & = (\boldsymbol{X}^{T} \boldsymbol{X})^{-1} \boldsymbol{X}^{T} \boldsymbol{y}, -\tag{7} -\end{align} -$$ - -has linearly dependent column vectors, we will not be able to compute the inverse -of \( \boldsymbol{X}^T\boldsymbol{X} \) and we cannot find the parameters (estimators) \( \beta_i \). -The estimators are only well-defined if \( (\boldsymbol{X}^{T}\boldsymbol{X})^{-1} \) exits. -This is more likely to happen when the matrix \( \boldsymbol{X} \) is high-dimensional. In this case it is likely to encounter a situation where -the regression parameters \( \beta_i \) cannot be estimated. +If we assume that \( n > p \), then our matrix \( \boldsymbol{U} \) has dimension \( n +\times n \). The last \( n-p \) columns of \( \boldsymbol{U} \) become however +irrelevant in our calculations since they are multiplied with the +zeros in \( \boldsymbol{\Sigma} \).
-A cheap ad hoc approach is simply to add a small diagonal component to the matrix to invert, that is we change -$$ -\boldsymbol{X}^{T} \boldsymbol{X} \rightarrow \boldsymbol{X}^{T} \boldsymbol{X}+\lambda \boldsymbol{I}, -$$ +The economy-size decomposition removes extra rows or columns of zeros +from the diagonal matrix of singular values, \( \boldsymbol{\Sigma} \), along with the columns +in either \( \boldsymbol{U} \) or \( \boldsymbol{V} \) that multiply those zeros in the expression. +Removing these zeros and columns can improve execution time +and reduce storage requirements without compromising the accuracy of +the decomposition. -where \( \boldsymbol{I} \) is the identity matrix. When we discuss Ridge regression this is actually what we end up evaluating. The parameter \( \lambda \) is called a hyperparameter. More about this later. +
+If \( n > p \), we keep only the first \( p \) columns of \( \boldsymbol{U} \) and \( \boldsymbol{\Sigma} \) has dimension \( p\times p \). +If \( p > n \), then only the first \( n \) columns of \( \boldsymbol{V} \) are computed and \( \boldsymbol{\Sigma} \) has dimension \( n\times n \). +The \( n=p \) case is obvious, we retain the full SVD. +In general the economy-size SVD leads to less FLOPS and still conserving the desired accuracy.
diff --git a/doc/pub/Regression/html/._Regression-bs037.html b/doc/pub/Regression/html/._Regression-bs037.html index 400c5aebf..6142e651b 100644 --- a/doc/pub/Regression/html/._Regression-bs037.html +++ b/doc/pub/Regression/html/._Regression-bs037.html @@ -111,112 +111,112 @@ Automatically generated HTML file from DocOnce source ('The Boston housing data example', 2, None, '___sec27'), ('Housing data, the code', 2, None, '___sec28'), ('The singular value decomposition', 2, None, '___sec29'), - ('The Ising model', 2, None, '___sec30'), - ('Reformulating the problem to suit regression', - 2, - None, - '___sec31'), - ('Linear regression', 2, None, '___sec32'), - ('Singular Value decomposition', 2, None, '___sec33'), - ('Linear Regression Problems', 2, None, '___sec34'), - ('Fixing the singularity', 2, None, '___sec35'), - ('Basic math of the SVD', 2, None, '___sec36'), - ('The SVD, a Fantastic Algorithm', 2, None, '___sec37'), - ('Another Example', 2, None, '___sec38'), - ('Economy-size SVD', 2, None, '___sec39'), - ('Mathematical Properties', 2, None, '___sec40'), - ('Ridge and LASSO Regression', 2, None, '___sec41'), - ('More on Ridge Regression', 2, None, '___sec42'), - ('Interpreting the Ridge results', 2, None, '___sec43'), - ('More interpretations', 2, None, '___sec44'), - ('Where are we going?', 2, None, '___sec45'), - ('Resampling methods', 2, None, '___sec46'), + ('Linear Regression Problems', 2, None, '___sec30'), + ('Fixing the singularity', 2, None, '___sec31'), + ('Basic math of the SVD', 2, None, '___sec32'), + ('The SVD, a Fantastic Algorithm', 2, None, '___sec33'), + ('Another Example', 2, None, '___sec34'), + ('Economy-size SVD', 2, None, '___sec35'), + ('Mathematical Properties', 2, None, '___sec36'), + ('Ridge and LASSO Regression', 2, None, '___sec37'), + ('More on Ridge Regression', 2, None, '___sec38'), + ('Interpreting the Ridge results', 2, None, '___sec39'), + ('More interpretations', 2, None, '___sec40'), + ('Where are we going?', 2, None, '___sec41'), + ('Resampling methods', 2, None, '___sec42'), ('Resampling approaches can be computationally expensive', 2, None, - '___sec47'), - ('Why resampling methods ?', 2, None, '___sec48'), - ('Statistical analysis', 2, None, '___sec49'), - ('Statistics', 2, None, '___sec50'), - ('Statistics, moments', 2, None, '___sec51'), - ('Statistics, central moments', 2, None, '___sec52'), - ('Statistics, covariance', 2, None, '___sec53'), - ('Statistics, more covariance', 2, None, '___sec54'), - ('Covariance example', 2, None, '___sec55'), - ('Covariance in numpy', 2, None, '___sec56'), - ('Statistics, independent variables', 2, None, '___sec57'), - ('Statistics, more variance', 2, None, '___sec58'), - ('Statistics and stochastic processes', 2, None, '___sec59'), - ('Statistics and sample variables', 2, None, '___sec60'), + '___sec43'), + ('Why resampling methods ?', 2, None, '___sec44'), + ('Statistical analysis', 2, None, '___sec45'), + ('Statistics', 2, None, '___sec46'), + ('Statistics, moments', 2, None, '___sec47'), + ('Statistics, central moments', 2, None, '___sec48'), + ('Statistics, covariance', 2, None, '___sec49'), + ('Statistics, more covariance', 2, None, '___sec50'), + ('Covariance example', 2, None, '___sec51'), + ('Covariance in numpy', 2, None, '___sec52'), + ('Statistics, independent variables', 2, None, '___sec53'), + ('Statistics, more variance', 2, None, '___sec54'), + ('Statistics and stochastic processes', 2, None, '___sec55'), + ('Statistics and sample variables', 2, None, '___sec56'), ('Statistics, sample variance and covariance', 2, None, - '___sec61'), - ('Statistics, law of large numbers', 2, None, '___sec62'), - ('Statistics, more on sample error', 2, None, '___sec63'), - ('Statistics', 2, None, '___sec64'), - ('Statistics, central limit theorem', 2, None, '___sec65'), - ('Statistics, more technicalities', 2, None, '___sec66'), - ('Statistics', 2, None, '___sec67'), - ('Statistics and sample variance', 2, None, '___sec68'), - ('Statistics, uncorrelated results', 2, None, '___sec69'), - ('Statistics, computations', 2, None, '___sec70'), + '___sec57'), + ('Statistics, law of large numbers', 2, None, '___sec58'), + ('Statistics, more on sample error', 2, None, '___sec59'), + ('Statistics', 2, None, '___sec60'), + ('Statistics, central limit theorem', 2, None, '___sec61'), + ('Statistics, more technicalities', 2, None, '___sec62'), + ('Statistics', 2, None, '___sec63'), + ('Statistics and sample variance', 2, None, '___sec64'), + ('Statistics, uncorrelated results', 2, None, '___sec65'), + ('Statistics, computations', 2, None, '___sec66'), ('Statistics, more on computations of errors', 2, None, - '___sec71'), - ('Statistics, wrapping up 1', 2, None, '___sec72'), - ('Statistics, final expression', 2, None, '___sec73'), + '___sec67'), + ('Statistics, wrapping up 1', 2, None, '___sec68'), + ('Statistics, final expression', 2, None, '___sec69'), ('Statistics, effective number of correlations', 2, None, - '___sec74'), + '___sec70'), ('Linking the regression analysis with a statistical ' 'interpretation', 2, None, - '___sec75'), - ('Assumptions made', 2, None, '___sec76'), - ('Expectation value and variance', 2, None, '___sec77'), + '___sec71'), + ('Assumptions made', 2, None, '___sec72'), + ('Expectation value and variance', 2, None, '___sec73'), ('Expectation value and variance for $\\boldsymbol{\\beta}$', 2, None, - '___sec78'), - ('Cross-validation', 2, None, '___sec79'), - ('Computationally expensive', 2, None, '___sec80'), - ('Various steps in cross-validation', 2, None, '___sec81'), + '___sec74'), + ('Cross-validation', 2, None, '___sec75'), + ('Computationally expensive', 2, None, '___sec76'), + ('Various steps in cross-validation', 2, None, '___sec77'), ('How to set up the cross-validation for Ridge and/or Lasso', 2, None, - '___sec82'), + '___sec78'), ('Resampling methods: Jackknife and Bootstrap', 2, None, - '___sec83'), - ('Resampling methods: Jackknife', 2, None, '___sec84'), - ('Jackknife code example', 2, None, '___sec85'), - ('Resampling methods: Bootstrap', 2, None, '___sec86'), - ('Resampling methods: Bootstrap background', 2, None, '___sec87'), + '___sec79'), + ('Resampling methods: Jackknife', 2, None, '___sec80'), + ('Jackknife code example', 2, None, '___sec81'), + ('Resampling methods: Bootstrap', 2, None, '___sec82'), + ('Resampling methods: Bootstrap background', 2, None, '___sec83'), ('Resampling methods: More Bootstrap background', 2, None, - '___sec88'), - ('Resampling methods: Bootstrap approach', 2, None, '___sec89'), - ('Resampling methods: Bootstrap steps', 2, None, '___sec90'), - ('Code example for the Bootstrap method', 2, None, '___sec91'), + '___sec84'), + ('Resampling methods: Bootstrap approach', 2, None, '___sec85'), + ('Resampling methods: Bootstrap steps', 2, None, '___sec86'), + ('Code example for the Bootstrap method', 2, None, '___sec87'), ('Code Example for Cross-validation and $k$-fold ' 'Cross-validation', 2, None, - '___sec92'), - ('The bias-variance tradeoff', 2, None, '___sec93'), - ('Example code for Bias-Variance tradeoff', 2, None, '___sec94'), - ('Understanding what happens', 2, None, '___sec95'), - ('Summing up', 2, None, '___sec96'), + '___sec88'), + ('The bias-variance tradeoff', 2, None, '___sec89'), + ('Example code for Bias-Variance tradeoff', 2, None, '___sec90'), + ('Understanding what happens', 2, None, '___sec91'), + ('Summing up', 2, None, '___sec92'), ("Another Example rom Scikit-Learn's Repository", 2, None, - '___sec97'), + '___sec93'), + ('The Ising model', 2, None, '___sec94'), + ('Reformulating the problem to suit regression', + 2, + None, + '___sec95'), + ('Linear regression', 2, None, '___sec96'), + ('Singular Value decomposition', 2, None, '___sec97'), ('The one-dimensional Ising model', 2, None, '___sec98'), ('Ridge regression', 2, None, '___sec99'), ('LASSO regression', 2, None, '___sec100'), @@ -295,74 +295,74 @@ MathJax.Hub.Config({
-From standard linear algebra we know that a square matrix \( \boldsymbol{X} \) can be diagonalized if and only it is -a so-called normal matrix, that is if \( \boldsymbol{X}\in {\mathbb{R}}^{n\times n} \) -we have \( \boldsymbol{X}\boldsymbol{X}^T=\boldsymbol{X}^T\boldsymbol{X} \) or if \( \boldsymbol{X}\in {\mathbb{C}}^{n\times n} \) we have \( \boldsymbol{X}\boldsymbol{X}^{\dagger}=\boldsymbol{X}^{\dagger}\boldsymbol{X} \). -The matrix has then a set of eigenpairs - -$$ -(\lambda_1,\boldsymbol{u}_1),\dots, (\lambda_n,\boldsymbol{u}_n), -$$ - -and the eigenvalues are given by the diagonal matrix -$$ -\boldsymbol{\Sigma}=\mathrm{Diag}(\lambda_1, \dots,\lambda_n). -$$ - -The matrix \( \boldsymbol{X} \) can be written in terms of an orthogonal/unitary transformation \( \boldsymbol{U} \) -$$ -\boldsymbol{X} = \boldsymbol{U}\boldsymbol{\Sigma}\boldsymbol{V}^T, -$$ - -with \( \boldsymbol{U}\boldsymbol{U}^T=\boldsymbol{I} \) or \( \boldsymbol{U}\boldsymbol{U}^{\dagger}=\boldsymbol{I} \). +There are several interesting mathematical properties which will be +relevant when we are going to discuss the differences between say +ordinary least squares (OLS) and Ridge regression.
-Not all square matrices are diagonalizable. A matrix like the one discussed above +We have from OLS that the parameters of the linear approximation are given by $$ -\boldsymbol{X} = \begin{bmatrix} -1& -1 \\ -1& -1\\ -\end{bmatrix} +\boldsymbol{\tilde{y}} = \boldsymbol{X}\boldsymbol{\beta} = \boldsymbol{X}\left(\boldsymbol{X}^T\boldsymbol{X}\right)^{-1}\boldsymbol{X}^T\boldsymbol{y}. $$ -is not diagonalizable, it is a so-called defective matrix. It is easy to see that the condition -\( \boldsymbol{X}\boldsymbol{X}^T=\boldsymbol{X}^T\boldsymbol{X} \) is not fulfilled. +
+The matrix to invert can be rewritten in terms of our SVD decomposition as + +$$ +\boldsymbol{X}^T\boldsymbol{X} = \boldsymbol{V}\boldsymbol{\Sigma}^T\boldsymbol{U}^T\boldsymbol{U}\boldsymbol{\Sigma}\boldsymbol{V}^T. +$$ + +Using the orthogonality properties of \( \boldsymbol{U} \) we have + +$$ +\boldsymbol{X}^T\boldsymbol{X} = \boldsymbol{V}\boldsymbol{\Sigma}^T\boldsymbol{\Sigma}\boldsymbol{V}^T = \boldsymbol{V}\boldsymbol{D}\boldsymbol{V}^T, +$$ + +with \( \boldsymbol{D} \) being a diagonal matrix with values along the diagonal given by the singular values squared. + +
+This means that +$$ +(\boldsymbol{X}^T\boldsymbol{X})\boldsymbol{V} = \boldsymbol{V}\boldsymbol{D}, +$$ + +that is the eigenvectors of \( (\boldsymbol{X}^T\boldsymbol{X}) \) are given by the columns of the right singular matrix of \( \boldsymbol{X} \) and the eigenvalues are the squared singular values. It is easy to show (show this) that +$$ +(\boldsymbol{X}\boldsymbol{X}^T)\boldsymbol{U} = \boldsymbol{U}\boldsymbol{D}, +$$ + +that is, the eigenvectors of \( (\boldsymbol{X}\boldsymbol{X})^T \) are the columns of the left singular matrix and the eigenvalues are the same. + +
+Going back to our OLS equation we have +$$ +\boldsymbol{X}\boldsymbol{\beta} = \boldsymbol{X}\left(\boldsymbol{V}\boldsymbol{D}\boldsymbol{V}^T \right)^{-1}\boldsymbol{X}^T\boldsymbol{y}=\boldsymbol{U\Sigma V^T}\left(\boldsymbol{V}\boldsymbol{D}\boldsymbol{V}^T \right)^{-1}(\boldsymbol{U\Sigma V^T})^T\boldsymbol{y}=\boldsymbol{U}\boldsymbol{U}^T\boldsymbol{y}. +$$ + +We will come back to this expression when we discuss Ridge regression.
diff --git a/doc/pub/Regression/html/._Regression-bs038.html b/doc/pub/Regression/html/._Regression-bs038.html index 13c587454..5b604d134 100644 --- a/doc/pub/Regression/html/._Regression-bs038.html +++ b/doc/pub/Regression/html/._Regression-bs038.html @@ -111,112 +111,112 @@ Automatically generated HTML file from DocOnce source ('The Boston housing data example', 2, None, '___sec27'), ('Housing data, the code', 2, None, '___sec28'), ('The singular value decomposition', 2, None, '___sec29'), - ('The Ising model', 2, None, '___sec30'), - ('Reformulating the problem to suit regression', - 2, - None, - '___sec31'), - ('Linear regression', 2, None, '___sec32'), - ('Singular Value decomposition', 2, None, '___sec33'), - ('Linear Regression Problems', 2, None, '___sec34'), - ('Fixing the singularity', 2, None, '___sec35'), - ('Basic math of the SVD', 2, None, '___sec36'), - ('The SVD, a Fantastic Algorithm', 2, None, '___sec37'), - ('Another Example', 2, None, '___sec38'), - ('Economy-size SVD', 2, None, '___sec39'), - ('Mathematical Properties', 2, None, '___sec40'), - ('Ridge and LASSO Regression', 2, None, '___sec41'), - ('More on Ridge Regression', 2, None, '___sec42'), - ('Interpreting the Ridge results', 2, None, '___sec43'), - ('More interpretations', 2, None, '___sec44'), - ('Where are we going?', 2, None, '___sec45'), - ('Resampling methods', 2, None, '___sec46'), + ('Linear Regression Problems', 2, None, '___sec30'), + ('Fixing the singularity', 2, None, '___sec31'), + ('Basic math of the SVD', 2, None, '___sec32'), + ('The SVD, a Fantastic Algorithm', 2, None, '___sec33'), + ('Another Example', 2, None, '___sec34'), + ('Economy-size SVD', 2, None, '___sec35'), + ('Mathematical Properties', 2, None, '___sec36'), + ('Ridge and LASSO Regression', 2, None, '___sec37'), + ('More on Ridge Regression', 2, None, '___sec38'), + ('Interpreting the Ridge results', 2, None, '___sec39'), + ('More interpretations', 2, None, '___sec40'), + ('Where are we going?', 2, None, '___sec41'), + ('Resampling methods', 2, None, '___sec42'), ('Resampling approaches can be computationally expensive', 2, None, - '___sec47'), - ('Why resampling methods ?', 2, None, '___sec48'), - ('Statistical analysis', 2, None, '___sec49'), - ('Statistics', 2, None, '___sec50'), - ('Statistics, moments', 2, None, '___sec51'), - ('Statistics, central moments', 2, None, '___sec52'), - ('Statistics, covariance', 2, None, '___sec53'), - ('Statistics, more covariance', 2, None, '___sec54'), - ('Covariance example', 2, None, '___sec55'), - ('Covariance in numpy', 2, None, '___sec56'), - ('Statistics, independent variables', 2, None, '___sec57'), - ('Statistics, more variance', 2, None, '___sec58'), - ('Statistics and stochastic processes', 2, None, '___sec59'), - ('Statistics and sample variables', 2, None, '___sec60'), + '___sec43'), + ('Why resampling methods ?', 2, None, '___sec44'), + ('Statistical analysis', 2, None, '___sec45'), + ('Statistics', 2, None, '___sec46'), + ('Statistics, moments', 2, None, '___sec47'), + ('Statistics, central moments', 2, None, '___sec48'), + ('Statistics, covariance', 2, None, '___sec49'), + ('Statistics, more covariance', 2, None, '___sec50'), + ('Covariance example', 2, None, '___sec51'), + ('Covariance in numpy', 2, None, '___sec52'), + ('Statistics, independent variables', 2, None, '___sec53'), + ('Statistics, more variance', 2, None, '___sec54'), + ('Statistics and stochastic processes', 2, None, '___sec55'), + ('Statistics and sample variables', 2, None, '___sec56'), ('Statistics, sample variance and covariance', 2, None, - '___sec61'), - ('Statistics, law of large numbers', 2, None, '___sec62'), - ('Statistics, more on sample error', 2, None, '___sec63'), - ('Statistics', 2, None, '___sec64'), - ('Statistics, central limit theorem', 2, None, '___sec65'), - ('Statistics, more technicalities', 2, None, '___sec66'), - ('Statistics', 2, None, '___sec67'), - ('Statistics and sample variance', 2, None, '___sec68'), - ('Statistics, uncorrelated results', 2, None, '___sec69'), - ('Statistics, computations', 2, None, '___sec70'), + '___sec57'), + ('Statistics, law of large numbers', 2, None, '___sec58'), + ('Statistics, more on sample error', 2, None, '___sec59'), + ('Statistics', 2, None, '___sec60'), + ('Statistics, central limit theorem', 2, None, '___sec61'), + ('Statistics, more technicalities', 2, None, '___sec62'), + ('Statistics', 2, None, '___sec63'), + ('Statistics and sample variance', 2, None, '___sec64'), + ('Statistics, uncorrelated results', 2, None, '___sec65'), + ('Statistics, computations', 2, None, '___sec66'), ('Statistics, more on computations of errors', 2, None, - '___sec71'), - ('Statistics, wrapping up 1', 2, None, '___sec72'), - ('Statistics, final expression', 2, None, '___sec73'), + '___sec67'), + ('Statistics, wrapping up 1', 2, None, '___sec68'), + ('Statistics, final expression', 2, None, '___sec69'), ('Statistics, effective number of correlations', 2, None, - '___sec74'), + '___sec70'), ('Linking the regression analysis with a statistical ' 'interpretation', 2, None, - '___sec75'), - ('Assumptions made', 2, None, '___sec76'), - ('Expectation value and variance', 2, None, '___sec77'), + '___sec71'), + ('Assumptions made', 2, None, '___sec72'), + ('Expectation value and variance', 2, None, '___sec73'), ('Expectation value and variance for $\\boldsymbol{\\beta}$', 2, None, - '___sec78'), - ('Cross-validation', 2, None, '___sec79'), - ('Computationally expensive', 2, None, '___sec80'), - ('Various steps in cross-validation', 2, None, '___sec81'), + '___sec74'), + ('Cross-validation', 2, None, '___sec75'), + ('Computationally expensive', 2, None, '___sec76'), + ('Various steps in cross-validation', 2, None, '___sec77'), ('How to set up the cross-validation for Ridge and/or Lasso', 2, None, - '___sec82'), + '___sec78'), ('Resampling methods: Jackknife and Bootstrap', 2, None, - '___sec83'), - ('Resampling methods: Jackknife', 2, None, '___sec84'), - ('Jackknife code example', 2, None, '___sec85'), - ('Resampling methods: Bootstrap', 2, None, '___sec86'), - ('Resampling methods: Bootstrap background', 2, None, '___sec87'), + '___sec79'), + ('Resampling methods: Jackknife', 2, None, '___sec80'), + ('Jackknife code example', 2, None, '___sec81'), + ('Resampling methods: Bootstrap', 2, None, '___sec82'), + ('Resampling methods: Bootstrap background', 2, None, '___sec83'), ('Resampling methods: More Bootstrap background', 2, None, - '___sec88'), - ('Resampling methods: Bootstrap approach', 2, None, '___sec89'), - ('Resampling methods: Bootstrap steps', 2, None, '___sec90'), - ('Code example for the Bootstrap method', 2, None, '___sec91'), + '___sec84'), + ('Resampling methods: Bootstrap approach', 2, None, '___sec85'), + ('Resampling methods: Bootstrap steps', 2, None, '___sec86'), + ('Code example for the Bootstrap method', 2, None, '___sec87'), ('Code Example for Cross-validation and $k$-fold ' 'Cross-validation', 2, None, - '___sec92'), - ('The bias-variance tradeoff', 2, None, '___sec93'), - ('Example code for Bias-Variance tradeoff', 2, None, '___sec94'), - ('Understanding what happens', 2, None, '___sec95'), - ('Summing up', 2, None, '___sec96'), + '___sec88'), + ('The bias-variance tradeoff', 2, None, '___sec89'), + ('Example code for Bias-Variance tradeoff', 2, None, '___sec90'), + ('Understanding what happens', 2, None, '___sec91'), + ('Summing up', 2, None, '___sec92'), ("Another Example rom Scikit-Learn's Repository", 2, None, - '___sec97'), + '___sec93'), + ('The Ising model', 2, None, '___sec94'), + ('Reformulating the problem to suit regression', + 2, + None, + '___sec95'), + ('Linear regression', 2, None, '___sec96'), + ('Singular Value decomposition', 2, None, '___sec97'), ('The one-dimensional Ising model', 2, None, '___sec98'), ('Ridge regression', 2, None, '___sec99'), ('LASSO regression', 2, None, '___sec100'), @@ -295,74 +295,74 @@ MathJax.Hub.Config({
-However, and this is the strength of the SVD algorithm, any general -matrix \( \boldsymbol{X} \) can be decomposed in terms of a diagonal matrix and -two orthogonal/unitary matrices. The Singular Value Decompostion -(SVD) theorem -states that a general \( m\times n \) matrix \( \boldsymbol{X} \) can be written in -terms of a diagonal matrix \( \boldsymbol{\Sigma} \) of dimensionality \( n\times n \) -and two orthognal matrices \( \boldsymbol{U} \) and \( \boldsymbol{V} \), where the first has -dimensionality \( m \times m \) and the last dimensionality \( n\times n \). -We have then +Let us remind ourselves about the expression for the standard Mean Squared Error (MSE) which we used to define our cost function and the equations for the ordinary least squares (OLS) method, that is +our optimization problem is +$$ +{\displaystyle \min_{\boldsymbol{\beta}\in {\mathbb{R}}^{p}}}\frac{1}{n}\left\{\left(\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\right)^T\left(\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\right)\right\}. +$$ -$$ -\boldsymbol{X} = \boldsymbol{U}\boldsymbol{\Sigma}\boldsymbol{V}^T +or we can state it as +$$ +{\displaystyle \min_{\boldsymbol{\beta}\in +{\mathbb{R}}^{p}}}\frac{1}{n}\sum_{i=0}^{n-1}\left(y_i-\tilde{y}_i\right)^2=\frac{1}{n}\vert\vert \boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\vert\vert_2^2, +$$ + +where we have used the definition of a norm-2 vector, that is +$$ +\vert\vert \boldsymbol{x}\vert\vert_2 = \sqrt{\sum_i x_i^2}. $$
-As an example, the above defective matrix can be decomposed as +By minimizing the above equation with respect to the parameters +\( \boldsymbol{\beta} \) we could then obtain an analytical expression for the +parameters \( \boldsymbol{\beta} \). We can add a regularization parameter \( \lambda \) by +defining a new cost function to be optimized, that is $$ -\boldsymbol{X} = \frac{1}{\sqrt{2}}\begin{bmatrix} 1& 1 \\ 1& -1\\ \end{bmatrix} \begin{bmatrix} 2& 0 \\ 0& 0\\ \end{bmatrix} \frac{1}{\sqrt{2}}\begin{bmatrix} 1& -1 \\ 1& 1\\ \end{bmatrix}=\boldsymbol{U}\boldsymbol{\Sigma}\boldsymbol{V}^T, +{\displaystyle \min_{\boldsymbol{\beta}\in +{\mathbb{R}}^{p}}}\frac{1}{n}\vert\vert \boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\vert\vert_2^2+\lambda\vert\vert \boldsymbol{\beta}\vert\vert_2^2 $$
-with eigenvalues \( \sigma_1=2 \) and \( \sigma_2=0 \). -The SVD exits always! +which leads to the Ridge regression minimization problem where we +require that \( \vert\vert \boldsymbol{\beta}\vert\vert_2^2\le t \), where \( t \) is +a finite number larger than zero. By defining + +$$ +C(\boldsymbol{X},\boldsymbol{\beta})=\frac{1}{n}\vert\vert \boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\vert\vert_2^2+\lambda\vert\vert \boldsymbol{\beta}\vert\vert_1, +$$ + +
+we have a new optimization equation +$$ +{\displaystyle \min_{\boldsymbol{\beta}\in +{\mathbb{R}}^{p}}}\frac{1}{n}\vert\vert \boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\vert\vert_2^2+\lambda\vert\vert \boldsymbol{\beta}\vert\vert_1 +$$ + +which leads to Lasso regression. Lasso stands for least absolute shrinkage and selection operator. + +
+Here we have defined the norm-1 as +$$ +\vert\vert \boldsymbol{x}\vert\vert_1 = \sum_i \vert x_i\vert. +$$
diff --git a/doc/pub/Regression/html/._Regression-bs039.html b/doc/pub/Regression/html/._Regression-bs039.html index d01066aec..c63c5f3ef 100644 --- a/doc/pub/Regression/html/._Regression-bs039.html +++ b/doc/pub/Regression/html/._Regression-bs039.html @@ -111,112 +111,112 @@ Automatically generated HTML file from DocOnce source ('The Boston housing data example', 2, None, '___sec27'), ('Housing data, the code', 2, None, '___sec28'), ('The singular value decomposition', 2, None, '___sec29'), - ('The Ising model', 2, None, '___sec30'), - ('Reformulating the problem to suit regression', - 2, - None, - '___sec31'), - ('Linear regression', 2, None, '___sec32'), - ('Singular Value decomposition', 2, None, '___sec33'), - ('Linear Regression Problems', 2, None, '___sec34'), - ('Fixing the singularity', 2, None, '___sec35'), - ('Basic math of the SVD', 2, None, '___sec36'), - ('The SVD, a Fantastic Algorithm', 2, None, '___sec37'), - ('Another Example', 2, None, '___sec38'), - ('Economy-size SVD', 2, None, '___sec39'), - ('Mathematical Properties', 2, None, '___sec40'), - ('Ridge and LASSO Regression', 2, None, '___sec41'), - ('More on Ridge Regression', 2, None, '___sec42'), - ('Interpreting the Ridge results', 2, None, '___sec43'), - ('More interpretations', 2, None, '___sec44'), - ('Where are we going?', 2, None, '___sec45'), - ('Resampling methods', 2, None, '___sec46'), + ('Linear Regression Problems', 2, None, '___sec30'), + ('Fixing the singularity', 2, None, '___sec31'), + ('Basic math of the SVD', 2, None, '___sec32'), + ('The SVD, a Fantastic Algorithm', 2, None, '___sec33'), + ('Another Example', 2, None, '___sec34'), + ('Economy-size SVD', 2, None, '___sec35'), + ('Mathematical Properties', 2, None, '___sec36'), + ('Ridge and LASSO Regression', 2, None, '___sec37'), + ('More on Ridge Regression', 2, None, '___sec38'), + ('Interpreting the Ridge results', 2, None, '___sec39'), + ('More interpretations', 2, None, '___sec40'), + ('Where are we going?', 2, None, '___sec41'), + ('Resampling methods', 2, None, '___sec42'), ('Resampling approaches can be computationally expensive', 2, None, - '___sec47'), - ('Why resampling methods ?', 2, None, '___sec48'), - ('Statistical analysis', 2, None, '___sec49'), - ('Statistics', 2, None, '___sec50'), - ('Statistics, moments', 2, None, '___sec51'), - ('Statistics, central moments', 2, None, '___sec52'), - ('Statistics, covariance', 2, None, '___sec53'), - ('Statistics, more covariance', 2, None, '___sec54'), - ('Covariance example', 2, None, '___sec55'), - ('Covariance in numpy', 2, None, '___sec56'), - ('Statistics, independent variables', 2, None, '___sec57'), - ('Statistics, more variance', 2, None, '___sec58'), - ('Statistics and stochastic processes', 2, None, '___sec59'), - ('Statistics and sample variables', 2, None, '___sec60'), + '___sec43'), + ('Why resampling methods ?', 2, None, '___sec44'), + ('Statistical analysis', 2, None, '___sec45'), + ('Statistics', 2, None, '___sec46'), + ('Statistics, moments', 2, None, '___sec47'), + ('Statistics, central moments', 2, None, '___sec48'), + ('Statistics, covariance', 2, None, '___sec49'), + ('Statistics, more covariance', 2, None, '___sec50'), + ('Covariance example', 2, None, '___sec51'), + ('Covariance in numpy', 2, None, '___sec52'), + ('Statistics, independent variables', 2, None, '___sec53'), + ('Statistics, more variance', 2, None, '___sec54'), + ('Statistics and stochastic processes', 2, None, '___sec55'), + ('Statistics and sample variables', 2, None, '___sec56'), ('Statistics, sample variance and covariance', 2, None, - '___sec61'), - ('Statistics, law of large numbers', 2, None, '___sec62'), - ('Statistics, more on sample error', 2, None, '___sec63'), - ('Statistics', 2, None, '___sec64'), - ('Statistics, central limit theorem', 2, None, '___sec65'), - ('Statistics, more technicalities', 2, None, '___sec66'), - ('Statistics', 2, None, '___sec67'), - ('Statistics and sample variance', 2, None, '___sec68'), - ('Statistics, uncorrelated results', 2, None, '___sec69'), - ('Statistics, computations', 2, None, '___sec70'), + '___sec57'), + ('Statistics, law of large numbers', 2, None, '___sec58'), + ('Statistics, more on sample error', 2, None, '___sec59'), + ('Statistics', 2, None, '___sec60'), + ('Statistics, central limit theorem', 2, None, '___sec61'), + ('Statistics, more technicalities', 2, None, '___sec62'), + ('Statistics', 2, None, '___sec63'), + ('Statistics and sample variance', 2, None, '___sec64'), + ('Statistics, uncorrelated results', 2, None, '___sec65'), + ('Statistics, computations', 2, None, '___sec66'), ('Statistics, more on computations of errors', 2, None, - '___sec71'), - ('Statistics, wrapping up 1', 2, None, '___sec72'), - ('Statistics, final expression', 2, None, '___sec73'), + '___sec67'), + ('Statistics, wrapping up 1', 2, None, '___sec68'), + ('Statistics, final expression', 2, None, '___sec69'), ('Statistics, effective number of correlations', 2, None, - '___sec74'), + '___sec70'), ('Linking the regression analysis with a statistical ' 'interpretation', 2, None, - '___sec75'), - ('Assumptions made', 2, None, '___sec76'), - ('Expectation value and variance', 2, None, '___sec77'), + '___sec71'), + ('Assumptions made', 2, None, '___sec72'), + ('Expectation value and variance', 2, None, '___sec73'), ('Expectation value and variance for $\\boldsymbol{\\beta}$', 2, None, - '___sec78'), - ('Cross-validation', 2, None, '___sec79'), - ('Computationally expensive', 2, None, '___sec80'), - ('Various steps in cross-validation', 2, None, '___sec81'), + '___sec74'), + ('Cross-validation', 2, None, '___sec75'), + ('Computationally expensive', 2, None, '___sec76'), + ('Various steps in cross-validation', 2, None, '___sec77'), ('How to set up the cross-validation for Ridge and/or Lasso', 2, None, - '___sec82'), + '___sec78'), ('Resampling methods: Jackknife and Bootstrap', 2, None, - '___sec83'), - ('Resampling methods: Jackknife', 2, None, '___sec84'), - ('Jackknife code example', 2, None, '___sec85'), - ('Resampling methods: Bootstrap', 2, None, '___sec86'), - ('Resampling methods: Bootstrap background', 2, None, '___sec87'), + '___sec79'), + ('Resampling methods: Jackknife', 2, None, '___sec80'), + ('Jackknife code example', 2, None, '___sec81'), + ('Resampling methods: Bootstrap', 2, None, '___sec82'), + ('Resampling methods: Bootstrap background', 2, None, '___sec83'), ('Resampling methods: More Bootstrap background', 2, None, - '___sec88'), - ('Resampling methods: Bootstrap approach', 2, None, '___sec89'), - ('Resampling methods: Bootstrap steps', 2, None, '___sec90'), - ('Code example for the Bootstrap method', 2, None, '___sec91'), + '___sec84'), + ('Resampling methods: Bootstrap approach', 2, None, '___sec85'), + ('Resampling methods: Bootstrap steps', 2, None, '___sec86'), + ('Code example for the Bootstrap method', 2, None, '___sec87'), ('Code Example for Cross-validation and $k$-fold ' 'Cross-validation', 2, None, - '___sec92'), - ('The bias-variance tradeoff', 2, None, '___sec93'), - ('Example code for Bias-Variance tradeoff', 2, None, '___sec94'), - ('Understanding what happens', 2, None, '___sec95'), - ('Summing up', 2, None, '___sec96'), + '___sec88'), + ('The bias-variance tradeoff', 2, None, '___sec89'), + ('Example code for Bias-Variance tradeoff', 2, None, '___sec90'), + ('Understanding what happens', 2, None, '___sec91'), + ('Summing up', 2, None, '___sec92'), ("Another Example rom Scikit-Learn's Repository", 2, None, - '___sec97'), + '___sec93'), + ('The Ising model', 2, None, '___sec94'), + ('Reformulating the problem to suit regression', + 2, + None, + '___sec95'), + ('Linear regression', 2, None, '___sec96'), + ('Singular Value decomposition', 2, None, '___sec97'), ('The one-dimensional Ising model', 2, None, '___sec98'), ('Ridge regression', 2, None, '___sec99'), ('LASSO regression', 2, None, '___sec100'), @@ -295,74 +295,74 @@ MathJax.Hub.Config({
-Consider the following matrix which can be SVD decomposed as +Using the matrix-vector expression for Ridge regression, $$ -\boldsymbol{X} = \frac{1}{15}\begin{bmatrix} 14 & 2\\ 4 & 22\\ 16 & 13\end{bmatrix}=\frac{1}{3}\begin{bmatrix} 1& 2 & 2 \\ 2& -1 & 1\\ 2 & 1& -2\end{bmatrix} \begin{bmatrix} 2& 0 \\ 0& 1\\ 0 & 0\end{bmatrix}\frac{1}{5}\begin{bmatrix} 3& 4 \\ 4& -3\end{bmatrix}=\boldsymbol{U}\boldsymbol{\Sigma}\boldsymbol{V}^T. +C(\boldsymbol{X},\boldsymbol{\beta})=\frac{1}{n}\left\{(\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta})^T(\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta})\right\}+\lambda\boldsymbol{\beta}^T\boldsymbol{\beta}, $$
-This is a \( 3\times 2 \) matrix which is decomposed in terms of a -\( 3\times 3 \) matrix \( \boldsymbol{U} \), and a \( 2\times 2 \) matrix \( \boldsymbol{V} \). It is easy to see -that \( \boldsymbol{U} \) and \( \boldsymbol{V} \) are orthogonal (how?). +by taking the derivatives with respect to \( \boldsymbol{\beta} \) we obtain then +a slightly modified matrix inversion problem which for finite values +of \( \lambda \) does not suffer from singularity problems. We obtain + +$$ +\boldsymbol{\beta}^{\mathrm{Ridge}} = \left(\boldsymbol{X}^T\boldsymbol{X}+\lambda\boldsymbol{I}\right)^{-1}\boldsymbol{X}^T\boldsymbol{y}, +$$
-And the SVD -decomposition (singular values) gives eigenvalues -\( \sigma_i\geq\sigma_{i+1} \) for all \( i \) and for dimensions larger than \( i=2 \), the -eigenvalues (singular values) are zero. +with \( \boldsymbol{I} \) being a \( p\times p \) identity matrix with the constraint that + +$$ +\sum_{i=0}^{p-1} \beta_i^2 \leq t, +$$
-In the general case, where our design matrix \( \boldsymbol{X} \) has dimension -\( n\times p \), the matrix is thus decomposed into an \( n\times n \) -orthogonal matrix \( \boldsymbol{U} \), a \( p\times p \) orthogonal matrix \( \boldsymbol{V} \) -and a diagonal matrix \( \boldsymbol{\Sigma} \) with \( r=\mathrm{min}(n,p) \) -singular values \( \sigma_i\lg 0 \) on the main diagonal and zeros filling -the rest of the matrix. There are at most \( p \) singular values -assuming that \( n > p \). In our regression examples for the nuclear -masses and the equation of state this is indeed the case, while for -the Ising model we have \( p > n \). These are often cases that lead to -near singular or singular matrices. +with \( t \) a finite positive number.
-The columns of \( \boldsymbol{U} \) are called the left singular vectors while the columns of \( \boldsymbol{V} \) are the right singular vectors. +We see that Ridge regression is nothing but the standard +OLS with a modified diagonal term added to \( \boldsymbol{X}^T\boldsymbol{X} \). The +consequences, in particular for our discussion of the bias-variance +are rather interesting. + +
+Furthermore, if we use the result above in terms of the SVD decomposition (our analysis was done for the OLS method), we had +$$ +(\boldsymbol{X}\boldsymbol{X}^T)\boldsymbol{U} = \boldsymbol{U}\boldsymbol{D}. +$$ + +
+We can analyse the OLS solutions in terms of the eigenvectors (the columns) of the right singular value matrix \( \boldsymbol{U} \) as +$$ +\boldsymbol{X}\boldsymbol{\beta} = \boldsymbol{X}\left(\boldsymbol{V}\boldsymbol{D}\boldsymbol{V}^T \right)^{-1}\boldsymbol{X}^T\boldsymbol{y}=\boldsymbol{U\Sigma V^T}\left(\boldsymbol{V}\boldsymbol{D}\boldsymbol{V}^T \right)^{-1}(\boldsymbol{U\Sigma V^T})^T\boldsymbol{y}=\boldsymbol{U}\boldsymbol{U}^T\boldsymbol{y} +$$ + +
+For Ridge regression this becomes + +$$ +\boldsymbol{X}\boldsymbol{\beta}^{\mathrm{Ridge}} = \boldsymbol{U\Sigma V^T}\left(\boldsymbol{V}\boldsymbol{D}\boldsymbol{V}^T+\lambda\boldsymbol{I} \right)^{-1}(\boldsymbol{U\Sigma V^T})^T\boldsymbol{y}=\sum_{j=0}^{p-1}\boldsymbol{u}_j\boldsymbol{u}_j^T\frac{\sigma_j^2}{\sigma_j^2+\lambda}\boldsymbol{y}, +$$ + +
+with the vectors \( \boldsymbol{u}_j \) being the columns of \( \boldsymbol{U} \).
diff --git a/doc/pub/Regression/html/._Regression-bs040.html b/doc/pub/Regression/html/._Regression-bs040.html index d1ac113a7..7fd678478 100644 --- a/doc/pub/Regression/html/._Regression-bs040.html +++ b/doc/pub/Regression/html/._Regression-bs040.html @@ -111,112 +111,112 @@ Automatically generated HTML file from DocOnce source ('The Boston housing data example', 2, None, '___sec27'), ('Housing data, the code', 2, None, '___sec28'), ('The singular value decomposition', 2, None, '___sec29'), - ('The Ising model', 2, None, '___sec30'), - ('Reformulating the problem to suit regression', - 2, - None, - '___sec31'), - ('Linear regression', 2, None, '___sec32'), - ('Singular Value decomposition', 2, None, '___sec33'), - ('Linear Regression Problems', 2, None, '___sec34'), - ('Fixing the singularity', 2, None, '___sec35'), - ('Basic math of the SVD', 2, None, '___sec36'), - ('The SVD, a Fantastic Algorithm', 2, None, '___sec37'), - ('Another Example', 2, None, '___sec38'), - ('Economy-size SVD', 2, None, '___sec39'), - ('Mathematical Properties', 2, None, '___sec40'), - ('Ridge and LASSO Regression', 2, None, '___sec41'), - ('More on Ridge Regression', 2, None, '___sec42'), - ('Interpreting the Ridge results', 2, None, '___sec43'), - ('More interpretations', 2, None, '___sec44'), - ('Where are we going?', 2, None, '___sec45'), - ('Resampling methods', 2, None, '___sec46'), + ('Linear Regression Problems', 2, None, '___sec30'), + ('Fixing the singularity', 2, None, '___sec31'), + ('Basic math of the SVD', 2, None, '___sec32'), + ('The SVD, a Fantastic Algorithm', 2, None, '___sec33'), + ('Another Example', 2, None, '___sec34'), + ('Economy-size SVD', 2, None, '___sec35'), + ('Mathematical Properties', 2, None, '___sec36'), + ('Ridge and LASSO Regression', 2, None, '___sec37'), + ('More on Ridge Regression', 2, None, '___sec38'), + ('Interpreting the Ridge results', 2, None, '___sec39'), + ('More interpretations', 2, None, '___sec40'), + ('Where are we going?', 2, None, '___sec41'), + ('Resampling methods', 2, None, '___sec42'), ('Resampling approaches can be computationally expensive', 2, None, - '___sec47'), - ('Why resampling methods ?', 2, None, '___sec48'), - ('Statistical analysis', 2, None, '___sec49'), - ('Statistics', 2, None, '___sec50'), - ('Statistics, moments', 2, None, '___sec51'), - ('Statistics, central moments', 2, None, '___sec52'), - ('Statistics, covariance', 2, None, '___sec53'), - ('Statistics, more covariance', 2, None, '___sec54'), - ('Covariance example', 2, None, '___sec55'), - ('Covariance in numpy', 2, None, '___sec56'), - ('Statistics, independent variables', 2, None, '___sec57'), - ('Statistics, more variance', 2, None, '___sec58'), - ('Statistics and stochastic processes', 2, None, '___sec59'), - ('Statistics and sample variables', 2, None, '___sec60'), + '___sec43'), + ('Why resampling methods ?', 2, None, '___sec44'), + ('Statistical analysis', 2, None, '___sec45'), + ('Statistics', 2, None, '___sec46'), + ('Statistics, moments', 2, None, '___sec47'), + ('Statistics, central moments', 2, None, '___sec48'), + ('Statistics, covariance', 2, None, '___sec49'), + ('Statistics, more covariance', 2, None, '___sec50'), + ('Covariance example', 2, None, '___sec51'), + ('Covariance in numpy', 2, None, '___sec52'), + ('Statistics, independent variables', 2, None, '___sec53'), + ('Statistics, more variance', 2, None, '___sec54'), + ('Statistics and stochastic processes', 2, None, '___sec55'), + ('Statistics and sample variables', 2, None, '___sec56'), ('Statistics, sample variance and covariance', 2, None, - '___sec61'), - ('Statistics, law of large numbers', 2, None, '___sec62'), - ('Statistics, more on sample error', 2, None, '___sec63'), - ('Statistics', 2, None, '___sec64'), - ('Statistics, central limit theorem', 2, None, '___sec65'), - ('Statistics, more technicalities', 2, None, '___sec66'), - ('Statistics', 2, None, '___sec67'), - ('Statistics and sample variance', 2, None, '___sec68'), - ('Statistics, uncorrelated results', 2, None, '___sec69'), - ('Statistics, computations', 2, None, '___sec70'), + '___sec57'), + ('Statistics, law of large numbers', 2, None, '___sec58'), + ('Statistics, more on sample error', 2, None, '___sec59'), + ('Statistics', 2, None, '___sec60'), + ('Statistics, central limit theorem', 2, None, '___sec61'), + ('Statistics, more technicalities', 2, None, '___sec62'), + ('Statistics', 2, None, '___sec63'), + ('Statistics and sample variance', 2, None, '___sec64'), + ('Statistics, uncorrelated results', 2, None, '___sec65'), + ('Statistics, computations', 2, None, '___sec66'), ('Statistics, more on computations of errors', 2, None, - '___sec71'), - ('Statistics, wrapping up 1', 2, None, '___sec72'), - ('Statistics, final expression', 2, None, '___sec73'), + '___sec67'), + ('Statistics, wrapping up 1', 2, None, '___sec68'), + ('Statistics, final expression', 2, None, '___sec69'), ('Statistics, effective number of correlations', 2, None, - '___sec74'), + '___sec70'), ('Linking the regression analysis with a statistical ' 'interpretation', 2, None, - '___sec75'), - ('Assumptions made', 2, None, '___sec76'), - ('Expectation value and variance', 2, None, '___sec77'), + '___sec71'), + ('Assumptions made', 2, None, '___sec72'), + ('Expectation value and variance', 2, None, '___sec73'), ('Expectation value and variance for $\\boldsymbol{\\beta}$', 2, None, - '___sec78'), - ('Cross-validation', 2, None, '___sec79'), - ('Computationally expensive', 2, None, '___sec80'), - ('Various steps in cross-validation', 2, None, '___sec81'), + '___sec74'), + ('Cross-validation', 2, None, '___sec75'), + ('Computationally expensive', 2, None, '___sec76'), + ('Various steps in cross-validation', 2, None, '___sec77'), ('How to set up the cross-validation for Ridge and/or Lasso', 2, None, - '___sec82'), + '___sec78'), ('Resampling methods: Jackknife and Bootstrap', 2, None, - '___sec83'), - ('Resampling methods: Jackknife', 2, None, '___sec84'), - ('Jackknife code example', 2, None, '___sec85'), - ('Resampling methods: Bootstrap', 2, None, '___sec86'), - ('Resampling methods: Bootstrap background', 2, None, '___sec87'), + '___sec79'), + ('Resampling methods: Jackknife', 2, None, '___sec80'), + ('Jackknife code example', 2, None, '___sec81'), + ('Resampling methods: Bootstrap', 2, None, '___sec82'), + ('Resampling methods: Bootstrap background', 2, None, '___sec83'), ('Resampling methods: More Bootstrap background', 2, None, - '___sec88'), - ('Resampling methods: Bootstrap approach', 2, None, '___sec89'), - ('Resampling methods: Bootstrap steps', 2, None, '___sec90'), - ('Code example for the Bootstrap method', 2, None, '___sec91'), + '___sec84'), + ('Resampling methods: Bootstrap approach', 2, None, '___sec85'), + ('Resampling methods: Bootstrap steps', 2, None, '___sec86'), + ('Code example for the Bootstrap method', 2, None, '___sec87'), ('Code Example for Cross-validation and $k$-fold ' 'Cross-validation', 2, None, - '___sec92'), - ('The bias-variance tradeoff', 2, None, '___sec93'), - ('Example code for Bias-Variance tradeoff', 2, None, '___sec94'), - ('Understanding what happens', 2, None, '___sec95'), - ('Summing up', 2, None, '___sec96'), + '___sec88'), + ('The bias-variance tradeoff', 2, None, '___sec89'), + ('Example code for Bias-Variance tradeoff', 2, None, '___sec90'), + ('Understanding what happens', 2, None, '___sec91'), + ('Summing up', 2, None, '___sec92'), ("Another Example rom Scikit-Learn's Repository", 2, None, - '___sec97'), + '___sec93'), + ('The Ising model', 2, None, '___sec94'), + ('Reformulating the problem to suit regression', + 2, + None, + '___sec95'), + ('Linear regression', 2, None, '___sec96'), + ('Singular Value decomposition', 2, None, '___sec97'), ('The one-dimensional Ising model', 2, None, '___sec98'), ('Ridge regression', 2, None, '___sec99'), ('LASSO regression', 2, None, '___sec100'), @@ -295,74 +295,74 @@ MathJax.Hub.Config({
-If we assume that \( n > p \), then our matrix \( \boldsymbol{U} \) has dimension \( n -\times n \). The last \( n-p \) columns of \( \boldsymbol{U} \) become however -irrelevant in our calculations since they are multiplied with the -zeros in \( \boldsymbol{\Sigma} \). +Since \( \lambda \geq 0 \), it means that compared to OLS, we have + +$$ +\frac{\sigma_j^2}{\sigma_j^2+\lambda} \leq 1. +$$
-The economy-size decomposition removes extra rows or columns of zeros -from the diagonal matrix of singular values, \( \boldsymbol{\Sigma} \), along with the columns -in either \( \boldsymbol{U} \) or \( \boldsymbol{V} \) that multiply those zeros in the expression. -Removing these zeros and columns can improve execution time -and reduce storage requirements without compromising the accuracy of -the decomposition. +Ridge regression finds the coordinates of \( \boldsymbol{y} \) with respect to the +orthonormal basis \( \boldsymbol{U} \), it then shrinks the coordinates by +\( \frac{\sigma_j^2}{\sigma_j^2+\lambda} \). Recall that the SVD has +eigenvalues ordered in a descending way, that is \( \sigma_i \geq +\sigma_{i+1} \).
-If \( n > p \), we keep only the first \( p \) columns of \( \boldsymbol{U} \) and \( \boldsymbol{\Sigma} \) has dimension \( p\times p \). -If \( p > n \), then only the first \( n \) columns of \( \boldsymbol{V} \) are computed and \( \boldsymbol{\Sigma} \) has dimension \( n\times n \). -The \( n=p \) case is obvious, we retain the full SVD. -In general the economy-size SVD leads to less FLOPS and still conserving the desired accuracy. +For small eigenvalues \( \sigma_i \) it means that their contributions become less important, a fact which can be used to reduce the number of degrees of freedom. +Actually, calculating the variance of \( \boldsymbol{X}\boldsymbol{v}_j \) shows that this quantity is equal to \( \sigma_j^2/n \). +With a parameter \( \lambda \) we can thus shrink the role of specific parameters.
diff --git a/doc/pub/Regression/html/._Regression-bs041.html b/doc/pub/Regression/html/._Regression-bs041.html index 2f58af5f0..ab77a840d 100644 --- a/doc/pub/Regression/html/._Regression-bs041.html +++ b/doc/pub/Regression/html/._Regression-bs041.html @@ -111,112 +111,112 @@ Automatically generated HTML file from DocOnce source ('The Boston housing data example', 2, None, '___sec27'), ('Housing data, the code', 2, None, '___sec28'), ('The singular value decomposition', 2, None, '___sec29'), - ('The Ising model', 2, None, '___sec30'), - ('Reformulating the problem to suit regression', - 2, - None, - '___sec31'), - ('Linear regression', 2, None, '___sec32'), - ('Singular Value decomposition', 2, None, '___sec33'), - ('Linear Regression Problems', 2, None, '___sec34'), - ('Fixing the singularity', 2, None, '___sec35'), - ('Basic math of the SVD', 2, None, '___sec36'), - ('The SVD, a Fantastic Algorithm', 2, None, '___sec37'), - ('Another Example', 2, None, '___sec38'), - ('Economy-size SVD', 2, None, '___sec39'), - ('Mathematical Properties', 2, None, '___sec40'), - ('Ridge and LASSO Regression', 2, None, '___sec41'), - ('More on Ridge Regression', 2, None, '___sec42'), - ('Interpreting the Ridge results', 2, None, '___sec43'), - ('More interpretations', 2, None, '___sec44'), - ('Where are we going?', 2, None, '___sec45'), - ('Resampling methods', 2, None, '___sec46'), + ('Linear Regression Problems', 2, None, '___sec30'), + ('Fixing the singularity', 2, None, '___sec31'), + ('Basic math of the SVD', 2, None, '___sec32'), + ('The SVD, a Fantastic Algorithm', 2, None, '___sec33'), + ('Another Example', 2, None, '___sec34'), + ('Economy-size SVD', 2, None, '___sec35'), + ('Mathematical Properties', 2, None, '___sec36'), + ('Ridge and LASSO Regression', 2, None, '___sec37'), + ('More on Ridge Regression', 2, None, '___sec38'), + ('Interpreting the Ridge results', 2, None, '___sec39'), + ('More interpretations', 2, None, '___sec40'), + ('Where are we going?', 2, None, '___sec41'), + ('Resampling methods', 2, None, '___sec42'), ('Resampling approaches can be computationally expensive', 2, None, - '___sec47'), - ('Why resampling methods ?', 2, None, '___sec48'), - ('Statistical analysis', 2, None, '___sec49'), - ('Statistics', 2, None, '___sec50'), - ('Statistics, moments', 2, None, '___sec51'), - ('Statistics, central moments', 2, None, '___sec52'), - ('Statistics, covariance', 2, None, '___sec53'), - ('Statistics, more covariance', 2, None, '___sec54'), - ('Covariance example', 2, None, '___sec55'), - ('Covariance in numpy', 2, None, '___sec56'), - ('Statistics, independent variables', 2, None, '___sec57'), - ('Statistics, more variance', 2, None, '___sec58'), - ('Statistics and stochastic processes', 2, None, '___sec59'), - ('Statistics and sample variables', 2, None, '___sec60'), + '___sec43'), + ('Why resampling methods ?', 2, None, '___sec44'), + ('Statistical analysis', 2, None, '___sec45'), + ('Statistics', 2, None, '___sec46'), + ('Statistics, moments', 2, None, '___sec47'), + ('Statistics, central moments', 2, None, '___sec48'), + ('Statistics, covariance', 2, None, '___sec49'), + ('Statistics, more covariance', 2, None, '___sec50'), + ('Covariance example', 2, None, '___sec51'), + ('Covariance in numpy', 2, None, '___sec52'), + ('Statistics, independent variables', 2, None, '___sec53'), + ('Statistics, more variance', 2, None, '___sec54'), + ('Statistics and stochastic processes', 2, None, '___sec55'), + ('Statistics and sample variables', 2, None, '___sec56'), ('Statistics, sample variance and covariance', 2, None, - '___sec61'), - ('Statistics, law of large numbers', 2, None, '___sec62'), - ('Statistics, more on sample error', 2, None, '___sec63'), - ('Statistics', 2, None, '___sec64'), - ('Statistics, central limit theorem', 2, None, '___sec65'), - ('Statistics, more technicalities', 2, None, '___sec66'), - ('Statistics', 2, None, '___sec67'), - ('Statistics and sample variance', 2, None, '___sec68'), - ('Statistics, uncorrelated results', 2, None, '___sec69'), - ('Statistics, computations', 2, None, '___sec70'), + '___sec57'), + ('Statistics, law of large numbers', 2, None, '___sec58'), + ('Statistics, more on sample error', 2, None, '___sec59'), + ('Statistics', 2, None, '___sec60'), + ('Statistics, central limit theorem', 2, None, '___sec61'), + ('Statistics, more technicalities', 2, None, '___sec62'), + ('Statistics', 2, None, '___sec63'), + ('Statistics and sample variance', 2, None, '___sec64'), + ('Statistics, uncorrelated results', 2, None, '___sec65'), + ('Statistics, computations', 2, None, '___sec66'), ('Statistics, more on computations of errors', 2, None, - '___sec71'), - ('Statistics, wrapping up 1', 2, None, '___sec72'), - ('Statistics, final expression', 2, None, '___sec73'), + '___sec67'), + ('Statistics, wrapping up 1', 2, None, '___sec68'), + ('Statistics, final expression', 2, None, '___sec69'), ('Statistics, effective number of correlations', 2, None, - '___sec74'), + '___sec70'), ('Linking the regression analysis with a statistical ' 'interpretation', 2, None, - '___sec75'), - ('Assumptions made', 2, None, '___sec76'), - ('Expectation value and variance', 2, None, '___sec77'), + '___sec71'), + ('Assumptions made', 2, None, '___sec72'), + ('Expectation value and variance', 2, None, '___sec73'), ('Expectation value and variance for $\\boldsymbol{\\beta}$', 2, None, - '___sec78'), - ('Cross-validation', 2, None, '___sec79'), - ('Computationally expensive', 2, None, '___sec80'), - ('Various steps in cross-validation', 2, None, '___sec81'), + '___sec74'), + ('Cross-validation', 2, None, '___sec75'), + ('Computationally expensive', 2, None, '___sec76'), + ('Various steps in cross-validation', 2, None, '___sec77'), ('How to set up the cross-validation for Ridge and/or Lasso', 2, None, - '___sec82'), + '___sec78'), ('Resampling methods: Jackknife and Bootstrap', 2, None, - '___sec83'), - ('Resampling methods: Jackknife', 2, None, '___sec84'), - ('Jackknife code example', 2, None, '___sec85'), - ('Resampling methods: Bootstrap', 2, None, '___sec86'), - ('Resampling methods: Bootstrap background', 2, None, '___sec87'), + '___sec79'), + ('Resampling methods: Jackknife', 2, None, '___sec80'), + ('Jackknife code example', 2, None, '___sec81'), + ('Resampling methods: Bootstrap', 2, None, '___sec82'), + ('Resampling methods: Bootstrap background', 2, None, '___sec83'), ('Resampling methods: More Bootstrap background', 2, None, - '___sec88'), - ('Resampling methods: Bootstrap approach', 2, None, '___sec89'), - ('Resampling methods: Bootstrap steps', 2, None, '___sec90'), - ('Code example for the Bootstrap method', 2, None, '___sec91'), + '___sec84'), + ('Resampling methods: Bootstrap approach', 2, None, '___sec85'), + ('Resampling methods: Bootstrap steps', 2, None, '___sec86'), + ('Code example for the Bootstrap method', 2, None, '___sec87'), ('Code Example for Cross-validation and $k$-fold ' 'Cross-validation', 2, None, - '___sec92'), - ('The bias-variance tradeoff', 2, None, '___sec93'), - ('Example code for Bias-Variance tradeoff', 2, None, '___sec94'), - ('Understanding what happens', 2, None, '___sec95'), - ('Summing up', 2, None, '___sec96'), + '___sec88'), + ('The bias-variance tradeoff', 2, None, '___sec89'), + ('Example code for Bias-Variance tradeoff', 2, None, '___sec90'), + ('Understanding what happens', 2, None, '___sec91'), + ('Summing up', 2, None, '___sec92'), ("Another Example rom Scikit-Learn's Repository", 2, None, - '___sec97'), + '___sec93'), + ('The Ising model', 2, None, '___sec94'), + ('Reformulating the problem to suit regression', + 2, + None, + '___sec95'), + ('Linear regression', 2, None, '___sec96'), + ('Singular Value decomposition', 2, None, '___sec97'), ('The one-dimensional Ising model', 2, None, '___sec98'), ('Ridge regression', 2, None, '___sec99'), ('LASSO regression', 2, None, '___sec100'), @@ -295,74 +295,74 @@ MathJax.Hub.Config({
-There are several interesting mathematical properties which will be -relevant when we are going to discuss the differences between say -ordinary least squares (OLS) and Ridge regression. +For the sake of simplicity, let us assume that the design matrix is orthonormal, that is -
-We have from OLS that the parameters of the linear approximation are given by $$ -\boldsymbol{\tilde{y}} = \boldsymbol{X}\boldsymbol{\beta} = \boldsymbol{X}\left(\boldsymbol{X}^T\boldsymbol{X}\right)^{-1}\boldsymbol{X}^T\boldsymbol{y}. +\boldsymbol{X}^T\boldsymbol{X}=(\boldsymbol{X}^T\boldsymbol{X})^{-1} =\boldsymbol{I}. $$
-The matrix to invert can be rewritten in terms of our SVD decomposition as - +In this case the standard OLS results in $$ -\boldsymbol{X}^T\boldsymbol{X} = \boldsymbol{V}\boldsymbol{\Sigma}^T\boldsymbol{U}^T\boldsymbol{U}\boldsymbol{\Sigma}\boldsymbol{V}^T. +\boldsymbol{\beta}^{\mathrm{OLS}} = \boldsymbol{X}^T\boldsymbol{y}=\sum_{i=0}^{p-1}\boldsymbol{u}_j\boldsymbol{u}_j^T\boldsymbol{y}, $$ -Using the orthogonality properties of \( \boldsymbol{U} \) we have - -$$ -\boldsymbol{X}^T\boldsymbol{X} = \boldsymbol{V}\boldsymbol{\Sigma}^T\boldsymbol{\Sigma}\boldsymbol{V}^T = \boldsymbol{V}\boldsymbol{D}\boldsymbol{V}^T, -$$ - -with \( \boldsymbol{D} \) being a diagonal matrix with values along the diagonal given by the singular values squared. -
-This means that -$$ -(\boldsymbol{X}^T\boldsymbol{X})\boldsymbol{V} = \boldsymbol{V}\boldsymbol{D}, -$$ +and -that is the eigenvectors of \( (\boldsymbol{X}^T\boldsymbol{X}) \) are given by the columns of the right singular matrix of \( \boldsymbol{X} \) and the eigenvalues are the squared singular values. It is easy to show (show this) that $$ -(\boldsymbol{X}\boldsymbol{X}^T)\boldsymbol{U} = \boldsymbol{U}\boldsymbol{D}, +\boldsymbol{\beta}^{\mathrm{Ridge}} = \left(\boldsymbol{I}+\lambda\boldsymbol{I}\right)^{-1}\boldsymbol{X}^T\boldsymbol{y}=\left(1+\lambda\right)^{-1}\boldsymbol{\beta}^{\mathrm{OLS}}, $$ -that is, the eigenvectors of \( (\boldsymbol{X}\boldsymbol{X})^T \) are the columns of the left singular matrix and the eigenvalues are the same. -
-Going back to our OLS equation we have -$$ -\boldsymbol{X}\boldsymbol{\beta} = \boldsymbol{X}\left(\boldsymbol{V}\boldsymbol{D}\boldsymbol{V}^T \right)^{-1}\boldsymbol{X}^T\boldsymbol{y}=\boldsymbol{U\Sigma V^T}\left(\boldsymbol{V}\boldsymbol{D}\boldsymbol{V}^T \right)^{-1}(\boldsymbol{U\Sigma V^T})^T\boldsymbol{y}=\boldsymbol{U}\boldsymbol{U}^T\boldsymbol{y}. -$$ +that is the Ridge estimator scales the OLS estimator by the inverse of a factor \( 1+\lambda \), and +the Ridge estimator converges to zero when the hyperparameter goes to +infinity. -We will come back to this expression when we discuss Ridge regression. +
+We will come back to more interpreations after we have gone through some of the statistical analysis part. + +
+For more discussions of Ridge and Lasso regression, Wessel van Wieringen's article is highly recommended. +Similarly, Mehta et al's article is also recommended.
diff --git a/doc/pub/Regression/html/._Regression-bs042.html b/doc/pub/Regression/html/._Regression-bs042.html index 49f2db109..8d2b5faee 100644 --- a/doc/pub/Regression/html/._Regression-bs042.html +++ b/doc/pub/Regression/html/._Regression-bs042.html @@ -111,112 +111,112 @@ Automatically generated HTML file from DocOnce source ('The Boston housing data example', 2, None, '___sec27'), ('Housing data, the code', 2, None, '___sec28'), ('The singular value decomposition', 2, None, '___sec29'), - ('The Ising model', 2, None, '___sec30'), - ('Reformulating the problem to suit regression', - 2, - None, - '___sec31'), - ('Linear regression', 2, None, '___sec32'), - ('Singular Value decomposition', 2, None, '___sec33'), - ('Linear Regression Problems', 2, None, '___sec34'), - ('Fixing the singularity', 2, None, '___sec35'), - ('Basic math of the SVD', 2, None, '___sec36'), - ('The SVD, a Fantastic Algorithm', 2, None, '___sec37'), - ('Another Example', 2, None, '___sec38'), - ('Economy-size SVD', 2, None, '___sec39'), - ('Mathematical Properties', 2, None, '___sec40'), - ('Ridge and LASSO Regression', 2, None, '___sec41'), - ('More on Ridge Regression', 2, None, '___sec42'), - ('Interpreting the Ridge results', 2, None, '___sec43'), - ('More interpretations', 2, None, '___sec44'), - ('Where are we going?', 2, None, '___sec45'), - ('Resampling methods', 2, None, '___sec46'), + ('Linear Regression Problems', 2, None, '___sec30'), + ('Fixing the singularity', 2, None, '___sec31'), + ('Basic math of the SVD', 2, None, '___sec32'), + ('The SVD, a Fantastic Algorithm', 2, None, '___sec33'), + ('Another Example', 2, None, '___sec34'), + ('Economy-size SVD', 2, None, '___sec35'), + ('Mathematical Properties', 2, None, '___sec36'), + ('Ridge and LASSO Regression', 2, None, '___sec37'), + ('More on Ridge Regression', 2, None, '___sec38'), + ('Interpreting the Ridge results', 2, None, '___sec39'), + ('More interpretations', 2, None, '___sec40'), + ('Where are we going?', 2, None, '___sec41'), + ('Resampling methods', 2, None, '___sec42'), ('Resampling approaches can be computationally expensive', 2, None, - '___sec47'), - ('Why resampling methods ?', 2, None, '___sec48'), - ('Statistical analysis', 2, None, '___sec49'), - ('Statistics', 2, None, '___sec50'), - ('Statistics, moments', 2, None, '___sec51'), - ('Statistics, central moments', 2, None, '___sec52'), - ('Statistics, covariance', 2, None, '___sec53'), - ('Statistics, more covariance', 2, None, '___sec54'), - ('Covariance example', 2, None, '___sec55'), - ('Covariance in numpy', 2, None, '___sec56'), - ('Statistics, independent variables', 2, None, '___sec57'), - ('Statistics, more variance', 2, None, '___sec58'), - ('Statistics and stochastic processes', 2, None, '___sec59'), - ('Statistics and sample variables', 2, None, '___sec60'), + '___sec43'), + ('Why resampling methods ?', 2, None, '___sec44'), + ('Statistical analysis', 2, None, '___sec45'), + ('Statistics', 2, None, '___sec46'), + ('Statistics, moments', 2, None, '___sec47'), + ('Statistics, central moments', 2, None, '___sec48'), + ('Statistics, covariance', 2, None, '___sec49'), + ('Statistics, more covariance', 2, None, '___sec50'), + ('Covariance example', 2, None, '___sec51'), + ('Covariance in numpy', 2, None, '___sec52'), + ('Statistics, independent variables', 2, None, '___sec53'), + ('Statistics, more variance', 2, None, '___sec54'), + ('Statistics and stochastic processes', 2, None, '___sec55'), + ('Statistics and sample variables', 2, None, '___sec56'), ('Statistics, sample variance and covariance', 2, None, - '___sec61'), - ('Statistics, law of large numbers', 2, None, '___sec62'), - ('Statistics, more on sample error', 2, None, '___sec63'), - ('Statistics', 2, None, '___sec64'), - ('Statistics, central limit theorem', 2, None, '___sec65'), - ('Statistics, more technicalities', 2, None, '___sec66'), - ('Statistics', 2, None, '___sec67'), - ('Statistics and sample variance', 2, None, '___sec68'), - ('Statistics, uncorrelated results', 2, None, '___sec69'), - ('Statistics, computations', 2, None, '___sec70'), + '___sec57'), + ('Statistics, law of large numbers', 2, None, '___sec58'), + ('Statistics, more on sample error', 2, None, '___sec59'), + ('Statistics', 2, None, '___sec60'), + ('Statistics, central limit theorem', 2, None, '___sec61'), + ('Statistics, more technicalities', 2, None, '___sec62'), + ('Statistics', 2, None, '___sec63'), + ('Statistics and sample variance', 2, None, '___sec64'), + ('Statistics, uncorrelated results', 2, None, '___sec65'), + ('Statistics, computations', 2, None, '___sec66'), ('Statistics, more on computations of errors', 2, None, - '___sec71'), - ('Statistics, wrapping up 1', 2, None, '___sec72'), - ('Statistics, final expression', 2, None, '___sec73'), + '___sec67'), + ('Statistics, wrapping up 1', 2, None, '___sec68'), + ('Statistics, final expression', 2, None, '___sec69'), ('Statistics, effective number of correlations', 2, None, - '___sec74'), + '___sec70'), ('Linking the regression analysis with a statistical ' 'interpretation', 2, None, - '___sec75'), - ('Assumptions made', 2, None, '___sec76'), - ('Expectation value and variance', 2, None, '___sec77'), + '___sec71'), + ('Assumptions made', 2, None, '___sec72'), + ('Expectation value and variance', 2, None, '___sec73'), ('Expectation value and variance for $\\boldsymbol{\\beta}$', 2, None, - '___sec78'), - ('Cross-validation', 2, None, '___sec79'), - ('Computationally expensive', 2, None, '___sec80'), - ('Various steps in cross-validation', 2, None, '___sec81'), + '___sec74'), + ('Cross-validation', 2, None, '___sec75'), + ('Computationally expensive', 2, None, '___sec76'), + ('Various steps in cross-validation', 2, None, '___sec77'), ('How to set up the cross-validation for Ridge and/or Lasso', 2, None, - '___sec82'), + '___sec78'), ('Resampling methods: Jackknife and Bootstrap', 2, None, - '___sec83'), - ('Resampling methods: Jackknife', 2, None, '___sec84'), - ('Jackknife code example', 2, None, '___sec85'), - ('Resampling methods: Bootstrap', 2, None, '___sec86'), - ('Resampling methods: Bootstrap background', 2, None, '___sec87'), + '___sec79'), + ('Resampling methods: Jackknife', 2, None, '___sec80'), + ('Jackknife code example', 2, None, '___sec81'), + ('Resampling methods: Bootstrap', 2, None, '___sec82'), + ('Resampling methods: Bootstrap background', 2, None, '___sec83'), ('Resampling methods: More Bootstrap background', 2, None, - '___sec88'), - ('Resampling methods: Bootstrap approach', 2, None, '___sec89'), - ('Resampling methods: Bootstrap steps', 2, None, '___sec90'), - ('Code example for the Bootstrap method', 2, None, '___sec91'), + '___sec84'), + ('Resampling methods: Bootstrap approach', 2, None, '___sec85'), + ('Resampling methods: Bootstrap steps', 2, None, '___sec86'), + ('Code example for the Bootstrap method', 2, None, '___sec87'), ('Code Example for Cross-validation and $k$-fold ' 'Cross-validation', 2, None, - '___sec92'), - ('The bias-variance tradeoff', 2, None, '___sec93'), - ('Example code for Bias-Variance tradeoff', 2, None, '___sec94'), - ('Understanding what happens', 2, None, '___sec95'), - ('Summing up', 2, None, '___sec96'), + '___sec88'), + ('The bias-variance tradeoff', 2, None, '___sec89'), + ('Example code for Bias-Variance tradeoff', 2, None, '___sec90'), + ('Understanding what happens', 2, None, '___sec91'), + ('Summing up', 2, None, '___sec92'), ("Another Example rom Scikit-Learn's Repository", 2, None, - '___sec97'), + '___sec93'), + ('The Ising model', 2, None, '___sec94'), + ('Reformulating the problem to suit regression', + 2, + None, + '___sec95'), + ('Linear regression', 2, None, '___sec96'), + ('Singular Value decomposition', 2, None, '___sec97'), ('The one-dimensional Ising model', 2, None, '___sec98'), ('Ridge regression', 2, None, '___sec99'), ('LASSO regression', 2, None, '___sec100'), @@ -295,74 +295,74 @@ MathJax.Hub.Config({
-Let us remind ourselves about the expression for the standard Mean Squared Error (MSE) which we used to define our cost function and the equations for the ordinary least squares (OLS) method, that is -our optimization problem is -$$ -{\displaystyle \min_{\boldsymbol{\beta}\in {\mathbb{R}}^{p}}}\frac{1}{n}\left\{\left(\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\right)^T\left(\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\right)\right\}. -$$ +Before we proceed, we need to rethink what we have been doing. In our +eager to fit the data, we have omitted several important elements in +our regression analysis. In what follows we will -or we can state it as -$$ -{\displaystyle \min_{\boldsymbol{\beta}\in -{\mathbb{R}}^{p}}}\frac{1}{n}\sum_{i=0}^{n-1}\left(y_i-\tilde{y}_i\right)^2=\frac{1}{n}\vert\vert \boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\vert\vert_2^2, -$$ +
-By minimizing the above equation with respect to the parameters -\( \boldsymbol{\beta} \) we could then obtain an analytical expression for the -parameters \( \boldsymbol{\beta} \). We can add a regularization parameter \( \lambda \) by -defining a new cost function to be optimized, that is - -$$ -{\displaystyle \min_{\boldsymbol{\beta}\in -{\mathbb{R}}^{p}}}\frac{1}{n}\vert\vert \boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\vert\vert_2^2+\lambda\vert\vert \boldsymbol{\beta}\vert\vert_2^2 -$$ - -
-which leads to the Ridge regression minimization problem where we -require that \( \vert\vert \boldsymbol{\beta}\vert\vert_2^2\le t \), where \( t \) is -a finite number larger than zero. By defining - -$$ -C(\boldsymbol{X},\boldsymbol{\beta})=\frac{1}{n}\vert\vert \boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\vert\vert_2^2+\lambda\vert\vert \boldsymbol{\beta}\vert\vert_1, -$$ - -
-we have a new optimization equation -$$ -{\displaystyle \min_{\boldsymbol{\beta}\in -{\mathbb{R}}^{p}}}\frac{1}{n}\vert\vert \boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\vert\vert_2^2+\lambda\vert\vert \boldsymbol{\beta}\vert\vert_1 -$$ - -which leads to Lasso regression. Lasso stands for least absolute shrinkage and selection operator. - -
-Here we have defined the norm-1 as -$$ -\vert\vert \boldsymbol{x}\vert\vert_1 = \sum_i \vert x_i\vert. -$$ +This will allow us to link the standard linear algebra methods we have discussed above to a statistical interpretation of the methods.
diff --git a/doc/pub/Regression/html/._Regression-bs043.html b/doc/pub/Regression/html/._Regression-bs043.html index 857579026..416da079f 100644 --- a/doc/pub/Regression/html/._Regression-bs043.html +++ b/doc/pub/Regression/html/._Regression-bs043.html @@ -111,112 +111,112 @@ Automatically generated HTML file from DocOnce source ('The Boston housing data example', 2, None, '___sec27'), ('Housing data, the code', 2, None, '___sec28'), ('The singular value decomposition', 2, None, '___sec29'), - ('The Ising model', 2, None, '___sec30'), - ('Reformulating the problem to suit regression', - 2, - None, - '___sec31'), - ('Linear regression', 2, None, '___sec32'), - ('Singular Value decomposition', 2, None, '___sec33'), - ('Linear Regression Problems', 2, None, '___sec34'), - ('Fixing the singularity', 2, None, '___sec35'), - ('Basic math of the SVD', 2, None, '___sec36'), - ('The SVD, a Fantastic Algorithm', 2, None, '___sec37'), - ('Another Example', 2, None, '___sec38'), - ('Economy-size SVD', 2, None, '___sec39'), - ('Mathematical Properties', 2, None, '___sec40'), - ('Ridge and LASSO Regression', 2, None, '___sec41'), - ('More on Ridge Regression', 2, None, '___sec42'), - ('Interpreting the Ridge results', 2, None, '___sec43'), - ('More interpretations', 2, None, '___sec44'), - ('Where are we going?', 2, None, '___sec45'), - ('Resampling methods', 2, None, '___sec46'), + ('Linear Regression Problems', 2, None, '___sec30'), + ('Fixing the singularity', 2, None, '___sec31'), + ('Basic math of the SVD', 2, None, '___sec32'), + ('The SVD, a Fantastic Algorithm', 2, None, '___sec33'), + ('Another Example', 2, None, '___sec34'), + ('Economy-size SVD', 2, None, '___sec35'), + ('Mathematical Properties', 2, None, '___sec36'), + ('Ridge and LASSO Regression', 2, None, '___sec37'), + ('More on Ridge Regression', 2, None, '___sec38'), + ('Interpreting the Ridge results', 2, None, '___sec39'), + ('More interpretations', 2, None, '___sec40'), + ('Where are we going?', 2, None, '___sec41'), + ('Resampling methods', 2, None, '___sec42'), ('Resampling approaches can be computationally expensive', 2, None, - '___sec47'), - ('Why resampling methods ?', 2, None, '___sec48'), - ('Statistical analysis', 2, None, '___sec49'), - ('Statistics', 2, None, '___sec50'), - ('Statistics, moments', 2, None, '___sec51'), - ('Statistics, central moments', 2, None, '___sec52'), - ('Statistics, covariance', 2, None, '___sec53'), - ('Statistics, more covariance', 2, None, '___sec54'), - ('Covariance example', 2, None, '___sec55'), - ('Covariance in numpy', 2, None, '___sec56'), - ('Statistics, independent variables', 2, None, '___sec57'), - ('Statistics, more variance', 2, None, '___sec58'), - ('Statistics and stochastic processes', 2, None, '___sec59'), - ('Statistics and sample variables', 2, None, '___sec60'), + '___sec43'), + ('Why resampling methods ?', 2, None, '___sec44'), + ('Statistical analysis', 2, None, '___sec45'), + ('Statistics', 2, None, '___sec46'), + ('Statistics, moments', 2, None, '___sec47'), + ('Statistics, central moments', 2, None, '___sec48'), + ('Statistics, covariance', 2, None, '___sec49'), + ('Statistics, more covariance', 2, None, '___sec50'), + ('Covariance example', 2, None, '___sec51'), + ('Covariance in numpy', 2, None, '___sec52'), + ('Statistics, independent variables', 2, None, '___sec53'), + ('Statistics, more variance', 2, None, '___sec54'), + ('Statistics and stochastic processes', 2, None, '___sec55'), + ('Statistics and sample variables', 2, None, '___sec56'), ('Statistics, sample variance and covariance', 2, None, - '___sec61'), - ('Statistics, law of large numbers', 2, None, '___sec62'), - ('Statistics, more on sample error', 2, None, '___sec63'), - ('Statistics', 2, None, '___sec64'), - ('Statistics, central limit theorem', 2, None, '___sec65'), - ('Statistics, more technicalities', 2, None, '___sec66'), - ('Statistics', 2, None, '___sec67'), - ('Statistics and sample variance', 2, None, '___sec68'), - ('Statistics, uncorrelated results', 2, None, '___sec69'), - ('Statistics, computations', 2, None, '___sec70'), + '___sec57'), + ('Statistics, law of large numbers', 2, None, '___sec58'), + ('Statistics, more on sample error', 2, None, '___sec59'), + ('Statistics', 2, None, '___sec60'), + ('Statistics, central limit theorem', 2, None, '___sec61'), + ('Statistics, more technicalities', 2, None, '___sec62'), + ('Statistics', 2, None, '___sec63'), + ('Statistics and sample variance', 2, None, '___sec64'), + ('Statistics, uncorrelated results', 2, None, '___sec65'), + ('Statistics, computations', 2, None, '___sec66'), ('Statistics, more on computations of errors', 2, None, - '___sec71'), - ('Statistics, wrapping up 1', 2, None, '___sec72'), - ('Statistics, final expression', 2, None, '___sec73'), + '___sec67'), + ('Statistics, wrapping up 1', 2, None, '___sec68'), + ('Statistics, final expression', 2, None, '___sec69'), ('Statistics, effective number of correlations', 2, None, - '___sec74'), + '___sec70'), ('Linking the regression analysis with a statistical ' 'interpretation', 2, None, - '___sec75'), - ('Assumptions made', 2, None, '___sec76'), - ('Expectation value and variance', 2, None, '___sec77'), + '___sec71'), + ('Assumptions made', 2, None, '___sec72'), + ('Expectation value and variance', 2, None, '___sec73'), ('Expectation value and variance for $\\boldsymbol{\\beta}$', 2, None, - '___sec78'), - ('Cross-validation', 2, None, '___sec79'), - ('Computationally expensive', 2, None, '___sec80'), - ('Various steps in cross-validation', 2, None, '___sec81'), + '___sec74'), + ('Cross-validation', 2, None, '___sec75'), + ('Computationally expensive', 2, None, '___sec76'), + ('Various steps in cross-validation', 2, None, '___sec77'), ('How to set up the cross-validation for Ridge and/or Lasso', 2, None, - '___sec82'), + '___sec78'), ('Resampling methods: Jackknife and Bootstrap', 2, None, - '___sec83'), - ('Resampling methods: Jackknife', 2, None, '___sec84'), - ('Jackknife code example', 2, None, '___sec85'), - ('Resampling methods: Bootstrap', 2, None, '___sec86'), - ('Resampling methods: Bootstrap background', 2, None, '___sec87'), + '___sec79'), + ('Resampling methods: Jackknife', 2, None, '___sec80'), + ('Jackknife code example', 2, None, '___sec81'), + ('Resampling methods: Bootstrap', 2, None, '___sec82'), + ('Resampling methods: Bootstrap background', 2, None, '___sec83'), ('Resampling methods: More Bootstrap background', 2, None, - '___sec88'), - ('Resampling methods: Bootstrap approach', 2, None, '___sec89'), - ('Resampling methods: Bootstrap steps', 2, None, '___sec90'), - ('Code example for the Bootstrap method', 2, None, '___sec91'), + '___sec84'), + ('Resampling methods: Bootstrap approach', 2, None, '___sec85'), + ('Resampling methods: Bootstrap steps', 2, None, '___sec86'), + ('Code example for the Bootstrap method', 2, None, '___sec87'), ('Code Example for Cross-validation and $k$-fold ' 'Cross-validation', 2, None, - '___sec92'), - ('The bias-variance tradeoff', 2, None, '___sec93'), - ('Example code for Bias-Variance tradeoff', 2, None, '___sec94'), - ('Understanding what happens', 2, None, '___sec95'), - ('Summing up', 2, None, '___sec96'), + '___sec88'), + ('The bias-variance tradeoff', 2, None, '___sec89'), + ('Example code for Bias-Variance tradeoff', 2, None, '___sec90'), + ('Understanding what happens', 2, None, '___sec91'), + ('Summing up', 2, None, '___sec92'), ("Another Example rom Scikit-Learn's Repository", 2, None, - '___sec97'), + '___sec93'), + ('The Ising model', 2, None, '___sec94'), + ('Reformulating the problem to suit regression', + 2, + None, + '___sec95'), + ('Linear regression', 2, None, '___sec96'), + ('Singular Value decomposition', 2, None, '___sec97'), ('The one-dimensional Ising model', 2, None, '___sec98'), ('Ridge regression', 2, None, '___sec99'), ('LASSO regression', 2, None, '___sec100'), @@ -295,74 +295,74 @@ MathJax.Hub.Config({
+Resampling methods are an indispensable tool in modern +statistics. They involve repeatedly drawing samples from a training +set and refitting a model of interest on each sample in order to +obtain additional information about the fitted model. For example, in +order to estimate the variability of a linear regression fit, we can +repeatedly draw different samples from the training data, fit a linear +regression to each new sample, and then examine the extent to which +the resulting fits differ. Such an approach may allow us to obtain +information that would not be available from fitting the model only +once using the original training sample. +
-Using the matrix-vector expression for Ridge regression, - -$$ -C(\boldsymbol{X},\boldsymbol{\beta})=\frac{1}{n}\left\{(\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta})^T(\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta})\right\}+\lambda\boldsymbol{\beta}^T\boldsymbol{\beta}, -$$ - -
-by taking the derivatives with respect to \( \boldsymbol{\beta} \) we obtain then -a slightly modified matrix inversion problem which for finite values -of \( \lambda \) does not suffer from singularity problems. We obtain - -$$ -\boldsymbol{\beta}^{\mathrm{Ridge}} = \left(\boldsymbol{X}^T\boldsymbol{X}+\lambda\boldsymbol{I}\right)^{-1}\boldsymbol{X}^T\boldsymbol{y}, -$$ - -
-with \( \boldsymbol{I} \) being a \( p\times p \) identity matrix with the constraint that - -$$ -\sum_{i=0}^{p-1} \beta_i^2 \leq t, -$$ - -
-with \( t \) a finite positive number. - -
-We see that Ridge regression is nothing but the standard -OLS with a modified diagonal term added to \( \boldsymbol{X}^T\boldsymbol{X} \). The -consequences, in particular for our discussion of the bias-variance -are rather interesting. - -
-Furthermore, if we use the result above in terms of the SVD decomposition (our analysis was done for the OLS method), we had -$$ -(\boldsymbol{X}\boldsymbol{X}^T)\boldsymbol{U} = \boldsymbol{U}\boldsymbol{D}. -$$ - -
-We can analyse the OLS solutions in terms of the eigenvectors (the columns) of the right singular value matrix \( \boldsymbol{U} \) as -$$ -\boldsymbol{X}\boldsymbol{\beta} = \boldsymbol{X}\left(\boldsymbol{V}\boldsymbol{D}\boldsymbol{V}^T \right)^{-1}\boldsymbol{X}^T\boldsymbol{y}=\boldsymbol{U\Sigma V^T}\left(\boldsymbol{V}\boldsymbol{D}\boldsymbol{V}^T \right)^{-1}(\boldsymbol{U\Sigma V^T})^T\boldsymbol{y}=\boldsymbol{U}\boldsymbol{U}^T\boldsymbol{y} -$$ - -
-For Ridge regression this becomes - -$$ -\boldsymbol{X}\boldsymbol{\beta}^{\mathrm{Ridge}} = \boldsymbol{U\Sigma V^T}\left(\boldsymbol{V}\boldsymbol{D}\boldsymbol{V}^T+\lambda\boldsymbol{I} \right)^{-1}(\boldsymbol{U\Sigma V^T})^T\boldsymbol{y}=\sum_{j=0}^{p-1}\boldsymbol{u}_j\boldsymbol{u}_j^T\frac{\sigma_j^2}{\sigma_j^2+\lambda}\boldsymbol{y}, -$$ - -
-with the vectors \( \boldsymbol{u}_j \) being the columns of \( \boldsymbol{U} \).
diff --git a/doc/pub/Regression/html/._Regression-bs044.html b/doc/pub/Regression/html/._Regression-bs044.html index 564661628..d3532c4de 100644 --- a/doc/pub/Regression/html/._Regression-bs044.html +++ b/doc/pub/Regression/html/._Regression-bs044.html @@ -111,112 +111,112 @@ Automatically generated HTML file from DocOnce source ('The Boston housing data example', 2, None, '___sec27'), ('Housing data, the code', 2, None, '___sec28'), ('The singular value decomposition', 2, None, '___sec29'), - ('The Ising model', 2, None, '___sec30'), - ('Reformulating the problem to suit regression', - 2, - None, - '___sec31'), - ('Linear regression', 2, None, '___sec32'), - ('Singular Value decomposition', 2, None, '___sec33'), - ('Linear Regression Problems', 2, None, '___sec34'), - ('Fixing the singularity', 2, None, '___sec35'), - ('Basic math of the SVD', 2, None, '___sec36'), - ('The SVD, a Fantastic Algorithm', 2, None, '___sec37'), - ('Another Example', 2, None, '___sec38'), - ('Economy-size SVD', 2, None, '___sec39'), - ('Mathematical Properties', 2, None, '___sec40'), - ('Ridge and LASSO Regression', 2, None, '___sec41'), - ('More on Ridge Regression', 2, None, '___sec42'), - ('Interpreting the Ridge results', 2, None, '___sec43'), - ('More interpretations', 2, None, '___sec44'), - ('Where are we going?', 2, None, '___sec45'), - ('Resampling methods', 2, None, '___sec46'), + ('Linear Regression Problems', 2, None, '___sec30'), + ('Fixing the singularity', 2, None, '___sec31'), + ('Basic math of the SVD', 2, None, '___sec32'), + ('The SVD, a Fantastic Algorithm', 2, None, '___sec33'), + ('Another Example', 2, None, '___sec34'), + ('Economy-size SVD', 2, None, '___sec35'), + ('Mathematical Properties', 2, None, '___sec36'), + ('Ridge and LASSO Regression', 2, None, '___sec37'), + ('More on Ridge Regression', 2, None, '___sec38'), + ('Interpreting the Ridge results', 2, None, '___sec39'), + ('More interpretations', 2, None, '___sec40'), + ('Where are we going?', 2, None, '___sec41'), + ('Resampling methods', 2, None, '___sec42'), ('Resampling approaches can be computationally expensive', 2, None, - '___sec47'), - ('Why resampling methods ?', 2, None, '___sec48'), - ('Statistical analysis', 2, None, '___sec49'), - ('Statistics', 2, None, '___sec50'), - ('Statistics, moments', 2, None, '___sec51'), - ('Statistics, central moments', 2, None, '___sec52'), - ('Statistics, covariance', 2, None, '___sec53'), - ('Statistics, more covariance', 2, None, '___sec54'), - ('Covariance example', 2, None, '___sec55'), - ('Covariance in numpy', 2, None, '___sec56'), - ('Statistics, independent variables', 2, None, '___sec57'), - ('Statistics, more variance', 2, None, '___sec58'), - ('Statistics and stochastic processes', 2, None, '___sec59'), - ('Statistics and sample variables', 2, None, '___sec60'), + '___sec43'), + ('Why resampling methods ?', 2, None, '___sec44'), + ('Statistical analysis', 2, None, '___sec45'), + ('Statistics', 2, None, '___sec46'), + ('Statistics, moments', 2, None, '___sec47'), + ('Statistics, central moments', 2, None, '___sec48'), + ('Statistics, covariance', 2, None, '___sec49'), + ('Statistics, more covariance', 2, None, '___sec50'), + ('Covariance example', 2, None, '___sec51'), + ('Covariance in numpy', 2, None, '___sec52'), + ('Statistics, independent variables', 2, None, '___sec53'), + ('Statistics, more variance', 2, None, '___sec54'), + ('Statistics and stochastic processes', 2, None, '___sec55'), + ('Statistics and sample variables', 2, None, '___sec56'), ('Statistics, sample variance and covariance', 2, None, - '___sec61'), - ('Statistics, law of large numbers', 2, None, '___sec62'), - ('Statistics, more on sample error', 2, None, '___sec63'), - ('Statistics', 2, None, '___sec64'), - ('Statistics, central limit theorem', 2, None, '___sec65'), - ('Statistics, more technicalities', 2, None, '___sec66'), - ('Statistics', 2, None, '___sec67'), - ('Statistics and sample variance', 2, None, '___sec68'), - ('Statistics, uncorrelated results', 2, None, '___sec69'), - ('Statistics, computations', 2, None, '___sec70'), + '___sec57'), + ('Statistics, law of large numbers', 2, None, '___sec58'), + ('Statistics, more on sample error', 2, None, '___sec59'), + ('Statistics', 2, None, '___sec60'), + ('Statistics, central limit theorem', 2, None, '___sec61'), + ('Statistics, more technicalities', 2, None, '___sec62'), + ('Statistics', 2, None, '___sec63'), + ('Statistics and sample variance', 2, None, '___sec64'), + ('Statistics, uncorrelated results', 2, None, '___sec65'), + ('Statistics, computations', 2, None, '___sec66'), ('Statistics, more on computations of errors', 2, None, - '___sec71'), - ('Statistics, wrapping up 1', 2, None, '___sec72'), - ('Statistics, final expression', 2, None, '___sec73'), + '___sec67'), + ('Statistics, wrapping up 1', 2, None, '___sec68'), + ('Statistics, final expression', 2, None, '___sec69'), ('Statistics, effective number of correlations', 2, None, - '___sec74'), + '___sec70'), ('Linking the regression analysis with a statistical ' 'interpretation', 2, None, - '___sec75'), - ('Assumptions made', 2, None, '___sec76'), - ('Expectation value and variance', 2, None, '___sec77'), + '___sec71'), + ('Assumptions made', 2, None, '___sec72'), + ('Expectation value and variance', 2, None, '___sec73'), ('Expectation value and variance for $\\boldsymbol{\\beta}$', 2, None, - '___sec78'), - ('Cross-validation', 2, None, '___sec79'), - ('Computationally expensive', 2, None, '___sec80'), - ('Various steps in cross-validation', 2, None, '___sec81'), + '___sec74'), + ('Cross-validation', 2, None, '___sec75'), + ('Computationally expensive', 2, None, '___sec76'), + ('Various steps in cross-validation', 2, None, '___sec77'), ('How to set up the cross-validation for Ridge and/or Lasso', 2, None, - '___sec82'), + '___sec78'), ('Resampling methods: Jackknife and Bootstrap', 2, None, - '___sec83'), - ('Resampling methods: Jackknife', 2, None, '___sec84'), - ('Jackknife code example', 2, None, '___sec85'), - ('Resampling methods: Bootstrap', 2, None, '___sec86'), - ('Resampling methods: Bootstrap background', 2, None, '___sec87'), + '___sec79'), + ('Resampling methods: Jackknife', 2, None, '___sec80'), + ('Jackknife code example', 2, None, '___sec81'), + ('Resampling methods: Bootstrap', 2, None, '___sec82'), + ('Resampling methods: Bootstrap background', 2, None, '___sec83'), ('Resampling methods: More Bootstrap background', 2, None, - '___sec88'), - ('Resampling methods: Bootstrap approach', 2, None, '___sec89'), - ('Resampling methods: Bootstrap steps', 2, None, '___sec90'), - ('Code example for the Bootstrap method', 2, None, '___sec91'), + '___sec84'), + ('Resampling methods: Bootstrap approach', 2, None, '___sec85'), + ('Resampling methods: Bootstrap steps', 2, None, '___sec86'), + ('Code example for the Bootstrap method', 2, None, '___sec87'), ('Code Example for Cross-validation and $k$-fold ' 'Cross-validation', 2, None, - '___sec92'), - ('The bias-variance tradeoff', 2, None, '___sec93'), - ('Example code for Bias-Variance tradeoff', 2, None, '___sec94'), - ('Understanding what happens', 2, None, '___sec95'), - ('Summing up', 2, None, '___sec96'), + '___sec88'), + ('The bias-variance tradeoff', 2, None, '___sec89'), + ('Example code for Bias-Variance tradeoff', 2, None, '___sec90'), + ('Understanding what happens', 2, None, '___sec91'), + ('Summing up', 2, None, '___sec92'), ("Another Example rom Scikit-Learn's Repository", 2, None, - '___sec97'), + '___sec93'), + ('The Ising model', 2, None, '___sec94'), + ('Reformulating the problem to suit regression', + 2, + None, + '___sec95'), + ('Linear regression', 2, None, '___sec96'), + ('Singular Value decomposition', 2, None, '___sec97'), ('The one-dimensional Ising model', 2, None, '___sec98'), ('Ridge regression', 2, None, '___sec99'), ('LASSO regression', 2, None, '___sec100'), @@ -295,74 +295,74 @@ MathJax.Hub.Config({
-Since \( \lambda \geq 0 \), it means that compared to OLS, we have - -$$ -\frac{\sigma_j^2}{\sigma_j^2+\lambda} \leq 1. -$$ +Resampling approaches can be computationally expensive, because they +involve fitting the same statistical method multiple times using +different subsets of the training data. However, due to recent +advances in computing power, the computational requirements of +resampling methods generally are not prohibitive. In this chapter, we +discuss two of the most commonly used resampling methods, +cross-validation and the bootstrap. Both methods are important tools +in the practical application of many statistical learning +procedures. For example, cross-validation can be used to estimate the +test error associated with a given statistical learning method in +order to evaluate its performance, or to select the appropriate level +of flexibility. The process of evaluating a model’s performance is +known as model assessment, whereas the process of selecting the proper +level of flexibility for a model is known as model selection. The +bootstrap is widely used.
-Ridge regression finds the coordinates of \( \boldsymbol{y} \) with respect to the -orthonormal basis \( \boldsymbol{U} \), it then shrinks the coordinates by -\( \frac{\sigma_j^2}{\sigma_j^2+\lambda} \). Recall that the SVD has -eigenvalues ordered in a descending way, that is \( \sigma_i \geq -\sigma_{i+1} \). +
-For small eigenvalues \( \sigma_i \) it means that their contributions become less important, a fact which can be used to reduce the number of degrees of freedom. -Actually, calculating the variance of \( \boldsymbol{X}\boldsymbol{v}_j \) shows that this quantity is equal to \( \sigma_j^2/n \). -With a parameter \( \lambda \) we can thus shrink the role of specific parameters.
diff --git a/doc/pub/Regression/html/._Regression-bs045.html b/doc/pub/Regression/html/._Regression-bs045.html index c8b667d8b..a2680f2fe 100644 --- a/doc/pub/Regression/html/._Regression-bs045.html +++ b/doc/pub/Regression/html/._Regression-bs045.html @@ -111,112 +111,112 @@ Automatically generated HTML file from DocOnce source ('The Boston housing data example', 2, None, '___sec27'), ('Housing data, the code', 2, None, '___sec28'), ('The singular value decomposition', 2, None, '___sec29'), - ('The Ising model', 2, None, '___sec30'), - ('Reformulating the problem to suit regression', - 2, - None, - '___sec31'), - ('Linear regression', 2, None, '___sec32'), - ('Singular Value decomposition', 2, None, '___sec33'), - ('Linear Regression Problems', 2, None, '___sec34'), - ('Fixing the singularity', 2, None, '___sec35'), - ('Basic math of the SVD', 2, None, '___sec36'), - ('The SVD, a Fantastic Algorithm', 2, None, '___sec37'), - ('Another Example', 2, None, '___sec38'), - ('Economy-size SVD', 2, None, '___sec39'), - ('Mathematical Properties', 2, None, '___sec40'), - ('Ridge and LASSO Regression', 2, None, '___sec41'), - ('More on Ridge Regression', 2, None, '___sec42'), - ('Interpreting the Ridge results', 2, None, '___sec43'), - ('More interpretations', 2, None, '___sec44'), - ('Where are we going?', 2, None, '___sec45'), - ('Resampling methods', 2, None, '___sec46'), + ('Linear Regression Problems', 2, None, '___sec30'), + ('Fixing the singularity', 2, None, '___sec31'), + ('Basic math of the SVD', 2, None, '___sec32'), + ('The SVD, a Fantastic Algorithm', 2, None, '___sec33'), + ('Another Example', 2, None, '___sec34'), + ('Economy-size SVD', 2, None, '___sec35'), + ('Mathematical Properties', 2, None, '___sec36'), + ('Ridge and LASSO Regression', 2, None, '___sec37'), + ('More on Ridge Regression', 2, None, '___sec38'), + ('Interpreting the Ridge results', 2, None, '___sec39'), + ('More interpretations', 2, None, '___sec40'), + ('Where are we going?', 2, None, '___sec41'), + ('Resampling methods', 2, None, '___sec42'), ('Resampling approaches can be computationally expensive', 2, None, - '___sec47'), - ('Why resampling methods ?', 2, None, '___sec48'), - ('Statistical analysis', 2, None, '___sec49'), - ('Statistics', 2, None, '___sec50'), - ('Statistics, moments', 2, None, '___sec51'), - ('Statistics, central moments', 2, None, '___sec52'), - ('Statistics, covariance', 2, None, '___sec53'), - ('Statistics, more covariance', 2, None, '___sec54'), - ('Covariance example', 2, None, '___sec55'), - ('Covariance in numpy', 2, None, '___sec56'), - ('Statistics, independent variables', 2, None, '___sec57'), - ('Statistics, more variance', 2, None, '___sec58'), - ('Statistics and stochastic processes', 2, None, '___sec59'), - ('Statistics and sample variables', 2, None, '___sec60'), + '___sec43'), + ('Why resampling methods ?', 2, None, '___sec44'), + ('Statistical analysis', 2, None, '___sec45'), + ('Statistics', 2, None, '___sec46'), + ('Statistics, moments', 2, None, '___sec47'), + ('Statistics, central moments', 2, None, '___sec48'), + ('Statistics, covariance', 2, None, '___sec49'), + ('Statistics, more covariance', 2, None, '___sec50'), + ('Covariance example', 2, None, '___sec51'), + ('Covariance in numpy', 2, None, '___sec52'), + ('Statistics, independent variables', 2, None, '___sec53'), + ('Statistics, more variance', 2, None, '___sec54'), + ('Statistics and stochastic processes', 2, None, '___sec55'), + ('Statistics and sample variables', 2, None, '___sec56'), ('Statistics, sample variance and covariance', 2, None, - '___sec61'), - ('Statistics, law of large numbers', 2, None, '___sec62'), - ('Statistics, more on sample error', 2, None, '___sec63'), - ('Statistics', 2, None, '___sec64'), - ('Statistics, central limit theorem', 2, None, '___sec65'), - ('Statistics, more technicalities', 2, None, '___sec66'), - ('Statistics', 2, None, '___sec67'), - ('Statistics and sample variance', 2, None, '___sec68'), - ('Statistics, uncorrelated results', 2, None, '___sec69'), - ('Statistics, computations', 2, None, '___sec70'), + '___sec57'), + ('Statistics, law of large numbers', 2, None, '___sec58'), + ('Statistics, more on sample error', 2, None, '___sec59'), + ('Statistics', 2, None, '___sec60'), + ('Statistics, central limit theorem', 2, None, '___sec61'), + ('Statistics, more technicalities', 2, None, '___sec62'), + ('Statistics', 2, None, '___sec63'), + ('Statistics and sample variance', 2, None, '___sec64'), + ('Statistics, uncorrelated results', 2, None, '___sec65'), + ('Statistics, computations', 2, None, '___sec66'), ('Statistics, more on computations of errors', 2, None, - '___sec71'), - ('Statistics, wrapping up 1', 2, None, '___sec72'), - ('Statistics, final expression', 2, None, '___sec73'), + '___sec67'), + ('Statistics, wrapping up 1', 2, None, '___sec68'), + ('Statistics, final expression', 2, None, '___sec69'), ('Statistics, effective number of correlations', 2, None, - '___sec74'), + '___sec70'), ('Linking the regression analysis with a statistical ' 'interpretation', 2, None, - '___sec75'), - ('Assumptions made', 2, None, '___sec76'), - ('Expectation value and variance', 2, None, '___sec77'), + '___sec71'), + ('Assumptions made', 2, None, '___sec72'), + ('Expectation value and variance', 2, None, '___sec73'), ('Expectation value and variance for $\\boldsymbol{\\beta}$', 2, None, - '___sec78'), - ('Cross-validation', 2, None, '___sec79'), - ('Computationally expensive', 2, None, '___sec80'), - ('Various steps in cross-validation', 2, None, '___sec81'), + '___sec74'), + ('Cross-validation', 2, None, '___sec75'), + ('Computationally expensive', 2, None, '___sec76'), + ('Various steps in cross-validation', 2, None, '___sec77'), ('How to set up the cross-validation for Ridge and/or Lasso', 2, None, - '___sec82'), + '___sec78'), ('Resampling methods: Jackknife and Bootstrap', 2, None, - '___sec83'), - ('Resampling methods: Jackknife', 2, None, '___sec84'), - ('Jackknife code example', 2, None, '___sec85'), - ('Resampling methods: Bootstrap', 2, None, '___sec86'), - ('Resampling methods: Bootstrap background', 2, None, '___sec87'), + '___sec79'), + ('Resampling methods: Jackknife', 2, None, '___sec80'), + ('Jackknife code example', 2, None, '___sec81'), + ('Resampling methods: Bootstrap', 2, None, '___sec82'), + ('Resampling methods: Bootstrap background', 2, None, '___sec83'), ('Resampling methods: More Bootstrap background', 2, None, - '___sec88'), - ('Resampling methods: Bootstrap approach', 2, None, '___sec89'), - ('Resampling methods: Bootstrap steps', 2, None, '___sec90'), - ('Code example for the Bootstrap method', 2, None, '___sec91'), + '___sec84'), + ('Resampling methods: Bootstrap approach', 2, None, '___sec85'), + ('Resampling methods: Bootstrap steps', 2, None, '___sec86'), + ('Code example for the Bootstrap method', 2, None, '___sec87'), ('Code Example for Cross-validation and $k$-fold ' 'Cross-validation', 2, None, - '___sec92'), - ('The bias-variance tradeoff', 2, None, '___sec93'), - ('Example code for Bias-Variance tradeoff', 2, None, '___sec94'), - ('Understanding what happens', 2, None, '___sec95'), - ('Summing up', 2, None, '___sec96'), + '___sec88'), + ('The bias-variance tradeoff', 2, None, '___sec89'), + ('Example code for Bias-Variance tradeoff', 2, None, '___sec90'), + ('Understanding what happens', 2, None, '___sec91'), + ('Summing up', 2, None, '___sec92'), ("Another Example rom Scikit-Learn's Repository", 2, None, - '___sec97'), + '___sec93'), + ('The Ising model', 2, None, '___sec94'), + ('Reformulating the problem to suit regression', + 2, + None, + '___sec95'), + ('Linear regression', 2, None, '___sec96'), + ('Singular Value decomposition', 2, None, '___sec97'), ('The one-dimensional Ising model', 2, None, '___sec98'), ('Ridge regression', 2, None, '___sec99'), ('LASSO regression', 2, None, '___sec100'), @@ -295,74 +295,74 @@ MathJax.Hub.Config({
-
-For the sake of simplicity, let us assume that the design matrix is orthonormal, that is +
-In this case the standard OLS results in -$$ -\boldsymbol{\beta}^{\mathrm{OLS}} = \boldsymbol{X}^T\boldsymbol{y}=\sum_{i=0}^{p-1}\boldsymbol{u}_j\boldsymbol{u}_j^T\boldsymbol{y}, -$$ - -
-and - -$$ -\boldsymbol{\beta}^{\mathrm{Ridge}} = \left(\boldsymbol{I}+\lambda\boldsymbol{I}\right)^{-1}\boldsymbol{X}^T\boldsymbol{y}=\left(1+\lambda\right)^{-1}\boldsymbol{\beta}^{\mathrm{OLS}}, -$$ - -
-that is the Ridge estimator scales the OLS estimator by the inverse of a factor \( 1+\lambda \), and -the Ridge estimator converges to zero when the hyperparameter goes to -infinity. - -
-We will come back to more interpreations after we have gone through some of the statistical analysis part. - -
-For more discussions of Ridge and Lasso regression, Wessel van Wieringen's article is highly recommended. -Similarly, Mehta et al's article is also recommended.
diff --git a/doc/pub/Regression/html/._Regression-bs046.html b/doc/pub/Regression/html/._Regression-bs046.html index 4ae502f5b..05142a991 100644 --- a/doc/pub/Regression/html/._Regression-bs046.html +++ b/doc/pub/Regression/html/._Regression-bs046.html @@ -111,112 +111,112 @@ Automatically generated HTML file from DocOnce source ('The Boston housing data example', 2, None, '___sec27'), ('Housing data, the code', 2, None, '___sec28'), ('The singular value decomposition', 2, None, '___sec29'), - ('The Ising model', 2, None, '___sec30'), - ('Reformulating the problem to suit regression', - 2, - None, - '___sec31'), - ('Linear regression', 2, None, '___sec32'), - ('Singular Value decomposition', 2, None, '___sec33'), - ('Linear Regression Problems', 2, None, '___sec34'), - ('Fixing the singularity', 2, None, '___sec35'), - ('Basic math of the SVD', 2, None, '___sec36'), - ('The SVD, a Fantastic Algorithm', 2, None, '___sec37'), - ('Another Example', 2, None, '___sec38'), - ('Economy-size SVD', 2, None, '___sec39'), - ('Mathematical Properties', 2, None, '___sec40'), - ('Ridge and LASSO Regression', 2, None, '___sec41'), - ('More on Ridge Regression', 2, None, '___sec42'), - ('Interpreting the Ridge results', 2, None, '___sec43'), - ('More interpretations', 2, None, '___sec44'), - ('Where are we going?', 2, None, '___sec45'), - ('Resampling methods', 2, None, '___sec46'), + ('Linear Regression Problems', 2, None, '___sec30'), + ('Fixing the singularity', 2, None, '___sec31'), + ('Basic math of the SVD', 2, None, '___sec32'), + ('The SVD, a Fantastic Algorithm', 2, None, '___sec33'), + ('Another Example', 2, None, '___sec34'), + ('Economy-size SVD', 2, None, '___sec35'), + ('Mathematical Properties', 2, None, '___sec36'), + ('Ridge and LASSO Regression', 2, None, '___sec37'), + ('More on Ridge Regression', 2, None, '___sec38'), + ('Interpreting the Ridge results', 2, None, '___sec39'), + ('More interpretations', 2, None, '___sec40'), + ('Where are we going?', 2, None, '___sec41'), + ('Resampling methods', 2, None, '___sec42'), ('Resampling approaches can be computationally expensive', 2, None, - '___sec47'), - ('Why resampling methods ?', 2, None, '___sec48'), - ('Statistical analysis', 2, None, '___sec49'), - ('Statistics', 2, None, '___sec50'), - ('Statistics, moments', 2, None, '___sec51'), - ('Statistics, central moments', 2, None, '___sec52'), - ('Statistics, covariance', 2, None, '___sec53'), - ('Statistics, more covariance', 2, None, '___sec54'), - ('Covariance example', 2, None, '___sec55'), - ('Covariance in numpy', 2, None, '___sec56'), - ('Statistics, independent variables', 2, None, '___sec57'), - ('Statistics, more variance', 2, None, '___sec58'), - ('Statistics and stochastic processes', 2, None, '___sec59'), - ('Statistics and sample variables', 2, None, '___sec60'), + '___sec43'), + ('Why resampling methods ?', 2, None, '___sec44'), + ('Statistical analysis', 2, None, '___sec45'), + ('Statistics', 2, None, '___sec46'), + ('Statistics, moments', 2, None, '___sec47'), + ('Statistics, central moments', 2, None, '___sec48'), + ('Statistics, covariance', 2, None, '___sec49'), + ('Statistics, more covariance', 2, None, '___sec50'), + ('Covariance example', 2, None, '___sec51'), + ('Covariance in numpy', 2, None, '___sec52'), + ('Statistics, independent variables', 2, None, '___sec53'), + ('Statistics, more variance', 2, None, '___sec54'), + ('Statistics and stochastic processes', 2, None, '___sec55'), + ('Statistics and sample variables', 2, None, '___sec56'), ('Statistics, sample variance and covariance', 2, None, - '___sec61'), - ('Statistics, law of large numbers', 2, None, '___sec62'), - ('Statistics, more on sample error', 2, None, '___sec63'), - ('Statistics', 2, None, '___sec64'), - ('Statistics, central limit theorem', 2, None, '___sec65'), - ('Statistics, more technicalities', 2, None, '___sec66'), - ('Statistics', 2, None, '___sec67'), - ('Statistics and sample variance', 2, None, '___sec68'), - ('Statistics, uncorrelated results', 2, None, '___sec69'), - ('Statistics, computations', 2, None, '___sec70'), + '___sec57'), + ('Statistics, law of large numbers', 2, None, '___sec58'), + ('Statistics, more on sample error', 2, None, '___sec59'), + ('Statistics', 2, None, '___sec60'), + ('Statistics, central limit theorem', 2, None, '___sec61'), + ('Statistics, more technicalities', 2, None, '___sec62'), + ('Statistics', 2, None, '___sec63'), + ('Statistics and sample variance', 2, None, '___sec64'), + ('Statistics, uncorrelated results', 2, None, '___sec65'), + ('Statistics, computations', 2, None, '___sec66'), ('Statistics, more on computations of errors', 2, None, - '___sec71'), - ('Statistics, wrapping up 1', 2, None, '___sec72'), - ('Statistics, final expression', 2, None, '___sec73'), + '___sec67'), + ('Statistics, wrapping up 1', 2, None, '___sec68'), + ('Statistics, final expression', 2, None, '___sec69'), ('Statistics, effective number of correlations', 2, None, - '___sec74'), + '___sec70'), ('Linking the regression analysis with a statistical ' 'interpretation', 2, None, - '___sec75'), - ('Assumptions made', 2, None, '___sec76'), - ('Expectation value and variance', 2, None, '___sec77'), + '___sec71'), + ('Assumptions made', 2, None, '___sec72'), + ('Expectation value and variance', 2, None, '___sec73'), ('Expectation value and variance for $\\boldsymbol{\\beta}$', 2, None, - '___sec78'), - ('Cross-validation', 2, None, '___sec79'), - ('Computationally expensive', 2, None, '___sec80'), - ('Various steps in cross-validation', 2, None, '___sec81'), + '___sec74'), + ('Cross-validation', 2, None, '___sec75'), + ('Computationally expensive', 2, None, '___sec76'), + ('Various steps in cross-validation', 2, None, '___sec77'), ('How to set up the cross-validation for Ridge and/or Lasso', 2, None, - '___sec82'), + '___sec78'), ('Resampling methods: Jackknife and Bootstrap', 2, None, - '___sec83'), - ('Resampling methods: Jackknife', 2, None, '___sec84'), - ('Jackknife code example', 2, None, '___sec85'), - ('Resampling methods: Bootstrap', 2, None, '___sec86'), - ('Resampling methods: Bootstrap background', 2, None, '___sec87'), + '___sec79'), + ('Resampling methods: Jackknife', 2, None, '___sec80'), + ('Jackknife code example', 2, None, '___sec81'), + ('Resampling methods: Bootstrap', 2, None, '___sec82'), + ('Resampling methods: Bootstrap background', 2, None, '___sec83'), ('Resampling methods: More Bootstrap background', 2, None, - '___sec88'), - ('Resampling methods: Bootstrap approach', 2, None, '___sec89'), - ('Resampling methods: Bootstrap steps', 2, None, '___sec90'), - ('Code example for the Bootstrap method', 2, None, '___sec91'), + '___sec84'), + ('Resampling methods: Bootstrap approach', 2, None, '___sec85'), + ('Resampling methods: Bootstrap steps', 2, None, '___sec86'), + ('Code example for the Bootstrap method', 2, None, '___sec87'), ('Code Example for Cross-validation and $k$-fold ' 'Cross-validation', 2, None, - '___sec92'), - ('The bias-variance tradeoff', 2, None, '___sec93'), - ('Example code for Bias-Variance tradeoff', 2, None, '___sec94'), - ('Understanding what happens', 2, None, '___sec95'), - ('Summing up', 2, None, '___sec96'), + '___sec88'), + ('The bias-variance tradeoff', 2, None, '___sec89'), + ('Example code for Bias-Variance tradeoff', 2, None, '___sec90'), + ('Understanding what happens', 2, None, '___sec91'), + ('Summing up', 2, None, '___sec92'), ("Another Example rom Scikit-Learn's Repository", 2, None, - '___sec97'), + '___sec93'), + ('The Ising model', 2, None, '___sec94'), + ('Reformulating the problem to suit regression', + 2, + None, + '___sec95'), + ('Linear regression', 2, None, '___sec96'), + ('Singular Value decomposition', 2, None, '___sec97'), ('The one-dimensional Ising model', 2, None, '___sec98'), ('Ridge regression', 2, None, '___sec99'), ('LASSO regression', 2, None, '___sec100'), @@ -295,74 +295,74 @@ MathJax.Hub.Config({
-
-Before we proceed, we need to rethink what we have been doing. In our -eager to fit the data, we have omitted several important elements in -our regression analysis. In what follows we will +
diff --git a/doc/pub/Regression/html/._Regression-bs047.html b/doc/pub/Regression/html/._Regression-bs047.html index a8564ed23..7fa3a3ae5 100644 --- a/doc/pub/Regression/html/._Regression-bs047.html +++ b/doc/pub/Regression/html/._Regression-bs047.html @@ -111,112 +111,112 @@ Automatically generated HTML file from DocOnce source ('The Boston housing data example', 2, None, '___sec27'), ('Housing data, the code', 2, None, '___sec28'), ('The singular value decomposition', 2, None, '___sec29'), - ('The Ising model', 2, None, '___sec30'), - ('Reformulating the problem to suit regression', - 2, - None, - '___sec31'), - ('Linear regression', 2, None, '___sec32'), - ('Singular Value decomposition', 2, None, '___sec33'), - ('Linear Regression Problems', 2, None, '___sec34'), - ('Fixing the singularity', 2, None, '___sec35'), - ('Basic math of the SVD', 2, None, '___sec36'), - ('The SVD, a Fantastic Algorithm', 2, None, '___sec37'), - ('Another Example', 2, None, '___sec38'), - ('Economy-size SVD', 2, None, '___sec39'), - ('Mathematical Properties', 2, None, '___sec40'), - ('Ridge and LASSO Regression', 2, None, '___sec41'), - ('More on Ridge Regression', 2, None, '___sec42'), - ('Interpreting the Ridge results', 2, None, '___sec43'), - ('More interpretations', 2, None, '___sec44'), - ('Where are we going?', 2, None, '___sec45'), - ('Resampling methods', 2, None, '___sec46'), + ('Linear Regression Problems', 2, None, '___sec30'), + ('Fixing the singularity', 2, None, '___sec31'), + ('Basic math of the SVD', 2, None, '___sec32'), + ('The SVD, a Fantastic Algorithm', 2, None, '___sec33'), + ('Another Example', 2, None, '___sec34'), + ('Economy-size SVD', 2, None, '___sec35'), + ('Mathematical Properties', 2, None, '___sec36'), + ('Ridge and LASSO Regression', 2, None, '___sec37'), + ('More on Ridge Regression', 2, None, '___sec38'), + ('Interpreting the Ridge results', 2, None, '___sec39'), + ('More interpretations', 2, None, '___sec40'), + ('Where are we going?', 2, None, '___sec41'), + ('Resampling methods', 2, None, '___sec42'), ('Resampling approaches can be computationally expensive', 2, None, - '___sec47'), - ('Why resampling methods ?', 2, None, '___sec48'), - ('Statistical analysis', 2, None, '___sec49'), - ('Statistics', 2, None, '___sec50'), - ('Statistics, moments', 2, None, '___sec51'), - ('Statistics, central moments', 2, None, '___sec52'), - ('Statistics, covariance', 2, None, '___sec53'), - ('Statistics, more covariance', 2, None, '___sec54'), - ('Covariance example', 2, None, '___sec55'), - ('Covariance in numpy', 2, None, '___sec56'), - ('Statistics, independent variables', 2, None, '___sec57'), - ('Statistics, more variance', 2, None, '___sec58'), - ('Statistics and stochastic processes', 2, None, '___sec59'), - ('Statistics and sample variables', 2, None, '___sec60'), + '___sec43'), + ('Why resampling methods ?', 2, None, '___sec44'), + ('Statistical analysis', 2, None, '___sec45'), + ('Statistics', 2, None, '___sec46'), + ('Statistics, moments', 2, None, '___sec47'), + ('Statistics, central moments', 2, None, '___sec48'), + ('Statistics, covariance', 2, None, '___sec49'), + ('Statistics, more covariance', 2, None, '___sec50'), + ('Covariance example', 2, None, '___sec51'), + ('Covariance in numpy', 2, None, '___sec52'), + ('Statistics, independent variables', 2, None, '___sec53'), + ('Statistics, more variance', 2, None, '___sec54'), + ('Statistics and stochastic processes', 2, None, '___sec55'), + ('Statistics and sample variables', 2, None, '___sec56'), ('Statistics, sample variance and covariance', 2, None, - '___sec61'), - ('Statistics, law of large numbers', 2, None, '___sec62'), - ('Statistics, more on sample error', 2, None, '___sec63'), - ('Statistics', 2, None, '___sec64'), - ('Statistics, central limit theorem', 2, None, '___sec65'), - ('Statistics, more technicalities', 2, None, '___sec66'), - ('Statistics', 2, None, '___sec67'), - ('Statistics and sample variance', 2, None, '___sec68'), - ('Statistics, uncorrelated results', 2, None, '___sec69'), - ('Statistics, computations', 2, None, '___sec70'), + '___sec57'), + ('Statistics, law of large numbers', 2, None, '___sec58'), + ('Statistics, more on sample error', 2, None, '___sec59'), + ('Statistics', 2, None, '___sec60'), + ('Statistics, central limit theorem', 2, None, '___sec61'), + ('Statistics, more technicalities', 2, None, '___sec62'), + ('Statistics', 2, None, '___sec63'), + ('Statistics and sample variance', 2, None, '___sec64'), + ('Statistics, uncorrelated results', 2, None, '___sec65'), + ('Statistics, computations', 2, None, '___sec66'), ('Statistics, more on computations of errors', 2, None, - '___sec71'), - ('Statistics, wrapping up 1', 2, None, '___sec72'), - ('Statistics, final expression', 2, None, '___sec73'), + '___sec67'), + ('Statistics, wrapping up 1', 2, None, '___sec68'), + ('Statistics, final expression', 2, None, '___sec69'), ('Statistics, effective number of correlations', 2, None, - '___sec74'), + '___sec70'), ('Linking the regression analysis with a statistical ' 'interpretation', 2, None, - '___sec75'), - ('Assumptions made', 2, None, '___sec76'), - ('Expectation value and variance', 2, None, '___sec77'), + '___sec71'), + ('Assumptions made', 2, None, '___sec72'), + ('Expectation value and variance', 2, None, '___sec73'), ('Expectation value and variance for $\\boldsymbol{\\beta}$', 2, None, - '___sec78'), - ('Cross-validation', 2, None, '___sec79'), - ('Computationally expensive', 2, None, '___sec80'), - ('Various steps in cross-validation', 2, None, '___sec81'), + '___sec74'), + ('Cross-validation', 2, None, '___sec75'), + ('Computationally expensive', 2, None, '___sec76'), + ('Various steps in cross-validation', 2, None, '___sec77'), ('How to set up the cross-validation for Ridge and/or Lasso', 2, None, - '___sec82'), + '___sec78'), ('Resampling methods: Jackknife and Bootstrap', 2, None, - '___sec83'), - ('Resampling methods: Jackknife', 2, None, '___sec84'), - ('Jackknife code example', 2, None, '___sec85'), - ('Resampling methods: Bootstrap', 2, None, '___sec86'), - ('Resampling methods: Bootstrap background', 2, None, '___sec87'), + '___sec79'), + ('Resampling methods: Jackknife', 2, None, '___sec80'), + ('Jackknife code example', 2, None, '___sec81'), + ('Resampling methods: Bootstrap', 2, None, '___sec82'), + ('Resampling methods: Bootstrap background', 2, None, '___sec83'), ('Resampling methods: More Bootstrap background', 2, None, - '___sec88'), - ('Resampling methods: Bootstrap approach', 2, None, '___sec89'), - ('Resampling methods: Bootstrap steps', 2, None, '___sec90'), - ('Code example for the Bootstrap method', 2, None, '___sec91'), + '___sec84'), + ('Resampling methods: Bootstrap approach', 2, None, '___sec85'), + ('Resampling methods: Bootstrap steps', 2, None, '___sec86'), + ('Code example for the Bootstrap method', 2, None, '___sec87'), ('Code Example for Cross-validation and $k$-fold ' 'Cross-validation', 2, None, - '___sec92'), - ('The bias-variance tradeoff', 2, None, '___sec93'), - ('Example code for Bias-Variance tradeoff', 2, None, '___sec94'), - ('Understanding what happens', 2, None, '___sec95'), - ('Summing up', 2, None, '___sec96'), + '___sec88'), + ('The bias-variance tradeoff', 2, None, '___sec89'), + ('Example code for Bias-Variance tradeoff', 2, None, '___sec90'), + ('Understanding what happens', 2, None, '___sec91'), + ('Summing up', 2, None, '___sec92'), ("Another Example rom Scikit-Learn's Repository", 2, None, - '___sec97'), + '___sec93'), + ('The Ising model', 2, None, '___sec94'), + ('Reformulating the problem to suit regression', + 2, + None, + '___sec95'), + ('Linear regression', 2, None, '___sec96'), + ('Singular Value decomposition', 2, None, '___sec97'), ('The one-dimensional Ising model', 2, None, '___sec98'), ('Ridge regression', 2, None, '___sec99'), ('LASSO regression', 2, None, '___sec100'), @@ -295,74 +295,74 @@ MathJax.Hub.Config({
-Resampling methods are an indispensable tool in modern -statistics. They involve repeatedly drawing samples from a training -set and refitting a model of interest on each sample in order to -obtain additional information about the fitted model. For example, in -order to estimate the variability of a linear regression fit, we can -repeatedly draw different samples from the training data, fit a linear -regression to each new sample, and then examine the extent to which -the resulting fits differ. Such an approach may allow us to obtain -information that would not be available from fitting the model only -once using the original training sample. +The probability distribution function (PDF) is a function +\( p(x) \) on the domain which, in the discrete case, gives us the +probability or relative frequency with which these values of \( X \) occur: +$$ +p(x) = \mathrm{prob}(X=x) +$$ + +In the continuous case, the PDF does not directly depict the +actual probability. Instead we define the probability for the +stochastic variable to assume any value on an infinitesimal interval +around \( x \) to be \( p(x)dx \). The continuous function \( p(x) \) then gives us +the density of the probability rather than the probability +itself. The probability for a stochastic variable to assume any value +on a non-infinitesimal interval \( [a,\,b] \) is then just the integral: +$$ +\mathrm{prob}(a\leq X\leq b) = \int_a^b p(x)dx +$$ + +Qualitatively speaking, a stochastic variable represents the values of +numbers chosen as if by chance from some specified PDF so that the +selection of a large set of these numbers reproduces this PDF.
+A particularly useful class of special expectation values are the +moments. The \( n \)-th moment of the PDF \( p \) is defined as +follows: +$$ +\langle x^n\rangle \equiv \int\! x^n p(x)\,dx +$$ -
-Resampling approaches can be computationally expensive, because they -involve fitting the same statistical method multiple times using -different subsets of the training data. However, due to recent -advances in computing power, the computational requirements of -resampling methods generally are not prohibitive. In this chapter, we -discuss two of the most commonly used resampling methods, -cross-validation and the bootstrap. Both methods are important tools -in the practical application of many statistical learning -procedures. For example, cross-validation can be used to estimate the -test error associated with a given statistical learning method in -order to evaluate its performance, or to select the appropriate level -of flexibility. The process of evaluating a model’s performance is -known as model assessment, whereas the process of selecting the proper -level of flexibility for a model is known as model selection. The -bootstrap is widely used. - -
+The zero-th moment \( \langle 1\rangle \) is just the normalization condition of +\( p \). The first moment, \( \langle x\rangle \), is called the mean of \( p \) +and often denoted by the letter \( \mu \): +$$ +\langle x\rangle = \mu \equiv \int\! x p(x)\,dx +$$
+A special version of the moments is the set of central moments, +the n-th central moment defined as: +$$ +\langle (x-\langle x \rangle )^n\rangle \equiv \int\! (x-\langle x\rangle)^n p(x)\,dx +$$ -
+Another important quantity is the so called covariance, a variant of +the above defined variance. Consider again the set \( \{X_i\} \) of \( n \) +stochastic variables (not necessarily uncorrelated) with the +multivariate PDF \( P(x_1,\dots,x_n) \). The covariance of two +of the stochastic variables, \( X_i \) and \( X_j \), is defined as follows: +$$ +\begin{align} +\mathrm{cov}(X_i,\,X_j) &\equiv \langle (x_i-\langle x_i\rangle)(x_j-\langle x_j\rangle)\rangle +\nonumber\\ +&= +\int\!\cdots\!\int\!(x_i-\langle x_i \rangle)(x_j-\langle x_j \rangle)\, +P(x_1,\dots,x_n)\,dx_1\dots dx_n +\tag{6} +\end{align} +$$ -
-The probability distribution function (PDF) is a function -\( p(x) \) on the domain which, in the discrete case, gives us the -probability or relative frequency with which these values of \( X \) occur: +If we consider the above covariance as a matrix \( C_{ij}=\mathrm{cov}(X_i,\,X_j) \), then the diagonal elements are just the familiar +variances, \( C_{ii} = \mathrm{cov}(X_i,\,X_i) = \mathrm{var}(X_i) \). It turns out that +all the off-diagonal elements are zero if the stochastic variables are +uncorrelated. This is easy to show, keeping in mind the linearity of +the expectation value. Consider the stochastic variables \( X_i \) and +\( X_j \), (\( i\neq j \)): $$ -p(x) = \mathrm{prob}(X=x) +\begin{align} +\mathrm{cov}(X_i,\,X_j) &= \langle(x_i-\langle x_i\rangle)(x_j-\langle x_j\rangle)\rangle +\tag{7}\\ +&=\langle x_i x_j - x_i\langle x_j\rangle - \langle x_i\rangle x_j + \langle x_i\rangle\langle x_j\rangle\rangle +\tag{8}\\ +&=\langle x_i x_j\rangle - \langle x_i\langle x_j\rangle\rangle - \langle \langle x_i\rangle x_j\rangle + +\langle \langle x_i\rangle\langle x_j\rangle\rangle +\tag{9}\\ +&=\langle x_i x_j\rangle - \langle x_i\rangle\langle x_j\rangle - \langle x_i\rangle\langle x_j\rangle + +\langle x_i\rangle\langle x_j\rangle +\tag{10}\\ +&=\langle x_i x_j\rangle - \langle x_i\rangle\langle x_j\rangle +\tag{11} +\end{align} $$ - -In the continuous case, the PDF does not directly depict the -actual probability. Instead we define the probability for the -stochastic variable to assume any value on an infinitesimal interval -around \( x \) to be \( p(x)dx \). The continuous function \( p(x) \) then gives us -the density of the probability rather than the probability -itself. The probability for a stochastic variable to assume any value -on a non-infinitesimal interval \( [a,\,b] \) is then just the integral: -$$ -\mathrm{prob}(a\leq X\leq b) = \int_a^b p(x)dx -$$ - -Qualitatively speaking, a stochastic variable represents the values of -numbers chosen as if by chance from some specified PDF so that the -selection of a large set of these numbers reproduces this PDF.
-A particularly useful class of special expectation values are the -moments. The \( n \)-th moment of the PDF \( p \) is defined as -follows: +
+Suppose we have defined three vectors \( \hat{x}, \hat{y}, \hat{z} \) with +\( n \) elements each. The covariance matrix is defined as + $$ -\langle x^n\rangle \equiv \int\! x^n p(x)\,dx +\hat{\Sigma} = \begin{bmatrix} \sigma_{xx} & \sigma_{xy} & \sigma_{xz} \\ + \sigma_{yx} & \sigma_{yy} & \sigma_{yz} \\ + \sigma_{zx} & \sigma_{zy} & \sigma_{zz} + \end{bmatrix}, $$ -The zero-th moment \( \langle 1\rangle \) is just the normalization condition of -\( p \). The first moment, \( \langle x\rangle \), is called the mean of \( p \) -and often denoted by the letter \( \mu \): +where for example $$ -\langle x\rangle = \mu \equiv \int\! x p(x)\,dx +\sigma_{xy} =\frac{1}{n} \sum_{i=0}^{n-1}(x_i- \overline{x})(y_i- \overline{y}). $$ -
+The Numpy function np.cov calculates the covariance elements using +the factor \( 1/(n-1) \) instead of \( 1/n \) since it assumes we do not have +the exact mean valu\ es. + +
+The following simple function uses the np.vstack function which +takes each vector of dimension \( 1\times n \) and produces a \( 3\times n \) +matrix \( \hat{W} \) + +$$ +\hat{W} = \begin{bmatrix} x_0 & y_0 & z_0 \\ + x_1 & y_1 & z_1 \\ + x_2 & y_2 & z_2 \\ + \dots & \dots & \dots \\ + x_{n-2} & y_{n-2} & z_{n-2} \\ + x_{n-1} & y_{n-1} & z_{n-1} + \end{bmatrix}, +$$ + +
+which in turn is converted into into the \( 3\times 3 \) covariance matrix +\( \hat{\Sigma} \) via the Numpy function np.cov(). We note that we can +also calculate the mean value of each set of samples \( \hat{x} \) etc +using the Numpy function np.mean(x). We can also extract the +eigenvalues of the covariance matrix through the np.linalg.eig() +function.
diff --git a/doc/pub/Regression/html/._Regression-bs053.html b/doc/pub/Regression/html/._Regression-bs053.html index 597c51e87..108f03bca 100644 --- a/doc/pub/Regression/html/._Regression-bs053.html +++ b/doc/pub/Regression/html/._Regression-bs053.html @@ -111,112 +111,112 @@ Automatically generated HTML file from DocOnce source ('The Boston housing data example', 2, None, '___sec27'), ('Housing data, the code', 2, None, '___sec28'), ('The singular value decomposition', 2, None, '___sec29'), - ('The Ising model', 2, None, '___sec30'), - ('Reformulating the problem to suit regression', - 2, - None, - '___sec31'), - ('Linear regression', 2, None, '___sec32'), - ('Singular Value decomposition', 2, None, '___sec33'), - ('Linear Regression Problems', 2, None, '___sec34'), - ('Fixing the singularity', 2, None, '___sec35'), - ('Basic math of the SVD', 2, None, '___sec36'), - ('The SVD, a Fantastic Algorithm', 2, None, '___sec37'), - ('Another Example', 2, None, '___sec38'), - ('Economy-size SVD', 2, None, '___sec39'), - ('Mathematical Properties', 2, None, '___sec40'), - ('Ridge and LASSO Regression', 2, None, '___sec41'), - ('More on Ridge Regression', 2, None, '___sec42'), - ('Interpreting the Ridge results', 2, None, '___sec43'), - ('More interpretations', 2, None, '___sec44'), - ('Where are we going?', 2, None, '___sec45'), - ('Resampling methods', 2, None, '___sec46'), + ('Linear Regression Problems', 2, None, '___sec30'), + ('Fixing the singularity', 2, None, '___sec31'), + ('Basic math of the SVD', 2, None, '___sec32'), + ('The SVD, a Fantastic Algorithm', 2, None, '___sec33'), + ('Another Example', 2, None, '___sec34'), + ('Economy-size SVD', 2, None, '___sec35'), + ('Mathematical Properties', 2, None, '___sec36'), + ('Ridge and LASSO Regression', 2, None, '___sec37'), + ('More on Ridge Regression', 2, None, '___sec38'), + ('Interpreting the Ridge results', 2, None, '___sec39'), + ('More interpretations', 2, None, '___sec40'), + ('Where are we going?', 2, None, '___sec41'), + ('Resampling methods', 2, None, '___sec42'), ('Resampling approaches can be computationally expensive', 2, None, - '___sec47'), - ('Why resampling methods ?', 2, None, '___sec48'), - ('Statistical analysis', 2, None, '___sec49'), - ('Statistics', 2, None, '___sec50'), - ('Statistics, moments', 2, None, '___sec51'), - ('Statistics, central moments', 2, None, '___sec52'), - ('Statistics, covariance', 2, None, '___sec53'), - ('Statistics, more covariance', 2, None, '___sec54'), - ('Covariance example', 2, None, '___sec55'), - ('Covariance in numpy', 2, None, '___sec56'), - ('Statistics, independent variables', 2, None, '___sec57'), - ('Statistics, more variance', 2, None, '___sec58'), - ('Statistics and stochastic processes', 2, None, '___sec59'), - ('Statistics and sample variables', 2, None, '___sec60'), + '___sec43'), + ('Why resampling methods ?', 2, None, '___sec44'), + ('Statistical analysis', 2, None, '___sec45'), + ('Statistics', 2, None, '___sec46'), + ('Statistics, moments', 2, None, '___sec47'), + ('Statistics, central moments', 2, None, '___sec48'), + ('Statistics, covariance', 2, None, '___sec49'), + ('Statistics, more covariance', 2, None, '___sec50'), + ('Covariance example', 2, None, '___sec51'), + ('Covariance in numpy', 2, None, '___sec52'), + ('Statistics, independent variables', 2, None, '___sec53'), + ('Statistics, more variance', 2, None, '___sec54'), + ('Statistics and stochastic processes', 2, None, '___sec55'), + ('Statistics and sample variables', 2, None, '___sec56'), ('Statistics, sample variance and covariance', 2, None, - '___sec61'), - ('Statistics, law of large numbers', 2, None, '___sec62'), - ('Statistics, more on sample error', 2, None, '___sec63'), - ('Statistics', 2, None, '___sec64'), - ('Statistics, central limit theorem', 2, None, '___sec65'), - ('Statistics, more technicalities', 2, None, '___sec66'), - ('Statistics', 2, None, '___sec67'), - ('Statistics and sample variance', 2, None, '___sec68'), - ('Statistics, uncorrelated results', 2, None, '___sec69'), - ('Statistics, computations', 2, None, '___sec70'), + '___sec57'), + ('Statistics, law of large numbers', 2, None, '___sec58'), + ('Statistics, more on sample error', 2, None, '___sec59'), + ('Statistics', 2, None, '___sec60'), + ('Statistics, central limit theorem', 2, None, '___sec61'), + ('Statistics, more technicalities', 2, None, '___sec62'), + ('Statistics', 2, None, '___sec63'), + ('Statistics and sample variance', 2, None, '___sec64'), + ('Statistics, uncorrelated results', 2, None, '___sec65'), + ('Statistics, computations', 2, None, '___sec66'), ('Statistics, more on computations of errors', 2, None, - '___sec71'), - ('Statistics, wrapping up 1', 2, None, '___sec72'), - ('Statistics, final expression', 2, None, '___sec73'), + '___sec67'), + ('Statistics, wrapping up 1', 2, None, '___sec68'), + ('Statistics, final expression', 2, None, '___sec69'), ('Statistics, effective number of correlations', 2, None, - '___sec74'), + '___sec70'), ('Linking the regression analysis with a statistical ' 'interpretation', 2, None, - '___sec75'), - ('Assumptions made', 2, None, '___sec76'), - ('Expectation value and variance', 2, None, '___sec77'), + '___sec71'), + ('Assumptions made', 2, None, '___sec72'), + ('Expectation value and variance', 2, None, '___sec73'), ('Expectation value and variance for $\\boldsymbol{\\beta}$', 2, None, - '___sec78'), - ('Cross-validation', 2, None, '___sec79'), - ('Computationally expensive', 2, None, '___sec80'), - ('Various steps in cross-validation', 2, None, '___sec81'), + '___sec74'), + ('Cross-validation', 2, None, '___sec75'), + ('Computationally expensive', 2, None, '___sec76'), + ('Various steps in cross-validation', 2, None, '___sec77'), ('How to set up the cross-validation for Ridge and/or Lasso', 2, None, - '___sec82'), + '___sec78'), ('Resampling methods: Jackknife and Bootstrap', 2, None, - '___sec83'), - ('Resampling methods: Jackknife', 2, None, '___sec84'), - ('Jackknife code example', 2, None, '___sec85'), - ('Resampling methods: Bootstrap', 2, None, '___sec86'), - ('Resampling methods: Bootstrap background', 2, None, '___sec87'), + '___sec79'), + ('Resampling methods: Jackknife', 2, None, '___sec80'), + ('Jackknife code example', 2, None, '___sec81'), + ('Resampling methods: Bootstrap', 2, None, '___sec82'), + ('Resampling methods: Bootstrap background', 2, None, '___sec83'), ('Resampling methods: More Bootstrap background', 2, None, - '___sec88'), - ('Resampling methods: Bootstrap approach', 2, None, '___sec89'), - ('Resampling methods: Bootstrap steps', 2, None, '___sec90'), - ('Code example for the Bootstrap method', 2, None, '___sec91'), + '___sec84'), + ('Resampling methods: Bootstrap approach', 2, None, '___sec85'), + ('Resampling methods: Bootstrap steps', 2, None, '___sec86'), + ('Code example for the Bootstrap method', 2, None, '___sec87'), ('Code Example for Cross-validation and $k$-fold ' 'Cross-validation', 2, None, - '___sec92'), - ('The bias-variance tradeoff', 2, None, '___sec93'), - ('Example code for Bias-Variance tradeoff', 2, None, '___sec94'), - ('Understanding what happens', 2, None, '___sec95'), - ('Summing up', 2, None, '___sec96'), + '___sec88'), + ('The bias-variance tradeoff', 2, None, '___sec89'), + ('Example code for Bias-Variance tradeoff', 2, None, '___sec90'), + ('Understanding what happens', 2, None, '___sec91'), + ('Summing up', 2, None, '___sec92'), ("Another Example rom Scikit-Learn's Repository", 2, None, - '___sec97'), + '___sec93'), + ('The Ising model', 2, None, '___sec94'), + ('Reformulating the problem to suit regression', + 2, + None, + '___sec95'), + ('Linear regression', 2, None, '___sec96'), + ('Singular Value decomposition', 2, None, '___sec97'), ('The one-dimensional Ising model', 2, None, '___sec98'), ('Ridge regression', 2, None, '___sec99'), ('LASSO regression', 2, None, '___sec100'), @@ -295,74 +295,74 @@ MathJax.Hub.Config({
-A special version of the moments is the set of central moments, -the n-th central moment defined as: -$$ -\langle (x-\langle x \rangle )^n\rangle \equiv \int\! (x-\langle x\rangle)^n p(x)\,dx -$$ +
-The square root of the variance, \( \sigma =\sqrt{\langle (x-\langle x\rangle)^2\rangle} \) is called the standard deviation of \( p \). It is clearly just the RMS (root-mean-square) -value of the deviation of the PDF from its mean value, interpreted -qualitatively as the spread of \( p \) around its mean. -
# Importing various packages
+import numpy as np
+n = 100
+x = np.random.normal(size=n)
+print(np.mean(x))
+y = 4+3*x+np.random.normal(size=n)
+print(np.mean(y))
+z = x**3+np.random.normal(size=n)
+print(np.mean(z))
+W = np.vstack((x, y, z))
+Sigma = np.cov(W)
+print(Sigma)
+Eigvals, Eigvecs = np.linalg.eig(Sigma)
+print(Eigvals)
++ +
import numpy as np
+import matplotlib.pyplot as plt
+from scipy import sparse
+eye = np.eye(4)
+print(eye)
+sparse_mtx = sparse.csr_matrix(eye)
+print(sparse_mtx)
+x = np.linspace(-10,10,100)
+y = np.sin(x)
+plt.plot(x,y,marker='x')
+plt.show()
+
diff --git a/doc/pub/Regression/html/._Regression-bs054.html b/doc/pub/Regression/html/._Regression-bs054.html index 2ca729a79..deb8b2308 100644 --- a/doc/pub/Regression/html/._Regression-bs054.html +++ b/doc/pub/Regression/html/._Regression-bs054.html @@ -111,112 +111,112 @@ Automatically generated HTML file from DocOnce source ('The Boston housing data example', 2, None, '___sec27'), ('Housing data, the code', 2, None, '___sec28'), ('The singular value decomposition', 2, None, '___sec29'), - ('The Ising model', 2, None, '___sec30'), - ('Reformulating the problem to suit regression', - 2, - None, - '___sec31'), - ('Linear regression', 2, None, '___sec32'), - ('Singular Value decomposition', 2, None, '___sec33'), - ('Linear Regression Problems', 2, None, '___sec34'), - ('Fixing the singularity', 2, None, '___sec35'), - ('Basic math of the SVD', 2, None, '___sec36'), - ('The SVD, a Fantastic Algorithm', 2, None, '___sec37'), - ('Another Example', 2, None, '___sec38'), - ('Economy-size SVD', 2, None, '___sec39'), - ('Mathematical Properties', 2, None, '___sec40'), - ('Ridge and LASSO Regression', 2, None, '___sec41'), - ('More on Ridge Regression', 2, None, '___sec42'), - ('Interpreting the Ridge results', 2, None, '___sec43'), - ('More interpretations', 2, None, '___sec44'), - ('Where are we going?', 2, None, '___sec45'), - ('Resampling methods', 2, None, '___sec46'), + ('Linear Regression Problems', 2, None, '___sec30'), + ('Fixing the singularity', 2, None, '___sec31'), + ('Basic math of the SVD', 2, None, '___sec32'), + ('The SVD, a Fantastic Algorithm', 2, None, '___sec33'), + ('Another Example', 2, None, '___sec34'), + ('Economy-size SVD', 2, None, '___sec35'), + ('Mathematical Properties', 2, None, '___sec36'), + ('Ridge and LASSO Regression', 2, None, '___sec37'), + ('More on Ridge Regression', 2, None, '___sec38'), + ('Interpreting the Ridge results', 2, None, '___sec39'), + ('More interpretations', 2, None, '___sec40'), + ('Where are we going?', 2, None, '___sec41'), + ('Resampling methods', 2, None, '___sec42'), ('Resampling approaches can be computationally expensive', 2, None, - '___sec47'), - ('Why resampling methods ?', 2, None, '___sec48'), - ('Statistical analysis', 2, None, '___sec49'), - ('Statistics', 2, None, '___sec50'), - ('Statistics, moments', 2, None, '___sec51'), - ('Statistics, central moments', 2, None, '___sec52'), - ('Statistics, covariance', 2, None, '___sec53'), - ('Statistics, more covariance', 2, None, '___sec54'), - ('Covariance example', 2, None, '___sec55'), - ('Covariance in numpy', 2, None, '___sec56'), - ('Statistics, independent variables', 2, None, '___sec57'), - ('Statistics, more variance', 2, None, '___sec58'), - ('Statistics and stochastic processes', 2, None, '___sec59'), - ('Statistics and sample variables', 2, None, '___sec60'), + '___sec43'), + ('Why resampling methods ?', 2, None, '___sec44'), + ('Statistical analysis', 2, None, '___sec45'), + ('Statistics', 2, None, '___sec46'), + ('Statistics, moments', 2, None, '___sec47'), + ('Statistics, central moments', 2, None, '___sec48'), + ('Statistics, covariance', 2, None, '___sec49'), + ('Statistics, more covariance', 2, None, '___sec50'), + ('Covariance example', 2, None, '___sec51'), + ('Covariance in numpy', 2, None, '___sec52'), + ('Statistics, independent variables', 2, None, '___sec53'), + ('Statistics, more variance', 2, None, '___sec54'), + ('Statistics and stochastic processes', 2, None, '___sec55'), + ('Statistics and sample variables', 2, None, '___sec56'), ('Statistics, sample variance and covariance', 2, None, - '___sec61'), - ('Statistics, law of large numbers', 2, None, '___sec62'), - ('Statistics, more on sample error', 2, None, '___sec63'), - ('Statistics', 2, None, '___sec64'), - ('Statistics, central limit theorem', 2, None, '___sec65'), - ('Statistics, more technicalities', 2, None, '___sec66'), - ('Statistics', 2, None, '___sec67'), - ('Statistics and sample variance', 2, None, '___sec68'), - ('Statistics, uncorrelated results', 2, None, '___sec69'), - ('Statistics, computations', 2, None, '___sec70'), + '___sec57'), + ('Statistics, law of large numbers', 2, None, '___sec58'), + ('Statistics, more on sample error', 2, None, '___sec59'), + ('Statistics', 2, None, '___sec60'), + ('Statistics, central limit theorem', 2, None, '___sec61'), + ('Statistics, more technicalities', 2, None, '___sec62'), + ('Statistics', 2, None, '___sec63'), + ('Statistics and sample variance', 2, None, '___sec64'), + ('Statistics, uncorrelated results', 2, None, '___sec65'), + ('Statistics, computations', 2, None, '___sec66'), ('Statistics, more on computations of errors', 2, None, - '___sec71'), - ('Statistics, wrapping up 1', 2, None, '___sec72'), - ('Statistics, final expression', 2, None, '___sec73'), + '___sec67'), + ('Statistics, wrapping up 1', 2, None, '___sec68'), + ('Statistics, final expression', 2, None, '___sec69'), ('Statistics, effective number of correlations', 2, None, - '___sec74'), + '___sec70'), ('Linking the regression analysis with a statistical ' 'interpretation', 2, None, - '___sec75'), - ('Assumptions made', 2, None, '___sec76'), - ('Expectation value and variance', 2, None, '___sec77'), + '___sec71'), + ('Assumptions made', 2, None, '___sec72'), + ('Expectation value and variance', 2, None, '___sec73'), ('Expectation value and variance for $\\boldsymbol{\\beta}$', 2, None, - '___sec78'), - ('Cross-validation', 2, None, '___sec79'), - ('Computationally expensive', 2, None, '___sec80'), - ('Various steps in cross-validation', 2, None, '___sec81'), + '___sec74'), + ('Cross-validation', 2, None, '___sec75'), + ('Computationally expensive', 2, None, '___sec76'), + ('Various steps in cross-validation', 2, None, '___sec77'), ('How to set up the cross-validation for Ridge and/or Lasso', 2, None, - '___sec82'), + '___sec78'), ('Resampling methods: Jackknife and Bootstrap', 2, None, - '___sec83'), - ('Resampling methods: Jackknife', 2, None, '___sec84'), - ('Jackknife code example', 2, None, '___sec85'), - ('Resampling methods: Bootstrap', 2, None, '___sec86'), - ('Resampling methods: Bootstrap background', 2, None, '___sec87'), + '___sec79'), + ('Resampling methods: Jackknife', 2, None, '___sec80'), + ('Jackknife code example', 2, None, '___sec81'), + ('Resampling methods: Bootstrap', 2, None, '___sec82'), + ('Resampling methods: Bootstrap background', 2, None, '___sec83'), ('Resampling methods: More Bootstrap background', 2, None, - '___sec88'), - ('Resampling methods: Bootstrap approach', 2, None, '___sec89'), - ('Resampling methods: Bootstrap steps', 2, None, '___sec90'), - ('Code example for the Bootstrap method', 2, None, '___sec91'), + '___sec84'), + ('Resampling methods: Bootstrap approach', 2, None, '___sec85'), + ('Resampling methods: Bootstrap steps', 2, None, '___sec86'), + ('Code example for the Bootstrap method', 2, None, '___sec87'), ('Code Example for Cross-validation and $k$-fold ' 'Cross-validation', 2, None, - '___sec92'), - ('The bias-variance tradeoff', 2, None, '___sec93'), - ('Example code for Bias-Variance tradeoff', 2, None, '___sec94'), - ('Understanding what happens', 2, None, '___sec95'), - ('Summing up', 2, None, '___sec96'), + '___sec88'), + ('The bias-variance tradeoff', 2, None, '___sec89'), + ('Example code for Bias-Variance tradeoff', 2, None, '___sec90'), + ('Understanding what happens', 2, None, '___sec91'), + ('Summing up', 2, None, '___sec92'), ("Another Example rom Scikit-Learn's Repository", 2, None, - '___sec97'), + '___sec93'), + ('The Ising model', 2, None, '___sec94'), + ('Reformulating the problem to suit regression', + 2, + None, + '___sec95'), + ('Linear regression', 2, None, '___sec96'), + ('Singular Value decomposition', 2, None, '___sec97'), ('The one-dimensional Ising model', 2, None, '___sec98'), ('Ridge regression', 2, None, '___sec99'), ('LASSO regression', 2, None, '___sec100'), @@ -295,74 +295,74 @@ MathJax.Hub.Config({
-Another important quantity is the so called covariance, a variant of -the above defined variance. Consider again the set \( \{X_i\} \) of \( n \) -stochastic variables (not necessarily uncorrelated) with the -multivariate PDF \( P(x_1,\dots,x_n) \). The covariance of two -of the stochastic variables, \( X_i \) and \( X_j \), is defined as follows: +If \( X_i \) and \( X_j \) are independent, we get +\( \langle x_i x_j\rangle =\langle x_i\rangle\langle x_j\rangle \), resulting in \( \mathrm{cov}(X_i, X_j) = 0\ \ (i\neq j) \). + +
+Also useful for us is the covariance of linear combinations of +stochastic variables. Let \( \{X_i\} \) and \( \{Y_i\} \) be two sets of +stochastic variables. Let also \( \{a_i\} \) and \( \{b_i\} \) be two sets of +scalars. Consider the linear combination: $$ -\begin{align} -\mathrm{cov}(X_i,\,X_j) &\equiv \langle (x_i-\langle x_i\rangle)(x_j-\langle x_j\rangle)\rangle -\nonumber\\ -&= -\int\!\cdots\!\int\!(x_i-\langle x_i \rangle)(x_j-\langle x_j \rangle)\, -P(x_1,\dots,x_n)\,dx_1\dots dx_n -\tag{12} -\end{align} +U = \sum_i a_i X_i \qquad V = \sum_j b_j Y_j $$ -with +By the linearity of the expectation value $$ -\langle x_i\rangle = -\int\!\cdots\!\int\!x_i\,P(x_1,\dots,x_n)\,dx_1\dots dx_n +\mathrm{cov}(U, V) = \sum_{i,j}a_i b_j \mathrm{cov}(X_i, Y_j) $$
-If we consider the above covariance as a matrix \( C_{ij}=\mathrm{cov}(X_i,\,X_j) \), then the diagonal elements are just the familiar -variances, \( C_{ii} = \mathrm{cov}(X_i,\,X_i) = \mathrm{var}(X_i) \). It turns out that -all the off-diagonal elements are zero if the stochastic variables are -uncorrelated. This is easy to show, keeping in mind the linearity of -the expectation value. Consider the stochastic variables \( X_i \) and -\( X_j \), (\( i\neq j \)): +Now, since the variance is just \( \mathrm{var}(X_i) = \mathrm{cov}(X_i, X_i) \), we get +the variance of the linear combination \( U = \sum_i a_i X_i \): $$ -\begin{align} -\mathrm{cov}(X_i,\,X_j) &= \langle(x_i-\langle x_i\rangle)(x_j-\langle x_j\rangle)\rangle -\tag{13}\\ -&=\langle x_i x_j - x_i\langle x_j\rangle - \langle x_i\rangle x_j + \langle x_i\rangle\langle x_j\rangle\rangle -\tag{14}\\ -&=\langle x_i x_j\rangle - \langle x_i\langle x_j\rangle\rangle - \langle \langle x_i\rangle x_j\rangle + -\langle \langle x_i\rangle\langle x_j\rangle\rangle -\tag{15}\\ -&=\langle x_i x_j\rangle - \langle x_i\rangle\langle x_j\rangle - \langle x_i\rangle\langle x_j\rangle + -\langle x_i\rangle\langle x_j\rangle -\tag{16}\\ -&=\langle x_i x_j\rangle - \langle x_i\rangle\langle x_j\rangle -\tag{17} -\end{align} +\begin{equation} +\mathrm{var}(U) = \sum_{i,j}a_i a_j \mathrm{cov}(X_i, X_j) +\tag{12} +\end{equation} $$ + +And in the special case when the stochastic variables are +uncorrelated, the off-diagonal elements of the covariance are as we +know zero, resulting in: +$$ +\mathrm{var}(U) = \sum_i a_i^2 \mathrm{cov}(X_i, X_i) = \sum_i a_i^2 \mathrm{var}(X_i) +$$ + +$$ +\mathrm{var}(\sum_i a_i X_i) = \sum_i a_i^2 \mathrm{var}(X_i) +$$ + +which will become very useful in our study of the error in the mean +value of a set of measurements.
-Suppose we have defined three vectors \( \hat{x}, \hat{y}, \hat{z} \) with -\( n \) elements each. The covariance matrix is defined as - +
+A stochastic process is a process that produces sequentially a +chain of values: $$ -\hat{\Sigma} = \begin{bmatrix} \sigma_{xx} & \sigma_{xy} & \sigma_{xz} \\ - \sigma_{yx} & \sigma_{yy} & \sigma_{yz} \\ - \sigma_{zx} & \sigma_{zy} & \sigma_{zz} - \end{bmatrix}, +\{x_1, x_2,\dots\,x_k,\dots\}. $$ -where for example -$$ -\sigma_{xy} =\frac{1}{n} \sum_{i=0}^{n-1}(x_i- \overline{x})(y_i- \overline{y}). -$$ +We will call these +values our measurements and the entire set as our measured +sample. The action of measuring all the elements of a sample +we will call a stochastic experiment since, operationally, +they are often associated with results of empirical observation of +some physical or mathematical phenomena; precisely an experiment. We +assume that these values are distributed according to some +PDF \( p_X^{\phantom X}(x) \), where \( X \) is just the formal symbol for the +stochastic variable whose PDF is \( p_X^{\phantom X}(x) \). Instead of +trying to determine the full distribution \( p \) we are often only +interested in finding the few lowest moments, like the mean +\( \mu_X^{\phantom X} \) and the variance \( \sigma_X^{\phantom X} \). +
-The Numpy function np.cov calculates the covariance elements using -the factor \( 1/(n-1) \) instead of \( 1/n \) since it assumes we do not have -the exact mean valu\ es. - -
-The following simple function uses the np.vstack function which -takes each vector of dimension \( 1\times n \) and produces a \( 3\times n \) -matrix \( \hat{W} \) - -$$ -\hat{W} = \begin{bmatrix} x_0 & y_0 & z_0 \\ - x_1 & y_1 & z_1 \\ - x_2 & y_2 & z_2 \\ - \dots & \dots & \dots \\ - x_{n-2} & y_{n-2} & z_{n-2} \\ - x_{n-1} & y_{n-1} & z_{n-1} - \end{bmatrix}, -$$ - -
-which in turn is converted into into the \( 3\times 3 \) covariance matrix -\( \hat{\Sigma} \) via the Numpy function np.cov(). We note that we can -also calculate the mean value of each set of samples \( \hat{x} \) etc -using the Numpy function np.mean(x). We can also extract the -eigenvalues of the covariance matrix through the np.linalg.eig() -function.
diff --git a/doc/pub/Regression/html/._Regression-bs057.html b/doc/pub/Regression/html/._Regression-bs057.html index e969faf61..01bd4d7b1 100644 --- a/doc/pub/Regression/html/._Regression-bs057.html +++ b/doc/pub/Regression/html/._Regression-bs057.html @@ -111,112 +111,112 @@ Automatically generated HTML file from DocOnce source ('The Boston housing data example', 2, None, '___sec27'), ('Housing data, the code', 2, None, '___sec28'), ('The singular value decomposition', 2, None, '___sec29'), - ('The Ising model', 2, None, '___sec30'), - ('Reformulating the problem to suit regression', - 2, - None, - '___sec31'), - ('Linear regression', 2, None, '___sec32'), - ('Singular Value decomposition', 2, None, '___sec33'), - ('Linear Regression Problems', 2, None, '___sec34'), - ('Fixing the singularity', 2, None, '___sec35'), - ('Basic math of the SVD', 2, None, '___sec36'), - ('The SVD, a Fantastic Algorithm', 2, None, '___sec37'), - ('Another Example', 2, None, '___sec38'), - ('Economy-size SVD', 2, None, '___sec39'), - ('Mathematical Properties', 2, None, '___sec40'), - ('Ridge and LASSO Regression', 2, None, '___sec41'), - ('More on Ridge Regression', 2, None, '___sec42'), - ('Interpreting the Ridge results', 2, None, '___sec43'), - ('More interpretations', 2, None, '___sec44'), - ('Where are we going?', 2, None, '___sec45'), - ('Resampling methods', 2, None, '___sec46'), + ('Linear Regression Problems', 2, None, '___sec30'), + ('Fixing the singularity', 2, None, '___sec31'), + ('Basic math of the SVD', 2, None, '___sec32'), + ('The SVD, a Fantastic Algorithm', 2, None, '___sec33'), + ('Another Example', 2, None, '___sec34'), + ('Economy-size SVD', 2, None, '___sec35'), + ('Mathematical Properties', 2, None, '___sec36'), + ('Ridge and LASSO Regression', 2, None, '___sec37'), + ('More on Ridge Regression', 2, None, '___sec38'), + ('Interpreting the Ridge results', 2, None, '___sec39'), + ('More interpretations', 2, None, '___sec40'), + ('Where are we going?', 2, None, '___sec41'), + ('Resampling methods', 2, None, '___sec42'), ('Resampling approaches can be computationally expensive', 2, None, - '___sec47'), - ('Why resampling methods ?', 2, None, '___sec48'), - ('Statistical analysis', 2, None, '___sec49'), - ('Statistics', 2, None, '___sec50'), - ('Statistics, moments', 2, None, '___sec51'), - ('Statistics, central moments', 2, None, '___sec52'), - ('Statistics, covariance', 2, None, '___sec53'), - ('Statistics, more covariance', 2, None, '___sec54'), - ('Covariance example', 2, None, '___sec55'), - ('Covariance in numpy', 2, None, '___sec56'), - ('Statistics, independent variables', 2, None, '___sec57'), - ('Statistics, more variance', 2, None, '___sec58'), - ('Statistics and stochastic processes', 2, None, '___sec59'), - ('Statistics and sample variables', 2, None, '___sec60'), + '___sec43'), + ('Why resampling methods ?', 2, None, '___sec44'), + ('Statistical analysis', 2, None, '___sec45'), + ('Statistics', 2, None, '___sec46'), + ('Statistics, moments', 2, None, '___sec47'), + ('Statistics, central moments', 2, None, '___sec48'), + ('Statistics, covariance', 2, None, '___sec49'), + ('Statistics, more covariance', 2, None, '___sec50'), + ('Covariance example', 2, None, '___sec51'), + ('Covariance in numpy', 2, None, '___sec52'), + ('Statistics, independent variables', 2, None, '___sec53'), + ('Statistics, more variance', 2, None, '___sec54'), + ('Statistics and stochastic processes', 2, None, '___sec55'), + ('Statistics and sample variables', 2, None, '___sec56'), ('Statistics, sample variance and covariance', 2, None, - '___sec61'), - ('Statistics, law of large numbers', 2, None, '___sec62'), - ('Statistics, more on sample error', 2, None, '___sec63'), - ('Statistics', 2, None, '___sec64'), - ('Statistics, central limit theorem', 2, None, '___sec65'), - ('Statistics, more technicalities', 2, None, '___sec66'), - ('Statistics', 2, None, '___sec67'), - ('Statistics and sample variance', 2, None, '___sec68'), - ('Statistics, uncorrelated results', 2, None, '___sec69'), - ('Statistics, computations', 2, None, '___sec70'), + '___sec57'), + ('Statistics, law of large numbers', 2, None, '___sec58'), + ('Statistics, more on sample error', 2, None, '___sec59'), + ('Statistics', 2, None, '___sec60'), + ('Statistics, central limit theorem', 2, None, '___sec61'), + ('Statistics, more technicalities', 2, None, '___sec62'), + ('Statistics', 2, None, '___sec63'), + ('Statistics and sample variance', 2, None, '___sec64'), + ('Statistics, uncorrelated results', 2, None, '___sec65'), + ('Statistics, computations', 2, None, '___sec66'), ('Statistics, more on computations of errors', 2, None, - '___sec71'), - ('Statistics, wrapping up 1', 2, None, '___sec72'), - ('Statistics, final expression', 2, None, '___sec73'), + '___sec67'), + ('Statistics, wrapping up 1', 2, None, '___sec68'), + ('Statistics, final expression', 2, None, '___sec69'), ('Statistics, effective number of correlations', 2, None, - '___sec74'), + '___sec70'), ('Linking the regression analysis with a statistical ' 'interpretation', 2, None, - '___sec75'), - ('Assumptions made', 2, None, '___sec76'), - ('Expectation value and variance', 2, None, '___sec77'), + '___sec71'), + ('Assumptions made', 2, None, '___sec72'), + ('Expectation value and variance', 2, None, '___sec73'), ('Expectation value and variance for $\\boldsymbol{\\beta}$', 2, None, - '___sec78'), - ('Cross-validation', 2, None, '___sec79'), - ('Computationally expensive', 2, None, '___sec80'), - ('Various steps in cross-validation', 2, None, '___sec81'), + '___sec74'), + ('Cross-validation', 2, None, '___sec75'), + ('Computationally expensive', 2, None, '___sec76'), + ('Various steps in cross-validation', 2, None, '___sec77'), ('How to set up the cross-validation for Ridge and/or Lasso', 2, None, - '___sec82'), + '___sec78'), ('Resampling methods: Jackknife and Bootstrap', 2, None, - '___sec83'), - ('Resampling methods: Jackknife', 2, None, '___sec84'), - ('Jackknife code example', 2, None, '___sec85'), - ('Resampling methods: Bootstrap', 2, None, '___sec86'), - ('Resampling methods: Bootstrap background', 2, None, '___sec87'), + '___sec79'), + ('Resampling methods: Jackknife', 2, None, '___sec80'), + ('Jackknife code example', 2, None, '___sec81'), + ('Resampling methods: Bootstrap', 2, None, '___sec82'), + ('Resampling methods: Bootstrap background', 2, None, '___sec83'), ('Resampling methods: More Bootstrap background', 2, None, - '___sec88'), - ('Resampling methods: Bootstrap approach', 2, None, '___sec89'), - ('Resampling methods: Bootstrap steps', 2, None, '___sec90'), - ('Code example for the Bootstrap method', 2, None, '___sec91'), + '___sec84'), + ('Resampling methods: Bootstrap approach', 2, None, '___sec85'), + ('Resampling methods: Bootstrap steps', 2, None, '___sec86'), + ('Code example for the Bootstrap method', 2, None, '___sec87'), ('Code Example for Cross-validation and $k$-fold ' 'Cross-validation', 2, None, - '___sec92'), - ('The bias-variance tradeoff', 2, None, '___sec93'), - ('Example code for Bias-Variance tradeoff', 2, None, '___sec94'), - ('Understanding what happens', 2, None, '___sec95'), - ('Summing up', 2, None, '___sec96'), + '___sec88'), + ('The bias-variance tradeoff', 2, None, '___sec89'), + ('Example code for Bias-Variance tradeoff', 2, None, '___sec90'), + ('Understanding what happens', 2, None, '___sec91'), + ('Summing up', 2, None, '___sec92'), ("Another Example rom Scikit-Learn's Repository", 2, None, - '___sec97'), + '___sec93'), + ('The Ising model', 2, None, '___sec94'), + ('Reformulating the problem to suit regression', + 2, + None, + '___sec95'), + ('Linear regression', 2, None, '___sec96'), + ('Singular Value decomposition', 2, None, '___sec97'), ('The one-dimensional Ising model', 2, None, '___sec98'), ('Ridge regression', 2, None, '___sec99'), ('LASSO regression', 2, None, '___sec100'), @@ -295,74 +295,74 @@ MathJax.Hub.Config({
- + -
+In practical situations a sample is always of finite size. Let that +size be \( n \). The expectation value of a sample, the sample mean, is then defined as follows: +$$ +\bar{x}_n \equiv \frac{1}{n}\sum_{k=1}^n x_k +$$ -
+The sample variance is: +$$ +\mathrm{var}(x) \equiv \frac{1}{n}\sum_{k=1}^n (x_k - \bar{x}_n)^2 +$$ - -
# Importing various packages
-import numpy as np
+its square root being the standard deviation of the sample. The
+sample covariance is:
+$$
+\mathrm{cov}(x)\equiv\frac{1}{n}\sum_{kl}(x_k - \bar{x}_n)(x_l - \bar{x}_n)
+$$
+- -
import numpy as np
-import matplotlib.pyplot as plt
-from scipy import sparse
-eye = np.eye(4)
-print(eye)
-sparse_mtx = sparse.csr_matrix(eye)
-print(sparse_mtx)
-x = np.linspace(-10,10,100)
-y = np.sin(x)
-plt.plot(x,y,marker='x')
-plt.show()
-
diff --git a/doc/pub/Regression/html/._Regression-bs058.html b/doc/pub/Regression/html/._Regression-bs058.html index b4a0d26a7..280ac6b3d 100644 --- a/doc/pub/Regression/html/._Regression-bs058.html +++ b/doc/pub/Regression/html/._Regression-bs058.html @@ -111,112 +111,112 @@ Automatically generated HTML file from DocOnce source ('The Boston housing data example', 2, None, '___sec27'), ('Housing data, the code', 2, None, '___sec28'), ('The singular value decomposition', 2, None, '___sec29'), - ('The Ising model', 2, None, '___sec30'), - ('Reformulating the problem to suit regression', - 2, - None, - '___sec31'), - ('Linear regression', 2, None, '___sec32'), - ('Singular Value decomposition', 2, None, '___sec33'), - ('Linear Regression Problems', 2, None, '___sec34'), - ('Fixing the singularity', 2, None, '___sec35'), - ('Basic math of the SVD', 2, None, '___sec36'), - ('The SVD, a Fantastic Algorithm', 2, None, '___sec37'), - ('Another Example', 2, None, '___sec38'), - ('Economy-size SVD', 2, None, '___sec39'), - ('Mathematical Properties', 2, None, '___sec40'), - ('Ridge and LASSO Regression', 2, None, '___sec41'), - ('More on Ridge Regression', 2, None, '___sec42'), - ('Interpreting the Ridge results', 2, None, '___sec43'), - ('More interpretations', 2, None, '___sec44'), - ('Where are we going?', 2, None, '___sec45'), - ('Resampling methods', 2, None, '___sec46'), + ('Linear Regression Problems', 2, None, '___sec30'), + ('Fixing the singularity', 2, None, '___sec31'), + ('Basic math of the SVD', 2, None, '___sec32'), + ('The SVD, a Fantastic Algorithm', 2, None, '___sec33'), + ('Another Example', 2, None, '___sec34'), + ('Economy-size SVD', 2, None, '___sec35'), + ('Mathematical Properties', 2, None, '___sec36'), + ('Ridge and LASSO Regression', 2, None, '___sec37'), + ('More on Ridge Regression', 2, None, '___sec38'), + ('Interpreting the Ridge results', 2, None, '___sec39'), + ('More interpretations', 2, None, '___sec40'), + ('Where are we going?', 2, None, '___sec41'), + ('Resampling methods', 2, None, '___sec42'), ('Resampling approaches can be computationally expensive', 2, None, - '___sec47'), - ('Why resampling methods ?', 2, None, '___sec48'), - ('Statistical analysis', 2, None, '___sec49'), - ('Statistics', 2, None, '___sec50'), - ('Statistics, moments', 2, None, '___sec51'), - ('Statistics, central moments', 2, None, '___sec52'), - ('Statistics, covariance', 2, None, '___sec53'), - ('Statistics, more covariance', 2, None, '___sec54'), - ('Covariance example', 2, None, '___sec55'), - ('Covariance in numpy', 2, None, '___sec56'), - ('Statistics, independent variables', 2, None, '___sec57'), - ('Statistics, more variance', 2, None, '___sec58'), - ('Statistics and stochastic processes', 2, None, '___sec59'), - ('Statistics and sample variables', 2, None, '___sec60'), + '___sec43'), + ('Why resampling methods ?', 2, None, '___sec44'), + ('Statistical analysis', 2, None, '___sec45'), + ('Statistics', 2, None, '___sec46'), + ('Statistics, moments', 2, None, '___sec47'), + ('Statistics, central moments', 2, None, '___sec48'), + ('Statistics, covariance', 2, None, '___sec49'), + ('Statistics, more covariance', 2, None, '___sec50'), + ('Covariance example', 2, None, '___sec51'), + ('Covariance in numpy', 2, None, '___sec52'), + ('Statistics, independent variables', 2, None, '___sec53'), + ('Statistics, more variance', 2, None, '___sec54'), + ('Statistics and stochastic processes', 2, None, '___sec55'), + ('Statistics and sample variables', 2, None, '___sec56'), ('Statistics, sample variance and covariance', 2, None, - '___sec61'), - ('Statistics, law of large numbers', 2, None, '___sec62'), - ('Statistics, more on sample error', 2, None, '___sec63'), - ('Statistics', 2, None, '___sec64'), - ('Statistics, central limit theorem', 2, None, '___sec65'), - ('Statistics, more technicalities', 2, None, '___sec66'), - ('Statistics', 2, None, '___sec67'), - ('Statistics and sample variance', 2, None, '___sec68'), - ('Statistics, uncorrelated results', 2, None, '___sec69'), - ('Statistics, computations', 2, None, '___sec70'), + '___sec57'), + ('Statistics, law of large numbers', 2, None, '___sec58'), + ('Statistics, more on sample error', 2, None, '___sec59'), + ('Statistics', 2, None, '___sec60'), + ('Statistics, central limit theorem', 2, None, '___sec61'), + ('Statistics, more technicalities', 2, None, '___sec62'), + ('Statistics', 2, None, '___sec63'), + ('Statistics and sample variance', 2, None, '___sec64'), + ('Statistics, uncorrelated results', 2, None, '___sec65'), + ('Statistics, computations', 2, None, '___sec66'), ('Statistics, more on computations of errors', 2, None, - '___sec71'), - ('Statistics, wrapping up 1', 2, None, '___sec72'), - ('Statistics, final expression', 2, None, '___sec73'), + '___sec67'), + ('Statistics, wrapping up 1', 2, None, '___sec68'), + ('Statistics, final expression', 2, None, '___sec69'), ('Statistics, effective number of correlations', 2, None, - '___sec74'), + '___sec70'), ('Linking the regression analysis with a statistical ' 'interpretation', 2, None, - '___sec75'), - ('Assumptions made', 2, None, '___sec76'), - ('Expectation value and variance', 2, None, '___sec77'), + '___sec71'), + ('Assumptions made', 2, None, '___sec72'), + ('Expectation value and variance', 2, None, '___sec73'), ('Expectation value and variance for $\\boldsymbol{\\beta}$', 2, None, - '___sec78'), - ('Cross-validation', 2, None, '___sec79'), - ('Computationally expensive', 2, None, '___sec80'), - ('Various steps in cross-validation', 2, None, '___sec81'), + '___sec74'), + ('Cross-validation', 2, None, '___sec75'), + ('Computationally expensive', 2, None, '___sec76'), + ('Various steps in cross-validation', 2, None, '___sec77'), ('How to set up the cross-validation for Ridge and/or Lasso', 2, None, - '___sec82'), + '___sec78'), ('Resampling methods: Jackknife and Bootstrap', 2, None, - '___sec83'), - ('Resampling methods: Jackknife', 2, None, '___sec84'), - ('Jackknife code example', 2, None, '___sec85'), - ('Resampling methods: Bootstrap', 2, None, '___sec86'), - ('Resampling methods: Bootstrap background', 2, None, '___sec87'), + '___sec79'), + ('Resampling methods: Jackknife', 2, None, '___sec80'), + ('Jackknife code example', 2, None, '___sec81'), + ('Resampling methods: Bootstrap', 2, None, '___sec82'), + ('Resampling methods: Bootstrap background', 2, None, '___sec83'), ('Resampling methods: More Bootstrap background', 2, None, - '___sec88'), - ('Resampling methods: Bootstrap approach', 2, None, '___sec89'), - ('Resampling methods: Bootstrap steps', 2, None, '___sec90'), - ('Code example for the Bootstrap method', 2, None, '___sec91'), + '___sec84'), + ('Resampling methods: Bootstrap approach', 2, None, '___sec85'), + ('Resampling methods: Bootstrap steps', 2, None, '___sec86'), + ('Code example for the Bootstrap method', 2, None, '___sec87'), ('Code Example for Cross-validation and $k$-fold ' 'Cross-validation', 2, None, - '___sec92'), - ('The bias-variance tradeoff', 2, None, '___sec93'), - ('Example code for Bias-Variance tradeoff', 2, None, '___sec94'), - ('Understanding what happens', 2, None, '___sec95'), - ('Summing up', 2, None, '___sec96'), + '___sec88'), + ('The bias-variance tradeoff', 2, None, '___sec89'), + ('Example code for Bias-Variance tradeoff', 2, None, '___sec90'), + ('Understanding what happens', 2, None, '___sec91'), + ('Summing up', 2, None, '___sec92'), ("Another Example rom Scikit-Learn's Repository", 2, None, - '___sec97'), + '___sec93'), + ('The Ising model', 2, None, '___sec94'), + ('Reformulating the problem to suit regression', + 2, + None, + '___sec95'), + ('Linear regression', 2, None, '___sec96'), + ('Singular Value decomposition', 2, None, '___sec97'), ('The one-dimensional Ising model', 2, None, '___sec98'), ('Ridge regression', 2, None, '___sec99'), ('LASSO regression', 2, None, '___sec100'), @@ -295,74 +295,74 @@ MathJax.Hub.Config({
-If \( X_i \) and \( X_j \) are independent, we get -\( \langle x_i x_j\rangle =\langle x_i\rangle\langle x_j\rangle \), resulting in \( \mathrm{cov}(X_i, X_j) = 0\ \ (i\neq j) \). +Note that the sample variance is the sample covariance without the +cross terms. In a similar manner as the covariance in Eq. (6) is a measure of the correlation between +two stochastic variables, the above defined sample covariance is a +measure of the sequential correlation between succeeding measurements +of a sample.
-Also useful for us is the covariance of linear combinations of -stochastic variables. Let \( \{X_i\} \) and \( \{Y_i\} \) be two sets of -stochastic variables. Let also \( \{a_i\} \) and \( \{b_i\} \) be two sets of -scalars. Consider the linear combination: -$$ -U = \sum_i a_i X_i \qquad V = \sum_j b_j Y_j -$$ - -By the linearity of the expectation value -$$ -\mathrm{cov}(U, V) = \sum_{i,j}a_i b_j \mathrm{cov}(X_i, Y_j) -$$ +These quantities, being known experimental values, differ +significantly from and must not be confused with the similarly named +quantities for stochastic variables, mean \( \mu_X \), variance \( \mathrm{var}(X) \) +and covariance \( \mathrm{cov}(X,Y) \).
-Now, since the variance is just \( \mathrm{var}(X_i) = \mathrm{cov}(X_i, X_i) \), we get -the variance of the linear combination \( U = \sum_i a_i X_i \): +The law of large numbers +states that as the size of our sample grows to infinity, the sample +mean approaches the true mean \( \mu_X^{\phantom X} \) of the chosen PDF: $$ -\begin{equation} -\mathrm{var}(U) = \sum_{i,j}a_i a_j \mathrm{cov}(X_i, X_j) -\tag{18} -\end{equation} +\lim_{n\to\infty}\bar{x}_n = \mu_X^{\phantom X} $$ -And in the special case when the stochastic variables are -uncorrelated, the off-diagonal elements of the covariance are as we -know zero, resulting in: -$$ -\mathrm{var}(U) = \sum_i a_i^2 \mathrm{cov}(X_i, X_i) = \sum_i a_i^2 \mathrm{var}(X_i) -$$ +The sample mean \( \bar{x}_n \) works therefore as an estimate of the true +mean \( \mu_X^{\phantom X} \). -$$ -\mathrm{var}(\sum_i a_i X_i) = \sum_i a_i^2 \mathrm{var}(X_i) -$$ - -which will become very useful in our study of the error in the mean -value of a set of measurements. +
+What we need to find out is how good an approximation \( \bar{x}_n \) is to +\( \mu_X^{\phantom X} \). In any stochastic measurement, an estimated +mean is of no use to us without a measure of its error. A quantity +that tells us how well we can reproduce it in another experiment. We +are therefore interested in the PDF of the sample mean itself. Its +standard deviation will be a measure of the spread of sample means, +and we will simply call it the error of the sample mean, or +just sample error, and denote it by \( \mathrm{err}_X^{\phantom X} \). In +practice, we will only be able to produce an estimate of the +sample error since the exact value would require the knowledge of the +true PDFs behind, which we usually do not have.
-A stochastic process is a process that produces sequentially a -chain of values: +Let us first take a look at what happens to the sample error as the +size of the sample grows. In a sample, each of the measurements \( x_i \) +can be associated with its own stochastic variable \( X_i \). The +stochastic variable \( \overline X_n \) for the sample mean \( \bar{x}_n \) is +then just a linear combination, already familiar to us: $$ -\{x_1, x_2,\dots\,x_k,\dots\}. +\overline X_n = \frac{1}{n}\sum_{i=1}^n X_i $$ -We will call these -values our measurements and the entire set as our measured -sample. The action of measuring all the elements of a sample -we will call a stochastic experiment since, operationally, -they are often associated with results of empirical observation of -some physical or mathematical phenomena; precisely an experiment. We -assume that these values are distributed according to some -PDF \( p_X^{\phantom X}(x) \), where \( X \) is just the formal symbol for the -stochastic variable whose PDF is \( p_X^{\phantom X}(x) \). Instead of -trying to determine the full distribution \( p \) we are often only -interested in finding the few lowest moments, like the mean -\( \mu_X^{\phantom X} \) and the variance \( \sigma_X^{\phantom X} \). +All the coefficients are just equal \( 1/n \). The PDF of \( \overline X_n \), +denoted by \( p_{\overline X_n}(x) \) is the desired PDF of the sample +means.
- + -
-In practical situations a sample is always of finite size. Let that -size be \( n \). The expectation value of a sample, the sample mean, is then defined as follows: +The probability density of obtaining a sample mean \( \bar x_n \) +is the product of probabilities of obtaining arbitrary values \( x_1, +x_2,\dots,x_n \) with the constraint that the mean of the set \( \{x_i\} \) +is \( \bar x_n \): $$ -\bar{x}_n \equiv \frac{1}{n}\sum_{k=1}^n x_k +p_{\overline X_n}(x) = \int p_X^{\phantom X}(x_1)\cdots +\int p_X^{\phantom X}(x_n)\ +\delta\!\left(x - \frac{x_1+x_2+\dots+x_n}{n}\right)dx_n \cdots dx_1 $$ -The sample variance is: -$$ -\mathrm{var}(x) \equiv \frac{1}{n}\sum_{k=1}^n (x_k - \bar{x}_n)^2 -$$ - -its square root being the standard deviation of the sample. The -sample covariance is: -$$ -\mathrm{cov}(x)\equiv\frac{1}{n}\sum_{kl}(x_k - \bar{x}_n)(x_l - \bar{x}_n) -$$ +And in particular we are interested in its variance \( \mathrm{var}(\overline X_n) \).
-Note that the sample variance is the sample covariance without the -cross terms. In a similar manner as the covariance in Eq. (12) is a measure of the correlation between -two stochastic variables, the above defined sample covariance is a -measure of the sequential correlation between succeeding measurements -of a sample. - -
-These quantities, being known experimental values, differ -significantly from and must not be confused with the similarly named -quantities for stochastic variables, mean \( \mu_X \), variance \( \mathrm{var}(X) \) -and covariance \( \mathrm{cov}(X,Y) \). +It is generally not possible to express \( p_{\overline X_n}(x) \) in a +closed form given an arbitrary PDF \( p_X^{\phantom X} \) and a number +\( n \). But for the limit \( n\to\infty \) it is possible to make an +approximation. The very important result is called the central limit theorem. It tells us that as \( n \) goes to infinity, +\( p_{\overline X_n}(x) \) approaches a Gaussian distribution whose mean +and variance equal the true mean and variance, \( \mu_{X}^{\phantom X} \) +and \( \sigma_{X}^{2} \), respectively: +$$ +\begin{equation} +\lim_{n\to\infty} p_{\overline X_n}(x) = +\left(\frac{n}{2\pi\mathrm{var}(X)}\right)^{1/2} +e^{-\frac{n(x-\bar x_n)^2}{2\mathrm{var}(X)}} +\tag{13} +\end{equation} +$$
-The law of large numbers -states that as the size of our sample grows to infinity, the sample -mean approaches the true mean \( \mu_X^{\phantom X} \) of the chosen PDF: +The desired variance +\( \mathrm{var}(\overline X_n) \), i.e. the sample error squared +\( \mathrm{err}_X^2 \), is given by: $$ -\lim_{n\to\infty}\bar{x}_n = \mu_X^{\phantom X} +\begin{equation} +\mathrm{err}_X^2 = \mathrm{var}(\overline X_n) = \frac{1}{n^2} +\sum_{ij} \mathrm{cov}(X_i, X_j) +\tag{14} +\end{equation} $$ -The sample mean \( \bar{x}_n \) works therefore as an estimate of the true -mean \( \mu_X^{\phantom X} \). - -
-What we need to find out is how good an approximation \( \bar{x}_n \) is to -\( \mu_X^{\phantom X} \). In any stochastic measurement, an estimated -mean is of no use to us without a measure of its error. A quantity -that tells us how well we can reproduce it in another experiment. We -are therefore interested in the PDF of the sample mean itself. Its -standard deviation will be a measure of the spread of sample means, -and we will simply call it the error of the sample mean, or -just sample error, and denote it by \( \mathrm{err}_X^{\phantom X} \). In -practice, we will only be able to produce an estimate of the -sample error since the exact value would require the knowledge of the -true PDFs behind, which we usually do not have. +We see now that in order to calculate the exact error of the sample +with the above expression, we would need the true means +\( \mu_{X_i}^{\phantom X} \) of the stochastic variables \( X_i \). To +calculate these requires that we know the true multivariate PDF of all +the \( X_i \). But this PDF is unknown to us, we have only got the measurements of +one sample. The best we can do is to let the sample itself be an +estimate of the PDF of each of the \( X_i \), estimating all properties of +\( X_i \) through the measurements of the sample.
-Let us first take a look at what happens to the sample error as the -size of the sample grows. In a sample, each of the measurements \( x_i \) -can be associated with its own stochastic variable \( X_i \). The -stochastic variable \( \overline X_n \) for the sample mean \( \bar{x}_n \) is -then just a linear combination, already familiar to us: +Our estimate of \( \mu_{X_i}^{\phantom X} \) is then the sample mean \( \bar x \) +itself, in accordance with the the central limit theorem: $$ -\overline X_n = \frac{1}{n}\sum_{i=1}^n X_i +\mu_{X_i}^{\phantom X} = \langle x_i\rangle \approx \frac{1}{n}\sum_{k=1}^n x_k = \bar x $$ -All the coefficients are just equal \( 1/n \). The PDF of \( \overline X_n \), -denoted by \( p_{\overline X_n}(x) \) is the desired PDF of the sample -means. +Using \( \bar x \) in place of \( \mu_{X_i}^{\phantom X} \) we can give an +estimate of the covariance in Eq. (14) +$$ +\mathrm{cov}(X_i, X_j) = \langle (x_i-\langle x_i\rangle)(x_j-\langle x_j\rangle)\rangle +\approx\langle (x_i - \bar x)(x_j - \bar{x})\rangle, +$$ + +resulting in +$$ +\frac{1}{n} \sum_{l}^n \left(\frac{1}{n}\sum_{k}^n (x_k -\bar x_n)(x_l - \bar x_n)\right)=\frac{1}{n}\frac{1}{n} \sum_{kl} (x_k -\bar x_n)(x_l - \bar x_n)=\frac{1}{n}\mathrm{cov}(x) +$$
-The probability density of obtaining a sample mean \( \bar x_n \) -is the product of probabilities of obtaining arbitrary values \( x_1, -x_2,\dots,x_n \) with the constraint that the mean of the set \( \{x_i\} \) -is \( \bar x_n \): +By the same procedure we can use the sample variance as an +estimate of the variance of any of the stochastic variables \( X_i \) $$ -p_{\overline X_n}(x) = \int p_X^{\phantom X}(x_1)\cdots -\int p_X^{\phantom X}(x_n)\ -\delta\!\left(x - \frac{x_1+x_2+\dots+x_n}{n}\right)dx_n \cdots dx_1 +\mathrm{var}(X_i)=\langle x_i - \langle x_i\rangle\rangle \approx \langle x_i - \bar x_n\rangle\nonumber, $$ -And in particular we are interested in its variance \( \mathrm{var}(\overline X_n) \). +which is approximated as +$$ +\begin{equation} +\mathrm{var}(X_i)\approx \frac{1}{n}\sum_{k=1}^n (x_k - \bar x_n)=\mathrm{var}(x) +\tag{15} +\end{equation} +$$ + +
+Now we can calculate an estimate of the error +\( \mathrm{err}_X^{\phantom X} \) of the sample mean \( \bar x_n \): +$$ +\begin{align} +\mathrm{err}_X^2 +&=\frac{1}{n^2}\sum_{ij} \mathrm{cov}(X_i, X_j) \nonumber \\ +&\approx&\frac{1}{n^2}\sum_{ij}\frac{1}{n}\mathrm{cov}(x) =\frac{1}{n^2}n^2\frac{1}{n}\mathrm{cov}(x)\nonumber\\ +&=\frac{1}{n}\mathrm{cov}(x) +\tag{16} +\end{align} +$$ + +which is nothing but the sample covariance divided by the number of +measurements in the sample.
-It is generally not possible to express \( p_{\overline X_n}(x) \) in a -closed form given an arbitrary PDF \( p_X^{\phantom X} \) and a number -\( n \). But for the limit \( n\to\infty \) it is possible to make an -approximation. The very important result is called the central limit theorem. It tells us that as \( n \) goes to infinity, -\( p_{\overline X_n}(x) \) approaches a Gaussian distribution whose mean -and variance equal the true mean and variance, \( \mu_{X}^{\phantom X} \) -and \( \sigma_{X}^{2} \), respectively: + +
+In the special case that the measurements of the sample are +uncorrelated (equivalently the stochastic variables \( X_i \) are +uncorrelated) we have that the off-diagonal elements of the covariance +are zero. This gives the following estimate of the sample error: +$$ +\mathrm{err}_X^2=\frac{1}{n^2}\sum_{ij} \mathrm{cov}(X_i, X_j) = +\frac{1}{n^2} \sum_i \mathrm{var}(X_i), +$$ + +resulting in $$ \begin{equation} -\lim_{n\to\infty} p_{\overline X_n}(x) = -\left(\frac{n}{2\pi\mathrm{var}(X)}\right)^{1/2} -e^{-\frac{n(x-\bar x_n)^2}{2\mathrm{var}(X)}} -\tag{19} +\mathrm{err}_X^2\approx \frac{1}{n^2} \sum_i \mathrm{var}(x)= \frac{1}{n}\mathrm{var}(x) +\tag{17} \end{equation} $$ + +where in the second step we have used Eq. (15). +The error of the sample is then just its standard deviation divided by +the square root of the number of measurements the sample contains. +This is a very useful formula which is easy to compute. It acts as a +first approximation to the error, but in numerical experiments, we +cannot overlook the always present correlations.
-The desired variance -\( \mathrm{var}(\overline X_n) \), i.e. the sample error squared -\( \mathrm{err}_X^2 \), is given by: +For computational purposes one usually splits up the estimate of +\( \mathrm{err}_X^2 \), given by Eq. (16), into two +parts +$$ +\mathrm{err}_X^2 = \frac{1}{n}\mathrm{var}(x) + \frac{1}{n}(\mathrm{cov}(x)-\mathrm{var}(x)), +$$ + +which equals $$ \begin{equation} -\mathrm{err}_X^2 = \mathrm{var}(\overline X_n) = \frac{1}{n^2} -\sum_{ij} \mathrm{cov}(X_i, X_j) -\tag{20} +\frac{1}{n^2}\sum_{k=1}^n (x_k - \bar x_n)^2 +\frac{2}{n^2}\sum_{k < l} (x_k - \bar x_n)(x_l - \bar x_n) +\tag{18} \end{equation} $$ -We see now that in order to calculate the exact error of the sample -with the above expression, we would need the true means -\( \mu_{X_i}^{\phantom X} \) of the stochastic variables \( X_i \). To -calculate these requires that we know the true multivariate PDF of all -the \( X_i \). But this PDF is unknown to us, we have only got the measurements of -one sample. The best we can do is to let the sample itself be an -estimate of the PDF of each of the \( X_i \), estimating all properties of -\( X_i \) through the measurements of the sample. +The first term is the same as the error in the uncorrelated case, +Eq. (17). This means that the second +term accounts for the error correction due to correlation between the +measurements. For uncorrelated measurements this second term is zero.
-Our estimate of \( \mu_{X_i}^{\phantom X} \) is then the sample mean \( \bar x \) -itself, in accordance with the the central limit theorem: +Computationally the uncorrelated first term is much easier to treat +efficiently than the second. $$ -\mu_{X_i}^{\phantom X} = \langle x_i\rangle \approx \frac{1}{n}\sum_{k=1}^n x_k = \bar x +\mathrm{var}(x) = \frac{1}{n}\sum_{k=1}^n (x_k - \bar x_n)^2 = +\left(\frac{1}{n}\sum_{k=1}^n x_k^2\right) - \bar x_n^2 $$ -Using \( \bar x \) in place of \( \mu_{X_i}^{\phantom X} \) we can give an -estimate of the covariance in Eq. (20) -$$ -\mathrm{cov}(X_i, X_j) = \langle (x_i-\langle x_i\rangle)(x_j-\langle x_j\rangle)\rangle -\approx\langle (x_i - \bar x)(x_j - \bar{x})\rangle, -$$ - -resulting in -$$ -\frac{1}{n} \sum_{l}^n \left(\frac{1}{n}\sum_{k}^n (x_k -\bar x_n)(x_l - \bar x_n)\right)=\frac{1}{n}\frac{1}{n} \sum_{kl} (x_k -\bar x_n)(x_l - \bar x_n)=\frac{1}{n}\mathrm{cov}(x) -$$ +We just accumulate separately the values \( x^2 \) and \( x \) for every +measurement \( x \) we receive. The correlation term, though, has to be +calculated at the end of the experiment since we need all the +measurements to calculate the cross terms. Therefore, all measurements +have to be stored throughout the experiment.
-By the same procedure we can use the sample variance as an -estimate of the variance of any of the stochastic variables \( X_i \) +Let us analyze the problem by splitting up the correlation term into +partial sums of the form: $$ -\mathrm{var}(X_i)=\langle x_i - \langle x_i\rangle\rangle \approx \langle x_i - \bar x_n\rangle\nonumber, +f_d = \frac{1}{n-d}\sum_{k=1}^{n-d}(x_k - \bar x_n)(x_{k+d} - \bar x_n) $$ -which is approximated as +The correlation term of the error can now be rewritten in terms of +\( f_d \) $$ -\begin{equation} -\mathrm{var}(X_i)\approx \frac{1}{n}\sum_{k=1}^n (x_k - \bar x_n)=\mathrm{var}(x) -\tag{21} -\end{equation} +\frac{2}{n}\sum_{k < l} (x_k - \bar x_n)(x_l - \bar x_n) = +2\sum_{d=1}^{n-1} f_d $$ -
-Now we can calculate an estimate of the error -\( \mathrm{err}_X^{\phantom X} \) of the sample mean \( \bar x_n \): +The value of \( f_d \) reflects the correlation between measurements +separated by the distance \( d \) in the sample samples. Notice that for +\( d=0 \), \( f \) is just the sample variance, \( \mathrm{var}(x) \). If we divide \( f_d \) +by \( \mathrm{var}(x) \), we arrive at the so called autocorrelation function $$ -\begin{align} -\mathrm{err}_X^2 -&=\frac{1}{n^2}\sum_{ij} \mathrm{cov}(X_i, X_j) \nonumber \\ -&\approx&\frac{1}{n^2}\sum_{ij}\frac{1}{n}\mathrm{cov}(x) =\frac{1}{n^2}n^2\frac{1}{n}\mathrm{cov}(x)\nonumber\\ -&=\frac{1}{n}\mathrm{cov}(x) -\tag{22} -\end{align} +\kappa_d = \frac{f_d}{\mathrm{var}(x)} $$ -which is nothing but the sample covariance divided by the number of -measurements in the sample. +which gives us a useful measure of pairwise correlations +starting always at \( 1 \) for \( d=0 \).
- -
-In the special case that the measurements of the sample are -uncorrelated (equivalently the stochastic variables \( X_i \) are -uncorrelated) we have that the off-diagonal elements of the covariance -are zero. This gives the following estimate of the sample error: +The sample error (see eq. (18)) can now be +written in terms of the autocorrelation function: $$ -\mathrm{err}_X^2=\frac{1}{n^2}\sum_{ij} \mathrm{cov}(X_i, X_j) = -\frac{1}{n^2} \sum_i \mathrm{var}(X_i), +\begin{align} +\mathrm{err}_X^2 &= +\frac{1}{n}\mathrm{var}(x)+\frac{2}{n}\cdot\mathrm{var}(x)\sum_{d=1}^{n-1} +\frac{f_d}{\mathrm{var}(x)}\nonumber\\ &=& +\left(1+2\sum_{d=1}^{n-1}\kappa_d\right)\frac{1}{n}\mathrm{var}(x)\nonumber\\ +&=\frac{\tau}{n}\cdot\mathrm{var}(x) +\tag{19} +\end{align} $$ -resulting in +and we see that \( \mathrm{err}_X \) can be expressed in terms the +uncorrelated sample variance times a correction factor \( \tau \) which +accounts for the correlation between measurements. We call this +correction factor the autocorrelation time: $$ \begin{equation} -\mathrm{err}_X^2\approx \frac{1}{n^2} \sum_i \mathrm{var}(x)= \frac{1}{n}\mathrm{var}(x) -\tag{23} +\tau = 1+2\sum_{d=1}^{n-1}\kappa_d +\tag{20} \end{equation} $$ - -where in the second step we have used Eq. (21). -The error of the sample is then just its standard deviation divided by -the square root of the number of measurements the sample contains. -This is a very useful formula which is easy to compute. It acts as a -first approximation to the error, but in numerical experiments, we -cannot overlook the always present correlations.
-For computational purposes one usually splits up the estimate of -\( \mathrm{err}_X^2 \), given by Eq. (22), into two -parts +For a correlation free experiment, \( \tau \) +equals 1. From the point of view of +eq. (19) we can interpret a sequential +correlation as an effective reduction of the number of measurements by +a factor \( \tau \). The effective number of measurements becomes: $$ -\mathrm{err}_X^2 = \frac{1}{n}\mathrm{var}(x) + \frac{1}{n}(\mathrm{cov}(x)-\mathrm{var}(x)), +n_\mathrm{eff} = \frac{n}{\tau} $$ -which equals -$$ -\begin{equation} -\frac{1}{n^2}\sum_{k=1}^n (x_k - \bar x_n)^2 +\frac{2}{n^2}\sum_{k < l} (x_k - \bar x_n)(x_l - \bar x_n) -\tag{24} -\end{equation} -$$ - -The first term is the same as the error in the uncorrelated case, -Eq. (23). This means that the second -term accounts for the error correction due to correlation between the -measurements. For uncorrelated measurements this second term is zero. +To neglect the autocorrelation time \( \tau \) will always cause our +simple uncorrelated estimate of \( \mathrm{err}_X^2\approx \mathrm{var}(x)/n \) to +be less than the true sample error. The estimate of the error will be +too good. On the other hand, the calculation of the full +autocorrelation time poses an efficiency problem if the set of +measurements is very large.
- + -
-Computationally the uncorrelated first term is much easier to treat -efficiently than the second. +
+Finally, we are going to discuss several statistical properties which can be obtained in terms of analytical expressions. +The +advantage of doing linear regression is that we actually end up with +analytical expressions for several statistical quantities. +Standard least squares and Ridge regression allow us to +derive quantities like the variance and other expectation values in a +rather straightforward way. + +
+It is assumed that \( \varepsilon_i +\sim \mathcal{N}(0, \sigma^2) \) and the \( \varepsilon_{i} \) are +independent, i.e.: $$ -\mathrm{var}(x) = \frac{1}{n}\sum_{k=1}^n (x_k - \bar x_n)^2 = -\left(\frac{1}{n}\sum_{k=1}^n x_k^2\right) - \bar x_n^2 +\begin{align*} +\mbox{Cov}(\varepsilon_{i_1}, +\varepsilon_{i_2}) & = \left\{ \begin{array}{lcc} \sigma^2 & \mbox{if} +& i_1 = i_2, \\ 0 & \mbox{if} & i_1 \not= i_2. \end{array} \right. +\end{align*} $$ -We just accumulate separately the values \( x^2 \) and \( x \) for every -measurement \( x \) we receive. The correlation term, though, has to be -calculated at the end of the experiment since we need all the -measurements to calculate the cross terms. Therefore, all measurements -have to be stored throughout the experiment. -
+Recall that \( \boldsymbol{X} \) is a matrix of dimensionality \( n\times p \). The +notation above \( \mathbf{X}_{i,\ast} \) means that we are looking at the +row number \( i \) and perform a sum over all values \( p \).
diff --git a/doc/pub/Regression/html/._Regression-bs073.html b/doc/pub/Regression/html/._Regression-bs073.html index 541bac035..ebcb6cf6c 100644 --- a/doc/pub/Regression/html/._Regression-bs073.html +++ b/doc/pub/Regression/html/._Regression-bs073.html @@ -111,112 +111,112 @@ Automatically generated HTML file from DocOnce source ('The Boston housing data example', 2, None, '___sec27'), ('Housing data, the code', 2, None, '___sec28'), ('The singular value decomposition', 2, None, '___sec29'), - ('The Ising model', 2, None, '___sec30'), - ('Reformulating the problem to suit regression', - 2, - None, - '___sec31'), - ('Linear regression', 2, None, '___sec32'), - ('Singular Value decomposition', 2, None, '___sec33'), - ('Linear Regression Problems', 2, None, '___sec34'), - ('Fixing the singularity', 2, None, '___sec35'), - ('Basic math of the SVD', 2, None, '___sec36'), - ('The SVD, a Fantastic Algorithm', 2, None, '___sec37'), - ('Another Example', 2, None, '___sec38'), - ('Economy-size SVD', 2, None, '___sec39'), - ('Mathematical Properties', 2, None, '___sec40'), - ('Ridge and LASSO Regression', 2, None, '___sec41'), - ('More on Ridge Regression', 2, None, '___sec42'), - ('Interpreting the Ridge results', 2, None, '___sec43'), - ('More interpretations', 2, None, '___sec44'), - ('Where are we going?', 2, None, '___sec45'), - ('Resampling methods', 2, None, '___sec46'), + ('Linear Regression Problems', 2, None, '___sec30'), + ('Fixing the singularity', 2, None, '___sec31'), + ('Basic math of the SVD', 2, None, '___sec32'), + ('The SVD, a Fantastic Algorithm', 2, None, '___sec33'), + ('Another Example', 2, None, '___sec34'), + ('Economy-size SVD', 2, None, '___sec35'), + ('Mathematical Properties', 2, None, '___sec36'), + ('Ridge and LASSO Regression', 2, None, '___sec37'), + ('More on Ridge Regression', 2, None, '___sec38'), + ('Interpreting the Ridge results', 2, None, '___sec39'), + ('More interpretations', 2, None, '___sec40'), + ('Where are we going?', 2, None, '___sec41'), + ('Resampling methods', 2, None, '___sec42'), ('Resampling approaches can be computationally expensive', 2, None, - '___sec47'), - ('Why resampling methods ?', 2, None, '___sec48'), - ('Statistical analysis', 2, None, '___sec49'), - ('Statistics', 2, None, '___sec50'), - ('Statistics, moments', 2, None, '___sec51'), - ('Statistics, central moments', 2, None, '___sec52'), - ('Statistics, covariance', 2, None, '___sec53'), - ('Statistics, more covariance', 2, None, '___sec54'), - ('Covariance example', 2, None, '___sec55'), - ('Covariance in numpy', 2, None, '___sec56'), - ('Statistics, independent variables', 2, None, '___sec57'), - ('Statistics, more variance', 2, None, '___sec58'), - ('Statistics and stochastic processes', 2, None, '___sec59'), - ('Statistics and sample variables', 2, None, '___sec60'), + '___sec43'), + ('Why resampling methods ?', 2, None, '___sec44'), + ('Statistical analysis', 2, None, '___sec45'), + ('Statistics', 2, None, '___sec46'), + ('Statistics, moments', 2, None, '___sec47'), + ('Statistics, central moments', 2, None, '___sec48'), + ('Statistics, covariance', 2, None, '___sec49'), + ('Statistics, more covariance', 2, None, '___sec50'), + ('Covariance example', 2, None, '___sec51'), + ('Covariance in numpy', 2, None, '___sec52'), + ('Statistics, independent variables', 2, None, '___sec53'), + ('Statistics, more variance', 2, None, '___sec54'), + ('Statistics and stochastic processes', 2, None, '___sec55'), + ('Statistics and sample variables', 2, None, '___sec56'), ('Statistics, sample variance and covariance', 2, None, - '___sec61'), - ('Statistics, law of large numbers', 2, None, '___sec62'), - ('Statistics, more on sample error', 2, None, '___sec63'), - ('Statistics', 2, None, '___sec64'), - ('Statistics, central limit theorem', 2, None, '___sec65'), - ('Statistics, more technicalities', 2, None, '___sec66'), - ('Statistics', 2, None, '___sec67'), - ('Statistics and sample variance', 2, None, '___sec68'), - ('Statistics, uncorrelated results', 2, None, '___sec69'), - ('Statistics, computations', 2, None, '___sec70'), + '___sec57'), + ('Statistics, law of large numbers', 2, None, '___sec58'), + ('Statistics, more on sample error', 2, None, '___sec59'), + ('Statistics', 2, None, '___sec60'), + ('Statistics, central limit theorem', 2, None, '___sec61'), + ('Statistics, more technicalities', 2, None, '___sec62'), + ('Statistics', 2, None, '___sec63'), + ('Statistics and sample variance', 2, None, '___sec64'), + ('Statistics, uncorrelated results', 2, None, '___sec65'), + ('Statistics, computations', 2, None, '___sec66'), ('Statistics, more on computations of errors', 2, None, - '___sec71'), - ('Statistics, wrapping up 1', 2, None, '___sec72'), - ('Statistics, final expression', 2, None, '___sec73'), + '___sec67'), + ('Statistics, wrapping up 1', 2, None, '___sec68'), + ('Statistics, final expression', 2, None, '___sec69'), ('Statistics, effective number of correlations', 2, None, - '___sec74'), + '___sec70'), ('Linking the regression analysis with a statistical ' 'interpretation', 2, None, - '___sec75'), - ('Assumptions made', 2, None, '___sec76'), - ('Expectation value and variance', 2, None, '___sec77'), + '___sec71'), + ('Assumptions made', 2, None, '___sec72'), + ('Expectation value and variance', 2, None, '___sec73'), ('Expectation value and variance for $\\boldsymbol{\\beta}$', 2, None, - '___sec78'), - ('Cross-validation', 2, None, '___sec79'), - ('Computationally expensive', 2, None, '___sec80'), - ('Various steps in cross-validation', 2, None, '___sec81'), + '___sec74'), + ('Cross-validation', 2, None, '___sec75'), + ('Computationally expensive', 2, None, '___sec76'), + ('Various steps in cross-validation', 2, None, '___sec77'), ('How to set up the cross-validation for Ridge and/or Lasso', 2, None, - '___sec82'), + '___sec78'), ('Resampling methods: Jackknife and Bootstrap', 2, None, - '___sec83'), - ('Resampling methods: Jackknife', 2, None, '___sec84'), - ('Jackknife code example', 2, None, '___sec85'), - ('Resampling methods: Bootstrap', 2, None, '___sec86'), - ('Resampling methods: Bootstrap background', 2, None, '___sec87'), + '___sec79'), + ('Resampling methods: Jackknife', 2, None, '___sec80'), + ('Jackknife code example', 2, None, '___sec81'), + ('Resampling methods: Bootstrap', 2, None, '___sec82'), + ('Resampling methods: Bootstrap background', 2, None, '___sec83'), ('Resampling methods: More Bootstrap background', 2, None, - '___sec88'), - ('Resampling methods: Bootstrap approach', 2, None, '___sec89'), - ('Resampling methods: Bootstrap steps', 2, None, '___sec90'), - ('Code example for the Bootstrap method', 2, None, '___sec91'), + '___sec84'), + ('Resampling methods: Bootstrap approach', 2, None, '___sec85'), + ('Resampling methods: Bootstrap steps', 2, None, '___sec86'), + ('Code example for the Bootstrap method', 2, None, '___sec87'), ('Code Example for Cross-validation and $k$-fold ' 'Cross-validation', 2, None, - '___sec92'), - ('The bias-variance tradeoff', 2, None, '___sec93'), - ('Example code for Bias-Variance tradeoff', 2, None, '___sec94'), - ('Understanding what happens', 2, None, '___sec95'), - ('Summing up', 2, None, '___sec96'), + '___sec88'), + ('The bias-variance tradeoff', 2, None, '___sec89'), + ('Example code for Bias-Variance tradeoff', 2, None, '___sec90'), + ('Understanding what happens', 2, None, '___sec91'), + ('Summing up', 2, None, '___sec92'), ("Another Example rom Scikit-Learn's Repository", 2, None, - '___sec97'), + '___sec93'), + ('The Ising model', 2, None, '___sec94'), + ('Reformulating the problem to suit regression', + 2, + None, + '___sec95'), + ('Linear regression', 2, None, '___sec96'), + ('Singular Value decomposition', 2, None, '___sec97'), ('The one-dimensional Ising model', 2, None, '___sec98'), ('Ridge regression', 2, None, '___sec99'), ('LASSO regression', 2, None, '___sec100'), @@ -295,74 +295,74 @@ MathJax.Hub.Config({
-Let us analyze the problem by splitting up the correlation term into -partial sums of the form: +
+The assumption we have made here can be summarized as (and this is going to useful when we discuss the bias-variance trade off) +that there exists a function \( f(\boldsymbol{x}) \) and a normal distributed error \( \boldsymbol{\varepsilon}\sim \mathcal{N}(0, \sigma^2) \) +which describes our data $$ -f_d = \frac{1}{n-d}\sum_{k=1}^{n-d}(x_k - \bar x_n)(x_{k+d} - \bar x_n) +\boldsymbol{y} = f(\boldsymbol{x})+\boldsymbol{\varepsilon} $$ -The correlation term of the error can now be rewritten in terms of -\( f_d \) +
+We approximate this function with our model from the solution of the linear regression equations, that is our +function \( f \) is approximated by \( \boldsymbol{\tilde{y}} \) where we want to minimize \( (\boldsymbol{y}-\boldsymbol{\tilde{y}})^2 \), our MSE, with $$ -\frac{2}{n}\sum_{k < l} (x_k - \bar x_n)(x_l - \bar x_n) = -2\sum_{d=1}^{n-1} f_d +\boldsymbol{\tilde{y}} = \boldsymbol{X}\boldsymbol{\beta}. $$ -The value of \( f_d \) reflects the correlation between measurements -separated by the distance \( d \) in the sample samples. Notice that for -\( d=0 \), \( f \) is just the sample variance, \( \mathrm{var}(x) \). If we divide \( f_d \) -by \( \mathrm{var}(x) \), we arrive at the so called autocorrelation function -$$ -\kappa_d = \frac{f_d}{\mathrm{var}(x)} -$$ - -which gives us a useful measure of pairwise correlations -starting always at \( 1 \) for \( d=0 \). -
diff --git a/doc/pub/Regression/html/._Regression-bs074.html b/doc/pub/Regression/html/._Regression-bs074.html index 4fe97e6f0..0732b2c11 100644 --- a/doc/pub/Regression/html/._Regression-bs074.html +++ b/doc/pub/Regression/html/._Regression-bs074.html @@ -111,112 +111,112 @@ Automatically generated HTML file from DocOnce source ('The Boston housing data example', 2, None, '___sec27'), ('Housing data, the code', 2, None, '___sec28'), ('The singular value decomposition', 2, None, '___sec29'), - ('The Ising model', 2, None, '___sec30'), - ('Reformulating the problem to suit regression', - 2, - None, - '___sec31'), - ('Linear regression', 2, None, '___sec32'), - ('Singular Value decomposition', 2, None, '___sec33'), - ('Linear Regression Problems', 2, None, '___sec34'), - ('Fixing the singularity', 2, None, '___sec35'), - ('Basic math of the SVD', 2, None, '___sec36'), - ('The SVD, a Fantastic Algorithm', 2, None, '___sec37'), - ('Another Example', 2, None, '___sec38'), - ('Economy-size SVD', 2, None, '___sec39'), - ('Mathematical Properties', 2, None, '___sec40'), - ('Ridge and LASSO Regression', 2, None, '___sec41'), - ('More on Ridge Regression', 2, None, '___sec42'), - ('Interpreting the Ridge results', 2, None, '___sec43'), - ('More interpretations', 2, None, '___sec44'), - ('Where are we going?', 2, None, '___sec45'), - ('Resampling methods', 2, None, '___sec46'), + ('Linear Regression Problems', 2, None, '___sec30'), + ('Fixing the singularity', 2, None, '___sec31'), + ('Basic math of the SVD', 2, None, '___sec32'), + ('The SVD, a Fantastic Algorithm', 2, None, '___sec33'), + ('Another Example', 2, None, '___sec34'), + ('Economy-size SVD', 2, None, '___sec35'), + ('Mathematical Properties', 2, None, '___sec36'), + ('Ridge and LASSO Regression', 2, None, '___sec37'), + ('More on Ridge Regression', 2, None, '___sec38'), + ('Interpreting the Ridge results', 2, None, '___sec39'), + ('More interpretations', 2, None, '___sec40'), + ('Where are we going?', 2, None, '___sec41'), + ('Resampling methods', 2, None, '___sec42'), ('Resampling approaches can be computationally expensive', 2, None, - '___sec47'), - ('Why resampling methods ?', 2, None, '___sec48'), - ('Statistical analysis', 2, None, '___sec49'), - ('Statistics', 2, None, '___sec50'), - ('Statistics, moments', 2, None, '___sec51'), - ('Statistics, central moments', 2, None, '___sec52'), - ('Statistics, covariance', 2, None, '___sec53'), - ('Statistics, more covariance', 2, None, '___sec54'), - ('Covariance example', 2, None, '___sec55'), - ('Covariance in numpy', 2, None, '___sec56'), - ('Statistics, independent variables', 2, None, '___sec57'), - ('Statistics, more variance', 2, None, '___sec58'), - ('Statistics and stochastic processes', 2, None, '___sec59'), - ('Statistics and sample variables', 2, None, '___sec60'), + '___sec43'), + ('Why resampling methods ?', 2, None, '___sec44'), + ('Statistical analysis', 2, None, '___sec45'), + ('Statistics', 2, None, '___sec46'), + ('Statistics, moments', 2, None, '___sec47'), + ('Statistics, central moments', 2, None, '___sec48'), + ('Statistics, covariance', 2, None, '___sec49'), + ('Statistics, more covariance', 2, None, '___sec50'), + ('Covariance example', 2, None, '___sec51'), + ('Covariance in numpy', 2, None, '___sec52'), + ('Statistics, independent variables', 2, None, '___sec53'), + ('Statistics, more variance', 2, None, '___sec54'), + ('Statistics and stochastic processes', 2, None, '___sec55'), + ('Statistics and sample variables', 2, None, '___sec56'), ('Statistics, sample variance and covariance', 2, None, - '___sec61'), - ('Statistics, law of large numbers', 2, None, '___sec62'), - ('Statistics, more on sample error', 2, None, '___sec63'), - ('Statistics', 2, None, '___sec64'), - ('Statistics, central limit theorem', 2, None, '___sec65'), - ('Statistics, more technicalities', 2, None, '___sec66'), - ('Statistics', 2, None, '___sec67'), - ('Statistics and sample variance', 2, None, '___sec68'), - ('Statistics, uncorrelated results', 2, None, '___sec69'), - ('Statistics, computations', 2, None, '___sec70'), + '___sec57'), + ('Statistics, law of large numbers', 2, None, '___sec58'), + ('Statistics, more on sample error', 2, None, '___sec59'), + ('Statistics', 2, None, '___sec60'), + ('Statistics, central limit theorem', 2, None, '___sec61'), + ('Statistics, more technicalities', 2, None, '___sec62'), + ('Statistics', 2, None, '___sec63'), + ('Statistics and sample variance', 2, None, '___sec64'), + ('Statistics, uncorrelated results', 2, None, '___sec65'), + ('Statistics, computations', 2, None, '___sec66'), ('Statistics, more on computations of errors', 2, None, - '___sec71'), - ('Statistics, wrapping up 1', 2, None, '___sec72'), - ('Statistics, final expression', 2, None, '___sec73'), + '___sec67'), + ('Statistics, wrapping up 1', 2, None, '___sec68'), + ('Statistics, final expression', 2, None, '___sec69'), ('Statistics, effective number of correlations', 2, None, - '___sec74'), + '___sec70'), ('Linking the regression analysis with a statistical ' 'interpretation', 2, None, - '___sec75'), - ('Assumptions made', 2, None, '___sec76'), - ('Expectation value and variance', 2, None, '___sec77'), + '___sec71'), + ('Assumptions made', 2, None, '___sec72'), + ('Expectation value and variance', 2, None, '___sec73'), ('Expectation value and variance for $\\boldsymbol{\\beta}$', 2, None, - '___sec78'), - ('Cross-validation', 2, None, '___sec79'), - ('Computationally expensive', 2, None, '___sec80'), - ('Various steps in cross-validation', 2, None, '___sec81'), + '___sec74'), + ('Cross-validation', 2, None, '___sec75'), + ('Computationally expensive', 2, None, '___sec76'), + ('Various steps in cross-validation', 2, None, '___sec77'), ('How to set up the cross-validation for Ridge and/or Lasso', 2, None, - '___sec82'), + '___sec78'), ('Resampling methods: Jackknife and Bootstrap', 2, None, - '___sec83'), - ('Resampling methods: Jackknife', 2, None, '___sec84'), - ('Jackknife code example', 2, None, '___sec85'), - ('Resampling methods: Bootstrap', 2, None, '___sec86'), - ('Resampling methods: Bootstrap background', 2, None, '___sec87'), + '___sec79'), + ('Resampling methods: Jackknife', 2, None, '___sec80'), + ('Jackknife code example', 2, None, '___sec81'), + ('Resampling methods: Bootstrap', 2, None, '___sec82'), + ('Resampling methods: Bootstrap background', 2, None, '___sec83'), ('Resampling methods: More Bootstrap background', 2, None, - '___sec88'), - ('Resampling methods: Bootstrap approach', 2, None, '___sec89'), - ('Resampling methods: Bootstrap steps', 2, None, '___sec90'), - ('Code example for the Bootstrap method', 2, None, '___sec91'), + '___sec84'), + ('Resampling methods: Bootstrap approach', 2, None, '___sec85'), + ('Resampling methods: Bootstrap steps', 2, None, '___sec86'), + ('Code example for the Bootstrap method', 2, None, '___sec87'), ('Code Example for Cross-validation and $k$-fold ' 'Cross-validation', 2, None, - '___sec92'), - ('The bias-variance tradeoff', 2, None, '___sec93'), - ('Example code for Bias-Variance tradeoff', 2, None, '___sec94'), - ('Understanding what happens', 2, None, '___sec95'), - ('Summing up', 2, None, '___sec96'), + '___sec88'), + ('The bias-variance tradeoff', 2, None, '___sec89'), + ('Example code for Bias-Variance tradeoff', 2, None, '___sec90'), + ('Understanding what happens', 2, None, '___sec91'), + ('Summing up', 2, None, '___sec92'), ("Another Example rom Scikit-Learn's Repository", 2, None, - '___sec97'), + '___sec93'), + ('The Ising model', 2, None, '___sec94'), + ('Reformulating the problem to suit regression', + 2, + None, + '___sec95'), + ('Linear regression', 2, None, '___sec96'), + ('Singular Value decomposition', 2, None, '___sec97'), ('The one-dimensional Ising model', 2, None, '___sec98'), ('Ridge regression', 2, None, '___sec99'), ('LASSO regression', 2, None, '___sec100'), @@ -295,74 +295,74 @@ MathJax.Hub.Config({
-The sample error (see eq. (24)) can now be -written in terms of the autocorrelation function: +
+We can calculate the expectation value of \( \boldsymbol{y} \) for a given element \( i \) $$ -\begin{align} -\mathrm{err}_X^2 &= -\frac{1}{n}\mathrm{var}(x)+\frac{2}{n}\cdot\mathrm{var}(x)\sum_{d=1}^{n-1} -\frac{f_d}{\mathrm{var}(x)}\nonumber\\ &=& -\left(1+2\sum_{d=1}^{n-1}\kappa_d\right)\frac{1}{n}\mathrm{var}(x)\nonumber\\ -&=\frac{\tau}{n}\cdot\mathrm{var}(x) -\tag{25} -\end{align} +\begin{align*} +\mathbb{E}(y_i) & = +\mathbb{E}(\mathbf{X}_{i, \ast} \, \boldsymbol{\beta}) + \mathbb{E}(\varepsilon_i) +\, \, \, = \, \, \, \mathbf{X}_{i, \ast} \, \beta, +\end{align*} $$ -and we see that \( \mathrm{err}_X \) can be expressed in terms the -uncorrelated sample variance times a correction factor \( \tau \) which -accounts for the correlation between measurements. We call this -correction factor the autocorrelation time: +while +its variance is $$ -\begin{equation} -\tau = 1+2\sum_{d=1}^{n-1}\kappa_d -\tag{26} -\end{equation} +\begin{align*} \mbox{Var}(y_i) & = \mathbb{E} \{ [y_i +- \mathbb{E}(y_i)]^2 \} \, \, \, = \, \, \, \mathbb{E} ( y_i^2 ) - +[\mathbb{E}(y_i)]^2 \\ & = \mathbb{E} [ ( \mathbf{X}_{i, \ast} \, +\beta + \varepsilon_i )^2] - ( \mathbf{X}_{i, \ast} \, \boldsymbol{\beta})^2 \\ & += \mathbb{E} [ ( \mathbf{X}_{i, \ast} \, \boldsymbol{\beta})^2 + 2 \varepsilon_i +\mathbf{X}_{i, \ast} \, \boldsymbol{\beta} + \varepsilon_i^2 ] - ( \mathbf{X}_{i, +\ast} \, \beta)^2 \\ & = ( \mathbf{X}_{i, \ast} \, \boldsymbol{\beta})^2 + 2 +\mathbb{E}(\varepsilon_i) \mathbf{X}_{i, \ast} \, \boldsymbol{\beta} + +\mathbb{E}(\varepsilon_i^2 ) - ( \mathbf{X}_{i, \ast} \, \boldsymbol{\beta})^2 +\\ & = \mathbb{E}(\varepsilon_i^2 ) \, \, \, = \, \, \, +\mbox{Var}(\varepsilon_i) \, \, \, = \, \, \, \sigma^2. +\end{align*} $$ -
diff --git a/doc/pub/Regression/html/._Regression-bs075.html b/doc/pub/Regression/html/._Regression-bs075.html index faf8babc4..0af6f2aa2 100644 --- a/doc/pub/Regression/html/._Regression-bs075.html +++ b/doc/pub/Regression/html/._Regression-bs075.html @@ -111,112 +111,112 @@ Automatically generated HTML file from DocOnce source ('The Boston housing data example', 2, None, '___sec27'), ('Housing data, the code', 2, None, '___sec28'), ('The singular value decomposition', 2, None, '___sec29'), - ('The Ising model', 2, None, '___sec30'), - ('Reformulating the problem to suit regression', - 2, - None, - '___sec31'), - ('Linear regression', 2, None, '___sec32'), - ('Singular Value decomposition', 2, None, '___sec33'), - ('Linear Regression Problems', 2, None, '___sec34'), - ('Fixing the singularity', 2, None, '___sec35'), - ('Basic math of the SVD', 2, None, '___sec36'), - ('The SVD, a Fantastic Algorithm', 2, None, '___sec37'), - ('Another Example', 2, None, '___sec38'), - ('Economy-size SVD', 2, None, '___sec39'), - ('Mathematical Properties', 2, None, '___sec40'), - ('Ridge and LASSO Regression', 2, None, '___sec41'), - ('More on Ridge Regression', 2, None, '___sec42'), - ('Interpreting the Ridge results', 2, None, '___sec43'), - ('More interpretations', 2, None, '___sec44'), - ('Where are we going?', 2, None, '___sec45'), - ('Resampling methods', 2, None, '___sec46'), + ('Linear Regression Problems', 2, None, '___sec30'), + ('Fixing the singularity', 2, None, '___sec31'), + ('Basic math of the SVD', 2, None, '___sec32'), + ('The SVD, a Fantastic Algorithm', 2, None, '___sec33'), + ('Another Example', 2, None, '___sec34'), + ('Economy-size SVD', 2, None, '___sec35'), + ('Mathematical Properties', 2, None, '___sec36'), + ('Ridge and LASSO Regression', 2, None, '___sec37'), + ('More on Ridge Regression', 2, None, '___sec38'), + ('Interpreting the Ridge results', 2, None, '___sec39'), + ('More interpretations', 2, None, '___sec40'), + ('Where are we going?', 2, None, '___sec41'), + ('Resampling methods', 2, None, '___sec42'), ('Resampling approaches can be computationally expensive', 2, None, - '___sec47'), - ('Why resampling methods ?', 2, None, '___sec48'), - ('Statistical analysis', 2, None, '___sec49'), - ('Statistics', 2, None, '___sec50'), - ('Statistics, moments', 2, None, '___sec51'), - ('Statistics, central moments', 2, None, '___sec52'), - ('Statistics, covariance', 2, None, '___sec53'), - ('Statistics, more covariance', 2, None, '___sec54'), - ('Covariance example', 2, None, '___sec55'), - ('Covariance in numpy', 2, None, '___sec56'), - ('Statistics, independent variables', 2, None, '___sec57'), - ('Statistics, more variance', 2, None, '___sec58'), - ('Statistics and stochastic processes', 2, None, '___sec59'), - ('Statistics and sample variables', 2, None, '___sec60'), + '___sec43'), + ('Why resampling methods ?', 2, None, '___sec44'), + ('Statistical analysis', 2, None, '___sec45'), + ('Statistics', 2, None, '___sec46'), + ('Statistics, moments', 2, None, '___sec47'), + ('Statistics, central moments', 2, None, '___sec48'), + ('Statistics, covariance', 2, None, '___sec49'), + ('Statistics, more covariance', 2, None, '___sec50'), + ('Covariance example', 2, None, '___sec51'), + ('Covariance in numpy', 2, None, '___sec52'), + ('Statistics, independent variables', 2, None, '___sec53'), + ('Statistics, more variance', 2, None, '___sec54'), + ('Statistics and stochastic processes', 2, None, '___sec55'), + ('Statistics and sample variables', 2, None, '___sec56'), ('Statistics, sample variance and covariance', 2, None, - '___sec61'), - ('Statistics, law of large numbers', 2, None, '___sec62'), - ('Statistics, more on sample error', 2, None, '___sec63'), - ('Statistics', 2, None, '___sec64'), - ('Statistics, central limit theorem', 2, None, '___sec65'), - ('Statistics, more technicalities', 2, None, '___sec66'), - ('Statistics', 2, None, '___sec67'), - ('Statistics and sample variance', 2, None, '___sec68'), - ('Statistics, uncorrelated results', 2, None, '___sec69'), - ('Statistics, computations', 2, None, '___sec70'), + '___sec57'), + ('Statistics, law of large numbers', 2, None, '___sec58'), + ('Statistics, more on sample error', 2, None, '___sec59'), + ('Statistics', 2, None, '___sec60'), + ('Statistics, central limit theorem', 2, None, '___sec61'), + ('Statistics, more technicalities', 2, None, '___sec62'), + ('Statistics', 2, None, '___sec63'), + ('Statistics and sample variance', 2, None, '___sec64'), + ('Statistics, uncorrelated results', 2, None, '___sec65'), + ('Statistics, computations', 2, None, '___sec66'), ('Statistics, more on computations of errors', 2, None, - '___sec71'), - ('Statistics, wrapping up 1', 2, None, '___sec72'), - ('Statistics, final expression', 2, None, '___sec73'), + '___sec67'), + ('Statistics, wrapping up 1', 2, None, '___sec68'), + ('Statistics, final expression', 2, None, '___sec69'), ('Statistics, effective number of correlations', 2, None, - '___sec74'), + '___sec70'), ('Linking the regression analysis with a statistical ' 'interpretation', 2, None, - '___sec75'), - ('Assumptions made', 2, None, '___sec76'), - ('Expectation value and variance', 2, None, '___sec77'), + '___sec71'), + ('Assumptions made', 2, None, '___sec72'), + ('Expectation value and variance', 2, None, '___sec73'), ('Expectation value and variance for $\\boldsymbol{\\beta}$', 2, None, - '___sec78'), - ('Cross-validation', 2, None, '___sec79'), - ('Computationally expensive', 2, None, '___sec80'), - ('Various steps in cross-validation', 2, None, '___sec81'), + '___sec74'), + ('Cross-validation', 2, None, '___sec75'), + ('Computationally expensive', 2, None, '___sec76'), + ('Various steps in cross-validation', 2, None, '___sec77'), ('How to set up the cross-validation for Ridge and/or Lasso', 2, None, - '___sec82'), + '___sec78'), ('Resampling methods: Jackknife and Bootstrap', 2, None, - '___sec83'), - ('Resampling methods: Jackknife', 2, None, '___sec84'), - ('Jackknife code example', 2, None, '___sec85'), - ('Resampling methods: Bootstrap', 2, None, '___sec86'), - ('Resampling methods: Bootstrap background', 2, None, '___sec87'), + '___sec79'), + ('Resampling methods: Jackknife', 2, None, '___sec80'), + ('Jackknife code example', 2, None, '___sec81'), + ('Resampling methods: Bootstrap', 2, None, '___sec82'), + ('Resampling methods: Bootstrap background', 2, None, '___sec83'), ('Resampling methods: More Bootstrap background', 2, None, - '___sec88'), - ('Resampling methods: Bootstrap approach', 2, None, '___sec89'), - ('Resampling methods: Bootstrap steps', 2, None, '___sec90'), - ('Code example for the Bootstrap method', 2, None, '___sec91'), + '___sec84'), + ('Resampling methods: Bootstrap approach', 2, None, '___sec85'), + ('Resampling methods: Bootstrap steps', 2, None, '___sec86'), + ('Code example for the Bootstrap method', 2, None, '___sec87'), ('Code Example for Cross-validation and $k$-fold ' 'Cross-validation', 2, None, - '___sec92'), - ('The bias-variance tradeoff', 2, None, '___sec93'), - ('Example code for Bias-Variance tradeoff', 2, None, '___sec94'), - ('Understanding what happens', 2, None, '___sec95'), - ('Summing up', 2, None, '___sec96'), + '___sec88'), + ('The bias-variance tradeoff', 2, None, '___sec89'), + ('Example code for Bias-Variance tradeoff', 2, None, '___sec90'), + ('Understanding what happens', 2, None, '___sec91'), + ('Summing up', 2, None, '___sec92'), ("Another Example rom Scikit-Learn's Repository", 2, None, - '___sec97'), + '___sec93'), + ('The Ising model', 2, None, '___sec94'), + ('Reformulating the problem to suit regression', + 2, + None, + '___sec95'), + ('Linear regression', 2, None, '___sec96'), + ('Singular Value decomposition', 2, None, '___sec97'), ('The one-dimensional Ising model', 2, None, '___sec98'), ('Ridge regression', 2, None, '___sec99'), ('LASSO regression', 2, None, '___sec100'), @@ -295,74 +295,74 @@ MathJax.Hub.Config({
-For a correlation free experiment, \( \tau \) -equals 1. From the point of view of -eq. (25) we can interpret a sequential -correlation as an effective reduction of the number of measurements by -a factor \( \tau \). The effective number of measurements becomes: +
+With the OLS expressions for the parameters \( \boldsymbol{\beta} \) we can evaluate the expectation value $$ -n_\mathrm{eff} = \frac{n}{\tau} +\mathbb{E}(\boldsymbol{\beta}) = \mathbb{E}[ (\mathbf{X}^{\top} \mathbf{X})^{-1}\mathbf{X}^{T} \mathbf{Y}]=(\mathbf{X}^{T} \mathbf{X})^{-1}\mathbf{X}^{T} \mathbb{E}[ \mathbf{Y}]=(\mathbf{X}^{T} \mathbf{X})^{-1} \mathbf{X}^{T}\mathbf{X}\boldsymbol{\beta}=\boldsymbol{\beta}. $$ -To neglect the autocorrelation time \( \tau \) will always cause our -simple uncorrelated estimate of \( \mathrm{err}_X^2\approx \mathrm{var}(x)/n \) to -be less than the true sample error. The estimate of the error will be -too good. On the other hand, the calculation of the full -autocorrelation time poses an efficiency problem if the set of -measurements is very large. -
+We can also calculate the variance + +
+The variance of \( \boldsymbol{\beta} \) is +$$ +\begin{eqnarray*} +\mbox{Var}(\boldsymbol{\beta}) & = & \mathbb{E} \{ [\boldsymbol{\beta} - \mathbb{E}(\boldsymbol{\beta})] [\boldsymbol{\beta} - \mathbb{E}(\boldsymbol{\beta})]^{T} \} +\\ +& = & \mathbb{E} \{ [(\mathbf{X}^{T} \mathbf{X})^{-1} \, \mathbf{X}^{T} \mathbf{Y} - \boldsymbol{\beta}] \, [(\mathbf{X}^{T} \mathbf{X})^{-1} \, \mathbf{X}^{T} \mathbf{Y} - \boldsymbol{\beta}]^{T} \} +\\ +% & = & \mathbb{E} \{ [(\mathbf{X}^{T} \mathbf{X})^{-1} \, \mathbf{X}^{T} \mathbf{Y}] \, [(\mathbf{X}^{T} \mathbf{X})^{-1} \, \mathbf{X}^{T} \mathbf{Y}]^{T} \} - \boldsymbol{\beta} \, \boldsymbol{\beta}^{T} +% \\ +% & = & \mathbb{E} \{ (\mathbf{X}^{T} \mathbf{X})^{-1} \, \mathbf{X}^{T} \mathbf{Y} \, \mathbf{Y}^{T} \, \mathbf{X} \, (\mathbf{X}^{T} \mathbf{X})^{-1} \} - \boldsymbol{\beta} \, \boldsymbol{\beta}^{T} +% \\ +& = & (\mathbf{X}^{T} \mathbf{X})^{-1} \, \mathbf{X}^{T} \, \mathbb{E} \{ \mathbf{Y} \, \mathbf{Y}^{T} \} \, \mathbf{X} \, (\mathbf{X}^{T} \mathbf{X})^{-1} - \boldsymbol{\beta} \, \boldsymbol{\beta}^{T} +\\ +& = & (\mathbf{X}^{T} \mathbf{X})^{-1} \, \mathbf{X}^{T} \, \{ \mathbf{X} \, \boldsymbol{\beta} \, \boldsymbol{\beta}^{T} \, \mathbf{X}^{T} + \sigma^2 \} \, \mathbf{X} \, (\mathbf{X}^{T} \mathbf{X})^{-1} - \boldsymbol{\beta} \, \boldsymbol{\beta}^{T} +% \\ +% & = & (\mathbf{X}^T \mathbf{X})^{-1} \, \mathbf{X}^T \, \mathbf{X} \, \boldsymbol{\beta} \, \boldsymbol{\beta}^T \, \mathbf{X}^T \, \mathbf{X} \, (\mathbf{X}^T % \mathbf{X})^{-1} +% \\ +% & & + \, \, \sigma^2 \, (\mathbf{X}^T \mathbf{X})^{-1} \, \mathbf{X}^T \, \mathbf{X} \, (\mathbf{X}^T \mathbf{X})^{-1} - \boldsymbol{\beta} \boldsymbol{\beta}^T +\\ +& = & \boldsymbol{\beta} \, \boldsymbol{\beta}^{T} + \sigma^2 \, (\mathbf{X}^{T} \mathbf{X})^{-1} - \boldsymbol{\beta} \, \boldsymbol{\beta}^{T} +\, \, \, = \, \, \, \sigma^2 \, (\mathbf{X}^{T} \mathbf{X})^{-1}, +\end{eqnarray*} +$$ + +
+where we have used that \( \mathbb{E} (\mathbf{Y} \mathbf{Y}^{T}) = +\mathbf{X} \, \boldsymbol{\beta} \, \boldsymbol{\beta}^{T} \, \mathbf{X}^{T} + +\sigma^2 \, \mathbf{I}_{nn} \). From \( \mbox{Var}(\boldsymbol{\beta}) = \sigma^2 +\, (\mathbf{X}^{T} \mathbf{X})^{-1} \), one obtains an estimate of the +variance of the estimate of the \( j \)-th regression coefficient: +\( \hat{\sigma}^2 (\hat{\beta}_j ) = \hat{\sigma}^2 \sqrt{ +[(\mathbf{X}^{T} \mathbf{X})^{-1}]_{jj} } \). This may be used to +construct a confidence interval for the estimates. + +
+In a similar way, we cna obtain analytical expressions for say the +expectation values of the parameters \( \boldsymbol{\beta} \) and their variance +when we employ Ridge regression, and thereby a confidence interval. + +
+It is rather straightforward to show that +$$ +\mathbb{E} \big[ \boldsymbol{\beta}^{\mathrm{Ridge}} \big]=(\mathbf{X}^{T} \mathbf{X} + \lambda \mathbf{I}_{pp})^{-1} (\mathbf{X}^{\top} \mathbf{X})\boldsymbol{\beta}^{\mathrm{OLS}}. +$$ + +We see clearly that +\( \mathbb{E} \big[ \boldsymbol{\beta}^{\mathrm{Ridge}} \big] \not= \boldsymbol{\beta}^{\mathrm{OLS}} \) for any \( \lambda > 0 \). We say then that the ridge estimator is biased. + +
+We can also compute the variance as + +$$ +\mbox{Var}[\boldsymbol{\beta}^{\mathrm{Ridge}}]=\sigma^2[ \mathbf{X}^{T} \mathbf{X} + \lambda \mathbf{I} ]^{-1} \mathbf{X}^{T} \mathbf{X} \{ [ \mathbf{X}^{\top} \mathbf{X} + \lambda \mathbf{I} ]^{-1}\}^{T}, +$$ + +and it is easy to see that if the parameter \( \lambda \) goes to infinity then the variance of Ridge parameters \( \boldsymbol{\beta} \) goes to zero. + +
+With this, we can compute the difference + +$$ +\mbox{Var}[\boldsymbol{\beta}^{\mathrm{OLS}}]-\mbox{Var}(\boldsymbol{\beta}^{\mathrm{Ridge}})=\sigma^2 [ \mathbf{X}^{T} \mathbf{X} + \lambda \mathbf{I} ]^{-1}[ 2\lambda\mathbf{I} + \lambda^2 (\mathbf{X}^{T} \mathbf{X})^{-1} ] \{ [ \mathbf{X}^{T} \mathbf{X} + \lambda \mathbf{I} ]^{-1}\}^{T}. +$$ + +The difference is non-negative definite since each component of the +matrix product is non-negative definite. +This means the variance we obtain with the standard OLS will always for \( \lambda > 0 \) be larger than the variance of \( \boldsymbol{\beta} \) obtained with the Ridge estimator. This has interesting consequences when we discuss the so-called bias-variance trade-off below.
diff --git a/doc/pub/Regression/html/._Regression-bs076.html b/doc/pub/Regression/html/._Regression-bs076.html index a2a888cab..435769825 100644 --- a/doc/pub/Regression/html/._Regression-bs076.html +++ b/doc/pub/Regression/html/._Regression-bs076.html @@ -111,112 +111,112 @@ Automatically generated HTML file from DocOnce source ('The Boston housing data example', 2, None, '___sec27'), ('Housing data, the code', 2, None, '___sec28'), ('The singular value decomposition', 2, None, '___sec29'), - ('The Ising model', 2, None, '___sec30'), - ('Reformulating the problem to suit regression', - 2, - None, - '___sec31'), - ('Linear regression', 2, None, '___sec32'), - ('Singular Value decomposition', 2, None, '___sec33'), - ('Linear Regression Problems', 2, None, '___sec34'), - ('Fixing the singularity', 2, None, '___sec35'), - ('Basic math of the SVD', 2, None, '___sec36'), - ('The SVD, a Fantastic Algorithm', 2, None, '___sec37'), - ('Another Example', 2, None, '___sec38'), - ('Economy-size SVD', 2, None, '___sec39'), - ('Mathematical Properties', 2, None, '___sec40'), - ('Ridge and LASSO Regression', 2, None, '___sec41'), - ('More on Ridge Regression', 2, None, '___sec42'), - ('Interpreting the Ridge results', 2, None, '___sec43'), - ('More interpretations', 2, None, '___sec44'), - ('Where are we going?', 2, None, '___sec45'), - ('Resampling methods', 2, None, '___sec46'), + ('Linear Regression Problems', 2, None, '___sec30'), + ('Fixing the singularity', 2, None, '___sec31'), + ('Basic math of the SVD', 2, None, '___sec32'), + ('The SVD, a Fantastic Algorithm', 2, None, '___sec33'), + ('Another Example', 2, None, '___sec34'), + ('Economy-size SVD', 2, None, '___sec35'), + ('Mathematical Properties', 2, None, '___sec36'), + ('Ridge and LASSO Regression', 2, None, '___sec37'), + ('More on Ridge Regression', 2, None, '___sec38'), + ('Interpreting the Ridge results', 2, None, '___sec39'), + ('More interpretations', 2, None, '___sec40'), + ('Where are we going?', 2, None, '___sec41'), + ('Resampling methods', 2, None, '___sec42'), ('Resampling approaches can be computationally expensive', 2, None, - '___sec47'), - ('Why resampling methods ?', 2, None, '___sec48'), - ('Statistical analysis', 2, None, '___sec49'), - ('Statistics', 2, None, '___sec50'), - ('Statistics, moments', 2, None, '___sec51'), - ('Statistics, central moments', 2, None, '___sec52'), - ('Statistics, covariance', 2, None, '___sec53'), - ('Statistics, more covariance', 2, None, '___sec54'), - ('Covariance example', 2, None, '___sec55'), - ('Covariance in numpy', 2, None, '___sec56'), - ('Statistics, independent variables', 2, None, '___sec57'), - ('Statistics, more variance', 2, None, '___sec58'), - ('Statistics and stochastic processes', 2, None, '___sec59'), - ('Statistics and sample variables', 2, None, '___sec60'), + '___sec43'), + ('Why resampling methods ?', 2, None, '___sec44'), + ('Statistical analysis', 2, None, '___sec45'), + ('Statistics', 2, None, '___sec46'), + ('Statistics, moments', 2, None, '___sec47'), + ('Statistics, central moments', 2, None, '___sec48'), + ('Statistics, covariance', 2, None, '___sec49'), + ('Statistics, more covariance', 2, None, '___sec50'), + ('Covariance example', 2, None, '___sec51'), + ('Covariance in numpy', 2, None, '___sec52'), + ('Statistics, independent variables', 2, None, '___sec53'), + ('Statistics, more variance', 2, None, '___sec54'), + ('Statistics and stochastic processes', 2, None, '___sec55'), + ('Statistics and sample variables', 2, None, '___sec56'), ('Statistics, sample variance and covariance', 2, None, - '___sec61'), - ('Statistics, law of large numbers', 2, None, '___sec62'), - ('Statistics, more on sample error', 2, None, '___sec63'), - ('Statistics', 2, None, '___sec64'), - ('Statistics, central limit theorem', 2, None, '___sec65'), - ('Statistics, more technicalities', 2, None, '___sec66'), - ('Statistics', 2, None, '___sec67'), - ('Statistics and sample variance', 2, None, '___sec68'), - ('Statistics, uncorrelated results', 2, None, '___sec69'), - ('Statistics, computations', 2, None, '___sec70'), + '___sec57'), + ('Statistics, law of large numbers', 2, None, '___sec58'), + ('Statistics, more on sample error', 2, None, '___sec59'), + ('Statistics', 2, None, '___sec60'), + ('Statistics, central limit theorem', 2, None, '___sec61'), + ('Statistics, more technicalities', 2, None, '___sec62'), + ('Statistics', 2, None, '___sec63'), + ('Statistics and sample variance', 2, None, '___sec64'), + ('Statistics, uncorrelated results', 2, None, '___sec65'), + ('Statistics, computations', 2, None, '___sec66'), ('Statistics, more on computations of errors', 2, None, - '___sec71'), - ('Statistics, wrapping up 1', 2, None, '___sec72'), - ('Statistics, final expression', 2, None, '___sec73'), + '___sec67'), + ('Statistics, wrapping up 1', 2, None, '___sec68'), + ('Statistics, final expression', 2, None, '___sec69'), ('Statistics, effective number of correlations', 2, None, - '___sec74'), + '___sec70'), ('Linking the regression analysis with a statistical ' 'interpretation', 2, None, - '___sec75'), - ('Assumptions made', 2, None, '___sec76'), - ('Expectation value and variance', 2, None, '___sec77'), + '___sec71'), + ('Assumptions made', 2, None, '___sec72'), + ('Expectation value and variance', 2, None, '___sec73'), ('Expectation value and variance for $\\boldsymbol{\\beta}$', 2, None, - '___sec78'), - ('Cross-validation', 2, None, '___sec79'), - ('Computationally expensive', 2, None, '___sec80'), - ('Various steps in cross-validation', 2, None, '___sec81'), + '___sec74'), + ('Cross-validation', 2, None, '___sec75'), + ('Computationally expensive', 2, None, '___sec76'), + ('Various steps in cross-validation', 2, None, '___sec77'), ('How to set up the cross-validation for Ridge and/or Lasso', 2, None, - '___sec82'), + '___sec78'), ('Resampling methods: Jackknife and Bootstrap', 2, None, - '___sec83'), - ('Resampling methods: Jackknife', 2, None, '___sec84'), - ('Jackknife code example', 2, None, '___sec85'), - ('Resampling methods: Bootstrap', 2, None, '___sec86'), - ('Resampling methods: Bootstrap background', 2, None, '___sec87'), + '___sec79'), + ('Resampling methods: Jackknife', 2, None, '___sec80'), + ('Jackknife code example', 2, None, '___sec81'), + ('Resampling methods: Bootstrap', 2, None, '___sec82'), + ('Resampling methods: Bootstrap background', 2, None, '___sec83'), ('Resampling methods: More Bootstrap background', 2, None, - '___sec88'), - ('Resampling methods: Bootstrap approach', 2, None, '___sec89'), - ('Resampling methods: Bootstrap steps', 2, None, '___sec90'), - ('Code example for the Bootstrap method', 2, None, '___sec91'), + '___sec84'), + ('Resampling methods: Bootstrap approach', 2, None, '___sec85'), + ('Resampling methods: Bootstrap steps', 2, None, '___sec86'), + ('Code example for the Bootstrap method', 2, None, '___sec87'), ('Code Example for Cross-validation and $k$-fold ' 'Cross-validation', 2, None, - '___sec92'), - ('The bias-variance tradeoff', 2, None, '___sec93'), - ('Example code for Bias-Variance tradeoff', 2, None, '___sec94'), - ('Understanding what happens', 2, None, '___sec95'), - ('Summing up', 2, None, '___sec96'), + '___sec88'), + ('The bias-variance tradeoff', 2, None, '___sec89'), + ('Example code for Bias-Variance tradeoff', 2, None, '___sec90'), + ('Understanding what happens', 2, None, '___sec91'), + ('Summing up', 2, None, '___sec92'), ("Another Example rom Scikit-Learn's Repository", 2, None, - '___sec97'), + '___sec93'), + ('The Ising model', 2, None, '___sec94'), + ('Reformulating the problem to suit regression', + 2, + None, + '___sec95'), + ('Linear regression', 2, None, '___sec96'), + ('Singular Value decomposition', 2, None, '___sec97'), ('The one-dimensional Ising model', 2, None, '___sec98'), ('Ridge regression', 2, None, '___sec99'), ('LASSO regression', 2, None, '___sec100'), @@ -295,74 +295,74 @@ MathJax.Hub.Config({
-Finally, we are going to discuss several statistical properties which can be obtained in terms of analytical expressions. -The -advantage of doing linear regression is that we actually end up with -analytical expressions for several statistical quantities. -Standard least squares and Ridge regression allow us to -derive quantities like the variance and other expectation values in a -rather straightforward way. +Instead of choosing the penalty parameter to balance model fit with +model complexity, cross-validation requires it (i.e. the penalty +parameter) to yield a model with good prediction +performance. Commonly, this performance is evaluated on novel +data. Novel data need not be easy to come by and one has to make do +with the data at hand.
-It is assumed that \( \varepsilon_i -\sim \mathcal{N}(0, \sigma^2) \) and the \( \varepsilon_{i} \) are -independent, i.e.: -$$ -\begin{align*} -\mbox{Cov}(\varepsilon_{i_1}, -\varepsilon_{i_2}) & = \left\{ \begin{array}{lcc} \sigma^2 & \mbox{if} -& i_1 = i_2, \\ 0 & \mbox{if} & i_1 \not= i_2. \end{array} \right. -\end{align*} -$$ - -The randomness of \( \varepsilon_i \) implies that -\( \mathbf{y}_i \) is also a random variable. In particular, -\( \mathbf{y}_i \) is normally distributed, because \( \varepsilon_i \sim -\mathcal{N}(0, \sigma^2) \) and \( \mathbf{X}_{i,\ast} \, \boldsymbol{\beta} \) is a -non-random scalar. To specify the parameters of the distribution of -\( \mathbf{y}_i \) we need to calculate its first two moments. - -
-Recall that \( \boldsymbol{X} \) is a matrix of dimensionality \( n\times p \). The -notation above \( \mathbf{X}_{i,\ast} \) means that we are looking at the -row number \( i \) and perform a sum over all values \( p \). +The setting of original and novel data is +then mimicked by sample splitting: the data set is divided into two +(groups of samples). One of these two data sets, called the +training set, plays the role of original data on which the model is +built. The second of these data sets, called the test set, plays the +role of the novel data and is used to evaluate the prediction +performance (often operationalized as the log-likelihood or the +prediction error or its square or the R2 score) of the model built on the training data set. This +procedure (model building and prediction evaluation on training and +test set, respectively) is done for a collection of possible penalty +parameter choices. The penalty parameter that yields the model with +the best prediction performance is to be preferred. The thus obtained +performance evaluation depends on the actual split of the data set. To +remove this dependence the data set is split many times into a +training and test set. For each split the model parameters are +estimated for all choices of \( \lambda \) using the training data and +estimated parameters are evaluated on the corresponding test set. The +penalty parameter that on average over the test sets performs best (in +some sense) is then selected.
diff --git a/doc/pub/Regression/html/._Regression-bs077.html b/doc/pub/Regression/html/._Regression-bs077.html index 1a192e2bd..61ce3a51d 100644 --- a/doc/pub/Regression/html/._Regression-bs077.html +++ b/doc/pub/Regression/html/._Regression-bs077.html @@ -111,112 +111,112 @@ Automatically generated HTML file from DocOnce source ('The Boston housing data example', 2, None, '___sec27'), ('Housing data, the code', 2, None, '___sec28'), ('The singular value decomposition', 2, None, '___sec29'), - ('The Ising model', 2, None, '___sec30'), - ('Reformulating the problem to suit regression', - 2, - None, - '___sec31'), - ('Linear regression', 2, None, '___sec32'), - ('Singular Value decomposition', 2, None, '___sec33'), - ('Linear Regression Problems', 2, None, '___sec34'), - ('Fixing the singularity', 2, None, '___sec35'), - ('Basic math of the SVD', 2, None, '___sec36'), - ('The SVD, a Fantastic Algorithm', 2, None, '___sec37'), - ('Another Example', 2, None, '___sec38'), - ('Economy-size SVD', 2, None, '___sec39'), - ('Mathematical Properties', 2, None, '___sec40'), - ('Ridge and LASSO Regression', 2, None, '___sec41'), - ('More on Ridge Regression', 2, None, '___sec42'), - ('Interpreting the Ridge results', 2, None, '___sec43'), - ('More interpretations', 2, None, '___sec44'), - ('Where are we going?', 2, None, '___sec45'), - ('Resampling methods', 2, None, '___sec46'), + ('Linear Regression Problems', 2, None, '___sec30'), + ('Fixing the singularity', 2, None, '___sec31'), + ('Basic math of the SVD', 2, None, '___sec32'), + ('The SVD, a Fantastic Algorithm', 2, None, '___sec33'), + ('Another Example', 2, None, '___sec34'), + ('Economy-size SVD', 2, None, '___sec35'), + ('Mathematical Properties', 2, None, '___sec36'), + ('Ridge and LASSO Regression', 2, None, '___sec37'), + ('More on Ridge Regression', 2, None, '___sec38'), + ('Interpreting the Ridge results', 2, None, '___sec39'), + ('More interpretations', 2, None, '___sec40'), + ('Where are we going?', 2, None, '___sec41'), + ('Resampling methods', 2, None, '___sec42'), ('Resampling approaches can be computationally expensive', 2, None, - '___sec47'), - ('Why resampling methods ?', 2, None, '___sec48'), - ('Statistical analysis', 2, None, '___sec49'), - ('Statistics', 2, None, '___sec50'), - ('Statistics, moments', 2, None, '___sec51'), - ('Statistics, central moments', 2, None, '___sec52'), - ('Statistics, covariance', 2, None, '___sec53'), - ('Statistics, more covariance', 2, None, '___sec54'), - ('Covariance example', 2, None, '___sec55'), - ('Covariance in numpy', 2, None, '___sec56'), - ('Statistics, independent variables', 2, None, '___sec57'), - ('Statistics, more variance', 2, None, '___sec58'), - ('Statistics and stochastic processes', 2, None, '___sec59'), - ('Statistics and sample variables', 2, None, '___sec60'), + '___sec43'), + ('Why resampling methods ?', 2, None, '___sec44'), + ('Statistical analysis', 2, None, '___sec45'), + ('Statistics', 2, None, '___sec46'), + ('Statistics, moments', 2, None, '___sec47'), + ('Statistics, central moments', 2, None, '___sec48'), + ('Statistics, covariance', 2, None, '___sec49'), + ('Statistics, more covariance', 2, None, '___sec50'), + ('Covariance example', 2, None, '___sec51'), + ('Covariance in numpy', 2, None, '___sec52'), + ('Statistics, independent variables', 2, None, '___sec53'), + ('Statistics, more variance', 2, None, '___sec54'), + ('Statistics and stochastic processes', 2, None, '___sec55'), + ('Statistics and sample variables', 2, None, '___sec56'), ('Statistics, sample variance and covariance', 2, None, - '___sec61'), - ('Statistics, law of large numbers', 2, None, '___sec62'), - ('Statistics, more on sample error', 2, None, '___sec63'), - ('Statistics', 2, None, '___sec64'), - ('Statistics, central limit theorem', 2, None, '___sec65'), - ('Statistics, more technicalities', 2, None, '___sec66'), - ('Statistics', 2, None, '___sec67'), - ('Statistics and sample variance', 2, None, '___sec68'), - ('Statistics, uncorrelated results', 2, None, '___sec69'), - ('Statistics, computations', 2, None, '___sec70'), + '___sec57'), + ('Statistics, law of large numbers', 2, None, '___sec58'), + ('Statistics, more on sample error', 2, None, '___sec59'), + ('Statistics', 2, None, '___sec60'), + ('Statistics, central limit theorem', 2, None, '___sec61'), + ('Statistics, more technicalities', 2, None, '___sec62'), + ('Statistics', 2, None, '___sec63'), + ('Statistics and sample variance', 2, None, '___sec64'), + ('Statistics, uncorrelated results', 2, None, '___sec65'), + ('Statistics, computations', 2, None, '___sec66'), ('Statistics, more on computations of errors', 2, None, - '___sec71'), - ('Statistics, wrapping up 1', 2, None, '___sec72'), - ('Statistics, final expression', 2, None, '___sec73'), + '___sec67'), + ('Statistics, wrapping up 1', 2, None, '___sec68'), + ('Statistics, final expression', 2, None, '___sec69'), ('Statistics, effective number of correlations', 2, None, - '___sec74'), + '___sec70'), ('Linking the regression analysis with a statistical ' 'interpretation', 2, None, - '___sec75'), - ('Assumptions made', 2, None, '___sec76'), - ('Expectation value and variance', 2, None, '___sec77'), + '___sec71'), + ('Assumptions made', 2, None, '___sec72'), + ('Expectation value and variance', 2, None, '___sec73'), ('Expectation value and variance for $\\boldsymbol{\\beta}$', 2, None, - '___sec78'), - ('Cross-validation', 2, None, '___sec79'), - ('Computationally expensive', 2, None, '___sec80'), - ('Various steps in cross-validation', 2, None, '___sec81'), + '___sec74'), + ('Cross-validation', 2, None, '___sec75'), + ('Computationally expensive', 2, None, '___sec76'), + ('Various steps in cross-validation', 2, None, '___sec77'), ('How to set up the cross-validation for Ridge and/or Lasso', 2, None, - '___sec82'), + '___sec78'), ('Resampling methods: Jackknife and Bootstrap', 2, None, - '___sec83'), - ('Resampling methods: Jackknife', 2, None, '___sec84'), - ('Jackknife code example', 2, None, '___sec85'), - ('Resampling methods: Bootstrap', 2, None, '___sec86'), - ('Resampling methods: Bootstrap background', 2, None, '___sec87'), + '___sec79'), + ('Resampling methods: Jackknife', 2, None, '___sec80'), + ('Jackknife code example', 2, None, '___sec81'), + ('Resampling methods: Bootstrap', 2, None, '___sec82'), + ('Resampling methods: Bootstrap background', 2, None, '___sec83'), ('Resampling methods: More Bootstrap background', 2, None, - '___sec88'), - ('Resampling methods: Bootstrap approach', 2, None, '___sec89'), - ('Resampling methods: Bootstrap steps', 2, None, '___sec90'), - ('Code example for the Bootstrap method', 2, None, '___sec91'), + '___sec84'), + ('Resampling methods: Bootstrap approach', 2, None, '___sec85'), + ('Resampling methods: Bootstrap steps', 2, None, '___sec86'), + ('Code example for the Bootstrap method', 2, None, '___sec87'), ('Code Example for Cross-validation and $k$-fold ' 'Cross-validation', 2, None, - '___sec92'), - ('The bias-variance tradeoff', 2, None, '___sec93'), - ('Example code for Bias-Variance tradeoff', 2, None, '___sec94'), - ('Understanding what happens', 2, None, '___sec95'), - ('Summing up', 2, None, '___sec96'), + '___sec88'), + ('The bias-variance tradeoff', 2, None, '___sec89'), + ('Example code for Bias-Variance tradeoff', 2, None, '___sec90'), + ('Understanding what happens', 2, None, '___sec91'), + ('Summing up', 2, None, '___sec92'), ("Another Example rom Scikit-Learn's Repository", 2, None, - '___sec97'), + '___sec93'), + ('The Ising model', 2, None, '___sec94'), + ('Reformulating the problem to suit regression', + 2, + None, + '___sec95'), + ('Linear regression', 2, None, '___sec96'), + ('Singular Value decomposition', 2, None, '___sec97'), ('The one-dimensional Ising model', 2, None, '___sec98'), ('Ridge regression', 2, None, '___sec99'), ('LASSO regression', 2, None, '___sec100'), @@ -295,74 +295,74 @@ MathJax.Hub.Config({
-The assumption we have made here can be summarized as (and this is going to useful when we discuss the bias-variance trade off) -that there exists a function \( f(\boldsymbol{x}) \) and a normal distributed error \( \boldsymbol{\varepsilon}\sim \mathcal{N}(0, \sigma^2) \) -which describes our data -$$ -\boldsymbol{y} = f(\boldsymbol{x})+\boldsymbol{\varepsilon} -$$ +The validation set approach is conceptually simple and is easy to implement. But it has two potential drawbacks: -
-We approximate this function with our model from the solution of the linear regression equations, that is our -function \( f \) is approximated by \( \boldsymbol{\tilde{y}} \) where we want to minimize \( (\boldsymbol{y}-\boldsymbol{\tilde{y}})^2 \), our MSE, with -$$ -\boldsymbol{\tilde{y}} = \boldsymbol{X}\boldsymbol{\beta}. -$$ +
- + -
-We can calculate the expectation value of \( \boldsymbol{y} \) for a given element \( i \) -$$ -\begin{align*} -\mathbb{E}(y_i) & = -\mathbb{E}(\mathbf{X}_{i, \ast} \, \boldsymbol{\beta}) + \mathbb{E}(\varepsilon_i) -\, \, \, = \, \, \, \mathbf{X}_{i, \ast} \, \beta, -\end{align*} -$$ - -while -its variance is -$$ -\begin{align*} \mbox{Var}(y_i) & = \mathbb{E} \{ [y_i -- \mathbb{E}(y_i)]^2 \} \, \, \, = \, \, \, \mathbb{E} ( y_i^2 ) - -[\mathbb{E}(y_i)]^2 \\ & = \mathbb{E} [ ( \mathbf{X}_{i, \ast} \, -\beta + \varepsilon_i )^2] - ( \mathbf{X}_{i, \ast} \, \boldsymbol{\beta})^2 \\ & -= \mathbb{E} [ ( \mathbf{X}_{i, \ast} \, \boldsymbol{\beta})^2 + 2 \varepsilon_i -\mathbf{X}_{i, \ast} \, \boldsymbol{\beta} + \varepsilon_i^2 ] - ( \mathbf{X}_{i, -\ast} \, \beta)^2 \\ & = ( \mathbf{X}_{i, \ast} \, \boldsymbol{\beta})^2 + 2 -\mathbb{E}(\varepsilon_i) \mathbf{X}_{i, \ast} \, \boldsymbol{\beta} + -\mathbb{E}(\varepsilon_i^2 ) - ( \mathbf{X}_{i, \ast} \, \boldsymbol{\beta})^2 -\\ & = \mathbb{E}(\varepsilon_i^2 ) \, \, \, = \, \, \, -\mbox{Var}(\varepsilon_i) \, \, \, = \, \, \, \sigma^2. -\end{align*} -$$ - -Hence, \( y_i \sim \mathcal{N}( \mathbf{X}_{i, \ast} \, \boldsymbol{\beta}, \sigma^2) \), that is \( \boldsymbol{y} \) follows a normal distribution with -mean value \( \boldsymbol{X}\boldsymbol{\beta} \) and variance \( \sigma^2 \) (not be confused with the singular values of the SVD). +When the repetitive splitting of the data set is done randomly, +samples may accidently end up in a fast majority of the splits in +either training or test set. Such samples may have an unbalanced +influence on either model building or prediction evaluation. To avoid +this \( k \)-fold cross-validation structures the data splitting. The +samples are divided into \( k \) more or less equally sized exhaustive and +mutually exclusive subsets. In turn (at each split) one of these +subsets plays the role of the test set while the union of the +remaining subsets constitutes the training set. Such a splitting +warrants a balanced representation of each sample in both training and +test set over the splits. Still the division into the \( k \) subsets +involves a degree of randomness. This may be fully excluded when +choosing \( k=n \). This particular case is referred to as leave-one-out +cross-validation (LOOCV).
diff --git a/doc/pub/Regression/html/._Regression-bs079.html b/doc/pub/Regression/html/._Regression-bs079.html index a725bb2f4..b0555066c 100644 --- a/doc/pub/Regression/html/._Regression-bs079.html +++ b/doc/pub/Regression/html/._Regression-bs079.html @@ -111,112 +111,112 @@ Automatically generated HTML file from DocOnce source ('The Boston housing data example', 2, None, '___sec27'), ('Housing data, the code', 2, None, '___sec28'), ('The singular value decomposition', 2, None, '___sec29'), - ('The Ising model', 2, None, '___sec30'), - ('Reformulating the problem to suit regression', - 2, - None, - '___sec31'), - ('Linear regression', 2, None, '___sec32'), - ('Singular Value decomposition', 2, None, '___sec33'), - ('Linear Regression Problems', 2, None, '___sec34'), - ('Fixing the singularity', 2, None, '___sec35'), - ('Basic math of the SVD', 2, None, '___sec36'), - ('The SVD, a Fantastic Algorithm', 2, None, '___sec37'), - ('Another Example', 2, None, '___sec38'), - ('Economy-size SVD', 2, None, '___sec39'), - ('Mathematical Properties', 2, None, '___sec40'), - ('Ridge and LASSO Regression', 2, None, '___sec41'), - ('More on Ridge Regression', 2, None, '___sec42'), - ('Interpreting the Ridge results', 2, None, '___sec43'), - ('More interpretations', 2, None, '___sec44'), - ('Where are we going?', 2, None, '___sec45'), - ('Resampling methods', 2, None, '___sec46'), + ('Linear Regression Problems', 2, None, '___sec30'), + ('Fixing the singularity', 2, None, '___sec31'), + ('Basic math of the SVD', 2, None, '___sec32'), + ('The SVD, a Fantastic Algorithm', 2, None, '___sec33'), + ('Another Example', 2, None, '___sec34'), + ('Economy-size SVD', 2, None, '___sec35'), + ('Mathematical Properties', 2, None, '___sec36'), + ('Ridge and LASSO Regression', 2, None, '___sec37'), + ('More on Ridge Regression', 2, None, '___sec38'), + ('Interpreting the Ridge results', 2, None, '___sec39'), + ('More interpretations', 2, None, '___sec40'), + ('Where are we going?', 2, None, '___sec41'), + ('Resampling methods', 2, None, '___sec42'), ('Resampling approaches can be computationally expensive', 2, None, - '___sec47'), - ('Why resampling methods ?', 2, None, '___sec48'), - ('Statistical analysis', 2, None, '___sec49'), - ('Statistics', 2, None, '___sec50'), - ('Statistics, moments', 2, None, '___sec51'), - ('Statistics, central moments', 2, None, '___sec52'), - ('Statistics, covariance', 2, None, '___sec53'), - ('Statistics, more covariance', 2, None, '___sec54'), - ('Covariance example', 2, None, '___sec55'), - ('Covariance in numpy', 2, None, '___sec56'), - ('Statistics, independent variables', 2, None, '___sec57'), - ('Statistics, more variance', 2, None, '___sec58'), - ('Statistics and stochastic processes', 2, None, '___sec59'), - ('Statistics and sample variables', 2, None, '___sec60'), + '___sec43'), + ('Why resampling methods ?', 2, None, '___sec44'), + ('Statistical analysis', 2, None, '___sec45'), + ('Statistics', 2, None, '___sec46'), + ('Statistics, moments', 2, None, '___sec47'), + ('Statistics, central moments', 2, None, '___sec48'), + ('Statistics, covariance', 2, None, '___sec49'), + ('Statistics, more covariance', 2, None, '___sec50'), + ('Covariance example', 2, None, '___sec51'), + ('Covariance in numpy', 2, None, '___sec52'), + ('Statistics, independent variables', 2, None, '___sec53'), + ('Statistics, more variance', 2, None, '___sec54'), + ('Statistics and stochastic processes', 2, None, '___sec55'), + ('Statistics and sample variables', 2, None, '___sec56'), ('Statistics, sample variance and covariance', 2, None, - '___sec61'), - ('Statistics, law of large numbers', 2, None, '___sec62'), - ('Statistics, more on sample error', 2, None, '___sec63'), - ('Statistics', 2, None, '___sec64'), - ('Statistics, central limit theorem', 2, None, '___sec65'), - ('Statistics, more technicalities', 2, None, '___sec66'), - ('Statistics', 2, None, '___sec67'), - ('Statistics and sample variance', 2, None, '___sec68'), - ('Statistics, uncorrelated results', 2, None, '___sec69'), - ('Statistics, computations', 2, None, '___sec70'), + '___sec57'), + ('Statistics, law of large numbers', 2, None, '___sec58'), + ('Statistics, more on sample error', 2, None, '___sec59'), + ('Statistics', 2, None, '___sec60'), + ('Statistics, central limit theorem', 2, None, '___sec61'), + ('Statistics, more technicalities', 2, None, '___sec62'), + ('Statistics', 2, None, '___sec63'), + ('Statistics and sample variance', 2, None, '___sec64'), + ('Statistics, uncorrelated results', 2, None, '___sec65'), + ('Statistics, computations', 2, None, '___sec66'), ('Statistics, more on computations of errors', 2, None, - '___sec71'), - ('Statistics, wrapping up 1', 2, None, '___sec72'), - ('Statistics, final expression', 2, None, '___sec73'), + '___sec67'), + ('Statistics, wrapping up 1', 2, None, '___sec68'), + ('Statistics, final expression', 2, None, '___sec69'), ('Statistics, effective number of correlations', 2, None, - '___sec74'), + '___sec70'), ('Linking the regression analysis with a statistical ' 'interpretation', 2, None, - '___sec75'), - ('Assumptions made', 2, None, '___sec76'), - ('Expectation value and variance', 2, None, '___sec77'), + '___sec71'), + ('Assumptions made', 2, None, '___sec72'), + ('Expectation value and variance', 2, None, '___sec73'), ('Expectation value and variance for $\\boldsymbol{\\beta}$', 2, None, - '___sec78'), - ('Cross-validation', 2, None, '___sec79'), - ('Computationally expensive', 2, None, '___sec80'), - ('Various steps in cross-validation', 2, None, '___sec81'), + '___sec74'), + ('Cross-validation', 2, None, '___sec75'), + ('Computationally expensive', 2, None, '___sec76'), + ('Various steps in cross-validation', 2, None, '___sec77'), ('How to set up the cross-validation for Ridge and/or Lasso', 2, None, - '___sec82'), + '___sec78'), ('Resampling methods: Jackknife and Bootstrap', 2, None, - '___sec83'), - ('Resampling methods: Jackknife', 2, None, '___sec84'), - ('Jackknife code example', 2, None, '___sec85'), - ('Resampling methods: Bootstrap', 2, None, '___sec86'), - ('Resampling methods: Bootstrap background', 2, None, '___sec87'), + '___sec79'), + ('Resampling methods: Jackknife', 2, None, '___sec80'), + ('Jackknife code example', 2, None, '___sec81'), + ('Resampling methods: Bootstrap', 2, None, '___sec82'), + ('Resampling methods: Bootstrap background', 2, None, '___sec83'), ('Resampling methods: More Bootstrap background', 2, None, - '___sec88'), - ('Resampling methods: Bootstrap approach', 2, None, '___sec89'), - ('Resampling methods: Bootstrap steps', 2, None, '___sec90'), - ('Code example for the Bootstrap method', 2, None, '___sec91'), + '___sec84'), + ('Resampling methods: Bootstrap approach', 2, None, '___sec85'), + ('Resampling methods: Bootstrap steps', 2, None, '___sec86'), + ('Code example for the Bootstrap method', 2, None, '___sec87'), ('Code Example for Cross-validation and $k$-fold ' 'Cross-validation', 2, None, - '___sec92'), - ('The bias-variance tradeoff', 2, None, '___sec93'), - ('Example code for Bias-Variance tradeoff', 2, None, '___sec94'), - ('Understanding what happens', 2, None, '___sec95'), - ('Summing up', 2, None, '___sec96'), + '___sec88'), + ('The bias-variance tradeoff', 2, None, '___sec89'), + ('Example code for Bias-Variance tradeoff', 2, None, '___sec90'), + ('Understanding what happens', 2, None, '___sec91'), + ('Summing up', 2, None, '___sec92'), ("Another Example rom Scikit-Learn's Repository", 2, None, - '___sec97'), + '___sec93'), + ('The Ising model', 2, None, '___sec94'), + ('Reformulating the problem to suit regression', + 2, + None, + '___sec95'), + ('Linear regression', 2, None, '___sec96'), + ('Singular Value decomposition', 2, None, '___sec97'), ('The one-dimensional Ising model', 2, None, '___sec98'), ('Ridge regression', 2, None, '___sec99'), ('LASSO regression', 2, None, '___sec100'), @@ -295,74 +295,74 @@ MathJax.Hub.Config({
- + -
-With the OLS expressions for the parameters \( \boldsymbol{\beta} \) we can evaluate the expectation value -$$ -\mathbb{E}(\boldsymbol{\beta}) = \mathbb{E}[ (\mathbf{X}^{\top} \mathbf{X})^{-1}\mathbf{X}^{T} \mathbf{Y}]=(\mathbf{X}^{T} \mathbf{X})^{-1}\mathbf{X}^{T} \mathbb{E}[ \mathbf{Y}]=(\mathbf{X}^{T} \mathbf{X})^{-1} \mathbf{X}^{T}\mathbf{X}\boldsymbol{\beta}=\boldsymbol{\beta}. -$$ - -This means that the estimator of the regression parameters is unbiased. - -
-We can also calculate the variance - -
-The variance of \( \boldsymbol{\beta} \) is -$$ -\begin{eqnarray*} -\mbox{Var}(\boldsymbol{\beta}) & = & \mathbb{E} \{ [\boldsymbol{\beta} - \mathbb{E}(\boldsymbol{\beta})] [\boldsymbol{\beta} - \mathbb{E}(\boldsymbol{\beta})]^{T} \} -\\ -& = & \mathbb{E} \{ [(\mathbf{X}^{T} \mathbf{X})^{-1} \, \mathbf{X}^{T} \mathbf{Y} - \boldsymbol{\beta}] \, [(\mathbf{X}^{T} \mathbf{X})^{-1} \, \mathbf{X}^{T} \mathbf{Y} - \boldsymbol{\beta}]^{T} \} -\\ -% & = & \mathbb{E} \{ [(\mathbf{X}^{T} \mathbf{X})^{-1} \, \mathbf{X}^{T} \mathbf{Y}] \, [(\mathbf{X}^{T} \mathbf{X})^{-1} \, \mathbf{X}^{T} \mathbf{Y}]^{T} \} - \boldsymbol{\beta} \, \boldsymbol{\beta}^{T} -% \\ -% & = & \mathbb{E} \{ (\mathbf{X}^{T} \mathbf{X})^{-1} \, \mathbf{X}^{T} \mathbf{Y} \, \mathbf{Y}^{T} \, \mathbf{X} \, (\mathbf{X}^{T} \mathbf{X})^{-1} \} - \boldsymbol{\beta} \, \boldsymbol{\beta}^{T} -% \\ -& = & (\mathbf{X}^{T} \mathbf{X})^{-1} \, \mathbf{X}^{T} \, \mathbb{E} \{ \mathbf{Y} \, \mathbf{Y}^{T} \} \, \mathbf{X} \, (\mathbf{X}^{T} \mathbf{X})^{-1} - \boldsymbol{\beta} \, \boldsymbol{\beta}^{T} -\\ -& = & (\mathbf{X}^{T} \mathbf{X})^{-1} \, \mathbf{X}^{T} \, \{ \mathbf{X} \, \boldsymbol{\beta} \, \boldsymbol{\beta}^{T} \, \mathbf{X}^{T} + \sigma^2 \} \, \mathbf{X} \, (\mathbf{X}^{T} \mathbf{X})^{-1} - \boldsymbol{\beta} \, \boldsymbol{\beta}^{T} -% \\ -% & = & (\mathbf{X}^T \mathbf{X})^{-1} \, \mathbf{X}^T \, \mathbf{X} \, \boldsymbol{\beta} \, \boldsymbol{\beta}^T \, \mathbf{X}^T \, \mathbf{X} \, (\mathbf{X}^T % \mathbf{X})^{-1} -% \\ -% & & + \, \, \sigma^2 \, (\mathbf{X}^T \mathbf{X})^{-1} \, \mathbf{X}^T \, \mathbf{X} \, (\mathbf{X}^T \mathbf{X})^{-1} - \boldsymbol{\beta} \boldsymbol{\beta}^T -\\ -& = & \boldsymbol{\beta} \, \boldsymbol{\beta}^{T} + \sigma^2 \, (\mathbf{X}^{T} \mathbf{X})^{-1} - \boldsymbol{\beta} \, \boldsymbol{\beta}^{T} -\, \, \, = \, \, \, \sigma^2 \, (\mathbf{X}^{T} \mathbf{X})^{-1}, -\end{eqnarray*} -$$ - -
-where we have used that \( \mathbb{E} (\mathbf{Y} \mathbf{Y}^{T}) = -\mathbf{X} \, \boldsymbol{\beta} \, \boldsymbol{\beta}^{T} \, \mathbf{X}^{T} + -\sigma^2 \, \mathbf{I}_{nn} \). From \( \mbox{Var}(\boldsymbol{\beta}) = \sigma^2 -\, (\mathbf{X}^{T} \mathbf{X})^{-1} \), one obtains an estimate of the -variance of the estimate of the \( j \)-th regression coefficient: -\( \hat{\sigma}^2 (\hat{\beta}_j ) = \hat{\sigma}^2 \sqrt{ -[(\mathbf{X}^{T} \mathbf{X})^{-1}]_{jj} } \). This may be used to -construct a confidence interval for the estimates. - -
-In a similar way, we cna obtain analytical expressions for say the -expectation values of the parameters \( \boldsymbol{\beta} \) and their variance -when we employ Ridge regression, and thereby a confidence interval. - -
-It is rather straightforward to show that -$$ -\mathbb{E} \big[ \boldsymbol{\beta}^{\mathrm{Ridge}} \big]=(\mathbf{X}^{T} \mathbf{X} + \lambda \mathbf{I}_{pp})^{-1} (\mathbf{X}^{\top} \mathbf{X})\boldsymbol{\beta}^{\mathrm{OLS}}. -$$ - -We see clearly that -\( \mathbb{E} \big[ \boldsymbol{\beta}^{\mathrm{Ridge}} \big] \not= \boldsymbol{\beta}^{\mathrm{OLS}} \) for any \( \lambda > 0 \). We say then that the ridge estimator is biased. - -
-We can also compute the variance as +
-With this, we can compute the difference +
+
- + -
-Instead of choosing the penalty parameter to balance model fit with -model complexity, cross-validation requires it (i.e. the penalty -parameter) to yield a model with good prediction -performance. Commonly, this performance is evaluated on novel -data. Novel data need not be easy to come by and one has to make do -with the data at hand. +Two famous +resampling methods are the independent bootstrap and the jackknife.
-The setting of original and novel data is -then mimicked by sample splitting: the data set is divided into two -(groups of samples). One of these two data sets, called the -training set, plays the role of original data on which the model is -built. The second of these data sets, called the test set, plays the -role of the novel data and is used to evaluate the prediction -performance (often operationalized as the log-likelihood or the -prediction error or its square or the R2 score) of the model built on the training data set. This -procedure (model building and prediction evaluation on training and -test set, respectively) is done for a collection of possible penalty -parameter choices. The penalty parameter that yields the model with -the best prediction performance is to be preferred. The thus obtained -performance evaluation depends on the actual split of the data set. To -remove this dependence the data set is split many times into a -training and test set. For each split the model parameters are -estimated for all choices of \( \lambda \) using the training data and -estimated parameters are evaluated on the corresponding test set. The -penalty parameter that on average over the test sets performs best (in -some sense) is then selected. +The jackknife is a special case of the independent bootstrap. Still, the jackknife was made +popular prior to the independent bootstrap. And as the popularity of +the independent bootstrap soared, new variants, such as the dependent bootstrap. + +
+The Jackknife and independent bootstrap work for +independent, identically distributed random variables. +If these conditions are not +satisfied, the methods will fail. Yet, it should be said that if the data are +independent, identically distributed, and we only want to estimate the +variance of \( \overline{X} \) (which often is the case), then there is no +need for bootstrapping.
diff --git a/doc/pub/Regression/html/._Regression-bs081.html b/doc/pub/Regression/html/._Regression-bs081.html index 271a6f60a..c2ff2409d 100644 --- a/doc/pub/Regression/html/._Regression-bs081.html +++ b/doc/pub/Regression/html/._Regression-bs081.html @@ -111,112 +111,112 @@ Automatically generated HTML file from DocOnce source ('The Boston housing data example', 2, None, '___sec27'), ('Housing data, the code', 2, None, '___sec28'), ('The singular value decomposition', 2, None, '___sec29'), - ('The Ising model', 2, None, '___sec30'), - ('Reformulating the problem to suit regression', - 2, - None, - '___sec31'), - ('Linear regression', 2, None, '___sec32'), - ('Singular Value decomposition', 2, None, '___sec33'), - ('Linear Regression Problems', 2, None, '___sec34'), - ('Fixing the singularity', 2, None, '___sec35'), - ('Basic math of the SVD', 2, None, '___sec36'), - ('The SVD, a Fantastic Algorithm', 2, None, '___sec37'), - ('Another Example', 2, None, '___sec38'), - ('Economy-size SVD', 2, None, '___sec39'), - ('Mathematical Properties', 2, None, '___sec40'), - ('Ridge and LASSO Regression', 2, None, '___sec41'), - ('More on Ridge Regression', 2, None, '___sec42'), - ('Interpreting the Ridge results', 2, None, '___sec43'), - ('More interpretations', 2, None, '___sec44'), - ('Where are we going?', 2, None, '___sec45'), - ('Resampling methods', 2, None, '___sec46'), + ('Linear Regression Problems', 2, None, '___sec30'), + ('Fixing the singularity', 2, None, '___sec31'), + ('Basic math of the SVD', 2, None, '___sec32'), + ('The SVD, a Fantastic Algorithm', 2, None, '___sec33'), + ('Another Example', 2, None, '___sec34'), + ('Economy-size SVD', 2, None, '___sec35'), + ('Mathematical Properties', 2, None, '___sec36'), + ('Ridge and LASSO Regression', 2, None, '___sec37'), + ('More on Ridge Regression', 2, None, '___sec38'), + ('Interpreting the Ridge results', 2, None, '___sec39'), + ('More interpretations', 2, None, '___sec40'), + ('Where are we going?', 2, None, '___sec41'), + ('Resampling methods', 2, None, '___sec42'), ('Resampling approaches can be computationally expensive', 2, None, - '___sec47'), - ('Why resampling methods ?', 2, None, '___sec48'), - ('Statistical analysis', 2, None, '___sec49'), - ('Statistics', 2, None, '___sec50'), - ('Statistics, moments', 2, None, '___sec51'), - ('Statistics, central moments', 2, None, '___sec52'), - ('Statistics, covariance', 2, None, '___sec53'), - ('Statistics, more covariance', 2, None, '___sec54'), - ('Covariance example', 2, None, '___sec55'), - ('Covariance in numpy', 2, None, '___sec56'), - ('Statistics, independent variables', 2, None, '___sec57'), - ('Statistics, more variance', 2, None, '___sec58'), - ('Statistics and stochastic processes', 2, None, '___sec59'), - ('Statistics and sample variables', 2, None, '___sec60'), + '___sec43'), + ('Why resampling methods ?', 2, None, '___sec44'), + ('Statistical analysis', 2, None, '___sec45'), + ('Statistics', 2, None, '___sec46'), + ('Statistics, moments', 2, None, '___sec47'), + ('Statistics, central moments', 2, None, '___sec48'), + ('Statistics, covariance', 2, None, '___sec49'), + ('Statistics, more covariance', 2, None, '___sec50'), + ('Covariance example', 2, None, '___sec51'), + ('Covariance in numpy', 2, None, '___sec52'), + ('Statistics, independent variables', 2, None, '___sec53'), + ('Statistics, more variance', 2, None, '___sec54'), + ('Statistics and stochastic processes', 2, None, '___sec55'), + ('Statistics and sample variables', 2, None, '___sec56'), ('Statistics, sample variance and covariance', 2, None, - '___sec61'), - ('Statistics, law of large numbers', 2, None, '___sec62'), - ('Statistics, more on sample error', 2, None, '___sec63'), - ('Statistics', 2, None, '___sec64'), - ('Statistics, central limit theorem', 2, None, '___sec65'), - ('Statistics, more technicalities', 2, None, '___sec66'), - ('Statistics', 2, None, '___sec67'), - ('Statistics and sample variance', 2, None, '___sec68'), - ('Statistics, uncorrelated results', 2, None, '___sec69'), - ('Statistics, computations', 2, None, '___sec70'), + '___sec57'), + ('Statistics, law of large numbers', 2, None, '___sec58'), + ('Statistics, more on sample error', 2, None, '___sec59'), + ('Statistics', 2, None, '___sec60'), + ('Statistics, central limit theorem', 2, None, '___sec61'), + ('Statistics, more technicalities', 2, None, '___sec62'), + ('Statistics', 2, None, '___sec63'), + ('Statistics and sample variance', 2, None, '___sec64'), + ('Statistics, uncorrelated results', 2, None, '___sec65'), + ('Statistics, computations', 2, None, '___sec66'), ('Statistics, more on computations of errors', 2, None, - '___sec71'), - ('Statistics, wrapping up 1', 2, None, '___sec72'), - ('Statistics, final expression', 2, None, '___sec73'), + '___sec67'), + ('Statistics, wrapping up 1', 2, None, '___sec68'), + ('Statistics, final expression', 2, None, '___sec69'), ('Statistics, effective number of correlations', 2, None, - '___sec74'), + '___sec70'), ('Linking the regression analysis with a statistical ' 'interpretation', 2, None, - '___sec75'), - ('Assumptions made', 2, None, '___sec76'), - ('Expectation value and variance', 2, None, '___sec77'), + '___sec71'), + ('Assumptions made', 2, None, '___sec72'), + ('Expectation value and variance', 2, None, '___sec73'), ('Expectation value and variance for $\\boldsymbol{\\beta}$', 2, None, - '___sec78'), - ('Cross-validation', 2, None, '___sec79'), - ('Computationally expensive', 2, None, '___sec80'), - ('Various steps in cross-validation', 2, None, '___sec81'), + '___sec74'), + ('Cross-validation', 2, None, '___sec75'), + ('Computationally expensive', 2, None, '___sec76'), + ('Various steps in cross-validation', 2, None, '___sec77'), ('How to set up the cross-validation for Ridge and/or Lasso', 2, None, - '___sec82'), + '___sec78'), ('Resampling methods: Jackknife and Bootstrap', 2, None, - '___sec83'), - ('Resampling methods: Jackknife', 2, None, '___sec84'), - ('Jackknife code example', 2, None, '___sec85'), - ('Resampling methods: Bootstrap', 2, None, '___sec86'), - ('Resampling methods: Bootstrap background', 2, None, '___sec87'), + '___sec79'), + ('Resampling methods: Jackknife', 2, None, '___sec80'), + ('Jackknife code example', 2, None, '___sec81'), + ('Resampling methods: Bootstrap', 2, None, '___sec82'), + ('Resampling methods: Bootstrap background', 2, None, '___sec83'), ('Resampling methods: More Bootstrap background', 2, None, - '___sec88'), - ('Resampling methods: Bootstrap approach', 2, None, '___sec89'), - ('Resampling methods: Bootstrap steps', 2, None, '___sec90'), - ('Code example for the Bootstrap method', 2, None, '___sec91'), + '___sec84'), + ('Resampling methods: Bootstrap approach', 2, None, '___sec85'), + ('Resampling methods: Bootstrap steps', 2, None, '___sec86'), + ('Code example for the Bootstrap method', 2, None, '___sec87'), ('Code Example for Cross-validation and $k$-fold ' 'Cross-validation', 2, None, - '___sec92'), - ('The bias-variance tradeoff', 2, None, '___sec93'), - ('Example code for Bias-Variance tradeoff', 2, None, '___sec94'), - ('Understanding what happens', 2, None, '___sec95'), - ('Summing up', 2, None, '___sec96'), + '___sec88'), + ('The bias-variance tradeoff', 2, None, '___sec89'), + ('Example code for Bias-Variance tradeoff', 2, None, '___sec90'), + ('Understanding what happens', 2, None, '___sec91'), + ('Summing up', 2, None, '___sec92'), ("Another Example rom Scikit-Learn's Repository", 2, None, - '___sec97'), + '___sec93'), + ('The Ising model', 2, None, '___sec94'), + ('Reformulating the problem to suit regression', + 2, + None, + '___sec95'), + ('Linear regression', 2, None, '___sec96'), + ('Singular Value decomposition', 2, None, '___sec97'), ('The one-dimensional Ising model', 2, None, '___sec98'), ('Ridge regression', 2, None, '___sec99'), ('LASSO regression', 2, None, '___sec100'), @@ -295,74 +295,74 @@ MathJax.Hub.Config({
-The validation set approach is conceptually simple and is easy to implement. But it has two potential drawbacks: +The Jackknife works by making many replicas of the estimator \( \widehat{\theta} \). +The jackknife is a resampling method where we systematically leave out one observation from the vector of observed values \( \boldsymbol{x} = (x_1,x_2,\cdots,X_n) \). +Let \( \boldsymbol{x}_i \) denote the vector +$$ +\boldsymbol{x}_i = (x_1,x_2,\cdots,x_{i-1},x_{i+1},\cdots,x_n), +$$ -
+which equals the vector \( \boldsymbol{x} \) with the exception that observation +number \( i \) is left out. Using this notation, define +\( \widehat{\theta}_i \) to be the estimator +\( \widehat{\theta} \) computed using \( \vec{X}_i \). +
- - -
-When the repetitive splitting of the data set is done randomly, -samples may accidently end up in a fast majority of the splits in -either training or test set. Such samples may have an unbalanced -influence on either model building or prediction evaluation. To avoid -this \( k \)-fold cross-validation structures the data splitting. The -samples are divided into \( k \) more or less equally sized exhaustive and -mutually exclusive subsets. In turn (at each split) one of these -subsets plays the role of the test set while the union of the -remaining subsets constitutes the training set. Such a splitting -warrants a balanced representation of each sample in both training and -test set over the splits. Still the division into the \( k \) subsets -involves a degree of randomness. This may be fully excluded when -choosing \( k=n \). This particular case is referred to as leave-one-out -cross-validation (LOOCV). + +
from numpy import *
+from numpy.random import randint, randn
+from time import time
+
+def jackknife(data, stat):
+ n = len(data);t = zeros(n); inds = arange(n); t0 = time()
+ ## 'jackknifing' by leaving out an observation for each i
+ for i in range(n):
+ t[i] = stat(delete(data,i) )
+
+ # analysis
+ print("Runtime: %g sec" % (time()-t0)); print("Jackknife Statistics :")
+ print("original bias std. error")
+ print("%8g %14g %15g" % (stat(data),(n-1)*mean(t)/n, (n*var(t))**.5))
+
+ return t
+
+
+# Returns mean of data samples
+def stat(data):
+ return mean(data)
+
+
+mu, sigma = 100, 15
+datapoints = 10000
+x = mu + sigma*random.randn(datapoints)
+# jackknife returns the data sample
+t = jackknife(x, stat)
+
diff --git a/doc/pub/Regression/html/._Regression-bs083.html b/doc/pub/Regression/html/._Regression-bs083.html index efc0e7d3c..45bf09ab0 100644 --- a/doc/pub/Regression/html/._Regression-bs083.html +++ b/doc/pub/Regression/html/._Regression-bs083.html @@ -111,112 +111,112 @@ Automatically generated HTML file from DocOnce source ('The Boston housing data example', 2, None, '___sec27'), ('Housing data, the code', 2, None, '___sec28'), ('The singular value decomposition', 2, None, '___sec29'), - ('The Ising model', 2, None, '___sec30'), - ('Reformulating the problem to suit regression', - 2, - None, - '___sec31'), - ('Linear regression', 2, None, '___sec32'), - ('Singular Value decomposition', 2, None, '___sec33'), - ('Linear Regression Problems', 2, None, '___sec34'), - ('Fixing the singularity', 2, None, '___sec35'), - ('Basic math of the SVD', 2, None, '___sec36'), - ('The SVD, a Fantastic Algorithm', 2, None, '___sec37'), - ('Another Example', 2, None, '___sec38'), - ('Economy-size SVD', 2, None, '___sec39'), - ('Mathematical Properties', 2, None, '___sec40'), - ('Ridge and LASSO Regression', 2, None, '___sec41'), - ('More on Ridge Regression', 2, None, '___sec42'), - ('Interpreting the Ridge results', 2, None, '___sec43'), - ('More interpretations', 2, None, '___sec44'), - ('Where are we going?', 2, None, '___sec45'), - ('Resampling methods', 2, None, '___sec46'), + ('Linear Regression Problems', 2, None, '___sec30'), + ('Fixing the singularity', 2, None, '___sec31'), + ('Basic math of the SVD', 2, None, '___sec32'), + ('The SVD, a Fantastic Algorithm', 2, None, '___sec33'), + ('Another Example', 2, None, '___sec34'), + ('Economy-size SVD', 2, None, '___sec35'), + ('Mathematical Properties', 2, None, '___sec36'), + ('Ridge and LASSO Regression', 2, None, '___sec37'), + ('More on Ridge Regression', 2, None, '___sec38'), + ('Interpreting the Ridge results', 2, None, '___sec39'), + ('More interpretations', 2, None, '___sec40'), + ('Where are we going?', 2, None, '___sec41'), + ('Resampling methods', 2, None, '___sec42'), ('Resampling approaches can be computationally expensive', 2, None, - '___sec47'), - ('Why resampling methods ?', 2, None, '___sec48'), - ('Statistical analysis', 2, None, '___sec49'), - ('Statistics', 2, None, '___sec50'), - ('Statistics, moments', 2, None, '___sec51'), - ('Statistics, central moments', 2, None, '___sec52'), - ('Statistics, covariance', 2, None, '___sec53'), - ('Statistics, more covariance', 2, None, '___sec54'), - ('Covariance example', 2, None, '___sec55'), - ('Covariance in numpy', 2, None, '___sec56'), - ('Statistics, independent variables', 2, None, '___sec57'), - ('Statistics, more variance', 2, None, '___sec58'), - ('Statistics and stochastic processes', 2, None, '___sec59'), - ('Statistics and sample variables', 2, None, '___sec60'), + '___sec43'), + ('Why resampling methods ?', 2, None, '___sec44'), + ('Statistical analysis', 2, None, '___sec45'), + ('Statistics', 2, None, '___sec46'), + ('Statistics, moments', 2, None, '___sec47'), + ('Statistics, central moments', 2, None, '___sec48'), + ('Statistics, covariance', 2, None, '___sec49'), + ('Statistics, more covariance', 2, None, '___sec50'), + ('Covariance example', 2, None, '___sec51'), + ('Covariance in numpy', 2, None, '___sec52'), + ('Statistics, independent variables', 2, None, '___sec53'), + ('Statistics, more variance', 2, None, '___sec54'), + ('Statistics and stochastic processes', 2, None, '___sec55'), + ('Statistics and sample variables', 2, None, '___sec56'), ('Statistics, sample variance and covariance', 2, None, - '___sec61'), - ('Statistics, law of large numbers', 2, None, '___sec62'), - ('Statistics, more on sample error', 2, None, '___sec63'), - ('Statistics', 2, None, '___sec64'), - ('Statistics, central limit theorem', 2, None, '___sec65'), - ('Statistics, more technicalities', 2, None, '___sec66'), - ('Statistics', 2, None, '___sec67'), - ('Statistics and sample variance', 2, None, '___sec68'), - ('Statistics, uncorrelated results', 2, None, '___sec69'), - ('Statistics, computations', 2, None, '___sec70'), + '___sec57'), + ('Statistics, law of large numbers', 2, None, '___sec58'), + ('Statistics, more on sample error', 2, None, '___sec59'), + ('Statistics', 2, None, '___sec60'), + ('Statistics, central limit theorem', 2, None, '___sec61'), + ('Statistics, more technicalities', 2, None, '___sec62'), + ('Statistics', 2, None, '___sec63'), + ('Statistics and sample variance', 2, None, '___sec64'), + ('Statistics, uncorrelated results', 2, None, '___sec65'), + ('Statistics, computations', 2, None, '___sec66'), ('Statistics, more on computations of errors', 2, None, - '___sec71'), - ('Statistics, wrapping up 1', 2, None, '___sec72'), - ('Statistics, final expression', 2, None, '___sec73'), + '___sec67'), + ('Statistics, wrapping up 1', 2, None, '___sec68'), + ('Statistics, final expression', 2, None, '___sec69'), ('Statistics, effective number of correlations', 2, None, - '___sec74'), + '___sec70'), ('Linking the regression analysis with a statistical ' 'interpretation', 2, None, - '___sec75'), - ('Assumptions made', 2, None, '___sec76'), - ('Expectation value and variance', 2, None, '___sec77'), + '___sec71'), + ('Assumptions made', 2, None, '___sec72'), + ('Expectation value and variance', 2, None, '___sec73'), ('Expectation value and variance for $\\boldsymbol{\\beta}$', 2, None, - '___sec78'), - ('Cross-validation', 2, None, '___sec79'), - ('Computationally expensive', 2, None, '___sec80'), - ('Various steps in cross-validation', 2, None, '___sec81'), + '___sec74'), + ('Cross-validation', 2, None, '___sec75'), + ('Computationally expensive', 2, None, '___sec76'), + ('Various steps in cross-validation', 2, None, '___sec77'), ('How to set up the cross-validation for Ridge and/or Lasso', 2, None, - '___sec82'), + '___sec78'), ('Resampling methods: Jackknife and Bootstrap', 2, None, - '___sec83'), - ('Resampling methods: Jackknife', 2, None, '___sec84'), - ('Jackknife code example', 2, None, '___sec85'), - ('Resampling methods: Bootstrap', 2, None, '___sec86'), - ('Resampling methods: Bootstrap background', 2, None, '___sec87'), + '___sec79'), + ('Resampling methods: Jackknife', 2, None, '___sec80'), + ('Jackknife code example', 2, None, '___sec81'), + ('Resampling methods: Bootstrap', 2, None, '___sec82'), + ('Resampling methods: Bootstrap background', 2, None, '___sec83'), ('Resampling methods: More Bootstrap background', 2, None, - '___sec88'), - ('Resampling methods: Bootstrap approach', 2, None, '___sec89'), - ('Resampling methods: Bootstrap steps', 2, None, '___sec90'), - ('Code example for the Bootstrap method', 2, None, '___sec91'), + '___sec84'), + ('Resampling methods: Bootstrap approach', 2, None, '___sec85'), + ('Resampling methods: Bootstrap steps', 2, None, '___sec86'), + ('Code example for the Bootstrap method', 2, None, '___sec87'), ('Code Example for Cross-validation and $k$-fold ' 'Cross-validation', 2, None, - '___sec92'), - ('The bias-variance tradeoff', 2, None, '___sec93'), - ('Example code for Bias-Variance tradeoff', 2, None, '___sec94'), - ('Understanding what happens', 2, None, '___sec95'), - ('Summing up', 2, None, '___sec96'), + '___sec88'), + ('The bias-variance tradeoff', 2, None, '___sec89'), + ('Example code for Bias-Variance tradeoff', 2, None, '___sec90'), + ('Understanding what happens', 2, None, '___sec91'), + ('Summing up', 2, None, '___sec92'), ("Another Example rom Scikit-Learn's Repository", 2, None, - '___sec97'), + '___sec93'), + ('The Ising model', 2, None, '___sec94'), + ('Reformulating the problem to suit regression', + 2, + None, + '___sec95'), + ('Linear regression', 2, None, '___sec96'), + ('Singular Value decomposition', 2, None, '___sec97'), ('The one-dimensional Ising model', 2, None, '___sec98'), ('Ridge regression', 2, None, '___sec99'), ('LASSO regression', 2, None, '___sec100'), @@ -295,74 +295,74 @@ MathJax.Hub.Config({
- + -
+Bootstrapping is a nonparametric approach to statistical inference +that substitutes computation for more traditional distributional +assumptions and asymptotic results. Bootstrapping offers a number of +advantages: -
-Two famous -resampling methods are the independent bootstrap and the jackknife. - -
-The jackknife is a special case of the independent bootstrap. Still, the jackknife was made -popular prior to the independent bootstrap. And as the popularity of -the independent bootstrap soared, new variants, such as the dependent bootstrap. - -
-The Jackknife and independent bootstrap work for -independent, identically distributed random variables. -If these conditions are not -satisfied, the methods will fail. Yet, it should be said that if the data are -independent, identically distributed, and we only want to estimate the -variance of \( \overline{X} \) (which often is the case), then there is no -need for bootstrapping. +Since \( \widehat{\theta} = \widehat{\theta}(\boldsymbol{X}) \) is a function of random variables, +\( \widehat{\theta} \) itself must be a random variable. Thus it has +a pdf, call this function \( p(\boldsymbol{t}) \). The aim of the bootstrap is to +estimate \( p(\boldsymbol{t}) \) by the relative frequency of +\( \widehat{\theta} \). You can think of this as using a histogram +in the place of \( p(\boldsymbol{t}) \). If the relative frequency closely +resembles \( p(\vec{t}) \), then using numerics, it is straight forward to +estimate all the interesting parameters of \( p(\boldsymbol{t}) \) using point +estimators.
diff --git a/doc/pub/Regression/html/._Regression-bs085.html b/doc/pub/Regression/html/._Regression-bs085.html index 832ab47e8..603c58d33 100644 --- a/doc/pub/Regression/html/._Regression-bs085.html +++ b/doc/pub/Regression/html/._Regression-bs085.html @@ -111,112 +111,112 @@ Automatically generated HTML file from DocOnce source ('The Boston housing data example', 2, None, '___sec27'), ('Housing data, the code', 2, None, '___sec28'), ('The singular value decomposition', 2, None, '___sec29'), - ('The Ising model', 2, None, '___sec30'), - ('Reformulating the problem to suit regression', - 2, - None, - '___sec31'), - ('Linear regression', 2, None, '___sec32'), - ('Singular Value decomposition', 2, None, '___sec33'), - ('Linear Regression Problems', 2, None, '___sec34'), - ('Fixing the singularity', 2, None, '___sec35'), - ('Basic math of the SVD', 2, None, '___sec36'), - ('The SVD, a Fantastic Algorithm', 2, None, '___sec37'), - ('Another Example', 2, None, '___sec38'), - ('Economy-size SVD', 2, None, '___sec39'), - ('Mathematical Properties', 2, None, '___sec40'), - ('Ridge and LASSO Regression', 2, None, '___sec41'), - ('More on Ridge Regression', 2, None, '___sec42'), - ('Interpreting the Ridge results', 2, None, '___sec43'), - ('More interpretations', 2, None, '___sec44'), - ('Where are we going?', 2, None, '___sec45'), - ('Resampling methods', 2, None, '___sec46'), + ('Linear Regression Problems', 2, None, '___sec30'), + ('Fixing the singularity', 2, None, '___sec31'), + ('Basic math of the SVD', 2, None, '___sec32'), + ('The SVD, a Fantastic Algorithm', 2, None, '___sec33'), + ('Another Example', 2, None, '___sec34'), + ('Economy-size SVD', 2, None, '___sec35'), + ('Mathematical Properties', 2, None, '___sec36'), + ('Ridge and LASSO Regression', 2, None, '___sec37'), + ('More on Ridge Regression', 2, None, '___sec38'), + ('Interpreting the Ridge results', 2, None, '___sec39'), + ('More interpretations', 2, None, '___sec40'), + ('Where are we going?', 2, None, '___sec41'), + ('Resampling methods', 2, None, '___sec42'), ('Resampling approaches can be computationally expensive', 2, None, - '___sec47'), - ('Why resampling methods ?', 2, None, '___sec48'), - ('Statistical analysis', 2, None, '___sec49'), - ('Statistics', 2, None, '___sec50'), - ('Statistics, moments', 2, None, '___sec51'), - ('Statistics, central moments', 2, None, '___sec52'), - ('Statistics, covariance', 2, None, '___sec53'), - ('Statistics, more covariance', 2, None, '___sec54'), - ('Covariance example', 2, None, '___sec55'), - ('Covariance in numpy', 2, None, '___sec56'), - ('Statistics, independent variables', 2, None, '___sec57'), - ('Statistics, more variance', 2, None, '___sec58'), - ('Statistics and stochastic processes', 2, None, '___sec59'), - ('Statistics and sample variables', 2, None, '___sec60'), + '___sec43'), + ('Why resampling methods ?', 2, None, '___sec44'), + ('Statistical analysis', 2, None, '___sec45'), + ('Statistics', 2, None, '___sec46'), + ('Statistics, moments', 2, None, '___sec47'), + ('Statistics, central moments', 2, None, '___sec48'), + ('Statistics, covariance', 2, None, '___sec49'), + ('Statistics, more covariance', 2, None, '___sec50'), + ('Covariance example', 2, None, '___sec51'), + ('Covariance in numpy', 2, None, '___sec52'), + ('Statistics, independent variables', 2, None, '___sec53'), + ('Statistics, more variance', 2, None, '___sec54'), + ('Statistics and stochastic processes', 2, None, '___sec55'), + ('Statistics and sample variables', 2, None, '___sec56'), ('Statistics, sample variance and covariance', 2, None, - '___sec61'), - ('Statistics, law of large numbers', 2, None, '___sec62'), - ('Statistics, more on sample error', 2, None, '___sec63'), - ('Statistics', 2, None, '___sec64'), - ('Statistics, central limit theorem', 2, None, '___sec65'), - ('Statistics, more technicalities', 2, None, '___sec66'), - ('Statistics', 2, None, '___sec67'), - ('Statistics and sample variance', 2, None, '___sec68'), - ('Statistics, uncorrelated results', 2, None, '___sec69'), - ('Statistics, computations', 2, None, '___sec70'), + '___sec57'), + ('Statistics, law of large numbers', 2, None, '___sec58'), + ('Statistics, more on sample error', 2, None, '___sec59'), + ('Statistics', 2, None, '___sec60'), + ('Statistics, central limit theorem', 2, None, '___sec61'), + ('Statistics, more technicalities', 2, None, '___sec62'), + ('Statistics', 2, None, '___sec63'), + ('Statistics and sample variance', 2, None, '___sec64'), + ('Statistics, uncorrelated results', 2, None, '___sec65'), + ('Statistics, computations', 2, None, '___sec66'), ('Statistics, more on computations of errors', 2, None, - '___sec71'), - ('Statistics, wrapping up 1', 2, None, '___sec72'), - ('Statistics, final expression', 2, None, '___sec73'), + '___sec67'), + ('Statistics, wrapping up 1', 2, None, '___sec68'), + ('Statistics, final expression', 2, None, '___sec69'), ('Statistics, effective number of correlations', 2, None, - '___sec74'), + '___sec70'), ('Linking the regression analysis with a statistical ' 'interpretation', 2, None, - '___sec75'), - ('Assumptions made', 2, None, '___sec76'), - ('Expectation value and variance', 2, None, '___sec77'), + '___sec71'), + ('Assumptions made', 2, None, '___sec72'), + ('Expectation value and variance', 2, None, '___sec73'), ('Expectation value and variance for $\\boldsymbol{\\beta}$', 2, None, - '___sec78'), - ('Cross-validation', 2, None, '___sec79'), - ('Computationally expensive', 2, None, '___sec80'), - ('Various steps in cross-validation', 2, None, '___sec81'), + '___sec74'), + ('Cross-validation', 2, None, '___sec75'), + ('Computationally expensive', 2, None, '___sec76'), + ('Various steps in cross-validation', 2, None, '___sec77'), ('How to set up the cross-validation for Ridge and/or Lasso', 2, None, - '___sec82'), + '___sec78'), ('Resampling methods: Jackknife and Bootstrap', 2, None, - '___sec83'), - ('Resampling methods: Jackknife', 2, None, '___sec84'), - ('Jackknife code example', 2, None, '___sec85'), - ('Resampling methods: Bootstrap', 2, None, '___sec86'), - ('Resampling methods: Bootstrap background', 2, None, '___sec87'), + '___sec79'), + ('Resampling methods: Jackknife', 2, None, '___sec80'), + ('Jackknife code example', 2, None, '___sec81'), + ('Resampling methods: Bootstrap', 2, None, '___sec82'), + ('Resampling methods: Bootstrap background', 2, None, '___sec83'), ('Resampling methods: More Bootstrap background', 2, None, - '___sec88'), - ('Resampling methods: Bootstrap approach', 2, None, '___sec89'), - ('Resampling methods: Bootstrap steps', 2, None, '___sec90'), - ('Code example for the Bootstrap method', 2, None, '___sec91'), + '___sec84'), + ('Resampling methods: Bootstrap approach', 2, None, '___sec85'), + ('Resampling methods: Bootstrap steps', 2, None, '___sec86'), + ('Code example for the Bootstrap method', 2, None, '___sec87'), ('Code Example for Cross-validation and $k$-fold ' 'Cross-validation', 2, None, - '___sec92'), - ('The bias-variance tradeoff', 2, None, '___sec93'), - ('Example code for Bias-Variance tradeoff', 2, None, '___sec94'), - ('Understanding what happens', 2, None, '___sec95'), - ('Summing up', 2, None, '___sec96'), + '___sec88'), + ('The bias-variance tradeoff', 2, None, '___sec89'), + ('Example code for Bias-Variance tradeoff', 2, None, '___sec90'), + ('Understanding what happens', 2, None, '___sec91'), + ('Summing up', 2, None, '___sec92'), ("Another Example rom Scikit-Learn's Repository", 2, None, - '___sec97'), + '___sec93'), + ('The Ising model', 2, None, '___sec94'), + ('Reformulating the problem to suit regression', + 2, + None, + '___sec95'), + ('Linear regression', 2, None, '___sec96'), + ('Singular Value decomposition', 2, None, '___sec97'), ('The one-dimensional Ising model', 2, None, '___sec98'), ('Ridge regression', 2, None, '___sec99'), ('LASSO regression', 2, None, '___sec100'), @@ -295,74 +295,74 @@ MathJax.Hub.Config({
-The Jackknife works by making many replicas of the estimator \( \widehat{\theta} \). -The jackknife is a resampling method where we systematically leave out one observation from the vector of observed values \( \boldsymbol{x} = (x_1,x_2,\cdots,X_n) \). -Let \( \boldsymbol{x}_i \) denote the vector -$$ -\boldsymbol{x}_i = (x_1,x_2,\cdots,x_{i-1},x_{i+1},\cdots,x_n), -$$ +In the case that \( \widehat{\theta} \) has +more than one component, and the components are independent, we use the +same estimator on each component separately. If the probability +density function of \( X_i \), \( p(x) \), had been known, then it would have +been straight forward to do this by: -
-which equals the vector \( \boldsymbol{x} \) with the exception that observation -number \( i \) is left out. Using this notation, define -\( \widehat{\theta}_i \) to be the estimator -\( \widehat{\theta} \) computed using \( \vec{X}_i \). +
diff --git a/doc/pub/Regression/html/._Regression-bs086.html b/doc/pub/Regression/html/._Regression-bs086.html index c52aa8363..efe41fa8a 100644 --- a/doc/pub/Regression/html/._Regression-bs086.html +++ b/doc/pub/Regression/html/._Regression-bs086.html @@ -111,112 +111,112 @@ Automatically generated HTML file from DocOnce source ('The Boston housing data example', 2, None, '___sec27'), ('Housing data, the code', 2, None, '___sec28'), ('The singular value decomposition', 2, None, '___sec29'), - ('The Ising model', 2, None, '___sec30'), - ('Reformulating the problem to suit regression', - 2, - None, - '___sec31'), - ('Linear regression', 2, None, '___sec32'), - ('Singular Value decomposition', 2, None, '___sec33'), - ('Linear Regression Problems', 2, None, '___sec34'), - ('Fixing the singularity', 2, None, '___sec35'), - ('Basic math of the SVD', 2, None, '___sec36'), - ('The SVD, a Fantastic Algorithm', 2, None, '___sec37'), - ('Another Example', 2, None, '___sec38'), - ('Economy-size SVD', 2, None, '___sec39'), - ('Mathematical Properties', 2, None, '___sec40'), - ('Ridge and LASSO Regression', 2, None, '___sec41'), - ('More on Ridge Regression', 2, None, '___sec42'), - ('Interpreting the Ridge results', 2, None, '___sec43'), - ('More interpretations', 2, None, '___sec44'), - ('Where are we going?', 2, None, '___sec45'), - ('Resampling methods', 2, None, '___sec46'), + ('Linear Regression Problems', 2, None, '___sec30'), + ('Fixing the singularity', 2, None, '___sec31'), + ('Basic math of the SVD', 2, None, '___sec32'), + ('The SVD, a Fantastic Algorithm', 2, None, '___sec33'), + ('Another Example', 2, None, '___sec34'), + ('Economy-size SVD', 2, None, '___sec35'), + ('Mathematical Properties', 2, None, '___sec36'), + ('Ridge and LASSO Regression', 2, None, '___sec37'), + ('More on Ridge Regression', 2, None, '___sec38'), + ('Interpreting the Ridge results', 2, None, '___sec39'), + ('More interpretations', 2, None, '___sec40'), + ('Where are we going?', 2, None, '___sec41'), + ('Resampling methods', 2, None, '___sec42'), ('Resampling approaches can be computationally expensive', 2, None, - '___sec47'), - ('Why resampling methods ?', 2, None, '___sec48'), - ('Statistical analysis', 2, None, '___sec49'), - ('Statistics', 2, None, '___sec50'), - ('Statistics, moments', 2, None, '___sec51'), - ('Statistics, central moments', 2, None, '___sec52'), - ('Statistics, covariance', 2, None, '___sec53'), - ('Statistics, more covariance', 2, None, '___sec54'), - ('Covariance example', 2, None, '___sec55'), - ('Covariance in numpy', 2, None, '___sec56'), - ('Statistics, independent variables', 2, None, '___sec57'), - ('Statistics, more variance', 2, None, '___sec58'), - ('Statistics and stochastic processes', 2, None, '___sec59'), - ('Statistics and sample variables', 2, None, '___sec60'), + '___sec43'), + ('Why resampling methods ?', 2, None, '___sec44'), + ('Statistical analysis', 2, None, '___sec45'), + ('Statistics', 2, None, '___sec46'), + ('Statistics, moments', 2, None, '___sec47'), + ('Statistics, central moments', 2, None, '___sec48'), + ('Statistics, covariance', 2, None, '___sec49'), + ('Statistics, more covariance', 2, None, '___sec50'), + ('Covariance example', 2, None, '___sec51'), + ('Covariance in numpy', 2, None, '___sec52'), + ('Statistics, independent variables', 2, None, '___sec53'), + ('Statistics, more variance', 2, None, '___sec54'), + ('Statistics and stochastic processes', 2, None, '___sec55'), + ('Statistics and sample variables', 2, None, '___sec56'), ('Statistics, sample variance and covariance', 2, None, - '___sec61'), - ('Statistics, law of large numbers', 2, None, '___sec62'), - ('Statistics, more on sample error', 2, None, '___sec63'), - ('Statistics', 2, None, '___sec64'), - ('Statistics, central limit theorem', 2, None, '___sec65'), - ('Statistics, more technicalities', 2, None, '___sec66'), - ('Statistics', 2, None, '___sec67'), - ('Statistics and sample variance', 2, None, '___sec68'), - ('Statistics, uncorrelated results', 2, None, '___sec69'), - ('Statistics, computations', 2, None, '___sec70'), + '___sec57'), + ('Statistics, law of large numbers', 2, None, '___sec58'), + ('Statistics, more on sample error', 2, None, '___sec59'), + ('Statistics', 2, None, '___sec60'), + ('Statistics, central limit theorem', 2, None, '___sec61'), + ('Statistics, more technicalities', 2, None, '___sec62'), + ('Statistics', 2, None, '___sec63'), + ('Statistics and sample variance', 2, None, '___sec64'), + ('Statistics, uncorrelated results', 2, None, '___sec65'), + ('Statistics, computations', 2, None, '___sec66'), ('Statistics, more on computations of errors', 2, None, - '___sec71'), - ('Statistics, wrapping up 1', 2, None, '___sec72'), - ('Statistics, final expression', 2, None, '___sec73'), + '___sec67'), + ('Statistics, wrapping up 1', 2, None, '___sec68'), + ('Statistics, final expression', 2, None, '___sec69'), ('Statistics, effective number of correlations', 2, None, - '___sec74'), + '___sec70'), ('Linking the regression analysis with a statistical ' 'interpretation', 2, None, - '___sec75'), - ('Assumptions made', 2, None, '___sec76'), - ('Expectation value and variance', 2, None, '___sec77'), + '___sec71'), + ('Assumptions made', 2, None, '___sec72'), + ('Expectation value and variance', 2, None, '___sec73'), ('Expectation value and variance for $\\boldsymbol{\\beta}$', 2, None, - '___sec78'), - ('Cross-validation', 2, None, '___sec79'), - ('Computationally expensive', 2, None, '___sec80'), - ('Various steps in cross-validation', 2, None, '___sec81'), + '___sec74'), + ('Cross-validation', 2, None, '___sec75'), + ('Computationally expensive', 2, None, '___sec76'), + ('Various steps in cross-validation', 2, None, '___sec77'), ('How to set up the cross-validation for Ridge and/or Lasso', 2, None, - '___sec82'), + '___sec78'), ('Resampling methods: Jackknife and Bootstrap', 2, None, - '___sec83'), - ('Resampling methods: Jackknife', 2, None, '___sec84'), - ('Jackknife code example', 2, None, '___sec85'), - ('Resampling methods: Bootstrap', 2, None, '___sec86'), - ('Resampling methods: Bootstrap background', 2, None, '___sec87'), + '___sec79'), + ('Resampling methods: Jackknife', 2, None, '___sec80'), + ('Jackknife code example', 2, None, '___sec81'), + ('Resampling methods: Bootstrap', 2, None, '___sec82'), + ('Resampling methods: Bootstrap background', 2, None, '___sec83'), ('Resampling methods: More Bootstrap background', 2, None, - '___sec88'), - ('Resampling methods: Bootstrap approach', 2, None, '___sec89'), - ('Resampling methods: Bootstrap steps', 2, None, '___sec90'), - ('Code example for the Bootstrap method', 2, None, '___sec91'), + '___sec84'), + ('Resampling methods: Bootstrap approach', 2, None, '___sec85'), + ('Resampling methods: Bootstrap steps', 2, None, '___sec86'), + ('Code example for the Bootstrap method', 2, None, '___sec87'), ('Code Example for Cross-validation and $k$-fold ' 'Cross-validation', 2, None, - '___sec92'), - ('The bias-variance tradeoff', 2, None, '___sec93'), - ('Example code for Bias-Variance tradeoff', 2, None, '___sec94'), - ('Understanding what happens', 2, None, '___sec95'), - ('Summing up', 2, None, '___sec96'), + '___sec88'), + ('The bias-variance tradeoff', 2, None, '___sec89'), + ('Example code for Bias-Variance tradeoff', 2, None, '___sec90'), + ('Understanding what happens', 2, None, '___sec91'), + ('Summing up', 2, None, '___sec92'), ("Another Example rom Scikit-Learn's Repository", 2, None, - '___sec97'), + '___sec93'), + ('The Ising model', 2, None, '___sec94'), + ('Reformulating the problem to suit regression', + 2, + None, + '___sec95'), + ('Linear regression', 2, None, '___sec96'), + ('Singular Value decomposition', 2, None, '___sec97'), ('The one-dimensional Ising model', 2, None, '___sec98'), ('Ridge regression', 2, None, '___sec99'), ('LASSO regression', 2, None, '___sec100'), @@ -295,74 +295,74 @@ MathJax.Hub.Config({
+But +unless there is enough information available about the process that +generated \( X_1,X_2,\cdots,X_n \), \( p(x) \) is in general +unknown. Therefore, Efron in 1979 asked the +question: What if we replace \( p(x) \) by the relative frequency +of the observation \( X_i \); if we draw observations in accordance with +the relative frequency of the observations, will we obtain the same +result in some asymptotic sense? The answer is yes. - -
from numpy import *
-from numpy.random import randint, randn
-from time import time
+
+Instead of generating the histogram for the relative
+frequency of the observation \( X_i \), just draw the values
+\( (X_1^*,X_2^*,\cdots,X_n^*) \) with replacement from the vector
+\( \boldsymbol{X} \).
-def jackknife(data, stat):
- n = len(data);t = zeros(n); inds = arange(n); t0 = time()
- ## 'jackknifing' by leaving out an observation for each i
- for i in range(n):
- t[i] = stat(delete(data,i) )
-
- # analysis
- print("Runtime: %g sec" % (time()-t0)); print("Jackknife Statistics :")
- print("original bias std. error")
- print("%8g %14g %15g" % (stat(data),(n-1)*mean(t)/n, (n*var(t))**.5))
-
- return t
-
-
-# Returns mean of data samples
-def stat(data):
- return mean(data)
-
-
-mu, sigma = 100, 15
-datapoints = 10000
-x = mu + sigma*random.randn(datapoints)
-# jackknife returns the data sample
-t = jackknife(x, stat)
-
diff --git a/doc/pub/Regression/html/._Regression-bs087.html b/doc/pub/Regression/html/._Regression-bs087.html index c26977d5f..c52711bcb 100644 --- a/doc/pub/Regression/html/._Regression-bs087.html +++ b/doc/pub/Regression/html/._Regression-bs087.html @@ -111,112 +111,112 @@ Automatically generated HTML file from DocOnce source ('The Boston housing data example', 2, None, '___sec27'), ('Housing data, the code', 2, None, '___sec28'), ('The singular value decomposition', 2, None, '___sec29'), - ('The Ising model', 2, None, '___sec30'), - ('Reformulating the problem to suit regression', - 2, - None, - '___sec31'), - ('Linear regression', 2, None, '___sec32'), - ('Singular Value decomposition', 2, None, '___sec33'), - ('Linear Regression Problems', 2, None, '___sec34'), - ('Fixing the singularity', 2, None, '___sec35'), - ('Basic math of the SVD', 2, None, '___sec36'), - ('The SVD, a Fantastic Algorithm', 2, None, '___sec37'), - ('Another Example', 2, None, '___sec38'), - ('Economy-size SVD', 2, None, '___sec39'), - ('Mathematical Properties', 2, None, '___sec40'), - ('Ridge and LASSO Regression', 2, None, '___sec41'), - ('More on Ridge Regression', 2, None, '___sec42'), - ('Interpreting the Ridge results', 2, None, '___sec43'), - ('More interpretations', 2, None, '___sec44'), - ('Where are we going?', 2, None, '___sec45'), - ('Resampling methods', 2, None, '___sec46'), + ('Linear Regression Problems', 2, None, '___sec30'), + ('Fixing the singularity', 2, None, '___sec31'), + ('Basic math of the SVD', 2, None, '___sec32'), + ('The SVD, a Fantastic Algorithm', 2, None, '___sec33'), + ('Another Example', 2, None, '___sec34'), + ('Economy-size SVD', 2, None, '___sec35'), + ('Mathematical Properties', 2, None, '___sec36'), + ('Ridge and LASSO Regression', 2, None, '___sec37'), + ('More on Ridge Regression', 2, None, '___sec38'), + ('Interpreting the Ridge results', 2, None, '___sec39'), + ('More interpretations', 2, None, '___sec40'), + ('Where are we going?', 2, None, '___sec41'), + ('Resampling methods', 2, None, '___sec42'), ('Resampling approaches can be computationally expensive', 2, None, - '___sec47'), - ('Why resampling methods ?', 2, None, '___sec48'), - ('Statistical analysis', 2, None, '___sec49'), - ('Statistics', 2, None, '___sec50'), - ('Statistics, moments', 2, None, '___sec51'), - ('Statistics, central moments', 2, None, '___sec52'), - ('Statistics, covariance', 2, None, '___sec53'), - ('Statistics, more covariance', 2, None, '___sec54'), - ('Covariance example', 2, None, '___sec55'), - ('Covariance in numpy', 2, None, '___sec56'), - ('Statistics, independent variables', 2, None, '___sec57'), - ('Statistics, more variance', 2, None, '___sec58'), - ('Statistics and stochastic processes', 2, None, '___sec59'), - ('Statistics and sample variables', 2, None, '___sec60'), + '___sec43'), + ('Why resampling methods ?', 2, None, '___sec44'), + ('Statistical analysis', 2, None, '___sec45'), + ('Statistics', 2, None, '___sec46'), + ('Statistics, moments', 2, None, '___sec47'), + ('Statistics, central moments', 2, None, '___sec48'), + ('Statistics, covariance', 2, None, '___sec49'), + ('Statistics, more covariance', 2, None, '___sec50'), + ('Covariance example', 2, None, '___sec51'), + ('Covariance in numpy', 2, None, '___sec52'), + ('Statistics, independent variables', 2, None, '___sec53'), + ('Statistics, more variance', 2, None, '___sec54'), + ('Statistics and stochastic processes', 2, None, '___sec55'), + ('Statistics and sample variables', 2, None, '___sec56'), ('Statistics, sample variance and covariance', 2, None, - '___sec61'), - ('Statistics, law of large numbers', 2, None, '___sec62'), - ('Statistics, more on sample error', 2, None, '___sec63'), - ('Statistics', 2, None, '___sec64'), - ('Statistics, central limit theorem', 2, None, '___sec65'), - ('Statistics, more technicalities', 2, None, '___sec66'), - ('Statistics', 2, None, '___sec67'), - ('Statistics and sample variance', 2, None, '___sec68'), - ('Statistics, uncorrelated results', 2, None, '___sec69'), - ('Statistics, computations', 2, None, '___sec70'), + '___sec57'), + ('Statistics, law of large numbers', 2, None, '___sec58'), + ('Statistics, more on sample error', 2, None, '___sec59'), + ('Statistics', 2, None, '___sec60'), + ('Statistics, central limit theorem', 2, None, '___sec61'), + ('Statistics, more technicalities', 2, None, '___sec62'), + ('Statistics', 2, None, '___sec63'), + ('Statistics and sample variance', 2, None, '___sec64'), + ('Statistics, uncorrelated results', 2, None, '___sec65'), + ('Statistics, computations', 2, None, '___sec66'), ('Statistics, more on computations of errors', 2, None, - '___sec71'), - ('Statistics, wrapping up 1', 2, None, '___sec72'), - ('Statistics, final expression', 2, None, '___sec73'), + '___sec67'), + ('Statistics, wrapping up 1', 2, None, '___sec68'), + ('Statistics, final expression', 2, None, '___sec69'), ('Statistics, effective number of correlations', 2, None, - '___sec74'), + '___sec70'), ('Linking the regression analysis with a statistical ' 'interpretation', 2, None, - '___sec75'), - ('Assumptions made', 2, None, '___sec76'), - ('Expectation value and variance', 2, None, '___sec77'), + '___sec71'), + ('Assumptions made', 2, None, '___sec72'), + ('Expectation value and variance', 2, None, '___sec73'), ('Expectation value and variance for $\\boldsymbol{\\beta}$', 2, None, - '___sec78'), - ('Cross-validation', 2, None, '___sec79'), - ('Computationally expensive', 2, None, '___sec80'), - ('Various steps in cross-validation', 2, None, '___sec81'), + '___sec74'), + ('Cross-validation', 2, None, '___sec75'), + ('Computationally expensive', 2, None, '___sec76'), + ('Various steps in cross-validation', 2, None, '___sec77'), ('How to set up the cross-validation for Ridge and/or Lasso', 2, None, - '___sec82'), + '___sec78'), ('Resampling methods: Jackknife and Bootstrap', 2, None, - '___sec83'), - ('Resampling methods: Jackknife', 2, None, '___sec84'), - ('Jackknife code example', 2, None, '___sec85'), - ('Resampling methods: Bootstrap', 2, None, '___sec86'), - ('Resampling methods: Bootstrap background', 2, None, '___sec87'), + '___sec79'), + ('Resampling methods: Jackknife', 2, None, '___sec80'), + ('Jackknife code example', 2, None, '___sec81'), + ('Resampling methods: Bootstrap', 2, None, '___sec82'), + ('Resampling methods: Bootstrap background', 2, None, '___sec83'), ('Resampling methods: More Bootstrap background', 2, None, - '___sec88'), - ('Resampling methods: Bootstrap approach', 2, None, '___sec89'), - ('Resampling methods: Bootstrap steps', 2, None, '___sec90'), - ('Code example for the Bootstrap method', 2, None, '___sec91'), + '___sec84'), + ('Resampling methods: Bootstrap approach', 2, None, '___sec85'), + ('Resampling methods: Bootstrap steps', 2, None, '___sec86'), + ('Code example for the Bootstrap method', 2, None, '___sec87'), ('Code Example for Cross-validation and $k$-fold ' 'Cross-validation', 2, None, - '___sec92'), - ('The bias-variance tradeoff', 2, None, '___sec93'), - ('Example code for Bias-Variance tradeoff', 2, None, '___sec94'), - ('Understanding what happens', 2, None, '___sec95'), - ('Summing up', 2, None, '___sec96'), + '___sec88'), + ('The bias-variance tradeoff', 2, None, '___sec89'), + ('Example code for Bias-Variance tradeoff', 2, None, '___sec90'), + ('Understanding what happens', 2, None, '___sec91'), + ('Summing up', 2, None, '___sec92'), ("Another Example rom Scikit-Learn's Repository", 2, None, - '___sec97'), + '___sec93'), + ('The Ising model', 2, None, '___sec94'), + ('Reformulating the problem to suit regression', + 2, + None, + '___sec95'), + ('Linear regression', 2, None, '___sec96'), + ('Singular Value decomposition', 2, None, '___sec97'), ('The one-dimensional Ising model', 2, None, '___sec98'), ('Ridge regression', 2, None, '___sec99'), ('LASSO regression', 2, None, '___sec100'), @@ -295,74 +295,74 @@ MathJax.Hub.Config({
-Bootstrapping is a nonparametric approach to statistical inference -that substitutes computation for more traditional distributional -assumptions and asymptotic results. Bootstrapping offers a number of -advantages: +
+The independent bootstrap works like this:
diff --git a/doc/pub/Regression/html/._Regression-bs088.html b/doc/pub/Regression/html/._Regression-bs088.html index 9bbc97bcb..24600b562 100644 --- a/doc/pub/Regression/html/._Regression-bs088.html +++ b/doc/pub/Regression/html/._Regression-bs088.html @@ -111,112 +111,112 @@ Automatically generated HTML file from DocOnce source ('The Boston housing data example', 2, None, '___sec27'), ('Housing data, the code', 2, None, '___sec28'), ('The singular value decomposition', 2, None, '___sec29'), - ('The Ising model', 2, None, '___sec30'), - ('Reformulating the problem to suit regression', - 2, - None, - '___sec31'), - ('Linear regression', 2, None, '___sec32'), - ('Singular Value decomposition', 2, None, '___sec33'), - ('Linear Regression Problems', 2, None, '___sec34'), - ('Fixing the singularity', 2, None, '___sec35'), - ('Basic math of the SVD', 2, None, '___sec36'), - ('The SVD, a Fantastic Algorithm', 2, None, '___sec37'), - ('Another Example', 2, None, '___sec38'), - ('Economy-size SVD', 2, None, '___sec39'), - ('Mathematical Properties', 2, None, '___sec40'), - ('Ridge and LASSO Regression', 2, None, '___sec41'), - ('More on Ridge Regression', 2, None, '___sec42'), - ('Interpreting the Ridge results', 2, None, '___sec43'), - ('More interpretations', 2, None, '___sec44'), - ('Where are we going?', 2, None, '___sec45'), - ('Resampling methods', 2, None, '___sec46'), + ('Linear Regression Problems', 2, None, '___sec30'), + ('Fixing the singularity', 2, None, '___sec31'), + ('Basic math of the SVD', 2, None, '___sec32'), + ('The SVD, a Fantastic Algorithm', 2, None, '___sec33'), + ('Another Example', 2, None, '___sec34'), + ('Economy-size SVD', 2, None, '___sec35'), + ('Mathematical Properties', 2, None, '___sec36'), + ('Ridge and LASSO Regression', 2, None, '___sec37'), + ('More on Ridge Regression', 2, None, '___sec38'), + ('Interpreting the Ridge results', 2, None, '___sec39'), + ('More interpretations', 2, None, '___sec40'), + ('Where are we going?', 2, None, '___sec41'), + ('Resampling methods', 2, None, '___sec42'), ('Resampling approaches can be computationally expensive', 2, None, - '___sec47'), - ('Why resampling methods ?', 2, None, '___sec48'), - ('Statistical analysis', 2, None, '___sec49'), - ('Statistics', 2, None, '___sec50'), - ('Statistics, moments', 2, None, '___sec51'), - ('Statistics, central moments', 2, None, '___sec52'), - ('Statistics, covariance', 2, None, '___sec53'), - ('Statistics, more covariance', 2, None, '___sec54'), - ('Covariance example', 2, None, '___sec55'), - ('Covariance in numpy', 2, None, '___sec56'), - ('Statistics, independent variables', 2, None, '___sec57'), - ('Statistics, more variance', 2, None, '___sec58'), - ('Statistics and stochastic processes', 2, None, '___sec59'), - ('Statistics and sample variables', 2, None, '___sec60'), + '___sec43'), + ('Why resampling methods ?', 2, None, '___sec44'), + ('Statistical analysis', 2, None, '___sec45'), + ('Statistics', 2, None, '___sec46'), + ('Statistics, moments', 2, None, '___sec47'), + ('Statistics, central moments', 2, None, '___sec48'), + ('Statistics, covariance', 2, None, '___sec49'), + ('Statistics, more covariance', 2, None, '___sec50'), + ('Covariance example', 2, None, '___sec51'), + ('Covariance in numpy', 2, None, '___sec52'), + ('Statistics, independent variables', 2, None, '___sec53'), + ('Statistics, more variance', 2, None, '___sec54'), + ('Statistics and stochastic processes', 2, None, '___sec55'), + ('Statistics and sample variables', 2, None, '___sec56'), ('Statistics, sample variance and covariance', 2, None, - '___sec61'), - ('Statistics, law of large numbers', 2, None, '___sec62'), - ('Statistics, more on sample error', 2, None, '___sec63'), - ('Statistics', 2, None, '___sec64'), - ('Statistics, central limit theorem', 2, None, '___sec65'), - ('Statistics, more technicalities', 2, None, '___sec66'), - ('Statistics', 2, None, '___sec67'), - ('Statistics and sample variance', 2, None, '___sec68'), - ('Statistics, uncorrelated results', 2, None, '___sec69'), - ('Statistics, computations', 2, None, '___sec70'), + '___sec57'), + ('Statistics, law of large numbers', 2, None, '___sec58'), + ('Statistics, more on sample error', 2, None, '___sec59'), + ('Statistics', 2, None, '___sec60'), + ('Statistics, central limit theorem', 2, None, '___sec61'), + ('Statistics, more technicalities', 2, None, '___sec62'), + ('Statistics', 2, None, '___sec63'), + ('Statistics and sample variance', 2, None, '___sec64'), + ('Statistics, uncorrelated results', 2, None, '___sec65'), + ('Statistics, computations', 2, None, '___sec66'), ('Statistics, more on computations of errors', 2, None, - '___sec71'), - ('Statistics, wrapping up 1', 2, None, '___sec72'), - ('Statistics, final expression', 2, None, '___sec73'), + '___sec67'), + ('Statistics, wrapping up 1', 2, None, '___sec68'), + ('Statistics, final expression', 2, None, '___sec69'), ('Statistics, effective number of correlations', 2, None, - '___sec74'), + '___sec70'), ('Linking the regression analysis with a statistical ' 'interpretation', 2, None, - '___sec75'), - ('Assumptions made', 2, None, '___sec76'), - ('Expectation value and variance', 2, None, '___sec77'), + '___sec71'), + ('Assumptions made', 2, None, '___sec72'), + ('Expectation value and variance', 2, None, '___sec73'), ('Expectation value and variance for $\\boldsymbol{\\beta}$', 2, None, - '___sec78'), - ('Cross-validation', 2, None, '___sec79'), - ('Computationally expensive', 2, None, '___sec80'), - ('Various steps in cross-validation', 2, None, '___sec81'), + '___sec74'), + ('Cross-validation', 2, None, '___sec75'), + ('Computationally expensive', 2, None, '___sec76'), + ('Various steps in cross-validation', 2, None, '___sec77'), ('How to set up the cross-validation for Ridge and/or Lasso', 2, None, - '___sec82'), + '___sec78'), ('Resampling methods: Jackknife and Bootstrap', 2, None, - '___sec83'), - ('Resampling methods: Jackknife', 2, None, '___sec84'), - ('Jackknife code example', 2, None, '___sec85'), - ('Resampling methods: Bootstrap', 2, None, '___sec86'), - ('Resampling methods: Bootstrap background', 2, None, '___sec87'), + '___sec79'), + ('Resampling methods: Jackknife', 2, None, '___sec80'), + ('Jackknife code example', 2, None, '___sec81'), + ('Resampling methods: Bootstrap', 2, None, '___sec82'), + ('Resampling methods: Bootstrap background', 2, None, '___sec83'), ('Resampling methods: More Bootstrap background', 2, None, - '___sec88'), - ('Resampling methods: Bootstrap approach', 2, None, '___sec89'), - ('Resampling methods: Bootstrap steps', 2, None, '___sec90'), - ('Code example for the Bootstrap method', 2, None, '___sec91'), + '___sec84'), + ('Resampling methods: Bootstrap approach', 2, None, '___sec85'), + ('Resampling methods: Bootstrap steps', 2, None, '___sec86'), + ('Code example for the Bootstrap method', 2, None, '___sec87'), ('Code Example for Cross-validation and $k$-fold ' 'Cross-validation', 2, None, - '___sec92'), - ('The bias-variance tradeoff', 2, None, '___sec93'), - ('Example code for Bias-Variance tradeoff', 2, None, '___sec94'), - ('Understanding what happens', 2, None, '___sec95'), - ('Summing up', 2, None, '___sec96'), + '___sec88'), + ('The bias-variance tradeoff', 2, None, '___sec89'), + ('Example code for Bias-Variance tradeoff', 2, None, '___sec90'), + ('Understanding what happens', 2, None, '___sec91'), + ('Summing up', 2, None, '___sec92'), ("Another Example rom Scikit-Learn's Repository", 2, None, - '___sec97'), + '___sec93'), + ('The Ising model', 2, None, '___sec94'), + ('Reformulating the problem to suit regression', + 2, + None, + '___sec95'), + ('Linear regression', 2, None, '___sec96'), + ('Singular Value decomposition', 2, None, '___sec97'), ('The one-dimensional Ising model', 2, None, '___sec98'), ('Ridge regression', 2, None, '___sec99'), ('LASSO regression', 2, None, '___sec100'), @@ -295,74 +295,74 @@ MathJax.Hub.Config({
-Since \( \widehat{\theta} = \widehat{\theta}(\boldsymbol{X}) \) is a function of random variables, -\( \widehat{\theta} \) itself must be a random variable. Thus it has -a pdf, call this function \( p(\boldsymbol{t}) \). The aim of the bootstrap is to -estimate \( p(\boldsymbol{t}) \) by the relative frequency of -\( \widehat{\theta} \). You can think of this as using a histogram -in the place of \( p(\boldsymbol{t}) \). If the relative frequency closely -resembles \( p(\vec{t}) \), then using numerics, it is straight forward to -estimate all the interesting parameters of \( p(\boldsymbol{t}) \) using point -estimators. +The following code starts with a Gaussian distribution with mean value +\( \mu =100 \) and variance \( \sigma=15 \). We use this to generate the data +used in the bootstrap analysis. The bootstrap analysis returns a data +set after a given number of bootstrap operations (as many as we have +data points). This data set consists of estimated mean values for each +bootstrap operation. The histogram generated by the bootstrap method +shows that the distribution for these mean values is also a Gaussian, +centered around the mean value \( \mu=100 \) but with standard deviation +\( \sigma/\sqrt{n} \), where \( n \) is the number of bootstrap samples (in +this case the same as the number of original data points). The value +of the standard deviation is what we expect from the central limit +theorem. +
+ + +
from numpy import *
+from numpy.random import randint, randn
+from time import time
+import matplotlib.mlab as mlab
+import matplotlib.pyplot as plt
+
+# Returns mean of bootstrap samples
+def stat(data):
+ return mean(data)
+
+# Bootstrap algorithm
+def bootstrap(data, statistic, R):
+ t = zeros(R); n = len(data); inds = arange(n); t0 = time()
+ # non-parametric bootstrap
+ for i in range(R):
+ t[i] = statistic(data[randint(0,n,n)])
+
+ # analysis
+ print("Runtime: %g sec" % (time()-t0)); print("Bootstrap Statistics :")
+ print("original bias std. error")
+ print("%8g %8g %14g %15g" % (statistic(data), std(data),mean(t),std(t)))
+ return t
+
+
+mu, sigma = 100, 15
+datapoints = 10000
+x = mu + sigma*random.randn(datapoints)
+# bootstrap returns the data sample
+t = bootstrap(x, stat, datapoints)
+# the histogram of the bootstrapped data
+n, binsboot, patches = plt.hist(t, 50, normed=1, facecolor='red', alpha=0.75)
+
+# add a 'best fit' line
+y = mlab.normpdf( binsboot, mean(t), std(t))
+lt = plt.plot(binsboot, y, 'r--', linewidth=1)
+plt.xlabel('Smarts')
+plt.ylabel('Probability')
+plt.axis([99.5, 100.6, 0, 3.0])
+plt.grid(True)
+
+plt.show()
+
diff --git a/doc/pub/Regression/html/._Regression-bs089.html b/doc/pub/Regression/html/._Regression-bs089.html index 183c6e607..8c221d91b 100644 --- a/doc/pub/Regression/html/._Regression-bs089.html +++ b/doc/pub/Regression/html/._Regression-bs089.html @@ -111,112 +111,112 @@ Automatically generated HTML file from DocOnce source ('The Boston housing data example', 2, None, '___sec27'), ('Housing data, the code', 2, None, '___sec28'), ('The singular value decomposition', 2, None, '___sec29'), - ('The Ising model', 2, None, '___sec30'), - ('Reformulating the problem to suit regression', - 2, - None, - '___sec31'), - ('Linear regression', 2, None, '___sec32'), - ('Singular Value decomposition', 2, None, '___sec33'), - ('Linear Regression Problems', 2, None, '___sec34'), - ('Fixing the singularity', 2, None, '___sec35'), - ('Basic math of the SVD', 2, None, '___sec36'), - ('The SVD, a Fantastic Algorithm', 2, None, '___sec37'), - ('Another Example', 2, None, '___sec38'), - ('Economy-size SVD', 2, None, '___sec39'), - ('Mathematical Properties', 2, None, '___sec40'), - ('Ridge and LASSO Regression', 2, None, '___sec41'), - ('More on Ridge Regression', 2, None, '___sec42'), - ('Interpreting the Ridge results', 2, None, '___sec43'), - ('More interpretations', 2, None, '___sec44'), - ('Where are we going?', 2, None, '___sec45'), - ('Resampling methods', 2, None, '___sec46'), + ('Linear Regression Problems', 2, None, '___sec30'), + ('Fixing the singularity', 2, None, '___sec31'), + ('Basic math of the SVD', 2, None, '___sec32'), + ('The SVD, a Fantastic Algorithm', 2, None, '___sec33'), + ('Another Example', 2, None, '___sec34'), + ('Economy-size SVD', 2, None, '___sec35'), + ('Mathematical Properties', 2, None, '___sec36'), + ('Ridge and LASSO Regression', 2, None, '___sec37'), + ('More on Ridge Regression', 2, None, '___sec38'), + ('Interpreting the Ridge results', 2, None, '___sec39'), + ('More interpretations', 2, None, '___sec40'), + ('Where are we going?', 2, None, '___sec41'), + ('Resampling methods', 2, None, '___sec42'), ('Resampling approaches can be computationally expensive', 2, None, - '___sec47'), - ('Why resampling methods ?', 2, None, '___sec48'), - ('Statistical analysis', 2, None, '___sec49'), - ('Statistics', 2, None, '___sec50'), - ('Statistics, moments', 2, None, '___sec51'), - ('Statistics, central moments', 2, None, '___sec52'), - ('Statistics, covariance', 2, None, '___sec53'), - ('Statistics, more covariance', 2, None, '___sec54'), - ('Covariance example', 2, None, '___sec55'), - ('Covariance in numpy', 2, None, '___sec56'), - ('Statistics, independent variables', 2, None, '___sec57'), - ('Statistics, more variance', 2, None, '___sec58'), - ('Statistics and stochastic processes', 2, None, '___sec59'), - ('Statistics and sample variables', 2, None, '___sec60'), + '___sec43'), + ('Why resampling methods ?', 2, None, '___sec44'), + ('Statistical analysis', 2, None, '___sec45'), + ('Statistics', 2, None, '___sec46'), + ('Statistics, moments', 2, None, '___sec47'), + ('Statistics, central moments', 2, None, '___sec48'), + ('Statistics, covariance', 2, None, '___sec49'), + ('Statistics, more covariance', 2, None, '___sec50'), + ('Covariance example', 2, None, '___sec51'), + ('Covariance in numpy', 2, None, '___sec52'), + ('Statistics, independent variables', 2, None, '___sec53'), + ('Statistics, more variance', 2, None, '___sec54'), + ('Statistics and stochastic processes', 2, None, '___sec55'), + ('Statistics and sample variables', 2, None, '___sec56'), ('Statistics, sample variance and covariance', 2, None, - '___sec61'), - ('Statistics, law of large numbers', 2, None, '___sec62'), - ('Statistics, more on sample error', 2, None, '___sec63'), - ('Statistics', 2, None, '___sec64'), - ('Statistics, central limit theorem', 2, None, '___sec65'), - ('Statistics, more technicalities', 2, None, '___sec66'), - ('Statistics', 2, None, '___sec67'), - ('Statistics and sample variance', 2, None, '___sec68'), - ('Statistics, uncorrelated results', 2, None, '___sec69'), - ('Statistics, computations', 2, None, '___sec70'), + '___sec57'), + ('Statistics, law of large numbers', 2, None, '___sec58'), + ('Statistics, more on sample error', 2, None, '___sec59'), + ('Statistics', 2, None, '___sec60'), + ('Statistics, central limit theorem', 2, None, '___sec61'), + ('Statistics, more technicalities', 2, None, '___sec62'), + ('Statistics', 2, None, '___sec63'), + ('Statistics and sample variance', 2, None, '___sec64'), + ('Statistics, uncorrelated results', 2, None, '___sec65'), + ('Statistics, computations', 2, None, '___sec66'), ('Statistics, more on computations of errors', 2, None, - '___sec71'), - ('Statistics, wrapping up 1', 2, None, '___sec72'), - ('Statistics, final expression', 2, None, '___sec73'), + '___sec67'), + ('Statistics, wrapping up 1', 2, None, '___sec68'), + ('Statistics, final expression', 2, None, '___sec69'), ('Statistics, effective number of correlations', 2, None, - '___sec74'), + '___sec70'), ('Linking the regression analysis with a statistical ' 'interpretation', 2, None, - '___sec75'), - ('Assumptions made', 2, None, '___sec76'), - ('Expectation value and variance', 2, None, '___sec77'), + '___sec71'), + ('Assumptions made', 2, None, '___sec72'), + ('Expectation value and variance', 2, None, '___sec73'), ('Expectation value and variance for $\\boldsymbol{\\beta}$', 2, None, - '___sec78'), - ('Cross-validation', 2, None, '___sec79'), - ('Computationally expensive', 2, None, '___sec80'), - ('Various steps in cross-validation', 2, None, '___sec81'), + '___sec74'), + ('Cross-validation', 2, None, '___sec75'), + ('Computationally expensive', 2, None, '___sec76'), + ('Various steps in cross-validation', 2, None, '___sec77'), ('How to set up the cross-validation for Ridge and/or Lasso', 2, None, - '___sec82'), + '___sec78'), ('Resampling methods: Jackknife and Bootstrap', 2, None, - '___sec83'), - ('Resampling methods: Jackknife', 2, None, '___sec84'), - ('Jackknife code example', 2, None, '___sec85'), - ('Resampling methods: Bootstrap', 2, None, '___sec86'), - ('Resampling methods: Bootstrap background', 2, None, '___sec87'), + '___sec79'), + ('Resampling methods: Jackknife', 2, None, '___sec80'), + ('Jackknife code example', 2, None, '___sec81'), + ('Resampling methods: Bootstrap', 2, None, '___sec82'), + ('Resampling methods: Bootstrap background', 2, None, '___sec83'), ('Resampling methods: More Bootstrap background', 2, None, - '___sec88'), - ('Resampling methods: Bootstrap approach', 2, None, '___sec89'), - ('Resampling methods: Bootstrap steps', 2, None, '___sec90'), - ('Code example for the Bootstrap method', 2, None, '___sec91'), + '___sec84'), + ('Resampling methods: Bootstrap approach', 2, None, '___sec85'), + ('Resampling methods: Bootstrap steps', 2, None, '___sec86'), + ('Code example for the Bootstrap method', 2, None, '___sec87'), ('Code Example for Cross-validation and $k$-fold ' 'Cross-validation', 2, None, - '___sec92'), - ('The bias-variance tradeoff', 2, None, '___sec93'), - ('Example code for Bias-Variance tradeoff', 2, None, '___sec94'), - ('Understanding what happens', 2, None, '___sec95'), - ('Summing up', 2, None, '___sec96'), + '___sec88'), + ('The bias-variance tradeoff', 2, None, '___sec89'), + ('Example code for Bias-Variance tradeoff', 2, None, '___sec90'), + ('Understanding what happens', 2, None, '___sec91'), + ('Summing up', 2, None, '___sec92'), ("Another Example rom Scikit-Learn's Repository", 2, None, - '___sec97'), + '___sec93'), + ('The Ising model', 2, None, '___sec94'), + ('Reformulating the problem to suit regression', + 2, + None, + '___sec95'), + ('Linear regression', 2, None, '___sec96'), + ('Singular Value decomposition', 2, None, '___sec97'), ('The one-dimensional Ising model', 2, None, '___sec98'), ('Ridge regression', 2, None, '___sec99'), ('LASSO regression', 2, None, '___sec100'), @@ -295,74 +295,74 @@ MathJax.Hub.Config({
-In the case that \( \widehat{\theta} \) has -more than one component, and the components are independent, we use the -same estimator on each component separately. If the probability -density function of \( X_i \), \( p(x) \), had been known, then it would have -been straight forward to do this by: +The code here uses Ridge regression with cross-validation (CV) resampling and \( k \)-fold CV in order to fit a specific polynomial. +
-
import numpy as np
+import matplotlib.pyplot as plt
+from sklearn.model_selection import KFold
+from sklearn.linear_model import Ridge
+from sklearn.model_selection import cross_val_score
+from sklearn.preprocessing import PolynomialFeatures
-By repeated use of (1) and (2), many
-estimates of \( \widehat{\theta} \) could have been obtained. The
-idea is to use the relative frequency of \( \widehat{\theta}^* \)
-(think of a histogram) as an estimate of \( p(\boldsymbol{t}) \).
+# A seed just to ensure that the random numbers are the same for every run.
+# Useful for eventual debugging.
+np.random.seed(3155)
+# Generate the data.
+nsamples = 100
+x = np.random.randn(nsamples)
+y = 3*x**2 + np.random.randn(nsamples)
+
+## Cross-validation on Ridge regression using KFold only
+
+# Decide degree on polynomial to fit
+poly = PolynomialFeatures(degree = 6)
+
+# Decide which values of lambda to use
+nlambdas = 500
+lambdas = np.logspace(-3, 5, nlambdas)
+
+# Initialize a KFold instance
+k = 5
+kfold = KFold(n_splits = k)
+
+# Perform the cross-validation to estimate MSE
+scores_KFold = np.zeros((nlambdas, k))
+
+i = 0
+for lmb in lambdas:
+ ridge = Ridge(alpha = lmb)
+ j = 0
+ for train_inds, test_inds in kfold.split(x):
+ xtrain = x[train_inds]
+ ytrain = y[train_inds]
+
+ xtest = x[test_inds]
+ ytest = y[test_inds]
+
+ Xtrain = poly.fit_transform(xtrain[:, np.newaxis])
+ ridge.fit(Xtrain, ytrain[:, np.newaxis])
+
+ Xtest = poly.fit_transform(xtest[:, np.newaxis])
+ ypred = ridge.predict(Xtest)
+
+ scores_KFold[i,j] = np.sum((ypred - ytest[:, np.newaxis])**2)/np.size(ypred)
+
+ j += 1
+ i += 1
+
+
+estimated_mse_KFold = np.mean(scores_KFold, axis = 1)
+
+## Cross-validation using cross_val_score from sklearn along with KFold
+
+# kfold is an instance initialized above as:
+# kfold = KFold(n_splits = k)
+
+estimated_mse_sklearn = np.zeros(nlambdas)
+i = 0
+for lmb in lambdas:
+ ridge = Ridge(alpha = lmb)
+
+ X = poly.fit_transform(x[:, np.newaxis])
+ estimated_mse_folds = cross_val_score(ridge, X, y[:, np.newaxis], scoring='neg_mean_squared_error', cv=kfold)
+
+ # cross_val_score return an array containing the estimated negative mse for every fold.
+ # we have to the the mean of every array in order to get an estimate of the mse of the model
+ estimated_mse_sklearn[i] = np.mean(-estimated_mse_folds)
+
+ i += 1
+
+## Plot and compare the slightly different ways to perform cross-validation
+
+plt.figure()
+
+plt.plot(np.log10(lambdas), estimated_mse_sklearn, label = 'cross_val_score')
+plt.plot(np.log10(lambdas), estimated_mse_KFold, 'r--', label = 'KFold')
+
+plt.xlabel('log10(lambda)')
+plt.ylabel('mse')
+
+plt.legend()
+
+plt.show()
+
diff --git a/doc/pub/Regression/html/._Regression-bs090.html b/doc/pub/Regression/html/._Regression-bs090.html index 6f523d063..b02a9ad7b 100644 --- a/doc/pub/Regression/html/._Regression-bs090.html +++ b/doc/pub/Regression/html/._Regression-bs090.html @@ -111,112 +111,112 @@ Automatically generated HTML file from DocOnce source ('The Boston housing data example', 2, None, '___sec27'), ('Housing data, the code', 2, None, '___sec28'), ('The singular value decomposition', 2, None, '___sec29'), - ('The Ising model', 2, None, '___sec30'), - ('Reformulating the problem to suit regression', - 2, - None, - '___sec31'), - ('Linear regression', 2, None, '___sec32'), - ('Singular Value decomposition', 2, None, '___sec33'), - ('Linear Regression Problems', 2, None, '___sec34'), - ('Fixing the singularity', 2, None, '___sec35'), - ('Basic math of the SVD', 2, None, '___sec36'), - ('The SVD, a Fantastic Algorithm', 2, None, '___sec37'), - ('Another Example', 2, None, '___sec38'), - ('Economy-size SVD', 2, None, '___sec39'), - ('Mathematical Properties', 2, None, '___sec40'), - ('Ridge and LASSO Regression', 2, None, '___sec41'), - ('More on Ridge Regression', 2, None, '___sec42'), - ('Interpreting the Ridge results', 2, None, '___sec43'), - ('More interpretations', 2, None, '___sec44'), - ('Where are we going?', 2, None, '___sec45'), - ('Resampling methods', 2, None, '___sec46'), + ('Linear Regression Problems', 2, None, '___sec30'), + ('Fixing the singularity', 2, None, '___sec31'), + ('Basic math of the SVD', 2, None, '___sec32'), + ('The SVD, a Fantastic Algorithm', 2, None, '___sec33'), + ('Another Example', 2, None, '___sec34'), + ('Economy-size SVD', 2, None, '___sec35'), + ('Mathematical Properties', 2, None, '___sec36'), + ('Ridge and LASSO Regression', 2, None, '___sec37'), + ('More on Ridge Regression', 2, None, '___sec38'), + ('Interpreting the Ridge results', 2, None, '___sec39'), + ('More interpretations', 2, None, '___sec40'), + ('Where are we going?', 2, None, '___sec41'), + ('Resampling methods', 2, None, '___sec42'), ('Resampling approaches can be computationally expensive', 2, None, - '___sec47'), - ('Why resampling methods ?', 2, None, '___sec48'), - ('Statistical analysis', 2, None, '___sec49'), - ('Statistics', 2, None, '___sec50'), - ('Statistics, moments', 2, None, '___sec51'), - ('Statistics, central moments', 2, None, '___sec52'), - ('Statistics, covariance', 2, None, '___sec53'), - ('Statistics, more covariance', 2, None, '___sec54'), - ('Covariance example', 2, None, '___sec55'), - ('Covariance in numpy', 2, None, '___sec56'), - ('Statistics, independent variables', 2, None, '___sec57'), - ('Statistics, more variance', 2, None, '___sec58'), - ('Statistics and stochastic processes', 2, None, '___sec59'), - ('Statistics and sample variables', 2, None, '___sec60'), + '___sec43'), + ('Why resampling methods ?', 2, None, '___sec44'), + ('Statistical analysis', 2, None, '___sec45'), + ('Statistics', 2, None, '___sec46'), + ('Statistics, moments', 2, None, '___sec47'), + ('Statistics, central moments', 2, None, '___sec48'), + ('Statistics, covariance', 2, None, '___sec49'), + ('Statistics, more covariance', 2, None, '___sec50'), + ('Covariance example', 2, None, '___sec51'), + ('Covariance in numpy', 2, None, '___sec52'), + ('Statistics, independent variables', 2, None, '___sec53'), + ('Statistics, more variance', 2, None, '___sec54'), + ('Statistics and stochastic processes', 2, None, '___sec55'), + ('Statistics and sample variables', 2, None, '___sec56'), ('Statistics, sample variance and covariance', 2, None, - '___sec61'), - ('Statistics, law of large numbers', 2, None, '___sec62'), - ('Statistics, more on sample error', 2, None, '___sec63'), - ('Statistics', 2, None, '___sec64'), - ('Statistics, central limit theorem', 2, None, '___sec65'), - ('Statistics, more technicalities', 2, None, '___sec66'), - ('Statistics', 2, None, '___sec67'), - ('Statistics and sample variance', 2, None, '___sec68'), - ('Statistics, uncorrelated results', 2, None, '___sec69'), - ('Statistics, computations', 2, None, '___sec70'), + '___sec57'), + ('Statistics, law of large numbers', 2, None, '___sec58'), + ('Statistics, more on sample error', 2, None, '___sec59'), + ('Statistics', 2, None, '___sec60'), + ('Statistics, central limit theorem', 2, None, '___sec61'), + ('Statistics, more technicalities', 2, None, '___sec62'), + ('Statistics', 2, None, '___sec63'), + ('Statistics and sample variance', 2, None, '___sec64'), + ('Statistics, uncorrelated results', 2, None, '___sec65'), + ('Statistics, computations', 2, None, '___sec66'), ('Statistics, more on computations of errors', 2, None, - '___sec71'), - ('Statistics, wrapping up 1', 2, None, '___sec72'), - ('Statistics, final expression', 2, None, '___sec73'), + '___sec67'), + ('Statistics, wrapping up 1', 2, None, '___sec68'), + ('Statistics, final expression', 2, None, '___sec69'), ('Statistics, effective number of correlations', 2, None, - '___sec74'), + '___sec70'), ('Linking the regression analysis with a statistical ' 'interpretation', 2, None, - '___sec75'), - ('Assumptions made', 2, None, '___sec76'), - ('Expectation value and variance', 2, None, '___sec77'), + '___sec71'), + ('Assumptions made', 2, None, '___sec72'), + ('Expectation value and variance', 2, None, '___sec73'), ('Expectation value and variance for $\\boldsymbol{\\beta}$', 2, None, - '___sec78'), - ('Cross-validation', 2, None, '___sec79'), - ('Computationally expensive', 2, None, '___sec80'), - ('Various steps in cross-validation', 2, None, '___sec81'), + '___sec74'), + ('Cross-validation', 2, None, '___sec75'), + ('Computationally expensive', 2, None, '___sec76'), + ('Various steps in cross-validation', 2, None, '___sec77'), ('How to set up the cross-validation for Ridge and/or Lasso', 2, None, - '___sec82'), + '___sec78'), ('Resampling methods: Jackknife and Bootstrap', 2, None, - '___sec83'), - ('Resampling methods: Jackknife', 2, None, '___sec84'), - ('Jackknife code example', 2, None, '___sec85'), - ('Resampling methods: Bootstrap', 2, None, '___sec86'), - ('Resampling methods: Bootstrap background', 2, None, '___sec87'), + '___sec79'), + ('Resampling methods: Jackknife', 2, None, '___sec80'), + ('Jackknife code example', 2, None, '___sec81'), + ('Resampling methods: Bootstrap', 2, None, '___sec82'), + ('Resampling methods: Bootstrap background', 2, None, '___sec83'), ('Resampling methods: More Bootstrap background', 2, None, - '___sec88'), - ('Resampling methods: Bootstrap approach', 2, None, '___sec89'), - ('Resampling methods: Bootstrap steps', 2, None, '___sec90'), - ('Code example for the Bootstrap method', 2, None, '___sec91'), + '___sec84'), + ('Resampling methods: Bootstrap approach', 2, None, '___sec85'), + ('Resampling methods: Bootstrap steps', 2, None, '___sec86'), + ('Code example for the Bootstrap method', 2, None, '___sec87'), ('Code Example for Cross-validation and $k$-fold ' 'Cross-validation', 2, None, - '___sec92'), - ('The bias-variance tradeoff', 2, None, '___sec93'), - ('Example code for Bias-Variance tradeoff', 2, None, '___sec94'), - ('Understanding what happens', 2, None, '___sec95'), - ('Summing up', 2, None, '___sec96'), + '___sec88'), + ('The bias-variance tradeoff', 2, None, '___sec89'), + ('Example code for Bias-Variance tradeoff', 2, None, '___sec90'), + ('Understanding what happens', 2, None, '___sec91'), + ('Summing up', 2, None, '___sec92'), ("Another Example rom Scikit-Learn's Repository", 2, None, - '___sec97'), + '___sec93'), + ('The Ising model', 2, None, '___sec94'), + ('Reformulating the problem to suit regression', + 2, + None, + '___sec95'), + ('Linear regression', 2, None, '___sec96'), + ('Singular Value decomposition', 2, None, '___sec97'), ('The one-dimensional Ising model', 2, None, '___sec98'), ('Ridge regression', 2, None, '___sec99'), ('LASSO regression', 2, None, '___sec100'), @@ -295,74 +295,74 @@ MathJax.Hub.Config({
-But -unless there is enough information available about the process that -generated \( X_1,X_2,\cdots,X_n \), \( p(x) \) is in general -unknown. Therefore, Efron in 1979 asked the -question: What if we replace \( p(x) \) by the relative frequency -of the observation \( X_i \); if we draw observations in accordance with -the relative frequency of the observations, will we obtain the same -result in some asymptotic sense? The answer is yes. +We will discuss the bias-variance tradeoff in the context of +continuous predictions such as regression. However, many of the +intuitions and ideas discussed here also carry over to classification +tasks. Consider a dataset \( \mathcal{L} \) consisting of the data +\( \mathbf{X}_\mathcal{L}=\{(y_j, \boldsymbol{x}_j), j=0\ldots n-1\} \).
-Instead of generating the histogram for the relative -frequency of the observation \( X_i \), just draw the values -\( (X_1^*,X_2^*,\cdots,X_n^*) \) with replacement from the vector -\( \boldsymbol{X} \). +Let us assume that the true data is generated from a noisy model + +$$ +\boldsymbol{y}=f(\boldsymbol{x}) + \boldsymbol{\epsilon} +$$ + +
+where \( \epsilon \) is normally distributed with mean zero and standard deviation \( \sigma^2 \). + +
+In our derivation of the ordinary least squares method we defined then +an approximation to the function \( f \) in terms of the parameters +\( \boldsymbol{\beta} \) and the design matrix \( \boldsymbol{X} \) which embody our model, +that is \( \boldsymbol{\tilde{y}}=\boldsymbol{X}\boldsymbol{\beta} \). + +
+Thereafter we found the parameters \( \boldsymbol{\beta} \) by optimizing the means squared error via the so-called cost function +$$ +C(\boldsymbol{X},\boldsymbol{\beta}) =\frac{1}{n}\sum_{i=0}^{n-1}(y_i-\tilde{y}_i)^2=\mathbb{E}\left[(\boldsymbol{y}-\boldsymbol{\tilde{y}})^2\right]. +$$ + +
+We can rewrite this as +$$ +\mathbb{E}\left[(\boldsymbol{y}-\boldsymbol{\tilde{y}})^2\right]=\frac{1}{n}\sum_i(f_i-\mathbb{E}\left[\boldsymbol{\tilde{y}}\right])^2+\frac{1}{n}\sum_i(\tilde{y}_i-\mathbb{E}\left[\boldsymbol{\tilde{y}}\right])^2+\sigma^2. +$$ + +
+The three terms represent the square of the bias of the learning +method, which can be thought of as the error caused by the simplifying +assumptions built into the method. The second term represents the +variance of the chosen model and finally the last terms is variance of +the error \( \boldsymbol{\epsilon} \). + +
+To derive this equation, we need to recall that the variance of \( \boldsymbol{y} \) and \( \boldsymbol{\epsilon} \) are both equal to \( \sigma^2 \). The mean value of \( \boldsymbol{\epsilon} \) is by definition equal to zero. Furthermore, the function \( f \) is not a stochastics variable, idem for \( \boldsymbol{\tilde{y}} \). +We use a more compact notation in terms of the expectation value +$$ +\mathbb{E}\left[(\boldsymbol{y}-\boldsymbol{\tilde{y}})^2\right]=\mathbb{E}\left[(\boldsymbol{f}+\boldsymbol{\epsilon}-\boldsymbol{\tilde{y}})^2\right], +$$ + +and adding and subtracting \( \mathbb{E}\left[\boldsymbol{\tilde{y}}\right] \) we get +$$ +\mathbb{E}\left[(\boldsymbol{y}-\boldsymbol{\tilde{y}})^2\right]=\mathbb{E}\left[(\boldsymbol{f}+\boldsymbol{\epsilon}-\boldsymbol{\tilde{y}}+\mathbb{E}\left[\boldsymbol{\tilde{y}}\right]-\mathbb{E}\left[\boldsymbol{\tilde{y}}\right])^2\right], +$$ + +which, using the abovementioned expectation values can be rewritten as +$$ +\mathbb{E}\left[(\boldsymbol{y}-\boldsymbol{\tilde{y}})^2\right]=\mathbb{E}\left[(\boldsymbol{y}-\mathbb{E}\left[\boldsymbol{\tilde{y}}\right])^2\right]+\mathrm{Var}\left[\boldsymbol{\tilde{y}}\right]+\sigma^2, +$$ + +that is the rewriting in terms of the so-called bias, the variance of the model \( \boldsymbol{\tilde{y}} \) and the variance of \( \boldsymbol{\epsilon} \).
diff --git a/doc/pub/Regression/html/._Regression-bs091.html b/doc/pub/Regression/html/._Regression-bs091.html index bdb92c3d0..04bca014d 100644 --- a/doc/pub/Regression/html/._Regression-bs091.html +++ b/doc/pub/Regression/html/._Regression-bs091.html @@ -111,112 +111,112 @@ Automatically generated HTML file from DocOnce source ('The Boston housing data example', 2, None, '___sec27'), ('Housing data, the code', 2, None, '___sec28'), ('The singular value decomposition', 2, None, '___sec29'), - ('The Ising model', 2, None, '___sec30'), - ('Reformulating the problem to suit regression', - 2, - None, - '___sec31'), - ('Linear regression', 2, None, '___sec32'), - ('Singular Value decomposition', 2, None, '___sec33'), - ('Linear Regression Problems', 2, None, '___sec34'), - ('Fixing the singularity', 2, None, '___sec35'), - ('Basic math of the SVD', 2, None, '___sec36'), - ('The SVD, a Fantastic Algorithm', 2, None, '___sec37'), - ('Another Example', 2, None, '___sec38'), - ('Economy-size SVD', 2, None, '___sec39'), - ('Mathematical Properties', 2, None, '___sec40'), - ('Ridge and LASSO Regression', 2, None, '___sec41'), - ('More on Ridge Regression', 2, None, '___sec42'), - ('Interpreting the Ridge results', 2, None, '___sec43'), - ('More interpretations', 2, None, '___sec44'), - ('Where are we going?', 2, None, '___sec45'), - ('Resampling methods', 2, None, '___sec46'), + ('Linear Regression Problems', 2, None, '___sec30'), + ('Fixing the singularity', 2, None, '___sec31'), + ('Basic math of the SVD', 2, None, '___sec32'), + ('The SVD, a Fantastic Algorithm', 2, None, '___sec33'), + ('Another Example', 2, None, '___sec34'), + ('Economy-size SVD', 2, None, '___sec35'), + ('Mathematical Properties', 2, None, '___sec36'), + ('Ridge and LASSO Regression', 2, None, '___sec37'), + ('More on Ridge Regression', 2, None, '___sec38'), + ('Interpreting the Ridge results', 2, None, '___sec39'), + ('More interpretations', 2, None, '___sec40'), + ('Where are we going?', 2, None, '___sec41'), + ('Resampling methods', 2, None, '___sec42'), ('Resampling approaches can be computationally expensive', 2, None, - '___sec47'), - ('Why resampling methods ?', 2, None, '___sec48'), - ('Statistical analysis', 2, None, '___sec49'), - ('Statistics', 2, None, '___sec50'), - ('Statistics, moments', 2, None, '___sec51'), - ('Statistics, central moments', 2, None, '___sec52'), - ('Statistics, covariance', 2, None, '___sec53'), - ('Statistics, more covariance', 2, None, '___sec54'), - ('Covariance example', 2, None, '___sec55'), - ('Covariance in numpy', 2, None, '___sec56'), - ('Statistics, independent variables', 2, None, '___sec57'), - ('Statistics, more variance', 2, None, '___sec58'), - ('Statistics and stochastic processes', 2, None, '___sec59'), - ('Statistics and sample variables', 2, None, '___sec60'), + '___sec43'), + ('Why resampling methods ?', 2, None, '___sec44'), + ('Statistical analysis', 2, None, '___sec45'), + ('Statistics', 2, None, '___sec46'), + ('Statistics, moments', 2, None, '___sec47'), + ('Statistics, central moments', 2, None, '___sec48'), + ('Statistics, covariance', 2, None, '___sec49'), + ('Statistics, more covariance', 2, None, '___sec50'), + ('Covariance example', 2, None, '___sec51'), + ('Covariance in numpy', 2, None, '___sec52'), + ('Statistics, independent variables', 2, None, '___sec53'), + ('Statistics, more variance', 2, None, '___sec54'), + ('Statistics and stochastic processes', 2, None, '___sec55'), + ('Statistics and sample variables', 2, None, '___sec56'), ('Statistics, sample variance and covariance', 2, None, - '___sec61'), - ('Statistics, law of large numbers', 2, None, '___sec62'), - ('Statistics, more on sample error', 2, None, '___sec63'), - ('Statistics', 2, None, '___sec64'), - ('Statistics, central limit theorem', 2, None, '___sec65'), - ('Statistics, more technicalities', 2, None, '___sec66'), - ('Statistics', 2, None, '___sec67'), - ('Statistics and sample variance', 2, None, '___sec68'), - ('Statistics, uncorrelated results', 2, None, '___sec69'), - ('Statistics, computations', 2, None, '___sec70'), + '___sec57'), + ('Statistics, law of large numbers', 2, None, '___sec58'), + ('Statistics, more on sample error', 2, None, '___sec59'), + ('Statistics', 2, None, '___sec60'), + ('Statistics, central limit theorem', 2, None, '___sec61'), + ('Statistics, more technicalities', 2, None, '___sec62'), + ('Statistics', 2, None, '___sec63'), + ('Statistics and sample variance', 2, None, '___sec64'), + ('Statistics, uncorrelated results', 2, None, '___sec65'), + ('Statistics, computations', 2, None, '___sec66'), ('Statistics, more on computations of errors', 2, None, - '___sec71'), - ('Statistics, wrapping up 1', 2, None, '___sec72'), - ('Statistics, final expression', 2, None, '___sec73'), + '___sec67'), + ('Statistics, wrapping up 1', 2, None, '___sec68'), + ('Statistics, final expression', 2, None, '___sec69'), ('Statistics, effective number of correlations', 2, None, - '___sec74'), + '___sec70'), ('Linking the regression analysis with a statistical ' 'interpretation', 2, None, - '___sec75'), - ('Assumptions made', 2, None, '___sec76'), - ('Expectation value and variance', 2, None, '___sec77'), + '___sec71'), + ('Assumptions made', 2, None, '___sec72'), + ('Expectation value and variance', 2, None, '___sec73'), ('Expectation value and variance for $\\boldsymbol{\\beta}$', 2, None, - '___sec78'), - ('Cross-validation', 2, None, '___sec79'), - ('Computationally expensive', 2, None, '___sec80'), - ('Various steps in cross-validation', 2, None, '___sec81'), + '___sec74'), + ('Cross-validation', 2, None, '___sec75'), + ('Computationally expensive', 2, None, '___sec76'), + ('Various steps in cross-validation', 2, None, '___sec77'), ('How to set up the cross-validation for Ridge and/or Lasso', 2, None, - '___sec82'), + '___sec78'), ('Resampling methods: Jackknife and Bootstrap', 2, None, - '___sec83'), - ('Resampling methods: Jackknife', 2, None, '___sec84'), - ('Jackknife code example', 2, None, '___sec85'), - ('Resampling methods: Bootstrap', 2, None, '___sec86'), - ('Resampling methods: Bootstrap background', 2, None, '___sec87'), + '___sec79'), + ('Resampling methods: Jackknife', 2, None, '___sec80'), + ('Jackknife code example', 2, None, '___sec81'), + ('Resampling methods: Bootstrap', 2, None, '___sec82'), + ('Resampling methods: Bootstrap background', 2, None, '___sec83'), ('Resampling methods: More Bootstrap background', 2, None, - '___sec88'), - ('Resampling methods: Bootstrap approach', 2, None, '___sec89'), - ('Resampling methods: Bootstrap steps', 2, None, '___sec90'), - ('Code example for the Bootstrap method', 2, None, '___sec91'), + '___sec84'), + ('Resampling methods: Bootstrap approach', 2, None, '___sec85'), + ('Resampling methods: Bootstrap steps', 2, None, '___sec86'), + ('Code example for the Bootstrap method', 2, None, '___sec87'), ('Code Example for Cross-validation and $k$-fold ' 'Cross-validation', 2, None, - '___sec92'), - ('The bias-variance tradeoff', 2, None, '___sec93'), - ('Example code for Bias-Variance tradeoff', 2, None, '___sec94'), - ('Understanding what happens', 2, None, '___sec95'), - ('Summing up', 2, None, '___sec96'), + '___sec88'), + ('The bias-variance tradeoff', 2, None, '___sec89'), + ('Example code for Bias-Variance tradeoff', 2, None, '___sec90'), + ('Understanding what happens', 2, None, '___sec91'), + ('Summing up', 2, None, '___sec92'), ("Another Example rom Scikit-Learn's Repository", 2, None, - '___sec97'), + '___sec93'), + ('The Ising model', 2, None, '___sec94'), + ('Reformulating the problem to suit regression', + 2, + None, + '___sec95'), + ('Linear regression', 2, None, '___sec96'), + ('Singular Value decomposition', 2, None, '___sec97'), ('The one-dimensional Ising model', 2, None, '___sec98'), ('Ridge regression', 2, None, '___sec99'), ('LASSO regression', 2, None, '___sec100'), @@ -295,74 +295,74 @@ MathJax.Hub.Config({
-The independent bootstrap works like this: -
import matplotlib.pyplot as plt
+import numpy as np
+from sklearn.linear_model import LinearRegression, Ridge, Lasso
+from sklearn.preprocessing import PolynomialFeatures
+from sklearn.model_selection import train_test_split
+from sklearn.pipeline import make_pipeline
+from sklearn.utils import resample
-When you are done, you can draw a histogram of the relative frequency
-of \( \widehat \theta^* \). This is your estimate of the probability
-distribution \( p(t) \). Using this probability distribution you can
-estimate any statistics thereof. In principle you never draw the
-histogram of the relative frequency of \( \widehat{\theta}^* \). Instead
-you use the estimators corresponding to the statistic of interest. For
-example, if you are interested in estimating the variance of \( \widehat
-\theta \), apply the etsimator \( \widehat \sigma^2 \) to the values
-\( \widehat \theta ^* \).
+np.random.seed(2018)
+n = 500
+n_boostraps = 100
+degree = 18 # A quite high value, just to show.
+noise = 0.1
+
+# Make data set.
+x = np.linspace(-1, 3, n).reshape(-1, 1)
+y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2) + np.random.normal(0, 0.1, x.shape)
+
+# Hold out some test data that is never used in training.
+x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2)
+
+# Combine x transformation and model into one operation.
+# Not neccesary, but convenient.
+model = make_pipeline(PolynomialFeatures(degree=degree), LinearRegression(fit_intercept=False))
+
+# The following (m x n_bootstraps) matrix holds the column vectors y_pred
+# for each bootstrap iteration.
+y_pred = np.empty((y_test.shape[0], n_boostraps))
+for i in range(n_boostraps):
+ x_, y_ = resample(x_train, y_train)
+
+ # Evaluate the new model on the same test data each time.
+ y_pred[:, i] = model.fit(x_, y_).predict(x_test).ravel()
+
+# Note: Expectations and variances taken w.r.t. different training
+# data sets, hence the axis=1. Subsequent means are taken across the test data
+# set in order to obtain a total value, but before this we have error/bias/variance
+# calculated per data point in the test set.
+# Note 2: The use of keepdims=True is important in the calculation of bias as this
+# maintains the column vector form. Dropping this yields very unexpected results.
+error = np.mean( np.mean((y_test - y_pred)**2, axis=1, keepdims=True) )
+bias = np.mean( (y_test - np.mean(y_pred, axis=1, keepdims=True))**2 )
+variance = np.mean( np.var(y_pred, axis=1, keepdims=True) )
+print('Error:', error)
+print('Bias^2:', bias)
+print('Var:', variance)
+print('{} >= {} + {} = {}'.format(error, bias, variance, bias+variance))
+
+plt.plot(x[::5, :], y[::5, :], label='f(x)')
+plt.scatter(x_test, y_test, label='Data points')
+plt.scatter(x_test, np.mean(y_pred, axis=1), label='Pred')
+plt.legend()
+plt.show()
+
diff --git a/doc/pub/Regression/html/._Regression-bs092.html b/doc/pub/Regression/html/._Regression-bs092.html index bd2d6da70..83aa75bc5 100644 --- a/doc/pub/Regression/html/._Regression-bs092.html +++ b/doc/pub/Regression/html/._Regression-bs092.html @@ -111,112 +111,112 @@ Automatically generated HTML file from DocOnce source ('The Boston housing data example', 2, None, '___sec27'), ('Housing data, the code', 2, None, '___sec28'), ('The singular value decomposition', 2, None, '___sec29'), - ('The Ising model', 2, None, '___sec30'), - ('Reformulating the problem to suit regression', - 2, - None, - '___sec31'), - ('Linear regression', 2, None, '___sec32'), - ('Singular Value decomposition', 2, None, '___sec33'), - ('Linear Regression Problems', 2, None, '___sec34'), - ('Fixing the singularity', 2, None, '___sec35'), - ('Basic math of the SVD', 2, None, '___sec36'), - ('The SVD, a Fantastic Algorithm', 2, None, '___sec37'), - ('Another Example', 2, None, '___sec38'), - ('Economy-size SVD', 2, None, '___sec39'), - ('Mathematical Properties', 2, None, '___sec40'), - ('Ridge and LASSO Regression', 2, None, '___sec41'), - ('More on Ridge Regression', 2, None, '___sec42'), - ('Interpreting the Ridge results', 2, None, '___sec43'), - ('More interpretations', 2, None, '___sec44'), - ('Where are we going?', 2, None, '___sec45'), - ('Resampling methods', 2, None, '___sec46'), + ('Linear Regression Problems', 2, None, '___sec30'), + ('Fixing the singularity', 2, None, '___sec31'), + ('Basic math of the SVD', 2, None, '___sec32'), + ('The SVD, a Fantastic Algorithm', 2, None, '___sec33'), + ('Another Example', 2, None, '___sec34'), + ('Economy-size SVD', 2, None, '___sec35'), + ('Mathematical Properties', 2, None, '___sec36'), + ('Ridge and LASSO Regression', 2, None, '___sec37'), + ('More on Ridge Regression', 2, None, '___sec38'), + ('Interpreting the Ridge results', 2, None, '___sec39'), + ('More interpretations', 2, None, '___sec40'), + ('Where are we going?', 2, None, '___sec41'), + ('Resampling methods', 2, None, '___sec42'), ('Resampling approaches can be computationally expensive', 2, None, - '___sec47'), - ('Why resampling methods ?', 2, None, '___sec48'), - ('Statistical analysis', 2, None, '___sec49'), - ('Statistics', 2, None, '___sec50'), - ('Statistics, moments', 2, None, '___sec51'), - ('Statistics, central moments', 2, None, '___sec52'), - ('Statistics, covariance', 2, None, '___sec53'), - ('Statistics, more covariance', 2, None, '___sec54'), - ('Covariance example', 2, None, '___sec55'), - ('Covariance in numpy', 2, None, '___sec56'), - ('Statistics, independent variables', 2, None, '___sec57'), - ('Statistics, more variance', 2, None, '___sec58'), - ('Statistics and stochastic processes', 2, None, '___sec59'), - ('Statistics and sample variables', 2, None, '___sec60'), + '___sec43'), + ('Why resampling methods ?', 2, None, '___sec44'), + ('Statistical analysis', 2, None, '___sec45'), + ('Statistics', 2, None, '___sec46'), + ('Statistics, moments', 2, None, '___sec47'), + ('Statistics, central moments', 2, None, '___sec48'), + ('Statistics, covariance', 2, None, '___sec49'), + ('Statistics, more covariance', 2, None, '___sec50'), + ('Covariance example', 2, None, '___sec51'), + ('Covariance in numpy', 2, None, '___sec52'), + ('Statistics, independent variables', 2, None, '___sec53'), + ('Statistics, more variance', 2, None, '___sec54'), + ('Statistics and stochastic processes', 2, None, '___sec55'), + ('Statistics and sample variables', 2, None, '___sec56'), ('Statistics, sample variance and covariance', 2, None, - '___sec61'), - ('Statistics, law of large numbers', 2, None, '___sec62'), - ('Statistics, more on sample error', 2, None, '___sec63'), - ('Statistics', 2, None, '___sec64'), - ('Statistics, central limit theorem', 2, None, '___sec65'), - ('Statistics, more technicalities', 2, None, '___sec66'), - ('Statistics', 2, None, '___sec67'), - ('Statistics and sample variance', 2, None, '___sec68'), - ('Statistics, uncorrelated results', 2, None, '___sec69'), - ('Statistics, computations', 2, None, '___sec70'), + '___sec57'), + ('Statistics, law of large numbers', 2, None, '___sec58'), + ('Statistics, more on sample error', 2, None, '___sec59'), + ('Statistics', 2, None, '___sec60'), + ('Statistics, central limit theorem', 2, None, '___sec61'), + ('Statistics, more technicalities', 2, None, '___sec62'), + ('Statistics', 2, None, '___sec63'), + ('Statistics and sample variance', 2, None, '___sec64'), + ('Statistics, uncorrelated results', 2, None, '___sec65'), + ('Statistics, computations', 2, None, '___sec66'), ('Statistics, more on computations of errors', 2, None, - '___sec71'), - ('Statistics, wrapping up 1', 2, None, '___sec72'), - ('Statistics, final expression', 2, None, '___sec73'), + '___sec67'), + ('Statistics, wrapping up 1', 2, None, '___sec68'), + ('Statistics, final expression', 2, None, '___sec69'), ('Statistics, effective number of correlations', 2, None, - '___sec74'), + '___sec70'), ('Linking the regression analysis with a statistical ' 'interpretation', 2, None, - '___sec75'), - ('Assumptions made', 2, None, '___sec76'), - ('Expectation value and variance', 2, None, '___sec77'), + '___sec71'), + ('Assumptions made', 2, None, '___sec72'), + ('Expectation value and variance', 2, None, '___sec73'), ('Expectation value and variance for $\\boldsymbol{\\beta}$', 2, None, - '___sec78'), - ('Cross-validation', 2, None, '___sec79'), - ('Computationally expensive', 2, None, '___sec80'), - ('Various steps in cross-validation', 2, None, '___sec81'), + '___sec74'), + ('Cross-validation', 2, None, '___sec75'), + ('Computationally expensive', 2, None, '___sec76'), + ('Various steps in cross-validation', 2, None, '___sec77'), ('How to set up the cross-validation for Ridge and/or Lasso', 2, None, - '___sec82'), + '___sec78'), ('Resampling methods: Jackknife and Bootstrap', 2, None, - '___sec83'), - ('Resampling methods: Jackknife', 2, None, '___sec84'), - ('Jackknife code example', 2, None, '___sec85'), - ('Resampling methods: Bootstrap', 2, None, '___sec86'), - ('Resampling methods: Bootstrap background', 2, None, '___sec87'), + '___sec79'), + ('Resampling methods: Jackknife', 2, None, '___sec80'), + ('Jackknife code example', 2, None, '___sec81'), + ('Resampling methods: Bootstrap', 2, None, '___sec82'), + ('Resampling methods: Bootstrap background', 2, None, '___sec83'), ('Resampling methods: More Bootstrap background', 2, None, - '___sec88'), - ('Resampling methods: Bootstrap approach', 2, None, '___sec89'), - ('Resampling methods: Bootstrap steps', 2, None, '___sec90'), - ('Code example for the Bootstrap method', 2, None, '___sec91'), + '___sec84'), + ('Resampling methods: Bootstrap approach', 2, None, '___sec85'), + ('Resampling methods: Bootstrap steps', 2, None, '___sec86'), + ('Code example for the Bootstrap method', 2, None, '___sec87'), ('Code Example for Cross-validation and $k$-fold ' 'Cross-validation', 2, None, - '___sec92'), - ('The bias-variance tradeoff', 2, None, '___sec93'), - ('Example code for Bias-Variance tradeoff', 2, None, '___sec94'), - ('Understanding what happens', 2, None, '___sec95'), - ('Summing up', 2, None, '___sec96'), + '___sec88'), + ('The bias-variance tradeoff', 2, None, '___sec89'), + ('Example code for Bias-Variance tradeoff', 2, None, '___sec90'), + ('Understanding what happens', 2, None, '___sec91'), + ('Summing up', 2, None, '___sec92'), ("Another Example rom Scikit-Learn's Repository", 2, None, - '___sec97'), + '___sec93'), + ('The Ising model', 2, None, '___sec94'), + ('Reformulating the problem to suit regression', + 2, + None, + '___sec95'), + ('Linear regression', 2, None, '___sec96'), + ('Singular Value decomposition', 2, None, '___sec97'), ('The one-dimensional Ising model', 2, None, '___sec98'), ('Ridge regression', 2, None, '___sec99'), ('LASSO regression', 2, None, '___sec100'), @@ -295,74 +295,74 @@ MathJax.Hub.Config({
-The following code starts with a Gaussian distribution with mean value -\( \mu =100 \) and variance \( \sigma=15 \). We use this to generate the data -used in the bootstrap analysis. The bootstrap analysis returns a data -set after a given number of bootstrap operations (as many as we have -data points). This data set consists of estimated mean values for each -bootstrap operation. The histogram generated by the bootstrap method -shows that the distribution for these mean values is also a Gaussian, -centered around the mean value \( \mu=100 \) but with standard deviation -\( \sigma/\sqrt{n} \), where \( n \) is the number of bootstrap samples (in -this case the same as the number of original data points). The value -of the standard deviation is what we expect from the central limit -theorem. - +
-
from numpy import *
-from numpy.random import randint, randn
-from time import time
-import matplotlib.mlab as mlab
-import matplotlib.pyplot as plt
+import matplotlib.pyplot as plt
+import numpy as np
+from sklearn.linear_model import LinearRegression, Ridge, Lasso
+from sklearn.preprocessing import PolynomialFeatures
+from sklearn.model_selection import train_test_split
+from sklearn.pipeline import make_pipeline
+from sklearn.utils import resample
-# Returns mean of bootstrap samples
-def stat(data):
- return mean(data)
+np.random.seed(2018)
-# Bootstrap algorithm
-def bootstrap(data, statistic, R):
- t = zeros(R); n = len(data); inds = arange(n); t0 = time()
- # non-parametric bootstrap
- for i in range(R):
- t[i] = statistic(data[randint(0,n,n)])
-
- # analysis
- print("Runtime: %g sec" % (time()-t0)); print("Bootstrap Statistics :")
- print("original bias std. error")
- print("%8g %8g %14g %15g" % (statistic(data), std(data),mean(t),std(t)))
- return t
+n = 40
+n_boostraps = 100
+maxdegree = 14
-mu, sigma = 100, 15
-datapoints = 10000
-x = mu + sigma*random.randn(datapoints)
-# bootstrap returns the data sample
-t = bootstrap(x, stat, datapoints)
-# the histogram of the bootstrapped data
-n, binsboot, patches = plt.hist(t, 50, normed=1, facecolor='red', alpha=0.75)
+# Make data set.
+x = np.linspace(-3, 3, n).reshape(-1, 1)
+y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)
+error = np.zeros(maxdegree)
+bias = np.zeros(maxdegree)
+variance = np.zeros(maxdegree)
+polydegree = np.zeros(maxdegree)
+x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2)
-# add a 'best fit' line
-y = mlab.normpdf( binsboot, mean(t), std(t))
-lt = plt.plot(binsboot, y, 'r--', linewidth=1)
-plt.xlabel('Smarts')
-plt.ylabel('Probability')
-plt.axis([99.5, 100.6, 0, 3.0])
-plt.grid(True)
+for degree in range(maxdegree):
+ model = make_pipeline(PolynomialFeatures(degree=degree), LinearRegression(fit_intercept=False))
+ y_pred = np.empty((y_test.shape[0], n_boostraps))
+ for i in range(n_boostraps):
+ x_, y_ = resample(x_train, y_train)
+ y_pred[:, i] = model.fit(x_, y_).predict(x_test).ravel()
+ polydegree[degree] = degree
+ error[degree] = np.mean( np.mean((y_test - y_pred)**2, axis=1, keepdims=True) )
+ bias[degree] = np.mean( (y_test - np.mean(y_pred, axis=1, keepdims=True))**2 )
+ variance[degree] = np.mean( np.var(y_pred, axis=1, keepdims=True) )
+ print('Polynomial degree:', degree)
+ print('Error:', error[degree])
+ print('Bias^2:', bias[degree])
+ print('Var:', variance[degree])
+ print('{} >= {} + {} = {}'.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree]))
+
+plt.plot(polydegree, error, label='Error')
+plt.plot(polydegree, bias, label='bias')
+plt.plot(polydegree, variance, label='Variance')
+plt.legend()
plt.show()
diff --git a/doc/pub/Regression/html/._Regression-bs093.html b/doc/pub/Regression/html/._Regression-bs093.html
index 36cb3cbc2..e97039030 100644
--- a/doc/pub/Regression/html/._Regression-bs093.html
+++ b/doc/pub/Regression/html/._Regression-bs093.html
@@ -111,112 +111,112 @@ Automatically generated HTML file from DocOnce source
('The Boston housing data example', 2, None, '___sec27'),
('Housing data, the code', 2, None, '___sec28'),
('The singular value decomposition', 2, None, '___sec29'),
- ('The Ising model', 2, None, '___sec30'),
- ('Reformulating the problem to suit regression',
- 2,
- None,
- '___sec31'),
- ('Linear regression', 2, None, '___sec32'),
- ('Singular Value decomposition', 2, None, '___sec33'),
- ('Linear Regression Problems', 2, None, '___sec34'),
- ('Fixing the singularity', 2, None, '___sec35'),
- ('Basic math of the SVD', 2, None, '___sec36'),
- ('The SVD, a Fantastic Algorithm', 2, None, '___sec37'),
- ('Another Example', 2, None, '___sec38'),
- ('Economy-size SVD', 2, None, '___sec39'),
- ('Mathematical Properties', 2, None, '___sec40'),
- ('Ridge and LASSO Regression', 2, None, '___sec41'),
- ('More on Ridge Regression', 2, None, '___sec42'),
- ('Interpreting the Ridge results', 2, None, '___sec43'),
- ('More interpretations', 2, None, '___sec44'),
- ('Where are we going?', 2, None, '___sec45'),
- ('Resampling methods', 2, None, '___sec46'),
+ ('Linear Regression Problems', 2, None, '___sec30'),
+ ('Fixing the singularity', 2, None, '___sec31'),
+ ('Basic math of the SVD', 2, None, '___sec32'),
+ ('The SVD, a Fantastic Algorithm', 2, None, '___sec33'),
+ ('Another Example', 2, None, '___sec34'),
+ ('Economy-size SVD', 2, None, '___sec35'),
+ ('Mathematical Properties', 2, None, '___sec36'),
+ ('Ridge and LASSO Regression', 2, None, '___sec37'),
+ ('More on Ridge Regression', 2, None, '___sec38'),
+ ('Interpreting the Ridge results', 2, None, '___sec39'),
+ ('More interpretations', 2, None, '___sec40'),
+ ('Where are we going?', 2, None, '___sec41'),
+ ('Resampling methods', 2, None, '___sec42'),
('Resampling approaches can be computationally expensive',
2,
None,
- '___sec47'),
- ('Why resampling methods ?', 2, None, '___sec48'),
- ('Statistical analysis', 2, None, '___sec49'),
- ('Statistics', 2, None, '___sec50'),
- ('Statistics, moments', 2, None, '___sec51'),
- ('Statistics, central moments', 2, None, '___sec52'),
- ('Statistics, covariance', 2, None, '___sec53'),
- ('Statistics, more covariance', 2, None, '___sec54'),
- ('Covariance example', 2, None, '___sec55'),
- ('Covariance in numpy', 2, None, '___sec56'),
- ('Statistics, independent variables', 2, None, '___sec57'),
- ('Statistics, more variance', 2, None, '___sec58'),
- ('Statistics and stochastic processes', 2, None, '___sec59'),
- ('Statistics and sample variables', 2, None, '___sec60'),
+ '___sec43'),
+ ('Why resampling methods ?', 2, None, '___sec44'),
+ ('Statistical analysis', 2, None, '___sec45'),
+ ('Statistics', 2, None, '___sec46'),
+ ('Statistics, moments', 2, None, '___sec47'),
+ ('Statistics, central moments', 2, None, '___sec48'),
+ ('Statistics, covariance', 2, None, '___sec49'),
+ ('Statistics, more covariance', 2, None, '___sec50'),
+ ('Covariance example', 2, None, '___sec51'),
+ ('Covariance in numpy', 2, None, '___sec52'),
+ ('Statistics, independent variables', 2, None, '___sec53'),
+ ('Statistics, more variance', 2, None, '___sec54'),
+ ('Statistics and stochastic processes', 2, None, '___sec55'),
+ ('Statistics and sample variables', 2, None, '___sec56'),
('Statistics, sample variance and covariance',
2,
None,
- '___sec61'),
- ('Statistics, law of large numbers', 2, None, '___sec62'),
- ('Statistics, more on sample error', 2, None, '___sec63'),
- ('Statistics', 2, None, '___sec64'),
- ('Statistics, central limit theorem', 2, None, '___sec65'),
- ('Statistics, more technicalities', 2, None, '___sec66'),
- ('Statistics', 2, None, '___sec67'),
- ('Statistics and sample variance', 2, None, '___sec68'),
- ('Statistics, uncorrelated results', 2, None, '___sec69'),
- ('Statistics, computations', 2, None, '___sec70'),
+ '___sec57'),
+ ('Statistics, law of large numbers', 2, None, '___sec58'),
+ ('Statistics, more on sample error', 2, None, '___sec59'),
+ ('Statistics', 2, None, '___sec60'),
+ ('Statistics, central limit theorem', 2, None, '___sec61'),
+ ('Statistics, more technicalities', 2, None, '___sec62'),
+ ('Statistics', 2, None, '___sec63'),
+ ('Statistics and sample variance', 2, None, '___sec64'),
+ ('Statistics, uncorrelated results', 2, None, '___sec65'),
+ ('Statistics, computations', 2, None, '___sec66'),
('Statistics, more on computations of errors',
2,
None,
- '___sec71'),
- ('Statistics, wrapping up 1', 2, None, '___sec72'),
- ('Statistics, final expression', 2, None, '___sec73'),
+ '___sec67'),
+ ('Statistics, wrapping up 1', 2, None, '___sec68'),
+ ('Statistics, final expression', 2, None, '___sec69'),
('Statistics, effective number of correlations',
2,
None,
- '___sec74'),
+ '___sec70'),
('Linking the regression analysis with a statistical '
'interpretation',
2,
None,
- '___sec75'),
- ('Assumptions made', 2, None, '___sec76'),
- ('Expectation value and variance', 2, None, '___sec77'),
+ '___sec71'),
+ ('Assumptions made', 2, None, '___sec72'),
+ ('Expectation value and variance', 2, None, '___sec73'),
('Expectation value and variance for $\\boldsymbol{\\beta}$',
2,
None,
- '___sec78'),
- ('Cross-validation', 2, None, '___sec79'),
- ('Computationally expensive', 2, None, '___sec80'),
- ('Various steps in cross-validation', 2, None, '___sec81'),
+ '___sec74'),
+ ('Cross-validation', 2, None, '___sec75'),
+ ('Computationally expensive', 2, None, '___sec76'),
+ ('Various steps in cross-validation', 2, None, '___sec77'),
('How to set up the cross-validation for Ridge and/or Lasso',
2,
None,
- '___sec82'),
+ '___sec78'),
('Resampling methods: Jackknife and Bootstrap',
2,
None,
- '___sec83'),
- ('Resampling methods: Jackknife', 2, None, '___sec84'),
- ('Jackknife code example', 2, None, '___sec85'),
- ('Resampling methods: Bootstrap', 2, None, '___sec86'),
- ('Resampling methods: Bootstrap background', 2, None, '___sec87'),
+ '___sec79'),
+ ('Resampling methods: Jackknife', 2, None, '___sec80'),
+ ('Jackknife code example', 2, None, '___sec81'),
+ ('Resampling methods: Bootstrap', 2, None, '___sec82'),
+ ('Resampling methods: Bootstrap background', 2, None, '___sec83'),
('Resampling methods: More Bootstrap background',
2,
None,
- '___sec88'),
- ('Resampling methods: Bootstrap approach', 2, None, '___sec89'),
- ('Resampling methods: Bootstrap steps', 2, None, '___sec90'),
- ('Code example for the Bootstrap method', 2, None, '___sec91'),
+ '___sec84'),
+ ('Resampling methods: Bootstrap approach', 2, None, '___sec85'),
+ ('Resampling methods: Bootstrap steps', 2, None, '___sec86'),
+ ('Code example for the Bootstrap method', 2, None, '___sec87'),
('Code Example for Cross-validation and $k$-fold '
'Cross-validation',
2,
None,
- '___sec92'),
- ('The bias-variance tradeoff', 2, None, '___sec93'),
- ('Example code for Bias-Variance tradeoff', 2, None, '___sec94'),
- ('Understanding what happens', 2, None, '___sec95'),
- ('Summing up', 2, None, '___sec96'),
+ '___sec88'),
+ ('The bias-variance tradeoff', 2, None, '___sec89'),
+ ('Example code for Bias-Variance tradeoff', 2, None, '___sec90'),
+ ('Understanding what happens', 2, None, '___sec91'),
+ ('Summing up', 2, None, '___sec92'),
("Another Example rom Scikit-Learn's Repository",
2,
None,
- '___sec97'),
+ '___sec93'),
+ ('The Ising model', 2, None, '___sec94'),
+ ('Reformulating the problem to suit regression',
+ 2,
+ None,
+ '___sec95'),
+ ('Linear regression', 2, None, '___sec96'),
+ ('Singular Value decomposition', 2, None, '___sec97'),
('The one-dimensional Ising model', 2, None, '___sec98'),
('Ridge regression', 2, None, '___sec99'),
('LASSO regression', 2, None, '___sec100'),
@@ -295,74 +295,74 @@ MathJax.Hub.Config({
- + -
-The code here uses Ridge regression with cross-validation (CV) resampling and \( k \)-fold CV in order to fit a specific polynomial. +The bias-variance tradeoff summarizes the fundamental tension in +machine learning, particularly supervised learning, between the +complexity of a model and the amount of training data needed to train +it. Since data is often limited, in practice it is often useful to +use a less-complex model with higher bias, that is a model whose asymptotic +performance is worse than another model because it is easier to +train and less sensitive to sampling noise arising from having a +finite-sized training dataset (smaller variance). +
+The above equations tell us that in +order to minimize the expected test error, we need to select a +statistical learning method that simultaneously achieves low variance +and low bias. Note that variance is inherently a nonnegative quantity, +and squared bias is also nonnegative. Hence, we see that the expected +test MSE can never lie below \( Var(\epsilon) \), the irreducible error. - -
import numpy as np
-import matplotlib.pyplot as plt
-from sklearn.model_selection import KFold
-from sklearn.linear_model import Ridge
-from sklearn.model_selection import cross_val_score
-from sklearn.preprocessing import PolynomialFeatures
+
+What do we mean by the variance and bias of a statistical learning
+method? The variance refers to the amount by which our model would change if we
+estimated it using a different training data set. Since the training
+data are used to fit the statistical learning method, different
+training data sets will result in a different estimate. But ideally the
+estimate for our model should not vary too much between training
+sets. However, if a method has high variance then small changes in
+the training data can result in large changes in the model. In general, more
+flexible statistical methods have higher variance.
-# A seed just to ensure that the random numbers are the same for every run.
-# Useful for eventual debugging.
-np.random.seed(3155)
-
-# Generate the data.
-nsamples = 100
-x = np.random.randn(nsamples)
-y = 3*x**2 + np.random.randn(nsamples)
-
-## Cross-validation on Ridge regression using KFold only
-
-# Decide degree on polynomial to fit
-poly = PolynomialFeatures(degree = 6)
-
-# Decide which values of lambda to use
-nlambdas = 500
-lambdas = np.logspace(-3, 5, nlambdas)
-
-# Initialize a KFold instance
-k = 5
-kfold = KFold(n_splits = k)
-
-# Perform the cross-validation to estimate MSE
-scores_KFold = np.zeros((nlambdas, k))
-
-i = 0
-for lmb in lambdas:
- ridge = Ridge(alpha = lmb)
- j = 0
- for train_inds, test_inds in kfold.split(x):
- xtrain = x[train_inds]
- ytrain = y[train_inds]
-
- xtest = x[test_inds]
- ytest = y[test_inds]
-
- Xtrain = poly.fit_transform(xtrain[:, np.newaxis])
- ridge.fit(Xtrain, ytrain[:, np.newaxis])
-
- Xtest = poly.fit_transform(xtest[:, np.newaxis])
- ypred = ridge.predict(Xtest)
-
- scores_KFold[i,j] = np.sum((ypred - ytest[:, np.newaxis])**2)/np.size(ypred)
-
- j += 1
- i += 1
-
-
-estimated_mse_KFold = np.mean(scores_KFold, axis = 1)
-
-## Cross-validation using cross_val_score from sklearn along with KFold
-
-# kfold is an instance initialized above as:
-# kfold = KFold(n_splits = k)
-
-estimated_mse_sklearn = np.zeros(nlambdas)
-i = 0
-for lmb in lambdas:
- ridge = Ridge(alpha = lmb)
-
- X = poly.fit_transform(x[:, np.newaxis])
- estimated_mse_folds = cross_val_score(ridge, X, y[:, np.newaxis], scoring='neg_mean_squared_error', cv=kfold)
-
- # cross_val_score return an array containing the estimated negative mse for every fold.
- # we have to the the mean of every array in order to get an estimate of the mse of the model
- estimated_mse_sklearn[i] = np.mean(-estimated_mse_folds)
-
- i += 1
-
-## Plot and compare the slightly different ways to perform cross-validation
-
-plt.figure()
-
-plt.plot(np.log10(lambdas), estimated_mse_sklearn, label = 'cross_val_score')
-plt.plot(np.log10(lambdas), estimated_mse_KFold, 'r--', label = 'KFold')
-
-plt.xlabel('log10(lambda)')
-plt.ylabel('mse')
-
-plt.legend()
-
-plt.show()
-
diff --git a/doc/pub/Regression/html/._Regression-bs094.html b/doc/pub/Regression/html/._Regression-bs094.html index c9e3650c3..bee332626 100644 --- a/doc/pub/Regression/html/._Regression-bs094.html +++ b/doc/pub/Regression/html/._Regression-bs094.html @@ -111,112 +111,112 @@ Automatically generated HTML file from DocOnce source ('The Boston housing data example', 2, None, '___sec27'), ('Housing data, the code', 2, None, '___sec28'), ('The singular value decomposition', 2, None, '___sec29'), - ('The Ising model', 2, None, '___sec30'), - ('Reformulating the problem to suit regression', - 2, - None, - '___sec31'), - ('Linear regression', 2, None, '___sec32'), - ('Singular Value decomposition', 2, None, '___sec33'), - ('Linear Regression Problems', 2, None, '___sec34'), - ('Fixing the singularity', 2, None, '___sec35'), - ('Basic math of the SVD', 2, None, '___sec36'), - ('The SVD, a Fantastic Algorithm', 2, None, '___sec37'), - ('Another Example', 2, None, '___sec38'), - ('Economy-size SVD', 2, None, '___sec39'), - ('Mathematical Properties', 2, None, '___sec40'), - ('Ridge and LASSO Regression', 2, None, '___sec41'), - ('More on Ridge Regression', 2, None, '___sec42'), - ('Interpreting the Ridge results', 2, None, '___sec43'), - ('More interpretations', 2, None, '___sec44'), - ('Where are we going?', 2, None, '___sec45'), - ('Resampling methods', 2, None, '___sec46'), + ('Linear Regression Problems', 2, None, '___sec30'), + ('Fixing the singularity', 2, None, '___sec31'), + ('Basic math of the SVD', 2, None, '___sec32'), + ('The SVD, a Fantastic Algorithm', 2, None, '___sec33'), + ('Another Example', 2, None, '___sec34'), + ('Economy-size SVD', 2, None, '___sec35'), + ('Mathematical Properties', 2, None, '___sec36'), + ('Ridge and LASSO Regression', 2, None, '___sec37'), + ('More on Ridge Regression', 2, None, '___sec38'), + ('Interpreting the Ridge results', 2, None, '___sec39'), + ('More interpretations', 2, None, '___sec40'), + ('Where are we going?', 2, None, '___sec41'), + ('Resampling methods', 2, None, '___sec42'), ('Resampling approaches can be computationally expensive', 2, None, - '___sec47'), - ('Why resampling methods ?', 2, None, '___sec48'), - ('Statistical analysis', 2, None, '___sec49'), - ('Statistics', 2, None, '___sec50'), - ('Statistics, moments', 2, None, '___sec51'), - ('Statistics, central moments', 2, None, '___sec52'), - ('Statistics, covariance', 2, None, '___sec53'), - ('Statistics, more covariance', 2, None, '___sec54'), - ('Covariance example', 2, None, '___sec55'), - ('Covariance in numpy', 2, None, '___sec56'), - ('Statistics, independent variables', 2, None, '___sec57'), - ('Statistics, more variance', 2, None, '___sec58'), - ('Statistics and stochastic processes', 2, None, '___sec59'), - ('Statistics and sample variables', 2, None, '___sec60'), + '___sec43'), + ('Why resampling methods ?', 2, None, '___sec44'), + ('Statistical analysis', 2, None, '___sec45'), + ('Statistics', 2, None, '___sec46'), + ('Statistics, moments', 2, None, '___sec47'), + ('Statistics, central moments', 2, None, '___sec48'), + ('Statistics, covariance', 2, None, '___sec49'), + ('Statistics, more covariance', 2, None, '___sec50'), + ('Covariance example', 2, None, '___sec51'), + ('Covariance in numpy', 2, None, '___sec52'), + ('Statistics, independent variables', 2, None, '___sec53'), + ('Statistics, more variance', 2, None, '___sec54'), + ('Statistics and stochastic processes', 2, None, '___sec55'), + ('Statistics and sample variables', 2, None, '___sec56'), ('Statistics, sample variance and covariance', 2, None, - '___sec61'), - ('Statistics, law of large numbers', 2, None, '___sec62'), - ('Statistics, more on sample error', 2, None, '___sec63'), - ('Statistics', 2, None, '___sec64'), - ('Statistics, central limit theorem', 2, None, '___sec65'), - ('Statistics, more technicalities', 2, None, '___sec66'), - ('Statistics', 2, None, '___sec67'), - ('Statistics and sample variance', 2, None, '___sec68'), - ('Statistics, uncorrelated results', 2, None, '___sec69'), - ('Statistics, computations', 2, None, '___sec70'), + '___sec57'), + ('Statistics, law of large numbers', 2, None, '___sec58'), + ('Statistics, more on sample error', 2, None, '___sec59'), + ('Statistics', 2, None, '___sec60'), + ('Statistics, central limit theorem', 2, None, '___sec61'), + ('Statistics, more technicalities', 2, None, '___sec62'), + ('Statistics', 2, None, '___sec63'), + ('Statistics and sample variance', 2, None, '___sec64'), + ('Statistics, uncorrelated results', 2, None, '___sec65'), + ('Statistics, computations', 2, None, '___sec66'), ('Statistics, more on computations of errors', 2, None, - '___sec71'), - ('Statistics, wrapping up 1', 2, None, '___sec72'), - ('Statistics, final expression', 2, None, '___sec73'), + '___sec67'), + ('Statistics, wrapping up 1', 2, None, '___sec68'), + ('Statistics, final expression', 2, None, '___sec69'), ('Statistics, effective number of correlations', 2, None, - '___sec74'), + '___sec70'), ('Linking the regression analysis with a statistical ' 'interpretation', 2, None, - '___sec75'), - ('Assumptions made', 2, None, '___sec76'), - ('Expectation value and variance', 2, None, '___sec77'), + '___sec71'), + ('Assumptions made', 2, None, '___sec72'), + ('Expectation value and variance', 2, None, '___sec73'), ('Expectation value and variance for $\\boldsymbol{\\beta}$', 2, None, - '___sec78'), - ('Cross-validation', 2, None, '___sec79'), - ('Computationally expensive', 2, None, '___sec80'), - ('Various steps in cross-validation', 2, None, '___sec81'), + '___sec74'), + ('Cross-validation', 2, None, '___sec75'), + ('Computationally expensive', 2, None, '___sec76'), + ('Various steps in cross-validation', 2, None, '___sec77'), ('How to set up the cross-validation for Ridge and/or Lasso', 2, None, - '___sec82'), + '___sec78'), ('Resampling methods: Jackknife and Bootstrap', 2, None, - '___sec83'), - ('Resampling methods: Jackknife', 2, None, '___sec84'), - ('Jackknife code example', 2, None, '___sec85'), - ('Resampling methods: Bootstrap', 2, None, '___sec86'), - ('Resampling methods: Bootstrap background', 2, None, '___sec87'), + '___sec79'), + ('Resampling methods: Jackknife', 2, None, '___sec80'), + ('Jackknife code example', 2, None, '___sec81'), + ('Resampling methods: Bootstrap', 2, None, '___sec82'), + ('Resampling methods: Bootstrap background', 2, None, '___sec83'), ('Resampling methods: More Bootstrap background', 2, None, - '___sec88'), - ('Resampling methods: Bootstrap approach', 2, None, '___sec89'), - ('Resampling methods: Bootstrap steps', 2, None, '___sec90'), - ('Code example for the Bootstrap method', 2, None, '___sec91'), + '___sec84'), + ('Resampling methods: Bootstrap approach', 2, None, '___sec85'), + ('Resampling methods: Bootstrap steps', 2, None, '___sec86'), + ('Code example for the Bootstrap method', 2, None, '___sec87'), ('Code Example for Cross-validation and $k$-fold ' 'Cross-validation', 2, None, - '___sec92'), - ('The bias-variance tradeoff', 2, None, '___sec93'), - ('Example code for Bias-Variance tradeoff', 2, None, '___sec94'), - ('Understanding what happens', 2, None, '___sec95'), - ('Summing up', 2, None, '___sec96'), + '___sec88'), + ('The bias-variance tradeoff', 2, None, '___sec89'), + ('Example code for Bias-Variance tradeoff', 2, None, '___sec90'), + ('Understanding what happens', 2, None, '___sec91'), + ('Summing up', 2, None, '___sec92'), ("Another Example rom Scikit-Learn's Repository", 2, None, - '___sec97'), + '___sec93'), + ('The Ising model', 2, None, '___sec94'), + ('Reformulating the problem to suit regression', + 2, + None, + '___sec95'), + ('Linear regression', 2, None, '___sec96'), + ('Singular Value decomposition', 2, None, '___sec97'), ('The one-dimensional Ising model', 2, None, '___sec98'), ('Ridge regression', 2, None, '___sec99'), ('LASSO regression', 2, None, '___sec100'), @@ -295,74 +295,74 @@ MathJax.Hub.Config({
-We will discuss the bias-variance tradeoff in the context of -continuous predictions such as regression. However, many of the -intuitions and ideas discussed here also carry over to classification -tasks. Consider a dataset \( \mathcal{L} \) consisting of the data -\( \mathbf{X}_\mathcal{L}=\{(y_j, \boldsymbol{x}_j), j=0\ldots n-1\} \). -
-Let us assume that the true data is generated from a noisy model + +
"""
+============================
+Underfitting vs. Overfitting
+============================
-$$
-\boldsymbol{y}=f(\boldsymbol{x}) + \boldsymbol{\epsilon}
-$$
+This example demonstrates the problems of underfitting and overfitting and
+how we can use linear regression with polynomial features to approximate
+nonlinear functions. The plot shows the function that we want to approximate,
+which is a part of the cosine function. In addition, the samples from the
+real function and the approximations of different models are displayed. The
+models have polynomial features of different degrees. We can see that a
+linear function (polynomial with degree 1) is not sufficient to fit the
+training samples. This is called **underfitting**. A polynomial of degree 4
+approximates the true function almost perfectly. However, for higher degrees
+the model will **overfit** the training data, i.e. it learns the noise of the
+training data.
+We evaluate quantitatively **overfitting** / **underfitting** by using
+cross-validation. We calculate the mean squared error (MSE) on the validation
+set, the higher, the less likely the model generalizes correctly from the
+training data.
+"""
-
-where \( \epsilon \) is normally distributed with mean zero and standard deviation \( \sigma^2 \).
+print(__doc__)
-
-In our derivation of the ordinary least squares method we defined then
-an approximation to the function \( f \) in terms of the parameters
-\( \boldsymbol{\beta} \) and the design matrix \( \boldsymbol{X} \) which embody our model,
-that is \( \boldsymbol{\tilde{y}}=\boldsymbol{X}\boldsymbol{\beta} \).
+import numpy as np
+import matplotlib.pyplot as plt
+from sklearn.pipeline import Pipeline
+from sklearn.preprocessing import PolynomialFeatures
+from sklearn.linear_model import LinearRegression
+from sklearn.model_selection import cross_val_score
-
-Thereafter we found the parameters \( \boldsymbol{\beta} \) by optimizing the means squared error via the so-called cost function
-$$
-C(\boldsymbol{X},\boldsymbol{\beta}) =\frac{1}{n}\sum_{i=0}^{n-1}(y_i-\tilde{y}_i)^2=\mathbb{E}\left[(\boldsymbol{y}-\boldsymbol{\tilde{y}})^2\right].
-$$
-
-We can rewrite this as
-$$
-\mathbb{E}\left[(\boldsymbol{y}-\boldsymbol{\tilde{y}})^2\right]=\frac{1}{n}\sum_i(f_i-\mathbb{E}\left[\boldsymbol{\tilde{y}}\right])^2+\frac{1}{n}\sum_i(\tilde{y}_i-\mathbb{E}\left[\boldsymbol{\tilde{y}}\right])^2+\sigma^2.
-$$
+def true_fun(X):
+ return np.cos(1.5 * np.pi * X)
-
-The three terms represent the square of the bias of the learning
-method, which can be thought of as the error caused by the simplifying
-assumptions built into the method. The second term represents the
-variance of the chosen model and finally the last terms is variance of
-the error \( \boldsymbol{\epsilon} \).
+np.random.seed(0)
-
-To derive this equation, we need to recall that the variance of \( \boldsymbol{y} \) and \( \boldsymbol{\epsilon} \) are both equal to \( \sigma^2 \). The mean value of \( \boldsymbol{\epsilon} \) is by definition equal to zero. Furthermore, the function \( f \) is not a stochastics variable, idem for \( \boldsymbol{\tilde{y}} \).
-We use a more compact notation in terms of the expectation value
-$$
-\mathbb{E}\left[(\boldsymbol{y}-\boldsymbol{\tilde{y}})^2\right]=\mathbb{E}\left[(\boldsymbol{f}+\boldsymbol{\epsilon}-\boldsymbol{\tilde{y}})^2\right],
-$$
+n_samples = 30
+degrees = [1, 4, 15]
-and adding and subtracting \( \mathbb{E}\left[\boldsymbol{\tilde{y}}\right] \) we get
-$$
-\mathbb{E}\left[(\boldsymbol{y}-\boldsymbol{\tilde{y}})^2\right]=\mathbb{E}\left[(\boldsymbol{f}+\boldsymbol{\epsilon}-\boldsymbol{\tilde{y}}+\mathbb{E}\left[\boldsymbol{\tilde{y}}\right]-\mathbb{E}\left[\boldsymbol{\tilde{y}}\right])^2\right],
-$$
+X = np.sort(np.random.rand(n_samples))
+y = true_fun(X) + np.random.randn(n_samples) * 0.1
-which, using the abovementioned expectation values can be rewritten as
-$$
-\mathbb{E}\left[(\boldsymbol{y}-\boldsymbol{\tilde{y}})^2\right]=\mathbb{E}\left[(\boldsymbol{y}-\mathbb{E}\left[\boldsymbol{\tilde{y}}\right])^2\right]+\mathrm{Var}\left[\boldsymbol{\tilde{y}}\right]+\sigma^2,
-$$
+plt.figure(figsize=(14, 5))
+for i in range(len(degrees)):
+ ax = plt.subplot(1, len(degrees), i + 1)
+ plt.setp(ax, xticks=(), yticks=())
-that is the rewriting in terms of the so-called bias, the variance of the model \( \boldsymbol{\tilde{y}} \) and the variance of \( \boldsymbol{\epsilon} \).
+ polynomial_features = PolynomialFeatures(degree=degrees[i],
+ include_bias=False)
+ linear_regression = LinearRegression()
+ pipeline = Pipeline([("polynomial_features", polynomial_features),
+ ("linear_regression", linear_regression)])
+ pipeline.fit(X[:, np.newaxis], y)
+ # Evaluate the models using crossvalidation
+ scores = cross_val_score(pipeline, X[:, np.newaxis], y,
+ scoring="neg_mean_squared_error", cv=10)
+
+ X_test = np.linspace(0, 1, 100)
+ plt.plot(X_test, pipeline.predict(X_test[:, np.newaxis]), label="Model")
+ plt.plot(X_test, true_fun(X_test), label="True function")
+ plt.scatter(X, y, edgecolor='b', s=20, label="Samples")
+ plt.xlabel("x")
+ plt.ylabel("y")
+ plt.xlim((0, 1))
+ plt.ylim((-2, 2))
+ plt.legend(loc="best")
+ plt.title("Degree {}\nMSE = {:.2e}(+/- {:.2e})".format(
+ degrees[i], -scores.mean(), scores.std()))
+plt.show()
+
diff --git a/doc/pub/Regression/html/._Regression-bs095.html b/doc/pub/Regression/html/._Regression-bs095.html index 63247350c..66f3b7357 100644 --- a/doc/pub/Regression/html/._Regression-bs095.html +++ b/doc/pub/Regression/html/._Regression-bs095.html @@ -111,112 +111,112 @@ Automatically generated HTML file from DocOnce source ('The Boston housing data example', 2, None, '___sec27'), ('Housing data, the code', 2, None, '___sec28'), ('The singular value decomposition', 2, None, '___sec29'), - ('The Ising model', 2, None, '___sec30'), - ('Reformulating the problem to suit regression', - 2, - None, - '___sec31'), - ('Linear regression', 2, None, '___sec32'), - ('Singular Value decomposition', 2, None, '___sec33'), - ('Linear Regression Problems', 2, None, '___sec34'), - ('Fixing the singularity', 2, None, '___sec35'), - ('Basic math of the SVD', 2, None, '___sec36'), - ('The SVD, a Fantastic Algorithm', 2, None, '___sec37'), - ('Another Example', 2, None, '___sec38'), - ('Economy-size SVD', 2, None, '___sec39'), - ('Mathematical Properties', 2, None, '___sec40'), - ('Ridge and LASSO Regression', 2, None, '___sec41'), - ('More on Ridge Regression', 2, None, '___sec42'), - ('Interpreting the Ridge results', 2, None, '___sec43'), - ('More interpretations', 2, None, '___sec44'), - ('Where are we going?', 2, None, '___sec45'), - ('Resampling methods', 2, None, '___sec46'), + ('Linear Regression Problems', 2, None, '___sec30'), + ('Fixing the singularity', 2, None, '___sec31'), + ('Basic math of the SVD', 2, None, '___sec32'), + ('The SVD, a Fantastic Algorithm', 2, None, '___sec33'), + ('Another Example', 2, None, '___sec34'), + ('Economy-size SVD', 2, None, '___sec35'), + ('Mathematical Properties', 2, None, '___sec36'), + ('Ridge and LASSO Regression', 2, None, '___sec37'), + ('More on Ridge Regression', 2, None, '___sec38'), + ('Interpreting the Ridge results', 2, None, '___sec39'), + ('More interpretations', 2, None, '___sec40'), + ('Where are we going?', 2, None, '___sec41'), + ('Resampling methods', 2, None, '___sec42'), ('Resampling approaches can be computationally expensive', 2, None, - '___sec47'), - ('Why resampling methods ?', 2, None, '___sec48'), - ('Statistical analysis', 2, None, '___sec49'), - ('Statistics', 2, None, '___sec50'), - ('Statistics, moments', 2, None, '___sec51'), - ('Statistics, central moments', 2, None, '___sec52'), - ('Statistics, covariance', 2, None, '___sec53'), - ('Statistics, more covariance', 2, None, '___sec54'), - ('Covariance example', 2, None, '___sec55'), - ('Covariance in numpy', 2, None, '___sec56'), - ('Statistics, independent variables', 2, None, '___sec57'), - ('Statistics, more variance', 2, None, '___sec58'), - ('Statistics and stochastic processes', 2, None, '___sec59'), - ('Statistics and sample variables', 2, None, '___sec60'), + '___sec43'), + ('Why resampling methods ?', 2, None, '___sec44'), + ('Statistical analysis', 2, None, '___sec45'), + ('Statistics', 2, None, '___sec46'), + ('Statistics, moments', 2, None, '___sec47'), + ('Statistics, central moments', 2, None, '___sec48'), + ('Statistics, covariance', 2, None, '___sec49'), + ('Statistics, more covariance', 2, None, '___sec50'), + ('Covariance example', 2, None, '___sec51'), + ('Covariance in numpy', 2, None, '___sec52'), + ('Statistics, independent variables', 2, None, '___sec53'), + ('Statistics, more variance', 2, None, '___sec54'), + ('Statistics and stochastic processes', 2, None, '___sec55'), + ('Statistics and sample variables', 2, None, '___sec56'), ('Statistics, sample variance and covariance', 2, None, - '___sec61'), - ('Statistics, law of large numbers', 2, None, '___sec62'), - ('Statistics, more on sample error', 2, None, '___sec63'), - ('Statistics', 2, None, '___sec64'), - ('Statistics, central limit theorem', 2, None, '___sec65'), - ('Statistics, more technicalities', 2, None, '___sec66'), - ('Statistics', 2, None, '___sec67'), - ('Statistics and sample variance', 2, None, '___sec68'), - ('Statistics, uncorrelated results', 2, None, '___sec69'), - ('Statistics, computations', 2, None, '___sec70'), + '___sec57'), + ('Statistics, law of large numbers', 2, None, '___sec58'), + ('Statistics, more on sample error', 2, None, '___sec59'), + ('Statistics', 2, None, '___sec60'), + ('Statistics, central limit theorem', 2, None, '___sec61'), + ('Statistics, more technicalities', 2, None, '___sec62'), + ('Statistics', 2, None, '___sec63'), + ('Statistics and sample variance', 2, None, '___sec64'), + ('Statistics, uncorrelated results', 2, None, '___sec65'), + ('Statistics, computations', 2, None, '___sec66'), ('Statistics, more on computations of errors', 2, None, - '___sec71'), - ('Statistics, wrapping up 1', 2, None, '___sec72'), - ('Statistics, final expression', 2, None, '___sec73'), + '___sec67'), + ('Statistics, wrapping up 1', 2, None, '___sec68'), + ('Statistics, final expression', 2, None, '___sec69'), ('Statistics, effective number of correlations', 2, None, - '___sec74'), + '___sec70'), ('Linking the regression analysis with a statistical ' 'interpretation', 2, None, - '___sec75'), - ('Assumptions made', 2, None, '___sec76'), - ('Expectation value and variance', 2, None, '___sec77'), + '___sec71'), + ('Assumptions made', 2, None, '___sec72'), + ('Expectation value and variance', 2, None, '___sec73'), ('Expectation value and variance for $\\boldsymbol{\\beta}$', 2, None, - '___sec78'), - ('Cross-validation', 2, None, '___sec79'), - ('Computationally expensive', 2, None, '___sec80'), - ('Various steps in cross-validation', 2, None, '___sec81'), + '___sec74'), + ('Cross-validation', 2, None, '___sec75'), + ('Computationally expensive', 2, None, '___sec76'), + ('Various steps in cross-validation', 2, None, '___sec77'), ('How to set up the cross-validation for Ridge and/or Lasso', 2, None, - '___sec82'), + '___sec78'), ('Resampling methods: Jackknife and Bootstrap', 2, None, - '___sec83'), - ('Resampling methods: Jackknife', 2, None, '___sec84'), - ('Jackknife code example', 2, None, '___sec85'), - ('Resampling methods: Bootstrap', 2, None, '___sec86'), - ('Resampling methods: Bootstrap background', 2, None, '___sec87'), + '___sec79'), + ('Resampling methods: Jackknife', 2, None, '___sec80'), + ('Jackknife code example', 2, None, '___sec81'), + ('Resampling methods: Bootstrap', 2, None, '___sec82'), + ('Resampling methods: Bootstrap background', 2, None, '___sec83'), ('Resampling methods: More Bootstrap background', 2, None, - '___sec88'), - ('Resampling methods: Bootstrap approach', 2, None, '___sec89'), - ('Resampling methods: Bootstrap steps', 2, None, '___sec90'), - ('Code example for the Bootstrap method', 2, None, '___sec91'), + '___sec84'), + ('Resampling methods: Bootstrap approach', 2, None, '___sec85'), + ('Resampling methods: Bootstrap steps', 2, None, '___sec86'), + ('Code example for the Bootstrap method', 2, None, '___sec87'), ('Code Example for Cross-validation and $k$-fold ' 'Cross-validation', 2, None, - '___sec92'), - ('The bias-variance tradeoff', 2, None, '___sec93'), - ('Example code for Bias-Variance tradeoff', 2, None, '___sec94'), - ('Understanding what happens', 2, None, '___sec95'), - ('Summing up', 2, None, '___sec96'), + '___sec88'), + ('The bias-variance tradeoff', 2, None, '___sec89'), + ('Example code for Bias-Variance tradeoff', 2, None, '___sec90'), + ('Understanding what happens', 2, None, '___sec91'), + ('Summing up', 2, None, '___sec92'), ("Another Example rom Scikit-Learn's Repository", 2, None, - '___sec97'), + '___sec93'), + ('The Ising model', 2, None, '___sec94'), + ('Reformulating the problem to suit regression', + 2, + None, + '___sec95'), + ('Linear regression', 2, None, '___sec96'), + ('Singular Value decomposition', 2, None, '___sec97'), ('The one-dimensional Ising model', 2, None, '___sec98'), ('Ridge regression', 2, None, '___sec99'), ('LASSO regression', 2, None, '___sec100'), @@ -295,74 +295,74 @@ MathJax.Hub.Config({
+The one-dimensional Ising model with nearest neighbor interaction, no +external field and a constant coupling constant \( J \) is given by + +$$ +\begin{align} + H = -J \sum_{k}^L s_k s_{k + 1}, +\tag{21} +\end{align} +$$ + +
+where \( s_i \in \{-1, 1\} \) and \( s_{N + 1} = s_1 \). The number of spins +in the system is determined by \( L \). For the one-dimensional system +there is no phase transition. + +
+We will look at a system of \( L = 40 \) spins with a coupling constant of +\( J = 1 \). To get enough training data we will generate 10000 states +with their respective energies. +
-
import matplotlib.pyplot as plt
-import numpy as np
-from sklearn.linear_model import LinearRegression, Ridge, Lasso
-from sklearn.preprocessing import PolynomialFeatures
+import numpy as np
+import matplotlib.pyplot as plt
+from mpl_toolkits.axes_grid1 import make_axes_locatable
+import seaborn as sns
+import scipy.linalg as scl
from sklearn.model_selection import train_test_split
-from sklearn.pipeline import make_pipeline
-from sklearn.utils import resample
+import tqdm
+sns.set(color_codes=True)
+cmap_args=dict(vmin=-1., vmax=1., cmap='seismic')
-np.random.seed(2018)
+L = 40
+n = int(1e4)
-n = 500
-n_boostraps = 100
-degree = 18 # A quite high value, just to show.
-noise = 0.1
+spins = np.random.choice([-1, 1], size=(n, L))
+J = 1.0
-# Make data set.
-x = np.linspace(-1, 3, n).reshape(-1, 1)
-y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2) + np.random.normal(0, 0.1, x.shape)
+energies = np.zeros(n)
-# Hold out some test data that is never used in training.
-x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2)
-
-# Combine x transformation and model into one operation.
-# Not neccesary, but convenient.
-model = make_pipeline(PolynomialFeatures(degree=degree), LinearRegression(fit_intercept=False))
-
-# The following (m x n_bootstraps) matrix holds the column vectors y_pred
-# for each bootstrap iteration.
-y_pred = np.empty((y_test.shape[0], n_boostraps))
-for i in range(n_boostraps):
- x_, y_ = resample(x_train, y_train)
-
- # Evaluate the new model on the same test data each time.
- y_pred[:, i] = model.fit(x_, y_).predict(x_test).ravel()
-
-# Note: Expectations and variances taken w.r.t. different training
-# data sets, hence the axis=1. Subsequent means are taken across the test data
-# set in order to obtain a total value, but before this we have error/bias/variance
-# calculated per data point in the test set.
-# Note 2: The use of keepdims=True is important in the calculation of bias as this
-# maintains the column vector form. Dropping this yields very unexpected results.
-error = np.mean( np.mean((y_test - y_pred)**2, axis=1, keepdims=True) )
-bias = np.mean( (y_test - np.mean(y_pred, axis=1, keepdims=True))**2 )
-variance = np.mean( np.var(y_pred, axis=1, keepdims=True) )
-print('Error:', error)
-print('Bias^2:', bias)
-print('Var:', variance)
-print('{} >= {} + {} = {}'.format(error, bias, variance, bias+variance))
-
-plt.plot(x[::5, :], y[::5, :], label='f(x)')
-plt.scatter(x_test, y_test, label='Data points')
-plt.scatter(x_test, np.mean(y_pred, axis=1), label='Pred')
-plt.legend()
-plt.show()
+for i in range(n):
+ energies[i] = - J * np.dot(spins[i], np.roll(spins[i], 1))
+
+Here we use ordinary least squares
+regression to predict the energy for the nearest neighbor
+one-dimensional Ising model on a ring, i.e., the endpoints wrap
+around. We will use linear regression to fit a value for
+the coupling constant to achieve this.
+
diff --git a/doc/pub/Regression/html/._Regression-bs096.html b/doc/pub/Regression/html/._Regression-bs096.html
index 06b8f4b93..79bd7416f 100644
--- a/doc/pub/Regression/html/._Regression-bs096.html
+++ b/doc/pub/Regression/html/._Regression-bs096.html
@@ -111,112 +111,112 @@ Automatically generated HTML file from DocOnce source
('The Boston housing data example', 2, None, '___sec27'),
('Housing data, the code', 2, None, '___sec28'),
('The singular value decomposition', 2, None, '___sec29'),
- ('The Ising model', 2, None, '___sec30'),
- ('Reformulating the problem to suit regression',
- 2,
- None,
- '___sec31'),
- ('Linear regression', 2, None, '___sec32'),
- ('Singular Value decomposition', 2, None, '___sec33'),
- ('Linear Regression Problems', 2, None, '___sec34'),
- ('Fixing the singularity', 2, None, '___sec35'),
- ('Basic math of the SVD', 2, None, '___sec36'),
- ('The SVD, a Fantastic Algorithm', 2, None, '___sec37'),
- ('Another Example', 2, None, '___sec38'),
- ('Economy-size SVD', 2, None, '___sec39'),
- ('Mathematical Properties', 2, None, '___sec40'),
- ('Ridge and LASSO Regression', 2, None, '___sec41'),
- ('More on Ridge Regression', 2, None, '___sec42'),
- ('Interpreting the Ridge results', 2, None, '___sec43'),
- ('More interpretations', 2, None, '___sec44'),
- ('Where are we going?', 2, None, '___sec45'),
- ('Resampling methods', 2, None, '___sec46'),
+ ('Linear Regression Problems', 2, None, '___sec30'),
+ ('Fixing the singularity', 2, None, '___sec31'),
+ ('Basic math of the SVD', 2, None, '___sec32'),
+ ('The SVD, a Fantastic Algorithm', 2, None, '___sec33'),
+ ('Another Example', 2, None, '___sec34'),
+ ('Economy-size SVD', 2, None, '___sec35'),
+ ('Mathematical Properties', 2, None, '___sec36'),
+ ('Ridge and LASSO Regression', 2, None, '___sec37'),
+ ('More on Ridge Regression', 2, None, '___sec38'),
+ ('Interpreting the Ridge results', 2, None, '___sec39'),
+ ('More interpretations', 2, None, '___sec40'),
+ ('Where are we going?', 2, None, '___sec41'),
+ ('Resampling methods', 2, None, '___sec42'),
('Resampling approaches can be computationally expensive',
2,
None,
- '___sec47'),
- ('Why resampling methods ?', 2, None, '___sec48'),
- ('Statistical analysis', 2, None, '___sec49'),
- ('Statistics', 2, None, '___sec50'),
- ('Statistics, moments', 2, None, '___sec51'),
- ('Statistics, central moments', 2, None, '___sec52'),
- ('Statistics, covariance', 2, None, '___sec53'),
- ('Statistics, more covariance', 2, None, '___sec54'),
- ('Covariance example', 2, None, '___sec55'),
- ('Covariance in numpy', 2, None, '___sec56'),
- ('Statistics, independent variables', 2, None, '___sec57'),
- ('Statistics, more variance', 2, None, '___sec58'),
- ('Statistics and stochastic processes', 2, None, '___sec59'),
- ('Statistics and sample variables', 2, None, '___sec60'),
+ '___sec43'),
+ ('Why resampling methods ?', 2, None, '___sec44'),
+ ('Statistical analysis', 2, None, '___sec45'),
+ ('Statistics', 2, None, '___sec46'),
+ ('Statistics, moments', 2, None, '___sec47'),
+ ('Statistics, central moments', 2, None, '___sec48'),
+ ('Statistics, covariance', 2, None, '___sec49'),
+ ('Statistics, more covariance', 2, None, '___sec50'),
+ ('Covariance example', 2, None, '___sec51'),
+ ('Covariance in numpy', 2, None, '___sec52'),
+ ('Statistics, independent variables', 2, None, '___sec53'),
+ ('Statistics, more variance', 2, None, '___sec54'),
+ ('Statistics and stochastic processes', 2, None, '___sec55'),
+ ('Statistics and sample variables', 2, None, '___sec56'),
('Statistics, sample variance and covariance',
2,
None,
- '___sec61'),
- ('Statistics, law of large numbers', 2, None, '___sec62'),
- ('Statistics, more on sample error', 2, None, '___sec63'),
- ('Statistics', 2, None, '___sec64'),
- ('Statistics, central limit theorem', 2, None, '___sec65'),
- ('Statistics, more technicalities', 2, None, '___sec66'),
- ('Statistics', 2, None, '___sec67'),
- ('Statistics and sample variance', 2, None, '___sec68'),
- ('Statistics, uncorrelated results', 2, None, '___sec69'),
- ('Statistics, computations', 2, None, '___sec70'),
+ '___sec57'),
+ ('Statistics, law of large numbers', 2, None, '___sec58'),
+ ('Statistics, more on sample error', 2, None, '___sec59'),
+ ('Statistics', 2, None, '___sec60'),
+ ('Statistics, central limit theorem', 2, None, '___sec61'),
+ ('Statistics, more technicalities', 2, None, '___sec62'),
+ ('Statistics', 2, None, '___sec63'),
+ ('Statistics and sample variance', 2, None, '___sec64'),
+ ('Statistics, uncorrelated results', 2, None, '___sec65'),
+ ('Statistics, computations', 2, None, '___sec66'),
('Statistics, more on computations of errors',
2,
None,
- '___sec71'),
- ('Statistics, wrapping up 1', 2, None, '___sec72'),
- ('Statistics, final expression', 2, None, '___sec73'),
+ '___sec67'),
+ ('Statistics, wrapping up 1', 2, None, '___sec68'),
+ ('Statistics, final expression', 2, None, '___sec69'),
('Statistics, effective number of correlations',
2,
None,
- '___sec74'),
+ '___sec70'),
('Linking the regression analysis with a statistical '
'interpretation',
2,
None,
- '___sec75'),
- ('Assumptions made', 2, None, '___sec76'),
- ('Expectation value and variance', 2, None, '___sec77'),
+ '___sec71'),
+ ('Assumptions made', 2, None, '___sec72'),
+ ('Expectation value and variance', 2, None, '___sec73'),
('Expectation value and variance for $\\boldsymbol{\\beta}$',
2,
None,
- '___sec78'),
- ('Cross-validation', 2, None, '___sec79'),
- ('Computationally expensive', 2, None, '___sec80'),
- ('Various steps in cross-validation', 2, None, '___sec81'),
+ '___sec74'),
+ ('Cross-validation', 2, None, '___sec75'),
+ ('Computationally expensive', 2, None, '___sec76'),
+ ('Various steps in cross-validation', 2, None, '___sec77'),
('How to set up the cross-validation for Ridge and/or Lasso',
2,
None,
- '___sec82'),
+ '___sec78'),
('Resampling methods: Jackknife and Bootstrap',
2,
None,
- '___sec83'),
- ('Resampling methods: Jackknife', 2, None, '___sec84'),
- ('Jackknife code example', 2, None, '___sec85'),
- ('Resampling methods: Bootstrap', 2, None, '___sec86'),
- ('Resampling methods: Bootstrap background', 2, None, '___sec87'),
+ '___sec79'),
+ ('Resampling methods: Jackknife', 2, None, '___sec80'),
+ ('Jackknife code example', 2, None, '___sec81'),
+ ('Resampling methods: Bootstrap', 2, None, '___sec82'),
+ ('Resampling methods: Bootstrap background', 2, None, '___sec83'),
('Resampling methods: More Bootstrap background',
2,
None,
- '___sec88'),
- ('Resampling methods: Bootstrap approach', 2, None, '___sec89'),
- ('Resampling methods: Bootstrap steps', 2, None, '___sec90'),
- ('Code example for the Bootstrap method', 2, None, '___sec91'),
+ '___sec84'),
+ ('Resampling methods: Bootstrap approach', 2, None, '___sec85'),
+ ('Resampling methods: Bootstrap steps', 2, None, '___sec86'),
+ ('Code example for the Bootstrap method', 2, None, '___sec87'),
('Code Example for Cross-validation and $k$-fold '
'Cross-validation',
2,
None,
- '___sec92'),
- ('The bias-variance tradeoff', 2, None, '___sec93'),
- ('Example code for Bias-Variance tradeoff', 2, None, '___sec94'),
- ('Understanding what happens', 2, None, '___sec95'),
- ('Summing up', 2, None, '___sec96'),
+ '___sec88'),
+ ('The bias-variance tradeoff', 2, None, '___sec89'),
+ ('Example code for Bias-Variance tradeoff', 2, None, '___sec90'),
+ ('Understanding what happens', 2, None, '___sec91'),
+ ('Summing up', 2, None, '___sec92'),
("Another Example rom Scikit-Learn's Repository",
2,
None,
- '___sec97'),
+ '___sec93'),
+ ('The Ising model', 2, None, '___sec94'),
+ ('Reformulating the problem to suit regression',
+ 2,
+ None,
+ '___sec95'),
+ ('Linear regression', 2, None, '___sec96'),
+ ('Singular Value decomposition', 2, None, '___sec97'),
('The one-dimensional Ising model', 2, None, '___sec98'),
('Ridge regression', 2, None, '___sec99'),
('LASSO regression', 2, None, '___sec100'),
@@ -295,74 +295,74 @@ MathJax.Hub.Config({
+A more general form for the one-dimensional Ising model is + +$$ +\begin{align} + H = - \sum_j^L \sum_k^L s_j s_k J_{jk}. +\tag{22} +\end{align} +$$ + +
+Here we allow for interactions beyond the nearest neighbors and a state dependent +coupling constant. This latter expression can be formulated as +a matrix-product +$$ +\begin{align} + \boldsymbol{H} = \boldsymbol{X} J, +\tag{23} +\end{align} +$$ + +
+where \( X_{jk} = s_j s_k \) and \( J \) is a matrix which consists of the +elements \( -J_{jk} \). This form of writing the energy fits perfectly +with the form utilized in linear regression, that is + +$$ +\begin{align} + \boldsymbol{y} = \boldsymbol{X}\boldsymbol{\beta} + \boldsymbol{\epsilon}, +\tag{24} +\end{align} +$$ + +
+We split the data in training and test data as discussed in the previous example +
-
import matplotlib.pyplot as plt
-import numpy as np
-from sklearn.linear_model import LinearRegression, Ridge, Lasso
-from sklearn.preprocessing import PolynomialFeatures
-from sklearn.model_selection import train_test_split
-from sklearn.pipeline import make_pipeline
-from sklearn.utils import resample
-
-np.random.seed(2018)
-
-n = 40
-n_boostraps = 100
-maxdegree = 14
-
-
-# Make data set.
-x = np.linspace(-3, 3, n).reshape(-1, 1)
-y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)
-error = np.zeros(maxdegree)
-bias = np.zeros(maxdegree)
-variance = np.zeros(maxdegree)
-polydegree = np.zeros(maxdegree)
-x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2)
-
-for degree in range(maxdegree):
- model = make_pipeline(PolynomialFeatures(degree=degree), LinearRegression(fit_intercept=False))
- y_pred = np.empty((y_test.shape[0], n_boostraps))
- for i in range(n_boostraps):
- x_, y_ = resample(x_train, y_train)
- y_pred[:, i] = model.fit(x_, y_).predict(x_test).ravel()
-
- polydegree[degree] = degree
- error[degree] = np.mean( np.mean((y_test - y_pred)**2, axis=1, keepdims=True) )
- bias[degree] = np.mean( (y_test - np.mean(y_pred, axis=1, keepdims=True))**2 )
- variance[degree] = np.mean( np.var(y_pred, axis=1, keepdims=True) )
- print('Polynomial degree:', degree)
- print('Error:', error[degree])
- print('Bias^2:', bias[degree])
- print('Var:', variance[degree])
- print('{} >= {} + {} = {}'.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree]))
-
-plt.plot(polydegree, error, label='Error')
-plt.plot(polydegree, bias, label='bias')
-plt.plot(polydegree, variance, label='Variance')
-plt.legend()
-plt.show()
+X = np.zeros((n, L ** 2))
+for i in range(n):
+ X[i] = np.outer(spins[i], spins[i]).ravel()
+y = energies
+X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)
diff --git a/doc/pub/Regression/html/._Regression-bs097.html b/doc/pub/Regression/html/._Regression-bs097.html
index 8e9f10846..548459001 100644
--- a/doc/pub/Regression/html/._Regression-bs097.html
+++ b/doc/pub/Regression/html/._Regression-bs097.html
@@ -111,112 +111,112 @@ Automatically generated HTML file from DocOnce source
('The Boston housing data example', 2, None, '___sec27'),
('Housing data, the code', 2, None, '___sec28'),
('The singular value decomposition', 2, None, '___sec29'),
- ('The Ising model', 2, None, '___sec30'),
- ('Reformulating the problem to suit regression',
- 2,
- None,
- '___sec31'),
- ('Linear regression', 2, None, '___sec32'),
- ('Singular Value decomposition', 2, None, '___sec33'),
- ('Linear Regression Problems', 2, None, '___sec34'),
- ('Fixing the singularity', 2, None, '___sec35'),
- ('Basic math of the SVD', 2, None, '___sec36'),
- ('The SVD, a Fantastic Algorithm', 2, None, '___sec37'),
- ('Another Example', 2, None, '___sec38'),
- ('Economy-size SVD', 2, None, '___sec39'),
- ('Mathematical Properties', 2, None, '___sec40'),
- ('Ridge and LASSO Regression', 2, None, '___sec41'),
- ('More on Ridge Regression', 2, None, '___sec42'),
- ('Interpreting the Ridge results', 2, None, '___sec43'),
- ('More interpretations', 2, None, '___sec44'),
- ('Where are we going?', 2, None, '___sec45'),
- ('Resampling methods', 2, None, '___sec46'),
+ ('Linear Regression Problems', 2, None, '___sec30'),
+ ('Fixing the singularity', 2, None, '___sec31'),
+ ('Basic math of the SVD', 2, None, '___sec32'),
+ ('The SVD, a Fantastic Algorithm', 2, None, '___sec33'),
+ ('Another Example', 2, None, '___sec34'),
+ ('Economy-size SVD', 2, None, '___sec35'),
+ ('Mathematical Properties', 2, None, '___sec36'),
+ ('Ridge and LASSO Regression', 2, None, '___sec37'),
+ ('More on Ridge Regression', 2, None, '___sec38'),
+ ('Interpreting the Ridge results', 2, None, '___sec39'),
+ ('More interpretations', 2, None, '___sec40'),
+ ('Where are we going?', 2, None, '___sec41'),
+ ('Resampling methods', 2, None, '___sec42'),
('Resampling approaches can be computationally expensive',
2,
None,
- '___sec47'),
- ('Why resampling methods ?', 2, None, '___sec48'),
- ('Statistical analysis', 2, None, '___sec49'),
- ('Statistics', 2, None, '___sec50'),
- ('Statistics, moments', 2, None, '___sec51'),
- ('Statistics, central moments', 2, None, '___sec52'),
- ('Statistics, covariance', 2, None, '___sec53'),
- ('Statistics, more covariance', 2, None, '___sec54'),
- ('Covariance example', 2, None, '___sec55'),
- ('Covariance in numpy', 2, None, '___sec56'),
- ('Statistics, independent variables', 2, None, '___sec57'),
- ('Statistics, more variance', 2, None, '___sec58'),
- ('Statistics and stochastic processes', 2, None, '___sec59'),
- ('Statistics and sample variables', 2, None, '___sec60'),
+ '___sec43'),
+ ('Why resampling methods ?', 2, None, '___sec44'),
+ ('Statistical analysis', 2, None, '___sec45'),
+ ('Statistics', 2, None, '___sec46'),
+ ('Statistics, moments', 2, None, '___sec47'),
+ ('Statistics, central moments', 2, None, '___sec48'),
+ ('Statistics, covariance', 2, None, '___sec49'),
+ ('Statistics, more covariance', 2, None, '___sec50'),
+ ('Covariance example', 2, None, '___sec51'),
+ ('Covariance in numpy', 2, None, '___sec52'),
+ ('Statistics, independent variables', 2, None, '___sec53'),
+ ('Statistics, more variance', 2, None, '___sec54'),
+ ('Statistics and stochastic processes', 2, None, '___sec55'),
+ ('Statistics and sample variables', 2, None, '___sec56'),
('Statistics, sample variance and covariance',
2,
None,
- '___sec61'),
- ('Statistics, law of large numbers', 2, None, '___sec62'),
- ('Statistics, more on sample error', 2, None, '___sec63'),
- ('Statistics', 2, None, '___sec64'),
- ('Statistics, central limit theorem', 2, None, '___sec65'),
- ('Statistics, more technicalities', 2, None, '___sec66'),
- ('Statistics', 2, None, '___sec67'),
- ('Statistics and sample variance', 2, None, '___sec68'),
- ('Statistics, uncorrelated results', 2, None, '___sec69'),
- ('Statistics, computations', 2, None, '___sec70'),
+ '___sec57'),
+ ('Statistics, law of large numbers', 2, None, '___sec58'),
+ ('Statistics, more on sample error', 2, None, '___sec59'),
+ ('Statistics', 2, None, '___sec60'),
+ ('Statistics, central limit theorem', 2, None, '___sec61'),
+ ('Statistics, more technicalities', 2, None, '___sec62'),
+ ('Statistics', 2, None, '___sec63'),
+ ('Statistics and sample variance', 2, None, '___sec64'),
+ ('Statistics, uncorrelated results', 2, None, '___sec65'),
+ ('Statistics, computations', 2, None, '___sec66'),
('Statistics, more on computations of errors',
2,
None,
- '___sec71'),
- ('Statistics, wrapping up 1', 2, None, '___sec72'),
- ('Statistics, final expression', 2, None, '___sec73'),
+ '___sec67'),
+ ('Statistics, wrapping up 1', 2, None, '___sec68'),
+ ('Statistics, final expression', 2, None, '___sec69'),
('Statistics, effective number of correlations',
2,
None,
- '___sec74'),
+ '___sec70'),
('Linking the regression analysis with a statistical '
'interpretation',
2,
None,
- '___sec75'),
- ('Assumptions made', 2, None, '___sec76'),
- ('Expectation value and variance', 2, None, '___sec77'),
+ '___sec71'),
+ ('Assumptions made', 2, None, '___sec72'),
+ ('Expectation value and variance', 2, None, '___sec73'),
('Expectation value and variance for $\\boldsymbol{\\beta}$',
2,
None,
- '___sec78'),
- ('Cross-validation', 2, None, '___sec79'),
- ('Computationally expensive', 2, None, '___sec80'),
- ('Various steps in cross-validation', 2, None, '___sec81'),
+ '___sec74'),
+ ('Cross-validation', 2, None, '___sec75'),
+ ('Computationally expensive', 2, None, '___sec76'),
+ ('Various steps in cross-validation', 2, None, '___sec77'),
('How to set up the cross-validation for Ridge and/or Lasso',
2,
None,
- '___sec82'),
+ '___sec78'),
('Resampling methods: Jackknife and Bootstrap',
2,
None,
- '___sec83'),
- ('Resampling methods: Jackknife', 2, None, '___sec84'),
- ('Jackknife code example', 2, None, '___sec85'),
- ('Resampling methods: Bootstrap', 2, None, '___sec86'),
- ('Resampling methods: Bootstrap background', 2, None, '___sec87'),
+ '___sec79'),
+ ('Resampling methods: Jackknife', 2, None, '___sec80'),
+ ('Jackknife code example', 2, None, '___sec81'),
+ ('Resampling methods: Bootstrap', 2, None, '___sec82'),
+ ('Resampling methods: Bootstrap background', 2, None, '___sec83'),
('Resampling methods: More Bootstrap background',
2,
None,
- '___sec88'),
- ('Resampling methods: Bootstrap approach', 2, None, '___sec89'),
- ('Resampling methods: Bootstrap steps', 2, None, '___sec90'),
- ('Code example for the Bootstrap method', 2, None, '___sec91'),
+ '___sec84'),
+ ('Resampling methods: Bootstrap approach', 2, None, '___sec85'),
+ ('Resampling methods: Bootstrap steps', 2, None, '___sec86'),
+ ('Code example for the Bootstrap method', 2, None, '___sec87'),
('Code Example for Cross-validation and $k$-fold '
'Cross-validation',
2,
None,
- '___sec92'),
- ('The bias-variance tradeoff', 2, None, '___sec93'),
- ('Example code for Bias-Variance tradeoff', 2, None, '___sec94'),
- ('Understanding what happens', 2, None, '___sec95'),
- ('Summing up', 2, None, '___sec96'),
+ '___sec88'),
+ ('The bias-variance tradeoff', 2, None, '___sec89'),
+ ('Example code for Bias-Variance tradeoff', 2, None, '___sec90'),
+ ('Understanding what happens', 2, None, '___sec91'),
+ ('Summing up', 2, None, '___sec92'),
("Another Example rom Scikit-Learn's Repository",
2,
None,
- '___sec97'),
+ '___sec93'),
+ ('The Ising model', 2, None, '___sec94'),
+ ('Reformulating the problem to suit regression',
+ 2,
+ None,
+ '___sec95'),
+ ('Linear regression', 2, None, '___sec96'),
+ ('Singular Value decomposition', 2, None, '___sec97'),
('The one-dimensional Ising model', 2, None, '___sec98'),
('Ridge regression', 2, None, '___sec99'),
('LASSO regression', 2, None, '___sec100'),
@@ -295,74 +295,74 @@ MathJax.Hub.Config({
- + -
-The bias-variance tradeoff summarizes the fundamental tension in -machine learning, particularly supervised learning, between the -complexity of a model and the amount of training data needed to train -it. Since data is often limited, in practice it is often useful to -use a less-complex model with higher bias, that is a model whose asymptotic -performance is worse than another model because it is easier to -train and less sensitive to sampling noise arising from having a -finite-sized training dataset (smaller variance). +In the ordinary least squares method we choose the cost function + +$$ +\begin{align} + C(\boldsymbol{X}, \boldsymbol{\beta})= \frac{1}{n}\left\{(\boldsymbol{X}\boldsymbol{\beta} - \boldsymbol{y})^T(\boldsymbol{X}\boldsymbol{\beta} - \boldsymbol{y})\right\}. +\tag{25} +\end{align} +$$
-The above equations tell us that in -order to minimize the expected test error, we need to select a -statistical learning method that simultaneously achieves low variance -and low bias. Note that variance is inherently a nonnegative quantity, -and squared bias is also nonnegative. Hence, we see that the expected -test MSE can never lie below \( Var(\epsilon) \), the irreducible error. +We then find the extremal point of \( C \) by taking the derivative with respect to \( \boldsymbol{\beta} \) as discussed above. +This yields the expression for \( \boldsymbol{\beta} \) to be + +$$ + \boldsymbol{\beta} = \frac{\boldsymbol{X}^T \boldsymbol{y}}{\boldsymbol{X}^T \boldsymbol{X}}, +$$
-What do we mean by the variance and bias of a statistical learning -method? The variance refers to the amount by which our model would change if we -estimated it using a different training data set. Since the training -data are used to fit the statistical learning method, different -training data sets will result in a different estimate. But ideally the -estimate for our model should not vary too much between training -sets. However, if a method has high variance then small changes in -the training data can result in large changes in the model. In general, more -flexible statistical methods have higher variance. +which immediately imposes some requirements on \( \boldsymbol{X} \) as there must exist +an inverse of \( \boldsymbol{X}^T \boldsymbol{X} \). If the expression we are modeling contains an +intercept, i.e., a constant term, we must make sure that the +first column of \( \boldsymbol{X} \) consists of \( 1 \). We do this here +
+ + +
X_train_own = np.concatenate(
+ (np.ones(len(X_train))[:, np.newaxis], X_train),
+ axis=1
+)
+X_test_own = np.concatenate(
+ (np.ones(len(X_test))[:, np.newaxis], X_test),
+ axis=1
+)
++ + +
def ols_inv(x: np.ndarray, y: np.ndarray) -> np.ndarray:
+ return scl.inv(x.T @ x) @ (x.T @ y)
+beta = ols_inv(X_train_own, y_train)
+
diff --git a/doc/pub/Regression/html/._Regression-bs098.html b/doc/pub/Regression/html/._Regression-bs098.html index 8b9924c73..0dd8464db 100644 --- a/doc/pub/Regression/html/._Regression-bs098.html +++ b/doc/pub/Regression/html/._Regression-bs098.html @@ -111,112 +111,112 @@ Automatically generated HTML file from DocOnce source ('The Boston housing data example', 2, None, '___sec27'), ('Housing data, the code', 2, None, '___sec28'), ('The singular value decomposition', 2, None, '___sec29'), - ('The Ising model', 2, None, '___sec30'), - ('Reformulating the problem to suit regression', - 2, - None, - '___sec31'), - ('Linear regression', 2, None, '___sec32'), - ('Singular Value decomposition', 2, None, '___sec33'), - ('Linear Regression Problems', 2, None, '___sec34'), - ('Fixing the singularity', 2, None, '___sec35'), - ('Basic math of the SVD', 2, None, '___sec36'), - ('The SVD, a Fantastic Algorithm', 2, None, '___sec37'), - ('Another Example', 2, None, '___sec38'), - ('Economy-size SVD', 2, None, '___sec39'), - ('Mathematical Properties', 2, None, '___sec40'), - ('Ridge and LASSO Regression', 2, None, '___sec41'), - ('More on Ridge Regression', 2, None, '___sec42'), - ('Interpreting the Ridge results', 2, None, '___sec43'), - ('More interpretations', 2, None, '___sec44'), - ('Where are we going?', 2, None, '___sec45'), - ('Resampling methods', 2, None, '___sec46'), + ('Linear Regression Problems', 2, None, '___sec30'), + ('Fixing the singularity', 2, None, '___sec31'), + ('Basic math of the SVD', 2, None, '___sec32'), + ('The SVD, a Fantastic Algorithm', 2, None, '___sec33'), + ('Another Example', 2, None, '___sec34'), + ('Economy-size SVD', 2, None, '___sec35'), + ('Mathematical Properties', 2, None, '___sec36'), + ('Ridge and LASSO Regression', 2, None, '___sec37'), + ('More on Ridge Regression', 2, None, '___sec38'), + ('Interpreting the Ridge results', 2, None, '___sec39'), + ('More interpretations', 2, None, '___sec40'), + ('Where are we going?', 2, None, '___sec41'), + ('Resampling methods', 2, None, '___sec42'), ('Resampling approaches can be computationally expensive', 2, None, - '___sec47'), - ('Why resampling methods ?', 2, None, '___sec48'), - ('Statistical analysis', 2, None, '___sec49'), - ('Statistics', 2, None, '___sec50'), - ('Statistics, moments', 2, None, '___sec51'), - ('Statistics, central moments', 2, None, '___sec52'), - ('Statistics, covariance', 2, None, '___sec53'), - ('Statistics, more covariance', 2, None, '___sec54'), - ('Covariance example', 2, None, '___sec55'), - ('Covariance in numpy', 2, None, '___sec56'), - ('Statistics, independent variables', 2, None, '___sec57'), - ('Statistics, more variance', 2, None, '___sec58'), - ('Statistics and stochastic processes', 2, None, '___sec59'), - ('Statistics and sample variables', 2, None, '___sec60'), + '___sec43'), + ('Why resampling methods ?', 2, None, '___sec44'), + ('Statistical analysis', 2, None, '___sec45'), + ('Statistics', 2, None, '___sec46'), + ('Statistics, moments', 2, None, '___sec47'), + ('Statistics, central moments', 2, None, '___sec48'), + ('Statistics, covariance', 2, None, '___sec49'), + ('Statistics, more covariance', 2, None, '___sec50'), + ('Covariance example', 2, None, '___sec51'), + ('Covariance in numpy', 2, None, '___sec52'), + ('Statistics, independent variables', 2, None, '___sec53'), + ('Statistics, more variance', 2, None, '___sec54'), + ('Statistics and stochastic processes', 2, None, '___sec55'), + ('Statistics and sample variables', 2, None, '___sec56'), ('Statistics, sample variance and covariance', 2, None, - '___sec61'), - ('Statistics, law of large numbers', 2, None, '___sec62'), - ('Statistics, more on sample error', 2, None, '___sec63'), - ('Statistics', 2, None, '___sec64'), - ('Statistics, central limit theorem', 2, None, '___sec65'), - ('Statistics, more technicalities', 2, None, '___sec66'), - ('Statistics', 2, None, '___sec67'), - ('Statistics and sample variance', 2, None, '___sec68'), - ('Statistics, uncorrelated results', 2, None, '___sec69'), - ('Statistics, computations', 2, None, '___sec70'), + '___sec57'), + ('Statistics, law of large numbers', 2, None, '___sec58'), + ('Statistics, more on sample error', 2, None, '___sec59'), + ('Statistics', 2, None, '___sec60'), + ('Statistics, central limit theorem', 2, None, '___sec61'), + ('Statistics, more technicalities', 2, None, '___sec62'), + ('Statistics', 2, None, '___sec63'), + ('Statistics and sample variance', 2, None, '___sec64'), + ('Statistics, uncorrelated results', 2, None, '___sec65'), + ('Statistics, computations', 2, None, '___sec66'), ('Statistics, more on computations of errors', 2, None, - '___sec71'), - ('Statistics, wrapping up 1', 2, None, '___sec72'), - ('Statistics, final expression', 2, None, '___sec73'), + '___sec67'), + ('Statistics, wrapping up 1', 2, None, '___sec68'), + ('Statistics, final expression', 2, None, '___sec69'), ('Statistics, effective number of correlations', 2, None, - '___sec74'), + '___sec70'), ('Linking the regression analysis with a statistical ' 'interpretation', 2, None, - '___sec75'), - ('Assumptions made', 2, None, '___sec76'), - ('Expectation value and variance', 2, None, '___sec77'), + '___sec71'), + ('Assumptions made', 2, None, '___sec72'), + ('Expectation value and variance', 2, None, '___sec73'), ('Expectation value and variance for $\\boldsymbol{\\beta}$', 2, None, - '___sec78'), - ('Cross-validation', 2, None, '___sec79'), - ('Computationally expensive', 2, None, '___sec80'), - ('Various steps in cross-validation', 2, None, '___sec81'), + '___sec74'), + ('Cross-validation', 2, None, '___sec75'), + ('Computationally expensive', 2, None, '___sec76'), + ('Various steps in cross-validation', 2, None, '___sec77'), ('How to set up the cross-validation for Ridge and/or Lasso', 2, None, - '___sec82'), + '___sec78'), ('Resampling methods: Jackknife and Bootstrap', 2, None, - '___sec83'), - ('Resampling methods: Jackknife', 2, None, '___sec84'), - ('Jackknife code example', 2, None, '___sec85'), - ('Resampling methods: Bootstrap', 2, None, '___sec86'), - ('Resampling methods: Bootstrap background', 2, None, '___sec87'), + '___sec79'), + ('Resampling methods: Jackknife', 2, None, '___sec80'), + ('Jackknife code example', 2, None, '___sec81'), + ('Resampling methods: Bootstrap', 2, None, '___sec82'), + ('Resampling methods: Bootstrap background', 2, None, '___sec83'), ('Resampling methods: More Bootstrap background', 2, None, - '___sec88'), - ('Resampling methods: Bootstrap approach', 2, None, '___sec89'), - ('Resampling methods: Bootstrap steps', 2, None, '___sec90'), - ('Code example for the Bootstrap method', 2, None, '___sec91'), + '___sec84'), + ('Resampling methods: Bootstrap approach', 2, None, '___sec85'), + ('Resampling methods: Bootstrap steps', 2, None, '___sec86'), + ('Code example for the Bootstrap method', 2, None, '___sec87'), ('Code Example for Cross-validation and $k$-fold ' 'Cross-validation', 2, None, - '___sec92'), - ('The bias-variance tradeoff', 2, None, '___sec93'), - ('Example code for Bias-Variance tradeoff', 2, None, '___sec94'), - ('Understanding what happens', 2, None, '___sec95'), - ('Summing up', 2, None, '___sec96'), + '___sec88'), + ('The bias-variance tradeoff', 2, None, '___sec89'), + ('Example code for Bias-Variance tradeoff', 2, None, '___sec90'), + ('Understanding what happens', 2, None, '___sec91'), + ('Summing up', 2, None, '___sec92'), ("Another Example rom Scikit-Learn's Repository", 2, None, - '___sec97'), + '___sec93'), + ('The Ising model', 2, None, '___sec94'), + ('Reformulating the problem to suit regression', + 2, + None, + '___sec95'), + ('Linear regression', 2, None, '___sec96'), + ('Singular Value decomposition', 2, None, '___sec97'), ('The one-dimensional Ising model', 2, None, '___sec98'), ('Ridge regression', 2, None, '___sec99'), ('LASSO regression', 2, None, '___sec100'), @@ -295,74 +295,74 @@ MathJax.Hub.Config({
+Doing the inversion directly turns out to be a bad idea since the matrix +\( \boldsymbol{X}^T\boldsymbol{X} \) is singular. An alternative approach is to use the singular +value decomposition. Using the definition of the Moore-Penrose +pseudoinverse we can write the equation for \( \boldsymbol{\beta} \) as + +$$ + \boldsymbol{\beta} = \boldsymbol{X}^{+}\boldsymbol{y}, +$$ + +
+where the pseudoinverse of \( \boldsymbol{X} \) is given by + +$$ + \boldsymbol{X}^{+} = \frac{\boldsymbol{X}^T}{\boldsymbol{X}^T\boldsymbol{X}}. +$$ + +
+Using singular value decomposition we can decompose the matrix \( \boldsymbol{X} = \boldsymbol{U}\boldsymbol{\Sigma} \boldsymbol{V}^T \), +where \( \boldsymbol{U} \) and \( \boldsymbol{V} \) are orthogonal(unitary) matrices and \( \boldsymbol{\Sigma} \) contains the singular values (more details below). +where \( X^{+} = V\Sigma^{+} U^T \). This reduces the equation for +\( \omega \) to +$$ +\begin{align} + \boldsymbol{\beta} = \boldsymbol{V}\boldsymbol{\Sigma}^{+} \boldsymbol{U}^T \boldsymbol{y}. +\tag{26} +\end{align} +$$ + +
+Note that solving this equation by actually doing the pseudoinverse +(which is what we will do) is not a good idea as this operation scales +as \( \mathcal{O}(n^3) \), where \( n \) is the number of elements in a +general matrix. Instead, doing \( QR \)-factorization and solving the +linear system as an equation would reduce this down to +\( \mathcal{O}(n^2) \) operations. +
-
"""
-============================
-Underfitting vs. Overfitting
-============================
+def ols_svd(x: np.ndarray, y: np.ndarray) -> np.ndarray:
+ u, s, v = scl.svd(x)
+ return v.T @ scl.pinv(scl.diagsvd(s, u.shape[0], v.shape[0])) @ u.T @ y
+
+
-This example demonstrates the problems of underfitting and overfitting and
-how we can use linear regression with polynomial features to approximate
-nonlinear functions. The plot shows the function that we want to approximate,
-which is a part of the cosine function. In addition, the samples from the
-real function and the approximations of different models are displayed. The
-models have polynomial features of different degrees. We can see that a
-linear function (polynomial with degree 1) is not sufficient to fit the
-training samples. This is called **underfitting**. A polynomial of degree 4
-approximates the true function almost perfectly. However, for higher degrees
-the model will **overfit** the training data, i.e. it learns the noise of the
-training data.
-We evaluate quantitatively **overfitting** / **underfitting** by using
-cross-validation. We calculate the mean squared error (MSE) on the validation
-set, the higher, the less likely the model generalizes correctly from the
-training data.
-"""
+
+
beta = ols_svd(X_train_own,y_train)
+
+
+When extracting the \( J \)-matrix we need to make sure that we remove the intercept, as is done here
-print(__doc__)
+
-import numpy as np
-import matplotlib.pyplot as plt
-from sklearn.pipeline import Pipeline
-from sklearn.preprocessing import PolynomialFeatures
-from sklearn.linear_model import LinearRegression
-from sklearn.model_selection import cross_val_score
+
+
J = beta[1:].reshape(L, L)
+
+
+A way of looking at the coefficients in \( J \) is to plot the matrices as images.
+
-def true_fun(X):
- return np.cos(1.5 * np.pi * X)
-
-np.random.seed(0)
-
-n_samples = 30
-degrees = [1, 4, 15]
-
-X = np.sort(np.random.rand(n_samples))
-y = true_fun(X) + np.random.randn(n_samples) * 0.1
-
-plt.figure(figsize=(14, 5))
-for i in range(len(degrees)):
- ax = plt.subplot(1, len(degrees), i + 1)
- plt.setp(ax, xticks=(), yticks=())
-
- polynomial_features = PolynomialFeatures(degree=degrees[i],
- include_bias=False)
- linear_regression = LinearRegression()
- pipeline = Pipeline([("polynomial_features", polynomial_features),
- ("linear_regression", linear_regression)])
- pipeline.fit(X[:, np.newaxis], y)
-
- # Evaluate the models using crossvalidation
- scores = cross_val_score(pipeline, X[:, np.newaxis], y,
- scoring="neg_mean_squared_error", cv=10)
-
- X_test = np.linspace(0, 1, 100)
- plt.plot(X_test, pipeline.predict(X_test[:, np.newaxis]), label="Model")
- plt.plot(X_test, true_fun(X_test), label="True function")
- plt.scatter(X, y, edgecolor='b', s=20, label="Samples")
- plt.xlabel("x")
- plt.ylabel("y")
- plt.xlim((0, 1))
- plt.ylim((-2, 2))
- plt.legend(loc="best")
- plt.title("Degree {}\nMSE = {:.2e}(+/- {:.2e})".format(
- degrees[i], -scores.mean(), scores.std()))
+
+
fig = plt.figure(figsize=(20, 14))
+im = plt.imshow(J, **cmap_args)
+plt.title("OLS", fontsize=18)
+plt.xticks(fontsize=18)
+plt.yticks(fontsize=18)
+cb = fig.colorbar(im)
+cb.ax.set_yticklabels(cb.ax.get_yticklabels(), fontsize=18)
plt.show()
+
+It is interesting to note that OLS
+considers both \( J_{j, j + 1} = -0.5 \) and \( J_{j, j - 1} = -0.5 \) as
+valid matrix elements for \( J \).
+In our discussion below on hyperparameters and Ridge and Lasso regression we will see that
+this problem can be removed, partly and only with Lasso regression.
+
+
+In this case our matrix inversion was actually possible. The obvious question now is what is the mathematics behind the SVD?
+
diff --git a/doc/pub/Regression/html/._Regression-bs099.html b/doc/pub/Regression/html/._Regression-bs099.html
index adbfdb03b..fcf2eeaf6 100644
--- a/doc/pub/Regression/html/._Regression-bs099.html
+++ b/doc/pub/Regression/html/._Regression-bs099.html
@@ -111,112 +111,112 @@ Automatically generated HTML file from DocOnce source
('The Boston housing data example', 2, None, '___sec27'),
('Housing data, the code', 2, None, '___sec28'),
('The singular value decomposition', 2, None, '___sec29'),
- ('The Ising model', 2, None, '___sec30'),
- ('Reformulating the problem to suit regression',
- 2,
- None,
- '___sec31'),
- ('Linear regression', 2, None, '___sec32'),
- ('Singular Value decomposition', 2, None, '___sec33'),
- ('Linear Regression Problems', 2, None, '___sec34'),
- ('Fixing the singularity', 2, None, '___sec35'),
- ('Basic math of the SVD', 2, None, '___sec36'),
- ('The SVD, a Fantastic Algorithm', 2, None, '___sec37'),
- ('Another Example', 2, None, '___sec38'),
- ('Economy-size SVD', 2, None, '___sec39'),
- ('Mathematical Properties', 2, None, '___sec40'),
- ('Ridge and LASSO Regression', 2, None, '___sec41'),
- ('More on Ridge Regression', 2, None, '___sec42'),
- ('Interpreting the Ridge results', 2, None, '___sec43'),
- ('More interpretations', 2, None, '___sec44'),
- ('Where are we going?', 2, None, '___sec45'),
- ('Resampling methods', 2, None, '___sec46'),
+ ('Linear Regression Problems', 2, None, '___sec30'),
+ ('Fixing the singularity', 2, None, '___sec31'),
+ ('Basic math of the SVD', 2, None, '___sec32'),
+ ('The SVD, a Fantastic Algorithm', 2, None, '___sec33'),
+ ('Another Example', 2, None, '___sec34'),
+ ('Economy-size SVD', 2, None, '___sec35'),
+ ('Mathematical Properties', 2, None, '___sec36'),
+ ('Ridge and LASSO Regression', 2, None, '___sec37'),
+ ('More on Ridge Regression', 2, None, '___sec38'),
+ ('Interpreting the Ridge results', 2, None, '___sec39'),
+ ('More interpretations', 2, None, '___sec40'),
+ ('Where are we going?', 2, None, '___sec41'),
+ ('Resampling methods', 2, None, '___sec42'),
('Resampling approaches can be computationally expensive',
2,
None,
- '___sec47'),
- ('Why resampling methods ?', 2, None, '___sec48'),
- ('Statistical analysis', 2, None, '___sec49'),
- ('Statistics', 2, None, '___sec50'),
- ('Statistics, moments', 2, None, '___sec51'),
- ('Statistics, central moments', 2, None, '___sec52'),
- ('Statistics, covariance', 2, None, '___sec53'),
- ('Statistics, more covariance', 2, None, '___sec54'),
- ('Covariance example', 2, None, '___sec55'),
- ('Covariance in numpy', 2, None, '___sec56'),
- ('Statistics, independent variables', 2, None, '___sec57'),
- ('Statistics, more variance', 2, None, '___sec58'),
- ('Statistics and stochastic processes', 2, None, '___sec59'),
- ('Statistics and sample variables', 2, None, '___sec60'),
+ '___sec43'),
+ ('Why resampling methods ?', 2, None, '___sec44'),
+ ('Statistical analysis', 2, None, '___sec45'),
+ ('Statistics', 2, None, '___sec46'),
+ ('Statistics, moments', 2, None, '___sec47'),
+ ('Statistics, central moments', 2, None, '___sec48'),
+ ('Statistics, covariance', 2, None, '___sec49'),
+ ('Statistics, more covariance', 2, None, '___sec50'),
+ ('Covariance example', 2, None, '___sec51'),
+ ('Covariance in numpy', 2, None, '___sec52'),
+ ('Statistics, independent variables', 2, None, '___sec53'),
+ ('Statistics, more variance', 2, None, '___sec54'),
+ ('Statistics and stochastic processes', 2, None, '___sec55'),
+ ('Statistics and sample variables', 2, None, '___sec56'),
('Statistics, sample variance and covariance',
2,
None,
- '___sec61'),
- ('Statistics, law of large numbers', 2, None, '___sec62'),
- ('Statistics, more on sample error', 2, None, '___sec63'),
- ('Statistics', 2, None, '___sec64'),
- ('Statistics, central limit theorem', 2, None, '___sec65'),
- ('Statistics, more technicalities', 2, None, '___sec66'),
- ('Statistics', 2, None, '___sec67'),
- ('Statistics and sample variance', 2, None, '___sec68'),
- ('Statistics, uncorrelated results', 2, None, '___sec69'),
- ('Statistics, computations', 2, None, '___sec70'),
+ '___sec57'),
+ ('Statistics, law of large numbers', 2, None, '___sec58'),
+ ('Statistics, more on sample error', 2, None, '___sec59'),
+ ('Statistics', 2, None, '___sec60'),
+ ('Statistics, central limit theorem', 2, None, '___sec61'),
+ ('Statistics, more technicalities', 2, None, '___sec62'),
+ ('Statistics', 2, None, '___sec63'),
+ ('Statistics and sample variance', 2, None, '___sec64'),
+ ('Statistics, uncorrelated results', 2, None, '___sec65'),
+ ('Statistics, computations', 2, None, '___sec66'),
('Statistics, more on computations of errors',
2,
None,
- '___sec71'),
- ('Statistics, wrapping up 1', 2, None, '___sec72'),
- ('Statistics, final expression', 2, None, '___sec73'),
+ '___sec67'),
+ ('Statistics, wrapping up 1', 2, None, '___sec68'),
+ ('Statistics, final expression', 2, None, '___sec69'),
('Statistics, effective number of correlations',
2,
None,
- '___sec74'),
+ '___sec70'),
('Linking the regression analysis with a statistical '
'interpretation',
2,
None,
- '___sec75'),
- ('Assumptions made', 2, None, '___sec76'),
- ('Expectation value and variance', 2, None, '___sec77'),
+ '___sec71'),
+ ('Assumptions made', 2, None, '___sec72'),
+ ('Expectation value and variance', 2, None, '___sec73'),
('Expectation value and variance for $\\boldsymbol{\\beta}$',
2,
None,
- '___sec78'),
- ('Cross-validation', 2, None, '___sec79'),
- ('Computationally expensive', 2, None, '___sec80'),
- ('Various steps in cross-validation', 2, None, '___sec81'),
+ '___sec74'),
+ ('Cross-validation', 2, None, '___sec75'),
+ ('Computationally expensive', 2, None, '___sec76'),
+ ('Various steps in cross-validation', 2, None, '___sec77'),
('How to set up the cross-validation for Ridge and/or Lasso',
2,
None,
- '___sec82'),
+ '___sec78'),
('Resampling methods: Jackknife and Bootstrap',
2,
None,
- '___sec83'),
- ('Resampling methods: Jackknife', 2, None, '___sec84'),
- ('Jackknife code example', 2, None, '___sec85'),
- ('Resampling methods: Bootstrap', 2, None, '___sec86'),
- ('Resampling methods: Bootstrap background', 2, None, '___sec87'),
+ '___sec79'),
+ ('Resampling methods: Jackknife', 2, None, '___sec80'),
+ ('Jackknife code example', 2, None, '___sec81'),
+ ('Resampling methods: Bootstrap', 2, None, '___sec82'),
+ ('Resampling methods: Bootstrap background', 2, None, '___sec83'),
('Resampling methods: More Bootstrap background',
2,
None,
- '___sec88'),
- ('Resampling methods: Bootstrap approach', 2, None, '___sec89'),
- ('Resampling methods: Bootstrap steps', 2, None, '___sec90'),
- ('Code example for the Bootstrap method', 2, None, '___sec91'),
+ '___sec84'),
+ ('Resampling methods: Bootstrap approach', 2, None, '___sec85'),
+ ('Resampling methods: Bootstrap steps', 2, None, '___sec86'),
+ ('Code example for the Bootstrap method', 2, None, '___sec87'),
('Code Example for Cross-validation and $k$-fold '
'Cross-validation',
2,
None,
- '___sec92'),
- ('The bias-variance tradeoff', 2, None, '___sec93'),
- ('Example code for Bias-Variance tradeoff', 2, None, '___sec94'),
- ('Understanding what happens', 2, None, '___sec95'),
- ('Summing up', 2, None, '___sec96'),
+ '___sec88'),
+ ('The bias-variance tradeoff', 2, None, '___sec89'),
+ ('Example code for Bias-Variance tradeoff', 2, None, '___sec90'),
+ ('Understanding what happens', 2, None, '___sec91'),
+ ('Summing up', 2, None, '___sec92'),
("Another Example rom Scikit-Learn's Repository",
2,
None,
- '___sec97'),
+ '___sec93'),
+ ('The Ising model', 2, None, '___sec94'),
+ ('Reformulating the problem to suit regression',
+ 2,
+ None,
+ '___sec95'),
+ ('Linear regression', 2, None, '___sec96'),
+ ('Singular Value decomposition', 2, None, '___sec97'),
('The one-dimensional Ising model', 2, None, '___sec98'),
('Ridge regression', 2, None, '___sec99'),
('LASSO regression', 2, None, '___sec100'),
@@ -295,74 +295,74 @@ MathJax.Hub.Config({
-
diff --git a/doc/pub/Regression/html/Regression-reveal.html b/doc/pub/Regression/html/Regression-reveal.html index 43534fae6..4f66cfa17 100644 --- a/doc/pub/Regression/html/Regression-reveal.html +++ b/doc/pub/Regression/html/Regression-reveal.html @@ -148,7 +148,7 @@ MathJax.Hub.Config({
-
@@ -1490,8 +1490,7 @@ The examples we have looked at so far are cases where we normally can invert the matrix \( \boldsymbol{X}^T\boldsymbol{X} \). Using a polynomial expansion as we did both for the masses and the fitting of the equation of state, leads to row vectors of the design matrix which are essentially -orthogonal due to the polynomial character of our model. Obtaining the inverse of the design matrix is then often done via a so-called LU, QR or Cholesky decomposition. -More material to come here. +orthogonal due to the polynomial character of our model. Obtaining the inverse of the design matrix is then often done via a so-called LU, QR or Cholesky decomposition.
This may @@ -1509,276 +1508,11 @@ inversion algorithm. Thereafter we dive into the math of the SVD.
-
-The one-dimensional Ising model with nearest neighbor interaction, no
-external field and a constant coupling constant \( J \) is given by
-
-
-where \( s_i \in \{-1, 1\} \) and \( s_{N + 1} = s_1 \). The number of spins
-in the system is determined by \( L \). For the one-dimensional system
-there is no phase transition.
-
-
-We will look at a system of \( L = 40 \) spins with a coupling constant of
-\( J = 1 \). To get enough training data we will generate 10000 states
-with their respective energies.
-
-
-
-
-
-Here we use ordinary least squares
-regression to predict the energy for the nearest neighbor
-one-dimensional Ising model on a ring, i.e., the endpoints wrap
-around. We will use linear regression to fit a value for
-the coupling constant to achieve this.
-
-A more general form for the one-dimensional Ising model is
-
-
-Here we allow for interactions beyond the nearest neighbors and a state dependent
-coupling constant. This latter expression can be formulated as
-a matrix-product
-
-where \( X_{jk} = s_j s_k \) and \( J \) is a matrix which consists of the
-elements \( -J_{jk} \). This form of writing the energy fits perfectly
-with the form utilized in linear regression, that is
-
-
-We split the data in training and test data as discussed in the previous example
-
-
-
-
-
-In the ordinary least squares method we choose the cost function
-
-
-We then find the extremal point of \( C \) by taking the derivative with respect to \( \boldsymbol{\beta} \) as discussed above.
-This yields the expression for \( \boldsymbol{\beta} \) to be
-
-
-which immediately imposes some requirements on \( \boldsymbol{X} \) as there must exist
-an inverse of \( \boldsymbol{X}^T \boldsymbol{X} \). If the expression we are modeling contains an
-intercept, i.e., a constant term, we must make sure that the
-first column of \( \boldsymbol{X} \) consists of \( 1 \). We do this here
-
-
-
-
-
-
-
-
-Doing the inversion directly turns out to be a bad idea since the matrix
-\( \boldsymbol{X}^T\boldsymbol{X} \) is singular. An alternative approach is to use the singular
-value decomposition. Using the definition of the Moore-Penrose
-pseudoinverse we can write the equation for \( \boldsymbol{\beta} \) as
-
-
-where the pseudoinverse of \( \boldsymbol{X} \) is given by
-
-
-Using singular value decomposition we can decompose the matrix \( \boldsymbol{X} = \boldsymbol{U}\boldsymbol{\Sigma} \boldsymbol{V}^T \),
-where \( \boldsymbol{U} \) and \( \boldsymbol{V} \) are orthogonal(unitary) matrices and \( \boldsymbol{\Sigma} \) contains the singular values (more details below).
-where \( X^{+} = V\Sigma^{+} U^T \). This reduces the equation for
-\( \omega \) to
-
-Note that solving this equation by actually doing the pseudoinverse
-(which is what we will do) is not a good idea as this operation scales
-as \( \mathcal{O}(n^3) \), where \( n \) is the number of elements in a
-general matrix. Instead, doing \( QR \)-factorization and solving the
-linear system as an equation would reduce this down to
-\( \mathcal{O}(n^2) \) operations.
-
-
-
-
-
-
-
-
-When extracting the \( J \)-matrix we need to make sure that we remove the intercept, as is done here
-
-
-
-
-
-A way of looking at the coefficients in \( J \) is to plot the matrices as images.
-
-
-
-
-
-It is interesting to note that OLS
-considers both \( J_{j, j + 1} = -0.5 \) and \( J_{j, j - 1} = -0.5 \) as
-valid matrix elements for \( J \).
-In our discussion below on hyperparameters and Ridge and Lasso regression we will see that
-this problem can be removed, partly and only with Lasso regression.
-
-
-In this case our matrix inversion was actually possible. The obvious question now is what is the mathematics behind the SVD?
-
One of the typical problems we encounter with linear regression, in particular
@@ -1813,7 +1547,7 @@ of linearly independent columns. In this particular case the matrix has rank 2.
Super-collinearity of an \( (n \times p) \)-dimensional design matrix \( \mathbf{X} \) implies
-that the inverse of the matrix \( \boldsymbol{X}^T\boldsymbol{x} \) (the matrix we need to invert to solve the linear regression equations) is non-invertible. If we have a square matrix that does not have an inverse, we say this matrix singular. The example here demonstrates this
+that the inverse of the matrix \( \boldsymbol{X}^T\boldsymbol{X} \) (the matrix we need to invert to solve the linear regression equations) is non-invertible. If we have a square matrix that does not have an inverse, we say this matrix singular. The example here demonstrates this
If our design matrix \( \boldsymbol{X} \) which enters the linear regression problem
@@ -1841,7 +1575,7 @@ If our design matrix \( \boldsymbol{X} \) which enters the linear regression pro
$$
\begin{align}
\boldsymbol{\beta} & = (\boldsymbol{X}^{T} \boldsymbol{X})^{-1} \boldsymbol{X}^{T} \boldsymbol{y},
-\tag{7}
+\tag{1}
\end{align}
$$
From standard linear algebra we know that a square matrix \( \boldsymbol{X} \) can be diagonalized if and only it is
@@ -1912,7 +1646,7 @@ is not diagonalizable, it is a so-called The SVD, a Fantastic Algorithm
+
However, and this is the strength of the SVD algorithm, any general
@@ -1947,7 +1681,7 @@ The SVD exits always!
Consider the following matrix which can be SVD decomposed as
@@ -1974,7 +1708,7 @@ In the general case, where our design matrix \( \boldsymbol{X} \) has dimension
\( n\times p \), the matrix is thus decomposed into an \( n\times n \)
orthogonal matrix \( \boldsymbol{U} \), a \( p\times p \) orthogonal matrix \( \boldsymbol{V} \)
and a diagonal matrix \( \boldsymbol{\Sigma} \) with \( r=\mathrm{min}(n,p) \)
-singular values \( \sigma_i\lg 0 \) on the main diagonal and zeros filling
+singular values \( \sigma_i\geq 0 \) on the main diagonal and zeros filling
the rest of the matrix. There are at most \( p \) singular values
assuming that \( n > p \). In our regression examples for the nuclear
masses and the equation of state this is indeed the case, while for
@@ -1987,7 +1721,7 @@ The columns of \( \boldsymbol{U} \) are called the left singular vectors while t
If we assume that \( n > p \), then our matrix \( \boldsymbol{U} \) has dimension \( n
@@ -2012,7 +1746,7 @@ In general the economy-size SVD leads to less FLOPS and still conserving the des
There are several interesting mathematical properties which will be
@@ -2076,7 +1810,7 @@ We will come back to this expression when we discuss Ridge regression.
Let us remind ourselves about the expression for the standard Mean Squared Error (MSE) which we used to define our cost function and the equations for the ordinary least squares (OLS) method, that is
@@ -2148,7 +1882,7 @@ $$
Using the matrix-vector expression for Ridge regression,
@@ -2219,7 +1953,7 @@ with the vectors \( \boldsymbol{u}_j \) being the columns of \( \boldsymbol{U} \
Since \( \lambda \geq 0 \), it means that compared to OLS, we have
@@ -2245,7 +1979,7 @@ With a parameter \( \lambda \) we can thus shrink the role of specific parameter
For the sake of simplicity, let us assume that the design matrix is orthonormal, that is
@@ -2288,7 +2022,7 @@ Similarly, Mehta et a
Before we proceed, we need to rethink what we have been doing. In our
@@ -2306,7 +2040,7 @@ This will allow us to link the standard linear algebra methods we have discussed
@@ -2325,7 +2059,7 @@ once using the original training sample.
@@ -2351,7 +2085,7 @@ bootstrap is widely used.
@@ -2418,7 +2152,7 @@ selection of a large set of these numbers reproduces this PDF.
@@ -2444,7 +2178,7 @@ $$
@@ -2465,14 +2199,14 @@ $$
\begin{align}
\sigma^2_X\ \ =\ \ \mathrm{var}(X) & = \langle (x-\langle x\rangle)^2\rangle =
\int\! (x-\langle x\rangle)^2 p(x)\,dx
-\tag{8}\\
+\tag{2}\\
& = \int\! \left(x^2 - 2 x \langle x\rangle^{2} +
\langle x\rangle^2\right)p(x)\,dx
-\tag{9}\\
+\tag{3}\\
& = \langle x^2\rangle - 2 \langle x\rangle\langle x\rangle + \langle x\rangle^2
-\tag{10}\\
+\tag{4}\\
& = \langle x^2\rangle - \langle x\rangle^2
-\tag{11}
+\tag{5}
\end{align}
$$
@@ -2502,7 +2236,7 @@ $$
&=
\int\!\cdots\!\int\!(x_i-\langle x_i \rangle)(x_j-\langle x_j \rangle)\,
P(x_1,\dots,x_n)\,dx_1\dots dx_n
-\tag{12}
+\tag{6}
\end{align}
$$
@@ -2533,17 +2267,17 @@ the expectation value. Consider the stochastic variables \( X_i \) and
$$
\begin{align}
\mathrm{cov}(X_i,\,X_j) &= \langle(x_i-\langle x_i\rangle)(x_j-\langle x_j\rangle)\rangle
-\tag{13}\\
+\tag{7}\\
&=\langle x_i x_j - x_i\langle x_j\rangle - \langle x_i\rangle x_j + \langle x_i\rangle\langle x_j\rangle\rangle
-\tag{14}\\
+\tag{8}\\
&=\langle x_i x_j\rangle - \langle x_i\langle x_j\rangle\rangle - \langle \langle x_i\rangle x_j\rangle +
\langle \langle x_i\rangle\langle x_j\rangle\rangle
-\tag{15}\\
+\tag{9}\\
&=\langle x_i x_j\rangle - \langle x_i\rangle\langle x_j\rangle - \langle x_i\rangle\langle x_j\rangle +
\langle x_i\rangle\langle x_j\rangle
-\tag{16}\\
+\tag{10}\\
&=\langle x_i x_j\rangle - \langle x_i\rangle\langle x_j\rangle
-\tag{17}
+\tag{11}
\end{align}
$$
Suppose we have defined three vectors \( \hat{x}, \hat{y}, \hat{z} \) with
@@ -2607,7 +2341,7 @@ function.
@@ -2647,7 +2381,7 @@ plt.show()
@@ -2676,7 +2410,7 @@ $$
@@ -2686,7 +2420,7 @@ the variance of the linear combination \( U = \sum_i a_i X_i \):
$$
\begin{equation}
\mathrm{var}(U) = \sum_{i,j}a_i a_j \mathrm{cov}(X_i, X_j)
-\tag{18}
+\tag{12}
\end{equation}
$$
@@ -2742,7 +2476,7 @@ interested in finding the few lowest moments, like the mean
@@ -2773,12 +2507,12 @@ $$
Note that the sample variance is the sample covariance without the
-cross terms. In a similar manner as the covariance in Eq. (12) is a measure of the correlation between
+cross terms. In a similar manner as the covariance in Eq. (6) is a measure of the correlation between
two stochastic variables, the above defined sample covariance is a
measure of the sequential correlation between succeeding measurements
of a sample.
@@ -2793,7 +2527,7 @@ and covariance \( \mathrm{cov}(X,Y) \).
@@ -2826,7 +2560,7 @@ true PDFs behind, which we usually do not have.
@@ -2849,7 +2583,7 @@ means.
@@ -2871,7 +2605,7 @@ And in particular we are interested in its variance \( \mathrm{var}(\overline X_
@@ -2888,7 +2622,7 @@ $$
\lim_{n\to\infty} p_{\overline X_n}(x) =
\left(\frac{n}{2\pi\mathrm{var}(X)}\right)^{1/2}
e^{-\frac{n(x-\bar x_n)^2}{2\mathrm{var}(X)}}
-\tag{19}
+\tag{13}
\end{equation}
$$
@@ -2909,7 +2643,7 @@ $$
\begin{equation}
\mathrm{err}_X^2 = \mathrm{var}(\overline X_n) = \frac{1}{n^2}
\sum_{ij} \mathrm{cov}(X_i, X_j)
-\tag{20}
+\tag{14}
\end{equation}
$$
@@ -2940,7 +2674,7 @@ $$
@@ -2976,7 +2710,7 @@ which is approximated as
$$
\begin{equation}
\mathrm{var}(X_i)\approx \frac{1}{n}\sum_{k=1}^n (x_k - \bar x_n)=\mathrm{var}(x)
-\tag{21}
+\tag{15}
\end{equation}
$$
@@ -3023,12 +2757,12 @@ resulting in
$$
\begin{equation}
\mathrm{err}_X^2\approx \frac{1}{n^2} \sum_i \mathrm{var}(x)= \frac{1}{n}\mathrm{var}(x)
-\tag{23}
+\tag{17}
\end{equation}
$$
For computational purposes one usually splits up the estimate of
-\( \mathrm{err}_X^2 \), given by Eq. (22), into two
+\( \mathrm{err}_X^2 \), given by Eq. (16), into two
parts
@@ -3094,7 +2828,7 @@ have to be stored throughout the experiment.
@@ -3132,11 +2866,11 @@ starting always at \( 1 \) for \( d=0 \).
-The sample error (see eq. (24)) can now be
+The sample error (see eq. (18)) can now be
written in terms of the autocorrelation function:
For a correlation free experiment, \( \tau \)
equals 1. From the point of view of
-eq. (25) we can interpret a sequential
+eq. (19) we can interpret a sequential
correlation as an effective reduction of the number of measurements by
a factor \( \tau \). The effective number of measurements becomes:
Finally, we are going to discuss several statistical properties which can be obtained in terms of analytical expressions.
@@ -3234,7 +2968,7 @@ row number \( i \) and perform a sum over all values \( p \).
The assumption we have made here can be summarized as (and this is going to useful when we discuss the bias-variance trade off)
@@ -3258,7 +2992,7 @@ $$
We can calculate the expectation value of \( \boldsymbol{y} \) for a given element \( i \)
@@ -3297,7 +3031,7 @@ mean value \( \boldsymbol{X}\boldsymbol{\beta} \) and variance \( \sigma^2 \) (n
With the OLS expressions for the parameters \( \boldsymbol{\beta} \) we can evaluate the expectation value
@@ -3392,7 +3126,7 @@ This means the variance we obtain with the standard OLS will always for \( \lamb
Instead of choosing the penalty parameter to balance model fit with
@@ -3426,7 +3160,7 @@ some sense) is then selected.
The validation set approach is conceptually simple and is easy to implement. But it has two potential drawbacks:
@@ -3439,7 +3173,7 @@ The validation set approach is conceptually simple and is easy to implement. But
When the repetitive splitting of the data set is done randomly,
@@ -3460,7 +3194,7 @@ cross-validation (LOOCV).
Two famous
@@ -3522,7 +3256,7 @@ need for bootstrapping.
The Jackknife works by making many replicas of the estimator \( \widehat{\theta} \).
@@ -3543,7 +3277,7 @@ number \( i \) is left out. Using this notation, define
@@ -3580,7 +3314,7 @@ t = jackknife(x, stat)
@@ -3602,7 +3336,7 @@ advantages:
Since \( \widehat{\theta} = \widehat{\theta}(\boldsymbol{X}) \) is a function of random variables,
@@ -3618,7 +3352,7 @@ estimators.
In the case that \( \widehat{\theta} \) has
@@ -3641,7 +3375,7 @@ idea is to use the relative frequency of \( \widehat{\theta}^* \)
But
@@ -3662,7 +3396,7 @@ frequency of the observation \( X_i \), just draw the values
The independent bootstrap works like this:
@@ -3688,7 +3422,7 @@ example, if you are interested in estimating the variance of \( \widehat
The following code starts with a Gaussian distribution with mean value
@@ -3753,7 +3487,7 @@ plt.show()
The code here uses Ridge regression with cross-validation (CV) resampling and \( k \)-fold CV in order to fit a specific polynomial.
@@ -3854,7 +3588,7 @@ plt.show()
We will discuss the bias-variance tradeoff in the context of
@@ -3932,7 +3666,7 @@ that is the rewriting in terms of the so-called bias, the variance of the model
@@ -3995,7 +3729,7 @@ plt.show()
@@ -4050,7 +3784,7 @@ plt.show()
The bias-variance tradeoff summarizes the fundamental tension in
@@ -4084,7 +3818,7 @@ flexible statistical methods have higher variance.
@@ -4163,6 +3897,268 @@ plt.show()
+The one-dimensional Ising model with nearest neighbor interaction, no
+external field and a constant coupling constant \( J \) is given by
+
+
+where \( s_i \in \{-1, 1\} \) and \( s_{N + 1} = s_1 \). The number of spins
+in the system is determined by \( L \). For the one-dimensional system
+there is no phase transition.
+
+
+We will look at a system of \( L = 40 \) spins with a coupling constant of
+\( J = 1 \). To get enough training data we will generate 10000 states
+with their respective energies.
+
+
+
+
+
+Here we use ordinary least squares
+regression to predict the energy for the nearest neighbor
+one-dimensional Ising model on a ring, i.e., the endpoints wrap
+around. We will use linear regression to fit a value for
+the coupling constant to achieve this.
+
+A more general form for the one-dimensional Ising model is
+
+
+Here we allow for interactions beyond the nearest neighbors and a state dependent
+coupling constant. This latter expression can be formulated as
+a matrix-product
+
+where \( X_{jk} = s_j s_k \) and \( J \) is a matrix which consists of the
+elements \( -J_{jk} \). This form of writing the energy fits perfectly
+with the form utilized in linear regression, that is
+
+
+We split the data in training and test data as discussed in the previous example
+
+
+
+
+
+In the ordinary least squares method we choose the cost function
+
+
+We then find the extremal point of \( C \) by taking the derivative with respect to \( \boldsymbol{\beta} \) as discussed above.
+This yields the expression for \( \boldsymbol{\beta} \) to be
+
+
+which immediately imposes some requirements on \( \boldsymbol{X} \) as there must exist
+an inverse of \( \boldsymbol{X}^T \boldsymbol{X} \). If the expression we are modeling contains an
+intercept, i.e., a constant term, we must make sure that the
+first column of \( \boldsymbol{X} \) consists of \( 1 \). We do this here
+
+
+
+
+
+
+
+
+Doing the inversion directly turns out to be a bad idea since the matrix
+\( \boldsymbol{X}^T\boldsymbol{X} \) is singular. An alternative approach is to use the singular
+value decomposition. Using the definition of the Moore-Penrose
+pseudoinverse we can write the equation for \( \boldsymbol{\beta} \) as
+
+
+where the pseudoinverse of \( \boldsymbol{X} \) is given by
+
+
+Using singular value decomposition we can decompose the matrix \( \boldsymbol{X} = \boldsymbol{U}\boldsymbol{\Sigma} \boldsymbol{V}^T \),
+where \( \boldsymbol{U} \) and \( \boldsymbol{V} \) are orthogonal(unitary) matrices and \( \boldsymbol{\Sigma} \) contains the singular values (more details below).
+where \( X^{+} = V\Sigma^{+} U^T \). This reduces the equation for
+\( \omega \) to
+
+Note that solving this equation by actually doing the pseudoinverse
+(which is what we will do) is not a good idea as this operation scales
+as \( \mathcal{O}(n^3) \), where \( n \) is the number of elements in a
+general matrix. Instead, doing \( QR \)-factorization and solving the
+linear system as an equation would reduce this down to
+\( \mathcal{O}(n^2) \) operations.
+
+
+
+
+
+
+
+
+When extracting the \( J \)-matrix we need to make sure that we remove the intercept, as is done here
+
+
+
+
+
+A way of looking at the coefficients in \( J \) is to plot the matrices as images.
+
+
+
+
+
+It is interesting to note that OLS
+considers both \( J_{j, j + 1} = -0.5 \) and \( J_{j, j - 1} = -0.5 \) as
+valid matrix elements for \( J \).
+In our discussion below on hyperparameters and Ridge and Lasso regression we will see that
+this problem can be removed, partly and only with Lasso regression.
+
+
+In this case our matrix inversion was actually possible. The obvious question now is what is the mathematics behind the SVD?
+
-
This may
@@ -1575,255 +1574,10 @@ inversion algorithm. Thereafter we dive into the math of the SVD.
-
-
-The one-dimensional Ising model with nearest neighbor interaction, no
-external field and a constant coupling constant \( J \) is given by
-
-$$
-\begin{align}
- H = -J \sum_{k}^L s_k s_{k + 1},
-\label{_auto1}
-\end{align}
-$$
-
-
-where \( s_i \in \{-1, 1\} \) and \( s_{N + 1} = s_1 \). The number of spins
-in the system is determined by \( L \). For the one-dimensional system
-there is no phase transition.
-
-
-We will look at a system of \( L = 40 \) spins with a coupling constant of
-\( J = 1 \). To get enough training data we will generate 10000 states
-with their respective energies.
-
-
-
-
-
-Here we use ordinary least squares
-regression to predict the energy for the nearest neighbor
-one-dimensional Ising model on a ring, i.e., the endpoints wrap
-around. We will use linear regression to fit a value for
-the coupling constant to achieve this.
-
-
-
-A more general form for the one-dimensional Ising model is
-
-$$
-\begin{align}
- H = - \sum_j^L \sum_k^L s_j s_k J_{jk}.
-\label{_auto2}
-\end{align}
-$$
-
-
-Here we allow for interactions beyond the nearest neighbors and a state dependent
-coupling constant. This latter expression can be formulated as
-a matrix-product
-$$
-\begin{align}
- \boldsymbol{H} = \boldsymbol{X} J,
-\label{_auto3}
-\end{align}
-$$
-
-
-where \( X_{jk} = s_j s_k \) and \( J \) is a matrix which consists of the
-elements \( -J_{jk} \). This form of writing the energy fits perfectly
-with the form utilized in linear regression, that is
-
-$$
-\begin{align}
- \boldsymbol{y} = \boldsymbol{X}\boldsymbol{\beta} + \boldsymbol{\epsilon},
-\label{_auto4}
-\end{align}
-$$
-
-
-We split the data in training and test data as discussed in the previous example
-
-
-
-
-
-
-In the ordinary least squares method we choose the cost function
-
-$$
-\begin{align}
- C(\boldsymbol{X}, \boldsymbol{\beta})= \frac{1}{n}\left\{(\boldsymbol{X}\boldsymbol{\beta} - \boldsymbol{y})^T(\boldsymbol{X}\boldsymbol{\beta} - \boldsymbol{y})\right\}.
-\label{_auto5}
-\end{align}
-$$
-
-
-We then find the extremal point of \( C \) by taking the derivative with respect to \( \boldsymbol{\beta} \) as discussed above.
-This yields the expression for \( \boldsymbol{\beta} \) to be
-
-$$
- \boldsymbol{\beta} = \frac{\boldsymbol{X}^T \boldsymbol{y}}{\boldsymbol{X}^T \boldsymbol{X}},
-$$
-
-
-which immediately imposes some requirements on \( \boldsymbol{X} \) as there must exist
-an inverse of \( \boldsymbol{X}^T \boldsymbol{X} \). If the expression we are modeling contains an
-intercept, i.e., a constant term, we must make sure that the
-first column of \( \boldsymbol{X} \) consists of \( 1 \). We do this here
-
-
-
-
-
-
-
-
-
-Doing the inversion directly turns out to be a bad idea since the matrix
-\( \boldsymbol{X}^T\boldsymbol{X} \) is singular. An alternative approach is to use the singular
-value decomposition. Using the definition of the Moore-Penrose
-pseudoinverse we can write the equation for \( \boldsymbol{\beta} \) as
-
-$$
- \boldsymbol{\beta} = \boldsymbol{X}^{+}\boldsymbol{y},
-$$
-
-
-where the pseudoinverse of \( \boldsymbol{X} \) is given by
-
-$$
- \boldsymbol{X}^{+} = \frac{\boldsymbol{X}^T}{\boldsymbol{X}^T\boldsymbol{X}}.
-$$
-
-
-Using singular value decomposition we can decompose the matrix \( \boldsymbol{X} = \boldsymbol{U}\boldsymbol{\Sigma} \boldsymbol{V}^T \),
-where \( \boldsymbol{U} \) and \( \boldsymbol{V} \) are orthogonal(unitary) matrices and \( \boldsymbol{\Sigma} \) contains the singular values (more details below).
-where \( X^{+} = V\Sigma^{+} U^T \). This reduces the equation for
-\( \omega \) to
-$$
-\begin{align}
- \boldsymbol{\beta} = \boldsymbol{V}\boldsymbol{\Sigma}^{+} \boldsymbol{U}^T \boldsymbol{y}.
-\label{_auto6}
-\end{align}
-$$
-
-
-Note that solving this equation by actually doing the pseudoinverse
-(which is what we will do) is not a good idea as this operation scales
-as \( \mathcal{O}(n^3) \), where \( n \) is the number of elements in a
-general matrix. Instead, doing \( QR \)-factorization and solving the
-linear system as an equation would reduce this down to
-\( \mathcal{O}(n^2) \) operations.
-
-
-
-
-
-
-
-
-When extracting the \( J \)-matrix we need to make sure that we remove the intercept, as is done here
-
-
-
-
-
-A way of looking at the coefficients in \( J \) is to plot the matrices as images.
-
-
-
-
-
-It is interesting to note that OLS
-considers both \( J_{j, j + 1} = -0.5 \) and \( J_{j, j - 1} = -0.5 \) as
-valid matrix elements for \( J \).
-In our discussion below on hyperparameters and Ridge and Lasso regression we will see that
-this problem can be removed, partly and only with Lasso regression.
-
-
-In this case our matrix inversion was actually possible. The obvious question now is what is the mathematics behind the SVD?
-
-
-
One of the typical problems we encounter with linear regression, in particular
@@ -1856,7 +1610,7 @@ of linearly independent columns. In this particular case the matrix has rank 2.
Super-collinearity of an \( (n \times p) \)-dimensional design matrix \( \mathbf{X} \) implies
-that the inverse of the matrix \( \boldsymbol{X}^T\boldsymbol{x} \) (the matrix we need to invert to solve the linear regression equations) is non-invertible. If we have a square matrix that does not have an inverse, we say this matrix singular. The example here demonstrates this
+that the inverse of the matrix \( \boldsymbol{X}^T\boldsymbol{X} \) (the matrix we need to invert to solve the linear regression equations) is non-invertible. If we have a square matrix that does not have an inverse, we say this matrix singular. The example here demonstrates this
$$
\begin{align*}
\boldsymbol{X} & = \left[
@@ -1874,14 +1628,14 @@ This is equivalent to saying that the matrix \( \boldsymbol{X} \) has at least a
If our design matrix \( \boldsymbol{X} \) which enters the linear regression problem
$$
\begin{align}
\boldsymbol{\beta} & = (\boldsymbol{X}^{T} \boldsymbol{X})^{-1} \boldsymbol{X}^{T} \boldsymbol{y},
-\label{_auto7}
+\label{_auto1}
\end{align}
$$
@@ -1902,7 +1656,7 @@ where \( \boldsymbol{I} \) is the identity matrix. When we discuss Ridge
From standard linear algebra we know that a square matrix \( \boldsymbol{X} \) can be diagonalized if and only it is
@@ -1941,7 +1695,7 @@ is not diagonalizable, it is a so-called The SVD, a Fantastic Algorithm
+
However, and this is the strength of the SVD algorithm, any general
@@ -1972,7 +1726,7 @@ The SVD exits always!
Consider the following matrix which can be SVD decomposed as
@@ -1997,7 +1751,7 @@ In the general case, where our design matrix \( \boldsymbol{X} \) has dimension
\( n\times p \), the matrix is thus decomposed into an \( n\times n \)
orthogonal matrix \( \boldsymbol{U} \), a \( p\times p \) orthogonal matrix \( \boldsymbol{V} \)
and a diagonal matrix \( \boldsymbol{\Sigma} \) with \( r=\mathrm{min}(n,p) \)
-singular values \( \sigma_i\lg 0 \) on the main diagonal and zeros filling
+singular values \( \sigma_i\geq 0 \) on the main diagonal and zeros filling
the rest of the matrix. There are at most \( p \) singular values
assuming that \( n > p \). In our regression examples for the nuclear
masses and the equation of state this is indeed the case, while for
@@ -2010,7 +1764,7 @@ The columns of \( \boldsymbol{U} \) are called the left singular vectors while t
If we assume that \( n > p \), then our matrix \( \boldsymbol{U} \) has dimension \( n
@@ -2035,7 +1789,7 @@ In general the economy-size SVD leads to less FLOPS and still conserving the des
There are several interesting mathematical properties which will be
@@ -2087,7 +1841,7 @@ We will come back to this expression when we discuss Ridge regression.
Let us remind ourselves about the expression for the standard Mean Squared Error (MSE) which we used to define our cost function and the equations for the ordinary least squares (OLS) method, that is
@@ -2145,7 +1899,7 @@ $$
Using the matrix-vector expression for Ridge regression,
@@ -2204,7 +1958,7 @@ with the vectors \( \boldsymbol{u}_j \) being the columns of \( \boldsymbol{U} \
Since \( \lambda \geq 0 \), it means that compared to OLS, we have
@@ -2228,7 +1982,7 @@ With a parameter \( \lambda \) we can thus shrink the role of specific parameter
For the sake of simplicity, let us assume that the design matrix is orthonormal, that is
@@ -2265,7 +2019,7 @@ Similarly, Mehta et a
Before we proceed, we need to rethink what we have been doing. In our
@@ -2282,7 +2036,7 @@ This will allow us to link the standard linear algebra methods we have discussed
@@ -2302,7 +2056,7 @@ once using the original training sample.
@@ -2331,7 +2085,7 @@ bootstrap is widely used.
@@ -2347,7 +2101,7 @@ bootstrap is widely used.
@@ -2369,7 +2123,7 @@ bootstrap is widely used.
@@ -2400,7 +2154,7 @@ selection of a large set of these numbers reproduces this PDF.
@@ -2423,7 +2177,7 @@ $$
@@ -2441,14 +2195,14 @@ $$
\begin{align}
\sigma^2_X\ \ =\ \ \mathrm{var}(X) & = \langle (x-\langle x\rangle)^2\rangle =
\int\! (x-\langle x\rangle)^2 p(x)\,dx
-\label{_auto8}\\
+\label{_auto2}\\
& = \int\! \left(x^2 - 2 x \langle x\rangle^{2} +
\langle x\rangle^2\right)p(x)\,dx
-\label{_auto9}\\
+\label{_auto3}\\
& = \langle x^2\rangle - 2 \langle x\rangle\langle x\rangle + \langle x\rangle^2
-\label{_auto10}\\
+\label{_auto4}\\
& = \langle x^2\rangle - \langle x\rangle^2
-\label{_auto11}
+\label{_auto5}
\end{align}
$$
@@ -2461,7 +2215,7 @@ qualitatively as the spread of \( p \) around its mean.
@@ -2492,7 +2246,7 @@ $$
@@ -2505,17 +2259,17 @@ the expectation value. Consider the stochastic variables \( X_i \) and
$$
\begin{align}
\mathrm{cov}(X_i,\,X_j) &= \langle(x_i-\langle x_i\rangle)(x_j-\langle x_j\rangle)\rangle
-\label{_auto12}\\
+\label{_auto6}\\
&=\langle x_i x_j - x_i\langle x_j\rangle - \langle x_i\rangle x_j + \langle x_i\rangle\langle x_j\rangle\rangle
-\label{_auto13}\\
+\label{_auto7}\\
&=\langle x_i x_j\rangle - \langle x_i\langle x_j\rangle\rangle - \langle \langle x_i\rangle x_j\rangle +
\langle \langle x_i\rangle\langle x_j\rangle\rangle
-\label{_auto14}\\
+\label{_auto8}\\
&=\langle x_i x_j\rangle - \langle x_i\rangle\langle x_j\rangle - \langle x_i\rangle\langle x_j\rangle +
\langle x_i\rangle\langle x_j\rangle
-\label{_auto15}\\
+\label{_auto9}\\
&=\langle x_i x_j\rangle - \langle x_i\rangle\langle x_j\rangle
-\label{_auto16}
+\label{_auto10}
\end{align}
$$
Suppose we have defined three vectors \( \hat{x}, \hat{y}, \hat{z} \) with
@@ -2573,7 +2327,7 @@ function.
@@ -2612,7 +2366,7 @@ plt.show()
@@ -2638,7 +2392,7 @@ $$
@@ -2670,7 +2424,7 @@ value of a set of measurements.
@@ -2698,7 +2452,7 @@ interested in finding the few lowest moments, like the mean
-
@@ -2724,7 +2478,7 @@ $$
@@ -2745,7 +2499,7 @@ and covariance \( \mathrm{cov}(X,Y) \).
@@ -2777,7 +2531,7 @@ true PDFs behind, which we usually do not have.
@@ -2799,7 +2553,7 @@ means.
@@ -2820,7 +2574,7 @@ And in particular we are interested in its variance \( \mathrm{var}(\overline X_
@@ -2845,7 +2599,7 @@ $$
@@ -2874,7 +2628,7 @@ estimate of the PDF of each of the \( X_i \), estimating all properties of
@@ -2901,7 +2655,7 @@ $$
@@ -2940,7 +2694,7 @@ measurements in the sample.
@@ -2975,7 +2729,7 @@ cannot overlook the always present correlations.
@@ -3004,7 +2758,7 @@ measurements. For uncorrelated measurements this second term is zero.
@@ -3026,7 +2780,7 @@ have to be stored throughout the experiment.
@@ -3059,7 +2813,7 @@ starting always at \( 1 \) for \( d=0 \).
@@ -3092,7 +2846,7 @@ $$
@@ -3117,7 +2871,7 @@ measurements is very large.
-
Finally, we are going to discuss several statistical properties which can be obtained in terms of analytical expressions.
@@ -3155,7 +2909,7 @@ row number \( i \) and perform a sum over all values \( p \).
The assumption we have made here can be summarized as (and this is going to useful when we discuss the bias-variance trade off)
@@ -3175,7 +2929,7 @@ $$
We can calculate the expectation value of \( \boldsymbol{y} \) for a given element \( i \)
@@ -3210,7 +2964,7 @@ mean value \( \boldsymbol{X}\boldsymbol{\beta} \) and variance \( \sigma^2 \) (n
With the OLS expressions for the parameters \( \boldsymbol{\beta} \) we can evaluate the expectation value
@@ -3295,7 +3049,7 @@ This means the variance we obtain with the standard OLS will always for \( \lamb
-
Instead of choosing the penalty parameter to balance model fit with
@@ -3329,7 +3083,7 @@ some sense) is then selected.
The validation set approach is conceptually simple and is easy to implement. But it has two potential drawbacks:
@@ -3341,7 +3095,7 @@ The validation set approach is conceptually simple and is easy to implement. But
-
When the repetitive splitting of the data set is done randomly,
@@ -3362,7 +3116,7 @@ cross-validation (LOOCV).
-
Two famous
@@ -3421,7 +3175,7 @@ need for bootstrapping.
The Jackknife works by making many replicas of the estimator \( \widehat{\theta} \).
@@ -3440,7 +3194,7 @@ number \( i \) is left out. Using this notation, define
@@ -3476,7 +3230,7 @@ t = jackknife(x, stat)
@@ -3497,7 +3251,7 @@ advantages:
Since \( \widehat{\theta} = \widehat{\theta}(\boldsymbol{X}) \) is a function of random variables,
@@ -3513,7 +3267,7 @@ estimators.
In the case that \( \widehat{\theta} \) has
@@ -3535,7 +3289,7 @@ idea is to use the relative frequency of \( \widehat{\theta}^* \)
But
@@ -3556,7 +3310,7 @@ frequency of the observation \( X_i \), just draw the values
The independent bootstrap works like this:
@@ -3581,7 +3335,7 @@ example, if you are interested in estimating the variance of \( \widehat
The following code starts with a Gaussian distribution with mean value
@@ -3645,7 +3399,7 @@ plt.show()
The code here uses Ridge regression with cross-validation (CV) resampling and \( k \)-fold CV in order to fit a specific polynomial.
@@ -3745,7 +3499,7 @@ plt.show()
We will discuss the bias-variance tradeoff in the context of
@@ -3811,7 +3565,7 @@ that is the rewriting in terms of the so-called bias, the variance of the model
@@ -3873,7 +3627,7 @@ plt.show()
@@ -3927,7 +3681,7 @@ plt.show()
-
The bias-variance tradeoff summarizes the fundamental tension in
@@ -3961,7 +3715,7 @@ flexible statistical methods have higher variance.
@@ -4040,6 +3794,248 @@ plt.show()
+The one-dimensional Ising model with nearest neighbor interaction, no
+external field and a constant coupling constant \( J \) is given by
+
+$$
+\begin{align}
+ H = -J \sum_{k}^L s_k s_{k + 1},
+\label{_auto11}
+\end{align}
+$$
+
+
+where \( s_i \in \{-1, 1\} \) and \( s_{N + 1} = s_1 \). The number of spins
+in the system is determined by \( L \). For the one-dimensional system
+there is no phase transition.
+
+
+We will look at a system of \( L = 40 \) spins with a coupling constant of
+\( J = 1 \). To get enough training data we will generate 10000 states
+with their respective energies.
+
+
+
+
+
+Here we use ordinary least squares
+regression to predict the energy for the nearest neighbor
+one-dimensional Ising model on a ring, i.e., the endpoints wrap
+around. We will use linear regression to fit a value for
+the coupling constant to achieve this.
+
+
+
+A more general form for the one-dimensional Ising model is
+
+$$
+\begin{align}
+ H = - \sum_j^L \sum_k^L s_j s_k J_{jk}.
+\label{_auto12}
+\end{align}
+$$
+
+
+Here we allow for interactions beyond the nearest neighbors and a state dependent
+coupling constant. This latter expression can be formulated as
+a matrix-product
+$$
+\begin{align}
+ \boldsymbol{H} = \boldsymbol{X} J,
+\label{_auto13}
+\end{align}
+$$
+
+
+where \( X_{jk} = s_j s_k \) and \( J \) is a matrix which consists of the
+elements \( -J_{jk} \). This form of writing the energy fits perfectly
+with the form utilized in linear regression, that is
+
+$$
+\begin{align}
+ \boldsymbol{y} = \boldsymbol{X}\boldsymbol{\beta} + \boldsymbol{\epsilon},
+\label{_auto14}
+\end{align}
+$$
+
+
+We split the data in training and test data as discussed in the previous example
+
+
+
+
+
+
+In the ordinary least squares method we choose the cost function
+
+$$
+\begin{align}
+ C(\boldsymbol{X}, \boldsymbol{\beta})= \frac{1}{n}\left\{(\boldsymbol{X}\boldsymbol{\beta} - \boldsymbol{y})^T(\boldsymbol{X}\boldsymbol{\beta} - \boldsymbol{y})\right\}.
+\label{_auto15}
+\end{align}
+$$
+
+
+We then find the extremal point of \( C \) by taking the derivative with respect to \( \boldsymbol{\beta} \) as discussed above.
+This yields the expression for \( \boldsymbol{\beta} \) to be
+
+$$
+ \boldsymbol{\beta} = \frac{\boldsymbol{X}^T \boldsymbol{y}}{\boldsymbol{X}^T \boldsymbol{X}},
+$$
+
+
+which immediately imposes some requirements on \( \boldsymbol{X} \) as there must exist
+an inverse of \( \boldsymbol{X}^T \boldsymbol{X} \). If the expression we are modeling contains an
+intercept, i.e., a constant term, we must make sure that the
+first column of \( \boldsymbol{X} \) consists of \( 1 \). We do this here
+
+
+
+
+
+
+
+
+
+Doing the inversion directly turns out to be a bad idea since the matrix
+\( \boldsymbol{X}^T\boldsymbol{X} \) is singular. An alternative approach is to use the singular
+value decomposition. Using the definition of the Moore-Penrose
+pseudoinverse we can write the equation for \( \boldsymbol{\beta} \) as
+
+$$
+ \boldsymbol{\beta} = \boldsymbol{X}^{+}\boldsymbol{y},
+$$
+
+
+where the pseudoinverse of \( \boldsymbol{X} \) is given by
+
+$$
+ \boldsymbol{X}^{+} = \frac{\boldsymbol{X}^T}{\boldsymbol{X}^T\boldsymbol{X}}.
+$$
+
+
+Using singular value decomposition we can decompose the matrix \( \boldsymbol{X} = \boldsymbol{U}\boldsymbol{\Sigma} \boldsymbol{V}^T \),
+where \( \boldsymbol{U} \) and \( \boldsymbol{V} \) are orthogonal(unitary) matrices and \( \boldsymbol{\Sigma} \) contains the singular values (more details below).
+where \( X^{+} = V\Sigma^{+} U^T \). This reduces the equation for
+\( \omega \) to
+$$
+\begin{align}
+ \boldsymbol{\beta} = \boldsymbol{V}\boldsymbol{\Sigma}^{+} \boldsymbol{U}^T \boldsymbol{y}.
+\label{_auto16}
+\end{align}
+$$
+
+
+Note that solving this equation by actually doing the pseudoinverse
+(which is what we will do) is not a good idea as this operation scales
+as \( \mathcal{O}(n^3) \), where \( n \) is the number of elements in a
+general matrix. Instead, doing \( QR \)-factorization and solving the
+linear system as an equation would reduce this down to
+\( \mathcal{O}(n^2) \) operations.
+
+
+
+
+
+
+
+
+When extracting the \( J \)-matrix we need to make sure that we remove the intercept, as is done here
+
+
+
+
+
+A way of looking at the coefficients in \( J \) is to plot the matrices as images.
+
+
+
+
+
+It is interesting to note that OLS
+considers both \( J_{j, j + 1} = -0.5 \) and \( J_{j, j - 1} = -0.5 \) as
+valid matrix elements for \( J \).
+In our discussion below on hyperparameters and Ridge and Lasso regression we will see that
+this problem can be removed, partly and only with Lasso regression.
+
+
+In this case our matrix inversion was actually possible. The obvious question now is what is the mathematics behind the SVD?
+
+
+
diff --git a/doc/pub/Regression/html/Regression.html b/doc/pub/Regression/html/Regression.html
index 5a5c74677..b03683500 100644
--- a/doc/pub/Regression/html/Regression.html
+++ b/doc/pub/Regression/html/Regression.html
@@ -136,112 +136,112 @@ div { text-align: justify; text-justify: inter-word; }
('The Boston housing data example', 2, None, '___sec27'),
('Housing data, the code', 2, None, '___sec28'),
('The singular value decomposition', 2, None, '___sec29'),
- ('The Ising model', 2, None, '___sec30'),
- ('Reformulating the problem to suit regression',
- 2,
- None,
- '___sec31'),
- ('Linear regression', 2, None, '___sec32'),
- ('Singular Value decomposition', 2, None, '___sec33'),
- ('Linear Regression Problems', 2, None, '___sec34'),
- ('Fixing the singularity', 2, None, '___sec35'),
- ('Basic math of the SVD', 2, None, '___sec36'),
- ('The SVD, a Fantastic Algorithm', 2, None, '___sec37'),
- ('Another Example', 2, None, '___sec38'),
- ('Economy-size SVD', 2, None, '___sec39'),
- ('Mathematical Properties', 2, None, '___sec40'),
- ('Ridge and LASSO Regression', 2, None, '___sec41'),
- ('More on Ridge Regression', 2, None, '___sec42'),
- ('Interpreting the Ridge results', 2, None, '___sec43'),
- ('More interpretations', 2, None, '___sec44'),
- ('Where are we going?', 2, None, '___sec45'),
- ('Resampling methods', 2, None, '___sec46'),
+ ('Linear Regression Problems', 2, None, '___sec30'),
+ ('Fixing the singularity', 2, None, '___sec31'),
+ ('Basic math of the SVD', 2, None, '___sec32'),
+ ('The SVD, a Fantastic Algorithm', 2, None, '___sec33'),
+ ('Another Example', 2, None, '___sec34'),
+ ('Economy-size SVD', 2, None, '___sec35'),
+ ('Mathematical Properties', 2, None, '___sec36'),
+ ('Ridge and LASSO Regression', 2, None, '___sec37'),
+ ('More on Ridge Regression', 2, None, '___sec38'),
+ ('Interpreting the Ridge results', 2, None, '___sec39'),
+ ('More interpretations', 2, None, '___sec40'),
+ ('Where are we going?', 2, None, '___sec41'),
+ ('Resampling methods', 2, None, '___sec42'),
('Resampling approaches can be computationally expensive',
2,
None,
- '___sec47'),
- ('Why resampling methods ?', 2, None, '___sec48'),
- ('Statistical analysis', 2, None, '___sec49'),
- ('Statistics', 2, None, '___sec50'),
- ('Statistics, moments', 2, None, '___sec51'),
- ('Statistics, central moments', 2, None, '___sec52'),
- ('Statistics, covariance', 2, None, '___sec53'),
- ('Statistics, more covariance', 2, None, '___sec54'),
- ('Covariance example', 2, None, '___sec55'),
- ('Covariance in numpy', 2, None, '___sec56'),
- ('Statistics, independent variables', 2, None, '___sec57'),
- ('Statistics, more variance', 2, None, '___sec58'),
- ('Statistics and stochastic processes', 2, None, '___sec59'),
- ('Statistics and sample variables', 2, None, '___sec60'),
+ '___sec43'),
+ ('Why resampling methods ?', 2, None, '___sec44'),
+ ('Statistical analysis', 2, None, '___sec45'),
+ ('Statistics', 2, None, '___sec46'),
+ ('Statistics, moments', 2, None, '___sec47'),
+ ('Statistics, central moments', 2, None, '___sec48'),
+ ('Statistics, covariance', 2, None, '___sec49'),
+ ('Statistics, more covariance', 2, None, '___sec50'),
+ ('Covariance example', 2, None, '___sec51'),
+ ('Covariance in numpy', 2, None, '___sec52'),
+ ('Statistics, independent variables', 2, None, '___sec53'),
+ ('Statistics, more variance', 2, None, '___sec54'),
+ ('Statistics and stochastic processes', 2, None, '___sec55'),
+ ('Statistics and sample variables', 2, None, '___sec56'),
('Statistics, sample variance and covariance',
2,
None,
- '___sec61'),
- ('Statistics, law of large numbers', 2, None, '___sec62'),
- ('Statistics, more on sample error', 2, None, '___sec63'),
- ('Statistics', 2, None, '___sec64'),
- ('Statistics, central limit theorem', 2, None, '___sec65'),
- ('Statistics, more technicalities', 2, None, '___sec66'),
- ('Statistics', 2, None, '___sec67'),
- ('Statistics and sample variance', 2, None, '___sec68'),
- ('Statistics, uncorrelated results', 2, None, '___sec69'),
- ('Statistics, computations', 2, None, '___sec70'),
+ '___sec57'),
+ ('Statistics, law of large numbers', 2, None, '___sec58'),
+ ('Statistics, more on sample error', 2, None, '___sec59'),
+ ('Statistics', 2, None, '___sec60'),
+ ('Statistics, central limit theorem', 2, None, '___sec61'),
+ ('Statistics, more technicalities', 2, None, '___sec62'),
+ ('Statistics', 2, None, '___sec63'),
+ ('Statistics and sample variance', 2, None, '___sec64'),
+ ('Statistics, uncorrelated results', 2, None, '___sec65'),
+ ('Statistics, computations', 2, None, '___sec66'),
('Statistics, more on computations of errors',
2,
None,
- '___sec71'),
- ('Statistics, wrapping up 1', 2, None, '___sec72'),
- ('Statistics, final expression', 2, None, '___sec73'),
+ '___sec67'),
+ ('Statistics, wrapping up 1', 2, None, '___sec68'),
+ ('Statistics, final expression', 2, None, '___sec69'),
('Statistics, effective number of correlations',
2,
None,
- '___sec74'),
+ '___sec70'),
('Linking the regression analysis with a statistical '
'interpretation',
2,
None,
- '___sec75'),
- ('Assumptions made', 2, None, '___sec76'),
- ('Expectation value and variance', 2, None, '___sec77'),
+ '___sec71'),
+ ('Assumptions made', 2, None, '___sec72'),
+ ('Expectation value and variance', 2, None, '___sec73'),
('Expectation value and variance for $\\boldsymbol{\\beta}$',
2,
None,
- '___sec78'),
- ('Cross-validation', 2, None, '___sec79'),
- ('Computationally expensive', 2, None, '___sec80'),
- ('Various steps in cross-validation', 2, None, '___sec81'),
+ '___sec74'),
+ ('Cross-validation', 2, None, '___sec75'),
+ ('Computationally expensive', 2, None, '___sec76'),
+ ('Various steps in cross-validation', 2, None, '___sec77'),
('How to set up the cross-validation for Ridge and/or Lasso',
2,
None,
- '___sec82'),
+ '___sec78'),
('Resampling methods: Jackknife and Bootstrap',
2,
None,
- '___sec83'),
- ('Resampling methods: Jackknife', 2, None, '___sec84'),
- ('Jackknife code example', 2, None, '___sec85'),
- ('Resampling methods: Bootstrap', 2, None, '___sec86'),
- ('Resampling methods: Bootstrap background', 2, None, '___sec87'),
+ '___sec79'),
+ ('Resampling methods: Jackknife', 2, None, '___sec80'),
+ ('Jackknife code example', 2, None, '___sec81'),
+ ('Resampling methods: Bootstrap', 2, None, '___sec82'),
+ ('Resampling methods: Bootstrap background', 2, None, '___sec83'),
('Resampling methods: More Bootstrap background',
2,
None,
- '___sec88'),
- ('Resampling methods: Bootstrap approach', 2, None, '___sec89'),
- ('Resampling methods: Bootstrap steps', 2, None, '___sec90'),
- ('Code example for the Bootstrap method', 2, None, '___sec91'),
+ '___sec84'),
+ ('Resampling methods: Bootstrap approach', 2, None, '___sec85'),
+ ('Resampling methods: Bootstrap steps', 2, None, '___sec86'),
+ ('Code example for the Bootstrap method', 2, None, '___sec87'),
('Code Example for Cross-validation and $k$-fold '
'Cross-validation',
2,
None,
- '___sec92'),
- ('The bias-variance tradeoff', 2, None, '___sec93'),
- ('Example code for Bias-Variance tradeoff', 2, None, '___sec94'),
- ('Understanding what happens', 2, None, '___sec95'),
- ('Summing up', 2, None, '___sec96'),
+ '___sec88'),
+ ('The bias-variance tradeoff', 2, None, '___sec89'),
+ ('Example code for Bias-Variance tradeoff', 2, None, '___sec90'),
+ ('Understanding what happens', 2, None, '___sec91'),
+ ('Summing up', 2, None, '___sec92'),
("Another Example rom Scikit-Learn's Repository",
2,
None,
- '___sec97'),
+ '___sec93'),
+ ('The Ising model', 2, None, '___sec94'),
+ ('Reformulating the problem to suit regression',
+ 2,
+ None,
+ '___sec95'),
+ ('Linear regression', 2, None, '___sec96'),
+ ('Singular Value decomposition', 2, None, '___sec97'),
('The one-dimensional Ising model', 2, None, '___sec98'),
('Ridge regression', 2, None, '___sec99'),
('LASSO regression', 2, None, '___sec100'),
@@ -294,7 +294,7 @@ MathJax.Hub.Config({
-
This may
@@ -1580,255 +1579,10 @@ inversion algorithm. Thereafter we dive into the math of the SVD.
-
-
-The one-dimensional Ising model with nearest neighbor interaction, no
-external field and a constant coupling constant \( J \) is given by
-
-$$
-\begin{align}
- H = -J \sum_{k}^L s_k s_{k + 1},
-\label{_auto1}
-\end{align}
-$$
-
-
-where \( s_i \in \{-1, 1\} \) and \( s_{N + 1} = s_1 \). The number of spins
-in the system is determined by \( L \). For the one-dimensional system
-there is no phase transition.
-
-
-We will look at a system of \( L = 40 \) spins with a coupling constant of
-\( J = 1 \). To get enough training data we will generate 10000 states
-with their respective energies.
-
-
-
-
-
-Here we use ordinary least squares
-regression to predict the energy for the nearest neighbor
-one-dimensional Ising model on a ring, i.e., the endpoints wrap
-around. We will use linear regression to fit a value for
-the coupling constant to achieve this.
-
-
-
-A more general form for the one-dimensional Ising model is
-
-$$
-\begin{align}
- H = - \sum_j^L \sum_k^L s_j s_k J_{jk}.
-\label{_auto2}
-\end{align}
-$$
-
-
-Here we allow for interactions beyond the nearest neighbors and a state dependent
-coupling constant. This latter expression can be formulated as
-a matrix-product
-$$
-\begin{align}
- \boldsymbol{H} = \boldsymbol{X} J,
-\label{_auto3}
-\end{align}
-$$
-
-
-where \( X_{jk} = s_j s_k \) and \( J \) is a matrix which consists of the
-elements \( -J_{jk} \). This form of writing the energy fits perfectly
-with the form utilized in linear regression, that is
-
-$$
-\begin{align}
- \boldsymbol{y} = \boldsymbol{X}\boldsymbol{\beta} + \boldsymbol{\epsilon},
-\label{_auto4}
-\end{align}
-$$
-
-
-We split the data in training and test data as discussed in the previous example
-
-
-
-
-
-
-In the ordinary least squares method we choose the cost function
-
-$$
-\begin{align}
- C(\boldsymbol{X}, \boldsymbol{\beta})= \frac{1}{n}\left\{(\boldsymbol{X}\boldsymbol{\beta} - \boldsymbol{y})^T(\boldsymbol{X}\boldsymbol{\beta} - \boldsymbol{y})\right\}.
-\label{_auto5}
-\end{align}
-$$
-
-
-We then find the extremal point of \( C \) by taking the derivative with respect to \( \boldsymbol{\beta} \) as discussed above.
-This yields the expression for \( \boldsymbol{\beta} \) to be
-
-$$
- \boldsymbol{\beta} = \frac{\boldsymbol{X}^T \boldsymbol{y}}{\boldsymbol{X}^T \boldsymbol{X}},
-$$
-
-
-which immediately imposes some requirements on \( \boldsymbol{X} \) as there must exist
-an inverse of \( \boldsymbol{X}^T \boldsymbol{X} \). If the expression we are modeling contains an
-intercept, i.e., a constant term, we must make sure that the
-first column of \( \boldsymbol{X} \) consists of \( 1 \). We do this here
-
-
-
-
-
-
-
-
-
-Doing the inversion directly turns out to be a bad idea since the matrix
-\( \boldsymbol{X}^T\boldsymbol{X} \) is singular. An alternative approach is to use the singular
-value decomposition. Using the definition of the Moore-Penrose
-pseudoinverse we can write the equation for \( \boldsymbol{\beta} \) as
-
-$$
- \boldsymbol{\beta} = \boldsymbol{X}^{+}\boldsymbol{y},
-$$
-
-
-where the pseudoinverse of \( \boldsymbol{X} \) is given by
-
-$$
- \boldsymbol{X}^{+} = \frac{\boldsymbol{X}^T}{\boldsymbol{X}^T\boldsymbol{X}}.
-$$
-
-
-Using singular value decomposition we can decompose the matrix \( \boldsymbol{X} = \boldsymbol{U}\boldsymbol{\Sigma} \boldsymbol{V}^T \),
-where \( \boldsymbol{U} \) and \( \boldsymbol{V} \) are orthogonal(unitary) matrices and \( \boldsymbol{\Sigma} \) contains the singular values (more details below).
-where \( X^{+} = V\Sigma^{+} U^T \). This reduces the equation for
-\( \omega \) to
-$$
-\begin{align}
- \boldsymbol{\beta} = \boldsymbol{V}\boldsymbol{\Sigma}^{+} \boldsymbol{U}^T \boldsymbol{y}.
-\label{_auto6}
-\end{align}
-$$
-
-
-Note that solving this equation by actually doing the pseudoinverse
-(which is what we will do) is not a good idea as this operation scales
-as \( \mathcal{O}(n^3) \), where \( n \) is the number of elements in a
-general matrix. Instead, doing \( QR \)-factorization and solving the
-linear system as an equation would reduce this down to
-\( \mathcal{O}(n^2) \) operations.
-
-
-
-
-
-
-
-
-When extracting the \( J \)-matrix we need to make sure that we remove the intercept, as is done here
-
-
-
-
-
-A way of looking at the coefficients in \( J \) is to plot the matrices as images.
-
-
-
-
-
-It is interesting to note that OLS
-considers both \( J_{j, j + 1} = -0.5 \) and \( J_{j, j - 1} = -0.5 \) as
-valid matrix elements for \( J \).
-In our discussion below on hyperparameters and Ridge and Lasso regression we will see that
-this problem can be removed, partly and only with Lasso regression.
-
-
-In this case our matrix inversion was actually possible. The obvious question now is what is the mathematics behind the SVD?
-
-
-
One of the typical problems we encounter with linear regression, in particular
@@ -1861,7 +1615,7 @@ of linearly independent columns. In this particular case the matrix has rank 2.
Super-collinearity of an \( (n \times p) \)-dimensional design matrix \( \mathbf{X} \) implies
-that the inverse of the matrix \( \boldsymbol{X}^T\boldsymbol{x} \) (the matrix we need to invert to solve the linear regression equations) is non-invertible. If we have a square matrix that does not have an inverse, we say this matrix singular. The example here demonstrates this
+that the inverse of the matrix \( \boldsymbol{X}^T\boldsymbol{X} \) (the matrix we need to invert to solve the linear regression equations) is non-invertible. If we have a square matrix that does not have an inverse, we say this matrix singular. The example here demonstrates this
$$
\begin{align*}
\boldsymbol{X} & = \left[
@@ -1879,14 +1633,14 @@ This is equivalent to saying that the matrix \( \boldsymbol{X} \) has at least a
If our design matrix \( \boldsymbol{X} \) which enters the linear regression problem
$$
\begin{align}
\boldsymbol{\beta} & = (\boldsymbol{X}^{T} \boldsymbol{X})^{-1} \boldsymbol{X}^{T} \boldsymbol{y},
-\label{_auto7}
+\label{_auto1}
\end{align}
$$
@@ -1907,7 +1661,7 @@ where \( \boldsymbol{I} \) is the identity matrix. When we discuss Ridge
The Ising model
-
-
-$$
-\begin{align}
- H = -J \sum_{k}^L s_k s_{k + 1},
-\tag{1}
-\end{align}
-$$
-
-
-import numpy as np
-import matplotlib.pyplot as plt
-from mpl_toolkits.axes_grid1 import make_axes_locatable
-import seaborn as sns
-import scipy.linalg as scl
-from sklearn.model_selection import train_test_split
-import tqdm
-sns.set(color_codes=True)
-cmap_args=dict(vmin=-1., vmax=1., cmap='seismic')
-
-L = 40
-n = int(1e4)
-
-spins = np.random.choice([-1, 1], size=(n, L))
-J = 1.0
-
-energies = np.zeros(n)
-
-for i in range(n):
- energies[i] = - J * np.dot(spins[i], np.roll(spins[i], 1))
-
Reformulating the problem to suit regression
-
-
-$$
-\begin{align}
- H = - \sum_j^L \sum_k^L s_j s_k J_{jk}.
-\tag{2}
-\end{align}
-$$
-
-
-
-$$
-\begin{align}
- \boldsymbol{H} = \boldsymbol{X} J,
-\tag{3}
-\end{align}
-$$
-
-
-
-$$
-\begin{align}
- \boldsymbol{y} = \boldsymbol{X}\boldsymbol{\beta} + \boldsymbol{\epsilon},
-\tag{4}
-\end{align}
-$$
-
-
-X = np.zeros((n, L ** 2))
-for i in range(n):
- X[i] = np.outer(spins[i], spins[i]).ravel()
-y = energies
-X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)
-
Linear regression
-
-
-$$
-\begin{align}
- C(\boldsymbol{X}, \boldsymbol{\beta})= \frac{1}{n}\left\{(\boldsymbol{X}\boldsymbol{\beta} - \boldsymbol{y})^T(\boldsymbol{X}\boldsymbol{\beta} - \boldsymbol{y})\right\}.
-\tag{5}
-\end{align}
-$$
-
-
-
-$$
- \boldsymbol{\beta} = \frac{\boldsymbol{X}^T \boldsymbol{y}}{\boldsymbol{X}^T \boldsymbol{X}},
-$$
-
-
-X_train_own = np.concatenate(
- (np.ones(len(X_train))[:, np.newaxis], X_train),
- axis=1
-)
-X_test_own = np.concatenate(
- (np.ones(len(X_test))[:, np.newaxis], X_test),
- axis=1
-)
-
def ols_inv(x: np.ndarray, y: np.ndarray) -> np.ndarray:
- return scl.inv(x.T @ x) @ (x.T @ y)
-beta = ols_inv(X_train_own, y_train)
-
Singular Value decomposition
-
-
-$$
- \boldsymbol{\beta} = \boldsymbol{X}^{+}\boldsymbol{y},
-$$
-
-
-
-$$
- \boldsymbol{X}^{+} = \frac{\boldsymbol{X}^T}{\boldsymbol{X}^T\boldsymbol{X}}.
-$$
-
-
-
-$$
-\begin{align}
- \boldsymbol{\beta} = \boldsymbol{V}\boldsymbol{\Sigma}^{+} \boldsymbol{U}^T \boldsymbol{y}.
-\tag{6}
-\end{align}
-$$
-
-
-def ols_svd(x: np.ndarray, y: np.ndarray) -> np.ndarray:
- u, s, v = scl.svd(x)
- return v.T @ scl.pinv(scl.diagsvd(s, u.shape[0], v.shape[0])) @ u.T @ y
-
beta = ols_svd(X_train_own,y_train)
-
J = beta[1:].reshape(L, L)
-fig = plt.figure(figsize=(20, 14))
-im = plt.imshow(J, **cmap_args)
-plt.title("OLS", fontsize=18)
-plt.xticks(fontsize=18)
-plt.yticks(fontsize=18)
-cb = fig.colorbar(im)
-cb.ax.set_yticklabels(cb.ax.get_yticklabels(), fontsize=18)
-plt.show()
-
Linear Regression Problems
+Linear Regression Problems
$$
\begin{align*}
@@ -1833,7 +1567,7 @@ This is equivalent to saying that the matrix \( \boldsymbol{X} \) has at least a
Fixing the singularity
+Fixing the singularity
@@ -1865,7 +1599,7 @@ where \( \boldsymbol{I} \) is the identity matrix. When we discuss Ridge
Basic math of the SVD
+Basic math of the SVD
The SVD, a Fantastic Algorithm
Another Example
+Another Example
Economy-size SVD
+Economy-size SVD
Mathematical Properties
+Mathematical Properties
Ridge and LASSO Regression
+Ridge and LASSO Regression
More on Ridge Regression
+More on Ridge Regression
Interpreting the Ridge results
+Interpreting the Ridge results
More interpretations
+More interpretations
Where are we going?
+Where are we going?
Resampling methods
+Resampling methods
Resampling approaches can be computationally expensive
+Resampling approaches can be computationally expensive
Why resampling methods ?
+Why resampling methods ?
@@ -2364,7 +2098,7 @@ bootstrap is widely used.
Statistical analysis
+Statistical analysis
@@ -2384,7 +2118,7 @@ bootstrap is widely used.
Statistics
+Statistics
Statistics, moments
+Statistics, moments
Statistics, central moments
+Statistics, central moments
@@ -2485,7 +2219,7 @@ qualitatively as the spread of \( p \) around its mean.
Statistics, covariance
+Statistics, covariance
@@ -2519,7 +2253,7 @@ $$
Statistics, more covariance
+Statistics, more covariance
@@ -2552,7 +2286,7 @@ $$
Covariance example
+Covariance example
Covariance in numpy
+Covariance in numpy
Statistics, independent variables
+Statistics, independent variables
Statistics, more variance
+Statistics, more variance
@@ -2713,7 +2447,7 @@ value of a set of measurements.
Statistics and stochastic processes
+Statistics and stochastic processes
Statistics and sample variables
+Statistics and sample variables
Statistics, sample variance and covariance
+Statistics, sample variance and covariance
Statistics, law of large numbers
+Statistics, law of large numbers
Statistics, more on sample error
+Statistics, more on sample error
Statistics
+Statistics
Statistics, central limit theorem
+Statistics, central limit theorem
@@ -2897,7 +2631,7 @@ $$
Statistics, more technicalities
+Statistics, more technicalities
@@ -2927,7 +2661,7 @@ estimate of the PDF of each of the \( X_i \), estimating all properties of
Statistics
+Statistics
Using \( \bar x \) in place of \( \mu_{X_i}^{\phantom X} \) we can give an
-estimate of the covariance in Eq. (20)
+estimate of the covariance in Eq. (14)
$$
\mathrm{cov}(X_i, X_j) = \langle (x_i-\langle x_i\rangle)(x_j-\langle x_j\rangle)\rangle
@@ -2959,7 +2693,7 @@ $$
Statistics and sample variance
+Statistics and sample variance
@@ -2991,7 +2725,7 @@ $$
&=\frac{1}{n^2}\sum_{ij} \mathrm{cov}(X_i, X_j) \nonumber \\
&\approx&\frac{1}{n^2}\sum_{ij}\frac{1}{n}\mathrm{cov}(x) =\frac{1}{n^2}n^2\frac{1}{n}\mathrm{cov}(x)\nonumber\\
&=\frac{1}{n}\mathrm{cov}(x)
-\tag{22}
+\tag{16}
\end{align}
$$
@@ -3003,7 +2737,7 @@ measurements in the sample.
Statistics, uncorrelated results
+Statistics, uncorrelated results
-where in the second step we have used Eq. (21).
+where in the second step we have used Eq. (15).
The error of the sample is then just its standard deviation divided by
the square root of the number of measurements the sample contains.
This is a very useful formula which is easy to compute. It acts as a
@@ -3039,12 +2773,12 @@ cannot overlook the always present correlations.
Statistics, computations
+Statistics, computations
$$
@@ -3057,13 +2791,13 @@ which equals
$$
\begin{equation}
\frac{1}{n^2}\sum_{k=1}^n (x_k - \bar x_n)^2 +\frac{2}{n^2}\sum_{k < l} (x_k - \bar x_n)(x_l - \bar x_n)
-\tag{24}
+\tag{18}
\end{equation}
$$
The first term is the same as the error in the uncorrelated case,
-Eq. (23). This means that the second
+Eq. (17). This means that the second
term accounts for the error correction due to correlation between the
measurements. For uncorrelated measurements this second term is zero.
Statistics, more on computations of errors
+Statistics, more on computations of errors
Statistics, wrapping up 1
+Statistics, wrapping up 1
Statistics, final expression
+Statistics, final expression
$$
@@ -3146,7 +2880,7 @@ $$
\frac{f_d}{\mathrm{var}(x)}\nonumber\\ &=&
\left(1+2\sum_{d=1}^{n-1}\kappa_d\right)\frac{1}{n}\mathrm{var}(x)\nonumber\\
&=\frac{\tau}{n}\cdot\mathrm{var}(x)
-\tag{25}
+\tag{19}
\end{align}
$$
@@ -3159,7 +2893,7 @@ correction factor the autocorrelation time:
$$
\begin{equation}
\tau = 1+2\sum_{d=1}^{n-1}\kappa_d
-\tag{26}
+\tag{20}
\end{equation}
$$
@@ -3168,13 +2902,13 @@ $$
Statistics, effective number of correlations
+Statistics, effective number of correlations
@@ -3194,7 +2928,7 @@ measurements is very large.
Linking the regression analysis with a statistical interpretation
+Linking the regression analysis with a statistical interpretation
Assumptions made
+Assumptions made
Expectation value and variance
+Expectation value and variance
Expectation value and variance for \( \boldsymbol{\beta} \)
+Expectation value and variance for \( \boldsymbol{\beta} \)
Cross-validation
+Cross-validation
Computationally expensive
+Computationally expensive
Various steps in cross-validation
+Various steps in cross-validation
How to set up the cross-validation for Ridge and/or Lasso
+How to set up the cross-validation for Ridge and/or Lasso
Resampling methods: Jackknife and Bootstrap
+Resampling methods: Jackknife and Bootstrap
Resampling methods: Jackknife
+Resampling methods: Jackknife
Jackknife code example
+Jackknife code example
Resampling methods: Bootstrap
+Resampling methods: Bootstrap
Resampling methods: Bootstrap background
+Resampling methods: Bootstrap background
Resampling methods: More Bootstrap background
+Resampling methods: More Bootstrap background
Resampling methods: Bootstrap approach
+Resampling methods: Bootstrap approach
Resampling methods: Bootstrap steps
+Resampling methods: Bootstrap steps
Code example for the Bootstrap method
+Code example for the Bootstrap method
Code Example for Cross-validation and \( k \)-fold Cross-validation
+Code Example for Cross-validation and \( k \)-fold Cross-validation
The bias-variance tradeoff
+The bias-variance tradeoff
Example code for Bias-Variance tradeoff
+Example code for Bias-Variance tradeoff
Understanding what happens
+Understanding what happens
Summing up
+Summing up
Another Example rom Scikit-Learn's Repository
+Another Example rom Scikit-Learn's Repository
The Ising model
+
+
+$$
+\begin{align}
+ H = -J \sum_{k}^L s_k s_{k + 1},
+\tag{21}
+\end{align}
+$$
+
+
+import numpy as np
+import matplotlib.pyplot as plt
+from mpl_toolkits.axes_grid1 import make_axes_locatable
+import seaborn as sns
+import scipy.linalg as scl
+from sklearn.model_selection import train_test_split
+import tqdm
+sns.set(color_codes=True)
+cmap_args=dict(vmin=-1., vmax=1., cmap='seismic')
+
+L = 40
+n = int(1e4)
+
+spins = np.random.choice([-1, 1], size=(n, L))
+J = 1.0
+
+energies = np.zeros(n)
+
+for i in range(n):
+ energies[i] = - J * np.dot(spins[i], np.roll(spins[i], 1))
+
Reformulating the problem to suit regression
+
+
+$$
+\begin{align}
+ H = - \sum_j^L \sum_k^L s_j s_k J_{jk}.
+\tag{22}
+\end{align}
+$$
+
+
+
+$$
+\begin{align}
+ \boldsymbol{H} = \boldsymbol{X} J,
+\tag{23}
+\end{align}
+$$
+
+
+
+$$
+\begin{align}
+ \boldsymbol{y} = \boldsymbol{X}\boldsymbol{\beta} + \boldsymbol{\epsilon},
+\tag{24}
+\end{align}
+$$
+
+
+X = np.zeros((n, L ** 2))
+for i in range(n):
+ X[i] = np.outer(spins[i], spins[i]).ravel()
+y = energies
+X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)
+
Linear regression
+
+
+$$
+\begin{align}
+ C(\boldsymbol{X}, \boldsymbol{\beta})= \frac{1}{n}\left\{(\boldsymbol{X}\boldsymbol{\beta} - \boldsymbol{y})^T(\boldsymbol{X}\boldsymbol{\beta} - \boldsymbol{y})\right\}.
+\tag{25}
+\end{align}
+$$
+
+
+
+$$
+ \boldsymbol{\beta} = \frac{\boldsymbol{X}^T \boldsymbol{y}}{\boldsymbol{X}^T \boldsymbol{X}},
+$$
+
+
+X_train_own = np.concatenate(
+ (np.ones(len(X_train))[:, np.newaxis], X_train),
+ axis=1
+)
+X_test_own = np.concatenate(
+ (np.ones(len(X_test))[:, np.newaxis], X_test),
+ axis=1
+)
+
def ols_inv(x: np.ndarray, y: np.ndarray) -> np.ndarray:
+ return scl.inv(x.T @ x) @ (x.T @ y)
+beta = ols_inv(X_train_own, y_train)
+
Singular Value decomposition
+
+
+$$
+ \boldsymbol{\beta} = \boldsymbol{X}^{+}\boldsymbol{y},
+$$
+
+
+
+$$
+ \boldsymbol{X}^{+} = \frac{\boldsymbol{X}^T}{\boldsymbol{X}^T\boldsymbol{X}}.
+$$
+
+
+
+$$
+\begin{align}
+ \boldsymbol{\beta} = \boldsymbol{V}\boldsymbol{\Sigma}^{+} \boldsymbol{U}^T \boldsymbol{y}.
+\tag{26}
+\end{align}
+$$
+
+
+def ols_svd(x: np.ndarray, y: np.ndarray) -> np.ndarray:
+ u, s, v = scl.svd(x)
+ return v.T @ scl.pinv(scl.diagsvd(s, u.shape[0], v.shape[0])) @ u.T @ y
+
beta = ols_svd(X_train_own,y_train)
+
J = beta[1:].reshape(L, L)
+fig = plt.figure(figsize=(20, 14))
+im = plt.imshow(J, **cmap_args)
+plt.title("OLS", fontsize=18)
+plt.xticks(fontsize=18)
+plt.yticks(fontsize=18)
+cb = fig.colorbar(im)
+cb.ax.set_yticklabels(cb.ax.get_yticklabels(), fontsize=18)
+plt.show()
+
The one-dimensional Ising model
diff --git a/doc/pub/Regression/html/Regression-solarized.html b/doc/pub/Regression/html/Regression-solarized.html
index 29262f9ae..0fd8fbde6 100644
--- a/doc/pub/Regression/html/Regression-solarized.html
+++ b/doc/pub/Regression/html/Regression-solarized.html
@@ -131,112 +131,112 @@ div { text-align: justify; text-justify: inter-word; }
('The Boston housing data example', 2, None, '___sec27'),
('Housing data, the code', 2, None, '___sec28'),
('The singular value decomposition', 2, None, '___sec29'),
- ('The Ising model', 2, None, '___sec30'),
- ('Reformulating the problem to suit regression',
- 2,
- None,
- '___sec31'),
- ('Linear regression', 2, None, '___sec32'),
- ('Singular Value decomposition', 2, None, '___sec33'),
- ('Linear Regression Problems', 2, None, '___sec34'),
- ('Fixing the singularity', 2, None, '___sec35'),
- ('Basic math of the SVD', 2, None, '___sec36'),
- ('The SVD, a Fantastic Algorithm', 2, None, '___sec37'),
- ('Another Example', 2, None, '___sec38'),
- ('Economy-size SVD', 2, None, '___sec39'),
- ('Mathematical Properties', 2, None, '___sec40'),
- ('Ridge and LASSO Regression', 2, None, '___sec41'),
- ('More on Ridge Regression', 2, None, '___sec42'),
- ('Interpreting the Ridge results', 2, None, '___sec43'),
- ('More interpretations', 2, None, '___sec44'),
- ('Where are we going?', 2, None, '___sec45'),
- ('Resampling methods', 2, None, '___sec46'),
+ ('Linear Regression Problems', 2, None, '___sec30'),
+ ('Fixing the singularity', 2, None, '___sec31'),
+ ('Basic math of the SVD', 2, None, '___sec32'),
+ ('The SVD, a Fantastic Algorithm', 2, None, '___sec33'),
+ ('Another Example', 2, None, '___sec34'),
+ ('Economy-size SVD', 2, None, '___sec35'),
+ ('Mathematical Properties', 2, None, '___sec36'),
+ ('Ridge and LASSO Regression', 2, None, '___sec37'),
+ ('More on Ridge Regression', 2, None, '___sec38'),
+ ('Interpreting the Ridge results', 2, None, '___sec39'),
+ ('More interpretations', 2, None, '___sec40'),
+ ('Where are we going?', 2, None, '___sec41'),
+ ('Resampling methods', 2, None, '___sec42'),
('Resampling approaches can be computationally expensive',
2,
None,
- '___sec47'),
- ('Why resampling methods ?', 2, None, '___sec48'),
- ('Statistical analysis', 2, None, '___sec49'),
- ('Statistics', 2, None, '___sec50'),
- ('Statistics, moments', 2, None, '___sec51'),
- ('Statistics, central moments', 2, None, '___sec52'),
- ('Statistics, covariance', 2, None, '___sec53'),
- ('Statistics, more covariance', 2, None, '___sec54'),
- ('Covariance example', 2, None, '___sec55'),
- ('Covariance in numpy', 2, None, '___sec56'),
- ('Statistics, independent variables', 2, None, '___sec57'),
- ('Statistics, more variance', 2, None, '___sec58'),
- ('Statistics and stochastic processes', 2, None, '___sec59'),
- ('Statistics and sample variables', 2, None, '___sec60'),
+ '___sec43'),
+ ('Why resampling methods ?', 2, None, '___sec44'),
+ ('Statistical analysis', 2, None, '___sec45'),
+ ('Statistics', 2, None, '___sec46'),
+ ('Statistics, moments', 2, None, '___sec47'),
+ ('Statistics, central moments', 2, None, '___sec48'),
+ ('Statistics, covariance', 2, None, '___sec49'),
+ ('Statistics, more covariance', 2, None, '___sec50'),
+ ('Covariance example', 2, None, '___sec51'),
+ ('Covariance in numpy', 2, None, '___sec52'),
+ ('Statistics, independent variables', 2, None, '___sec53'),
+ ('Statistics, more variance', 2, None, '___sec54'),
+ ('Statistics and stochastic processes', 2, None, '___sec55'),
+ ('Statistics and sample variables', 2, None, '___sec56'),
('Statistics, sample variance and covariance',
2,
None,
- '___sec61'),
- ('Statistics, law of large numbers', 2, None, '___sec62'),
- ('Statistics, more on sample error', 2, None, '___sec63'),
- ('Statistics', 2, None, '___sec64'),
- ('Statistics, central limit theorem', 2, None, '___sec65'),
- ('Statistics, more technicalities', 2, None, '___sec66'),
- ('Statistics', 2, None, '___sec67'),
- ('Statistics and sample variance', 2, None, '___sec68'),
- ('Statistics, uncorrelated results', 2, None, '___sec69'),
- ('Statistics, computations', 2, None, '___sec70'),
+ '___sec57'),
+ ('Statistics, law of large numbers', 2, None, '___sec58'),
+ ('Statistics, more on sample error', 2, None, '___sec59'),
+ ('Statistics', 2, None, '___sec60'),
+ ('Statistics, central limit theorem', 2, None, '___sec61'),
+ ('Statistics, more technicalities', 2, None, '___sec62'),
+ ('Statistics', 2, None, '___sec63'),
+ ('Statistics and sample variance', 2, None, '___sec64'),
+ ('Statistics, uncorrelated results', 2, None, '___sec65'),
+ ('Statistics, computations', 2, None, '___sec66'),
('Statistics, more on computations of errors',
2,
None,
- '___sec71'),
- ('Statistics, wrapping up 1', 2, None, '___sec72'),
- ('Statistics, final expression', 2, None, '___sec73'),
+ '___sec67'),
+ ('Statistics, wrapping up 1', 2, None, '___sec68'),
+ ('Statistics, final expression', 2, None, '___sec69'),
('Statistics, effective number of correlations',
2,
None,
- '___sec74'),
+ '___sec70'),
('Linking the regression analysis with a statistical '
'interpretation',
2,
None,
- '___sec75'),
- ('Assumptions made', 2, None, '___sec76'),
- ('Expectation value and variance', 2, None, '___sec77'),
+ '___sec71'),
+ ('Assumptions made', 2, None, '___sec72'),
+ ('Expectation value and variance', 2, None, '___sec73'),
('Expectation value and variance for $\\boldsymbol{\\beta}$',
2,
None,
- '___sec78'),
- ('Cross-validation', 2, None, '___sec79'),
- ('Computationally expensive', 2, None, '___sec80'),
- ('Various steps in cross-validation', 2, None, '___sec81'),
+ '___sec74'),
+ ('Cross-validation', 2, None, '___sec75'),
+ ('Computationally expensive', 2, None, '___sec76'),
+ ('Various steps in cross-validation', 2, None, '___sec77'),
('How to set up the cross-validation for Ridge and/or Lasso',
2,
None,
- '___sec82'),
+ '___sec78'),
('Resampling methods: Jackknife and Bootstrap',
2,
None,
- '___sec83'),
- ('Resampling methods: Jackknife', 2, None, '___sec84'),
- ('Jackknife code example', 2, None, '___sec85'),
- ('Resampling methods: Bootstrap', 2, None, '___sec86'),
- ('Resampling methods: Bootstrap background', 2, None, '___sec87'),
+ '___sec79'),
+ ('Resampling methods: Jackknife', 2, None, '___sec80'),
+ ('Jackknife code example', 2, None, '___sec81'),
+ ('Resampling methods: Bootstrap', 2, None, '___sec82'),
+ ('Resampling methods: Bootstrap background', 2, None, '___sec83'),
('Resampling methods: More Bootstrap background',
2,
None,
- '___sec88'),
- ('Resampling methods: Bootstrap approach', 2, None, '___sec89'),
- ('Resampling methods: Bootstrap steps', 2, None, '___sec90'),
- ('Code example for the Bootstrap method', 2, None, '___sec91'),
+ '___sec84'),
+ ('Resampling methods: Bootstrap approach', 2, None, '___sec85'),
+ ('Resampling methods: Bootstrap steps', 2, None, '___sec86'),
+ ('Code example for the Bootstrap method', 2, None, '___sec87'),
('Code Example for Cross-validation and $k$-fold '
'Cross-validation',
2,
None,
- '___sec92'),
- ('The bias-variance tradeoff', 2, None, '___sec93'),
- ('Example code for Bias-Variance tradeoff', 2, None, '___sec94'),
- ('Understanding what happens', 2, None, '___sec95'),
- ('Summing up', 2, None, '___sec96'),
+ '___sec88'),
+ ('The bias-variance tradeoff', 2, None, '___sec89'),
+ ('Example code for Bias-Variance tradeoff', 2, None, '___sec90'),
+ ('Understanding what happens', 2, None, '___sec91'),
+ ('Summing up', 2, None, '___sec92'),
("Another Example rom Scikit-Learn's Repository",
2,
None,
- '___sec97'),
+ '___sec93'),
+ ('The Ising model', 2, None, '___sec94'),
+ ('Reformulating the problem to suit regression',
+ 2,
+ None,
+ '___sec95'),
+ ('Linear regression', 2, None, '___sec96'),
+ ('Singular Value decomposition', 2, None, '___sec97'),
('The one-dimensional Ising model', 2, None, '___sec98'),
('Ridge regression', 2, None, '___sec99'),
('LASSO regression', 2, None, '___sec100'),
@@ -289,7 +289,7 @@ MathJax.Hub.Config({
Aug 29, 2019
Aug 30, 2019
@@ -1554,8 +1554,7 @@ The examples we have looked at so far are cases where we normally can
invert the matrix \( \boldsymbol{X}^T\boldsymbol{X} \). Using a polynomial expansion as we
did both for the masses and the fitting of the equation of state,
leads to row vectors of the design matrix which are essentially
-orthogonal due to the polynomial character of our model. Obtaining the inverse of the design matrix is then often done via a so-called LU, QR or Cholesky decomposition.
-More material to come here.
+orthogonal due to the polynomial character of our model. Obtaining the inverse of the design matrix is then often done via a so-called LU, QR or Cholesky decomposition.
-The Ising model
-
-import numpy as np
-import matplotlib.pyplot as plt
-from mpl_toolkits.axes_grid1 import make_axes_locatable
-import seaborn as sns
-import scipy.linalg as scl
-from sklearn.model_selection import train_test_split
-import tqdm
-sns.set(color_codes=True)
-cmap_args=dict(vmin=-1., vmax=1., cmap='seismic')
-
-L = 40
-n = int(1e4)
-
-spins = np.random.choice([-1, 1], size=(n, L))
-J = 1.0
-
-energies = np.zeros(n)
-
-for i in range(n):
- energies[i] = - J * np.dot(spins[i], np.roll(spins[i], 1))
-
-
-Reformulating the problem to suit regression
-
-X = np.zeros((n, L ** 2))
-for i in range(n):
- X[i] = np.outer(spins[i], spins[i]).ravel()
-y = energies
-X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)
-
-
-Linear regression
-
-X_train_own = np.concatenate(
- (np.ones(len(X_train))[:, np.newaxis], X_train),
- axis=1
-)
-X_test_own = np.concatenate(
- (np.ones(len(X_test))[:, np.newaxis], X_test),
- axis=1
-)
-
def ols_inv(x: np.ndarray, y: np.ndarray) -> np.ndarray:
- return scl.inv(x.T @ x) @ (x.T @ y)
-beta = ols_inv(X_train_own, y_train)
-
-
-Singular Value decomposition
-
-def ols_svd(x: np.ndarray, y: np.ndarray) -> np.ndarray:
- u, s, v = scl.svd(x)
- return v.T @ scl.pinv(scl.diagsvd(s, u.shape[0], v.shape[0])) @ u.T @ y
-
beta = ols_svd(X_train_own,y_train)
-
J = beta[1:].reshape(L, L)
-fig = plt.figure(figsize=(20, 14))
-im = plt.imshow(J, **cmap_args)
-plt.title("OLS", fontsize=18)
-plt.xticks(fontsize=18)
-plt.yticks(fontsize=18)
-cb = fig.colorbar(im)
-cb.ax.set_yticklabels(cb.ax.get_yticklabels(), fontsize=18)
-plt.show()
-
-
-Linear Regression Problems
+Linear Regression Problems
-Fixing the singularity
+Fixing the singularity
-Basic math of the SVD
+Basic math of the SVD
The SVD, a Fantastic Algorithm
-Another Example
+Another Example
-Economy-size SVD
+Economy-size SVD
-Mathematical Properties
+Mathematical Properties
-Ridge and LASSO Regression
+Ridge and LASSO Regression
-More on Ridge Regression
+More on Ridge Regression
-Interpreting the Ridge results
+Interpreting the Ridge results
-More interpretations
+More interpretations
-Where are we going?
+Where are we going?
-Resampling methods
+Resampling methods
-Resampling approaches can be computationally expensive
+Resampling approaches can be computationally expensive
-Why resampling methods ?
+Why resampling methods ?
-Statistical analysis
+Statistical analysis
-Statistics
+Statistics
-Statistics, moments
+Statistics, moments
-Statistics, central moments
+Statistics, central moments
-Statistics, covariance
+Statistics, covariance
-Statistics, more covariance
+Statistics, more covariance
-Covariance example
+Covariance example
-Covariance in numpy
+Covariance in numpy
-Statistics, independent variables
+Statistics, independent variables
-Statistics, more variance
+Statistics, more variance
-Statistics and stochastic processes
+Statistics and stochastic processes
Statistics and sample variables
+Statistics and sample variables
-Statistics, sample variance and covariance
+Statistics, sample variance and covariance
-Statistics, law of large numbers
+Statistics, law of large numbers
-Statistics, more on sample error
+Statistics, more on sample error
-Statistics
+Statistics
-Statistics, central limit theorem
+Statistics, central limit theorem
-Statistics, more technicalities
+Statistics, more technicalities
-Statistics
+Statistics
-Statistics and sample variance
+Statistics and sample variance
-Statistics, uncorrelated results
+Statistics, uncorrelated results
-Statistics, computations
+Statistics, computations
-Statistics, more on computations of errors
+Statistics, more on computations of errors
-Statistics, wrapping up 1
+Statistics, wrapping up 1
-Statistics, final expression
+Statistics, final expression
-Statistics, effective number of correlations
+Statistics, effective number of correlations
Linking the regression analysis with a statistical interpretation
+Linking the regression analysis with a statistical interpretation
-Assumptions made
+Assumptions made
-Expectation value and variance
+Expectation value and variance
-Expectation value and variance for \( \boldsymbol{\beta} \)
+Expectation value and variance for \( \boldsymbol{\beta} \)
Cross-validation
+Cross-validation
-Computationally expensive
+Computationally expensive
Various steps in cross-validation
+Various steps in cross-validation
How to set up the cross-validation for Ridge and/or Lasso
+How to set up the cross-validation for Ridge and/or Lasso
-Resampling methods: Jackknife and Bootstrap
+Resampling methods: Jackknife and Bootstrap
-Resampling methods: Jackknife
+Resampling methods: Jackknife
-Jackknife code example
+Jackknife code example
-Resampling methods: Bootstrap
+Resampling methods: Bootstrap
-Resampling methods: Bootstrap background
+Resampling methods: Bootstrap background
-Resampling methods: More Bootstrap background
+Resampling methods: More Bootstrap background
-Resampling methods: Bootstrap approach
+Resampling methods: Bootstrap approach
-Resampling methods: Bootstrap steps
+Resampling methods: Bootstrap steps
-Code example for the Bootstrap method
+Code example for the Bootstrap method
-Code Example for Cross-validation and \( k \)-fold Cross-validation
+Code Example for Cross-validation and \( k \)-fold Cross-validation
-The bias-variance tradeoff
+The bias-variance tradeoff
-Example code for Bias-Variance tradeoff
+Example code for Bias-Variance tradeoff
-Understanding what happens
+Understanding what happens
Summing up
+Summing up
-Another Example rom Scikit-Learn's Repository
+Another Example rom Scikit-Learn's Repository
+The Ising model
+
+import numpy as np
+import matplotlib.pyplot as plt
+from mpl_toolkits.axes_grid1 import make_axes_locatable
+import seaborn as sns
+import scipy.linalg as scl
+from sklearn.model_selection import train_test_split
+import tqdm
+sns.set(color_codes=True)
+cmap_args=dict(vmin=-1., vmax=1., cmap='seismic')
+
+L = 40
+n = int(1e4)
+
+spins = np.random.choice([-1, 1], size=(n, L))
+J = 1.0
+
+energies = np.zeros(n)
+
+for i in range(n):
+ energies[i] = - J * np.dot(spins[i], np.roll(spins[i], 1))
+
+
+Reformulating the problem to suit regression
+
+X = np.zeros((n, L ** 2))
+for i in range(n):
+ X[i] = np.outer(spins[i], spins[i]).ravel()
+y = energies
+X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)
+
+
+Linear regression
+
+X_train_own = np.concatenate(
+ (np.ones(len(X_train))[:, np.newaxis], X_train),
+ axis=1
+)
+X_test_own = np.concatenate(
+ (np.ones(len(X_test))[:, np.newaxis], X_test),
+ axis=1
+)
+
def ols_inv(x: np.ndarray, y: np.ndarray) -> np.ndarray:
+ return scl.inv(x.T @ x) @ (x.T @ y)
+beta = ols_inv(X_train_own, y_train)
+
+
+Singular Value decomposition
+
+def ols_svd(x: np.ndarray, y: np.ndarray) -> np.ndarray:
+ u, s, v = scl.svd(x)
+ return v.T @ scl.pinv(scl.diagsvd(s, u.shape[0], v.shape[0])) @ u.T @ y
+
beta = ols_svd(X_train_own,y_train)
+
J = beta[1:].reshape(L, L)
+fig = plt.figure(figsize=(20, 14))
+im = plt.imshow(J, **cmap_args)
+plt.title("OLS", fontsize=18)
+plt.xticks(fontsize=18)
+plt.yticks(fontsize=18)
+cb = fig.colorbar(im)
+cb.ax.set_yticklabels(cb.ax.get_yticklabels(), fontsize=18)
+plt.show()
+
+
The one-dimensional Ising model
Aug 29, 2019
Aug 30, 2019
@@ -1559,8 +1559,7 @@ The examples we have looked at so far are cases where we normally can
invert the matrix \( \boldsymbol{X}^T\boldsymbol{X} \). Using a polynomial expansion as we
did both for the masses and the fitting of the equation of state,
leads to row vectors of the design matrix which are essentially
-orthogonal due to the polynomial character of our model. Obtaining the inverse of the design matrix is then often done via a so-called LU, QR or Cholesky decomposition.
-More material to come here.
+orthogonal due to the polynomial character of our model. Obtaining the inverse of the design matrix is then often done via a so-called LU, QR or Cholesky decomposition.
-The Ising model
-
-import numpy as np
-import matplotlib.pyplot as plt
-from mpl_toolkits.axes_grid1 import make_axes_locatable
-import seaborn as sns
-import scipy.linalg as scl
-from sklearn.model_selection import train_test_split
-import tqdm
-sns.set(color_codes=True)
-cmap_args=dict(vmin=-1., vmax=1., cmap='seismic')
-
-L = 40
-n = int(1e4)
-
-spins = np.random.choice([-1, 1], size=(n, L))
-J = 1.0
-
-energies = np.zeros(n)
-
-for i in range(n):
- energies[i] = - J * np.dot(spins[i], np.roll(spins[i], 1))
-
-
-Reformulating the problem to suit regression
-
-X = np.zeros((n, L ** 2))
-for i in range(n):
- X[i] = np.outer(spins[i], spins[i]).ravel()
-y = energies
-X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)
-
-
-Linear regression
-
-X_train_own = np.concatenate(
- (np.ones(len(X_train))[:, np.newaxis], X_train),
- axis=1
-)
-X_test_own = np.concatenate(
- (np.ones(len(X_test))[:, np.newaxis], X_test),
- axis=1
-)
-
def ols_inv(x: np.ndarray, y: np.ndarray) -> np.ndarray:
- return scl.inv(x.T @ x) @ (x.T @ y)
-beta = ols_inv(X_train_own, y_train)
-
-
-Singular Value decomposition
-
-def ols_svd(x: np.ndarray, y: np.ndarray) -> np.ndarray:
- u, s, v = scl.svd(x)
- return v.T @ scl.pinv(scl.diagsvd(s, u.shape[0], v.shape[0])) @ u.T @ y
-
beta = ols_svd(X_train_own,y_train)
-J = beta[1:].reshape(L, L)
-
fig = plt.figure(figsize=(20, 14))
-im = plt.imshow(J, **cmap_args)
-plt.title("OLS", fontsize=18)
-plt.xticks(fontsize=18)
-plt.yticks(fontsize=18)
-cb = fig.colorbar(im)
-cb.ax.set_yticklabels(cb.ax.get_yticklabels(), fontsize=18)
-plt.show()
-
-
-Linear Regression Problems
+Linear Regression Problems
-Fixing the singularity
+Fixing the singularity
-Basic math of the SVD
+Basic math of the SVD