updating
This commit is contained in:
Morten Hjorth-Jensen
@@ -7,48 +7,3 @@ Translating doconce text in chapter1.do.txt to ipynb
|
||||
Failed to remove ans_at_end environment
|
||||
Failed to remove sol_at_end environment
|
||||
output in chapter1.ipynb
|
||||
Translating doconce text in chapter1.do.txt to ipynb
|
||||
*** replacing \bm{...} by \boldsymbol{...} (\bm is not supported by MathJax)
|
||||
|
||||
*** warning: latex envir \begin{bmatrix} does not work well in Markdown. Stick to \[ ... \], equation, equation*, align, or align* environments in math environments.
|
||||
|
||||
*** warning: latex envir \begin{bmatrix} does not work well in Markdown. Stick to \[ ... \], equation, equation*, align, or align* environments in math environments.
|
||||
Failed to remove ans_at_end environment
|
||||
Failed to remove sol_at_end environment
|
||||
output in chapter1.ipynb
|
||||
Translating doconce text in chapter1.do.txt to ipynb
|
||||
*** replacing \bm{...} by \boldsymbol{...} (\bm is not supported by MathJax)
|
||||
|
||||
*** warning: latex envir \begin{bmatrix} does not work well in Markdown. Stick to \[ ... \], equation, equation*, align, or align* environments in math environments.
|
||||
|
||||
*** warning: latex envir \begin{bmatrix} does not work well in Markdown. Stick to \[ ... \], equation, equation*, align, or align* environments in math environments.
|
||||
Failed to remove ans_at_end environment
|
||||
Failed to remove sol_at_end environment
|
||||
output in chapter1.ipynb
|
||||
Translating doconce text in chapter1.do.txt to ipynb
|
||||
*** replacing \bm{...} by \boldsymbol{...} (\bm is not supported by MathJax)
|
||||
|
||||
*** warning: latex envir \begin{bmatrix} does not work well in Markdown. Stick to \[ ... \], equation, equation*, align, or align* environments in math environments.
|
||||
|
||||
*** warning: latex envir \begin{bmatrix} does not work well in Markdown. Stick to \[ ... \], equation, equation*, align, or align* environments in math environments.
|
||||
Failed to remove ans_at_end environment
|
||||
Failed to remove sol_at_end environment
|
||||
output in chapter1.ipynb
|
||||
Translating doconce text in chapter1.do.txt to ipynb
|
||||
*** replacing \bm{...} by \boldsymbol{...} (\bm is not supported by MathJax)
|
||||
|
||||
*** warning: latex envir \begin{bmatrix} does not work well in Markdown. Stick to \[ ... \], equation, equation*, align, or align* environments in math environments.
|
||||
|
||||
*** warning: latex envir \begin{bmatrix} does not work well in Markdown. Stick to \[ ... \], equation, equation*, align, or align* environments in math environments.
|
||||
Failed to remove ans_at_end environment
|
||||
Failed to remove sol_at_end environment
|
||||
output in chapter1.ipynb
|
||||
Translating doconce text in chapter1.do.txt to ipynb
|
||||
*** replacing \bm{...} by \boldsymbol{...} (\bm is not supported by MathJax)
|
||||
|
||||
*** warning: latex envir \begin{bmatrix} does not work well in Markdown. Stick to \[ ... \], equation, equation*, align, or align* environments in math environments.
|
||||
|
||||
*** warning: latex envir \begin{bmatrix} does not work well in Markdown. Stick to \[ ... \], equation, equation*, align, or align* environments in math environments.
|
||||
Failed to remove ans_at_end environment
|
||||
Failed to remove sol_at_end environment
|
||||
output in chapter1.ipynb
|
||||
|
||||
@@ -230,7 +230,9 @@ How to evaluate which model fits best the data is something we will come back to
|
||||
|
||||
===== Simple linear regression model using _scikit-learn_ =====
|
||||
|
||||
We start with perhaps our simplest possible example, using _Scikit-Learn_ to perform linear regression analysis on a data set produced by us.
|
||||
We start with perhaps our simplest possible example, using
|
||||
_Scikit-Learn_ to perform linear regression analysis on a data set
|
||||
produced by us.
|
||||
|
||||
What follows is a simple Python code where we have defined a function
|
||||
$y$ in terms of the variable $x$. Both are defined as vectors with $100$ entries.
|
||||
@@ -373,7 +375,8 @@ We can modify easily the above Python code and plot the relative error instead
|
||||
import numpy as np
|
||||
import matplotlib.pyplot as plt
|
||||
from sklearn.linear_model import LinearRegression
|
||||
|
||||
# Number of data points
|
||||
n = 100
|
||||
x = np.random.rand(100,1)
|
||||
y = 5*x+0.01*np.random.randn(100,1)
|
||||
linreg = LinearRegression()
|
||||
@@ -391,7 +394,8 @@ plt.show()
|
||||
Depending on the parameter in front of the normal distribution, we may
|
||||
have a small or larger relative error. Try to play around with
|
||||
different training data sets and study (graphically) the value of the
|
||||
relative error.
|
||||
relative error. Note also that _Scikit-Learn_ requires a matrix as input for the input values $x$ and $y$. In the above code we have
|
||||
solved this by declaring $x$ and $y$ as arrays of dimension $n\times 1$.
|
||||
|
||||
As mentioned above, _Scikit-Learn_ has an impressive functionality.
|
||||
We can for example extract the values of $\alpha$ and $\beta$ and
|
||||
@@ -493,13 +497,14 @@ ways of dealing with outliers.
|
||||
The Huber cost function is defined as
|
||||
!bt
|
||||
\[
|
||||
H_{\delta}(\bm{a})=\left\{\begin{array}{cc}\frac{1}{2} \bm{a}^{2}& \text{for }|\bm{a}|\leq \delta\\ \delta (|\b\
|
||||
m{a}|-\frac{1}{2}\delta ),&\text{otherwise}.\end{array}\right.
|
||||
H_{\delta}(\bm{a})=\left\{\begin{array}{cc}\frac{1}{2} \bm{a}^{2}& \text{for }|\bm{a}|\leq \delta\\ \delta (|\bm{a}|-\frac{1}{2}\delta ),&\text{otherwise}.\end{array}\right.
|
||||
\]
|
||||
!et
|
||||
Here $\bm{a}=\bm{y} - \bm{\tilde{y}}$.
|
||||
|
||||
|
||||
|
||||
|
||||
We will discuss in more
|
||||
detail these and other functions in the various lectures. We conclude this part with another example. Instead of
|
||||
a linear $x$-dependence we study now a cubic polynomial and use the polynomial regression analysis tools of scikit-learn.
|
||||
@@ -761,6 +766,9 @@ X[:,2] = A**(2.0/3.0)
|
||||
X[:,3] = A**(-1.0/3.0)
|
||||
X[:,4] = A**(-1.0)
|
||||
!ec
|
||||
|
||||
Note well that we have made life simple here. We perform a fit in terms of the number of nucleons only. A more sophisticated fit can be done by including an explicit dependence on the number of protons and neutrons in the asymmetry and Coulomb terms.
|
||||
|
||||
With _scikitlearn_ we are now ready to use linear regression and fit our data.
|
||||
!bc pycod
|
||||
clf = skl.LinearRegression().fit(X, Energies)
|
||||
@@ -775,7 +783,6 @@ print("Mean squared error: %.2f" % mean_squared_error(Energies, fity))
|
||||
print('Variance score: %.2f' % r2_score(Energies, fity))
|
||||
# Mean absolute error
|
||||
print('Mean absolute error: %.2f' % mean_absolute_error(Energies, fity))
|
||||
print(clf.coef_, clf.intercept_)
|
||||
|
||||
Masses['Eapprox'] = fity
|
||||
# Generate a plot comparing the experimental with the fitted values values.
|
||||
@@ -1874,6 +1881,93 @@ plt.show()
|
||||
!ec
|
||||
|
||||
|
||||
===== Splitting our Data in Training and Test data =====
|
||||
|
||||
|
||||
It is normal in essentially all Machine Learning studies to split the
|
||||
data in a training set and a test set (sometimes also an additional
|
||||
validation set). _Scikit-Learn_ has an own function for this. There
|
||||
is no explicit recipe for how much data should be included as training
|
||||
data and say test data. An accepted rule of thumb is to use
|
||||
approximately $2/3$ to $4/5$ of the data as training data. We will
|
||||
postpone a discussion of this splitting to the end of these notes and
|
||||
our discussion of the so-called _bias-variance_ tradeoff. Here we
|
||||
limit ourselves to repeat the above equation of state fitting example
|
||||
but now splitting the data into a training set and a test set.
|
||||
|
||||
Let us study some examples. The first code here takes a simple
|
||||
one-dimensional second-order polynomial and we fit it to a
|
||||
second-order polynomial. Depending on the strength of the added noise,
|
||||
the various measures like the $R2$ score or the mean-squared error,
|
||||
the fit becomes better or worse.
|
||||
|
||||
!bc pycod
|
||||
import os
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import matplotlib.pyplot as plt
|
||||
from sklearn.model_selection import train_test_split
|
||||
|
||||
|
||||
def R2(y_data, y_model):
|
||||
return 1 - np.sum((y_data - y_model) ** 2) / np.sum((y_data - np.mean(y_data)) ** 2)
|
||||
def MSE(y_data,y_model):
|
||||
n = np.size(y_model)
|
||||
return np.sum((y_data-y_model)**2)/n
|
||||
|
||||
x = np.random.rand(100)
|
||||
y = 2.0+5*x*x+0.1*np.random.randn(100)
|
||||
|
||||
|
||||
# The design matrix now as function of a given polynomial
|
||||
X = np.zeros((len(x),3))
|
||||
X[:,0] = 1.0
|
||||
X[:,1] = x
|
||||
X[:,2] = x**2
|
||||
# We split the data in test and training data
|
||||
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)
|
||||
# matrix inversion to find beta
|
||||
beta = np.linalg.inv(X_train.T @ X_train) @ X_train.T @ y_train
|
||||
print(beta)
|
||||
# and then make the prediction
|
||||
ytilde = X_train @ beta
|
||||
print("Training R2")
|
||||
print(R2(y_train,ytilde))
|
||||
print("Training MSE")
|
||||
print(MSE(y_train,ytilde))
|
||||
ypredict = X_test @ beta
|
||||
print("Test R2")
|
||||
print(R2(y_test,ypredict))
|
||||
print("Test MSE")
|
||||
print(MSE(y_test,ypredict))
|
||||
!ec
|
||||
|
||||
Alternatively, you could write your own test-train splitting function as shown here.
|
||||
|
||||
!bc pycod
|
||||
# equivalently in numpy
|
||||
def train_test_split_numpy(inputs, labels, train_size, test_size):
|
||||
n_inputs = len(inputs)
|
||||
inputs_shuffled = inputs.copy()
|
||||
labels_shuffled = labels.copy()
|
||||
|
||||
np.random.shuffle(inputs_shuffled)
|
||||
np.random.shuffle(labels_shuffled)
|
||||
|
||||
train_end = int(n_inputs*train_size)
|
||||
X_train, X_test = inputs_shuffled[:train_end], inputs_shuffled[train_end:]
|
||||
Y_train, Y_test = labels_shuffled[:train_end], labels_shuffled[train_end:]
|
||||
|
||||
return X_train, X_test, Y_train, Y_test
|
||||
!ec
|
||||
|
||||
But since _scikit-learn_ has its own function for doing this and since
|
||||
it interfaces easily with _tensorflow_ and other libraries, we
|
||||
normally recommend using the latter functionality.
|
||||
|
||||
|
||||
|
||||
|
||||
===== Reducing the number of degrees of freedom, overarching view =====
|
||||
|
||||
Many Machine Learning problems involve thousands or even millions of
|
||||
@@ -1897,6 +1991,7 @@ is one of the most used tools in data modeling, compression and
|
||||
visualization.
|
||||
|
||||
|
||||
|
||||
Before we proceed however, we will discuss how to preprocess our
|
||||
data. Till now and in connection with our previous examples we have
|
||||
not met so many cases where we are too sensitive to the scaling of our
|
||||
@@ -1904,6 +1999,15 @@ data. Normally the data may need a rescaling and/or may be sensitive
|
||||
to extreme values. Scaling the data renders our inputs much more
|
||||
suitable for the algorithms we want to employ.
|
||||
|
||||
For data sets gathered for real world applications, it is rather normal that
|
||||
different features have very different units and
|
||||
numerical scales. For example, a data set detailing health habits may include
|
||||
features such as _age_ in the range $0-80$, and _caloric intake_ of order $2000$.
|
||||
Many machine learning methods sensitive to the scales of the features and may perform poorly if they
|
||||
are very different scales. Therefore, it is typical to scale
|
||||
the features in a way to avoid such outlier values.
|
||||
|
||||
|
||||
_Scikit-Learn_ has several functions which allow us to rescale the
|
||||
data, normally resulting in much better results in terms of various
|
||||
accuracy scores. The _StandardScaler_ function in _Scikit-Learn_
|
||||
@@ -1933,107 +2037,134 @@ outliers, and might often lead to trouble for other scaling
|
||||
techniques.
|
||||
|
||||
|
||||
=== Simple preprocessing examples, Franke function and regression ===
|
||||
Many features are often scaled using standardization to improve
|
||||
performance. In _Scikit-Learn_ this is given by the _StandardScaler_
|
||||
function as discussed above. It is easy however to write your own.
|
||||
Mathematically, this involves subtracting the mean and divide by the
|
||||
standard deviation over the data set, for each feature:
|
||||
|
||||
!bt
|
||||
\[
|
||||
x_j^{(i)} \rightarrow \frac{x_j^{(i)} - \overline{x}_j}{\sigma(x_j)},
|
||||
\]
|
||||
!et
|
||||
|
||||
where $\overline{x}_j$ and $\sigma(x_j)$ are the mean and standard
|
||||
deviation, respectively, of the feature $x_j$. This ensures that each
|
||||
feature has zero mean and unit standard deviation. For data sets
|
||||
where we do not have the standard deviation or don't wish to calculate
|
||||
it, it is then common to simply set it to one.
|
||||
|
||||
|
||||
|
||||
Let us consider the following vanilla example where we use both
|
||||
_Scikit-Learn_ and write our own function as well. We produce a
|
||||
simple test design matrix with random numbers. Each column could then
|
||||
represent a specific feature whose mean value is subracted.
|
||||
|
||||
|
||||
!bc pycod
|
||||
# Common imports
|
||||
import os
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import matplotlib.pyplot as plt
|
||||
import sklearn.linear_model as skl
|
||||
from sklearn.metrics import mean_squared_error
|
||||
from sklearn.model_selection import train_test_split
|
||||
from sklearn.preprocessing import MinMaxScaler, StandardScaler, Normalizer
|
||||
|
||||
# Where to save the figures and data files
|
||||
PROJECT_ROOT_DIR = "Results"
|
||||
FIGURE_ID = "Results/FigureFiles"
|
||||
DATA_ID = "DataFiles/"
|
||||
|
||||
if not os.path.exists(PROJECT_ROOT_DIR):
|
||||
os.mkdir(PROJECT_ROOT_DIR)
|
||||
|
||||
if not os.path.exists(FIGURE_ID):
|
||||
os.makedirs(FIGURE_ID)
|
||||
|
||||
if not os.path.exists(DATA_ID):
|
||||
os.makedirs(DATA_ID)
|
||||
|
||||
def image_path(fig_id):
|
||||
return os.path.join(FIGURE_ID, fig_id)
|
||||
|
||||
def data_path(dat_id):
|
||||
return os.path.join(DATA_ID, dat_id)
|
||||
|
||||
def save_fig(fig_id):
|
||||
plt.savefig(image_path(fig_id) + ".png", format='png')
|
||||
|
||||
|
||||
def FrankeFunction(x,y):
|
||||
term1 = 0.75*np.exp(-(0.25*(9*x-2)**2) - 0.25*((9*y-2)**2))
|
||||
term2 = 0.75*np.exp(-((9*x+1)**2)/49.0 - 0.1*(9*y+1))
|
||||
term3 = 0.5*np.exp(-(9*x-7)**2/4.0 - 0.25*((9*y-3)**2))
|
||||
term4 = -0.2*np.exp(-(9*x-4)**2 - (9*y-7)**2)
|
||||
return term1 + term2 + term3 + term4
|
||||
|
||||
|
||||
def create_X(x, y, n ):
|
||||
if len(x.shape) > 1:
|
||||
x = np.ravel(x)
|
||||
y = np.ravel(y)
|
||||
|
||||
N = len(x)
|
||||
l = int((n+1)*(n+2)/2) # Number of elements in beta
|
||||
X = np.ones((N,l))
|
||||
|
||||
for i in range(1,n+1):
|
||||
q = int((i)*(i+1)/2)
|
||||
for k in range(i+1):
|
||||
X[:,q+k] = (x**(i-k))*(y**k)
|
||||
|
||||
return X
|
||||
|
||||
|
||||
# Making meshgrid of datapoints and compute Franke's function
|
||||
n = 5
|
||||
N = 1000
|
||||
x = np.sort(np.random.uniform(0, 1, N))
|
||||
y = np.sort(np.random.uniform(0, 1, N))
|
||||
z = FrankeFunction(x, y)
|
||||
X = create_X(x, y, n=n)
|
||||
# split in training and test data
|
||||
X_train, X_test, y_train, y_test = train_test_split(X,z,test_size=0.2)
|
||||
|
||||
|
||||
clf = skl.LinearRegression().fit(X_train, y_train)
|
||||
|
||||
# The mean squared error and R2 score
|
||||
print("MSE before scaling: {:.2f}".format(mean_squared_error(clf.predict(X_test), y_test)))
|
||||
print("R2 score before scaling {:.2f}".format(clf.score(X_test,y_test)))
|
||||
|
||||
scaler = StandardScaler()
|
||||
scaler.fit(X_train)
|
||||
X_train_scaled = scaler.transform(X_train)
|
||||
X_test_scaled = scaler.transform(X_test)
|
||||
|
||||
print("Feature min values before scaling:\n {}".format(X_train.min(axis=0)))
|
||||
print("Feature max values before scaling:\n {}".format(X_train.max(axis=0)))
|
||||
|
||||
print("Feature min values after scaling:\n {}".format(X_train_scaled.min(axis=0)))
|
||||
print("Feature max values after scaling:\n {}".format(X_train_scaled.max(axis=0)))
|
||||
|
||||
clf = skl.LinearRegression().fit(X_train_scaled, y_train)
|
||||
|
||||
|
||||
print("MSE after scaling: {:.2f}".format(mean_squared_error(clf.predict(X_test_scaled), y_test)))
|
||||
print("R2 score for scaled data: {:.2f}".format(clf.score(X_test_scaled,y_test)))
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from IPython.display import display
|
||||
np.random.seed(100)
|
||||
# setting up a 10 x 5 matrix
|
||||
rows = 10
|
||||
cols = 5
|
||||
X = np.random.randn(rows,cols)
|
||||
XPandas = pd.DataFrame(X)
|
||||
display(XPandas)
|
||||
print(XPandas.mean())
|
||||
print(XPandas.std())
|
||||
XPandas = (XPandas -XPandas.mean())
|
||||
display(XPandas)
|
||||
# This option does not include the standard deviation
|
||||
scaler = StandardScaler(with_std=False)
|
||||
scaler.fit(X)
|
||||
Xscaled = scaler.transform(X)
|
||||
display(XPandas-Xscaled)
|
||||
!ec
|
||||
|
||||
Small exercise: perform the standard scaling by including the standard deviation and compare with what Scikit-Learn gives.
|
||||
|
||||
|
||||
|
||||
Another commonly used scaling method is min-max scaling. This is very
|
||||
useful for when we want the features to lie in a certain interval. To
|
||||
scale the feature $x_j$ to the interval $[a, b]$, we can apply the
|
||||
transformation
|
||||
|
||||
!bt
|
||||
\[
|
||||
x_j^{(i)} \rightarrow (b-a)\frac{x_j^{(i)} - \min(x_j)}{\max(x_j) - \min(x_j)} - a
|
||||
\]
|
||||
!et
|
||||
where $\min(x_j)$ and $\max(x_j)$ return the minimum and maximum value of $x_j$ over the data set, respectively.
|
||||
|
||||
|
||||
|
||||
|
||||
===== Testing the Means Squared Error as function of Complexity =====
|
||||
|
||||
|
||||
Before we proceed with a more detailed analysis of the so-called
|
||||
Bias-Variance tradeoff, we present here an example of the relation
|
||||
between model complexity and the mean squared error for the triaining
|
||||
data and the test data.
|
||||
|
||||
The results here tell us clearly that for the data not included in the
|
||||
training, there is an optimal model as function of the complexity of
|
||||
ourmodel (here in terms of the polynomial degree of the model).
|
||||
|
||||
The results here will vary as function of model complexity and the amount od data used for training.
|
||||
|
||||
|
||||
Our data is defined by $x\in [-3,3]$ with a total of for example $100$ data points.
|
||||
|
||||
!bc pycod
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
from sklearn.linear_model import LinearRegression, Ridge, Lasso
|
||||
from sklearn.preprocessing import PolynomialFeatures
|
||||
from sklearn.model_selection import train_test_split
|
||||
from sklearn.pipeline import make_pipeline
|
||||
|
||||
|
||||
np.random.seed(2018)
|
||||
n = 100
|
||||
maxdegree = 14
|
||||
# Make data set.
|
||||
x = np.linspace(-3, 3, n).reshape(-1, 1)
|
||||
y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)
|
||||
TestError = np.zeros(maxdegree)
|
||||
TrainError = np.zeros(maxdegree)
|
||||
polydegree = np.zeros(maxdegree)
|
||||
x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2)
|
||||
scaler = StandardScaler()
|
||||
scaler.fit(x_train)
|
||||
x_train_scaled = scaler.transform(x_train)
|
||||
x_test_scaled = scaler.transform(x_test)
|
||||
|
||||
for degree in range(maxdegree):
|
||||
model = make_pipeline(PolynomialFeatures(degree=degree), LinearRegression(fit_intercept=False))
|
||||
clf = model.fit(x_train_scaled,y_train)
|
||||
y_fit = clf.predict(x_train_scaled)
|
||||
y_pred = clf.predict(x_test_scaled)
|
||||
polydegree[degree] = degree
|
||||
TestError[degree] = np.mean( np.mean((y_test - y_pred)**2) )
|
||||
TrainError[degree] = np.mean( np.mean((y_train - y_fit)**2) )
|
||||
|
||||
plt.plot(polydegree, TestError, label='Test Error')
|
||||
plt.plot(polydegree, TrainError, label='Train Error')
|
||||
plt.legend()
|
||||
plt.show()
|
||||
!ec
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -2235,3 +2366,6 @@ your results. For which polynomial degree do you find an optimal MSE
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -1,19 +0,0 @@
|
||||
*** error: file has a mako construction ${0:.1f}'
|
||||
but seemingly no definition in <%...%>'
|
||||
(it is not a command-line given mako variable either).
|
||||
However, if this is a variable in a Makefile or Bash script
|
||||
run with --no_mako - and you cannot use mako and Makefile or Bash variables
|
||||
in the same document!
|
||||
|
||||
*** error: file has a mako construction ${0:.1f}'
|
||||
but seemingly no definition in <%...%>'
|
||||
(it is not a command-line given mako variable either).
|
||||
However, if this is a variable in a Makefile or Bash script
|
||||
run with --no_mako - and you cannot use mako and Makefile or Bash variables
|
||||
in the same document!
|
||||
|
||||
avoided abortion because of --no-abort
|
||||
Translating doconce text in chapter10.do.txt to ipynb
|
||||
Failed to remove ans_at_end environment
|
||||
Failed to remove sol_at_end environment
|
||||
output in chapter10.ipynb
|
||||
@@ -1,29 +0,0 @@
|
||||
Translating doconce text in chapter11.do.txt to ipynb
|
||||
*** replacing \bm{...} by \boldsymbol{...} (\bm is not supported by MathJax)
|
||||
|
||||
*** warning: latex envir \begin{aligned} does not work well in Markdown. Stick to \[ ... \], equation, equation*, align, or align* environments in math environments.
|
||||
|
||||
*** warning: latex envir \begin{aligned} does not work well in Markdown. Stick to \[ ... \], equation, equation*, align, or align* environments in math environments.
|
||||
|
||||
*** warning: latex envir \begin{aligned} does not work well in Markdown. Stick to \[ ... \], equation, equation*, align, or align* environments in math environments.
|
||||
|
||||
*** warning: latex envir \begin{pmatrix} does not work well in Markdown. Stick to \[ ... \], equation, equation*, align, or align* environments in math environments.
|
||||
|
||||
*** warning: latex envir \begin{aligned} does not work well in Markdown. Stick to \[ ... \], equation, equation*, align, or align* environments in math environments.
|
||||
|
||||
*** warning: latex envir \begin{aligned} does not work well in Markdown. Stick to \[ ... \], equation, equation*, align, or align* environments in math environments.
|
||||
|
||||
*** warning: latex envir \begin{aligned} does not work well in Markdown. Stick to \[ ... \], equation, equation*, align, or align* environments in math environments.
|
||||
|
||||
*** warning: latex envir \begin{aligned} does not work well in Markdown. Stick to \[ ... \], equation, equation*, align, or align* environments in math environments.
|
||||
|
||||
*** warning: latex envir \begin{aligned} does not work well in Markdown. Stick to \[ ... \], equation, equation*, align, or align* environments in math environments.
|
||||
|
||||
*** warning: latex envir \begin{aligned} does not work well in Markdown. Stick to \[ ... \], equation, equation*, align, or align* environments in math environments.
|
||||
|
||||
*** warning: latex envir \begin{aligned} does not work well in Markdown. Stick to \[ ... \], equation, equation*, align, or align* environments in math environments.
|
||||
|
||||
*** warning: latex envir \begin{aligned} does not work well in Markdown. Stick to \[ ... \], equation, equation*, align, or align* environments in math environments.
|
||||
Failed to remove ans_at_end environment
|
||||
Failed to remove sol_at_end environment
|
||||
output in chapter11.ipynb
|
||||
@@ -1,10 +0,0 @@
|
||||
Translating doconce text in chapter12.do.txt to ipynb
|
||||
*** error: figure file "figslides/nn.jpeg" does not exist!
|
||||
Translating doconce text in chapter12.do.txt to ipynb
|
||||
Failed to remove ans_at_end environment
|
||||
Failed to remove sol_at_end environment
|
||||
output in chapter12.ipynb
|
||||
Translating doconce text in chapter12.do.txt to ipynb
|
||||
Failed to remove ans_at_end environment
|
||||
Failed to remove sol_at_end environment
|
||||
output in chapter12.ipynb
|
||||
+2462
-864
File diff suppressed because it is too large
Load Diff
+867
-2648
File diff suppressed because it is too large
Load Diff
@@ -1,30 +0,0 @@
|
||||
*** error: file has a mako construction ${\cal L}'
|
||||
but seemingly no definition in <%...%>'
|
||||
(it is not a command-line given mako variable either).
|
||||
However, if this is a variable in a Makefile or Bash script
|
||||
run with --no_mako - and you cannot use mako and Makefile or Bash variables
|
||||
in the same document!
|
||||
|
||||
Translating doconce text in chapter5.do.txt to ipynb
|
||||
*** replacing \bm{...} by \boldsymbol{...} (\bm is not supported by MathJax)
|
||||
|
||||
*** warning: latex envir \begin{bmatrix} does not work well in Markdown. Stick to \[ ... \], equation, equation*, align, or align* environments in math environments.
|
||||
|
||||
*** warning: latex envir \begin{bmatrix} does not work well in Markdown. Stick to \[ ... \], equation, equation*, align, or align* environments in math environments.
|
||||
|
||||
*** warning: latex envir \begin{bmatrix} does not work well in Markdown. Stick to \[ ... \], equation, equation*, align, or align* environments in math environments.
|
||||
|
||||
*** warning: latex envir \begin{bmatrix} does not work well in Markdown. Stick to \[ ... \], equation, equation*, align, or align* environments in math environments.
|
||||
|
||||
*** warning: latex envir \begin{bmatrix} does not work well in Markdown. Stick to \[ ... \], equation, equation*, align, or align* environments in math environments.
|
||||
|
||||
*** warning: latex envir \begin{bmatrix} does not work well in Markdown. Stick to \[ ... \], equation, equation*, align, or align* environments in math environments.
|
||||
|
||||
*** warning: latex envir \begin{bmatrix} does not work well in Markdown. Stick to \[ ... \], equation, equation*, align, or align* environments in math environments.
|
||||
|
||||
*** warning: latex envir \begin{bmatrix} does not work well in Markdown. Stick to \[ ... \], equation, equation*, align, or align* environments in math environments.
|
||||
|
||||
*** warning: latex envir \begin{bmatrix} does not work well in Markdown. Stick to \[ ... \], equation, equation*, align, or align* environments in math environments.
|
||||
Failed to remove ans_at_end environment
|
||||
Failed to remove sol_at_end environment
|
||||
output in chapter5.ipynb
|
||||
@@ -1,5 +0,0 @@
|
||||
Translating doconce text in chapter6.do.txt to ipynb
|
||||
ERROR: 0 !bblock do not match 1 !eblock directives
|
||||
Failed to remove ans_at_end environment
|
||||
Failed to remove sol_at_end environment
|
||||
output in chapter6.ipynb
|
||||
@@ -1,12 +0,0 @@
|
||||
*** error: file has a mako construction ${\cal C}'
|
||||
but seemingly no definition in <%...%>'
|
||||
(it is not a command-line given mako variable either).
|
||||
However, if this is a variable in a Makefile or Bash script
|
||||
run with --no_mako - and you cannot use mako and Makefile or Bash variables
|
||||
in the same document!
|
||||
|
||||
Translating doconce text in chapter7.do.txt to ipynb
|
||||
*** replacing \bm{...} by \boldsymbol{...} (\bm is not supported by MathJax)
|
||||
Failed to remove ans_at_end environment
|
||||
Failed to remove sol_at_end environment
|
||||
output in chapter7.ipynb
|
||||
@@ -1,33 +0,0 @@
|
||||
Translating doconce text in chapter8.do.txt to ipynb
|
||||
*** replacing \bm{...} by \boldsymbol{...} (\bm is not supported by MathJax)
|
||||
|
||||
*** warning: latex envir \begin{bmatrix} does not work well in Markdown. Stick to \[ ... \], equation, equation*, align, or align* environments in math environments.
|
||||
|
||||
*** warning: latex envir \begin{bmatrix} does not work well in Markdown. Stick to \[ ... \], equation, equation*, align, or align* environments in math environments.
|
||||
|
||||
*** warning: latex envir \begin{bmatrix} does not work well in Markdown. Stick to \[ ... \], equation, equation*, align, or align* environments in math environments.
|
||||
|
||||
*** warning: latex envir \begin{bmatrix} does not work well in Markdown. Stick to \[ ... \], equation, equation*, align, or align* environments in math environments.
|
||||
|
||||
*** warning: latex envir \begin{bmatrix} does not work well in Markdown. Stick to \[ ... \], equation, equation*, align, or align* environments in math environments.
|
||||
|
||||
*** warning: latex envir \begin{bmatrix} does not work well in Markdown. Stick to \[ ... \], equation, equation*, align, or align* environments in math environments.
|
||||
|
||||
*** warning: latex envir \begin{bmatrix} does not work well in Markdown. Stick to \[ ... \], equation, equation*, align, or align* environments in math environments.
|
||||
|
||||
*** warning: latex envir \begin{bmatrix} does not work well in Markdown. Stick to \[ ... \], equation, equation*, align, or align* environments in math environments.
|
||||
|
||||
*** warning: latex envir \begin{bmatrix} does not work well in Markdown. Stick to \[ ... \], equation, equation*, align, or align* environments in math environments.
|
||||
|
||||
*** warning: latex envir \begin{bmatrix} does not work well in Markdown. Stick to \[ ... \], equation, equation*, align, or align* environments in math environments.
|
||||
|
||||
*** warning: latex envir \begin{bmatrix} does not work well in Markdown. Stick to \[ ... \], equation, equation*, align, or align* environments in math environments.
|
||||
|
||||
*** warning: latex envir \begin{bmatrix} does not work well in Markdown. Stick to \[ ... \], equation, equation*, align, or align* environments in math environments.
|
||||
|
||||
*** warning: latex envir \begin{bmatrix} does not work well in Markdown. Stick to \[ ... \], equation, equation*, align, or align* environments in math environments.
|
||||
|
||||
*** warning: latex envir \begin{bmatrix} does not work well in Markdown. Stick to \[ ... \], equation, equation*, align, or align* environments in math environments.
|
||||
Failed to remove ans_at_end environment
|
||||
Failed to remove sol_at_end environment
|
||||
output in chapter8.ipynb
|
||||
@@ -1,4 +0,0 @@
|
||||
Translating doconce text in chapter9.do.txt to ipynb
|
||||
Failed to remove ans_at_end environment
|
||||
Failed to remove sol_at_end environment
|
||||
output in chapter9.ipynb
|
||||
@@ -1,22 +0,0 @@
|
||||
Translating doconce text in linalg.do.txt to ipynb
|
||||
|
||||
*** warning: latex envir \begin{bmatrix} does not work well in Markdown. Stick to \[ ... \], equation, equation*, align, or align* environments in math environments.
|
||||
|
||||
*** warning: latex envir \begin{bmatrix} does not work well in Markdown. Stick to \[ ... \], equation, equation*, align, or align* environments in math environments.
|
||||
|
||||
*** warning: latex envir \begin{bmatrix} does not work well in Markdown. Stick to \[ ... \], equation, equation*, align, or align* environments in math environments.
|
||||
|
||||
*** warning: latex envir \begin{bmatrix} does not work well in Markdown. Stick to \[ ... \], equation, equation*, align, or align* environments in math environments.
|
||||
|
||||
*** warning: latex envir \begin{bmatrix} does not work well in Markdown. Stick to \[ ... \], equation, equation*, align, or align* environments in math environments.
|
||||
|
||||
*** warning: latex envir \begin{bmatrix} does not work well in Markdown. Stick to \[ ... \], equation, equation*, align, or align* environments in math environments.
|
||||
|
||||
*** warning: latex envir \begin{bmatrix} does not work well in Markdown. Stick to \[ ... \], equation, equation*, align, or align* environments in math environments.
|
||||
|
||||
*** warning: latex envir \begin{bmatrix} does not work well in Markdown. Stick to \[ ... \], equation, equation*, align, or align* environments in math environments.
|
||||
|
||||
*** warning: latex envir \begin{bmatrix} does not work well in Markdown. Stick to \[ ... \], equation, equation*, align, or align* environments in math environments.
|
||||
Failed to remove ans_at_end environment
|
||||
Failed to remove sol_at_end environment
|
||||
output in linalg.ipynb
|
||||
@@ -1,4 +0,0 @@
|
||||
Translating doconce text in statistics.do.txt to ipynb
|
||||
Failed to remove ans_at_end environment
|
||||
Failed to remove sol_at_end environment
|
||||
output in statistics.ipynb
|
||||
@@ -0,0 +1,100 @@
|
||||
1, 21, 1, 0
|
||||
2, 23, 1, 0
|
||||
3, 25, 1, 1
|
||||
4, 29, 1, 0
|
||||
5, 21, 1, 0
|
||||
6, 24, 1, 0
|
||||
7, 27, 1, 0
|
||||
8, 29, 1, 0
|
||||
9, 28, 1, 0
|
||||
10, 26, 1, 0
|
||||
11, 30, 2, 0
|
||||
12, 31, 2, 0
|
||||
13, 31, 2, 0
|
||||
14, 31, 2, 1
|
||||
15, 32, 2, 0
|
||||
16, 34, 2, 0
|
||||
17, 34, 2, 0
|
||||
18, 31, 2, 0
|
||||
19, 32, 2, 0
|
||||
20, 32, 2, 0
|
||||
21, 33, 2, 0
|
||||
22, 34, 2, 0
|
||||
23, 31, 2, 1
|
||||
24, 30, 2, 0
|
||||
25, 33, 2, 0
|
||||
26, 36, 3, 1
|
||||
27, 35, 3, 0
|
||||
28, 35, 3, 0
|
||||
29, 38, 3, 0
|
||||
30, 37, 3, 1
|
||||
31, 36, 3, 0
|
||||
32, 35, 3, 0
|
||||
33, 39, 3, 0
|
||||
34, 39, 3, 0
|
||||
35, 38, 3, 1
|
||||
36, 37, 3, 0
|
||||
37, 37, 3, 0
|
||||
38, 40, 4, 0
|
||||
39, 41, 4, 1
|
||||
40, 44, 4, 0
|
||||
41, 44, 4, 0
|
||||
42, 43, 4, 1
|
||||
43, 42, 4, 0
|
||||
44, 41, 4, 0
|
||||
45, 40, 4, 1
|
||||
46, 42, 4, 0
|
||||
47, 42, 4, 0
|
||||
48, 43, 4, 0
|
||||
49, 44, 4, 1
|
||||
50, 44, 4, 0
|
||||
51, 42, 4, 0
|
||||
52, 41, 4, 1
|
||||
53, 45, 5, 0
|
||||
54, 45, 5, 1
|
||||
55, 49, 5, 0
|
||||
56, 48, 5, 1
|
||||
57, 47, 5, 0
|
||||
58, 49, 5, 1
|
||||
59, 46, 5, 1
|
||||
60, 45, 5, 0
|
||||
61, 49, 5, 1
|
||||
62, 48, 5, 0
|
||||
63, 47, 5, 1
|
||||
64, 46, 5, 0
|
||||
65, 47, 5, 0
|
||||
66, 50, 6, 1
|
||||
67, 51, 6, 1
|
||||
68, 51, 6, 0
|
||||
69, 54, 6, 1
|
||||
70, 53, 6, 1
|
||||
71, 51, 6, 0
|
||||
72, 52, 6, 1
|
||||
73, 54, 6, 0
|
||||
74, 55, 7, 1
|
||||
75, 56, 7, 1
|
||||
76, 58, 7, 0
|
||||
77, 59, 7, 1
|
||||
78, 59, 7, 1
|
||||
79, 58, 7, 0
|
||||
80, 55, 7, 1
|
||||
81, 56, 7, 1
|
||||
82, 57, 7, 1
|
||||
83, 58, 7, 1
|
||||
84, 59, 7, 0
|
||||
85, 55, 7, 1
|
||||
86, 56, 7, 1
|
||||
87, 57, 7, 1
|
||||
88, 58, 7, 0
|
||||
89, 59, 7, 1
|
||||
90, 56, 7, 1
|
||||
91, 60, 8, 1
|
||||
92, 65, 8, 1
|
||||
93, 67, 8, 1
|
||||
94, 66, 8, 0
|
||||
95, 63, 8, 1
|
||||
96, 61, 8, 1
|
||||
97, 69, 8, 1
|
||||
98, 65, 8, 1
|
||||
99, 64, 8, 1
|
||||
100, 63, 8, 0
|
||||
|
+271
-85
@@ -6,8 +6,6 @@
|
||||
"source": [
|
||||
"# Linear Regression\n",
|
||||
"\n",
|
||||
"[Video of Lecture](https://www.uio.no/studier/emner/matnat/fys/FYS-STK3155/h20/forelesningsvideoer/LectureAug21.mp4?vrtx=view-as-webpage)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"## Introduction\n",
|
||||
"\n",
|
||||
@@ -15,7 +13,7 @@
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"Our emphasis throughout this series of lectures (small change) \n",
|
||||
"Our emphasis throughout this series of lectures \n",
|
||||
"is on understanding the mathematical aspects of\n",
|
||||
"different algorithms used in the fields of data analysis and machine learning. \n",
|
||||
"\n",
|
||||
@@ -235,7 +233,9 @@
|
||||
"\n",
|
||||
"## Simple linear regression model using **scikit-learn**\n",
|
||||
"\n",
|
||||
"We start with perhaps our simplest possible example, using **Scikit-Learn** to perform linear regression analysis on a data set produced by us. \n",
|
||||
"We start with perhaps our simplest possible example, using\n",
|
||||
"**Scikit-Learn** to perform linear regression analysis on a data set\n",
|
||||
"produced by us.\n",
|
||||
"\n",
|
||||
"What follows is a simple Python code where we have defined a function\n",
|
||||
"$y$ in terms of the variable $x$. Both are defined as vectors with $100$ entries. \n",
|
||||
@@ -437,7 +437,8 @@
|
||||
"import numpy as np\n",
|
||||
"import matplotlib.pyplot as plt\n",
|
||||
"from sklearn.linear_model import LinearRegression\n",
|
||||
"\n",
|
||||
"# Number of data points\n",
|
||||
"n = 100\n",
|
||||
"x = np.random.rand(100,1)\n",
|
||||
"y = 5*x+0.01*np.random.randn(100,1)\n",
|
||||
"linreg = LinearRegression()\n",
|
||||
@@ -459,7 +460,8 @@
|
||||
"Depending on the parameter in front of the normal distribution, we may\n",
|
||||
"have a small or larger relative error. Try to play around with\n",
|
||||
"different training data sets and study (graphically) the value of the\n",
|
||||
"relative error.\n",
|
||||
"relative error. Note also that **Scikit-Learn** requires a matrix as input for the input values $x$ and $y$. In the above code we have\n",
|
||||
"solved this by declaring $x$ and $y$ as arrays of dimension $n\\times 1$.\n",
|
||||
"\n",
|
||||
"As mentioned above, **Scikit-Learn** has an impressive functionality.\n",
|
||||
"We can for example extract the values of $\\alpha$ and $\\beta$ and\n",
|
||||
@@ -629,8 +631,7 @@
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"$$\n",
|
||||
"H_{\\delta}(\\boldsymbol{a})=\\left\\{\\begin{array}{cc}\\frac{1}{2} \\boldsymbol{a}^{2}& \\text{for }|\\boldsymbol{a}|\\leq \\delta\\\\ \\delta (|\\b\\\n",
|
||||
"m{a}|-\\frac{1}{2}\\delta ),&\\text{otherwise}.\\end{array}\\right.\n",
|
||||
"H_{\\delta}(\\boldsymbol{a})=\\left\\{\\begin{array}{cc}\\frac{1}{2} \\boldsymbol{a}^{2}& \\text{for }|\\boldsymbol{a}|\\leq \\delta\\\\ \\delta (|\\boldsymbol{a}|-\\frac{1}{2}\\delta ),&\\text{otherwise}.\\end{array}\\right.\n",
|
||||
"$$"
|
||||
]
|
||||
},
|
||||
@@ -641,6 +642,8 @@
|
||||
"Here $\\boldsymbol{a}=\\boldsymbol{y} - \\boldsymbol{\\tilde{y}}$.\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"We will discuss in more\n",
|
||||
"detail these and other functions in the various lectures. We conclude this part with another example. Instead of \n",
|
||||
"a linear $x$-dependence we study now a cubic polynomial and use the polynomial regression analysis tools of scikit-learn."
|
||||
@@ -1061,6 +1064,8 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Note well that we have made life simple here. We perform a fit in terms of the number of nucleons only. A more sophisticated fit can be done by including an explicit dependence on the number of protons and neutrons in the asymmetry and Coulomb terms.\n",
|
||||
"\n",
|
||||
"With **scikitlearn** we are now ready to use linear regression and fit our data."
|
||||
]
|
||||
},
|
||||
@@ -1100,7 +1105,6 @@
|
||||
"print('Variance score: %.2f' % r2_score(Energies, fity))\n",
|
||||
"# Mean absolute error \n",
|
||||
"print('Mean absolute error: %.2f' % mean_absolute_error(Energies, fity))\n",
|
||||
"print(clf.coef_, clf.intercept_)\n",
|
||||
"\n",
|
||||
"Masses['Eapprox'] = fity\n",
|
||||
"# Generate a plot comparing the experimental with the fitted values values.\n",
|
||||
@@ -3046,6 +3050,119 @@
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Splitting our Data in Training and Test data\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"It is normal in essentially all Machine Learning studies to split the\n",
|
||||
"data in a training set and a test set (sometimes also an additional\n",
|
||||
"validation set). **Scikit-Learn** has an own function for this. There\n",
|
||||
"is no explicit recipe for how much data should be included as training\n",
|
||||
"data and say test data. An accepted rule of thumb is to use\n",
|
||||
"approximately $2/3$ to $4/5$ of the data as training data. We will\n",
|
||||
"postpone a discussion of this splitting to the end of these notes and\n",
|
||||
"our discussion of the so-called **bias-variance** tradeoff. Here we\n",
|
||||
"limit ourselves to repeat the above equation of state fitting example\n",
|
||||
"but now splitting the data into a training set and a test set.\n",
|
||||
"\n",
|
||||
"Let us study some examples. The first code here takes a simple\n",
|
||||
"one-dimensional second-order polynomial and we fit it to a\n",
|
||||
"second-order polynomial. Depending on the strength of the added noise,\n",
|
||||
"the various measures like the $R2$ score or the mean-squared error,\n",
|
||||
"the fit becomes better or worse."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"editable": true
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import os\n",
|
||||
"import numpy as np\n",
|
||||
"import pandas as pd\n",
|
||||
"import matplotlib.pyplot as plt\n",
|
||||
"from sklearn.model_selection import train_test_split\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def R2(y_data, y_model):\n",
|
||||
" return 1 - np.sum((y_data - y_model) ** 2) / np.sum((y_data - np.mean(y_data)) ** 2)\n",
|
||||
"def MSE(y_data,y_model):\n",
|
||||
" n = np.size(y_model)\n",
|
||||
" return np.sum((y_data-y_model)**2)/n\n",
|
||||
"\n",
|
||||
"x = np.random.rand(100)\n",
|
||||
"y = 2.0+5*x*x+0.1*np.random.randn(100)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# The design matrix now as function of a given polynomial\n",
|
||||
"X = np.zeros((len(x),3))\n",
|
||||
"X[:,0] = 1.0\n",
|
||||
"X[:,1] = x\n",
|
||||
"X[:,2] = x**2\n",
|
||||
"# We split the data in test and training data\n",
|
||||
"X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)\n",
|
||||
"# matrix inversion to find beta\n",
|
||||
"beta = np.linalg.inv(X_train.T @ X_train) @ X_train.T @ y_train\n",
|
||||
"print(beta)\n",
|
||||
"# and then make the prediction\n",
|
||||
"ytilde = X_train @ beta\n",
|
||||
"print(\"Training R2\")\n",
|
||||
"print(R2(y_train,ytilde))\n",
|
||||
"print(\"Training MSE\")\n",
|
||||
"print(MSE(y_train,ytilde))\n",
|
||||
"ypredict = X_test @ beta\n",
|
||||
"print(\"Test R2\")\n",
|
||||
"print(R2(y_test,ypredict))\n",
|
||||
"print(\"Test MSE\")\n",
|
||||
"print(MSE(y_test,ypredict))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Alternatively, you could write your own test-train splitting function as shown here."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"editable": true
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# equivalently in numpy\n",
|
||||
"def train_test_split_numpy(inputs, labels, train_size, test_size):\n",
|
||||
" n_inputs = len(inputs)\n",
|
||||
" inputs_shuffled = inputs.copy()\n",
|
||||
" labels_shuffled = labels.copy()\n",
|
||||
"\n",
|
||||
" np.random.shuffle(inputs_shuffled)\n",
|
||||
" np.random.shuffle(labels_shuffled)\n",
|
||||
"\n",
|
||||
" train_end = int(n_inputs*train_size)\n",
|
||||
" X_train, X_test = inputs_shuffled[:train_end], inputs_shuffled[train_end:]\n",
|
||||
" Y_train, Y_test = labels_shuffled[:train_end], labels_shuffled[train_end:]\n",
|
||||
"\n",
|
||||
" return X_train, X_test, Y_train, Y_test"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"But since **scikit-learn** has its own function for doing this and since\n",
|
||||
"it interfaces easily with **tensorflow** and other libraries, we\n",
|
||||
"normally recommend using the latter functionality.\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"## Reducing the number of degrees of freedom, overarching view\n",
|
||||
"\n",
|
||||
"Many Machine Learning problems involve thousands or even millions of\n",
|
||||
@@ -3069,6 +3186,7 @@
|
||||
"visualization.\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"Before we proceed however, we will discuss how to preprocess our\n",
|
||||
"data. Till now and in connection with our previous examples we have\n",
|
||||
"not met so many cases where we are too sensitive to the scaling of our\n",
|
||||
@@ -3076,6 +3194,15 @@
|
||||
"to extreme values. Scaling the data renders our inputs much more\n",
|
||||
"suitable for the algorithms we want to employ.\n",
|
||||
"\n",
|
||||
"For data sets gathered for real world applications, it is rather normal that\n",
|
||||
"different features have very different units and\n",
|
||||
"numerical scales. For example, a data set detailing health habits may include\n",
|
||||
"features such as **age** in the range $0-80$, and **caloric intake** of order $2000$.\n",
|
||||
"Many machine learning methods sensitive to the scales of the features and may perform poorly if they\n",
|
||||
"are very different scales. Therefore, it is typical to scale\n",
|
||||
"the features in a way to avoid such outlier values.\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"**Scikit-Learn** has several functions which allow us to rescale the\n",
|
||||
"data, normally resulting in much better results in terms of various\n",
|
||||
"accuracy scores. The **StandardScaler** function in **Scikit-Learn**\n",
|
||||
@@ -3105,7 +3232,38 @@
|
||||
"techniques.\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"### Simple preprocessing examples, Franke function and regression"
|
||||
"Many features are often scaled using standardization to improve\n",
|
||||
"performance. In **Scikit-Learn** this is given by the **StandardScaler**\n",
|
||||
"function as discussed above. It is easy however to write your own.\n",
|
||||
"Mathematically, this involves subtracting the mean and divide by the\n",
|
||||
"standard deviation over the data set, for each feature:"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"$$\n",
|
||||
"x_j^{(i)} \\rightarrow \\frac{x_j^{(i)} - \\overline{x}_j}{\\sigma(x_j)},\n",
|
||||
"$$"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"where $\\overline{x}_j$ and $\\sigma(x_j)$ are the mean and standard\n",
|
||||
"deviation, respectively, of the feature $x_j$. This ensures that each\n",
|
||||
"feature has zero mean and unit standard deviation. For data sets\n",
|
||||
"where we do not have the standard deviation or don't wish to calculate\n",
|
||||
"it, it is then common to simply set it to one.\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"Let us consider the following vanilla example where we use both\n",
|
||||
"**Scikit-Learn** and write our own function as well. We produce a\n",
|
||||
"simple test design matrix with random numbers. Each column could then\n",
|
||||
"represent a specific feature whose mean value is subracted."
|
||||
]
|
||||
},
|
||||
{
|
||||
@@ -3117,98 +3275,126 @@
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Common imports\n",
|
||||
"import os\n",
|
||||
"import numpy as np\n",
|
||||
"import pandas as pd\n",
|
||||
"import matplotlib.pyplot as plt\n",
|
||||
"import sklearn.linear_model as skl\n",
|
||||
"from sklearn.metrics import mean_squared_error\n",
|
||||
"from sklearn.model_selection import train_test_split\n",
|
||||
"from sklearn.preprocessing import MinMaxScaler, StandardScaler, Normalizer\n",
|
||||
"\n",
|
||||
"# Where to save the figures and data files\n",
|
||||
"PROJECT_ROOT_DIR = \"Results\"\n",
|
||||
"FIGURE_ID = \"Results/FigureFiles\"\n",
|
||||
"DATA_ID = \"DataFiles/\"\n",
|
||||
"\n",
|
||||
"if not os.path.exists(PROJECT_ROOT_DIR):\n",
|
||||
" os.mkdir(PROJECT_ROOT_DIR)\n",
|
||||
"\n",
|
||||
"if not os.path.exists(FIGURE_ID):\n",
|
||||
" os.makedirs(FIGURE_ID)\n",
|
||||
"\n",
|
||||
"if not os.path.exists(DATA_ID):\n",
|
||||
" os.makedirs(DATA_ID)\n",
|
||||
"\n",
|
||||
"def image_path(fig_id):\n",
|
||||
" return os.path.join(FIGURE_ID, fig_id)\n",
|
||||
"\n",
|
||||
"def data_path(dat_id):\n",
|
||||
" return os.path.join(DATA_ID, dat_id)\n",
|
||||
"\n",
|
||||
"def save_fig(fig_id):\n",
|
||||
" plt.savefig(image_path(fig_id) + \".png\", format='png')\n",
|
||||
"import numpy as np\n",
|
||||
"import pandas as pd\n",
|
||||
"from IPython.display import display\n",
|
||||
"np.random.seed(100)\n",
|
||||
"# setting up a 10 x 5 matrix\n",
|
||||
"rows = 10\n",
|
||||
"cols = 5\n",
|
||||
"X = np.random.randn(rows,cols)\n",
|
||||
"XPandas = pd.DataFrame(X)\n",
|
||||
"display(XPandas)\n",
|
||||
"print(XPandas.mean())\n",
|
||||
"print(XPandas.std())\n",
|
||||
"XPandas = (XPandas -XPandas.mean())\n",
|
||||
"display(XPandas)\n",
|
||||
"# This option does not include the standard deviation\n",
|
||||
"scaler = StandardScaler(with_std=False)\n",
|
||||
"scaler.fit(X)\n",
|
||||
"Xscaled = scaler.transform(X)\n",
|
||||
"display(XPandas-Xscaled)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Small exercise: perform the standard scaling by including the standard deviation and compare with what Scikit-Learn gives.\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def FrankeFunction(x,y):\n",
|
||||
"\tterm1 = 0.75*np.exp(-(0.25*(9*x-2)**2) - 0.25*((9*y-2)**2))\n",
|
||||
"\tterm2 = 0.75*np.exp(-((9*x+1)**2)/49.0 - 0.1*(9*y+1))\n",
|
||||
"\tterm3 = 0.5*np.exp(-(9*x-7)**2/4.0 - 0.25*((9*y-3)**2))\n",
|
||||
"\tterm4 = -0.2*np.exp(-(9*x-4)**2 - (9*y-7)**2)\n",
|
||||
"\treturn term1 + term2 + term3 + term4\n",
|
||||
"\n",
|
||||
"Another commonly used scaling method is min-max scaling. This is very\n",
|
||||
"useful for when we want the features to lie in a certain interval. To\n",
|
||||
"scale the feature $x_j$ to the interval $[a, b]$, we can apply the\n",
|
||||
"transformation"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"$$\n",
|
||||
"x_j^{(i)} \\rightarrow (b-a)\\frac{x_j^{(i)} - \\min(x_j)}{\\max(x_j) - \\min(x_j)} - a\n",
|
||||
"$$"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"where $\\min(x_j)$ and $\\max(x_j)$ return the minimum and maximum value of $x_j$ over the data set, respectively.\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def create_X(x, y, n ):\n",
|
||||
"\tif len(x.shape) > 1:\n",
|
||||
"\t\tx = np.ravel(x)\n",
|
||||
"\t\ty = np.ravel(y)\n",
|
||||
"\n",
|
||||
"\tN = len(x)\n",
|
||||
"\tl = int((n+1)*(n+2)/2)\t\t# Number of elements in beta\n",
|
||||
"\tX = np.ones((N,l))\n",
|
||||
"\n",
|
||||
"\tfor i in range(1,n+1):\n",
|
||||
"\t\tq = int((i)*(i+1)/2)\n",
|
||||
"\t\tfor k in range(i+1):\n",
|
||||
"\t\t\tX[:,q+k] = (x**(i-k))*(y**k)\n",
|
||||
"\n",
|
||||
"\treturn X\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# Making meshgrid of datapoints and compute Franke's function\n",
|
||||
"n = 5\n",
|
||||
"N = 1000\n",
|
||||
"x = np.sort(np.random.uniform(0, 1, N))\n",
|
||||
"y = np.sort(np.random.uniform(0, 1, N))\n",
|
||||
"z = FrankeFunction(x, y)\n",
|
||||
"X = create_X(x, y, n=n) \n",
|
||||
"# split in training and test data\n",
|
||||
"X_train, X_test, y_train, y_test = train_test_split(X,z,test_size=0.2)\n",
|
||||
"## Testing the Means Squared Error as function of Complexity\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"clf = skl.LinearRegression().fit(X_train, y_train)\n",
|
||||
"Before we proceed with a more detailed analysis of the so-called\n",
|
||||
"Bias-Variance tradeoff, we present here an example of the relation\n",
|
||||
"between model complexity and the mean squared error for the triaining\n",
|
||||
"data and the test data.\n",
|
||||
"\n",
|
||||
"# The mean squared error and R2 score\n",
|
||||
"print(\"MSE before scaling: {:.2f}\".format(mean_squared_error(clf.predict(X_test), y_test)))\n",
|
||||
"print(\"R2 score before scaling {:.2f}\".format(clf.score(X_test,y_test)))\n",
|
||||
"The results here tell us clearly that for the data not included in the\n",
|
||||
"training, there is an optimal model as function of the complexity of\n",
|
||||
"ourmodel (here in terms of the polynomial degree of the model).\n",
|
||||
"\n",
|
||||
"The results here will vary as function of model complexity and the amount od data used for training. \n",
|
||||
"\n",
|
||||
"\n",
|
||||
"Our data is defined by $x\\in [-3,3]$ with a total of for example $100$ data points."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"editable": true
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import matplotlib.pyplot as plt\n",
|
||||
"import numpy as np\n",
|
||||
"from sklearn.linear_model import LinearRegression, Ridge, Lasso\n",
|
||||
"from sklearn.preprocessing import PolynomialFeatures\n",
|
||||
"from sklearn.model_selection import train_test_split\n",
|
||||
"from sklearn.pipeline import make_pipeline\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"np.random.seed(2018)\n",
|
||||
"n = 100\n",
|
||||
"maxdegree = 14\n",
|
||||
"# Make data set.\n",
|
||||
"x = np.linspace(-3, 3, n).reshape(-1, 1)\n",
|
||||
"y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)\n",
|
||||
"TestError = np.zeros(maxdegree)\n",
|
||||
"TrainError = np.zeros(maxdegree)\n",
|
||||
"polydegree = np.zeros(maxdegree)\n",
|
||||
"x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2)\n",
|
||||
"scaler = StandardScaler()\n",
|
||||
"scaler.fit(X_train)\n",
|
||||
"X_train_scaled = scaler.transform(X_train)\n",
|
||||
"X_test_scaled = scaler.transform(X_test)\n",
|
||||
"scaler.fit(x_train)\n",
|
||||
"x_train_scaled = scaler.transform(x_train)\n",
|
||||
"x_test_scaled = scaler.transform(x_test)\n",
|
||||
"\n",
|
||||
"print(\"Feature min values before scaling:\\n {}\".format(X_train.min(axis=0)))\n",
|
||||
"print(\"Feature max values before scaling:\\n {}\".format(X_train.max(axis=0)))\n",
|
||||
"for degree in range(maxdegree):\n",
|
||||
" model = make_pipeline(PolynomialFeatures(degree=degree), LinearRegression(fit_intercept=False))\n",
|
||||
" clf = model.fit(x_train_scaled,y_train)\n",
|
||||
" y_fit = clf.predict(x_train_scaled)\n",
|
||||
" y_pred = clf.predict(x_test_scaled) \n",
|
||||
" polydegree[degree] = degree\n",
|
||||
" TestError[degree] = np.mean( np.mean((y_test - y_pred)**2) )\n",
|
||||
" TrainError[degree] = np.mean( np.mean((y_train - y_fit)**2) )\n",
|
||||
"\n",
|
||||
"print(\"Feature min values after scaling:\\n {}\".format(X_train_scaled.min(axis=0)))\n",
|
||||
"print(\"Feature max values after scaling:\\n {}\".format(X_train_scaled.max(axis=0)))\n",
|
||||
"\n",
|
||||
"clf = skl.LinearRegression().fit(X_train_scaled, y_train)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"print(\"MSE after scaling: {:.2f}\".format(mean_squared_error(clf.predict(X_test_scaled), y_test)))\n",
|
||||
"print(\"R2 score for scaled data: {:.2f}\".format(clf.score(X_test_scaled,y_test)))"
|
||||
"plt.plot(polydegree, TestError, label='Test Error')\n",
|
||||
"plt.plot(polydegree, TrainError, label='Train Error')\n",
|
||||
"plt.legend()\n",
|
||||
"plt.show()"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
||||
+461
-531
File diff suppressed because it is too large
Load Diff
+964
-4214
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user