From b42b287094ea55a1121716d1366e6fc5b18e7719 Mon Sep 17 00:00:00 2001 From: mhjensen Date: Fri, 24 Aug 2018 06:02:33 +0200 Subject: [PATCH] added html files --- .../Regression/html/._Regression-bs019.html | 285 +++++++++++++++ .../Regression/html/._Regression-bs020.html | 322 +++++++++++++++++ .../Regression/html/._Regression-bs021.html | 276 +++++++++++++++ .../Regression/html/._Regression-bs022.html | 267 ++++++++++++++ .../Regression/html/._Regression-bs023.html | 276 +++++++++++++++ .../Regression/html/._Regression-bs024.html | 275 +++++++++++++++ .../Regression/html/._Regression-bs025.html | 294 ++++++++++++++++ .../Regression/html/._Regression-bs026.html | 299 ++++++++++++++++ .../Regression/html/._Regression-bs027.html | 271 +++++++++++++++ .../Regression/html/._Regression-bs028.html | 276 +++++++++++++++ .../Regression/html/._Regression-bs029.html | 276 +++++++++++++++ .../Regression/html/._Regression-bs030.html | 298 ++++++++++++++++ .../Regression/html/._Regression-bs031.html | 293 ++++++++++++++++ .../Regression/html/._Regression-bs032.html | 273 +++++++++++++++ .../Regression/html/._Regression-bs033.html | 278 +++++++++++++++ .../Regression/html/._Regression-bs034.html | 327 ++++++++++++++++++ .../Regression/html/._Regression-bs035.html | 263 ++++++++++++++ .../Regression/html/._Regression-bs036.html | 252 ++++++++++++++ .../Regression/html/._Regression-bs037.html | 252 ++++++++++++++ doc/src/Regression/Regression.do.txt | 2 +- 20 files changed, 5354 insertions(+), 1 deletion(-) create mode 100644 doc/pub/Regression/html/._Regression-bs019.html create mode 100644 doc/pub/Regression/html/._Regression-bs020.html create mode 100644 doc/pub/Regression/html/._Regression-bs021.html create mode 100644 doc/pub/Regression/html/._Regression-bs022.html create mode 100644 doc/pub/Regression/html/._Regression-bs023.html create mode 100644 doc/pub/Regression/html/._Regression-bs024.html create mode 100644 doc/pub/Regression/html/._Regression-bs025.html create mode 100644 doc/pub/Regression/html/._Regression-bs026.html create mode 100644 doc/pub/Regression/html/._Regression-bs027.html create mode 100644 doc/pub/Regression/html/._Regression-bs028.html create mode 100644 doc/pub/Regression/html/._Regression-bs029.html create mode 100644 doc/pub/Regression/html/._Regression-bs030.html create mode 100644 doc/pub/Regression/html/._Regression-bs031.html create mode 100644 doc/pub/Regression/html/._Regression-bs032.html create mode 100644 doc/pub/Regression/html/._Regression-bs033.html create mode 100644 doc/pub/Regression/html/._Regression-bs034.html create mode 100644 doc/pub/Regression/html/._Regression-bs035.html create mode 100644 doc/pub/Regression/html/._Regression-bs036.html create mode 100644 doc/pub/Regression/html/._Regression-bs037.html diff --git a/doc/pub/Regression/html/._Regression-bs019.html b/doc/pub/Regression/html/._Regression-bs019.html new file mode 100644 index 000000000..98985f9d7 --- /dev/null +++ b/doc/pub/Regression/html/._Regression-bs019.html @@ -0,0 +1,285 @@ + + + + + + + +Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Simple regression model, now using scikit-learn

+ +

+We can repeat the above algorithm using scikit-learn as follows +

+ + +

# Importing various packages
+from random import random, seed
+import numpy as np
+import matplotlib.pyplot as plt
+from sklearn.linear_model import LinearRegression
+
+x = 2*np.random.rand(100,1)
+y = 4+3*x+np.random.randn(100,1)
+linreg = LinearRegression()
+linreg.fit(x,y)
+xnew = np.array([[0],[2]])
+ypredict = linreg.predict(xnew)
+
+plt.plot(xnew, ypredict, "r-")
+plt.plot(x, y ,'ro')
+plt.axis([0,2.0,0, 15.0])
+plt.xlabel(r'$x$')
+plt.ylabel(r'$y$')
+plt.title(r'Random numbers ')
+plt.show()
+
+

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/Regression/html/._Regression-bs020.html b/doc/pub/Regression/html/._Regression-bs020.html new file mode 100644 index 000000000..420ad0471 --- /dev/null +++ b/doc/pub/Regression/html/._Regression-bs020.html @@ -0,0 +1,322 @@ + + + + + + + +Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Simple linear regression model using scikit-learn

+ +

+We start with perhaps our simplest possible example, using scikit-learn to perform linear regression analysis on a data set produced by us. +What follows is a simple Python code where we have defined function \( y \) in terms of the variable \( x \). Both are defined as vectors of dimension \( 1\times 100 \). The entries to the vector \( \hat{x} \) are given by random numbers generated with a uniform distribution with entries \( x_i \in [0,1] \) (more about probability distribution functions later). These values are then used to define a function \( y(x) \) (tabulated again as a vector) with a linear dependence on \( x \) plus a random noise added via the normal distribution. + +

+The Numpy functions are imported used the import numpy as np +statement and the random number generator for the uniform distribution +is called using the function np.random.rand(), where we specificy +that we want \( 100 \) random variables. Using Numpy we define +automatically an array with the specified number of elements, \( 100 \) in +our case. With the Numpy function randn() we can compute random +numbers with the normal distribution (mean value \( \mu \) equal to zero and +variance \( \sigma^2 \) set to one) and produce the values of \( y \) assuming a linear +dependence as function of \( x \) + +$$ +y = 2x+N(0,1), +$$ + +

+where \( N(0,1) \) represents random numbers generated by the normal +distribution. From scikit-learn we import then the +LinearRegression functionality and make a prediction \( \tilde{y} = +\alpha + \beta x \) using the function fit(x,y). We call the set of +data \( (\hat{x},\hat{y}) \) for our training data. The Python package +scikit-learn has also a functionality which extracts the above +fitting parameters \( \alpha \) and \( \beta \) (see below). Later we will +distinguish between training data and test data. + +

+For plotting we use the Python package +matplotlib which produces publication +quality figures. Feel free to explore the extensive +gallery of examples. In +this example we plot our original values of \( x \) and \( y \) as well as the +prediction ypredict (\( \tilde{y} \)), which attempts at fitting our +data with a straight line. + +

+The Python code follows here. +

+ + +

# Importing various packages
+import numpy as np
+import matplotlib.pyplot as plt
+from sklearn.linear_model import LinearRegression
+
+x = np.random.rand(100,1)
+y = 2*x+np.random.randn(100,1)
+linreg = LinearRegression()
+linreg.fit(x,y)
+xnew = np.array([[0],[1]])
+ypredict = linreg.predict(xnew)
+
+plt.plot(xnew, ypredict, "r-")
+plt.plot(x, y ,'ro')
+plt.axis([0,1.0,0, 5.0])
+plt.xlabel(r'$x$')
+plt.ylabel(r'$y$')
+plt.title(r'Simple Linear Regression')
+plt.show()
+
+

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/Regression/html/._Regression-bs021.html b/doc/pub/Regression/html/._Regression-bs021.html new file mode 100644 index 000000000..c1f975f55 --- /dev/null +++ b/doc/pub/Regression/html/._Regression-bs021.html @@ -0,0 +1,276 @@ + + + + + + + +Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Simple linear regression model

+ +

+This example serves several aims. It allows us to demonstrate several +aspects of data analysis and later machine learning algorithms. The +immediate visualization shows that our linear fit is not +impressive. It goes through the data points, but there are many +outliers which are not reproduced by our linear regression. We could +now play around with this small program and change for example the +factor in front of \( x \) and the normal distribution. Try to change the +function \( y \) to + +$$ +y = 10x+0.01 \times N(0,1), +$$ + +

+where \( x \) is defined as before. + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/Regression/html/._Regression-bs022.html b/doc/pub/Regression/html/._Regression-bs022.html new file mode 100644 index 000000000..2487b3f53 --- /dev/null +++ b/doc/pub/Regression/html/._Regression-bs022.html @@ -0,0 +1,267 @@ + + + + + + + +Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Less noise

+ +

+Does the fit look better? Indeed, by +reducing the role of the normal distribution we see immediately that +our linear prediction seemingly reproduces better the training +set. However, this testing 'by the eye' is obviouly not satisfactory in the +long run. Here we have only defined the training data and our model, and +have not discussed a more rigorous approach to the cost function. + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/Regression/html/._Regression-bs023.html b/doc/pub/Regression/html/._Regression-bs023.html new file mode 100644 index 000000000..bd1b9f6a5 --- /dev/null +++ b/doc/pub/Regression/html/._Regression-bs023.html @@ -0,0 +1,276 @@ + + + + + + + +Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

How to study our fits

+ +

+We need more rigorous criteria in defining whether we have succeeded or +not in modeling our training data. You will be surprised to see that +many scientists seldomly venture beyond this 'by the eye' approach. A +standard approach for the cost function is the so-called \( \chi^2 \) +function + +$$ \chi^2 = \frac{1}{n} +\sum_{i=0}^{n-1}\frac{(y_i-\tilde{y}_i)^2}{\sigma_i^2}, +$$ + +

+where \( \sigma_i^2 \) is the variance (to be defined later) of the entry +\( y_i \). We may not know the explicit value of \( \sigma_i^2 \), it serves +however the aim of scaling the equations and make the cost function +dimensionless. + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/Regression/html/._Regression-bs024.html b/doc/pub/Regression/html/._Regression-bs024.html new file mode 100644 index 000000000..4979e3750 --- /dev/null +++ b/doc/pub/Regression/html/._Regression-bs024.html @@ -0,0 +1,275 @@ + + + + + + + +Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Minimizing the cost function

+ +

+Minimizing the cost function is a central aspect of +our discussions to come. Finding its minima as function of the model +parameters (\( \alpha \) and \( \beta \) in our case) will be a recurring +theme in these series of lectures. Essentially all machine learning +algorithms we will discuss center around the minimization of the +chosen cost function. This depends in turn on our specific +model for describing the data, a typical situation in supervised +learning. Automatizing the search for the minima of the cost function is a +central ingredient in all algorithms. Typical methods which are +employed are various variants of gradient methods. These will be +discussed in more detail later. Again, you'll be surprised to hear that +many practitioners minimize the above function ''by the eye', popularly dubbed as +'chi by the eye'. That is, change a parameter and see (visually and numerically) that +the \( \chi^2 \) function becomes smaller. + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/Regression/html/._Regression-bs025.html b/doc/pub/Regression/html/._Regression-bs025.html new file mode 100644 index 000000000..3596923c1 --- /dev/null +++ b/doc/pub/Regression/html/._Regression-bs025.html @@ -0,0 +1,294 @@ + + + + + + + +Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Relative error

+ +

+There are many ways to define the cost function. A simpler approach is to look at the relative difference between the training data and the predicted data, that is we define +the relative error as + +$$ +\epsilon_{\mathrm{relative}}= \frac{\vert \hat{y} -\hat{\tilde{y}}\vert}{\vert \hat{y}\vert}. +$$ + +We can modify easily the above Python code and plot the relative error instead +

+ + +

import numpy as np
+import matplotlib.pyplot as plt
+from sklearn.linear_model import LinearRegression
+
+x = np.random.rand(100,1)
+y = 5*x+0.01*np.random.randn(100,1)
+linreg = LinearRegression()
+linreg.fit(x,y)
+ypredict = linreg.predict(x)
+
+plt.plot(x, np.abs(ypredict-y)/abs(y), "ro")
+plt.axis([0,1.0,0.0, 0.5])
+plt.xlabel(r'$x$')
+plt.ylabel(r'$\epsilon_{\mathrm{relative}}$')
+plt.title(r'Relative error')
+plt.show()
+
+

+Depending on the parameter in front of the normal distribution, we may +have a small or larger relative error. Try to play around with +different training data sets and study (graphically) the value of the +relative error. + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/Regression/html/._Regression-bs026.html b/doc/pub/Regression/html/._Regression-bs026.html new file mode 100644 index 000000000..f827affc9 --- /dev/null +++ b/doc/pub/Regression/html/._Regression-bs026.html @@ -0,0 +1,299 @@ + + + + + + + +Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

The richness of scikit-learn

+ +

+As mentioned above, scikit-learn has an impressive functionality. +We can for example extract the values of \( \alpha \) and \( \beta \) and +their error estimates, or the variance and standard deviation and many +other properties from the statistical data analysis. + +

+Here we show an +example of the functionality of scikit-learn. +

+ + +

import numpy as np 
+import matplotlib.pyplot as plt 
+from sklearn.linear_model import LinearRegression 
+from sklearn.metrics import mean_squared_error, r2_score, mean_squared_log_error, mean_absolute_error
+
+x = np.random.rand(100,1)
+y = 2.0+ 5*x+0.5*np.random.randn(100,1)
+linreg = LinearRegression()
+linreg.fit(x,y)
+ypredict = linreg.predict(x)
+print('The intercept alpha: \n', linreg.intercept_)
+print('Coefficient beta : \n', linreg.coef_)
+# The mean squared error                               
+print("Mean squared error: %.2f" % mean_squared_error(y, ypredict))
+# Explained variance score: 1 is perfect prediction                                 
+print('Variance score: %.2f' % r2_score(y, ypredict))
+# Mean squared log error                                                        
+print('Mean squared log error: %.2f' % mean_squared_log_error(y, ypredict) )
+# Mean absolute error                                                           
+print('Mean absolute error: %.2f' % mean_absolute_error(y, ypredict))
+plt.plot(x, ypredict, "r-")
+plt.plot(x, y ,'ro')
+plt.axis([0.0,1.0,1.5, 7.0])
+plt.xlabel(r'$x$')
+plt.ylabel(r'$y$')
+plt.title(r'Linear Regression fit ')
+plt.show()
+
+

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/Regression/html/._Regression-bs027.html b/doc/pub/Regression/html/._Regression-bs027.html new file mode 100644 index 000000000..d68ae908b --- /dev/null +++ b/doc/pub/Regression/html/._Regression-bs027.html @@ -0,0 +1,271 @@ + + + + + + + +Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Functions in scikit-learn

+ +

+The function coef gives us the parameter \( \beta \) of our fit while intercept yields +\( \alpha \). Depending on the constant in front of the normal distribution, we get values near or far from \( alpha =2 \) and \( \beta =5 \). Try to play around with different parameters in front of the normal distribution. The function meansquarederror gives us the mean square error, a risk metric corresponding to the expected value of the squared (quadratic) error or loss defined as +$$ MSE(\hat{y},\hat{\tilde{y}}) = \frac{1}{n} +\sum_{i=0}^{n-1}(y_i-\tilde{y}_i)^2, +$$ + +

+The smaller the value, the better the fit. Ideally we would like to +have an MSE equal zero. The attentive reader has probably recognized +this function as being similar to the \( \chi^2 \) function defined above. + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/Regression/html/._Regression-bs028.html b/doc/pub/Regression/html/._Regression-bs028.html new file mode 100644 index 000000000..39f493b62 --- /dev/null +++ b/doc/pub/Regression/html/._Regression-bs028.html @@ -0,0 +1,276 @@ + + + + + + + +Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Other functions in scikit-learn

+ +

+The r2score function computes \( R^2 \), the coefficient of +determination. It provides a measure of how well future samples are +likely to be predicted by the model. Best possible score is 1.0 and it +can be negative (because the model can be arbitrarily worse). A +constant model that always predicts the expected value of \( \hat{y} \), +disregarding the input features, would get a \( R^2 \) score of \( 0.0 \). + +

+If \( \tilde{\hat{y}}_i \) is the predicted value of the \( i-th \) sample and \( y_i \) is the corresponding true value, then the score \( R^2 \) is defined as +$$ +R^2(\hat{y}, \tilde{\hat{y}}) = 1 - \frac{\sum_{i=0}^{n - 1} (y_i - \tilde{y}_i)^2}{\sum_{i=0}^{n - 1} (y_i - \bar{y})^2}, +$$ + +where we have defined the mean value of \( \hat{y} \) as +$$ +\bar{y} = \frac{1}{n} \sum_{i=0}^{n - 1} y_i. +$$ + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/Regression/html/._Regression-bs029.html b/doc/pub/Regression/html/._Regression-bs029.html new file mode 100644 index 000000000..6a182d377 --- /dev/null +++ b/doc/pub/Regression/html/._Regression-bs029.html @@ -0,0 +1,276 @@ + + + + + + + +Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

The mean absolute error and other functions in scikit-learn

+ +

+Another quantity will meet again in our discussions of regression analysis is + mean absolute error (MAE), a risk metric corresponding to the expected value of the absolute error loss or what we call the \( l1 \)-norm loss. In our discussion above we presented the relative error. +The MAE is defined as follows +$$ +\text{MAE}(\hat{y}, \hat{\tilde{y}}) = \frac{1}{n} \sum_{i=0}^{n-1} \left| y_i - \tilde{y}_i \right|. +$$ + +Finally we present the +squared logarithmic (quadratic) error +$$ +\text{MSLE}(\hat{y}, \hat{\tilde{y}}) = \frac{1}{n} \sum_{i=0}^{n - 1} (\log_e (1 + y_i) - \log_e (1 + \tilde{y}_i) )^2, +$$ + +

+where \( \log_e (x) \) stands for the natural logarithm of \( x \). This error +estimate is best to use when targets having exponential growth, such +as population counts, average sales of a commodity over a span of +years etc. + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/Regression/html/._Regression-bs030.html b/doc/pub/Regression/html/._Regression-bs030.html new file mode 100644 index 000000000..a0e4dfa92 --- /dev/null +++ b/doc/pub/Regression/html/._Regression-bs030.html @@ -0,0 +1,298 @@ + + + + + + + +Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Cubic polynomial in scikit-learn

+ +

+We will discuss in more +detail these and other functions in the various lectures. We conclude this part with another example. Instead of +a linear \( x \)-dependence we study now a cubic polynomial and use the polynomial regression analysis tools of scikit-learn. + +

+ + +

import matplotlib.pyplot as plt
+import numpy as np
+import random
+from sklearn.linear_model import Ridge
+from sklearn.preprocessing import PolynomialFeatures
+from sklearn.pipeline import make_pipeline
+from sklearn.linear_model import LinearRegression
+
+x=np.linspace(0.02,0.98,200)
+noise = np.asarray(random.sample((range(200)),200))
+y=x**3*noise
+yn=x**3*100
+poly3 = PolynomialFeatures(degree=3)
+X = poly3.fit_transform(x[:,np.newaxis])
+clf3 = LinearRegression()
+clf3.fit(X,y)
+
+Xplot=poly3.fit_transform(x[:,np.newaxis])
+poly3_plot=plt.plot(x, clf3.predict(Xplot), label='Cubic Fit')
+plt.plot(x,yn, color='red', label="True Cubic")
+plt.scatter(x, y, label='Data', color='orange', s=15)
+plt.legend()
+plt.show()
+
+def error(a):
+    for i in y:
+        err=(y-yn)/yn
+    return abs(np.sum(err))/len(err)
+
+print (error(y))
+
+

+Similarly, using R, we can perform similar studies. +(more details on R will be inserted later). + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/Regression/html/._Regression-bs031.html b/doc/pub/Regression/html/._Regression-bs031.html new file mode 100644 index 000000000..3ad7cabb6 --- /dev/null +++ b/doc/pub/Regression/html/._Regression-bs031.html @@ -0,0 +1,293 @@ + + + + + + + +Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Simple regression model with gradient descent

+Add info about the equations, play around with different learning rates +

+ + +

# Importing various packages
+from math import exp, sqrt
+from random import random, seed
+import numpy as np
+import matplotlib.pyplot as plt
+
+x = 2*np.random.rand(100,1)
+y = 4+3*x+np.random.randn(100,1)
+
+xb = np.c_[np.ones((100,1)), x]
+theta_linreg = np.linalg.inv(xb.T.dot(xb)).dot(xb.T).dot(y)
+print(theta_linreg)
+theta = np.random.randn(2,1)
+
+eta = 0.1
+Niterations = 1000
+m = 100
+
+for iter in range(Niterations):
+    gradients = 2.0/m*xb.T.dot(xb.dot(theta)-y)
+    theta -= eta*gradients
+
+print(theta)
+xnew = np.array([[0],[2]])
+xbnew = np.c_[np.ones((2,1)), xnew]
+ypredict = xbnew.dot(theta)
+ypredict2 = xbnew.dot(theta_linreg)
+plt.plot(xnew, ypredict, "r-")
+plt.plot(xnew, ypredict2, "b-")
+plt.plot(x, y ,'ro')
+plt.axis([0,2.0,0, 15.0])
+plt.xlabel(r'$x$')
+plt.ylabel(r'$y$')
+plt.title(r'Random numbers ')
+plt.show()
+
+

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/Regression/html/._Regression-bs032.html b/doc/pub/Regression/html/._Regression-bs032.html new file mode 100644 index 000000000..cae569fdf --- /dev/null +++ b/doc/pub/Regression/html/._Regression-bs032.html @@ -0,0 +1,273 @@ + + + + + + + +Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Simple regression model with stochastic gradient descent

+Add info about the equations, play around with different learning rates +

+ + +

# Importing various packages
+from math import exp, sqrt
+from random import random, seed
+import numpy as np
+import matplotlib.pyplot as plt
+from sklearn.linear_model import SGDRegressor
+
+x = 2*np.random.rand(100,1)
+y = 4+3*x+np.random.randn(100,1)
+
+xb = np.c_[np.ones((100,1)), x]
+theta_linreg = np.linalg.inv(xb.T.dot(xb)).dot(xb.T).dot(y)
+print(theta_linreg)
+sgdreg = SGDRegressor(n_iter = 50, penalty=None, eta0=0.1)
+sgdreg.fit(x,y.ravel())
+print(sgdreg.intercept_, sgdreg.coef_)
+
+

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/Regression/html/._Regression-bs033.html b/doc/pub/Regression/html/._Regression-bs033.html new file mode 100644 index 000000000..21186180b --- /dev/null +++ b/doc/pub/Regression/html/._Regression-bs033.html @@ -0,0 +1,278 @@ + + + + + + + +Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Polynomial Regression

+

+ + +

# Importing various packages
+from math import exp, sqrt
+from random import random, seed
+import numpy as np
+import matplotlib.pyplot as plt
+
+m = 100
+x = 2*np.random.rand(m,1)+4.
+y = 4+3*x*x+ +x-np.random.randn(m,1)
+
+xb = np.c_[np.ones((m,1)), x]
+theta = np.linalg.inv(xb.T.dot(xb)).dot(xb.T).dot(y)
+xnew = np.array([[0],[2]])
+xbnew = np.c_[np.ones((2,1)), xnew]
+ypredict = xbnew.dot(theta)
+
+plt.plot(xnew, ypredict, "r-")
+plt.plot(x, y ,'ro')
+plt.axis([0,2.0,0, 15.0])
+plt.xlabel(r'$x$')
+plt.ylabel(r'$y$')
+plt.title(r'Random numbers ')
+plt.show()
+
+

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/Regression/html/._Regression-bs034.html b/doc/pub/Regression/html/._Regression-bs034.html new file mode 100644 index 000000000..364fa6719 --- /dev/null +++ b/doc/pub/Regression/html/._Regression-bs034.html @@ -0,0 +1,327 @@ + + + + + + + +Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Ridge and Lasso Regression

+ +

+ + +

import matplotlib.pyplot as plt
+import numpy as np
+from sklearn import linear_model
+from sklearn.linear_model import LinearRegression
+from sklearn.metrics import mean_squared_error, r2_score
+
+#creating data with random noise
+x=np.arange(50)
+
+delta=np.random.uniform(-2.5,2.5, size=(50))
+np.random.shuffle(delta)
+y =0.5*x+5+delta
+
+#arranging data into 2x50 matrix
+a=np.array(x) #inputs
+b=np.array(y) #outputs
+
+#Split into training and test
+X_train=a[:37, np.newaxis]
+X_test=a[37:, np.newaxis]
+y_train=b[:37]
+y_test=b[37:]
+
+print ("X_train: ", X_train.shape)
+print ("y_train: ", y_train.shape)
+print ("X_test: ", X_test.shape)
+print ("y_test: ", y_test.shape)
+
+print ("------------------------------------")
+
+print ("Ordinary Least Squares")
+#Add Ordinary Least Squares fit
+reg=LinearRegression()
+reg.fit(X_train, y_train)
+pred=reg.predict(X_test)
+print ("Prediction Shape: ", pred.shape)
+
+print('Coefficients: \n', reg.coef_)
+# The mean squared error
+print("Mean squared error: %.2f"
+      % mean_squared_error(y_test, pred))
+# Explained variance score: 1 is perfect prediction
+print('Variance score: %.2f' % r2_score(y_test, pred))
+
+#plot
+plt.scatter(X_test,y_test,color='green', label="Training Data")
+plt.plot(X_test, pred, color='black', label="Fit Line")
+plt.legend()
+plt.show()
+
+print ("------------------------------------")
+
+print ("Ridge Regression")
+
+ridge=linear_model.RidgeCV(alphas=[0.1,1.0,10.0])
+ridge.fit(X_train,y_train)
+print ("Ridge Coefficient: ",ridge.coef_)
+print ("Ridge Intercept: ", ridge.intercept_)
+#Look into graphing with Ridge fit
+
+print ("------------------------------------")
+
+print ("Lasso")
+lasso=linear_model.Lasso(alpha=0.1)
+lasso.fit(X_train,y_train)
+predl=lasso.predict(X_test)
+print("Lasso Coefficient: ", lasso.coef_)
+print("Lasso Intercept: ", lasso.intercept_)
+plt.scatter(X_test,y_test,color='green', label="Training Data")
+plt.plot(X_test, predl, color='blue', label="Lasso")
+plt.legend()
+plt.show()
+
+

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/Regression/html/._Regression-bs035.html b/doc/pub/Regression/html/._Regression-bs035.html new file mode 100644 index 000000000..476608042 --- /dev/null +++ b/doc/pub/Regression/html/._Regression-bs035.html @@ -0,0 +1,263 @@ + + + + + + + +Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

The singular value decompostion

+
+
+

+How can we use the singular value decomposition to find the parameters \( \beta_j \)? More details will come. We first note that a general \( m\times n \) matrix \( \hat{A} \) can be written in terms of a diagonal matrix \( \hat{\Sigma} \) of dimensionality \( n\times n \) and two orthognal matrices \( \hat{U} \) and \( \hat{V} \), where the first has dimensionality \( m \times n \) and the last dimensionality \( n\times n \). We have then +$$ +\hat{A} = \hat{U}\hat{\Sigma}\hat{V} +$$ +

+
+ + +

+Add codes and discuss this in connection with lasso and ridge, show example where the standard inversion of a matrix fails and where SVD comes to rescue + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/Regression/html/._Regression-bs036.html b/doc/pub/Regression/html/._Regression-bs036.html new file mode 100644 index 000000000..b5551cea1 --- /dev/null +++ b/doc/pub/Regression/html/._Regression-bs036.html @@ -0,0 +1,252 @@ + + + + + + + +Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Lasso and Ridge regression

+ +

+Discuss the mathematics here + +

+

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/pub/Regression/html/._Regression-bs037.html b/doc/pub/Regression/html/._Regression-bs037.html new file mode 100644 index 000000000..1a3204fc2 --- /dev/null +++ b/doc/pub/Regression/html/._Regression-bs037.html @@ -0,0 +1,252 @@ + + + + + + + +Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + Data Analysis and Machine Learning: Linear Regression and more Advanced Regression Analysis +
+ +
+ +
+
+ + +
+ +

 

 

 

+ + + + +

Logistic regression

+Add discussion about classification versus regression, show examples of more than two cases and why regression is not the best approach. Motivate for k-nearest neighbors + +

+Add examples on classification problems + +

+ +

+ +

+ + +
+ + + + + + + +
+ +
+ + + + + + diff --git a/doc/src/Regression/Regression.do.txt b/doc/src/Regression/Regression.do.txt index 783a0975e..b0c6e086f 100644 --- a/doc/src/Regression/Regression.do.txt +++ b/doc/src/Regression/Regression.do.txt @@ -393,7 +393,7 @@ and We define then !bt \[ -\gamma = \sum_{i=0}^{1}\frac{n-1}{\sigma_i^2}, +\gamma = \sum_{i=0}^{n-1}\frac{n-1}{\sigma_i^2}, \] !et