lars/FYS-STK4155
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

small typo in bias-variance code

Browse Source
This commit is contained in:
mhjensen
2019-08-21 21:45:24 +02:00
parent 369597638e
commit 931bced261
19 changed files with 744 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files
doc/LectureNotes/fig/pandas.jpg
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 101 KiB

doc/pub/How2ReadData/ipynb/How2ReadData.ipynb
+79 -19
View File
@@ -2325,21 +2325,9 @@
},
{
"cell_type": "code",
"execution_count": 10,
"execution_count": 12,
"metadata": {},
"outputs": [
{
"ename": "FileNotFoundError",
"evalue": "[Errno 2] No such file or directory: 'DataFiles/MassEval2016.dat'",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mFileNotFoundError\u001b[0m Traceback (most recent call last)",
"\u001b[0;32m<ipython-input-10-3cd19a0768e1>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m 31\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msavefig\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mimage_path\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfig_id\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m\".png\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mformat\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'png'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 32\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 33\u001b[0;31m \u001b[0minfile\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata_path\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"MassEval2016.dat\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'r'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
"\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: 'DataFiles/MassEval2016.dat'"
]
}
],
"outputs": [],
"source": [
"# Common imports\n",
"import numpy as np\n",
@@ -2385,7 +2373,7 @@
},
{
"cell_type": "code",
"execution_count": 29,
"execution_count": 13,
"metadata": {},
"outputs": [],
"source": [
@@ -2446,7 +2434,7 @@
},
{
"cell_type": "code",
"execution_count": 31,
"execution_count": 14,
"metadata": {},
"outputs": [],
"source": [
@@ -2488,9 +2476,81 @@
},
{
"cell_type": "code",
"execution_count": 32,
"execution_count": 15,
"metadata": {},
"outputs": [],
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
" N Z A Element Ebinding\n",
"A \n",
"1 0 0 1 1 H 0.000000\n",
"2 1 1 1 2 H 1.112283\n",
"3 2 2 1 3 H 2.827265\n",
"4 6 2 2 4 He 7.073915\n",
"5 9 3 2 5 He 5.512132\n",
"6 14 3 3 6 Li 5.332331\n",
"7 19 4 3 7 Li 5.606439\n",
"8 24 4 4 8 Be 7.062435\n",
"9 29 5 4 9 Be 6.462668\n",
"10 34 6 4 10 Be 6.497630\n",
"11 40 6 5 11 B 6.927732\n",
"12 46 6 6 12 C 7.680144\n",
"13 52 7 6 13 C 7.469849\n",
"14 57 8 6 14 C 7.520319\n",
"15 64 8 7 15 N 7.699460\n",
"16 72 8 8 16 O 7.976206\n",
"17 78 9 8 17 O 7.750728\n",
"18 85 10 8 18 O 7.767097\n",
"19 93 10 9 19 F 7.779018\n",
"20 102 10 10 20 Ne 8.032240\n",
"21 110 11 10 21 Ne 7.971713\n",
"22 118 12 10 22 Ne 8.080465\n",
"23 128 12 11 23 Na 8.111493\n",
"24 137 12 12 24 Mg 8.260709\n",
"25 146 13 12 25 Mg 8.223502\n",
"26 154 14 12 26 Mg 8.333870\n",
"27 164 14 13 27 Al 8.331553\n",
"28 174 14 14 28 Si 8.447744\n",
"29 183 15 14 29 Si 8.448635\n",
"30 192 16 14 30 Si 8.520654\n",
"... ... ... ... ... ...\n",
"238 3089 146 92 238 U 7.570125\n",
"239 3099 146 93 239 Np 7.560567\n",
"240 3109 146 94 240 Pu 7.556042\n",
"241 3118 147 94 241 Pu 7.546439\n",
"242 3127 148 94 242 Pu 7.541327\n",
"243 3136 149 94 243 Pu 7.531008\n",
"244 3144 150 94 244 Pu 7.524815\n",
"245 3154 149 96 245 Cm 7.515767\n",
"246 3162 150 96 246 Cm 7.511471\n",
"247 3170 151 96 247 Cm 7.501931\n",
"248 3177 152 96 248 Cm 7.496728\n",
"249 3186 152 97 249 Bk 7.486040\n",
"250 3194 152 98 250 Cf 7.479956\n",
"251 3201 153 98 251 Cf 7.470500\n",
"252 3209 154 98 252 Cf 7.465347\n",
"253 3216 155 98 253 Cf 7.454829\n",
"254 3224 156 98 254 Cf 7.449225\n",
"255 3232 156 99 255 Es 7.437821\n",
"256 3241 156 100 256 Fm 7.431780\n",
"257 3248 157 100 257 Fm 7.422194\n",
"258 3256 157 101 258 Md 7.409675\n",
"259 3264 157 102 259 No 7.399974\n",
"260 3275 154 106 260 Sg 7.342562\n",
"261 3280 157 104 261 Rf 7.371384\n",
"262 3289 156 106 262 Sg 7.341185\n",
"264 3304 156 108 264 Hs 7.298375\n",
"265 3310 157 108 265 Hs 7.296247\n",
"266 3317 158 108 266 Hs 7.298273\n",
"269 3338 159 110 269 Ds 7.250154\n",
"270 3344 160 110 270 Ds 7.253775\n",
"\n",
"[267 rows x 5 columns]\n"
]
}
],
"source": [
"A = Masses['A']\n",
"Z = Masses['Z']\n",
@@ -2510,7 +2570,7 @@
},
{
"cell_type": "code",
"execution_count": 33,
"execution_count": 16,
"metadata": {},
"outputs": [],
"source": [
doc/pub/Regression/html/._Regression-bs000.html
+1 -1
View File
@@ -405,7 +405,7 @@ MathJax.Hub.Config({
<center>[2] <b>Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University</b></center>
<br>
<p>
<center><h4>Aug 19, 2019</h4></center> <!-- date -->
<center><h4>Aug 21, 2019</h4></center> <!-- date -->
<br>
<p>
doc/pub/Regression/html/._Regression-bs092.html
+1 -1
View File
@@ -431,7 +431,7 @@ x_train, x_test, y_train, y_test <span style="color: #666666">=</span> train_tes
<span style="color: #008000; font-weight: bold">print</span>(<span style="color: #BA2121">&#39;Var:&#39;</span>, variance[degree])
<span style="color: #008000; font-weight: bold">print</span>(<span style="color: #BA2121">&#39;{} &gt;= {} + {} = {}&#39;</span><span style="color: #666666">.</span>format(error[degree], bias[degree], variance[degree], bias[degree]<span style="color: #666666">+</span>variance[degree]))
plt<span style="color: #666666">.</span>plot(polydegree, np<span style="color: #666666">.</span>log10(error), label<span style="color: #666666">=</span><span style="color: #BA2121">&#39;Error&#39;</span>)
plt<span style="color: #666666">.</span>plot(polydegree, error, label<span style="color: #666666">=</span><span style="color: #BA2121">&#39;Error&#39;</span>)
plt<span style="color: #666666">.</span>plot(polydegree, bias, label<span style="color: #666666">=</span><span style="color: #BA2121">&#39;bias&#39;</span>)
plt<span style="color: #666666">.</span>plot(polydegree, variance, label<span style="color: #666666">=</span><span style="color: #BA2121">&#39;Variance&#39;</span>)
plt<span style="color: #666666">.</span>legend()
doc/pub/Regression/html/Regression-bs.html
+1 -1
View File
@@ -405,7 +405,7 @@ MathJax.Hub.Config({
<center>[2] <b>Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University</b></center>
<br>
<p>
<center><h4>Aug 19, 2019</h4></center> <!-- date -->
<center><h4>Aug 21, 2019</h4></center> <!-- date -->
<br>
<p>
doc/pub/Regression/html/Regression-reveal.html
+2 -2
View File
@@ -148,7 +148,7 @@ MathJax.Hub.Config({
<center>[2] <b>Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University</b></center>
<br>
<p>&nbsp;<br>
<center><h4>Aug 19, 2019</h4></center> <!-- date -->
<center><h4>Aug 21, 2019</h4></center> <!-- date -->
<br>
<p>
@@ -3747,7 +3747,7 @@ x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=<span style=
<span style="color: #8B008B; font-weight: bold">print</span>(<span style="color: #CD5555">&#39;Var:&#39;</span>, variance[degree])
<span style="color: #8B008B; font-weight: bold">print</span>(<span style="color: #CD5555">&#39;{} &gt;= {} + {} = {}&#39;</span>.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree]))
plt.plot(polydegree, np.log10(error), label=<span style="color: #CD5555">&#39;Error&#39;</span>)
plt.plot(polydegree, error, label=<span style="color: #CD5555">&#39;Error&#39;</span>)
plt.plot(polydegree, bias, label=<span style="color: #CD5555">&#39;bias&#39;</span>)
plt.plot(polydegree, variance, label=<span style="color: #CD5555">&#39;Variance&#39;</span>)
plt.legend()
doc/pub/Regression/html/Regression-solarized.html
+2 -2
View File
@@ -291,7 +291,7 @@ MathJax.Hub.Config({
<center>[2] <b>Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University</b></center>
<br>
<p>
<center><h4>Aug 19, 2019</h4></center> <!-- date -->
<center><h4>Aug 21, 2019</h4></center> <!-- date -->
<br>
<p>
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
@@ -3636,7 +3636,7 @@ x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=<span style=
<span style="color: #8B008B; font-weight: bold">print</span>(<span style="color: #CD5555">&#39;Var:&#39;</span>, variance[degree])
<span style="color: #8B008B; font-weight: bold">print</span>(<span style="color: #CD5555">&#39;{} &gt;= {} + {} = {}&#39;</span>.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree]))
plt.plot(polydegree, np.log10(error), label=<span style="color: #CD5555">&#39;Error&#39;</span>)
plt.plot(polydegree, error, label=<span style="color: #CD5555">&#39;Error&#39;</span>)
plt.plot(polydegree, bias, label=<span style="color: #CD5555">&#39;bias&#39;</span>)
plt.plot(polydegree, variance, label=<span style="color: #CD5555">&#39;Variance&#39;</span>)
plt.legend()
doc/pub/Regression/html/Regression.html
+2 -2
View File
@@ -296,7 +296,7 @@ MathJax.Hub.Config({
<center>[2] <b>Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University</b></center>
<br>
<p>
<center><h4>Aug 19, 2019</h4></center> <!-- date -->
<center><h4>Aug 21, 2019</h4></center> <!-- date -->
<br>
<p>
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
@@ -3641,7 +3641,7 @@ x_train, x_test, y_train, y_test <span style="color: #666666">=</span> train_tes
<span style="color: #008000; font-weight: bold">print</span>(<span style="color: #BA2121">&#39;Var:&#39;</span>, variance[degree])
<span style="color: #008000; font-weight: bold">print</span>(<span style="color: #BA2121">&#39;{} &gt;= {} + {} = {}&#39;</span><span style="color: #666666">.</span>format(error[degree], bias[degree], variance[degree], bias[degree]<span style="color: #666666">+</span>variance[degree]))
plt<span style="color: #666666">.</span>plot(polydegree, np<span style="color: #666666">.</span>log10(error), label<span style="color: #666666">=</span><span style="color: #BA2121">&#39;Error&#39;</span>)
plt<span style="color: #666666">.</span>plot(polydegree, error, label<span style="color: #666666">=</span><span style="color: #BA2121">&#39;Error&#39;</span>)
plt<span style="color: #666666">.</span>plot(polydegree, bias, label<span style="color: #666666">=</span><span style="color: #BA2121">&#39;bias&#39;</span>)
plt<span style="color: #666666">.</span>plot(polydegree, variance, label<span style="color: #666666">=</span><span style="color: #BA2121">&#39;Variance&#39;</span>)
plt<span style="color: #666666">.</span>legend()
doc/pub/Regression/ipynb/Regression.ipynb
+2 -2
View File
@@ -10,7 +10,7 @@
"<!-- Author: --> \n",
"**Morten Hjorth-Jensen**, Department of Physics, University of Oslo and Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University\n",
"\n",
"Date: **Aug 19, 2019**\n",
"Date: **Aug 21, 2019**\n",
"\n",
"Copyright 1999-2019, Morten Hjorth-Jensen. Released under CC Attribution-NonCommercial 4.0 license\n",
"\n",
@@ -4926,7 +4926,7 @@
" print('Var:', variance[degree])\n",
" print('{} >= {} + {} = {}'.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree]))\n",
"\n",
"plt.plot(polydegree, np.log10(error), label='Error')\n",
"plt.plot(polydegree, error, label='Error')\n",
"plt.plot(polydegree, bias, label='bias')\n",
"plt.plot(polydegree, variance, label='Variance')\n",
"plt.legend()\n",
doc/pub/Regression/ipynb/ipynb-Regression-src.tar.gz
BIN
View File
Binary file not shown.
doc/pub/Regression/pdf/Regression-minted.pdf
BIN
View File
Binary file not shown.
doc/src/Projects/2019/Exercises/clean.sh
Executable
+3
View File
@@ -0,0 +1,3 @@
#!/bin/sh
doconce clean
rm -rf *.pdf *.tex ipynb*.tar.gz *.html ._*.html *~ reveal.js Trash README.txt
doc/src/Projects/2019/Exercises/hw1.do.txt
+157
View File
@@ -0,0 +1,157 @@
TITLE: Homework 1 Fall Semester 2019
AUTHOR: "Data Analysis and Machine Learning FYS-STK3155/FYS4155":"http://www.uio.no/studier/emner/matnat/fys/FYS3155/index-eng.html" {copyright, 1999-present|CC BY-NC} at Department of Physics, University of Oslo, Norway
DATE:Today
===== Exercise 1 =====
The first exercise here is of a mere technical art. We want you to have
* git as a version control software and to establish a user account on a provider like GitHub. Other providers like GitLab etc are equally fine. You can also use the University of Oslo "GitHub facilities":"https://www.uio.no/tjenester/it/maskin/filer/versjonskontroll/github.html".
* Install various Python packages
We will make extensive use of Python as programming language and its
myriad of available libraries. You will find
IPython/Jupyter notebooks invaluable in your work. You can run _R_
codes in the Jupyter/IPython notebooks, with the immediate benefit of
visualizing your data. You can also use compiled languages like C++,
Rust, Fortran etc if you prefer. The focus in these lectures will be
on Python.
If you have Python installed (we recommend Python3) and you feel
pretty familiar with installing different packages, we recommend that
you install the following Python packages via _pip_ as
o pip install numpy scipy matplotlib ipython scikit-learn sympy pandas pillow
For _Tensorflow_, we recommend following the instructions in the text of
"Aurelien Geron, Hands‑On Machine Learning with Scikit‑Learn and TensorFlow, O'Reilly":"http://shop.oreilly.com/product/0636920052289.do"
We will come back to _tensorflow_ later.
For Python3, replace _pip_ with _pip3_.
For OSX users we recommend, after having installed Xcode, to
install _brew_. Brew allows for a seamless installation of additional
software via for example
o brew install python3
For Linux users, with its variety of distributions like for example the widely popular Ubuntu distribution,
you can use _pip_ as well and simply install Python as
o sudo apt-get install python3 (or python for pyhton2.7)
If you don't want to perform these operations separately and venture
into the hassle of exploring how to set up dependencies and paths, we
recommend two widely used distrubutions which set up all relevant
dependencies for Python, namely
* "Anaconda":"https://docs.anaconda.com/",
which is an open source
distribution of the Python and R programming languages for large-scale
data processing, predictive analytics, and scientific computing, that
aims to simplify package management and deployment. Package versions
are managed by the package management system _conda_.
* "Enthought canopy":"https://www.enthought.com/product/canopy/"
is a Python
distribution for scientific and analytic computing distribution and
analysis environment, available for free and under a commercial
license.
We recommend using _Anaconda_.
===== Exercise 2 =====
We will generate our own dataset for a function $y(x)$ where $x \in [0,1]$ and defined by random numbers computed with the uniform distribution. The function $y$ is a quadratic polynomial in $x$ with added stochastic noise according to the normal distribution $\cal {N}(0,1)$.
The following simple Python instructions define our $x$ and $y$ values (with 100 data points).
!bc pycod
x = np.random.rand(100,1)
y = 5*x*x+0.1*np.random.randn(100,1)
!ec
o Write your own code (following the examples under the "regression slides":"https://compphysics.github.io/MachineLearning/doc/pub/Regression/html/Regression-bs.html") for computing the parametrization of the data set fitting a second-order polynomial.
o Use thereafter _scikit-learn_ (see again the examples in the regression slides) and compare with your own code.
o Using scikit-learn, compute also the mean square error, a risk metric corresponding to the expected value of the squared (quadratic) error defined as
!bt
\[ MSE(\hat{y},\hat{\tilde{y}}) = \frac{1}{n}
\sum_{i=0}^{n-1}(y_i-\tilde{y}_i)^2,
\]
!et
and the $R^2$ score function.
If $\tilde{\hat{y}}_i$ is the predicted value of the $i-th$ sample and $y_i$ is the corresponding true value, then the score $R^2$ is defined as
!bt
\[
R^2(\hat{y}, \tilde{\hat{y}}) = 1 - \frac{\sum_{i=0}^{n - 1} (y_i - \tilde{y}_i)^2}{\sum_{i=0}^{n - 1} (y_i - \bar{y})^2},
\]
!et
where we have defined the mean value of $\hat{y}$ as
!bt
\[
\bar{y} = \frac{1}{n} \sum_{i=0}^{n - 1} y_i.
\]
!et
You can use the functionality included in scikit-learn. If you feel for it, you can use your own program and define functions which compute the above two functions.
Discuss the meaning of these results. Try also to vary the coefficient in front of the added stochastic noise term and discuss the quality of the fits.
===== Exercise 3, mean values and variances in linear regression =====
This exercise deals with various mean values ad variances in linear regression method (here it may be useful to look up chapter 3, equation (3.8) of "Trevor Hastie, Robert Tibshirani, Jerome H. Friedman, The Elements of Statistical Learning, Springer":"https://www.springer.com/gp/book/9780387848570").
The assumption we have made is
that there exists a function $f(\bm{x})$ and a normal distributed error $\bm{\varepsilon}\sim \mathcal{N}(0, \sigma^2)$
which describes our data
!bt
\[
\bm{y} = f(\bm{x})+\bm{\varepsilon}
\]
!et
We then approximate this function with our model from the solution of the linear regression equations (ordinary least squares OLS), that is our
function $f$ is approximated by $\bm{\tilde{y}}$ where we minimized $(\bm{y}-\bm{\tilde{y}})^2$, with
!bt
\[
\bm{\tilde{y}} = \bm{X}\bm{\beta}.
\]
!et
The matrix $\bm{X}$ is the so-called design matrix.
Show that the expectation value of $\bm{y}$ for a given element $i$
!bt
\begin{align*}
\mathbb{E}(y_i) & =\mathbf{X}_{i, \ast} \, \beta,
\end{align*}
!et
and that
its variance is
!bt
\begin{align*} \mbox{Var}(y_i) & = \sigma^2.
\end{align*}
!et
Hence, $y_i \sim \mathcal{N}( \mathbf{X}_{i, \ast} \, \bm{\beta}, \sigma^2)$, that is $\bm{y}$ follows a normal distribution with
mean value $\bm{X}\bm{\beta}$ and variance $\sigma^2$.
With the OLS expressions for the parameters $\bm{\beta}$ show that
!bt
\[
\mathbb{E}(\bm{\beta}) = \bm{\beta}.
\]
!et
This means that the estimator of the regression parameters is unbiased.
Show finally that the variance of $\bm{\beta}$ is
!bt
\begin{eqnarray*}
\mbox{Var}(\bm{\beta}) & = & \sigma^2 \, (\mathbf{X}^{T} \mathbf{X})^{-1}.
\end{eqnarray*}
!et
doc/src/Projects/2019/Exercises/hw2.do.txt
+65
View File
@@ -0,0 +1,65 @@
TITLE: Homework 2
AUTHOR: "Data Analysis and Machine Learning FYS-STK3155/FYS4155":"http://www.uio.no/studier/emner/matnat/fys/FYS3155/index-eng.html" {copyright, 1999-present|CC BY-NC} at Department of Physics, University of Oslo, Norway
DATE:Today
===== Exercise 4 =====
This exercise is a continuation of exercise 2 from homework 1. We will
use the same function to generate our data set, still staying with a
simple function $y(x)$ which we want to fit using linear regression,
but now extending the analysis to include the Ridge and the Lasso
regression methods. You can use the code under the Regression as an example on how to use the Ridge and the Lasso methods, see the "regression slides":"https://compphysics.github.io/MachineLearning/doc/pub/Regression/html/Regression-bs.html").
We will thus again generate our own dataset for a function $y(x)$ where
$x \in [0,1]$ and defined by random numbers computed with the uniform
distribution. The function $y$ is a quadratic polynomial in $x$ with
added stochastic noise according to the normal distribution $\cal{N}(0,1)$.
The following simple Python instructions define our $x$ and $y$ values (with 100 data points).
!bc pycod
x = np.random.rand(100,1)
y = 5*x*x+0.1*np.random.randn(100,1)
!ec
o Write your own code for the Ridge method (see chapter 3.4 of Hastie *et al.*, equations (3.43) and (3.44)) and compute the parametrization for different values of $\lambda$. Compare and analyze your results with those from exercise 2. Study the dependence on $\lambda$ while also varying the strength of the noise in your expression for $y(x)$.
o Repeat the above but using the functionality of _scikit-learn_. Compare your code with the results from _scikit-learn_. Remember to run with the same random numbers for generating $x$ and $y$.
o Our next step is to study the variance of the parameters $\beta_1$ and $\beta_2$ (assuming that we are parametrizing our function with a second-order polynomial. We will use standard linear regression and the Ridge regression. You can now opt for either writing your own function that calculates the variance of these paramaters (recall that this is equal to the diagonal elements of the matrix $(\hat{X}^T\hat{X})+\lambda\hat{I})^{-1}$) or use the functionality of _scikit-learn_ and compute their variances. Discuss the results of these variances as functions of $\lambda$. In particular, try to link your discussion with the discussion in Hastie *et al.* and their figure 3.11.
o Repeat the previous step but add now the Lasso method, see equation (3.53) of Hastie *et al.*. Discuss your results and compare with standard regression and the Ridge regression results. You can write your own code or use the functionality of _scikit-learn_.
o Finally, using _scikit-learn_ or your own code, compute also the mean square error, a risk metric corresponding to the expected value of the squared (quadratic) error defined as
!bt
\[ MSE(\hat{y},\hat{\tilde{y}}) = \frac{1}{n}
\sum_{i=0}^{n-1}(y_i-\tilde{y}_i)^2,
\]
!et
and the $R^2$ score function.
If $\tilde{\hat{y}}_i$ is the predicted value of the $i-th$ sample and $y_i$ is the corresponding true value, then the score $R^2$ is defined as
!bt
\[
R^2(\hat{y}, \tilde{\hat{y}}) = 1 - \frac{\sum_{i=0}^{n - 1} (y_i - \tilde{y}_i)^2}{\sum_{i=0}^{n - 1} (y_i - \bar{y})^2},
\]
!et
where we have defined the mean value of $\hat{y}$ as
!bt
\[
\bar{y} = \frac{1}{n} \sum_{i=0}^{n - 1} y_i.
\]
!et
Discuss these quantities as functions of the variable $\lambda$ in the Ridge and Lasso regression methods.
===== Exercise 5 =====
Using the singular value decomposition, show that the variance of the direction vector
$\hat{z}_i=\hat{X}\hat{v}_i=\hat{u}_1d_1$ is equal to (equation (3.49) of Hastie *et al.*)
!bt
\[
\mathrm{Var}(\hat{z}_i)=\frac{d_i^2}{N},
\]
!et
where $d_i$ are the singular values of the matrix $\hat{X}$. In Hastie *et al*, the matrix elements of $X$ are centered. The consequence is that the mean values of for example $\hat{u}_i$ are zero.
Give an interpretation of these results, in particular in connection with the variance of the coefficients you obtained in the previous exercise.
doc/src/Projects/2019/Exercises/make.sh
Executable
+79
View File
@@ -0,0 +1,79 @@
#!/bin/sh
set -x
function system {
"$@"
if [ $? -ne 0 ]; then
echo "make.sh: unsuccessful command $@"
echo "abort!"
exit 1
fi
}
if [ $# -eq 0 ]; then
echo 'bash make.sh slides1|slides2'
exit 1
fi
name=$1
rm -f *.tar.gz
opt="--encoding=utf-8"
opt=
rm -f *.aux
# Plain HTML documents
html=${name}
system doconce format html $name --pygments_html_style=default --html_style=bloodish --html_links_in_new_window --html_output=$html $opt
system doconce split_html $html.html --method=space10
# Bootstrap style
html=${name}-bs
system doconce format html $name --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=$html $opt
system doconce split_html $html.html --method=split --pagination --nav_button=bottom
# Ordinary plain LaTeX document
system doconce format pdflatex $name --print_latex_style=trac --latex_admon=paragraph $opt
system doconce ptex2tex $name envir=print
# Add special packages
doconce subst "% Add user's preamble" "\g<1>\n\\usepackage{simplewick}" $name.tex
doconce replace 'section{' 'section*{' $name.tex
pdflatex -shell-escape $name
pdflatex -shell-escape $name
mv -f $name.pdf ${name}.pdf
cp $name.tex ${name}.tex
# Publish
dest=../../../../Projects/2019
if [ ! -d $dest/$name ]; then
mkdir $dest/$name
mkdir $dest/$name/pdf
mkdir $dest/$name/html
mkdir $dest/$name/ipynb
fi
cp ${name}*.tex $dest/$name/pdf
cp ${name}*.pdf $dest/$name/pdf
cp -r ${name}*.html ._${name}*.html $dest/$name/html
# Figures: cannot just copy link, need to physically copy the files
if [ -d fig-${name} ]; then
if [ ! -d $dest/$name/html/fig-$name ]; then
mkdir $dest/$name/html/fig-$name
fi
cp -r fig-${name}/* $dest/$name/html/fig-$name
fi
cp ${name}.ipynb $dest/$name/ipynb
ipynb_tarfile=ipynb-${name}-src.tar.gz
if [ ! -f ${ipynb_tarfile} ]; then
cat > README.txt <<EOF
This IPython notebook ${name}.ipynb does not require any additional
programs.
EOF
tar czf ${ipynb_tarfile} README.txt
fi
cp ${ipynb_tarfile} $dest/$name/ipynb
doc/src/Projects/2019/Project1/Project1.do.txt
+267
View File
@@ -0,0 +1,267 @@
TITLE: Project 1 on Machine Learning, deadline October 1
AUTHOR: "Data Analysis and Machine Learning FYS-STK3155/FYS4155":"http://www.uio.no/studier/emner/matnat/fys/FYS3155/index-eng.html" {copyright, 1999-present|CC BY-NC} at Department of Physics, University of Oslo, Norway
DATE: today
===== Regression analysis and resampling methods =====
The main aim of this project is to study in more detail various
regression methods, including the Ordinary Least Squares (OLS) method,
Ridge regression and finally Lasso regression.
The methods are in turn combined with resampling techniques.
We will first study how
to fit polynomials to a specific two-dimensional function called
"Franke's
function":"http://www.dtic.mil/dtic/tr/fulltext/u2/a081688.pdf". This
is a function which has been widely used when testing various interpolation and fitting
algorithms. Furthermore, after having etsablished the model and the
method, we will employ resamling techniques such as the cross-validation and/or
the bootstrap methods, in order to perform a proper assessment of our models.
The Franke function, which is a weighted sum of four exponentials reads as follows
!bt
\begin{align*}
f(x,y) &= \frac{3}{4}\exp{\left(-\frac{(9x-2)^2}{4} - \frac{(9y-2)^2}{4}\right)}+\frac{3}{4}\exp{\left(-\frac{(9x+1)^2}{49}- \frac{(9y+1)}{10}\right)} \\
&+\frac{1}{2}\exp{\left(-\frac{(9x-7)^2}{4} - \frac{(9y-3)^2}{4}\right)} -\frac{1}{5}\exp{\left(-(9x-4)^2 - (9y-7)^2\right) }.
\end{align*}
!et
The function will be defined for $x,y\in [0,1]$. Our first step will
be to perform an OLS regression analysis of this function, trying out
a polynomial fit with an $x$ and $y$ dependence of the form $[x, y,
x^2, y^2, xy, \dots]$. We will also include cross-validation and
bootstrap as resampling techniques. As in homeworks 1 and 2, we
can use a uniform distribution to set up the arrays of values for $x$
and $y$, or as in the example below just a fix values for $x$ and $y$ with a given step size.
In this case we will have two predictors and need to fit a
function (for example a polynomial) of $x$ and $y$. Thereafter we will
repeat much of the same procedure using the the Ridge and
Lasso regression methods, introducing thus a dependence on the bias
(penalty) $\lambda$.
Thereafter we are going to use (real) digital terrain data and try to
reproduce these data using the same methods. We will also try to go
beyond the second-order polynomials metioned above and explore
which polynomial fits the data best.
The Python fucntion for the Franke function is included here (it performs also a three-dimensional plot of it)
!bc pycod
from mpl_toolkits.mplot3d import Axes3D
import matplotlib.pyplot as plt
from matplotlib import cm
from matplotlib.ticker import LinearLocator, FormatStrFormatter
import numpy as np
from random import random, seed
fig = plt.figure()
ax = fig.gca(projection='3d')
# Make data.
x = np.arange(0, 1, 0.05)
y = np.arange(0, 1, 0.05)
x, y = np.meshgrid(x,y)
def FrankeFunction(x,y):
term1 = 0.75*np.exp(-(0.25*(9*x-2)**2) - 0.25*((9*y-2)**2))
term2 = 0.75*np.exp(-((9*x+1)**2)/49.0 - 0.1*(9*y+1))
term3 = 0.5*np.exp(-(9*x-7)**2/4.0 - 0.25*((9*y-3)**2))
term4 = -0.2*np.exp(-(9*x-4)**2 - (9*y-7)**2)
return term1 + term2 + term3 + term4
z = FrankeFunction(x, y)
# Plot the surface.
surf = ax.plot_surface(x, y, z, cmap=cm.coolwarm,
linewidth=0, antialiased=False)
# Customize the z axis.
ax.set_zlim(-0.10, 1.40)
ax.zaxis.set_major_locator(LinearLocator(10))
ax.zaxis.set_major_formatter(FormatStrFormatter('%.02f'))
# Add a color bar which maps values to colors.
fig.colorbar(surf, shrink=0.5, aspect=5)
plt.show()
!ec
=== Part a): Ordinary Least Square on the Franke function with resampling ===
We will thus again generate our own dataset for a function $\mathrm{FrankeFunction}(x,y)$ where
$x,y \in [0,1]$ could be defined by random numbers computed with the uniform
distribution. The function $f(x,y)$ is the Franke function. You should explore also the addition
an added stochastic noise to this function using the normal distribution $\cal{N}(0,1)$.
Write your own code (using either a matrix inversion or a singular value decomposition from e.g., _numpy_ ) or use your code from homeworks 1 and 2
and perform a standard least square regression analysis using polynomials in $x$ and $y$ up to fifth order. Find the confidence intervals of the parameters $\beta$ by computing their variances, evaluate the Mean Squared error (MSE)
!bt
\[ MSE(\hat{y},\hat{\tilde{y}}) = \frac{1}{n}
\sum_{i=0}^{n-1}(y_i-\tilde{y}_i)^2,
\]
!et
and the $R^2$ score function.
If $\tilde{\hat{y}}_i$ is the predicted value of the $i-th$ sample and $y_i$ is the corresponding true value, then the score $R^2$ is defined as
!bt
\[
R^2(\hat{y}, \tilde{\hat{y}}) = 1 - \frac{\sum_{i=0}^{n - 1} (y_i - \tilde{y}_i)^2}{\sum_{i=0}^{n - 1} (y_i - \bar{y})^2},
\]
!et
where we have defined the mean value of $\hat{y}$ as
!bt
\[
\bar{y} = \frac{1}{n} \sum_{i=0}^{n - 1} y_i.
\]
!et
Perform a resampling of the data where you split the data in training data and test data. Implement the $k$-fold cross-validation algorithm and/or the bootstrap algorithm
and evaluate again the MSE and the $R^2$ functions resulting from the test data. Evaluate also the bias and variance of the final models using for example equation (7.9) in the textbook of Hastie *et al.*
=== Part b): Ridge Regression on the Franke function with resampling ===
Write your own code for the Ridge method, either using matrix inversion or the singular value decomposition as done in the previous exercise or howework 2 (see also chapter 3.4 of Hastie *et al.*, equations (3.43) and (3.44)). Perform the same analysis as in the previous exercise (for the same polynomials and include resampling techniques) but now for different values of $\lambda$. Compare and analyze your results with those obtained in part a). Study the dependence on $\lambda$ while also varying eventually the strength of the noise in your expression for $\mathrm{FrankeFunction}(x,y)$.
=== Part c): Lasso Regression on the Franke function with resampling ===
This part is essentially a repeat of the previous two ones, but now with Lasso regression. Write either your own code or, in this case, you can also use the functionalities of _scikit-learn_. Give a critical discussion of the three methods and a judgement of which model fits the data best.
=== Part d): Introducing real data ===
With our codes functioning and having been tested properly on a simpler function we are now ready to look at real data. We will essentially repeat in part e) what was done in parts a-c). However, we need first to download the data and prepare properly the inputs to our codes.
We are going to download digital terrain data from the website URL:"https://earthexplorer.usgs.gov/",
In order to obtain data for a specific region, you need to register as a user (free) at this website and then decide upon which area you want to fetch the digital terrain data from. In order to be able to read the data properly, you need to specify that the format should be _SRTM Arc-Second Global_ and download the data as a _GeoTIF_ file.
The files are then stored in *tif* format which can be imported into a Python program using
!bc pycod
scipy.misc.imread
!ec
Here is a simple part of a Python code which reads and plots the data from such files
!bc pycod
import numpy as np
from imageio import imread
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
from matplotlib import cm
# Load the terrain
terrain1 = imread('SRTM_data_Norway_1.tif')
# Show the terrain
plt.figure()
plt.title('Terrain over Norway 1')
plt.imshow(terrain1, cmap='gray')
plt.xlabel('X')
plt.ylabel('Y')
plt.show()
!ec
If you should have problems in downloading the digital terrain data, we provide two examples under the data folder of project 1. One is from a region close to Stavanger in Norway and the other Møsvatn Austfjell, again in Norway.
=== Part e) OLS, Ridge and Lasso regression with resampling ===
Our final part deals with the parameterization of your digital terrain data. We will apply all three methods for linear regression as in parts a-c), the same type (or higher order) of polynomial approximation and the same resampling techniques to evaluate which model fits the data best.
At the end, you should pesent a critical evaluation of your results and discuss the applicability of these regression methods to the type of data presented here.
===== Background literature =====
o For a discussion and derivation of the variances and mean squared errors using linear regression, see the "Lecture notes on ridge regression by Wessel N. van Wieringen":"https://arxiv.org/abs/1509.09169"
o The textbook of "Trevor Hastie, Robert Tibshirani, Jerome H. Friedman, The Elements of Statistical Learning, Springer":"https://www.springer.com/gp/book/9780387848570", chapters 3 and 7 are the most relevant ones for the analysis here.
===== Introduction to numerical projects =====
Here follows a brief recipe and recommendation on how to write a report for each
project.
* Give a short description of the nature of the problem and the eventual numerical methods you have used.
* Describe the algorithm you have used and/or developed. Here you may find it convenient to use pseudocoding. In many cases you can describe the algorithm in the program itself.
* Include the source code of your program. Comment your program properly.
* If possible, try to find analytic solutions, or known limits in order to test your program when developing the code.
* Include your results either in figure form or in a table. Remember to label your results. All tables and figures should have relevant captions and labels on the axes.
* Try to evaluate the reliabilty and numerical stability/precision of your results. If possible, include a qualitative and/or quantitative discussion of the numerical stability, eventual loss of precision etc.
* Try to give an interpretation of you results in your answers to the problems.
* Critique: if possible include your comments and reflections about the exercise, whether you felt you learnt something, ideas for improvements and other thoughts you've made when solving the exercise. We wish to keep this course at the interactive level and your comments can help us improve it.
* Try to establish a practice where you log your work at the computerlab. You may find such a logbook very handy at later stages in your work, especially when you don't properly remember what a previous test version of your program did. Here you could also record the time spent on solving the exercise, various algorithms you may have tested or other topics which you feel worthy of mentioning.
===== Format for electronic delivery of report and programs =====
The preferred format for the report is a PDF file. You can also use DOC or postscript formats or as an ipython notebook file. As programming language we prefer that you choose between C/C++, Fortran2008 or Python. The following prescription should be followed when preparing the report:
* Use Devilry to hand in your projects, log in at URL:"http://devilry.ifi.uio.no" with your normal UiO username and password and choose either 'fysstk3155' or 'fysstk4155'. There you can load up the files within the deadline.
* Upload _only_ the report file! For the source code file(s) you have developed please provide us with your link to your github domain. The report file should include all of your discussions and a list of the codes you have developed. Do not include library files which are available at the course homepage, unless you have made specific changes to them.
* In your git repository, please include a folder which contains selected results. These can be in the form of output from your code for a selected set of runs and input parameters.
* In this and all later projects, you should include tests (for example unit tests) of your code(s).
* Comments from us on your projects, approval or not, corrections to be made etc can be found under your Devilry domain and are only visible to you and the teachers of the course.
Finally,
we encourage you to collaborate. Optimal working groups consist of
2-3 students. You can then hand in a common report.
===== Software and needed installations =====
If you have Python installed (we recommend Python3) and you feel pretty familiar with installing different packages,
we recommend that you install the following Python packages via _pip_ as
o pip install numpy scipy matplotlib ipython scikit-learn tensorflow sympy pandas pillow
For Python3, replace _pip_ with _pip3_.
See below for a discussion of _tensorflow_ and _scikit-learn_.
For OSX users we recommend also, after having installed Xcode, to install _brew_. Brew allows
for a seamless installation of additional software via for example
o brew install python3
For Linux users, with its variety of distributions like for example the widely popular Ubuntu distribution
you can use _pip_ as well and simply install Python as
o sudo apt-get install python3 (or python for python2.7)
etc etc.
If you don't want to install various Python packages with their dependencies separately, we recommend two widely used distrubutions which set up all relevant dependencies for Python, namely
o "Anaconda":"https://docs.anaconda.com/" Anaconda is an open source distribution of the Python and R programming languages for large-scale data processing, predictive analytics, and scientific computing, that aims to simplify package management and deployment. Package versions are managed by the package management system _conda_
o "Enthought canopy":"https://www.enthought.com/product/canopy/" is a Python distribution for scientific and analytic computing distribution and analysis environment, available for free and under a commercial license.
Popular software packages written in Python for ML are
* "Scikit-learn":"http://scikit-learn.org/stable/",
* "Tensorflow":"https://www.tensorflow.org/",
* "PyTorch":"http://pytorch.org/" and
* "Keras":"https://keras.io/".
These are all freely available at their respective GitHub sites. They
encompass communities of developers in the thousands or more. And the number
of code developers and contributors keeps increasing.
doc/src/Projects/2019/Project1/clean.sh
Executable
+3
View File
@@ -0,0 +1,3 @@
#!/bin/sh
doconce clean
rm -rf *.pdf *.tex ipynb*.tar.gz *.html ._*.html *~ reveal.js Trash README.txt
doc/src/Projects/2019/Project1/make.sh
Executable
+79
View File
@@ -0,0 +1,79 @@
#!/bin/sh
set -x
function system {
"$@"
if [ $? -ne 0 ]; then
echo "make.sh: unsuccessful command $@"
echo "abort!"
exit 1
fi
}
if [ $# -eq 0 ]; then
echo 'bash make.sh slides1|slides2'
exit 1
fi
name=$1
rm -f *.tar.gz
opt="--encoding=utf-8"
opt=
rm -f *.aux
# Plain HTML documents
html=${name}
system doconce format html $name --pygments_html_style=default --html_style=bloodish --html_links_in_new_window --html_output=$html $opt
system doconce split_html $html.html --method=space10
# Bootstrap style
html=${name}-bs
system doconce format html $name --html_style=bootstrap --pygments_html_style=default --html_admon=bootstrap_panel --html_output=$html $opt
system doconce split_html $html.html --method=split --pagination --nav_button=bottom
# Ordinary plain LaTeX document
system doconce format pdflatex $name --print_latex_style=trac --latex_admon=paragraph $opt
system doconce ptex2tex $name envir=print
# Add special packages
doconce subst "% Add user's preamble" "\g<1>\n\\usepackage{simplewick}" $name.tex
doconce replace 'section{' 'section*{' $name.tex
pdflatex -shell-escape $name
pdflatex -shell-escape $name
mv -f $name.pdf ${name}.pdf
cp $name.tex ${name}.tex
# Publish
dest=../../../../Projects/2018
if [ ! -d $dest/$name ]; then
mkdir $dest/$name
mkdir $dest/$name/pdf
mkdir $dest/$name/html
mkdir $dest/$name/ipynb
fi
cp ${name}*.tex $dest/$name/pdf
cp ${name}*.pdf $dest/$name/pdf
cp -r ${name}*.html ._${name}*.html $dest/$name/html
# Figures: cannot just copy link, need to physically copy the files
if [ -d fig-${name} ]; then
if [ ! -d $dest/$name/html/fig-$name ]; then
mkdir $dest/$name/html/fig-$name
fi
cp -r fig-${name}/* $dest/$name/html/fig-$name
fi
cp ${name}.ipynb $dest/$name/ipynb
ipynb_tarfile=ipynb-${name}-src.tar.gz
if [ ! -f ${ipynb_tarfile} ]; then
cat > README.txt <<EOF
This IPython notebook ${name}.ipynb does not require any additional
programs.
EOF
tar czf ${ipynb_tarfile} README.txt
fi
cp ${ipynb_tarfile} $dest/$name/ipynb
doc/src/Regression/Regression.do.txt
+1 -1
View File
@@ -3017,7 +3017,7 @@ for degree in range(maxdegree):
print('Var:', variance[degree])
print('{} >= {} + {} = {}'.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree]))
plt.plot(polydegree, np.log10(error), label='Error')
plt.plot(polydegree, error, label='Error')
plt.plot(polydegree, bias, label='bias')
plt.plot(polydegree, variance, label='Variance')
plt.legend()