diff --git a/doc/LectureNotes/_build/.doctrees/chapter1.doctree b/doc/LectureNotes/_build/.doctrees/chapter1.doctree
index 30c0bee2e..90c4c8184 100644
Binary files a/doc/LectureNotes/_build/.doctrees/chapter1.doctree and b/doc/LectureNotes/_build/.doctrees/chapter1.doctree differ
diff --git a/doc/LectureNotes/_build/.doctrees/chapter2.doctree b/doc/LectureNotes/_build/.doctrees/chapter2.doctree
index 55068749b..5622196e5 100644
Binary files a/doc/LectureNotes/_build/.doctrees/chapter2.doctree and b/doc/LectureNotes/_build/.doctrees/chapter2.doctree differ
diff --git a/doc/LectureNotes/_build/.doctrees/chapter3.doctree b/doc/LectureNotes/_build/.doctrees/chapter3.doctree
index ceb0eabba..677298c6a 100644
Binary files a/doc/LectureNotes/_build/.doctrees/chapter3.doctree and b/doc/LectureNotes/_build/.doctrees/chapter3.doctree differ
diff --git a/doc/LectureNotes/_build/.doctrees/chapter4.doctree b/doc/LectureNotes/_build/.doctrees/chapter4.doctree
index fc69eb650..1bb3dd684 100644
Binary files a/doc/LectureNotes/_build/.doctrees/chapter4.doctree and b/doc/LectureNotes/_build/.doctrees/chapter4.doctree differ
diff --git a/doc/LectureNotes/_build/.doctrees/environment.pickle b/doc/LectureNotes/_build/.doctrees/environment.pickle
index 47d7ee608..a1fb76610 100644
Binary files a/doc/LectureNotes/_build/.doctrees/environment.pickle and b/doc/LectureNotes/_build/.doctrees/environment.pickle differ
diff --git a/doc/LectureNotes/_build/html/_images/chapter1_11_0.png b/doc/LectureNotes/_build/html/_images/chapter1_11_0.png
index d56e1d6d0..8585b7491 100644
Binary files a/doc/LectureNotes/_build/html/_images/chapter1_11_0.png and b/doc/LectureNotes/_build/html/_images/chapter1_11_0.png differ
diff --git a/doc/LectureNotes/_build/html/_images/chapter1_13_1.png b/doc/LectureNotes/_build/html/_images/chapter1_13_1.png
index 3fd1831e1..b145ce82a 100644
Binary files a/doc/LectureNotes/_build/html/_images/chapter1_13_1.png and b/doc/LectureNotes/_build/html/_images/chapter1_13_1.png differ
diff --git a/doc/LectureNotes/_build/html/_images/chapter1_214_0.png b/doc/LectureNotes/_build/html/_images/chapter1_214_0.png
new file mode 100644
index 000000000..cbac4fd70
Binary files /dev/null and b/doc/LectureNotes/_build/html/_images/chapter1_214_0.png differ
diff --git a/doc/LectureNotes/_build/html/_images/chapter1_27_0.png b/doc/LectureNotes/_build/html/_images/chapter1_27_0.png
index ab7dda310..bef273ce3 100644
Binary files a/doc/LectureNotes/_build/html/_images/chapter1_27_0.png and b/doc/LectureNotes/_build/html/_images/chapter1_27_0.png differ
diff --git a/doc/LectureNotes/_build/html/_images/chapter1_3_0.png b/doc/LectureNotes/_build/html/_images/chapter1_3_0.png
index 4ad4c8690..0be2e475d 100644
Binary files a/doc/LectureNotes/_build/html/_images/chapter1_3_0.png and b/doc/LectureNotes/_build/html/_images/chapter1_3_0.png differ
diff --git a/doc/LectureNotes/_build/html/_images/chapter1_61_11.png b/doc/LectureNotes/_build/html/_images/chapter1_61_11.png
new file mode 100644
index 000000000..916bfd378
Binary files /dev/null and b/doc/LectureNotes/_build/html/_images/chapter1_61_11.png differ
diff --git a/doc/LectureNotes/_build/html/_images/chapter2_245_2.png b/doc/LectureNotes/_build/html/_images/chapter2_245_2.png
new file mode 100644
index 000000000..dc3a3ce3e
Binary files /dev/null and b/doc/LectureNotes/_build/html/_images/chapter2_245_2.png differ
diff --git a/doc/LectureNotes/_build/html/_images/chapter2_247_1.png b/doc/LectureNotes/_build/html/_images/chapter2_247_1.png
new file mode 100644
index 000000000..ac2df9e69
Binary files /dev/null and b/doc/LectureNotes/_build/html/_images/chapter2_247_1.png differ
diff --git a/doc/LectureNotes/_build/html/_images/chapter2_249_1.png b/doc/LectureNotes/_build/html/_images/chapter2_249_1.png
new file mode 100644
index 000000000..9666ad291
Binary files /dev/null and b/doc/LectureNotes/_build/html/_images/chapter2_249_1.png differ
diff --git a/doc/LectureNotes/_build/html/_images/chapter2_313_2.png b/doc/LectureNotes/_build/html/_images/chapter2_313_2.png
new file mode 100644
index 000000000..07584f6bb
Binary files /dev/null and b/doc/LectureNotes/_build/html/_images/chapter2_313_2.png differ
diff --git a/doc/LectureNotes/_build/html/_images/chapter3_25_2.png b/doc/LectureNotes/_build/html/_images/chapter3_25_2.png
index 98e9e3a97..b77a1febb 100644
Binary files a/doc/LectureNotes/_build/html/_images/chapter3_25_2.png and b/doc/LectureNotes/_build/html/_images/chapter3_25_2.png differ
diff --git a/doc/LectureNotes/_build/html/_images/chapter4_13_0.png b/doc/LectureNotes/_build/html/_images/chapter4_13_0.png
new file mode 100644
index 000000000..cde80495d
Binary files /dev/null and b/doc/LectureNotes/_build/html/_images/chapter4_13_0.png differ
diff --git a/doc/LectureNotes/_build/html/_images/chapter4_13_1.png b/doc/LectureNotes/_build/html/_images/chapter4_13_1.png
new file mode 100644
index 000000000..aee9096ee
Binary files /dev/null and b/doc/LectureNotes/_build/html/_images/chapter4_13_1.png differ
diff --git a/doc/LectureNotes/_build/html/_images/chapter4_13_2.png b/doc/LectureNotes/_build/html/_images/chapter4_13_2.png
new file mode 100644
index 000000000..0a17d7002
Binary files /dev/null and b/doc/LectureNotes/_build/html/_images/chapter4_13_2.png differ
diff --git a/doc/LectureNotes/_build/html/_images/chapter4_51_0.png b/doc/LectureNotes/_build/html/_images/chapter4_51_0.png
new file mode 100644
index 000000000..903d98702
Binary files /dev/null and b/doc/LectureNotes/_build/html/_images/chapter4_51_0.png differ
diff --git a/doc/LectureNotes/_build/html/_images/chapter4_51_1.png b/doc/LectureNotes/_build/html/_images/chapter4_51_1.png
new file mode 100644
index 000000000..42ba3f001
Binary files /dev/null and b/doc/LectureNotes/_build/html/_images/chapter4_51_1.png differ
diff --git a/doc/LectureNotes/_build/html/_images/chapter4_5_1.png b/doc/LectureNotes/_build/html/_images/chapter4_5_1.png
new file mode 100644
index 000000000..89d6b14ac
Binary files /dev/null and b/doc/LectureNotes/_build/html/_images/chapter4_5_1.png differ
diff --git a/doc/LectureNotes/_build/html/_images/chapter4_7_0.png b/doc/LectureNotes/_build/html/_images/chapter4_7_0.png
new file mode 100644
index 000000000..de0c567ed
Binary files /dev/null and b/doc/LectureNotes/_build/html/_images/chapter4_7_0.png differ
diff --git a/doc/LectureNotes/_build/html/_sources/chapter1.ipynb b/doc/LectureNotes/_build/html/_sources/chapter1.ipynb
index 756ce7708..c930e0a6b 100644
--- a/doc/LectureNotes/_build/html/_sources/chapter1.ipynb
+++ b/doc/LectureNotes/_build/html/_sources/chapter1.ipynb
@@ -6,8 +6,6 @@
"source": [
"# Linear Regression\n",
"\n",
- "[Video of Lecture](https://www.uio.no/studier/emner/matnat/fys/FYS-STK3155/h20/forelesningsvideoer/LectureAug21.mp4?vrtx=view-as-webpage)\n",
- "\n",
"\n",
"## Introduction\n",
"\n",
@@ -15,7 +13,7 @@
"\n",
"\n",
"\n",
- "Our emphasis throughout this series of lectures (small change) \n",
+ "Our emphasis throughout this series of lectures \n",
"is on understanding the mathematical aspects of\n",
"different algorithms used in the fields of data analysis and machine learning. \n",
"\n",
@@ -235,7 +233,9 @@
"\n",
"## Simple linear regression model using **scikit-learn**\n",
"\n",
- "We start with perhaps our simplest possible example, using **Scikit-Learn** to perform linear regression analysis on a data set produced by us. \n",
+ "We start with perhaps our simplest possible example, using\n",
+ "**Scikit-Learn** to perform linear regression analysis on a data set\n",
+ "produced by us.\n",
"\n",
"What follows is a simple Python code where we have defined a function\n",
"$y$ in terms of the variable $x$. Both are defined as vectors with $100$ entries. \n",
@@ -437,7 +437,8 @@
"import numpy as np\n",
"import matplotlib.pyplot as plt\n",
"from sklearn.linear_model import LinearRegression\n",
- "\n",
+ "# Number of data points\n",
+ "n = 100\n",
"x = np.random.rand(100,1)\n",
"y = 5*x+0.01*np.random.randn(100,1)\n",
"linreg = LinearRegression()\n",
@@ -459,7 +460,8 @@
"Depending on the parameter in front of the normal distribution, we may\n",
"have a small or larger relative error. Try to play around with\n",
"different training data sets and study (graphically) the value of the\n",
- "relative error.\n",
+ "relative error. Note also that **Scikit-Learn** requires a matrix as input for the input values $x$ and $y$. In the above code we have\n",
+ "solved this by declaring $x$ and $y$ as arrays of dimension $n\\times 1$.\n",
"\n",
"As mentioned above, **Scikit-Learn** has an impressive functionality.\n",
"We can for example extract the values of $\\alpha$ and $\\beta$ and\n",
@@ -629,8 +631,7 @@
"metadata": {},
"source": [
"$$\n",
- "H_{\\delta}(\\boldsymbol{a})=\\left\\{\\begin{array}{cc}\\frac{1}{2} \\boldsymbol{a}^{2}& \\text{for }|\\boldsymbol{a}|\\leq \\delta\\\\ \\delta (|\\b\\\n",
- "m{a}|-\\frac{1}{2}\\delta ),&\\text{otherwise}.\\end{array}\\right.\n",
+ "H_{\\delta}(\\boldsymbol{a})=\\left\\{\\begin{array}{cc}\\frac{1}{2} \\boldsymbol{a}^{2}& \\text{for }|\\boldsymbol{a}|\\leq \\delta\\\\ \\delta (|\\boldsymbol{a}|-\\frac{1}{2}\\delta ),&\\text{otherwise}.\\end{array}\\right.\n",
"$$"
]
},
@@ -641,6 +642,8 @@
"Here $\\boldsymbol{a}=\\boldsymbol{y} - \\boldsymbol{\\tilde{y}}$.\n",
"\n",
"\n",
+ "\n",
+ "\n",
"We will discuss in more\n",
"detail these and other functions in the various lectures. We conclude this part with another example. Instead of \n",
"a linear $x$-dependence we study now a cubic polynomial and use the polynomial regression analysis tools of scikit-learn."
@@ -1061,6 +1064,8 @@
"cell_type": "markdown",
"metadata": {},
"source": [
+ "Note well that we have made life simple here. We perform a fit in terms of the number of nucleons only. A more sophisticated fit can be done by including an explicit dependence on the number of protons and neutrons in the asymmetry and Coulomb terms.\n",
+ "\n",
"With **scikitlearn** we are now ready to use linear regression and fit our data."
]
},
@@ -1100,7 +1105,6 @@
"print('Variance score: %.2f' % r2_score(Energies, fity))\n",
"# Mean absolute error \n",
"print('Mean absolute error: %.2f' % mean_absolute_error(Energies, fity))\n",
- "print(clf.coef_, clf.intercept_)\n",
"\n",
"Masses['Eapprox'] = fity\n",
"# Generate a plot comparing the experimental with the fitted values values.\n",
@@ -3046,6 +3050,119 @@
"cell_type": "markdown",
"metadata": {},
"source": [
+ "## Splitting our Data in Training and Test data\n",
+ "\n",
+ "\n",
+ "It is normal in essentially all Machine Learning studies to split the\n",
+ "data in a training set and a test set (sometimes also an additional\n",
+ "validation set). **Scikit-Learn** has an own function for this. There\n",
+ "is no explicit recipe for how much data should be included as training\n",
+ "data and say test data. An accepted rule of thumb is to use\n",
+ "approximately $2/3$ to $4/5$ of the data as training data. We will\n",
+ "postpone a discussion of this splitting to the end of these notes and\n",
+ "our discussion of the so-called **bias-variance** tradeoff. Here we\n",
+ "limit ourselves to repeat the above equation of state fitting example\n",
+ "but now splitting the data into a training set and a test set.\n",
+ "\n",
+ "Let us study some examples. The first code here takes a simple\n",
+ "one-dimensional second-order polynomial and we fit it to a\n",
+ "second-order polynomial. Depending on the strength of the added noise,\n",
+ "the various measures like the $R2$ score or the mean-squared error,\n",
+ "the fit becomes better or worse."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": false,
+ "editable": true
+ },
+ "outputs": [],
+ "source": [
+ "import os\n",
+ "import numpy as np\n",
+ "import pandas as pd\n",
+ "import matplotlib.pyplot as plt\n",
+ "from sklearn.model_selection import train_test_split\n",
+ "\n",
+ "\n",
+ "def R2(y_data, y_model):\n",
+ " return 1 - np.sum((y_data - y_model) ** 2) / np.sum((y_data - np.mean(y_data)) ** 2)\n",
+ "def MSE(y_data,y_model):\n",
+ " n = np.size(y_model)\n",
+ " return np.sum((y_data-y_model)**2)/n\n",
+ "\n",
+ "x = np.random.rand(100)\n",
+ "y = 2.0+5*x*x+0.1*np.random.randn(100)\n",
+ "\n",
+ "\n",
+ "# The design matrix now as function of a given polynomial\n",
+ "X = np.zeros((len(x),3))\n",
+ "X[:,0] = 1.0\n",
+ "X[:,1] = x\n",
+ "X[:,2] = x**2\n",
+ "# We split the data in test and training data\n",
+ "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)\n",
+ "# matrix inversion to find beta\n",
+ "beta = np.linalg.inv(X_train.T @ X_train) @ X_train.T @ y_train\n",
+ "print(beta)\n",
+ "# and then make the prediction\n",
+ "ytilde = X_train @ beta\n",
+ "print(\"Training R2\")\n",
+ "print(R2(y_train,ytilde))\n",
+ "print(\"Training MSE\")\n",
+ "print(MSE(y_train,ytilde))\n",
+ "ypredict = X_test @ beta\n",
+ "print(\"Test R2\")\n",
+ "print(R2(y_test,ypredict))\n",
+ "print(\"Test MSE\")\n",
+ "print(MSE(y_test,ypredict))"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Alternatively, you could write your own test-train splitting function as shown here."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": false,
+ "editable": true
+ },
+ "outputs": [],
+ "source": [
+ "# equivalently in numpy\n",
+ "def train_test_split_numpy(inputs, labels, train_size, test_size):\n",
+ " n_inputs = len(inputs)\n",
+ " inputs_shuffled = inputs.copy()\n",
+ " labels_shuffled = labels.copy()\n",
+ "\n",
+ " np.random.shuffle(inputs_shuffled)\n",
+ " np.random.shuffle(labels_shuffled)\n",
+ "\n",
+ " train_end = int(n_inputs*train_size)\n",
+ " X_train, X_test = inputs_shuffled[:train_end], inputs_shuffled[train_end:]\n",
+ " Y_train, Y_test = labels_shuffled[:train_end], labels_shuffled[train_end:]\n",
+ "\n",
+ " return X_train, X_test, Y_train, Y_test"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "But since **scikit-learn** has its own function for doing this and since\n",
+ "it interfaces easily with **tensorflow** and other libraries, we\n",
+ "normally recommend using the latter functionality.\n",
+ "\n",
+ "\n",
+ "\n",
+ "\n",
"## Reducing the number of degrees of freedom, overarching view\n",
"\n",
"Many Machine Learning problems involve thousands or even millions of\n",
@@ -3069,6 +3186,7 @@
"visualization.\n",
"\n",
"\n",
+ "\n",
"Before we proceed however, we will discuss how to preprocess our\n",
"data. Till now and in connection with our previous examples we have\n",
"not met so many cases where we are too sensitive to the scaling of our\n",
@@ -3076,6 +3194,15 @@
"to extreme values. Scaling the data renders our inputs much more\n",
"suitable for the algorithms we want to employ.\n",
"\n",
+ "For data sets gathered for real world applications, it is rather normal that\n",
+ "different features have very different units and\n",
+ "numerical scales. For example, a data set detailing health habits may include\n",
+ "features such as **age** in the range $0-80$, and **caloric intake** of order $2000$.\n",
+ "Many machine learning methods sensitive to the scales of the features and may perform poorly if they\n",
+ "are very different scales. Therefore, it is typical to scale\n",
+ "the features in a way to avoid such outlier values.\n",
+ "\n",
+ "\n",
"**Scikit-Learn** has several functions which allow us to rescale the\n",
"data, normally resulting in much better results in terms of various\n",
"accuracy scores. The **StandardScaler** function in **Scikit-Learn**\n",
@@ -3105,7 +3232,38 @@
"techniques.\n",
"\n",
"\n",
- "### Simple preprocessing examples, Franke function and regression"
+ "Many features are often scaled using standardization to improve\n",
+ "performance. In **Scikit-Learn** this is given by the **StandardScaler**\n",
+ "function as discussed above. It is easy however to write your own.\n",
+ "Mathematically, this involves subtracting the mean and divide by the\n",
+ "standard deviation over the data set, for each feature:"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "$$\n",
+ "x_j^{(i)} \\rightarrow \\frac{x_j^{(i)} - \\overline{x}_j}{\\sigma(x_j)},\n",
+ "$$"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "where $\\overline{x}_j$ and $\\sigma(x_j)$ are the mean and standard\n",
+ "deviation, respectively, of the feature $x_j$. This ensures that each\n",
+ "feature has zero mean and unit standard deviation. For data sets\n",
+ "where we do not have the standard deviation or don't wish to calculate\n",
+ "it, it is then common to simply set it to one.\n",
+ "\n",
+ "\n",
+ "\n",
+ "Let us consider the following vanilla example where we use both\n",
+ "**Scikit-Learn** and write our own function as well. We produce a\n",
+ "simple test design matrix with random numbers. Each column could then\n",
+ "represent a specific feature whose mean value is subracted."
]
},
{
@@ -3117,98 +3275,126 @@
},
"outputs": [],
"source": [
- "# Common imports\n",
- "import os\n",
- "import numpy as np\n",
- "import pandas as pd\n",
- "import matplotlib.pyplot as plt\n",
"import sklearn.linear_model as skl\n",
"from sklearn.metrics import mean_squared_error\n",
"from sklearn.model_selection import train_test_split\n",
"from sklearn.preprocessing import MinMaxScaler, StandardScaler, Normalizer\n",
- "\n",
- "# Where to save the figures and data files\n",
- "PROJECT_ROOT_DIR = \"Results\"\n",
- "FIGURE_ID = \"Results/FigureFiles\"\n",
- "DATA_ID = \"DataFiles/\"\n",
- "\n",
- "if not os.path.exists(PROJECT_ROOT_DIR):\n",
- " os.mkdir(PROJECT_ROOT_DIR)\n",
- "\n",
- "if not os.path.exists(FIGURE_ID):\n",
- " os.makedirs(FIGURE_ID)\n",
- "\n",
- "if not os.path.exists(DATA_ID):\n",
- " os.makedirs(DATA_ID)\n",
- "\n",
- "def image_path(fig_id):\n",
- " return os.path.join(FIGURE_ID, fig_id)\n",
- "\n",
- "def data_path(dat_id):\n",
- " return os.path.join(DATA_ID, dat_id)\n",
- "\n",
- "def save_fig(fig_id):\n",
- " plt.savefig(image_path(fig_id) + \".png\", format='png')\n",
+ "import numpy as np\n",
+ "import pandas as pd\n",
+ "from IPython.display import display\n",
+ "np.random.seed(100)\n",
+ "# setting up a 10 x 5 matrix\n",
+ "rows = 10\n",
+ "cols = 5\n",
+ "X = np.random.randn(rows,cols)\n",
+ "XPandas = pd.DataFrame(X)\n",
+ "display(XPandas)\n",
+ "print(XPandas.mean())\n",
+ "print(XPandas.std())\n",
+ "XPandas = (XPandas -XPandas.mean())\n",
+ "display(XPandas)\n",
+ "# This option does not include the standard deviation\n",
+ "scaler = StandardScaler(with_std=False)\n",
+ "scaler.fit(X)\n",
+ "Xscaled = scaler.transform(X)\n",
+ "display(XPandas-Xscaled)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Small exercise: perform the standard scaling by including the standard deviation and compare with what Scikit-Learn gives.\n",
"\n",
"\n",
- "def FrankeFunction(x,y):\n",
- "\tterm1 = 0.75*np.exp(-(0.25*(9*x-2)**2) - 0.25*((9*y-2)**2))\n",
- "\tterm2 = 0.75*np.exp(-((9*x+1)**2)/49.0 - 0.1*(9*y+1))\n",
- "\tterm3 = 0.5*np.exp(-(9*x-7)**2/4.0 - 0.25*((9*y-3)**2))\n",
- "\tterm4 = -0.2*np.exp(-(9*x-4)**2 - (9*y-7)**2)\n",
- "\treturn term1 + term2 + term3 + term4\n",
+ "\n",
+ "Another commonly used scaling method is min-max scaling. This is very\n",
+ "useful for when we want the features to lie in a certain interval. To\n",
+ "scale the feature $x_j$ to the interval $[a, b]$, we can apply the\n",
+ "transformation"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "$$\n",
+ "x_j^{(i)} \\rightarrow (b-a)\\frac{x_j^{(i)} - \\min(x_j)}{\\max(x_j) - \\min(x_j)} - a\n",
+ "$$"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "where $\\min(x_j)$ and $\\max(x_j)$ return the minimum and maximum value of $x_j$ over the data set, respectively.\n",
"\n",
"\n",
- "def create_X(x, y, n ):\n",
- "\tif len(x.shape) > 1:\n",
- "\t\tx = np.ravel(x)\n",
- "\t\ty = np.ravel(y)\n",
- "\n",
- "\tN = len(x)\n",
- "\tl = int((n+1)*(n+2)/2)\t\t# Number of elements in beta\n",
- "\tX = np.ones((N,l))\n",
- "\n",
- "\tfor i in range(1,n+1):\n",
- "\t\tq = int((i)*(i+1)/2)\n",
- "\t\tfor k in range(i+1):\n",
- "\t\t\tX[:,q+k] = (x**(i-k))*(y**k)\n",
- "\n",
- "\treturn X\n",
"\n",
"\n",
- "# Making meshgrid of datapoints and compute Franke's function\n",
- "n = 5\n",
- "N = 1000\n",
- "x = np.sort(np.random.uniform(0, 1, N))\n",
- "y = np.sort(np.random.uniform(0, 1, N))\n",
- "z = FrankeFunction(x, y)\n",
- "X = create_X(x, y, n=n) \n",
- "# split in training and test data\n",
- "X_train, X_test, y_train, y_test = train_test_split(X,z,test_size=0.2)\n",
+ "## Testing the Means Squared Error as function of Complexity\n",
"\n",
"\n",
- "clf = skl.LinearRegression().fit(X_train, y_train)\n",
+ "Before we proceed with a more detailed analysis of the so-called\n",
+ "Bias-Variance tradeoff, we present here an example of the relation\n",
+ "between model complexity and the mean squared error for the triaining\n",
+ "data and the test data.\n",
"\n",
- "# The mean squared error and R2 score\n",
- "print(\"MSE before scaling: {:.2f}\".format(mean_squared_error(clf.predict(X_test), y_test)))\n",
- "print(\"R2 score before scaling {:.2f}\".format(clf.score(X_test,y_test)))\n",
+ "The results here tell us clearly that for the data not included in the\n",
+ "training, there is an optimal model as function of the complexity of\n",
+ "ourmodel (here in terms of the polynomial degree of the model).\n",
"\n",
+ "The results here will vary as function of model complexity and the amount od data used for training. \n",
+ "\n",
+ "\n",
+ "Our data is defined by $x\\in [-3,3]$ with a total of for example $100$ data points."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": false,
+ "editable": true
+ },
+ "outputs": [],
+ "source": [
+ "import matplotlib.pyplot as plt\n",
+ "import numpy as np\n",
+ "from sklearn.linear_model import LinearRegression, Ridge, Lasso\n",
+ "from sklearn.preprocessing import PolynomialFeatures\n",
+ "from sklearn.model_selection import train_test_split\n",
+ "from sklearn.pipeline import make_pipeline\n",
+ "\n",
+ "\n",
+ "np.random.seed(2018)\n",
+ "n = 100\n",
+ "maxdegree = 14\n",
+ "# Make data set.\n",
+ "x = np.linspace(-3, 3, n).reshape(-1, 1)\n",
+ "y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)\n",
+ "TestError = np.zeros(maxdegree)\n",
+ "TrainError = np.zeros(maxdegree)\n",
+ "polydegree = np.zeros(maxdegree)\n",
+ "x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2)\n",
"scaler = StandardScaler()\n",
- "scaler.fit(X_train)\n",
- "X_train_scaled = scaler.transform(X_train)\n",
- "X_test_scaled = scaler.transform(X_test)\n",
+ "scaler.fit(x_train)\n",
+ "x_train_scaled = scaler.transform(x_train)\n",
+ "x_test_scaled = scaler.transform(x_test)\n",
"\n",
- "print(\"Feature min values before scaling:\\n {}\".format(X_train.min(axis=0)))\n",
- "print(\"Feature max values before scaling:\\n {}\".format(X_train.max(axis=0)))\n",
+ "for degree in range(maxdegree):\n",
+ " model = make_pipeline(PolynomialFeatures(degree=degree), LinearRegression(fit_intercept=False))\n",
+ " clf = model.fit(x_train_scaled,y_train)\n",
+ " y_fit = clf.predict(x_train_scaled)\n",
+ " y_pred = clf.predict(x_test_scaled) \n",
+ " polydegree[degree] = degree\n",
+ " TestError[degree] = np.mean( np.mean((y_test - y_pred)**2) )\n",
+ " TrainError[degree] = np.mean( np.mean((y_train - y_fit)**2) )\n",
"\n",
- "print(\"Feature min values after scaling:\\n {}\".format(X_train_scaled.min(axis=0)))\n",
- "print(\"Feature max values after scaling:\\n {}\".format(X_train_scaled.max(axis=0)))\n",
- "\n",
- "clf = skl.LinearRegression().fit(X_train_scaled, y_train)\n",
- "\n",
- "\n",
- "print(\"MSE after scaling: {:.2f}\".format(mean_squared_error(clf.predict(X_test_scaled), y_test)))\n",
- "print(\"R2 score for scaled data: {:.2f}\".format(clf.score(X_test_scaled,y_test)))"
+ "plt.plot(polydegree, TestError, label='Test Error')\n",
+ "plt.plot(polydegree, TrainError, label='Train Error')\n",
+ "plt.legend()\n",
+ "plt.show()"
]
},
{
diff --git a/doc/LectureNotes/_build/html/_sources/chapter2.ipynb b/doc/LectureNotes/_build/html/_sources/chapter2.ipynb
index 99ead382c..651eadf4a 100644
--- a/doc/LectureNotes/_build/html/_sources/chapter2.ipynb
+++ b/doc/LectureNotes/_build/html/_sources/chapter2.ipynb
@@ -543,6 +543,143 @@
"example\n",
"\n",
"\n",
+ "## Code for SVD and Inversion of Matrices\n",
+ "\n",
+ "How do we use the SVD to invert a matrix $\\boldsymbol{X}^\\boldsymbol{X}$ which is singular or near singular?\n",
+ "The simple answer is to use the linear algebra function for the pseudoinverse, that is"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": false,
+ "editable": true
+ },
+ "outputs": [],
+ "source": [
+ "#Ainv = np.linlag.pinv(A)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Let us first look at a matrix which does not causes problems and write our own function where we just use the SVD."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": false,
+ "editable": true
+ },
+ "outputs": [],
+ "source": [
+ "import numpy as np\n",
+ "# SVD inversion\n",
+ "def SVDinv(A):\n",
+ " ''' Takes as input a numpy matrix A and returns inv(A) based on singular value decomposition (SVD).\n",
+ " SVD is numerically more stable than the inversion algorithms provided by\n",
+ " numpy and scipy.linalg at the cost of being slower.\n",
+ " '''\n",
+ " U, s, VT = np.linalg.svd(A)\n",
+ " print('test U')\n",
+ " print( (np.transpose(U) @ U - U @np.transpose(U)))\n",
+ " print('test VT')\n",
+ " print( (np.transpose(VT) @ VT - VT @np.transpose(VT)))\n",
+ "\n",
+ "\n",
+ " D = np.zeros((len(U),len(VT)))\n",
+ " D = np.diag(s)\n",
+ " UT = np.transpose(U); V = np.transpose(VT); invD = np.linalg.inv(D)\n",
+ " return np.matmul(V,np.matmul(invD,UT))\n",
+ "\n",
+ "\n",
+ "#X = np.array([ [1.0, -1.0, 2.0], [1.0, 0.0, 1.0], [1.0, 2.0, -1.0], [1.0, 1.0, 0.0] ])\n",
+ "# Non-singular square matrix\n",
+ "X = np.array( [ [1,2,3],[2,4,5],[3,5,6]])\n",
+ "print(X)\n",
+ "A = np.transpose(X) @ X\n",
+ "# Brute force inversion\n",
+ "B = np.linalg.pinv(A) # here we could use np.linalg.inv(A), try it!\n",
+ "C = SVDinv(A)\n",
+ "print(np.abs(B-C))"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Although our matrix to invert $\\boldsymbol{X}^T\\boldsymbol{X}$ is a square matrix, our matrix may be singular. \n",
+ "\n",
+ "The pseudoinverse is the generalization of the matrix inverse for square matrices to\n",
+ "rectangular matrices where the number of rows and columns are not equal.\n",
+ "\n",
+ "It is also called the the Moore-Penrose Inverse after two independent discoverers of the method or the Generalized Inverse.\n",
+ "It is used for the calculation of the inverse for singular or near singular matrices and for rectangular matrices.\n",
+ "\n",
+ "Using the SVD we can obtain the pseudoinverse of a matrix $\\boldsymbol{A}$ (labeled here as $\\boldsymbol{A}_{\\mathrm{PI}}$"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "$$\n",
+ "\\boldsymbol{A}_{\\mathrm{PI}}= \\boldsymbol{V}\\boldsymbol{D}_{\\mathrm{PI}}\\boldsymbol{U}^T,\n",
+ "$$"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "where $\\boldsymbol{D}_{\\mathrm{PI}}$ can be calculated by creating a diagonal matrix from $\\boldsymbol{Sigma}$ where we only keep the singular values (the non-zero values). The following code computes the pseudoinvers of the matrix based on the SVD."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": false,
+ "editable": true
+ },
+ "outputs": [],
+ "source": [
+ "import numpy as np\n",
+ "# SVD inversion\n",
+ "def SVDinv(A):\n",
+ " U, s, VT = np.linalg.svd(A)\n",
+ " # reciprocals of singular values of s\n",
+ " d = 1.0 / s\n",
+ " # create m x n D matrix\n",
+ " D = np.zeros(A.shape)\n",
+ " # populate D with n x n diagonal matrix\n",
+ " D[:A.shape[1], :A.shape[1]] = np.diag(d)\n",
+ " UT = np.transpose(U)\n",
+ " V = np.transpose(VT)\n",
+ " return np.matmul(V,np.matmul(D.T,UT))\n",
+ "\n",
+ "\n",
+ "A = np.array([ [0.3, 0.4], [0.5, 0.6], [0.7, 0.8],[0.9, 1.0]])\n",
+ "print(A)\n",
+ "# Brute force inversion of super-collinear matrix\n",
+ "B = np.linalg.pinv(A)\n",
+ "print(B)\n",
+ "# Compare our own algorithm with pinv\n",
+ "C = SVDinv(A)\n",
+ "print(np.abs(C-B))"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "As you can see from these examples, our own decomposition based on the SVD agrees the pseudoinverse algorithm provided by **Numpy**.\n",
+ "\n",
+ "\n",
"\n",
"\n",
"\n",
@@ -1740,535 +1877,8 @@
"\n",
"\n",
"\n",
- "## Ridge and LASSO Regression\n",
"\n",
- "Let us remind ourselves about the expression for the standard Mean Squared Error (MSE) which we used to define our cost function and the equations for the ordinary least squares (OLS) method, that is \n",
- "our optimization problem is"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "$$\n",
- "{\\displaystyle \\min_{\\boldsymbol{\\beta}\\in {\\mathbb{R}}^{p}}}\\frac{1}{n}\\left\\{\\left(\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta}\\right)^T\\left(\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta}\\right)\\right\\}.\n",
- "$$"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "or we can state it as"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "$$\n",
- "{\\displaystyle \\min_{\\boldsymbol{\\beta}\\in\n",
- "{\\mathbb{R}}^{p}}}\\frac{1}{n}\\sum_{i=0}^{n-1}\\left(y_i-\\tilde{y}_i\\right)^2=\\frac{1}{n}\\vert\\vert \\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta}\\vert\\vert_2^2,\n",
- "$$"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "where we have used the definition of a norm-2 vector, that is"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "$$\n",
- "\\vert\\vert \\boldsymbol{x}\\vert\\vert_2 = \\sqrt{\\sum_i x_i^2}.\n",
- "$$"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "By minimizing the above equation with respect to the parameters\n",
- "$\\boldsymbol{\\beta}$ we could then obtain an analytical expression for the\n",
- "parameters $\\boldsymbol{\\beta}$. We can add a regularization parameter $\\lambda$ by\n",
- "defining a new cost function to be optimized, that is"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "$$\n",
- "{\\displaystyle \\min_{\\boldsymbol{\\beta}\\in\n",
- "{\\mathbb{R}}^{p}}}\\frac{1}{n}\\vert\\vert \\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta}\\vert\\vert_2^2+\\lambda\\vert\\vert \\boldsymbol{\\beta}\\vert\\vert_2^2\n",
- "$$"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "which leads to the Ridge regression minimization problem where we\n",
- "require that $\\vert\\vert \\boldsymbol{\\beta}\\vert\\vert_2^2\\le t$, where $t$ is\n",
- "a finite number larger than zero. By defining"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "$$\n",
- "C(\\boldsymbol{X},\\boldsymbol{\\beta})=\\frac{1}{n}\\vert\\vert \\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta}\\vert\\vert_2^2+\\lambda\\vert\\vert \\boldsymbol{\\beta}\\vert\\vert_1,\n",
- "$$"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "we have a new optimization equation"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "$$\n",
- "{\\displaystyle \\min_{\\boldsymbol{\\beta}\\in\n",
- "{\\mathbb{R}}^{p}}}\\frac{1}{n}\\vert\\vert \\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta}\\vert\\vert_2^2+\\lambda\\vert\\vert \\boldsymbol{\\beta}\\vert\\vert_1\n",
- "$$"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "which leads to Lasso regression. Lasso stands for least absolute shrinkage and selection operator. \n",
- "\n",
- "Here we have defined the norm-1 as"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "$$\n",
- "\\vert\\vert \\boldsymbol{x}\\vert\\vert_1 = \\sum_i \\vert x_i\\vert.\n",
- "$$"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Using the matrix-vector expression for Ridge regression and dropping the parameter $1/n$ in front of the standard means squared error equation, we have"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "$$\n",
- "C(\\boldsymbol{X},\\boldsymbol{\\beta})=\\left\\{(\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta})^T(\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta})\\right\\}+\\lambda\\boldsymbol{\\beta}^T\\boldsymbol{\\beta},\n",
- "$$"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "and \n",
- "taking the derivatives with respect to $\\boldsymbol{\\beta}$ we obtain then\n",
- "a slightly modified matrix inversion problem which for finite values\n",
- "of $\\lambda$ does not suffer from singularity problems. We obtain\n",
- "the optimal parameters"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "$$\n",
- "\\hat{\\boldsymbol{\\beta}}_{\\mathrm{Ridge}} = \\left(\\boldsymbol{X}^T\\boldsymbol{X}+\\lambda\\boldsymbol{I}\\right)^{-1}\\boldsymbol{X}^T\\boldsymbol{y},\n",
- "$$"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "with $\\boldsymbol{I}$ being a $p\\times p$ identity matrix with the constraint that"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "$$\n",
- "\\sum_{i=0}^{p-1} \\beta_i^2 \\leq t,\n",
- "$$"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "with $t$ a finite positive number. \n",
- "\n",
- "When we compare this with the ordinary least squares result we have"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "$$\n",
- "\\hat{\\boldsymbol{\\beta}}_{\\mathrm{OLS}} = \\left(\\boldsymbol{X}^T\\boldsymbol{X}\\right)^{-1}\\boldsymbol{X}^T\\boldsymbol{y},\n",
- "$$"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "which can lead to singular matrices. However, with the SVD, we can always compute the inverse of the matrix $\\boldsymbol{X}^T\\boldsymbol{X}$.\n",
- "\n",
- "\n",
- "We see that Ridge regression is nothing but the standard OLS with a\n",
- "modified diagonal term added to $\\boldsymbol{X}^T\\boldsymbol{X}$. The consequences, in\n",
- "particular for our discussion of the bias-variance tradeoff are rather\n",
- "interesting. We will see that for specific values of $\\lambda$, we may\n",
- "even reduce the variance of the optimal parameters $\\boldsymbol{\\beta}$. These topics and other related ones, will be discussed after the more linear algebra oriented analysis here.\n",
- "\n",
- "Using our insights about the SVD of the design matrix $\\boldsymbol{X}$ \n",
- "We have already analyzed the OLS solutions in terms of the eigenvectors (the columns) of the right singular value matrix $\\boldsymbol{U}$ as"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "$$\n",
- "\\tilde{\\boldsymbol{y}}_{\\mathrm{OLS}}=\\boldsymbol{X}\\boldsymbol{\\beta} =\\boldsymbol{U}\\boldsymbol{U}^T\\boldsymbol{y}.\n",
- "$$"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "For Ridge regression this becomes"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "$$\n",
- "\\tilde{\\boldsymbol{y}}_{\\mathrm{Ridge}}=\\boldsymbol{X}\\boldsymbol{\\beta}_{\\mathrm{Ridge}} = \\boldsymbol{U\\Sigma V^T}\\left(\\boldsymbol{V}\\boldsymbol{\\Sigma}^2\\boldsymbol{V}^T+\\lambda\\boldsymbol{I} \\right)^{-1}(\\boldsymbol{U\\Sigma V^T})^T\\boldsymbol{y}=\\sum_{j=0}^{p-1}\\boldsymbol{u}_j\\boldsymbol{u}_j^T\\frac{\\sigma_j^2}{\\sigma_j^2+\\lambda}\\boldsymbol{y},\n",
- "$$"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "with the vectors $\\boldsymbol{u}_j$ being the columns of $\\boldsymbol{U}$ from the SVD of the matrix $\\boldsymbol{X}$. \n",
- "\n",
- "\n",
- "Since $\\lambda \\geq 0$, it means that compared to OLS, we have"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "$$\n",
- "\\frac{\\sigma_j^2}{\\sigma_j^2+\\lambda} \\leq 1.\n",
- "$$"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Ridge regression finds the coordinates of $\\boldsymbol{y}$ with respect to the\n",
- "orthonormal basis $\\boldsymbol{U}$, it then shrinks the coordinates by\n",
- "$\\frac{\\sigma_j^2}{\\sigma_j^2+\\lambda}$. Recall that the SVD has\n",
- "eigenvalues ordered in a descending way, that is $\\sigma_i \\geq\n",
- "\\sigma_{i+1}$.\n",
- "\n",
- "For small eigenvalues $\\sigma_i$ it means that their contributions become less important, a fact which can be used to reduce the number of degrees of freedom. More about this when we have covered the material on a statistical interpretation of various linear regression methods.\n",
- "\n",
- "\n",
- "\n",
- "For the sake of simplicity, let us assume that the design matrix is orthonormal, that is"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "$$\n",
- "\\boldsymbol{X}^T\\boldsymbol{X}=(\\boldsymbol{X}^T\\boldsymbol{X})^{-1} =\\boldsymbol{I}.\n",
- "$$"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "In this case the standard OLS results in"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "$$\n",
- "\\boldsymbol{\\beta}^{\\mathrm{OLS}} = \\boldsymbol{X}^T\\boldsymbol{y}=\\sum_{i=0}^{p-1}\\boldsymbol{u}_j\\boldsymbol{u}_j^T\\boldsymbol{y},\n",
- "$$"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "and"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "$$\n",
- "\\boldsymbol{\\beta}^{\\mathrm{Ridge}} = \\left(\\boldsymbol{I}+\\lambda\\boldsymbol{I}\\right)^{-1}\\boldsymbol{X}^T\\boldsymbol{y}=\\left(1+\\lambda\\right)^{-1}\\boldsymbol{\\beta}^{\\mathrm{OLS}},\n",
- "$$"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "that is the Ridge estimator scales the OLS estimator by the inverse of a factor $1+\\lambda$, and\n",
- "the Ridge estimator converges to zero when the hyperparameter goes to\n",
- "infinity.\n",
- "\n",
- "We will come back to more interpreations after we have gone through some of the statistical analysis part. \n",
- "\n",
- "For more discussions of Ridge and Lasso regression, [Wessel van Wieringen's](https://arxiv.org/abs/1509.09169) article is highly recommended.\n",
- "Similarly, [Mehta et al's article](https://arxiv.org/abs/1803.08823) is also recommended.\n",
- "\n",
- "\n",
- "Using the matrix-vector expression for Lasso regression and dropping the parameter $1/n$ in front of the standard means squared error equation, we have the following **cost** function"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "$$\n",
- "C(\\boldsymbol{X},\\boldsymbol{\\beta})=\\left\\{(\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta})^T(\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta})\\right\\}+\\lambda\\vert\\vert\\boldsymbol{\\beta}\\vert\\vert_1,\n",
- "$$"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Taking the derivative with respect to $\\boldsymbol{\\beta}$ and recalling that the derivative of the absolute value is (we drop the boldfaced vector symbol for simplicty)"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "$$\n",
- "\\frac{d \\vert \\beta\\vert}{d \\boldsymbol{\\beta}}=\\mathrm{sgn}(\\boldsymbol{\\beta})=\\left\\{\\begin{array}{cc} 1 & \\beta > 0 \\\\ 0 & \\beta =0\\\\-1 & \\beta < 0, \\end{array}\\right.\n",
- "$$"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "we have that the derivative of the cost function is"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "$$\n",
- "\\frac{\\partial C(\\boldsymbol{X},\\boldsymbol{\\beta})}{\\partial \\boldsymbol{\\beta}}=-2\\boldsymbol{X}^T(\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta})+\\lambda sgn(\\boldsymbol{\\beta})=0,\n",
- "$$"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "and reordering we have"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "$$\n",
- "\\boldsymbol{X}^T\\boldsymbol{X}\\boldsymbol{\\beta})+\\lambda sgn(\\boldsymbol{\\beta})=2\\boldsymbol{X}^T(\\boldsymbol{y}.\n",
- "$$"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "This equation does not lead to a nice analytical equation as in either Ridge regression or ordinary least squares. This equation can however be solved by using standard convex optimization algorithms using for example the Python package [CVXOPT](https://cvxopt.org/). We will discuss this later. \n",
- "\n",
- "## Code for SVD and Inversion of Matrices\n",
- "\n",
- "How do we use the SVD to invert a matrix $\\boldsymbol{X}^\\boldsymbol{X}$ which is singular or near singular?\n",
- "The simple answer is to use the linear algebra function for the pseudoinverse, that is"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "collapsed": false,
- "editable": true
- },
- "outputs": [],
- "source": [
- "#Ainv = np.linlag.pinv(A)"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Let us first look at a matrix which does not causes problems and write our own function where we just use the SVD."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "collapsed": false,
- "editable": true
- },
- "outputs": [],
- "source": [
- "import numpy as np\n",
- "# SVD inversion\n",
- "def SVDinv(A):\n",
- " ''' Takes as input a numpy matrix A and returns inv(A) based on singular value decomposition (SVD).\n",
- " SVD is numerically more stable than the inversion algorithms provided by\n",
- " numpy and scipy.linalg at the cost of being slower.\n",
- " '''\n",
- " U, s, VT = np.linalg.svd(A)\n",
- " print('test U')\n",
- " print( (np.transpose(U) @ U - U @np.transpose(U)))\n",
- " print('test VT')\n",
- " print( (np.transpose(VT) @ VT - VT @np.transpose(VT)))\n",
- "\n",
- "\n",
- " D = np.zeros((len(U),len(VT)))\n",
- " D = np.diag(s)\n",
- " UT = np.transpose(U); V = np.transpose(VT); invD = np.linalg.inv(D)\n",
- " return np.matmul(V,np.matmul(invD,UT))\n",
- "\n",
- "\n",
- "#X = np.array([ [1.0, -1.0, 2.0], [1.0, 0.0, 1.0], [1.0, 2.0, -1.0], [1.0, 1.0, 0.0] ])\n",
- "# Non-singular square matrix\n",
- "X = np.array( [ [1,2,3],[2,4,5],[3,5,6]])\n",
- "print(X)\n",
- "A = np.transpose(X) @ X\n",
- "# Brute force inversion\n",
- "B = np.linalg.inv(A) # here we could use np.linalg.pinv(A)\n",
- "C = SVDinv(A)\n",
- "print(np.abs(B-C))"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Although our matrix to invert $\\boldsymbol{X}^T\\boldsymbol{X}$ is a square matrix, our matrix may be singular. \n",
- "\n",
- "The pseudoinverse is the generalization of the matrix inverse for square matrices to\n",
- "rectangular matrices where the number of rows and columns are not equal.\n",
- "\n",
- "It is also called the the Moore-Penrose Inverse after two independent discoverers of the method or the Generalized Inverse.\n",
- "It is used for the calculation of the inverse for singular or near singular matrices and for rectangular matrices.\n",
- "\n",
- "Using the SVD we can obtain the pseudoinverse of a matrix $\\boldsymbol{A}$ (labeled here as $\\boldsymbol{A}_{\\mathrm{PI}}$"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "$$\n",
- "\\boldsymbol{A}_{\\mathrm{PI}}= \\boldsymbol{V}\\boldsymbol{D}_{\\mathrm{PI}}\\boldsymbol{U}^T,\n",
- "$$"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "where $\\boldsymbol{D}_{\\mathrm{PI}}$ can be calculated by creating a diagonal matrix from $\\boldsymbol{Sigma}$ where we only keep the singular values (the non-zero values). The following code computes the pseudoinvers of the matrix based on the SVD."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "collapsed": false,
- "editable": true
- },
- "outputs": [],
- "source": [
- "import numpy as np\n",
- "# SVD inversion\n",
- "def SVDinv(A):\n",
- " U, s, VT = np.linalg.svd(A)\n",
- " # reciprocals of singular values of s\n",
- " d = 1.0 / s\n",
- " # create m x n D matrix\n",
- " D = np.zeros(A.shape)\n",
- " # populate D with n x n diagonal matrix\n",
- " D[:A.shape[1], :A.shape[1]] = np.diag(d)\n",
- " UT = np.transpose(U)\n",
- " V = np.transpose(VT)\n",
- " return np.matmul(V,np.matmul(D.T,UT))\n",
- "\n",
- "\n",
- "A = np.array([ [0.3, 0.4], [0.5, 0.6], [0.7, 0.8],[0.9, 1.0]])\n",
- "print(A)\n",
- "# Brute force inversion of super-collinear matrix\n",
- "B = np.linalg.pinv(A)\n",
- "print(B)\n",
- "# Compare our own algorithm with pinv\n",
- "C = SVDinv(A)\n",
- "print(np.abs(C-B))"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "As you can see from this example, our own decomposition based on the SVD agrees the pseudoinverse algorithm provided by **Numpy**.\n",
- "\n",
- "\n",
- "\n",
- "## Deriving the Ridge Regression Equations\n",
+ "## Ridge and Lasso Regression\n",
"\n",
"Let us remind ourselves about the expression for the standard Mean Squared Error (MSE) which we used to define our cost function and the equations for the ordinary least squares (OLS) method, that is \n",
"our optimization problem is"
@@ -3089,7 +2699,8 @@
"\n",
"Here we set up the OLS, Ridge and Lasso functionality in order to study the above example. Note that here we have opted for a set of values of $\\lambda$, meaning that we need to perform a search in order to find the optimal values.\n",
"\n",
- "First we study and compare the OLS and Ridge results. The next code compares all three methods."
+ "First we study and compare the OLS and Ridge results. The next code compares all three methods.\n",
+ "We select values of the hyperparameter $\\lambda\\in [10^{-4},10^4]$ and compute the predicted values for ordinary least squares and Ridge regression."
]
},
{
@@ -3158,7 +2769,13 @@
"cell_type": "markdown",
"metadata": {},
"source": [
- "We see here that we reach a plateau. What is actually happening?"
+ "We see here that we reach a plateau for the Ridge results. Writing out the coefficients $\\boldsymbol{\\beta}$, we that they are getting smaller and smaller and our error stabilizes since the predicted values of $\\tilde{\\boldsymbol{y}}$ approach zero.\n",
+ "\n",
+ "This happens also for Lasso regression, as seen from the next code\n",
+ "output. The difference is that Lasso shrinks the values of $\\beta$ to\n",
+ "zero at a much earlier stage and the results flatten out. We see that\n",
+ "Lasso gives also an excellent fit for small values of $\\lambda$ and\n",
+ "shows rthe best performance of the three regression methods."
]
},
{
@@ -3232,7 +2849,15 @@
"cell_type": "markdown",
"metadata": {},
"source": [
- "Another Example, now with a polynomial fit."
+ "We bring then back our exponential function example and study all\n",
+ "three regression methods. Depending on the level of noise, we note\n",
+ "that for small values of the hyperparameter $\\lambda$ all three\n",
+ "methods produce the same mean squared error. Again, Lasso shrinks the\n",
+ "parameter values to zero much earlier than Ridge regression and the\n",
+ "Lasso results flatten out much earlier since all $\\beta_j=0$ (check\n",
+ "this by printing the values). This case is an example of where OLS\n",
+ "performs best. Lasso and Ridge reproduce the OLS results for a limited\n",
+ "set of $\\lambda$ values."
]
},
{
@@ -3324,6 +2949,311 @@
"plt.show()"
]
},
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Both these example send a clear message. The addition of a\n",
+ "shrinkage/regularization term implies that we need to perform a search\n",
+ "for the optimal values of $\\lambda$. We will see this throughout these\n",
+ "series of lectures.\n",
+ "\n",
+ "\n",
+ "As a small addendum, we note that you can also solve this problem using the convex optimization package [CVXOPT](https://cvxopt.org/examples/mlbook/l1regls.html). This requires, in addition to having installed **CVXOPT**, you need to download the file *l1regl.py*.\n",
+ "The following code example solves the simpler problem we discussed above, where we have added the latter python file."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": false,
+ "editable": true
+ },
+ "outputs": [],
+ "source": [
+ "from cvxopt import matrix, spdiag, mul, div, sqrt, normal, setseed\n",
+ "from cvxopt import blas, lapack, solvers, sparse, spmatrix\n",
+ "import math\n",
+ "\n",
+ "try:\n",
+ " import mosek\n",
+ " import sys\n",
+ " __MOSEK = True\n",
+ "except: __MOSEK = False\n",
+ "\n",
+ "if __MOSEK:\n",
+ "\n",
+ " def l1regls_mosek(A, b):\n",
+ " \"\"\"\n",
+ "\n",
+ " Returns the solution of l1-norm regularized least-squares problem\n",
+ "\n",
+ " minimize || A*x - b ||_2^2 + e'*u\n",
+ "\n",
+ " subject to -u <= x <= u\n",
+ "\n",
+ " \"\"\"\n",
+ "\n",
+ " m, n = A.size\n",
+ "\n",
+ " env = mosek.Env()\n",
+ " task = env.Task(0,0)\n",
+ " task.set_Stream(mosek.streamtype.log, lambda x: sys.stdout.write(x))\n",
+ "\n",
+ " task.appendvars( 2*n) # number of variables\n",
+ " task.appendcons( 2*n) # number of constraints\n",
+ "\n",
+ " # input quadratic objective\n",
+ " Q = matrix(0.0, (n,n)) \n",
+ " blas.syrk(A, Q, alpha = 2.0, trans='T')\n",
+ "\n",
+ " I = []\n",
+ " for i in range(n):\n",
+ " I.extend(range(i,n))\n",
+ "\n",
+ " J = []\n",
+ " for i in range(n):\n",
+ " J.extend((n-i)*[i])\n",
+ "\n",
+ " task.putqobj(I, J, list(Q[matrix(I) + matrix(J)*n]))\n",
+ " task.putclist(range(2*n), list(-2*A.T*b) + n*[1.0]) # setup linear objective\n",
+ "\n",
+ " # input constraint matrix row by row\n",
+ " for i in range(n):\n",
+ " task.putarow( i, [i, n+i], [1.0, -1.0])\n",
+ " task.putarow( n+i, [i, n+i], [1.0, 1.0])\n",
+ "\n",
+ " # setup bounds on constraints\n",
+ " task.putboundslice(mosek.accmode.con,\n",
+ " 0, n, n*[mosek.boundkey.up], n*[0.0], n*[0.0])\n",
+ " task.putboundslice(mosek.accmode.con,\n",
+ " n, 2*n, n*[mosek.boundkey.lo], n*[0.0], n*[0.0])\n",
+ "\n",
+ " # setup variable bounds\n",
+ " task.putboundslice(mosek.accmode.var,\n",
+ " 0, 2*n, 2*n*[mosek.boundkey.fr], 2*n*[0.0], 2*n*[0.0])\n",
+ "\n",
+ " # optimize the task\n",
+ " task.putobjsense(mosek.objsense.minimize)\n",
+ " task.optimize()\n",
+ " task.solutionsummary(mosek.streamtype.log)\n",
+ " x = n*[0.0]\n",
+ " task.getsolutionslice(mosek.soltype.itr, mosek.solitem.xx, 0, n, x)\n",
+ "\n",
+ " return matrix(x)\n",
+ "\n",
+ " def l1regls_mosek2(A, b):\n",
+ " \"\"\"\n",
+ "\n",
+ " Returns the solution of l1-norm regularized least-squares problem\n",
+ "\n",
+ " minimize w'*w + e'*u\n",
+ "\n",
+ " subject to -u <= x <= u\n",
+ "\n",
+ " A*x - w = b\n",
+ "\n",
+ " \"\"\"\n",
+ "\n",
+ " m, n = A.size\n",
+ "\n",
+ " env = mosek.Env()\n",
+ " task = env.Task(0,0)\n",
+ " task.set_Stream(mosek.streamtype.log, lambda x: sys.stdout.write(x))\n",
+ "\n",
+ " task.appendvars(2*n + m) # number of variables\n",
+ " task.appendcons(2*n + m) # number of constraints\n",
+ "\n",
+ " # input quadratic objective\n",
+ " task.putqobj(range(2*n,2*n+m), range(2*n,2*n+m), m*[2.0])\n",
+ "\n",
+ " task.putclist(range(2*n+m), n*[0.0] + n*[1.0] + m*[0.0]) # setup linear objective\n",
+ "\n",
+ " # input constraint matrix row by row\n",
+ " for i in range(n):\n",
+ " task.putarow( i, [i, n+i], [1.0, -1.0])\n",
+ " task.putarow( n+i, [i, n+i], [1.0, 1.0])\n",
+ "\n",
+ " for i in range(m):\n",
+ " task.putarow( 2*n+i, range(n) + [2*n+i], list(A[i,:]) + [-1.0])\n",
+ "\n",
+ " # setup bounds on constraints\n",
+ " task.putboundslice(mosek.accmode.con,\n",
+ " 0, n, n*[mosek.boundkey.up], n*[0.0], n*[0.0])\n",
+ " task.putboundslice(mosek.accmode.con,\n",
+ " n, 2*n, n*[mosek.boundkey.lo], n*[0.0], n*[0.0])\n",
+ " task.putboundslice(mosek.accmode.con,\n",
+ " 2*n, 2*n+m, m*[mosek.boundkey.fx], list(b), list(b))\n",
+ "\n",
+ " # setup variable bounds\n",
+ " task.putboundslice(mosek.accmode.var, 0, 2*n+m, (2*n+m)*[mosek.boundkey.fr], \n",
+ " (2*n+m)*[0.0], (2*n+m)*[0.0])\n",
+ "\n",
+ " # optimize the task\n",
+ " task.putobjsense(mosek.objsense.minimize)\n",
+ " task.optimize()\n",
+ " task.solutionsummary(mosek.streamtype.log)\n",
+ " x = n*[0.0]\n",
+ " task.getsolutionslice(mosek.soltype.itr, mosek.solitem.xx, 0, n, x)\n",
+ "\n",
+ " return matrix(x)\n",
+ "\n",
+ "def l1regls(A, b):\n",
+ " \"\"\"\n",
+ " \n",
+ " Returns the solution of l1-norm regularized least-squares problem\n",
+ " \n",
+ " minimize || A*x - b ||_2^2 + || x ||_1.\n",
+ "\n",
+ " \"\"\"\n",
+ "\n",
+ " m, n = A.size\n",
+ " q = matrix(1.0, (2*n,1))\n",
+ " q[:n] = -2.0 * A.T * b\n",
+ "\n",
+ " def P(u, v, alpha = 1.0, beta = 0.0 ):\n",
+ " \"\"\"\n",
+ " v := alpha * 2.0 * [ A'*A, 0; 0, 0 ] * u + beta * v \n",
+ " \"\"\"\n",
+ " v *= beta\n",
+ " v[:n] += alpha * 2.0 * A.T * (A * u[:n])\n",
+ "\n",
+ "\n",
+ " def G(u, v, alpha=1.0, beta=0.0, trans='N'):\n",
+ " \"\"\"\n",
+ " v := alpha*[I, -I; -I, -I] * u + beta * v (trans = 'N' or 'T')\n",
+ " \"\"\"\n",
+ "\n",
+ " v *= beta\n",
+ " v[:n] += alpha*(u[:n] - u[n:])\n",
+ " v[n:] += alpha*(-u[:n] - u[n:])\n",
+ "\n",
+ " h = matrix(0.0, (2*n,1))\n",
+ "\n",
+ "\n",
+ " # Customized solver for the KKT system \n",
+ " #\n",
+ " # [ 2.0*A'*A 0 I -I ] [x[:n] ] [bx[:n] ]\n",
+ " # [ 0 0 -I -I ] [x[n:] ] = [bx[n:] ].\n",
+ " # [ I -I -D1^-1 0 ] [zl[:n]] [bzl[:n]]\n",
+ " # [ -I -I 0 -D2^-1 ] [zl[n:]] [bzl[n:]]\n",
+ " #\n",
+ " # where D1 = W['di'][:n]**2, D2 = W['di'][:n]**2.\n",
+ " # \n",
+ " # We first eliminate zl and x[n:]:\n",
+ " #\n",
+ " # ( 2*A'*A + 4*D1*D2*(D1+D2)^-1 ) * x[:n] = \n",
+ " # bx[:n] - (D2-D1)*(D1+D2)^-1 * bx[n:] + \n",
+ " # D1 * ( I + (D2-D1)*(D1+D2)^-1 ) * bzl[:n] - \n",
+ " # D2 * ( I - (D2-D1)*(D1+D2)^-1 ) * bzl[n:] \n",
+ " #\n",
+ " # x[n:] = (D1+D2)^-1 * ( bx[n:] - D1*bzl[:n] - D2*bzl[n:] ) \n",
+ " # - (D2-D1)*(D1+D2)^-1 * x[:n] \n",
+ " #\n",
+ " # zl[:n] = D1 * ( x[:n] - x[n:] - bzl[:n] )\n",
+ " # zl[n:] = D2 * (-x[:n] - x[n:] - bzl[n:] ).\n",
+ " #\n",
+ " # The first equation has the form\n",
+ " #\n",
+ " # (A'*A + D)*x[:n] = rhs\n",
+ " #\n",
+ " # and is equivalent to\n",
+ " #\n",
+ " # [ D A' ] [ x:n] ] = [ rhs ]\n",
+ " # [ A -I ] [ v ] [ 0 ].\n",
+ " #\n",
+ " # It can be solved as \n",
+ " #\n",
+ " # ( A*D^-1*A' + I ) * v = A * D^-1 * rhs\n",
+ " # x[:n] = D^-1 * ( rhs - A'*v ).\n",
+ "\n",
+ " S = matrix(0.0, (m,m))\n",
+ " Asc = matrix(0.0, (m,n))\n",
+ " v = matrix(0.0, (m,1))\n",
+ "\n",
+ " def Fkkt(W):\n",
+ "\n",
+ " # Factor \n",
+ " #\n",
+ " # S = A*D^-1*A' + I \n",
+ " #\n",
+ " # where D = 2*D1*D2*(D1+D2)^-1, D1 = d[:n]**-2, D2 = d[n:]**-2.\n",
+ "\n",
+ " d1, d2 = W['di'][:n]**2, W['di'][n:]**2\n",
+ "\n",
+ " # ds is square root of diagonal of D\n",
+ " ds = math.sqrt(2.0) * div( mul( W['di'][:n], W['di'][n:]), \n",
+ " sqrt(d1+d2) )\n",
+ " d3 = div(d2 - d1, d1 + d2)\n",
+ " \n",
+ " # Asc = A*diag(d)^-1/2\n",
+ " Asc = A * spdiag(ds**-1)\n",
+ "\n",
+ " # S = I + A * D^-1 * A'\n",
+ " blas.syrk(Asc, S)\n",
+ " S[::m+1] += 1.0 \n",
+ " lapack.potrf(S)\n",
+ "\n",
+ " def g(x, y, z):\n",
+ "\n",
+ " x[:n] = 0.5 * ( x[:n] - mul(d3, x[n:]) + \n",
+ " mul(d1, z[:n] + mul(d3, z[:n])) - mul(d2, z[n:] - \n",
+ " mul(d3, z[n:])) )\n",
+ " x[:n] = div( x[:n], ds) \n",
+ "\n",
+ " # Solve\n",
+ " #\n",
+ " # S * v = 0.5 * A * D^-1 * ( bx[:n] - \n",
+ " # (D2-D1)*(D1+D2)^-1 * bx[n:] + \n",
+ " # D1 * ( I + (D2-D1)*(D1+D2)^-1 ) * bzl[:n] - \n",
+ " # D2 * ( I - (D2-D1)*(D1+D2)^-1 ) * bzl[n:] )\n",
+ " \n",
+ " blas.gemv(Asc, x, v)\n",
+ " lapack.potrs(S, v)\n",
+ " \n",
+ " # x[:n] = D^-1 * ( rhs - A'*v ).\n",
+ " blas.gemv(Asc, v, x, alpha=-1.0, beta=1.0, trans='T')\n",
+ " x[:n] = div(x[:n], ds)\n",
+ "\n",
+ " # x[n:] = (D1+D2)^-1 * ( bx[n:] - D1*bzl[:n] - D2*bzl[n:] ) \n",
+ " # - (D2-D1)*(D1+D2)^-1 * x[:n] \n",
+ " x[n:] = div( x[n:] - mul(d1, z[:n]) - mul(d2, z[n:]), d1+d2 )\\\n",
+ " - mul( d3, x[:n] )\n",
+ " \n",
+ " # zl[:n] = D1^1/2 * ( x[:n] - x[n:] - bzl[:n] )\n",
+ " # zl[n:] = D2^1/2 * ( -x[:n] - x[n:] - bzl[n:] ).\n",
+ " z[:n] = mul( W['di'][:n], x[:n] - x[n:] - z[:n] ) \n",
+ " z[n:] = mul( W['di'][n:], -x[:n] - x[n:] - z[n:] ) \n",
+ "\n",
+ " return g\n",
+ "\n",
+ " return solvers.coneqp(P, q, G, h, kktsolver = Fkkt)['x'][:n]"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Then we call the above functions and solve the problem, as done here"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": false,
+ "editable": true
+ },
+ "outputs": [],
+ "source": [
+ "from cvxopt import matrix, normal\n",
+ "\n",
+ "X = matrix( [ [ 2, 0, 1], [0, 1, 3]])\n",
+ "y = matrix( [4, 2, 3])\n",
+ "x = l1regls(X,y)"
+ ]
+ },
{
"cell_type": "markdown",
"metadata": {},
diff --git a/doc/LectureNotes/_build/html/chapter1.html b/doc/LectureNotes/_build/html/chapter1.html
index 244b648c0..3e61ffcbd 100644
--- a/doc/LectureNotes/_build/html/chapter1.html
+++ b/doc/LectureNotes/_build/html/chapter1.html
@@ -54,7 +54,7 @@
-
+
@@ -144,12 +144,12 @@
Our emphasis throughout this series of lectures (small change)
+
Our emphasis throughout this series of lectures
is on understanding the mathematical aspects of
different algorithms used in the fields of data analysis and machine learning.
However, where possible we will emphasize the
@@ -585,7 +587,9 @@ may first try the simplest class of models, namely linear models, followed obvio
3.3. Simple linear regression model using scikit-learn¶
-
We start with perhaps our simplest possible example, using Scikit-Learn to perform linear regression analysis on a data set produced by us.
+
We start with perhaps our simplest possible example, using
+Scikit-Learn to perform linear regression analysis on a data set
+produced by us.
What follows is a simple Python code where we have defined a function
\(y\) in terms of the variable \(x\). Both are defined as vectors with \(100\) entries.
The numbers in the vector \(\boldsymbol{x}\) are given
@@ -717,7 +721,8 @@ to be dominated by outliers.
importnumpyasnpimportmatplotlib.pyplotaspltfromsklearn.linear_modelimportLinearRegression
-
+# Number of data points
+n=100x=np.random.rand(100,1)y=5*x+0.01*np.random.randn(100,1)linreg=LinearRegression()
@@ -740,7 +745,8 @@ to be dominated by outliers.
Depending on the parameter in front of the normal distribution, we may
have a small or larger relative error. Try to play around with
different training data sets and study (graphically) the value of the
-relative error.
+relative error. Note also that Scikit-Learn requires a matrix as input for the input values \(x\) and \(y\). In the above code we have
+solved this by declaring \(x\) and \(y\) as arrays of dimension \(n\times 1\).
As mentioned above, Scikit-Learn has an impressive functionality.
We can for example extract the values of \(\alpha\) and \(\beta\) and
their error estimates, or the variance and standard deviation and many
@@ -781,13 +787,13 @@ example of the functionality of Scikit-Learn.
Here \(\boldsymbol{a}=\boldsymbol{y} - \boldsymbol{\tilde{y}}\).
We will discuss in more
@@ -888,7 +893,7 @@ a linear \(x\)-dependence we s
-
0.005000000000000007
+
0.0050000000000000044
@@ -1126,6 +1131,7 @@ It has dimensionality \(p\times n\)
+
Note well that we have made life simple here. We perform a fit in terms of the number of nucleons only. A more sophisticated fit can be done by including an explicit dependence on the number of protons and neutrons in the asymmetry and Coulomb terms.
With scikitlearn we are now ready to use linear regression and fit our data.
@@ -1145,7 +1151,6 @@ Now we can print measures of how our fit is doing, the coefficients from the fit
print('Variance score: %.2f'%r2_score(Energies,fity))# Mean absolute error print('Mean absolute error: %.2f'%mean_absolute_error(Energies,fity))
-print(clf.coef_,clf.intercept_)Masses['Eapprox']=fity# Generate a plot comparing the experimental with the fitted values values.
@@ -1166,8 +1171,6 @@ Now we can print measures of how our fit is doing, the coefficients from the fit
Mean squared error: 0.04
Variance score: 0.95
Mean absolute error: 0.05
-[ 0.00000000e+00 7.06492086e-03 -1.73091052e-01 -1.66020213e+01
- 1.17385778e+00] 15.212327334149492
@@ -1225,7 +1228,7 @@ A
270 3344 160 110 270 Ds 7.253775 7.253775
[267 rows x 6 columns]
-0.009883615646716182
+0.009883615646716186
@@ -1273,8 +1276,6 @@ functionality.
warnings.warn(
/Users/mhjensen/opt/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:582: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (100) reached and the optimization hasn't converged yet.
warnings.warn(
-/Users/mhjensen/opt/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:582: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (100) reached and the optimization hasn't converged yet.
- warnings.warn(
/Users/mhjensen/opt/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:582: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (100) reached and the optimization hasn't converged yet.
@@ -1339,7 +1340,9 @@ functionality.
/Users/mhjensen/opt/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:582: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (100) reached and the optimization hasn't converged yet.
warnings.warn(
-/Users/mhjensen/opt/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:582: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (100) reached and the optimization hasn't converged yet.
+
+
+
/Users/mhjensen/opt/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:582: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (100) reached and the optimization hasn't converged yet.
warnings.warn(
/Users/mhjensen/opt/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:582: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (100) reached and the optimization hasn't converged yet.
warnings.warn(
@@ -1357,7 +1360,7 @@ functionality.
warnings.warn(
-
+
@@ -2332,13 +2335,13 @@ but now splitting the data into a training set and a test set.
Training R2
-0.9999878260589065
+0.9999869956119286
Training MSE
-5.415634483874318
+5.745136489050356
Test R2
-0.999974281880048
+0.9999787681537219
Test MSE
-11.653095390463358
+9.712199063818309
@@ -2600,8 +2603,107 @@ R2 score is 0.6628996975186952
+
+
3.8. Splitting our Data in Training and Test data¶
+
It is normal in essentially all Machine Learning studies to split the
+data in a training set and a test set (sometimes also an additional
+validation set). Scikit-Learn has an own function for this. There
+is no explicit recipe for how much data should be included as training
+data and say test data. An accepted rule of thumb is to use
+approximately \(2/3\) to \(4/5\) of the data as training data. We will
+postpone a discussion of this splitting to the end of these notes and
+our discussion of the so-called bias-variance tradeoff. Here we
+limit ourselves to repeat the above equation of state fitting example
+but now splitting the data into a training set and a test set.
+
Let us study some examples. The first code here takes a simple
+one-dimensional second-order polynomial and we fit it to a
+second-order polynomial. Depending on the strength of the added noise,
+the various measures like the \(R2\) score or the mean-squared error,
+the fit becomes better or worse.
+
+
+
importos
+importnumpyasnp
+importpandasaspd
+importmatplotlib.pyplotasplt
+fromsklearn.model_selectionimporttrain_test_split
+
+
+defR2(y_data,y_model):
+ return1-np.sum((y_data-y_model)**2)/np.sum((y_data-np.mean(y_data))**2)
+defMSE(y_data,y_model):
+ n=np.size(y_model)
+ returnnp.sum((y_data-y_model)**2)/n
+
+x=np.random.rand(100)
+y=2.0+5*x*x+0.1*np.random.randn(100)
+
+
+# The design matrix now as function of a given polynomial
+X=np.zeros((len(x),3))
+X[:,0]=1.0
+X[:,1]=x
+X[:,2]=x**2
+# We split the data in test and training data
+X_train,X_test,y_train,y_test=train_test_split(X,y,test_size=0.2)
+# matrix inversion to find beta
+beta=np.linalg.inv(X_train.T@X_train)@X_train.T@y_train
+print(beta)
+# and then make the prediction
+ytilde=X_train@beta
+print("Training R2")
+print(R2(y_train,ytilde))
+print("Training MSE")
+print(MSE(y_train,ytilde))
+ypredict=X_test@beta
+print("Test R2")
+print(R2(y_test,ypredict))
+print("Test MSE")
+print(MSE(y_test,ypredict))
+
But since scikit-learn has its own function for doing this and since
+it interfaces easily with tensorflow and other libraries, we
+normally recommend using the latter functionality.
+
-
3.8. Reducing the number of degrees of freedom, overarching view¶
+
3.9. Reducing the number of degrees of freedom, overarching view¶
Many Machine Learning problems involve thousands or even millions of
features for each training instance. Not only does this make training
extremely slow, it can also make it much harder to find a good
@@ -2624,6 +2726,13 @@ not met so many cases where we are too sensitive to the scaling of our
data. Normally the data may need a rescaling and/or may be sensitive
to extreme values. Scaling the data renders our inputs much more
suitable for the algorithms we want to employ.
+
For data sets gathered for real world applications, it is rather normal that
+different features have very different units and
+numerical scales. For example, a data set detailing health habits may include
+features such as age in the range \(0-80\), and caloric intake of order \(2000\).
+Many machine learning methods sensitive to the scales of the features and may perform poorly if they
+are very different scales. Therefore, it is typical to scale
+the features in a way to avoid such outlier values.
Scikit-Learn has several functions which allow us to rescale the
data, normally resulting in much better results in terms of various
accuracy scores. The StandardScaler function in Scikit-Learn
@@ -2648,142 +2757,465 @@ RobustScaler ignore data points that are very different from the rest
(like measurement errors). These odd data points are also called
outliers, and might often lead to trouble for other scaling
techniques.
-
-
3.8.1. Simple preprocessing examples, Franke function and regression¶
+
Many features are often scaled using standardization to improve
+performance. In Scikit-Learn this is given by the StandardScaler
+function as discussed above. It is easy however to write your own.
+Mathematically, this involves subtracting the mean and divide by the
+standard deviation over the data set, for each feature:
where \(\overline{x}_j\) and \(\sigma(x_j)\) are the mean and standard
+deviation, respectively, of the feature \(x_j\). This ensures that each
+feature has zero mean and unit standard deviation. For data sets
+where we do not have the standard deviation or don’t wish to calculate
+it, it is then common to simply set it to one.
+
Let us consider the following vanilla example where we use both
+Scikit-Learn and write our own function as well. We produce a
+simple test design matrix with random numbers. Each column could then
+represent a specific feature whose mean value is subracted.
-
# Common imports
-importos
-importnumpyasnp
-importpandasaspd
-importmatplotlib.pyplotasplt
-importsklearn.linear_modelasskl
+
importsklearn.linear_modelassklfromsklearn.metricsimportmean_squared_errorfromsklearn.model_selectionimporttrain_test_splitfromsklearn.preprocessingimportMinMaxScaler,StandardScaler,Normalizer
-
-# Where to save the figures and data files
-PROJECT_ROOT_DIR="Results"
-FIGURE_ID="Results/FigureFiles"
-DATA_ID="DataFiles/"
-
-ifnotos.path.exists(PROJECT_ROOT_DIR):
- os.mkdir(PROJECT_ROOT_DIR)
-
-ifnotos.path.exists(FIGURE_ID):
- os.makedirs(FIGURE_ID)
-
-ifnotos.path.exists(DATA_ID):
- os.makedirs(DATA_ID)
-
-defimage_path(fig_id):
- returnos.path.join(FIGURE_ID,fig_id)
-
-defdata_path(dat_id):
- returnos.path.join(DATA_ID,dat_id)
-
-defsave_fig(fig_id):
- plt.savefig(image_path(fig_id)+".png",format='png')
-
-
-defFrankeFunction(x,y):
- term1=0.75*np.exp(-(0.25*(9*x-2)**2)-0.25*((9*y-2)**2))
- term2=0.75*np.exp(-((9*x+1)**2)/49.0-0.1*(9*y+1))
- term3=0.5*np.exp(-(9*x-7)**2/4.0-0.25*((9*y-3)**2))
- term4=-0.2*np.exp(-(9*x-4)**2-(9*y-7)**2)
- returnterm1+term2+term3+term4
-
-
-defcreate_X(x,y,n):
- iflen(x.shape)>1:
- x=np.ravel(x)
- y=np.ravel(y)
-
- N=len(x)
- l=int((n+1)*(n+2)/2)# Number of elements in beta
- X=np.ones((N,l))
-
- foriinrange(1,n+1):
- q=int((i)*(i+1)/2)
- forkinrange(i+1):
- X[:,q+k]=(x**(i-k))*(y**k)
-
- returnX
-
-
-# Making meshgrid of datapoints and compute Franke's function
-n=5
-N=1000
-x=np.sort(np.random.uniform(0,1,N))
-y=np.sort(np.random.uniform(0,1,N))
-z=FrankeFunction(x,y)
-X=create_X(x,y,n=n)
-# split in training and test data
-X_train,X_test,y_train,y_test=train_test_split(X,z,test_size=0.2)
-
-
-clf=skl.LinearRegression().fit(X_train,y_train)
-
-# The mean squared error and R2 score
-print("MSE before scaling: {:.2f}".format(mean_squared_error(clf.predict(X_test),y_test)))
-print("R2 score before scaling {:.2f}".format(clf.score(X_test,y_test)))
-
-scaler=StandardScaler()
-scaler.fit(X_train)
-X_train_scaled=scaler.transform(X_train)
-X_test_scaled=scaler.transform(X_test)
-
-print("Feature min values before scaling:\n{}".format(X_train.min(axis=0)))
-print("Feature max values before scaling:\n{}".format(X_train.max(axis=0)))
-
-print("Feature min values after scaling:\n{}".format(X_train_scaled.min(axis=0)))
-print("Feature max values after scaling:\n{}".format(X_train_scaled.max(axis=0)))
-
-clf=skl.LinearRegression().fit(X_train_scaled,y_train)
-
-
-print("MSE after scaling: {:.2f}".format(mean_squared_error(clf.predict(X_test_scaled),y_test)))
-print("R2 score for scaled data: {:.2f}".format(clf.score(X_test_scaled,y_test)))
+importnumpyasnp
+importpandasaspd
+fromIPython.displayimportdisplay
+np.random.seed(100)
+# setting up a 10 x 5 matrix
+rows=10
+cols=5
+X=np.random.randn(rows,cols)
+XPandas=pd.DataFrame(X)
+display(XPandas)
+print(XPandas.mean())
+print(XPandas.std())
+XPandas=(XPandas-XPandas.mean())
+display(XPandas)
+# This option does not include the standard deviation
+scaler=StandardScaler(with_std=False)
+scaler.fit(X)
+Xscaled=scaler.transform(X)
+display(XPandas-Xscaled)
Small exercise: perform the standard scaling by including the standard deviation and compare with what Scikit-Learn gives.
+
Another commonly used scaling method is min-max scaling. This is very
+useful for when we want the features to lie in a certain interval. To
+scale the feature \(x_j\) to the interval \([a, b]\), we can apply the
+transformation
where \(\min(x_j)\) and \(\max(x_j)\) return the minimum and maximum value of \(x_j\) over the data set, respectively.
+
+
+
3.10. Testing the Means Squared Error as function of Complexity¶
+
Before we proceed with a more detailed analysis of the so-called
+Bias-Variance tradeoff, we present here an example of the relation
+between model complexity and the mean squared error for the triaining
+data and the test data.
+
The results here tell us clearly that for the data not included in the
+training, there is an optimal model as function of the complexity of
+ourmodel (here in terms of the polynomial degree of the model).
+
The results here will vary as function of model complexity and the amount od data used for training.
+
Our data is defined by \(x\in [-3,3]\) with a total of for example \(100\) data points.
3.9.1. Exercise: Setting up various Python environments¶
+
3.11.1. Exercise: Setting up various Python environments¶
The first exercise here is of a mere technical art. We want you to have
git as a version control software and to establish a user account on a provider like GitHub. Other providers like GitLab etc are equally fine. You can also use the University of Oslo GitHub facilities.
@@ -2839,7 +3271,7 @@ license.
We recommend using Anaconda if you are not too familiar with setting paths in a terminal environment.
-
3.9.2. Exercise: making your own data and exploring scikit-learn¶
+
3.11.2. Exercise: making your own data and exploring scikit-learn¶
We will generate our own dataset for a function \(y(x)\) where \(x \in [0,1]\) and defined by random numbers computed with the uniform distribution. The function \(y\) is a quadratic polynomial in \(x\) with added stochastic noise according to the normal distribution \(\cal {N}(0,1)\).
The following simple Python instructions define our \(x\) and \(y\) values (with 100 data points).
@@ -2875,7 +3307,7 @@ R^2(\boldsymbol{y}, \tilde{\boldsymbol{y}}) = 1 - \frac{\sum_{i=0}^{n - 1} (y_i
Discuss the meaning of these results. Try also to vary the coefficient in front of the added stochastic noise term and discuss the quality of the fits.
A much used approach before starting to train the data is to preprocess our
data. Normally the data may need a rescaling and/or may be sensitive
to extreme values. Scaling the data renders our inputs much more
@@ -2987,7 +3419,7 @@ your results. For which polynomial degree do you find an optimal MSE
How do we use the SVD to invert a matrix \(\boldsymbol{X}^\boldsymbol{X}\) which is singular or near singular?
+The simple answer is to use the linear algebra function for the pseudoinverse, that is
+
+
+
#Ainv = np.linlag.pinv(A)
+
+
+
+
+
Let us first look at a matrix which does not causes problems and write our own function where we just use the SVD.
+
+
+
importnumpyasnp
+# SVD inversion
+defSVDinv(A):
+ ''' Takes as input a numpy matrix A and returns inv(A) based on singular value decomposition (SVD).
+ SVD is numerically more stable than the inversion algorithms provided by
+ numpy and scipy.linalg at the cost of being slower.
+ '''
+ U,s,VT=np.linalg.svd(A)
+ print('test U')
+ print((np.transpose(U)@U-U@np.transpose(U)))
+ print('test VT')
+ print((np.transpose(VT)@VT-VT@np.transpose(VT)))
+
+
+ D=np.zeros((len(U),len(VT)))
+ D=np.diag(s)
+ UT=np.transpose(U);V=np.transpose(VT);invD=np.linalg.inv(D)
+ returnnp.matmul(V,np.matmul(invD,UT))
+
+
+#X = np.array([ [1.0, -1.0, 2.0], [1.0, 0.0, 1.0], [1.0, 2.0, -1.0], [1.0, 1.0, 0.0] ])
+# Non-singular square matrix
+X=np.array([[1,2,3],[2,4,5],[3,5,6]])
+print(X)
+A=np.transpose(X)@X
+# Brute force inversion
+B=np.linalg.pinv(A)# here we could use np.linalg.inv(A), try it!
+C=SVDinv(A)
+print(np.abs(B-C))
+
Although our matrix to invert \(\boldsymbol{X}^T\boldsymbol{X}\) is a square matrix, our matrix may be singular.
+
The pseudoinverse is the generalization of the matrix inverse for square matrices to
+rectangular matrices where the number of rows and columns are not equal.
+
It is also called the the Moore-Penrose Inverse after two independent discoverers of the method or the Generalized Inverse.
+It is used for the calculation of the inverse for singular or near singular matrices and for rectangular matrices.
+
Using the SVD we can obtain the pseudoinverse of a matrix \(\boldsymbol{A}\) (labeled here as \(\boldsymbol{A}_{\mathrm{PI}}\)
where \(\boldsymbol{D}_{\mathrm{PI}}\) can be calculated by creating a diagonal matrix from \(\boldsymbol{Sigma}\) where we only keep the singular values (the non-zero values). The following code computes the pseudoinvers of the matrix based on the SVD.
+
+
+
importnumpyasnp
+# SVD inversion
+defSVDinv(A):
+ U,s,VT=np.linalg.svd(A)
+ # reciprocals of singular values of s
+ d=1.0/s
+ # create m x n D matrix
+ D=np.zeros(A.shape)
+ # populate D with n x n diagonal matrix
+ D[:A.shape[1],:A.shape[1]]=np.diag(d)
+ UT=np.transpose(U)
+ V=np.transpose(VT)
+ returnnp.matmul(V,np.matmul(D.T,UT))
+
+
+A=np.array([[0.3,0.4],[0.5,0.6],[0.7,0.8],[0.9,1.0]])
+print(A)
+# Brute force inversion of super-collinear matrix
+B=np.linalg.pinv(A)
+print(B)
+# Compare our own algorithm with pinv
+C=SVDinv(A)
+print(np.abs(C-B))
+
Let us take a closer look at the mathematics of the SVD and the various implications for machine learning studies.
Our starting point is our design matrix \(\boldsymbol{X}\) of dimension \(n\times p\)
@@ -780,7 +894,7 @@ decomposition of the design matrix.
It means that the ordinary least square model (with the optimal parameters) \(\boldsymbol{\tilde{y}}\), corresponds to an orthogonal transformation of the output (or target) vector \(\boldsymbol{y}\) by the vectors of the matrix \(\boldsymbol{U}\).
-
4.6. Further properties (important for our analyses later)¶
+
4.7. Further properties (important for our analyses later)¶
Let us study again \(\boldsymbol{X}^T\boldsymbol{X}\) in terms of our SVD,
\[
@@ -823,7 +937,7 @@ other texts you may find the opposite notation. This has consequences
for the definition of for example the covariance matrix and its relation to the SVD.
Before we move on to a discussion of Ridge and Lasso regression, we want to show an important example of the above.
We have already noted that the matrix \(\boldsymbol{X}^T\boldsymbol{X}\) in ordinary
least squares is proportional to the second derivative of the cost
@@ -982,10 +1096,10 @@ covariance matrix through the np.linalg.eig() function.
Let us remind ourselves about the expression for the standard Mean Squared Error (MSE) which we used to define our cost function and the equations for the ordinary least squares (OLS) method, that is
-our optimization problem is
By minimizing the above equation with respect to the parameters
-\(\boldsymbol{\beta}\) we could then obtain an analytical expression for the
-parameters \(\boldsymbol{\beta}\). We can add a regularization parameter \(\lambda\) by
-defining a new cost function to be optimized, that is
which leads to the Ridge regression minimization problem where we
-require that \(\vert\vert \boldsymbol{\beta}\vert\vert_2^2\le t\), where \(t\) is
-a finite number larger than zero. By defining
Using the matrix-vector expression for Ridge regression and dropping the parameter \(1/n\) in front of the standard means squared error equation, we have
and
-taking the derivatives with respect to \(\boldsymbol{\beta}\) we obtain then
-a slightly modified matrix inversion problem which for finite values
-of \(\lambda\) does not suffer from singularity problems. We obtain
-the optimal parameters
which can lead to singular matrices. However, with the SVD, we can always compute the inverse of the matrix \(\boldsymbol{X}^T\boldsymbol{X}\).
-
We see that Ridge regression is nothing but the standard OLS with a
-modified diagonal term added to \(\boldsymbol{X}^T\boldsymbol{X}\). The consequences, in
-particular for our discussion of the bias-variance tradeoff are rather
-interesting. We will see that for specific values of \(\lambda\), we may
-even reduce the variance of the optimal parameters \(\boldsymbol{\beta}\). These topics and other related ones, will be discussed after the more linear algebra oriented analysis here.
-
Using our insights about the SVD of the design matrix \(\boldsymbol{X}\)
-We have already analyzed the OLS solutions in terms of the eigenvectors (the columns) of the right singular value matrix \(\boldsymbol{U}\) as
Ridge regression finds the coordinates of \(\boldsymbol{y}\) with respect to the
-orthonormal basis \(\boldsymbol{U}\), it then shrinks the coordinates by
-\(\frac{\sigma_j^2}{\sigma_j^2+\lambda}\). Recall that the SVD has
-eigenvalues ordered in a descending way, that is \(\sigma_i \geq
-\sigma_{i+1}\).
-
For small eigenvalues \(\sigma_i\) it means that their contributions become less important, a fact which can be used to reduce the number of degrees of freedom. More about this when we have covered the material on a statistical interpretation of various linear regression methods.
-
For the sake of simplicity, let us assume that the design matrix is orthonormal, that is
that is the Ridge estimator scales the OLS estimator by the inverse of a factor \(1+\lambda\), and
-the Ridge estimator converges to zero when the hyperparameter goes to
-infinity.
-
We will come back to more interpreations after we have gone through some of the statistical analysis part.
Using the matrix-vector expression for Lasso regression and dropping the parameter \(1/n\) in front of the standard means squared error equation, we have the following cost function
Taking the derivative with respect to \(\boldsymbol{\beta}\) and recalling that the derivative of the absolute value is (we drop the boldfaced vector symbol for simplicty)
This equation does not lead to a nice analytical equation as in either Ridge regression or ordinary least squares. This equation can however be solved by using standard convex optimization algorithms using for example the Python package CVXOPT. We will discuss this later.
How do we use the SVD to invert a matrix \(\boldsymbol{X}^\boldsymbol{X}\) which is singular or near singular?
-The simple answer is to use the linear algebra function for the pseudoinverse, that is
-
-
-
#Ainv = np.linlag.pinv(A)
-
-
-
-
-
Let us first look at a matrix which does not causes problems and write our own function where we just use the SVD.
-
-
-
importnumpyasnp
-# SVD inversion
-defSVDinv(A):
- ''' Takes as input a numpy matrix A and returns inv(A) based on singular value decomposition (SVD).
- SVD is numerically more stable than the inversion algorithms provided by
- numpy and scipy.linalg at the cost of being slower.
- '''
- U,s,VT=np.linalg.svd(A)
- print('test U')
- print((np.transpose(U)@U-U@np.transpose(U)))
- print('test VT')
- print((np.transpose(VT)@VT-VT@np.transpose(VT)))
-
-
- D=np.zeros((len(U),len(VT)))
- D=np.diag(s)
- UT=np.transpose(U);V=np.transpose(VT);invD=np.linalg.inv(D)
- returnnp.matmul(V,np.matmul(invD,UT))
-
-
-#X = np.array([ [1.0, -1.0, 2.0], [1.0, 0.0, 1.0], [1.0, 2.0, -1.0], [1.0, 1.0, 0.0] ])
-# Non-singular square matrix
-X=np.array([[1,2,3],[2,4,5],[3,5,6]])
-print(X)
-A=np.transpose(X)@X
-# Brute force inversion
-B=np.linalg.inv(A)# here we could use np.linalg.pinv(A)
-C=SVDinv(A)
-print(np.abs(B-C))
-
Although our matrix to invert \(\boldsymbol{X}^T\boldsymbol{X}\) is a square matrix, our matrix may be singular.
-
The pseudoinverse is the generalization of the matrix inverse for square matrices to
-rectangular matrices where the number of rows and columns are not equal.
-
It is also called the the Moore-Penrose Inverse after two independent discoverers of the method or the Generalized Inverse.
-It is used for the calculation of the inverse for singular or near singular matrices and for rectangular matrices.
-
Using the SVD we can obtain the pseudoinverse of a matrix \(\boldsymbol{A}\) (labeled here as \(\boldsymbol{A}_{\mathrm{PI}}\)
where \(\boldsymbol{D}_{\mathrm{PI}}\) can be calculated by creating a diagonal matrix from \(\boldsymbol{Sigma}\) where we only keep the singular values (the non-zero values). The following code computes the pseudoinvers of the matrix based on the SVD.
-
-
-
importnumpyasnp
-# SVD inversion
-defSVDinv(A):
- U,s,VT=np.linalg.svd(A)
- # reciprocals of singular values of s
- d=1.0/s
- # create m x n D matrix
- D=np.zeros(A.shape)
- # populate D with n x n diagonal matrix
- D[:A.shape[1],:A.shape[1]]=np.diag(d)
- UT=np.transpose(U)
- V=np.transpose(VT)
- returnnp.matmul(V,np.matmul(D.T,UT))
-
-
-A=np.array([[0.3,0.4],[0.5,0.6],[0.7,0.8],[0.9,1.0]])
-print(A)
-# Brute force inversion of super-collinear matrix
-B=np.linalg.pinv(A)
-print(B)
-# Compare our own algorithm with pinv
-C=SVDinv(A)
-print(np.abs(C-B))
-
Let us remind ourselves about the expression for the standard Mean Squared Error (MSE) which we used to define our cost function and the equations for the ordinary least squares (OLS) method, that is
our optimization problem is
Using the constraint on \(\beta_0\) and \(\beta_1\) we can then find the optimal value of \(\lambda\) for the different cases. We leave this as an exercise to you.
Here we set up the OLS, Ridge and Lasso functionality in order to study the above example. Note that here we have opted for a set of values of \(\lambda\), meaning that we need to perform a search in order to find the optimal values.
-
First we study and compare the OLS and Ridge results. The next code compares all three methods.
+
First we study and compare the OLS and Ridge results. The next code compares all three methods.
+We select values of the hyperparameter \(\lambda\in [10^{-4},10^4]\) and compute the predicted values for ordinary least squares and Ridge regression.
We see here that we reach a plateau. What is actually happening?
+
We see here that we reach a plateau for the Ridge results. Writing out the coefficients \(\boldsymbol{\beta}\), we that they are getting smaller and smaller and our error stabilizes since the predicted values of \(\tilde{\boldsymbol{y}}\) approach zero.
+
This happens also for Lasso regression, as seen from the next code
+output. The difference is that Lasso shrinks the values of \(\beta\) to
+zero at a much earlier stage and the results flatten out. We see that
+Lasso gives also an excellent fit for small values of \(\lambda\) and
+shows rthe best performance of the three regression methods.
importos
@@ -2181,10 +2040,18 @@ Training MSE for OLS
[ 0. -0.]
-
+
-
Another Example, now with a polynomial fit.
+
We bring then back our exponential function example and study all
+three regression methods. Depending on the level of noise, we note
+that for small values of the hyperparameter \(\lambda\) all three
+methods produce the same mean squared error. Again, Lasso shrinks the
+parameter values to zero much earlier than Ridge regression and the
+Lasso results flatten out much earlier since all \(\beta_j=0\) (check
+this by printing the values). This case is an example of where OLS
+performs best. Lasso and Ridge reproduce the OLS results for a limited
+set of \(\lambda\) values.
importos
@@ -2276,12 +2143,304 @@ Test MSE OLS
0.008675369724976777
-
+
+
+
+
Both these example send a clear message. The addition of a
+shrinkage/regularization term implies that we need to perform a search
+for the optimal values of \(\lambda\). We will see this throughout these
+series of lectures.
+
As a small addendum, we note that you can also solve this problem using the convex optimization package CVXOPT. This requires, in addition to having installed CVXOPT, you need to download the file l1regl.py.
+The following code example solves the simpler problem we discussed above, where we have added the latter python file.
+
+
+
fromcvxoptimportmatrix,spdiag,mul,div,sqrt,normal,setseed
+fromcvxoptimportblas,lapack,solvers,sparse,spmatrix
+importmath
+
+try:
+ importmosek
+ importsys
+ __MOSEK=True
+except:__MOSEK=False
+
+if__MOSEK:
+
+ defl1regls_mosek(A,b):
+ """
+
+ Returns the solution of l1-norm regularized least-squares problem
+
+ minimize || A*x - b ||_2^2 + e'*u
+
+ subject to -u <= x <= u
+
+ """
+
+ m,n=A.size
+
+ env=mosek.Env()
+ task=env.Task(0,0)
+ task.set_Stream(mosek.streamtype.log,lambdax:sys.stdout.write(x))
+
+ task.appendvars(2*n)# number of variables
+ task.appendcons(2*n)# number of constraints
+
+ # input quadratic objective
+ Q=matrix(0.0,(n,n))
+ blas.syrk(A,Q,alpha=2.0,trans='T')
+
+ I=[]
+ foriinrange(n):
+ I.extend(range(i,n))
+
+ J=[]
+ foriinrange(n):
+ J.extend((n-i)*[i])
+
+ task.putqobj(I,J,list(Q[matrix(I)+matrix(J)*n]))
+ task.putclist(range(2*n),list(-2*A.T*b)+n*[1.0])# setup linear objective
+
+ # input constraint matrix row by row
+ foriinrange(n):
+ task.putarow(i,[i,n+i],[1.0,-1.0])
+ task.putarow(n+i,[i,n+i],[1.0,1.0])
+
+ # setup bounds on constraints
+ task.putboundslice(mosek.accmode.con,
+ 0,n,n*[mosek.boundkey.up],n*[0.0],n*[0.0])
+ task.putboundslice(mosek.accmode.con,
+ n,2*n,n*[mosek.boundkey.lo],n*[0.0],n*[0.0])
+
+ # setup variable bounds
+ task.putboundslice(mosek.accmode.var,
+ 0,2*n,2*n*[mosek.boundkey.fr],2*n*[0.0],2*n*[0.0])
+
+ # optimize the task
+ task.putobjsense(mosek.objsense.minimize)
+ task.optimize()
+ task.solutionsummary(mosek.streamtype.log)
+ x=n*[0.0]
+ task.getsolutionslice(mosek.soltype.itr,mosek.solitem.xx,0,n,x)
+
+ returnmatrix(x)
+
+ defl1regls_mosek2(A,b):
+ """
+
+ Returns the solution of l1-norm regularized least-squares problem
+
+ minimize w'*w + e'*u
+
+ subject to -u <= x <= u
+
+ A*x - w = b
+
+ """
+
+ m,n=A.size
+
+ env=mosek.Env()
+ task=env.Task(0,0)
+ task.set_Stream(mosek.streamtype.log,lambdax:sys.stdout.write(x))
+
+ task.appendvars(2*n+m)# number of variables
+ task.appendcons(2*n+m)# number of constraints
+
+ # input quadratic objective
+ task.putqobj(range(2*n,2*n+m),range(2*n,2*n+m),m*[2.0])
+
+ task.putclist(range(2*n+m),n*[0.0]+n*[1.0]+m*[0.0])# setup linear objective
+
+ # input constraint matrix row by row
+ foriinrange(n):
+ task.putarow(i,[i,n+i],[1.0,-1.0])
+ task.putarow(n+i,[i,n+i],[1.0,1.0])
+
+ foriinrange(m):
+ task.putarow(2*n+i,range(n)+[2*n+i],list(A[i,:])+[-1.0])
+
+ # setup bounds on constraints
+ task.putboundslice(mosek.accmode.con,
+ 0,n,n*[mosek.boundkey.up],n*[0.0],n*[0.0])
+ task.putboundslice(mosek.accmode.con,
+ n,2*n,n*[mosek.boundkey.lo],n*[0.0],n*[0.0])
+ task.putboundslice(mosek.accmode.con,
+ 2*n,2*n+m,m*[mosek.boundkey.fx],list(b),list(b))
+
+ # setup variable bounds
+ task.putboundslice(mosek.accmode.var,0,2*n+m,(2*n+m)*[mosek.boundkey.fr],
+ (2*n+m)*[0.0],(2*n+m)*[0.0])
+
+ # optimize the task
+ task.putobjsense(mosek.objsense.minimize)
+ task.optimize()
+ task.solutionsummary(mosek.streamtype.log)
+ x=n*[0.0]
+ task.getsolutionslice(mosek.soltype.itr,mosek.solitem.xx,0,n,x)
+
+ returnmatrix(x)
+
+defl1regls(A,b):
+ """
+
+ Returns the solution of l1-norm regularized least-squares problem
+
+ minimize || A*x - b ||_2^2 + || x ||_1.
+
+ """
+
+ m,n=A.size
+ q=matrix(1.0,(2*n,1))
+ q[:n]=-2.0*A.T*b
+
+ defP(u,v,alpha=1.0,beta=0.0):
+ """
+ v := alpha * 2.0 * [ A'*A, 0; 0, 0 ] * u + beta * v
+ """
+ v*=beta
+ v[:n]+=alpha*2.0*A.T*(A*u[:n])
+
+
+ defG(u,v,alpha=1.0,beta=0.0,trans='N'):
+ """
+ v := alpha*[I, -I; -I, -I] * u + beta * v (trans = 'N' or 'T')
+ """
+
+ v*=beta
+ v[:n]+=alpha*(u[:n]-u[n:])
+ v[n:]+=alpha*(-u[:n]-u[n:])
+
+ h=matrix(0.0,(2*n,1))
+
+
+ # Customized solver for the KKT system
+ #
+ # [ 2.0*A'*A 0 I -I ] [x[:n] ] [bx[:n] ]
+ # [ 0 0 -I -I ] [x[n:] ] = [bx[n:] ].
+ # [ I -I -D1^-1 0 ] [zl[:n]] [bzl[:n]]
+ # [ -I -I 0 -D2^-1 ] [zl[n:]] [bzl[n:]]
+ #
+ # where D1 = W['di'][:n]**2, D2 = W['di'][:n]**2.
+ #
+ # We first eliminate zl and x[n:]:
+ #
+ # ( 2*A'*A + 4*D1*D2*(D1+D2)^-1 ) * x[:n] =
+ # bx[:n] - (D2-D1)*(D1+D2)^-1 * bx[n:] +
+ # D1 * ( I + (D2-D1)*(D1+D2)^-1 ) * bzl[:n] -
+ # D2 * ( I - (D2-D1)*(D1+D2)^-1 ) * bzl[n:]
+ #
+ # x[n:] = (D1+D2)^-1 * ( bx[n:] - D1*bzl[:n] - D2*bzl[n:] )
+ # - (D2-D1)*(D1+D2)^-1 * x[:n]
+ #
+ # zl[:n] = D1 * ( x[:n] - x[n:] - bzl[:n] )
+ # zl[n:] = D2 * (-x[:n] - x[n:] - bzl[n:] ).
+ #
+ # The first equation has the form
+ #
+ # (A'*A + D)*x[:n] = rhs
+ #
+ # and is equivalent to
+ #
+ # [ D A' ] [ x:n] ] = [ rhs ]
+ # [ A -I ] [ v ] [ 0 ].
+ #
+ # It can be solved as
+ #
+ # ( A*D^-1*A' + I ) * v = A * D^-1 * rhs
+ # x[:n] = D^-1 * ( rhs - A'*v ).
+
+ S=matrix(0.0,(m,m))
+ Asc=matrix(0.0,(m,n))
+ v=matrix(0.0,(m,1))
+
+ defFkkt(W):
+
+ # Factor
+ #
+ # S = A*D^-1*A' + I
+ #
+ # where D = 2*D1*D2*(D1+D2)^-1, D1 = d[:n]**-2, D2 = d[n:]**-2.
+
+ d1,d2=W['di'][:n]**2,W['di'][n:]**2
+
+ # ds is square root of diagonal of D
+ ds=math.sqrt(2.0)*div(mul(W['di'][:n],W['di'][n:]),
+ sqrt(d1+d2))
+ d3=div(d2-d1,d1+d2)
+
+ # Asc = A*diag(d)^-1/2
+ Asc=A*spdiag(ds**-1)
+
+ # S = I + A * D^-1 * A'
+ blas.syrk(Asc,S)
+ S[::m+1]+=1.0
+ lapack.potrf(S)
+
+ defg(x,y,z):
+
+ x[:n]=0.5*(x[:n]-mul(d3,x[n:])+
+ mul(d1,z[:n]+mul(d3,z[:n]))-mul(d2,z[n:]-
+ mul(d3,z[n:])))
+ x[:n]=div(x[:n],ds)
+
+ # Solve
+ #
+ # S * v = 0.5 * A * D^-1 * ( bx[:n] -
+ # (D2-D1)*(D1+D2)^-1 * bx[n:] +
+ # D1 * ( I + (D2-D1)*(D1+D2)^-1 ) * bzl[:n] -
+ # D2 * ( I - (D2-D1)*(D1+D2)^-1 ) * bzl[n:] )
+
+ blas.gemv(Asc,x,v)
+ lapack.potrs(S,v)
+
+ # x[:n] = D^-1 * ( rhs - A'*v ).
+ blas.gemv(Asc,v,x,alpha=-1.0,beta=1.0,trans='T')
+ x[:n]=div(x[:n],ds)
+
+ # x[n:] = (D1+D2)^-1 * ( bx[n:] - D1*bzl[:n] - D2*bzl[n:] )
+ # - (D2-D1)*(D1+D2)^-1 * x[:n]
+ x[n:]=div(x[n:]-mul(d1,z[:n])-mul(d2,z[n:]),d1+d2)\
+ -mul(d3,x[:n])
+
+ # zl[:n] = D1^1/2 * ( x[:n] - x[n:] - bzl[:n] )
+ # zl[n:] = D2^1/2 * ( -x[:n] - x[n:] - bzl[n:] ).
+ z[:n]=mul(W['di'][:n],x[:n]-x[n:]-z[:n])
+ z[n:]=mul(W['di'][n:],-x[:n]-x[n:]-z[n:])
+
+ returng
+
+ returnsolvers.coneqp(P,q,G,h,kktsolver=Fkkt)['x'][:n]
+
+
+
+
+
Then we call the above functions and solve the problem, as done here
4.12. Linking the regression analysis with a statistical interpretation¶
+
4.11. Linking the regression analysis with a statistical interpretation¶
We will now couple the discussions of ordinary least squares, Ridge
and Lasso regression with a statistical interpretation, that is we
move from a linear algebra analysis to a statistical analysis. In
@@ -2420,7 +2579,7 @@ matrix product is non-negative definite.
This means the variance we obtain with the standard OLS will always for \(\lambda > 0\) be larger than the variance of \(\boldsymbol{\beta}\) obtained with the Ridge estimator. This has interesting consequences when we discuss the so-called bias-variance trade-off below.
-
4.13. Deriving OLS from a probability distribution¶
+
4.12. Deriving OLS from a probability distribution¶
Our basic assumption when we derived the OLS equations was to assume
that our output is determined by a given continuous function
\(f(\boldsymbol{x})\) and a random noise \(\boldsymbol{\epsilon}\) given by the normal
@@ -2576,7 +2735,7 @@ p(Y=1\vert X=1)=\frac{p(X=1\vert Y=1)p(Y=1)}{p(X=1\vert Y=1)p(Y=1)+p(X=1\vert Y=
That is, in case of a positive test, there is only a \(3\%\) chance of having breast cancer!
-
4.14. Bayes’ Theorem and Ridge and Lasso Regression¶
+
4.13. Bayes’ Theorem and Ridge and Lasso Regression¶
Hitherto we have discussed Ridge and Lasso regression in terms of a
linear analysis. This may to many of you feel rather technical and
perhaps not that intuitive. The question is whether we can develop a
@@ -2662,7 +2821,9 @@ Test MSE OLS
0.958228616652075
0.001 [ 1.0170259 0.27852549 -1.40702 1.03193199 0. ]
0.001 [ 1.034342 -0.18063928 -0. 0. 0. ]
-0.021544346900318832 [ 1.01825571 0.26412372 -1.37186301 1.00890601 0. ]
+
@@ -725,18 +725,112 @@ the probability of a given category. This leads us to the logistic function.
-
---------------------------------------------------------------------------
-FileNotFoundErrorTraceback (most recent call last)
-<ipython-input-1-a77d5ac269b2>in<module>
-38plt.savefig(image_path(fig_id)+".png",format='png')
-39
----> 40infile=open(data_path("chddata.csv"),'r')
-41
-42# Read the chd data as csv file and organize the data into arrays with age group, age, and chd
+
+
+
+
+
+
+
ID
+
Age
+
Agegroup
+
CHD
+
+
+
+
+
0
+
1
+
21
+
1
+
0
+
+
+
1
+
2
+
23
+
1
+
0
+
+
+
2
+
3
+
25
+
1
+
1
+
+
+
3
+
4
+
29
+
1
+
0
+
+
+
4
+
5
+
21
+
1
+
0
+
+
+
...
+
...
+
...
+
...
+
...
+
+
+
95
+
96
+
61
+
8
+
1
+
+
+
96
+
97
+
69
+
8
+
1
+
+
+
97
+
98
+
65
+
8
+
1
+
+
+
98
+
99
+
64
+
8
+
1
+
+
+
99
+
100
+
63
+
8
+
0
+
+
+
+
100 rows × 4 columns
+
What we could attempt however is to plot the mean value for each group.
@@ -753,6 +847,9 @@ the probability of a given category. This leads us to the logistic function.
+
+
+
We are now trying to find a function \(f(y\vert x)\), that is a function which gives us an expected value for the output \(y\) with a given input \(x\).
In standard linear regression with a linear dependence on \(x\), we would write this in terms of our model
We assume now that we have two classes with \(y_i\) either \(0\) or \(1\). Furthermore we assume also that we have only two parameters \(\beta\) in our fitting of the Sigmoid function, that is we define probabilities
@@ -1016,6 +1118,24 @@ classification.
+
+
(426, 30)
+(143, 30)
+Test set accuracy with Logistic Regression: 0.94
+Test set accuracy Logistic Regression with scaled data: 0.96
+
+
+
/Users/mhjensen/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_logistic.py:762: ConvergenceWarning: lbfgs failed to converge (status=1):
+STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.
+
+Increase the number of iterations (max_iter) or scale the data as shown in:
+ https://scikit-learn.org/stable/modules/preprocessing.html
+Please also refer to the documentation for alternative solver options:
+ https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression
+ n_iter_i = _check_optimize_result(
+
+
+
In addition to the above scores, we could also study the covariance (and the correlation matrix).
We use Pandas to compute the correlation matrix.
@@ -1058,6 +1178,10 @@ We use Pandas to compute the correlation matrix.
+
+
+
+
In the above example we note two things. In the first plot we display
the overlap of benign and malignant tumors as functions of the various
@@ -1138,6 +1262,39 @@ applications. This will be discussed later this semester (
+
diff --git a/doc/LectureNotes/_build/html/reports/chapter4.log b/doc/LectureNotes/_build/html/reports/chapter4.log
index f2d559b52..64735e6fa 100644
--- a/doc/LectureNotes/_build/html/reports/chapter4.log
+++ b/doc/LectureNotes/_build/html/reports/chapter4.log
@@ -17,73 +17,61 @@ Traceback (most recent call last):
raise CellExecutionError.from_cell_and_msg(cell, exec_reply['content'])
nbclient.exceptions.CellExecutionError: An error occurred while executing the following cell:
------------------
-%matplotlib inline
-
-# Common imports
-import os
-import numpy as np
-import pandas as pd
import matplotlib.pyplot as plt
-from sklearn.linear_model import LinearRegression, Ridge, Lasso
-from sklearn.model_selection import train_test_split
-from sklearn.utils import resample
-from sklearn.metrics import mean_squared_error
-from IPython.display import display
-from pylab import plt, mpl
-plt.style.use('seaborn')
-mpl.rcParams['font.family'] = 'serif'
+import numpy as np
+from sklearn.model_selection import train_test_split
+from sklearn.datasets import load_breast_cancer
+from sklearn.linear_model import LogisticRegression
-# Where to save the figures and data files
-PROJECT_ROOT_DIR = "Results"
-FIGURE_ID = "Results/FigureFiles"
-DATA_ID = "DataFiles/"
+# Load the data
+cancer = load_breast_cancer()
-if not os.path.exists(PROJECT_ROOT_DIR):
- os.mkdir(PROJECT_ROOT_DIR)
+X_train, X_test, y_train, y_test = train_test_split(cancer.data,cancer.target,random_state=0)
+print(X_train.shape)
+print(X_test.shape)
+# Logistic Regression
+logreg = LogisticRegression(solver='lbfgs')
+logreg.fit(X_train, y_train)
+print("Test set accuracy with Logistic Regression: {:.2f}".format(logreg.score(X_test,y_test)))
+#now scale the data
+from sklearn.preprocessing import StandardScaler
+scaler = StandardScaler()
+scaler.fit(X_train)
+X_train_scaled = scaler.transform(X_train)
+X_test_scaled = scaler.transform(X_test)
+# Logistic Regression
+logreg.fit(X_train_scaled, y_train)
+print("Test set accuracy Logistic Regression with scaled data: {:.2f}".format(logreg.score(X_test_scaled,y_test)))
-if not os.path.exists(FIGURE_ID):
- os.makedirs(FIGURE_ID)
-if not os.path.exists(DATA_ID):
- os.makedirs(DATA_ID)
+from sklearn.preprocessing import LabelEncoder
+from sklearn.model_selection import cross_validate
+#Cross validation
+accuracy = cross_validate(logreg,X_test_scaled,y_test,cv=10)['test_score']
+print(accuracy)
+print("Test set accuracy with Logistic Regression and scaled data: {:.2f}".format(logreg.score(X_test_scaled,y_test)))
-def image_path(fig_id):
- return os.path.join(FIGURE_ID, fig_id)
-def data_path(dat_id):
- return os.path.join(DATA_ID, dat_id)
-
-def save_fig(fig_id):
- plt.savefig(image_path(fig_id) + ".png", format='png')
-
-infile = open(data_path("chddata.csv"),'r')
-
-# Read the chd data as csv file and organize the data into arrays with age group, age, and chd
-chd = pd.read_csv(infile, names=('ID', 'Age', 'Agegroup', 'CHD'))
-chd.columns = ['ID', 'Age', 'Agegroup', 'CHD']
-output = chd['CHD']
-age = chd['Age']
-agegroup = chd['Agegroup']
-numberID = chd['ID']
-display(chd)
-
-plt.scatter(age, output, marker='o')
-plt.axis([18,70.0,-0.1, 1.2])
-plt.xlabel(r'Age')
-plt.ylabel(r'CHD')
-plt.title(r'Age distribution and Coronary heart disease')
+import scikitplot as skplt
+y_pred = logreg.predict(X_test_scaled)
+skplt.metrics.plot_confusion_matrix(y_test, y_pred, normalize=True)
+plt.show()
+y_probas = logreg.predict_proba(X_test_scaled)
+skplt.metrics.plot_roc(y_test, y_probas)
+plt.show()
+skplt.metrics.plot_cumulative_gain(y_test, y_probas)
plt.show()
------------------
[0;31m---------------------------------------------------------------------------[0m
-[0;31mFileNotFoundError[0m Traceback (most recent call last)
-[0;32m[0m in [0;36m[0;34m[0m
-[1;32m 38[0m [0mplt[0m[0;34m.[0m[0msavefig[0m[0;34m([0m[0mimage_path[0m[0;34m([0m[0mfig_id[0m[0;34m)[0m [0;34m+[0m [0;34m".png"[0m[0;34m,[0m [0mformat[0m[0;34m=[0m[0;34m'png'[0m[0;34m)[0m[0;34m[0m[0;34m[0m[0m
-[1;32m 39[0m [0;34m[0m[0m
-[0;32m---> 40[0;31m [0minfile[0m [0;34m=[0m [0mopen[0m[0;34m([0m[0mdata_path[0m[0;34m([0m[0;34m"chddata.csv"[0m[0;34m)[0m[0;34m,[0m[0;34m'r'[0m[0;34m)[0m[0;34m[0m[0;34m[0m[0m
-[0m[1;32m 41[0m [0;34m[0m[0m
-[1;32m 42[0m [0;31m# Read the chd data as csv file and organize the data into arrays with age group, age, and chd[0m[0;34m[0m[0;34m[0m[0;34m[0m[0m
+[0;31mModuleNotFoundError[0m Traceback (most recent call last)
+[0;32m[0m in [0;36m[0;34m[0m
+[1;32m 34[0m [0;34m[0m[0m
+[1;32m 35[0m [0;34m[0m[0m
+[0;32m---> 36[0;31m [0;32mimport[0m [0mscikitplot[0m [0;32mas[0m [0mskplt[0m[0;34m[0m[0;34m[0m[0m
+[0m[1;32m 37[0m [0my_pred[0m [0;34m=[0m [0mlogreg[0m[0;34m.[0m[0mpredict[0m[0;34m([0m[0mX_test_scaled[0m[0;34m)[0m[0;34m[0m[0;34m[0m[0m
+[1;32m 38[0m [0mskplt[0m[0;34m.[0m[0mmetrics[0m[0;34m.[0m[0mplot_confusion_matrix[0m[0;34m([0m[0my_test[0m[0;34m,[0m [0my_pred[0m[0;34m,[0m [0mnormalize[0m[0;34m=[0m[0;32mTrue[0m[0;34m)[0m[0;34m[0m[0;34m[0m[0m
-[0;31mFileNotFoundError[0m: [Errno 2] No such file or directory: 'DataFiles/chddata.csv'
-FileNotFoundError: [Errno 2] No such file or directory: 'DataFiles/chddata.csv'
+[0;31mModuleNotFoundError[0m: No module named 'scikitplot'
+ModuleNotFoundError: No module named 'scikitplot'
diff --git a/doc/LectureNotes/_build/html/searchindex.js b/doc/LectureNotes/_build/html/searchindex.js
index 5185a8cf4..08e6b3d20 100644
--- a/doc/LectureNotes/_build/html/searchindex.js
+++ b/doc/LectureNotes/_build/html/searchindex.js
@@ -1 +1 @@
-Search.setIndex({docnames:["Clustering","chapter1","chapter10","chapter2","chapter3","chapter4","chapter5","chapter6","chapter7","chapter8","chapter9","content","intro","linalg","schedule","statistics","teachers","textbooks"],envversion:{"sphinx.domains.c":2,"sphinx.domains.changeset":1,"sphinx.domains.citation":1,"sphinx.domains.cpp":3,"sphinx.domains.index":1,"sphinx.domains.javascript":2,"sphinx.domains.math":2,"sphinx.domains.python":2,"sphinx.domains.rst":2,"sphinx.domains.std":1,"sphinx.ext.intersphinx":1,sphinx:56},filenames:["Clustering.ipynb","chapter1.ipynb","chapter10.ipynb","chapter2.ipynb","chapter3.ipynb","chapter4.ipynb","chapter5.ipynb","chapter6.ipynb","chapter7.ipynb","chapter8.ipynb","chapter9.ipynb","content.md","intro.md","linalg.ipynb","schedule.md","statistics.ipynb","teachers.md","textbooks.md"],objects:{},objnames:{},objtypes:{},terms:{"000":2,"000000":[1,3,9],"00000000e":[1,3],"0001":2,"00015921913736794912":7,"00019998":3,"00024087":3,"00029012":3,"00034944":3,"00042089":3,"00050694":3,"00061058":3,"00073541":3,"00079968":3,"00088573":3,"00096314":3,"001":[2,3,5,6],"00106677":3,"00109273":[],"00115999":3,"00128479":3,"00139705":3,"00154733":3,"00168251":3,"00186347":3,"00200":6,"00202624":3,"00224413":3,"0024401":3,"00270244":3,"00293838":3,"0032542":3,"0032873138755776365":15,"00353823":3,"003704":1,"003717":1,"003759":1,"003774":1,"003788":1,"00391839":3,"004":3,"00426027":3,"00433417":9,"004456043243408203":0,"00445655":9,"00471782":3,"0049999999999999845":[],"004999999999999996":[],"004999999999999997":[],"0050000000000000044":[],"005000000000000007":1,"00512927":3,"0056799":3,"00617499":3,"00683748":3,"00727646693":1,"00728323e":3,"0074331":3,"007789":[],"007824":1,"00823002":3,"0086649156":1,"008675369724976777":3,"008885578722629236":[],"00890601":3,"00894639":3,"009154":1,"009163470508352228":3,"009735":9,"00973536":9,"009790":1,"009883615646716182":1,"009883615646716184":[],"009883615646716186":[],"00990475":3,"009911":1,"01057384067458835":[],"010679893512872646":[],"01076611":3,"0110":15,"011347":1,"01191824":3,"01210814993019007":[],"01295356":3,"0140617":[],"01433809":3,"01503674":[],"015037":[],"01514394":[],"015144":[],"01558197":3,"0169643":3,"0170259":3,"01724499":3,"01825571":3,"01867234e":15,"01873344":9,"01873869":3,"02073509":3,"021544346900318832":3,"02161783e":15,"021901":9,"02190139":9,"02252765":3,"02492265":3,"02493044e":[],"02568378":[],"025709":9,"02625193":6,"02707227":3,"02730126581656065":15,"02881304":[],"029688":[],"02968834":[],"02994311":3,"03019554":3,"030196":3,"030670":[],"03067028":[],"03099776":3,"031":3,"03193199":3,"03251863":3,"03256632e":2,"03303359":[],"033037772005753835":3,"03308408":3,"034342":3,"0344707":3,"0359565":3,"03607832":[],"03652792":3,"036528":3,"03697069":[],"03707133":9,"03750367":13,"03787596148305236":1,"03791824e":15,"038300":9,"03871832":[],"039039":3,"04205220e":15,"04260073e":[],"04315108":3,"04330858":[],"04346721":3,"04421672e":[],"044402":1,"04455272":3,"044553":3,"04529095":3,"0458":7,"04648335":3,"04683565":3,"04757618e":[],"04829457939163212":15,"051115":[],"051649":9,"0517473":3,"052645":[],"053053":3,"0530533":3,"05318162":[],"053182":[],"05364854":6,"05396545e":[],"054187":[],"054276":[],"054674":[],"055":4,"055085":3,"055744":[],"055774":3,"055831":[],"055887":[],"055916":[],"05614483":3,"056326":[],"056329":9,"056484":3,"057035":[],"057088709963637":1,"057320":[],"057332":[],"05755374":[],"057571":[],"057654":[],"057919":[],"058178":[],"058282":[],"058805":[],"058892":3,"059245":[],"059344":9,"059406":[],"059455":3,"059531":[],"059571":[],"059806":[],"059865":3,"059908":[],"059994":[],"060064":9,"060070":9,"060308":[],"06041458e":[],"060520":3,"060919":[],"061164":[],"061176":[],"061255":[],"06126507e":15,"061330":[],"061601":9,"061652":[],"061774":[],"061842":[],"061855":[],"061896":[],"06200174":3,"06209126":[],"062188":[],"062327":3,"062376":[],"062639":[],"062659":3,"062660":9,"062731":3,"062822":[],"062829":[],"062879":[],"062885":[],"062923":9,"062979":[],"063055":3,"063097":[],"063122":[],"063178":9,"063324":9,"063395":[],"063415":[],"063526":3,"063615":[],"063670":3,"063727":[],"063759":[],"063763":[],"063882":[],"063982":[],"064000":3,"06412177e":15,"064304":[],"064354":3,"064410":[],"064472":[],"064648":9,"064686":[],"064783":9,"06492086e":1,"064929":3,"064964":[],"064982":[],"065":4,"065003":[],"065086":[],"065110":9,"065144":[],"065151":[],"065200":[],"065202":[],"065204":3,"065355":[],"065591":[],"065618":[],"065660":[],"065680":[],"065762":[],"065860":9,"065994":[],"066002":9,"06600226":9,"066030":9,"066047":3,"066074":9,"066092":[],"066116":3,"06619182206626131":[],"066249":3,"066272":3,"066294":[],"066312":9,"066389":[],"066403":9,"066487":[],"066629":[],"066683":9,"066761":[],"066788":[],"066835":[],"066837":9,"066951":9,"066963":[],"067043":3,"067061":[],"067123":[],"067132":9,"067175":9,"06724062":3,"067440":9,"067504":3,"067531":[],"067577":[],"067601":[],"067610":3,"067667":9,"067685":9,"067769":3,"067777":[],"067844":[],"067865":[],"067892":[],"067985":[],"067990":[],"068073":3,"068185":3,"068276":[],"068351":9,"068460":[],"068514":[],"068654":[],"068697":9,"068818":9,"068838":9,"068840":[],"068920":[],"069005":[],"069028":[],"069081":[],"069098":[],"069195":9,"069243":9,"069282":3,"069369":[],"069413":9,"069503":[],"069510":3,"069516":[],"069528":[],"069635":3,"069641":9,"069775":9,"069814":[],"069848":3,"069859":3,"069872":3,"069890":[],"069902":[],"069988":9,"070103":3,"070129":[],"070131":[],"070138":[],"070182":[],"070188":3,"070254":[],"070337":9,"070456":[],"070517":3,"070548":9,"070653":3,"070762":9,"070791":9,"070812":[],"070815":[],"070867":9,"070889":9,"070967":[],"071016":[],"071049":9,"07106781e":[],"071096":[],"07110274":13,"071248":[],"071268":3,"0713":1,"071313":[],"071325":[],"071435":9,"07145103":9,"071481":[],"071502":[],"071547":9,"07155335":[],"071576":[],"071579":[],"07159175":[],"07160764":3,"071660":9,"071667":3,"071681":[],"071684":[],"071761":[],"071855":3,"071887":[],"071917":[],"072094":[],"072198":[],"072279":3,"072424":[],"072471":[],"072555":9,"072589":[],"072598":3,"072620":[],"072654":9,"072710":9,"072826":[],"072879":9,"072910":3,"073":[],"073020":[],"073035":3,"073096":9,"073310":9,"073333":3,"073352":[],"073371":9,"073372":[],"073483":[],"073567":[],"073598":9,"073602":[],"073656":9,"073765":9,"073851":[],"073902":3,"073915":1,"073966":3,"074":[],"074027":[],"074067":9,"074096":3,"074099":3,"074134":[],"074170":[],"074191":[],"074201":9,"074207":3,"07421084":3,"074230":[],"07423848370736122":[],"074248":3,"074286":[],"074331":9,"074432":3,"074453":[],"074488":[],"07456491":3,"074568":[],"074618":9,"074665":3,"074696":3,"074848":[],"074979":9,"075084":[],"075119":[],"075163":3,"075194":9,"075212":[],"07521771":[],"075218":[],"075261":[],"075266":[],"075289":3,"075296":9,"075309":3,"075526":9,"075545":9,"0755452":9,"075562":[],"075615":9,"075754":3,"075772":[],"07581375":3,"075814":3,"075821":[],"075865":3,"075993":9,"076062":[],"076189":3,"076283":3,"076320":[],"076367":9,"076398":[],"076408":[],"076410":9,"076466":9,"076469":[],"076730":[],"076938":9,"077010":9,"077046":9,"077143":9,"077160":[],"077212":3,"07729012413236423":9,"077303":3,"077305":9,"077386":3,"07744472306026946":7,"077469":[],"077533":[],"077623":[],"077771":9,"07777777777777778":2,"077891":[],"077927":9,"078049":[],"078076":3,"078099":9,"078103":9,"078244":9,"078279":3,"078311":[],"078388":3,"078399":3,"07839924":3,"078451":3,"078674":3,"078693":9,"078710":9,"078746":9,"078927":[],"078930":[],"078937":3,"078966":[],"079001":[],"079037":3,"079137":3,"079139":[],"079333":3,"079365":[],"07944154":13,"079624":[],"079785":9,"079848":9,"079956":[],"080024":9,"080084":1,"080115":[],"080223":3,"080264":9,"080325":9,"080336":3,"080381":3,"080387":3,"080398":[],"08043851":3,"080570":9,"080626":9,"080675":9,"080845":[],"080871":3,"081077":9,"081103":3,"081129":9,"081210":9,"081227":3,"081260":[],"081402":[],"081480":3,"081489":9,"081514":[],"08170444":[],"081816":[],"081886":[],"082211":9,"082225":9,"082247":9,"08248290e":[],"08251898":15,"082687":3,"08271336":[],"082866":3,"082990":[],"083128":[],"08318298e":2,"0832":[],"083295":9,"083376":[],"083577":[],"083618":3,"083657":3,"083658":9,"083728":3,"083775":3,"084092":[],"08415761":[],"084167":[],"084181":3,"084249":9,"084251":3,"084604":9,"084813":9,"084873":[],"084931":3,"08501147":1,"085121":9,"085285":[],"085289":3,"085597":3,"085646":9,"085676":[],"085928":3,"085951":[],"08611111111111111":2,"086116":[],"086325":3,"086465":3,"086592":[],"08665060086846632":3,"086652":9,"086974":9,"087202":9,"087482":[],"087563":9,"0881981":3,"08839750e":1,"088660":3,"08871404":3,"088871":1,"08888888888888889":2,"089300":3,"089425":9,"089504":3,"089916":3,"089931":3,"090274":9,"09103481e":15,"091159":3,"09166666666666666":2,"0917":7,"091979":3,"093755":9,"09487315108590391":9,"095173":3,"095472":3,"09609807":3,"096726":[],"09672604":[],"09678277e":15,"09741585e":[],"097452":3,"09903804":6,"099763":[],"09976319":[],"100":[0,1,2,3,4,5,6,7,8,9,13,15,16],"1000":[0,1,2,5,6,9,12,15],"10000":[0,3,4,8,9,15],"100000":6,"10001":8,"1001":15,"1002":15,"1003":15,"1005":15,"1007125":1,"1009":15,"10094646e":[],"1011":15,"1013":15,"10131725":[],"1013904243":15,"1015":15,"102":1,"1022964509394572":2,"1023":15,"1026":15,"1027":15,"103":2,"1030":15,"10307631":[],"10327559":13,"1037":15,"10378326e":2,"1038":15,"1040":15,"10405456":9,"1047":15,"105137868830763":15,"10555555555555556":2,"10589577":3,"106095":9,"108":1,"10806972":[],"108070":[],"10896672e":15,"10898112e":[],"10th":7,"10x":1,"110":1,"1100":15,"1101":15,"11022302e":3,"111":[2,5,10],"1111111111111111":2,"112283":1,"112383":9,"11304709e":15,"11388888888888889":2,"11456076e":[],"11462415":3,"11507992e":2,"11666666666666667":2,"117":6,"11944444444444445":2,"12002944":[],"1203284":6,"121":[6,7,8],"122":[6,7,8],"12222222222222222":2,"12283463":[],"123190":1,"123459876":15,"12366979e":15,"124":1,"12497018e":1,"12618549":3,"127773":15,"12777777777777777":2,"1298":7,"13055555555555556":2,"13060772":1,"13162821e":3,"13209041":[],"13266452e":1,"133":5,"13328820e":[],"13457922":[],"13519106":1,"13579199e":[],"1361111111111111":2,"13646574":3,"137268":[],"137400784702912":1,"137652":9,"13803928":1,"13865173":3,"138775":9,"13891144e":3,"139462":4,"139475":[],"1404":1,"1437":2,"1440501043841336":2,"1445":4,"14459063":[],"1446":4,"1447":4,"1448":4,"1449":4,"14662074":1,"14722222222222223":2,"147400":9,"147420":9,"147722":4,"148":4,"149366":[],"149667":[],"149894":4,"149903":[],"14g":4,"150":6,"15005476":3,"15024669":[],"151":4,"152636":[],"1527777777777778":2,"153106":1,"15324631":1,"15332528e":[],"154720":1,"15475232":1,"154911":1,"155491":1,"155687":1,"155883":1,"156":1,"157":1,"15751596":3,"158":1,"159":1,"15979239e":15,"15g":4,"160":1,"16111111111111112":2,"16211139":3,"16342407":3,"16496581e":[],"16553696":[],"16637855e":15,"16666666666666666":2,"16807":15,"16831012":3,"16861772":3,"168618":3,"16b8e3cda33a":2,"17117385":[],"17174962e":2,"17234827e":[],"17248441e":3,"17385778e":1,"174497":3,"17449733":3,"1758506":1,"17654307":[],"17707436":[],"17777777777777778":2,"17801022":3,"17917768":3,"17949575":3,"17953942":9,"1797":2,"180092462880674":[],"18029127":3,"18063928":3,"18220995":1,"18224665":3,"18333333333333332":2,"18404906e":[],"1856411":[],"18611111111111112":2,"18620601e":1,"18673098":9,"18954529":[],"18968431e":15,"1914224774238273":[],"19207979":3,"1940":1,"1943":10,"197":1,"1970":13,"1973":7,"197370":9,"19742904e":[],"1979":4,"1989":15,"19937":15,"19955871":[],"199559":[],"19972087e":15,"1_1":10,"1_2":10,"1_3":10,"1cm":[1,6,8,15],"1e10":0,"200":[1,6,7,8,15],"200000":1,"2004":5,"2006":17,"2010":2,"2011":2,"2015":2,"2016":1,"2018":4,"201841":[],"20184113":[],"2021":0,"20277777777777778":2,"203757":3,"20375701":3,"205466494327873":[],"20632410e":3,"207545":9,"20772452":[],"20819609e":[],"20833333333333334":2,"20843563e":[],"20906175e":15,"210340":9,"21058097":3,"212":[],"21208310e":[],"212327334149492":1,"2125":[],"2126":[],"2127":[],"2128":[],"2129":[],"213":[],"213103":9,"21347282":[],"213743":9,"214":[],"2147483647":15,"215":[],"216":[],"21623344":1,"216290":9,"216683":9,"22044605e":3,"22076362":1,"221":6,"22209371e":1,"22443803":3,"22527008e":15,"22690428":3,"22717936e":15,"22847924":3,"22999784":[],"229998":[],"23117916e":15,"23167717":3,"23333333333333334":2,"2335879":[],"23382086e":[],"23438810e":1,"24444444444444444":2,"24829908":3,"250":[5,7],"25000":1,"250000":1,"250154":1,"251559":3,"2515593":3,"25190225e":1,"25226753e":15,"25240108":[],"253":[],"25303483":[],"253775":1,"254":[],"254509":[],"25450941":[],"25457052":[],"255":[],"2551":1,"256":[],"256962":1,"257":[],"25726439e":[],"2572e3a4b38d":2,"25803281":[],"259107":[],"25910749":[],"259153":9,"2627588":13,"26301436":3,"26381865":[],"263819":[],"264":1,"26412372":3,"265":1,"2653678":1,"266":1,"267":1,"26710969":3,"26780278":3,"2683":4,"2684":4,"2685":4,"2686":4,"2687":4,"26890510e":[],"269":1,"26974938e":[],"270":1,"27296891e":[],"276263":9,"27753165e":15,"27852549":3,"27924636":3,"27n_":15,"280647":9,"28166741":[],"282":1,"282727":9,"28310983":[],"283110":[],"2836":15,"28475098":6,"2861":15,"2873":7,"2882":15,"2886":15,"2890":1,"2892":15,"28971976":9,"289720":9,"290":[],"291":[],"2915":15,"291614":[],"29167186":3,"292":[],"293":[],"2931":[],"294":[],"295656121491569":[],"296247":1,"2968":[],"297219777724628":9,"297260":1,"2980":[],"298273":1,"2983233":13,"298375":1,"2990":[],"299444":9,"29944428":9,"2_1":10,"2_2":10,"2_3":10,"2_i":10,"2_m":15,"2_x":15,"2cm":6,"2ff97f4bf03b":15,"2nd":7,"2x_ix_jy_iy_j":6,"2x_j":6,"2y_i":8,"2y_j":6,"30000":1,"300162456113691":[],"30119421":6,"306854":[],"30685416":[],"30879705":1,"30968181":1,"31109204e":1,"31318084":3,"31457796":3,"3155":[3,4],"3156929654100207":7,"31608475e":[],"31718909":9,"31730641":[],"317367":9,"31853484":[],"31896852":6,"3200":2,"32047562":1,"323291478597321":15,"3250":2,"32521615e":15,"327631":1,"32938847":13,"32945844e":15,"3304":1,"33066907e":3,"3310":1,"3317":1,"333":5,"333333":1,"3338":1,"3344":1,"33443859e":[],"33544681":[],"33569998e":1,"33861512":[],"338869":3,"33886902":3,"339535706819584":15,"33953571":15,"340782":9,"34114547":3,"34114641e":1,"34172919":[],"3436":1,"3437":1,"344172":1,"34569596":3,"346433":9,"34643337":9,"34902789e":15,"350387":[],"35038714":[],"351636":9,"35176067":[],"35182854":3,"35216172e":1,"35367281e":3,"358869339268145":15,"35886934":15,"359":3,"359640894899012":[],"360":2,"360688":1,"3621311":3,"36436520e":[],"36468301":[],"3655222":3,"369139":9,"36941772":[],"369418":[],"37186301":3,"372889":3,"3728892":3,"37416969":9,"374170":9,"37738324":[],"38207279e":[],"38216436":[],"38629436":13,"38777878e":3,"38937995e":15,"38986237":[],"39579407":3,"396740":9,"39674043":9,"39706038":3,"397700":9,"39792608e":[],"3cd19a0768e1":[],"4000":17,"401842":9,"40212127":[],"404":1,"40425078e":15,"405890":9,"40702":3,"40708470e":1,"40859":4,"4087793":3,"40902095":[],"41433969":3,"41511965e":2,"415634483874318":1,"416694683938511":9,"4171578884124756":0,"41754964":[],"41770932":[],"418506":9,"41876428e":[],"42441033":3,"42847770e":15,"42937310e":15,"43054282":3,"435490":[],"43549028":[],"43766686":9,"44089210e":3,"442600":9,"44395541":[],"44625466e":15,"44970586e":2,"45013332e":[],"45019484":[],"450257":9,"4557763":9,"458078":9,"45937170e":[],"45960079":3,"46016532e":3,"461":15,"462":5,"46323168e":15,"46415888336127775":3,"46423858e":1,"465564":[],"46556436":[],"466":15,"46675058":1,"46914544e":[],"46929603e":[],"47042744":3,"47079457e":15,"47125748":3,"47132891":3,"47441766":3,"47566390e":15,"47654764e":[],"47815203":9,"48019541":3,"48154187202453613":0,"48257387":16,"48471852e":[],"48476997":9,"48608063e":[],"48994188":3,"4940954":1,"497221":[],"49722108":[],"49865673":[],"4990":15,"4992":15,"4997":15,"4c4c7f":[7,8],"4y_i":8,"500":[2,4,5,7,8,15],"500000":1,"50000455":3,"50000553":3,"50000718":3,"50000855":3,"50000969":3,"50001063":3,"50001142":3,"50001207":3,"50001261":3,"50001306":3,"50001343":3,"50001374":3,"500014":3,"50001414":3,"50001422":3,"50001439":3,"50001454":3,"50001466":3,"50001476":3,"50001485":3,"50001492":3,"50001498":3,"50001502":3,"50001506":3,"5000151":3,"50001512":3,"50001515":3,"50001517":3,"50001518":3,"50001519":3,"50001521":3,"50001522":3,"50001523":3,"50001524":3,"50001525":3,"5018":15,"50321091":3,"50394742":[],"506":1,"507d50":[7,8],"5098779":3,"509878":3,"50j":5,"50x10":2,"510":2,"5120":0,"512132":1,"51257863e":[],"51345668e":[],"51363731e":[],"51523276e":[],"515768":[],"51576807":[],"51893804e":[],"51943726":9,"519842":1,"52158335":1,"5222222222222223":2,"526744":9,"52687171":[],"526872":[],"52722156":[],"52874252":3,"52d2c51caad1":[],"5303329":9,"5305555555555556":2,"535457":3,"53545715":3,"5378811":9,"539261":9,"54039921":3,"54096582":[],"540966":[],"54121682":[],"541217":[],"54152940e":[],"54237024":15,"54702088e":[],"55111512e":3,"55138385":9,"551384":9,"55280484":[],"5555555555555556":2,"55707065":[],"557071":[],"557795":9,"55854694":9,"56033697":3,"56198284":3,"56216797e":1,"564374":9,"56536":1,"569":2,"56939714":3,"57051369":[],"571":3,"574465":9,"5755035":1,"57781668":[],"57871326":[],"581766":9,"58176612":9,"582":[1,2],"58228342e":15,"58239999":[],"582400":[],"584804":1,"58521266":9,"585213":9,"58596975":[],"585970":[],"587401":1,"58836420e":15,"5888888888888889":2,"59007674e":15,"59304755e":1,"5944444444444444":2,"59480085":[],"5cm":15,"60122668e":15,"60293962":3,"60394236e":1,"60420593":3,"60673226":9,"60949193":3,"609492":3,"60999846":[],"61069091e":15,"6111111111111112":2,"61124978":[],"61234223":[],"614808":9,"61505887e":[],"61745046e":[],"618":1,"6183694":3,"61869821":[],"622539":9,"62253933":9,"6226921":[],"62316154e":15,"62359224e":15,"62373464":9,"625":5,"62783293":[],"62856593":1,"62894215":3,"629961":1,"6300745149331701":1,"63315151":1,"63339159":[],"63374631":[],"63437572":[],"63442451e":1,"63498144":3,"636323":9,"63632311":9,"63680118":[],"637129335071195":1,"63993205e":[],"64012627":3,"64166831e":[],"64447921":[],"64580686":[],"646283":9,"64669382":[],"646694":[],"647473":9,"64857826e":[],"649382":9,"64x50":2,"650024":[],"65002433":[],"6510573774179256":15,"65105738":15,"65238878":13,"65245958":[],"653095390463358":1,"653702":13,"65482578":[],"65572035":[],"65599927":[],"65766387":[],"65885453":3,"65933852":[],"65939208e":[],"66020213e":1,"6614":[],"66183486":9,"661835":9,"6628996975186952":1,"66302359":[],"66383151":[],"66677842":[],"66800261":[],"66880047":[],"67006792":[],"67060602":[],"671089":1,"6714":[],"67171347e":15,"67279536":[],"67303655":9,"673037":9,"67407338e":15,"67450955":[],"67708423":1,"677149":[],"67714918":[],"6796265324852733":3,"68002363":3,"680024":3,"68034946e":[],"6813":4,"6814":4,"6815":4,"6816":4,"6817":4,"6818813252071303":15,"68188133":15,"68192193":3,"68316185":1,"68342382":[],"68542204":3,"68616263":13,"68729414":[],"68937695":1,"689519":9,"69069n_":15,"693361":1,"69347005":[],"693850":9,"69385025":9,"69519693":[],"695197":[],"69573183":1,"69695259":3,"69843037":3,"69981195e":15,"6999536":9,"6ea927cc6e88":2,"6n_":15,"7022283":1,"70234019":[],"70415861":[],"70523024e":[],"70589906":[],"70710678":3,"70790937":[],"70832814":3,"70885528e":3,"70886748":1,"70946493e":[],"712018":9,"71281409":[],"71351486":[],"71442781":13,"71606852":[],"7162":[],"71669651":1,"7172":[],"72108703":[],"72174172":9,"72218808":[],"72312577":[],"7236674":3,"7240496":[],"725394195434945":[],"72780613e":15,"72859758":3,"72879865e":[],"72981762":6,"73091052e":1,"73453972":[],"7371165871823337":3,"73921714":[],"74081822":6,"74107697":9,"74143127e":1,"7432283":[],"74382593":1,"74495014":[],"74840212":3,"74845978":[],"74921867":[],"751699":9,"75170092":3,"75195757":3,"7522047280566193":[],"75382481":[],"75524378":[],"75629493":[],"75841112":[],"75932862":[],"76172241e":[],"762":9,"7621419":1,"76290332":[],"763880":[],"76388013":[],"76497666":[],"765":5,"76504618":[],"76570177":1,"76648901e":[],"76936315":3,"7693978131030923":9,"77152076":3,"77156117e":3,"7718":7,"772b904ae9cb":4,"77317984":1,"7733":4,"77350269e":[],"774300":1,"77661393e":15,"77714169":6,"78080633":1,"78195":[],"78857629":1,"78941903":3,"78944806":[],"7899453":[],"79111643":3,"79295029e":15,"79326583e":15,"79328828":[],"793701":1,"794282":9,"79459035":1,"79648291":1,"79902342":1,"7c394b1e8b71":7,"7d7d58":[7,8],"800":5,"80004454e":[],"800266":[],"80026635":[],"80121":[],"80469739":3,"8055555555555556":2,"81160425":3,"81620806":[],"81633628":9,"81781888":9,"82198978":3,"8265786":3,"827265":1,"82889306e":15,"8305555555555556":2,"83425361":9,"834254":9,"83512277":3,"83614019":[],"8388888888888889":2,"83935285":[],"84087101":[],"842":1,"84251485":3,"842515":3,"84355903e":2,"84372853":1,"84443254e":2,"84658093e":15,"8479552268981934":0,"84994524":3,"8520127":[],"85278450e":1,"85396354":[],"85450859":9,"85463934e":[],"85497163e":[],"85546305e":[],"85601992":3,"861":1,"86117291":3,"86134827":3,"86145244":9,"8638888888888889":2,"86436607":[],"86574276":[],"8666666666666667":2,"86692943":[],"8718475896381779":15,"87184759":15,"8722222222222222":2,"87381451":3,"875":2,"87761937":[],"8777777777777778":2,"87972591":[],"8802":[],"88046261":3,"88046462":[],"8805555555555555":2,"88297395":[],"88336879":3,"88559559":[],"8867467323038865":[],"88693966e":15,"88712946":[],"8888888888888888":2,"8901":[],"8914984":[],"8921171964770647":9,"8923":1,"89288636":9,"89383322":[],"89410423":3,"8944444444444445":2,"89742056":3,"897421":3,"898500":9,"89850037":9,"89928965":3,"89942598":1,"89975818":[],"8x8":2,"90075537":3,"90220243":3,"90233874":[],"90266948":3,"90316476":[],"9040":7,"9055555555555556":2,"90618734e":15,"907307":9,"90730735":9,"90825063":1,"91012519e":[],"9111111111111111":2,"91128596":3,"91145266e":15,"91417278":[],"91549644":1,"9166666666666666":2,"916978":[],"91697817":[],"9171356":1,"91760278":3,"91812702":3,"9222222222222223":2,"922312":[],"92231203":[],"92280994":3,"924018":1,"925":2,"92507116e":2,"92578916":3,"92605247":1,"92645039":9,"92646965":9,"926470":9,"92732185e":[],"9277777777777778":2,"9279671770201344":15,"92919670e":15,"9305555555555556":2,"931":1,"93100040e":15,"93155188":3,"93158979":3,"932734":[],"93273404":[],"933":3,"9354":[],"9361111111111111":2,"937":15,"93799826":3,"938":15,"9388888888888889":2,"939":[1,15],"94019247e":1,"94034531":1,"94284104":3,"94320205":3,"9444444444444444":2,"94536341":15,"94591015":13,"94639099":9,"946893955211749":1,"947543":9,"9482527":3,"948729":[],"95014575":[],"95231424":3,"9527777777777777":2,"95284275":3,"95351665":3,"954":15,"95443703e":3,"9547578478889096":1,"95517094":3,"955171":3,"9555555555555556":2,"95628168":3,"956282":3,"956563":9,"95684892":3,"958228616652075":3,"9583333333333334":2,"959247":3,"960":15,"96024953":3,"96084663":3,"961":15,"96104648":[],"9611111111111111":2,"962":15,"963499":9,"96349948":9,"9640435":3,"965885569080809":[],"96688672":3,"9674916":3,"967809":9,"96793117e":[],"97005689":3,"97101567e":15,"9722222222222222":2,"97243128":3,"97300836":3,"975":2,"97507735":3,"9756404":1,"976":6,"97622676":[],"976227":[],"97705827":3,"97723801":[],"97758848":3,"9777777777777777":2,"9780387310732":17,"9780387848570":17,"9781492032632":17,"97926491":3,"9804422":1,"98046438":1,"9805555555555555":2,"98091621":3,"98139097":3,"98275501":3,"98314755":[],"983148":[],"98413059":3,"98452685":[],"98454786":3,"985":15,"98526763":1,"98528992":1,"98531221":1,"98566191":3,"986":15,"9860803":[],"98609175":15,"986091753050161":15,"9861111111111112":2,"98661465":[],"98680716":3,"98716878":3,"9877742":[],"987902":1,"98808176":3,"988663":[],"9888888888888889":2,"98892195e":15,"98893512":[],"989":15,"9890348":3,"98927731":[],"9893447":3,"9898ff":[7,8],"99009525":3,"99009739":[],"9901168":1,"99013921":1,"99016161":1,"99018401":1,"99043999":[],"99088801":3,"9909252":[],"991":15,"991072":9,"99107239":9,"99115119":3,"99126104":[],"99133007":[],"99160404":[],"99176998":3,"99190487":[],"99194716":[],"992":15,"99218987":[],"99219378":[],"99242605":[],"99242921":3,"99248001":[],"99265097":3,"99273355":[],"99276945":[],"993":15,"99305549":[],"99305802":[],"99311297":[],"99316252":3,"99346398":[],"99363129":[],"99363383":[],"99371056":3,"99389612":3,"99393624":[],"994":4,"99400444":[],"99418903":9,"99420743":[],"99420997":[],"99428016":[],"9943201":3,"9945452":[],"99462421":[],"99473581":[],"9947756":3,"99478645":[],"99478898":[],"99492986":3,"99498985":1,"995":4,"99501236":1,"99503487":1,"99505739":1,"9950799":1,"99527696":[],"99528218":3,"9953048353087299":[],"99536326":[],"9953658":[],"99539415":3,"99544872":[],"9954538761021741":[],"99566069":3,"99578809":3,"99579317":[],"99581841":[],"99594294":[],"996":[3,4],"99600927":[],"99608161":3,"99636015":[],"9963961":3,"99650061":3,"99652042":[],"99652296":[],"99655111":[],"9967458":3,"99696351":[],"997":4,"99700706":3,"99709215":3,"99709325":[],"99710078":[],"99723611":1,"99728435":[],"99729756":3,"99730848":[],"99751458":3,"99758326":3,"99763569":[],"99767893":[],"99775587":3,"99782689":[],"99793613":3,"99799099":3,"998":4,"99813653":3,"99817842":[],"99825997":[],"99828624":3,"9983295":3,"99836973":[],"99845267":3,"9984806":[],"998577":3,"99861053":3,"99871521":3,"99881845":3,"99883879":[],"99884384":3,"99891285":[],"99893323":3,"999":[7,15],"99901896":3,"99903755":3,"99911427":3,"99918546":3,"99919837":3,"99926459":3,"9993237":3,"99933188":3,"99938942":3,"9994385":3,"99944272":3,"99945628":[],"99949306":3,"99953381":3,"99953475":3,"99957911":3,"99961294":3,"99965056":3,"99967865":3,"99970988":3,"9997332":3,"99975913":3,"99977849":3,"99980002":3,"9998161":3,"99984732":3,"99987324":3,"99988687":1,"99989476":3,"9999095":1,"99991263":3,"99992746":3,"99993212":1,"99993978":3,"99995":3,"99995475":1,"99995818594196":[],"999974281880048":1,"99997737":1,"9999794306626945":[],"9999822527140678":[],"9999864543345858":[],"9999868619217517":[],"9999878260589065":1,"9999910208315801":[],"9b9cf4fa1a95":[],"\u00f8yvind":16,"abstract":2,"break":[0,1,9],"byte":13,"case":[1,2,3,4,9,10,12,13,14],"catch":1,"char":15,"class":[1,2,4,5,6,7,9,10,15],"const":15,"default":[1,2,5,13],"ekstr\u00f8m":16,"export":7,"f\u00f8470":16,"final":[0,1,2,3,4,6,7,8,9,14,15,16],"float":[0,1,3,7,9,13,15],"function":[0,3,4,7,12,13],"import":[0,1,2,4,5,6,7,8,9,10],"int":[0,1,2,3,5,9,13,15],"long":[1,2,5,10,15],"new":[0,1,2,3,4,5,6,7,8,9,13,15],"null":15,"public":[1,12],"return":[0,1,2,3,4,5,6,7,9,13,15],"s\u00f8rli":16,"sch\u00f8yen":16,"short":[0,3,11],"steinsv\u00e5g":16,"super":3,"switch":[0,1],"throw":15,"true":[0,1,2,3,4,5,6,7,8,10,15],"try":[0,1,2,3,5,6,7,8,9,12,13,15],"var":[3,4,8,9,15],"while":[0,1,2,3,4,5,6,7,9,10,15],AGE:1,Adding:2,Age:5,And:[0,1,4,7,12,15],Are:9,Being:5,But:[0,2,3,4,7,8,15],CAS:[],DIS:1,Doing:[3,5,8],EoS:[1,4],FYS:14,For:[0,1,2,3,4,5,6,7,8,9,10,12,13,14,15,17],Going:2,Ising:[3,10],Its:[2,9],MDS:9,NNs:10,N_s:6,Not:[1,2,3,4],OLS:[1,4],One:[1,2,3,4,5,6,9,10,15],PCs:[9,12],RMS:15,Such:[4,10,15],That:[0,1,3,5,8,9,10,15],The:[8,13,14,16,17],Then:[0,1,2,4,5,6,7,8,9,10,13,15],There:[0,1,3,6,7,9,10,11,13,14,15,16],These:[0,1,3,6,7,8,9,10,13,15],Use:[1,7],Useful:[3,4,13],Using:[1,3,4,6,8,10,13],With:[1,3,4,6,7,8,9,10,13,15],__class__:8,__doc__:4,__future__:[6,7],__getattr__:[],__init__:[2,4],__name__:[4,8],_auto10:10,_auto1:[3,5,10,13,15],_auto2:[10,13,15],_auto3:[10,13],_auto4:[10,13],_auto5:[10,13],_auto6:[10,13],_auto7:[10,13],_auto8:10,_auto9:10,_ax:4,_base:6,_build:[12,17],_check_optimize_result:9,_compon:9,_datafram:[],_depth:7,_fraction:7,_lambda:1,_leaf:7,_logist:9,_make_index:[],_multilayer_perceptron:[1,2],_node:7,_num_sampl:[],_ratio:9,_sampl:7,_split:7,_varianc:9,_weight:7,a0faa0:[7,8],a77d5ac269b2:5,a_0:1,a_1a:1,a_2a:1,a_3:1,a_3a:1,a_4:1,a_4a:1,a_h:2,a_i:[1,2,10],a_j:[2,10],a_k:[2,10],aaron:17,ab_channel:12,abandon:2,abbrevi:14,abid:15,abil:[1,8],abl:[2,3,5,8,10,15],abort:15,about:[0,1,2,3,4,5,6,7,8,9,10,12,13,17],abov:[0,1,2,3,4,5,6,7,8,9,10,13,15],abovement:4,abs:[0,1,3],abscissa:5,absolut:[1,3,4],acccess:[],accept:[1,7],access:[1,9,15],accid:4,accompani:1,accomplish:[6,7],accord:[0,1,2,3,4,5,7,10,15],accordingli:9,account:[1,3,15],accumul:[10,15],accur:[4,8,15],accuraci:[1,2,3,5,7,8,9,10],accuracy_scor:[1,2,8],accuracy_score_numpi:2,achiev:[1,2,3,4,6,10,13],aco:15,acquaint:12,acquir:[2,12],acr:1,across:[2,4,7,12],act:[2,13],action:15,activ:[0,1,7,14],actual:[1,2,3,4,6,9,13],ada_clf:8,adaboostclassifi:8,adam:2,adapt:[1,4,5,17],add:[1,2,3,4,6,8,9,10,15],add_subplot:[0,2,5,10],added:[1,2,3,5,6,13],adding:[0,2,4,13],addit:[0,1,3,4,5,6,7,8,10,12,13,15,16,17],addition:[5,10],address:[2,5,7,9,17],adjac:10,adjoint:3,adjust:[5,10],admir:1,advanc:[4,10,17],advantag:[2,3,4,5,8,13],afecionado:[],affect:[],affin:[1,6,9],aficionado:[],aforement:0,african:1,after:[0,1,2,3,4,5,7,9,10,12,13,15],afterward:1,again:[0,1,2,3,4,5,6,8,9,10,15],against:[2,5,8],age:[1,5],agegroup:5,agegroupmean:5,aggreg:[7,8],agorithm:8,agre:[3,15],ahead:7,aid:9,aim:[0,1,2,4,5,9,12,13],ainv:3,aka:3,alarm:3,albeit:0,algebra:[1,3,5,12,14],algo:15,algorithm:[1,2,3,4,5,6,12,13,14,15,17],align:[1,3,4,5,6,15],all:[0,1,2,3,4,5,7,8,9,10,12,13,14,15,16,17],allevi:[2,5],alloc:13,allow:[1,2,3,4,5,6,8,12,13],almost:[1,2,4,5,6,9,15],alon:7,along:[0,3,4,7,8,9,12,13],alpha:[0,1,2,4,5,6,7,8,15],alpha_:8,alpha_i:5,alpha_k:5,alpha_m:8,alpha_opt:5,alreadi:[3,8,10,12,13,15],also:[0,1,2,3,4,5,6,7,8,9,10,12,13,14,15,17],alter:[2,15],altern:[1,2,3,6,7,9,13],although:[2,3,4,6,8],alwai:[0,1,3,4,5,10,15],ame2016:1,american:1,among:[1,3,7,8,10],amongst:3,amount:[0,2,4,6,8,12],an_:15,anaconda3:[1,2,4,6,9],anaconda:[1,2,12],analys:[4,15],analysi:[2,4,5,13,14,17],analyt:[1,3,4,5,10,12],analyz:[1,2,3,15],andrew:2,angl:[1,7,15],ani:[0,1,2,3,4,6,7,8,10,15],anim:10,ann:10,annot:[1,2,5,6],anoth:[1,2,3,4,5,6,8,9,10,13,15],ans:15,ansatz:1,answer:[1,2,3,4,13],anymor:[2,6],anyon:6,anyth:[2,15],anytim:16,apach:2,apart:[5,9],api:[2,12],appear:[1,2,13,15],append:[2,4,5,6,7,15],appli:[1,2,4,5,6,7,8,9,10,15,17],applic:[1,2,3,4,5,7,10,15,17],approach:[2,3,4,7,8,9,10,12,15,17],appropri:[4,7,10,12,15],approx:[1,5,8,9,15],approxim:[1,2,3,4,5,8,9,15],apt:[1,12],aptli:0,aragorn:[],arang:[2,4,5,7,8,10],arbitrari:[2,5,6,10,15],arbitrarili:[1,2,9],architectur:[10,17],area:[1,17],arg:4,argc:15,argmax:[2,9],argmin:[0,8],argsort:9,argu:2,arguabl:0,argument:[1,3,9,10,15],argv:15,aris:[1,4,5,10,15],arithmet:[1,13],arma:15,armadillo:[13,15],around:[1,2,3,4,9,15],arrai:[0,1,2,3,4,5,6,7,9,10,12,15],arriv:[1,7,9,13,15],arrow:10,arrowprop:6,art:[1,2,12],articl:[0,1,3,4,8,15],artifici:[1,5,10,17],artificialneuron:10,artist:4,asarrai:[1,7],ascii:15,ask:[3,4,9,10],aspect:[1,12],assembl:1,assess:[1,4],assign:[0,1,5,6,7,10,14,17],assign_points_to_clust:0,associ:[0,1,4,7,10,15],assum:[0,1,2,3,4,5,6,7,8,9,10,13,15],assumpt:[1,3,4,7,9,15],ast:[1,3,4],astyp:[7,8],asymmetri:1,asymptot:4,atoi:15,atom:1,attempt:[1,5,6,8],attend:14,attent:[1,13],attr:[],attract:[1,8],attribut:[1,7],attributeerror:4,audi:1,aurelien:[1,14,17],author:[1,2,8,15],authour:1,auto:[7,8,15],autocor:15,autocorrelation_tim:15,autocorrelform:15,autocovari:15,autoencod:12,autoencond:12,autograd:12,autom:[1,12],automac:13,automag:[],automat:[1,2,9,12,13],autonom:17,avail:[1,2,4,8,9,12,13,14,17],averag:[0,1,2,4,7,8,15,16],avg:15,avoid:[0,1,3,4,7,9,13],awai:15,awar:8,award:16,axes3d:5,axes:[1,4,5,6,7,8,9],axessubplot:1,axhlin:6,axi:[0,1,2,4,5,6,7,8,9,10,15],axiom:3,axlabel:1,axvlin:6,b_1:[5,10],b_5:5,b_group:7,b_i:[1,2,10],b_ia_:1,b_index:7,b_j:[2,10],b_k:[2,5,10],b_m:10,b_score:7,b_valu:7,bachelor:14,back:[1,3,6,7,8,13,15],backbon:13,backend:2,background:17,backpropag:2,backtrack:7,backup:13,backward:[2,10,13],bad:15,badli:15,bag:[7,12,14],bag_clf:8,baggin:[],baggingboot:8,baggingclassifi:8,baggingtre:8,balanc:4,band:13,bandwidth:13,bar:[1,9],barber:17,bare:8,barebon:0,base:[0,1,2,3,5,6,7,8,12,15,16,17],basi:[3,5,6,8,9,10,13],basic:[6,10,12,14,15],batch:[5,9,10],batch_siz:2,bay:5,bayesian:[3,12,17],becaus:[0,1,2,3,4,5,6,7,10],becom:[0,1,2,3,4,5,7,10,15],been:[1,2,4,9,10,12,13],befor:[0,1,2,3,4,5,6,10,13,15],beforehand:[0,1,15],begin:[0,1,2,3,4,5,6,7,9,10,13,15],behav:[2,4,5],behavior:[1,2,5],behaviour:10,behind:[1,2,5,6],being:[0,1,2,3,5,6,8,9,10,15],believ:[7,13],belong:[0,3,5,6,7],below:[1,2,3,4,5,6,7,8,9,10,13,15],benchmark:8,bendik:16,benefici:2,benefit:[1,2,5,9,12],bengio:[2,14,17],benign:[2,5],besid:3,bessel:3,best:[0,1,2,3,4,5,6,7,8,10,15,16],beta:[1,2,3,4,5,8,9],beta_0:[1,2,3,5],beta_0x_:1,beta_1:[1,2,3,5,8],beta_1x_0:1,beta_1x_1:[1,5],beta_1x_2:1,beta_1x_:1,beta_1x_i:5,beta_2:1,beta_2x_0:1,beta_2x_1:1,beta_2x_2:[1,5],beta_2x_:1,beta_:[1,5],beta_i:[1,3],beta_j:[1,3,5],beta_k:5,beta_linreg:5,beta_m:8,beta_mg_m:8,beta_p:5,beta_px_p:5,better:[0,1,2,3,7,8,9,10],between:[0,1,2,3,4,5,6,7,9,10,15],beyond:[1,2,3,5,6],bia:[1,2,3,6,7,8,10,14,15],bias:[2,3,4,7,10],big:[0,1,2,3,4],bigger:2,bigr:10,bike:7,bilbo:[],bilek:16,billion:[10,12],bin:[1,4,5,15],binari:[1,3,5,7,8,10,14,15],bind:[1,4],binomi:12,binsboot:4,bioinformat:1,biolog:[2,10,17],bios1100:12,bird:1,birth:[],bishop:[14,17],bit:[0,2,13,15],bitwis:15,bla:13,black:[0,6,7],block:[0,4,8,12,13],blockingavg:15,blockingstd:15,blockingvar:15,blocksiz:15,blocksizemax:15,blocksizemin:15,blue:1,bmatrix:[1,2,3,5,6,9,13],bmi:2,bodi:[1,2,10],bold:2,boldfac:[1,3],boldsymbol:[0,1,2,3,4,5,6,8,9],boltzmann:[10,12],book:17,bool:0,boost:[2,7,12,14],boostrap:8,bootavg:15,bootstd:15,bootstrap:[2,12,14],bootvar:15,bootvec:15,borrow:[],boston_dataset:1,bot:6,both:[0,1,2,4,5,6,7,8,12,13,15,16],bottl:5,bottom:4,bound:[1,4,6,10],boundari:[6,9,10],box:7,boyd:[5,6],bracket:15,brain:[2,5,10],branch:7,breast:[3,5,9],brew:[1,12],brg:6,briefli:1,bring:[1,8],broad:1,broadcast:0,browser:[],brute:[3,9],bsol:[],bsubex:[],build:[1,3,4,8,13,15],built:[1,2,4],bunch:9,busi:1,c46dd114b2af:6,c_0:15,c_1:10,c_2:10,c_3:10,c_4:10,c_i:[5,10],c_k:15,cach:8,cal:[1,5,6,8,10],calcul:[0,1,2,3,4,5,6,7,8,9,10,13],call:[0,1,2,3,4,5,6,7,8,9,10,12,13,15,17],callabl:4,cambridg:[5,17],came:0,can:[0,1,2,3,4,5,6,7,8,9,10,12,13,15,17],cancel:1,cancer:[3,8],cancerpd:5,candid:[6,7,8],cannot:[1,2,3,5,6,7,14,15],canopi:[1,12],cap:3,capabl:[1,2,6,12],capita:1,captur:[9,10],card:[1,5],cardin:2,care:[0,9],carefulli:5,carlo:[1,4,12,15,17],carri:[4,5],cart:8,casella:17,cast:2,categor:[1,2,7,9],categori:[0,1,2,5,8,10,14],categorical_crossentropi:2,caus:[1,3,4,15],causal:1,causat:1,cax:2,cbar:2,ccc:[3,10],cdf:15,cdot:[0,1,4,5,10,13,15],celebr:5,center:[0,1,2,4,5,6,7,9,15],centr:17,central:[1,3,4,6,13],centroid:[0,15],centroid_differ:0,centroid_list:0,certain:[0,1,4,5,7,15],cha:1,chain:[2,12,15],challeng:[0,5],chanc:[2,3,5,15],chang:[0,1,2,3,4,5,6,7,9,10,13,15],chapter:[0,4,8,9,13,14,17],charact:[1,3,6,15],character:[6,7,8,10,15],characterist:[1,2,8],charg:1,charl:1,chd:5,chddata:5,cheap:3,cheaper:[2,5],chebychev:0,check:[1,2,5,9,13],check_consistent_length:[],chemic:15,chen:8,choic:[0,1,2,4,5,7,10,13,15],choleski:[3,13],choos:[0,4,5,7,8,9,15],chosen:[1,2,4,5,6,7,8,15],chosen_datapoint:2,christian:17,christoph:[14,17],cin:15,circ:[2,10],circl:[1,6,10],circumfer:7,circumv:[2,3],clariti:[0,15],class_nam:7,class_val:7,class_valu:7,classic:[5,7],classif:[1,3,4,5,6,9,10,12,14,17],classifi:[1,2,5,7,8,9],classificaton:2,classifii:8,clean:2,clear:[2,8,10],clearer:0,clearli:[3,4,5,6,15],clever:[0,2,8],clf3:1,clf:[1,6,7,8],clf_ridg:1,clip:15,close:[0,1,2,4,5,6,7,9,10,15,17],closer:3,closest:[0,5,6,9],closur:12,cloud:12,clust:0,cluster:[1,2,4,9,12,14],cluster_label:0,cmap:[1,2,6,7,8],cmath:15,cmb:14,cmd:7,cn_:15,cnn:10,cntk:12,code:[4,6,12,13,14,17],coef0:6,coef:1,coef_:[1,3,5,6,7],coeffici:[1,3,4,5,6,7,13],coerc:[1,4],coin:[8,15],coin_toss:8,col:[1,9],colab:12,cold:7,colinear:1,collaps:6,collect:[1,4,8,9,12,15,17],collinear:3,color:[1,4,6,7,8,15],colorbar:2,colsample_bytre:8,colsaobject:8,column:[1,2,3,4,5,6,7,9,10,13],columntransform:7,com:[12,16,17],combin:[2,3,4,5,8,15],come:[0,1,2,3,5,10],comma:[],command:[1,2,15],comment:1,commerci:[1,12],commod:1,common:[0,1,2,3,4,5,7,9,15],commonli:[0,2,4,5,7],commun:[1,10],compact:[0,1,2,3,4,5,7,9,10],compar:[0,1,3,4,5,9,13],compat:5,compet:1,competit:8,compil:[1,2,12,13,15],complet:[1,7,10],completenn:10,complex:[1,2,3,4,6,7,9,10],complic:[0,1,2,4,5,7],compon:[0,1,2,3,4,5,7,12,14],components_:9,compos:[0,7,10],compphys:[12,14,17],compress:1,compris:4,compromis:3,compulsori:12,comput:[0,1,2,3,4,6,8,9,10,12,13,14,17],computation:[1,4,5,7,15],concaten:0,concav:[2,5],concentr:[1,8],concept:[0,1,12],conceptu:[5,10],concern:[0,1,2,5],concic:[],conclud:[1,3],conclus:2,conda:[1,2,12],condit:[1,3,4,6,7,9,15],conduct:12,confid:[1,3,4,5,6],confirm:[3,10],confus:[3,4,8,13],confusion_matrix:7,congruenti:15,conjug:6,conjugaci:5,connect:[1,2,5,7,9,10,13],consequ:[3,4,5,6,8,10],conserv:[0,3],consid:[0,1,2,3,4,5,6,7,8,10,13,15],consider:[1,2,3,5],consist:[1,2,4,5,10,15],constant:[1,3,5,6,10,15],constitu:1,constitut:4,constrain:[2,3,5,9],constraint:[3,5,6],construct:[1,2,3,4,5,6,7,8,9,13,15,17],contact:1,contain:[0,1,3,4,5,6,7,9,10,13,15,17],contemporari:17,content:[2,12,13],context:[4,5,8],continu:[1,2,3,4,5,6,7,8,10,13],contour:[5,7,8],contourf:[6,7,8],contrast:[2,7,8,10],contribut:[1,3,15],contributor:1,control:[1,2,7,12],conveni:[1,3,4,5,10,13],convent:10,converg:[0,1,2,3,5,6,9],convergencewarn:[1,2,6,9],convert:[1,2,3,7,9,13],convex:3,convinc:5,convolut:[2,12,14],cool:7,coordin:[0,3,10],coorel:1,copi:[0,2],core:8,corel:1,coronari:5,corr:[1,3,5,9],correalt:[9,12],correct:[0,1,2,3,13,15],correctli:[2,4,8],correl:[1,2,3,5,8,10,12],correlation_matrix:[1,3,5,9],correspond:[0,1,3,4,6,7,9,10,12,13,15],cortex:10,cos:[1,4,7],cosin:[0,4],cost:[1,3,4,6,7,10,15],could:[1,2,3,4,5,6,7,8,9,10,13,15],coulomb:1,count:[1,7,14,15,16],countor:5,coupl:3,cours:[1,2,3,9,14,15],courvil:[14,17],cout:15,cov:[3,4,9,13,15],cov_xi:[3,9],cov_xx:[3,9],cov_yi:[3,9],covari:[5,12,13],covariance_matrix:[0,3,9],cover:[1,3,11,12,17],covert:1,covxi:15,covxx:15,covxz:15,covyi:15,covyz:15,covzz:15,cpu:2,creat:[2,3,7,8,9,10,12],create_biases_and_weight:2,create_neural_network_kera:2,create_x:[1,3,9],credit:[1,5],crim:1,crime:1,criteria:[0,1,7,8,15],criterion:[5,7,8],cross:[1,2,5,7,8,12,14,15],cross_val_scor:4,cross_valid:[5,8],crossvalid:4,crucial:[2,15],csr_matrix:13,cstdlib:15,csv:[1,4,5,7],ctnk:2,cubic:1,cumbersom:3,cumsum:[8,9],cumul:[4,8],cumulative_heads_ratio:8,cup:3,current:[0,2,5],curs:1,curv:[5,8,10],curvatur:5,custom:0,custom_cmap2:[7,8],custom_cmap:[7,8],cutpoint:7,cvxbook:5,cvxopt:[3,6],cyber:17,cycl:[0,2,10,15],d_f:5,dagger:[3,13],dai:[2,7,12],dalen:16,darget:7,darkr:15,dat:1,dat_id:[1,4,5,7],data1:0,data2:0,data3:0,data4:0,data:[0,3,4,6,8,10,13,17],data_id:[1,4,5,7],data_indic:2,data_panda:[],data_path:[1,4,5,7],databas:2,datafil:[1,4,5,7],datafram:[1,3,5,7,9],datapoint:[1,2,3,4,5,9],dataset:[0,1,4,5,6,7,8,9],date:1,daughter:8,david:17,dbh:2,dbo:2,dcomposit:13,dead:2,deal:[0,1,2,3,5,6,9,13,15],debt:5,debug:[3,4],decad:1,decai:[1,5,15],decent:8,decid:[3,4,7],decim:1,decis:[1,2,6,9,12,14,17],decision_funct:6,decision_tre:7,decisiontreeclassifi:[7,8],decisiontreeregressor:[1,7,8],declar:[1,13],decompos:[3,13],decomposit:[1,10],decompost:3,decorrel:8,decreas:[2,3,4,5,8,9],deduc:1,deep:[5,10,12,14,17],deep_tree_clf1:7,deep_tree_clf2:7,deep_tree_clf:[7,8],deepen:12,deeper:0,deeplearningbook:17,def:[0,1,2,3,4,5,6,7,8,9,15],def_covari:15,defect:3,defici:3,defin:[0,1,3,4,5,6,7,8,9,10,13,15],definit:[0,2,3,4,5,6,8,9,10,13],defint:15,degre:[3,4,6,7,8,9,15],del:2,delet:[4,15],deliv:14,delta:[0,1,6,10,15],delta_:[2,13],delta_h:[1,2],delta_j:10,delta_k:10,delta_l:2,delta_n:1,delug:12,delv:1,demand:5,demonstr:[1,3,4,5,9,10,12],denomin:[2,3],denot:[2,4,5,15],dens:[0,2],densiti:[0,1,4,15],depart:16,depend:[0,1,2,3,4,5,6,9,10,12,15],depict:15,deploy:[1,12],deprec:1,depth:[7,8,13],deriv:[1,2,4,6,8,9,12],descend:[3,7,9],descent:[1,2,6,10],descr:1,describ:[0,1,3,4,6,8,9,10,13],descript:[0,1,6,7],design:[1,2,3,4,5,8,9,10],designmatrix:1,desir:[0,1,3,5],despit:[2,10],destroi:13,det:[3,13],detail:[0,1,5,9,13],detect:[6,10],determin:[1,3,5,6,7,8,9,10,13,15],determinist:[5,15],dev:[2,15],develop:[0,1,3,6,8,9,10,12,13],deviat:[1,2,3,4],devis:10,df1:[],diag:[3,6],diagnost:[2,8],diagon:[1,3,5,13,15],diagonaliz:3,diagram:8,dict:6,dict_kei:1,dictionari:1,did:[0,1,2,3,5,8,9],die:2,diffeent:6,differ:[0,1,2,3,4,7,8,9,10,12,13,15,17],differenti:[5,12,13],difficult:[0,1,2,4,8,15],difficulti:[1,2,5],digit:[1,2,14,16],dilut:2,dim:[0,9],dimens:[0,1,2,3,6,9,13],dimension:[0,1,3,4,5,7,9,12,13],dimensionless:1,diment:13,dimes:0,direct:[0,1,2,5,9,10],directli:[0,2,3,15],directori:5,disadvantag:1,discard:[4,9],disciplin:[1,10],disclaim:15,discourag:5,discov:1,discover:3,discret:[2,3,5],discrimin:[5,8,9],discuss:[0,1,2,3,4,5,6,7,8,9,10,12,13,14,15,17],diseas:5,disk:[],disord:[2,5],displai:[0,1,2,4,5,6,7,8,9,10,15],displaystyl:[1,3],displot:1,disregard:1,dissimilar:[0,9],dist:0,distanc:[0,1,6,7,9,15],distance_list:7,distinct:[0,5,6,7,8],distinctli:6,distinguish:[1,5,6,15],distplot:1,distribut:[0,1,2,4,5,8,9,12,13],distrubut:[1,12],dive:[1,6],diverg:[2,5],divid:[1,2,3,4,6,7,9,10,15],divis:[4,6,7,13,15],dna:5,dnn:[1,2,10],dnn_kera:2,dnn_model:2,dnn_numpi:2,dnn_scikit:[1,2],doamin:3,doc:[12,14,17],doconc:[],document:9,doe:[0,1,2,3,4,5,6,8,9,10,13,15],doesn:[7,10],dog:2,doi:0,doing:[1,3,4,9],domain:[3,5,6],domin:1,don:[0,1,2,6,9,12,15],done:[0,1,3,4,7,8,9,13,15],dot:[1,3,5,6,7,8,9,10,13,15],doubl:[13,15],doubli:2,down:[1,5,7,9,10],download:[1,2,13,17],dozen:2,dramat:9,draw:[4,5,8],drawback:[1,2,5],drawn:[2,4,5,9,15],drop:[1,2,3,4,9,15],dropna:[1,4],dtype:[0,1,2,13],dub:1,due:[0,2,3,4,5,6,8,10],dummi:1,dure:[1,2,6,7,9,12],dwell:1,dwh:2,dwo:2,dx_1:15,dx_n:15,dying:2,each:[0,1,2,3,4,5,6,7,8,9,10,12,13,14,15,16],eapprox:1,earli:2,earlier:[1,3,5,6,7,9,10],eas:[0,7],easi:[1,3,4,5,6,7,8,9,10,12,13],easier:[3,4,6,7,15],easiest:5,easili:[1,2,3,5,6,7,8,9,10,13],eastern:16,ebind:1,eblock:7,econometr:[],economi:3,ecosystem:12,ect:14,edgecolor:4,edit:[],edu:5,educ:1,eface79dac2c:8,eff:15,effect:[2,8,15],effic:2,effici:[1,5,8,12,13,15],efron:[4,15],eig:[3,5,9,13,15],eigen:15,eigenpair:[3,9],eigenvalu:[3,5,6,9,13],eigenvector:[3,5,9],eight:13,eigval:[13,15],eigvalu:[5,9],eigvec:[13,15],eigvector:[5,9],eispack:13,either:[1,2,3,4,5,6,7,8,9,15],ekstrom:16,elabor:15,electr:[1,10],electur:17,eleg:9,element:[2,3,4,5,6,9,10,12,13,14,17],elementari:[8,13],elessar:[],elif:0,elim:13,elimin:6,els:[2,5,7,10,15],elu:2,elus:1,email:[14,16],embed:[1,9],embodi:4,emit:15,emner:17,emphas:[1,8,12],emphasi:[1,12,17],empir:[2,9,15],emploi:[1,2,3,4,5,9,15],employ:1,empti:[4,8],emul:10,enabl:9,encapsul:0,encod:[0,1,3,7,9],encompass:[1,15],encount:[1,2,3,5,15],end:[0,1,2,3,4,5,6,7,8,9,10,13,15],endl:15,endpoint:15,energi:[1,4],enforc:10,eng:17,engin:[1,2,12,15],english:17,enough:[1,4,5],ensembl:[2,7,14,15],ensur:[1,2,3,4,5,9,15],enter:[3,15],enthought:[1,12],entir:[2,5,7,12,15],entiti:[7,10,13],entri:[1,3,6,9,10,13],entropi:[2,5,8],enumer:[1,2,6],env:15,environ:[12,17],eol:1,eosfit:1,epoch:[1,2,5,10],epsilon:[1,3,4,5],epsilon_0:1,epsilon_1:1,epsilon_2:1,epsilon_:1,epsilon_i:1,eqnarrai:[3,4],equal:[0,1,2,3,4,5,6,7,9,10,13,15],equat:[0,2,4,6,7,8,9,13,15],equilibrium:10,equiv:[5,13,15],equival:[2,3,6,9,12,13],erf:15,eriador:[],eridg:1,err:[1,8],err_:4,errat:5,errno:5,error:[1,2,3,4,5,7,9,10,12,15],error_estimate_corr_tim:15,error_hidden:2,error_output:2,escap:5,esl:0,esol:[],especi:[2,7,10],essenti:[0,1,3,7,8,10,15],establish:[1,8,9],estim:[1,2,3,4,5,8,9,12,15],estimated_mse_fold:4,estimated_mse_kfold:4,estimated_mse_sklearn:4,esubex:[],eta0:[5,6],eta:[1,2,5,6,10],eta_v:[1,2],etc:[0,1,2,3,5,6,7,9,10,12,13,15],ethic:12,etsim:4,euclidean:[0,1],evalu:[1,3,4,5,7,15],even:[0,1,2,3,4,5,6,7,8,9,10,12,13,15],event:[3,5,8],eventu:[3,4,5,9,10,16],everi:[0,1,2,3,4,5,7,8,9,10,12,15],everyth:10,everywher:5,evolv:1,exact:[1,3,5,9,10,13,15],exactli:[1,10,12],examin:4,exampl:[3,4,9,10,12,13,14,17],exce:[2,10],excel:[0,1,2,8,17],except:[4,6,7,15],excess:1,excit:1,exclud:[2,4,10],exclus:[1,2,4,15],execut:3,exemplifi:5,exercis:[3,12,14],exhaust:4,exhibit:[1,6],exist:[0,1,2,3,4,5,6,7,13,17],exit:[3,13,15],exp:[1,2,3,4,5,6,8,9,10,15],exp_term:2,expand:[3,5,9],expans:[1,3,5,6,8,10],expect:[1,2,3,4,5,9,10,12],expectation_value_of_h_wrt_p:15,expens:[4,5,8,15],experi:[1,2,4,5,6,12],experiment:[1,4,7,15],expert:[2,7],explain:[0,1,5,7,8,9,15],explained_variance_ratio_:9,explanatori:1,explicit:[1,13],explicitli:[0,1],explod:2,exploit:[1,10],explor:[2,5,6,12],expon:2,exponenti:[1,2,3,5,8],export_graphviz:7,export_text:7,exporttext:7,expos:12,express:[1,3,4,8,10,13,15],exptmean:15,exptvari:15,extend:[9,12],extens:[1,10,12],extent:[1,2,4,17],extern:7,extra:[2,3],extract:[1,3,5,6,9,13],extrapol:1,extrem:[0,1,2,3,5,6,7,13],extremum:5,extrins:9,eye:[0,1,3,5,13],f11:1,f12:1,f13:1,f1d:5,f6d7a289d493:13,f_0:8,f_1:[5,8],f_2:[5,10],f_3:10,f_d:15,f_i:[4,10],f_m:8,face:5,facecolor:[4,6,15],facil:[1,12],facilit:10,fact:[1,2,3,5,7,9,10],factor:[1,2,3,7,8,9,13,15],fafab0:[7,8],fail:[4,5,6,9,16],failur:5,fairli:[0,2,15],fall:[6,7,14],fals:[0,1,2,3,4,5,7,8],famili:[1,5,6,15],familiar:[1,3,6,12,13,15],famou:[4,10,13],far:[0,1,3,5,6,9,10,15],fashion:[1,7,8],fast:[2,4,5,8,10,12,15],faster:[2,9],fastest:5,favor:5,favorit:15,featur:[1,2,3,4,5,6,8,9,10,12,15],feature_nam:[1,2,5,7],feautur:7,fed:2,feed:[1,9,12,14],feed_forward:2,feed_forward_out:2,feed_forward_train:2,feedforward:[2,10],feel:[0,1,3,9,12,16],feet:1,few:[0,2,7,11,15],fewer:[1,7,9],ffnn:[2,10],field:[1,10,12],fifth:1,fig:[0,1,2,5,10],fig_id:[1,4,5,7],figaxi:15,figsiz:[1,2,4,5,6,7,8],figur:[0,1,2,3,4,5,6,7,8,10,12],figure_id:[1,4,5,7],figurefil:[1,4,5,7],file:[1,2,4,5,6,7,13,15],filenam:[1,15],filenotfounderror:5,fileout:15,fill:[3,7],financ:1,find:[0,1,2,3,4,5,6,7,8,9,10,12,15],fine:[0,1],finit:[3,4,10,15],first:[0,1,2,3,4,6,7,8,9,13,15,17],firsteigvector:9,fit:[2,3,4,5,6,7,9,10,15],fit_intercept:[3,4],fit_mod:7,fit_transform:[1,4,6,7,9],fiti:1,five:[1,7],fix:[1,4,5,8,9,10],flag:0,flat:[5,10],flatten:2,flexibl:[1,2,4,6,8,10],float32:7,float64:[1,13],flop:[3,13],flow:[2,10],fluctuat:3,fly:9,flyvbjerg:15,fmesh:5,focu:[1,3,4,12,17],focus:[2,5,13],fold:[4,7],folder:1,follow:[0,1,2,3,4,5,6,7,8,9,10,12,13,14,15,16,17],font:[1,5,15],fontdict:15,fontsiz:[2,6,7,8,15],fontweight:2,foral:6,forc:[1,3,8,9],forecast:10,forest:[1,2,7,12,14],forget:9,form:[1,3,4,5,6,7,9,10,12,13,15],formal:[0,15],format:[1,2,4,5,6,7,8,9,12,15,17],formatstrformatt:5,formul:[0,9],formula:[5,15],forth:10,fortran2003:12,fortran90:15,fortran:[1,12,13],fortun:[1,9],forward:[1,4,12,13,14],found:[2,3,4,5,10],foundat:12,four:[3,6,10,13,14],fourier:1,fourth:10,frac:[0,1,2,3,4,5,6,7,8,9,10,13,15],fraction:7,frame:5,framework:[2,6,8,15],frank:[3,9],frankefunct:[1,3,9],free:[1,9,12,13,15,16,17],freecodecamp:12,freedom:3,freeli:1,frequenc:[4,5,15],frequent:[1,5,6,7],frequentist:12,fresh:8,fret:0,fridai:14,friedman:[14,17],frodo:[],from:[0,1,2,4,5,6,7,9,12,13,14,15,16,17],from_cod:7,front:[1,3],fruit:0,fstream:15,fulfil:[3,10],full:[1,2,3,5,7,8,15],full_matric:3,fulli:[4,10,14,15],fun:12,func:4,functionali:9,fundament:[1,4,12],further:7,furthermor:[1,3,4,5,9,10,12],futur:[1,6,7],futurewarn:1,fys:16,g_1:8,g_2:8,g_m:8,gain:[0,2,3,7,8],galleri:1,gamge:[],gamma1:6,gamma2:6,gamma:[1,5,6,7,8,9],gamma_0:8,gamma_1:8,gamma_1x:8,gamma_:1,gamma_i:[1,6,15],gamma_j:5,gamma_k:5,gamma_m:8,gamma_x:1,gap:6,gate:10,gather:[2,10],gaug:10,gaussbacksub:13,gaussian:[0,3,4,6],gaussian_point:0,gaussian_rbf:6,gave:5,gbc:14,gca:[4,5,6],gd_clf:8,gdclassiffiercgain:8,gdclassiffierconfus:8,gdclassiffierroc:8,gdregress:8,gen:15,gender:1,gener:[0,1,2,3,4,5,6,8,9,10,13,17],generallay:10,generate_simple_clustering_dataset:0,genom:12,geodes:9,geometr:1,geometri:3,georg:17,geq:[3,5,6,7],geron:[1,14,17],get:[0,1,2,3,4,5,7,8,9,12,13,15],get_distances_to_clust:0,get_dummi:7,get_split:7,get_yaxi:6,getattr:4,gibb:12,gini:8,gini_index:7,git:[1,12],github:[1,12,14,17],gitlab:[1,12],give:[0,1,2,3,5,6,7,8,10,12,15,17],given:[0,1,2,3,4,5,6,7,8,9,10,13,15],glare:0,global:[5,15],glorot:2,gmail:16,goal:[1,5,7],goe:[0,1,2,3,4,5,13],going:[1,2,3,4,5,6,7,9,10],golden:5,gone:3,gong:2,good:[0,2,3,5,7,8,9,12,15,17],goodfellow:[14,17],googl:[0,2,12],got:2,gpu:2,grade:14,gradient:[1,6,7,10,12,14],gradientboostingclassifi:8,gradientboostingregressor:8,gradual:[0,2],graph:[2,5,7,9,10],graph_from_dot_data:7,graphic:[1,2,7],grasp:1,gray_r:2,great:5,greater:[2,5,15],greedi:7,green:[1,7,15],grid:[2,4,5,6,10,15],grossli:5,ground:1,group:[0,1,4,5,7,12,14],groupbi:1,grow:[2,7,8],growth:1,guarante:[1,5,15],guess:[0,2,5,8],guestrin:8,guid:2,h_1:5,h_2:5,h_m:8,had:[1,2,4,5],hadamard:[2,10],half:[2,6,7],halv:8,hand:[1,2,3,5,9,10,12,13,14,15,17],handl:[1,2,3,7,9,12],handle_unknown:7,handsid:10,handwrit:10,handwritten:[2,3,5],happen:[0,2,3,5,8,15],hard:[2,5,6,8],hardcopi:12,harder:[1,2],has:[0,1,2,3,4,5,6,7,8,9,10,13,15],hasn:[1,2],hassl:[1,12],hast:12,hasti:[0,1,14,17],hat:[2,3,5,7,8,9,10,13,15],have:[0,1,2,3,4,5,6,7,8,9,10,12,13,15],haven:2,hdf5:[],head:[1,8,15],header:1,heads_proba:8,hear:1,heart:[1,5],heatmap:[1,2,5],heavili:1,heavisid:2,height:2,help:[0,1,2,10],helper:0,henc:[1,3,4,5,6,7,8,10],her:5,here:[0,1,2,3,4,5,6,7,8,9,10,12,13,15,17],hereaft:[1,6,10],hermitian:13,hessenberg:13,hessian:3,heterogen:[7,8],hidden:[2,10],hidden_bia:2,hidden_bias_gradi:2,hidden_layer_s:[1,2],hidden_weight:2,hidden_weights_gradi:2,hierarch:[0,3],high:[0,1,2,3,4,5,7,8,9,12,13],higher:[1,2,3,4,5,6],highest:2,highli:[1,3,8,12,13,15,17],highwai:1,hing:6,hint:5,hip:12,hire:1,his:5,hist:[4,5,15],histogram:[1,4,5,15],histor:[5,9],histori:10,histplot:1,histtyp:4,hitherto:3,hjorth:16,hobbi:15,hoc:3,hoff:17,hold:[0,2,4,5,15],holder:1,home:1,homogen:[2,7,8],hopefulli:[1,9,15],horizont:9,hors:5,hot:[2,7],hour:[2,12,14,15,16],how:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,17],howev:[0,1,2,3,4,5,6,7,8,9,10,12,13,15],hspace:[1,6,8,15],hstack:2,htf:14,html:[9,12,14,17],http:[5,9,12,13,14,17],huang:1,huber:1,huge:[2,12],human:[1,2,7,10],humid:7,hundr:2,hungri:2,hybrid:14,hydrogen:1,hyperbol:[2,10],hyperparam:6,hyperparamet:[0,1,3,7],hyperplan:9,i_1:[3,4],i_2:[3,4],ian:17,idea:[1,2,4,5,7,8,10,13,15],ideal:[0,1,4,6,15],idem:4,ident:[3,4,10,13],identical:3,identifi:[0,1,2,5,6,7,9,10],idum:15,ieor:15,ifi:17,ifs:12,ignor:[1,2,7],iii:13,ijca2016907841:0,illustr:[0,3,5,8,10,12],imag:[2,7,9,10,17],image_path:[1,4,5,7],imagin:2,immedi:[1,12],implement:[0,1,3,5,6,7,8,9,10],impli:[3,4,5,13],implicitli:[9,15],impos:[1,9,10],imposs:[1,3],impress:[1,10],improv:[0,3,7,8,9],impur:7,imshow:2,in3050:17,in4080:17,in4300:17,in5400:17,inaccur:5,inact:10,inadequ:1,includ:[1,2,3,5,9,10,12,15,16,17],include_bia:[4,7],incom:10,inconsist:[],incorrect:2,incoveni:6,increas:[1,2,3,4,6,7,9,10,15],increasingli:15,ind:4,inde:[1,3],indent:15,indentationerror:15,independ:[1,3,4,5,6,10,15],index:[0,1,2,8,12,15,17],index_col:1,indic:[1,2,3,7,8,9],indispens:4,individu:[2,5,8,10,15],indu:1,indx:13,ineffici:0,inequ:6,inequaltii:5,inf1000:12,inf1100:12,inf1100l:12,inf1110:12,inf3000:17,inf4490:17,inf5860:17,inf:1,infeas:7,infer:[1,2,4,17],inferenc:2,infil:[1,4,5,7],infin:[3,4,5,9],infinitesim:15,influenc:[4,8],influenti:2,info:[],inform:[0,1,2,4,5,7,9,10,13,17],infti:[5,15],ingeni:5,ingredi:[1,7],inher:4,inherit:13,initi:[0,1,2,4,5,8,13,15],initialis:15,inject:0,inlin:[0,1,2,3,4,5,6,7,8,9,10,13,15],inner:[4,5],innov:17,input:[0,1,2,3,4,5,6,7,8,10,13,15],input_dim:2,inputs:2,inputs_shuffl:2,insert:[3,6,8,15],insid:[1,5],insight:[1,2,3,12,17],insist:5,inspir:[1,2,10,17],instal:[1,2,7,14],instanc:[1,2,4,5,7,9],instanti:8,instead:[0,1,2,3,4,5,6,7,9,13,15],institut:2,instruct:[1,2],int32:8,int64:1,int_0:15,int_:15,int_a:15,integ:[0,2,13,15],integer_vector:2,integr:15,intellig:[0,1,17],intend:8,intens:2,intention:0,interact:[1,7,10,12],intercept:[1,5,6,9],intercept_:[1,5,6,7],interchang:[3,10,13],interconnect:2,interest:[1,2,3,4,5,6,7,10,12,15],interfac:[2,13],interior:[1,7],intermedi:13,intern:[2,8,10],interpol:[2,10],interpr:3,interpret:[1,2,7,8,10,13,14,15],interv:[1,3,4,5,15],intial:5,intimid:[],intract:1,intrins:[9,13,15],intro:[12,17],introduc:[1,2,3,5,6,8,10,13,15],introduct:[2,5,14,17],introductori:[1,13,17],intuit:[1,3,4,6,10],inv:[1,3,5],invalid:[2,6,13],invalu:[1,5,12],invari:2,invd:3,inver:6,invers:[1,5],invers_period:15,inverse_transform:6,invert:[1,3,5,8],invok:[1,6],involv:[1,4,5,9,10],iomanip:15,ios:15,iostream:15,ipca:9,ipynb:12,ipython:[0,1,2,4,5,6,7,8,9,12,13,15],irani:0,iri:[6,7],irreduc:4,irrelev:3,irrespect:1,isn:3,isnul:1,isomap:9,issu:[2,7,13],it_arrai:5,item:1,items:13,iter:[0,1,2,4,6,9,15],its:[0,1,2,3,4,5,6,7,8,9,10,12,13,17],itself:[3,4,10,15],jackknavg:15,jackknif:[4,12],jackknstd:15,jackknvar:15,jackknvec:15,jacobian:5,jargon:15,jensen:16,jerom:17,job:[6,8],join:[1,4,5,7],joint:3,journal:15,judg:5,julia:[12,13],jump:[0,15],jupyt:[0,1,12,17],just:[0,1,2,3,4,5,6,7,8,9,10,15],justif:1,justifi:8,k_mean:0,kappa_d:15,karim:16,karlsen:16,karush:6,keep:[0,1,2,3,5,9,13],keepdim:[2,4,8],kei:[1,2,10,17],kept:0,kera:[1,12,14],kernel:[1,2,12],kernel_regular:2,kernelpca:9,kev:1,kevin:17,keyword:13,kfold:4,kick:[2,5],kind:[0,1,6,10],kjm:12,kkt:6,kmeanspoint:0,kn_k:0,know:[0,1,2,3,5,6,12,15],knowledg:[1,12],known:[2,3,4,5,6,7,10,13,15,17],kondev:1,kpca:9,kroneck:0,kuhn:6,kwarg:4,kwown:1,l1_l2:2,l_1:5,l_2:5,l_j:10,la_i:10,la_k:10,lab:[12,14],label:[0,1,2,3,4,5,6,7,8,10,12,13,15],labelencod:[5,8],labels:[6,7],labels_shuffl:2,labor:0,laboratori:14,lack:[0,1],lagrang:[6,9],lambda:[1,2,3,4,5,6,8,10,15],lambda_0:9,lambda_1:[3,6,9],lambda_2:[6,9],lambda_:9,lambda_i:[6,9],lambda_iy_i:6,lambda_jy_iy_j:6,lambda_k:6,lambda_n:[3,6],lamda:2,land:[1,6],landmark:6,landscap:5,langl:[1,9,15],languag:[1,2,6,12,13,17],lapack:13,laplac:3,laptop:12,larg:[1,2,3,4,5,6,7,8,9,12,13,15,17],larger:[1,3,4,5,6,8,9,15],largest:[6,9],lasso:[1,4,5,12,14],last:[0,1,2,3,4,5,6,7,8,10,13,15],later:[0,1,2,5,6,10,12,15],latex:[],latter:[1,5,6,9,13,15],lattic:10,law:1,layer:1,lbfg:[5,7,8,9],lbl:4,lcc:[3,4],lda:9,ldot:[1,4,9,15],lead:[1,2,3,4,5,6,7,8,9,10,13,15],leaf:7,leaki:2,lear:5,learn:[3,4,6,7,8,10,13,14,17],learner:8,learning_r:[6,8],learning_rate_init:[1,2],learning_schedul:5,least:[0,1,4,5,6,8,9,12,13,15],leav:[1,2,3,4,7,9],lectur:[1,2,3,5,8,9,10,12,13,14,17],lecturenot:[12,17],left:[1,2,3,4,5,6,7,8,9,10,13,15],leftarrow:[6,10],legend:[1,3,4,5,6,7,8],len:[0,1,2,3,4,6,7,8,9,10,13,15],length:[1,2,5,6,7,12,15],leq:[0,1,3,5,6,15],less:[1,2,3,4,6,7,15],lessen:2,let:[0,1,2,3,4,5,6,7,8,9,10,13,15],letter:[1,13,15],level:[1,2,4,7,12,13,14],lib:[1,2,4,6,9],liblinear:[6,8],librari:[1,2,3,7,8,9,13,15,17],licens:[1,2,12],lie:[4,9,15],lies:[6,9],life:[1,2,6,10],lift:0,light:[],like:[0,1,2,3,4,5,7,8,9,10,12,13,15],likelihood:[1,2,3,7],lim_:15,limit:[1,4,6,9,10,13],lin_clf:6,lin_model:1,lin_reg:7,linalg:[1,3,5,6,9,13,15],line1:6,line2:6,line3:6,line:[1,2,4,5,6,9,13,15],linear:[2,3,4,5,7,8,9,10,12,14,15],linear_model:[1,3,4,5,6,7,8,9],linear_regress:4,linearli:3,linearloc:5,linearregress:[1,4,5,7],linearsvc:6,liner:2,linerar:8,linewidth:[1,4,6,7,8],link:[1,5,7,10,12],linlag:3,linpack:13,linreg:1,linspac:[1,4,6,7,8,13,15],linu:16,linuek:16,linux:[1,2,12],liquid:1,list:[0,1,2,7,12],listedcolormap:[7,8],literatur:[0,2,5,17],littl:[2,7,10],live:6,lle:1,lloyd:0,lmb:[3,4],lmbd:[1,2],lmbd_val:[1,2],lmbda:5,load:[1,2,5,7,8],load_boston:1,load_breast_canc:[2,5,7,8,9],load_digit:2,load_iri:[6,7],loc:[1,4,5,6,7,8],local:[1,2,5,10],locat:6,log10:[3,4],log:[1,2,3,4,5,7,8,9,13],log_:1,log_clf:8,logarithm:[1,3,5,13],logic:[1,2,7],logist:[1,2,6,7,8,9,10,12,14],logisticregress:[5,7,8,9],logit:5,logreg:[5,7,8,9],logspac:[1,2,3,4],longer:[0,6,8,13,15],longest:0,loocv:4,look:[0,1,2,3,4,5,6,7,8,9,13,15],loop:[0,2,4,8,10,12,13,15],lose:2,loss:[1,2,3,4,6,8,9,13],lot:[0,1,2,4],low:[1,4,7,8,9,15],lower:[1,2,7,8,13],lowercas:13,lowest:[5,7,15],lstat:1,lstsq:1,lubksb:13,ludcmp:13,lux:13,lvert:2,m_1:0,m_h:1,m_k:0,m_l:10,m_n:1,m_p:1,machin:[2,3,4,7,8,9,10,13,14,17],machinelearn:[12,14,17],mackai:17,made:[1,2,3,4,5,7,9,10],mae:1,magic:0,magnitud:[2,5],mai:[1,2,3,4,5,6,7,9,10,12,13,15],mail:14,main:[1,2,3,5,7,13,17],mainli:[1,3,4,5,7],maintain:4,major:[2,4,5,7,8,13],make:[0,2,3,4,5,6,9,10,12,13,15,17],make_moon:[6,7,8],make_pipelin:[1,4,8],makedir:[1,4,5,7],makeplot:1,malcondit:13,malign:[2,5,7],mammographi:3,manag:[1,12],manhattan:0,mani:[0,1,2,3,4,5,6,7,9,11,12,13,15,17],manifold:9,manual:0,map:[0,1,2,4,5,6,9,10,15],margin:[1,3,6],marit:1,mark:[],marker:[1,5,13],markov:12,marsaglia:15,mask:15,mass:[1,2,3],massag:1,masses2016:1,masses2016ol:1,masses2016tre:1,masseval2016:1,master:14,mat1100:12,mat1110:12,mat1120:12,mat3155:14,mat4155:14,mat:12,match:[0,2,3,5],materi:[3,5,13],math:[0,5,10,13,15,17],mathbb:[0,1,3,4,5,6,9,10,13,15],mathbf:[1,3,4,5,6,13,15],mathcal:[2,3,4,5],mathemat:[0,1,5,9,10,12,13,15,17],mathemati:[],mathemt:[],mathrm:[0,1,2,3,4,5,6,7,8,9,10,15],matmul:[2,3],matnat:[16,17],matplotlib:[0,1,2,3,4,5,6,7,8,9,10,12,13,15],matric:[1,2,5,6,9,12,13],matrix:[1,4,6,8],matshow:2,matter:5,max:[1,2,5,7,8,10],max_depth:[1,7,8],max_it:[1,2,5,6,9],max_iter:0,max_leaf_nod:8,max_sampl:8,maxdegre:[1,4,8],maxdepth:8,maxim:[2,3,5,6,9],maximum:[0,1,2,3,5,6,7,8],maxpolydegre:[3,4],mbox:[3,4],mcculloch:10,mcint:15,mcintsqr2:15,mean:[1,2,3,4,5,7,8,9,10,12,13,14],mean_absolute_error:1,mean_divisor:0,mean_i:15,mean_matrix:0,mean_squared_error:[1,4,5,8],mean_squared_log_error:1,mean_vector:0,mean_x:15,meaning:[1,5],meansquarederror:1,meant:[5,8],meantempvec:15,meanvec:15,measur:[0,1,2,3,4,7,9,10,15],mechan:[1,15],median:1,medicin:10,medium:6,medv:1,meet:[1,16],mehta:[1,3],memori:[9,10,13],mention:[0,1,5,10,15],mere:1,mersienn:15,meshgrid:[1,3,6,7,8,9],met:[1,6],meteorolog:7,method:[0,1,2,3,6,9,10,12,13,14,17],metric:[0,1,2,4,5,7,8],metropoli:12,mev:[1,15],mglearn:12,mgrid:5,mhjensen:[1,2,6,9],microsoft:17,mid:2,midpoint:7,might:[1,2,5,7],mild:7,miller:15,million:1,mimic:10,min:[1,3,6,7],min_:[0,1,3],min_samples_leaf:7,mind:[0,1,5],mine:12,mini:[2,5,9,10],minibatch:[2,5,9],minibathc:5,minim:[0,1,2,3,4,5,6,7,8,9,10,15],minima:[1,2,5],minimum:[1,2,4,5,6,7,9],minkowski:0,minmaxscal:1,minor:15,minst:2,minu:5,mirror:7,misclassif:[6,7,8],misclassifi:[6,8],mismatch:2,miss:[1,8],mit:17,mix:2,mkdir:[1,4,5,7],mlab:[4,15],mle:[3,5],mlp:2,mlpclassifi:2,mlpregressor:1,mnist:[2,9],mod:15,mode:[14,15],model:[0,3,4,5,6,7,8,9,12,15,17],model_select:[1,2,3,4,5,7,8,9],moder:8,modern:[1,4,5,12],modif:10,modifi:[1,2,3,5,6,8,10],modul:[1,4,5,7,8,9,13],modular:15,modulenotfounderror:7,modulo:15,moe:[3,9],moment:[3,4],monoton:[3,10,15],mont:[1,4,12,15,17],montecarlocycl:15,moor:3,more:[1,2,3,4,6,7,8,9,10,12,14,15],moreov:1,morten:16,most:[0,1,2,3,4,5,6,7,8,9,10,12,15],mostli:[2,9],motion:1,motiv:2,move:[0,1,3,5,7,10,15],mpl:[1,5],mpl_toolkit:5,mplot3d:5,mplregressor:2,mse:[1,3,4,7,8],mse_simpletre:8,mselassopredict:3,mselassotrain:3,msepredict:3,mseridgepredict:3,msetrain:3,msg:1,msle:1,mt19937_64:15,mu0:15,mu1:15,mu2:15,mu_:15,mu_n:9,mu_x:15,much:[1,2,3,4,5,6,7,8,9,10,13,15],multi:[1,2,5,12],multiclass:[2,5],multidimension:[9,10],multilay:2,multinomi:5,multipl:[3,4,5,10,15],multipli:[3,5,9,13,15],multiplum:6,multitud:[],multivari:[1,8,9,12,15],multivariate_norm:[0,9],murphi:[9,17],must:[0,2,3,4,5,6,8,10,15],mutat:5,mutual:[2,4,5],mx_:15,myriad:[1,12],mz1:15,mz2:15,n_0:[10,15],n_b:15,n_boostrap:[4,8],n_bootstrap:4,n_categori:2,n_cluster:0,n_compon:9,n_epoch:5,n_estim:8,n_featur:2,n_hidden_neuron:[1,2],n_i:15,n_input:2,n_instanc:7,n_iter_i:9,n_job:8,n_k:0,n_l:[10,15],n_layer:2,n_m:7,n_neuron:2,n_neurons_layer1:2,n_neurons_layer2:2,n_point:0,n_sampl:[0,4,6,7,8],n_split:4,nabla:[2,5],nabla_:5,naimi:1,naiv:[0,5],nall:1,name:[0,1,2,3,4,5,6,7,8,10,12,13,16],nameerror:8,namespac:15,nation:[2,3],nativ:12,natur:[1,2,5,6,7,10,15,17],navier:10,nb_:13,nbconvert:[],nboot:15,nearest:[2,9],nearli:5,neat:[],neccesari:4,necessari:[0,1,2,6],necessarili:[1,9,15],necesserali:3,neck:5,need:[0,1,2,3,4,5,6,7,8,9,10,13,15],neg:[1,2,3,4,5,8,15],neg_mean_squared_error:4,neglect:15,neglig:15,neighbor:9,neq:[0,5,15],nervou:10,nest:[7,10],net:10,netlib:13,network:[1,7,12,14,17],neural:[1,5,12,14,17],neural_network:[1,2],neuralnetwork:2,neuron:[2,10],neutral:1,neutron:1,never:[2,4,7,15],new_hobbit:[],new_sig:4,newaxi:[1,4,7],newli:1,newton:[2,6,15],next:[0,1,2,3,5,6,7,15],next_guess:5,nian:16,nice:[0,1,2,3,9],nichola:16,nicholaskarlsen1102:16,niter:5,nitric:1,nlambda:[3,4],nm_n:1,nmse:4,nn_model:2,node:[2,7,8,10],nois:[1,3,4,5,6,7,8],noisi:[2,4],non:[0,1,2,3,4,5,7,8,9,10,13,15],none:[0,1,2,3,4,5,7,8,15],nonetheless:0,nonlinear:[4,6,7,9,10],nonneg:[4,5,7],nonparametr:4,nonsens:15,nonsingular:13,nonumb:[5,6,13],nor:2,norm:[1,2,3,4,5,6,9],normal:[3,4,5,6,7,8,9,10,12,13],normali:13,normpdf:4,notat:[0,1,3,4,15],note:[0,1,2,3,4,5,6,9,10,12,13,14,15,17],notebook:[0,1,2,7,12],noth:[0,2,3,6,10,15],notic:[3,10,13,15],novel:[4,8],novemb:2,now:[0,1,3,4,5,6,8,9,10,13,15],nowadai:[1,2,7,12],nox:1,np_assign_points_to_clust:0,np_get_distances_to_clust:0,np_k_mean:0,nsampl:4,nspin:15,nthi:1,nuclear:3,nuclei:[1,15],nucleon:1,nucleu:1,num_tre:8,number:[0,2,3,4,5,6,7,8,9,10,13,14,16],numberid:5,numer:[1,3,4,5,7,8,9,10,12,13,17],numpi:[0,1,2,3,4,5,6,7,8,9,10,12,15],nunmpi:3,obei:[5,9],object:[1,2,4,6,8,13],obliqu:3,observ:[0,2,3,4,5,6,7,8,9,10],obtain:[0,1,2,3,4,5,6,7,8,10,13,15],obviou:[3,9,15],obviouli:1,obvious:[1,3,13],occupi:1,occur:[1,6,7,15],odd:[1,5],off:[2,3,4,7,15],offend:[],offer:[4,9,12,13,14],offic:16,offici:14,ofil:15,ofstream:15,often:[0,1,2,3,4,5,6,7,8,9,10,12,13,15],ofter:13,old:[2,3,5,8],olsbeta:3,omit:[1,3],onc:[0,2,4,7,9,15],one:[0,1,2,3,4,6,7,8,9,12,13,15],onehot:2,onehot_vector:2,onehotencod:7,ones:[1,3,5,6,7,8,9,13],onli:[0,1,2,3,4,5,6,7,8,9,10,13,15],onlin:[9,14],onto:[3,9],open:[1,2,4,5,7,12,14,15],oper:[1,2,3,4,8,9,10,12,13,15],operation:15,opinion:0,oplu:15,opmiz:5,opportun:1,opposit:[2,3,6],opt:[1,2,3,4,6,9],optim:[0,1,3,4,7,8,9,14],optimis:2,option:[2,3,6,9],optmiz:[2,6],orang:1,order:[1,2,3,4,5,6,7,8,9,10,13,15],ordinari:[1,4,5,9,12,14],oreilli:17,org:[9,12,13,17],organ:[0,4,5,8,13],orient:[2,3,4,15],origin:[1,3,4,6,9,10,13,15],orthogn:3,orthogon:[1,3,5,6,9,13],orthonorm:3,oscar:2,oslo:[1,14,16],osx:[1,12],other:[0,1,2,3,4,5,6,8,12,13,14,17],otherwis:[1,2,5],ouput:[3,5,10],our:[2,4,6,7,8,10,12,13],ourselv:[0,1,3,5,6,9],out:[0,1,2,3,4,5,6,7,8,9,10,12,13,15],out_fil:7,outcom:[1,5,7,8,10,15],outdoor:7,outer:10,outfilenam:15,outlier:[1,6],outlin:[4,8,9],outlook:7,outperform:8,output:[1,2,3,5,6,7,8,10,13],output_bia:2,output_bias_gradi:2,output_weight:2,output_weights_gradi:2,outputlayer1:10,outputlayer2:10,over:[0,1,2,3,4,5,7,8,10,15],overal:[2,8],overcast:7,overcom:10,overdetermin:1,overfit:[2,4,7,8],overflow:[2,3],overhead:10,overlap:[5,6,7],overlin:[0,3,4,7,8,9,13],overst:1,overview:[0,17],own:[3,5,6,10,12,13],owner:1,oxid:1,oyvinssc:16,p_i:[3,15],p_j:15,p_n:15,p_x:15,pack:1,packag:[1,2,3,4,5,6,9,12,15],page:[1,12],pai:[2,7],painless:[],pair:[1,7,12,15],panda:[1,3,4,5,7,9,12],panel:[],paper:2,paradigm:1,parallel:[8,13],paramet:[1,2,3,4,5,6,7,8,10,15],parameter:[1,8],parametr:[1,4],paramt:3,park:15,part:[0,1,2,3,4,8,13,14,15,17],partial:[1,2,3,5,6,8,9,10,15],particip:[12,14],particl:[1,15],particular:[1,2,3,4,5,7,8,9,10,15,17],particularli:[3,4,5,6,9,15],partit:[2,7],pass:[0,10,15],past:[8,15],patch:[4,15],path:[1,4,5,7,12],patient:5,pattern:[1,10,14,17],pauli:1,pca:[1,5,12,14],pdf:[1,3,4,7,17],pedagog:1,penalti:[4,5],penros:3,pentagon:5,peopl:[1,2,7,12],per:[1,2,4,14],percentag:[1,8,9],perceptron:[1,2,5],peregrin:[],perfect:[1,2],perfectli:4,perform:[0,1,3,4,5,6,8,9,10,12,13,15],perhap:[1,3,5],perimet:2,period:2,permut:9,person:[3,5,14,16],perspect:17,pertin:10,petal:[6,7],peter:17,petersen:15,phantom:15,phase:10,phatak:0,phenomena:15,phi:6,phi_k:6,philip:16,philosophi:5,phone:16,phrase:1,physic:[1,2,5,10,15,16,17],pick:[0,2,5,7,8,9,15],pickl:2,pictur:1,pie:12,piec:[0,9],pillow:[1,12],pinv:3,pip3:[1,2],pip:[1,2,12],pipelin:[1,4,6,8],pippin:[],pise:0,pitt:10,pixel:2,pixel_height:2,pixel_width:2,place:[1,4,5,6,13],plai:[1,3,4,6,9,12],plain:[5,6,8,10],plan:[4,7,16,17],plane:[6,7],plateau:[3,15],platform:12,plausibl:10,pleas:[1,9],plenti:2,plethora:10,plot:[0,1,2,3,4,5,6,7,8,9,10,12,13,15],plot_confusion_matrix:[5,8],plot_cumulative_gain:[5,8],plot_data:2,plot_dataset:6,plot_decision_boundari:[7,8],plot_import:8,plot_predict:6,plot_regression_predict:7,plot_roc:[5,8],plot_surfac:5,plot_train:7,plot_tre:[7,8],plt:[0,1,2,3,4,5,6,7,8,9,10,13,15],plu:[1,3,5],png:[1,4,5,7],point:[0,1,2,3,4,5,6,7,8,9,13,15,16],points_in_clust:0,poisson:12,poli:[4,6],poly100_kernel_svm_clf:6,poly3:1,poly3_plot:1,poly_featur:[6,7],poly_features10:7,poly_fit10:7,poly_fit:7,poly_kernel_svm_clf:6,polydegre:[3,4,8],polygon:5,polym:10,polynomi:[1,3,4,5,6,7,8,9],polynomial_featur:4,polynomial_svm_clf:6,polynomialfeatur:[1,4,6,7],polytrop:[1,4],poor:[2,5],popul:[1,3],popular:[1,2,4,5,6,7,9,10,12,13,15],popularli:1,portabl:8,portion:9,pose:[1,9,15],posit:[0,1,2,3,5,6,8,9,13,15],possibl:[1,2,3,4,5,6,7,8,9,10,12,13,15,16],posterior:3,postpon:1,postul:3,potenti:[1,3,10],pott:10,power:[1,2,3,4,6,7,10],practic:[1,3,4,5,6,15],practition:[1,2],preced:[2,9,10,15],preceq:6,precis:[1,3,9,13,15],pred:4,predict:[1,2,3,4,5,6,7,8,12,17],predict_prob:2,predict_proba:[5,8],predictor:[1,3,5,7,8,9],prefer:[1,2,6,7,9,12],prepar:1,preprocess:[4,5,6,7,8,9],prerequisit:1,present:[1,3,7,10,15],preserv:9,press:[5,17],pretrain:2,pretti:[1,6,7,12],prev_centroid:0,prevent:15,previou:[1,2,3,5,6,8,9,10,13,15],previous:[0,7,8,15],price:[1,7],primal:6,primari:[1,5],primarili:0,prime:15,princip:[1,3,5,12,14],principl:[0,1,4,5,6],print:[0,1,2,3,4,5,6,7,8,9,13,15],print_funct:[6,7],printout:1,prior:[1,3,4],privat:1,prob:[2,15],probabilist:[1,17],probabl:[1,2,4,5,8,12],problem:[1,3,4,6,7,8,9,10,12,13,14,15],proce:[1,3,5,6,7,8,9,13],procedur:[3,4,5,6,8,9],proceed:13,process:[0,1,4,5,7,8,10,12,13,15,17],prod:17,prod_:[2,3,5],produc:[0,1,3,7,8,9,10,12,13,15],product:[1,2,3,4,5,6,10,12,13],profess:1,profil:0,program:[0,1,2,3,6,10,12,13,14,15],programm:13,progress:[0,2],progression_plot:0,prohibit:4,project:[1,2,3,5,9,12,14,15,16],project_root_dir:[1,4,5,7],promin:10,promis:6,prone:7,pronounc:12,proof:[1,5,9,10],prop:4,proper:[1,4],properli:[0,2,6,8],properti:[1,2,4,5,10,13],proport:[1,2,3,7,9,15],propos:[2,8],propto:[3,5],proton:1,prove:5,provid:[1,2,3,4,5,6,7,8,10,12,13,15,17],proxi:2,prun:0,prune:7,pseudoinv:3,pseudoinvers:3,pseudorandom:15,psycholog:1,ptratio:1,punish:[1,2],pure:[7,15],purest:7,puriti:7,purpos:[0,1,8,10],put:2,pycod:[],pydata:12,pydot:7,pyhton2:[],pylab:[1,5],pypi:12,pyplot:[0,1,2,3,4,5,6,7,8,9,10,13,15],pythagora:[0,3],python2:1,python3:[1,2,4,6,9,12],python:[2,3,6,9,10,14],pytorch:[1,12],qquad:[9,13],quad:[2,5,13],quadrat:[1,5,6,7],qualit:[7,15],qualiti:[1,7,12],quantifi:2,quantil:8,quantit:[1,4,7],quantiti:[0,1,3,4,5,7,8,9,10,13,15],quantum:10,quartil:1,queri:7,question:[1,3,4,5,7,9,10],quick:15,quickli:[2,5,7,9],quirk:0,quit:[2,3,4,7,8,10],quot:15,r2_score:1,r2score:1,r_1:7,r_2:7,r_j:7,r_m:7,rad:1,radial:[1,6,10],radioact:15,radiu:[1,2],rain:7,rais:4,ramp:2,ran1:15,ran2:15,ran3:15,rand:[1,3,4,5,7,8,13,15],rand_max:15,randint:[4,5,7],randn:[1,2,3,4,5,7,9],random:[0,1,2,3,4,5,6,7,12,13,14],random_devic:15,random_forest_model:8,random_index:5,random_indic:2,random_st:[1,5,6,7,8,9],randomforestclassifi:8,randomli:[0,2,4,5,7,15],randomnumbergener:15,rang:[0,1,2,3,4,5,7,8,9,10,13,15],rangl:[1,9,15],rangle_x:15,rank:3,raphson:[2,6],rapidli:1,rare:2,rate:[1,2,5,6,7,8,10],rather:[1,2,3,4,5,6,7,8,9,10,13,15],ratio:[5,7,8,9],rational:1,ravel:[1,3,4,5,6,7,8,9],raw:15,rbf:[6,9,10],rbf_kernel_svm_clf:6,rbf_pca:9,rcond:1,rcparam:[1,2,5,6,7,8,15],reach:[0,1,2,3,4,5,7,8,9,10,15],read:[0,1,3,4,5,6,9,10,13,14,15,17],read_csv:[1,4,5,7],read_fwf:1,readabl:0,reader:[1,13,15],readi:[0,1,2,3,6,8,9,10,13],readili:2,real:[1,2,4,5,8,9,10,13],realist:6,realiti:15,realiz:[2,10],realli:2,reason:[0,1,2,5,8,17],reassign:2,recal:[3,4,5,7,8,9,10,13,15],recalcul:15,receiv:[2,8,10,15],recent:[1,4,5,7,8],recept:10,recip:[1,5,13],reciproc:3,recogn:[1,3,8],recognit:[1,2,10,14,17],recommend:[1,3,6,12,13,14,15,17],reconsid:7,reconstruct:9,record:[8,14],recreat:15,rectangl:[4,5,7],rectangular:3,rectifi:[2,10],recur:[1,12],recurr:[2,12,14],recurs:[7,12,13],recycl:15,red:[1,4,6,7],redefin:[1,8],reduc:[2,3,5,7,8,9],reduct:[1,8,9,12,15],refer:[0,1,2,3,4,5,9,10,13,17],refin:10,refit:4,reflect:[1,2,15],refresh:[12,14],reg:[8,9],regard:[2,5,7],regardless:10,region:[7,10],regist:15,reglasso:3,regr_1:[1,7],regr_2:[1,7],regr_3:[1,7],regress:[2,4,6,9,10,12,14],regressor:[1,5,8],regridg:3,regular:[1,3,5,7],reilli:[1,17],reinforc:[1,6,12],reiter:2,rel:[1,4,5,7,10,15],relat:[0,1,2,3,5,9,13,15],relationship:[1,7],relativeerror:1,releas:[2,12],relev:[1,2,3,5,9,12,15],reli:[1,6],reliabl:[5,15],remain:[2,4,10,13,15],remaind:15,remark:2,rememb:[1,6,13],remind:[1,3,9,13,15],remov:[1,3],render:1,reorder:[3,5],reorgan:1,repeat:[0,1,2,3,4,5,7,8,9,13,15],repeated:1,repeatedli:[4,8,15],repetit:[4,14],rephras:5,replac:[0,1,2,3,4,5,8,10],replica:4,repositori:1,repres:[1,2,3,4,5,6,7,8,10,15],represent:[1,2,4,15],reproduc:[1,7,10,12,15],repuls:1,request:1,requir:[1,2,3,4,5,6,7,9,10,13],resampl:[1,5,8,12,14,15],rescal:[1,9,10],rescu:3,research:[1,12,17],resembl:[4,15],reserv:[2,3,4,15],reset:15,reshap:[0,1,2,4,6,7,8],residenti:1,residu:[1,3,5],resiz:3,respect:[0,1,2,3,4,5,6,8,9,10,15],respond:10,respons:[1,5,7,10],rest:[1,3],restat:[1,10],restrict:[1,7,10],result:[0,1,2,3,4,5,6,7,8,9,10,12,13,15],ret:4,retail:1,retain:[3,4],return_data:0,return_x_i:7,reus:2,reveal:[1,10],revers:2,review:[12,13],revisit:0,reward:1,rewrit:[1,3,4,5,6,8,9,10,13,15],rewritten:[4,6,8,15],rewrot:5,rgoj5yh7evk:12,rho:[1,8],rho_1:8,rho_2:8,rho_m:8,rich:1,rid:0,ride:7,rideclass:7,ridedata:7,ridg:[1,4,9,12,14],ridgebeta:3,right:[1,2,3,4,5,6,7,8,10,13,15],rightarrow:[1,2,3,5,6,9,10,15],rigor:1,rise:1,risk:[1,5],river:1,rmse:1,rmsprop:2,rnd_clf:8,rnn:10,rntrick1:15,rntrick2:15,rntrick3:15,rntrick4:15,robert:17,robust:1,robustscal:1,roc:8,role:[1,3,4,6,12],room:[1,16],root:[1,3,5,7,15],rot:[],rotat:[2,6,7,8],rotation_matrix:7,roughli:2,round:[1,5,7],routin:[5,13],row:[1,2,3,4,7,9,13],rrr:3,rug:5,rule:[1,2,3],run:[0,1,2,3,4,5,6,7,9,12],runtim:[0,2,4],runtimewarn:2,rust:[1,12,13],rvert:2,rvert_2:2,rwidth:4,s_i:5,saddl:5,safe:15,sai:[0,1,2,3,4,5,6,7,8,9,10,13,15],said:[4,5,7],sake:[1,3,5,9],sale:1,sam:[],same:[0,1,2,3,4,5,6,7,9,10,13,15],samm:8,sampl:[0,1,2,3,4,5,6,7,8,12,13],sample_vari:0,sampleexptvari:15,samwis:[],sanitize_sequ:4,sastri:9,satisfactori:1,satisfi:[2,4,5,6,13,15],satur:[2,4],save:[1,4,5,7],save_fig:[1,4,5,7,8],savefig:[1,4,5,7,15],saw:3,scalabl:8,scalar:[3,4,8],scale:[1,2,3,5,6,7,8,9,10,12,16],scaler:[1,5,6,7,8,9],scan:[3,5],scari:3,scatter:[0,1,2,4,5,6,7],scenario:5,scheme:[2,5],schrage:15,scienc:[1,2,5,8,10,12,14,15,17],scientif:[1,12],scientist:[0,1],scikit:[3,6,7,8,12,13,14,17],scikitlearn:1,scikitplot:[5,8],scipi:[1,3,5,12,13],score:[1,2,4,5,7,8,9,16],scores_kfold:4,scratch:2,sdg:5,seaborn:[1,2,5],seamless:[1,12],search:[1,2,3,5,7],sec:4,second:[0,1,3,4,5,6,7,9,10,12,13,15],secondeigvector:9,secondli:10,section:[0,9,11,13,14,15],sector:1,see:[0,1,2,3,4,5,6,8,9,10,12,13,14,15],seed:[0,1,2,3,4,5,6,7,9,15],seek:[2,6],seem:2,seemingli:1,seen:[1,2,8,10,15],segment:5,seldomli:1,select:[2,3,4,6,7,8,9,14,17],self:[2,3,4,15,17],semest:[5,14],semi:[5,6],send:[10,16],senior:14,sens:[1,4,6],sensit:[1,3,4,7],sentenc:10,separ:[0,1,2,4,6,7,10,12,15],sequenc:[0,5,7,8,10,12,13,15],sequenti:[2,8,10,15],seri:[1,2,3,5,8,9,10,13,14],serif:[1,5,15],serv:[1,2,3,5,17],session:[2,14],set:[0,2,3,4,5,6,8,9,12,13,15],set_:4,set_label:4,set_tick:[2,6],set_ticklabel:2,set_titl:[0,1,2,5,10],set_xlabel:[1,2,5,10],set_xlim:[5,10],set_xticklabel:2,set_ylabel:[1,2,5],set_ylim:[5,10],set_ytick:5,set_yticklabel:2,setiosflag:15,setminu:4,setosa:[6,7],setosa_or_versicolor:6,setp:4,setprecis:15,setup:[2,6,12],setw:15,sever:[1,3,4,5,6,7,9,10,12,13,14,15],sgd:2,sgd_clf:6,sgdclassifi:6,sgdreg:5,sgdregressor:5,sgn:3,shape:[0,1,2,3,4,5,6,7,8,9,13],share:2,she:5,shift:[2,10,15],shire:[],shortcom:5,shorter:15,shorthand:[],shortli:13,should:[0,1,3,4,6,7,9,10,13],show:[0,1,2,3,4,5,6,7,8,9,10,13,15],shown:[3,5,6,9,10,15],showpoint:15,shrink:[3,6,9],shrinkag:3,shrunk:9,shuffl:[2,4],side:[1,3,5,6,10,13],sigh:12,sigma0:15,sigma1:15,sigma2:15,sigma:[1,2,3,4,5,8,9,10,13,15],sigma_0:3,sigma_1:3,sigma_2:3,sigma_:[3,13,15],sigma_fn:[5,10],sigma_i:[1,3],sigma_j:3,sigma_m:15,sigma_n:[9,15],sigma_x:15,sigmoid:[2,5,6,8,10],sigmundson:16,sign:[2,5,6,8,15],signal:[2,8,10],signific:2,significantli:[2,5,15],sim:[3,4,15],similar:[0,1,2,3,4,5,6,7,8,9,12,13],similarli:[1,2,3,6,8,15],simpl:[0,2,3,4,6,8,9,10,12,13],simplepredict:8,simpler:[0,1,2,3,12],simplest:[0,1,2,7,8,10],simpletre:8,simpli:[1,2,3,4,6,7,8,9,10,12,13,15],simplic:[0,3,5,6,7,8,9,10],simplicti:3,simplifi:[1,4,7,12],simplist:15,simul:[4,15],simultan:4,sin:[1,2,7,10,13],sinc:[1,2,3,4,5,6,7,8,9,13,15,17],sine:10,singl:[2,3,5,6,7,10,13,15],singular:[1,4,5,13],site:[1,2,4,6,9,14],situat:[1,3,5],six:15,size:[1,2,3,4,5,6,7,8,9,13,15],sketch:8,ski:7,skill:1,skip:9,skl:1,sklearn:[1,2,3,4,5,6,7,8,9],skplt:[5,8],slack:6,slice:13,slide:[1,15],slight:4,slightli:[2,3,4,8,15],slope:[6,9,10],slow:[1,5,6],slower:[3,13],slowli:10,slp:2,small:[0,1,2,3,4,5,6,7,8,9,10,12,13,15],smaller:[1,2,4,5,6,7,9,15],smallest:[0,1],smallest_row_index:0,smart:4,smooth:[1,5],sne:9,sneak:1,sns:[1,2,5],soar:4,social:1,soft:[2,5,8,10],soften:6,softmax:5,softwar:[1,6,12,13,14],sol:6,sole:1,solid:[1,5],solut:[1,2,3,4,5,6,8,9,13,15],solv:[1,2,3,6,8,9,10,13],solver:[5,6,7,8,9,13],some:[0,1,2,3,4,6,7,8,9,10,13,15],someth:[1,2,5,7,9,15],sometim:[0,1,2,9,10],soon:13,sophist:1,sopt:5,sort:[1,3,4,7,9,15],sound:3,sourc:[0,1,2,4,12,13,15],space:[0,1,2,3,5,6,7,9,10,15],span:[1,3,7,9,13],spare:2,spars:13,sparse_mtx:13,sparsiti:8,spatial:[2,10],speak:15,special:[4,5,8,10,13,15],specif:[1,2,3,4,5,6,7,9,10,12,13,15],specifi:[0,1,3,4,5,7,9,15],specifici:[1,8],spectral:2,speech:[1,2,10],speed:2,spend:15,sphere:1,spite:1,spline:6,split:[0,2,3,4,6,7,8,9,15],splitter:[2,8],spontan:15,spread:[1,9,15],springer:17,sqquar:3,sqrt:[1,3,4,5,6,8,9,15],squar:[0,1,2,4,5,6,7,9,12,13,15],squarederror:8,squaredeuclidean:0,squash:10,srand:15,stabl:[1,3,7,9,12],stack:4,stage:5,stai:[1,3,9],stand:[1,3,7,10],standard:[1,2,3,4,6,8,10,13],standardscal:[1,5,6,7,8,9],stanford:5,start:[0,1,2,3,4,5,6,7,8,9,10,13,15],start_tim:0,startpoint:15,stat:4,state:[0,2,3,5,6,8,9,10,12,15],statement:[1,5,13],statis:[],statist:[0,1,2,5,7,8,9,10,13,14,17],statu:[1,5,9],std:[4,15],stdev:15,steep:5,step:[0,1,2,7,8,9,10,13,15],step_fn:[5,10],step_length:5,steps_list:7,stian:16,still:[3,4,5,9,15],stimuli:10,stk2100:17,stk4021:17,stk4051:17,stk5000:17,stk:17,stochast:[1,2,3,4,6,9,10],stoke:10,stone:[1,5],stop:[0,2,7,9,15],storag:3,store:[1,2,5,9,15],str:2,straight:[1,4,5,6],straightforward:[1,3,4,5,6,7,8,13],strategi:[1,2,7],stratifi:4,strength:[0,3],stretch:9,strict:[5,6],strictli:[5,6],string:[2,15],stroke:5,strong:[7,8,10,15],strongli:[1,6,12,13],stronli:1,structur:[0,1,2,4,7,8,10,12],stuck:[2,5],student:[1,14,16,17],studi:[0,1,3,5,6,9,10,12,17],studier:17,style:[1,5,7],sub:[7,10],subdivid:[1,13],subfield:1,subject:[6,15],subplot:[0,1,2,4,5,6,7,8],subplots_adjust:[6,15],subprogram:13,subroutin:1,subscript:2,subsequ:[2,3,4,10,13,15],subset:[2,4,5,7,10,12],subspac:[1,6,9],substanti:[7,8],substep:9,substitut:[4,10,13],subsubset:7,subtl:2,subtract:[3,4,9,13,15],subtre:7,succeed:1,success:[5,7,15],successfulli:7,sucess:15,sudo:[1,12],suffer:[1,2,3,8],suffici:[2,4,5,6,9],suggest:[2,5,17],suit:[6,10],suitabl:[1,15],sum:[0,1,2,3,4,5,6,7,8,9,10,15],sum_:[0,1,2,3,4,5,6,7,8,9,10,13,15],sum_i:[3,4,5,6],sum_k:[6,10,13],summar:[0,3,4,7],summari:[0,2,8,14],summat:[],sunni:7,superscript:[2,10],supervis:[1,3,4,5,7,10,12],supplement:5,support:[1,2,7,8,9,12],suppos:[1,3,4,5,6,8,9,10,13],suppress:3,sure:2,surfac:1,surpris:1,surround:12,survei:[1,3],svc:[6,7,8],svd:[1,4,9],svdinv:3,svm:[6,7,8,9],svm_clf:[6,8],swath:3,symbol:[2,3,9,12,15],symmeteri:2,symmetr:[1,3,5,6,9,10,13],sympi:[1,12],synonim:15,syntax:[2,13],syntaxerror:[2,6,13],sys:5,system:[1,2,5,7,8,10,12,13,17],systemat:[4,15],t_0:[5,7],t_1:5,t_b:8,t_i:[2,10],t_j:[3,10],t_k:7,tabl:[7,15,16],tabul:1,tabular:[],tackl:0,tag:[0,3,5,10,13,15],taht:1,tail:15,tailor:[6,9],taiwan:1,take:[0,1,2,3,4,5,6,7,8,9,10,12,13,15],taken:[1,2,4,8,13],tangent:[2,5,10],tanh:[2,5,6,10],target:[1,2,3,5,6,7,8,9,10],target_nam:7,task:[0,1,2,4,7,9,10],tau:[3,15],tax:1,taylor:5,taylornr:5,team:2,teaser:1,technic:[0,1,3],techniqu:[1,2,6,8,12,15,17],technolog:[1,2],tek5040:17,tell:[4,5,8,9,15],temp1:2,temp2:2,temp:2,temperatur:[1,7],temporarili:2,ten:[],tend:[0,3,4,6,7,8,10],tendenc:1,tension:4,tensorflow:[0,1,6,12,13,14,17],term1:[1,3,9],term2:[1,3,9],term3:[1,3,9],term4:[1,3,9],term:[0,1,2,3,4,5,6,7,8,9,10,15],termin:[1,3,7,8],test:[3,4,5,6,7,8,15],test_accuraci:2,test_data:0,test_ind:4,test_pr:2,test_predict:2,test_scor:[5,8],test_siz:[1,2,3,4,8],test_split:7,tester:[],testerror:4,text:[1,2,3,5,6,7,9,13,15,17],textual:7,textur:2,than:[1,2,3,4,7,8,9,10,12,15],thats:0,theano:[2,12],thei:[0,1,2,3,4,5,6,7,9,10,13,15],them:[1,2,5,6,7,8,9,10,13],theme:1,themselv:[1,15],thenc:4,theorem:[4,5],theoret:[1,8],theori:[1,2,5,6,7,10,12,17],thereaft:[1,4,9,10,13],therebi:[1,3,5,9,15],therefor:[1,2,4,5,6,9,15],therein:9,thereof:[1,4,5],theta:[2,4,5,15],theta_:2,theta_i:2,theta_k:15,theta_linreg:5,thi:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,17],thing:[0,1,2,3,5,7,15],think:[0,1,2,4,5,7,10,15],third:[1,5],thirti:5,thorough:0,those:[0,3,6,7,8,9,13,14],though:[2,13,15],thought:[0,4,15],thousand:[1,2],three:[1,2,3,4,6,7,10,13,14,16],threshold:[2,5,7,8,9,10],through:[0,1,2,3,5,6,9,10,12,13,15],throughout:[0,1,12,13,15],thu:[1,2,3,4,5,6,8,9,10,15,16],thumb:1,thursdai:14,tibshirani:[14,17],ticker:[5,15],tight_layout:[2,5],tightli:9,tild:[1,3,4,9,15],till:[1,5,6,7,8,10,13],time:[0,1,2,3,4,5,6,7,8,9,10,13,14,15],timefunct:15,tini:2,tip:11,titl:[1,2,4,5,6,7,8,15],to_categor:2,to_categorical_numpi:2,to_numer:[1,4],to_str:15,togeth:[1,6,9],toi:0,toler:0,tomographi:10,too:[1,3,4,5,7,9,15,17],took:6,tool:[0,1,2,4,12,15],toolbox:6,top:[1,3,4,7,8,12],topic:[0,1,3,5,6,12,17],topolog:[2,10],toss:8,total:[0,1,2,4,5,6,8,9,10,15,16],totalclustervari:0,totalscatt:0,totalvari:15,toward:[2,5,10],town:1,tpng:7,traceback:[4,5,7,8],track:[0,5,13],tract:1,tractabl:1,trade:[3,4,7],tradeoff:[1,3,14],tradit:[1,2,4],train:[3,4,5,6,7,8,9,10],train_accuraci:[1,2],train_end:2,train_ind:4,train_pr:2,train_siz:2,train_test_split:[1,2,3,4,5,7,8,9],train_test_split_numpi:2,trainingerror:4,trait:1,transfer:7,transform:[1,3,4,5,6,7,8,9,10,12,13],transit:10,translat:[2,8],transpos:[2,3,9],treat:[1,2,4,5,10,15],tree:[1,2,12,14],tree_clf:[7,8],tree_clf_:7,tree_clf_sr:7,tree_reg1:7,tree_reg2:7,tree_reg:7,trend:15,trevor:17,tri:7,trial:[1,4,5,15],triangl:5,triangular:13,trick:[6,9,15],trickier:15,tridiagon:13,trillion:12,trivial:[1,2,3,9,15],troubl:[1,6,10],true_divid:2,true_fun:4,tucker:6,tumor:[5,7],tumour:5,tunabl:2,tune:[7,13],turn:[1,2,3,4,5,6,7,8,9,10,13,15],tutori:2,tweak:[2,8,15],twice:5,twist:9,twister:15,two:[0,1,2,3,4,5,7,8,9,10,13,14,15,17],tx_1:5,type:[1,2,4,5,6,8,13,15],typeerror:[],typic:[1,2,3,5,7,8,10,15],u_i:10,u_m:8,ubuntu:[1,12],uci:1,uio:[16,17],unari:13,unbalanc:[4,7],unbias:[1,3,4,15],uncertainti:[1,3],uncertitud:15,unchang:2,uncorrel:8,undefin:3,under:[1,2,3,4,5,8,12],underdetermin:1,underfit:[2,4],underflowproblem:3,undergo:3,undergradu:14,underli:[1,2,5,7,15],underset:0,understand:[0,1,2,3,5,8,12],understood:[0,6],undesir:6,undetermin:[3,6],unexpect:[4,15],unexpected:15,unfortun:[2,6,7,8],unicode_liter:[6,7],uniform:[1,2,3,5,9],uniform_real_distribut:15,uniformli:[5,15],unifrompdf:15,unimport:5,union:[3,4],uniqu:[0,1,4,5,13],unique_cluster_label:0,unit:[1,2,3,8,10,15],unitari:[3,13],unitarili:13,uniti:15,univari:15,univers:[1,2,5,14,16],unix:2,unknow:[1,13],unknown:[1,2,3,4,6,8,13],unknowwn:10,unlabel:2,unless:[1,4,5,9],unlik:[2,5,6,15],unnecessarili:7,unravel:2,unrol:9,unseen:[5,7],unstabl:2,unsupervis:[1,2,10,12,14],unsymmetr:13,until:[0,2,5,7,10,15],unusu:10,updat:[0,2,4,8,10],upload:[12,17],upon:[2,9,13],upper:[1,6,7,13],uppercas:[13,15],ups:[],usag:[1,6,12,15],usd10000:1,usd:1,use:[0,1,3,4,5,6,7,8,9,10,12,13,14],usecol:1,used:[0,1,2,3,4,6,7,8,9,10,12,13,15,17],useful:[1,2,3,4,5,7,9,10,12,13,15,17],useless:2,user:[1,2,5,6,9,12,13],uses:[1,2,3,4,7,9,10,13,15],usetex:15,using:[0,2,3,4,6,7,8,9,10,13],usr:15,usual:[0,1,5,10],util:[0,2,4,5,8],v_0:9,valid:[1,2,5,7,8,12,14,15],valu:[0,1,2,4,5,6,7,8,10,12,13],valuat:7,valueerror:[],van:[1,3],vandenbergh:[5,6],vandermond:1,vanilla:9,vanish:[2,5,15],var_x:15,varabl:6,varepsilon:[3,4],varepsilon_:[3,4],varepsilon_i:[3,4],vari:[1,2,4,8],variabl:[0,1,2,3,4,6,8,9,10,13],varianc:[0,1,2,3,5,7,8,9,12,13,14],variance_i:[3,9],variance_x:[3,9],variant:[1,2,4,5,6,10],variat:9,varieti:[1,10,12],variou:[2,3,5,6,7,9,10,12,13,15],vartempvec:15,varvec:15,vaue:2,vdot:5,vec:[4,15],vector:[0,1,2,3,4,5,7,8,9,12],vector_mean:0,ventur:[1,6,12],verbos:2,veri:[0,1,2,3,4,5,6,7,8,9,10,15,17],verifi:[9,13],versatil:6,versicolor:[6,7],version:[0,1,8,12,13,15],versu:2,vert:[1,2,3,5,6,7,9],vert_1:3,vert_2:[3,9],vertic:4,via:[1,3,4,5,6,7,8,9,10,12,13,14,15],vidal:9,video:[1,2,5,10,12,14],view:[2,3,10,15,17],violat:6,virginica:7,viridi:[1,2],virtual:2,vision:1,visual:[1,9,10,12],visualis:2,viz:[6,15],vmax:2,vmc:15,vmin:2,volum:1,vote:8,voting_clf:8,votingclassifi:8,votingsimpl:8,vstack:[3,9,13,15],w_1:[6,13],w_1x_1:6,w_1x_:6,w_2:[6,13],w_2x_2:6,w_2x_:6,w_3:13,w_4:13,w_i:[2,8],w_ix_i:10,w_j:13,w_m:13,w_px_:6,w_px_p:6,wai:[0,1,2,3,4,5,6,8,9,10,11,13,15],walk:7,walker:15,wang:1,want:[0,1,2,3,4,5,6,7,8,9,10,12,15],warn:[1,2,6],warrant:4,watch:12,wavelet:6,weak:[0,7,8],weather:[2,10],web:[12,14],websit:[13,14],wedg:[6,15],wednesdai:14,wee:9,week:[3,5],weekli:[12,17],weight:[1,2,4,5,7,8,10,15],welcom:[6,12],well:[1,2,3,4,5,6,7,8,10,12,13,15,17],went:6,were:[0,1,2,3,4,5,6,8,9,10,15],wessel:[1,3],what:[0,2,3,4,5,6,7,8,9,10,12,13,14],when:[0,1,2,3,4,6,7,8,9,10,13,15],whenev:15,where:[0,1,2,3,4,5,6,7,8,9,10,12,13,15,16],wherea:[4,15],wherein:[2,10],whether:[1,3,5,7,15],which:[0,1,3,4,5,6,7,8,9,10,12,13,14,16],whichev:2,white:7,who:[1,14],whole:[0,2,3,7,9],whose:[4,8,15],whow:[3,9],why:[1,2,5],wide:[1,2,4,5,10,12,13],widehat:4,width:[1,6,7],wieringen:[1,3],win:8,wind:7,wing:16,wiscons:5,wisconsin:8,wise:[1,2,3,10],wish:[0,1,3,5,6,9,13],within:[0,1,5,7,10,15,17],withinclust:0,without:[1,2,3,5,6,7,9,10],won:1,wonder:6,word:[0,1,2,3,15],work:[0,1,2,4,5,6,7,12,14,15],world:[1,6],worldwid:1,wors:[1,2,4],worth:7,would:[1,2,3,4,5,6,7,8,9,10,13,15],wrap:13,write:[1,2,3,5,6,10,11,13,14,15],written:[1,3,5,9,10,12,13,15],wrong:[2,6],wrongli:[8,15],wrote:[3,9],wrt:8,wth:8,www:[12,13,17],wx_1:6,x0s:6,x1_exampl:6,x1d:6,x1s:[6,7,8],x2d:[6,9],x2d_train:9,x2dsl:9,x2s:[6,7,8],x3s:6,x_0:[1,3,9,13],x_1:[1,3,4,5,6,7,8,9,13,15],x_2:[1,3,4,5,6,7,8,9,13,15],x_3:[6,13,15],x_4:13,x_center:9,x_data:2,x_data_ful:2,x_i:[0,1,2,3,4,5,6,7,8,9,10,13,15],x_ix_:1,x_iy_i:6,x_j:[6,7,10,15],x_jy_j:6,x_k:[0,10,13,15],x_l:15,x_m:[10,13,15],x_n:[1,4,5,6,9,10,13,15],x_new:[7,8],x_p:[5,7],x_poli:7,x_poly10:7,x_reduc:9,x_scale:6,x_test:[1,2,3,4,5,7,8,9],x_test_scal:[1,5,7,8,9],x_train:[1,2,3,4,5,7,8,9],x_train_scal:[1,5,7,8,9],x_val:2,xarrai:12,xavier:2,xbnew:5,xcode:[1,12],xdclassiffierconfus:8,xdclassiffierroc:8,xg_clf:8,xgb:8,xgbclassifi:8,xgboost:7,xgboot:8,xgbregressor:8,xgparam:8,xgtree:8,xi_1:6,xi_:6,xi_i:6,xlabel:[1,2,3,4,5,6,7,8,15],xlim:[4,8],xmesh:5,xnew:[1,5],xpd:[3,9],xplot:1,xsr:7,xt_x:5,xtest:4,xtick:[4,6,7],xtrain:4,xytext:6,y_0:[1,3,9,13],y_1:[1,3,5,6,7,9,13],y_1y_1:6,y_1y_1k:6,y_1y_2:6,y_1y_2k:6,y_1y_n:6,y_1y_nk:6,y_2:[1,3,6,7,9,13],y_2y_1:6,y_2y_1k:6,y_2y_2:6,y_2y_2k:6,y_3:[1,7,13],y_4:13,y_data:[1,2,3],y_data_ful:2,y_decis:6,y_i:[1,2,3,4,5,6,7,8,9,10,13],y_if_:8,y_ix_:1,y_ix_i:[5,6],y_iy_jk:6,y_j:[4,6,10],y_k:10,y_m:13,y_model:[1,3],y_n:[5,6],y_ny_1:6,y_ny_1k:6,y_ny_2:6,y_ny_2k:6,y_ny_n:6,y_ny_nk:6,y_plot:7,y_pred1:7,y_pred2:7,y_pred:[1,2,4,5,6,7,8],y_pred_rf:8,y_pred_tre:8,y_proba:[5,8],y_test:[1,2,3,4,5,7,8,9],y_test_onehot:2,y_test_predict:1,y_train:[1,2,3,4,5,7,8,9],y_train_onehot:2,y_train_predict:1,y_val:2,year:[1,12],yes:[4,5],yet:[1,2,4,6,9],yield:[0,1,3,4,5,6,8,10,13,15],ylabel:[1,2,3,4,5,6,7,8,15],ylim:4,ymesh:5,yoshua:[2,17],you:[0,1,2,3,4,5,6,7,8,9,12,13,15,17],young:1,your:[0,2,4,5,6,9,12,13,15],yourself:[5,9],youtub:12,ypred:4,ypredict2:5,ypredict:[1,5],ypredictlasso:3,ypredictol:3,ypredictridg:3,yridg:1,ytest:4,ytick:[4,6,7],ytild:[1,4],ytildelasso:3,ytildenp:1,ytildeol:3,ytilderidg:3,ytrain:4,z_0:13,z_1:13,z_2:13,z_c:2,z_h:2,z_i:[2,10],z_j:[2,10],z_k:10,z_m:2,z_mod:7,z_o:2,zaman:15,zero:[0,1,2,3,4,5,6,7,8,9,10,13,15],zm_h:1,zone:1},titles:["11. Clustering Analysis","3. Linear Regression","13. Building a Feed Forward Neural Network","4. Ridge and Lasso Regression","5. Resampling Methods","6. Logistic Regression","7. Support Vector Machines, overarching aims","8. Decision trees, overarching aims","9. Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods","10. Basic ideas of the Principal Component Analysis (PCA)","12. Neural networks","Content in Jupyter Book","Applied Data Analysis and Machine Learning, FYS-STK3155/4155 at the University of Oslo, Norway","2. Linear Algebra, Handling of Arrays and more Python Features","Teaching schedule with links to material","1. Elements of Probability Theory and Statistical Data Analysis","Teachers and Grading","Textbooks"],titleterms:{"2021":16,"4155":12,"case":[5,6,8,15],"final":[5,10],"function":[1,2,5,6,8,9,10,15],"import":[3,13,15],And:5,Eye:8,FYS:12,OLS:3,The:[0,1,2,3,4,5,6,7,9,10,12,15],Useful:12,Using:5,activ:[2,10],actual:15,adaboost:8,adapt:8,adjust:2,again:7,aim:[6,7],algebra:13,algorithm:[0,7,8,9,10],algortithm:5,all:6,analys:3,analysi:[0,1,3,9,12,15],ani:5,anoth:7,appli:12,approach:[1,5,6],approxim:10,architectur:2,arrai:13,assist:16,august:14,autocorrel:15,back:[2,9,10],background:12,bag:8,basic:[0,1,3,5,7,8,9,13],batch:2,bay:3,befor:9,better:[6,15],bia:4,binari:2,binomi:15,bird:8,block:15,book:11,boost:8,bootstrap:[4,8,15],boston:1,breast:2,brief:5,bring:10,build:[2,7],calcul:15,cancer:[2,5,7,9],cart:7,central:[5,12,15],chain:10,chang:8,chi:1,choos:2,classic:9,classif:[2,7,8],classifi:6,clip:2,cluster:0,code:[0,1,2,3,5,7,9,10,15],collect:2,compar:8,compon:9,comput:[5,7,15],con:7,concept:15,condit:5,conjug:5,content:11,continu:15,convex:[5,6],convolut:10,correl:[9,15],correspond:5,cost:[2,5,8],cours:[12,17],covari:[3,9,15],cross:4,cumul:15,cython:[],data:[1,2,5,7,9,12,15],dataset:2,decemb:14,decis:[7,8],decomposit:[3,9,13],deep:2,defin:2,definit:15,degre:1,demonstr:15,dens:1,deriv:[3,5,10],descent:[5,8],develop:2,deviat:15,diagon:9,dice:15,differ:[5,6],dimension:6,disadvantag:7,discret:15,disguis:15,distribut:[3,15],doing:2,domain:15,down:2,dropout:2,economi:[],element:[1,15],elimin:13,ensembl:8,entropi:7,environ:1,equat:[1,3,5,10],error:8,etc:[],evalu:2,event:15,exampl:[1,2,5,6,7,8,15],exercis:1,expect:15,experi:15,explor:1,exponenti:15,express:5,extend:5,extrem:8,fall:16,famili:2,famou:15,fantast:[],featur:[7,13],feed:[2,10],fine:2,first:[5,10],fit:[1,8],forest:8,forward:[2,10],frank:1,freedom:1,frequentist:1,fridai:5,from:[3,8,10],further:3,gaussian:[13,15],gener:[7,15],geometr:[5,9],gini:7,good:1,grade:16,gradient:[2,5,8],handl:13,has:12,hessian:5,homework:5,hous:1,how:15,hyperparamet:2,hyperplan:6,id3:7,idea:[0,9],ideal:5,implement:[2,15],implic:3,improv:2,increment:9,index:7,inform:16,instal:12,instructor:16,interpret:[3,5,9],introduc:9,introduct:[1,4,12,13],invers:[3,13],iter:[5,8],its:15,jackknif:15,julia:[],jungl:8,jupyt:11,kera:2,kernel:[6,9],lagrangian:6,lasso:3,later:3,layer:[2,10],learn:[1,2,5,9,12],least:3,level:8,librari:12,likelihood:5,limit:[2,5,15],linear:[1,6,13],link:[3,9,14,17],logist:5,loss:5,machin:[1,5,6,12],main:15,make:[1,7,8],mani:[8,10],materi:14,math:3,mathemat:[3,6],matric:3,matrix:[2,3,5,9,10,13,15],matter:1,mean:[0,15],meet:[3,8,15],mercer:6,mersenn:15,method:[4,5,7,8,15],mlp:10,model:[1,2,10],moment:15,moon:[6,7],more:[0,5,13],multilay:10,multipl:2,multipli:6,name:15,need:[],network:[2,5,10],neural:[2,10],newton:5,non:6,normal:[1,2,15],norwai:12,notat:10,novemb:14,now:[2,7],nuclear:1,nueral:5,numba:[],number:[1,15],numer:15,numpi:13,numpython:0,observ:15,obtain:9,octob:14,one:[5,10],optim:[2,5,6,12],ordinari:3,organ:1,oslo:[12,17],other:[7,9,10,15],our:[0,1,3,5,9,15],outcom:12,output:15,overarch:[1,6,7],overview:8,own:[0,1,8,9],packag:13,panda:[],part:[5,12],pass:2,pca:9,pdf:15,perceptron:10,perform:[2,7],period:15,perspect:2,poisson:15,pre:2,preprocess:1,prerequisit:12,princip:9,pro:7,probabl:[3,15],problem:[2,5],procedur:7,process:2,program:5,propag:[2,10],properti:[3,15],pseudo:15,python:[0,1,7,12,13,15],quick:6,ran0:15,random:[8,9,15],raphson:5,read:7,recip:15,recurr:10,reduc:1,regress:[1,3,5,7,8],regular:2,relev:17,relu:2,remind:[4,5,6],requir:12,resampl:4,revisit:5,ridg:[3,5],rng:15,rule:10,sampl:[9,15],schedul:14,schemat:7,scikit:[1,2,5,9],select:15,semest:16,sensit:5,septemb:[5,14],set:[1,7,10],sgd:5,should:[2,15],simpl:[1,5,7,15],singl:8,singular:[3,9],situat:15,size:[],slightli:5,soft:6,softmax:2,softwar:[],solv:5,some:5,split:1,squar:[3,8],standard:[5,15],state:1,statist:[3,4,12,15],steepest:[5,8],step:[4,5],stk3155:12,stochast:[5,15],stop:5,supervis:2,support:6,svd:3,teach:[14,16],teacher:16,techniqu:9,technolog:12,tensorflow:2,test:[1,2],textbook:17,than:5,theorem:[3,6,9,10,15],theori:15,three:15,togeth:10,top:2,toss:15,toward:[0,9],tradeoff:4,train:[1,2],tree:[7,8],tune:2,two:[6,12],type:10,uncorrel:15,understand:[],uniform:15,univers:[10,12,17],use:[2,15],used:5,using:[1,5,15],valid:4,valu:[3,9,15],variabl:[5,15],varianc:[4,15],variou:[1,4],vector:[6,10,13],view:[1,8],visual:[2,7],wai:7,week:14,weekli:14,what:[1,15],when:5,which:[2,15],why:15,wisconsin:5,write:[0,9],xgboost:8,your:[1,8]}})
\ No newline at end of file
+Search.setIndex({docnames:["Clustering","chapter1","chapter10","chapter2","chapter3","chapter4","chapter5","chapter6","chapter7","chapter8","chapter9","content","intro","linalg","schedule","statistics","teachers","textbooks"],envversion:{"sphinx.domains.c":2,"sphinx.domains.changeset":1,"sphinx.domains.citation":1,"sphinx.domains.cpp":3,"sphinx.domains.index":1,"sphinx.domains.javascript":2,"sphinx.domains.math":2,"sphinx.domains.python":2,"sphinx.domains.rst":2,"sphinx.domains.std":1,"sphinx.ext.intersphinx":1,sphinx:56},filenames:["Clustering.ipynb","chapter1.ipynb","chapter10.ipynb","chapter2.ipynb","chapter3.ipynb","chapter4.ipynb","chapter5.ipynb","chapter6.ipynb","chapter7.ipynb","chapter8.ipynb","chapter9.ipynb","content.md","intro.md","linalg.ipynb","schedule.md","statistics.ipynb","teachers.md","textbooks.md"],objects:{},objnames:{},objtypes:{},terms:{"000":2,"000000":[1,3,9],"00000000e":3,"0001":2,"00015921913736794912":7,"00019998":3,"00024087":3,"00029012":3,"00034944":3,"00042089":3,"00050694":3,"00061058":3,"00073541":3,"00079968":3,"00088573":3,"00096314":3,"001":[2,3,5,6],"00106677":3,"00109273":[],"00115999":3,"00128479":3,"00139705":3,"00154733":3,"00168251":3,"00186347":3,"00200":6,"00202624":3,"00224413":3,"0024401":3,"00270244":3,"00293838":3,"0032542":3,"0032873138755776365":15,"00353823":3,"003704":1,"003717":1,"003759":1,"003774":1,"003788":1,"00391839":3,"004":3,"00426027":3,"00433417":9,"004456043243408203":0,"00445655":9,"00471782":3,"0049999999999999845":[],"004999999999999996":[],"004999999999999997":[],"0050000000000000044":1,"005000000000000007":[],"00512927":3,"0056799":3,"00617499":3,"00683748":3,"006968283076248407":1,"00727646693":1,"00728323e":[],"007315":1,"0074331":3,"007789":[],"007824":1,"007931713723267314":1,"00823002":3,"0086649156":1,"008675369724976777":3,"008885578722629236":[],"00890601":3,"00894639":3,"009154":1,"009163470508352228":3,"009735":9,"00973536":9,"009790":1,"009883615646716182":[],"009883615646716184":[],"009883615646716186":1,"00990475":3,"009911":1,"01057384067458835":[],"010679893512872646":[],"01076611":3,"0110":15,"011347":1,"01191824":3,"01210814993019007":[],"01295356":3,"0140617":[],"01433809":3,"01503674":[],"015037":[],"01514394":[],"015144":[],"01558197":3,"0169643":3,"0170259":3,"01724499":3,"018232":1,"01825571":3,"01867234e":15,"01873344":9,"01873869":3,"02073509":3,"021544346900318832":3,"02161783e":15,"021901":9,"02190139":9,"02252765":3,"02492265":3,"02493044e":[],"02568378":[],"025709":[1,9],"02625193":6,"02707227":3,"02730126581656065":15,"02881304":[],"029688":[],"02968834":[],"029733":1,"02994311":3,"03019554":[],"030196":[],"030670":[],"03067028":[],"03099776":3,"031":3,"03193199":3,"03251863":3,"03256632e":2,"03303359":[],"033037772005753835":[],"03308408":3,"034342":3,"0344707":3,"0359565":3,"03607832":[],"03652792":[],"036528":[],"03697069":[],"03707133":9,"03750367":13,"03787596148305236":1,"03791824e":15,"038300":[1,9],"03871832":[],"039039":3,"04205220e":15,"04260073e":[],"04315108":3,"04330858":[],"04346721":3,"04421672e":[],"044334":1,"044402":1,"04455272":[],"044553":[],"04529095":3,"0458":7,"04648335":3,"04683565":3,"04757618e":[],"04829457939163212":15,"051115":[],"051649":[1,9],"0517473":3,"052645":[],"053053":[],"0530533":[],"05318162":[],"053182":[],"05364854":6,"05396545e":[],"054187":[],"054276":[],"054674":[],"055":[],"055085":[],"05531364":3,"055314":3,"055676":1,"055744":[],"055774":[],"055831":[],"055887":[],"055916":[],"05614483":3,"056326":[],"056329":9,"056484":[],"057035":[],"057088709963637":1,"057320":[],"057332":[],"05755374":[],"057571":[],"057654":[],"057919":[],"05802766":1,"058178":[],"058247":3,"058282":[],"058719":3,"058805":[],"058892":[],"059225":3,"059245":[],"059344":9,"059406":[],"059455":[],"059531":[],"059571":[],"059680303635344434":3,"059765":3,"059806":[],"059865":[],"059908":[],"059994":[],"060064":9,"060070":9,"060308":[],"060335":3,"06041458e":[],"060520":[],"060919":[],"061164":[],"061176":[],"061255":[],"06126507e":15,"061330":[],"061601":9,"061652":[],"061774":[],"061842":[],"061855":[],"061896":[],"06200174":3,"06209126":[],"062188":[],"062327":[],"062376":[],"062639":[],"062659":[],"062660":9,"062731":[],"062822":[],"062829":[],"062879":[],"062885":[],"062923":9,"062979":[],"063055":[],"063097":[],"063122":[],"063178":9,"063190":3,"063324":9,"063395":[],"063415":[],"063523":3,"063526":[],"063615":[],"063670":[],"063727":[],"063759":[],"063763":[],"063880":3,"063882":[],"063925":3,"063982":[],"064000":[],"064013":3,"064075":3,"064078":3,"064101":3,"06412177e":15,"064156":3,"064257":3,"064273":3,"064304":[],"064354":[],"064357":3,"064393":3,"064410":[],"064449":3,"064472":[],"064591":3,"064647":3,"064648":9,"064686":[],"064689":3,"064783":9,"064795":3,"06492086e":[],"064929":[],"064958":3,"064964":[],"064982":[],"065":[],"065003":[],"065086":[],"065110":9,"065144":[],"065151":[],"065188":3,"065200":[],"065202":[],"065204":[],"065355":[],"065591":[],"065618":[],"065660":[],"065680":[],"065762":[],"065860":9,"065994":[],"066002":9,"06600226":9,"066030":9,"066047":[],"066074":9,"066092":[],"066116":[],"06619182206626131":[],"066249":[],"066272":[],"066294":[],"066312":9,"066389":[],"066403":9,"066487":[],"066629":[],"066683":9,"066705":3,"066761":[],"066788":[],"066835":[],"066837":[1,9],"066951":9,"066963":[],"067043":[],"067061":[],"067123":[],"067132":9,"067175":9,"067202":3,"06724062":3,"067355":3,"067440":9,"067504":[],"067531":[],"067577":[],"067601":[],"067610":[],"067667":9,"067685":9,"067711":3,"067718":3,"067769":[],"067777":[],"067794":3,"067844":[],"067865":[],"067889":3,"067892":[],"067985":[],"067990":[],"068056":3,"068073":[],"068165":3,"068185":[],"068213":3,"068244":3,"068276":[],"068293":3,"068351":9,"068385":3,"068427":3,"068460":[],"068491":3,"068514":[],"068593":3,"068640":3,"068654":[],"068667":3,"068673":3,"068686":3,"068697":9,"068717":3,"068818":9,"068821":3,"068828":3,"068838":9,"068840":[],"068873":3,"068896":3,"068920":[],"068928":3,"068945":3,"069005":[],"069028":[],"069071":3,"069081":[],"069098":[],"069195":9,"069243":9,"069282":[],"069369":[],"069413":9,"069496":3,"069503":[],"069510":[],"069516":[],"069528":[],"069542":3,"069584":1,"069635":[],"069641":9,"069775":9,"069814":[],"069848":[],"069853":3,"069859":[],"069872":[],"069890":[],"069902":[],"069988":9,"070043":1,"070103":[],"070129":[],"070131":[],"070138":[],"070166":3,"070182":[],"070188":[],"070254":[],"070337":9,"070456":[],"070473":3,"070517":[],"070548":9,"070653":[],"070762":9,"070764":3,"070791":9,"070812":[],"070815":[],"070867":9,"070889":[1,9],"070949":3,"070967":[],"071016":[],"071049":9,"07106781e":[],"071096":[],"07110274":13,"071156":3,"071248":[],"071268":[],"0713":1,"071313":[],"071325":[],"071359":3,"071435":9,"07145103":9,"071481":[],"071502":[],"071547":9,"071549":3,"07155335":[],"071576":[],"071579":[],"07159175":[],"07160764":3,"071660":9,"071667":[],"071681":[],"071684":[],"071714":3,"071761":[],"071855":[],"071887":[],"071917":[],"072012":3,"072094":[],"072198":[],"072246":3,"072279":[],"072424":[],"072468":3,"072471":[],"072555":9,"072589":[],"072598":[],"072620":[],"072654":9,"072661":3,"072710":9,"072826":[],"072879":9,"072910":[],"073":[],"073020":[],"073035":[],"073096":9,"073310":9,"073333":[],"073352":[],"073371":9,"073372":[],"073483":[],"073567":[],"073591":3,"073598":[1,9],"073602":[],"073607":3,"073656":9,"073681":3,"073695":3,"073765":9,"073851":[],"073902":[],"073915":1,"073926":3,"073966":[],"074":[],"074027":[],"074044":3,"074067":[1,9],"074077":3,"074096":[],"074099":[],"074134":[],"074170":[],"074191":[],"074201":9,"074207":[],"07421084":3,"074230":[],"07423848370736122":[],"074248":[],"074285":3,"074286":[],"074331":9,"074432":[],"074434":3,"074453":[],"074488":[],"07456491":3,"074568":[],"074618":9,"074665":[],"074693":3,"074696":[],"074848":[],"074979":9,"075084":[],"075119":[],"075163":[],"075194":9,"075212":[],"07521771":[],"075218":[],"075261":[],"075266":[],"075289":[],"075296":9,"075309":[],"075526":9,"075545":9,"0755452":9,"075562":[],"075567":3,"075615":9,"075754":[],"075772":[],"07581375":[],"075814":[],"075818":3,"075821":[],"075865":[],"075975":3,"075993":9,"076026":3,"076062":[],"076166":3,"076169":3,"076189":[],"076251":3,"076283":[],"076320":[],"076367":9,"076398":[],"076408":[],"076410":9,"076466":9,"076469":[],"076730":[],"076822":3,"076938":[1,9],"077010":9,"077046":9,"077143":9,"077160":[],"077212":[],"07729012413236423":9,"077303":[],"077305":9,"077338":3,"077339":3,"077386":[],"077407":3,"077416":3,"07744472306026946":7,"077469":[],"077533":[],"077623":[],"077771":9,"07777777777777778":2,"077891":[],"077927":9,"078049":[],"078076":[],"078099":9,"078103":9,"078197":3,"078244":9,"078279":[],"078311":[],"078388":[],"078399":[],"07839924":[],"078451":[],"078474":3,"078649":3,"078674":[],"078693":9,"078710":9,"078746":9,"078750":3,"078892":3,"078927":[],"078930":[],"078937":[],"078966":[],"079001":[],"079037":[],"079137":[],"079139":[],"079333":[],"079365":[],"07944154":13,"079624":[],"079785":9,"079848":9,"079956":[],"080024":9,"080084":1,"080115":[],"080182":3,"080223":[],"080264":9,"080325":9,"080336":[],"080381":[],"080387":[],"080398":[],"08043851":3,"080570":9,"080626":9,"080675":9,"080738":3,"080845":[],"080871":[],"081077":9,"081102":3,"081103":[],"081129":9,"081210":9,"081227":[],"081260":[],"081309":3,"081402":[],"081480":[],"081489":9,"081514":[],"08170444":[],"081816":[],"081886":[],"082091":3,"082211":9,"082225":9,"082247":9,"082272":3,"08248290e":[],"08251898":15,"082616":3,"082687":[],"08271336":[],"082866":[],"082990":[],"083128":[],"08318298e":2,"0832":[],"083246":3,"083295":9,"083376":[],"083527":1,"083577":[],"083618":[],"08362077210702115":3,"083657":[],"083658":9,"083728":[],"083775":[],"084092":[],"084103":3,"08415761":[],"084167":[],"084181":[],"084249":9,"084251":[],"084604":9,"084813":9,"084873":[],"084931":[],"08501147":[],"085121":9,"085259":3,"085285":[],"085289":[],"085597":[],"085646":9,"085676":[],"085928":[],"085951":[],"08611111111111111":2,"086116":[],"086325":[],"086465":[],"086592":[],"08665060086846632":[],"086652":9,"086974":9,"087202":9,"087482":[],"087563":9,"0881981":3,"08839750e":[],"088660":[],"08871404":3,"088871":[],"08888888888888889":2,"089300":[],"089425":9,"089504":[],"089916":[],"089931":[],"090274":9,"09103481e":15,"091159":[],"09166666666666666":2,"0917":7,"091979":[],"093755":9,"09487315108590391":9,"095173":[],"095472":[],"09609807":3,"096726":[],"09672604":[],"09678277e":15,"09741585e":[],"097452":[],"09903804":6,"099763":[],"09976319":[],"100":[0,1,2,3,4,5,6,7,8,9,13,15,16],"1000":[0,1,2,5,6,9,12,15],"10000":[0,3,4,8,9,15],"100000":6,"10001":8,"1001":15,"1002":15,"1003":15,"1005":15,"1007125":[],"1009":15,"10094646e":[],"1011":15,"1013":15,"10131725":[],"1013904243":15,"1015":15,"102":1,"1022964509394572":2,"1023":15,"1026":15,"1027":15,"103":2,"1030":15,"10307631":[],"10327559":13,"1037":15,"10378326e":2,"1038":15,"1040":15,"10405456":9,"104411":1,"1047":15,"105137868830763":15,"10555555555555556":2,"10589577":3,"106095":[1,9],"108":1,"10806972":[],"108070":[],"10896672e":15,"10898112e":[],"10th":7,"10x":1,"110":1,"1100":15,"1101":15,"11022302e":3,"111":[2,5,10],"1111111111111111":2,"112283":1,"112383":[1,9],"11304709e":15,"11388888888888889":2,"11456076e":[],"11462415":3,"11507992e":2,"11666666666666667":2,"117":6,"118318":1,"11944444444444445":2,"12002944":[],"1203284":6,"121":[6,7,8],"122":[6,7,8],"12222222222222222":2,"12283463":[],"123190":1,"123459876":15,"12366979e":15,"124":1,"12497018e":[],"12618549":3,"127773":15,"12777777777777777":2,"1298":7,"12adb44b1c20":5,"13055555555555556":2,"13060772":[],"13162821e":[],"13209041":[],"13266452e":[],"133":5,"13328820e":[],"13457922":[],"13519106":1,"13579199e":[],"1361111111111111":2,"13646574":3,"136687":4,"137268":[],"137400784702912":1,"137652":[1,9],"13803928":[],"13865173":3,"138775":[1,9],"13891144e":[],"139462":[],"139475":[],"1404":1,"143":5,"1437":2,"1440501043841336":2,"1445":4,"14459063":[],"1446":4,"1447":4,"1448":4,"1449":4,"14662074":[],"14722222222222223":2,"147400":[1,9],"147420":[1,9],"147722":[],"148":[],"149076":4,"149366":[],"149667":[],"149832":4,"149894":[],"149903":[],"14g":4,"150":6,"15005476":3,"15024669":[],"151":[],"152636":[],"1527777777777778":2,"153036":1,"153106":1,"15324631":[],"15332528e":[],"154720":1,"15475232":[],"154911":1,"155491":1,"155687":1,"155883":1,"156":1,"157":1,"15751596":3,"158":1,"159":1,"15979239e":15,"15g":4,"160":1,"16111111111111112":2,"16211139":3,"16342407":3,"16496581e":[],"16553696":[],"16637855e":15,"16666666666666666":2,"16807":15,"16831012":3,"16861772":[],"168618":[],"16b8e3cda33a":2,"17117385":[],"17174962e":2,"17234827e":[],"17248441e":[],"17385778e":[],"174497":[],"17449733":[],"175300":1,"1758506":[],"17654307":[],"17707436":[],"17777777777777778":2,"17801022":3,"17917768":3,"17949575":3,"17953942":9,"1797":2,"180092462880674":[],"18029127":3,"18063928":3,"18220995":1,"18224665":3,"18333333333333332":2,"18404906e":[],"184519":1,"1856411":[],"18611111111111112":2,"18620601e":[],"18673098":9,"189496":1,"18954529":[],"189622":1,"18968431e":15,"1914224774238273":[],"19207979":3,"1940":1,"1943":10,"197":1,"1970":13,"1973":7,"197370":[1,9],"19742904e":[],"1979":4,"1989":15,"19937":15,"19955871":[],"199559":[],"19972087e":15,"1_1":10,"1_2":10,"1_3":10,"1cm":[1,6,8,15],"1e10":0,"200":[1,6,7,8,15],"2000":1,"200000":1,"2004":5,"2006":17,"2010":2,"2011":2,"2015":2,"2016":1,"2018":[1,4],"201841":[],"20184113":[],"2021":0,"20277777777777778":2,"203757":[],"20375701":[],"205466494327873":[],"20632410e":[],"207545":[1,9],"20772452":[],"20819609e":[],"20833333333333334":2,"20843563e":[],"20906175e":15,"210340":[1,9],"21058097":3,"212":[],"21208310e":[],"212327334149492":[],"2125":[],"2126":[],"2127":[],"2128":[],"2129":[],"213":[],"213103":[1,9],"21347282":[],"213743":[1,9],"214":[],"2147483647":15,"215":[],"216":[],"21623344":[],"216290":[1,9],"216683":[1,9],"22044605e":3,"22076362":[],"221":6,"221180":1,"22209371e":[],"222400":1,"22443803":[],"22527008e":15,"22690428":3,"22717936e":15,"22847924":3,"22999784":[],"229998":[],"23117916e":15,"23167717":3,"232435":1,"23333333333333334":2,"2335879":[],"23382086e":[],"23438810e":[],"241262":3,"24126232":3,"2433e":3,"24340751":3,"24444444444444444":2,"24829908":3,"250":[5,7],"25000":1,"250000":1,"250154":1,"251559":[],"2515593":[],"251879":1,"25190225e":[],"25226753e":15,"25240108":[],"252436":1,"253":[],"25303483":[],"253775":1,"254":[],"254509":[],"25450941":[],"25457052":[],"255":[],"255001":1,"2551":1,"256":[],"256962":1,"257":[],"2571699":1,"25726439e":[],"2572e3a4b38d":2,"25803281":[],"259107":[],"25910749":[],"259153":[1,9],"259815":3,"25981533":3,"260227742627244":3,"2627588":13,"26301436":3,"26381865":[],"263819":[],"264":1,"26412372":3,"265":1,"2653678":[],"266":1,"267":1,"26710969":3,"26780278":3,"2683":4,"2684":4,"2685":4,"2686":4,"2687":4,"26890510e":[],"269":1,"26974938e":[],"270":1,"27262964":1,"27296891e":[],"276263":[1,9],"27753165e":15,"27852549":3,"27924636":3,"27n_":15,"280647":[1,9],"28166741":[],"282":1,"282727":[1,9],"28310983":[],"283110":[],"2836":15,"28475098":6,"2861":15,"2873":7,"2882":15,"2886":15,"2890":1,"2892":15,"28971976":9,"289720":9,"290":[],"291":[],"2915":15,"291614":[],"29167186":3,"292":[],"293":[],"2931":[],"294":[],"295656121491569":[],"296247":1,"2968":[],"297219777724628":9,"297260":1,"2980":[],"298273":1,"2983233":13,"298375":1,"2990":[],"299444":9,"29944428":9,"299748":1,"2_1":10,"2_2":10,"2_3":10,"2_i":10,"2_m":15,"2_x":15,"2cm":6,"2ff97f4bf03b":15,"2nd":7,"2x_ix_jy_iy_j":6,"2x_j":6,"2y_i":8,"2y_j":6,"30000":1,"300162456113691":[],"30119421":6,"30626706":1,"306854":[],"30685416":[],"30879705":1,"30968181":[],"31109204e":[],"31318084":3,"31457796":3,"3155":[3,4],"3156929654100207":7,"31608475e":[],"31718909":9,"31730641":[],"317367":9,"31853484":[],"31896852":6,"3200":2,"32047562":1,"323291478597321":15,"3250":2,"32521615e":15,"326238":1,"327631":1,"32938847":13,"32945844e":15,"3304":1,"33066907e":3,"3310":1,"3317":1,"331939":1,"333":5,"333333":1,"3338":1,"3344":1,"33443859e":[],"33544681":[],"33569998e":[],"33861512":[],"338869":[],"33886902":[],"339535706819584":15,"33953571":15,"340782":[1,9],"34114547":3,"34114641e":[],"34172919":[],"342680":1,"3436":1,"3437":1,"344172":1,"34569596":3,"346433":9,"34643337":9,"34902789e":15,"350387":[],"35038714":[],"351636":[1,9],"35176067":[],"35182854":3,"35216172e":[],"35367281e":[],"356399":1,"357508":1,"358869339268145":15,"35886934":15,"359":3,"359640894899012":[],"360":2,"360688":1,"361556":1,"3621311":3,"36436520e":[],"36468301":[],"3655222":3,"369139":[1,9],"36941772":[],"369418":[],"37186301":3,"372889":[],"3728892":[],"37416969":9,"374170":9,"37738324":[],"38207279e":[],"38216436":[],"38629436":13,"38777878e":3,"3893239":3,"389324":3,"38937995e":15,"38986237":[],"39579407":3,"396740":9,"39674043":9,"39706038":3,"397700":[1,9],"39792608e":[],"399836":1,"3cd19a0768e1":[],"4000":17,"401842":[1,9],"40212127":[],"404":1,"40425078e":15,"405890":[1,9],"40702":3,"40708470e":[],"40859":[],"4087793":3,"40902095":[],"412374":3,"41237437":3,"4140e":3,"41433969":3,"41511965e":2,"415634483874318":[],"416694683938511":9,"4171578884124756":0,"41754964":[],"41770932":[],"418506":[1,9],"41876428e":[],"4200e":3,"4201e":3,"4203e":3,"42441033":3,"426":5,"42847770e":15,"42937310e":15,"43054282":3,"4332e":3,"435163":1,"435490":[],"43549028":[],"43766686":9,"438136":1,"44089210e":3,"442600":[1,9],"443217":1,"44395541":[],"44625466e":15,"44970586e":2,"45013332e":[],"45019484":[],"450257":[1,9],"4557763":9,"455947":1,"458027":1,"458078":[1,9],"45937170e":[],"45960079":3,"46016532e":[],"461":15,"461838":3,"46183815":3,"462":5,"46323168e":15,"4632e":3,"46415888336127775":3,"46423858e":[],"465564":[],"46556436":[],"466":15,"46675058":[],"46914544e":[],"46929603e":[],"469730":3,"46973015":3,"47042744":3,"470714":1,"47079457e":15,"47125748":3,"47132891":3,"472652":3,"47265243":3,"47441766":[],"47566390e":15,"47654764e":[],"47815203":9,"47862383":3,"478624":3,"48019541":3,"48154187202453613":0,"48257387":16,"48471852e":[],"48476997":9,"48608063e":[],"48994188":3,"491837":3,"4918372":3,"4940954":1,"497221":[],"49722108":[],"49865673":[],"4990":15,"4992":15,"4997":15,"4c4c7f":[7,8],"4y_i":8,"500":[2,4,5,7,8,15],"500000":1,"50000455":3,"50000553":3,"50000718":3,"50000855":3,"50000969":3,"50001063":3,"50001142":3,"50001207":3,"50001261":3,"50001306":3,"50001343":3,"50001374":3,"500014":3,"50001414":3,"50001422":3,"50001439":3,"50001454":3,"50001466":3,"50001476":3,"50001485":3,"50001492":3,"50001498":3,"50001502":3,"50001506":3,"5000151":3,"50001512":3,"50001515":3,"50001517":3,"50001518":3,"50001519":3,"50001521":3,"50001522":3,"50001523":3,"50001524":3,"50001525":3,"5018":15,"50321091":3,"50394742":[],"506":1,"507d50":[7,8],"5098779":[],"509878":[],"50j":5,"50x10":2,"510":2,"5120":0,"512132":1,"51257863e":[],"51345668e":[],"51363731e":[],"514219":1,"51523276e":[],"515768":[],"51576807":[],"51707172":3,"517072":3,"51893804e":[],"51943726":9,"519842":1,"52158335":[],"5222222222222223":2,"52570079":3,"525701":3,"526744":[1,9],"52687171":[],"526872":[],"52722156":[],"52874252":3,"52d2c51caad1":[],"5303329":9,"5305555555555556":2,"531280":1,"535457":[],"53545715":[],"5378811":9,"539261":[1,9],"54039921":3,"54096582":[],"540966":[],"54121682":[],"541217":[],"54152940e":[],"541605":1,"54237024":15,"544439":1,"546972":3,"54697204":3,"54702088e":[],"55111512e":3,"55138385":9,"551384":9,"55280484":[],"5555555555555556":2,"55707065":[],"557071":[],"557795":[1,9],"55795935":3,"55854694":9,"56033697":3,"56198284":3,"56216797e":[],"564374":[1,9],"56536":1,"569":2,"56939714":3,"57051369":[],"571":3,"574465":[1,9],"5755035":[],"57781668":[],"57871326":[],"581766":9,"58176612":9,"582":[1,2],"58228342e":15,"58239999":[],"582400":[],"583595":1,"584804":1,"58521266":9,"585213":9,"58596975":[],"585970":[],"587401":1,"58836420e":15,"5888888888888889":2,"59007674e":15,"59304755e":[],"5944444444444444":2,"59480085":[],"5cm":15,"60122668e":15,"60293962":3,"60394236e":[],"60420593":3,"60673226":9,"60949193":[],"609492":[],"60999846":[],"61069091e":15,"6111111111111112":2,"61124978":[],"61234223":[],"612939":1,"613579":1,"614808":[1,9],"61505887e":[],"61745046e":[],"618":1,"6183694":[],"61869821":[],"618982":1,"622539":9,"62253933":9,"622625":3,"62262506":3,"6226921":[],"62316154e":15,"62359224e":15,"62373464":9,"625":5,"62783293":[],"62856593":[],"62894215":3,"629961":1,"6300745149331701":1,"63315151":[],"63339159":[],"63374631":[],"63437572":[],"63442451e":[],"63498144":3,"636323":9,"63632311":9,"63680118":[],"637129335071195":1,"63993205e":[],"64012627":3,"64166831e":[],"64447921":[],"64580686":[],"646283":[1,9],"64669382":[],"646694":[],"647473":[1,9],"64857826e":[],"649382":[1,9],"64x50":2,"650024":[],"65002433":[],"6510573774179256":15,"65105738":15,"65238878":13,"65245958":[],"653095390463358":[],"653702":13,"6544e":3,"65482578":[],"65572035":[],"65599927":[],"65766387":[],"65885453":3,"65933852":[],"65939208e":[],"66020213e":[],"6614":[],"66183486":9,"661835":9,"6628996975186952":1,"66302359":[],"66383151":[],"66677842":[],"66800261":[],"668172":1,"66880047":[],"67006792":[],"67060602":[],"671089":1,"6714":[],"67171347e":15,"672721":1,"67279536":[],"67303655":9,"673037":9,"67407338e":15,"67450955":[],"67708423":[],"677149":[],"67714918":[],"6796265324852733":[],"68002363":[],"680024":[],"68034946e":[],"6813":4,"6814":4,"6815":4,"6816":4,"6817":4,"6818813252071303":15,"68188133":15,"68192193":3,"68316185":[],"68342382":[],"68542204":3,"68616263":13,"68729414":[],"68937695":[],"689519":[1,9],"690617":1,"69069n_":15,"693361":1,"69347005":[],"693850":9,"69385025":9,"69519693":[],"695197":[],"69573183":[],"69695259":3,"69843037":3,"6996584":3,"69981195e":15,"6999536":9,"6ea927cc6e88":2,"6n_":15,"7022283":[],"70234019":[],"70415861":[],"70523024e":[],"70589906":[],"7070e":3,"70710678":3,"70790937":[],"70832814":3,"70885528e":[],"70886748":[],"70946493e":[],"712018":[1,9],"712199063818309":1,"71281409":[],"71351486":[],"71442781":13,"71606852":[],"7162":[],"71669651":[],"7172":[],"72108703":[],"72174172":9,"72218808":[],"72312577":[],"7236674":3,"7240496":[],"725394195434945":[],"72780613e":15,"72859758":3,"72879865e":[],"72981762":6,"73091052e":[],"731000":1,"733096":1,"73453972":[],"7371165871823337":[],"737349":3,"73734906":3,"73921714":[],"74081822":6,"74107697":9,"74143127e":[],"7432283":[],"74382593":[],"74495014":[],"745136489050356":1,"74840212":3,"74845978":[],"74921867":[],"749765":1,"750445":1,"751699":[1,9],"75170092":3,"75195757":[],"7522047280566193":[],"75382481":[],"75524378":[],"75629493":[],"756352":1,"75841112":[],"75932862":[],"76172241e":[],"762":[5,9],"7621419":[],"76290332":[],"763880":[],"76388013":[],"76497666":[],"765":5,"76504618":[],"76570177":[],"76648901e":[],"76936315":3,"7693978131030923":9,"77152076":3,"77156117e":3,"7718":7,"772b904ae9cb":4,"77317984":[],"7733":[],"77350269e":[],"774300":1,"776223":3,"77622336":3,"77661393e":15,"77714169":6,"78080633":[],"78082":4,"78195":[],"78857629":[],"78941903":3,"78944806":[],"7899453":[],"79111643":3,"79295029e":15,"793167":1,"79326583e":15,"79328828":[],"793701":1,"794282":[1,9],"79459035":[],"79648291":[],"79902342":1,"7c394b1e8b71":7,"7d7d58":[7,8],"800":5,"80004454e":[],"800266":[],"80026635":[],"80121":[],"80469739":3,"8055555555555556":2,"81160425":3,"81620806":[],"81633628":9,"816454":1,"816847":1,"81781888":9,"819202":3,"81920231":3,"82198978":3,"8265786":3,"827265":1,"82889306e":15,"8305555555555556":2,"83425361":9,"834254":9,"83512277":3,"83614019":[],"8388888888888889":2,"83935285":[],"84087101":[],"842":1,"842436":1,"84251485":[],"842515":[],"8429949116841184":3,"84355903e":2,"84372853":[],"84443254e":2,"84658093e":15,"8479552268981934":0,"84860939":1,"849766":3,"84976606":3,"84994524":3,"85058354":3,"850584":3,"8520127":[],"85278450e":[],"85396354":[],"85450859":9,"85463934e":[],"85497163e":[],"85546305e":[],"85601992":3,"861":1,"86117291":3,"86134827":3,"86145244":9,"8638888888888889":2,"86436607":[],"86574276":[],"8666666666666667":2,"86692943":[],"87030083":3,"870301":3,"8718475896381779":15,"87184759":15,"8722222222222222":2,"87381451":3,"875":2,"87761937":[],"8777777777777778":2,"87972591":[],"8802":[],"88046261":3,"88046462":[],"8805555555555555":2,"881323":3,"88297395":[],"88336879":3,"88559559":[],"8867467323038865":[],"88693966e":15,"88712946":[],"8888888888888888":2,"8901":[],"8914984":[],"8921171964770647":9,"8923":[],"89288636":9,"89383322":[],"89410423":3,"8944444444444445":2,"89704131":3,"89742056":[],"897421":[],"898500":9,"89850037":9,"89928965":3,"89942598":[],"89975818":[],"8x8":2,"90075537":3,"90220243":3,"90233874":[],"90266948":3,"90316476":[],"9040":7,"9055555555555556":2,"90618734e":15,"907307":9,"90730735":9,"90825063":[],"9096":4,"91012519e":[],"9111111111111111":2,"91128596":3,"91145266e":15,"91417278":[],"91549644":1,"9166666666666666":2,"916978":[],"91697817":[],"9171356":[],"91760278":3,"91812702":3,"918992":1,"9196":4,"9222222222222223":2,"922312":[],"92231203":[],"92280994":3,"924018":1,"925":2,"92507116e":2,"92578916":3,"92605247":[],"92645039":9,"92646965":9,"926470":9,"92732185e":[],"9277777777777778":2,"9279671770201344":15,"92857143":5,"92919670e":15,"9305555555555556":2,"931":1,"93100040e":15,"93155188":3,"93158979":3,"932734":[],"93273404":[],"933":3,"9354":[],"9361111111111111":2,"937":15,"937082":1,"93799826":3,"938":15,"9388888888888889":2,"939":[1,15],"94019247e":[],"94034531":[],"94284104":3,"94320205":3,"9444444444444444":2,"94536341":15,"94591015":13,"94639099":9,"946893955211749":1,"947543":9,"9482527":3,"948729":[],"95014575":[],"95231424":3,"9527777777777777":2,"95284275":3,"95351665":3,"954":15,"95443703e":[],"9547578478889096":1,"95517094":[],"955171":[],"9555555555555556":2,"95628168":[],"956282":[],"956563":[1,9],"95684892":3,"958228616652075":3,"9583333333333334":2,"959247":[],"960":15,"96024953":3,"96084663":3,"961":15,"96104648":[],"9611111111111111":2,"962":15,"963499":9,"96349948":9,"9640435":3,"965548":1,"965885569080809":[],"96688672":3,"9674916":3,"967809":[1,9],"96793117e":[],"97005689":3,"97101567e":15,"9722222222222222":2,"97243128":3,"97300836":3,"975":2,"97507735":3,"9756404":[],"976":6,"97622676":[],"976227":[],"9765":4,"9769":4,"97705827":3,"97723801":[],"97758848":3,"9777777777777777":2,"9780387310732":17,"9780387848570":17,"9781492032632":17,"97926491":3,"9804422":[],"98046438":[],"9805555555555555":2,"98091621":3,"981321":1,"98139097":3,"98275501":3,"98314755":[],"983148":[],"983310":1,"98413059":3,"98452685":[],"98454786":3,"985":15,"98526763":[],"98528992":[],"98531221":[],"98566191":3,"986":15,"9860803":[],"98609175":15,"986091753050161":15,"9861111111111112":2,"98661465":[],"98680716":3,"98716878":3,"9877742":[],"987902":[],"98808176":3,"988663":[],"9888888888888889":2,"98892195e":15,"98893512":[],"989":15,"9890348":3,"98927731":[],"9893447":3,"9898ff":[7,8],"99009525":3,"99009739":[],"9901168":[],"99013921":[],"99016161":[],"99018401":[],"99043999":[],"99088801":3,"9909252":[],"991":15,"991072":9,"99107239":9,"99115119":3,"99126104":[],"99133007":[],"99160404":[],"99176998":3,"99190487":[],"99194716":[],"992":15,"99218987":[],"99219378":[],"99242605":[],"99242921":3,"99248001":[],"99265097":3,"99273355":[],"99276945":[],"993":15,"99305549":[],"99305802":[],"99311297":[],"99316252":3,"99346398":[],"99363129":[],"99363383":[],"99371056":3,"99389612":3,"99393624":[],"994":4,"99400444":[],"99418903":9,"99420743":[],"99420997":[],"99428016":[],"9943201":3,"9945452":[],"99462421":[],"99473581":[],"9947756":3,"99478645":[],"99478898":[],"99492986":3,"99498985":[],"995":4,"99501236":[],"99503487":[],"99505739":[],"9950799":[],"99527696":[],"99528218":3,"9953048353087299":[],"99536326":[],"9953658":[],"99539415":3,"99544872":[],"9954538761021741":[],"99566069":3,"99578809":3,"99579317":[],"99581841":[],"99594294":[],"996":[3,4],"99600927":[],"99608161":3,"99636015":[],"9963961":3,"99650061":3,"99652042":[],"99652296":[],"99655111":[],"9966744029509663":1,"9967292151090247":1,"9967458":3,"9969":4,"99696351":[],"997":4,"99700706":3,"99709215":3,"99709325":[],"99710078":[],"99723611":[],"99728435":[],"99729756":3,"99730848":[],"99751458":3,"99758326":3,"99763569":[],"99767893":[],"99775587":3,"99782689":[],"99793613":3,"99799099":3,"998":4,"99813653":3,"99817842":[],"99825997":[],"99828624":3,"9983295":3,"99836973":[],"99845267":3,"9984806":[],"998577":3,"99861053":3,"99871521":3,"99881845":3,"99883879":[],"99884384":3,"99891285":[],"99893323":3,"999":[7,15],"99901896":3,"99903755":3,"99911427":3,"99918546":3,"99919837":3,"99926459":3,"9993237":3,"99933188":3,"99938942":3,"9994385":3,"99944272":3,"99945628":[],"99949306":3,"99953381":3,"99953475":3,"99957911":3,"99961294":3,"99965056":3,"99967865":3,"99970988":3,"9997332":3,"99975913":3,"99977849":3,"99980002":3,"9998161":3,"99984732":3,"99987324":3,"99988687":[],"99989476":3,"9999095":[],"99991263":3,"99992746":3,"99993212":[],"99993978":3,"99995":3,"99995475":[],"99995818594196":[],"999974281880048":[],"99997737":[],"9999787681537219":1,"9999794306626945":[],"9999822527140678":[],"9999864543345858":[],"9999868619217517":[],"9999869956119286":1,"9999878260589065":[],"9999910208315801":[],"9b9cf4fa1a95":[],"\u00f8yvind":16,"abstract":2,"break":[0,1,9],"byte":13,"case":[1,2,3,4,9,10,12,13,14],"catch":1,"char":15,"class":[1,2,4,5,6,7,9,10,15],"const":15,"default":[1,2,5,13],"ekstr\u00f8m":16,"export":7,"f\u00f8470":16,"final":[0,1,2,3,4,6,7,8,9,14,15,16],"float":[0,1,3,7,9,13,15],"function":[0,3,4,7,12,13],"import":[0,1,2,4,5,6,7,8,9,10],"int":[0,1,2,3,5,9,13,15],"long":[1,2,5,10,15],"new":[0,1,2,3,4,5,6,7,8,9,13,15],"null":15,"public":[1,12],"return":[0,1,2,3,4,5,6,7,9,13,15],"s\u00f8rli":16,"sch\u00f8yen":16,"short":[0,3,11],"steinsv\u00e5g":16,"super":3,"switch":[0,1],"throw":15,"true":[0,1,2,3,4,5,6,7,8,10,15],"try":[0,1,2,3,5,6,7,8,9,12,13,15],"var":[3,4,8,9,15],"while":[0,1,2,3,4,5,6,7,9,10,15],AGE:1,Adding:2,Age:5,And:[0,1,4,7,12,15],Are:9,Being:5,But:[0,1,2,3,4,7,8,15],CAS:[],DIS:1,Doing:[3,5,8],EoS:[1,4],FYS:14,For:[0,1,2,3,4,5,6,7,8,9,10,12,13,14,15,17],Going:2,Ising:[3,10],Its:[2,9],MDS:9,NNs:10,N_s:6,Not:[1,2,3,4],OLS:[1,4],One:[1,2,3,4,5,6,9,10,15],PCs:[9,12],RMS:15,Such:[4,10,15],That:[0,1,3,5,8,9,10,15],The:[8,13,14,16,17],Then:[0,1,2,3,4,5,6,7,8,9,10,13,15],There:[0,1,3,6,7,9,10,11,13,14,15,16],These:[0,1,3,6,7,8,9,10,13,15],Use:[1,7],Useful:[3,4,13],Using:[1,3,4,6,8,10,13],With:[1,3,4,6,7,8,9,10,13,15],__class__:8,__doc__:4,__future__:[6,7],__getattr__:[],__init__:[2,4],__mosek:3,__name__:[4,8],_auto10:10,_auto1:[3,5,10,13,15],_auto2:[10,13,15],_auto3:[10,13],_auto4:[10,13],_auto5:[10,13],_auto6:[10,13],_auto7:[10,13],_auto8:10,_auto9:10,_ax:4,_base:6,_build:[12,17],_check_optimize_result:[5,9],_compon:9,_datafram:[],_depth:7,_fraction:7,_lambda:1,_leaf:7,_logist:[5,9],_make_index:[],_multilayer_perceptron:[1,2],_node:7,_num_sampl:[],_ratio:9,_sampl:7,_split:7,_varianc:9,_weight:7,a0faa0:[7,8],a77d5ac269b2:[],a_0:1,a_1a:1,a_2a:1,a_3:1,a_3a:1,a_4:1,a_4a:1,a_h:2,a_i:[1,2,10],a_j:[2,10],a_k:[2,10],aaron:17,ab_channel:12,abandon:2,abbrevi:14,abid:15,abil:[1,8],abl:[2,3,5,8,10,15],abort:15,about:[0,1,2,3,4,5,6,7,8,9,10,12,13,17],abov:[0,1,2,3,4,5,6,7,8,9,10,13,15],abovement:4,abs:[0,1,3],abscissa:5,absolut:[1,3,4],acccess:[],accept:[1,7],access:[1,9,15],accid:4,accmod:3,accompani:1,accomplish:[6,7],accord:[0,1,2,3,4,5,7,10,15],accordingli:9,account:[1,3,15],accumul:[10,15],accur:[4,8,15],accuraci:[1,2,3,5,7,8,9,10],accuracy_scor:[1,2,8],accuracy_score_numpi:2,achiev:[1,2,3,4,6,10,13],aco:15,acquaint:12,acquir:[2,12],acr:1,across:[2,4,7,12],act:[2,13],action:15,activ:[0,1,7,14],actual:[1,2,3,4,6,9,13],ada_clf:8,adaboostclassifi:8,adam:2,adapt:[1,4,5,17],add:[1,2,3,4,6,8,9,10,15],add_subplot:[0,2,5,10],added:[1,2,3,5,6,13],addendum:3,adding:[0,2,4,13],addit:[0,1,3,4,5,6,7,8,10,12,13,15,16,17],addition:[5,10],address:[2,5,7,9,17],adjac:10,adjoint:3,adjust:[5,10],admir:1,advanc:[4,10,17],advantag:[2,3,4,5,8,13],afecionado:[],affect:[],affin:[1,6,9],aficionado:[],aforement:0,african:1,after:[0,1,2,3,4,5,7,9,10,12,13,15],afterward:1,again:[0,1,2,3,4,5,6,8,9,10,15],against:[2,5,8],age:[1,5],agegroup:5,agegroupmean:5,aggreg:[7,8],agorithm:8,agre:[3,15],ahead:7,aid:9,aim:[0,1,2,4,5,9,12,13],ainv:3,aka:3,alarm:3,albeit:0,algebra:[1,3,5,12,14],algo:15,algorithm:[1,2,3,4,5,6,12,13,14,15,17],align:[1,3,4,5,6,15],all:[0,1,2,3,4,5,7,8,9,10,12,13,14,15,16,17],allevi:[2,5],alloc:13,allow:[1,2,3,4,5,6,8,12,13],almost:[1,2,4,5,6,9,15],alon:7,along:[0,3,4,7,8,9,12,13],alpha:[0,1,2,3,4,5,6,7,8,15],alpha_:8,alpha_i:5,alpha_k:5,alpha_m:8,alpha_opt:5,alreadi:[3,8,10,12,13,15],also:[0,1,2,3,4,5,6,7,8,9,10,12,13,14,15,17],alter:[2,15],altern:[1,2,3,5,6,7,9,13],although:[2,3,4,6,8],alwai:[0,1,3,4,5,10,15],ame2016:1,american:1,among:[1,3,7,8,10],amongst:3,amount:[0,1,2,4,6,8,12],an_:15,anaconda3:[1,2,4,5,6,9],anaconda:[1,2,12],analys:[4,15],analysi:[2,4,5,13,14,17],analyt:[1,3,4,5,10,12],analyz:[1,2,3,15],andrew:2,angl:[1,7,15],ani:[0,1,2,3,4,6,7,8,10,15],anim:10,ann:10,annot:[1,2,5,6],anoth:[1,2,3,4,5,6,8,9,10,13,15],ans:15,ansatz:1,answer:[1,2,3,4,13],anymor:[2,6],anyon:6,anyth:[2,15],anytim:16,apach:2,apart:[5,9],api:[2,12],appear:[1,2,13,15],append:[2,4,5,6,7,15],appendcon:3,appendvar:3,appli:[1,2,4,5,6,7,8,9,10,15,17],applic:[1,2,3,4,5,7,10,15,17],approach:[2,3,4,7,8,9,10,12,15,17],appropri:[4,7,10,12,15],approx:[1,5,8,9,15],approxim:[1,2,3,4,5,8,9,15],apt:[1,12],aptli:0,aragorn:[],arang:[2,4,5,7,8,10],arbitrari:[2,5,6,10,15],arbitrarili:[1,2,9],architectur:[10,17],area:[1,17],arg:4,argc:15,argmax:[2,9],argmin:[0,8],argsort:9,argu:2,arguabl:0,argument:[1,3,9,10,15],argv:15,aris:[1,4,5,10,15],arithmet:[1,13],arma:15,armadillo:[13,15],around:[1,2,3,4,9,15],arrai:[0,1,2,3,4,5,6,7,9,10,12,15],arriv:[1,7,9,13,15],arrow:10,arrowprop:6,art:[1,2,12],articl:[0,1,4,8,15],artifici:[1,5,10,17],artificialneuron:10,artist:4,asarrai:[1,7],asc:3,ascii:15,ask:[3,4,9,10],aspect:[1,12],assembl:1,assess:[1,4],assign:[0,1,5,6,7,10,14,17],assign_points_to_clust:0,associ:[0,1,4,7,10,15],assum:[0,1,2,3,4,5,6,7,8,9,10,13,15],assumpt:[1,3,4,7,9,15],ast:[1,3,4],astyp:[7,8],asymmetri:1,asymptot:4,atoi:15,atom:1,attempt:[1,5,6,8],attend:14,attent:[1,13],attr:[],attract:[1,8],attribut:[1,7],attributeerror:4,audi:1,aurelien:[1,14,17],author:[1,2,8,15],authour:1,auto:[7,8,15],autocor:15,autocorrelation_tim:15,autocorrelform:15,autocovari:15,autoencod:12,autoencond:12,autograd:12,autom:[1,12],automac:13,automag:[],automat:[1,2,9,12,13],autonom:17,avail:[1,2,4,8,9,12,13,14,17],averag:[0,1,2,4,7,8,15,16],avg:15,avoid:[0,1,3,4,7,9,13],awai:15,awar:8,award:16,axes3d:5,axes:[1,4,5,6,7,8,9],axessubplot:1,axhlin:6,axi:[0,1,2,4,5,6,7,8,9,10,15],axiom:3,axlabel:1,axvlin:6,b_1:[5,10],b_5:5,b_group:7,b_i:[1,2,10],b_ia_:1,b_index:7,b_j:[2,10],b_k:[2,5,10],b_m:10,b_score:7,b_valu:7,bachelor:14,back:[1,3,6,7,8,13,15],backbon:13,backend:2,background:17,backpropag:2,backtrack:7,backup:13,backward:[2,10,13],bad:15,badli:15,bag:[7,12,14],bag_clf:8,baggin:[],baggingboot:8,baggingclassifi:8,baggingtre:8,balanc:4,band:13,bandwidth:13,bar:[1,9],barber:17,bare:8,barebon:0,base:[0,1,2,3,5,6,7,8,12,15,16,17],basi:[3,5,6,8,9,10,13],basic:[6,10,12,14,15],batch:[5,9,10],batch_siz:2,bay:5,bayesian:[3,12,17],becaus:[0,1,2,3,4,5,6,7,10],becom:[0,1,2,3,4,5,7,10,15],been:[1,2,4,9,10,12,13],befor:[0,1,2,3,4,5,6,10,13,15],beforehand:[0,1,15],begin:[0,1,2,3,4,5,6,7,9,10,13,15],behav:[2,4,5],behavior:[1,2,5],behaviour:10,behind:[1,2,5,6],being:[0,1,2,3,5,6,8,9,10,15],believ:[7,13],belong:[0,3,5,6,7],below:[1,2,3,4,5,6,7,8,9,10,13,15],benchmark:8,bendik:16,benefici:2,benefit:[1,2,5,9,12],bengio:[2,14,17],benign:[2,5],besid:3,bessel:3,best:[0,1,2,3,4,5,6,7,8,10,15,16],beta:[1,2,3,4,5,8,9],beta_0:[1,2,3,5],beta_0x_:1,beta_1:[1,2,3,5,8],beta_1x_0:1,beta_1x_1:[1,5],beta_1x_2:1,beta_1x_:1,beta_1x_i:5,beta_2:1,beta_2x_0:1,beta_2x_1:1,beta_2x_2:[1,5],beta_2x_:1,beta_:[1,5],beta_i:[1,3],beta_j:[1,3,5],beta_k:5,beta_linreg:5,beta_m:8,beta_mg_m:8,beta_p:5,beta_px_p:5,better:[0,1,2,3,7,8,9,10],between:[0,1,2,3,4,5,6,7,9,10,15],beyond:[1,2,3,5,6],bia:[1,2,3,6,7,8,10,14,15],bias:[2,3,4,7,10],big:[0,1,2,3,4],bigger:2,bigr:10,bike:7,bilbo:[],bilek:16,billion:[10,12],bin:[1,4,5,15],binari:[1,3,5,7,8,10,14,15],bind:[1,4],binomi:12,binsboot:4,bioinformat:1,biolog:[2,10,17],bios1100:12,bird:1,birth:[],bishop:[14,17],bit:[0,2,13,15],bitwis:15,bla:[3,13],black:[0,6,7],block:[0,4,8,12,13],blockingavg:15,blockingstd:15,blockingvar:15,blocksiz:15,blocksizemax:15,blocksizemin:15,blue:1,bmatrix:[1,2,3,5,6,9,13],bmi:2,bodi:[1,2,10],bold:2,boldfac:[1,3],boldsymbol:[0,1,2,3,4,5,6,8,9],boltzmann:[10,12],book:17,bool:0,boost:[2,7,12,14],boostrap:8,bootavg:15,bootstd:15,bootstrap:[2,12,14],bootvar:15,bootvec:15,borrow:[],boston_dataset:1,bot:6,both:[0,1,2,3,4,5,6,7,8,12,13,15,16],bottl:5,bottom:4,bound:[1,3,4,6,10],boundari:[6,9,10],boundkei:3,box:7,boyd:[5,6],bracket:15,brain:[2,5,10],branch:7,breast:[3,5,9],brew:[1,12],brg:6,briefli:1,bring:[1,3,8],broad:1,broadcast:0,browser:[],brute:[3,9],bsol:[],bsubex:[],build:[1,3,4,8,13,15],built:[1,2,4],bunch:9,busi:1,bzl:3,c46dd114b2af:6,c_0:15,c_1:10,c_2:10,c_3:10,c_4:10,c_i:[5,10],c_k:15,cach:8,cal:[1,5,6,8,10],calcul:[0,1,2,3,4,5,6,7,8,9,10,13],call:[0,1,2,3,4,5,6,7,8,9,10,12,13,15,17],callabl:4,calor:1,cambridg:[5,17],came:0,can:[0,1,2,3,4,5,6,7,8,9,10,12,13,15,17],cancel:1,cancer:[3,8],cancerpd:5,candid:[6,7,8],cannot:[1,2,3,5,6,7,14,15],canopi:[1,12],cap:3,capabl:[1,2,6,12],capita:1,captur:[9,10],card:[1,5],cardin:2,care:[0,9],carefulli:5,carlo:[1,4,12,15,17],carri:[4,5],cart:8,casella:17,cast:2,categor:[1,2,7,9],categori:[0,1,2,5,8,10,14],categorical_crossentropi:2,caus:[1,3,4,15],causal:1,causat:1,cax:2,cbar:2,ccc:[3,10],cdf:15,cdot:[0,1,4,5,10,13,15],celebr:5,center:[0,1,2,4,5,6,7,9,15],centr:17,central:[1,3,4,6,13],centroid:[0,15],centroid_differ:0,centroid_list:0,certain:[0,1,4,5,7,15],cha:1,chain:[2,12,15],challeng:[0,5],chanc:[2,3,5,15],chang:[0,1,2,3,4,5,6,7,9,10,13,15],chapter:[0,4,8,9,13,14,17],charact:[1,3,6,15],character:[6,7,8,10,15],characterist:[1,2,8],charg:1,charl:1,chd:5,chddata:5,cheap:3,cheaper:[2,5],chebychev:0,check:[1,2,3,5,9,13],check_consistent_length:[],chemic:15,chen:8,choic:[0,1,2,4,5,7,10,13,15],choleski:[3,13],choos:[0,4,5,7,8,9,15],chosen:[1,2,4,5,6,7,8,15],chosen_datapoint:2,christian:17,christoph:[14,17],cin:15,circ:[2,10],circl:[1,6,10],circumfer:7,circumv:[2,3],clariti:[0,15],class_nam:7,class_val:7,class_valu:7,classic:[5,7],classif:[1,3,4,5,6,9,10,12,14,17],classifi:[1,2,5,7,8,9],classificaton:2,classifii:8,clean:2,clear:[2,3,8,10],clearer:0,clearli:[1,3,4,5,6,15],clever:[0,2,8],clf3:1,clf:[1,6,7,8],clf_ridg:1,clip:15,close:[0,1,2,4,5,6,7,9,10,15,17],closer:3,closest:[0,5,6,9],closur:12,cloud:12,clust:0,cluster:[1,2,4,9,12,14],cluster_label:0,cmap:[1,2,6,7,8],cmath:15,cmb:14,cmd:7,cn_:15,cnn:10,cntk:12,code:[4,6,12,13,14,17],coef0:6,coef:1,coef_:[1,3,5,6,7],coeffici:[1,3,4,5,6,7,13],coerc:[1,4],coin:[8,15],coin_toss:8,col:[1,9],colab:12,cold:7,colinear:1,collaps:6,collect:[1,4,8,9,12,15,17],collinear:3,color:[1,4,6,7,8,15],colorbar:2,colsample_bytre:8,colsaobject:8,column:[1,2,3,4,5,6,7,9,10,13],columntransform:7,com:[12,16,17],combin:[2,3,4,5,8,15],come:[0,1,2,3,5,10],comma:[],command:[1,2,15],comment:1,commerci:[1,12],commod:1,common:[0,1,2,3,4,5,7,9,15],commonli:[0,1,2,4,5,7],commun:[1,10],compact:[0,1,2,3,4,5,7,9,10],compar:[0,1,3,4,5,9,13],compat:5,compet:1,competit:8,compil:[1,2,12,13,15],complet:[1,7,10],completenn:10,complex:[2,3,4,6,7,9,10],complic:[0,1,2,4,5,7],compon:[0,1,2,3,4,5,7,12,14],components_:9,compos:[0,7,10],compphys:[12,14,17],compress:1,compris:4,compromis:3,compulsori:12,comput:[0,1,2,3,4,6,8,9,10,12,13,14,17],computation:[1,4,5,7,15],con:3,concaten:0,concav:[2,5],concentr:[1,8],concept:[0,1,12],conceptu:[5,10],concern:[0,1,2,5],concic:[],conclud:[1,3],conclus:2,conda:[1,2,12],condit:[1,3,4,6,7,9,15],conduct:12,coneqp:3,confid:[1,3,4,5,6],confirm:[3,10],confus:[3,4,8,13],confusion_matrix:7,congruenti:15,conjug:6,conjugaci:5,connect:[1,2,5,7,9,10,13],consequ:[3,4,5,6,8,10],conserv:[0,3],consid:[0,1,2,3,4,5,6,7,8,10,13,15],consider:[1,2,3,5],consist:[1,2,4,5,10,15],constant:[1,3,5,6,10,15],constitu:1,constitut:4,constrain:[2,3,5,9],constraint:[3,5,6],construct:[1,2,3,4,5,6,7,8,9,13,15,17],contact:1,contain:[0,1,3,4,5,6,7,9,10,13,15,17],contemporari:17,content:[2,12,13],context:[4,5,8],continu:[1,2,3,4,5,6,7,8,10,13],contour:[5,7,8],contourf:[6,7,8],contrast:[2,7,8,10],contribut:[1,3,15],contributor:1,control:[1,2,7,12],conveni:[1,3,4,5,10,13],convent:10,converg:[0,1,2,3,5,6,9],convergencewarn:[1,2,5,6,9],convert:[1,2,3,7,9,13],convex:3,convinc:5,convolut:[2,12,14],cool:7,coordin:[0,3,10],coorel:1,copi:[0,1,2],core:8,corel:1,coronari:5,corr:[1,3,5,9],correalt:[9,12],correct:[0,1,2,3,13,15],correctli:[2,4,8],correl:[1,2,3,5,8,10,12],correlation_matrix:[1,3,5,9],correspond:[0,1,3,4,6,7,9,10,12,13,15],cortex:10,cos:[1,4,7],cosin:[0,4],cost:[1,3,4,6,7,10,15],could:[1,2,3,4,5,6,7,8,9,10,13,15],coulomb:1,count:[1,7,14,15,16],countor:5,coupl:3,cours:[1,2,3,9,14,15],courvil:[14,17],cout:15,cov:[3,4,9,13,15],cov_xi:[3,9],cov_xx:[3,9],cov_yi:[3,9],covari:[5,12,13],covariance_matrix:[0,3,9],cover:[1,3,11,12,17],covert:1,covxi:15,covxx:15,covxz:15,covyi:15,covyz:15,covzz:15,cpu:2,creat:[2,3,7,8,9,10,12],create_biases_and_weight:2,create_neural_network_kera:2,create_x:[3,9],credit:[1,5],crim:1,crime:1,criteria:[0,1,7,8,15],criterion:[5,7,8],cross:[1,2,5,7,8,12,14,15],cross_val_scor:4,cross_valid:[5,8],crossvalid:4,crucial:[2,15],csr_matrix:13,cstdlib:15,csv:[1,4,5,7],ctnk:2,cubic:1,cumbersom:3,cumsum:[8,9],cumul:[4,8],cumulative_heads_ratio:8,cup:3,current:[0,2,5],curs:1,curv:[5,8,10],curvatur:5,custom:[0,3],custom_cmap2:[7,8],custom_cmap:[7,8],cutpoint:7,cvxbook:5,cvxopt:[3,6],cyber:17,cycl:[0,2,10,15],d_f:5,dagger:[3,13],dai:[2,7,12],dalen:16,darget:7,darkr:15,dat:1,dat_id:[1,4,5,7],data1:0,data2:0,data3:0,data4:0,data:[0,3,4,6,8,10,13,17],data_id:[1,4,5,7],data_indic:2,data_panda:[],data_path:[1,4,5,7],databas:2,datafil:[1,4,5,7],datafram:[1,3,5,7,9],datapoint:[2,3,4,5,9],dataset:[0,1,4,5,6,7,8,9],date:1,daughter:8,david:17,dbh:2,dbo:2,dcomposit:13,dcost:3,dead:2,deal:[0,1,2,3,5,6,9,13,15],debt:5,debug:[3,4],decad:1,decai:[1,5,15],decent:8,decid:[3,4,7],decim:1,decis:[1,2,6,9,12,14,17],decision_funct:6,decision_tre:7,decisiontreeclassifi:[7,8],decisiontreeregressor:[1,7,8],declar:[1,13],decompos:[3,13],decomposit:[1,10],decompost:3,decorrel:8,decreas:[2,3,4,5,8,9],deduc:1,deep:[5,10,12,14,17],deep_tree_clf1:7,deep_tree_clf2:7,deep_tree_clf:[7,8],deepen:12,deeper:0,deeplearningbook:17,def:[0,1,2,3,4,5,6,7,8,9,15],def_covari:15,defect:3,defici:3,defin:[0,1,3,4,5,6,7,8,9,10,13,15],definit:[0,2,3,4,5,6,8,9,10,13],defint:15,degre:[3,4,6,7,8,9,15],del:2,delet:[4,15],deliv:14,delta:[0,1,6,10,15],delta_:[2,13],delta_h:[1,2],delta_j:10,delta_k:10,delta_l:2,delta_n:1,delug:12,delv:1,demand:5,demonstr:[1,3,4,5,9,10,12],denomin:[2,3],denot:[2,4,5,15],dens:[0,2],densiti:[0,1,4,15],depart:16,depend:[0,1,2,3,4,5,6,9,10,12,15],depict:15,deploy:[1,12],deprec:1,depth:[7,8,13],deriv:[1,2,4,6,8,9,12],descend:[3,7,9],descent:[1,2,6,10],descr:1,describ:[0,1,3,4,6,8,9,10,13],descript:[0,1,6,7],design:[1,2,3,4,5,8,9,10],designmatrix:1,desir:[0,1,3,5],despit:[2,10],destroi:13,det:[3,13],detail:[0,1,5,9,13],detect:[6,10],determin:[1,3,5,6,7,8,9,10,13,15],determinist:[5,15],dev:[2,15],develop:[0,1,3,6,8,9,10,12,13],deviat:[1,2,3,4],devis:10,df1:[],diag:[3,6],diagnost:[2,8],diagon:[1,3,5,13,15],diagonaliz:3,diagram:8,dict:6,dict_kei:1,dictionari:1,did:[0,1,2,3,5,8,9],die:2,diffeent:6,differ:[0,1,2,3,4,7,8,9,10,12,13,15,17],differenti:[5,12,13],difficult:[0,1,2,4,8,15],difficulti:[1,2,5],digit:[1,2,14,16],dilut:2,dim:[0,9],dimens:[0,1,2,3,6,9,13],dimension:[0,1,3,4,5,7,9,12,13],dimensionless:1,diment:13,dimes:0,direct:[0,1,2,5,9,10],directli:[0,2,3,15],directori:[],disadvantag:1,discard:[4,9],disciplin:[1,10],disclaim:15,discourag:5,discov:1,discover:3,discret:[2,3,5],discrimin:[5,8,9],discuss:[0,1,2,3,4,5,6,7,8,9,10,12,13,14,15,17],diseas:5,disk:[],disord:[2,5],displai:[0,1,2,4,5,6,7,8,9,10,15],displaystyl:[1,3],displot:1,disregard:1,dissimilar:[0,9],dist:0,distanc:[0,1,6,7,9,15],distance_list:7,distinct:[0,5,6,7,8],distinctli:6,distinguish:[1,5,6,15],distplot:1,distribut:[0,1,2,4,5,8,9,12,13],distrubut:[1,12],div:3,dive:[1,6],diverg:[2,5],divid:[1,2,3,4,6,7,9,10,15],divis:[4,6,7,13,15],dna:5,dnn:[1,2,10],dnn_kera:2,dnn_model:2,dnn_numpi:2,dnn_scikit:[1,2],doamin:3,doc:[12,14,17],doconc:[],document:[5,9],doe:[0,1,2,3,4,5,6,8,9,10,13,15],doesn:[7,10],dog:2,doi:0,doing:[1,3,4,9],domain:[3,5,6],domin:1,don:[0,1,2,6,9,12,15],done:[0,1,3,4,7,8,9,13,15],dot:[1,3,5,6,7,8,9,10,13,15],doubl:[13,15],doubli:2,down:[1,5,7,9,10],download:[1,2,3,13,17],dozen:2,dramat:9,draw:[4,5,8],drawback:[1,2,5],drawn:[2,4,5,9,15],dre:3,drop:[1,2,3,4,9,15],dropna:[1,4],dtype:[0,1,2,13],dub:1,due:[0,2,3,4,5,6,8,10],dummi:1,dure:[1,2,6,7,9,12],dwell:1,dwh:2,dwo:2,dx_1:15,dx_n:15,dying:2,each:[0,1,2,3,4,5,6,7,8,9,10,12,13,14,15,16],eapprox:1,earli:2,earlier:[1,3,5,6,7,9,10],eas:[0,7],easi:[1,3,4,5,6,7,8,9,10,12,13],easier:[3,4,6,7,15],easiest:5,easili:[1,2,3,5,6,7,8,9,10,13],eastern:16,ebind:1,eblock:7,econometr:[],economi:3,ecosystem:12,ect:14,edgecolor:4,edit:[],edu:5,educ:1,eface79dac2c:8,eff:15,effect:[2,8,15],effic:2,effici:[1,5,8,12,13,15],efron:[4,15],eig:[3,5,9,13,15],eigen:15,eigenpair:[3,9],eigenvalu:[3,5,6,9,13],eigenvector:[3,5,9],eight:13,eigval:[13,15],eigvalu:[5,9],eigvec:[13,15],eigvector:[5,9],eispack:13,either:[1,2,3,4,5,6,7,8,9,15],ekstrom:16,elabor:15,electr:[1,10],electur:17,eleg:9,element:[2,3,4,5,6,9,10,12,13,14,17],elementari:[8,13],elessar:[],elif:0,elim:13,elimin:[3,6],els:[2,5,7,10,15],elu:2,elus:1,email:[14,16],embed:[1,9],embodi:4,emit:15,emner:17,emphas:[1,8,12],emphasi:[1,12,17],empir:[2,9,15],emploi:[1,2,3,4,5,9,15],employ:1,empti:[4,8],emul:10,enabl:9,encapsul:0,encod:[0,1,3,7,9],encompass:[1,15],encount:[1,2,3,5,15],end:[0,1,2,3,4,5,6,7,8,9,10,13,15],endl:15,endpoint:15,energi:[1,4],enforc:10,eng:17,engin:[1,2,12,15],english:17,enough:[1,4,5],ensembl:[2,7,14,15],ensur:[1,2,3,4,5,9,15],enter:[3,15],enthought:[1,12],entir:[2,5,7,12,15],entiti:[7,10,13],entri:[1,3,6,9,10,13],entropi:[2,5,8],enumer:[1,2,6],env:[3,15],environ:[12,17],eol:1,eosfit:1,epoch:[1,2,5,10],epsilon:[1,3,4,5],epsilon_0:1,epsilon_1:1,epsilon_2:1,epsilon_:1,epsilon_i:1,eqnarrai:[3,4],equal:[0,1,2,3,4,5,6,7,9,10,13,15],equat:[0,2,3,4,6,7,8,9,13,15],equilibrium:10,equiv:[5,13,15],equival:[1,2,3,6,9,12,13],erf:15,eriador:[],eridg:1,err:[1,8],err_:4,errat:5,errno:[],error:[2,3,4,5,7,9,10,12,15],error_estimate_corr_tim:15,error_hidden:2,error_output:2,escap:5,esl:0,esol:[],especi:[2,7,10],essenti:[0,1,3,7,8,10,15],establish:[1,8,9],estim:[1,2,3,4,5,8,9,12,15],estimated_mse_fold:4,estimated_mse_kfold:4,estimated_mse_sklearn:4,esubex:[],eta0:[5,6],eta:[1,2,5,6,10],eta_v:[1,2],etc:[0,1,2,3,5,6,7,9,10,12,13,15],ethic:12,etsim:4,euclidean:[0,1],evalu:[1,3,4,5,7,15],even:[0,1,2,3,4,5,6,7,8,9,10,12,13,15],event:[3,5,8],eventu:[3,4,5,9,10,16],everi:[0,1,2,3,4,5,7,8,9,10,12,15],everyth:10,everywher:5,evolv:1,exact:[1,3,5,9,10,13,15],exactli:[1,10,12],examin:4,exampl:[3,4,9,10,12,13,14,17],exce:[2,10],excel:[0,1,2,3,8,17],except:[3,4,6,7,15],excess:1,excit:1,exclud:[2,4,10],exclus:[1,2,4,15],execut:3,exemplifi:5,exercis:[3,12,14],exhaust:4,exhibit:[1,6],exist:[0,1,2,3,4,5,6,7,13,17],exit:[3,13,15],exp:[1,2,3,4,5,6,8,9,10,15],exp_term:2,expand:[3,5,9],expans:[1,3,5,6,8,10],expect:[1,2,3,4,5,9,10,12],expectation_value_of_h_wrt_p:15,expens:[4,5,8,15],experi:[1,2,4,5,6,12],experiment:[1,4,7,15],expert:[2,7],explain:[0,1,5,7,8,9,15],explained_variance_ratio_:9,explanatori:1,explicit:[1,13],explicitli:[0,1],explod:2,exploit:[1,10],explor:[2,5,6,12],expon:2,exponenti:[1,2,3,5,8],export_graphviz:7,export_text:7,exporttext:7,expos:12,express:[1,3,4,8,10,13,15],exptmean:15,exptvari:15,extend:[3,9,12],extens:[1,10,12],extent:[1,2,4,17],extern:7,extra:[2,3],extract:[1,3,5,6,9,13],extrapol:1,extrem:[0,1,2,3,5,6,7,13],extremum:5,extrins:9,eye:[0,1,3,5,13],f11:1,f12:1,f13:1,f1d:5,f6d7a289d493:13,f_0:8,f_1:[5,8],f_2:[5,10],f_3:10,f_d:15,f_i:[4,10],f_m:8,face:5,facecolor:[4,6,15],facil:[1,12],facilit:10,fact:[1,2,3,5,7,9,10],factor:[1,2,3,7,8,9,13,15],fafab0:[7,8],fail:[4,5,6,9,16],failur:5,fairli:[0,2,15],fall:[6,7,14],fals:[0,1,2,3,4,5,7,8],famili:[1,5,6,15],familiar:[1,3,6,12,13,15],famou:[4,10,13],far:[0,1,3,5,6,9,10,15],fashion:[1,7,8],fast:[2,4,5,8,10,12,15],faster:[2,9],fastest:5,favor:5,favorit:15,featur:[1,2,3,4,5,6,8,9,10,12,15],feature_nam:[1,2,5,7],feautur:7,fed:2,feed:[1,9,12,14],feed_forward:2,feed_forward_out:2,feed_forward_train:2,feedforward:[2,10],feel:[0,1,3,9,12,16],feet:1,few:[0,2,7,11,15],fewer:[1,7,9],ffnn:[2,10],field:[1,10,12],fifth:1,fig:[0,1,2,5,10],fig_id:[1,4,5,7],figaxi:15,figsiz:[1,2,4,5,6,7,8],figur:[0,1,2,3,4,5,6,7,8,10,12],figure_id:[1,4,5,7],figurefil:[1,4,5,7],file:[1,2,3,4,5,6,7,13,15],filenam:[1,15],filenotfounderror:[],fileout:15,fill:[3,7],financ:1,find:[0,1,2,3,4,5,6,7,8,9,10,12,15],fine:[0,1],finit:[3,4,10,15],first:[0,1,2,3,4,6,7,8,9,13,15,17],firsteigvector:9,fit:[2,3,4,5,6,7,9,10,15],fit_intercept:[1,3,4],fit_mod:7,fit_transform:[1,4,6,7,9],fiti:1,five:[1,7],fix:[1,4,5,8,9,10],fkkt:3,flag:0,flat:[5,10],flatten:[2,3],flexibl:[1,2,4,6,8,10],float32:7,float64:[1,13],flop:[3,13],flow:[2,10],fluctuat:3,fly:9,flyvbjerg:15,fmesh:5,focu:[1,3,4,12,17],focus:[2,5,13],fold:[4,7],folder:1,follow:[0,1,2,3,4,5,6,7,8,9,10,12,13,14,15,16,17],font:[1,5,15],fontdict:15,fontsiz:[2,6,7,8,15],fontweight:2,foral:6,forc:[1,3,8,9],forecast:10,forest:[1,2,7,12,14],forget:9,form:[1,3,4,5,6,7,9,10,12,13,15],formal:[0,15],format:[1,2,4,5,6,7,8,9,12,15,17],formatstrformatt:5,formul:[0,9],formula:[5,15],forth:10,fortran2003:12,fortran90:15,fortran:[1,12,13],fortun:[1,9],forward:[1,4,12,13,14],found:[2,3,4,5,10],foundat:12,four:[3,6,10,13,14],fourier:1,fourth:10,frac:[0,1,2,3,4,5,6,7,8,9,10,13,15],fraction:7,frame:5,framework:[2,6,8,15],frank:[3,9],frankefunct:[3,9],free:[1,9,12,13,15,16,17],freecodecamp:12,freedom:3,freeli:1,frequenc:[4,5,15],frequent:[1,5,6,7],frequentist:12,fresh:8,fret:0,fridai:14,friedman:[14,17],frodo:[],from:[0,1,2,4,5,6,7,9,12,13,14,15,16,17],from_cod:7,front:[1,3],fruit:0,fstream:15,fulfil:[3,10],full:[1,2,3,5,7,8,15],full_matric:3,fulli:[4,10,14,15],fun:12,func:4,functionali:9,fundament:[1,4,12],further:7,furthermor:[1,3,4,5,9,10,12],futur:[1,6,7],futurewarn:1,fys:16,g_1:8,g_2:8,g_m:8,gain:[0,2,3,7,8],galleri:1,gamge:[],gamma1:6,gamma2:6,gamma:[1,5,6,7,8,9],gamma_0:8,gamma_1:8,gamma_1x:8,gamma_:1,gamma_i:[1,6,15],gamma_j:5,gamma_k:5,gamma_m:8,gamma_x:1,gap:[3,6],gate:10,gather:[1,2,10],gaug:10,gaussbacksub:13,gaussian:[0,3,4,6],gaussian_point:0,gaussian_rbf:6,gave:5,gbc:14,gca:[4,5,6],gd_clf:8,gdclassiffiercgain:8,gdclassiffierconfus:8,gdclassiffierroc:8,gdregress:8,gemv:3,gen:15,gender:1,gener:[0,1,2,3,4,5,6,8,9,10,13,17],generallay:10,generate_simple_clustering_dataset:0,genom:12,geodes:9,geometr:1,geometri:3,georg:17,geq:[3,5,6,7],geron:[1,14,17],get:[0,1,2,3,4,5,7,8,9,12,13,15],get_distances_to_clust:0,get_dummi:7,get_split:7,get_yaxi:6,getattr:4,getsolutionslic:3,gibb:12,gini:8,gini_index:7,git:[1,12],github:[1,12,14,17],gitlab:[1,12],give:[0,1,2,3,5,6,7,8,10,12,15,17],given:[0,1,2,3,4,5,6,7,8,9,10,13,15],glare:0,global:[5,15],glorot:2,gmail:16,goal:[1,5,7],goe:[0,1,2,3,4,5,13],going:[1,2,3,4,5,6,7,9,10],golden:5,gone:3,gong:2,good:[0,2,3,5,7,8,9,12,15,17],goodfellow:[14,17],googl:[0,2,12],got:2,gpu:2,grade:14,gradient:[1,6,7,10,12,14],gradientboostingclassifi:8,gradientboostingregressor:8,gradual:[0,2],graph:[2,5,7,9,10],graph_from_dot_data:7,graphic:[1,2,7],grasp:1,gray_r:2,great:5,greater:[2,5,15],greedi:7,green:[1,7,15],grid:[2,4,5,6,10,15],grossli:5,ground:1,group:[0,1,4,5,7,12,14],groupbi:1,grow:[2,7,8],growth:1,guarante:[1,5,15],guess:[0,2,5,8],guestrin:8,guid:2,h_1:5,h_2:5,h_m:8,habit:1,had:[1,2,4,5],hadamard:[2,10],half:[2,6,7],halv:8,hand:[1,2,3,5,9,10,12,13,14,15,17],handl:[1,2,3,7,9,12],handle_unknown:7,handsid:10,handwrit:10,handwritten:[2,3,5],happen:[0,2,3,5,8,15],hard:[2,5,6,8],hardcopi:12,harder:[1,2],has:[0,1,2,3,4,5,6,7,8,9,10,13,15],hasn:[1,2],hassl:[1,12],hast:12,hasti:[0,1,14,17],hat:[2,3,5,7,8,9,10,13,15],have:[0,1,2,3,4,5,6,7,8,9,10,12,13,15],haven:2,hdf5:[],head:[1,8,15],header:1,heads_proba:8,health:1,hear:1,heart:[1,5],heatmap:[1,2,5],heavili:1,heavisid:2,height:2,help:[0,1,2,10],helper:0,henc:[1,3,4,5,6,7,8,10],her:5,here:[0,1,2,3,4,5,6,7,8,9,10,12,13,15,17],hereaft:[1,6,10],hermitian:13,hessenberg:13,hessian:3,heterogen:[7,8],hidden:[2,10],hidden_bia:2,hidden_bias_gradi:2,hidden_layer_s:[1,2],hidden_weight:2,hidden_weights_gradi:2,hierarch:[0,3],high:[0,1,2,3,4,5,7,8,9,12,13],higher:[1,2,3,4,5,6],highest:2,highli:[1,8,12,13,15,17],highwai:1,hing:6,hint:5,hip:12,hire:1,his:5,hist:[4,5,15],histogram:[1,4,5,15],histor:[5,9],histori:10,histplot:1,histtyp:4,hitherto:3,hjorth:16,hobbi:15,hoc:3,hoff:17,hold:[0,2,4,5,15],holder:1,home:1,homogen:[2,7,8],hopefulli:[1,9,15],horizont:9,hors:5,hot:[2,7],hour:[2,12,14,15,16],how:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,17],howev:[0,1,2,3,4,5,6,7,8,9,10,12,13,15],hspace:[1,6,8,15],hstack:2,htf:14,html:[5,9,12,14,17],http:[5,9,12,13,14,17],huang:1,huber:1,huge:[2,12],human:[1,2,7,10],humid:7,hundr:2,hungri:2,hybrid:14,hydrogen:1,hyperbol:[2,10],hyperparam:6,hyperparamet:[0,1,3,7],hyperplan:9,i_1:[3,4],i_2:[3,4],ian:17,idea:[1,2,4,5,7,8,10,13,15],ideal:[0,1,4,6,15],idem:4,ident:[3,4,10,13],identical:3,identifi:[0,1,2,5,6,7,9,10],idum:15,ieor:15,ifi:17,ifs:12,ignor:[1,2,7],iii:13,ijca2016907841:0,illustr:[0,3,5,8,10,12],imag:[2,7,9,10,17],image_path:[1,4,5,7],imagin:2,immedi:[1,12],implement:[0,1,3,5,6,7,8,9,10],impli:[3,4,5,13],implicitli:[9,15],impos:[1,9,10],imposs:[1,3],impress:[1,10],improv:[0,1,3,7,8,9],impur:7,imshow:2,in3050:17,in4080:17,in4300:17,in5400:17,inaccur:5,inact:10,inadequ:1,includ:[1,2,3,5,9,10,12,15,16,17],include_bia:[4,7],incom:10,inconsist:[],incorrect:2,incoveni:6,increas:[1,2,3,4,5,6,7,9,10,15],increasingli:15,ind:4,inde:[1,3],indent:15,indentationerror:15,independ:[1,3,4,5,6,10,15],index:[0,1,2,8,12,15,17],index_col:1,indic:[1,2,3,7,8,9],indispens:4,individu:[2,5,8,10,15],indu:1,indx:13,ineffici:0,inequ:6,inequaltii:5,inf1000:12,inf1100:12,inf1100l:12,inf1110:12,inf3000:17,inf4490:17,inf5860:17,inf:1,infeas:7,infer:[1,2,4,17],inferenc:2,infil:[1,4,5,7],infin:[3,4,5,9],infinitesim:15,influenc:[4,8],influenti:2,info:[],inform:[0,1,2,4,5,7,9,10,13,17],infti:[5,15],ingeni:5,ingredi:[1,7],inher:4,inherit:13,initi:[0,1,2,4,5,8,13,15],initialis:15,inject:0,inlin:[0,1,2,3,4,5,6,7,8,9,10,13,15],inner:[4,5],innov:17,input:[0,1,2,3,4,5,6,7,8,10,13,15],input_dim:2,inputs:2,inputs_shuffl:[1,2],insert:[3,6,8,15],insid:[1,5],insight:[1,2,3,12,17],insist:5,inspir:[1,2,10,17],instal:[1,2,3,7,14],instanc:[1,2,4,5,7,9],instanti:8,instead:[0,1,2,3,4,5,6,7,9,13,15],institut:2,instruct:[1,2],int32:8,int64:1,int_0:15,int_:15,int_a:15,intak:1,integ:[0,2,13,15],integer_vector:2,integr:15,intellig:[0,1,17],intend:8,intens:2,intention:0,interact:[1,7,10,12],intercept:[1,5,6,9],intercept_:[1,5,6,7],interchang:[3,10,13],interconnect:2,interest:[1,2,3,4,5,6,7,10,12,15],interfac:[1,2,13],interior:[1,7],intermedi:13,intern:[2,8,10],interpol:[2,10],interpr:3,interpret:[1,2,7,8,10,13,14,15],interv:[1,3,4,5,15],intial:5,intimid:[],intract:1,intrins:[9,13,15],intro:[12,17],introduc:[1,2,3,5,6,8,10,13,15],introduct:[2,5,14,17],introductori:[1,13,17],intuit:[1,3,4,6,10],inv:[1,3,5],invalid:[2,6,13],invalu:[1,5,12],invari:2,invd:3,inver:6,invers:[1,5],invers_period:15,inverse_transform:6,invert:[1,3,5,8],invok:[1,6],involv:[1,4,5,9,10],iomanip:15,ios:15,iostream:15,ipca:9,ipynb:12,ipython:[0,1,2,4,5,6,7,8,9,12,13,15],irani:0,iri:[6,7],irreduc:4,irrelev:3,irrespect:1,isn:3,isnul:1,isomap:9,issu:[2,7,13],it_arrai:5,item:1,items:13,iter:[0,1,2,4,6,9,15],itr:3,its:[0,1,2,3,4,5,6,7,8,9,10,12,13,17],itself:[3,4,10,15],jackknavg:15,jackknif:[4,12],jackknstd:15,jackknvar:15,jackknvec:15,jacobian:5,jargon:15,jensen:16,jerom:17,job:[6,8],join:[1,4,5,7],joint:3,journal:15,judg:5,julia:[12,13],jump:[0,15],jupyt:[0,1,12,17],just:[0,1,2,3,4,5,6,7,8,9,10,15],justif:1,justifi:8,k_mean:0,kappa_d:15,karim:16,karlsen:16,karush:6,keep:[0,1,2,3,5,9,13],keepdim:[2,4,8],kei:[1,2,10,17],kept:0,kera:[1,12,14],kernel:[1,2,12],kernel_regular:2,kernelpca:9,kev:1,kevin:17,keyword:13,kfold:4,kick:[2,5],kind:[0,1,6,10],kjm:12,kkt:[3,6],kktsolver:3,kmeanspoint:0,kn_k:0,know:[0,1,2,3,5,6,12,15],knowledg:[1,12],known:[2,3,4,5,6,7,10,13,15,17],kondev:1,kpca:9,kroneck:0,kuhn:6,kwarg:4,kwown:1,l1_l2:2,l1regl:3,l1regls_mosek2:3,l1regls_mosek:3,l_1:5,l_2:5,l_j:10,la_i:10,la_k:10,lab:[12,14],label:[0,1,2,3,4,5,6,7,8,10,12,13,15],labelencod:[5,8],labels:[6,7],labels_shuffl:[1,2],labor:0,laboratori:14,lack:[0,1],lagrang:[6,9],lambda:[1,2,3,4,5,6,8,10,15],lambda_0:9,lambda_1:[3,6,9],lambda_2:[6,9],lambda_:9,lambda_i:[6,9],lambda_iy_i:6,lambda_jy_iy_j:6,lambda_k:6,lambda_n:[3,6],lamda:2,land:[1,6],landmark:6,landscap:5,langl:[1,9,15],languag:[1,2,6,12,13,17],lapack:[3,13],laplac:3,laptop:12,larg:[1,2,3,4,5,6,7,8,9,12,13,15,17],larger:[1,3,4,5,6,8,9,15],largest:[6,9],lasso:[1,4,5,12,14],last:[0,1,2,3,4,5,6,7,8,10,13,15],later:[0,1,2,5,6,10,12,15],latex:[],latter:[1,3,5,6,9,13,15],lattic:10,law:1,layer:1,lbfg:[5,7,8,9],lbl:4,lcc:[3,4],lda:9,ldot:[1,4,9,15],lead:[1,2,3,4,5,6,7,8,9,10,13,15],leaf:7,leaki:2,lear:5,learn:[3,4,6,7,8,10,13,14,17],learner:8,learning_r:[6,8],learning_rate_init:[1,2],learning_schedul:5,least:[0,1,4,5,6,8,9,12,13,15],leav:[1,2,3,4,7,9],lectur:[1,2,3,5,8,9,10,12,13,14,17],lecturenot:[12,17],left:[1,2,3,4,5,6,7,8,9,10,13,15],leftarrow:[6,10],legend:[1,3,4,5,6,7,8],len:[0,1,2,3,4,6,7,8,9,10,13,15],length:[1,2,5,6,7,12,15],leq:[0,1,3,5,6,15],less:[1,2,3,4,6,7,15],lessen:2,let:[0,1,2,3,4,5,6,7,8,9,10,13,15],letter:[1,13,15],level:[1,2,3,4,7,12,13,14],lib:[1,2,4,5,6,9],liblinear:[6,8],librari:[1,2,3,7,8,9,13,15,17],licens:[1,2,12],lie:[1,4,9,15],lies:[6,9],life:[1,2,6,10],lift:0,light:[],like:[0,1,2,3,4,5,7,8,9,10,12,13,15],likelihood:[1,2,3,7],lim_:15,limit:[1,3,4,6,9,10,13],lin_clf:6,lin_model:1,lin_reg:7,linalg:[1,3,5,6,9,13,15],line1:6,line2:6,line3:6,line:[1,2,4,5,6,9,13,15],linear:[2,3,4,5,7,8,9,10,12,14,15],linear_model:[1,3,4,5,6,7,8,9],linear_regress:4,linearli:3,linearloc:5,linearregress:[1,4,5,7],linearsvc:6,liner:2,linerar:8,linewidth:[1,4,6,7,8],link:[1,5,7,10,12],linlag:3,linpack:13,linreg:1,linspac:[1,4,6,7,8,13,15],linu:16,linuek:16,linux:[1,2,12],liquid:1,list:[0,1,2,3,7,12],listedcolormap:[7,8],literatur:[0,2,5,17],littl:[2,7,10],live:6,lle:1,lloyd:0,lmb:[3,4],lmbd:[1,2],lmbd_val:[1,2],lmbda:5,load:[1,2,5,7,8],load_boston:1,load_breast_canc:[2,5,7,8,9],load_digit:2,load_iri:[6,7],loc:[1,4,5,6,7,8],local:[1,2,5,10],locat:6,log10:[3,4],log:[1,2,3,4,5,7,8,9,13],log_:1,log_clf:8,logarithm:[1,3,5,13],logic:[1,2,7],logist:[1,2,6,7,8,9,10,12,14],logisticregress:[5,7,8,9],logit:5,logreg:[5,7,8,9],logspac:[1,2,3,4],longer:[0,6,8,13,15],longest:0,loocv:4,look:[0,1,2,3,4,5,6,7,8,9,13,15],loop:[0,2,4,8,10,12,13,15],lose:2,loss:[1,2,3,4,6,8,9,13],lot:[0,1,2,4],low:[1,4,7,8,9,15],lower:[1,2,7,8,13],lowercas:13,lowest:[5,7,15],lstat:1,lstsq:1,lubksb:13,ludcmp:13,lux:13,lvert:2,m_1:0,m_h:1,m_k:0,m_l:10,m_n:1,m_p:1,machin:[2,3,4,7,8,9,10,13,14,17],machinelearn:[12,14,17],mackai:17,made:[1,2,3,4,5,7,9,10],mae:1,magic:0,magnitud:[2,5],mai:[1,2,3,4,5,6,7,9,10,12,13,15],mail:14,main:[1,2,3,5,7,13,17],mainli:[1,3,4,5,7],maintain:4,major:[2,4,5,7,8,13],make:[0,2,3,4,5,6,9,10,12,13,15,17],make_moon:[6,7,8],make_pipelin:[1,4,8],makedir:[1,4,5,7],makeplot:1,malcondit:13,malign:[2,5,7],mammographi:3,manag:[1,12],manhattan:0,mani:[0,1,2,3,4,5,6,7,9,11,12,13,15,17],manifold:9,manual:0,map:[0,1,2,4,5,6,9,10,15],margin:[1,3,6],marit:1,mark:[],marker:[1,5,13],markov:12,marsaglia:15,mask:15,mass:[1,2,3],massag:1,masses2016:1,masses2016ol:1,masses2016tre:1,masseval2016:1,master:14,mat1100:12,mat1110:12,mat1120:12,mat3155:14,mat4155:14,mat:12,match:[0,2,3,5],materi:[3,5,13],math:[0,5,10,13,15,17],mathbb:[0,1,3,4,5,6,9,10,13,15],mathbf:[1,3,4,5,6,13,15],mathcal:[2,3,4,5],mathemat:[0,1,5,9,10,12,13,15,17],mathemati:[],mathemt:[],mathrm:[0,1,2,3,4,5,6,7,8,9,10,15],matmul:[2,3],matnat:[16,17],matplotlib:[0,1,2,3,4,5,6,7,8,9,10,12,13,15],matric:[1,2,5,6,9,12,13],matrix:[1,4,6,8],matshow:2,matter:5,max:[1,2,5,7,8,10],max_depth:[1,7,8],max_it:[1,2,5,6,9],max_iter:0,max_leaf_nod:8,max_sampl:8,maxdegre:[1,4,8],maxdepth:8,maxim:[2,3,5,6,9],maximum:[0,1,2,3,5,6,7,8],maxpolydegre:[3,4],mbox:[3,4],mcculloch:10,mcint:15,mcintsqr2:15,mean:[2,3,4,5,7,8,9,10,12,13,14],mean_absolute_error:1,mean_divisor:0,mean_i:15,mean_matrix:0,mean_squared_error:[1,4,5,8],mean_squared_log_error:1,mean_vector:0,mean_x:15,meaning:[1,5],meansquarederror:1,meant:[5,8],meantempvec:15,meanvec:15,measur:[0,1,2,3,4,7,9,10,15],mechan:[1,15],median:1,medicin:10,medium:6,medv:1,meet:[1,16],mehta:1,memori:[9,10,13],mention:[0,1,5,10,15],mere:1,mersienn:15,meshgrid:[3,6,7,8,9],messag:3,met:[1,6],meteorolog:7,method:[0,1,2,3,6,9,10,12,13,14,17],metric:[0,1,2,4,5,7,8],metropoli:12,mev:[1,15],mglearn:12,mgrid:5,mhjensen:[1,2,5,6,9],microsoft:17,mid:2,midpoint:7,might:[1,2,5,7],mild:7,miller:15,million:1,mimic:10,min:[1,3,6,7],min_:[0,1,3],min_samples_leaf:7,mind:[0,1,5],mine:12,mini:[2,5,9,10],minibatch:[2,5,9],minibathc:5,minim:[0,1,2,3,4,5,6,7,8,9,10,15],minima:[1,2,5],minimum:[1,2,4,5,6,7,9],minkowski:0,minmaxscal:1,minor:15,minst:2,minu:5,mirror:7,misclassif:[6,7,8],misclassifi:[6,8],mismatch:2,miss:[1,8],mit:17,mix:2,mkdir:[1,4,5,7],mlab:[4,15],mle:[3,5],mlp:2,mlpclassifi:2,mlpregressor:1,mnist:[2,9],mod:15,mode:[14,15],model:[0,3,4,5,6,7,8,9,12,15,17],model_select:[1,2,3,4,5,7,8,9],moder:8,modern:[1,4,5,12],modif:10,modifi:[1,2,3,5,6,8,10],modul:[1,4,5,7,8,9,13],modular:15,modulenotfounderror:[5,7],modulo:15,moe:[3,9],moment:[3,4],monoton:[3,10,15],mont:[1,4,12,15,17],montecarlocycl:15,moor:3,more:[1,2,3,4,6,7,8,9,10,12,14,15],moreov:1,morten:16,mosek:3,most:[0,1,2,3,4,5,6,7,8,9,10,12,15],mostli:[2,9],motion:1,motiv:2,move:[0,1,3,5,7,10,15],mpl:[1,5],mpl_toolkit:5,mplot3d:5,mplregressor:2,mse:[1,3,4,7,8],mse_simpletre:8,mselassopredict:3,mselassotrain:3,msepredict:3,mseridgepredict:3,msetrain:3,msg:1,msle:1,mt19937_64:15,mu0:15,mu1:15,mu2:15,mu_:15,mu_n:9,mu_x:15,much:[1,2,3,4,5,6,7,8,9,10,13,15],mul:3,multi:[1,2,5,12],multiclass:[2,5],multidimension:[9,10],multilay:2,multinomi:5,multipl:[3,4,5,10,15],multipli:[3,5,9,13,15],multiplum:6,multitud:[],multivari:[1,8,9,12,15],multivariate_norm:[0,9],murphi:[9,17],must:[0,2,3,4,5,6,8,10,15],mutat:5,mutual:[2,4,5],mx_:15,myriad:[1,12],mz1:15,mz2:15,n_0:[10,15],n_b:15,n_boostrap:[4,8],n_bootstrap:4,n_categori:2,n_cluster:0,n_compon:9,n_epoch:5,n_estim:8,n_featur:2,n_hidden_neuron:[1,2],n_i:15,n_input:[1,2],n_instanc:7,n_iter_i:[5,9],n_job:8,n_k:0,n_l:[10,15],n_layer:2,n_m:7,n_neuron:2,n_neurons_layer1:2,n_neurons_layer2:2,n_point:0,n_sampl:[0,4,6,7,8],n_split:4,nabla:[2,5],nabla_:5,naimi:1,naiv:[0,5],nall:1,name:[0,1,2,3,4,5,6,7,8,10,12,13,16],nameerror:8,namespac:15,nation:[2,3],nativ:12,natur:[1,2,5,6,7,10,15,17],navier:10,nb_:13,nbconvert:[],nboot:15,nearest:[2,9],nearli:5,neat:[],neccesari:4,necessari:[0,1,2,6],necessarili:[1,9,15],necesserali:3,neck:5,need:[0,1,2,3,4,5,6,7,8,9,10,13,15],neg:[1,2,3,4,5,8,15],neg_mean_squared_error:4,neglect:15,neglig:15,neighbor:9,neq:[0,5,15],nervou:10,nest:[7,10],net:10,netlib:13,network:[1,7,12,14,17],neural:[1,5,12,14,17],neural_network:[1,2],neuralnetwork:2,neuron:[2,10],neutral:1,neutron:1,never:[2,4,7,15],new_hobbit:[],new_sig:4,newaxi:[1,4,7],newli:1,newton:[2,6,15],next:[0,1,2,3,5,6,7,15],next_guess:5,nian:16,nice:[0,1,2,3,9],nichola:16,nicholaskarlsen1102:16,niter:5,nitric:1,nlambda:[3,4],nm_n:1,nmse:4,nn_model:2,node:[2,7,8,10],nois:[1,3,4,5,6,7,8],noisi:[2,4],non:[0,1,2,3,4,5,7,8,9,10,13,15],none:[0,1,2,3,4,5,7,8,15],nonetheless:0,nonlinear:[4,6,7,9,10],nonneg:[4,5,7],nonparametr:4,nonsens:15,nonsingular:13,nonumb:[5,6,13],nor:2,norm:[1,2,3,4,5,6,9],normal:[3,4,5,6,7,8,9,10,12,13],normali:13,normpdf:4,notat:[0,1,3,4,15],note:[0,1,2,3,4,5,6,9,10,12,13,14,15,17],notebook:[0,1,2,7,12],noth:[0,2,3,6,10,15],notic:[3,10,13,15],novel:[4,8],novemb:2,now:[0,1,3,4,5,6,8,9,10,13,15],nowadai:[1,2,7,12],nox:1,np_assign_points_to_clust:0,np_get_distances_to_clust:0,np_k_mean:0,nsampl:4,nspin:15,nthi:1,nuclear:3,nuclei:[1,15],nucleon:1,nucleu:1,num_tre:8,number:[0,2,3,4,5,6,7,8,9,10,13,14,16],numberid:5,numer:[1,3,4,5,7,8,9,10,12,13,17],numpi:[0,1,2,3,4,5,6,7,8,9,10,12,15],nunmpi:3,obei:[5,9],object:[1,2,3,4,6,8,13],objsens:3,obliqu:3,observ:[0,2,3,4,5,6,7,8,9,10],obtain:[0,1,2,3,4,5,6,7,8,10,13,15],obviou:[3,9,15],obviouli:1,obvious:[1,3,13],occupi:1,occur:[1,6,7,15],odd:[1,5],off:[2,3,4,7,15],offend:[],offer:[4,9,12,13,14],offic:16,offici:14,ofil:15,ofstream:15,often:[0,1,2,3,4,5,6,7,8,9,10,12,13,15],ofter:13,old:[2,3,5,8],olsbeta:3,omit:[1,3],onc:[0,2,4,7,9,15],one:[0,1,2,3,4,6,7,8,9,12,13,15],onehot:2,onehot_vector:2,onehotencod:7,ones:[1,3,5,6,7,8,9,13],onli:[0,1,2,3,4,5,6,7,8,9,10,13,15],onlin:[9,14],onto:[3,9],open:[1,2,4,5,7,12,14,15],oper:[1,2,3,4,8,9,10,12,13,15],operation:15,opinion:0,oplu:15,opmiz:5,opportun:1,opposit:[2,3,6],opt:[1,2,3,4,5,6,9],optim:[0,1,3,4,7,8,9,14],optimis:2,option:[1,2,3,5,6,9],optmiz:[2,6],orang:1,order:[1,2,3,4,5,6,7,8,9,10,13,15],ordinari:[1,4,5,9,12,14],oreilli:17,org:[5,9,12,13,17],organ:[0,4,5,8,13],orient:[2,3,4,15],origin:[1,3,4,6,9,10,13,15],orthogn:3,orthogon:[1,3,5,6,9,13],orthonorm:3,oscar:2,oslo:[1,14,16],osx:[1,12],other:[0,1,2,3,4,5,6,8,12,13,14,17],otherwis:[1,2,5],ouput:[3,5,10],our:[2,4,6,7,8,10,12,13],ourmodel:1,ourselv:[0,1,3,5,6,9],out:[0,1,2,3,4,5,6,7,8,9,10,12,13,15],out_fil:7,outcom:[1,5,7,8,10,15],outdoor:7,outer:10,outfilenam:15,outlier:[1,6],outlin:[4,8,9],outlook:7,outperform:8,output:[1,2,3,5,6,7,8,10,13],output_bia:2,output_bias_gradi:2,output_weight:2,output_weights_gradi:2,outputlayer1:10,outputlayer2:10,over:[0,1,2,3,4,5,7,8,10,15],overal:[2,8],overcast:7,overcom:10,overdetermin:1,overfit:[2,4,7,8],overflow:[2,3],overhead:10,overlap:[5,6,7],overlin:[0,1,3,4,7,8,9,13],overst:1,overview:[0,17],own:[3,5,6,10,12,13],owner:1,oxid:1,oyvinssc:16,p_i:[3,15],p_j:15,p_n:15,p_x:15,pack:1,packag:[1,2,3,4,5,6,9,12,15],page:[1,12],pai:[2,7],painless:[],pair:[1,7,12,15],panda:[1,3,4,5,7,9,12],panel:[],paper:2,paradigm:1,parallel:[8,13],paramet:[1,2,3,4,5,6,7,8,10,15],parameter:[1,8],parametr:[1,4],paramt:3,park:15,part:[0,1,2,3,4,8,13,14,15,17],partial:[1,2,3,5,6,8,9,10,15],particip:[12,14],particl:[1,15],particular:[1,2,3,4,5,7,8,9,10,15,17],particularli:[3,4,5,6,9,15],partit:[2,7],pass:[0,10,15],past:[8,15],patch:[4,15],path:[1,4,5,7,12],patient:5,pattern:[1,10,14,17],pauli:1,pca:[1,5,12,14],pcost:3,pdf:[1,3,4,7,17],pedagog:1,penalti:[4,5],penros:3,pentagon:5,peopl:[1,2,7,12],per:[1,2,4,14],percentag:[1,8,9],perceptron:[1,2,5],peregrin:[],perfect:[1,2],perfectli:4,perform:[0,1,3,4,5,6,8,9,10,12,13,15],perhap:[1,3,5],perimet:2,period:2,permut:9,person:[3,5,14,16],perspect:17,pertin:10,petal:[6,7],peter:17,petersen:15,phantom:15,phase:10,phatak:0,phenomena:15,phi:6,phi_k:6,philip:16,philosophi:5,phone:16,phrase:1,physic:[1,2,5,10,15,16,17],pick:[0,2,5,7,8,9,15],pickl:2,pictur:1,pie:12,piec:[0,9],pillow:[1,12],pinv:3,pip3:[1,2],pip:[1,2,12],pipelin:[1,4,6,8],pippin:[],pise:0,pitt:10,pixel:2,pixel_height:2,pixel_width:2,place:[1,4,5,6,13],plai:[1,3,4,6,9,12],plain:[5,6,8,10],plan:[4,7,16,17],plane:[6,7],plateau:[3,15],platform:12,plausibl:10,pleas:[1,5,9],plenti:2,plethora:10,plot:[0,1,2,3,4,5,6,7,8,9,10,12,13,15],plot_confusion_matrix:[5,8],plot_cumulative_gain:[5,8],plot_data:2,plot_dataset:6,plot_decision_boundari:[7,8],plot_import:8,plot_predict:6,plot_regression_predict:7,plot_roc:[5,8],plot_surfac:5,plot_train:7,plot_tre:[7,8],plt:[0,1,2,3,4,5,6,7,8,9,10,13,15],plu:[1,3,5],png:[1,4,5,7],point:[0,1,2,3,4,5,6,7,8,9,13,15,16],points_in_clust:0,poisson:12,poli:[4,6],poly100_kernel_svm_clf:6,poly3:1,poly3_plot:1,poly_featur:[6,7],poly_features10:7,poly_fit10:7,poly_fit:7,poly_kernel_svm_clf:6,polydegre:[1,3,4,8],polygon:5,polym:10,polynomi:[1,3,4,5,6,7,8,9],polynomial_featur:4,polynomial_svm_clf:6,polynomialfeatur:[1,4,6,7],polytrop:[1,4],poor:[2,5],poorli:1,popul:[1,3],popular:[1,2,4,5,6,7,9,10,12,13,15],popularli:1,portabl:8,portion:9,pose:[1,9,15],posit:[0,1,2,3,5,6,8,9,13,15],possibl:[1,2,3,4,5,6,7,8,9,10,12,13,15,16],posterior:3,postpon:1,postul:3,potenti:[1,3,10],potr:3,potrf:3,pott:10,power:[1,2,3,4,6,7,10],practic:[1,3,4,5,6,15],practition:[1,2],pre:3,preced:[2,9,10,15],preceq:6,precis:[1,3,9,13,15],pred:4,predict:[1,2,3,4,5,6,7,8,12,17],predict_prob:2,predict_proba:[5,8],predictor:[1,3,5,7,8,9],prefer:[1,2,6,7,9,12],prepar:1,preprocess:[1,4,5,6,7,8,9],prerequisit:1,present:[1,3,7,10,15],preserv:9,press:[5,17],pretrain:2,pretti:[1,6,7,12],prev_centroid:0,prevent:15,previou:[1,2,3,5,6,8,9,10,13,15],previous:[0,7,8,15],price:[1,7],primal:6,primari:[1,5],primarili:0,prime:15,princip:[1,3,5,12,14],principl:[0,1,4,5,6],print:[0,1,2,3,4,5,6,7,8,9,13,15],print_funct:[6,7],printout:1,prior:[1,3,4],privat:1,prob:[2,15],probabilist:[1,17],probabl:[1,2,4,5,8,12],problem:[1,3,4,6,7,8,9,10,12,13,14,15],proce:[1,3,5,6,7,8,9,13],procedur:[3,4,5,6,8,9],proceed:13,process:[0,1,4,5,7,8,10,12,13,15,17],prod:17,prod_:[2,3,5],produc:[0,1,3,7,8,9,10,12,13,15],product:[1,2,3,4,5,6,10,12,13],profess:1,profil:0,program:[0,1,2,3,6,10,12,13,14,15],programm:13,progress:[0,2],progression_plot:0,prohibit:4,project:[1,2,3,5,9,12,14,15,16],project_root_dir:[1,4,5,7],promin:10,promis:6,prone:7,pronounc:12,proof:[1,5,9,10],prop:4,proper:[1,4],properli:[0,2,6,8],properti:[1,2,4,5,10,13],proport:[1,2,3,7,9,15],propos:[2,8],propto:[3,5],proton:1,prove:5,provid:[1,2,3,4,5,6,7,8,10,12,13,15,17],proxi:2,prun:0,prune:7,pseudoinv:3,pseudoinvers:3,pseudorandom:15,psycholog:1,ptratio:1,punish:[1,2],pure:[7,15],purest:7,puriti:7,purpos:[0,1,8,10],put:2,putarow:3,putboundslic:3,putclist:3,putobjsens:3,putqobj:3,pycod:[],pydata:12,pydot:7,pyhton2:[],pylab:[1,5],pypi:12,pyplot:[0,1,2,3,4,5,6,7,8,9,10,13,15],pythagora:[0,3],python2:1,python3:[1,2,4,5,6,9,12],python:[2,3,6,9,10,14],pytorch:[1,12],qquad:[9,13],quad:[2,5,13],quadrat:[1,3,5,6,7],qualit:[7,15],qualiti:[1,7,12],quantifi:2,quantil:8,quantit:[1,4,7],quantiti:[0,1,3,4,5,7,8,9,10,13,15],quantum:10,quartil:1,queri:7,question:[1,3,4,5,7,9,10],quick:15,quickli:[2,5,7,9],quirk:0,quit:[2,3,4,7,8,10],quot:15,r2_score:1,r2score:1,r_1:7,r_2:7,r_j:7,r_m:7,rad:1,radial:[1,6,10],radioact:15,radiu:[1,2],rain:7,rais:4,ramp:2,ran1:15,ran2:15,ran3:15,rand:[1,3,4,5,7,8,13,15],rand_max:15,randint:[4,5,7],randn:[1,2,3,4,5,7,9],random:[0,1,2,3,4,5,6,7,12,13,14],random_devic:15,random_forest_model:8,random_index:5,random_indic:2,random_st:[1,5,6,7,8,9],randomforestclassifi:8,randomli:[0,2,4,5,7,15],randomnumbergener:15,rang:[0,1,2,3,4,5,7,8,9,10,13,15],rangl:[1,9,15],rangle_x:15,rank:3,raphson:[2,6],rapidli:1,rare:2,rate:[1,2,5,6,7,8,10],rather:[1,2,3,4,5,6,7,8,9,10,13,15],ratio:[5,7,8,9],rational:1,ravel:[3,4,5,6,7,8,9],raw:15,rbf:[6,9,10],rbf_kernel_svm_clf:6,rbf_pca:9,rcond:1,rcparam:[1,2,5,6,7,8,15],reach:[0,1,2,3,4,5,7,8,9,10,15],read:[0,1,3,4,5,6,9,10,13,14,15,17],read_csv:[1,4,5,7],read_fwf:1,readabl:0,reader:[1,13,15],readi:[0,1,2,3,6,8,9,10,13],readili:2,real:[1,2,4,5,8,9,10,13],realist:6,realiti:15,realiz:[2,10],realli:2,reason:[0,1,2,5,8,17],reassign:2,recal:[3,4,5,7,8,9,10,13,15],recalcul:15,receiv:[2,8,10,15],recent:[1,4,5,7,8],recept:10,recip:[1,5,13],reciproc:3,recogn:[1,3,8],recognit:[1,2,10,14,17],recommend:[1,6,12,13,14,15,17],reconsid:7,reconstruct:9,record:[8,14],recreat:15,rectangl:[4,5,7],rectangular:3,rectifi:[2,10],recur:[1,12],recurr:[2,12,14],recurs:[7,12,13],recycl:15,red:[1,4,6,7],redefin:[1,8],reduc:[2,3,5,7,8,9],reduct:[1,8,9,12,15],refer:[0,1,2,3,4,5,9,10,13,17],refin:10,refit:4,reflect:[1,2,15],refresh:[12,14],reg:[8,9],regard:[2,5,7],regardless:10,region:[7,10],regist:15,reglasso:3,regr_1:[1,7],regr_2:[1,7],regr_3:[1,7],regress:[2,4,6,9,10,12,14],regressor:[1,5,8],regridg:3,regular:[1,3,5,7],reilli:[1,17],reinforc:[1,6,12],reiter:2,rel:[1,4,5,7,10,15],relat:[0,1,2,3,5,9,13,15],relationship:[1,7],relativeerror:1,releas:[2,12],relev:[1,2,3,5,9,12,15],reli:[1,6],reliabl:[5,15],remain:[2,4,10,13,15],remaind:15,remark:2,rememb:[1,6,13],remind:[1,3,9,13,15],remov:[1,3],render:1,reorder:[3,5],reorgan:1,repeat:[0,1,2,3,4,5,7,8,9,13,15],repeated:1,repeatedli:[4,8,15],repetit:[4,14],rephras:5,replac:[0,1,2,3,4,5,8,10],replica:4,repositori:1,repres:[1,2,3,4,5,6,7,8,10,15],represent:[1,2,4,15],reproduc:[1,3,7,10,12,15],repuls:1,request:1,requir:[1,2,3,4,5,6,7,9,10,13],resampl:[1,5,8,12,14,15],rescal:[1,9,10],rescu:3,research:[1,12,17],resembl:[4,15],reserv:[2,3,4,15],reset:15,reshap:[0,1,2,4,6,7,8],residenti:1,residu:[1,3,5],resiz:3,respect:[0,1,2,3,4,5,6,8,9,10,15],respond:10,respons:[1,5,7,10],rest:[1,3],restat:[1,10],restrict:[1,7,10],result:[0,1,2,3,4,5,6,7,8,9,10,12,13,15],ret:4,retail:1,retain:[3,4],return_data:0,return_x_i:7,reus:2,reveal:[1,10],revers:2,review:[12,13],revisit:0,reward:1,rewrit:[1,3,4,5,6,8,9,10,13,15],rewritten:[4,6,8,15],rewrot:5,rgoj5yh7evk:12,rho:[1,8],rho_1:8,rho_2:8,rho_m:8,rhs:3,rich:1,rid:0,ride:7,rideclass:7,ridedata:7,ridg:[1,4,9,12,14],ridgebeta:3,right:[1,2,3,4,5,6,7,8,10,13,15],rightarrow:[1,2,3,5,6,9,10,15],rigor:1,rise:1,risk:[1,5],river:1,rmse:1,rmsprop:2,rnd_clf:8,rnn:10,rntrick1:15,rntrick2:15,rntrick3:15,rntrick4:15,robert:17,robust:1,robustscal:1,roc:8,role:[1,3,4,6,12],room:[1,16],root:[1,3,5,7,15],rot:[],rotat:[2,6,7,8],rotation_matrix:7,roughli:2,round:[1,5,7],routin:[5,13],row:[1,2,3,4,5,7,9,13],rrr:3,rthe:3,rug:5,rule:[1,2,3],run:[0,1,2,3,4,5,6,7,9,12],runtim:[0,2,4],runtimewarn:2,rust:[1,12,13],rvert:2,rvert_2:2,rwidth:4,s_i:5,saddl:5,safe:15,sai:[0,1,2,3,4,5,6,7,8,9,10,13,15],said:[4,5,7],sake:[1,3,5,9],sale:1,sam:[],same:[0,1,2,3,4,5,6,7,9,10,13,15],samm:8,sampl:[0,1,2,3,4,5,6,7,8,12,13],sample_vari:0,sampleexptvari:15,samwis:[],sanitize_sequ:4,sastri:9,satisfactori:1,satisfi:[2,4,5,6,13,15],satur:[2,4],save:[1,4,5,7],save_fig:[1,4,5,7,8],savefig:[1,4,5,7,15],saw:3,scalabl:8,scalar:[3,4,8],scale:[1,2,3,5,6,7,8,9,10,12,16],scaler:[1,5,6,7,8,9],scan:[3,5],scari:3,scatter:[0,1,2,4,5,6,7],scenario:5,scheme:[2,5],schrage:15,scienc:[1,2,5,8,10,12,14,15,17],scientif:[1,12],scientist:[0,1],scikit:[3,6,7,8,12,13,14,17],scikitlearn:1,scikitplot:[5,8],scipi:[1,3,5,12,13],score:[1,2,4,5,7,8,9,16],scores_kfold:4,scratch:2,sdg:5,seaborn:[1,2,5],seamless:[1,12],search:[1,2,3,5,7],sec:4,second:[0,1,3,4,5,6,7,9,10,12,13,15],secondeigvector:9,secondli:10,section:[0,9,11,13,14,15],sector:1,see:[0,1,2,3,4,5,6,8,9,10,12,13,14,15],seed:[0,1,2,3,4,5,6,7,9,15],seek:[2,6],seem:2,seemingli:1,seen:[1,2,3,8,10,15],segment:5,seldomli:1,select:[2,3,4,6,7,8,9,14,17],self:[2,3,4,15,17],semest:[5,14],semi:[5,6],send:[3,10,16],senior:14,sens:[1,4,6],sensit:[1,3,4,7],sentenc:10,separ:[0,1,2,4,6,7,10,12,15],sequenc:[0,5,7,8,10,12,13,15],sequenti:[2,8,10,15],seri:[1,2,3,5,8,9,10,13,14],serif:[1,5,15],serv:[1,2,3,5,17],session:[2,14],set:[0,2,3,4,5,6,8,9,12,13,15],set_:4,set_label:4,set_stream:3,set_tick:[2,6],set_ticklabel:2,set_titl:[0,1,2,5,10],set_xlabel:[1,2,5,10],set_xlim:[5,10],set_xticklabel:2,set_ylabel:[1,2,5],set_ylim:[5,10],set_ytick:5,set_yticklabel:2,setiosflag:15,setminu:4,setosa:[6,7],setosa_or_versicolor:6,setp:4,setprecis:15,setse:3,setup:[2,3,6,12],setw:15,sever:[1,3,4,5,6,7,9,10,12,13,14,15],sgd:2,sgd_clf:6,sgdclassifi:6,sgdreg:5,sgdregressor:5,sgn:3,shape:[0,1,2,3,4,5,6,7,8,9,13],share:2,she:5,shift:[2,10,15],shire:[],shortcom:5,shorter:15,shorthand:[],shortli:13,should:[0,1,3,4,6,7,9,10,13],show:[0,1,2,3,4,5,6,7,8,9,10,13,15],shown:[1,3,5,6,9,10,15],showpoint:15,shrink:[3,6,9],shrinkag:3,shrunk:9,shuffl:[1,2,4],side:[1,3,5,6,10,13],sigh:12,sigma0:15,sigma1:15,sigma2:15,sigma:[1,2,3,4,5,8,9,10,13,15],sigma_0:3,sigma_1:3,sigma_2:3,sigma_:[3,13,15],sigma_fn:[5,10],sigma_i:[1,3],sigma_j:3,sigma_m:15,sigma_n:[9,15],sigma_x:15,sigmoid:[2,5,6,8,10],sigmundson:16,sign:[2,5,6,8,15],signal:[2,8,10],signific:2,significantli:[2,5,15],sim:[3,4,15],similar:[0,1,2,3,4,5,6,7,8,9,12,13],similarli:[1,2,3,6,8,15],simpl:[0,2,3,4,6,8,9,10,12,13],simplepredict:8,simpler:[0,1,2,3,12],simplest:[0,1,2,7,8,10],simpletre:8,simpli:[1,2,3,4,6,7,8,9,10,12,13,15],simplic:[0,3,5,6,7,8,9,10],simplicti:3,simplifi:[1,4,7,12],simplist:15,simul:[4,15],simultan:4,sin:[1,2,7,10,13],sinc:[1,2,3,4,5,6,7,8,9,13,15,17],sine:10,singl:[2,3,5,6,7,10,13,15],singular:[1,4,5,13],site:[1,2,4,5,6,9,14],situat:[1,3,5],six:15,size:[1,2,3,4,5,6,7,8,9,13,15],sketch:8,ski:7,skill:1,skip:9,skl:1,sklearn:[1,2,3,4,5,6,7,8,9],skplt:[5,8],slack:6,slice:13,slide:[1,15],slight:4,slightli:[2,3,4,8,15],slope:[6,9,10],slow:[1,5,6],slower:[3,13],slowli:10,slp:2,small:[0,1,2,3,4,5,6,7,8,9,10,12,13,15],smaller:[1,2,3,4,5,6,7,9,15],smallest:[0,1],smallest_row_index:0,smart:4,smooth:[1,5],sne:9,sneak:1,sns:[1,2,5],soar:4,social:1,soft:[2,5,8,10],soften:6,softmax:5,softwar:[1,6,12,13,14],sol:6,sole:1,solid:[1,5],solitem:3,soltyp:3,solut:[1,2,3,4,5,6,8,9,13,15],solutionsummari:3,solv:[1,2,3,6,8,9,10,13],solver:[3,5,6,7,8,9,13],some:[0,1,2,3,4,6,7,8,9,10,13,15],someth:[1,2,5,7,9,15],sometim:[0,1,2,9,10],soon:13,sophist:1,sopt:5,sort:[3,4,7,9,15],sound:3,sourc:[0,1,2,4,12,13,15],space:[0,1,2,3,5,6,7,9,10,15],span:[1,3,7,9,13],spare:2,spars:[3,13],sparse_mtx:13,sparsiti:8,spatial:[2,10],spdiag:3,speak:15,special:[4,5,8,10,13,15],specif:[1,2,3,4,5,6,7,9,10,12,13,15],specifi:[0,1,3,4,5,7,9,15],specifici:[1,8],spectral:2,speech:[1,2,10],speed:2,spend:15,sphere:1,spite:1,spline:6,split:[0,2,3,4,6,7,8,9,15],splitter:[2,8],spmatrix:3,spontan:15,spread:[1,9,15],springer:17,sqquar:3,sqrt:[1,3,4,5,6,8,9,15],squar:[0,2,4,5,6,7,9,12,13,15],squarederror:8,squaredeuclidean:0,squash:10,srand:15,stabil:3,stabl:[1,3,5,7,9,12],stack:4,stage:[3,5],stai:[1,3,9],stand:[1,3,7,10],standard:[1,2,3,4,6,8,10,13],standardscal:[1,5,6,7,8,9],stanford:5,start:[0,1,2,3,4,5,6,7,8,9,10,13,15],start_tim:0,startpoint:15,stat:4,state:[0,2,3,5,6,8,9,10,12,15],statement:[1,5,13],statis:[],statist:[0,1,2,5,7,8,9,10,13,14,17],statu:[1,5,9],std:[1,4,15],stdev:15,stdout:3,steep:5,step:[0,1,2,7,8,9,10,13,15],step_fn:[5,10],step_length:5,steps_list:7,stian:16,still:[3,4,5,9,15],stimuli:10,stk2100:17,stk4021:17,stk4051:17,stk5000:17,stk:17,stochast:[1,2,3,4,6,9,10],stoke:10,stone:[1,5],stop:[0,2,7,9,15],storag:3,store:[1,2,5,9,15],str:2,straight:[1,4,5,6],straightforward:[1,3,4,5,6,7,8,13],strategi:[1,2,7],stratifi:4,streamtyp:3,strength:[0,1,3],stretch:9,strict:[5,6],strictli:[5,6],string:[2,15],stroke:5,strong:[7,8,10,15],strongli:[1,6,12,13],stronli:1,structur:[0,1,2,4,7,8,10,12],stuck:[2,5],student:[1,14,16,17],studi:[0,1,3,5,6,9,10,12,17],studier:17,style:[1,5,7],sub:[7,10],subdivid:[1,13],subfield:1,subject:[3,6,15],subplot:[0,1,2,4,5,6,7,8],subplots_adjust:[6,15],subprogram:13,subract:1,subroutin:1,subscript:2,subsequ:[2,3,4,10,13,15],subset:[2,4,5,7,10,12],subspac:[1,6,9],substanti:[7,8],substep:9,substitut:[4,10,13],subsubset:7,subtl:2,subtract:[1,3,4,9,13,15],subtre:7,succeed:1,success:[5,7,15],successfulli:7,sucess:15,sudo:[1,12],suffer:[1,2,3,8],suffici:[2,4,5,6,9],suggest:[2,5,17],suit:[6,10],suitabl:[1,15],sum:[0,1,2,3,4,5,6,7,8,9,10,15],sum_:[0,1,2,3,4,5,6,7,8,9,10,13,15],sum_i:[3,4,5,6],sum_k:[6,10,13],summar:[0,3,4,7],summari:[0,2,8,14],summat:[],sunni:7,superscript:[2,10],supervis:[1,3,4,5,7,10,12],supplement:5,support:[1,2,7,8,9,12],suppos:[1,3,4,5,6,8,9,10,13],suppress:3,sure:2,surfac:1,surpris:1,surround:12,survei:[1,3],svc:[6,7,8],svd:[1,4,9],svdinv:3,svm:[6,7,8,9],svm_clf:[6,8],swath:3,symbol:[2,3,9,12,15],symmeteri:2,symmetr:[1,3,5,6,9,10,13],sympi:[1,12],synonim:15,syntax:[2,13],syntaxerror:[2,6,13],syrk:3,sys:[3,5],system:[1,2,3,5,7,8,10,12,13,17],systemat:[4,15],t_0:[5,7],t_1:5,t_b:8,t_i:[2,10],t_j:[3,10],t_k:7,tabl:[7,15,16],tabul:1,tabular:[],tackl:0,tag:[0,3,5,10,13,15],taht:1,tail:15,tailor:[6,9],taiwan:1,take:[0,1,2,3,4,5,6,7,8,9,10,12,13,15],taken:[1,2,4,8,13],tangent:[2,5,10],tanh:[2,5,6,10],target:[1,2,3,5,6,7,8,9,10],target_nam:7,task:[0,1,2,3,4,7,9,10],tau:[3,15],tax:1,taylor:5,taylornr:5,team:2,teaser:1,technic:[0,1,3],techniqu:[1,2,6,8,12,15,17],technolog:[1,2],tek5040:17,tell:[1,4,5,8,9,15],temp1:2,temp2:2,temp:2,temperatur:[1,7],temporarili:2,ten:[],tend:[0,3,4,6,7,8,10],tendenc:1,tension:4,tensorflow:[0,1,6,12,13,14,17],term1:[3,9],term2:[3,9],term3:[3,9],term4:[3,9],term:[0,1,2,3,4,5,6,7,8,9,10,15],termin:[1,3,7,8],test:[3,4,5,6,7,8,15],test_accuraci:2,test_data:0,test_ind:4,test_pr:2,test_predict:2,test_scor:[5,8],test_siz:[1,2,3,4,8],test_split:7,tester:[],testerror:[1,4],text:[1,2,3,5,6,7,9,13,15,17],textual:7,textur:2,than:[1,2,3,4,7,8,9,10,12,15],thats:0,theano:[2,12],thei:[0,1,2,3,4,5,6,7,9,10,13,15],them:[1,2,5,6,7,8,9,10,13],theme:1,themselv:[1,15],thenc:4,theorem:[4,5],theoret:[1,8],theori:[1,2,5,6,7,10,12,17],thereaft:[1,4,9,10,13],therebi:[1,3,5,9,15],therefor:[1,2,4,5,6,9,15],therein:9,thereof:[1,4,5],theta:[2,4,5,15],theta_:2,theta_i:2,theta_k:15,theta_linreg:5,thi:[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,17],thing:[0,1,2,3,5,7,15],think:[0,1,2,4,5,7,10,15],third:[1,5],thirti:5,thorough:0,those:[0,3,6,7,8,9,13,14],though:[2,13,15],thought:[0,4,15],thousand:[1,2],three:[1,2,3,4,6,7,10,13,14,16],threshold:[2,5,7,8,9,10],through:[0,1,2,3,5,6,9,10,12,13,15],throughout:[0,1,3,12,13,15],thu:[1,2,3,4,5,6,8,9,10,15,16],thumb:1,thursdai:14,tibshirani:[14,17],ticker:[5,15],tight_layout:[2,5],tightli:9,tild:[1,3,4,9,15],till:[1,5,6,7,8,10,13],time:[0,1,2,3,4,5,6,7,8,9,10,13,14,15],timefunct:15,tini:2,tip:11,titl:[1,2,4,5,6,7,8,15],to_categor:2,to_categorical_numpi:2,to_numer:[1,4],to_str:15,togeth:[1,6,9],toi:0,toler:0,tomographi:10,too:[1,3,4,5,7,9,15,17],took:6,tool:[0,1,2,4,12,15],toolbox:6,top:[1,3,4,7,8,12],topic:[0,1,3,5,6,12,17],topolog:[2,10],toss:8,total:[0,1,2,4,5,6,8,9,10,15,16],totalclustervari:0,totalscatt:0,totalvari:15,toward:[2,5,10],town:1,tpng:7,traceback:[4,5,7,8],track:[0,5,13],tract:1,tractabl:1,trade:[3,4,7],tradeoff:[1,3,14],tradit:[1,2,4],train:[3,4,5,6,7,8,9,10],train_accuraci:[1,2],train_end:[1,2],train_ind:4,train_pr:2,train_siz:[1,2],train_test_split:[1,2,3,4,5,7,8,9],train_test_split_numpi:[1,2],trainerror:1,trainingerror:4,trait:1,tran:3,transfer:7,transform:[1,3,4,5,6,7,8,9,10,12,13],transit:10,translat:[2,8],transpos:[2,3,9],treat:[1,2,4,5,10,15],tree:[1,2,12,14],tree_clf:[7,8],tree_clf_:7,tree_clf_sr:7,tree_reg1:7,tree_reg2:7,tree_reg:7,trend:15,trevor:17,tri:7,triain:1,trial:[1,4,5,15],triangl:5,triangular:13,trick:[6,9,15],trickier:15,tridiagon:13,trillion:12,trivial:[1,2,3,9,15],troubl:[1,6,10],true_divid:2,true_fun:4,tucker:6,tumor:[5,7],tumour:5,tunabl:2,tune:[7,13],turn:[1,2,3,4,5,6,7,8,9,10,13,15],tutori:2,tweak:[2,8,15],twice:5,twist:9,twister:15,two:[0,1,2,3,4,5,7,8,9,10,13,14,15,17],tx_1:5,type:[1,2,4,5,6,8,13,15],typeerror:[],typic:[1,2,3,5,7,8,10,15],u_i:10,u_m:8,ubuntu:[1,12],uci:1,uio:[16,17],unari:13,unbalanc:[4,7],unbias:[1,3,4,15],uncertainti:[1,3],uncertitud:15,unchang:2,uncorrel:8,undefin:3,under:[1,2,3,4,5,8,12],underdetermin:1,underfit:[2,4],underflowproblem:3,undergo:3,undergradu:14,underli:[1,2,5,7,15],underset:0,understand:[0,1,2,3,5,8,12],understood:[0,6],undesir:6,undetermin:[3,6],unexpect:[4,15],unexpected:15,unfortun:[2,6,7,8],unicode_liter:[6,7],uniform:[1,2,3,5,9],uniform_real_distribut:15,uniformli:[5,15],unifrompdf:15,unimport:5,union:[3,4],uniqu:[0,1,4,5,13],unique_cluster_label:0,unit:[1,2,3,8,10,15],unitari:[3,13],unitarili:13,uniti:15,univari:15,univers:[1,2,5,14,16],unix:2,unknow:[1,13],unknown:[1,2,3,4,6,8,13],unknowwn:10,unlabel:2,unless:[1,4,5,9],unlik:[2,5,6,15],unnecessarili:7,unravel:2,unrol:9,unseen:[5,7],unstabl:2,unsupervis:[1,2,10,12,14],unsymmetr:13,until:[0,2,5,7,10,15],unusu:10,updat:[0,2,4,8,10],upload:[12,17],upon:[2,9,13],upper:[1,6,7,13],uppercas:[13,15],ups:[],usag:[1,6,12,15],usd10000:1,usd:1,use:[0,1,3,4,5,6,7,8,9,10,12,13,14],usecol:1,used:[0,1,2,3,4,6,7,8,9,10,12,13,15,17],useful:[1,2,3,4,5,7,9,10,12,13,15,17],useless:2,user:[1,2,5,6,9,12,13],uses:[1,2,3,4,7,9,10,13,15],usetex:15,using:[0,2,3,4,6,7,8,9,10,13],usr:15,usual:[0,1,5,10],util:[0,2,4,5,8],v_0:9,valid:[1,2,5,7,8,12,14,15],valu:[0,1,2,4,5,6,7,8,10,12,13],valuat:7,valueerror:[],van:1,vandenbergh:[5,6],vandermond:1,vanilla:[1,9],vanish:[2,5,15],var_x:15,varabl:6,varepsilon:[3,4],varepsilon_:[3,4],varepsilon_i:[3,4],vari:[1,2,4,8],variabl:[0,1,2,3,4,6,8,9,10,13],varianc:[0,1,2,3,5,7,8,9,12,13,14],variance_i:[3,9],variance_x:[3,9],variant:[1,2,4,5,6,10],variat:9,varieti:[1,10,12],variou:[2,3,5,6,7,9,10,12,13,15],vartempvec:15,varvec:15,vaue:2,vdot:5,vec:[4,15],vector:[0,1,2,3,4,5,7,8,9,12],vector_mean:0,ventur:[1,6,12],verbos:2,veri:[0,1,2,3,4,5,6,7,8,9,10,15,17],verifi:[9,13],versatil:6,versicolor:[6,7],version:[0,1,8,12,13,15],versu:2,vert:[1,2,3,5,6,7,9],vert_1:3,vert_2:[3,9],vertic:4,via:[1,3,4,5,6,7,8,9,10,12,13,14,15],vidal:9,video:[1,2,5,10,12,14],view:[2,3,10,15,17],violat:6,virginica:7,viridi:[1,2],virtual:2,vision:1,visual:[1,9,10,12],visualis:2,viz:[6,15],vmax:2,vmc:15,vmin:2,volum:1,vote:8,voting_clf:8,votingclassifi:8,votingsimpl:8,vstack:[3,9,13,15],w_1:[6,13],w_1x_1:6,w_1x_:6,w_2:[6,13],w_2x_2:6,w_2x_:6,w_3:13,w_4:13,w_i:[2,8],w_ix_i:10,w_j:13,w_m:13,w_px_:6,w_px_p:6,wai:[0,1,2,3,4,5,6,8,9,10,11,13,15],walk:7,walker:15,wang:1,want:[0,1,2,3,4,5,6,7,8,9,10,12,15],warn:[1,2,6],warrant:4,watch:12,wavelet:6,weak:[0,7,8],weather:[2,10],web:[12,14],websit:[13,14],wedg:[6,15],wednesdai:14,wee:9,week:[3,5],weekli:[12,17],weight:[1,2,4,5,7,8,10,15],welcom:[6,12],well:[1,2,3,4,5,6,7,8,10,12,13,15,17],went:6,were:[0,1,2,3,4,5,6,8,9,10,15],wessel:1,what:[0,2,3,4,5,6,7,8,9,10,12,13,14],when:[0,1,2,3,4,6,7,8,9,10,13,15],whenev:15,where:[0,1,2,3,4,5,6,7,8,9,10,12,13,15,16],wherea:[4,15],wherein:[2,10],whether:[1,3,5,7,15],which:[0,1,3,4,5,6,7,8,9,10,12,13,14,16],whichev:2,white:7,who:[1,14],whole:[0,2,3,7,9],whose:[1,4,8,15],whow:[3,9],why:[1,2,5],wide:[1,2,4,5,10,12,13],widehat:4,width:[1,6,7],wieringen:1,win:8,wind:7,wing:16,wiscons:5,wisconsin:8,wise:[1,2,3,10],wish:[0,1,3,5,6,9,13],with_std:1,within:[0,1,5,7,10,15,17],withinclust:0,without:[1,2,3,5,6,7,9,10],won:1,wonder:6,word:[0,1,2,3,15],work:[0,1,2,4,5,6,7,12,14,15],world:[1,6],worldwid:1,wors:[1,2,4],worth:7,would:[1,2,3,4,5,6,7,8,9,10,13,15],wrap:13,write:[1,2,3,5,6,10,11,13,14,15],written:[1,3,5,9,10,12,13,15],wrong:[2,6],wrongli:[8,15],wrote:[3,9],wrt:8,wth:8,www:[12,13,17],wx_1:6,x0s:6,x1_exampl:6,x1d:6,x1s:[6,7,8],x2d:[6,9],x2d_train:9,x2dsl:9,x2s:[6,7,8],x3s:6,x_0:[1,3,9,13],x_1:[1,3,4,5,6,7,8,9,13,15],x_2:[1,3,4,5,6,7,8,9,13,15],x_3:[6,13,15],x_4:13,x_center:9,x_data:2,x_data_ful:2,x_i:[0,1,2,3,4,5,6,7,8,9,10,13,15],x_ix_:1,x_iy_i:6,x_j:[1,6,7,10,15],x_jy_j:6,x_k:[0,10,13,15],x_l:15,x_m:[10,13,15],x_n:[1,4,5,6,9,10,13,15],x_new:[7,8],x_p:[5,7],x_poli:7,x_poly10:7,x_reduc:9,x_scale:6,x_test:[1,2,3,4,5,7,8,9],x_test_scal:[1,5,7,8,9],x_train:[1,2,3,4,5,7,8,9],x_train_scal:[1,5,7,8,9],x_val:2,xarrai:12,xavier:2,xbnew:5,xcode:[1,12],xdclassiffierconfus:8,xdclassiffierroc:8,xg_clf:8,xgb:8,xgbclassifi:8,xgboost:7,xgboot:8,xgbregressor:8,xgparam:8,xgtree:8,xi_1:6,xi_:6,xi_i:6,xlabel:[1,2,3,4,5,6,7,8,15],xlim:[4,8],xmesh:5,xnew:[1,5],xpanda:1,xpd:[3,9],xplot:1,xscale:1,xsr:7,xt_x:5,xtest:4,xtick:[4,6,7],xtrain:4,xytext:6,y_0:[1,3,9,13],y_1:[1,3,5,6,7,9,13],y_1y_1:6,y_1y_1k:6,y_1y_2:6,y_1y_2k:6,y_1y_n:6,y_1y_nk:6,y_2:[1,3,6,7,9,13],y_2y_1:6,y_2y_1k:6,y_2y_2:6,y_2y_2k:6,y_3:[1,7,13],y_4:13,y_data:[1,2,3],y_data_ful:2,y_decis:6,y_fit:1,y_i:[1,2,3,4,5,6,7,8,9,10,13],y_if_:8,y_ix_:1,y_ix_i:[5,6],y_iy_jk:6,y_j:[4,6,10],y_k:10,y_m:13,y_model:[1,3],y_n:[5,6],y_ny_1:6,y_ny_1k:6,y_ny_2:6,y_ny_2k:6,y_ny_n:6,y_ny_nk:6,y_plot:7,y_pred1:7,y_pred2:7,y_pred:[1,2,4,5,6,7,8],y_pred_rf:8,y_pred_tre:8,y_proba:[5,8],y_test:[1,2,3,4,5,7,8,9],y_test_onehot:2,y_test_predict:1,y_train:[1,2,3,4,5,7,8,9],y_train_onehot:2,y_train_predict:1,y_val:2,year:[1,12],yes:[4,5],yet:[1,2,4,6,9],yield:[0,1,3,4,5,6,8,10,13,15],ylabel:[1,2,3,4,5,6,7,8,15],ylim:4,ymesh:5,yoshua:[2,17],you:[0,1,2,3,4,5,6,7,8,9,12,13,15,17],young:1,your:[0,2,4,5,6,9,12,13,15],yourself:[5,9],youtub:12,ypred:4,ypredict2:5,ypredict:[1,5],ypredictlasso:3,ypredictol:3,ypredictridg:3,yridg:1,ytest:4,ytick:[4,6,7],ytild:[1,4],ytildelasso:3,ytildenp:1,ytildeol:3,ytilderidg:3,ytrain:4,z_0:13,z_1:13,z_2:13,z_c:2,z_h:2,z_i:[2,10],z_j:[2,10],z_k:10,z_m:2,z_mod:7,z_o:2,zaman:15,zero:[0,1,2,3,4,5,6,7,8,9,10,13,15],zm_h:1,zone:1},titles:["11. Clustering Analysis","3. Linear Regression","13. Building a Feed Forward Neural Network","4. Ridge and Lasso Regression","5. Resampling Methods","6. Logistic Regression","7. Support Vector Machines, overarching aims","8. Decision trees, overarching aims","9. Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods","10. Basic ideas of the Principal Component Analysis (PCA)","12. Neural networks","Content in Jupyter Book","Applied Data Analysis and Machine Learning, FYS-STK3155/4155 at the University of Oslo, Norway","2. Linear Algebra, Handling of Arrays and more Python Features","Teaching schedule with links to material","1. Elements of Probability Theory and Statistical Data Analysis","Teachers and Grading","Textbooks"],titleterms:{"2021":16,"4155":12,"case":[5,6,8,15],"final":[5,10],"function":[1,2,5,6,8,9,10,15],"import":[3,13,15],And:5,Eye:8,FYS:12,OLS:3,The:[0,1,2,3,4,5,6,7,9,10,12,15],Useful:12,Using:5,activ:[2,10],actual:15,adaboost:8,adapt:8,adjust:2,again:7,aim:[6,7],algebra:13,algorithm:[0,7,8,9,10],algortithm:5,all:6,analys:3,analysi:[0,1,3,9,12,15],ani:5,anoth:7,appli:12,approach:[1,5,6],approxim:10,architectur:2,arrai:13,assist:16,august:14,autocorrel:15,back:[2,9,10],background:12,bag:8,basic:[0,1,3,5,7,8,9,13],batch:2,bay:3,befor:9,better:[6,15],bia:4,binari:2,binomi:15,bird:8,block:15,book:11,boost:8,bootstrap:[4,8,15],boston:1,breast:2,brief:5,bring:10,build:[2,7],calcul:15,cancer:[2,5,7,9],cart:7,central:[5,12,15],chain:10,chang:8,chi:1,choos:2,classic:9,classif:[2,7,8],classifi:6,clip:2,cluster:0,code:[0,1,2,3,5,7,9,10,15],collect:2,compar:8,complex:1,compon:9,comput:[5,7,15],con:7,concept:15,condit:5,conjug:5,content:11,continu:15,convex:[5,6],convolut:10,correl:[9,15],correspond:5,cost:[2,5,8],cours:[12,17],covari:[3,9,15],cross:4,cumul:15,cython:[],data:[1,2,5,7,9,12,15],dataset:2,decemb:14,decis:[7,8],decomposit:[3,9,13],deep:2,defin:2,definit:15,degre:1,demonstr:15,dens:1,deriv:[3,5,10],descent:[5,8],develop:2,deviat:15,diagon:9,dice:15,differ:[5,6],dimension:6,disadvantag:7,discret:15,disguis:15,distribut:[3,15],doing:2,domain:15,down:2,dropout:2,economi:[],element:[1,15],elimin:13,ensembl:8,entropi:7,environ:1,equat:[1,5,10],error:[1,8],etc:[],evalu:2,event:15,exampl:[1,2,5,6,7,8,15],exercis:1,expect:15,experi:15,explor:1,exponenti:15,express:5,extend:5,extrem:8,fall:16,famili:2,famou:15,fantast:[],featur:[7,13],feed:[2,10],fine:2,first:[5,10],fit:[1,8],forest:8,forward:[2,10],frank:[],freedom:1,frequentist:1,fridai:5,from:[3,8,10],further:3,gaussian:[13,15],gener:[7,15],geometr:[5,9],gini:7,good:1,grade:16,gradient:[2,5,8],handl:13,has:12,hessian:5,homework:5,hous:1,how:15,hyperparamet:2,hyperplan:6,id3:7,idea:[0,9],ideal:5,implement:[2,15],implic:3,improv:2,increment:9,index:7,inform:16,instal:12,instructor:16,interpret:[3,5,9],introduc:9,introduct:[1,4,12,13],invers:[3,13],iter:[5,8],its:15,jackknif:15,julia:[],jungl:8,jupyt:11,kera:2,kernel:[6,9],lagrangian:6,lasso:3,later:3,layer:[2,10],learn:[1,2,5,9,12],least:3,level:8,librari:12,likelihood:5,limit:[2,5,15],linear:[1,6,13],link:[3,9,14,17],logist:5,loss:5,machin:[1,5,6,12],main:15,make:[1,7,8],mani:[8,10],materi:14,math:3,mathemat:[3,6],matric:3,matrix:[2,3,5,9,10,13,15],matter:1,mean:[0,1,15],meet:[3,8,15],mercer:6,mersenn:15,method:[4,5,7,8,15],mlp:10,model:[1,2,10],moment:15,moon:[6,7],more:[0,5,13],multilay:10,multipl:2,multipli:6,name:15,need:[],network:[2,5,10],neural:[2,10],newton:5,non:6,normal:[1,2,15],norwai:12,notat:10,novemb:14,now:[2,7],nuclear:1,nueral:5,numba:[],number:[1,15],numer:15,numpi:13,numpython:0,observ:15,obtain:9,octob:14,one:[5,10],optim:[2,5,6,12],ordinari:3,organ:1,oslo:[12,17],other:[7,9,10,15],our:[0,1,3,5,9,15],outcom:12,output:15,overarch:[1,6,7],overview:8,own:[0,1,8,9],packag:13,panda:[],part:[5,12],pass:2,pca:9,pdf:15,perceptron:10,perform:[2,7],period:15,perspect:2,poisson:15,pre:2,preprocess:[],prerequisit:12,princip:9,pro:7,probabl:[3,15],problem:[2,5],procedur:7,process:2,program:5,propag:[2,10],properti:[3,15],pseudo:15,python:[0,1,7,12,13,15],quick:6,ran0:15,random:[8,9,15],raphson:5,read:7,recip:15,recurr:10,reduc:1,regress:[1,3,5,7,8],regular:2,relev:17,relu:2,remind:[4,5,6],requir:12,resampl:4,revisit:5,ridg:[3,5],rng:15,rule:10,sampl:[9,15],schedul:14,schemat:7,scikit:[1,2,5,9],select:15,semest:16,sensit:5,septemb:[5,14],set:[1,7,10],sgd:5,should:[2,15],simpl:[1,5,7,15],singl:8,singular:[3,9],situat:15,size:[],slightli:5,soft:6,softmax:2,softwar:[],solv:5,some:5,split:1,squar:[1,3,8],standard:[5,15],state:1,statist:[3,4,12,15],steepest:[5,8],step:[4,5],stk3155:12,stochast:[5,15],stop:5,supervis:2,support:6,svd:3,teach:[14,16],teacher:16,techniqu:9,technolog:12,tensorflow:2,test:[1,2],textbook:17,than:5,theorem:[3,6,9,10,15],theori:15,three:15,togeth:10,top:2,toss:15,toward:[0,9],tradeoff:4,train:[1,2],tree:[7,8],tune:2,two:[6,12],type:10,uncorrel:15,understand:[],uniform:15,univers:[10,12,17],use:[2,15],used:5,using:[1,5,15],valid:4,valu:[3,9,15],variabl:[5,15],varianc:[4,15],variou:[1,4],vector:[6,10,13],view:[1,8],visual:[2,7],wai:7,week:14,weekli:14,what:[1,15],when:5,which:[2,15],why:15,wisconsin:5,write:[0,9],xgboost:8,your:[1,8]}})
\ No newline at end of file
diff --git a/doc/LectureNotes/_build/jupyter_execute/chapter1.ipynb b/doc/LectureNotes/_build/jupyter_execute/chapter1.ipynb
index 48cd1e448..0dbf33b5b 100644
--- a/doc/LectureNotes/_build/jupyter_execute/chapter1.ipynb
+++ b/doc/LectureNotes/_build/jupyter_execute/chapter1.ipynb
@@ -6,8 +6,6 @@
"source": [
"# Linear Regression\n",
"\n",
- "[Video of Lecture](https://www.uio.no/studier/emner/matnat/fys/FYS-STK3155/h20/forelesningsvideoer/LectureAug21.mp4?vrtx=view-as-webpage)\n",
- "\n",
"\n",
"## Introduction\n",
"\n",
@@ -15,7 +13,7 @@
"\n",
"\n",
"\n",
- "Our emphasis throughout this series of lectures (small change) \n",
+ "Our emphasis throughout this series of lectures \n",
"is on understanding the mathematical aspects of\n",
"different algorithms used in the fields of data analysis and machine learning. \n",
"\n",
@@ -235,7 +233,9 @@
"\n",
"## Simple linear regression model using **scikit-learn**\n",
"\n",
- "We start with perhaps our simplest possible example, using **Scikit-Learn** to perform linear regression analysis on a data set produced by us. \n",
+ "We start with perhaps our simplest possible example, using\n",
+ "**Scikit-Learn** to perform linear regression analysis on a data set\n",
+ "produced by us.\n",
"\n",
"What follows is a simple Python code where we have defined a function\n",
"$y$ in terms of the variable $x$. Both are defined as vectors with $100$ entries. \n",
@@ -301,7 +301,7 @@
"outputs": [
{
"data": {
- "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAAmVElEQVR4nO3debxVZf0v8M+X4TAjMmiCcjCFTFFRDoJTmGUqmmb3V9kP+2l6RS3Rcuqmdc2CBpNyuP1USiuF1NT0RWaZ94oSKsgBlUTNjBkRGUQBZTjnfO4fz9rtfQ573mvc6/N+vfbrnD2t9ay1936+az3DdxlJiIhIenWKugAiIhItBQIRkZRTIBARSTkFAhGRlFMgEBFJOQUCEZGUUyCQosxsopn9NaBl/8bMpvi8zMDKmzZmttXMPhp1OSR4CgQCMzvOzJ4zs/fMbJOZPWtmYwCA5EySn4m6jB2ZGc3swI6Px6m8XqDb6VWom8zsSTM7KOpylYtkb5JLoy6HBE+BIOXMrC+AxwDcBqA/gCEAbgCwI8pyJY2ZdSnw1I0ke8Pt1zUA7gpx3SJlUSCQEQBA8j6SrSQ/JPlXkosBwMzOM7O5mRd7R+JfM7N/mtkWM/uBmR3gnVG8b2a/N7MG77UnmNlqM7vWzDaY2XIzm1ioIGZ2upm9ZGabveUdVunGFCjvxV55N5vZL8zMcp4/38xeM7N3zewJM2vMee4WM1vlbddCMzs+57nvmdlDZjbDzN4HcF6xcpH8EMDvAYzKWcZgM3vYzNab2TIzuyznuR5m9luvXK+Z2TVmtjrn+eVm9i0zWwxgm5l1MbNx3n7bbGYvm9kJHfbLUu8zW5b5HMzsQDN7xjsb3GBmD3TYdwd6/+9hZvd4ZV1hZt8xs065+9zMbvLKu8zMTi3n85J4UCCQNwC0epXOqWa2ZxnvORnAaADjAFwDYDqAcwDsB2AkgC/nvPYjAAbCHRGfC2C6mX2s4wLN7AgAdwO4CMAAAHcCmGVm3ardsBynAxgD4DAAX/TKDzM7E8C1AD4PYBCAvwG4L+d9C+Aq7v4AfgfgQTPrnvP8mQAeAtAPwMxiBTCzXnD75U3vficAfwTwMty++RSAb5jZyd5brgcwDMBHAZwEt387+jKA07z17w3gTwCmeOW9CsDDZjbIW/etAE4l2QfAMQBe8pbxAwB/BbAngH3hzgzzuQ3AHl55xgP4LwBfzXl+LIB/wH3WNwK4KzfgSrwpEKQcyfcBHAeAAH4JYL2ZzTKzvYu87UaS75NcAuAVAH8luZTkewD+DOCIDq//LskdJJ+Bq6y+mGeZkwDcSXK+d2byW7jmqXG1bSEA4MckN5NcCWA2skflFwP4EcnXSLYA+CGAUZmzApIzSG4k2UJyGoBuAHKD2PMkHyXZ5h3x53OVmW0GsAVuP3/Fe3wMgEEkv09yp9cW/0sAZ3vPfxHAD0m+S3I1XEXe0a0kV3nrPgfA4yQf98rzJIBmABO817YBGGlmPUiu9T47ANgFoBHAYJLbSc7tuBIz6+yV69skt5BcDmBazrYAwAqSvyTZCuC3APaBC06SAAoEAq8iPI/kvnBH9IMB3FzkLety/v8wz/3eOfffJbkt5/4Kb/kdNQK40mvW2OxVnvsVeG2l3s75/4Oc8jUCuCVnfZsAGNwROszsKq9Z5j3v+T3gjngzVpWx7ptI9oM7uv8Q2UDSCGBwh+29FtnKc3CH5edbV+5jjQC+0GF5xwHYx9v/X4ILfGvN7E+W7bS+xtvmF8xsiZmdn2c9AwF0hfvsMlbA20+ef+9jkh94/+Z+DyTGFAikHZKvA/gNXEDww55e00TGUABv5XndKgBTSfbLufUkeV+e1/plFYCLOqyzB8nnvP6Aa+COzPf0KvP34CrNjLJT93pnI5fDBZ4e3rqXdVh3H5KZI/i1cE01GfvlW2yHbbm3w/J6kfyxt/4nSJ4Ed6T+OtzZB0i+TfJCkoPhmuX+23YfjbUB2TOHjKFwnd9SBxQIUs7MDjKzK81sX+/+fnBtz/N8XM0NZtbgVa6nA3gwz2t+CeBiMxtrTi8zO83M+hRZboOZdc+5da6wXHcA+LaZHQL8u0P0C95zfQC0AFgPoIuZ/W8AfStcfjtec81bcM1gLwDY4nX49jCzzmY20rxhu3Ady982sz3NbAiAS0ssfgaAz5rZyd6yupvrrN/XzPY2szO9gLwDwFa4piKY2Rcynz2Ad+GCS1uHcrd65ZlqZn28prMrvHVKHVAgkC1wHX3zzWwbXAB4BcCVPi3/bbgK5i24DtWLvbOOdkg2A7gQwP/xXv8mSozEAbAErrklc/tq8Zfvts5HAPwEwP3mRv68AiAz2uUJAH+B60xfAWA7ymsKKuWncGcaXeCC4igAy+COun8F1/wEAN8HsNp77v/CdUoXHNJLchVc5/W1cMFrFYCr4X7jneAq7rfgmr/GA7jEe+sYuM9+K4BZAC4vMHdgMoBtAJYCmAvXeX53hdsuMWW6MI0ExRu+OMPre5AamNklAM4mOT7qskj90RmBSAyZ2T5mdqyZdfKG214J4JGoyyX1KZQZiWa2HK4JohVAC8mmMNYrkmANcHMp9gewGcD9AP47ygJJ/QqlacgLBE0kNwS+MhERqYiahkREUi6sM4JlyA5Nu5Pk9DyvmQQ3rA69evUafdBBiUnSKCISuYULF24gOaia94YVCIaQXGNmewF4EsBkknMKvb6pqYnNzc2Bl0tEpF6Y2cJq+19DaRoiucb7+w7cyIejwliviIiUFngg8GaI9sn8D+AzcBN3REQkBsIYPro3gEe8jLRdAPyO5F9CWK+IiJQh8EDgTVc/POj1iIhIdTR8VEQk5RQIRERSToFARCTlFAhERFJOgUBEJOUUCEREUk6BQEQk5RQIRERSToFARCTlFAhERFJOgUBEJOUUCEREUk6BQEQk5RQIRERSToFARCTlFAhERFJOgUBEJOUUCEREUk6BQEQk5RQIRERSToFARCTlFAhERFJOgUBEJOUUCEREUk6BQEQk5RQIRERSToFARCTlFAhERFJOgUBEJOUUCEREUk6BQEQk5RQIRERSToFARCTlFAhERFJOgUBEJOVCCwRm1tnMXjSzx8Jap4iIlBbmGcHlAF4LcX0iIlKGUAKBme0L4DQAvwpjfSIiUr6wzghuBnANgLZCLzCzSWbWbGbN69evD6lYIiISeCAws9MBvENyYbHXkZxOsolk06BBg4IuloiIeMI4IzgWwBlmthzA/QBONLMZIaxXRETKEHggIPltkvuSHAbgbABPkTwn6PWKiEh5NI9ARCTluoS5MpJPA3g6zHWKiEhxOiMQEUk5BQIRkZRTIBARSTkFAhGRlFMgEBFJOQUCEZGUUyAQEUk5BQIRkZRTIBARSTkFAhGRlFMgEBFJOQUCEZGUUyAQEUk5BQIRkZRTIBARSTkFAhGRlFMgEBFJOQUCEZGUUyAQEUk5BQIRkZRTIBARSTkFAhGRlFMgEBFJOQUCEZGUUyAQEUk5BQIRkZRTIBARSTkFAhGRsMycCQwbBnTq5P7OnBl1iQAAXaIugIhIKsycCUyaBHzwgbu/YoW7DwATJ0ZXLuiMQEQkHNddlw0CGR984B6PmAKBiEgYVq6s7PEQKRCIiIRh6NDKHg+RAoGISBimTgV69mz/WM+e7vGIKRCIiIRh4kRg+nSgsREwc3+nT4+8oxgIYdSQmXUHMAdAN299D5G8Puj1iojEzsSJsaj4Owpj+OgOACeS3GpmXQHMNbM/k5wXwrpFRKSEwAMBSQLY6t3t6t0Y9HpFRKQ8ofQRmFlnM3sJwDsAniQ5P89rJplZs5k1r1+/PoxiifgnpjNGRcoRSiAg2UpyFIB9ARxlZiPzvGY6ySaSTYMGDQqjWCL+yMwYXbECILMzRhUMJCFCHTVEcjOA2QBOCXO9IoEKY8aozjgkQIEHAjMbZGb9vP97ADgJwOtBr1ckNEHPGNUZR7hSGHTDOCPYB8BsM1sMYAFcH8FjIaxXJBxBzxiNS46aNFSQKQ265gb1xEtTUxObm5ujLoZIeTpmlQTcjFG/Jgt16uQqpY7MgLa22pdfjqC3MS6GDXOVf0eNjcDy5WGXpiJmtpBkUzXv1cxikVoFPWM0Djlq4nJWErQYJ4YLkgKBiB8mTnRHjG1t7q+fR8lxyFGTlgoyDkE3AgoEInEXhxw1aakg4xB0I6BAIJIEQZ5xlCMtFWQlQbeOOs8VCETSqNJKLA5nJWEpJ+jW2egijRoSSZu0jAAKUgxHF2nUkIiULy0jgIJUZ53nCgQiaVNnlVgk6qzzXIFAxA9J6jiMWyWWpH2XUW3n+cyZwMCBrp/FzP0fh+0lGbvb6NGjKZIYM2aQPXuSrtvQ3Xr2dI/HUZzKG6eyVGrGDLKxkTRzf0uVecYMsmvX9tsKkA0NvmwvgGZWWeeqs1ikVjHsOCxp5kzXJ7BypTsTmDo1mo7iJO67ahXaVsCX7a2ls1iBQKRWccgFlFRp2neFthXwZXs1akgkSnFrc0+SNO27YtsU8fYqEIjUKi2zboOQpn03dSrQtevujzc0RL69CgQitUrTrFu/pWnfTZwI/PrXwIAB2ccGDADuvjvy7S3ZR2BmTwK4iuTL4RRJfQQiIpUKuo/gWwBuNrNfm9k+1axERETiq2QgILmI5CcBPAbgL2Z2vXftYRERqQNl9RGYmQH4B4DbAUwG8E8z+0qQBRMRkXCUDARm9iyANQB+DmAIgPMAnADgKDObHmThREQkeOWcEUwCMITkSSS/S/Ixkm+SnAzg+IDLJyKSDhHmXCqnj2AJCw8tOs3n8oiI35KY1K1WSdvmiC90U9M8ApJL/SqIiASgzq6kVZYkbnOpa0QEHNiUa0iknqUpqVtGEre5WM6le+8t64pyyjUkIvml8SI0xbY5rk1GxXIuhXBFOQUCkXqWpqRuGYW2rX//+DYZFcu5FEIwVyAQqWdpSuqWUWibgfheq7lYzqUQgrkCgUg9S1NSt4xC27xpU/7Xh9FMVk6T1MSJrg+jrc39zXxGIQRzdRaLSDpE1YmcGcVUorO35DJKXFFOncUiIqVE1UwWQmdvrRQIRCQdomomq7WzN4R5EWoaEhEJUq1NUmW+X01DIiJxVWuTlIaPiojUKOpJZLU2SWn4qPgm6h+DSBTikneo0NDQcoTQyR14IDCz/cxstpm9amZLzOzyoNcpHcTlxyAStgSM2CkphE7uwDuLvesc70NykZn1AbAQwOdIvlroPeos9lkSk3CJ+KFYMre2tvDLE6BYdxaTXEtykff/FgCvwV3pTMKSxsRjUr6omw2DXH8acy1VIdQ+AjMbBuAIAPPzPDfJzJrNrHn9+vVhFqv+6ccghUTdbBj0+tOYa6kaJEO5AegN1yz0+VKvHT16NMVHM2aQPXuS7qfmbj17uscl3Rob238vMrfGxvpZ/4wZbnlm7m+dfu8BNLPK+jmUCWVm1hXAYwCeIPmzUq9XH0EAyshVIikUdRt61OuvI7HuIzAzA3AXgNfKCQISkFqGryVZ1O3f1QizzFE3G/bvH/z6k/gdCFu1pxLl3gAcB4AAFgN4ybtNKPYeNQ2JL5LYJBZ2maPcRzNmkA0NuzcLde3q3/qT+B2oEmpoGgqtj6CSmwKB+CLq9u9qRFHmqNrQC23rgAGVL6vQNiTxO1Cud98lH32UvOwy8tBDawoEXSI6EREJXhKHzYZd5ij7jgptU6ELyBTSMd9/ZuRRsXXE+TtQyJYtwNy5wOzZwFNPAYsWubDWowdw7LHA3/9e9aIVCKR+DR2afyJdnIfNhlnmYhVoGMHAr20tNns4id+BjA8/BJ57LlvxL1gAtLQADQ3AuHHA9dcDJ54IHHUU0K2b62CvVrWnEkHe1DQkvijVPhzHYYXltmn7Ufaom038ar83y78dZsnqI9ixg5wzh7zhBnL8+Gz/SefO5NFHk9deSz75JLltW963Q30EIgUUqjDjXEGUquTDqEDDEkZAi2PAJ8ldu8h588gf/Yg86aTsZ2pGHnkkedVV5OOPk++/X9biFAhEKhX10TBZfQXlV9njsA/8EOegnqu1lVy0iJw2jTztNLJPn2x5R450nb6PPEJu2lTV4hUIRCoV9dFwLZWXX2WPSwXqxxF77jIGDHC3qM8A2trIV14hb7uNPOsssn//7H4eMYK8+GLygQfIdet8WZ0CgUiloj4armX9fpY96mYTv4NRlMGtrY184w3yzjvJL32J3HvvbBmGDSPPP5+8915y9epAVl9LINA1iyWdOo6YAVwysjAuZg7Ulloh6rL7ye8U6WGnXF+xIjuqZ/ZsYPVq9/jgwW5Ezyc/6W777+//ujuoJcWEho9KOmUqzKjG0NcyrDHqsvvJ73H+Qc8bWLu2fcW/dKl7fNCgbKV/4onA8OG1DecMmc4IRKJQT0f1tYj7GcEddwDf/S6wYQPQpYsbxw8A/foBJ5yQrfgPOSTyij/WSedEJI8QLj+YCH5fL6DW5W3eDMyaBXzjG+5M65JLXBAAXBDo2hWYMsU99sgjwGWXASNHRh4EalZt50KQN3UWhyTqjkIR0v/vYSXL27KF/POfyWuuIZuayE6dXOdu9+7ulqDhtVBnsVRMTRPF6foN9enDD4Hnn8+287/wQvZIf9w418xz4onA2LEuh0+1HfoRqKVpSIEgrXRB+8IUJOvHzp2uss9U/M8/D+zYAXTuDDQ1ZUf2HHvs7k1KCfuNKBBI5XRlqMISVgFIjpYWl5UzU/HPnesCuhkwalS24j/+eKBv3+LLivqAoMKz0loCQeT9Aflu6iMIQdQTquIs6lnHlUhTP0++bW1tJV98kfzZz8jTTyf79s1+XoccQk6eTP7hD+TGjf6tMwzlTIzrULaBwFJWWedGXunnuykQhMDPGZj1VhklJUjGJUVEkDLfrUwgzt3Wzp3JXr2y94cPJy+6iLz/fvLtt6MueW3KSaTX4bM/EmilAoFUzK8cL/VWGSVlm5ISsKqV73PoeOvVi7znHnLVqqhL669SZ6V5PvvRAFllnas+AqlNvbanJ2HUUL3286xc6dr4v/51YNu24q9N+rYWUup3leezbwLQTFY1oUETyqQ2habu5/sSJ8nEie4H19bm/sYtCACF01Ek4epbud5+G7jvPuDCC4EDD3SV3XnnlQ4CQPFtnTnTVaidOrm/M2f6VOAQlJoY5/NnrECQK8lfnKgU+kKaaf8Fze9ZuWHZuBF4+GHg0kuBgw8G9tkH+M//BB580M3Svflm4OWXS1d2xbY1M+JnxQp35LxiBXDOOcDAgcn4XpaaeZ7nsydQ/alRtW1KQd4i6SNISrtwMVF02s6YUbg9s17aqvOJSwd5XMpRzObN5KxZ5De/SR5+ePv2/VNOIW+8kWxuJlta2r8v328y810rta2F+k+S+LsuRKOGApD0jrcoA1mhH1wch1v6oR4OGoK0dSv5l7+Q3/oWOWZM+7QNJ55ITplCPvssuXNn6WVVG+gKHZwk7XddASjFhA+S3vEWZadtvXYYF5K27S1l+/bd0zbs2uXSNowdm53ENW4c0L17OGUq9BllJOV3XQFdj8APteSHj4Og87AXM3Vq/hmYcW+rrlaU+zoOdu4EFizIVvzPPefSNnTq5NI2XHllNm1Dr17RlDHfdzJXUn7XIVEgyEh6ZRZlIKunC6WUI+kHDZVqbd09bcO2be6o+vDD3TDPTNqGPfaIurRO5rt3+eWuczpXkn7XYam2TSnIW2QTypLQ8VaI2q3DU+/7urWVfOkl8uc/Jz/7WXKPPbLbefDB5KWXkg8/TG7YEHVJy5Pk33UFoD4CAZCMSVD1op72NQm8/nr2iP/pp9sfRffuDZx7LvCd7wAf+UhkxZTilH1URMpHumvt5l579+233XNDh7pO7/nzXV9AhtJwx54uVSkixa1aBdxzj5uxO2yYm8F74YUuCOy/P9C/f/Zyi6++2j4IAK7v7Lrrwi518iR0UqoCgUg9WrcOuP9+NwBi+HB3pH/uucBjjwFjxgC/+AXw2mvATTe5WbybNrkzhZUrd+9czfB7VFRCK82C8s1mnjQpEdulpiGRerBxI/DMM9nmnldfdY/37QuMH58dy3/ooa7izSg13j6Xn/Mkor7oS7WK9Q1FPL9EF6YRSZv33iP/+EfyiivIUaOyM2l79SJPPpn8yU/IF14gd+0qvpxSM3CLjYqqZTROEmfylxotFvEFjaBRQyJ1bts24Nlns0f8Cxe68f3dugHHHJM94h8zBmhoKH+5hY5iBwxwo4UKjYqq9Yg+iTP5Sx3xJ/iMIPAJZWZ2N4DTAbxDcmTQ6xOpC9u3A/PmZSv++fNd2oYuXVzahmuvdRX/0UfXlrah0ETKW24pXqFfd93us3YzHcrlBIIkTsorNaN8wgTg9tt3f37ChODK5JMwOot/A+CUENYjkly7drlUDVOnAp/6FLDnnq6inzLFpW+44grgmmtcyubnnnMjgN56q/bcPaXSHRdSa5qNJKbQLnX9h8cfz/98ocdLCbMzvdo2pUpuAIYBeKXc16uPQOpeSwu5YIFLwXzKKe2vvXv44S5l86xZLoUzGb/ZzH608Sdtxm+YfQRVfN6ooY8gNoEAwCQAzQCahw4dWvmOE4mz1lby5ZfJm28mzzijfdqGj3+c/PrXyYceItevz//+uHWuxi0whaVY8PLzM6piWbUEglA6i81sGIDHWGYfgTqLJfFI4B//aJ+2YcMG99wBB2Q7d084wTX3lBLHztV6SrPhBz+HxFbxece6s1gkNZYtc5V+Jm3D2rXu8f32cx2Gmcq/mg7ROHauTpyY7oq/Iz+z8Ib8eSsQiFRr9WpX4WeO+jM/3L33dhV+puI/4IBs+oZqJT1Nelr4FRxD/rzDGD56H4ATAAw0s9UArid5V9DrFfHdunWuiSdT8f/zn+7x/v1dE8/VV7uK/+Mfr73i7yht13xIu5A/b00ok/Qpt21706b2aRuWLHGP9+0LfOIT2SP+ww5rn7ZBJALKPuqnekuEJe0VSwz2/vvAn/4EXHUVcOSRwMCBwOc/D9x1FzBkCPDjH7uJXRs3An/8I/DNbwKjRikIJJF+5+3ojCBXUhNhSfkKpQFoaHApGzJpG44+OnvEf9RRlaVtkHir09+5Lkzjl4hzhUjAduwAevTIPywPcFfgyqRt6NEj3LJJeKL8nQc45FZNQ36pddp8kHQqW7ldu4Dnnwd++EPg058G+vUrHAQaG4Ef/MCdBSgI1LeofuelrlcQ5W+82ploQd6qSjHhx3T1uM3ezChnFmfSpusHoaWFbG4mf/pT8tRTyd69s/vrsMPIb3zDpW3u0SN9M2IlK6rfebH1+jBTG3FPMVHpreJA4Nd097hOmy/1xY1ruYPW2kouXkzecgt55plkv37Z7T/oIPJrXyMffJB8553271PQTLeofi/FchH5EJwUCPyM8HGsJEols4rrmYzf2trI118nb7+d/MIXyEGDstv60Y+SF1xAzpxJrlkTdUkl7qL4nRf7nfqQsE6BIOIrAwWuVEVfz9u/bBl5113kxInk4MHZbdt3X/IrXyF//Wty+fKoSymlxPEAK2zFzkQiPiOojxQTcczD4qdS083rafvXrGmftiEzimOvvdqnbTjwQP9n70owOg7XzHSSAokerlmxUrOFo0whUm0ECfIWWR9BnBU7okry9q9bRz7wAHnxxeSIEdny77knedZZ5G23ka+84pqFJJnS0nRZqxrPmpD6piEy2lPPOJz2xqEM5di0iXzkEfKyy8iRI7OVQp8+5GmnkdOmkYsWuY5gKS4pn3k9N13GSC2BQBPKalWnsxR9s2UL8Le/ZZt6XnzRVQM9egDHHZdt6hk92l2PV8oT5+9dx0lTW7e6tBwdaaKmr2qZUBb50X++W6IuVZnG095iR6LbtpFPPkleey05bhzZubPbHw0N5Pjx5A03kHPmkNu3R1T4OhHX712+ZsqGBrJr1/aPJaXpslIRnqVBZwQRiuOVo4KU70i0WzfgtNPcFbjmzQN27gQ6d3Y5ejJH/Mccoxm7forr965Q+oYBA4Deves7hXbEZ2nKNRSltOUnamwsPBV/9OhsxX/ccUCfPuGWLU3i+r2La4AKQ8SfiXINRWnqVBf1c0V15aggcpW0tgKLFgHTprmj/kJBwAxobgZuvBE49VQFgaDF6XuXq9CQ5SQOZa5UnHOVlVJtm1KQt0T1EZDxGL3h1xDStjY3XPPWW8nPfc4N48ws72Mfa5+/J05t02kUh+9dR0keylyriPttoOGjUvWXsK2NfOMN8o47yC99idxrr+x799+fPP989yPOpG1I8w9dyhPHABWGiH8btQQC9RHUi0raZlescEM5M0M616xxjw8e7Nr4M+38w4blX1eAOdVFEi3C34Y6i6V4R9Vzz7VP27BsmXtu0KD2aRuGD1faBpGEUmex5O887NLFXZVryBDgnHOAhx8GDj8cuPVW4O9/B9atAx54ALjoImDECAUBqX/VDqiodSBG3C8sVW2bUpA39RFU6N13yUcfJU8+uf3EnW7dyAkTyJtuIhcudBduiVJa244lHqptw6+17T+kvgOojyBltm7dPW1DW5ubsHXsse3TNnTtGnVpnTinRJB0qHacf63zA0KaX6A+gnr34YfZdv6nngIWLABaWlwlf/TR2Yp/7Fg3yzeO4joBStKj2slutU6SC2mSXS2BQFm+4mjnTmD+/GzF//zz2bQNY8YAV1/tKv9jjtm9XyCukjzZRupDtdftqPV6Hwm4XogCQRy0tLjZu0895W7PPuuaUMyAI44AJk92Ff9xxwF9+0Zd2uok4Mcgda7UBZ78fp9f7w9DtZ0LQd7qvrO4tdXl3J82zeXg79Mn24k0ciQ5ebLL2b9xY9Ql9Y8mokkcVDtgodaBDiEMlIBmFhcQl1EqbW3kkiXualtnnUX275+tDEeMIC+6yF2la926aMoXlrh8HmmgfZ06tQSC+u0sjnKUCgn861/Z2buzZ7sx+4DrHM2dvTtkSLBlkfTRCK1U0qihfMIepbJiRfvZu6tXu8cHD24/e3f//f1ft0gujdBKJY0ayifoUSpr17av+JcudY8PHNi+4teMXQmbRmhJheo3EPg9SmXDBuDpp7MV/+uvu8f79QPGjwcuu8xV/occ4sYNi0RFI7SkQvUbCGodsrV5MzBnTrbiX7zYPd6rF/CJTwAXXOCO+EeNcuP7ReIiCcMVJVbqNxBkOsXKTQm7dSswd2624l+0yM36697dpW2YMsUd8Tc1xSdtg0g+lX73JfXqt7O4lO3b3YzdzCSuF17Ipm0YNy7bzj92rAsGIpIcKbxmRuw7i83sFAC3AOgM4FckfxzGetvZudNV9rlpG3bscO35Y8YAV12VTdvQq1dly8790vXv7x7btCk1X0CRWOk4fHbFCncf0G+xgMDPCMysM4A3AJwEYDWABQC+TPLVQu/x5YygpcVl5cwc8c+dm03bMGpU9oj/+ONrS9uQb8x2riSN307hUZTUoZQOn431PAIzOxrA90ie7N3/NgCQ/FGh91QVCNra3MVWMpO4nnkGeP9999whh2Qr/vHjs0ftfij0pcuVhC+gJiFJvQgp22fcxD0Q/AeAU0j+T+/+VwCMJXlph9dNAuCdv2EkgFcCLZhPRgOjy3ndQmBhlasYCGBDle8t22HAoV2Bho6P7wJ2Lgb+HvT6yxDKfkgI7Yus3fZFAr7LQfkYyT7VvDE2o4ZITgcwHQDMrLnayFZvtC8c7Ycs7Yss7YssM6u6PT2MmU9rAOyXc39f7zEREYmBMALBAgDDzWx/M2sAcDaAWSGsV0REyhB40xDJFjO7FMATcMNH7ya5pMTbpgddrgTRvnC0H7K0L7K0L7Kq3hexnFAmIiLhUXY0EZGUUyAQEUm5yAKBmZ1iZv8wszfN7H/leb6bmT3gPT/fzIZFUMxQlLEvrjCzV81ssZn9PzNrjKKcYSi1L3Je9z/MjGZWt0MHy9kXZvZF77uxxMx+F3YZw1LGb2Somc02sxe938mEKMoZBjO728zeMbO8c63MudXbV4vN7MiSC632Gpe13OA6jf8F4KNwEz9eBnBwh9d8DcAd3v9nA3ggirLGZF98EkBP7/9L0rwvvNf1ATAHwDwATVGXO8LvxXAALwLY07u/V9TljnBfTAdwiff/wQCWR13uAPfHJwAcCeCVAs9PAPBnAAZgHID5pZYZ1RnBUQDeJLmU5E4A9wM4s8NrzgTwW+//hwB8yqwuL/VVcl+QnE0yk/thHtxcjHpUzvcCAH4A4CcAtodZuJCVsy8uBPALku8CAMl3Qi5jWMrZFwSQSRq2B4C3QixfqEjOAbCpyEvOBHAPnXkA+pnZPsWWGVUgGAJgVc791d5jeV9DsgXAewAGhFK6cJWzL3JdABft61HJfeGd5u5H8k9hFiwC5XwvRgAYYWbPmtk8L8tvPSpnX3wPwDlmthrA4wAmh1O0WKq0TolPigkpzczOAdAEYHzUZYmCmXUC8DMA50VclLjoAtc8dALcWeIcMzuU5OYoCxWRLwP4DclpXqLLe81sJMn6zTLno6jOCMpJO/Hv15hZF7jTvY2hlC5cZaXgMLNPA7gOwBkkd4RUtrCV2hd94BISPm1my+HaP2fVaYdxOd+L1QBmkdxFchlcuvfhIZUvTOXsiwsA/B4ASD4PoDtcQro0qjitT1SBoJy0E7MAnOv9/x8AnqLXE1JnSu4LMzsCwJ1wQaBe24GBEvuC5HskB5IcRnIYXH/JGSQDvpxdJMr5jTwKdzYAMxsI11S0NMQyhqWcfbESwKcAwMw+DhcI1odayviYBeC/vNFD4wC8R3JtsTdE0jTEAmknzOz7AJpJzgJwF9zp3ZtwHSNnR1HWoJW5L34KoDeAB73+8pUkz4is0AEpc1+kQpn74gkAnzGzVwG0AriaZN2dNZe5L64E8Esz+yZcx/F5dXrgCDO7D+4AYKDXJ3I9gK4AQPIOuD6SCQDeBPABgK+WXGad7isRESmTZhaLiKScAoGISMopEIiIpJwCgYhIyikQiIiknAKBiEjKKRCIiKScAoFIGbxc9yd5/08xs9uiLpOIX5R0TqQ81wP4vpntBeAIAHU3s1vSSzOLRcpkZs/Apfo4geSWqMsj4hc1DYmUwcwOBbAPgJ0KAlJvFAhESvCu7jQT7spPW+v4AjCSUgoEIkWYWU8AfwBwJcnX4C6TeX20pRLxl/oIRERSTmcEIiIpp0AgIpJyCgQiIimnQCAiknIKBCIiKadAICKScgoEIiIp9/8BxclT43b7YQsAAAAASUVORK5CYII=\n",
+ "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAAkhUlEQVR4nO3deZhcVbku8PftdCIJhIQMYBi6I6IHY0CQBuJVLhwVUUDjuSrCExAQiaAg3oOiGBUQ44j3elS4EhDBkwYc8XBxAAcUQYI0MpgEGQTCbBJCBghD0nnPH2sXVd1dVV3dVXuovd/f89TT3VXVtVft2rW+tb619tqUBDMzK66OtAtgZmbpciAwMys4BwIzs4JzIDAzKzgHAjOzgnMgMDMrOAcCq4vkPJLXxfTal5L8YotfM7byFg3JZ0jumnY5LH4OBAaSbyL5Z5LrSK4heRPJfQFAUq+kt6VdxsFIiuRug+/PUnmjQPdiVKGuIfkbkrunXa5GSdpG0gNpl8Pi50BQcCS3BXANgG8DmAJgJwDnAHghzXK1G5KdNR76mqRtEPbrYwC+l+C2zRriQGCvBgBJV0jql/ScpOsk3QUAJI8jeWPpyVFL/CMk7yO5geS5JF8Z9SjWk/wRyXHRcw8i+SjJz5BcTfIhkvNqFYTk4STvILk2er09R/pmapT3pKi8a0meT5IVj3+Q5N0knyZ5Lcnuisf+g+Qj0fu6jeQBFY+dTfInJBeTXA/guHrlkvQcgB8B2KviNXYk+VOSq0g+SPJjFY+NJ3lZVK67SZ5B8tGKxx8i+SmSdwF4lmQnyTnRfltL8k6SBw3aLw9En9mDpc+B5G4k/xj1BleT/OGgfbdb9Pskkj+IyrqC5GdJdlTuc5LnReV9kOQ7Gvm8LBscCOxeAP1RpfMOkts18D+HANgHwBwAZwBYBOBoALsAmA3gqIrnvhzANIQW8bEAFpH8l8EvSHJvAJcA+DCAqQAuBHA1yZeN9o1VOBzAvgD2BHBEVH6QnAvgMwD+F4DpAP4E4IqK/7sVoeKeAuByAD8muVXF43MB/ATAZAC99QpAcmuE/XJ/9HcHgP8P4E6EffMWAB8neUj0L2cBmAlgVwAHI+zfwY4CcFi0/R0A/ALAF6PyfgLAT0lOj7b9LQDvkDQRwP8AcEf0GucCuA7AdgB2RugZVvNtAJOi8hwI4AMAjq94fH8A9yB81l8D8L3KgGvZ5kBQcJLWA3gTAAG4CMAqkleT3KHOv31N0npJywAsBXCdpAckrQPwKwB7D3r+5yS9IOmPCJXVEVVecz6ACyXdEvVMLkNIT81p7h0CAL4iaa2khwFcj3Kr/CQAX5Z0t6TNAL4EYK9Sr0DSYklPSdos6RsAXgagMojdLOnnkrZELf5qPkFyLYANCPv5mOj+fQFMl/QFSS9GufiLABwZPX4EgC9JelrSowgV+WDfkvRItO2jAfxS0i+j8vwGQB+AQ6PnbgEwm+R4SU9Enx0AbALQDWBHSc9LunHwRkiOicp1pqQNkh4C8I2K9wIAKyRdJKkfwGUAZiAEJ2sDDgSGqCI8TtLOCC36HQF8s86//LPi9+eq/L1Nxd9PS3q24u8V0esP1g3g9CitsTaqPHep8dyRerLi940V5esG8B8V21sDgAgtdJD8RJSWWRc9PgmhxVvySAPbPk/SZITW/XMoB5JuADsOer+fQbny3HHQ61fbVuV93QDeN+j13gRgRrT/348Q+J4g+QuWB63PiN7zX0guI/nBKtuZBmAswmdXsgLRfoq8tI8lbYx+rTwOLMMcCGwASX8HcClCQGiF7aLUREkXgMerPO8RAAslTa64TZB0RZXntsojAD48aJvjJf05Gg84A6Flvl1Uma9DqDRLGl66N+qNnIYQeMZH235w0LYnSiq14J9ASNWU7FLtZQe9l/8c9HpbS/pKtP1rJR2M0FL/O0LvA5KelHSipB0R0nIXcOhsrNUo9xxKuhAGvy0HHAgKjuTuJE8nuXP09y4IueclLdzMOSTHRZXr4QB+XOU5FwE4ieT+DLYmeRjJiXVedxzJrSpuY0ZYru8COJPka4GXBkTfFz02EcBmAKsAdJL8PIBtR/j6A0TpmscR0mB/AbAhGvAdT3IMydmMpu0iDCyfSXI7kjsBOGWYl18M4J0kD4leayuGwfqdSe5Acm4UkF8A8AxCqggk31f67AE8jRBctgwqd39UnoUkJ0aps3+Ptmk54EBgGxAG+m4h+SxCAFgK4PQWvf6TCBXM4wgDqidFvY4BJPUBOBHAd6Ln349hZuIAWIaQbindjq//9CHbvArAVwFcyTDzZymA0myXawH8GmEwfQWA59FYKmg4X0foaXQiBMW9ADyI0Oq+GCH9BABfAPBo9NhvEQala07plfQIwuD1ZxCC1yMAPonwHe9AqLgfR0h/HQjg5Ohf90X47J8BcDWA02qcO3AqgGcBPADgRoTB80tG+N4to+gL01hcoumLi6OxB2sCyZMBHCnpwLTLYvnjHoFZBpGcQfKNJDui6banA7gq7XJZPiVyRiLJhxBSEP0ANkvqSWK7Zm1sHMK5FK8AsBbAlQAuSLNAll+JpIaiQNAjaXXsGzMzsxFxasjMrOCS6hE8iPLUtAslLarynPkI0+qw9dZb77P77m2zSKOZWepuu+221ZKmj+Z/kwoEO0l6jOT2AH4D4FRJN9R6fk9Pj/r6+mIvl5lZXpC8bbTjr4mkhiQ9Fv1ciTDzYb8ktmtmZsOLPRBEZ4hOLP0O4G0IJ+6YmVkGJDF9dAcAV0Ur0nYCuFzSrxPYrpmZNSD2QBCdrv66uLdjZmaj4+mjZmYF50BgZlZwDgRmZgXnQGBmVnAOBGZmBedAYGZWcA4EZmYF50BgZlZwDgRmZgXnQGBmVnAOBGZmBedAYGZWcA4EZmYF50BgZlZwDgRmZgXnQGBmVnAOBGZmBedAYGZWcA4EZmYF50BgZlZwDgRmZgXnQGBmVnAOBGZmBedAYGZWcA4EZmYF50BgZlZwDgRmZgXnQGDJ6u0FZs4EOjrCz97etEtkVnidaRfACqS3F5g/H9i4Mfy9YkX4GwDmzUuvXGYF5x6BJWfBgnIQKNm4MdxvZqlxILDkPPzwyO43s0Q4EFhyurpGdr+ZJcKBwJKzcCEwYcLA+yZMCPebWWocCCw58+YBixYB3d0AGX4uWuSBYrOUedaQJWvePFf8ZhmTWI+A5BiSt5O8JqltmpnZ8JJMDZ0G4O4Et2dmZg1IJBCQ3BnAYQAuTmJ7ZmbWuKR6BN8EcAaALbWeQHI+yT6SfatWrUqoWGZmFnsgIHk4gJWSbqv3PEmLJPVI6pk+fXrcxTIzs0gSPYI3AngXyYcAXAngzSQXJ7BdMzNrQOyBQNKZknaWNBPAkQB+L+nouLdrZmaN8QllZmYFl+gJZZL+AOAPSW7TzMzqc4/AzKzgHAjMzArOgcDMrOAcCMzMCs6BwMys4BwIzMwKzoHAzKzgHAjMzArOgcDMrOAcCMzMCs6BwMys4BwIzMwKzoHAzKzgHAjMzArOgcDMrOAcCMzMCs6BwMys4BwIzMwKzoHAzKzgHAjMzArOgcDMrOAcCMzMCs6BwMys4BwIzMwKzoHAzKzgHAjMzArOgcDMrOAcCMzMCs6BwMys4BwIzMwKzoHAzOLX2wvMnAl0dISfvb1pl8gqdKZdADPLud5eYP58YOPG8PeKFeFvAJg3L71y2UvcIzCzeC1YUA4CJRs3hvvzIAe9HQcCM2u9yspxxYrqz3n44USLFItSb2fFCkAq93baLBjEHghIbkXyLyTvJLmM5Dlxb9PMUjS4cqylqyu5MsUlJ72dJMYIXgDwZknPkBwL4EaSv5K0JIFtm1nSqlWOg02YACxcmEx54lSrV9NmvZ3YewQKnon+HBvd6jQTzKyt1asESaC7G1i0KB8DxbV6NW3W20lkjIDkGJJ3AFgJ4DeSbqnynPkk+0j2rVq1KolimVkcalWC3d3Ali3AQw/lIwgAoVczYcLA+9qwt5NIIJDUL2kvADsD2I/k7CrPWSSpR1LP9OnTkyiWmcUhJ5VjQ+bNC72b7u627u0keh6BpLUkrwfwdgBLk9y2mSWkVAkuWBDSRF1dIQi0WeXYsHnz2v69xR4ISE4HsCkKAuMBHAzgq3Fv18xSlIPKsUiS6BHMAHAZyTEIqagfSbomge2amVkDYg8Eku4CsHfc2zEzs9HxmcVmZgXnQGCWdTlYy8ZaJKZjwauPmmWZV+60khiPBareWiAp6enpUV9fX9rFMEvfzJnVF23r7g4nZllxDHMskLxNUs9oXtqpIbMsy8laNtYCtVZxrXX/CDgQmGVZTtaysRYYM2Zk94+AA4FZ3JoZ4CvScg1WX3//yO4fAQcCszg1e+GSnKxlYy3Q3T2y+0fAgcAsLr29wLHHNn/hknnzwsBw3lbutJGJsXfoQGAWh1JPoFa33YO9NlIx9g49fdQsDrWm+pV4+qe1mKePmmVNvRa/B3stYxwIzOJQa3rnmDEe7LXMcSAwi0Otgb3LLnMQsMxxIDCLg6d9WhsZdtE5kr8B8AlJdyZQHrP88FW6rE000iP4FIBvkvw+yRlxF8gsk7wUtOXYsIFA0l8l/SuAawD8muRZ0bWHzYqh2bODzTKuoTECkgRwD4D/B+BUAPeRPCbOgpllxoIFzZ8dbJZhwwYCkjcBeAzA/wWwE4DjABwEYD+Si+IsnFkmeCloa0YbpBUb6RHMB7CTpIMlfU7SNZLul3QqgANiLp+NRBsccG3JS0HbaLVJWrGRMYJlqr0OxWEtLo+NVpsccG3JS0HbaLVJWrGp8wgkPdCqgliT2uSAa0s+J8BGq03Sij6hLC/a5IBrW14KunlFTF3GkVaMYT86EOSF89iWZXGnLrMaZFqdVoxrP0rK3G2fffaRjdDixdKECVI4PMJtwoRwv1naursHHpulW3d386+d9WN/8eLwPsnws5ly1dmPAPo0yjrX1yPIk97eMCbw8MOhJ7BwoVMYlg0dHaHKGowM6bZm1Lr2Qx6v+VBnP1Ly9QgM7ZvHzmq33lonztRlkcbHYtqPDgSWLk97LYY4p+AWaXwspv3oQGC1JdFS97TX7Ijz845zCm6RzvOIaz+OdnAhzpsHizMgqQE4svrgF9na7Vh9WR9wHU4rB2TbFDxYbC2X1ABckQb6siyNz8GTG1rKF6+31ktqAK5I3fosS3rA1WNDmeJAYNUlNQDn5RuyIekBV48NZYoDgVWXZEu9Xae95knSPbMiTflsA7EHApK7kLye5HKSy0ieFvc2rQXcUi+WpD/vIk35bIWYZ/DFPlgcXed4hqS/kpwI4DYA75a0vNb/eLDYLOdKYwSV6aEJE9zYqKbBfZXpwWJJT0j6a/T7BgB3I1zpzMyKyj3OxiUwnpLoGAHJmQD2BnBLlcfmk+wj2bdq1aokixUvL59gWZfWMeqxocYkMJ6SWCAguQ2AnwL4uKT1gx+XtEhSj6Se6dOnJ1WseHmKnGWdj9HsmzKl+v0tHE9JJBCQHIsQBHol/SyJbWaCp8hZ1vkYzbbeXmDDhqH3jx3b0hldSQwWE8BlANZI+ngj/5ObweI4l941awUfo9lW64zvqVOB1asH3JXpwWIAbwRwDIA3k7wjuh2awHbT5ylylnU+RpsX5xhLrXGAp55q3TYAdLb01aqQdCMAxr2dTFq4sPq0Ly+fYFnhY7Q5g6d2lsZYgNYMfnd1Ve8RkMD55wMzZgA33wwsWdLcdka7Wl2ct1ytPupVES3rfIyOXpyX4JTCZ1Frhd7Sbdw4ac6cplYf9RITcfMUOU+hzbq8HKNpHGdxTe188kngqquAO++sPoZT8uc/A+vWhV5BE2JPDVnBxd11NgPSO85qpW5GMsayaRNwxx3lFM/NN5eX/h47Fhg3DnjxxaH/190NvOENoyn1EO4RWLw8PdGSUOs4O+20eHsJo1msr9TaP+MM4IADgG23BfbbL5T1hhuAnh7gvPOAm24C1q8HLrkk/gUBR5tTivOWqzGCkchjrtZXILMkDJdHj/Oqa/W+ty++KN16q/Stb0lHHSXNnFkuy9ix0v77Sx//uPTDH0oPPzy6bUTgK5TlQF4X4fIVyIop6auP1TrOqonz2HvyyZDaKaV5+vqA554Lj+24Y0jllG6vfz2w1VYt23Qz5xGk3vqvdstkjyDu1nrcsw/S0u7Xws2CdusppvGZV9tmrVureqONtPZPO0268kppxQppy5bWbLcGNNEjSL3Sr3bLXCBI4sBOO4USZ2XTbhVZlixeHKYHDp4umOV9mFajZvBxNnVqa8vx5JPSVVdJZ5whHXCANH58+TV33FF6z3uk886TbrpJeu651r2vBjUTCJwaakQS6Y00Uyh5TUvlwbRp1c8irbLEQGZkZdmKasf1uHHAxInAmjX1U1abNgF33VVO89x8M/Dgg+GxsWOBvfcemObZZZfw/lLUTGrIgaARSRzYaVbGzuNnV73KJYPfXQDZOp4qxyqmTAmzcDZtKj9e+o699a0DK/3K3P6MGQMr/X32aWluv1U8RhC3pLq6aaVQ0k5LtZOkP6N6ue6syuq4UK3v8Zgx5d/HjpX22692bj/DaU54jCBmWT2wWyWvA9WtlsZxUCvPPXVqfNtshSxVmP/8p/Tzn9cPql//unTjjdLGjbVfJ+P1gANBErJ0YLdaxg/wzEgjYC5eHFqpldsbO9afTS0vvij19Unf/rY0b5606671A8BIPr+MN5iaCQQeI7Ag6Xnf7SitQVB/NrWtXDk0t18aZ6vM7c+ZA9x3H3DKKdXH4YDh93FWBsFr8GCxWRKyNAhaRJs3D53J88AD4bHOzqEzebq6hg62VwuqQGMTNTL++XuwOGlpponynKLKOqfQkvXPf0r/9V/Spz8tHXjgwH3/8pdL//Zv0te+Jv3pT/Vz+8NpNOWT8c8fHiNIUJoHQ8YPxEJwII7Hpk3SbbdJ3/nO0Nx+Z6e0777Sxz4mXXGF9OCDrT1Lt96sucGf98knZ/bzbyYQODU0Uml2DzPeNTVr2KpVA1M8t95aTs28/OVD5+2PHx9fWepdF/i559rmRMusX7M4Xxq5EEVcF8iI6yIYNrxWf6ZFuljP5s3A7bcDF1wAHHMMsNtuwPbbA3PnhuWWn30WOOEE4PLLw9m7jz8O/OxnwCc/CbzpTfEGAaD2UtJAcZZQH21XIs5bplNDw+UT40zfZHz6Wm61+jPNe4pv5crauf0ddpDe/W7pq1+VbrhBevbZtEsbVEv5tdmJlvAYQYKG+xLHWVnnvQLJqlZ/pnkK6Js2SX/9q3T++dLRR0uvfGX5/XR2Sj090qmnSpdf3vrcftza7HNyIEhavQHDuFsRRRysTPs9t/ozbbOW5gArV0pXXy2deaZ00EHS1luXy5/V1v5otVnDq5lA4MHiVvOAbmtlYWXUVn+m7XKMbN4M/O1vA6+le//94bHOTmCvvcKJWqVB3ZkzU1+Bs+Xa6GQ+n0eQJUm0IlrRQk67ld2oLHTPizJGsGpVcVr7OQSnhjIm7ou8NFuJZLUiqiYraZRWf6ZpB+JNm6Tbb5cuuEA65hhpt93K+3bMGGmffaRTTpF6e6UHHmiv3H5BNRMInBpqN61IK7RLagJor7Jm2erV5fTOzTcDf/lLmLYJhKmclfP2e3qGTqccThulUPKqmdRQZ6sLYzFrxbkE7XQ+wsKF1ccISmvE2FCbNwNLlw68iPp994XHxowJuf3jjy8vxvaKVzSX2x88jrNiRfgbcDBoEw4E7aarq3oLuasr2ddISqkiKVprcyQt7EZa+yecMPrW/nAWLKh94lXeP6eccGqo3bRiFk0WZuJYbfU+n/e/H1i2bODyDJWt/de9bmCap9nWfiMyvjxzUXjWUNEMN9DYyEBk2oOVVlutmVIve9nAmTzbby/NnSt95SvSH/8oPfNMtsqb0ROv8gqeNWQvaacZQTbQ5s3SHXdUr1RLt49+NHyW//hHdmby+JirLuHGVjOBwKmhvPEsm/bx1FNDc/vPPFP7+Vn+DD1raKAU0q++QpmVOV+bTf395Zk8pcr/3nvDY6Xcfuks3VWrgM9+1mM47SyFBpmnj1pZO80IyrN6rf3p00OFX5rC2dMDbL31wP/ffnu3sNtZO03RRgKBgOQlAA4HsFLS7Li3V3ied5+8/v6hM3kqW/t77gl84APlmTy77jr8TJ5581zxt7M2a5Al0SO4FMB3APwggW1ZUefdJ2nNmqGt/Q0bwmOl1v5xx4Wf++47tLVvteVlrKHNGmSxBwJJN5CcGfd2rIJbk60zuLW/ZAlwzz3hsVJr/5hjRtbat+rydIZyuzXIRjvdaCQ3ADMBLB3mOfMB9AHo6+rqau28qrR4rn77eeop6Re/kD77Wektb5EmTixPiZw2TXrnO6UvfUm6/vr05u3nVaPnI+ThexXDe0DWp49GPYJr1OAYQS5mDfns3ezr7weWLx+Y2y+19js6Qmu/8izdV77Srf04NTLjLQ/fq5jeQ+anjxYyEHg+f/bUy+1PmzZ0Bc5ttkm3vEXTyHcmD9+rmN6Dp49mUZtNH8udRlr7Rx/t1n6WNDLA2uz3KguD0VmsG0abU2r0BuAKAE8A2ATgUQAnDPc/uVhiwuuvJGvNGumXv5Q+9znprW8dmts//HBp4ULp97+XNmxIu7RWy3C582a+V9WWwhg7Vpo6NdnxhpjqBnitoQwabv2VPAx4pWXzZumuu6QLL5SOP17afffyPu7okPbaSzr5ZOkHP5Duuy87a/JY85pZ16hWBZz0Gkkxrc3kQFBNFiraWmUo0iJdrfgcBrf2t922vN+mTnVrv2hGe0zVuuxpGr32jM0aSr3Sr3ZrOBC0a0VblLTRaD6H/n7pb3+TFi2q3tp/3eukk06SLrtMuvdet/atcY30CNK4HnaLFDMQ1Ktksl7RZuWC7HFr5HNYs0b61a+kz39eOvjg6q39L35R+t3vpPXr4y1vFnqRFp9qdUaW64kRKmYgqFfJZL2izXqgapV6XfHjj5de85ry32m39rPei7TWqAz2U6dK48a17jNPuSFRzEBQr7LPekVblEpn551rB4KpU6XDDkuutT+crB8zFo9WVd4Z+E4XMxDU++Jm4EMZVt7SEP390tKl0kUXSR/84MDWfuVt3DjpvPOyl9vPei9ypPJ2fGVdBhoSxQwEnp6Zrqefln79a+mss6S3vU2aNKn8OUyZUm7tn3mmtMsu6X0OjR4HGfgit0w7NITyJgMNiWIGAsmVfVLqtfY7OqQ995Q+/GHp0kule+7JTmt/JBVinirPPAW1dpGBfV7cQFB0cQXC4Vr7hx4qnXuu9Nvfpp/br2ekX868NCwy0DotnAw0JJoJBL5mcbtq1QqGW7YAd989cDG25cvDYx0dwOzZAxdje9Wr2mdNnqJevzkPC7O1o5TXMcr86qMj5UDQgNF+2deuBW65pVzp33ILsG5deGzKlPIF1N/wBmC//YCJE2MofEKKWiHmYanmuGVh8bkWayYQpJ4GqnZzaqgBjXT/+/ulZcukiy+WTjhBmjWr/H+ktMce0vz50ve/L/3979nJ7Y9EvXROBrrrqclLmisOOT0u4NRQAdVq7W6/PfCRj5Rb+2vXhvvz1toHGmv55rDlZ03KaU/RqaEiWrwYOPFE4Pnnhz5GDs3tv/rV7ZPbb1ROv9AWs5yOHfnCNHlSqwW7bt3Q3H5lENhqK+Cww4CTTgqt/W23Te89JCWLF/iw7Ovqqt6A6OpKviwZ4UCQJYNTHStWAMceC3zqU8Djj4dWTKm1f8QRoaU/Z05o7Xd0pFv2NPgLbaPRyJXQCsaBIAvWrQvXz/3oRwcenEC45OJTTwHnnFPO7Rehtd8If6FtNEpjRB47eonHCJK2ZUu4dm7ltXSXL6+esyxp89xlrDwYbAbAYwSt1eqKZf36obn9p58Oj223XUjtlNI8J5wAPPLI0NdwqqO2efNc8Zs1yYGgUrUc/fz54fdGKpstW4B77x3Y2l+2rJzbf+1rgfe+d+BMnsrc/pe/7FSHmSXOqaFKI52OWK+1P3ny0Hn7kyYNXwanOtqfP0NLgc8sbpV6Z+v290t33y1dcol04onS7NkDz9KdPVv60Iek731PWr48PN+Kp9mzVlt5oRSfWVwo8OqjLVJrtcqttpK226789+TJ0tvfLp1zjnTdddLatemU17KnmeWIW7X0QdpLKDgIpcKBoFlbtoTW/oknSp2dQ7/EO+3k1r41ppkloFu1pn2aa+OnHYSyKoHg2EwgKOYYwfr1Yd5+Kbe/ZEk5tz9+fDh8n38+rNuzcCHwoQ/FVxbLl2aWvWjV0gdpLqHgZT+GSmg1WE8frUcaOpNn6dLyTJ5Zs4D3vCcM6K5eDZx/fpjC2d3tQT4buWZOcmvVmdJpnnHtZT+GWrBg6ImiGzeG+7NSv4y2KxHnranU0Lp14cpZ554brqQ1ZUq5izppknTIIdLZZ0vXXjswt+8urbXKaNMAeRgjyMAlGzMnoSvGobBjBFu2hHX0v//9sK7+HnsM3OmzZoV1+C++OKzLXy+3n9UD2ANvxdLus4bcoBoqobqlmUDQXmMEGzYMze2vWRMemzRp6Lz9yZMb32gWl6b1laasHfk8ioHaYIwgu4Hg1luB++4bmtsvVcqzZg1cb3/33ZtbgTOLg1xZLJOZjVwCwTF/gWDyZPV1doZVN4HQ2t9//3Klv//+I2vtNyKLre8s9lKsOreCLWXNBIJsLmL/wgvA3LnARReFXsCaNcC11wJnnw0cckjrgwAQvrSLFoXWNhl+pp2CqTXLw4vQtU5vb+h5dXSEn729o3uN+fND700qr1E1mtcyS0E2ewR5XoZ6JLLYS8mTVu1fp/AsA/LXI7Agi72UPKk3v3skPHfe2lz+Tyhrd15vPz6tqsB9yUxrc4n0CEi+neQ9JO8n+ekktmk2rFaNwSxcGFJKlXwdCWsjsQcCkmMAnA/gHQBmATiK5Ky4t2s2rFZV4E7hWZtLIjW0H4D7JT0AACSvBDAXwPIEtm1WWysvYu4UnrWxJALBTgAqL8T7KID9Bz+J5HwA0XUh8QLJpQmUrR1MA7A67UJkQPz7YcUK4Oijw23ghqfsCOw0Fhi3CXjxceCx1cCaWMtSn4+JMu+Lsn8Z7T9mZrBY0iIAiwCAZN9op0HljfdF4P1Q5n1R5n1RRnLUc+6TGCx+DMAuFX/vHN1nZmYZkEQguBXAq0i+guQ4AEcCuDqB7ZqZWQNiTw1J2kzyFADXAhgD4BJJy4b5t0Vxl6uNeF8E3g9l3hdl3hdlo94XmVxiwszMkuMlJszMCs6BwMys4FILBMMtO0HyZSR/GD1+C8mZKRQzEQ3si38nuZzkXSR/R7I7jXImodHlSEi+h6RI5nbqYCP7guQR0bGxjOTlSZcxKQ18R7pIXk/y9uh7cmga5UwCyUtIrqx1rhWDb0X76i6Srx/2RUd7jctmbgiDxv8AsCuAcQDuBDBr0HM+AuC70e9HAvhhGmXNyL74VwATot9PLvK+iJ43EcANAJYA6Em73CkeF68CcDuA7aK/t0+73Cnui0UATo5+nwXgobTLHeP++J8AXg9gaY3HDwXwKwAEMAfALcO9Zlo9gpeWnZD0IoDSshOV5gK4LPr9JwDeQpIJljEpw+4LSddLKq2XvAThXIw8auS4AIBzAXwVwPNJFi5hjeyLEwGcL+lpAJC0MuEyJqWRfSEA20a/TwLweILlS5SkG1D/zPa5AH6gYAmAySRn1HvNtAJBtWUndqr1HEmbAawDMDWR0iWrkX1R6QSEaJ9Hw+6LqJu7i6RfJFmwFDRyXLwawKtJ3kRyCcm3J1a6ZDWyL84GcDTJRwH8EsCpyRQtk0Zap2RniQkbHsmjAfQAODDtsqSBZAeA/wPguJSLkhWdCOmhgxB6iTeQ3EPS2jQLlZKjAFwq6Rsk3wDgP0nOluSLezcgrR5BI8tOvPQckp0I3b2nEildshpagoPkWwEsAPAuSS8kVLakDbcvJgKYDeAPJB9CyH9endMB40aOi0cBXC1pk6QHAdyLEBjyppF9cQKAHwGApJsBbIWwIF0RjXhZn7QCQSPLTlwN4Njo9/cC+L2ikZCcGXZfkNwbwIUIQSCveWBgmH0haZ2kaZJmSpqJMF7yLkl5vMB1I9+RnyP0BkByGkKq6IEEy5iURvbFwwDeAgAkX4MQCFYlWsrsuBrAB6LZQ3MArJP0RL1/SCU1pBrLTpD8AoA+SVcD+B5C9+5+hIGRI9Moa9wa3BdfB7ANgB9H4+UPS3pXaoWOSYP7ohAa3BfXAngbyeUA+gF8UlLues0N7ovTAVxE8n8jDBwfl9OGI0hegdAAmBaNiZwFYCwASPouwhjJoQDuB7ARwPHDvmZO95WZmTXIZxabmRWcA4GZWcE5EJiZFZwDgZlZwTkQmJkVnAOBmVnBORCYmRWcA4FZA6K17g+Ofv8iyW+nXSazVvGic2aNOQvAF0huD2BvALk7s9uKy2cWmzWI5B8Rlvo4SNKGtMtj1ipODZk1gOQeAGYAeNFBwPLGgcBsGNHVnXoRrvz0TI4vAGMF5UBgVgfJCQB+BuB0SXcjXCbzrHRLZdZaHiMwMys49wjMzArOgcDMrOAcCMzMCs6BwMys4BwIzMwKzoHAzKzgHAjMzAruvwHii9RYGRvveQAAAABJRU5ErkJggg==\n",
"text/plain": [
"
"
]
@@ -469,7 +469,8 @@
"import numpy as np\n",
"import matplotlib.pyplot as plt\n",
"from sklearn.linear_model import LinearRegression\n",
- "\n",
+ "# Number of data points\n",
+ "n = 100\n",
"x = np.random.rand(100,1)\n",
"y = 5*x+0.01*np.random.randn(100,1)\n",
"linreg = LinearRegression()\n",
@@ -491,7 +492,8 @@
"Depending on the parameter in front of the normal distribution, we may\n",
"have a small or larger relative error. Try to play around with\n",
"different training data sets and study (graphically) the value of the\n",
- "relative error.\n",
+ "relative error. Note also that **Scikit-Learn** requires a matrix as input for the input values $x$ and $y$. In the above code we have\n",
+ "solved this by declaring $x$ and $y$ as arrays of dimension $n\\times 1$.\n",
"\n",
"As mentioned above, **Scikit-Learn** has an impressive functionality.\n",
"We can for example extract the values of $\\alpha$ and $\\beta$ and\n",
@@ -515,18 +517,18 @@
"output_type": "stream",
"text": [
"The intercept alpha: \n",
- " [1.99723611]\n",
+ " [1.84860939]\n",
"Coefficient beta : \n",
- " [[5.1007125]]\n",
- "Mean squared error: 0.22\n",
+ " [[5.2571699]]\n",
+ "Mean squared error: 0.25\n",
"Variance score: 0.90\n",
"Mean squared log error: 0.01\n",
- "Mean absolute error: 0.36\n"
+ "Mean absolute error: 0.40\n"
]
},
{
"data": {
- "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAAj3klEQVR4nO3de7hddX3n8fcnIQESUIRERfAcvEIRL5io9JlqtUofoYyOteNjnwSBOs2jVh/q0FGcaGEc42gdW7Uzo+CVmqPFC+0wFO83lFHGcFG5CAVMuFpCEiQQBZJ854+1d7PPPmvtvfZlXfZen9fz7Oecs/baa/3Oysn6rt/v+7soIjAzs+ZaVHUBzMysWg4EZmYN50BgZtZwDgRmZg3nQGBm1nAOBGZmDedAYGMj6YWSbqy6HNNA0nWSXlzAcR8n6TJJOyV9UNJ/lvSJcZ/HJos8jsAGJWkz8B8i4ptVlyWLpHOB9cBDwG7geuCsiPhhleWqmqR3AccDr46u//ySjgJ+ASyJiN0VFM8q4hqBTTxJ+2W8dWFEHASsAL4DfLGAc0vSJP0/mgWu7w4C1myT9AdsNSfpxZLu6Ph5s6S/kPRTSb+SdKGkAzreP0XSNZLuk/R/JT2r472zJd3SasK4XtKrOt47XdLlkv5G0jbg3F7laj3dzgFHSFrZOsajJX1S0t2S7pT0HkmLW+8tbjWb3CvpF5LeLCnaAUfSdyVtkHQ5sAt4sqRjJH1D0nZJN0p6TUd5T279Djtb5/qL1vYVki5p/f7bJX2/HVRa1+5lre/3l/QhSXe1Xh+StH/nNZd0lqR7Wr/PGRn/Pp8BTgPeJukBSS+TdK6kja1dLmt9va/1/m/3uq42PbKepMzG5TXAy4HfAJcDpwMfk3Q88Cng3wKbgLXAxZKOjoiHgFuAFwK/BP49sFHSUyPi7tZxXwD8PfA4YEmvAkhaCrwO2AbsaG3+DHAP8FRgOXAJcDtwHvCnwEnAc4AHSa9JnNra58bW568F/rK17ZnANyRdGxHXA58EXhMR35f0GOBJrWOcBdwBrGz9fAKQ9qS+vvXec1rv/2/gncC7Wu8/Hng0cARwIvAlSf8YETs6DxIRp0sCuCMi3tm6Nr/TscuLSJqGDnHTULO4RmBF+0hE3BUR24H/Q3IzA1gHnBcRV0TEnoi4gKQ9/wSAiPhi63N7I+JC4J+B53cc966I+NuI2B0Rv84492sk3Qf8muTm/kcRsVvS44CTgT+PiAcj4h7gb4DXtj8HfDgi7mjdTN+XcuzPRMR1rRvmy4HNEfHpVnmuBr5MEsAAHgGOlfSoiNgREVd1bD8cmI2IRyLi+xlNNmuAd0fEPRGxFfgvJIGIjuO8u3WMS4EHgKMzronZAg4EVrRfdny/Czio9f0scFarWeS+1g37icATACS9rqPZ6D7gOJK2/rbbc5z7CxFxCEmt4VpgVce5lwB3dxz/POCxrfef0HX8tHN1bpsFXtD1u6wheVIHeDVJ4Nki6XsdTS4fAG4Gvi7pVklnZ/weTwC2dPy8pbWtbVvXE3zndTbry01DVpXbgQ0RsaH7DUmzwMeBlwI/jIg9kq4B1LFb7mRnRNwraR2wSdLnWud+CFiR0QRyN3Bkx89PTDts1+/yvYg4MeP8PwZeKWkJ8GbgC8ATI2InSfPQWZKOA74t6ccR8a2uQ9xFEmyua/0809o2bk4gN5RrBDasJZIO6HgN+lDxceANkl7Q6nmzXNIfSDqYpM09gK0AreTncaMUNiJuBL4GvK2VZ/g68EFJj5K0SNJTJP1ua/cvAGdKOkLSIcDb+xz+EuDpkk6VtKT1ep6k35K0VNIaSY+OiEeA+4G9rd/rFElPVdJw/ytgT/u9Lp8H3ilppaQVJLmIjSn7jWpr6/xPLuDYVmMOBDasS0na3tuvcwf5cERsImm3/x8kCdybSRLJtBKsHwR+CPwLSfL18jGU+QPAOkmPJUkeLyUZX7AD+BJJez0kQerrwE+Bq0l+190kN+q032Un8PskOYa7SJrD3g/s39rlVGCzpPuBN5A0GwE8DfgmSZv+D4H/FRHfSTnFe0gS6j8FfgZc1do2VhGxC9gAXN5q4jph3OewevKAMrM+JJ0EfCwiZqsui1kRXCMw6yLpwFbf//0kHQGcA/xD1eUyK0rhgUDS0a3eH+3X/ZL+vOjzmo1AJF00d5A0Dd1A0i5vNpVKbRpSMnLzTuAFEbGl3/5mZla8spuGXgrc4iBgZlYfZY8jeC1JV7gFWv281wEsX7581THHHFNmuczMhvezn8HDDy/cvnQpPPOZ+Y+zfTts2QJ7O3oRL1oEj3883NUxdGTFCpid33fhyiuvvDciVjKE0pqGWvO93AU8IyL+pde+q1evjk2bNpVSLjOzkS1aBGn3Umn+Tb2fo45KAkEv27fDYx6TcipdGRGr859snzKbhk4CruoXBMzMJs7MzGDbs9x2W/Z7X/taEmxSgsCoygwEf0xGs5CZ2UTbsAGWLZu/bdmyZPsgsgKHBFu3Dle2HEoJBJKWk0yPe1EZ5zMzK9WaNXD++Um7vZR8Pf/8ZHte116b3SwUAevXj6esKWo5stg5AjNrFCnfPj3yDZOSIzAza565uSQJvGhR8nVubt970sIgMK58wwAcCMzMijI3B+vWJU0+EcnXdeuS3EF3AHjjG5N93vve8eQbBuD1CMzMirJ+PezaNX/brl3wznfO39bZRN/OK6xfn/QimplJgsAg+YYBORCYmRWlV3dQgEcegf1SbsNr1hR64+/mpiEzs6L0atefnU0PAhVwIDAzK0qvUcL9agslqkc4MjObJnm6gxbYC2hQrhGYmY3LTTctDAIHH1x6L6BBORCYmY2DBEcfPX9bBNx//+ijjgvmpiEzs1GkNQM98AAsX77v55J7AQ3KNQIzs2Hcemt6EIiYHwQmgGsEZmaDygoAE8o1AjOzvNLmBtq6daKDADgQmJn1d/fd2bWAFSvKL8+YuWnIzKyXKWsGSuMagZlZmrRmoNtvn7ogAK4RmJnNt307HHbYwu1TGADaHAjMzNoa0AyUxk1DZmZpzUA33dSIIAAOBGbWZA88kF0LeNrTijtvr+UrK+CmITNrpqqagdrLV7ZXLmsvXwmVTUPhGoGZNUtaM9A115TXDJS1fOX69eWcP4UDgZmNrmZNHal+85v0WsDGjfDsZ5dXjqwFaSpcqMaBwMxG027q2LIleapuN3XUKRhIcOCB6e+VXdasBWkOPbSyYOpAYGajqWFTx79KawbqVnZZN2xYuFDN0qXJugUVBVMHAjMbTQ2bOti9Oz0AZAWFMsu6Zs3ChWoOPhgeeWT+fiUGKAcCMxtNVlNHVWvySrBkyfxtEcmrLmVdswY2b4a9e5Ov27en71dSgHIgMLPRpDV1VLEmb1oz0EUXze8NVJeydqs4QDkQmNlo0po6ylyTd+/e7DEBr3pVvcqapeIApajhEOrVq1fHpk2bqi6GmdXdNM0NNDeX5ARuuy2pCWzYMFCAknRlRKwe5tSuEZjZ5Dn88IVB4FOfyh8EOsc9rFiRvKoeA9GdNyixllJKIJB0iKQvSfq5pBsk/XYZ5zWzKRORBIBf/nLh9jPOyHeM7nEP27Ylr7qMgahgcF5ZNYIPA1+NiGOAZwM3lHReM6uLUW9wUvLZTu3eQINIG/fQqcoxEBUNzis8EEh6NPAi4JMAEfFwRNxX9HnNrEZGucE973kLm4E++MHhcwF5umRWNQaiosF5hSeLJT0HOB+4nqQ2cCVwZkQ82LXfOmAdwMzMzKotW7YUWi4zK9FRRyU3/26zs0l7eJYiksFZZenUr1xFWbQo/feTktxBD3VPFu8HPBf4aEQcDzwInN29U0ScHxGrI2L1ypUrSyiWWUNVMUHcoKOP08YEDNMMlCatq2anKscVVDSeoIxAcAdwR0Rc0fr5SySBwczKVtUEcXlvcKecsjAArF8/3i6h3WMJDjssedVhXEFV4wkiovAX8H3g6Nb35wIf6LX/qlWrwswKMDvbfq6e/5qdLfa8GzdGLFs2/5zLliXb29LKlXac2dkIKfna+flpMeTvCGyKIe/RpQwoa+UJPgEsBW4FzoiIHVn7e0CZWUFGaIMeWdaAqbx5gO6VvSB5Wq7DyOAaqHuOgIi4JpL2/2dFxL/rFQTMrEBVzmnTPWDqW99aGATWrctuBqrzdNcTzmsWmzXJhg3pT9VVTBDXrV/rRB2nu54SnmLCbFrk6Q1U9aRro/QGqssU0lPIgcBsGgzSG6iKOW3OOGNhAHjpSwfrDVTXKaSngGcfNZsGww7YKsM4B4WNOEPnNKt9stjMCjbO9vNeTUyDDEYrYlBYhTN0LlDFwLyCOFlsNg1mZtJrBIO2n3d30Ww3MbVlvdd5Q3772+Gv/mr+cZ/wBLjzzsHKUme9rtME1lDcNGRWV4M0g4yrj32vJibo3/w0TQvF9FLDprhRmoZcIzCro0GfONvbRm0/H6aJ6bbbmhMA2qasK6trBGZ1lPXEuXgxXHBBcc0Pw9QI0tTwvjJWU1YjcLLYrI6yniz37Cl2krheXTT7zdoJ45shtO6mrCurA4FZHfVK8hY5rUKvAWft99Ls3duMANBW9cC8MXMgMKubuTl44IHe+xTZFp3VRfM974G1axfuv2wZfO5zxZVnEGV26axTV9YRORCY1Uk7SbxtW+/9ippWIetGKsG73pX+mbpM/FbVWgtTwMliszrJs4xiUVMvp3VBzauMaaz7qWECt0xOFptNi15NPkW3RadN89yt3XuoWx0mfpuyLp1lciAwq5OsG+rsbPFt0b1umO3eQGm9ZaTkSbzqaRY8O+nQHAjM6qTKbolZzcSdtYDO3jKQBIH256puk5+yLp1lciAwq5M1a+C005KBY5B8Pe20YnukXHhh+shgSL+RtnvLzM4uDB5VJo6nrEtnmZwsNquTstflTQsAs7P5pqmocv1jW8DJYrNpMcq6vKNOEb1tW3Jjz9s33m3yU8OBwKxOhu35krcP/Ve/mj1B3KGHDlbWOieObSAOBGZVSXuCz3qaXrSo95N+npqEBCedNH+fUeYGqnPi2AbiHIFZFbJyAaedlswu2qs/f1rOoFd7fdr2LVvG24TT8MFcdeAcgdmkyXqCv/TS+T1f2r2Huvfrzhlk3dTTgkDE+NvxPZhrojkQmFWh142zczKzrN433Z+veopoJ44nmgOBWRXy3jjz7tdriuhrry1+iuhJG8w1RQvPj4MDgTVbVTeEvDfOvPtdf336FNER8IxnjF7efiZpMJdnKV0oImr3WrVqVZgVbuPGiGXL2g0myWvZsmR7WeefnY2Qkq9Z5+23X2f52y/LNjubfs1mZ6su2UiATTHkPde9hqy5Jr2nS9p4gMsugxe+sPyyTJIpHRHtXkNmw5ikni6dTVhHHpk9KMxBoD8nthcoJRBI2izpZ5KukeRHfauHSbkhdLdp33nn/PfrsGD8JCVfJy2xXYIyawQviYjnDFt1MRu7SbkhZC0Ys3Jl9QEAJi/5OkmJ7ZKUkiOQtBlYHRH35tnfOQIrzdxccqPNM9tmFbZtgxUr0t+rS5v2pOdapsQoOYKyAsEvgB1AAOdFxIIOz5LWAesAZmZmVm3pt26r2bTLWiOgrS432ilNvk6aSUgW/05EPBc4CfgzSS/q3iEizo+I1RGxeuXKlSUVy6yG0qaIXrp04T4nn1xemXrJmrW0brkWy1RKIIiIO1tf7wH+AXh+Gec1G7u0pOi4EqUPPpjdG+j1r5//XkQyOV3V7fBzc7Bz58LtS5bUL9dimQpvGpK0HFgUETtb338DeHdEfDXrM84RWC2lzRi6ZElyg3744X3bhllRLCsAtNW1HT6rXIcdBvfmSgnamNS9aehxwA8k/QT4f8A/9QoCZrWV1nvnkUfmBwEYbN3etGagd7xjYZt7Xcc8ZJ1/+/Zyy2Ej2a/oE0TErcCziz6PWeEGuen22/eRRxa2+0NSm0ibG2hmJv3Ju+p2+LqWywbikcVmeQ1yc1u0CN70pvTcgZQeBCC7NlHXMQ91LZcNxIHALK+0m96SJek39T174KMfnT/Iau3a/l1CIb02UddBUHUtlw3Ek86ZDSJtABokS0zu2TPYsWZn65kAtolU92Sx2fToXD1s8+bk5zVrBhs4tXFjUktws4rVhAOB2TgMkj9o5wDcrGI14UBgNg551gxu68wBpNUwzErmQGA2DmvWpM8QmubQQydnymZrBAcCs1GlDQo7/PAkF9BdS1i6FO6/f3KmbLZGcCAwG0XW1BB33ZWeAzj44GQwWadBRiKbFcCBwGwYabWAtJXCunMAWVMvjGOqiElaJcxqxYHArJfum2taAID8K4UVtTzmpK0SZrXiQGD1VfUTbtrNtdug6wUXNXYgbUI8NzlZTg4EVk91eMLNWisYhl8wvqixA3WdndQmgqeYsHqqw/z7WfMC1XEJxjpcL6uUp5iw6VP1E26vyeHqOMWyp6uwETgQWD0VlVTtJysZ3FbXm6unq7AROBBYPQ37hDtsgjkrAGzcODk3V09XYUMqfIUys6G0b2LdUz73url1ryncTjB3Hi9Nv+6gvqHalHONwEZXVDfPQZ9wB+1CmXdQmNmUc43ARjPsU3gR8iaYDzoIHnxw4X4OANZQrhHYaKoYyJRVA8mTYJYWBoG0WkDVg9nMSuQagY2m7G6evWogGzbMfw/2JZjT8gB796Zvr1Mtx6wEHlBmoyl7IFO/83WvKbzffnDLLQv37/V378FZNoEKHVAm6RuSnj3Mwa0Byh7I1K8G0plg3rJlYRDIkwyuejCbWcny5AjeDnxI0qclHV50gWzClD2QKW8eoLvJZ/fu6mcINaupvoEgIq6KiJcAlwBflXSOpAOLL5pNjDIHMvWqgZx+evaYgMWLx3MOsymUq9eQJAE3Ah8F3gL8s6RTiyyYWaqsGsjatXDBBfP3rdsMoWY11TdZLOly4EnAdcCPgCuAnwNnAvtHxLpxF8rJ4hrpTr72G91btrQawK9/DQccUH5ZzCpU9Oyj64AjIuLEiHhXRFwSETdHxFuAFw5zUpsQZa8JMEjf/bPPzm4GchAwG0ieHMF1kV1t+IMxl8eGVcQAqDIHiw0SdCR4//vnb4tIJojzIDCzgXkcwTToHgAFSXJz1HbtRYvS29iLWJglT9/9tBrAjh1wyCHFXQOzCTERC9NIWizpakmXlHXOxijqyb3MbpS9+u5/5CPZzUCHHJJ87zV7zYZW5lxDZwI3lHi+5ihqAFSZ3SizgksEnHnmwm3dNRUPAjMbWimBQNKRJPmET5RxvsYp6sm9zG6UaUGn2+23JwEgLR/iQWBmQyurRvAh4G1AZsOypHWSNknatHXr1pKKNSWKfHIva7BYO+isWJH+fgQceWR2Uvnkkz0IzGxIhQcCSacA90TElb32i4jzI2J1RKxeuXJl0cWaLtMyAGrtWrj33vnbupuBsnIBl146HdfArAKF9xqS9N+AU4HdwAHAo4CLImJt1mfca6hh0hLB110Hxx67cHtWT6b2ceo46M2sBLXuNRQR74iIIyPiKOC1wLd7BQGrUNmLsXzzm9m9gdKCAPRu8y9j0JvZFPIKZf00ZaWqskcRS3DiifO35ZkbKE9S2d1GzQZSaiCIiO9GxCllnnMkZd8cq1RWP/y0KaIvuyz/5HDd+ZAs7jZqlptrBL00aZDSqP3w+9Wcrr46uxnohQNOWdXZk2l2Nn0fdxs1y82BoJcmDVIapR9+v5qTBM997vzPtJuBRm1689oBZiNzIOilSYOURrmhZtWc1q5dWAu46KJ9zUDjaHqblq6zZhXypHO9NG0is2HXHujVpbNT9z5eJN5sbEbpPupA0E/dF2apg6wbelvW31iZs5uaTblajyOYeGWuxzupspqPTj+9d01hmKa3pnTnNSuRA4GNZtu2JBfQbeNG+PSne3920LzEsDkFBw+zntw0ZMPL6g46iEGa3obJKTQtz2ON5RyBlSstAJx7LpxzTrHnHSan4IS0NcQogWC/cRfGptiuXbB8+cLtZT1MzMyk39R75RSaNBbEbEjOEVg+0sIgkGduoHEaZqxDk8aCmA3JgcB6S5sb6PWvLzcAtA0zeMwjj836ctOQpdu9G5YsWbi96pzSmjWDJXnb+3osiFkmBwJbaBy9gepk0OBh1jBuGrJ90pqBXvWqyQ4CZtaXA4ElN/qsWsBFF+U/jgdumU0kB4Kmk5Ibd6es3kC9bvRNWsTHbMo4EDRVWjPQMcdkNwP1u9E3aREfsynjkcVNNEwyuN8IXc8kalYpzz5q+aTVAvIOCus3QtcDt8wmlgNBE6QFgIMPHqw3UL8bvQdumU0sB4Jpl9UMdP/9gx2n343eS0aaTSwHgrxG6RpZVLfKuTlYsWLfE/9BByU/L1o0WjNQmjw3ei/iYzaRnCzOY5Q57YuaD39uDv7kT+Dhh/PtX8N/ZzMbH69HULRR5rQvaj78fusEj+s8ZjYR3GuoaKPMaV/UfPh5P+95982sDweCPEbpGllUt8q8n3f3TTPrw4Egj1G6RhbVrTJPs5C7b5pZDg4EeYzSNXKYz/bqZZTWG6ht+XI47LDRum964jizxnGyuG6yehk94xnw4x8v3H+c/35F9XAys8LVuteQpAOAy4D9SRbC+VJEnNPrM40OBHl7AxXx71ZUDyczK9wogaCMFcoeAn4vIh6QtAT4gaSvRMSPSjj35OnXy6fIwF1UDyczq7XCcwSReKD145LWq37tUXXRq5ePVGy7vSeOM2ukUpLFkhZLuga4B/hGRFyRss86SZskbdq6dWsZxaqnXs1CRS/4kqeHk5PJZlOnlEAQEXsi4jnAkcDzJR2Xss/5EbE6IlavXLmyjGIVb5CbZlpvIAkWL164b1ELvvTr4eRVyMymUum9hiT9JbArIv571j5TkSzO2wPnvPPgDW9Y+Pn2v0udFnxxMtmstmo9xYSklZIOaX1/IHAi8POizztWwzSH5Fm6UVoYBDpnCJ2bW7iecFtWu32RTTdOJptNpTJ6DR0OXCBpMUng+UJEXFLCecej+8m+3RwCvfvW97pppg0I27Nn/k2/fd49exbumzVieNiy5jUzk14jcDLZbKJ5QFk/wzaH5B0PAOlNP1mfX7wYLrhg4Y19bg5OOy09cIyr6cYDzsxqq9ZNQxNv2OaQtB443XotFJN1/L1704NAVu2h17EGbUbyKmRmU6mMpqHJNmxzSPvmuHbtwvceegiWLh3tvHNzSb7httuSG3lWEMgq67DNSGvW+MZvNmWmr0Yw7mTpsLOH/uAH6UEgon8Q6Hfe7m6cvYJAVlnzJLPNrBkionavVatWxVA2boxYtqzd4JK8li1Lto9i48aI2dkIKfna73id52+/xnne2dn0c3S/Fi/OLquU/hlpuLKaWaWATTHkPXe6ksVV93NP6w20axcceOB4z5M1tqBTvyRu1dfKzMbKyeK2qvq5X3NNehCIGH8QgOz8xOLF+ZO4RS2YY2YTZ7oCQRWTpklw/PHzt/XqDTQOWTfxCy5IehVt3tw/oeseQGbWMl2BoMyn3LS5ge67r9gA0Daum/iaNUnQyBs8zGwqTVf30faNrN2tcmYmCQLjvMHdeis85SkLt5eda3E3TjMbk+mqEcD4n3Lb3VHbNYDuIFB0M5CZWcGmq0YwbmlTKrQdeCB8/OPll8nMbMymr0YwrLSBaO94R3oQAPj1rz34ysymgmsEkD7dQtqo4G6eftnMpoBrBJA+3UIenn7ZzKaAAwH0frLPmkHUg6/MbEo4EEB2r592//zZ2eTn9vrBHnxlZlOk2YHgy19OnxoC9j3xt7ujRsDu3clXD74ysynSzGTx7t2wZMn8bSeeCDfdVNxANDOzmmpeIEirAUhJEPDN38waqDlNQ5dcsjAItGcGjdi3QlfehWwGWQBn3IvlFH1cM2uWYRcyKPI19MI0afbsWbj4yvvel724y+xs/2MOsgBOkYvlFHFcM5tIeGGaDI96FOzcOX9b+/fNWtxFSuYp6mWQRV2KWgDGC8uYWQcvTNPtm99MbuidQWDHjvk3/lHWLhhkAZyiFssZ13HdvGTWeNMVCCKSAHDiifu2nXtusv2QQ+bvO8raBYMEkaIWyxnHcdtTa2zZMlyexMymwvQEgs9/Pnmq7RQB55yTvv8oi7sMEkSKWixnHMdNm1pj1y5PpmfWNMMmF4p8DZQsvueehQnf++/P//lhbdyYJJal5GuvJO0g+xZVhjRSesJcGk/5zKw0NDJZHAGnnQaf/ey+bT/5CTzrWcUWbpo44Ww2NZqXLP7KV5JmoHYQeO97k8DgIDCYMtd4NrPamqxAsG1b0p5/8snJz096UrJAzMyMe74MY5Q8iZlNjXoHgs6ujQcdBCtW7HvvqquSheS//OXx9HxpajfKca/xbGYTp745gre+NX294D/8w+Tm3zaOdu60tYmXLfPTsZlNjFFyBIUHAklPBP4OeBwQwPkR8eFen1m9enVsuvfefDf4UUYItzlpamYTru7J4t3AWRFxLHAC8GeSju37qbwjZ8cxsKqo0b9mZhOg8EAQEXdHxFWt73cCNwBH9P1g3hv8OHq+FDX618xsApSaLJZ0FHA8cEXKe+skbZK0aevWrflv8OPo+eJulGbWYKUliyUdBHwP2BARF/Xa918HlM3NJdMdlLFqWJnnMjMbs1oniwEkLQEuAb4WEX/db/+xTUNtZtYQtU4WSxLwSeCGPEHAzMzKVUaO4N8ApwK/J+ma1uvkEs5rZmY5FL54fUT8AEhZMd7MzOqg3lNMmJlZ4RwIzMwazoHAzKzhHAjMzBrOgcDMrOEcCMzMGs6BwMys4RwIzMwazoHAzKzhHAjMzBrOgcDMrOEcCMzMGs6BwMys4RwIzMwarrSlKgchaSdwY9XlqIkVwL1VF6IGfB328bXYx9din6Mj4uBhPlj4egRDunHYJdemjaRNvha+Dp18LfbxtdhH0tDr+7ppyMys4RwIzMwarq6B4PyqC1AjvhYJX4d9fC328bXYZ+hrUctksZmZlaeuNQIzMyuJA4GZWcNVFggkvVzSjZJulnR2yvv7S7qw9f4Vko6qoJilyHEt/qOk6yX9VNK3JM1WUc4y9LsWHfu9WlJImtqug3muhaTXtP42rpP0ubLLWJYc/0dmJH1H0tWt/ycnV1HOMkj6lKR7JF2b8b4kfaR1rX4q6bl9DxoRpb+AxcAtwJOBpcBPgGO79nkT8LHW968FLqyirDW5Fi8BlrW+f2OTr0Vrv4OBy4AfAaurLneFfxdPA64GHtP6+bFVl7vCa3E+8MbW98cCm6sud4HX40XAc4FrM94/GfgKIOAE4Ip+x6yqRvB84OaIuDUiHgb+Hnhl1z6vBC5off8l4KWSVGIZy9L3WkTEdyJiV+vHHwFHllzGsuT5uwD4r8D7gd+UWbiS5bkWfwr8z4jYARAR95RcxrLkuRYBPKr1/aOBu0osX6ki4jJge49dXgn8XSR+BBwi6fBex6wqEBwB3N7x8x2tban7RMRu4FfAYaWUrlx5rkWn15NE+2nU91q0qrlPjIh/KrNgFcjzd/F04OmSLpf0I0kvL6105cpzLc4F1kq6A7gUeEs5RaulQe8ptZ1iwlJIWgusBn636rJUQdIi4K+B0ysuSl3sR9I89GKSWuJlkp4ZEfdVWaiK/DHwmYj4oKTfBj4r6biI2Ft1wSZBVTWCO4Endvx8ZGtb6j6S9iOp7m0rpXTlynMtkPQyYD3wioh4qKSyla3ftTgYOA74rqTNJO2fF09pwjjP38UdwMUR8UhE/AK4iSQwTJs81+L1wBcAIuKHwAEkE9I1Ua57SqeqAsGPgadJepKkpSTJ4Iu79rkYOK31/R8B345WJmTK9L0Wko4HziMJAtPaDgx9rkVE/CoiVkTEURFxFEm+5BURMfRkWzWW5//IP5LUBpC0gqSp6NYSy1iWPNfiNuClAJJ+iyQQbC21lPVxMfC6Vu+hE4BfRcTdvT5QSdNQROyW9GbgayQ9Aj4VEddJejewKSIuBj5JUr27mSQx8toqylq0nNfiA8BBwBdb+fLbIuIVlRW6IDmvRSPkvBZfA35f0vXAHuA/RcTU1ZpzXouzgI9LeitJ4vj0KX1wRNLnSR4AVrRyIucASwAi4mMkOZKTgZuBXcAZfY85pdfKzMxy8shiM7OGcyAwM2s4BwIzs4ZzIDAzazgHAjOzhnMgMDNrOAcCM7OGcyAwy6E11/2Jre/fI+lvqy6T2bh40jmzfM4B3i3pscDxwNSN7Lbm8shis5wkfY9kqo8XR8TOqstjNi5uGjLLQdIzgcOBhx0EbNo4EJj10VrdaY5k5acHpngBGGsoBwKzHiQtAy4CzoqIG0iWyTyn2lKZjZdzBGZmDecagZlZwzkQmJk1nAOBmVnDORCYmTWcA4GZWcM5EJiZNZwDgZlZw/1/MB/eQrEhaToAAAAASUVORK5CYII=\n",
+ "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAAk5UlEQVR4nO3de5QdVZ0v8O+3OwnSIZAhD0SgOzwEDDiAyUUYGUURF6LiuktlmGlCoqO9BGXplZFhjAjDtZ0ZufhgvAjhJaZ7lIjMXC6DrxkUGC4whDchw7080uGl6TwgwQB5/e4fVWf69DlVp+rUqceuU9/PWr26u06dquqC1O/s3977t2lmEBGR6uop+gJERKRYCgQiIhWnQCAiUnEKBCIiFadAICJScQoEIiIVp0AgqSH5xySfLPo6ugHJVSRPzOC4+5C8k+QWkpeR/ArJa9I+j5QLNY9A2kVyDYBPm9m/FH0tYUheDGApgDcA7ADwBIDzzOyeIq+raCQvBHAMgI9Zwz9+kvMAPAtgqpntKODypCBqEUjpkZwS8tKNZrYHgNkAfg3gJxmcmyTL9O9oAMATjUFAqq1M/wOL40ieSPL5ut/XkPwLko+SfIXkjSTfVPf6h0k+TPJlkv+H5B/WvXYByaf9FMYTJP9r3WtLSN5N8tskNwC4uNV1+Z9uRwHsR3KOf4y9SF5L8iWSL5D8Osle/7VeP22ynuSzJD9P0moBh+RvSA6TvBvAVgAHkTyc5K9IbiT5JMnT6673VP9v2OKf6y/87bNJ3ur//RtJ3lULKv69e7//824kv0PyRf/rOyR3q7/nJM8juc7/ez4Z8t/nBwAWAzif5Ksk30/yYpIj/i53+t9f9l8/vtV9le4R9klKJC2nAzgFwOsA7gawBMCVJI8BcB2AjwBYCeBMALeQPMzM3gDwNIA/BvBbAJ8AMELyEDN7yT/uOwH8GMA+AKa2ugCS0wCcBWADgE3+5h8AWAfgEADTAdwK4DkAVwH4DIAPAjgawO8R3JJY5O/zpP/+xwF8zd/2dgC/Ivm4mT0B4FoAp5vZXST/AMCB/jHOA/A8gDn+78cBCPqkvtR/7Wj/9f8F4KsALvRffzOAvQDsB+BkADeR/Ccz21R/EDNbQhIAnjezr/r35oS6Xd4NLzU0U6mhalGLQLJ2uZm9aGYbAfxveA8zABgCcJWZ3WdmO83sBnj5/OMAwMx+4r9vl5ndCOD/ATi27rgvmtnfm9kOM3st5Nynk3wZwGvwHu4fN7MdJPcBcCqAL5rZ781sHYBvAzij9j4A3zWz5/2H6d8GHPsHZrbKf2CeAmCNmV3vX89DAH4KL4ABwHYA80nuaWabzOzBuu37Ahgws+1mdldIymYQwCVmts7MxgH8NbxAhLrjXOIf4zYArwI4LOSeiDRRIJCs/bbu560A9vB/HgBwnp8Wedl/YB8A4C0AQPKsurTRywCOhJfrr3kuxrlXmNlMeK2GxwEsqDv3VAAv1R3/KgBz/dff0nD8oHPVbxsA8M6Gv2UQ3id1APgYvMAzRvKOupTLpQCeAvBLks+QvCDk73gLgLG638f8bTUbGj7B199nkUhKDUlRngMwbGbDjS+QHABwNYCTANxjZjtJPgyAdbvF7uw0s/UkhwCsJPkP/rnfADA7JAXyEoD9634/IOiwDX/LHWZ2csj57wfwUZJTAXwewAoAB5jZFnjpofNIHgngdpL3m9m/NhziRXjBZpX/e7+/LW3qQK4otQgkqakk31T31e6HiqsBfJbkO/2RN9NJfojkDHg5dwMwDgB+5+eRnVysmT0J4BcAzvf7GX4J4DKSe5LsIXkwyff4u68A8AWS+5GcCeAvIw5/K4BDSS4iOdX/+i8k30ZyGslBknuZ2XYAmwHs8v+uD5M8hF7i/hUAO2uvNfgRgK+SnENyNry+iJGA/To17p//oAyOLQ5TIJCkboOXe699XdzOm81sJby8/ffgdeA+Ba8jGX4H62UA7gHwO3idr3encM2XAhgiORde5/E0ePMLNgG4CV6+HvCC1C8BPArgIXh/6w54D+qgv2ULgA/A62N4EV467O8A7ObvsgjAGpKbAXwWXtoIAN4K4F/g5fTvAXCFmf064BRfh9eh/iiAxwA86G9LlZltBTAM4G4/xXVc2ucQN2lCmUgEkh8EcKWZDRR9LSJZUItApAHJ3f2x/1NI7gfgIgD/WPR1iWQl80BA8jB/9EftazPJL2Z9XpEOEN4QzU3wUkOr4eXlRbpSrqkhejM3XwDwTjMbi9pfRESyl3dq6CQATysIiIi4I+95BGfAGwrXxB/nPQQA06dPX3D44YfneV0iIu564IHmbQsWNOzywHozm9O8Y7TcUkN+vZcXARxhZr9rte/ChQtt5cqVuVyXiIizvvAF4PLLJ2/btQsggdFRYOlSYO1aoL8fc8bGnh03SzQHJM/U0AcBPBgVBEREYhsdBebNA3p6vO+jo0VfUTpqD/v6IPDlLwNmE0FgaAgYG/O2jY2h35t9nkieqaE/RUhaSESkbbWH4dat3u9jY97vADA4GP4+15HN2xozN0uXTvzdtbd18ME+lxYByenwyuPenMf5RKQCAh6G2LrV215G993XHATWrGkOAoCXDkpRLoHAL/U7y8xeyeN8IlIBYQ/DlB+SuSCB4xoqepgBA3XZnvo0WE+6j27NLBaRcurvb2+7iw46qLkVYNbcCmjsE9jZXPbKggsWxqJAICLlNDwM9PVN3tbX52133WuveQHg2Wcntl1ySXAaCAhOgwFAb693nIEBrJ28ZkVbtB6BiJRTrUO4bgglhofd7yiO0xncKCzdtWuX9wVgPbkx6SWpRSAi5TU46HWo7trlfXc5CPzwh81BYP366CAAZJ4GU4tARCRrSVoB9YaHJw+VBVJNg6lFICISR5LJa2S8zuAog4PAsmXeKCK/TwDLlqXWAlIgEJFo3TqDN66AmbwYGgq/D+PjzQFgdLT9AFAvwzSYAoGItNbuQ9A1aQSxdiavkcDcuZO3mQF/9mftnzcnCgQi0lqZZ/CmFcTiTF772teaWwGvv95ZKyAnTq5ZrOqjIg7p6Ql+mJH/OXTRWfPmeQ//RgMDXnolreM0BoDDDwdWr45//BSQfMDMFiZ5r1oEItJa2Wbw1qeCgh7eQPtlKMImr42NBXcG5xwEOqVAICKtlWkGb2MqKEy7Qaw2amfWrIltjemyO+8sRRooiOYRiEhrZZrBG1aKoV4nQey114K3lzQA1KiPQES6R1h/Rs3AQPIgFjQprHbMdvobMtJJH4FaBCLSPfr7w/sFyGQP7F27vOJuYcpY9rqB+ghEpHsMD4d/ck/SuU22DgJJj+sYBQIRKbf6UUJLlwLve19zMGi3X+D7328+xtKl5ek0b5NSQyJSXkHrFo+PA5/9LHDbbck6t1sViHvb28rRad4mdRaLSHmlNWEM6LxCaME0oUxEul9QzaA01i3esqU5CMyf734QaLgfs4G9kx5KqSERcV9QCmhoCNh7b2DDhub943bglrUVEHA/+oGB1m8KpxaBiLgvrPAdkKwD90/+pDkIPPNMOYIAEHg/2MHzXIFARNwXlurZuLH9BVtIYMWKydvMgAMPDH+Pa+sxpDx3QakhEXFf2ESx/n7voR9n5E7SNFBYWgoobsRQq4lzCahFICLxFPmpuJPCd08+2RwETj89fhrIxfUYAu6HAYlrgqtFICLRiv5UnLTwXRqdwWmMTEpbwP1YO5a8iaAWgYhEc+FTcW3N3uXLvd8XLQpvmfT0NAeBTZuSdQa7uh5DwxrG64GNSQ+lQCAi0Vz5VBxn6Umy+YFvBsycmeycZVqPISEFApFOuDaaJCuufCpu1TIhg1cL63RIaG1RmnZGJpWMAoFIUmktjF4GrnwqDmuBNKbHL7oo3TkBDWmYbgoCgAKBSHIu5M3z4sqn4jgtEDPg4oszv5RukksgIDmT5E0k/4PkapLH53FekUy5kjfPiwufioNaJjU7dpRnZrBj8moRfBfAz83scABHAVid03lFsuNK3rybNfbBmAWvSWwWvYCMhMo8EJDcC8C7AVwLAGa2zcxezvq8IplzJW/erYL6YBYtmrxPGp3BkkuL4EAA4wCuJ/kQyWtITm/cieQQyZUkV46Pj+dwWSIdciVv3q2C+mBqli9XAEhR5gvTkFwI4F4A7zKz+0h+F8BmM7sw7D1amEZE0NMT/LAnvX4KmcT1hWmeB/C8md3n/34TgHfkcF4RKaugSWE1afTBVGX+R0yZBwIz+y2A50ge5m86CcATWZ9XREpo/frg+kA1afTBVGn+R0x5jRo6F8AoyUcBHA3gGzmdV0TKggTmzJm8bWQk/T6YKs3/iCmX6qNm9jCARLkrEelyQX0Bd90FnHCC93Pane9Vm/8Rg2YWi0hxwgrE1YJAFtKa/9FF/QwKBCKSvcaHZlYF4uJIY/5Hl/UzKBCISLaCHpqN8pwTkMb8jy7rZ8h8HkESmkcg0kXmzQtfX9fB508sDs5xcH0egYhUWVgQaDVM1HVdVmdKgUBEstPqYV/ShyaArqszpUAgEqWLRofkJqgzuF6ch6bL973b6kyZmXNfCxYsMBEnjIyY9fXVxrN4X3193nZp9q1vTb5XgNlRR3n3a2DAjPS+R90/3fe2AVhpCZ+56iwWaSWso3NgwFucRSYEtQCSPl9039umzmKRrGgWarSgNNC2bZ2NCNJ9z5UCgUgrXTY6JFW7doW3AqZO7ezYuu+5UiAQaaXLRoekhmxeGjLNmcG677lSIBBppRtGh6Q5+ubkk5tbARdemP7EsG647yWizmKRblYr71BfDqGvL9lDNc3OYEmdOotFsubymPZW0qiJU2SBOMmFAoFIlDJXmuxk9M26dWoFVIQCgUiUMleaTDr6hgT22WfyNrUCupYCgUiUMo9pb3f0TVAa6JZbFAC6nAKBSJQyj2lvZ/RNWBroIx/J/jqB8vbDdAEFApEoZR/TPjjolWXYtcv73hgEXOgMLnM/TBdQIBCJ0q1j2u+4w53O4Fb9MEEtBbUeUqV5BCJZGx31Hmhr13rppOHh4oOIKwGgJmzFL8BrfdUHialTvevftm3yPt0QnDugeQQirnIt5RGUBnr66eI7g8P6W3p7m1sK27dPDgJAeUZxOUqBQCRLLg09DWsFHHRQ/tfSKKwfZufO+McowyguRykQiGTJhaGnLnQGRwnrhxkYiH+MMozicpQCgUinWnVcFjn09G/+pjkAzJjhVgCoFzS6KailMHUqMG3a5G1lGsXlIAUCkU5E9QEEPchIb78sR7uQwFe+MnmbGbB5czbnC9Pp6J6glsL11wPXXdd9o7gKpFFDIp2Is6RibdTQ2Jj34Kr/N5f2aJegfoDXXwd22y2d47cjzcqnEqmTUUMKBCKdCBv2SHopjnpZrsO7a1fzQjFAsWkgrTucKw0fFSlKO30AaXQcB6Vasl4tLCkXOsolllwCAck1JB8j+TBJfdSX7tFO+YlOO46D+iPOPHPyPp/+dPEBoKbMNZoqJs8WwXvN7OikTRcRJ7VTfqLTmkVBcxLqmQFXXx3/2rNW9hpNFaLUkHSXImrQRBV1q9+vk5pFYSmVxg5oV3RrjaYulEtnMclnAWwCYACuMrNlAfsMARgCgP7+/gVjQZ1MIq108yiV8XFg7tzg19T5KihHZ/EJZvYOAB8E8DmS727cwcyWmdlCM1s4Z86cnC5LuopL5RzSbJmQ4UEAAE49NfmxRZBTIDCzF/zv6wD8I4Bj8zivVIwro1TSKjQXVBoiyG23JbvORirtXFmZBwKS00nOqP0M4AMAHs/6vFJBroxSSaNlEhQAwoJCGoHOtSqpkqs8WgT7APg3ko8A+HcA/2xmP8/hvFI1roxS6aRl0qpAXJaBzqW0muQu80BgZs+Y2VH+1xFmprFjkg1XRqlEPbCDUjC33x69WEyWgc6VtJoUQiUmRNLWavQS0PxakLB/l1mtdqZyEKVXhlFDItXRqmUSNSls9erWcwLizllolytpNSnElKIvQKQrDQ4GP6RbpVqKbJ3XrtW1tZUlFwoEInkKe9i3sxJXVsKCl3Q9pYZE8nD++eHDP5WCkYIpEIhkjQQuvTT8tcWLvZ81mUsKotSQSFbizAo2A1asAG64YaITuTaZC1CqRnKhFoFI2kZG2psZvGGDJnNJoRQIRNJEAosWTd7W1+cFh3ZnAGsyl+REgUAkDQcfHP6Jv/bpPmys/qxZwe/TSl6SE/URiHQqTl/A2Fj4WH0geCayRhJJTtQikGrrpPRyUIG4xkXkG7cHzQwOmom8eLEXMNIaRaQS09KKmTn3tWDBAhPJ3MiIWV9frban99XX521vZXx88ntqX2bB2+tfz/K68jqeOAnASkv4zFXROamuJIXWoiqEhh2TBJYvjzccNO0CcCooVwkqOieSRDull4PSQFdc0VwyYng4PFjEHQ6adklolZiWCAoEUl1xF3oJe7CffXbz9sHB8HpCcR+8aS9A48rKbeIsBQKprrBF32vbW60W1kpYAbm4D960S0KrxLREUCCQ6gpb9P3mm6P7Alrp9MGb9kprrqzcJs5SZ7FUV09PvId7kn8jWa0kJhJCncXiHhfHrTde0957t97/kUeSLxaT1UpiIhnQzGJJX+OavS5U0wy6pmnTgKlTge3bm/d3sKUskhW1CCR9QevyFl1NM+iatm1rDgIjIwoCUjlqEUj6XBy3HufcCgBSUZEtApK/InlUHhcjXcLFcetR5+7ra68fo4g+EBf7XaQrxEkN/SWA75C8nuS+WV+QdAEXx60HlVio107qqtbfMDbmtSJqfSBZPpiLOKdURmQgMLMHzey9AG4F8HOSF5HcPftLk9Jyady6Wbwy0UD81FWcPpC0P7272O8iXSPWPAKSBHAEgBMAfB3A6wD+ysyWZ3FRmkcgqQibFNZpEbaw+QekN1y0cYQS4LWIOgmGUeeUyst0HgHJuwG8AODbAPYDsATAiQCOJbksyUlFMjV/fnMQOO64iQdpp6mrqD6QLD69u9jvIl0jzqihIQBPWHPT4VySqzO4JpHk4pSGCFspLO6n9eHh1iuKZTFqKuqcIh2I00ewKiAI1Hwo5euRMnFpFEu7BeI6mfkb1QeSxad3l/pdpOuo1pAkk0UePIlNm4JLRRT5/7Ur90YqpRS1hkj2knyI5K15nVMy5MIoFrI5CMQpE501fXqXksmzxMQXAKhPoVt0kgfvNKUUlAa6/PLiA0A9FZ2TEsklEJDcH15/wjV5nE9yEJbv7ulp/YDvdGJUWGfwuefGvnQRmSyvFsF3AJwPIHTAM8khkitJrhwfH8/psiSxoCGYALBzZ/gDfnQUWLw4WUqp3c5glzqyRRyXeSAg+WEA68zsgVb7mdkyM1toZgvnzJmT9WVJuxofrMDkPHhvb/N76h/wtZbAzp3Bxw9LKd1/f/urhakcg0hb8mgRvAvAaSTXAPgxgPeRHMnhvJKWsAcrMJEHD5vdWnvAB3Uu1wtKNZHAscc2b48qEOdCR7ZIiWQeCMzsr8xsfzObB+AMALeb2ZlZn1dSFOfBGjV2vlUncuPEqKA0UOO5Fy8OT/u4WAZbxGFamEaixXmwRpVtCAsUvb2Th1bGLRDXqi9C5RhE2pJrIDCz35jZh/M8p6QgzoM1aux8WKC44QZvn7DO4IGB6OtrbJ24WAZbxGFqEUi0uA/WVmPnwwLFhg2tO4PDRic1qm+duDShS6OXpARUYkLiGR1NXqQtTNzRQPXn7ukJHnkUt4R0nlRqQnJUihITUnJxZ8rG+QQclAbavDlegbgbbihP2kejl6QkFAiqLs3URdT4/dHR8FbAjBnxzuFS2ieKRi9JSSg1VGVppy5arfwVtL3b0ySdroQm0galhiSZtFMXYZ90wxaO7+RcYS0ZlzpnNXpJSiLOCmXSrdJOXfT3hz/0272GVhpbMrUU1N13e30IjduBYlodna6EJpITpYaqLO3URVCqqaa/P/ihn+RcYdfd21ueEUUiKVNqSJJJO3Vx2mnBQcAM+MY30jtXWCui3YJ2IgJAgaDa0hyBQwJ77jl5W32Z6DTP1apcRTv7iwgABQLpdCWtoDkB118fPCcgrVW7wloyQ0PqnBVJQIFAkgubE7BkifdzViN4wloXV1xRnjkGIg5RZ7G0L05pCJVXEMmVOoslH6tWxa8PpPIKIqWheQQST7vLRaq8gkhpqEUgrQV1Bj/+eOsgAGhxGJESUSCQcGGtgCOOiH6vyiuIlIYCgTQLWy2snYEFZaoSKlJxCgQyYcWK5gAwc2Z7AaBeWvMGRCRT6iwWT7udwSLSNdQiqLqgNNCWLQoCIhWiFkFVmXkzfoO2i0ilqEXgmjwWViGbg0C7ncEi0jUUCLKQ9GEeteZvpz73ueY00Kc+pQAgUnGqNZS2TmrsZLnGrTqDRbpadWsNubQ+bU0nNXayKMsQ1Bm8a5eCgIj8p/IGgqzTKElFPcxbBa80yzJs3RreCgjaLiKVVd5A4Gp1y1YP86jglVZZBhKYPn3yNnUGi0iI8gYCV6tbtnqYRwWvTssyHHpo86f9kZH8A4CLKTsRCVXezuIsO1Y7NTrqPdzXrvVaAsPD3sO8pyf4oUx6eftOuNIZrAVpRArhdGcxyTeR/HeSj5BcRfKvUzmwy9Utw2rsZFGaOY0CcWlyNWUnIqHySA29AeB9ZnYUgKMBnELyuI6PWsbqlmkGr6efdqcVUM/VlJ2IhMo8EJjnVf/Xqf5XOk+rslW3TCt4kcAhh0zelqQVkEUuXwvSiJROLp3FJHtJPgxgHYBfmdl9AfsMkVxJcuX4+Hgel1WMToJXUBpo1apkrYCgEUxnngnMnt1ZQHA5ZScigXIJBGa208yOBrA/gGNJHhmwzzIzW2hmC+fMmZPHZZVLWBpo/vxkxwvK5QPAhg2dzccoY8pOpOJyHzVE8msAtprZ/wjbp9QlJtKWVT9A2AimGhdGX4lIbK6PGppDcqb/8+4ATgbwH1mft/R+9rPmIHDwwel1Bkfl7LPs3NU8AxGn5LEewb4AbiDZCy/wrDCzW3M4b3nlMRpoeLh5vH+9rDp3G+cZ1GZXA0ofiRQkj1FDj5rZMWb2h2Z2pJldkvU5SyuoM3jz5vaCQNxP27Vc/qxZza9l2bmreQYizilviYluE9YKmDEj/jHaLcQ3OAisX++Vocirc1fzDEScU94SE90izTSQy2U3aspwjSIl5HRnsYS48srmIHDRRZ31BZTh07bmGYg4R4GgCCRw9tmTt5kBF18c/p44uf8yzOrVPAMR5ygQ5CnpamFxc/9l+bRdttIgIl1OgSBKGmPe33ijs9XC4o600adtEUlAncWtpFFbP+hBPzLS3sM5y3UMRKQrqLM4K52MeT/rrPBP++3W8ilD7l9ESkuBoJWko3BIYPny8NfbnUBVlty/iJSSAkEr7X4SD+oMDmsVBAWTsP4I5f5FJEMKBK3E/SQ+Pt78wD/wQC+vHzeYRI0Mihppo0JuIpKQAkErcT6Jk8DcuZPfZwY884z3c9xg0kl/RLulJURE6mjUUFLHHw/ce+/kbWNjwS2A0VHvgb52rff68HDzJ/pORgapbINI5XUyakiBIIksykSHPcxnzfIKw7Wi4aUilafho3kJ6gxOsmh8kOFhYOrU5u1btkSneDS8VEQ6UM5AMDrqLbJeezB3uuB6lEceaQ4AS5aku1jM4CCw557N27dti+4n0PBSEelA+QLB6CjwyU96i6zXbNgAfOpTnQWDc84BpkzxHvhTpni/A97vRx89eV8z4Prrk58rzMaNwduj5i1oeKmIdKB8gWDpUmD79ubtQZ+c4w6pPOcc4PvfB3bu9H7fudP7vbEV8Pvfp79kZL1OUjwq5CYiCZUvELT6dFz/WjtDKpctiz6vWXP6JW1K8YhIAcoXCFp9Oq5/rZ1x+bWWQJC0OoPjUIpHRApQnkBQS/MEDbEEgGnTJn9yDtsvaHtPyG3o7W3rElOhFI+I5KwcgaA+zRNk1izguusmPzTDHuKN21uNtR8aav9as6ZSEiKSsilFX0AsQWkeoPXM2bB0T2170KSw3l7v9d5eLwhccUWiy81M4/oItX4PQC0HEUmsHDOLk8ycDUsj9fc3dzjPnAls2pT0cvOjUhIiEqL7ZxYnGVYZNAIHaA4CZsD3vleOdEvS9RFERFooRyBIMqyyfgROkAce8IJA0ZU7azn/2kQ2MjwYqZSEiGTBzJz7WrBggTUZGTEbGDAjve8jI837BJkYADrxVW9gIHifgYF4x+/EyIhZX1/w+fv6mv/GoP2D9hORygGw0hI+c8vRR5BE3AqhRVbubDUcFgjO/ccpaS0ildP9fQTtePXV5iDwpS+FTworMt0SldsPel3zDEQkZd0VCEhgxozJ28yAyy4Lf0+RZR2igo1y/yKSg+4IBN/8ZnMrYHw8XmmIIss6hI1sAlRjSERyk/mEMpIHAPghgH0AGIBlZvbdFE/QvK3dfo/BwWJSLLVzLl3q9RXUJrQNDCj3LyK5yaNFsAPAeWY2H8BxAD5Hcn7HR/34x7NbLSxPtZy/GbBjh/dduX8RyVHmgcDMXjKzB/2ftwBYDWC/xAd8/XUvAPz0pxPbfvSj8gUAERFH5NpHQHIegGMA3Bfw2hDJlSRXjo+PBx/gE58Adt+9/k3e1wUXuDsbWETEcbkFApJ7APgpgC+a2ebG181smZktNLOFc+bMmfzi737nPfBvumliW1/fRCoo79nAUVQhVERKJJdAQHIqvCAwamY3t/Xmz3wGePObJ35/7DGvMzXuojN5K7pkhYhImzIPBCQJ4FoAq83sW7HfeM89Xivgmmu837/5Te/BeuSRbhdfa2dlNBERB+SxHsG7ACwC8BjJh/1tXzGz21q+64/+yPu+997Ac89NHm/f3x9eYrpoLgcpEZEAmQcCM/s3AAGD/SPceiuwxx7Ae94zefvoqFdGopErE7BcDlIiIgHcnVn8oQ8FB4GhIWDDhsnbZ81yZ5H3IktWiIgk4G4gCBK2ZOUee7gRBIBiS1aIiCRQrjLURZaMFhFxWHXKUGuFLhGR1JUrECj/LiKSunIFAuXfRURSl8c8gnQVVTJaRKRLlatFICIiqVMgEBGpOAUCEZGKczMQbNyoMs4iIjlxs7N4bGxiglitjDOgTmIRkQy42SJonCWsMs4iIplxMxAEURlnEZFMOFlraDZp8xq2bQe2PQo8VsT1FGw2gPVFX4QDdB8m6F5M0L2YcJiZzUjyRif7CDYAD6xPWDyp25BcmbSQVDfRfZigezFB92ICyYBKnfGUJzUkIiKZUCAQEak4VwPBsqIvwCG6Fx7dhwm6FxN0LyYkvhdOdhaLiEh+XG0RiIhIThQIREQqrrBAQPIUkk+SfIrkBQGv70byRv/1+0jOK+AycxHjXnyJ5BMkHyX5ryQHirjOPETdi7r9PkbSSHbt0ME494Lk6f7/G6tI/kPe15iXGP9G+kn+muRD/r+TU4u4zjyQvI7kOpKPh7xOkpf79+pRku+IPKiZ5f4FoBfA0wAOAjANwCMA5jfscw6AK/2fzwBwYxHX6si9eC+APv/ns6t8L/z9ZgC4E8C9ABYWfd0F/n/xVgAPAfgD//e5RV93gfdiGYCz/Z/nA1hT9HVneD/eDeAdAB4Pef1UAD8DQADHAbgv6phFtQiOBfCUmT1jZtsA/BjARxv2+SiAG/yfbwJwEknmeI15ibwXZvZrM9vq/3ovgP1zvsa8xPn/AgD+O4C/A/B6nheXszj34jMA/qeZbQIAM1uX8zXmJc69MAB7+j/vBeDFHK8vV2Z2J4CNLXb5KIAfmudeADNJ7tvqmEUFgv0APFf3+/P+tsB9zGwHgFcAzMrl6vIV517U+3N40b4bRd4Lv5l7gJn9c54XVoA4/18cCuBQkneTvJfkKbldXb7i3IuLAZxJ8nkAtwE4N59Lc1K7zxQ3S0xIMJJnAlgI4D1FX0sRSPYA+BaAJQVfiiumwEsPnQivlXgnybeb2ctFXlRB/hTAD8zsMpLHA1hO8kgz2xX1RimuRfACgAPqft/f3xa4D8kp8Jp7G3K5unzFuRcg+X4ASwGcZmZv5HRteYu6FzMAHAngNyTXwMt/3tKlHcZx/r94HsAtZrbdzJ4F8H/hBYZuE+de/DmAFQBgZvcAeBO8gnRVFOuZUq+oQHA/gLeSPJDkNHidwbc07HMLgMX+zx8HcLv5PSFdJvJekDwGwFXwgkC35oGBiHthZq+Y2Wwzm2dm8+D1l5xmZomLbTkszr+Rf4LXGgDJ2fBSRc/keI15iXMv1gI4CQBIvg1eIBjP9SrdcQuAs/zRQ8cBeMXMXmr1hkJSQ2a2g+TnAfwC3oiA68xsFclLAKw0s1sAXAuvefcUvI6RM4q41qzFvBeXAtgDwE/8/vK1ZnZaYRedkZj3ohJi3otfAPgAyScA7ATwZTPrulZzzHtxHoCrSf43eB3HS7r0gyNI/gjeB4DZfp/IRQCmAoCZXQmvj+RUAE8B2Argk5HH7NJ7JSIiMWlmsYhIxSkQiIhUnAKBiEjFKRCIiFScAoGISMUpEIiIVJwCgYhIxSkQiMTg17o/2f/56yT/vuhrEkmLis6JxHMRgEtIzgVwDICum9kt1aWZxSIxkbwDXqmPE81sS9HXI5IWpYZEYiD5dgD7AtimICDdRoFAJIK/utMovJWfXu3iBWCkohQIRFog2QfgZgDnmdlqeMtkXlTsVYmkS30EIiIVpxaBiEjFKRCIiFScAoGISMUpEIiIVJwCgYhIxSkQiIhUnAKBiEjF/X//C34IVOuzyAAAAABJRU5ErkJggg==\n",
"text/plain": [
"
"
]
@@ -691,8 +693,7 @@
"metadata": {},
"source": [
"$$\n",
- "H_{\\delta}(\\boldsymbol{a})=\\left\\{\\begin{array}{cc}\\frac{1}{2} \\boldsymbol{a}^{2}& \\text{for }|\\boldsymbol{a}|\\leq \\delta\\\\ \\delta (|\\b\\\n",
- "m{a}|-\\frac{1}{2}\\delta ),&\\text{otherwise}.\\end{array}\\right.\n",
+ "H_{\\delta}(\\boldsymbol{a})=\\left\\{\\begin{array}{cc}\\frac{1}{2} \\boldsymbol{a}^{2}& \\text{for }|\\boldsymbol{a}|\\leq \\delta\\\\ \\delta (|\\boldsymbol{a}|-\\frac{1}{2}\\delta ),&\\text{otherwise}.\\end{array}\\right.\n",
"$$"
]
},
@@ -703,6 +704,8 @@
"Here $\\boldsymbol{a}=\\boldsymbol{y} - \\boldsymbol{\\tilde{y}}$.\n",
"\n",
"\n",
+ "\n",
+ "\n",
"We will discuss in more\n",
"detail these and other functions in the various lectures. We conclude this part with another example. Instead of \n",
"a linear $x$-dependence we study now a cubic polynomial and use the polynomial regression analysis tools of scikit-learn."
@@ -718,7 +721,7 @@
"outputs": [
{
"data": {
- "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD6CAYAAABamQdMAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAA2PElEQVR4nO3deXhTVfrA8e9J0g0oa0vZLAUBZS+LbCLjLjI4igoj6LiMGyo/t3FGdMYRlZlh3HAXcdwVZXFXZhQQWURENgEFEUqBYmlLC7TQNcn5/XHTkrZJm33r+3mePElubm7ODfS9J+997zlKa40QQojYYgp3A4QQQgSeBHchhIhBEtyFECIGSXAXQogYJMFdCCFikAR3IYSIQY0Gd6XUq0qpfKXUNqdl85VSmx23bKXUZsfyDKVUmdNrc4LYdiGEEG5YPFjndeA54M3qBVrr31c/Vko9ARx1Wn+31jrTm0akpKTojIwMb94ihBBN3oYNGw5prVNdvdZocNdar1RKZbh6TSmlgEnA2f40MCMjg/Xr1/uzCSGEaHKUUnvdveZvzv0MIE9r/YvTsm5KqU1KqRVKqTP83L4QQggfeJKWachk4F2n57lAuta6UCk1BPhIKdVXa11c941KqZuAmwDS09P9bIYQQghnPvfclVIW4FJgfvUyrXWF1rrQ8XgDsBvo5er9Wuu5WuuhWuuhqakuU0ZCCCF85E/P/Vxgh9Y6p3qBUioVKNJa25RS3YGeQJYvG6+qqiInJ4fy8nI/mihcSUxMpEuXLsTFxYW7KUKIIGk0uCul3gXOBFKUUjnAg1rrV4ArqJ2SARgDPKyUqgLswFStdZEvDcvJySE5OZmMjAyM87YiELTWFBYWkpOTQ7du3cLdHCFEkHhSLTPZzfJrXSx7H3jf/2ZBeXm5BPYgUErRrl07CgoKwt0UIUQQRfQVqhLYg0O+VyGCpCwPloyBBS2N+7K8sDUlooO7EEJEldUT4dC3YC0x7ldPDFtTJLg34ODBg1xxxRWcfPLJDBkyhHHjxrFz584G35ORkcGhQ4fqLZ8zZw5vvvmmi3e4ZjabyczMrLllZ2czatQoALKzs5k3b553OyOECL7Dm0FbjcfaajwPE3/r3GOW1poJEyZwzTXX8N577wHwww8/kJeXR69eLqs7GzR16lSv1k9KSmLz5s21lq1ZswY4EdynTJnidTuEEEHUJtPosWsrKIvxPEyk5+7G8uXLiYuLqxWUBw4cyBlnnMHXX3/N+PHja5ZPmzaN119/veb5o48+Sv/+/Rk2bBi7du0CYMaMGTz++OMA7Nq1i3PPPZeBAwcyePBgdu/e7VGbWrRoAcD06dNZtWoVmZmZzJ49299dFUIEyuiFkDISLMnG/eiFYWtKVPTcH/r0R376td5Frn7p06klD17U1+3r27ZtY8iQIT5tu1WrVmzdupU333yTO++8k88++6zW61deeSXTp09nwoQJlJeXY7fb622jrKyMzMxMALp168aHH35Y89qsWbN4/PHH621XCBFmSWlw3spwtwKIkuAebSZPnlxzf9ddd9V6raSkhAMHDjBhwgTAuKDIFVdpGSGE8FRUBPeGetjB0rdvXxYtWuTyNYvFUqu3XfcqWudSQyk7FEKEg+Tc3Tj77LOpqKhg7ty5Ncu2bNnCqlWr6Nq1Kz/99BMVFRUcOXKEZcuW1Xrv/Pnza+5HjhxZ67Xk5GS6dOnCRx99BEBFRQWlpaVetS05OZmSkhIf9koI0VRIcHdDKcWHH37I0qVLOfnkk+nbty/33XcfHTp04KSTTmLSpEn069ePSZMmMWjQoFrvPXz4MAMGDODpp592ecLzrbfe4plnnmHAgAGMGjWKgwcPetW2AQMGYDabGThwoJxQFUK4pLTW4W4DQ4cO1XUn69i+fTu9e/cOU4tin3y/QkQ/pdQGrfVQV69Jz10IIWKQBHchhIhBEtyFECIGSXAXQogYJMFdCCFikAR3IYSIQRLc3SgsLKwZbrdDhw507ty55nllZWVAPqOqqorp06fTs2dPBg8ezMiRI/nvf//b4HuuvfZal1fOrl+/nttvvz0g7RJCRL+oGH4gHNq1a1cztsuMGTNo0aIF99xzT83rVqsVi8W/r++BBx4gNzeXbdu2kZCQQF5eHitWrPBpW0OHDmXoUJflrkKIJkh67l649tprmTp1KsOHD+cvf/lLrWF8Afr160d2djYAb7/9NsOGDSMzM5Obb74Zm81Wa1ulpaW8/PLLPPvssyQkJACQlpbGpEmTgBPD+wIsWrSIa6+9tub50qVLGTp0KL169aoZGdJ5GOJjx45x3XXX0b9/fwYMGMD77wdkWlshRBRptOuplHoVGA/ka637OZbNAG4EqmdZvl9rvdjx2n3A9YANuF1r/YXfrbzzTgj0CImZmfDUU16/LScnhzVr1mA2m5kxY4bLdbZv3878+fP55ptviIuL49Zbb+Wdd97h6quvrlln165dpKen07JlS6/bkJ2dzbp169i9ezdnnXVWzZjx1R555JGaYYfBGA5BCNG0eJJXeB14Dqg7R9xsrfXjzguUUn2AK4C+QCdgqVKql9baRoyYOHEiZrO5wXWWLVvGhg0bOO200wBjbPb27dsHrA2TJk3CZDLRs2dPunfvzo4dO2q9vnTp0prZowDatGkTsM8WQkSHRoO71nqlUirDw+1dDLynta4A9iildgHDgG99byI+9bCDpXnz5jWP3Q39q7Xmmmuu4V//+pfb7fTo0YN9+/ZRXFzssvfuPFRwQ0MKu3ouhBD+5NynKaW2KKVeVUpVdw07A/ud1slxLItJGRkZbNy4EYCNGzeyZ88eAM455xwWLVpEfn4+AEVFRezdu7fWe5s1a8b111/PHXfcUVN9U1BQwMKFxrRcaWlpbN++HbvdXmsWJoCFCxdit9vZvXs3WVlZnHLKKbVeP++883j++edrnktaRoimx9fg/iJwMpAJ5AJPeLsBpdRNSqn1Sqn1BQUFjb8hAl122WUUFRXRt29fnnvuuZqJs/v06cPMmTM5//zzGTBgAOeddx65ubn13j9z5kxSU1Pp06cP/fr1Y/z48TW9+FmzZjF+/HhGjRpFx44da70vPT2dYcOGceGFFzJnzpx6szn97W9/4/Dhw/Tr14+BAweyfPnyIH0DQohI5dGQv460zGfVJ1TdveY4mYrW+l+O174AZmitG0zLyJC/oSffrxDRL+BD/iqlnLuSE4BtjsefAFcopRKUUt2AnsA6Xz5DCCGE7zwphXwXOBNIUUrlAA8CZyqlMgENZAM3A2itf1RKLQB+AqzAbbFUKSOEENHCk2qZyS4Wv9LA+v8A/uFPo4QQIuaV5cHqiXB4M7TJhNELISktYJuXK1SFEMJbZXmwZAwsaGncl+V5v43VE+HQt2AtMe5XTwxoEyW4CyGEtwIRmA9vBm01Hmur8TyAJLgLIYQzT3rlgQjMbTJBOTLjymI8DyAJ7g0wm81kZmbSt29fBg4cyBNPPFHrilRXsrOzmTdvXohaKIQIOE965YEIzKMXQspIsCQb96MX+tPqemTI3wYkJSXVDPubn5/PlClTKC4u5qGHHnL7nurgPmXKlBC1UggRUJ70ykcvrH8y1FtJaXDeSt/b2QjpuXuoffv2zJ07l+eeew6tNdnZ2ZxxxhkMHjyYwYMHs2bNGgCmT5/OqlWryMzMZPbs2W7XE0JEKE965dWBeVKxcR/AKpdAiZ2ee5DLigC6d++OzWYjPz+f9u3bs2TJEhITE/nll1+YPHky69evZ9asWTz++OM146yXlpa6XE8IEaEC0SuPALET3KvzZNp6Ik8WxJ88VVVVTJs2jc2bN2M2m9m5c6df6wkhIkSQ0yWhEjvBPchlRQBZWVmYzWbat2/PQw89RFpaGj/88AN2u73e4F3VZs+e7dF6QggRSLGTcw9yWVFBQQFTp05l2rRpKKU4evQoHTt2xGQy8dZbb9VMo5ecnExJSUnN+9ytJ4QQwRQ7wT0IZUVlZWU1pZDnnnsu559/Pg8++CAAt956K2+88QYDBw5kx44dNZN4DBgwALPZzMCBA5k9e7bb9YQQIpg8GvI32GTI39CT71eI6BfwIX+FEEJENgnuQggRgyI6uEdCyigWyfcqROyL2OCemJhIYWGhBKIA01pTWFgoJZlCxLiIrXPv0qULOTk5ROvk2ZEsMTGRLl26hLsZQoggitjgHhcXR7du3cLdDCGEiEoRm5YRQgjhOwnuQggRgyS4CyFEDGo0uCulXlVK5Sultjkte0wptUMptUUp9aFSqrVjeYZSqkwptdlxmxPEtgshhHDDk57768DYOsuWAP201gOAncB9Tq/t1lpnOm5TA9NMIYQQ3mg0uGutVwJFdZZ9qXX1+LqsBaSuTgghIkggcu5/BP7r9LybUmqTUmqFUuoMd29SSt2klFqvlFovtexCCBFYfgV3pdRfASvwjmNRLpCutR4E3A3MU0q1dPVerfVcrfVQrfXQ1NRUf5ohhBCiDp+Du1LqWmA8cKV2jBGgta7QWhc6Hm8AdgO9AtBOIYQQXvApuCulxgJ/AX6ntS51Wp6qlDI7HncHegJZgWioEEIIzzU6/IBS6l3gTCBFKZUDPIhRHZMALFFKAax1VMaMAR5WSlUBdmCq1rrI5YaFEEIETaPBXWs92cXiV9ys+z7wvr+NEkKIoCnLg9UT4fBmY67l0QshKS16P8cNuUJVCNG0rJ4Ih74Fa4lxv3pidH+OGxLchRBNy+HNUH2ZjrYaz6P5c9yQ4C6EaFraZIJyZKSVxXgezZ/jhgR3IUTTMnohpIwES7JxP3phdH+OGxE7WYcQQgRFUhqctzJ2PscN6bkLIUQMkuAuhBAxSIK7EELEIAnuQggRgyS4CyFEDJLgLoQQMUiCuxBCxCAJ7kKIpqssD5aMgQUtjfuyvHC3KGAkuAshmq4wD+4VTBLchRCxraHeeZgH9womCe5CiNjWUO88zIN7BZMEdyFEbGuodx7mwb2CSQYOE0LEtjaZRo9dW+v3zsM8uFcwSc9dCBHbYrh33hDpuQshYlsM984b4lHPXSn1qlIqXym1zWlZW6XUEqXUL477No7lSin1jFJql1Jqi1JqcLAaL4QQwjVP0zKvA2PrLJsOLNNa9wSWOZ4DXAj0dNxuAl70v5lCCCG84VFw11qvBIrqLL4YeMPx+A3gEqflb2rDWqC1UqpjANoqhBDCQ/6cUE3TWuc6Hh8E0hyPOwP7ndbLcSyrRSl1k1JqvVJqfUFBgR/NEEIIUVdAqmW01hrQXr5nrtZ6qNZ6aGpqaiCaIYQQwsGf4J5XnW5x3Oc7lh8ATnJar4tjmRBCiBDxJ7h/AlzjeHwN8LHT8qsdVTMjgKNO6RshhBAh4FGdu1LqXeBMIEUplQM8CMwCFiilrgf2ApMcqy8GxgG7gFLgugC3WQghRCM8Cu5a68luXjrHxboauM2fRgkhhPCPDD8ghBDh8uyz8OWXQdm0BHchhAiHuXPh9tvhjTcaX9cHEtyFECLU3n4bpk6FcePgtdeC8hES3IUQ4RPDc5i69f77cM01cNZZsGgRxMcH5WMkuAshwieG5zB1afFimDwZRoyAjz+GpKSgfZQEdyFE+ETTHKb+/spYtgwuvRT69zeCfIsWwWmngwR3IUT4RNMcpv78yvjqK7joIujZE774Alq1Cl47HSS4CyHCJ5pmSfL1V8ZXX8H48dC9u9F7T0kJVgtrkZmYhBDhE02zJDU0F6s7X399IrB/9RW0bx/kRp4gPXchROSJxCoab39lrFgBv/0tdOsW8sAO0nMXQkSi6vy2tp7Ib4e7h+/Nr4wVK4wa9q5dwxLYQXruQohQ86RXHi1VNK72ZeVKI7CnpxuBPS2t8e0EgQR3IUTwOQfBT3s1XnUSLVU0dStoXjz/RGBfvhw6dKj/nhClnCS4CyECy1Xwcg6C1uLGe+XRUkXj/AvjRyvcvwW6dDF67K4CO9T6LqoK1lG+wt2gu/6RnLsQIrBc5cudg6Azd73yaKmiqa6g2WSFp4BOzYwee8eO7t/j+C72VHTizn1/ol/zffxjbOCbJj13IURgucqXO6dZMIOlZeT3yj0xeiFs7wlPAl2bw6rvGw7sgG6dybyicYzb+QzZlZ0Y1eF4UJomPXchRGC5qgcfvfBED776eVJ4TjQG1KIv4R8/w6nJcI+GHVMhxf2+HTpWwfScf7I05yijk7fy+ICVdDg3OKNCKmPipPAaOnSoXr9+fbibIYQIhOoce6wF8rpeegluuQUGtIQ7jkGCzTiYpYx0mVJatj2Pe9/fQnG5lXvHnsp1ozIwmZRfTVBKbdBaD3X1mvTchRCBFS35cn/Mng13321UxkxeCSabsdzFCeLSSiszP9/OvO/2cWqHZN65YQSndEgOehMluAshhKe0hpkz4e9/h8svh3fegRXnuh2WYPP+I9w1fzPZhce5aUx3/nR+LxIs5pA01efgrpQ6BZjvtKg78HegNXAjUOBYfr/WerGvnyOEiALVqZiijaDMgB3aDIqtlIzWcO+98NhjcPXV8MorYLG4PJ9QabXzzLJfeOHrXXRomcg7Nwxn1MmhGTCsWkBy7kopM3AAGA5cBxzTWj/u6fsl5y5ElFsy5kTvtYYZLM0BHf2596oquPFGY77TW26B554Dk+tiw+25xdy94Ae25xZz+ZAu/P2iPrRMjAtKs0KRcz8H2K213quUfycIhBBRyGUdu824YAkiZ3wYXxw/DpMmGRNsPPQQPPAAuIhzVpuduauymL1kJ62S4nn56qGc1yd8B7NABfcrgHednk9TSl0NrAf+pLU+HKDPEUJEIufyR1cieXyYhhQWGkP2rlsHc+bAzTe7XC2r4Bh/WvgDm/YdYVz/Dsy8pD9tmwdnblRP+X0Rk1IqHvgdUH0lwovAyUAmkAs84eZ9Nyml1iul1hcUFLhaRQgRLaqHCzA3d1yg1MK4j4bxYdzZtw9Gj4ZNm4yJrF0Edrtd89o3exj3zCqyCo7zzORBPD9lcNgDOwSm534hsFFrnQdQfQ+glHoZ+MzVm7TWc4G5YOTcA9AOIUS4uCp/dFXvHi22bYOxY+HYMfjySxgzpt4qOYdL+fPCLXybVchZp6Qy67IBpLVMDENjXQtEcJ+MU0pGKdVRa53reDoB2BaAzxBCRJtorXdfvdqY7zQpyRi+d8CAWi9rrVm4PoeHP/sJrTX/vqw/k4aeRKSdb/QruCulmgPnAc6/Vx5VSmUCGsiu85oQQkSuDz+EKVOMIXu/+AIyMmq9fOBIGfd9sJWVOwsY0b0tj10+kJPaNgtPWxvhV3DXWh8H2tVZ9ge/WiSEEKHgnDZqPRA2ngP3PwzDhsGnn0Jqas2qdrtm3rp9/GvxdjTw0O/68ocRXf0ePiCY5ApVIUTTVDOuuhUe/QaWrYaJE41a9qSkmtX2Fh5n+vtb+TarkNE9UvjXpf0jtrfuTIK7ECJ2eDNo2eHNcNwKzwJbNFwcD++9V3Nxks2ueX1NNo99sYM4k8m73Lq7doRwUDUZz10IEbm8nZKu7rR3rqbwq1Z1KjyMUfJxowluG14T2HflH2PinDU88tlPjDo5hS/vHsPvT0v3/KSpu3Z40z4/Sc9dCBG5XM3q1FAFjqcTa2/YAH/aCyVmuC8efjMURi+sucr0qaW/0CzezFO/z+TizE7eV8K4a0cIJ/6W4C6ECC5/UhHeBkNXE4XU9fHHRkVMaios+wH69gWMMWH+smgNWw8c5cJ+HXj44n6kJid41k5P2+FJ+wJE0jJCiODyJxXhPD2fJ8GwoYm1tYZ//hMmTDAC+tq10Lcv5VU2Hv3fDi56djW5R8t44crBvHjVEN8De0PtCOHE3zITkxAiuBa0NAJ7NUsyTCr27L2BOgF5/Dj88Y+wYAFMngz/+Q80a8Y3uw5x/4db2VtYyuVDuvDXcb1pEwFDB3hKZmISQgSWN0HXk1SEu+0F4irX7Gy45BLYsgUefRTuuYei0ipmLtjMBxsPkNGuGfNuGM6oHqEdbz3YJLgLIbznzYlOV5Nj+7M9b3z9tVG7XlUFn3+OHjuWDzYeYObnP1FSbmXaWT2YdnYPEuNCMztSKElwF0K4565H7c2JTk9634GuItEaXngB7rgDevaEjz8mu21n/vrKd3yzq5DB6a3516UDQjKXabjICVUhhHvuToZ6e6KzMc7bwwwoz2vb66qoMGZNmjYNLryQqm/W8PwBExc8tZIt+4/yyCX9WDR1VEwHdpDgLoRoiLsedaCrPpy3Z2kOtlLfqmv274czzzTmN/3LnWz4QzPGP/Mej33xM2f3bMXSP/0m4seECRRJywgh3HN3MjTQw/k6b29BS99SNF98AVdeCZWVlLw1j0cKs1mw8Vw6xhXwcsY/OK+TBVpG4RDEPpKeuxDCvRDWZdfwNuVjs8GMGXDhhehOnfjklY8ZvbsdH+T24ebU91na6xbOa/ltdE7z5wfpuQsh3AvHhBueVNdUKyiAq66CL7/k8OVXcNPI6/l+QynDu7XlkfZz6FX+aUiuBo1EEtyFEP4J9EiHnh5Qvv0WJk1CFxTw8S1/566Wp9GulBPjwZS/CKsPRec0fwEgwV0I4Z9g1ai7ozU8+U/0vQ9wvHUiN181g29b9eeakRncfX4vWibGGetF6zR/ASLBXQjhnxCOdEhhIdxwA3z0EetP6cP1Y/9Oj7Y5fNrvNfr+blHwPjcKSXAXIlb5mi7x9n2hGunw66+xX3kVOj+PWWf9kfdHnM3fOr3C5W2WYrK2CM5nRjGplhEiVq2eCAVrjHrxglXwaS/PLgjydhTHYFfUVFVhv+9+9Nlns69CcclVj1M2IYOvek9jUtslmEzmJney1BN+99yVUtlACWADrFrroUqptsB8IAPIBiZprQ/7+1lCCC8c3ozxZ+lgLfYsH+4uzRLMwb3cycqi5NJJJP+wgfn9z2PxDffy6MTT6N26DFYvabInSz0RqJ77WVrrTKehJ6cDy7TWPYFljudCiFBy1Zv1JB/urs48hFPEARTOfY2yvgPQO3bw9ykP0GreG7z+f2fTu2PLEweUScXGfZDmIY1mwcq5Xwyc6Xj8BvA1cG+QPksI4ay6h120EWOcFkfv3dN8uLs682CfOHW0u3Tfj+x5uTN9v9vKxi592Prv57h/4piYHLkxmAIR3DXwpVJKAy9precCaVrrXMfrBwE5rAoRKs6liZjB0hLQJwJ1YydM3aVZAnHitIHP1qsmsuazEk5+A04t+ZEvzj+HgfM+YnA7OVnqi0AE99Fa6wNKqfbAEqXUDucXtdbaEfhrUUrdBNwEkJ6eHoBmCCGA2j1sbICuPfPRkjG+1aV7c+WoO25q4jdtzyH3ATvj1m0mp10qO+89iQsGrQMJ7D7zO7hrrQ847vOVUh8Cw4A8pVRHrXWuUqojkO/ifXOBuWBMs+dvO4QQDo31sH1NrwTixGmdz87OzWXRg69w+TMPMOhILjvPSqfHVfswJVmgzUj/PquJ8+uEqlKquVIqufoxcD6wDfgEuMax2jXAx/58jhDCC42VJgZ6LHZvOD77sDWZmXuvZ8kL/bn74Rtpk2im/NOF9LqvK6YWIRykLIb5NUG2Uqo78KHjqQWYp7X+h1KqHbAASAf2YpRCFrnbjkyQLUQIBXosGC+Ul+Tyxvwn+GpNOx7+dA6nHNpH2TV/JOnZpyDZzeQZYWxvpAvaBNla6yxgoIvlhcA5/mxbCBEkYRhzxW7XfLrlV575ZDO//yyLd9fPxt6+PSxeTNKFFzb85lCPXdOYKDnYyPADQoig+nZ3If9cvJ1Wq7/m7WUv0LEwF266CdO//w2tWze+gVCOXeOJSDvYuCHBXQgRFNtzi3n8i5/ZsGkX/1z1GuM2LUH36gUfvAdjxni+oVCNXeMpb6/gDRMZW0YIUV9ZnlEy6cMk1XsLj3PHe5sY9/RKUj59nzVvTuPCrcvhb39D/fCDd4EdQjMblDf7GyFX8DZGeu5CiPp8SD3kFZfzzLJfmP/9froW57F03euc/P1KGDYM/vMf6N/ft7aE4hyBN/sbrit4vSTBXQhRnxeB6khpJS+u2M0ba7Ixl5czN2cpZ330KspshqefhttuA3OEDx3gTWAO5hW8ASTBXQhRnweB6niFlde+2cNLK7M4VmHlPtsurlv4FHF7s+GKK+Dxx6Fz51C33BCOMekDcQVvAElwFyLaBeNEXgOBqqzSxttr9/LSyt0cOlbJFe2q+Nual2ix5H/Quzd89RWcdZZ/n+8vb9NKgQjMETatnwR3IaKdP6V5XozRXl5lBPU5K7I4dKyCs9Jb8I9DS+n0wNMQF2f01G+/3Xgcbt7mvyMsMAeCBHchop0/J/I8ODCUV9mY990+Xlyxm4KSCkZ1b8t7rfbS47GZkJ0NkyfDY4+FLwXjSoTlv8NBgrsQ0c6fQNbAgaG8ysb87/fz/PJd5JdUMKJ7W147xUq/J+6GtWth4MDISMG4EmH573CQ4C5EOAUiX+5PIHNxYCivsrFw/X6eX76bg8XlDMtoy4unt2HInMdg/nzo2BFeeQWuuSZyq2BiMM3iLb8GDgsUGThMNFnOY6sri3GRTiiDktPB5XjyabwT/yQvry2goKSC0zLacM/wDgx7dw7q6aeNQP7nPxu3FjLOeiQI2sBhQggvOE9/p8yAHWxloB3T4IXjwpekNI6evozX12Tz2po9HCnN4fQe7XhmQm9GLH0fdf4jcOiQ0UufORO6dAlt+8ItwoYU8IYEdyFCpdb0dy6E+MRfQUkFr6zew9tr93Kswsq5vdtz6xkZDF7xGVz4e9i3D84806iCGTIkZO2KKFEySJgrEtyFCJVa0985M4GlechO/B04UsbcFbt57/v9VNrs/LZ/R277TXd6r1kC4/4AP/8MQ4caQwacey4oFfQ2RawIG1LAGxLchQgV55OX1UKYZ99xsJiXV+7hkx8OoDVMGNSZW37Tne4bVsNFN8CmTdCnD3zwAVxySdMO6tWiuKRSgrsQoVJd1eKcc28zqH5vPYB5Xq01a3YXMndlFit2FpAUZ+bK4V25YXQGXTZ+C5dNg9WroVs3ePNNmDIlcitgwiGKSyoluAsRKp6W5wUgz2u12fl8ay4vr8pi24FiUlokcM/5vbhqeDqtv14Kv73BqFXv1AleeAGuvx7i433csRgWxSWVEtyFiDR+5HmPV1h57/v9vLp6DweOlNE9tTmzLu3PJQM7krj4MxhzBWzcCF27wosvwnXXQUKC522LxOqRSGxTBJDgLkSk8SHPm3O4lLfW7uXd7/ZRXG5lWEZbHvpdX87u2Q7TB+/D9TNh2zbo0QNefRWuusq3MWAisXokEtsUAXwO7kqpk4A3gTRAA3O11k8rpWYANwIFjlXv11ov9rehQkS0QPYePczzaq1Zt6eI177J5sufDqKU4oK+adx4RncGpSYaOfQJT8LOncZojW+/Db//PVj86NNFYvVIJLYpAvjTc7cCf9Jab1RKJQMblFJLHK/N1lo/7n/zhIgSgew9NpLnLa+y8cnmX3ltTTbbc4tp3SyOm39zMleN6ErnqmPwwjPw3HPGxUdDh8LChXDppWAKwKya4awecXcAjeKKlmDyObhrrXOBXMfjEqXUdiCChoUTIoRC0HvMPVrG22v38u66/RQdr+TUDslGPn1QZxL37Ib77obXX4fycrjoIrjnHjjjjMCWNIazesTdATSKK1qCKSA5d6VUBjAI+A44HZimlLoaWI/Ruz8ciM8RImIFqfdot2u+zSrkne/28sWPeWitObd3Gted3o0R3dqgVq6ESXfAxx8b1S5XXw133w2nnhqQz68nnNUj7g6gUVzREkx+B3elVAvgfeBOrXWxUupF4BGMPPwjwBPAH1287ybgJoD09HR/myFEaLhLDQS493j4eCWLNuQwb90+9hw6TutmcVw/uht/GNGVk+Js8NZbRgnjjz9C27bwt78Zc5WmxXCViKRfvOLXqJBKqTjgM+ALrfWTLl7PAD7TWvdraDsyKqSIGt6M4ujlSVatNRv2Huad7/bx+dZcKq12hnZtw5Uj0rmwX0cSd+00Avobb0BJiTHey223GfOVJiUFZXcjipQ81tPQqJA+B3ellALeAIq01nc6Le/oyMejlLoLGK61vqKhbUlwF1FjQUuwltRelnqG60Dj4YGguLyKjzYd4J21+/g5r4TkBAsTBndmyvB0Tm2XBJ9+Cs8/b0yMER9vVLzcdhv07wrfTJJgVy1YwT+CDyrBCu6jgVXAVsDuWHw/MBnIxEjLZAM3Vwd7dyS4i7Dx9g/XOWBXcxe46x4ILMkwqRgwculrswpZuCGH/27LpbzKTv/OrbhqRDoXDexEsz27jQkx3nwT8vIgPR2mTjWuJG3fvn5bwjEWfKQJ1vcRwd9zUMZz11qvBlydhpeadhE9vC1hrM6tF6w6scxddYyLHHHO4VIWbchh0YYccg6XkZxo4bLBXfj9aScxoE0cLFgAd70C33xjjPEyfrwR0C+8sH59utR31xas7yNKv2e5QlU0bd7+4VZXZtTtzbk6uec4EJQX/sgXVZezIOtK1ny1HK3h9B7t+PMFp3BBnzQSN3wPD/8F3nsXSo5BRwXXnwTnpID5K0gsgqphYEmr/UsDZXy2nGA0BOuEa5SeyJXgLpo2T/5wXaVuGqmOsds16w/G8dGx5/l0x6+UlFvp0sbKHef05LLBXTgpfx+8MwcumwdZWdCsGYxOhhFl0MsGaj/o/calggVr4NNeGJlOBbZSxwHJbIwDj268QieC88YBE6x69yito5c5VEXT4SrAQeNBz4uc688HS/jo+518suFnDpS3ItFUxdi+aUwa3pMRSZWYFsw3hgHYuNG4YvTss+HKK40rSP/Xpf7J2sY45fEbFIl546ZwwAkymUNVCHCfX28syDWSuvn1SBmf/PArH206wI6DJZiVndEtdnFP6nLOV9/TfEMneL2rUe1itxsljE8+aZQwdux4YkOuJvNoiDcpgkjMG/syZIOvB4QmeCCR4C6aDl8DnIvUTdHxSv637SAfbz7AuuwitIZB6a156Hd9Gb9xJO02HITvgZ8A/QucbIe//tXopZ9yiuvPcf7536qPsezoT/icimlkH8LOl38PX8fw8eR9zgcA5+8/Sg8GEtxF9PKmN1aWR+3iLrPxfEHLxt/rCLqH8rP4omICi7Mmsnb5Umx2TffU5tx1bi8uTbXTZfn/4N7p8M1BIz3eCbhYwQWZcPOGxsd4cXUZfVkerLwYCr83Zm9qOxTGfOx9oInEvLEvBxxfD9CevM/5AFD43YnlgRhGOAy/HCS4i+jlTS9u9USwHq+9rLo37HzCss4fXn5xOf/7sYzFe/7Nuj1F2DV0S7Ez9YwMLrUdpPvapah7PofNm41t9u8Pf/0zdF0GLX85sT1fB+9aPRGKNmBcSmIBU7zRNm+DRSSOv+LLAcfXXyCevM/dBOaBSGOFYcx5Ce4iennTizu8GbA5LbAZvevqx1bHSclD35Kz5HqWtHuW/249yPd7jZRLj/YtuGtERyYUbqfzqkWoxxZDfr5xUnTUKJg1CyZMgF69QrOPgQ4WvvQs/e2N+nLA8fUXiCfvc3fOIxBprDCc85DgLqJXY72xhmrCzc1qeu52rdhS1pOlxcNYWjycHeXdgJ/o3b45M7taueDAFlL+uxJWroTKSmjVyrioaPx4GDsW2rVrvK11A+HQF2D9rY0HRnf7GOhg4cvBIhwzIPn6C8ST97k75xGINFYYznlIKaSIPtWBsmijkYfGDm0G1Q+QtYYKqH0ismzgc6xe+hTLDrRh2dFBFFjbYMLGebZvmXxgCUMOHCR5ezkUFBrb6tsXLrjAGCf99NO9n6Kubimi08GlwdJEd73juttrO8RI2fjai25gqAS3bWjoPaK2IOXcpRRSxBbnHmNDgbFWDtXG3opUVvZcxtc78lm94gAV1svoVnmUqZW7OevnWXT9aTvmXx3DJLUEBsTBP56GcZdB5wbmofHkD7duT9vqFAQb6nm763HWTTPYK/3rRTfUs3TXQ4/ECpxIFYZzHhLcRfh526tpLCXh2F5JhZ1vjw1n1bFBrCwZwt7KjnRe8xVji3ZyW9FOeu/aTFJ2lvGe5s3hZAVnAv2AkwBTFaQugs63N9x+T9ITdQNh3Z67t4GxbrBY0NK/NE1DOWl333ckVuA0pgnVu0twF+HnTe62bkmjU2C02zXbfj3KysVPsjJvPFuK7+DkghyGHdzGrIKX6H/gAC3yHAOUtmljTEE37VYYMwYGDYLlZ9ceEAw8C5Ke5L/rBkJXOXd/+NuLbqhn6W7bkViB05hwnCcIEwnuIvzqBseCVfCuxXVNt1NJo02b2F7Zm7XmZ/nuje85sGk73ff8SOavPzMjbws9c/cRZ3VUyKSlGUF8zBgY1hcO/x2OLoc2h6HfVcaIi6MXGiWR1SkTT4OkJ4HVVSAMZFAJZi86Gnvo7kTilbpBIidUReA19tO37uv2SqOW21WNceoZNUHQZtdsf3Mga49045cD6ZRlJ9It91f6H/yFIQd/oc3xIwDoeBMqww4nAz1McNoQuOq7E7XmDY2z4unJWm/2N1JFa7v9EYlj7PghKJN1BJIE9yjmKkC4O+FZvW7BN9TM7+Jc6VEnJXLMlsSWY/3ZZ7qf0nUbiPtxKz1/3UXv/D20qjB671qBtecpxI0cAcOHG7ce7eG7Ke6DlidVHjEWBFxqCvtYV4wd0KRaRgSPqxxmYxfe1Ezc5Xj96E/oi/LIeWkcOT8conR/InG5NtLyizitaAOj7JcAUBln4VjnZlhOV9A9AfqfipryAXEp3eu3q6Eg5e3VitpqHJDK8rwPBJEcTKI9ReHLdxuN5wl8JD134X7AJHcXcjj/QR3eBNZjJ7ZlSa4fPKt7hPOT4dAx9EEo+rUlR/e3wH7ARIu8ClKLCjFrI+jbURS0bcPxzm2wDDuXlNQ1NGu3FToAJsfnOKVr/Nrfhob5rXty1ZfP9KR3HK4DQKBr5UOtKf7yqEPSMqJhruYFdUWZwdzccULT5mIFM+hmcMgK+QrybNiPpFJa0htb1j6S9mURX1lVs3alyUJ2204c7NyNyl6n0mxgPzqPHMxJwzMxNW/mlMZZVf+j6qZSAh0gy/Lgo86gnfbT3BzaDm74M+q2w9XBL1JSQA2d+4iGYBktF1EF8eAtaRnhXlkeHFpTO4i5Y7NBUTEcAoqAwhM3XQi2Qo3laO3JJiot+fzaysTeNh3IyfwtlWlJxLcvI6VHIl0zSunFanq1s8Poh+qfdHWuXKmrbirFXYmbr39YSWmQMqp20FXmxsvo6rbD3KzxqfDClR4JdK18qEXLRVRhKr+M6uCee7SMmZ9vp3tKc7o5bt1TWtCqmZeXhkc6TwOUc6VH9RjgSp0oKYT6r2s7lAHFwBHgqNPtSJ3lR6jXYa+IjyOvZVv2JnfgQNc0cpNT+LVlKsdTEonrqEnpkkgfy3r6tTdxVrODWEp+rN9LLFhl9JJTRp3Yt9UTXQf26mFv65bjuSqnXDKm9ud4+4dVtwTw8KbGg1/ddmib0QP2dMCqYAapxv4fRUuwrBYtJZphOngHLS2jlBoLPI0xcPZ/tNaz3K3ra1pma85R/u/djew/XIbNfmI/2jaPrwn2Ge2a0blNEp1bG/cdWiZiNvk4/KovGhowqjqnfWTbibK7Vn2NZc4578LvOXES0gzmpPqB22aDRadCSYkRrI853Y4DxxSUWqDECsf0iddKgMr6zdYKypPjKWnRnMLmrTiY1I6sFp3Z06ILB1qm1gTx5BbH6JmUQ8+E/fRM3E/PxGx6JOwn2VJl9FpdpXCUBdD1fy04pwLq/uQG9zlvd718V5/jz093T9InvqRYvD14+/rzvrG2RfLJ32gWxLRbyHPuSikzsBM4D8jBmJNmstb6J1fr+5xzd0xkUFmwif2VaeyxZrCnLJWsik7sqezCnvKO5Flrj9hnwUqHuEI6x+fTOaGIjpY8Ui2HaR9/jFRzPu0th0lNKKMZRx1vaGb0bjEZARXAnAi2cg8f242hZW2OW6XjVuX0uKFl1bcyoNxx7/y4+r6i4a9Km0G3UFibmSlvFs/xpCSONGtBUWJrfm2Wwv6kNHYndiErsTP5zdtxOCkZu8mMCTudE4/S1bKX9Phcusbn0jUhl67xeaQnHaG5xWrsI6bagdzS0ljunG+uxQTKVD/PXx18l4wxxll33t5FO10Hm7rruvscf/+wPBnZEYIXIP09ORstOepYE6ace7CC+0hghtb6Asfz+wC01v9ytb7Pwf3NYfD69yfG5dZON8dzq91EmS2BUnsiZbYEym0JlNlP3FfZLaCNC9qVNt5s0po4rMSrKuKoIh4rcVhJ0FXE26uIs1cRZ7dhsVsx2+2YbHbMdjvKpjHZNMquwQrKhhH3PEhnN0RbgCSwJyrsiSZsSSaqEixUJMZRnpBAeXwCx+OTOJaQRHF8c47EJZOf2Jbc+BT2x7dnX1xHDphTsav6WbhW5hI6xBUat/hiOljy6GjJp2N8EV07dKTz+I+IM5saDyyuAkebTNcnQ2vVttepea9bE+/JH4SrXn7dzwlXsA0kf+vzpbok5oTjhGpnYL/T8xxgeJ1G3QTcBJCenu7bpxz8ETZVb9Dp5vTcouwkqzKSKTtRRue0jlZgw4QVM1YsVGGmCovxWFuwKrNxj5njpiQqVDxWk5mqeAs2k5kqkxmryYLVZDLuzWaqTBZsJhN2swm7WaFNCrvjeVWcGWuchao4MxWWeMotCZTHxVNujjeex8VTZknkuDmRYnMLis3NKVeJHn8lFqy0NB+nreUobS0ltDUf4QzLZtpZjhrLzEdJjTtCh4QyOphzaGaurJ+TrwmEL4PZ8aU1lt90la+tezk/GOmnlJEngqyrIA7e1SPXmmTBxRyjwUothDqX6kt9vnOboiVHLQIiWD33y4GxWusbHM//AAzXWk9ztb7PPXdXtchBZtMmSmzNOGJLptjW3PhFYE+k1G78Oii1J1FqT6DMnkiltmDVFqzabBw8tIUqbXYsM2FCY1IaE3YUCqXsjscas7KRaKokQVWSoKpINFWRYKoiQZWToCppYS4j2VRKC3MpyebjJJtKSTaXkmCyoVJH1Q6edcdLCUaPzV1POxR53EipEw92T9jT+nzpnTcZsZuWcZ48GLzMhXv42N+cu1LQeiCY4tyfOHWVr617AZG7/G7N5MnK/eTJcqIsOCLxe43ENomgCUdwt2CcUD0HOIBxQnWK1vpHV+vLRUxCCOG9kOfctdZWpdQ04AuMUshX3QV2IYQQgRe0i5i01ouBxcHavhBCCPdMja8ihBAi2khwF0KIGCTBXQghYpAEdyGEiEES3IUQIgZFxGQdSqkCYG8jq6VgjCTeVDXl/Zd9b7qa8v57su9dtdaprl6IiODuCaXUenfF+k1BU95/2femue/QtPff332XtIwQQsQgCe5CCBGDoim4zw13A8KsKe+/7HvT1ZT33699j5qcuxBCCM9FU89dCCGEhyIuuCulxiqlflZK7VJKTXfxeoJSar7j9e+UUhlhaGZQeLDvdyulflJKbVFKLVNKdQ1HO4Olsf13Wu8ypZRWSsVMFYUn+66UmuT49/9RKTUv1G0MJg/+76crpZYrpTY5/v+PC0c7A00p9apSKl8ptc3N60op9Yzje9milBrs8ca11hFzwxgeeDfQHYgHfgD61FnnVmCO4/EVwPxwtzuE+34W0Mzx+JZY2XdP99+xXjKwElgLDA13u0P4b98TY1LJNo7n7cPd7hDv/1zgFsfjPkB2uNsdoH0fAwwGtrl5fRzwX4zJQUcA33m67UjruQ8Ddmmts7TWlcB7wMV11rkYeMPxeBFwjlJKEf0a3Xet9XKttWM6KNYCXULcxmDy5N8e4BHg30B5KBsXZJ7s+43A81rrwwBa6/wQtzGYPNl/DbR0PG4F/BrC9gWN1nolUNTAKhcDb2rDWqC1UqqjJ9uOtODuamLtzu7W0VpbgaNAu5C0Lrg82Xdn12Mc0WNFo/vv+El6ktb681A2LAQ8+bfvBfRSSn2jlFqrlBobstYFnyf7PwO4SimVgzFPxP+Fpmlh521cqBG0yTpE8CilrgKGAr8Jd1tCRSllAp4Erg1zU8LFgpGaORPjF9tKpVR/rfWRcDYqhCYDr2utn3DM0fyWUqqf1toe7oZFqkjruR8ATnJ63sWxzOU6jrlaWwGFIWldcHmy7yilzgX+CvxOa10RoraFQmP7nwz0A75WSmVj5B8/iZGTqp782+cAn2itq7TWezDmKO4ZovYFmyf7fz2wAEBr/S2QiDH2SqzzKC64EmnB/Xugp1Kqm1IqHuOE6Sd11vkEuMbx+HLgK+048xDlGt13pdQg4CWMwB5LOVdoZP+11ke11ila6wytdQbGOYffaa1jYWZ1T/7ff4TRa0cplYKRpskKYRuDyZP93wecA6CU6o0R3AtC2srw+AS42lE1MwI4qrXO9eid4T5b7Obs8E6Ms+d/dSx7GOMPGYx/1IXALmAd0D3cbQ7hvi8F8oDNjtsn4W5zKPe/zrpfEyPVMh7+2yuMtNRPwFbginC3OcT73wf4BqOSZjNwfrjbHKD9fhfIBaowfp1dD0wFpjr9uz/v+F62evN/Xq5QFUKIGBRpaRkhhBABIMFdCCFikAR3IYSIQRLchRAiBklwF0KIGCTBXQghYpAEdyGEiEES3IUQIgb9P3RTPrJ4cV2EAAAAAElFTkSuQmCC\n",
+ "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAA0L0lEQVR4nO3deXxU5dXA8d+ZmawQEnaRRRaBCggRIgKiRQFXlPK2INhWUFpESltrF9Fqi1Vffa1LtVoVW/e14FKroiKibKIGZAn7FiAQQgghgSRkm+f9407CJJlJJpk1k/P9fOYzM/feufe5k8yZZ859FjHGoJRSKrrYwl0ApZRSgafBXSmlopAGd6WUikIa3JVSKgppcFdKqSjkCHcBADp06GB69uwZ7mIopVSzsnbt2qPGmI6e1kVEcO/Zsyfp6enhLoZSSjUrIrLP2zpNyyilVBTS4K6UUlFIg7tSSkWhiMi5e1JeXk5WVhanTp0Kd1GiTnx8PN26dSMmJibcRVFKBUnEBvesrCySkpLo2bMnIhLu4kQNYwx5eXlkZWXRq1evcBdHKRUkEZuWOXXqFO3bt9fAHmAiQvv27fUXkVJRLmKDO6CBPUj0fVUqApTkwJKL4d9trPuSnIDuPqKDu1JKRa2Vk+HoV1BxwrpfOTmgu9fgXo/Dhw8zdepU+vTpw7Bhw7jqqqvYsWNHva/p2bMnR48erbP8mWee4eWXX/b52Ha7ndTU1OpbZmYmo0aNAiAzM5PXX3+9cSejlIos+evBVFiPTYX1PIAi9oJquBljmDRpEtOnT+fNN98EYMOGDeTk5NCvX79G72/27NmN2j4hIYH169fXWLZ69WrgdHC//vrrG10OpVSEaJtq1dhNBYjDeh5AWnP3YtmyZcTExNQIykOGDOGiiy7iiy++YMKECdXL586dy4svvlj9/KGHHuLcc89l+PDh7Nq1C4D58+fz8MMPA7Br1y7GjRvHkCFDGDp0KLt37/apTK1btwZg3rx5rFixgtTUVB577DF/T1UpFQ6jF0KHkeBIsu5HLwzo7ptFzf2e/25my6HCgO5zwJlt+PM1A72uz8jIYNiwYU3ad3JyMps2beLll1/m1ltv5YMPPqix/sc//jHz5s1j0qRJnDp1CqfTWWcfJSUlpKamAtCrVy/efffd6nUPPvggDz/8cJ39KqWakYTOMH550HbfLIJ7czNt2rTq+9/85jc11p04cYKDBw8yadIkwOpQ5ImntIxSSvmqWQT3+mrYwTJw4EAWLVrkcZ3D4ahR267dZty9qaE2O1RKhUODOXcReV5EjohIhtuyt0RkveuWKSLrXct7ikiJ27pnglj2oLr00kspLS1lwYIF1cs2btzIihUrOOuss9iyZQulpaUcP36cpUuX1njtW2+9VX0/cuTIGuuSkpLo1q0b7733HgClpaUUFxc3qmxJSUmcOHGiCWellGopfLmg+iJwhfsCY8x1xphUY0wq8Dbwjtvq3VXrjDGNayISQUSEd999l88++4w+ffowcOBA7rjjDs444wy6d+/OlClTGDRoEFOmTOG8886r8dr8/HwGDx7M448/7vGC5yuvvMITTzzB4MGDGTVqFIcPH25U2QYPHozdbmfIkCF6QVUp5ZEYYxreSKQn8IExZlCt5QLsBy41xuz0tl1D0tLSTO3JOrZu3co555zTmN2oRtD3V6nmT0TWGmPSPK3ztynkRUCOMWan27JeIvKdiHwpIhfVU6hZIpIuIum5ubl+FkMppSJYkIca8MTf4D4NeMPteTbQwxhzHnAb8LqItPH0QmPMAmNMmjEmrWNHj1MAKqVUdAjyUAOeNDm4i4gD+B/graplxphSY0ye6/FaYDfQ+O6cSikVTYI81IAn/tTcxwHbjDFZVQtEpKOI2F2PewN9gT3+FVEppZq5tqnWEAMQlKEGPPGlKeQbwFdAfxHJEpGZrlVTqZmSAbgY2OhqGrkImG2MORbA8iqlVPMT5KEGPGmwE5MxZpqX5TM8LHsbq2mkUkqpKkEeasATHTjMi7y8vOrhds844wy6du1a/bysrCwgxygvL2fevHn07duXoUOHMnLkSBYvXlzva2bMmOGx52x6ejq/+tWvAlIupVTz1yyGHwiH9u3bV4/tMn/+fFq3bs3vfve76vUVFRU4HP69fXfffTfZ2dlkZGQQFxdHTk4OX375ZZP2lZaWRlqax+auSqkWSGvujTBjxgxmz57NBRdcwB/+8Icaw/gCDBo0iMzMTABeffVVhg8fTmpqKjfffDOVlZU19lVcXMxzzz3H3//+d+Li4gDo3LkzU6ZMAU4P7wuwaNEiZsyYUf38s88+Iy0tjX79+lWPDOk+DPHJkye58cYbOffccxk8eDBvv62ZMqVamuZRc7/1Vgj0CImpqfC3vzX6ZVlZWaxevRq73c78+fM9brN161beeustVq1aRUxMDHPmzOG1117jhhtuqN5m165d9OjRgzZtPHYDqFdmZibffPMNu3fv5pJLLqkeM77KvffeWz3sMFjDISilWpbmEdwjyOTJk7Hb7fVus3TpUtauXcv5558PWGOzd+rUKWBlmDJlCjabjb59+9K7d2+2bdtWY/1nn31WPXsUQNu2bQN2bKVU89A8gnsTatjB0qpVq+rH3ob+NcYwffp0HnjgAa/7Ofvss9m/fz+FhYUea+/uQwXXN6Swp+dKKaU5dz/07NmTdevWAbBu3Tr27t0LwNixY1m0aBFHjhwB4NixY+zbt6/GaxMTE5k5cya//vWvq1vf5ObmsnCh1f61c+fObN26FafTWWMWJoCFCxfidDrZvXs3e/bsoX///jXWjx8/nqeeeqr6uaZllGp5NLj74Yc//CHHjh1j4MCBPPnkk9UTZw8YMID77ruPyy67jMGDBzN+/Hiys7PrvP6+++6jY8eODBgwgEGDBjFhwoTqWvyDDz7IhAkTGDVqFF26dKnxuh49ejB8+HCuvPJKnnnmmTqzOd11113k5+czaNAghgwZwrJly4L0DiilIpVPQ/4Gmw75G3r6/irV/AVzyF+llFIRSIO7UkpFIQ3uSikVhTS4K6VUFNLgrpRSUUiDu1JKRSEN7vWw2+2kpqYycOBAhgwZwiOPPFKjR6onmZmZvP766yEqoVJKeabBvR4JCQmsX7+ezZs3s2TJEhYvXsw999xT72s0uCulIoEGdx916tSJBQsW8OSTT2KMITMzk4suuoihQ4cydOhQVq9eDcC8efNYsWIFqampPPbYY163U0qpoDLG1HsDngeOABluy+YDB4H1rttVbuvuAHYB24HLG9q/MYZhw4aZ2rZs2VJnWb2KDxvz6UXGvJVk3RcfbtzrPWjVqlWdZcnJyebw4cOmqKjIlJSUGGOM2bFjh6k6h2XLlpmrr766entv24Vbo99fpVTEAdKNl7jqy6iQLwJPAi/XWv6YMeZh9wUiMgBr4uyBwJnAZyLSzxhTSbCtnAxHvwJTYd2vnBzUOQvLy8uZO3cu69evx263s2PHDr+2U0qpQPJlguzlItLTx/1NBN40xpQCe0VkFzAc+KrpRfRR/norsIN1n78+4IfYs2cPdrudTp06cc8999C5c2c2bNiA0+msM3hXlccee8yn7ZRSKpD8ybnPFZGNIvK8iFTNBtEVOOC2TZZrWR0iMktE0kUkPTc3149iuLRNBXF9V4nDeh5Aubm5zJ49m7lz5yIiFBQU0KVLF2w2G6+88kr1NHpJSUmcOHGi+nXetlNKqWBqanB/GugDpALZwCON3YExZoExJs0Yk9axY8cmFsPN6IXQYSQ4kqz70Qv93mVJSUl1U8hx48Zx2WWX8ec//xmAOXPm8NJLLzFkyBC2bdtWPYnH4MGDsdvtDBkyhMcee8zrdkopFUw+DfnrSst8YIwZVN86EbkDwBjzgGvdJ8B8Y0y9aRkd8jf09P1VqvkL+JC/IuI+e8QkIMP1+H1gqojEiUgvoC/wTVOOoZRSqukavKAqIm8AY4AOIpIF/BkYIyKpgAEygZsBjDGbReTfwBagAvhFSFrKKKWUqsGX1jLTPCz+Vz3b3w/c70+h3Palkz8HgS+pOKWUS0mO1bQ6f73VUGP0QkjoHO5SNShie6jGx8eTl5engSjAjDHk5eVpk0ylfFXVh6bixOk+NM2AL52YwqJbt25kZWURkGaSqob4+Hi6desW7mIo1Tw0pg9NBNXyIza4x8TE0KtXr3AXQynV0rVNPd37vaE+NCHuKV+fiE3LKKVURGhMH5oQ9JT3VcTW3JVSKiIkdPa99t2YWn6Qac1dKaUCJQg95ZtKa+5KKRUojanlB5nW3JVSKgppcFdKqSikwV0ppaKQBnellIpCGtyVUioKaXBXSqkopMFdKaWikAZ3pZSKQhrclVIqCmlwV0qpKKTBXSmlolCDwV1EnheRIyKS4bbsryKyTUQ2isi7IpLiWt5TREpEZL3r9kwQy66UUsoLX2ruLwJX1Fq2BBhkjBkM7ADucFu32xiT6rrNDkwxlVJKNUaDwd0Ysxw4VmvZp8ZUjUjPGkDnbFNKqQgSiJz7TcBit+e9ROQ7EflSRC7y9iIRmSUi6SKSrvOkKqVUYPkV3EXkj0AF8JprUTbQwxhzHnAb8LqItPH0WmPMAmNMmjEmrWPHjv4UQymlVC1NDu4iMgOYAPzYGGMAjDGlxpg81+O1wG6gXwDKqZRSqhGaFNxF5ArgD8C1xphit+UdRcTuetwb6AvsCURBlVJK+a7BafZE5A1gDNBBRLKAP2O1jokDlogIwBpXy5iLgb+ISDngBGYbY4553LFSSqmgaTC4G2OmeVj8Ly/bvg287W+hlFJK+Ud7qCqlVEkOLLkY/t3Gui/JCXeJ/KbBXSmlVk6Go19BxQnrfuXkcJfIbxrclVIqfz1U9cs0FdbzZk6Du1Iq+jWUdmmbCuK6BCkO63kz1+AFVaWUanZKcqzUSv56K1A7y+DYWqtWXpV2Gb/89PajF9bcfvTC8JQ7gDS4K6WiT1UOvSqYY8BUWus8pV0SOtcM9lFA0zJKqehTO4duTNSlXRqiNXelVPSoSsdUFJ1eJg5oNwxssVGVdmmIBnelVPSoSsfgtJ6LHTqMtIJ5QuewFi3UNC2jlIoe7ukYAHuilUsPdGBvBp2eNLgrpaJHqJo0NoNOTxrclVLRY/RCKw3jSDqdjgmGZtDpSXPuSqno0VCTxtrt35uai2+berqpZYS2vtGau1Kq5QhUOiVUvxD8oDV3pVTLEah0SjPo9KQ1d6VUyxGFY8h4o8FdKdVyNIN0SqBoWkYp1XK4p1MCdXE1QvlUcxeR50XkiIhkuC1rJyJLRGSn676ta7mIyBMisktENorI0GAVXimlmqwZtFX3h69pmReBK2otmwcsNcb0BZa6ngNcCfR13WYBT/tfTKWUCrBm0FbdHz4Fd2PMcuBYrcUTgZdcj18CfuC2/GVjWQOkiEiXAJRVKaUCJ8ovrvpzQbWzMSbb9fgwUJWs6goccNsuy7WsBhGZJSLpIpKem5vrRzGUUlEnFGO3RPnF1YBcUDXGGBExjXzNAmABQFpaWqNeq5SKcrUn26g9c1IgNIO26v7wp+aeU5Vucd0fcS0/CHR3266ba5lSSvkmyvPhoeBPcH8fmO56PB34j9vyG1ytZkYABW7pG6WUaliU58NDwdemkG8AXwH9RSRLRGYCDwLjRWQnMM71HOAjYA+wC3gOmBPwUiuloluU58NDQYwJf7o7LS3NpKenh7sYSikVWitXwplnQu/eTXq5iKw1xqR5WqfDDyilVDgsXQqXXw5zgpPc0OCulIpOkTwV3scfw4QJVo39pZca3r4JNLgrpaJTpA4vsGgRXHstnHMOLFsGnYMzno0Gd6VUdIrE5pQvvgjXXQfDh8Pnn0OHDkE7lAZ3pVR0irTmlH//O9x4I4wdC598AikpQT2cBnelVHSKlOaUxsD//i/86lfwgx/Af/8LrVoF/bA6nrtSKjpFwvACxsC8efDQQ/CTn8ALL4AjNGFXg7tSSgWD0wm/+AU88wzccgs8+STYQpcs0bSMUipyRXJzxvqUl8P06VZgnzcPnnoqpIEdNLgrpSJZpDZnrM/Jk3DNNfDqq1au/YEHQMTjpp9vy2Hd/vygFEODu1IqcnlrzhipNfojR+CSS+Czz+Cf/4Q77vC4WUWlk4c+3sZNL6bz1Oe7glIUDe5KqcjlrTljOGr0DX2h7NkDF14ImzfDe+/BzJked5NdUML1z33NP77YzdTzu/PUj4MzzbQGd6VU5PLWnDEcHZTq+0JZtw5GjoRjx6zOSRMmeNzFsm1HuOrxFWQcKuDRKUN48IeDiY+xB6W42lpGKRW5vDVnbJt6eqamUHVQqv2FkrvCqsGXzoVpM6F9e2vMmO99r85LyyudPPzJdp5dvodzurThyevPo0/H1kEtrtbclVLNTzg6KLmniKq8twp+MNUaAGz1ao+B/cCxYqY8+xXPLt/DT0ecxbtzRgU9sIPW3JVSzVE4OiiNXmilYnJXgAE+BN5wwgA7LF8Oycl1XvJxxmH+sGgDxsBT1w/l6sFdQlZcDe5KKeWLqi+UxaPhr6thmYERAneNqBPYSysqeeCjbby4OpPB3ZJ5ctpQerRPDGlxNbgrpZSvjh+Hh+1WYJ8UCzcPh4sX1dgk82gRc99YR8bBQmaO7sXtV3yPWEfoM+BNDu4i0h94y21Rb+BPQArwcyDXtfxOY8xHTT2OUkpFhD17rFYwu3ZZQ/dOn15jtTGGhelZzP/vZmLsNp67IY3xA4IzVrsvmhzcjTHbgVQAEbEDB4F3gRuBx4wxDweigEopFVAlOVbuPH+9dZF09EIr5VKf1autER0rKuDTT2HMmBqr84vKuOOdTXy8+TCj+rTnkSlD6JKcEKQT8E2gfiuMBXYbY/YFaH9KKRUcje0A9eabcOmlVl59zZo6gX3lzqNc8fhylm7L4c6rvserMy8Ie2CHwAX3qcAbbs/nishGEXleRNp6eoGIzBKRdBFJz83N9bSJUkoFnq8doIyB++6DadOsmZPWrIF+/apXl1ZUcv+HW/jJv74mKT6Gd+dcyKyL+2CzeR5HJtT8Du4iEgtcC1Q1NH0a6IOVsskGHvH0OmPMAmNMmjEmrWPHjv4WQymlfOPLDE3FxXD99XD33XDDDbBkidVJyWVHzgl+8NRqnluxl5+OOIv/zh3NoK51m0KGUyBay1wJrDPG5ABU3QOIyHPABwE4hlJKBUZVe3X3nLu7zEyYNAk2bLBGdLz99upRHY0xvPzVPv73o620jnPwr+lpjD0nfBdN6xOI4D4Nt5SMiHQxxmS7nk4CMgJwDKVUS9WUC6D1qa8D1BdfwOTJ1njsH34IV15ZvepwwSluf3sjX+7I5ZL+HXnoR0PomBTX9HIEmV9pGRFpBYwH3nFb/JCIbBKRjcAlwG/8OYZSKoo0ZajeUIwAaYw1gfW4cdCxI3zzTXVgN8bwzrosLnvsS77Ze4y/TBzI8zPOj+jADn7W3I0xRUD7Wst+6leJlFLRqypQm4rTgbqhYQSCPQLkqVMwZ441v+m118Irr0CbNgDknijlznc3sWRLDmlnteXhyUPo2SH4k1sHgvZQVUqFTp2RFVdZtfj60i3BHAHy0CH4n/+Br7+GP/0J/vzn6unwPtyYzV3vbaKorJI/XnUON43uhT1CWsL4QkeFVEqFTp2RFZ0Np1uCNQLkl1/CsGGQkQFvvw333AM2G/lFZcx9fR2/eH0dPdol8tGvRvPzi3s3q8AOWnNXSoWSe0uVymIwldby+tItgR4B0umEv/4V7rwTzj7bauY4aBAAS7bkcMc7mygoKeN3l/Vj9vf74LA3zzqwBnelVOi4B+olF4dhwo18q936Bx/AlCnWPKdJSRwvLuMvH2zhnXUHOadLG16+aTgDzmwT/PIEkQZ3pVR4NNTePNDS061mjgcPwhNPwNy51rDsGw8x//3NHC8u55eXns0vL+0bllEcA02Du1IquLy1Uw/VhBvGwLPPwq9/DZ07WxNrjBhBdkEJd7+XwWdbjzC4WzIv33RBs6+tu2v+X09KqchWXzv1prR7b4yTJ+GnP4VbboFLLoF163AOv4BX1+zjskeXs3LXUf541Tm8c8uowAf2YJ9bAzS4K6WCq7526sHsoPTdd5CWBq+/Dn/5C3z0EXtMPFOfW8Nd72VwbrdkPrn1Yn5+ce/gXDQNReeremhaRikVXPW1Uw9GByWnEx5/HObNswb7WrKE8jGXsODLPTy+dCfxDhsP/XAwk9O6IRLE5o3B7nzVAK25K6WCq7526r6M0NgYOTlw9dVw221w+eWwcSPf9R3KtU+u4q+fbGfs9zrx2W3fZ8r53YMb2CHw59ZIWnNXSgVXfRdOA9li5pNPrKnvjh+HJ5/k+Iyf8X+f7ODNb/fTKSmOZ34yjCsGndH0/TdWqFsD1aLBXSllCfToi74IRIuZ0lKrQ9Kjj8LAgZhPP+Xt8nY88OhyjpeUc9OFvfjN+H60jgtxuPN2biF6nzUto5SyhPkCYJNs3QqjRlmB/ZZb2PnB51y3uojfLdzAWe0T+e/c0dw9YUDoA3t9QvQ+R9AZK6XCytsFwHDU6BtSddH0zjshMZHSfy/i0bie/OvZNbS2FfPg91YyZcpd2BIjsN16iC60as1dKWXxdgEw0mr0e/dabdZvuw0zbhxfvLOMS3e35dnVh5mUsoyl/W9matyz2FZNCW85vQnRhVatuSulLN4uAIa5SV81Y6yxYG67DUTI+ds/+H3roSxffID+nZNY2G8+58enu7YlfOVsSIgutGpwVyqa+JNC8XYBMJjjqfvq0CH42c9g8WIqvj+Gf9zwR57YVUbCsePcdfU5TB/Vk5jPE+CoI7zl9EWIhl3Q4K5UNGnKTEcNCWeTPmPgjTesQb5OnWLdb+dzc9II8naWMvX87vz2sv50aB0X/nJGIL+Du4hkAieASqDCGJMmIu2At4CeQCYwxRiT7++xlFINCEYKpbE1zapfD8fWgdgBJ7Q9r/EXYg8csKa/++ADTp6Xxm1X/4ZPy5NJ65TEi9cOZFDXZP/KGeUCdUH1EmNMqjEmzfV8HrDUGNMXWOp6rpQKtjD3igRO/3qoLIKKQqg42bgLsU4nPPUUDBiA8/PPeecntzF43N1sSuzM41NTWTh7ZN3AruoIVlpmIjDG9fgl4Avg9iAdSylVJRJSE+6/Hqo09Cuiqra/aS38ywFbCskcNpqfjZjJgZTOzL24N7PH9CExVjPJvgrEO2WAT0XEAM8aYxYAnY0x2a71h4E6v8VEZBYwC6BHjx4BKIZSKiJSE+4XYKs09Cti2Q/hha8w7zkpi43h/olzeLn/lVw9+ExeuPJ7dG+XGOxSR51ABPfRxpiDItIJWCIi29xXGmOMK/BTa/kCYAFAWlpanfVKqWaq6teDp5y7J6tXY27+CslysmxQGr8fcyu9Oh3mnekXMrRH25AWPZr4HdyNMQdd90dE5F1gOJAjIl2MMdki0gU44u9xlFIRrClNMHNz4fbb4YUXyEtO4fc/+jX7zunC/Wc+zeW9bUiP34ai5FHLrwuqItJKRJKqHgOXARnA+8B012bTgf/4cxylVASob2ahxvRirayEp5/G2bcflS+/wjMX/JAf/OIFLr0qn08GzuOKPjbkopbdjDEQ/K25dwbedY2L7ABeN8Z8LCLfAv8WkZnAPiBC+wErpXxWXxt6X5tgfvst5bN+Rsz6jXzTYxD3TpnFJRMvZPHYc0mK/wHwePDPo4XwK7gbY/YAQzwszwPG+rNvpVSEqS+AN9SLNS+P0tvvIPb5f5LfKoX7r/ktcaPK+GeX++kS2w/itX16oGm7IqWak3CO0FhfAPfWBLOigtJ/PI3z7j8Rc6KQ54ddy9YJZ/Or7m/SJ/6gtU1+cWjK38JocFeqOQnG8AK+qq8NvYcmmGUffEjRL39D28ydrOk+iI9/fwdTbprAzM0T4GiO1Yg6kseAaeY0uCvVnIRzhEYf29CXZ2wmd9ZczvzqCw6ldOGFWx5gzO2zmH9WO2uDtgHuaBWJ481HAA3uSjUnkTBCoxdlh4+Q+cvf0/udV2ntiOfFSXPof++d3Dawa80NA93Ryp9fM1H8xaCTdSjVnIxeCB1GgiPJuo+AkQ9Lc/eSft1oSnv2pM/br/LpyAms//wbpr/9JCNrB/Zg8OfXTKRNRBJAWnNXqjmJhOEFXE6VlJL+l7/R9x8PkVZ4jHVn96f8+hiuvDAfuXBg8AtQVeuuKDq9zNOvmfpq55EyEUkQaM1dqXCrr3NQBCouLWfJfU+T3aMvox+cR2GbVmy6tTfnzd/OBf0zkKOrQnMuVbVunNZzsXv+NVNf7TwSRtEMEg3uSoVbpKUGvHzZFJSU85+HFrCn9zmMv3sOMbZytj/xFH3/dRbnDt+P1ZcRwBmac6k9+qQ90fpVUztnXl/tPALTXIGiaRmlwi3SUgO1LlAe/uxGFu+aTv8nH2LinnXkJSezd0YXeo07Ap3ehNGLTqc9KovBVFr7Cfa5+Hpxub7tIijNFWga3JUKN0/Bxz1PnDzA2q5gS2hadLi+bHae6s7H6SMZ8vFObtw7lZNJKWRP60KXy7JpH1twelv3ALnk4tC15vF17Ppwj3EfphY5Ykz4R9tNS0sz6enp4S6GUoHnywfb0zbutWd34rDSB0GqbRpj+Hbhj1m6NJ4LP13PxZnfUdQ6kfLf3kHK726Fr66qGbxrlyWKmxY2yNu51/7CC+DfT0TWus2AV3OdBnelgsjTB9tTTbJ2APx3Gytv7U3HiwIaOCudhiVbclj+/Dtc9c4CRu/bQHHreJjSg8T/WwwdelsbhiJ4h+sLwt/jegvitf+WjiSYUhiQImtwVyqU3IOEew4arA927TSMp5qce6DwJEA1wIKSchZ+u58tL7/N5CWvMnL/JkradcBx++3EzJ0DiWGYASmINd2gHtdbEA9TzV1z7koFmreUCnZAIHfF6UXeLjq61+6rcu55Xzf8Onf11ER3HTnJKyt2UfTam9y4aiE/O7KHU53OoPKRR0mYfXN4gnqVcF1g9ve43i7chinnr8FdqUCrM0G0DRytALFq8u68XXT01IqjsRcra7V6ca6YzJdnLeK1z7fS9Z03+Nm379G9IIdTZ/eDB58n/vrrIS6u0acbcOEaYsHf43oL4mFqkaPBXalA85Z2+XeburX5xrSt9rUGWFVjd/1COFGZwDv5Y3l33SWM+epWHv7uQ1KKCykfMRLuWED8hAlgi6AuL+Fq3eLvcSOsWaUGd6UCqSQHnGVgnFaPyXbDTgcJX3Lt9fE1eLhq7JuK+/DasavYvr0H16Uv4a0t84irKMd5zTVw++3EXHhhk04x6MIVJN2PGwWtfjS4KxVIKyfDsbVYXeIdYIs9HRRCUCMtKq3gvzvb8sbhB+myKY+Za//D+Qe2UBkr2L8PTBuGbeb7AT9u1AnnuPkB0uTgLiLdgZex5lE1wAJjzOMiMh/4OZDr2vROY8xH/hZUqWahvotyQayRbs0u5PWv9/P5qq1c8207nvvufjoV5uPsIHA92McYaAU4dgTl+FEn0noNN4E/NfcK4LfGmHUikgSsFZElrnWPGWMe9r94SoVRU36ah/Bi4MnSCj7amM2b3+yDNWv48cZP+NPWFcSUl2JSk2FcPLZhMWBKInL894gWrL9jCNM9TQ7uxphsINv1+ISIbAVCMHizUiHSlJ/mQU69OJ2GNXvzWJSexapvd3LFhqU8nPEpvQ/vxbRujdw0A+bORQYNsl7gKZi4i4LcclAE6+8YwnRPQDoxiUhPYDkwCLgNmAEUAulYtft8D6+ZBcwC6NGjx7B9+/b5XQ6l/NZQB6QA9SxsrP15xSxal8Xb6Qc4Y/M6bsj4lKu2riCmrBRz/vnIrFkwdSq0bt24HYerw1BLFeDeqkHtxCQirYG3gVuNMYUi8jRwL1Ye/l7gEeCm2q8zxiwAFoDVQ9XfcigVEPWN6VLfT/Mg1ICLSiv4aFM2i9ZmkblxB5O2fMFbO76k26G9mKQkZOZN8POfI+ed1/SDeMsta40+OEKYtvMruItIDFZgf80Y8w6AMSbHbf1zwAd+lVCpUPLWAamhn+YB+rldWlHJl9tz+c+GQ6zesI+Lt6zid9u/JG3XWsQYGDkS7r0Lue46aNXKepE/gdhbsImC1iIRKYRt+P1pLSPAv4CtxphH3ZZ3ceXjASYBGf4VUakQampbdD9aV1Q6DWv25PGf9Qf5ZNMhBuxYz7TtX/Do1pXEnSrG9OyJ3H03/PSncPbZdXfgTyD2FmwaOh+t2dfP2/sTwjb8/tTcLwR+CmwSkfWuZXcC00QkFSstkwnc7McxlAoObx++ptasGvlz2xjDdweO8/76Q3y48RBdd2xi0o6V3LFrNW3zcqy0y4+nwQ03IKNH19+DtHYgzl1lnZ8vwdZbsGnofLRmX78IeH/8aS2zEhAPq7RNu4p83j58DdWs/PhScDoN6/bn83HGYT7OyCZlWwYTt6/g412raX80GxMbi1x+OUybhkycaA3eVZIDS8fUX0Num1pzMDKc/geThs4nCtqBB1UEvD/aQ1W1TE398DXyS6G80sk3e4+xOCObTzIO02nXFibs/Iq3d62m85EDGIcDGT8ernvACugpKb4dz93ohfBe15otezydT2NSKQ19yYVrcK/mIgLeHw3uqmVq6ofPhy+FU+WVrNp1lMUZh/ki4yD9tn/HVbu/5ra939AuLwdjsyGXXgrX/QmZNAnat/freCR0hg6jGj6fQKYKwj11XaSLgPdHg7tqmRrz4auq8R5bB5WnTi93C6JHTpzii225LN2Ww7rNBxi2/Vuu3vM1f9n9LYlFJzDx8VbKZeJEZMIE6NjRt3L6+iXky/kEMlXQ2AuDLe0CbASMEKkzMano1diA4sscmC5OI2wuH8zSzi+wbEc+RZu2M2ZPOpdlfsPQA9txVJRj2rdHrrkGJk6E8eNPN11szPEhcEGxznnYwNEacELb84IbcLWzVFDoNHuqZfIWUBo7kbGrV+GJygRWnxzCshPn89XRQfTbfYAxe9IZn/ktnY7nAWC6gozsCnNfgwsvBIfrx7EvXzTBDoDV47yvwhq10k2wA24Q5xFtyXSaPdX8NRQcPa331kTQW+651vYVxzayYd8xluf/gjUHkzF7YMS+DP5n/1LuO/gUjkonJsGODKiEIcBgkI6AoxC+//2a5fcl3x3sFhZVqQJPk28Hu0VHBFxgbGk0uKvmwVNwdM8zV01h577eWxNBL0HUpKSy99A+Vhaey8rCVAp3t2HIM79l1L6N3JKVQXx5GUaAnjZkUhdIS0LO3GlNjVrFW+DyJXCHKgC6H6dKsANuBFxgbGk0LaOaB08/6z0FKff11+ys20TQ7XXGWcH+8m58LdeSHnMjx1Z+xVnbNzJi/yZGHNhEUqk132nlOQOwj70ULr3UqpG3a+e5TAAdL2p6yiVQ1wga4n6BWOyEJOfeXDXlF2MI30PNuavmp/aHxllmzXDkHhzz19cNrlAzeLoFVYOD3YlX83X7+9j05SIqNhXQe38Ww7K2MuTwDuIrygEo79kLx7ixyNixMGYMnHGG5zI2Jkde+3zS/gHpc/wLCv4cv7FfBC2llUttDb3HYb5QrMFdNT+1PzTthllT1rnXNrGdTsVgtwb4wlQHobKYjmzeu5f1nz9O9uYSKvfY6Ld/L2kHt9LnWBYATruN0h4xxPcvRfrbYEQaTP3atzL6E/gCERQac5GyqccLZfCKxC+Sht7jMF8o1guqqvmpnaMu2GJ9aGo05zsd0E1KKtmDX2P9ITiwKp3yl/5G222bGHhoJ9cf2UtcpVUrL22TQuWIkZgxc5ALL8S2/2oSbCddB3WCbavvZfSnLXMgLp42Jkff1OOFsht9BIzHUkdD73EEXyjW4K7Cz1ONzduHxhVs8iuS2JZ/Foczu1BccD6xWzLoc2A8lx7ZS3xFGQClcTGc7NWeimtnEPf9S2DYMOLOPrvmIFxLzmvah9PfWmYggkJjLlI29XihDF4RMB5LHQ29xxF8oVjTMir8PP30d/vQ5MaNYI/z1+Sv3YlzzQe02Z/HWUey6V5QPXUAp+ITKeg/ENv559O23ec4Ou2EzpVgD1Iu2t90RX0XNSHw6YnmkHPXjk6Npjl3FdlcectThQ4O7e9IflYHyk6NIHb3Ttpm7aVr3kHiKq0aXaXYONKpA0Vd44np25b21/ya1kMHwv7fQsEGSB4Aed9So5NOMPKggcq1egpo0DKDnC9fJJGYlw8jzbkr3wT6g1OSA8snuoItYE+AE1BW1J88+wwKd2ZTvm0L8dtT6JBrSCk+SW+ygWzKbVs43P5MCjq3ZvugXrQ6q5gz+hylVfcKunTpX7fFwrE1VjDM83AxNBiphEClK7ylIiItPeFJoP9ffLmGEYq8fJR8gWjNvSXwtRmer931q15fO6Uw6i04IbBrA87PfkPhjgMU5cTjPGIj9mg5ycdOEl9WXqNoua1SONS+I4WdWuHsIiR2d9LpnLac2e47YhzGrTWMG3sraDf0dHnyv4OKk3hmg0mHAt9ePFC1zEipuTflPQhHGiUUrVOaUXpI0zLRyv0DmTzAWlawpW4Ad++9KQ6wJ0JFEVDVuccOjgSoLKnb4aeqhUrWaiiohAI7HI+Fo6WUHrVx6mgczmOCI7+SxIJS7M6aY5YUxcRzILkzh9p2pLBdIhUdHDg6V9D6jGLOPPMovZMPEW8rc3uFDcTmuWMSnC5/7fPx9CXg6wczWB/mpnZcgtDXHJvyHoS6GWBJDvy3H1S4jhGswNuMxsHRtExz0lDvQfdg4B603dMRuStg8bme928qoKwQSoCiqlslFJ6EQqAATAFUFDqoPGmHua1xFJTgKKsK2pVYLwanI4a8pGQOJ3Xg8JntyfteMmVndMe0OkpC+yJSOhVyZvuj9I4/RH/HPsTTvF012K39G6fn1WJ3dV76rmbawlSe7tRU+0vOvfWCt9ppsFpp+DoWu6fg1FDACnTqoCnvQaibAa6c7KqUuNgTg9M6pbHnFaFpnKAFdxG5Angc6xP7T2PMg4E+xqHjJTyxdCcpibG0TYwhJTHG9dh6npwYQ0pCLLGOeuaf9EednHL86fG+3R87Eq375IHWfcEWKwg5y09/iByJVlCrLOV0jdoldwW8e6b1uNIJp6h7KwFK3R4Xud1OgikCZ5ENisBW7ES8/GCrFBt5rZI5mphi3TqlkNczmbzEFE4mxWPaCI52ThI6nKJ9cgHd4o7QPSaHAbF7SLafrCeA26gzEmFtjlana2XVy9rg3jHJY/qo3VDfam/e8rXBClLBDH6Bzj03payhbgaYv56anw0TnCDa2POKxPb5BCm4i4gdeAoYD2QB34rI+8aYLYE8Tt7RQ3zx3SYKyuKoMA7EgGDAGART/by1FJNiKyTFfpJW9lKSKCLRVkIrexlJnCDRdopW9nJay0kSpYRWjgoSnEXEShmxdjtxlBIrlcRWniDOWU4MNmIqihGnsf7Xqm7OYqjAimGVxW7rTlrLKr6Gclw31+Oyqucn3R6DKQdnmWDKBMpAToGcMti8ZCtqq7DZOBmfSGF8K47HJ3Esrg0F7ZM43q01BXGtKYhvTWG8dV+eaMeZZMOW7KR1Ugmd4vI5IyaPTo48usYcYWjMNjrHHCPBVtq4P5DYrRmC6hkX3drObTiBGmxwzY66H+CmBJWSHDi6+nTayb126ml/gaiNBTP4BfrXRlPKGuoJKUL1S6Gx5xWJ7fMJXs19OLDLGLMHQETeBCYCAQ3u5y7+EWvuXx/IXYaME6E0JoZSR6x1s1uPTzliKbXHUuKIo9QRQ2mStawoNpHi2HhOxiZQHJNg3cfGUxSbQFFsAsUxcZh4sMcbbIlOYuPLSbKX0MZ+kiR7MSn2E7RzFNDOUcjZ9izaOU7QPr6CtvbjxDj9ySfWkyO3J9b8kLgHEE/pE/caUFXA9xRQmxJUVk6ueT0BTgcHT/tz/yJqam0smMEv0IEuAmYOalCkdhiK0F6qwQruXYEDbs+zgAvcNxCRWcAsgB49ejTtKDE74UdVO3TdvD2uvcxtXSVCmYnhlImllFhKTSzlxkE5dipwUI6DCuyUSQxlthjKJIZSWwylEkupxFBhd1AhdirsdiptdirERoXdToXNQYXNTrnNToU4qLDbMLF2nA5B7AaHzYkNJw6pxC6VOKjEJtbzGKkgwVZKgpQSZyujra2QM225xEspCbZS4m2ueymjtb2Y1rYSbGKzUhum8nS+vnYqCKBgT4MzDdVhbw0pA2u2IXcf88XTBBC1/8kbCiDBrunWYKt//xFaG6sWqYEumCL1CyhC/xZhu6BqjFkALACrtUyTdtJnKLRZ0fB2DbBjSKCMBMoa3tgrm/ecO1B/vtl2OucuDqovoro3Oay6eCoCKUPAllwrYG8JTPqgan/HMxq+oOt+PPfrDyLQLq3x/+ShrOl6+1XgbfsIqY1Vi9RA1xJF6N8iKE0hRWQkMN8Yc7nr+R0AxpgHPG3f5KaQvl7Q9OdxVdCtGoHQfX1VELv4P763ow5EMFaNF6qx0pUKoZC3cxcRB7ADGAscBL4FrjfGbPa0vbZzV0qpxgt5O3djTIWIzAU+wWoK+by3wK6UUirwgpZzN8Z8BHwUrP0rpZTyLki9e5RSSoWTBnellIpCGtyVUioKaXBXSqkoFBFD/opILrCvgc06AEdDUJxI1ZLPX8+95WrJ5+/LuZ9ljOnoaUVEBHdfiEi6t/acLUFLPn8995Z57tCyz9/fc9e0jFJKRSEN7kopFYWaU3BfEO4ChFlLPn8995arJZ+/X+febHLuSimlfNecau5KKaV8pMFdKaWiUMQFdxG5QkS2i8guEZnnYX2ciLzlWv+1iPQMQzGDwodzv01EtojIRhFZKiJnhaOcwdLQ+btt90MRMSISNU3kfDl3EZni+vtvFpHXQ13GYPLhf7+HiCwTke9c//9XhaOcgSYiz4vIERHJ8LJeROQJ1/uyUUSG+rxzY0zE3LCGB94N9AZigQ3AgFrbzAGecT2eCrwV7nKH8NwvARJdj2+JlnP39fxd2yUBy4E1QFq4yx3Cv31f4Dugret5p3CXO8TnvwC4xfV4AJAZ7nIH6NwvBoYCGV7WXwUsxpoUdATwta/7jrSae/XE2saYMqBqYm13E4GXXI8XAWNFRGj+Gjx3Y8wyY4xrOijWAN1CXMZg8uVvD3Av8H/AKQ/rmitfzv3nwFPGmHwAY8yREJcxmHw5fwO0cT1OBg6FsHxBY4xZDhyrZ5OJwMvGsgZIEZEuvuw70oK7p4m1u3rbxhhTARQA7UNSuuDy5dzdzcT6Ro8WDZ6/6ydpd2PMh6EsWAj48rfvB/QTkVUiskZErghZ6YLPl/OfD/xERLKw5on4ZWiKFnaNjQvVwjZBtmo6EfkJkAZ8P9xlCRURsQGPAjPCXJRwcWClZsZg/WJbLiLnGmOOh7NQITQNeNEY84hrjuZXRGSQMaa+medbtEiruR8Eurs97+Za5nEb11ytyUBeSEoXXL6cOyIyDvgjcK0xpjREZQuFhs4/CRgEfCEimVj5x/ej5KKqL3/7LOB9Y0y5MWYv1hzFfUNUvmDz5fxnAv8GMMZ8BcRjDawV7XyKC55EWnD/FugrIr1EJBbrgun7tbZ5H5juevwj4HPjuvLQzDV47iJyHvAsVmCPppwrNHD+xpgCY0wHY0xPY0xPrGsO1xpjomFmdV/+79/DqrUjIh2w0jR7QljGYPLl/PcDYwFE5Bys4J4b0lKGx/vADa5WMyOAAmNMtk+vDPfVYi9Xh3dgXT3/o2vZX7A+yGD9URcCu4BvgN7hLnMIz/0zIAdY77q9H+4yh/L8a237BVHSWsbHv71gpaW2AJuAqeEuc4jPfwCwCqslzXrgsnCXOUDn/QaQDZRj/TqbCcwGZrv93Z9yvS+bGvM/r8MPKKVUFIq0tIxSSqkA0OCulFJRSIO7UkpFIQ3uSikVhTS4K6VUFNLgrpRSUUiDu1JKRaH/B/3lCXFtiwzMAAAAAElFTkSuQmCC\n",
"text/plain": [
"
"
]
@@ -735,7 +738,7 @@
"name": "stdout",
"output_type": "stream",
"text": [
- "0.005000000000000007\n"
+ "0.0050000000000000044\n"
]
}
],
@@ -1179,6 +1182,8 @@
"cell_type": "markdown",
"metadata": {},
"source": [
+ "Note well that we have made life simple here. We perform a fit in terms of the number of nucleons only. A more sophisticated fit can be done by including an explicit dependence on the number of protons and neutrons in the asymmetry and Coulomb terms.\n",
+ "\n",
"With **scikitlearn** we are now ready to use linear regression and fit our data."
]
},
@@ -1217,9 +1222,7 @@
"text": [
"Mean squared error: 0.04\n",
"Variance score: 0.95\n",
- "Mean absolute error: 0.05\n",
- "[ 0.00000000e+00 7.06492086e-03 -1.73091052e-01 -1.66020213e+01\n",
- " 1.17385778e+00] 15.212327334149492\n"
+ "Mean absolute error: 0.05\n"
]
},
{
@@ -1244,7 +1247,6 @@
"print('Variance score: %.2f' % r2_score(Energies, fity))\n",
"# Mean absolute error \n",
"print('Mean absolute error: %.2f' % mean_absolute_error(Energies, fity))\n",
- "print(clf.coef_, clf.intercept_)\n",
"\n",
"Masses['Eapprox'] = fity\n",
"# Generate a plot comparing the experimental with the fitted values values.\n",
@@ -1308,7 +1310,7 @@
"270 3344 160 110 270 Ds 7.253775 7.253775\n",
"\n",
"[267 rows x 6 columns]\n",
- "0.009883615646716182\n"
+ "0.009883615646716186\n"
]
}
],
@@ -1370,8 +1372,6 @@
"/Users/mhjensen/opt/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:582: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (100) reached and the optimization hasn't converged yet.\n",
" warnings.warn(\n",
"/Users/mhjensen/opt/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:582: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (100) reached and the optimization hasn't converged yet.\n",
- " warnings.warn(\n",
- "/Users/mhjensen/opt/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:582: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (100) reached and the optimization hasn't converged yet.\n",
" warnings.warn(\n"
]
},
@@ -1464,7 +1464,13 @@
"output_type": "stream",
"text": [
"/Users/mhjensen/opt/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:582: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (100) reached and the optimization hasn't converged yet.\n",
- " warnings.warn(\n",
+ " warnings.warn(\n"
+ ]
+ },
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
"/Users/mhjensen/opt/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:582: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (100) reached and the optimization hasn't converged yet.\n",
" warnings.warn(\n",
"/Users/mhjensen/opt/anaconda3/lib/python3.8/site-packages/sklearn/neural_network/_multilayer_perceptron.py:582: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (100) reached and the optimization hasn't converged yet.\n",
@@ -1493,14 +1499,14 @@
},
{
"data": {
- "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlgAAAJjCAYAAADQ22+yAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAABi+ElEQVR4nO3deZyN5f/H8ddZZsY6xjJjrRDRIhUtUg0ixj6VSElUWmwRkULK2FL2SiGyU5mhLKEQqVBftCqR3Qyqsc1ylt8fd2bMz7jNMfecM8v76TGPh3OfM+e+Pue+z32u876u+x6b1+v1IiIiIiKWsQe6ASIiIiL5jTpYIiIiIhZTB0tERETEYupgiYiIiFhMHSwRERERi6mDJSIiImIxZ6AbIFKQDR8+nC1btgCwe/duKlasSKFChQBYuHBh2v8v5amnnmLAgAFUq1btoo+ZMGECV111FW3bts12u8/p2bMn3333HevWraNw4cKWPa+ISF5n03WwRHKHRo0aMWHCBGrVqhXopmTJ0aNHadOmDbVr16ZBgwY8/PDDgW6SiEiuoSFCkVxq0qRJPPHEE7Rq1Yp+/fpx7NgxnnvuOdq3b0+jRo3o1KkTx48fB4zO2c6dO/n222/p0KED/fv3p23btjRv3pxvvvkGgIEDBzJ9+nQAatWqxaRJk+jQoQONGjVi5syZALjdbkaOHEmTJk24//77efXVV+nUqVOm7Vu0aBH16tUjOjqaDz/8kPO/q23fvp127drRsmVLoqOj2bx5s+nyGjVqcOLEibTfP3f722+/pXXr1nTo0IHWrVuTkpLC8OHDadeuHc2bNycqKopt27YBcPr0aV566SWaNm1K8+bNeeuttzhz5gy33nore/bsSXvuLl26sGbNGis2kYjIRamDJZKLHTx4kCVLljB27Fg+++wzbrrpJhYuXMjatWspVKgQcXFxF/zOjh076Nq1K7GxsTz44INMnjz5gsekpKRQsmRJFixYwMSJE3nzzTdJTk5m8eLF/PTTT3z66acsWLCA/fv3Z9oul8vFokWLaN26NY0aNeL48eNs2LABgNTUVLp370737t359NNPef311xkxYgQpKSmZLvd4PKavwe+//86bb77J0qVL+emnn4iPj2fhwoUsX76c6Oho3n//fQAmTpxIcnIyy5cvJzY2lu+//56dO3fStm1bFi9eDMC+ffvYs2cPDRs29Gk7iIj4SnOwRHKxm266CafTeJt27tyZrVu38sEHH7B3715+//13ateufcHvVKhQgWuvvRaA6667jiVLlmT63Pfeey8A119/PSkpKZw5c4b169fTpk0bQkJCAGjfvj2zZ8++4HfXrl2Lx+Ph7rvvxul00rx5c2bNmkVkZCS7du3CbrfToEEDAG644QaWLVvGTz/9lOnySylfvjwVK1YE4Oabb6ZEiRJpnb9vv/2WokWLAvD111/z0ksv4XA4cDgczJkzB4CIiAgeffRR+vTpw8KFC3nwwQdxOByXXK+ISHaogyWSixUpUiTt/2+88QY7duzggQce4Pbbb8flcpHZFMrzJ8bbbLZMHwOkdaJsNhsAXq83rTN3jt2eecg9f/58kpKSuO+++wAjEUtISOD333/H4XCkPec5u3btuujyqlWrZliWkpKS4fb5r8G6deuIiYmhS5cu3HvvvVStWpWlS5cC4HQ6Mzz/4cOHKVSoEFWqVKFGjRqsXbuWZcuWpaVZIiI5SUOEInnExo0b6dy5M23btqV06dJ8/fXXuN1uS9cRGRnJ0qVLSUlJweVyZZp+7dmzh++++44lS5bwxRdf8MUXX7Bx40bq1q3LrFmzqFq1KjabjU2bNgHw008/0blz54su93g8lCpVip07dwKwevXqi7Zv06ZNNGzYkI4dO1KrVi3WrFmT9hrUq1ePJUuW4PF4SElJoVevXmlnaHbs2JExY8ZQu3ZtypYta+lrJiKSGSVYInlE9+7dGTNmDG+//TYOh4NbbrmFffv2WbqO+++/nz179tC2bVuKFClCpUqVLrj8wvz582ncuDFXXnllhuU9evTg6aefpm/fvkyaNIkRI0YwZswYgoKCmDRpEsHBwRdd/sorr/Daa68RGhrKnXfeSXh4eKbt69ChA/369aNVq1Y4HA7q1q3L559/jsfjoUePHsTExNCmTRvcbjfNmzdPS9gaNmzIK6+8QocOHSx9vURELkaXaRCRNBs3buT48eO0adMGMK7TFRISQv/+/QPcsuz5/vvvGTx4MJ9++ukFw5QiIjlBHSwRSXP06FEGDhzI8ePHcbvd1KxZk1dffZXixYsHummXbcCAAXz33XeMHj2a2267LdDNEZECQh0sERERyfOWLVvGO++8Q2pqKo8//jiPPPJIQNujOVgiIiKSpx09epRx48bxySefEBwcTIcOHbj99ttN/3xYTlMHS0RERHKNxMREEhMTL1geGhpKaGhopr/z9ddfc8cddxAWFgZA06ZNWblyJT169MjJpprK8x2shEMVA92EgLFTsCfrOgp4/QV9+3vQ7IaCKhXzq/8XBOEVDvp1fZ4j1/htXbMW98z0L1D06NGDnj17Zvo78fHxGc4+joiIYMeOHTnWxqzI8x0sERERyT86d+5MdHT0Bcsvll4BmV5QOdBnDKuDJSIiIrmG2VDgxZQtW5atW7em3Y6PjyciIsLqpvlEV3IXERERUx4//rscd955J5s3b+bEiROcPXuWzz//nHvuucfiV8E3SrBEREQkTytbtix9+vThscceIzU1lQcffJAbb7wxoG3K89fB0iT3gkuT3At2/ZrkXnBpkrv/J7knH6566QdZJKT8n35bV07SEKGIiIiIxTREKCIiIqaUGPtOCZaIiIiIxZRgiYiIiKnLPbuvIFOCJSIiImIxJVgiIiJiyp23LzgQEEqwRERERCymBEtERERM6SxC3ynBEhEREbGYOlgiIiIiFtMQoYiIiJhya4jQZ0qwRERERCymBEtERERMaZK775RgiYiIiFhMCZaIiIiY0oVGfacES0RERMRiSrBERETElP7Us++UYImIiIhYTAmWiIiImNJ1sHynBEtERETEYkqwRERExJRbAZbPlGCJiIiIWEwJloiIiJjSWYS+U4IlIiIiYjElWCIiImLKjS3QTchzlGCJiIiIWEwdLBERERGLaYhQRERETHl0mQafKcESERERsZgSLB999EkQH8cGERLs5aqrPLzQO5nQUOO+T2KdfLo8iORkGzWucTOwfzLBwYFtr9XWf+Vg2sxg7DYvxYvDwH7JVKqY/tXmaLyNp7oX4sNpZwkrEcCGWmz5KgfzFgel3T51GuITbCxbdJbSpaBp28KEl0l/HR5tn0qzJu5ANDXH/PGnjTcmBnHqtA2HHV7qm8K1NYyaO3ULITkFnP8dUaIau+nUwRXA1lrvjz9tvDkxmFOnbdjtXgb+V39KCrw5KYitPzgoUhjuqufmqcdTseejr68Fff+/2HHf7YZJbwfz3RYnbjc83D6Ftq3z135/jia5+04dLB98/4ODufODmPr2WSLCvaz83MmYNwsxfFgS6zc4+HhJMO9MOkOxYjD41UIs/CiITh1TA91syyQnw7ARIXw47SyVKnpZsNjJuEnBvDkqGYAVq5y8/0EQx47lo0+W/zRv6qZ5U+MDw+WCp3uH8NjDLkqXgr/22She3MucaUkBbmXOSUqCHv1DGNw/hfp3eFi/0c7gmGA++jCZs2fhwCEbq2OT0jpY+U1SEvTqX4iX+yf/V7+DoTEhLPowiZlzgzhy1M68GUkEOWHUW8F8HOekXXT++aAtyPu/2XE/bpmTAwftfPjBGc6cgWe6F+Ga6h6uu1ZXjRI/drB2797NqlWrOHLkCHa7nYiICO6++25q1arlryZk26+77NSt4yYi3PimFnm3i9FjQ0hNhZWfB9HhoZS0NKtf3yRcqfmrx+/2gNcLp07ZAC9nz9oI+S+hSzhmY8MmB2+OSuKRLkUC2s6c9uF8JyXDvNz/3zfVHT/Zcdjh2T4hJCbaaHiPiy6PunA4AtxQC32z1U6lCh7q32F8cNxT30OF8ikA/PSrnSKF4fmXgjl23MZtdTw892QqhUIC2WJrfbvVQcUM9bupUN74/6+77DRp5Ep7L9xT38XchUH5qoN1voK2/5sd9zdsdNK6ZSpOB4QWh3sbufh8tZPrrk0JcKutpwTLd36JGubOnUvfvn0BqFWrFtdffz0AgwcPZsaMGf5ogiWuq+lm2w8OjhwxdrTlK4NITbXxb6KN/Qfs/P23jb4vFqLzE4WZMTOYYsXy16zAIoXhxT4pPN2zEK0fLMxHsUE897RxIAkv42Xka8lUqZy/av7//vkX5i0Kom+P9GTS7Ybb6riZMDqZdyck8e0WB4uW5K8oZ99+O6VLwetjgnjs6RC69wvG/d8I0OkzUOdmN6NeTWHWu8kcOWpjyvtB5k+Yx+zbb6N0KS/DxwTT+ekQevYLSav/+mvdrPnSyZmzkJoKn691cuxE/vwwKoj7v9lxPz7eTkRE+jEvItxDfD5M8OXy+OVd8OGHHxIbG0vhwoUzLO/SpQvR0dF07drVH83Itptqe+j6WAqDhhTCZoMWUamEhnoJcnpxuWDLNgejhicRHAzDR4Xw3vRgevfIP99kdv9pY8aHQcz9wBgiXPSxk0FDQpg1LQlb/vw8uUDsMud/6UX6QbVtSzdgfNoGB8PD7Vws+sTJww/mnwTD5YZN39p5961kbrjOy/qNdnoPDGHZgiQi63uIrJ8+JNLlkVReHBrMCz0C2GCLudzw9bcO3n4rmRuuM4YI+wwsRNyCszz2sIu3p9l4snshihf30riBmz/+zJ8fsgVx/zc77md2Zl1+mnt3Po+3gBzkLeSXDpbT6cTluvDNlpSURFBQ7v6mO21GMBu/NrLuW252c3/bVFq2MGo5ccLGtA8gNBTKlPZyz11uihY1fq9pYxcffJj3Z7i/PyMorf59B+w0inSlTWp/oK2LiW8H828i+WpC+zlTZwTx1X+1332nm6e7prL6Sycv9MzYaV7+uYPqV3uofrXxunghX8xFeneGkw3/1f9vIlS+0ssN1/03THKXh+Fj4eBhG/sP2ihWFG6pbXSyvIAzHwwPnb/90+s3aoy8y82I/+oPLe7lkYdS6f2skeqs/sJBpYp5fw5OQd7/s3rcLxvh4fjx9I5HwjEbEeF5f9uLNfzyNnjmmWdo27Yt9erVIzw8HICEhAS++eYb+vTp448mXLYnu6bw5H8B2759Nno+X5g5M89QtCjMnB1E40ap2GzQINLFF+uM8fjgYPhqk5Nra+b9s2ie6prKU12ND46t39sZMSaEEyegVCnYsNFB+XLefNm5Ani6aypPd00fCkk8aUzmvvGGjAfQP/fY+XKDg1HDUkh1wUdLnDRtnPe/vT/T1cUzXY06jp2A9o8X4pffbFxbw8v32+3YbFChvJdt/7MzfbaTqeOTCXLCvEVOmjTM+/v++dv/+Ano8HjhtPp/OK/+5ascbPrGwRvDUzibBPMXO3kkH5xBWZD3/6we9++u7+azFUHUv9PN2bOw9osg+vXJn5P9NQfLdzav1+uXSTNHjx5l8+bNxMfH4/V6KVu2LPXq1aNs2bLZet6EQxUtamHWfLwkiE9ig/B44cYb3PTtnUxIiDEPYdacIL74Mgi3B66p7ubFvslpiVZOsAdgh/94iZOPYoMIcnoJDYW+vZKpWiXjLnRnw6Isjz2d4x0vh5/r//lXO4NfD+bjuRkPoElJ8MaEYH78xY7LBfdGunn2ydQcHzb19/b/frudie8GcTYJgoPghZ6p3FTLg8cDk95zsnGzA7cb6tzkoX+v1By/RIkH/873+2G7nUnvBnE2yUZQEPTtmcJNtTy43DDqzWB+/NmOxwNtWrp45KG83cHITG7a/1Pxb0p0seO+yw1T3glmy1YHLpeN1q1S6djeP2eOh1c46Jf1nPP9viv9tq5brtznt3XlJL91sHKKvztYuUkgOli5ib87WLlNQd/+/u5gSe7h7w5WbuTvDtaWfZX9tq5br9zrt3XlpHw6HU9EREQkcPL4VEQRERHJaTqL0HdKsEREREQspgRLRERETOksQt8pwRIRERGxmDpYIiIiIhbTEKGIiIiYcnuVx/hKr5iIiIiIxZRgiYiIiCmP8hif6RUTERERsZgSLBERETGlyzT4TgmWiIiIiMWUYImIiIgpnUXoO71iIiIiIhZTgiUiIiKmPJqD5TMlWCIiIiIWU4IlIiIiptzKY3ymV0xERETEYkqwRERExJTOIvSdXjERERERiynBEhEREVP6W4S+0ysmIiIiYjF1sEREREQspiFCERERMeX26kKjvlKCJSIiImIxJVgiIiJiShca9Z1eMRERERGLKcESERERUx5daNRnesVERERELKYES0RERExpDpbv9IqJiIiIWEwJloiIiJjSdbB8pwRLRERExGJ5PsEqZgsOdBMkQDx4At0ECSAHBfsbtdvrDXQTAqaILc9/dOU5+mPPvtMrJiIiImIxfQ0QERERU25dB8tnesVERERELKYES0REREx5Cvicx8uhBEtERETEYupgiYiIiFhMQ4QiIiJiSpPcfadXTERERMRiSrBERETElP7Ys+/0iomIiIhYTAmWiIiImPLojz37TAmWiIiIiMWUYImIiIgpzcHynV4xEREREYspwRIRERFTnjx6HazY2FjGjh1L6dKlAWjQoAF9+vTh0KFD9O/fn+PHj1OlShXGjh1L0aJFSUxMpF+/fuzfv59SpUoxfvx4wsPDL2vdefMVExEREbmEnTt3MnDgQOLi4oiLi6NPnz4ADBs2jI4dO7Jy5UpuuOEG3n77bQDGjx9P3bp1WbFiBe3atSMmJuay160OloiIiJhyY/PbT2JiIgcOHLjgJzEx0ed279y5k9jYWFq3bk2/fv34999/SU1NZcuWLTRt2hSA+++/n5UrVwKwbt06WrVqBUDLli3ZsGEDqampl/WaqYMlIiIiucasWbO49957L/iZNWuWz88VHh5Oz549iYuLo3z58rz22mv8/fffFCtWDKfTmfaYo0ePAhAfH582JOh0OilWrBgnTpy4rDo0B0tERERM+XMOVufOnYmOjr5geWho6EV/Z8WKFYwcOTLDsqpVqzJz5sy0208++SSNGzfmxRdfvOD3bbaLX+fLbr+82tXBEhERkVwjNDTUtDOVmaioKKKiojIsO3nyJDNnzuTxxx8HwOv14nQ6KVWqFKdOncLtduNwOEhISCAiIgKAiIgIjh07Rrly5XC5XJw6dYqwsLDLqkNDhCIiImLKn3OwrFKkSBGmTZvG9u3bAZgzZw5NmjQhKCiIunXrsnz5csA40/Cee+4BIDIyktjYWACWL19O3bp1CQoKuqz127xerzf7ZQTO2cNVAt0ECRAPnkA3QSRg3Hn70J0tDpPhnIKiaPm//Lq+kT8399u6XrpuuWXPtXXrVmJiYkhKSqJy5cqMGTOG4sWLc/DgQQYOHMjx48cpX748b731FiVKlOCff/5h4MCB7N+/n+LFizN27FgqVap0WetWB0vyLHWwpCBTB6tg83cHK+anln5b18vXf+q3deUkDRGKiIiIWEwdLBERERGL6SxCERERMeXOo38qJ5D0iomIiIhYTAmWiIiImPJYePmEgkIJloiIiIjFlGCJiIiIKc3B8p1eMRERERGLKcESERERUx6v5mD5SgmWiIiIiMWUYImIiIgpt/IYn+kVExEREbGYEqzL9MVXNgaPdLJpeWqG5Ykn4YneTl590c31NfPnH2P9/7V7PDDhPQdffWPDboMrK8ErL7goFRbYdlrtrbedrFnnILS4sV2vutLL6KHp2//kSXiidzCvvpjKdfl02wN8+ZWdISOD+Gp5ctqyjt2CSU6GoP+OKFFN3HTu4A5QC3OG2fafPsfBZ6scuNzQvImHpx93kd/+HvG4t52sXe8gtLhx+6orPIwcmsrYiUF8vyP9u3rCMRtlSnuZPz35Is+Ud2W2758zdrKTfQdsTByVmslv5n2ag+U7dbAuw18HYNw7TjyejMu/+sbGG5OdHDoSmHb5Q2a1xy6388suGwvecxEcDOPedfDW2w6GD8pfH7Dbf7QzckgKtW+4sPO08Rs7Yyc7OXQkfx+E9h2wXbD9z56FAwdtrI1LTutg5UcX2/4bv7GzZp2Due+lYLdD9xeDqbrOzn0NPRd5prxp508OYganUvuGjHX165XeoTh0xMZTvYJ59aUUfzcvx2W275/z+Zd2lq92cMO1+WubS/ZoiNBHZ5Pg5RgnL3R3XXDf/I8dvP6Si/DSAWiYH1ys9qureOnzjJvgYOP2dTW8HD6avzoaKSnw2+82PlzopP0TwfQbEsTho+n3z//YwbCXUvPttgdj+78SE3TB9v/xVztFCkPvgUE81CWYsZOdJOWz8MJs+3/5lZ1mjd0ULgwhIdC6mZvlqx2BbbDFztU/Z5GTjk+E8OKQYI5k8h6PGRvEI+1c1KiWvxLci+37AH/+ZWPWfCfdOl94X37iwe63n/wi/1TiJ8PfdPBgKw/Vq154AHn7DRe1r89fB5bzXaz22td7ufYaY1niSXhvlp0mkfnrm1zCcRu33uKh51MuFkxLodZ1Hvq+HIz3v5diyhup+XrbA8S8GcT9rdwXbP8zZ6DuzR7GDEtl9rspHIm3Mem9/BVlmW3/I/E2yoWnvyZlw73EJ+SvLxgJx23UvcVD9ydTmTstmVrXeXjhlfT9H2DTt3aOxttof3/+Sq7BfN8fHBPEsIGpFCkcoMZJruWXDtahQ4dMf/KKhbF2HA5o2zx/dR6yIiu17z8IXXs5ubmWl/bR+es1qljey6TRqVS+0ovNBo+1d3PgkC3fDwmesyjWgcPhpW3zCz88I+t7GP5yKsWKGglO10dcfLkxfyU4Ztvfm0m/2p7PvrpWLO9lwqiUtPofbe/i4P/b/+d/5KRzRxeO/LXpTff9YW8E0eF+F9Uy+cKd37i9Nr/95Bd++Zr59NNPs3fvXiIiIvD+v6ORzWZj7dq1/mjGZXl7hoN1m4wNHhQESUnw0BNOXC4bySnG/yePdhFRJsANzQG+1L7lBxsvDnPyeAc3nTvkj87VOzOcrN9kfFJWrOCl4d1uWt6XXpvXC05H/j2wnl//ue3f4YlgUl2QnGL8f9LoFH7eZadYUS91av/3WuST1yWr279chJeEE+kfCvHHjBQrr3t3hpMNXxu9pYoVvDS8y03z+9I7GV4vOJ1GnX//Az/+YueN1/LH3Kus7PvjRqTwww47f+2zMXcxJJ60ceo09BwQxKTR+XOiu/jGLx2s+fPn07FjR4YOHUqdOnX8sUrLPNfVzXNdL1x+8DA82CWIRdPz77h7Vmv/3482+rziZPQQF/Vvz/sfLOc829XFs//V/8efNp7oFczNtVKoWN7L4jgH1a/2UjYisG3MSefXf75Dh2206xLMgunGh2n8JhvTPnTy/oQUgpwwZ7GT+xrl/U52Vrd/5F0e3pvp5IGWbhwOWLbSQatmeX+Y7JmuLp7parzH/9hjo1uvEGrX8lCxvJeP4hxUq+qhbLjx2O0/2rmuhofC+WSYLKv7/ucfp082XLrCwZr19nx7FqH4zi8drGLFijF8+HAWL16c5zpYcmnvfGB8y53wnoMJ7xnLKpaHccPzT+ezWlUvL/ZK5fmXgnB7jIRixOD88W09ux5oZQyXdXwqGLfbmI/V7bH8s+3BfPtH3unhjz/ddHo2mNRUaFDfQ8umeb+Deb5qVbz065VK30HBeDwQEe4lZnB6R2L/ATsVyuWfL1dyIV2mwXc27/8fs8tjzh6uEugmSIB4yF8fYiK+cOftQ3e2OPLbRcYuQ9Hyf/l1fb1/eNhv65pw83y/rSsn5a9TfURERMRyHm8+O3PDD/SKiYiIiFhMCZaIiIiYcqNhWV8pwRIRERGxmBIsERERMaWzCH2nBEtERETEYkqwRERExJTOIvSdXjERERERiynBEhEREVMenUXoMyVYIiIiIhZTgiUiIiKm3DqL0GdKsEREREQspgRLRERETOksQt/pFRMRERGxmDpYIiIiIhbTEKGIiIiY0p/K8Z0SLBERERGLKcESERERU7rQqO+UYImIiIhYTAmWiIiImNIcLN8pwRIRERGxmBIsERERMaULjfpOr5iIiIiIxZRgiYiIiCnNwfKdEiwRERERiynBEhEREVO6DpbvlGCJiIiIWEwJloiIiJjSHCzfKcESERERsZgSLBERETGlBMt3SrBERERELKYOloiIiIjFNEQoIiIipjRE6Ls838EKsQUFugkiIv6nzzuRXC3Pd7BEREQkZynB8p3mYImIiIhYTAmWiIiImNKfyvGdEiwRERERiynBEhEREVOag+U7JVgiIiIiFlOCJSIiIqaUYPlOCZaIiIiIxZRgiYiIiCklWL5TgiUiIiJiMSVYIiIiYkoJlu+UYImIiIhYTAmWiIiImPIqwfKZEiwRERERi6mDJSIiImIxDRGKiIiIKf2xZ98pwRIRERGxmBIsERERMaXLNPhOCZaIiIiIxZRgiYiIiCldpsF3SrBERERELKYES0RERExpDpbvlGCJiIiIWEwJloiIiJjSHCzfKcESERERsZgSLBERETGlOVi+U4IlIiIiYjElWCIiImLK6w10C/IeJVgiIiIiFlOCJSIiIqY8aA6Wr5RgiYiIiFhMHSwRERHJ1yZMmMCkSZPSbicmJtKtWzeioqJ45JFHSEhIACAlJYX+/fsTFRVFdHQ0u3fvBsDr9TJ69GiaNWtG8+bN2bZt2yXXqQ6WiIiImPJ6bX77sdLJkycZNGgQM2bMyLB8/Pjx1K1blxUrVtCuXTtiYmIAmD17NoULF2bFihUMGjSIgQMHArBq1Sp2797N8uXLmTJlCgMHDsTlcpmuWx0sERERyTUSExM5cODABT+JiYk+P9fatWupXLkyXbp0ybB83bp1tGrVCoCWLVuyYcMGUlNTWbduHa1btwbg1ltv5e+//+bQoUOsX7+e5s2bY7fbqVKlChUqVOCHH34wXbcmuV/Eus0w7j1ISYUaVWH4AChWNONjRk+BVeugRKhxu/IVMO5V4/8PPAXJyRAUZNxu2RieeNhfrc++rNR/zpqvYOAI2LoifdnkD2DFF2B3wPXXwLAXICTEP23PruzUHjMBtu5Iv/9oAoSXhrgPcr7dVslO/SkpEDMRvvkeihaGBndCjy5gz0Nf5bLz3k9JNfaBbf/tA3ffDv2eAYfDnxVkT1bq37Ubhk+EU6eM9/iwF+D6GhkfM3Iy/HUA3h3lv7ZbLbvHwfzEnxcanTVrFpMnT75geY8ePejZs6dPz9W2bVuADMODAPHx8YSHhwPgdDopVqwYJ06cyLAcIDw8nCNHjhAfH09ERMQFy82og5WJE//Ay6Ng7hSoXAnGvgtvToWhfTM+7ocf4c2hcPMNGZefOQv7D8GmOAjKg69wVusH2HsA3ngn4zVSvvsBln8Bn0yDkGDo+QrM+SRvdDCzW/vLvdP/f/AwPNoTRg3K8WZbJrv1T50Dh47C0g+MLxdDx8L8WHjkfn9VkD3Zfe/P+8R4jqUzweMxtv/KL6FFYz8VkE1Zqf9sEjzRz+hsRN4BazdC/+GwfHb6Y1Z8Acs+hxuv83sJlsnue0EuX+fOnYmOjr5geWho6EV/Z8WKFYwcOTLDsqpVqzJz5swsr9d+kW+CdrsdbyYb92KPT7s/y2vOpjVr1jB79mz27duXYfnChQv91YQs27QFbqhpvKkAHm4Dn67J+OZJSYFf/oAZC6BtV+g12PhgAdj5CxQpDM8MgNaPG9/kkpL9XsZly0r9YBxoBwyHAd0zLnd7IDnFqDnVZbxWIcH+aXt2Zbf28w1+Azo/BNdWz7n2Wi279f+0C5o3MtJKux3uvRtWrfdP262Q3ff+4+3hrVeN2v9JhJOn0lOuvCAr9W/aAldWNDpXAI3qpyf3ALv3wvQF8Fxnf7U6Z1h5LMgPvF7//YSGhlKpUqULfsw6WFFRUWzYsCHDj1nnKiIigmPHjgHgcrk4deoUYWFhREREpE14B0hISCAiIoKyZctmutyMXzpYY8eOZc6cOezdu5cOHToQFxeXdt+CBQv80QSfHImH8ue9bmXD4dRpG6fPpC+LPw633wx9u8GS6VD7OugxyNg5Tp+F226CCa/B4qlw+Ci89Z7fy7hsWakfjHTioVZGdH6+enXgzjpw70NwTzQknoKHWud8u62Q3drP2fANHEmATg/kXFtzQnbrv/FaI704fcYYVvlsDSQcz/l2WyW7730wUus3p0LTjlC6FNS50b81ZEdW6t+7H8qUgpdHw4PdoOsL4HIb950+AwNGwIiBULSIf9tuNauOBZI7RUZGEhsbC8Dy5cupW7cuQUFBREZGpvVRtm7dSkhICBUqVOCee+5h2bJluN1u/vrrL/bu3UutWrVM1+GXDtb69euZNm0agwcPZt68eUyYMIEVK4yB6sxit0DzeDJffn4aWKk8vDcGqlwJNht07QD7DsHBI8Y3ujGvGGP1ISHw9KPG+HxekZX65y0BpwMeaHHh4z7+DA4cgQ2fGD+VysOYKTnTVqtlt/ZzZi2Gpzrmrbk3kP36n+wI1arAw89B175w0w15a5g8u+/9c154Gr75FCqWg2Fv5WybrZSV+l1u4wvEQ63go/fg0fuNtD4lBV4ZY9y+Jh90Nqw6FuQXefUswovp3bs3//vf/2jRogXz5s1jyJAhAHTq1ImUlBRatGhBTEwMY8aMAaBZs2ZUr16d1q1b89xzzxETE0OhQoVM1+GXQ5/X68VmM160ypUrM3XqVLp06UKpUqXSlucm5cvCjl/Sbx89BiWKeylSOH3Zb7vh1z+gTdP0ZV6v8Wb7chMUKwa31k5fnpc+ZLJSf+xKOJsM0U9AaqoxHBj9BEwdDau/glaN07/BPtQKXp/g3xouV3ZrjyhjzN3Y8QtMGu735mdbduu32YxhshefMx67/AtjOCmvyO57//udUDIMqlxhvOejm8HwPLLvQ9bqjyhtdC5r/ze/6t67YPAY4/e27TASrlmL4d9EOHkaur1odEjzGiuOBZJ7/P/J8WFhYbz77rsXPC4kJITRo0dfsNxmszFgwAAGDBiQ5XX6JcFq1qwZnTp1YscO49Sa6tWrM2HCBJ5//vkL5mTlBvVvhe0/GxMXARYuNVKp89lsMGIiHDhs3J4fCzWuhnIRxtDQG28bbza3G2YugqiGfi0hW7JS/6KpsGymMUQydTQUCjH+H1EGrqsOqzeAy2V88KzekH4wzu2yWzsYH7I31CTDgTivyG79X26CV8f+N1R+BmYtglZN/F7GZcvue/+b72HUZGPf93hg2Wq44xa/lpAtWan/7tvh0BH46Tfj9pbtxmtSq6aRWC+Zbvz07GoMj+bFzhVYcyzIT/JbguUPfslVevToQZ06dShaNP381jp16vDJJ59ccPGv3KB0SYgZCM8PMb6VXFHROBPsx1+NictLphsR+Mu94dmXwOM2xufHGgkj7VsbB98HnjTi9NtvzlsTPrNSv5mnH4VRU6BlZwgOgprVYPDzfml6tmW3djBOTa9YLufbmhOyW//9zWH7L9DqcaOD0a4lNG3gj5ZbI7vv/Sc7wshJ0PYJsNvgllrQp1tga/JFVuoPLw2TYuC1cXAmyXiPT3w971yGJausOBZIwWbz5sZJUD7wHLkm0E0QERHxK3u5XX5d3/Vxr/ptXT+18d+6clIeuvyfiIiISN6Qh6Zei4iISCDk7bGuwFCCJSIiImIxJVgiIiJiKj+d3ecvSrBERERELKYOloiIiIjFNEQoIiIipjRE6DslWCIiIiIWU4IlIiIipnSVBt8pwRIRERGxmBIsERERMaU5WL5TgiUiIiJiMSVYIiIiYk6TsHymBEtERETEYkqwRERExJTmYPlOCZaIiIiIxZRgiYiIiCmv5mD5TAmWiIiIiMWUYImIiIgpzcHynRIsEREREYspwRIRERFzSrB8pgRLRERExGLqYImIiIhYTEOEIiIiYkqXafCdEiwRERERiynBEhEREXNKsHymBEtERETEYkqwRERExJQuNOo7JVgiIiIiFlOCJSIiIuY0B8tnSrBERERELKYES0RERExpDpbvlGCJiIiIWEwJloiIiJjTHCyfKcESERERsZgSLBEREbkEzcHylRIsEREREYspwRIRERFzmoPlMyVYIiIiIhZTB0tERETEYhoiFBEREXMaIvSZEiwRERERiynBEhEREXP6Uzk+U4IlIiIiYjElWCIiImLKqzlYPlOCJSIiImIxJVgiIiJiTgmWz5RgiYiIiFhMCZaIiIiY01mEPlOCJSIiImIxJVgiIiJiyqY5WD5TgiUiIiJiMSVYIiIiYk4Jls+UYImIiIhYTAmWiIiImNNZhD5TgiUiIiJiMXWwRERERCymIUIRERExp0nuPlOCJSIiImIxJVgiIiJiTgmWz5RgiYiIiFhMCZaIiIiYU4LlMyVYIiIiIhZTgiUiIiLmdKFRnynBEhEREbGYEiwRERExZdMcLJ8pwRIRERGxmBIsERERMacEy2dKsEREREQspgTrItZthnHvQUoq1KgKwwdAsaIZH7N6A0z+AOx2CC0Gr78IV1bM+Jier0BEGRj8vN+abolL1R+7EmYtTr998hQcTYAvP4IypdKX58X6C3LtkL36nU4Y9hb8+gcULgT3R8GjD/i/BqtYdRzIq1R/wa5fskcJViZO/AMvj4IJr8OKOVCpArw5NeNjkpJhQAxMfB2WTIeG9SFmYsbHTJsH23b4rdmWyUr9bZsZdS+ZDoumGh2LV57P2MHIi/UX5Noh+/WPmgxFCsOns2DBO7DhW/jy60BUkn1WHQfyKtVfsOuX7PNbB2vv3r0cPXoUgMWLFzN8+HCWL1/ur9X7ZNMWuKEmVK5k3H64DXy6BrznjUG73cbtU6eN22fOQkhw+v3ffg8bv4P2bfzXbqtkpf7zTZsHpUtC+9bpy/Jq/QW5dsh+/T/tgjb3gcMBwUEQWQ8+X++ftlvNiuNAXqb6C3b9/5/N67+f/OKSQ4QzZsxg8uTJuN1uKlSoQI0aNdJ+rrnmGipVqnTJlcycOZPZs2fj8Xi44447OHz4ME2aNOHjjz9mz549dO/e3ZJirHIkHspHpN8uGw6nTts4fcabFg8XLQJD+8LD3SEsFDwemDvZuC/+GIyYBO+PhUVL/d/+7MpK/ef8/Q/MXAgfT0tflpfrL8i1Q/brv/FaiPscbq4FKSmwer0xbJgXZfc4kNep/oJdv2TfJQ99U6dOZcyYMdx4443s37+fXbt28dtvv7FhwwZ+//13AKpXr878+fMv+hwff/wxy5cv59ixY7Rs2ZJvvvmGkJAQ2rVrx4MPPpjrOlgeT+bL7eflfbt2wzsfGkMhV1aE2R9B7yGw+D3oOwxe6gkRpf3TXqtlpf5zFi2DRndBpfLG7VRX3q6/INcO2asfYMBzMOYduP9JCC8Fd9aFH37MmbbmtOwcB5ZMB1sev/C16s98eUGp/wK6krvPLtnBKlasGA0aNMDpdBIREUGdOnUy3H/gwIG0jtbFeDwegoODqVixIl27diUkJCTtPrfbfZlNzznly8KOX9JvHz0GJYp7KVI4fdnGLXDzDemTGTtGw6gpsP0nOHgYRk8xlh87YcTIySkw/EX/1ZAdWan/nBVfwqBe6bd//DVv11+Qa4fs1Q9w6gz0e8b4Ng/w/jy48tIhd66UnePAP/9CyTC/Ntdyqr9g1y/Zd8k5WN26dWPx4sUXvb9SpUo0bNjQ9Dnuu+8+Hn30UdxuNz179gTg119/pWPHjkRFRfnY5JxX/1bY/jPsPWDcXrgUGtXP+JjrroEt240PUYC1G41v8nVrG2dTnZsE3L41RDXKOx+wkLX6Af49CfsOGgeYc26+IW/XX5Brh+zVD7AwDibNMP5/7AR89Cm0vDdn25xTsnMcyA8frqq/YNcv2XfJBGvUqFGkpqby1Vdfcffdd3PttddSo0YNChfO5CvtRfTu3ZstW7bgcDjSlgUHB9OzZ08iIyMvr+U5qHRJiBkIzw+B1FS4oiKMGmQkFIPfMD4877gFunaAzr0hKAhKFIfJMYFuuTWyUj/AvgMQXhqC8ugcm8wU5Noh+/V3e9Q4q6rV48bk3+6PQ61r/V2FNXQcUP0Fuf4L5KPJ5/5i83ovdn6QYf/+/fz666/89ttv/Pbbb/z6668cOnSISpUqsWrVKn+186I8R64JdBNERET8yl5ul1/XV3X8W35b15/P9/XbunLSJb9/X3HFFVxxxRU0adIkbdmZM2fYtcu/G1dEREQCRAmWzy7rOlhFihThpptusrgpIiIiIvlDPptBIiIiIlbLTxcA9Rf9qRwRERERiynBEhEREXNKsHymDpaIiIjkaxMmTMBut6ddi3PLli306NGDcuXKAXDdddcxcuRIEhMT6devH/v376dUqVKMHz+e8PBwUlJSePnll/nxxx8pVKgQY8eO5eqrrzZdp4YIRURExJzXjz8WOnnyJIMGDWLGjBkZlu/cuZOuXbsSFxdHXFwcI0eOBGD8+PHUrVuXFStW0K5dO2JijAubzZ49m8KFC7NixQoGDRrEwIEDL7ludbBEREQk10hMTOTAgQMX/CQmJvr8XGvXrqVy5cp06dIlw/KdO3eyadMm2rZtyzPPPMPhw4cBWLduHa1atQKgZcuWbNiwgdTUVNatW0fr1q0BuPXWW/n77785dOiQ6bo1RCgiIiKm/HkW4axZs5g8efIFy3v06JE2xJdVbdu2BWDSpEkZlhcvXpwWLVrQuHFj5s+fT58+fViwYAHx8fGEh4cD4HQ6KVasGCdOnMiwHCA8PJwjR45QoUKFi65bHSwRERHJNTp37kx0dPQFy0NDQy/6OytWrEgb5junatWqzJw5M9PHv/baa2n/f/jhh3nzzTc5efJkpo+12zMf7LvY8nPUwRIRERFzXpvfVhUaGmramcpMVFQUUVFRWXqsx+Nh6tSpdOvWLcPfSHY6nURERHDs2DHKlSuHy+Xi1KlThIWFERERQUJCAldddRUACQkJREREmK5Hc7BERESkwLDb7axevTrt7ynHxsZSu3ZtChcuTGRkJLGxsQAsX76cunXrEhQURGRkJHFxcQBs3bqVkJAQ0+FBUIIlIiIil5LProM1evRoBg8ezJQpUyhVqhRjxowBoHfv3gwcOJAWLVpQvHhxxo4dC0CnTp0YMmQILVq0IDg4OO3xZmxerzdPv2yeI9cEugkiIiJ+ZS+3y6/rqzZ6nN/W9ceAPn5bV07SEKGIiIiIxTREKCIiIqb0x559pwRLRERExGJKsERERMScEiyfKcESERERsZgSLBERETGlOVi+U4IlIiIiYjElWCIiImJOCZbPlGCJiIiIWEwJloiIiJhTguUzJVgiIiIiFlOCJSIiIqZ0FqHvlGCJiIiIWEwdLBERERGLqYMlIiIiYjHNwRIRERFzmoPlMyVYIiIiIhZTB0tERETEYhoiFBEREVO6TIPvlGCJiIiIWEwJloiIiJhTguUzJVgiIiIiFlOCJSIiIuaUYPlMCZaIiIiIxZRgiYiIiCmdReg7JVgiIiIiFlOCJSIiIuaUYPlMCZaIiIiIxZRgiYiIiCnNwfKdEiwRERERiynBEhEREXNKsHymBEtERETEYkqwRERExJwSLJ8pwRIRERGxmDpYIiIiIhbTEKGIiIiY0mUafKcES0RERMRiSrBERETEnBIsnynBEhEREbGYEiwRERExpwTLZ0qwRERERCymBEtERERM6SxC3ynBEhEREbGYEiwRERExpwTLZ0qwRERERCymBEtERERMaQ6W75RgiYiIiFhMCZaIiIiYU4LlMyVYIiIiIhZTgiUiIiLmlGD5TAmWiIiIiMXUwRIRERGxmIYIRURExJQt0A3Ig5RgiYiIiFhMCZaIiIiY0yR3n6mDdRHrNsO49yAlFWpUheEDoFjR9PtjV8Ksxem3T56Cownw5UdQplT68p6vQEQZGPy835puiezWf2drKBuefn/XDtCqif/anx3a9gV324M12/9wPHR4FmKnQ8kwvzY/2wpy/ZeqHWD0FFi1DkqEGrcrXwHjXjX+//l6mDrH+P0KZWHUIChZwp8VSG5i83q9ebpf6jlyjeXPeeIfaNUZ5k6BypVg7Ltw+gwM7Zv541Nd0KknREdB+9bpy6fNgxkLIKpR3vqQzW79e/bBsy/Byrl+bbYltO0L7rYHa7Z/7EqY9AEcOmLj6zhvnupgFOT6s1p7h2dhQHe4+YaMy3/81dj3F7wNFcvDyMmQnAyvvpAz7bWX25UzT3wRtZ8f57d1bR/fx2/rykkBmYM1atSoQKw2yzZtgRtqGm8ygIfbwKdr4GJd0WnzoHTJjB+w334PG7+D9m1yvr1Wy279P/wIDjt07g1tusCUmeB2+6Xp2aZtX3C3PWS//vhjsHYjTB3tn/ZarSDXn5XaU1Lglz+ML09tu0KvwXDoqHHf0tXwQAujcwXQ43F44mG/liC5TI4PEb700ksXLPviiy/4999/ARg5cmRON8FnR+KhfET67bLhcOq0jdNnvBfExX//AzMXwsfT0pfFH4MRk+D9sbBoqV+abKns1u9yw511of+zkJQMzww0YvbO7fzS/GzRti+42x6yX39EGZg03C9NzREFuf6s1B5/HG6/Gfp2M4YGZyyAHoOM12DvfqhxNXQfBAePwDVVYWCPwNSSI/L0WFdg5HiCFRYWxrp166hZsya33XYbt912G0WKFEn7f27k8WS+3J7Jq7VoGTS6Cyr9960l1QV9h8FLPSGidM61MSdlp36Ah1rBy70hOBhCi8PjD8Gar3KmrVbTts98eUHY9pD9+vO6glx/VmqvVB7eGwNVrgSbzZhfuO+Q0aFyueDLr40hwU+mGfPRhrzhn7ZL7pTjHawBAwbw1ltvsXz5cipUqEB0dDQlSpQgOjqa6OjonF79ZSlfFhKOp98+egxKFPdSpPCFj13xpTH/4Jwff4WDh42JkNFPwMKlsOILeGVMzrfbKtmpHyBuFfy2O/221wvOPHI6hbZ9wd32kP3687qCXH9Wav9tt7GPn8/rBafDSO/uuhXCSxudsugo+N9P/mm7X3j9+JNP+GUOVr169Zg6dSrz5s1j9OjRuHP5pIz6t8L2n2HvAeP2wqXQqP6Fj/v3JOw7mHGy4803GGfTLJlu/LRvbUx0Hv6if9puhezUD/D7Hpg0w5h7k5QMc5dAVMOcb7cVtO0L7raH7Nef1xXk+rNSu80GIybCgcPG7fmxxrBguQhoGgnrv4G/jdkvrN5gzOmSgstvk9zDwsKYMGECVatWJTw8/NK/EEClS0LMQHh+CLToBLv+hBe7GwlF9BPpj9t3wPi2EpSHvqFnRXbr7/44lChuTHJu0wVuvh7atfRrCZdN277gbnvQ9i/I9Wel9muqGkPgz75kPGbNVzB2iHFfw/rw2IPwWG9o+Rj870d4rV/g6rGazeu/n/xCl2kQERHJY/x9mYabevrvMg3/m5Q/LtOQj75/iIiISI7I01FMYOhvEYqIiIhYTAmWiIiImMpPc6P8RQmWiIiIiMXUwRIRERGxmIYIRURExJyGCH2mBEtERETEYkqwRERExJQmuftOCZaIiIiIxZRgiYiIiDklWD5TgiUiIiJiMSVYIiIiYk4Jls+UYImIiIhYTAmWiIiImNJZhL5TgiUiIiJiMSVYIiIiYk4Jls+UYImIiIhYTAmWiIiImLJ5FWH5SgmWiIiIiMWUYImIiIg5BVg+U4IlIiIiYjF1sERERCRf2rZtGw888ABt2rShc+fOHDx4EIDExES6detGVFQUjzzyCAkJCQCkpKTQv39/oqKiiI6OZvfu3QB4vV5Gjx5Ns2bNaN68Odu2bbvkutXBEhEREVM2r/9+rNS/f39iYmKIi4ujVatWDB8+HIDx48dTt25dVqxYQbt27YiJiQFg9uzZFC5cmBUrVjBo0CAGDhwIwKpVq9i9ezfLly9nypQpDBw4EJfLZbpudbBEREQk10hMTOTAgQMX/CQmJvr0PCkpKfTu3ZuaNWsCUKNGDQ4fPgzAunXraNWqFQAtW7Zkw4YNpKamsm7dOlq3bg3Arbfeyt9//82hQ4dYv349zZs3x263U6VKFSpUqMAPP/xgun5NchcRERFzfpzkPmvWLCZPnnzB8h49etCzZ88sP09wcDBt2rQBwOPxMHnyZBo3bgxAfHw84eHhADidTooVK8aJEycyLAcIDw/nyJEjxMfHExERccFyM+pgiYiISK7RuXNnoqOjL1geGhp60d9ZsWIFI0eOzLCsatWqzJw5k5SUlLQhvaeffvqiz2G3Zz6oZ7fb8WZyHbCLPf4cdbBERETElD//2HNoaKhpZyozUVFRREVFXbD89OnTPPvss4SFhfHOO+8QFBQEQEREBMeOHaNcuXK4XC5OnTpFWFgYERERJCQkcNVVVwGQkJBAREQEZcuWTZsIf/5yM5qDJSIiIvlS//79ueqqq5gwYQLBwcFpyyMjI4mNjQVg+fLl1K1bl6CgICIjI4mLiwNg69athISEUKFCBe655x6WLVuG2+3mr7/+Yu/evdSqVct03UqwRERExFwevNDozz//zNq1a6lWrRpt27YFjOTq/fffp3fv3gwcOJAWLVpQvHhxxo4dC0CnTp0YMmQILVq0IDg4mDFjxgDQrFkzduzYkTYBPiYmhkKFCpmu3+bNbGAxD/EcuSbQTRAREfEre7ldfl3fbZ3f8tu6vpvV12/ryklKsERERMSUP+dg5ReagyUiIiJiMSVYIiIiYk4Jls+UYImIiIhYTAmWiIiImNIcLN8pwRIRERGxmBIsERERMZe3r+gUEEqwRERERCymDpaIiIiIxTREKCIiIqY0yd13SrBERERELKYES0RERMwpwfKZEiwRERERiynBEhEREVM2T6BbkPcowRIRERGxmBIsERERMac5WD5TgiUiIiJiMSVYIiIiYkrXwfKdEiwRERERiynBEhEREXP6Y88+U4IlIiIiYjElWCIiImJKc7B8pwRLRERExGJKsERERMScEiyfKcESERERsZg6WCIiIiIW0xChiIiImNIkd98pwRIRERGxmBIsERERMacLjfpMCZaIiIiIxZRgiYiIiCnNwfKdEiwRERERiynBEhEREXNKsHymBEtERETEYkqwRERExJTmYPlOCZaIiIiIxZRgiYiIiDmPIixfKcESERERsZgSrPOs2wzj3oOUVKhRFYYPgGJFfX/c4Xjo8CzEToeSYRl/98BhePApmDYWbqiZo+VYJiuvy+gpsGodlAg1ble+Asa96u+WWiMr9c79BObHgc0GV1aA1/pD6ZLGffOWwEefQXIyXF8Dhr8IwcH+r+NyZbf+O1tD2fD0x3btAK2a+K/92ZWV+pd+DjMWGPUXCoGXe6W/nz9fD1PnGL9foSyMGgQlS/i/jsuV1eMgwJqvYOAI2LrCuB0zAbbuSL//aAKEl4a4D3K+3TnFbFufb+N3MH4auN3GY/t2g7tu8397c4wCLJ/ZvN68ff17z5FrLHmeE/9Aq84wdwpUrgRj34XTZ2BoX98eF7sSJn0Ah47Y+DrOm6GDlZwMXfrCb7th1vi80cHK6uvS4VkY0B1uviEgzbRMVur96TfoNcToQBcvBmPeNh4zrB98vgEmvG/8fmgxeH4o1KoJTz0SsJJ8kt369+yDZ1+ClXMDVkK2ZKX+Pfvgsd7w8TSIKA3rv4Fhb8IXi+HHX436F7wNFcvDyMnG+/7VFwJWkk+y+n4H2HsAnn4Rjp2AbSsvvP/gYXi0J7w9Eq6tnuNNzxFm2/p8J09Bk4dh9kSoXsU4xnfqBV8uhqJFcqZt9nK7cuaJLyKy+Ri/rWv98hf9tq6c5Jchwh070r/SbN68mVGjRjF27Fi2b9/uj9VnyaYtRoenciXj9sNt4NM1F/75JbPHxR+DtRth6ujM1/HaeGjbDMLy0LfZrLwuKSnwyx/Gt7y2XaHXYDh0NDDtza6s1Ht9DaMDUbyY8eF5NAHC/kvulq6Cx9sbt+1244O19X3+r+NyZbf+H34Ehx0694Y2XWDKTOMbfV6RlfqDg+D1F40PXIAbahidjJRUWLoaHmhhdK4AejwOTzzs1xKyJavHwbNJMGC48aXqYga/AZ0fyrudKzDf1udzuWBIH6NzBXD1VcZr9ve//m1vTrJ5/feTX/ilgzV06FAA5s6dy4gRIyhXrhxlypRhyJAhzJkzxx9NuKQj8VA+Iv122XA4ddrG6TNZf1xEGZg0HKpVvvD5F39qvAkfapUjzc8xWXld4o/D7TcbkfiS6VD7OugxKG/+bdCs7gdBTmN4pEE7Y0gkurmxfO9+OPE3PNXf6GBM/sDoiOQV2a3f5YY768L7bxjf5jdtgTmf+K/92ZWV+iuWhwb1jP97vcbweMP6xofx3v1Gh7L7IOPLxuvjcy7ByAlZ3f5DxxrHshpVM3+eDd/AkQTo9EDOtdUfzLb1+UqGQfNG6bcnzTA6qZXK+62pkgv5dZL7okWL+PDDD3n88cd5/PHHmTt3bq7pYHk8mS+32y/vcef7aRcsXJp3hgnOl5V6K5WH98ZAlSuNuQddO8C+Q3DwiH/aaCVftm/ju2HzUuj+ODzVz/jdVBd8vdWYf7b4Pfg30ZiXkVdkt/6HWsHLvY05Z6HF4fGHjI5YXuFL/WfOQp+h8NdBeL2/sczlgi+/Nt7rn0yDMqVgyBs5116rZaX+eUvA6TCSuouZtRie6ggOh7XtC5TMtnVmXC5jHtrKdTDhdb81T3Ipv3SwXC4XHo+H0qVLU6RI+te54OBg7GY9kxw2cTpEP2H8fPQZJBxPv+/oMShR3EuRwhl/p3zZrD3ufHGr4NRp6NjdWFfCMeg/HL7YZG09OSEr9f6226jxfF6vcRDOa7JS718HYNt5E3kfaG4Mif570kgxG99tTAoODoJW98H2n/zX/uzKbv1xq4z94RyvF5x56FSarL6/Dx013s92hzGfMrS4sTyiDNx1qzGx226H6Cj4Xz7b/rErYedvxrHs6QGQlGz8P/6Ycf+Jf2DHL9C0gT9bbp3zPxcmTr/4tv7//j1pJNe/74EF7xgnOOQrXq//fvIJv/RuSpYsSWRkJH/88UfacOHmzZvp0KEDzZo180cTMtXrCWNIa8l04w2x/Wdj4iYYiVOj+hf+Tv1bs/a48w3qacxZObeu8DLwxiuX/r3cICv12mwwYqJxhiTA/FiocTWUiyDPyUq9Ccfhhdfg73+M28tWG3MvSpaAppHG2ZRJycZxYu1XeeNkhnOyW//ve4zhEbfbeA3mLoGohn4tIVuyUv8/ifBYL2hyD7w11Diz7JymkcZE6HNzb1ZvyH/bf9FUWDbTOJZNHW3Uv2S60bkE+H6nUbPZl87c7PzPhcfaXXxbny8lBZ58wRhSnDY2b501KjnHr2cR/vnnnyQmJnLTTTexbds2Tp48SYMGDbL1nFadRQjGgXHce5CaCldUNE6vDgs1zgwa/IbxhjN73PmujbzwLMJz7m0PE4blnQNvZvUeOJTxNVn6Obw/DzxuY97G8AF59xtcVuqdHwvzYo2ULry0McG1UnmjY/HubFjxBbg9cF114+y6i53mnhtlp/6zSTB8vPEhneqCZg3g+aeMTnhecan6351tdCKv+X/zj2a8ZXywnnttvB7jPTB8QHrnIy/IyvY/5+BhaN0l41mE0+fDnv3G5UnyOrNtnZpqJHhTR8O3P8CLw21Ur+LNMCw6ehBcc3XOtM3fZxE2bHqRs7dywJerBvhtXTlJl2kQERHJY9TByv3y0OwIERERCYg8HcUEhv5UjoiIiIjFlGCJiIiIKVvenk0UEEqwRERERCymBEtERETMXeQitHJxSrBERERELKYES0RERExpDpbvlGCJiIiIWEwJloiIiJhTgOUzJVgiIiIiFlOCJSIiIuY0B8tnSrBERERELKYES0REREzZFGD5TAmWiIiIiMXUwRIRERGxmIYIRURExJwmuftMCZaIiIiIxZRgiYiIiCmb/tizz5RgiYiIiFhMCZaIiIiY0xwsnynBEhEREbGYEiwRERExpwDLZ0qwRERERCymBEtERERM2TQHy2dKsEREREQspgRLREREzCnB8pkSLBERERGLKcESERERc7qSu8+UYImIiIhYTAmWiIiImNJZhL5TgiUiIiJiMXWwRERERCymIUIRERExpyFCnynBEhEREbGYEizJs35KPRPoJgRUm0/6BroJAfXLQ5MD3YSAalmxTqCbEDD/dqoX6CYE3Hez/LxCJVg+U4IlIiIiYjElWCIiImJOFxr1mRIsEREREYspwRIRERFTutCo79TBEhERkXxp27ZtjBgxApfLRVhYGCNGjKBixYps2bKFHj16UK5cOQCuu+46Ro4cSWJiIv369WP//v2UKlWK8ePHEx4eTkpKCi+//DI//vgjhQoVYuzYsVx99dWm69YQoYiIiJjzev33Y6H+/fsTExNDXFwcrVq1Yvjw4QDs3LmTrl27EhcXR1xcHCNHjgRg/Pjx1K1blxUrVtCuXTtiYmIAmD17NoULF2bFihUMGjSIgQMHXnLd6mCJiIhIrpGYmMiBAwcu+ElMTPTpeVJSUujduzc1a9YEoEaNGhw+fBgwOlibNm2ibdu2PPPMM2nL161bR6tWrQBo2bIlGzZsIDU1lXXr1tG6dWsAbr31Vv7++28OHTpkun4NEYqIiIg5P87BmjVrFpMnX3idux49etCzZ88sP09wcDBt2rQBwOPxMHnyZBo3bgxA8eLFadGiBY0bN2b+/Pn06dOHBQsWEB8fT3h4OABOp5NixYpx4sSJDMsBwsPDOXLkCBUqVLjo+tXBEhERkVyjc+fOREdHX7A8NDT0or+zYsWKtGG+c6pWrcrMmTNJSUlh4MCBuFwunn76aQBee+21tMc9/PDDvPnmm5w8eTLT57bbMx/su9jyc9TBEhEREXN+TLBCQ0NNO1OZiYqKIioq6oLlp0+f5tlnnyUsLIx33nmHoKAgPB4PU6dOpVu3bjgcjrTHOp1OIiIiOHbsGOXKlcPlcnHq1CnCwsKIiIggISGBq666CoCEhAQiIiJM26Q5WCIiIpIv9e/fn6uuuooJEyYQHBwMGMnT6tWrWbVqFQCxsbHUrl2bwoULExkZSWxsLADLly+nbt26BAUFERkZSVxcHABbt24lJCTEdHgQlGCJiIjIpeTBK7n//PPPrF27lmrVqtG2bVsAIiIieP/99xk9ejSDBw9mypQplCpVijFjxgDQu3dvBg4cSIsWLShevDhjx44FoFOnTgwZMoQWLVoQHByc9ngz6mCJiIhIvnPdddfx22+/ZXpf9erVWbBgwQXLw8LCePfddy9YHhISwujRo31av4YIRURERCymBEtERERM6U/l+E4JloiIiIjFlGCJiIiIOSVYPlOCJSIiImIxJVgiIiJizqMEy1dKsEREREQspgRLREREzGkOls+UYImIiIhYTAmWiIiImFOC5TMlWCIiIiIWU4IlIiIi5pRg+UwJloiIiIjFlGCJiIiIOV0Hy2dKsEREREQspgRLREREzHk9gW5BnqMES0RERMRi6mCJiIiIWExDhCIiImJOl2nwmRIsEREREYspwRIRERFzukyDz9TBMrFuM4x7D1JSoUZVGD4AihXN/LFeLwwaBdWrQNcO6cvvbA1lw9Nvd+0ArZrkbLsvV1brvdjjkpLh9XGw81fjvVj7WhjcBwqFwK9/wGvj4ORpKFYEej8Jd9zi/xozs2GNg7hFQdhsEBwCXbunUK3G5Z8xk5wE77wZwp7ddrweePSpFG6r7wbgf1sczPsgCI8bbDZ45IlUbrrVbVUpWTamSVN2HT/GtO+3mT6uSdWrGXtfM2q/OyVb6yvkdDKq8X1cFx6O3WZj9MavWP3nbgCaV7+GHrfdDsCJs2d55Ys17P3nn2ytLyfM/8TOoqVG6H9FRS9D+7kpXdLY70eMd/Djrza8Xqh1rZdBz7spFBLgBv8/9z5yN+36tQYvJJ1J5u3eM9i17c8LHndPu3p0GtIOt8vNsQMnmNj9feL3Hbvs9YYUDqbv+89S7ebK2Ox2pg2cw9dxW9LW9cjLDwDw77GTTHhmKgf/OHLZ6/JVszuvpVNUXbxeL0kpLt6c8yW/7D162c8XEuzkla73cc1Vxn4+edFXrP/e2M8b33YNXVob+/m/J88ycuYa9h/9x4oyJJfSEOFFnPgHXh4FE16HFXOgUgV4c2rmj929F7r0gZVfZly+Zx+EFocl09N/cmvnKqv1mj3u3dngckPsDIibYXzwvDfHuK/7y/BgC1g2Eya+DsPegoTjfirOxMH9Nj58L5hXRiYzdmoSDz6SythXs/fJuOjDIAoV9jJhxlkGj07i/YnBHE+wcfoUjB8RQs8XjXV1fzGZt4aHcPaMRcVkwdUlSzHn/gdpUf2aSz62clgYL90did1my/Z6e99ej9MpKdw3exaPLfmY1xreS7lixShTpAjDGzXmibglNJ87m1V//MGrDRple31W+/k3mLXQzoeTXSyZ6eLKil6mzDAOn+/PtuNyw0fTXXw03UVSMkyfm7sOrZWuqcBTYzoxKCqGZ27pz7yYjxn6cf8LHlexWjmef7cbMQ+P55mb+7NwTCxDFr+QrXV3evUhzp5O4onr+zDgvtfpNeVJylQsRVhECXq/041XWo7k6Zv6sWnJt/SY9ES21uWLK8uVpFf7u+k19hMeHTKHGUu/ZXSvVtl6zm5t63EmOYX2L82ix5iPefGxe4koWYxSoUUY0Lkxfd5awiOvzObLbX/Qv1Pu289Neb3++8kn/HYU+Oqrr0hMTAQgNjaW1157jY8//thfq/fZpi1wQ02oXMm4/XAb+HRN5tt+XixER0GzhhmX//AjOOzQuTe06QJTZoLb/2FFlmS1XrPH3VobnnkM7HZwOODa6nDoKPz9DxyJhzZNjd8JL20kXxu/81t5FxUUBM/2TaFkaaPQq69x88/fNlJTjfs/nhtE/2cK0e/pQoweEsKJYxk7GwtnBbFwVlCGZd9uctK4uQuA8LJeatdx8/V6B243PNUrmSsqG+u64ioveCHx3+x3YLKqU+2b+Ojnn/js912mjyvkdPJW0yhiNqy74L7nbr2NpQ8/wqcdH+Xdlq2JKJox5ux9ez16314vw7Km1aqx8KedABw6eZKv9v1Fi+rXcOzMGW57/10OnzqFw2ajYmgo/yQlZa/IHHBdDVg210XxYpCcDPHHbJQINe6rU9tLt07utP2+ZnUvh476b5tmRWpyKm899S4njvwDwK6tuylZLgxnUMZBjKq1K/Pn9r/Y++M+AHZ+9QtlK0dQ9iojhu846H7e3jqad79/g1c/6U/p8iUz/H6noe3oNLRdhmV3tb2N5e+vASBh/zG2fb6DyIfu5J/4f3mo3JMkHDiO3WEn4qpwEo+fyonyM5XqchMzYzXH/z0NwC97jlC6RFGcDuNjsUur2/hw2CPMee1R3ujVmjJhGffzp9rW46m2GffzyDrViF1n7OdHT5zk2x//ovFt13Ai8QzNer1L/IlTOOw2ypUO5d9TuW8/F2v5pYMVExPD1KlTSU5OZvz48Sxbtoxq1aqxevVqhg8f7o8m+OxIPJSPSL9dNhxOnbZxOpO0YfDz6Z2H87nccGddeP8NmD3R6JzM+STHmpwtWa3X7HH1b4UqVxjLDx6BDz+Cpg2gZBhUKg+xK4379h+CbTtzR4IVUc5LnTuMXq/XCzPfDaFuPTdBQbDucyd/7bEzakoSY6cmccttbt55K/iSz3k83kbpiPSeaekyXo4n2AktAfUbpvewF8wMonwlD2XL++8b26vrviD2118u+biYRo2Zv3MHvx7LODQUXfNaapQuQ/SCebScN4d1e/cwqvF9l3y+8sWKc+jkybTbR06dpFyx4gC4PB5qRZTl6ye68fANtZj1vx98rMo/gpzwxVc2mrRz8v0OG22jjGHkO2/1Uvm//f7QEZj7kZ37InPXRRmP/pXAd8u/T7v99Jud2bx0K65UV4bH/fHDHirfcAVX164MwB0t6xBauhilypekcad7qHzDlfS4/SWeuaU/3634gb7vP3PJdYdfUZqE/elv9oSDxylTqTQAbpeba+pUZf7+qbR4qjGxk1dYUG3WHD6WyKbte9JuP9+xARt+2I3L7aF5/Wu5ulIZugybx6ND5rBpxx5e7nrp/bxsqeLEn0jfz+P/PklEKWM/d7s9XFu5LJ+O60Z0g1osWp079/OLUoLlM7/Mwdq0aRPLli3D4XCwbt06Fi1aRHBwMO3bt6dly5b+aILPPBc5Ptp96JI+dF7aHBwMjz8Esz+Gzu0u/juBktV6s/K4n36Dnq/AI9HQ8E5j2ZQRMOZtmLUYalaDe+4w0qPcIuksTH4jhOPxNl4ZZXyz3PaNgz9+szPguUIAeNw2kpONx48ZGkL8ERv/nDCSii1fO4go5+XFYcmZHh/s9vSFbjfMfCeYH7Y4GDom932LffTG2ri9Hhb//BMVi4dmuK9RlarULleOuIcfAcBhs1PIaRxG3mnRmitKhBJexPim3+Tqq9n/byLPfrYUWybDjO7zXqid8Ue5fdpU7rmqMtPbRBP5wXROpiTnVImXrdHdXhrd7eKjT20809/JZ3Ndafv+z7/B84OddIj2EHln7vyQKFQkhP4fdCf8itK8FBVzwf2H/zzKm0+8Q+93niIoJIivl27hz+1/4UpxcUeLOtS4rRpvbxkNgN1hJ6SI8YVj6Mf9KV8lgpLlwgCo3+Y2Du+JZ9gDb2DL5KDpcacfSHZt+5P2FZ6ibtObGP7pSzx2dXdO/+u/cfNCwU6GPtWMiFLF6f2m8Q34rpuqcl2Vcsx61djP7XY7hUKM/XxMr9ZUKBNK6RLGfh55y9UcOpbIixOXYrdnsp+fNzn8l71Hieo9lTtqVeatvtFE95/OqTO5bz8Xa/ilg1WoUCGOHz9OREQEpUuX5syZMwQHB3P27Fmcztwzz37idPjya+P/p07DNVXT7zt6DEoU91KkcNafL26V0ZmocbVx2+uFXFTuZdVbvizs+OXij/tsrTHR/ZXe0PK8+WYeD7w9Ir3+bv2hUX3ra8qKBTOD2LrZAUDdem7ujXIxanAIFa/08uqbSYSEpLe5bftUmrY2vuWnpsCpU8YB9MVhxkHx3PBg+86pac9fJsLL38dtlCxlHFhPHLdR+WrjA+XUSRj7WiHwwsiJZyleImdrff6OO2lc1diwa/78k/HffH3J33ng2usp5HTyacdHCbI70v7fNW4JDrudqVu3MHfnDgCCHQ5C/3vBnv1sKUDa8OCEbzenPeehkyeJKFqUY2eMD85yxYrxc0ICEUWLUqN0Gb7a9xcAG/7ay6mUZK4KK8GP8fEWvQqXZ8oMO+s2GZ2DqpW9tG/j4ZYbjW0aHeVl+FuQeBLCSsCKtTZixjt4qbebFo1zR+eq87D21GtVF4DNy7ayfNpaXl86gH2/HKRfo2GkJKVc8DtBwU4O/nGEXne+DBidqPt7t+DInnjsDjsLx8Tx6bufpz22WMliAAx74A2AtOHB2cMWpz1n/L5jlCofxt//TeguU6EUu7fvpXT5klSpdSVbP98OwNZV/+NM4hkqXF2O37+/cPK9FbpF38k9Nxvvhw0//Enc+p281actew4d57lRi0n+L9Gz2+zMXr6Fj78w9vMgp4PQosZ+/uJEYz8/Nzz4fmz6fn7k+ElKlyjK8f86iOEli7HrrwTKhBWlWqUyfPOjsZ9/s3Mvp88mUymiBL/uDex+nmX5KFnyF78MEfbo0YMHH3yQ0aNHU7VqVTp16sSIESN46KGH6NKliz+akCW9nkifjL7gHdj+M+w9YNy3cKnvHYLf98CkGUZikZQMc5dAVMNL/56/XE699W+9+ONWrYMRE2Ha2IydK4ChY2HtRuP/P/xovDb16uRIWZfU4fFUxk41hv1a3J/K0BcKcftdbvq+kpzWuQK4qa6btSucnDGmaLBgVhCTRl16Avytd7pZ85nRkzyeYOOHLQ7q3OEmNQVeH1CIsuU8DB6dlOOdK4Dx33xNy3lzaDlvTpY6VwDRC+cRNfdDWs6bQ9e4JSS5XLScN4f406fZ8NdeHrq+FsWCjeSizx138lbTqEs+55o//+DhG24EjM7VPVdV5os9fxLicDKpeQuuKhEGwB2VrsBhs/PHiROXV7CFunf1sHi6i8XTXTzUxsOLrzn4+x/jvs/W2KhWxehcfb7OxqhJDqa+kXs6VwCzhi7kmVv688wt/flk/Ge8uW4YG5d8y4iO4zPtXAEEhQQxfuPrhP83hPdAn5b8uPFXTv59iq2fbyfqiXspUtz4NtX5tfYM+LDnJduxeekWWnQzDghlKpaibrOb+ObTbQQVCuLlBX2ocHU5AGo3uB6H08G+Xw5YUX6m3lvyNY8OmcOjQ+aw4PPvmTroIb7c9juvvLM8rXMF8M2Pe2kdWYuihYz9/On77+TVbpfezzd8/wfRDY39PKJkMerVqszG7X8SHOQk5rkWVIoIA6BOzStwOOzsORT4/Vxyjs3r9U+3dP/+/axZs4a//voLt9tNmTJlaNiwITfeeGO2ntdz5NJnQ12u9d8YlyNITYUrKsKoQRAWCj/+CoPfMDom53tpZMbLNJxNguHjjQ5JqguaNYDnnzJOz8+NslrvxR7XtCOcPJXxshQ33wBD+sCuP43nOHsWihQ25q1dXyN77f0pNfvDCB/PNSapX1kl49jn0DFJFC0Oi2cHsXm9E2xQJsLDsy+kULqM+Vvm7Fl4f0IIf/5ux+OBBx9J4Z7GbjasdTBxZCGurOzB7kh/jp4Dkrmqqu9vwzaf9PX5d875/5dpqBVRlpGNm9By3pwMj6tYPJSVjz5GrXcmA2ADet9Rj6hq1+DFy6GTJ3lpzWqOnjafnFwkKIjXG97L9REROGx2Jn/3LXG/GVFos2rV6XnbHXjxkpiczPAN6/g5IeGSNfzy0OTLqPzyLYyzs2CJHacDwssYl2KoVB5aPuLk5CmIKJP+2JtqeXj5+Zydh9WyYta/oXQcdD+PDWvP3p37Mizv33gYwSFBxHw2iJdbjOD44b+5+4E76DS0HQ6HnX2/HGTc01NJPH4Sm81Gp6HtuOfBeni9XuL3HeOtp97l+CU6CYWKFqL3209R7ZYq2B125sV8zNq5XwFw1/238+jgB8ELp/45zTt9Z7L7f3svWc+/nepd8jGX0qXVbXS7/0527884z7D76I9IPJPEk23rce+t1wBejhw/ScyM1ST8bb6fFw4JYkDne6lZOQK7zc6MZd+y8mtjP29YtzpPtrkDr9fLyTPJjJu3jl37Lr2fX8x3sy7//X85osp399u6VhzO3mVhcgu/dbBySk52sCR3s6KDlZdlp4OVH/i7g5Xb+NLBym+s6GDldepg5X65aEaQiIiI5Ep5O4sJiNx1NTwRERGRfEAJloiIiJhTguUzJVgiIiIiFlMHS0RERMRiGiIUERERcx4NEfpKCZaIiIiIxZRgiYiIiCmvN3f9AfO8QAmWiIiIiMWUYImIiIg5zcHymRIsEREREYspwRIRERFzutCoz5RgiYiIiFhMCZaIiIiY8+gsQl8pwRIRERGxmBIsERERMac5WD5TgiUiIiJiMSVYIiIiYsqrOVg+U4IlIiIiYjElWCIiImJOc7B8pgRLRERExGLqYImIiIhYTEOEIiIiYk5/7NlnSrBERERELKYES0RERMx5dZkGXynBEhEREbGYEiwREREx5dUcLJ8pwRIRERGxmBIsERERMac5WD5TgiUiIiJiMSVYIiIiYkpzsHynBEtERETEYkqwRERExJzmYPlMCZaIiIiIxWxer1cDqyIiIiIWUoIlIiIiYjF1sEREREQspg6WiIiIiMXUwRIRERGxmDpYIiIiIhZTB0tERETEYupgiYiIiFhMHSwRERERi6mDJSIiImIxdbBERERELKYOloiIiIjF1MESERERsZg6WCIiIiIWUwcrG5YtW0bz5s1p0qQJc+fODXRz/O7UqVO0bNmSAwcOBLopfjd58mRatGhBixYtGDNmTKCb43cTJkygefPmtGjRgg8++CDQzQmI0aNHM3DgwEA3w+8ee+wxWrRoQZs2bWjTpg3bt28PdJP86osvvuD++++nWbNmDB8+PNDNkVzMGegG5FVHjx5l3LhxfPLJJwQHB9OhQwduv/12qlWrFuim+cX27dt55ZVX2Lt3b6Cb4ndff/01GzduZMmSJdhsNp588klWr15NkyZNAt00v/juu+/45ptvWLp0KS6Xi+bNmxMZGUnVqlUD3TS/2bx5M0uWLKFBgwaBbopfeb1e/vzzT9atW4fTWfA+Pvbv38/QoUNZvHgxpUuXpnPnzqxfv57IyMhAN01yISVYl+nrr7/mjjvuICwsjCJFitC0aVNWrlwZ6Gb5zaJFixg6dCgRERGBborfhYeHM3DgQIKDgwkKCuLqq6/m0KFDgW6W39x22218+OGHOJ1Ojh8/jtvtpkiRIoFult/8888/jBs3jmeeeSbQTfG7P//8E5vNxlNPPUXr1q2ZM2dOoJvkV6tXr6Z58+aUK1eOoKAgxo0bR+3atQPdLMmlCt5XEIvEx8cTHh6edjsiIoIdO3YEsEX+FRMTE+gmBEz16tXT/r93716WL1/OggULAtgi/wsKCmLixInMmDGDZs2aUbZs2UA3yW+GDBlCnz59OHz4cKCb4neJiYnUq1ePV199laSkJB577DGqVKlC/fr1A900v/jrr78ICgriiSeeICEhgYYNG/L8888HulmSSynBukxer/eCZTabLQAtkUD5/fff6dq1KwMGDKBy5cqBbo7f9erVi82bN3P48GEWLVoU6Ob4xeLFiylfvjz16tULdFMC4uabb2bMmDEUKVKEUqVK8eCDD7J+/fpAN8tv3G43mzdv5o033mDRokXs3LmTJUuWBLpZkkupg3WZypYty7Fjx9Jux8fHF8jhsoJq27ZtPP7447zwwgtER0cHujl+tXv3bn755RcAChcuzH333cdvv/0W4Fb5x/Lly9m0aRNt2rRh4sSJfPHFF4wYMSLQzfKbrVu3snnz5rTbXq+3QM3FKlOmDPXq1aNUqVIUKlSIe++9t0CNXIhv1MG6THfeeSebN2/mxIkTnD17ls8//5x77rkn0M0SPzh8+DDdu3dn7NixtGjRItDN8bsDBw7wyiuvkJKSQkpKCmvXrqVOnTqBbpZffPDBB3z66afExcXRq1cvGjVqxKBBgwLdLL85efIkY8aMITk5mVOnTrFkyZICc3IHQMOGDdm4cSOJiYm43W6++uorrr/++kA3S3KpgvPVw2Jly5alT58+PPbYY6SmpvLggw9y4403BrpZ4gfTp08nOTmZUaNGpS3r0KEDDz/8cABb5T+RkZFs376dtm3b4nA4uO+++wpkR7MgatiwYdq293g8dOzYkZtvvjnQzfKb2rVr8+STT9KxY0dSU1OpX78+DzzwQKCbJbmUzZvZZCIRERERuWwaIhQRERGxmDpYIiIiIhZTB0tERETEYupgiYiIiFhMHSwRERERi6mDJSIiImIxdbBERERELKYOloj4LCoqinvuuYfff/890E0REcmV1MESEZ99+umnVK5cmVWrVgW6KSIiuZI6WCLiM4fDQZ06dQrMH3kWEfGV/hahiPgsKSmJzz77DP2lLRGRzCnBEhGfjRs3jrJly7J//35Onz4d6OaIiOQ66mCJiE9++OEHVq5cyaRJkyhevDi7du0KdJNERHIddbBEJMuSk5N56aWXGDZsGGFhYdSsWVPzsEREMqEOlohk2YQJE7j55ptp0KABADVr1uTXX38NbKNERHIhdbBEJEt27NjBypUrGTRoUNqya6+9VgmWiEgmbF6dBiQiIiJiKSVYIiIiIhZTB0tERETEYupgiYiIiFhMHSwRERERi6mDJSIiImIxdbBERERELKYOloiIiIjF/g+IQKgMNEb4VAAAAABJRU5ErkJggg==\n",
+ "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlgAAAJjCAYAAADQ22+yAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAABfx0lEQVR4nO3dd3hUZd6H8XsmCU1AFAlFbKyKrmJZUVddRbARQIqKIhYQFRuIDYkoYAEFZAUEdO1gF1YBcUFUFGxYUF+xrGVRVKQkiBipKTPvH2dpC4wZcjIh4f5wzXUxZ87kPL85U575Ps85E4nH43EkSZIUmmhZN0CSJKmisYMlSZIUMjtYkiRJIbODJUmSFDI7WJIkSSGzgyVJkhSy9LJugLQjGzhwIB9++CEA8+bNY/fdd6dKlSoAPPfcc+v//0cuvfRS+vTpw7777rvVdUaOHMlee+1F+/btS9zudXr27MkHH3zAzJkzqVq1amh/V5LKu4jnwZK2Dy1atGDkyJE0adKkrJtSLEuWLKFdu3YceuihnHjiiZx77rll3SRJ2m44RChtp0aNGsXFF1/M6aefzg033MDSpUu58sorOeecc2jRogUXXHABv/zyCxB0zj777DPef/99OnXqRO/evWnfvj2tWrXivffeAyA7O5tHHnkEgCZNmjBq1Cg6depEixYtGDt2LABFRUXcddddnHLKKZxxxhnceuutXHDBBVts3/jx4znmmGPo0KEDjz/+OBt/V/v000/p2LEjbdq0oUOHDsyePTvh8saNG7Ns2bL19193/f3336dt27Z06tSJtm3bkp+fz8CBA+nYsSOtWrUiKyuLjz76CICVK1dy0003cdppp9GqVSvuueceVq1axZFHHsn333+//m9fdNFFvPbaa2HsIknaKjtY0nbs559/ZuLEiQwbNox//etfHHbYYTz33HPMmDGDKlWqMHny5M3uM3fuXLp168akSZM466yzGD169Gbr5Ofns8suu/Dss89y77338ve//521a9cyYcIEvvjiC1566SWeffZZfvrppy22q7CwkPHjx9O2bVtatGjBL7/8wptvvglAQUEBV111FVdddRUvvfQSd9xxB3feeSf5+flbXB6LxRI+Bt9++y1///vfefHFF/niiy/IycnhueeeY+rUqXTo0IGHHnoIgHvvvZe1a9cydepUJk2axMcff8xnn31G+/btmTBhAgA//vgj33//Pc2bN09qP0hSspyDJW3HDjvsMNLTg5dply5dmDNnDo899hjz58/n22+/5dBDD93sPg0aNODAAw8E4M9//jMTJ07c4t8+6aSTADjooIPIz89n1apVzJo1i3bt2lG5cmUAzjnnHJ544onN7jtjxgxisRjHH3886enptGrVinHjxtGsWTO++eYbotEoJ554IgAHH3wwU6ZM4Ysvvtji8j9Sv359dt99dwAOP/xwdt555/Wdv/fff5+ddtoJgHfffZebbrqJtLQ00tLSePLJJwHIzMzk/PPP59prr+W5557jrLPOIi0t7Q+3K0klYQdL2o5Vq1Zt/f/vvvtu5s6dy5lnnsnRRx9NYWEhW5pCufHE+EgkssV1gPWdqEgkAkA8Hl/fmVsnGt1yyP3MM8+wZs0aTj31VCBIxHJzc/n2229JS0tb/zfX+eabb7a6vFGjRpssy8/P3+T6xo/BzJkzGTRoEBdddBEnnXQSjRo14sUXXwQgPT19k7+/aNEiqlSpwj777EPjxo2ZMWMGU6ZMWZ9mSVJpcohQKifefvttunTpQvv27alduzbvvvsuRUVFoW6jWbNmvPjii+Tn51NYWLjF9Ov777/ngw8+YOLEibz++uu8/vrrvP322zRt2pRx48bRqFEjIpEI77zzDgBffPEFXbp02eryWCzGrrvuymeffQbAq6++utX2vfPOOzRv3pzOnTvTpEkTXnvttfWPwTHHHMPEiROJxWLk5+dz9dVXrz9Cs3PnzgwdOpRDDz2UunXrhvqYSdKWmGBJ5cRVV13F0KFDue+++0hLS+Mvf/kLP/74Y6jbOOOMM/j+++9p37491apVo2HDhpudfuGZZ57h5JNPZs8999xkeY8ePbjsssu47rrrGDVqFHfeeSdDhw4lIyODUaNGUalSpa0uv+WWW7j99tupWbMmxx57LHXq1Nli+zp16sQNN9zA6aefTlpaGk2bNuWVV14hFovRo0cPBg0aRLt27SgqKqJVq1brE7bmzZtzyy230KlTp1AfL0naGk/TIGm9t99+m19++YV27doBwXm6KleuTO/evcu4ZSXz8ccf069fP1566aXNhiklqTTYwZK03pIlS8jOzuaXX36hqKiIAw44gFtvvZUaNWqUddO2WZ8+ffjggw8YMmQIRx11VFk3R9IOwg6WJEkq96ZMmcL9999PQUEBXbt25bzzzivT9jgHS5IklWtLlixh+PDhvPDCC1SqVIlOnTpx9NFHJ/z5sNJmB0uSJG038vLyyMvL22x5zZo1qVmz5hbv8+677/LXv/6VWrVqAXDaaafx8ssv06NHj9JsakLlvoO1etE+Zd0ESZJSqmr97/94pRDFFu+fsm2Nm9Bzi79A0aNHD3r27LnF++Tk5Gxy9HFmZiZz584ttTYWR7nvYEmSpIqjS5cudOjQYbPlW0uvgC2eULmsjxi2gyVJkrYbiYYCt6Zu3brMmTNn/fWcnBwyMzPDblpSPJO7JElKKJbCf9vi2GOPZfbs2SxbtozVq1fzyiuvcMIJJ4T8KCTHBEuSJJVrdevW5dprr+XCCy+koKCAs846i0MOOaRM21Tuz4PlJHdJ0o4m1ZPc1y5q9McrhaRy/e9Stq3S5BChJElSyBwilCRJCcUo14NdZcIES5IkKWQmWJIkKaFtPbpvR2aCJUmSFDITLEmSlFBR+T7hQJkwwZIkSQqZCZYkSUrIowiTZ4IlSZIUMjtYkiRJIXOIUJIkJVTkEGHSTLAkSZJCZoIlSZIScpJ78kywJEmSQmaCJUmSEvJEo8kzwZIkSQqZCZYkSUrIn3pOngmWJElSyEywJElSQp4HK3kmWJIkSSEzwZIkSQkVGWAlzQRLkiQpZCZYkiQpIY8iTJ4JliRJUshMsCRJUkJFRMq6CeWOCZYkSVLI7GBJkiSFzCFCSZKUUMzTNCTNBEuSJClkJlhJev2tCPc/lkYkAjVrwIDeheyxe3Dbc5OiTPxXlLVrIxy4f4xbbyyiUqWybW/YtlZ/UREMHpnGR58Gffa/HR3j2iuKiFSQeZFTpkd5YvyG7yMrVkbIyYXpEwpIT4dBw9P4+j9RqlaJ0y4rxrlnVKyDmhPVX3tXaN4ugzq7bfiK26VTjNanVJzH4I/qX+e6funUqR3npmuKyqCVpWtrr/0b+qfz488b1lu4OMIRh8YZeWdh2TU2RDv6c38dJ7knLxKPx8t18Ld60T4p29aatXBiuwzGP1zAng3hiQlR3v8oyujBhcx4M8Loh9MZO7qAGtWh963pHNQ4RrfzKs4LLVH9k6dFeemVKP8YVkgsDl2uSqfruUWcemK5fnptUUEhXHx1Om1bxjirbYxb7kojPQ36XV9ELAbX3JLOOe2KOOHYilc7bF7//B/h6r4ZvPhkQVk3LSX+t/51HnsmyuPPpnFq81iF62Aleu1v7POvIvQekM5jowqol1lGjS1F29Nzv2r971O6vS9/2j1l2/rzHj//8UrlQMoSrHnz5jF9+nQWL15MNBolMzOT448/niZNmqSqCSUWKwLiwTcYiLN6NVT+b0I1ZXoaF5xdxM41g+s3X1dIQQX7vElUfywGq9dAfgHEY1BYuOG2imbs01F23YX1H67//jpCdq8i0tIgLQ2O/2uMV2dFOeHYivUhu87/1v9/X0RJi8Il16TzWx6c3CzGJefHSEsr44aWkv+tH+DDTyK8+0GUs9rGyPu9DBtXShK99tcpKID+d6XTu0dhhexcwY793DfBSl5KOlhPPfUU48eP57TTTlvfocrNzaVfv360bduWbt26paIZJVatGtx8XRFdeqRTqyYUxWDsqKAX9eOCCMuWw5W908n9BQ4/JM61l1WsD9hE9bdtGePVmVFOPSuDoiI45sg4zSpggvPrcnh8fBrPPrSh99zkz3H+9WqUw5oUUZAPM96Mkl5BB9+3VH9REfy1aYxrLy9izVroeVM6O1WD8ztWnPR2nS3Vn7MUho5K4767C/nnixXwk5XEr/11Jk6NUqd2nBbHV7zXPfjcV/JS8jHw+OOPM2nSJKpWrbrJ8osuuogOHTqUmw7Wt99FeHBcGi+MLWCP3eHp56PcMCCd5x4upKAQ3psTZcSgQipXgn53pTHq4TRu7FlxOlmJ6n9gXBq71Irz+sRC1qyFa29J5/Hnolx4TsV6o3n+pSgnHhdj9/obll13RRHD70+j0yXp7FY7eMP99POKefzIluo/s82GfVypElzQsYhnXkirkB8y/1t/QSFk355O7x5F1Kldtm0rTYle++vmWT45IY1+11eMeVdbsqM/92NxE6xkpaSDlZ6eTmHh5i+8NWvWkJGRkYombLP7Hk1j5jvBE+vHBRFOPjG2flL7Oe1jDBuTxvLfIHO3OC2Oj1F9p+C2VqfEePDx8v9ttrj1z3gzGCbLyICMDDj9tBivzSrfHayNaz/xuDhXdivilTeim3WaV66Cay7fMDz82NNR9ti9/H+LL279L70SZf8/xdn/T0HN8Tikl/+nfrHq//LrCD8vijBsTFDwL8sixGKQnw8DbizfX66K+9rfpRZ89W2EoiJoelj5f96Dz32FIyUdrMsvv5z27dtzzDHHUKdOHSAYInzvvfe49tprU9GEbXZltyKu/G/A9sHHEQYMSeeXZUXU3hXeeDvC7vWCN5iTm8V45Y0oZ7SJUbkSvPF2lIMal/83m+LWf+D+cV55I8qRhxdRUAiz3o1yyJ/Lb+cKNq0dIO93+PHnCIcevOl+nfBiGitXwk3XFPHLMnjhpTTu6l/+v8kXt/7/fB/htVlR/n57kOQ+OzGNVhXgKKri1H/oQXGmT9gwZHT/Y0GnoyJMci/uax9gzv9FOOrwWIU5anhHf+5viXOwkpeSDtbpp5/OUUcdxezZs8nJySEej9O0aVN69uxJ3bp1U9GEUBz1lzhdOhVxyTUZZGTEqVkDhg8KPkjPbhfjtzzo3D2DohgcuF+M668s/2+yG0tU/w1XFTH43jTaX5BBNC3O0X+J07VzxXqj+fHnCHV2hYz/edVcfF4RNw9K58yu6cSBy7sWcfAB5b9z/b+2Vv9lXYoYPDKNjt0yKCiEU06McUbrirXvYev17wgSvfYheGwa1Kt4z/l1dvTnvraNp2mQJKmcSfVpGj78ce+UbevIPeenbFulqWLOxJUkSSpDO2DYLUmSkuFRhMkzwZIkSQqZCZYkSUrIowiTZ4IlSZIUMjtYkiRJIXOIUJIkJVQUN49Jlo+YJElSyEywJElSQjHzmKT5iEmSJIXMBEuSJCXkaRqSZ4IlSZIUMhMsSZKUkEcRJs9HTJIkKWQmWJIkKaGYc7CSZoIlSZIUMhMsSZKUUJF5TNJ8xCRJkkJmgiVJkhLyKMLk+YhJkiSFzARLkiQl5G8RJs9HTJIkKWR2sCRJkkLmEKEkSUqoKO6JRpNlgiVJkhQyEyxJkpSQJxpNno+YJElSyEywJElSQjFPNJo0HzFJkqSQmWBJkqSEnIOVPB8xSZKkkJlgSZKkhDwPVvJMsCRJkkJW7hOsjEhaWTdBUhkoisfKugllKi3i92Oljj/2nDwfMUmSpJCV+wRLkiSVriLPg5U0HzFJkqSQmWBJkqSEYngUYbJMsCRJkkJmB0uSJClkdrAkSVJCRfFoyi6lYeTIkYwaNWr99by8PLp3705WVhbnnXceubm5AOTn59O7d2+ysrLo0KED8+bNAyAejzNkyBBatmxJq1at+Oijj/5wm3awJElShfT777/Tt29fHn300U2WjxgxgqZNmzJt2jQ6duzIoEGDAHjiiSeoWrUq06ZNo2/fvmRnZwMwffp05s2bx9SpUxkzZgzZ2dkUFhYm3LYdLEmSlFAR0ZRd8vLyWLBgwWaXvLy8pNs9Y8YM9t57by666KJNls+cOZPTTz8dgDZt2vDmm29SUFDAzJkzadu2LQBHHnkkv/76KwsXLmTWrFm0atWKaDTKPvvsQ4MGDfjkk08SbtujCCVJ0nZj3LhxjB49erPlPXr0oGfPnkn9rfbt2wNsMjwIkJOTQ506dQBIT0+nevXqLFu2bJPlAHXq1GHx4sXk5OSQmZm52fJE7GBJkqSEYin8secuXbrQoUOHzZbXrFlzq/eZNm0ad9111ybLGjVqxNixY4u93Wh0y4N60WiUeDxe7PXXsYMlSZK2GzVr1kzYmdqSrKwssrKyir1+ZmYmS5cupV69ehQWFrJixQpq1apFZmYmubm57LXXXgDk5uaSmZlJ3bp110+E33h5Is7BkiRJCaVyDlYqNGvWjEmTJgEwdepUmjZtSkZGBs2aNWPy5MkAzJkzh8qVK9OgQQNOOOEEpkyZQlFRET/88APz58+nSZMmCbdhgiVJknYovXr1Ijs7m9atW1OjRg2GDRsGwAUXXED//v1p3bo1lSpVYujQoQC0bNmSuXPnrp8AP2jQIKpUqZJwG5H4lgYWy5HCxfuWdRMklYGieKysm1Cm0iIOQOzI0uv9J6XbG/7vU1O2rWsPfCVl2ypNvkIlSZJC5hChJElKqMgfe06aCZYkSVLITLAkSVJCsVL6jcCKzEdMkiQpZCZYkiQpIedgJc8ES5IkKWQmWJIkKSHnYCXPR0ySJClkdrAkSZJC5hChJElKqMghwqT5iEmSJIXMBEuSJCUU8zQNSTPBkiRJCpkJliRJSsg5WMnzEZMkSQqZCZYkSUooFncOVrJMsCRJkkJmgiVJkhIqMo9Jmo+YJElSyEywttGMt+CmO6N8MC0GwJ0jI8yZu2GMOicX6tSGiY/FyqqJpeKpFyI8NzlCJAJ7NIDbeseovQv8vgL6DY3y/Y8Qi0G7lnEu6Rwv6+aGamu1b6zXLVHq7BbnlmsqVu0b+9/nPkDHS6OsWQsZGcH1NifH6XZuxXkMXpwe4Ynxaeuv/74yeI2/OqGQ2rsGyxbnwPlXpjPh4UJ2qVU27UyF/93/+QXB+99H/33/O/7oONdfHictLdFfKZ925Nqdg5U8O1jb4IcFcPf9UWIbfX707RUHggU/L4ILe0a5s2/F6lx98TWMfS7CC4/EqFEd7r4vwqhHItx6Q5xRj0SoWyfOiNvjrFoN7bpGaXpInMMOLutWhyNR7es88nSEj+ZCyxZl2NBStqXn/qrV8NNCeGtyjIwK+o7S9rQ4bU8rBKCgEC66Oo2LO8fXd65enB7hvkfTyFlasT+EtrT/n34hwq/LI0weGyMWC977Xn4DWp9ccTrYsGPXrm3jEGGSVq+BPgOj3HjV1jtPA+6OcuHZcQ7cL4UNS4GDGsPUp4IOxtq1sCQXatUMbrvp6ji9rwjeVHJ/gfx8qF69DBsbskS1A7z/Mbz9QYSz21XcN9atPfc/+zdUqwpX9onSvmuUwaMjrFlbRo1MgceejrLrLtCxbfA45CyFN96OMmZIYRm3rHRtbf93PSfOsFtjRKOwPC9Is3euWbFeBzty7evEiKbsUlFUnEpS5LZhEc4+PU7jRlu+/a33YHEunH9mxXyRZaQHMXmLjlE+mhuhQ6ugzkgE0tOhz8AI7S+KcuRhcfbZo4wbG7Kt1Z6zFAaPijK0X4y0CvyK2tpzf+VqOOqwOMNvj/HcAzEWLYkw/MGKmeT8uhzGjY9yY4+i9csyd4PhdxTxp73LrFkpkei9LyMd7nkgQsvOUWrvCkcckvr2laYduXZtu5R8HCxcuDDhpbx4ZmKEtDQ4o/XWO0+PT4hyceeKOQa/zknHwzsvxriya5zuN0SJbfSlbsgtcd6eHOO33yPcP67ifcj+b+35+XDDbVH69IxRp3ZZt670JHrutzgOBt8Sp/pOULkydD8/xoy3Kt6+B/jnS1GaHxenYf2ybklqFee977rL4sx+KUaDenFuv6fi7P8dufaNFcUjKbtUFCmZMXHZZZcxf/58MjMzicc3fZJGIhFmzJiRimZsk1GPRHjj3WCHZ6TDmrVwxsURCgqCoaIzLo7yjyExMneDZcth7r9h5MCKk15tXP+f9orTqX18/Te0M1oFbyZ5v8PnX8P+jYJv8ztVg1YnxXl1VoR189LKo+LU/vnXwZy7oWOC7ypLl0GsKBgivf3G8ls7FP+5/8XXUKM6ND00uF88HqSZ5d2YR6PMfCfYryceF+OqbjGmvxElu2fRH9yzYiju/l+wCHatBXvvEazXvmWcO0dGqSiv/R2tdoUnEv/fHk8pWLFiBZ07d2bAgAEcccQRof7twsX7hvr3iuvnRdDuoihzXt4Q38x4C56eGOWReyrW5PZ1PvoUet8R5fmHY+xSCya/HGHs+AgTH43Rb0jwLW/A9XEKCuDqW6Ic0zROl7MrxhtNoto3NuaxCL/+RoU+ivB/n/vPToowcVqEcfcGk9yzB0VoWB96XVq6j0FRPLWvs7zf4eSO6bzzUuFWJ/MfcmIGsyYVpOQowrRI2YxH/+/+v39chLlfRhg1KJiLdOuwCFWrBPMyK5rtqfb0ev8p9W1s7Nr/65SybQ0/7NmUbas0peR7ZvXq1Rk4cCATJkwIvYO1PflhQYTd61W8N5V1jjgUup8fp+s1UdLSILM2jBoUvNH0vjJIdNpfFCUCtDg+zgVnVZzHIlHtO7qz28ZZsAg6XhKlsAiOOjzOFV0qzr5f58efI9TZlQp7pOS2urhznMGjglQnGoG/NIlzTfeKt/+3ZEeq3dM0JC8lCVZpKqsES1LZSnWCtb0pqwRL24dUJ1i9Pjk3ZdsaefgzKdtWafK7mCRJSigWt0OfLB8xSZKkkJlgSZKkhIpwDlayTLAkSZJCZoIlSZIS8ijC5JlgSZIkhcwES5IkJeRRhMnzEZMkSQqZCZYkSUoo5lGESTPBkiRJCpkJliRJSqjIowiTZoIlSZIUMhMsSZKUkEcRJs9HTJIkKWR2sCRJkkLmEKEkSUrIn8pJngmWJElSyEywJElSQp5oNHkmWJIkSSEzwZIkSQk5Byt5JliSJEkhM8GSJEkJeaLR5PmISZIkhcwES5IkJeQcrOSZYEmSJIXMBEuSJCXkebCSZ4IlSZIUMhMsSZKUkHOwkmeCJUmSFDITLEmSlJAJVvJMsCRJkkJmB0uSJClkDhFKkqSEHCJMXrnvYEUN4aQdUjTia1/S9qvcd7AkSVLpMsFKnl8BJUmSQmaCJUmSEvKncpJngiVJkhQyEyxJkpSQc7CSZ4IlSZIUMhMsSZKUkAlW8kywJEmSQmaCJUmSEjLBSp4JliRJUshMsCRJUkImWMkzwZIkSQqZCZYkSUooboKVNBMsSZKkkNnBkiRJFdJHH33EmWeeSbt27ejSpQs///wzAHl5eXTv3p2srCzOO+88cnNzAcjPz6d3795kZWXRoUMH5s2bB0A8HmfIkCG0bNmSVq1a8dFHH/3htu1gSZKkhGJEUnYJU+/evRk0aBCTJ0/m9NNPZ+DAgQCMGDGCpk2bMm3aNDp27MigQYMAeOKJJ6hatSrTpk2jb9++ZGdnAzB9+nTmzZvH1KlTGTNmDNnZ2RQWFibcth0sSZK03cjLy2PBggWbXfLy8pL6O/n5+fTq1YsDDjgAgMaNG7No0SIAZs6cyemnnw5AmzZtePPNNykoKGDmzJm0bdsWgCOPPJJff/2VhQsXMmvWLFq1akU0GmWfffahQYMGfPLJJwm37yR3SZKUUCpP0zBu3DhGjx692fIePXrQs2fPYv+dSpUq0a5dOwBisRijR4/m5JNPBiAnJ4c6deoAkJ6eTvXq1Vm2bNkmywHq1KnD4sWLycnJITMzc7PlidjBkiRJ240uXbrQoUOHzZbXrFlzq/eZNm0ad9111ybLGjVqxNixY8nPz18/pHfZZZdt9W9Eo1se1ItGo8Tj8WKvv44dLEmSlFAqT9NQs2bNhJ2pLcnKyiIrK2uz5StXruSKK66gVq1a3H///WRkZACQmZnJ0qVLqVevHoWFhaxYsYJatWqRmZlJbm4ue+21FwC5ublkZmZSt27d9RPhN16eiHOwJElShdS7d2/22msvRo4cSaVKldYvb9asGZMmTQJg6tSpNG3alIyMDJo1a8bkyZMBmDNnDpUrV6ZBgwaccMIJTJkyhaKiIn744Qfmz59PkyZNEm47Et9S7lWOxBbvX9ZNkCQppaL1vknp9o55JTtl25p96uBQ/s6XX35Jhw4d2HfffUlPDwbsMjMzeeihh1i+fDnZ2dn89NNP1KhRg2HDhtGwYUPWrl1L//79+fzzz6lUqRIDBw7koIMOIh6PM3ToUN58800AbrrpJv72t78l3L4dLEmSyhk7WNs/52BJkqSE/Kmc5DkHS5IkKWQmWJIkKaFUngerojDBkiRJCpkJliRJSqh8Hw5XNkywJEmSQmaCJUmSEorhHKxkmWBJkiSFzA6WJElSyBwilCRJCXmi0eSZYEmSJIXMBGsrZs6G4Q9CfgE0bgQD+0D1nba87mtvQfadMGdacH3QSJgzd8PtS3KhTm2Y/Fjptzssxan/m3kw8F5YsQKiaXDb9XBQ4+C2V2bBA08G929QFwb3hV12Tn0d26I4tQ8ZA9Nnws41g+t77wHDb4X8fBh0L7z3MexUFU48FnpcBNFy9FWmJPt+R3nub3X/FwSPwUf/fQyOPxpuuBzS0lJZQcmUpP5YDP7+AMx6D6IR2Ksh3HYD7ForxUWEpCSPRUXjiUaT5489b8Gy5XB6F3hqDOzdEIb9A1auggHXbb7u/AVw2Y2wdBl89PLmt/+8CM7vCffdBQfuF3pTS0Vx6l+9Bk49N3jDafZXmPF28MY69Qn4/Cu44iZ49j7YvT7cNRrWroVbry+zkoqtuPu+0xXQ5yo4/OBNl496FOb+G0YPhIwMGDAMDtgXzjsjZSWUSEn3/cYq6nMftr7/xz4HH30GI28POhvn94QLzoTWJ6eshBIpaf0TXoKpM+CBIVCpEtx9f/DeOOTmlJUQmpI+FqUt1T/2fNi/+qVsW//X+o6Ubas0pex79WuvvcYTTzzBjz/+uMny5557LlVNKLZ3PoSDDwheVADntoOXXtv8RGur10CfgcGLa2v63Q1dzi4/HzBQvPrf+RD23D34gAVocdyGb20vvgpntg46VwA9usLF56aq9SVTnNrz8+Hf/4FHn4X23eDqfrBwSXDbF99AqxZQuXKQWp10PEyflfo6tlVJ9/3GKupzP9H+73oO3HNrsO+X58HvKzYkG+VBSevfd2+44YqgcwVwcOMNt5U3JX0sKpp4PHWXiiIlHaxhw4bx5JNPMn/+fDp16sTkyZPX3/bss8+moglJWZwD9TM3XK9bB1asjLBy1abrDRgGZ58eRMdb8uZ7sDg3+AZbnhSn/vk/wW67ws1D4Kzu0O16KCzacFtREVzVN3jTuWME7FQtpSVss+LUnvMLHH04XNcdJj4Ch/4ZevQN3hgOORCmvR58080vgH+9Brm/pL6ObVXSfb9ORX7uJ9r/ABnpQaJ3WmeovSsccUhqayiJktZ/+MFw0H8HFX77He57HFqemNISQhPGc0E7tpR0sGbNmsXDDz9Mv379ePrppxk5ciTTpgUTlrbHEcpYbMvLN55H8/RESE8LkpqtGTcBLu1cvuZfQPHqLywKPkTPPh3++SCcfwZc3if4RldYCG+8GwwJvvBw8GHc/+7UtL2kilN7w/rw4FDYZ0+IRKBbJ/hxIfy8GC7pDPvuA+deCd2ug8MODj5wy4uS7vt1KvJzP9H+X+f6y+C9l2D3enDbPaXb5jCFVf+PP8MFPeGIJtC5Q+m2ubSE9VhUFPF4JGWXiiIlHax4PE4kEjxoe++9Nw888ACDBg3i/fffX798e1K/7qapw5KlsHONONWqblg26WX47GvocDFc1gfWrA3+n7M0uH3Z8mAuzmknprLl4ShO/Zm1gzeVQ/8cXD/pb0Fq9dNCyNwN/nZkMLk5GoUOWfB/X6S2hm1VnNq/ngeTp296v3g86HD/lhcME704Fp4cFUzs33P3lDQ9FCXd91Dxn/uJ9v/Hn8H3PwXLMtKhQ0v4MrVTZUqkpPUDvP9x8AWjfcvgS9Z2+BZfLGE8FtqxpaSD1bJlSy644ALmzg0Ordlvv/0YOXIk11xzzWZzsrYHxx0Jn34ZTGAHeO7FYJ7JxsY/AFPGBrHwA0OgSuXg/5m7Bbd//Fkwfr/xi7G8KE79xx8NCxfDF18H1z/8NHgjbVgfTmsWHEX062/Bba++GTwW5UFxao9E4M57YcGi4Pozk6Dxn6BeJrzxDtw6LHiTXbkKxo2H009JaQklUtJ9DxX/uZ9o/7/3MQweHaS4sRhMeRX++peUllAiJa3/k8+hZ7/gqOFunVLa9NCV9LGoaEywkpeyowhnz55NZmYmf/rTn9YvW7RoEY8++ig337zth5iUxlGEEHQQhj8IBQWwx+7BG8aChcHE3YmPbLruz4ug7UWbHkX4yDPBN9mBN5ZK80pdcer/8FMYdj+sWgOVMqBvzw3zTZ6ZBE9PgngsOE3DwD4bOp/bu+LU/uIr8NDTECsK5mYM7BPUWVgIA/4evDHHYtCxDVx0TtnWk6yS7vsd4bm/tf2fXwB3jQoen2gE/tIkOAimapWyrSkZJam/23Xw2VcbOtsQDJOOHlQ2tZRUSR6L0pbqowibvDggZdv6rO1tKdtWafI0DZIklTOp7mAdNPnWlG3ri3ap21ZpKkenP5QkSSofytHxTZIkqSyU77GusmGCJUmSFDITLEmSlFBFOrovVUywJEmSQmYHS5IkKWQOEUqSpIQcIkyeCZYkSVLITLAkSVJCnqUheSZYkiRJITPBkiRJCTkHK3kmWJIkSSEzwZIkSYk5CStpJliSJEkhM8GSJEkJOQcreSZYkiRJITPBkiRJCcWdg5U0EyxJkqSQmWBJkqSEnIOVPBMsSZKkkJlgSZKkxEywkmaCJUmSFDI7WJIkSSFziFCSJCXkaRqSZ4IlSZIUMhMsSZKUmAlW0kywJEmSQmaCJUmSEvJEo8kzwZIkSQqZCZYkSUrMOVhJM8GSJEkKmQmWJElKyDlYyTPBkiRJCpkJliRJSsw5WEkzwZIkSQqZCZYkSfoDzsFKlgmWJElSyEywJElSYs7BSpoJliRJUsjsYEmSJIXMIUJJkpSYQ4RJM8GSJEkKmQmWJElKzJ/KSZoJliRJUshMsCRJUkJx52AlzQRLkiQpZCZYkiQpMROspJlgSZIkhcwES5IkJeZRhEkzwZIkSQqZCZYkSUoo4hyspJlgSZIkhcwES5IkJWaClTQTLEmSpJCZYEmSpMQ8ijBpJliSJEkhs4MlSZIUMjtYkiQpsXgKLyGaM2cOZ5xxBqeffjqXX345v/32GwB5eXl0796drKwszjvvPHJzcwHIz8+nd+/eZGVl0aFDB+bNmxeUH48zZMgQWrZsSatWrfjoo4/+cNt2sCRJUoV00003MXToUKZMmcK+++7LI488AsCIESNo2rQp06ZNo2PHjgwaNAiAJ554gqpVqzJt2jT69u1LdnY2ANOnT2fevHlMnTqVMWPGkJ2dTWFhYcJt28GSJEmJpTDBysvLY8GCBZtd8vLykm721KlT2XfffSkoKGDJkiXUrFkTgJkzZ3L66acD0KZNG958800KCgqYOXMmbdu2BeDII4/k119/ZeHChcyaNYtWrVoRjUbZZ599aNCgAZ988knCbXsUoSRJ2m6MGzeO0aNHb7a8R48e9OzZM6m/lZGRwddff81FF11Eeno61113HQA5OTnUqVMHgPT0dKpXr86yZcs2WQ5Qp04dFi9eTE5ODpmZmZstT8QOliRJSiyFJxrt0qULHTp02Gz5uvRpS6ZNm8Zdd921ybJGjRoxduxYGjduzLvvvsuzzz7Ltddey7PPPrvFvxGNbnlQLxqNEo9v/gBsbf117GBJkqTtRs2aNRN2prYkKyuLrKysTZatXbuW1157jZNPPhmAtm3bMmTIEAAyMzNZunQp9erVo7CwkBUrVlCrVi0yMzPJzc1lr732AiA3N5fMzEzq1q27fiL8xssTcQ6WJElKLB5J3SUk6enp3HbbbXz++edAkHL95S9/AaBZs2ZMmjQJCOZpNW3alIyMDJo1a8bkyZOB4AjEypUr06BBA0444QSmTJlCUVERP/zwA/Pnz6dJkyaJtx9aJZIkSduJtLQ0hg8fTv/+/SkqKqJu3brrjxbs1asX2dnZtG7dmho1ajBs2DAALrjgAvr370/r1q2pVKkSQ4cOBaBly5bMnTt3/QT4QYMGUaVKlYTbj8S3NLBYjsQW71/WTZAkKaWi9b5J6fb2Gf33lG3r+x7Xp2xbpckhQkmSpJA5RChJkhIr12NdZcMES5IkKWQmWFsxczYMfxDyC6BxIxjYB6rvtOk6r74Jox+DaBRqVoc7boQ9d4c1a+GO4fDZVxCLw6EHQr9roUrlsqllW5SkfoBj20LdDedqo1snOP2U1LW/JP6o9kkvw7gJG67/vgKW5MIb/4Tddi3ftUPJ6r9jBPz484bbFiyCIw+F+zY9Pc12zf2/7fVXrgS3DIXvfoR4DNq1hEs7p76GkihJ/enpcNs98NV/oGoVOCMLzj8z9TVo++Ak9y1YthxO7wJPjYG9G8Kwf8DKVTDgug3rrFkbvJFOfAT2aghjx8Psj+CBITDiYVi0BO66CeJxuHFgsM7VF4fe1FJR0vq//xGuuAlefqrMSthmxal9YwWFcEFP6JAF57Qt37VDyevf2Gf/hl4D4KnRUD/x6WK2G+7/ktU/aCREotC3J6xaDad3hWH94PCDU1nFtitp/dl3Qloa3H4DFMWgx81wbntofmz4bU35JPdRKZzk3tNJ7kmZP38+S5YsAWDChAkMHDiQqVOnpmrzSXnnQzj4gOAFBnBuO3jptaCztE5RUXB9xcrg+qrVwbc3CL6xX35hkOykpcGB+8HCJamtoSRKWv8nn0NaFLr0gnYXwZixwfrlQXFq39jDT0PtXTZ0Lspz7VDy+tfJL4Cb7oKbepSfzhW4/0taf9+r4cYrgv/n/gL5+VCjeum3Oywlrf+Lb6DdqcH7fqUMaHYMvDIrNW0vbZF46i4VxR8OET766KOMHj2aoqIiGjRoQOPGjddf9t9/fxo2bPiHGxk7dixPPPEEsViMv/71ryxatIhTTjmF559/nu+//56rrroqlGLCsjhn0w+FunVgxcoIK1fF10fFO1ULvtWcexXUqgmxWPBNHeC4Izfc9+fF8Pg/4bYbUtf+kipp/YVFcGxT6H1FkHRdnh1E7F06pr6WZBWn9nV+XQ5jn4PnH96wrDzXDiWvf53n/wV1doNTTijV5obO/V+y+iORYJjsxoEwfRac/DfYZ4+UND0UJa3/kANh8itweJOgc/nqrODx0I7pD3f9Aw88wNChQznkkEP46aef+Oabb/j666958803+fbbbwHYb7/9eOaZZ7b6N55//nmmTp3K0qVLadOmDe+99x6VK1emY8eOnHXWWdtdBysW2/LyjX926Jt5cP/j8NK4YN7RE/+EXv2DIbPIf09E+8XX0PMWOK9D6UTEpaWk9Z99+ob1KlWCrmfDE8+Xjw+Z4tS+zvgp0OJv0LD+hmXluXYoef3rjJsQDJOUN+7/LS9Pdv8PvSX4AtarP9w3Dnp2C7edpaWk9fe5EobeD2dcAnV2DTrbn3xeOm1NuRDPsL6j+MMOVvXq1TnxxBNJT08nMzOTI444YpPbFyxYsL6jtTWxWIxKlSqx++67061bNypX3jDbu2g7zM/r14W5/95wfclS2LlGnGpVNyx7+8NgXsG6Sd2dO8DgMbD8N9ilFvxrRjDR/ZZe0KYcTXCFktf/5vtwwL7Q+E/BbfF4+fkWV5za15n2RjAksrHJ08tv7VDy+gG+/CYYFjvysFJrZqlx/5es/rc/gP0bQeZuQcrd+qTyNURW0vpXrIIbLg9SfYCHnoY9/3iQRxXUH87B6t69OxMmTNjq7Q0bNqR58+YJ/8app57K+eefT1FRET179gTgq6++onPnzpv9OOP24Lgj4dMvYf6C4PpzL0KL4zZd58/7w4efwtJlwfUZbwffZHapBdNnwp33wsPDyl/nCkpe/7ffw6hHgw/ZNWvhqYmQlfgpst0oTu0Av/0eHC33v5N3y3PtUPL6IXheHP2XDUlueeL+L1n9094I5p3F48EQ2bQ3gudCeVHS+p+bHOx/CN4b//kStDmpdNus7dcfHkV4+OGHU1BQwAknnMDxxx/PgQceSOPGjaladQtd+gQ+/PBDjjxyw+Sk7777jp9++olmzZptW8v/q7R+KmfWe8GhugUFsMfuMLgvLFgI/e4OhsEgePN8+gXIyICda8At18B++8BpnYNDdzc+VPvwg6H/taXS1FJRkvpXr4GBI4I3qoJCaHkiXHNp+fnALU7tn/0bbrgDpj+96X3Le+1QsvoBbh8OdWrDFRemtt1hcf9ve/15v8Ot9wQdzQhw0vHQ86ItD7Ftr0pS/8pV0GcQ/PBz0Mnsfh60PbV02pnqowgbjbgnZdv67pqtHLZZzvxhB+unn37iq6++4uuvv+brr7/mq6++YuHChTRs2JDp06enqp1b5W8RSpJ2NHawtn9/ODtgjz32YI899uCUUzaMda1atYpvvkntzpUkSWWkAp0+IVW2KbitVq0ahx12WMhNkSRJqhjK0fEtkiSpLFSkE4CmSjmaeihJklQ+mGBJkqTETLCSZoIlSZIUMhMsSZKUmAlW0kywJEmSQmaCJUmSEvIowuSZYEmSJIXMBEuSJCUWL0c/qLmdMMGSJEkKmQmWJElKzDlYSTPBkiRJCpkdLEmSpJA5RChJkhLyNA3JM8GSJEkKmQmWJElKzAQraSZYkiRJITPBkiRJCTkHK3kmWJIkSSEzwZIkSYmZYCXNBEuSJClkJliSJCkxE6ykmWBJkiSFzARLkiQl5FGEyTPBkiRJCpkdLEmSpJDZwZIkSQqZc7AkSVJizsFKmgmWJElSyOxgSZIkhcwhQkmSlJCnaUieCZYkSVLITLAkSVJiJlhJM8GSJEkKmQmWJElKzAQraSZYkiRJITPBkiRJCXkUYfJMsCRJkkJmgiVJkhIzwUqaCZYkSVLITLAkSVJCzsFKngmWJElSyEywJElSYiZYSTPBkiRJCpkJliRJSswEK2kmWJIkSSGzgyVJkhQyhwglSVJCnqYheSZYkiRJITPBkiRJiZlgJc0ES5IkKWQmWJIkKTETrKSZYEmSJIXMBEuSJCXkUYTJM8GSJEkKmQmWJElKzAQraSZYkiRJITPBkiRJCTkHK3kmWJIkSSEzwZIkSYmZYCXNBEuSJClkJliSJCkxE6ykmWBJkiSFzA6WJElSyOxgSZKkhCIpvJSGL7/8koMPPnj99fz8fHr37k1WVhYdOnRg3rx5AMTjcYYMGULLli1p1aoVH3300fr7PProo7Rs2ZLTTjuNV1555Q+36RwsSZJUYa1evZrbb7+dgoKC9cueeOIJqlatyrRp0/jwww/Jzs5mwoQJTJ8+nXnz5jF16lR++OEHunfvzrRp0/jyyy958cUXmTx5MitWrOCcc87hqKOOolatWlvdrgmWJElKLJ66S15eHgsWLNjskpeXt01NHzx4MF27dt1k2cyZM2nbti0ARx55JL/++isLFy5k1qxZtGrVimg0yj777EODBg345JNPePPNNznllFOoXLkytWvX5qijjmLmzJkJt2uCtRUzZ8PwByG/ABo3goF9oPpOG26f9DKMm7Dh+u8rYEkuvPFP2G3XDct73gKZu0G/a1LW9FDsyPXvyLVDyes/ti3UrbPh9m6d4PRTUtf+knL/u/8T1b+x196C7DthzrTgen4+DLoX3vsYdqoKJx4LPS6CqFFGUsaNG8fo0aM3W96jRw969uyZ1N+aMWMGa9asoWXLlpssz8nJoU6dDU/UOnXqsHjxYnJycsjMzNzi8iZNmmy2PBE7WFuwbDncPBieGgN7N4Rh/4C/PwADrtuwTvuWwQWgoBAu6AmXnrfpG+zDT8NHcyGrRUqbX2I7cv07cu1Q8vq//xFq1oCJj5RJ80vM/e/+/6P615m/AO6+H+Ibnb7ggSdh4RJ48THIyIABw+CZSXDeGamqoPSk8qdyunTpQocOHTZbXrNmza3eZ9q0adx1112bLGvUqBErVqxg7NixxdpuNBolHt+80ETLE/69Ym01ZIMHDy6LzRbbOx/CwQcELzCAc9vBS69t+kLa2MNPQ+1d4Jy2G5a9/zG8/QGc06702xu2Hbn+Hbl2KHn9n3wOaVHo0gvaXQRjxkJRUUqaHgr3v/u/OPWvXgN9BkKfqzZd/sU30KoFVK4cpFYnHQ/TZ6Wm7RVJzZo1adiw4WaXRB2srKws3nzzzU0urVu3Zvny5Zx33nm0axe8INu1a8eKFSvIzMwkNzd3/f1zc3PJzMykbt26SS1PpNQTrJtuummzZa+//jq//fYbwGY9zu3B4hyov9HjVrcOrFgZYeWq+GZR8a/LYexz8PzDG5blLIU7R8FDw2D8iylpcqh25Pp35Nqh5PUXFsGxTaH3FbBmLVyeHQyvdOmYkuaXmPvf/V+c+gcMg7NPD4YQN3bIgTDtdTi1WZBg/es1yP0lNW0vdeXwRKMdO3akY8cNT77GjRszefJkAJo1a8bkyZNp2rQpc+bMoXLlyjRo0IATTjiB559/njZt2rBgwQLmz59PkyZNqFy5Mv379+eiiy5i9erVvPfee/Tq1Svh9ku9g1WrVi0mTZrE5Zdfvr73+d5773HUUUeV9qa3WSy25eVbSgPHT4EWf4OG9YPrBYVw3W1wU0/IrF16bSxNO3L9O3LtULL6IfjQWadSJeh6NjzxfPn5gHX/b3m5+3/D/5+eCOlpcGZr+HnRputd0hlGPATnXhkMlWa1gK/nlV57te0uuOAC+vfvT+vWralUqRJDhw4FoGXLlsydO3f9BPhBgwZRpUoVDjnkENq2bctZZ51FYWEhV199NXXr1k24jVLvYPXp04cTTjiBESNGcN1113H00Uczbty4LY6vbi/q14W5/95wfclS2LlGnGpVN1932hvQ9+oN1z//KnjRDRkTXF+6LIjI1+bDwBtLt91h2ZHr35Frh5LVDzB5OhywLzT+U3A9Hof0cjTT0/3v/v+j+ie9DKvXQoeLoaAgSOo6XAwPDIFIBLqeAzdeGaw79XXYc/fU1lBqymGC9b++/vrr9f+vXLkyQ4YM2WydSCRCnz596NOnz2a3devWjW7duhV7eyl56h9zzDEceOCBDBgwgJkzZ1K0nQ/KH3ckDL0vmMS4d0N47kVocdzm6/32O/z4Mxy+4dxlHH5wcDTNOqMfg19/K19HEu3I9e/ItUPJ6gf49nt49U0YeXuQ6Dw1EdqcnJq2h8H97/7/o/rHP7Dh/z8vgrYXbZjUP/7F4CjEMXfCqtUwbnxwFKV2TCmb5F6rVi1GjhxJo0aNNjk0cntUexcYlA3X9IfWF8A338GNVwXfUDtcvGG9HxdAndqQUY6+oRXHjlz/jlw7lLz+q7rCzjWCCc7tLoLDD4KObVJaQom4/93/xal/a85oBbvUgtO7QsfLoGVzOO3EUm50ikTiqbtUFJH4lo49LEdii/cv6yZIkpRS0XrfpHR7h/UcnrJt/d+oa1O2rdJUwb5/SZKk0JXrKKZseH5ZSZKkkJlgSZKkhCrS3KhUMcGSJEkKmR0sSZKkkDlEKEmSEnOIMGkmWJIkSSEzwZIkSQk5yT15JliSJEkhM8GSJEmJmWAlzQRLkiQpZCZYkiQpMROspJlgSZIkhcwES5IkJeRRhMkzwZIkSQqZCZYkSUrMBCtpJliSJEkhM8GSJEkJReJGWMkywZIkSQqZCZYkSUrMACtpJliSJEkhs4MlSZIUMocIJUlSQp5oNHkmWJIkSSEzwZIkSYmZYCXNBEuSJClkJliSJCkh52AlzwRLkiQpZCZYkiQpMROspJlgSZIkhcwES5IkJeQcrOSZYEmSJIXMBEuSJCVmgpU0EyxJkqSQmWBJkqSEnIOVPBMsSZKkkJlgSZKkxOJGWMkywZIkSQqZHSxJkqSQOUQoSZIScpJ78kywJEmSQmaCJUmSEjPBSpoJliRJUshMsCRJUkKRWFm3oPwxwZIkSQqZCZYkSUrMOVhJM8GSJEkKmQmWJElKyPNgJc8ES5IkKWQmWJIkKTF/7DlpJliSJEkhM8GSJEkJOQcreSZYkiRJITPBkiRJiZlgJc0ES5IkKWR2sCRJkkLmEKEkSUrISe7JM8GSJEkKmQmWJElKzBONJs0ES5IkKWQmWJIkKSHnYCXPBEuSJClkJliSJCkxE6ykmWBJkiSFzARLkiQl5Bys5JlgSZIkhcwES5IkJRYzwkqWCZYkSVLIdvgEa+ZsGP4g5BdA40YwsA9U36n46xUVwZAx8PaHwf8vOgc6tQvu8/7HMOS+YHmtmnBTTzhg3+C2x56D56dCehrsUgtuux723D1lZf+h4jwu38yDgffCihUQTQtqOKjxhtvzfocLroZBfeDgA1Lb/m1V0ufDOotyoNMVMOmRYP+u886HMOwfMPGRUi9lm+2o+36d4tQ/ZAxMnwk71wyu770HDL81+P8rs+CBJ4P7N6gLg/vCLjunsoKSKe5rAOC1tyD7TpgzLbgei8HfH4BZ70E0Ans1hNtugF1rpaz5oXv1TRj9GESjULM63HHj5u/Vk16GcRM2XP99BSzJhTf+Cbvtmtr2lhoDrKTt0AnWsuVw82AYeQdMexIaNgjeHJJZ77kXYf4CePExGP8APP5PmPvv4AV2dT/ofQVMfgwGXAfX3gr5+fDuHHj+X/DsfTDpUTjleOg7OIWF/4HiPC6r18DFN8DF58ILj8AVF0LvgRtun/UenH05fP9jSpteImE8HyB4sz2/J+QsjaxftmYtjHg4eA4UFpV2JdtuR9336xT3OfDJ5/D3AUFHeeIjGzpXn38Fd4yAe2+HKWODjtfIh1PX/pIqbv0QvO/dff+mv6Dy/FT48ht44SF4cWzQERkyJhUtLx1r1kKfQXDvHcF+bn4cDLp38/Xat9zwXBj/QNCpuuWaCtS50jZJSQdr7ty56/8/e/ZsBg8ezLBhw/j0009TsfmteufD4Nv13g2D6+e2g5de2/wnlxKt99pbcEYWpKfDzjWgVQuY8gr8sACqV4djjgju02gvqF4N/u+L4EXX/7oN3woPPgAWLklNzcVRnMflnQ+DN89mfw2utzhuw4cMwJPPw103QZ3dUtbsEgvj+ZCzFGa8DQ8M2fQ+b38QdEwG9in9OkpiR9336xSn/vx8+Pd/4NFnoX234IvUutfvi6/Cma1h9/rB9R5dg45oeVHc18DqNdBnIPS5atPl++4NN1wBlSoF1w9uvH29tyWrqCiofcXK4Pqq1VC5UuL7PPw01N4Fzmlb+u1LpUg8dZeKIiUdrAEDBgDw1FNPceedd1KvXj122203+vfvz5NPPpmKJmzR4hyon7nhet06sGJlhJWrir/e4lyo9z+3Lc4NvrmuWh28YQF89m/4z3zI/QX2bwRHHRYsz8+Hex6AlieWQoHbqDiPy/yfgo7izUPgrO7Q7fpNk5mH7obDD05dm8MQxvMhczcYNTD4oNnYycfDTT2gVo1Sa34odtR9v05x6s/5BY4+HK7rHiQWh/4ZevQNPojn/xR8KF/VN+h83TECdqqW8jK2WXFfAwOGwdmnB0OIGzv8YDho/+D/v/0O9z2+fb23JWunasHow7lXwQlnwNMT4frLtr7+r8th7HPBdBAppUOE48eP5/HHH6dr16507dqVp556qkw7WLHYlpdHo8Vfb0u3pUWDdGrMoGAuRvtuMHk6HP0XyMjYsN6y5cFQS7WqcM2l21RCqSjO41JYBG++F7zJ/vNBOP8MuLxP0GEsr8J4PpR3O+q+X6c49TesDw8OhX32hEgEunWCHxfCz4uhsBDeeBduvR5eePi/afXdqWl7GIpT/9MTg7mjZ7be+t/58We4oCcc0QQ6dwi3jan0zTy4/3F4aRy8+QJcdj706r95orfO+CnQ4m/Bc0RKyST3wsJCYrEYtWvXplq1DV/nKlWqRDTFn0r3PhK8AUIQ++6/0TewJUth5xpxqlXd9D716wbzqra0Xv26QSq1Ts5SqJsZvFFVqwqPj9xwW+sLNkyO/HoeXNk3SDZuvALS0sKtsyQS1btOZu3gA+bQPwfXT/ob9BsKPy2EP+2d0uaWSNjPh/JuR9r3W1Kc+r+eB1/9B9qdtmFZPB50OjJ3C55DdWoHyztkwUXXpqbtYShO/ZNehtVrocPFUFAQzFPqcHEwLJ65W3Bwz3W3BUOj3TqlvoaS2vg9Yf5PcNqJG963O3eAwWNg+W+bHryyzrQ3oO/VqWppim2tV6mtSknvZpdddqFZs2b85z//WT9cOHv2bDp16kTLli1T0YT1rr54w2TEZ++HT78MJmtCMGG9xXGb3+e4I7e+3knHwQtTg2+ueb/D1BnBB04kApf1CSa9Arz8RjBPq/GfgvlZXa6BKy8Mho22p84VJK53neOPhoWL4Yuvg+sffhrUXN6+uYX9fCjvdqR9vyXFqT8SgTvvhQWLguvPTApe1/Uy4bRmwST/X38Lbnv1zfJ1FGVx6h//QDCBf+IjQaeqSuXg/5m7BZP/e/YLjpwsj50r2PQ94f7BwfN76bLgthlvB8/zLXWufvs9SO7K6/B4RTVp0iT+9re/0a5dO9q1a8fw4cMBWLhwIeeddx4tW7bkiiuuYOXKYKJdXl4e3bt3Jysri/POO4/c3FwA8vPz6d27N1lZWXTo0IF58+b94bZTkmA9/vjjAHz33Xfk5eUBQXp19dVXc+KJJ6aiCVtUexcYlA3X9A++ie2xe/DGAEHHqN/dwYss0Xqd2gXDA+3/+23unLYb5lfd3S8YHsgvDL7Rjh4UvDk//DSsWQNPvhBcACplwHP/SPlDsEVbq3fjx6RObRg1CG4fDqvWBO2/9w6oXLmsW7/twng+lHc76r5fpzj1798Ibu4FV9wEsaJgntKw/sH9mx8XzMG8sBfEY8FpGrb3Axs2Vpz6Exn1aBB03PNgcAHYvV7w3lce/fUvQUexS69gesfONTbU8r+PyY8LgtdGRgU9+VF5nXz+2WefkZ2dTZs2bTZZftttt9G5c2dat27NmDFjuO++++jduzcjRoygadOmPPjgg0yaNIlBgwYxYsQInnjiCapWrcq0adP48MMPyc7OZsKECVvZaiASj5fv3C+2eP+yboIkSSkVrfdNSrfX/LQhf7xSSCZPuGJ9GLOxmjVrUrNmzaT+1tlnn03NmjXJyclh//33p1+/flSrVo2jjz6aDz74gPT0dBYtWsT555/PjBkzaNGiBU899RT169ensLCQo446ivfff59u3brRq1cvmjZtCsDJJ5/M448/ToMGDba67QowLVeSJJWqeOou48aN46STTtrsMm7cuKSbXadOHXr27MnkyZOpX78+t99+O7/++ivVq1cnPT19/TpLlgTnE8nJyaFOnToApKenU716dZYtW7bJ8nX3Wbx4ccJtV9AwU5IklUddunShQ4fNDz9NlF5NmzaNu+66a5NljRo1YuzYseuvX3LJJZx88snceOONm90/EolstmydrR2M90cH6dnBkiRJCUVSOJtoW4YCs7KyyMrK2mTZ77//ztixY+natSsA8Xic9PR0dt11V1asWEFRURFpaWnk5uaSmRmcAC4zM5OlS5dSr149CgsLWbFiBbVq1SIzM5Pc3Fz22msvgE3uszUOEUqSpAqnWrVqPPzww+t/NebJJ5/klFNOISMjg6ZNmzJ16lQgONLwhBNOAKBZs2ZMmjQJgKlTp9K0aVMyMjJo1qwZkydPBmDOnDlUrlw54fwrcJK7JEnlTqonubc4KXU/mPv6jOzQ/tacOXMYNGgQa9asYe+992bo0KHUqFGDn3/+mezsbH755Rfq16/PPffcw84778zy5cvJzs7mp59+okaNGgwbNoyGDRuydu1a+vfvz+eff06lSpUYOHAgBx10UMJt28GSJKmcsYO1/XMOliRJSiiVc7AqCudgSZIkhcwES5IkJWaAlTQTLEmSpJCZYEmSpMScg5U0EyxJkqSQmWBJkqSEIgZYSTPBkiRJCpkdLEmSpJA5RChJkhJzknvSTLAkSZJCZoIlSZISisTKugXljwmWJElSyEywJElSYs7BSpoJliRJUshMsCRJUmIGWEkzwZIkSQqZCZYkSUoo4hyspJlgSZIkhcwES5IkJWaClTQTLEmSpJCZYEmSpMQ8k3vSTLAkSZJCZoIlSZIS8ijC5JlgSZIkhcwOliRJUsgcIpQkSYk5RJg0EyxJkqSQmWBJ5VRsBz9uuii+Y9ffZvcjyroJZWZ5l2PKugll7sPHUrxBE6ykmWBJkiSFzARLkiQltmMHxtvEBEuSJClkJliSJCkhTzSaPBMsSZKkkJlgSZKkxEywkmaCJUmSFDITLEmSlJgJVtJMsCRJkkJmgiVJkhIzwUqaCZYkSVLITLAkSVJinsk9aSZYkiRJIbODJUmSFDKHCCVJUkL+VE7yTLAkSZJCZoIlSZISM8FKmgmWJElSyEywJElSYjETrGSZYEmSJIXMBEuSJCXmHKykmWBJkiSFzARLkiQlZoKVNBMsSZKkkJlgSZKkxEywkmaCJUmSFDITLEmSlJjnwUqaCZYkSVLITLAkSVJi8VhZt6DcMcGSJEkKmR0sSZKkkDlEKEmSEvM0DUkzwZIkSQqZCZYkSUrM0zQkzQ7W/5g5G4Y/CPkF0LgRDOwD1XdKfr1FOdDpCpj0COxSa8Pydz6EYf+AiY+UeilJK2nta9bCHcPhs6+C1+KhB0K/ayE/H7pcs+nf+OY76H05dD0nJaUVS0nrX54Ht90DX/0HqlaBM7Lg/DOD+3z2b7hrNKxeA0VFcElnaHtqausrqSmvRHj02QiRCFStDDddHePgA+DOkRHmzI2sXy8nF+rUhomPVayjjr75DgaPTGPFygjRaJz+1xfx58bBbed0T2ftWsj47ztqq1NiXNRp+6p/74P3pMe93dhp52rEimKMuPxBvv34u83WO6HjMZx3c/DE/W3p74y8/AF+/s/ibd5u5aqVuO6hK9j38L2JRKM8nP0k707+sFS2VRzNDv8Tt17akuZXjkm43p/3qcdDN51Nq+se5LcVa7Z5e9FIhGvPbcZfD96btGiUJ1+ewwsz5wJwxAF7cPXZx5OelsbagkKGPfUGX35fuvUrdRwi3Miy5XDzYBh5B0x7Eho2gL8/kPx6k16G83tCztINHzpr1sKIh+HaW6GwqLQrSV4Ytf/jiaC2SY/C5EeDmh98EmrWCDqU6y5ntoK/NIHzzkxlhYmFUf/g0VCtKrw0Dp69H958H954N5i60Ks/9LgoqP/BoTBkDMxfkMoKS+b7H2HY/REevDvGC4/EuOzCGL36BW8ffXvFeeGRYPmogTEqV4I7+25fnYuSWr0GLr8hnYvOjTH+4UIuuzBG9sCgN7VqNSz4GSY8Urj+sr11ripXrcTg6bcw/u7JXHHEjTw58J9kP3n1ZuvVytyZXvd355Y2d3HZYTfwzsT36THq4hJt+4Jbz2b1yjVcfNC19Dn1Dq4ecwm77b5rqWzrj+xRtxa9zmlGNBJJuN7O1auQfeFJVMooeQZxxomHsEfdXeh0yzi63P4U5576F/68Tz3S06LceUVrBo19lfMGPMGjU97j9ktblnh7pSYeT92lgkhZB+utt94iLy8PgEmTJnH77bfz/PPPp2rzxfLOh3DwAbB3w+D6ue3gpdc239+J1stZCjPehgeGbHqftz8I3qQH9in9OrZFGLUfeShcfiFEo5CWBgfuBwuXbHr/HxYEHbEhN2/4tr89CKP+L76BdqcGtVfKgGbHwCuzggTvyq5wbNPgPvUyYZedYUlOysorsUoZcPuNMerUDq4f1BiWLgtSvI0NuDvKhWfHOXC/1LexNM3+MMIeDeIc/9fgCXHicXGG3VoIwOdfRahaFXpkp3HGRekMHR1lzdqybO3mjjj1UBbNW8IH0z4BYPaLcxh4zj2brbc85zfOrncJuQt+IZoWJXOvOuT9smL97Z37nsF9c4bwj4/v5tYXelO7/i6b3P+CAR25YEDHTZb9rf1RTH3oNQByf1rKR6/MpdnZx/7htsJWuVI6t1+axYhnZyZcLxKBO7q34r7n397strbHH8zjA87jyVvPZ8wNZ7JXvU3rb3Pcnxlw8WmbLDvxiH2Z8tbnFMXi/L5qLa+8/zVZxxxIYVGMVtc9yDc/5gKwe51a/LZy25MybX9S0sEaNGgQDzzwAGvXrmXEiBFMmTKFfffdl1dffZWBAwemognFsjgH6mduuF63DqxYGWHlquKvl7kbjBoI++696X1OPh5u6gG1apRa80skjNqPOxL22SNY/vNiePyfcNqJm95/xMNw3hnQoG6plLHNwqj/kANh8itQUAgrV8GrsyD3F6hcGc5qveE+418MUo9DDyrdmsK0e/2gwwhBZ3LomAjNjws6Xuu89R4szoXzz6w430DXmb8gQu1dYcDQNDp1T6P79Wnrk+iVq+Cow+P8/bYinvlHIYtyIox8cPsaHGi4f32WLV7OdQ9fwZgPBjPklX6kpadtcd2iwiL2P6IRz/z0AK0vPZlJo6cBcPIFJwTDjEffxOV/6c0H0z7huocu/8Nt19mjNrk//bL+eu7Pv7Bbw9oJt1Ua+nY5mRdmzeXbn5YmXO+yDsfyxXeLee/zHzZZ/pfGDWl93J+59K7nOP/WJ3l82hyG9mj7h9utu2sNlizb0HHM+fV36u5aHYCiohi71qzGv+7pztVnH8/jUz/chspSxAQraSnJEN555x2mTJlCWloaM2fOZPz48VSqVIlzzjmHNm3apKIJxRLbSqofjW7beuVJmLV/8TX0vAXO6wDNj92wfFFOkAANvLFkbS0NYdTf50oYej+ccQnU2TVIrD75fNP1Hnoq6Hg+dDdUqVzydqfaqtVw811RFufCA0M3fTAenxDl4s5x0rb8uV2uFRbC2+9HeHh4EYf8Oc4bb0e4qk86058rpPlxcZoft2Hc/9Lziri2fzp9em4/w4TpGekc1epwere4la8++A/HtG3KoH/15fy9r6Agv3Cz9b/56DvOaXApTU87jIEv3cSFf7qKv7Y+gsZH7ct9HwbxfDQtSuVqlQAY8Hxv6u+TyS71agFwXLujWPR9DredeTeRLbwxxopiCbe18rdVm92nJM5qfihFRTGmvPUF9WvX3Op6xx2yDwc3qk/Pv28+unLcIfuwR2YtHr250/plO1evQs2dqnB5h2M5dL8G1NypClUrV+Kp284nv6CIiwY+Q2QLw5FFG00YX5a3itbXPUjjvTK5r/dZXHTH0/y4ZHnJCtZ2ISUdrCpVqvDLL7+QmZlJ7dq1WbVqFZUqVWL16tWkp5ftONG9jwTzZABWrIT9G224bclS2LlGnGpVN71P/bow999/vN72rjRq/9eMYKL7Lb2gzSmb3nf6TDj5b7BTtdBL2SZh179wCdxwOdT67/v3Q0/Dnv8dSszPh5vugnk/wLP3BYnQ9m7UIxHeeDf4cGh+bJwz28S56qYof9orzmMj4pt0EJctDx6XkQMrzrfPMY9GmflO0DlYngd77wmH/Dmor/nf4tx6NyxYBD/+HKH6TtD00OC2eDzCVsKhlOpy2zkcc3owLl2jdnV++upnvvrgP0AwRHjdQ1dQv1Fdfvzq5/X3qV1/F/ZpsidzXvkUgDnT/49Veato8Kd6RNOiPDd0Mi/94xUAMiqlU32XIIm57cy7AdYPDz5x24T1fzPnx6XsWr8Wv/6307Bbg12Z9+n8hNva0uT7ZF3W/lhOODx4URcUxqhSKZ2nbjuf9LQ0Kv/3/72GT2Tp8pXr79P2+IPJ3KU6T956/vpl/+hzNrc/Mp20aISps//N6AlvAcFQYp1a1clbuYahT74OBEOERxywB7c9Mn39/Zf8ksdutTYcLVNnl+rkLPudnapW4sgD92Tmx8E++fqHHL79KZc/NayzfXawKlCylCop6d306NGDs846i9atW9OoUSMuuOACjjnmGN5++20uueSSVDRhq66+OLgA/PIrtLsomHy8d0N47kVocdzm9znuSBh63x+vt70Lu/bpM+HOe+HhYcE8pf8151M4tVmplZO0sOt/bjKsWAX9rgnmJ/3zJRjWL7jtmgFQFIOnx1BuOuI9L47T8+LgTXV5Hpx9aZT2WXGu7Lr5G+0nnwX7vLzUVhxXdYtxVbcgaVn6C7Tvms6XX8OfG8OcTyMQgd3rwYf/F+HBxyM8OrKIjPQgyTutRdmnV+MGPMe4Ac8BsEvdWjz8xXD2+0sjvv34O5ocfyDxeJxF3286ETCjSgY3P3stVx2ZzcJ5izn0xINIS0/jx38vYM4rn9K6+ym8/tRbrPp9NV1uP4d9D29E9ml3JGzH7Bc/pHX3U7j3yofYbfddadryMJ4a9HzCbYXhgUnv8sCkdzdbXr92TZ4deCHnDXhys9v6jJmyyfUPH7uOy4eM57cVa9i5ehVu7noKz7zyMb/8tpIzTzyUTqcczll9xyZsx6xP5tH2+IN46//mUbVyJU49qjGDH59BLBanX7dTWZa3irn/WUijBrXZu/6ufPHdohLVre1HSjpYLVq0YL/99uO1117jhx9+4LDDDmOnnXZi8ODBHHLIIaloQrHU3gUGZcM1/aGgAPbYHQb3DW77/Cvod3dwFFii9cqrMGq/58HgS06/uzf83cMPhv7XBv+fvyD4QNoehVF/9/OhzyA4vWvwOFzVFZocCB9/Bm+8G2HvPeKc12PDNq+/DP52VKor3TbPTY6wKAdeeyvCa29tGPJ49J4YtXaGHxZE2L1exf2Gu1ttGDGwiIEj0li9OkKlSjD89iIqV4aOp8dYsDDKOZemU1QERx4e5/ILy76DtbFflyzn1g5D6TnmEqrsVJmCtYXcduYwCtYWULv+Lgz6V19ubn0ni7/P4e+X3E//f14PcVixfCX92g5m7ep8pj08g91235V7Z99JPB4n58el3H3Rpqc62Di5WmfcgPH0uu9SHvrsHqJpUR668QkWfRcc/bK1baXSbrV2YuS1HTZLs/7Xe5//wONTP2TMDWcSi8dZuTqfG0dv2iF76Z0veemdLzdZ9vwbn9IwsxZP334B6elpTJw5l4+/DjqRvUe9yPWdTyQ9LUp+QRG3PDCVnF9Lb6J/iWxtfoS2KhKPl+/cL7Z4/7JuglQmYuzYb3hF8R27/ja7H1HWTSgzy7scU9ZNKHMfPnZdSreXVf+qlG1r2qLE5ygrL7ajA+UlSdJ2qXxnMWWiHB/3JkmStH0ywZIkSYmZYCXNBEuSJClkdrAkSZJCZgdLkiQlFoun7hKinJwcunfvTvv27enUqRMLFgSnyMjLy6N79+5kZWVx3nnnkZsb/CZkfn4+vXv3Jisriw4dOjBv3jwA4vE4Q4YMoWXLlrRq1YqPPvroD7dtB0uSJFVIN954I82bN2fSpEm0a9eOYcOGATBixAiaNm3KtGnT6NixI4MGDQLgiSeeoGrVqkybNo2+ffuSnZ0NwPTp05k3bx5Tp05lzJgxZGdnU1i4+c9MbcxJ7pIkKaF4Cs87l5eXR15e3mbLa9asSc2aW/8tyf+1bNkyvvrqKx577DEAzjzzTI45JjiH2syZM3nqqacAaNOmDbfffjsFBQXMnDmTXr16AXDkkUfy66+/snDhQmbNmkWrVq2IRqPss88+NGjQgE8++YQjjzxyq9u3gyVJkrYb48aNY/To0Zst79GjBz179iz23/npp59o0KABd955J++//z4NGjSgX7/g98tycnKoU6cOAOnp6VSvXp1ly5ZtshygTp06LF68mJycHDIzMzdbnogdLEmSlFjIc6MS6dKlCx06dNhseaL0atq0adx1112bLNtrr7348ssv6dmzJzfffDMTJkwgOzubJ554Yot/Ixrd8qypaDTKln70Zmvrr2MHS5IkbTeSHQoEyMrKIisra5NlP/74Ix06dKB58+ZAMBQ4cOBAADIzM1m6dCn16tWjsLCQFStWUKtWLTIzM8nNzWWvvfYCIDc3l8zMTOrWrbt+IvzGyxNxkrskSUosHk/dJSR77rkndevWZdasWQC88cYbHHTQQQA0a9aMSZMmATB16lSaNm1KRkYGzZo1Y/LkyQDMmTOHypUr06BBA0444QSmTJlCUVERP/zwA/Pnz6dJkyYJt2+CJUmSKqTRo0czYMAA7r77bqpXr87gwYMB6NWrF9nZ2bRu3ZoaNWqsP7rwggsuoH///rRu3ZpKlSoxdOhQAFq2bMncuXNp27YtAIMGDaJKlSoJtx2Jb2lgsRyJLd6/rJsglYkYqTuqZ3tUlMKjmrZHbXY/oqybUGaWdzmmrJtQ5j587LqUbq/lzt1Stq2Xf3s0ZdsqTQ4RSpIkhcwhQkmSlFj5HuwqEyZYkiRJITPBkiRJCcVjO/acx21hgiVJkhQyEyxJkpSYc7CSZoIlSZIUMjtYkiRJIXOIUJIkJZbCH3uuKEywJEmSQmaCJUmSEtvBf5pqW5hgSZIkhcwES5IkJRR3DlbSTLAkSZJCZoIlSZIScw5W0kywJEmSQmaCJUmSEnIOVvJMsCRJkkJmgiVJkhJzDlbSTLAkSZJCFonH4w6sSpIkhcgES5IkKWR2sCRJkkJmB0uSJClkdrAkSZJCZgdLkiQpZHawJEmSQmYHS5IkKWR2sCRJkkJmB0uSJClkdrAkSZJCZgdLkiQpZHawJEmSQmYHS5IkKWR2sEpgypQptGrVilNOOYWnnnqqrJuTcitWrKBNmzYsWLCgrJuScqNHj6Z169a0bt2aoUOHlnVzUm7kyJG0atWK1q1b89hjj5V1c8rEkCFDyM7OLutmpNyFF15I69atadeuHe3atePTTz8t6yal1Ouvv84ZZ5xBy5YtGThwYFk3R9ux9LJuQHm1ZMkShg8fzgsvvEClSpXo1KkTRx99NPvuu29ZNy0lPv30U2655Rbmz59f1k1JuXfffZe3336biRMnEolEuOSSS3j11Vc55ZRTyrppKfHBBx/w3nvv8eKLL1JYWEirVq1o1qwZjRo1Kuumpczs2bOZOHEiJ554Ylk3JaXi8TjfffcdM2fOJD19x/v4+OmnnxgwYAATJkygdu3adOnShVmzZtGsWbOybpq2QyZY2+jdd9/lr3/9K7Vq1aJatWqcdtppvPzyy2XdrJQZP348AwYMIDMzs6ybknJ16tQhOzubSpUqkZGRwZ/+9CcWLlxY1s1KmaOOOorHH3+c9PR0fvnlF4qKiqhWrVpZNytlli9fzvDhw7n88svLuikp99133xGJRLj00ktp27YtTz75ZFk3KaVeffVVWrVqRb169cjIyGD48OEceuihZd0sbad2vK8gIcnJyaFOnTrrr2dmZjJ37twybFFqDRo0qKybUGb222+/9f+fP38+U6dO5dlnny3DFqVeRkYG9957L48++igtW7akbt26Zd2klOnfvz/XXnstixYtKuumpFxeXh7HHHMMt956K2vWrOHCCy9kn3324bjjjivrpqXEDz/8QEZGBhdffDG5ubk0b96ca665pqybpe2UCdY2isfjmy2LRCJl0BKVlW+//ZZu3brRp08f9t5777JuTspdffXVzJ49m0WLFjF+/Piybk5KTJgwgfr163PMMceUdVPKxOGHH87QoUOpVq0au+66K2eddRazZs0q62alTFFREbNnz+buu+9m/PjxfPbZZ0ycOLGsm6XtlB2sbVS3bl2WLl26/npOTs4OOVy2o/roo4/o2rUr119/PR06dCjr5qTUvHnz+Pe//w1A1apVOfXUU/n666/LuFWpMXXqVN555x3atWvHvffey+uvv86dd95Z1s1KmTlz5jB79uz11+Px+A41F2u33XbjmGOOYdddd6VKlSqcdNJJO9TIhZJjB2sbHXvsscyePZtly5axevVqXnnlFU444YSybpZSYNGiRVx11VUMGzaM1q1bl3VzUm7BggXccsst5Ofnk5+fz4wZMzjiiCPKulkp8dhjj/HSSy8xefJkrr76alq0aEHfvn3Lulkp8/vvvzN06FDWrl3LihUrmDhx4g5zcAdA8+bNefvtt8nLy6OoqIi33nqLgw46qKybpe3UjvPVI2R169bl2muv5cILL6SgoICzzjqLQw45pKybpRR45JFHWLt2LYMHD16/rFOnTpx77rll2KrUadasGZ9++int27cnLS2NU089dYfsaO6Imjdvvn7fx2IxOnfuzOGHH17WzUqZQw89lEsuuYTOnTtTUFDAcccdx5lnnlnWzdJ2KhLf0mQiSZIkbTOHCCVJkkJmB0uSJClkdrAkSZJCZgdLkiQpZHawJEmSQmYHS5IkKWR2sCRJkkJmB0tS0rKysjjhhBP49ttvy7opkrRdsoMlKWkvvfQSe++9N9OnTy/rpkjSdskOlqSkpaWlccQRR+wwP/IsScnytwglJW3NmjX861//wl/akqQtM8GSlLThw4dTt25dfvrpJ1auXFnWzZGk7Y4dLElJ+eSTT3j55ZcZNWoUNWrU4JtvvinrJknSdscOlqRiW7t2LTfddBO33XYbtWrV4oADDnAeliRtgR0sScU2cuRIDj/8cE488UQADjjgAL766quybZQkbYfsYEkqlrlz5/Lyyy/Tt2/f9csOPPBAEyxJ2oJI3MOAJEmSQmWCJUmSFDI7WJIkSSGzgyVJkhQyO1iSJEkhs4MlSZIUMjtYkiRJIbODJUmSFLL/B9J07RddsyZ8AAAAAElFTkSuQmCC\n",
"text/plain": [
"
"
]
},
"metadata": {
"filenames": {
- "image/png": "/Users/mhjensen/Teaching/MachineLearning/doc/LectureNotes/_build/jupyter_execute/chapter1_61_10.png"
+ "image/png": "/Users/mhjensen/Teaching/MachineLearning/doc/LectureNotes/_build/jupyter_execute/chapter1_61_11.png"
}
},
"output_type": "display_data"
@@ -3223,13 +3229,13 @@
"output_type": "stream",
"text": [
"Training R2\n",
- "0.9999878260589065\n",
+ "0.9999869956119286\n",
"Training MSE\n",
- "5.415634483874318\n",
+ "5.745136489050356\n",
"Test R2\n",
- "0.999974281880048\n",
+ "0.9999787681537219\n",
"Test MSE\n",
- "11.653095390463358\n"
+ "9.712199063818309\n"
]
}
],
@@ -3761,6 +3767,135 @@
"cell_type": "markdown",
"metadata": {},
"source": [
+ "## Splitting our Data in Training and Test data\n",
+ "\n",
+ "\n",
+ "It is normal in essentially all Machine Learning studies to split the\n",
+ "data in a training set and a test set (sometimes also an additional\n",
+ "validation set). **Scikit-Learn** has an own function for this. There\n",
+ "is no explicit recipe for how much data should be included as training\n",
+ "data and say test data. An accepted rule of thumb is to use\n",
+ "approximately $2/3$ to $4/5$ of the data as training data. We will\n",
+ "postpone a discussion of this splitting to the end of these notes and\n",
+ "our discussion of the so-called **bias-variance** tradeoff. Here we\n",
+ "limit ourselves to repeat the above equation of state fitting example\n",
+ "but now splitting the data into a training set and a test set.\n",
+ "\n",
+ "Let us study some examples. The first code here takes a simple\n",
+ "one-dimensional second-order polynomial and we fit it to a\n",
+ "second-order polynomial. Depending on the strength of the added noise,\n",
+ "the various measures like the $R2$ score or the mean-squared error,\n",
+ "the fit becomes better or worse."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 36,
+ "metadata": {
+ "collapsed": false,
+ "editable": true
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "[ 2.05802766 -0.30626706 5.27262964]\n",
+ "Training R2\n",
+ "0.9967292151090247\n",
+ "Training MSE\n",
+ "0.007931713723267314\n",
+ "Test R2\n",
+ "0.9966744029509663\n",
+ "Test MSE\n",
+ "0.006968283076248407\n"
+ ]
+ }
+ ],
+ "source": [
+ "import os\n",
+ "import numpy as np\n",
+ "import pandas as pd\n",
+ "import matplotlib.pyplot as plt\n",
+ "from sklearn.model_selection import train_test_split\n",
+ "\n",
+ "\n",
+ "def R2(y_data, y_model):\n",
+ " return 1 - np.sum((y_data - y_model) ** 2) / np.sum((y_data - np.mean(y_data)) ** 2)\n",
+ "def MSE(y_data,y_model):\n",
+ " n = np.size(y_model)\n",
+ " return np.sum((y_data-y_model)**2)/n\n",
+ "\n",
+ "x = np.random.rand(100)\n",
+ "y = 2.0+5*x*x+0.1*np.random.randn(100)\n",
+ "\n",
+ "\n",
+ "# The design matrix now as function of a given polynomial\n",
+ "X = np.zeros((len(x),3))\n",
+ "X[:,0] = 1.0\n",
+ "X[:,1] = x\n",
+ "X[:,2] = x**2\n",
+ "# We split the data in test and training data\n",
+ "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)\n",
+ "# matrix inversion to find beta\n",
+ "beta = np.linalg.inv(X_train.T @ X_train) @ X_train.T @ y_train\n",
+ "print(beta)\n",
+ "# and then make the prediction\n",
+ "ytilde = X_train @ beta\n",
+ "print(\"Training R2\")\n",
+ "print(R2(y_train,ytilde))\n",
+ "print(\"Training MSE\")\n",
+ "print(MSE(y_train,ytilde))\n",
+ "ypredict = X_test @ beta\n",
+ "print(\"Test R2\")\n",
+ "print(R2(y_test,ypredict))\n",
+ "print(\"Test MSE\")\n",
+ "print(MSE(y_test,ypredict))"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Alternatively, you could write your own test-train splitting function as shown here."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 37,
+ "metadata": {
+ "collapsed": false,
+ "editable": true
+ },
+ "outputs": [],
+ "source": [
+ "# equivalently in numpy\n",
+ "def train_test_split_numpy(inputs, labels, train_size, test_size):\n",
+ " n_inputs = len(inputs)\n",
+ " inputs_shuffled = inputs.copy()\n",
+ " labels_shuffled = labels.copy()\n",
+ "\n",
+ " np.random.shuffle(inputs_shuffled)\n",
+ " np.random.shuffle(labels_shuffled)\n",
+ "\n",
+ " train_end = int(n_inputs*train_size)\n",
+ " X_train, X_test = inputs_shuffled[:train_end], inputs_shuffled[train_end:]\n",
+ " Y_train, Y_test = labels_shuffled[:train_end], labels_shuffled[train_end:]\n",
+ "\n",
+ " return X_train, X_test, Y_train, Y_test"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "But since **scikit-learn** has its own function for doing this and since\n",
+ "it interfaces easily with **tensorflow** and other libraries, we\n",
+ "normally recommend using the latter functionality.\n",
+ "\n",
+ "\n",
+ "\n",
+ "\n",
"## Reducing the number of degrees of freedom, overarching view\n",
"\n",
"Many Machine Learning problems involve thousands or even millions of\n",
@@ -3784,6 +3919,7 @@
"visualization.\n",
"\n",
"\n",
+ "\n",
"Before we proceed however, we will discuss how to preprocess our\n",
"data. Till now and in connection with our previous examples we have\n",
"not met so many cases where we are too sensitive to the scaling of our\n",
@@ -3791,6 +3927,15 @@
"to extreme values. Scaling the data renders our inputs much more\n",
"suitable for the algorithms we want to employ.\n",
"\n",
+ "For data sets gathered for real world applications, it is rather normal that\n",
+ "different features have very different units and\n",
+ "numerical scales. For example, a data set detailing health habits may include\n",
+ "features such as **age** in the range $0-80$, and **caloric intake** of order $2000$.\n",
+ "Many machine learning methods sensitive to the scales of the features and may perform poorly if they\n",
+ "are very different scales. Therefore, it is typical to scale\n",
+ "the features in a way to avoid such outlier values.\n",
+ "\n",
+ "\n",
"**Scikit-Learn** has several functions which allow us to rescale the\n",
"data, normally resulting in much better results in terms of various\n",
"accuracy scores. The **StandardScaler** function in **Scikit-Learn**\n",
@@ -3820,143 +3965,593 @@
"techniques.\n",
"\n",
"\n",
- "### Simple preprocessing examples, Franke function and regression"
+ "Many features are often scaled using standardization to improve\n",
+ "performance. In **Scikit-Learn** this is given by the **StandardScaler**\n",
+ "function as discussed above. It is easy however to write your own.\n",
+ "Mathematically, this involves subtracting the mean and divide by the\n",
+ "standard deviation over the data set, for each feature:"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "$$\n",
+ "x_j^{(i)} \\rightarrow \\frac{x_j^{(i)} - \\overline{x}_j}{\\sigma(x_j)},\n",
+ "$$"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "where $\\overline{x}_j$ and $\\sigma(x_j)$ are the mean and standard\n",
+ "deviation, respectively, of the feature $x_j$. This ensures that each\n",
+ "feature has zero mean and unit standard deviation. For data sets\n",
+ "where we do not have the standard deviation or don't wish to calculate\n",
+ "it, it is then common to simply set it to one.\n",
+ "\n",
+ "\n",
+ "\n",
+ "Let us consider the following vanilla example where we use both\n",
+ "**Scikit-Learn** and write our own function as well. We produce a\n",
+ "simple test design matrix with random numbers. Each column could then\n",
+ "represent a specific feature whose mean value is subracted."
]
},
{
"cell_type": "code",
- "execution_count": 36,
+ "execution_count": 38,
"metadata": {
"collapsed": false,
"editable": true
},
"outputs": [
+ {
+ "data": {
+ "text/html": [
+ "
"
+ ],
+ "text/plain": [
+ " 0 1 2 3 4\n",
+ "0 0.0 0.0 0.0 0.0 0.0\n",
+ "1 0.0 0.0 0.0 0.0 0.0\n",
+ "2 0.0 0.0 0.0 0.0 0.0\n",
+ "3 0.0 0.0 0.0 0.0 0.0\n",
+ "4 0.0 0.0 0.0 0.0 0.0\n",
+ "5 0.0 0.0 0.0 0.0 0.0\n",
+ "6 0.0 0.0 0.0 0.0 0.0\n",
+ "7 0.0 0.0 0.0 0.0 0.0\n",
+ "8 0.0 0.0 0.0 0.0 0.0\n",
+ "9 0.0 0.0 0.0 0.0 0.0"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
}
],
"source": [
- "# Common imports\n",
- "import os\n",
- "import numpy as np\n",
- "import pandas as pd\n",
- "import matplotlib.pyplot as plt\n",
"import sklearn.linear_model as skl\n",
"from sklearn.metrics import mean_squared_error\n",
"from sklearn.model_selection import train_test_split\n",
"from sklearn.preprocessing import MinMaxScaler, StandardScaler, Normalizer\n",
- "\n",
- "# Where to save the figures and data files\n",
- "PROJECT_ROOT_DIR = \"Results\"\n",
- "FIGURE_ID = \"Results/FigureFiles\"\n",
- "DATA_ID = \"DataFiles/\"\n",
- "\n",
- "if not os.path.exists(PROJECT_ROOT_DIR):\n",
- " os.mkdir(PROJECT_ROOT_DIR)\n",
- "\n",
- "if not os.path.exists(FIGURE_ID):\n",
- " os.makedirs(FIGURE_ID)\n",
- "\n",
- "if not os.path.exists(DATA_ID):\n",
- " os.makedirs(DATA_ID)\n",
- "\n",
- "def image_path(fig_id):\n",
- " return os.path.join(FIGURE_ID, fig_id)\n",
- "\n",
- "def data_path(dat_id):\n",
- " return os.path.join(DATA_ID, dat_id)\n",
- "\n",
- "def save_fig(fig_id):\n",
- " plt.savefig(image_path(fig_id) + \".png\", format='png')\n",
+ "import numpy as np\n",
+ "import pandas as pd\n",
+ "from IPython.display import display\n",
+ "np.random.seed(100)\n",
+ "# setting up a 10 x 5 matrix\n",
+ "rows = 10\n",
+ "cols = 5\n",
+ "X = np.random.randn(rows,cols)\n",
+ "XPandas = pd.DataFrame(X)\n",
+ "display(XPandas)\n",
+ "print(XPandas.mean())\n",
+ "print(XPandas.std())\n",
+ "XPandas = (XPandas -XPandas.mean())\n",
+ "display(XPandas)\n",
+ "# This option does not include the standard deviation\n",
+ "scaler = StandardScaler(with_std=False)\n",
+ "scaler.fit(X)\n",
+ "Xscaled = scaler.transform(X)\n",
+ "display(XPandas-Xscaled)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Small exercise: perform the standard scaling by including the standard deviation and compare with what Scikit-Learn gives.\n",
"\n",
"\n",
- "def FrankeFunction(x,y):\n",
- "\tterm1 = 0.75*np.exp(-(0.25*(9*x-2)**2) - 0.25*((9*y-2)**2))\n",
- "\tterm2 = 0.75*np.exp(-((9*x+1)**2)/49.0 - 0.1*(9*y+1))\n",
- "\tterm3 = 0.5*np.exp(-(9*x-7)**2/4.0 - 0.25*((9*y-3)**2))\n",
- "\tterm4 = -0.2*np.exp(-(9*x-4)**2 - (9*y-7)**2)\n",
- "\treturn term1 + term2 + term3 + term4\n",
+ "\n",
+ "Another commonly used scaling method is min-max scaling. This is very\n",
+ "useful for when we want the features to lie in a certain interval. To\n",
+ "scale the feature $x_j$ to the interval $[a, b]$, we can apply the\n",
+ "transformation"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "$$\n",
+ "x_j^{(i)} \\rightarrow (b-a)\\frac{x_j^{(i)} - \\min(x_j)}{\\max(x_j) - \\min(x_j)} - a\n",
+ "$$"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "where $\\min(x_j)$ and $\\max(x_j)$ return the minimum and maximum value of $x_j$ over the data set, respectively.\n",
"\n",
"\n",
- "def create_X(x, y, n ):\n",
- "\tif len(x.shape) > 1:\n",
- "\t\tx = np.ravel(x)\n",
- "\t\ty = np.ravel(y)\n",
- "\n",
- "\tN = len(x)\n",
- "\tl = int((n+1)*(n+2)/2)\t\t# Number of elements in beta\n",
- "\tX = np.ones((N,l))\n",
- "\n",
- "\tfor i in range(1,n+1):\n",
- "\t\tq = int((i)*(i+1)/2)\n",
- "\t\tfor k in range(i+1):\n",
- "\t\t\tX[:,q+k] = (x**(i-k))*(y**k)\n",
- "\n",
- "\treturn X\n",
"\n",
"\n",
- "# Making meshgrid of datapoints and compute Franke's function\n",
- "n = 5\n",
- "N = 1000\n",
- "x = np.sort(np.random.uniform(0, 1, N))\n",
- "y = np.sort(np.random.uniform(0, 1, N))\n",
- "z = FrankeFunction(x, y)\n",
- "X = create_X(x, y, n=n) \n",
- "# split in training and test data\n",
- "X_train, X_test, y_train, y_test = train_test_split(X,z,test_size=0.2)\n",
+ "## Testing the Means Squared Error as function of Complexity\n",
"\n",
"\n",
- "clf = skl.LinearRegression().fit(X_train, y_train)\n",
+ "Before we proceed with a more detailed analysis of the so-called\n",
+ "Bias-Variance tradeoff, we present here an example of the relation\n",
+ "between model complexity and the mean squared error for the triaining\n",
+ "data and the test data.\n",
"\n",
- "# The mean squared error and R2 score\n",
- "print(\"MSE before scaling: {:.2f}\".format(mean_squared_error(clf.predict(X_test), y_test)))\n",
- "print(\"R2 score before scaling {:.2f}\".format(clf.score(X_test,y_test)))\n",
+ "The results here tell us clearly that for the data not included in the\n",
+ "training, there is an optimal model as function of the complexity of\n",
+ "ourmodel (here in terms of the polynomial degree of the model).\n",
"\n",
+ "The results here will vary as function of model complexity and the amount od data used for training. \n",
+ "\n",
+ "\n",
+ "Our data is defined by $x\\in [-3,3]$ with a total of for example $100$ data points."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 39,
+ "metadata": {
+ "collapsed": false,
+ "editable": true
+ },
+ "outputs": [
+ {
+ "data": {
+ "image/png": "iVBORw0KGgoAAAANSUhEUgAAAr0AAAHmCAYAAAB+hTZxAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAABXwElEQVR4nO3deZxU1Z3///e9t6q6u3qhq7qrih2kW0UFXEIiosEVEQRRNBmXSJyJS6KTRL7R6CRxRI2jSVScLGZiTOJvIk40iUIwBDFjYkIgxmWEFldQdujqBXpfqqvu74/qLrrprXqrjdfz8ch037VOneGBb4/nfI5h27YtAAAAIIOZyW4AAAAAMNIIvQAAAMh4hF4AAABkPEIvAAAAMh6hFwAAABmP0AsAAICM50h2Awbi4MEGRSKJq7BWVJSnqqr6hH3e0Y7+Thz6OnHo68SivxOHvk4c+jp+pmnI48nt8Vpahd5IxE5o6O34TCQO/Z049HXi0NeJRX8nDn2dOPT10DG9AQAAABmP0AsAAICMR+gFAABAxiP0AgAAIOMRegEAAJDxCL0AAADIeIReAAAAZDxCLwAAADJeXJtTrFmzRj/+8Y8VCoV03XXX6Zprruly/aWXXtL3v/99RSIRTZ8+Xffee69cLpdWrVqlhx56SEVFRZKkc845R8uWLRv+bwEAAICYhx/+jsrKNqutLaQ9e3Zr8uQpkqTPfOZKXXzxJXG9o76+Xvfff7ceeODhbtfOOmumSkuP63LujDPO1E033TL0xo+QfkNveXm5VqxYoeeee04ul0tXXnmlTj/9dJWWlkqSGhsbde+99+r5559XcXGxli1bpueff17/9E//pLKyMt15551auHDhiH8RAAAARH3ta3dIkvbv36cvf/kmPfnk0wN+R11drT788INerw/mncnUb+jduHGjZs2apcLCQknSvHnztG7dOv3rv/6rJMntduvll1+W0+lUY2OjqqqqVFBQIEkqKyvTzp079fjjj+u4447TXXfdpVGjRo3ctwEAAEgBfyvbrw1b9g/Lu5wuS6HWcOz4rBljdOb0MYN61549u/XQQw+otrZGWVnZWrbsdh133FStX79OTz/93zJNU2PHjtVdd92nRx/9niorK/Rv/3abHnjgobg/44orFunEE6fpww/f11133atvf/tujRpVKJcrSytW/FDf//7Dev3112QY0rx5C/S5z12nN998XT/+8fcVDkc0ZUqJvvWtewb1/frSb+gNBoPy+XyxY7/fry1btnS5x+l06pVXXtHXv/51+f1+nXXWWZIkn8+nG2+8UTNmzNAjjzyie++9Vw8/3H2IPF5FRXmDfnawfL78hH/m0Yz+Thz6OnHo68SivxOHvu5dQcFBOV3WsL2v87sKCrLj7vuWllxZlhm7/8tfvlf//u//rhNPPFHbtm3TLbfcohdffFE///l/6dlnn1VRUZFWrFihuroK3Xvvci1dulRPPPGTHt99/fWf63J822236dOf/rQsy9Tcuefpxz/+ofbs2aNdu3bqf//35xo/frxWrlypQ4eqtHbtC2ptbdW1116rU0+drsJCt/bs2a0//elPys8fmT9X/YZe27a7nTMMo9u5s88+W6+++qoeeeQRLV++XA8//LB+9KMfxa5ff/31uuCCC4bU2KqqekUi3dszUny+fFVU1CXs84529Hfi0NeJQ18nFv2dOPR136ZP8mj6JM+wvKunvo6376urGxQOR1RRUafGxkaVlZXp9tvviF2vr2/Qtm27NWvWWfrsZ/9Jn/70OTr77PNUXDxe+/fviz3bkyeeeKrbuYqKOoXDEU2YUKqKijpVVzfI4/EqK2uUKirq9Je/bNC8eQtUXd0oSTr33Av18suv6Mwz52j8+Ilqbpaamwf/58o0jV4HSfut3hAIBFRZWRk7DgaD8vv9seNDhw5pw4YNseNFixbp/fffV11dnZ588snYedu25XDEtW4OAAAAwywSicjlytKTTz4d+9/jjz+pgoJRuvXW2/Ttb39XBQUFuu++u/Tii2uH9FlZWVk9/t598NJWOBzudt9I6Df0zp49W5s2bVJ1dbWampq0fv16zZkzJ3bdtm3dfvvt2rdvnyTpD3/4g0477TS53W498cQT2rx5syTpqaee0ty5c0foawAAAKAveXl5Gj9+QizQvvba33XLLTcqHA7ryisvU2Fhoa699p910UUX64MP3pdlWbFAOlw+8YmZ+sMffq9wOKzm5matX79Op546c1g/ozf9Dr0GAgEtW7ZMS5cuVSgU0hVXXKEZM2bohhtu0Fe+8hVNnz5d9913n2666SYZhqHS0lLdc889sixLjz76qJYvX67m5mZNnjxZ3/3udxPxnQAAANCDu+/+tr73vf/Q00//txwOp+699z/kcDj0hS/cpFtvvVlZWdnKy8vXt761XB6PV4HAaH35yzfpBz/oPq/3uuuu7nI8fvx4ffvbfWe9xYsv1+7du3TddVepra1N8+Yt0Nlnn6s333x9WL9nTwy7p0m7KSrRc3qLi/NUWVmfsM872jE/LHHo68ShrxOL/k4c+jpx6Ov4DWlO79Hqg92HdO3ydapvCiW7KQAAABgiQm8vQm0R1dS3ak+QkV4AAIB0R+jtRcCTI0kKHmpKcksAAAAwVITeXngLsuWwTJW315EDAABA+iL09sI0DY0ucqv8ICO9AAAA6Y7Q24exxXkKHmSkFwAAIN0RevswpjhXwYNNiqRPVTcAAAD0gH2B+zDWl6vWtogO1bXIW5Cd7OYAAADE5eGHv6Oyss1qawtpz57dmjx5iiTpM5+5UhdffElc77juuqv15JNPx3Xvm2++rjvuWKZx4yZ0Of/P/3yDzj773IE1foQQevswtjhXkhQ82EToBQAAaeNrX7tDkrR//z59+cs3xR1eOxvoM8cff4J++MPHB/w5iULo7cOY4uiOHuUHGzV1kifJrQEAAOki9MHfFHr/L8Pyrn1Oh0Khttix8/g5ch535qDfd8UVi3TiidP04Yfv67HHntCzz/6P3njjNdXW1qqwsFD33/9dFRUV66yzZmrDhtf1s5/9RJWVFdq9e5fKyw9o4cLF+vznvxD35+3fv09f+9qXNWpUoVyuLM2bN19/+MMLqqk5pDPPnKPPfOZKPfjgfSovPyDLsnTjjbdo1qzZ+tnPfqKtW99WMHhAS5Z8VkuWfGbQ31ki9PapuDBHDstQkAoOAAAgg8yaNVv33vuA9uzZrV27dui//uvnMk1T993371q/fp2uuupzXe7ftu1DPfbYE6qvr9NnP3upliz5rPLz87vc8/777+q6667ucu4///MxSdKuXTv161//QGPGjNXatWtUURHUU0/9Wg6HQ3fddadOO22mrrzyc9q7d49uvvl6/eIXKyVJra0teuqpXw/Ldyb09sEyDfkKcyhbBgAABsR53JlDGo3tzOfLV0VF3bC8q8OJJ06TJI0fP0H/+q/LtGbNKu3atVNbt5Zp3Ljx3e4/7bSZcjqd8ni8KigoUENDfbfQ29v0hsbGRnk8Xo0ZMzZ27rjjpsrhiMbQN998TXfc8S1J0rhx43XiidP0zjtvd2nncKB6Qz8CHrfKKVsGAAAySFZWliTpvffe1bJl/yrbjujcc8/XnDnnyO6hapXL5Yr9bhhGj/fE83k9HUciR77LVjgc7vG5oSD09sPvyVEFZcsAAEAGeuutN3TqqZ/QpZdeocmTp+gf/3hVkUgkoW34xCdm6oUXVkmS9u7do7KyzTrppBnD/jlMb+hHwJND2TIAAJCRzj//Qn3jG7fr85+/UpblUElJqfbv3zeod/U0p/f88y/UBRdc2Odzt956u7773fu1du0aGYahO+74loqLiwfVhr4Y9kDHp5Ooqqq+hyHwkePz5evPr+3Uw796S7dfdapOoILDiBqJOUvoGX2dOPR1YtHfiUNfJw59HT/TNFRUlNfztQS3Je0EPDmSxHbEAAAAaYzQ2w9vfrYclkEFBwAAgDRG6O2H2V62jFq9AAAA6YvQGwfKlgEAAKQ3Qm8c/J7oSC9lywAAANIToTcOAa9bofayZQAAAEg/hN44+NsrOLCYDQAAID0ReuMQiIVe5vUCAACkI0JvHLwF2XJYJhUcAAAA0hShNw6mYchXmK3yakZ6AQAA0hGhN04Bj1vBQ4z0AgAApCNCb5wCXsqWAQAApCtCb5z8HsqWAQAApCtCb5wClC0DAABIW4TeOPkpWwYAAJC2CL1xomwZAABA+iL0xsk0DPk9OZQtAwAASEOE3gHwF+Yw0gsAAJCGCL0DEPDmKHiIsmUAAADphtA7AAHKlgEAAKQlQu8AxCo4MK8XAAAgrRB6ByDgcUuSytmOGAAAIK0QegfAU5AVLVtWTegFAABIJ4TeAYiVLWODCgAAgLRC6B2ggIeyZQAAAOmG0DtAfg9lywAAANINoXeAKFsGAACQfgi9AxSgbBkAAEDaIfQOkL+jbBnzegEAANIGoXeAYmXLCL0AAABpg9A7QJQtAwAASD+E3kGgbBkAAEB6IfQOQsDjpmwZAABAGiH0DoLfk6NQW0QHaylbBgAAkA4IvYPQUbYsyLxeAACAtEDoHYSAl7JlAAAA6SSu0LtmzRotWLBAc+fO1cqVK7tdf+mll7Ro0SJdfPHFuvPOO9Xa2ipJ2rdvn6655hpddNFF+tKXvqSGhobhbX2SFOZnyemgbBkAAEC66Df0lpeXa8WKFXr66ae1evVqPfPMM9q2bVvsemNjo+6991794he/0O9//3u1tLTo+eeflyTdc889uvrqq7Vu3TpNmzZNjz322Mh9kwQyDUP+QsqWAQAApIt+Q+/GjRs1a9YsFRYWyu12a968eVq3bl3sutvt1ssvv6zi4mI1NjaqqqpKBQUFCoVCeu211zRv3jxJ0pIlS7o8l+6itXoZ6QUAAEgHjv5uCAaD8vl8sWO/368tW7Z0ucfpdOqVV17R17/+dfn9fp111lk6ePCg8vLy5HBEP8Ln86m8vHxIjS0qyhvS84Ph8+X3eH7yuEK9/XG1ioryZJpGgluVuXrrbww/+jpx6OvEor8Th75OHPp66PoNvXYPtWgNo3vIO/vss/Xqq6/qkUce0fLly/X1r389rucGoqqqXpFI4mrj+nz5qqio6/FafralUFtEH3xUqaJR2QlrUybrq78xvOjrxKGvE4v+Thz6OnHo6/iZptHrIGm/0xsCgYAqKytjx8FgUH6/P3Z86NAhbdiwIXa8aNEivf/++/J6vaqvr1c4HJYkVVRUdHku3QUKo2XLmNcLAACQ+voNvbNnz9amTZtUXV2tpqYmrV+/XnPmzIldt21bt99+u/bt2ydJ+sMf/qDTTjtNTqdTM2fO1Nq1ayVJq1at6vJcuusoW0YFBwAAgNQX10jvsmXLtHTpUl166aVauHChZsyYoRtuuEFlZWXyeDy67777dNNNN+mSSy7Rjh07dPvtt0uS7r77bj377LNasGCBXn/9dd16660j/X0SpqNsGSO9AAAAqc+we5q0m6JSaU6vJN31xKvye3L05ctnJKxNmYw5S4lDXycOfZ1Y9Hfi0NeJQ1/Hb0hzetE7ypYBAACkB0LvEAQ8bgUPNimSPoPlAAAARyVC7xD4vTlqC0d0sLYl2U0BAABAHwi9QxDwRCs4sJgNAAAgtRF6hyDg6ajVy7xeAACAVEboHYKOsmVBRnoBAABSGqF3CEzDkL8wR+XVjPQCAACkMkLvEPk9OQoeIvQCAACkMkLvEAW8lC0DAABIdYTeIfJ7omXLqmubk90UAAAA9ILQO0QdZcuCVHAAAABIWYTeIaJsGQAAQOoj9A5RR9my8mrKlgEAAKQqQu8QmYYRreDASC8AAEDKIvQOA39hDlsRAwAApDBC7zAIeN2qONSkSISyZQAAAKmI0DsMAp4ctYVtVddRtgwAACAVEXqHgb+9bBkVHAAAAFIToXcYdJQtYzEbAABAaiL0DoPC/Cy5KFsGAACQsgi9w8A0DPkoWwYAAJCyCL3DJOBxU7YMAAAgRRF6h4nfk0PZMgAAgBRF6B0mlC0DAABIXYTeYRKgbBkAAEDKIvQOE39H2TIqOAAAAKQcQu8wiZUtY6QXAAAg5RB6h4lpGPJTtgwAACAlEXqHkZ+yZQAAACmJ0DuMApQtAwAASEmE3mHk7yhbVkvZMgAAgFRC6B1GsbJlh5jXCwAAkEoIvcMo4I2GXsqWAQAApBZC7zAaleeibBkAAEAKIvQOI8qWAQAApCZC7zALULYMAAAg5RB6h5mfsmUAAAAph9A7zAJeN2XLAAAAUgyhd5j5C3MkicVsAAAAKYTQO8xiZcuY1wsAAJAyCL3DrJCyZQAAACmH0DvMjPayZeVsUAEAAJAyCL0jIOBxK8hWxAAAACmD0DsC/F7KlgEAAKQSQu8ICHgoWwYAAJBKCL0jIOChbBkAAEAqIfSOAL8nWraM7YgBAABSA6F3BBTmueRymgoy0gsAAJASCL0jwDAM+QvdlC0DAABIEYTeERLw5FC2DAAAIEUQekcIZcsAAABSR1yhd82aNVqwYIHmzp2rlStXdrv+xz/+UYsXL9Yll1yim2++WTU1NZKkVatW6ayzztLixYu1ePFirVixYnhbn8I6ypZVUbYMAAAg6Rz93VBeXq4VK1boueeek8vl0pVXXqnTTz9dpaWlkqT6+notX75cv/3tbxUIBPSf//mf+sEPfqBvfetbKisr05133qmFCxeO+BdJNR1ly4IHm+QrzElyawAAAI5u/Y70bty4UbNmzVJhYaHcbrfmzZundevWxa6HQiEtX75cgUBAknT88cdr//79kqSysjKtWrVKl1xyiW677bbYCPDRgLJlAAAAqaPfkd5gMCifzxc79vv92rJlS+zY4/HoggsukCQ1Nzfr8ccf17XXXitJ8vl8uvHGGzVjxgw98sgjuvfee/Xwww8PurFFRXmDfnawfL78QT1XXJynLJel2ua2Qb/jaERfJQ59nTj0dWLR34lDXycOfT10/YZe2+6+EMswjG7n6urqdPPNN2vq1Km67LLLJEk/+tGPYtevv/76WDgerKqq+oQuDPP58lVRUTf450flaOfemiG942gy1P5G/OjrxKGvE4v+Thz6OnHo6/iZptHrIGm/0xsCgYAqKytjx8FgUH6/v8s9wWBQV199taZOnar7779fUjQEP/nkk7F7bNuWw9Fvxs4oAW8OWxEDAACkgH5D7+zZs7Vp0yZVV1erqalJ69ev15w5c2LXw+GwvvjFL2r+/Pn65je/GRsFdrvdeuKJJ7R582ZJ0lNPPaW5c+eO0NdITX4PZcsAAABSQb9Dr4FAQMuWLdPSpUsVCoV0xRVXaMaMGbrhhhv0la98RQcOHNA777yjcDisF198UZI0bdo03X///Xr00Ue1fPlyNTc3a/Lkyfrud7874l8olQQ8boUj0bJlVHAAAABInrjmGyxatEiLFi3qcu6nP/2pJGn69Ol67733enxu5syZev7554fYxPTVUbas/GAjoRcAACCJ2JFtBHWULQsyrxcAACCpCL0jqDDPJZfTVHk1oRcAACCZCL0jyDAM+QvdbFABAACQZITeERbw5jC9AQAAIMkIvb2wW5vU8MFrQ35PwONWxaEmhSORYWgVAAAABoPQ24vwgfdV/usHFa7cOaT3+D05CkdsVde2DFPLAAAAMFCE3l6YnnGSpHD5tiG9p3PZMgAAACQHobcXRl6xrNzCIYfejrJlVHAAAABIHkJvLwzDUNb44xUObh/SewrzXMpyWixmAwAASCJCbx+yxx0nuzaoSFPtoN9hGIb8nhymNwAAACQRobcP2eOPlzT0eb3R0MtILwAAQLIQevvgGj1FMixFhryYza1KypYBAAAkDaG3D6YzS2bxxCHP6w20ly2romwZAABAUhB6+2EFShUOfiw70jbod/jby5YFmdcLAACQFITeflj+EincqkjVnkG/I+ClbBkAAEAyEXr7YY0+VtLQFrONyo2WLaOCAwAAQHIQevth5HpluAsVDg4+9HaULaNWLwAAQHIQevthGEZ0Xm/50BezUbYMAAAgOQi9cbACJbLrKhRprBn0O/yULQMAAEgaQm8cLH+pJA1pigNlywAAAJKH0BsHs3iSZFqKDGGKQ0cFh2A1i9kAAAASjdAbB8Phklk8aUgVHDpq9TKvFwAAIPEIvXGy/KUKV3wsOzy4TSooWwYAAJA8hN44WYFSKRxSpGrXoJ6nbBkAAEDyEHrjZAVKJEnh4BDm9VK2DAAAICkIvXEy84pk5HqGNK834KVsGQAAQDIQegfA8pcMbTFbIWXLAAAAkoHQOwBW4FjZ9VWKNB4a1POULQMAAEgOQu8AxOb1DnK0N0DZMgAAgKQg9A5AdJMKx6BDb0GuS1kuypYBAAAkGqF3AAzLKdM3edA7sxmGoUAhZcsAAAASjdA7QJa/ROHKwW9S4ffkqJw5vQAAAAlF6B2g6CYVbYPepCLgdauyppmyZQAAAAlE6B0gK1AqafCL2fye9rJlNc3D2SwAAAD0gdA7QGauR0audwgVHNrLljGvFwAAIGEIvYNgBUopWwYAAJBGCL2DYAVKZDdUK9JwcMDPxsqWsZgNAAAgYQi9gzCUeb2xsmWHGOkFAABIFELvIJhFkyRr8JtU+L1uRnoBAAASiNA7CIblkFV8jMLBwW1SEfDkULYMAAAggQi9g2QGShSp2CE7HBrws5QtAwAASCxC7yBZgVIp0qZI5c4BP9tRtowKDgAAAIlB6B2kw4vZBj7FoaNsGbV6AQAAEoPQO0imu1BGXpHCwYEvZqNsGQAAQGIReocguknFwEd6DcNQwJPD9AYAAIAEIfQOgRUojW5SUV814Gf9HreCBxnpBQAASARC7xBY/hJJg5/XS9kyAACAxCD0DoFZNFGynIPapIKyZQAAAIlD6B0Cw3LI8h0zqMVslC0DAABIHELvEJn+EkUqd8puax3QcwFve+ilggMAAMCII/QOkTW6VIqEB7xJRYHbqSyXRa1eAACABIgr9K5Zs0YLFizQ3LlztXLlym7X//jHP2rx4sW65JJLdPPNN6umpkaStG/fPl1zzTW66KKL9KUvfUkNDQ3D2/oUYPnbN6kY4BQHypYBAAAkTr+ht7y8XCtWrNDTTz+t1atX65lnntG2bYcDXn19vZYvX67HH39cv/vd73T88cfrBz/4gSTpnnvu0dVXX61169Zp2rRpeuyxx0bumySJ6R4lI983yAoObpVTtgwAAGDE9Rt6N27cqFmzZqmwsFBut1vz5s3TunXrYtdDoZCWL1+uQCAgSTr++OO1f/9+hUIhvfbaa5o3b54kacmSJV2eyyRWoETh8m2ybXtAz/k9OaqqaVZbmLJlAAAAI8nR3w3BYFA+ny927Pf7tWXLltixx+PRBRdcIElqbm7W448/rmuvvVYHDx5UXl6eHI7oR/h8PpWXlw+psUVFeUN6fjB8vvx+76mZcpKqtv1d3qwWOUb5+r2/Q+lEr8Kbdsp2WPIVJ/67paJ4+hvDg75OHPo6sejvxKGvE4e+Hrp+Q29Po5eGYXQ7V1dXp5tvvllTp07VZZdd1mPA7em5gaiqqlckMrDR1KHw+fJVUVHX733h3AmSpOC7m+UsOT3u9+c4ov3x3vZKOQc4SpyJ4u1vDB19nTj0dWLR34lDXycOfR0/0zR6HSTtd3pDIBBQZWVl7DgYDMrv93e5JxgM6uqrr9bUqVN1//33S5K8Xq/q6+sVDoclSRUVFd2eyxRm0XjJcg14kwrKlgEAACRGv6F39uzZ2rRpk6qrq9XU1KT169drzpw5sevhcFhf/OIXNX/+fH3zm9+MjeY6nU7NnDlTa9eulSStWrWqy3OZxDAdsvzHDDj0FridynZZVHAAAAAYYf1ObwgEAlq2bJmWLl2qUCikK664QjNmzNANN9ygr3zlKzpw4IDeeecdhcNhvfjii5KkadOm6f7779fdd9+tO++8Uz/+8Y81ZswYPfLIIyP+hZLF8peodcuLsttaZThccT1jGIb8nhxq9QIAAIywfkOvJC1atEiLFi3qcu6nP/2pJGn69Ol67733enxu3Lhx+uUvfznEJqYHM1Aq2WsVrtwhx+jj4n4u4HFrZznzdAAAAEYSO7INEysQ3aQiMsApDn5PjioPUbYMAABgJBF6h4mZUzCoTSoCHrcitq2q2uYRahkAAAAIvcPICpQOeJOKgDdHklRezbxeAACAkULoHUZWoFR2U43s+sr+b27n90TLlgXZjhgAAGDEEHqHUce83oFMcaBsGQAAwMgj9A4j0ztecmQNqF6vYRgKeNwqZ6QXAABgxBB6h5FhWrJ8xygcHNhiNmr1AgAAjCxC7zCzAqWKVO6S3dYa9zMBL2XLAAAARhKhd5hZgRLJDitc8XHcz/gLKVsGAAAwkgi9w8z0l0jSgOb1UrYMAABgZBF6h5mZUyCjIDCgndkC7WXLWMwGAAAwMgi9I8AKlCgc3B73JhX57WXLWMwGAAAwMgi9IyC6SUWt7LqKuO6nbBkAAMDIIvSOAGuQ83qDzOkFAAAYEYTeEWB6x0vO7AHtzOb35KiyhrJlAAAAI4HQOwIOb1IxsMVsEdtWVQ1lywAAAIYboXeEWIFSRap2yw61xHW/39NetozFbAAAAMOO0DtCoptUROLepIKyZQAAACOH0DtCLH+pJMU9xSHf7VROlsViNgAAgBFA6B0hRnaejFGjFYlzMZthGPIXulV+iJFeAACA4UboHUFWoETh8m1xb1JB2TIAAICRQegdQZa/VHZznezaYFz3+z1uypYBAACMAELvCLIC7fN649ykIuDJoWwZAADACCD0jiDTMy66SUUwvnm9VHAAAAAYGYTeEWSYpiz/lLhHeqnVCwAAMDIIvSPMCpQqUr1bdqj/KQuULQMAABgZhN4RZvlLJduOa5MKwzDk97iZ3gAAADDMCL0jzAqUSBrYYrYg0xsAAACGFaF3hBlZuTILxwxgXi9lywAAAIYboTcBTH+pIsGP4tqkoqNsWSVlywAAAIYNoTcBrEBJ+yYV5f3e21G2LMi8XgAAgGFD6E2Aw5tU9F+v1+9tL1tGBQcAAIBhQ+hNANMzVnLmxDWvNz+nvWwZi9kAAACGDaE3AQwj/k0qKFsGAAAw/Ai9CWIFShU5uEd2a/8juAFPDqEXAABgGBF6E8QKlMS9SQVlywAAAIYXoTdBLH/8m1QEPDmybVG2DAAAYJgQehPEyMqV6RkbX+j1UrYMAABgOBF6E8jylyoc3N7vJhV+D2XLAAAAhhOhN4HMQInU0iC75kCf90XLljlYzAYAADBMCL0JdHiTir6nOETLluVQqxcAAGCYEHoTyCwcI7ncce3MRtkyAACA4UPoTaDYJhXBeCo4ULYMAABguBB6E8wKlCpSvbffTSr8lC0DAAAYNoTeBIvO67UVDn7U530dZcvKq5niAAAAMFSE3gSz/FMkGf0uZgu0ly1jMRsAAMDQEXoTzHC5o5tU9DOvN4+yZQAAAMOG0JsEVqBE4fLtsu3eF6kZhtFewYGRXgAAgKEi9CaB5S+VWhsVOdT3JhXRWr2M9AIAAAwVoTcJzNHRTSoi/c7rpWwZAADAcCD0JoE5arSUldvvvN6AN1q2rOIQUxwAAACGgtCbBNFNKkr63ZnN74mWLaOCAwAAwNDEFXrXrFmjBQsWaO7cuVq5cmWv991xxx167rnnYserVq3SWWedpcWLF2vx4sVasWLF0FucIaxAiSIH98lu7X3ObkfZMhazAQAADI2jvxvKy8u1YsUKPffcc3K5XLryyit1+umnq7S0tMs9d999tzZt2qTTTz89dr6srEx33nmnFi5cODKtT2OW//AmFY7x03q8Jy/HKXeWg8VsAAAAQ9TvSO/GjRs1a9YsFRYWyu12a968eVq3bl2Xe9asWaPzzz9f8+fP73K+rKxMq1at0iWXXKLbbrtNNTU1w9v6NBbPJhWGYchP2TIAAIAh63ekNxgMyufzxY79fr+2bNnS5Z7rr79ekvTGG290Oe/z+XTjjTdqxowZeuSRR3Tvvffq4YcfHnRji4ryBv3sYPl8+SP05ny1+ifIOrijz8+YOKZA7+88OILtSC1Hy/dMBfR14tDXiUV/Jw59nTj09dD1G3pt2+52zjCMuF7+ox/9KPb79ddfrwsuuGAATeuuqqpekUj39owUny9fFRV1I/Z+2ztFTR+9qmCwRobR86D7qBynggcbtf9AjRxWZq87HOn+xmH0deLQ14lFfycOfZ049HX8TNPodZC03xQVCARUWVkZOw4Gg/L7/f1+aF1dnZ588snYsW3bcjj6zdhHFStQIrU2KXJwf6/3ULYMAABg6PoNvbNnz9amTZtUXV2tpqYmrV+/XnPmzOn3xW63W0888YQ2b94sSXrqqac0d+7cobc4g1iB6GLAvur1BihbBgAAMGRxjfQuW7ZMS5cu1aWXXqqFCxdqxowZuuGGG1RWVtbrc5Zl6dFHH9Xy5cs1f/58bd26VbfffvuwNj7dGe2bVPS1M5ufsmUAAABDFtd8g0WLFmnRokVdzv30pz/tdt+DDz7Y5XjmzJl6/vnnh9C8zGYYRr+bVHSULSunbBkAAMCgZfbKqDRgBUoVObRPdktDj9c7ypYxvQEAAGDwCL1Jdnheb++jvQGvW+XVjPQCAAAMFqE3ySz/FMkw+pziEPDkqKq2WW3hSAJbBgAAkDkIvUlmOLNlesf3uTOb30PZMgAAgKEg9KYAy1+qcPAj2XbPI7kdZcuo4AAAADA4hN4UYAVKpVCTIgf39Xg94KVWLwAAwFAQelOAFSiRpF6nOORmOyhbBgAAMASE3hRgFARkZOf3upjNMAwFvDkKUsEBAABgUAi9KcAwDJn+KYqUf9jrPX6Pmzm9AAAAg0ToTRFWoFSRmgOym+t7vE7ZMgAAgMEj9KaI/japCHjclC0DAAAYJEJvirB8x7RvUtHzYja/J0cSZcsAAAAGg9CbIqKbVEzsfaS3o2wZi9kAAAAGjNCbQqxASXSTikj3ebt5OU7lZjtUzvQGAACAASP0ppDoJhXNihzc2+N1v4eyZQAAAINB6E0hscVsvczrDVC2DAAAYFAIvSnEyPdFN6kI9r6Yraq2WaE2ypYBAAAMBKE3hRiGIStQ2uvObB1lyyprGO0FAAAYCEJvijEDJbJ72aTC720vW1ZN6AUAABgIQm+Ksfwdm1R0n+IQ8LSXLTvIYjYAAICBIPSmmOgmFabCB7qH3ljZMhazAQAADAihN8UYziyZRRN63aTC73Ez0gsAADBAhN4UZPlL2zepCHe7FvDkMNILAAAwQITeFGSNLpXaWhSp3tPtGmXLAAAABo7Qm4IOL2brPsWBsmUAAAADR+hNQUZ+sYycgh53ZqNsGQAAwMARelNQX5tUULYMAABg4Ai9Kcr0l8quLVekqbbLecqWAQAADByhN0VZgRJJUqSHeb1+j1vljPQCAADEjdCboqKbVFg9T3Hw5ijISC8AAEDcCL0pynC4ZBZP7HkxWyFlywAAAAaC0JvCLH+JwhXdN6kIeKNlyyoOMdoLAAAQD0JvCrMCpVJbqyLVu7uc93uiZcuY4gAAABAfQm8K61jMduQUh46yZSxmAwAAiA+hN4UZecUyckZ1W8zWUbaMkV4AAID4EHpT2OFNKnpYzEbZMgAAgLgRelOcFSiVXVfRbZOKgDeHrYgBAADiROhNcWagVFLP83qrKVsGAAAQF0JvirOKJ0mmpcgRodfvyZEtypYBAADEg9Cb4gyHS2bRJIWP2I6YCg4AAADxI/SmAStQonDwY9mRttg5avUCAADEj9CbBqxAqRRuVaRqT+xcR9myckIvAABAvwi9acDqbTGb160g0xsAAAD6RehNA0auV4a7sFvo9XsoWwYAABAPQm8aiG1SEeytbFk4SS0DAABID4TeNGEFSmTXVSrSeCh2LhArW9actHYBAACkA0JvmrD8HfN6D5cu81O2DAAAIC6E3jRhFk+STEeXeb0Bb7RsGfN6AQAA+kboTROGwyWzeJIinTapyM2Oli0LsisbAABAnwi9acQKlCpc8bHs8OFNKgJet8qrmd4AAADQF0JvGrECJVI4pEjVrti5gCeHXdkAAAD6EVfoXbNmjRYsWKC5c+dq5cqVvd53xx136Lnnnosd79u3T9dcc40uuugifelLX1JDQ8PQW3wUiy1mC3ZdzEbZMgAAgL71G3rLy8u1YsUKPf3001q9erWeeeYZbdu2rds9X/ziF7Vu3bou5++55x5dffXVWrdunaZNm6bHHntseFt/lDHzvDJyvV0Xs7WXLQtStgwAAKBX/YbejRs3atasWSosLJTb7da8efO6hds1a9bo/PPP1/z582PnQqGQXnvtNc2bN0+StGTJkm7PYeCsQMkRFRyiZcvYjhgAAKB3jv5uCAaD8vl8sWO/368tW7Z0uef666+XJL3xxhuxcwcPHlReXp4cjuhH+Hw+lZeXD6mxRUV5Q3p+MHy+/IR/Zl8OTTlJ1R+9Jk92SI58r3JysyRJDa2RlGvrYGTCd0gX9HXi0NeJRX8nDn2dOPT10PUbem3b7nbOMIx+XzzY5/pSVVWvSKT7e0eKz5evioq6hH1ePMK54yVJwXfeknPKJyVJeTlOfbTnUMq1daBSsb8zFX2dOPR1YtHfiUNfJw59HT/TNHodJO13ekMgEFBlZWXsOBgMyu/39/uhXq9X9fX1CoejC6wqKirieg59i21S0WUxWw5lywAAAPrQb+idPXu2Nm3apOrqajU1NWn9+vWaM2dOvy92Op2aOXOm1q5dK0latWpVXM+hb4bllOmb3G0xG3N6AQAAehfXSO+yZcu0dOlSXXrppVq4cKFmzJihG264QWVlZX0+e/fdd+vZZ5/VggUL9Prrr+vWW28drnYf1Sx/iSKVO2KbVAQ8blXXtlC2DAAAoBf9zumVpEWLFmnRokVdzv30pz/tdt+DDz7Y5XjcuHH65S9/OYTmoSdWoFShshcVqdopy18if6eyZeOKc5PdPAAAgJTDjmxpyAq0b1LRPsUhVraMeb0AAAA9IvSmITPXIyOvSOHy6GI2vydHklTOdsQAAAA9IvSmKct/eJOK3Gyn8nKcLGYDAADoBaE3TVmBUtkN1Yo0HJQUreCQriO9EdvWvsoGhRNYgxkAABxd4lrIhtTTeV6vOeWT8nty9MHuQ8lt1AA0Nrdp645qbdlWqbKPqlTbGNLcT03UVeeVJrtpAAAgAxF605RZNFGynAqXb5NzyicV8Lj1963lCrWF5XRYyW5eN3b7aO6Wj6q0ZVuVtu2tUThiKzfboWlTimQahl76xy6NL3Lr0yePTXZzAQBAhiH0pinDcsgqnhzbmc3vTb2yZS2hsN7beVBbtldpy/YqVdU2S5Im+PN00ekTNaOkSFPGFsgyTUUitppaw3rqpQ80MZCvSaPZYxwAAAwfQm8aMwMlCr39R9nhkAKew2XLkhl6Kw41xULue7sOKtQWUZbT0omTPbp49iTNmFIkb0F2t+dM09Btn/uEvvzQn/TYqjLdfd0n5c52JuEbAACATEToTWNWoFShLesUqdwpv2eSpMSXLWsLR/ThnhqVba/S5u2V2l8VrSDh9+To7FPG6uSSYh03oVBOR/9rJkflZelLl07Td1a+qSdeeFf/evl0mYYx0l8BAAAcBQi9aazzYrbcQGnCypbV1LdE5+Zur9I7O6rV1BKWwzJ0/IRCnX3KOJ1cUhTbMGOgSseN0mfPK9X//PFDrXt1lxbMmjTMrQcAAEcjQm8aM92F7ZtUtO/MNkJlyyK2rY/312rLtipt+ahKOw/USZI8+Vn65NSATi4p0gmTPcp2Dc8fpws+MV7b99bot69s1zFjCnTCJM+wvBcAABy9CL1pzgocq/CB9yVJfo9bH+w+OCzvbWgOaevH1dq8rUpvf1ylusaQDEMqGTdKS+ZM0YySIk3w58kYgekHhmHouvlTtTtYr5+sflt3//On5MnPGvbPAQAARw9Cb5qzAiVq2/53ReqrFPDk6O9bDwyqbJlt29pb0VFSrFLb9tYqYkdLik0vKdKMKUWaNqVIeTmJWVyW7XLo5sum69v/3+v68eq39fWrTpXDYi8VAAAwOITeNHd4Xu92+b0To2XLDjZpnC+v32dbWsN6d+dBbdleqS0fVam6tkWSNNGfpwVnTNSMkmJNGVMg00zOYrJxxbm6bv5U/eR3W/WbP2/Xlecfm5R2AACA9EfoTXNm0QTJcilcvk2BY6ZK6jv0Bg82diopdkht4YiyXJZOmuzVJWcWafqUopSaSnD6iQFt21Oj9a/tVum4UZo51Z/sJgEAgDRE6E1zhumQ5ZuscHCbAqflSOpatqwtHNEHuw/Fgu6B6mh1h4DXrfNOG6fpJUU6bnx8JcWS5Z/OL9XHB2r187Xvarw/T6MHWRkCAAAcvQi9GcAKlKq17EXlOWzl5Tj10f5a/WXzPm3ZXqWtO6rV0tpeUmyiR+eeNk4zSopim1mkA4dl6uZLp2n5L17Tj54v07eunaksV+pttQwAAFIXoTcDmIESaXNYkcqdCnhz9Pp7Qb3+XlCe/CydcWJA00uKdOIkb1oHRW9Btm665CQ98sxb+u8X39P1C08ckcoRAAAgMxF6M4Dlb1/MFtymy+fM0sf7azVtSpHG+3IzKhiedIxXiz99jFb99WOVji/UuaeOS3aTAABAmiD0ZgDTPUpGvk/hA9s0dcZ8Tc3gzRwWzp6s7Xtr9T9//ECTR+frmDEFyW4SAABIA6m7egkDYgVKFA5ul23byW7KiDINQzcsOlGjcrP02PNlqm8KJbtJAAAgDRB6M4TlL5XdeEh2fVWymzLi8nKcuvmyaappaNXja7YqkuFBHwAADB2hN0NYozs2qdiW5JYkxjFjCnT1Bcfp7Y+q9cLfdiS7OQAAIMURejOE6Z0gOVwKB7cnuykJc/YpY3XGSaO1esPHevvjzB/hBgAAg0fozRCGacnyHXPUjPRKkmEYWnrR8Rrny9Xjv3tHVTXNyW4SAABIUYTeDGIFShWp3CW7rTXZTUmYLKelmy+brrZwRD9e/bbawpFkNwkAAKQgQm8Gsfylkh1WuHJHspuSUKO9bn3h4hP00b5aPfO/R89INwAAiB+hN4OYgRJJUuQomuLQ4RPH+zXvUxP0v2/u0d+3Hkh2cwAAQIoh9GYQM6dARoFfoR1vym5tTHZzEu7ys0t07PhRenLde9pbUZ/s5gAAgBRC6M0wrmlzFSnfroZnv6HQ9n9k/GYVnTksU19cPE3ZLod+9PzbamppS3aTAABAiiD0ZhjXtLlyX3qXDPcoNf/vY2r6w8OK1AaT3ayE8eRn6YuXnKTyg4168g/vHVWhHwAA9I7Qm4Es/xS5L71bWbOvUbh8mxp+/Q21vLladvjo2LJ36iSPrji7RK+9F9QfX9+T7OYAAIAUQOjNUIZpyjVtrnI/+4Ack05T6+vPq/E3d6lt37vJblpCXHT6RJ16bLGe/dM2bdtTk+zmAACAJCP0Zjgz16OcC25Wzvz/JzsSVtML31HTnx5XpKk22U0bUYZh6AsXn6Cigmz9ePXbqm04emoXAwCA7gi9RwnHhBnK/cz9cp26SG3bX1XDM3eq9d0/y7YzdzMHd7ZTN182TfVNIf3kd1sViTC/FwCAoxWh9yhiOFzK+uTlcl9+n6yiiWr565NqXH2/wlW7kt20ETMxkK/PXXic3t15UKs2fJTs5gAAgCQh9B6FLM9Y5Sy8Q9nn3CC7NqjG55ar+e+/kh1qTnbTRsSnZ4zVnJPH6IWNO/XWtspkNwcAACQBofcoZRiGnMedqdzPPiDn8Z9WaMu6aG3fHW9kZJmva+Yep4mBPD2x5h1VHGpKdnMAAECCEXqPckZ2nrLn/LPcl3xTRpZbzet/oKYX/1ORuswaEXU6LN182XRJ0mPPv61QWzjJLQIAAIlE6IUkyRp9rNxLlivr9H9SeN870dq+b62VHcmcXc38hTm6ftGJ2llep5UvfZjs5gAAgAQi9CLGMB1ynTw/Wtt3/DS1/uNZNf52udoOfJDspg2bU0qLdfEZk/SXzfu0Ycv+ZDcHAAAkCKEX3Zh5Rcq58CvKufCrslsb1fS7/1DzKz+X3Vyf7KYNi0s/fYxOmOTRL9e/r13ldcluDgAASABCL3rlmHyqcj/7H3LOmK/QBxvU8MydCr3/17Rf6GaZpm665CTlZjv02PNvq7H56NieGQCAoxmhF30ynNnKnvVPcl9+j4zC0Wp+5WdqeuFBhQ/uTXbThqQg16UvXTpNVbXN+tnv3037IA8AAPpG6EVcLO8EuS/5hrLm/LPC1XvU+Jt/V8s/fiO7rSXZTRu0Y8cX6jPnlur/PqzUun9k7gYdAABAciS7AUgfhmHKNfVsOSadqpZXn1HrWy8otP3vyj7zWjkmnpzs5g3K3JnjtW1vjX775480ZUyBjp/oSXaTAADACGCkFwNm5hQo55wblLPwDhmWU03rVqjppR8qUl+d7KYNmGEY+uf5U+X35OjHq7fqUH36jlwDAIDeEXoxaI6xJ8h9+X1yffJyte3arIZff0OtZS/KjqTXxg85WQ7dctk0Nbe26b9Wva22cCTZTQIAAMOM0IshMSyHsk5dpNzP3C9r9HFq2fQ/anz+HoWDHyW7aQMyzpenz180VR/sqdFzr6RX2wEAQP8IvRgWZoFfORctU/YFt8huqlXjqvvUvOG/Zbc0JLtpcTvjpNE697RxWvePXXrj/YpkNwcAAAwjQi+GjWEYck75pHI/+4Cc0y5Q6N0/qeHZf1No29/TpiTYlecdq2PGFOjna99ReXVjspsDAACGCaEXw85w5Sh79jVyX3a3jLwiNb/8X2pa+5AiNQeS3bR+OR2mbr50mizT1I+eL1NLKL3mJwMAgJ7FFXrXrFmjBQsWaO7cuVq5cmW36++++64uv/xyzZs3T9/85jfV1tYmSVq1apXOOussLV68WIsXL9aKFSuGt/VIaVbxZLkX36WsM69VOPiRGn79LbW8/rzsttZkN61PRaOydeOiE7W3okG/fPH9tBmlBgAAveu3Tm95eblWrFih5557Ti6XS1deeaVOP/10lZaWxu65/fbb9e1vf1unnHKKvvGNb+jZZ5/V1VdfrbKyMt15551auHDhiH4JpC7DNOU66Xw5jvmEWjb9Sq1vrm6v7btUjvEnJbt5vZo2pUiXnHWMVm/4WKXjR+mcU8Ylu0kAAGAI+h3p3bhxo2bNmqXCwkK53W7NmzdP69ati13fu3evmpubdcopp0iSlixZErteVlamVatW6ZJLLtFtt92mmpqakfkWSHmmu1A5539ROQtuk2ypae331PTyfynSeCjZTevVojMna9oxXj390gf6eH9tspsDAACGoN+R3mAwKJ/PFzv2+/3asmVLr9d9Pp/Ky8tjv994442aMWOGHnnkEd177716+OGHB93YoqK8QT87WD5ffsI/M6P5zlBk+id0aOPzOrTxOUV2b5H33GuUf+rc6OUU6+87r/uUlj36in7yu6169P+do3y3K9lNGjap1teZjL5OLPo7cejrxKGvh67f0NvTfEbDMOK6/qMf/Sh27vrrr9cFF1wwqEZ2qKqqVySSuPmVPl++KirqEvZ5R5UTFih3zGlq/tt/q3LdT1X9xh81ev71qstOvWkENy06SQ889YYefPIf+soVM2R2+vOfrviznTj0dWLR34lDXycOfR0/0zR6HSTtd3pDIBBQZWVl7DgYDMrv9/d6vaKiQn6/X3V1dXryySdj523blsPRb8bGUcQsHK2cBbcr+7ybZNdXad9/f1ONax9SuHxbspvWxZSxBbr6gmO1ZXuVfr9pZ7KbAwAABqHf0Dt79mxt2rRJ1dXVampq0vr16zVnzpzY9XHjxikrK0tvvPGGpGjFhjlz5sjtduuJJ57Q5s2bJUlPPfWU5s6dO0JfA+nKMAw5S89Q7pXfk/f8pYpU7lTj6m+r8Q+PpNSubuecOk6zTgpo1V8+0tYd1cluDgAAGKC4RnqXLVumpUuX6tJLL9XChQs1Y8YM3XDDDSorK5MkPfTQQ3rggQc0f/58NTU1aenSpbIsS48++qiWL1+u+fPna+vWrbr99ttH/AshPRnOLBXOWqzcq74n16c+o3BwuxpX3avGdSsUrtyR7ObJMAx9ft5UjS3O1U9Wb1V1bXOymwQAAAbAsNOoCClzejNb5/62W5vUuvWPat2yTmppkGPSqXJ94lJZxZOS2sb9VQ267/97XeOKc3XHNafJYaXn/i782U4c+jqx6O/Eoa8Th76O35Dm9ALJYLhylHXqIuVd9T25Zl6mtv3vqfG5u9W0/gcKV+9OWrvGFOXqXxacoO37avXsy6k19xgAAPSOlWVIaYbLrazTFst10gVqLVuv1rL1atvxhhxTPinXaZfK8ia+2sPMqX5d+MkJWv/abpWMG6VPneDvUtEEAACkHkIv0oKRlausmZfJNW2uWsteVOvbL6nto9flKPmUXJ9YLKtwbELbc8U5Jfpof61+8rutevx3W+VwmHJaZvtPQw7LlNNhyhE7d/g4+tOI3t/rfcbh+3u4p6fPcTpMWaZBAAcAoAeEXqQVIztPWZ+8XK7p89S6ZV17+P2HHCWzlHXaYpmFoxPSDodl6stLpmvDlv1qag2rLRxRW1tEbeGIQm0RhcIRtYVthTrOhSNqbGmL3ReK/Tx8z3AwJDk6heCOYNw1LJsqmVCoTx7n0wR/4jd8AQAgGVjI1gcmjifWYPo70lSr0JZ1at36RykckqN0trJOu0TmqMAItXJk2LatcMQ+HJg7Bei2sK1Q7PfDoTnUzz29BfDWtrB2ldcr1BbRlLEFOvvksfrUCQFluaxkd0NG4u+RxKK/E4e+Thz6On59LWRjpBdpzcwpUNbpn5VzxkVq3bxWoa3/q7Ztm+Q49kxlnbZIZoG//5ekAMMw5Ggflc1JwOdlubO05pVteuWtvfrFH97Tr17+ULNOHK2zTxmriQG2ugQAZB5CLzKCmVOg7FlXyjXjIrW+tVahd19W24cb5Tz+TLlOXSQz35fsJqaUglyXLvzkBM2dOV4f7qnRK2/t01+37Nef/m+vjhmTr7NPGadPneBXtou/IgAAmYHpDX3gPyck1nD2d6ThoFrf+r1C7/5Zki3n8Z+Oht+8omF5f7rrqa/rm0LatPWAXnlrn/ZVNijbZWnWSaN19sljNWk0o7+Dxd8jiUV/Jw59nTj0dfyY3oCjjpnrUfaZn5Pr5AVqfesFhd57RaH3N8g5dY5cpyyUmedNdhNTTl6OU3NnTtAFnxiv7Xtr9cpbe/W3sv368//t1aTR+TrnlOjc35ws/toAAKQfRnr7wL9ZJdZI9nekvkqt/7dGoff+KpmGnFPPkeuUi2Xmekbk81JdvH3d0BzS37eW689v7dXeigZluSzNOjGgs08Zq8mjCxLQ0vTH3yOJRX8nDn2dOPR1/BjpxVHPzCtS9qevk+uUi6Ph952XFXrvFTlPOFeuUxbIdBcmu4kpKTfbqfM/MV7nnTZO2/dFR383vR2dAjEpkK+zTxmr009k9BcAkPoY6e0D/2aVWIns70htUC1vrlHbh3+TTIecJ50n18kLZOYcHaOXQ+nrxuaQNm0t1ytv7dWeigZlOS2dfqJfZ58yTpNH57M5xhH4eySx6O/Eoa8Th76OHyO9wBHMAr9yzvmCIqcuVMubv1Oo7EWF3nlZrpMukPPk+TKzWbjVG3en0d+P9tfqlbf26e/vlOsvm/droj+vffR3tNzZ/PUCAEgdjPT2gX+zSqxk9nfk0H61vPk7tW37u+TMkuukC+SacZGM7MzcsWy4+7qxuU2vvnNAf35rn3YH6+VymvrUCdG5v1PGFBzVo7/8PZJY9Hfi0NeJQ1/Hj5FeoB9m4RjlnHeTwqcuUuubq9X61u/VuvWPck2bGw2/WbnJbmJKc2c7dO5p43XOqeO040CdXnlrr159J6gNW/ZrvC86+nvGSQG5s53JbioA4CjFSG8f+DerxEql/g5X71Xrm6vU9tFrkjNHrukXyjX9wowJv4no66aWNr36TrTyw67yerkcpj55QnTub8nYo2f0N5X+XB8N6O/Eoa8Th76OHyO9wABZ3nHKueAWhat2q/WNVdHR37dfkmvGPLmmXSjDlYjNgtNbTpZD55w6rn309/Dc37+VHdA4X67OPnmszpg2WrmM/gIAEoCR3j7wb1aJlcr9Ha7cqdY3Vqlt5/9JWblyzbhIrpMuSNvwm6y+bmpp0z/eLdcrb+3TjgN1cjpMfWpq++jvuMwc/U3lP9eZiP5OHPo6cejr+PU10kvo7QN/yBIrHfo7XLFDLW88r/CuzTKy8uQ84ZzolAdDiv4fo/339uNYiDMO39P5nHT4uP2n0fkew1CnF/Z83ujhc7udN9rfG/3dMyagWsMrw5k1lO4Ykp0H6vTK5n36+9YDam4Na1xxruacMlazM2z0Nx3+XGcS+jtx6OvEoa/jR+gdJP6QJVY69Xc4+JFa3lil8O4tyW7KEBgyCvyyiibI9E6QWTRelneijPwiGYaZsFY0t7bpH+8G9cpbe/Xx/ujo78zj/Tr7lLE6dvyotB/9Tac/15mA/k4c+jpx6Ov4MacXGGaWf4rc8/+f7LYWKRJpP2tLnf8dsv13u/N5247e1+l6z+ft9sMe7rE73trpni73dj7f87l8R4sOfvyBItW7Fa7erbaP3zj8vDNbpne8rKKJ0Z/eCTK940dsKke2y6E5J4/VnJPHaueBOv1l8z5t2npAm7Ye0Jgit84+ZZxmTxutvJzMGf0FACQeoRcYAsPR//SAVBynzPXlq9F7QuzYDjUrcnCvwlW7FanerUjVboW2bZJam2L3GPm+TqPCE2R5J8go8A3rqPCk0fm6dvTx+uy5pdG5v5v36Vf/+6F+8+ftmnm8TydM9miiP19ji3PldCRuNBoAkP4IvQBkOLNl+Utk+Uti52zblt1QrUjVrvYwvEeRql3RxXwdI8iOrMOjwUXjZRZNlOUdL8PlHlJ7slyWPn3yWH365LHaVd4x+luuv79TLkmyTENjinI1MZCnif48TQjka4I/j9FgAECvCL0AemQYhoy8Ipl5RXJMOjV23m5rUeTgPoWrdrUH4d0Kffya9N6fDz+bVxSbHhEdFZ4oo8Avwxz46OzEQL4+d+HxuvqC4xQ81KRd5XXaHazXrvJ6bd1RrY1vH4jdW1SQpQn+fE0M5MV+Fo/KTvt5wQCAoSP0AhgQw5Ely3eMLN8xsXPRUeGD0TnCHVMkqnerbddmyW6f82y5ZHrHHZ4i4Z0gq2hC3Bt+mKah0V63Rnvd+tQJgdj5moZW7Q7WaXd5vXYF67WrvE6bt1fGBqNzsixN8EVHgyf68zQxwPQIADgaEXoBDFl0VNgrM88rx8STY+fttlZFDu1TpGq3wh3TIz5+U/Z7fzn8bK43NkfY7AjEowIyTCuuzx6V69KoY4o07Zii2LmWUFj7Khu0q7xOu4L12l1erw1b9qslFJbUMT3CHRsN7pgiwfQIAMhchF4AI8ZwuGQVT5ZVPFkdcdK2bdlNNe1zhffERoVbd78t2dFQKssp0zOufTS4Y67wBBnZPZehOVKW09IxYwp0zJiC2LmIbaviYFNsNHh3sF7v7qzWpq2Hp0d4C7I00R+dHzwxEA3CxaOyZTI9AgDSHqEXQEIZhiHDXSjTXSjHhBmx83Y4pMih/e2jwtEKEuHdm9X2wV8PP5yVK7MgILPAL7PAFx0RLvDLLPDLyOm7pq9pGAp43Qp43frkVH/sfG1Da3SOcKcpEkdOjxjvy4uG4UA0DI8rzpXTEd9INAAgNRB6AaQEw3LKKpooq2iiOk8yiDTWxEaDIzVBRWqDCge3q+2jV7vWOnZkRYNwQUBGga89GEcDspHn7XW6REGuSycd49VJx3hj51pDYe2tbGhfMBedIrHh7f1qeTM6Em0ahsYUu6PTImIL5/KU73aNRNcAAIYBoRdASjPdo2S6R0njp3U5b4fbZNdXKlIbjIXhSG0wOlq8e7MUbjt8s2HJKChuD8KH/2cUBGTmF8twdA2rrt6mRxxqah8NrtOu8nq9t+uQNm0tj93jyc+KzQ+e2D5ForedgQAAiUXoBZCWDMshY9RomaNGSxO6XrPtSLSaRHsQtjsCcU1QoQPbpFBT5zfJyPUcDsKjjgjG7TWHTcNQwONWwOPWzM7TIxqj0yM6wvDu8nqVfVStSPsodJbLUlFBtopHZR/+OSpbxaNyVDQqWwVuJyXVACABCL0AMo5hmLEawxp7Qpdrtm3LbqmXXVN+eHS4/X9tu96S3VTb9V3Z+e3TJQJHjBL7ZeQUqMDt0kmTvTppcs/TI6obWrXnQJ0qa5q0fW+NGprburzf5TBV1BGECw4H4uJR0YBckOsiFAPAMCD0AjiqGIYhIztfys6XFSjtdt1ubeoUhCtk10bDcfjAB2rb9ndJneYRO7Oj84jz/V0W1TkK/Joc8OqYMQXy+fJVUVEXe6SppU1VNc2qrGlWZU2TKmuaVVUbPd6xv071TaEu7XFYZnsQ7jpa3DFSPCrPRXUJAIgDoRcAOjFcObKKJ8kqntTtmh0Oya6rVKS2XJHaCkU6RosP7lXbrrekSPjwzaZDZn6x2orHqM1ZEK1jnOuVM9ejMblejZvkkeEq7vYZza2dQ3Fz9PfaZlXVNOnN8jrVNR4Zig15C44MxTmxoFyYlyXTJBQDAKEXAOJkWE4ZhWNkFo7pds2ORGQ3VHedR1xTrnB9tdpqtsluruv+QmeOzDxPdIOO3MM/A7lejfZ6ZE7wSi53l+kNLa3h9hAcDcKxcFzbrLe2Vam2obXLR1imIW9BVpcg3Hm0uDDfJWsQ20MDQLoh9ALAMDBMU0Z+scz8YmncibHzHdMb7LZW2Y2HFGk4GA3H9dGfdsNBRRqq1Va9R3ZjjbpMn5AkhysahvO80QV3uV752oOxMdEjIy8gIysvFoxbQmFVt4fiyk7TKKpqm1X2UZVq6ruHYk9+VtcFdgXZ8uRnKd/tVF6OU/luF9s2A0h7hF4ASADD4YrN+e2NHWmT3Vgju776cDju9DOy9121NR6S7EjXBy1Hl9Fib65HRbleHe/xyBzvlZE7XkZOgQzDVKgtrKralvZQ3NRlCsU7Ow7qUN2BI2O3JCnbZSnfHQ3A0SAc/b1zMI6dy3Eq22WxAA9ASiH0AkCKMEyHjLwiKa9Ive33ZkfCsptq20eLD48U2w0HZTccVLj8Q7U1HOw6v1iSTCu6E15ekUblelSY69GxuV4ZEzwyT/DKyB0rI2eU2iJSdV2zaupbVdfYqrqmkOoaQ6prbFV9+89D9S3aHaxXXWNIbeFIj+10WGY0BHcKyH2F5dwcJwvyAIwoQi8ApBHDtGTkeqRcjyx/SY/32HZEdlNdp0DcHo7bQ3K4YofsHW9K4dCRL5fhLlRerkcF+T6ZhdE6yOb4MTJHTZLhzDric2w1t4ZV1xSKBeK6xpDqmjoCcigWnIOHalTfFFJTyxFhvOOjDSkvp+dR4zy3s8txx3WHxZQLAPEj9AJAhjEMU4Z7lOQeJcs3ucd7bNuWWhpiobhjjnHHdIpwcJvatr+qznOMjVyvzMIx0SBcOFpm4RhlFY5R9iiP/IU5cbUt1BZRfVPXgFzXHpDrO40s76tsUF3jITU0hXqcbiFJOVmW8nMOjxrnuZ3KdWepubm12712Ly/p7d19XbT7emown6No6DcNQ4ZhyDSNTsfRn6ZhyDA73dP5utlx7ojnzMP3drluHvHejnvMzu/t9FyX9hgyTcmQofpQRAcPNkbb3/5/jPYvY7R/p+ih0fP12Dkj1gcd93ccd76/4/lu12XEnj183uh0f9d7ul9P7//CELFthcMRtYVttbX/DIcjaotEj8Ox8+3n2trviUS63h+21RY54rjj+U739vTOnu4Ph21dcU6JzpzefeFvshB6AeAoZBiGlJ0nKztPKprY4z12W2u0LFvNfkUOHYhu8VxzQKEP/yaFmg/faLlkFgZkjopWtoiOEEd/Gs7sLu90Okx58rPkyc9SPCIRWw3NnUaNG0NHhObo7wfrWrQrWC/DMBSJ9DzlYjDhprdH+n5Tz1d7e5dtR4O0bUe/r23bithq/9n+e6Trud5CPAavcwjvHJA7AnOXkN4pMB8Z8DueG2hgPzKMd/6XB9M01RpqiwXSUKdgGhmhPwwOy5BlmXKYhhyWefi445wj+tNpmcpxOTpdN+Qwoz/H+1JrG3ZCLwCgR4bDJatogqyirvs827Ytu6kmGoI7heFwxcdq+/i1LsOqRq6nfWS46wixkVckw+h/eoJpGu3TGVyScvu9/8jNQDKVbUfHm6MhWe1B2FYkEg3QXc93DtOHzx8O053P9R6+D5+P3pufn63a2qbY/7vt9nZF29c+Im63j3J3CvZHXu94tvP3Ovyc3f7evq+r8z2dPrdrezrd39v1XtrY13ew7U7Xem273akNR7Yx1upe25Wd5VC4Ldw1dFqmLCsaOi2rI5i2/94eOh2dr5mdQmt7QHXG7jkcVi3LkNX+XwkyDaEXADAghmFEF8W5C7tv89zWGt2449A+RWoORENxzX6Ftm2SWpsO32g5ZY4KHA7EsVA8RoYrvqkSR7OOEUbTSl4wOVr+BSMV0NfDg9ALABg2hsMlyztOlndcl/PR0eHa9iC8P/YzXLVbbTve7FKGzcgZ1W2aRHR0uFgGG2kAGCRCLwBgxEVHh0fJdI+Sxhzf5ZodbovuZFez//CUiZoDCn30mtTScPhGyyGz4IjR4fYKE0ZW/1MfABzdCL0AgKQyLIcsz1hZnrFdztu2Lbu57vDo8KEDsmsOKHxwr9p2viXZh8ufGTkFMgsC2u92q7W1LcHfYBgcWX5ARx7HDtpPGUdc6+X+Xt9ldDrdy/1G52tGbFFXx7WKbJeam0Nd39HrOzs78t7O53qartHLtT7u7XE+ak991dt7+r238/fr9ssA7u1auaK3vjmUl6XWzrsp9tR3XV90hJ7aMJhne1vZ2f28IUPWxJOj/6KbIgi9AICUZBhGNMzmFEijj+tyzY60ya6tUOTQAYUP7Zddsz86WtzaLLut51rAKatjJZR0eKVTl2vt52Kn7S6LwI681uO7Oi346nZ/l9X/nVZp9dSGTseNpqFIxO5+vdt3637cY9m3nt5jd/ulh/fafdzby7WB3JsCWpLdgEFyzVyirNMuSXYzYgi9AIC0Y5gOGe1THBw6NXaeBT+Jc7T1tR1XKO8jOHcrLdZbIO8etouLclVZVd/r9b7O2322oafzA3t/X88aud6en0kSQi8AAEA/jD6nK4wsMztXhqvn+tP9ybzCY4PHMlgAAABkPEIvAAAAMl5coXfNmjVasGCB5s6dq5UrV3a7/u677+ryyy/XvHnz9M1vflNtbdGVs/v27dM111yjiy66SF/60pfU0NDQ7VkAAABgpPUbesvLy7VixQo9/fTTWr16tZ555hlt27atyz2333677rrrLr344ouybVvPPvusJOmee+7R1VdfrXXr1mnatGl67LHHRuZbAAAAAH3oN/Ru3LhRs2bNUmFhodxut+bNm6d169bFru/du1fNzc065ZRTJElLlizRunXrFAqF9Nprr2nevHldzgMAAACJ1m/1hmAwKJ/PFzv2+/3asmVLr9d9Pp/Ky8t18OBB5eXlyeFwdDk/FEVFeUN6fjB8vvyEf+bRjP5OHPo6cejrxKK/E4e+Thz6euj6Db12D3XZOu900tv1/p4bjKqq+mgh7AQ52moQJhv9nTj0deLQ14lFfycOfZ049HX8TNPodZC03+kNgUBAlZWVseNgMCi/39/r9YqKCvn9fnm9XtXX1yscDnc5DwAAACRav6F39uzZ2rRpk6qrq9XU1KT169drzpw5sevjxo1TVlaW3njjDUnSqlWrNGfOHDmdTs2cOVNr167tch4AAABItLhGepctW6alS5fq0ksv1cKFCzVjxgzdcMMNKisrkyQ99NBDeuCBBzR//nw1NTVp6dKlkqS7775bzz77rBYsWKDXX39dt95664h+GQAAAKAnht3T5NsUxZzezEZ/Jw59nTj0dWLR34lDXycOfR2/Ic3pBQAAANIdoRcAAAAZj9ALAACAjEfoBQAAQMYj9AIAACDjEXoBAACQ8frdhjiVmObQtjFOl888mtHfiUNfJw59nVj0d+LQ14lDX8enr35Kqzq9AAAAwGAwvQEAAAAZj9ALAACAjEfoBQAAQMYj9AIAACDjEXoBAACQ8Qi9AAAAyHiEXgAAAGQ8Qi8AAAAyHqEXAAAAGY/QCwAAgIxH6AUAAEDGI/QCAAAg4xF6AQAAkPEIvb1Ys2aNFixYoLlz52rlypXJbk5G++EPf6iLL75YF198sb773e8muzlHhe985zu68847k92MjPfyyy9ryZIluuiii/Ttb3872c3JaKtXr479PfKd73wn2c3JSPX19Vq4cKH27NkjSdq4caMWLVqkCy+8UCtWrEhy6zLLkX39zDPPaOHChVq0aJH+7d/+Ta2trUluYXoi9PagvLxcK1as0NNPP63Vq1frmWee0bZt25LdrIy0ceNGbdiwQc8//7xWrVqlrVu36qWXXkp2szLapk2b9Pzzzye7GRlv9+7duvvuu/XYY49pzZo1euedd/TKK68ku1kZqampSffff79++ctfavXq1Xr99de1cePGZDcro2zevFlXXXWVduzYIUlqbm7WN77xDT322GNau3at3n77bf58D5Mj+/rjjz/Wz372M/3qV7/S7373O0UiET399NPJbWSaIvT2YOPGjZo1a5YKCwvldrs1b948rVu3LtnNykg+n0933nmnXC6XnE6nSkpKtG/fvmQ3K2MdOnRIK1as0Be/+MVkNyXjvfTSS1qwYIFGjx4tp9OpFStW6OSTT052szJSOBxWJBJRU1OT2tra1NbWpqysrGQ3K6M8++yzuvvuu+X3+yVJW7Zs0aRJkzRhwgQ5HA4tWrSIf04OkyP72uVyafny5crLy5NhGDruuOP45+QgOZLdgFQUDAbl8/lix36/X1u2bEliizLXscceG/t9x44dWrt2rX71q18lsUWZ7d///d+1bNky7d+/P9lNyXg7d+6U0+nUF77wBVVUVOjcc8/VrbfemuxmZaS8vDx99atf1fz585Wdna1PfepTOu2005LdrIxy//33dznu6Z+T5eXliW5WRjqyr8eNG6dx48ZJkqqrq7Vy5Uo98MADyWha2mOktwe2bXc7ZxhGElpy9Pjwww/1L//yL7rjjjs0efLkZDcnI/3617/WmDFjdMYZZyS7KUeFcDisTZs26Xvf+56effZZlZWVMa1khLz33nv67W9/qz/96U/asGGDTNPUz372s2Q3K6Pxz8nEKy8v1+c//3ldfvnlOv3005PdnLRE6O1BIBBQZWVl7DgYDMb+MwOG3xtvvKHrrrtOX/va13TZZZcluzkZa+3atfrb3/6mxYsX6/vf/75efvll/cd//Eeym5WxiouLdcYZZ8jr9So7O1vnn38+/8VohGzYsEFnnHGGioqK5HK5tGTJEv3jH/9IdrMyGv+cTKzt27frqquu0mWXXaZbbrkl2c1JW4TeHsyePVubNm1SdXW1mpqatH79es2ZMyfZzcpI+/fv1y233KKHHnpIF198cbKbk9F+8Ytf6IUXXtDq1av1la98Reedd56+8Y1vJLtZGevcc8/Vhg0bVFtbq3A4rL/+9a866aSTkt2sjDR16lRt3LhRjY2Nsm1bL7/8sqZPn57sZmW0k08+WR9//LF27typcDisF154gX9OjpD6+np94Qtf0Fe/+lX9y7/8S7Kbk9aY09uDQCCgZcuWaenSpQqFQrriiis0Y8aMZDcrI/3sZz9TS0uLHnzwwdi5K6+8UldddVUSWwUM3cknn6zrr79eV199tUKhkM4880xdfvnlyW5WRjrrrLP0zjvvaMmSJXI6nZo+fbpuvPHGZDcro2VlZenBBx/Ul7/8ZbW0tOjss8/WRRddlOxmZaTf/OY3qqys1M9//nP9/Oc/lySdd955+upXv5rklqUfw+5pYg4AAACQQZjeAAAAgIxH6AUAAEDGI/QCAAAg4xF6AQAAkPEIvQAAAMh4hF4AAABkPEIvAAAAMt7/D7vNunJDLlVlAAAAAElFTkSuQmCC\n",
+ "text/plain": [
+ "
"
+ ]
+ },
+ "metadata": {
+ "filenames": {
+ "image/png": "/Users/mhjensen/Teaching/MachineLearning/doc/LectureNotes/_build/jupyter_execute/chapter1_214_0.png"
+ }
+ },
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "import matplotlib.pyplot as plt\n",
+ "import numpy as np\n",
+ "from sklearn.linear_model import LinearRegression, Ridge, Lasso\n",
+ "from sklearn.preprocessing import PolynomialFeatures\n",
+ "from sklearn.model_selection import train_test_split\n",
+ "from sklearn.pipeline import make_pipeline\n",
+ "\n",
+ "\n",
+ "np.random.seed(2018)\n",
+ "n = 100\n",
+ "maxdegree = 14\n",
+ "# Make data set.\n",
+ "x = np.linspace(-3, 3, n).reshape(-1, 1)\n",
+ "y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)\n",
+ "TestError = np.zeros(maxdegree)\n",
+ "TrainError = np.zeros(maxdegree)\n",
+ "polydegree = np.zeros(maxdegree)\n",
+ "x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2)\n",
"scaler = StandardScaler()\n",
- "scaler.fit(X_train)\n",
- "X_train_scaled = scaler.transform(X_train)\n",
- "X_test_scaled = scaler.transform(X_test)\n",
+ "scaler.fit(x_train)\n",
+ "x_train_scaled = scaler.transform(x_train)\n",
+ "x_test_scaled = scaler.transform(x_test)\n",
"\n",
- "print(\"Feature min values before scaling:\\n {}\".format(X_train.min(axis=0)))\n",
- "print(\"Feature max values before scaling:\\n {}\".format(X_train.max(axis=0)))\n",
+ "for degree in range(maxdegree):\n",
+ " model = make_pipeline(PolynomialFeatures(degree=degree), LinearRegression(fit_intercept=False))\n",
+ " clf = model.fit(x_train_scaled,y_train)\n",
+ " y_fit = clf.predict(x_train_scaled)\n",
+ " y_pred = clf.predict(x_test_scaled) \n",
+ " polydegree[degree] = degree\n",
+ " TestError[degree] = np.mean( np.mean((y_test - y_pred)**2) )\n",
+ " TrainError[degree] = np.mean( np.mean((y_train - y_fit)**2) )\n",
"\n",
- "print(\"Feature min values after scaling:\\n {}\".format(X_train_scaled.min(axis=0)))\n",
- "print(\"Feature max values after scaling:\\n {}\".format(X_train_scaled.max(axis=0)))\n",
- "\n",
- "clf = skl.LinearRegression().fit(X_train_scaled, y_train)\n",
- "\n",
- "\n",
- "print(\"MSE after scaling: {:.2f}\".format(mean_squared_error(clf.predict(X_test_scaled), y_test)))\n",
- "print(\"R2 score for scaled data: {:.2f}\".format(clf.score(X_test_scaled,y_test)))"
+ "plt.plot(polydegree, TestError, label='Test Error')\n",
+ "plt.plot(polydegree, TrainError, label='Train Error')\n",
+ "plt.legend()\n",
+ "plt.show()"
]
},
{
@@ -4037,7 +4632,7 @@
},
{
"cell_type": "code",
- "execution_count": 37,
+ "execution_count": 40,
"metadata": {
"collapsed": false,
"editable": true
@@ -4154,7 +4749,7 @@
},
{
"cell_type": "code",
- "execution_count": 38,
+ "execution_count": 41,
"metadata": {
"collapsed": false,
"editable": true
@@ -4174,7 +4769,7 @@
},
{
"cell_type": "code",
- "execution_count": 39,
+ "execution_count": 42,
"metadata": {
"collapsed": false,
"editable": true
@@ -4205,7 +4800,7 @@
},
{
"cell_type": "code",
- "execution_count": 40,
+ "execution_count": 43,
"metadata": {
"collapsed": false,
"editable": true
diff --git a/doc/LectureNotes/_build/jupyter_execute/chapter1.py b/doc/LectureNotes/_build/jupyter_execute/chapter1.py
index b828ec607..f7147cccd 100644
--- a/doc/LectureNotes/_build/jupyter_execute/chapter1.py
+++ b/doc/LectureNotes/_build/jupyter_execute/chapter1.py
@@ -1,7 +1,5 @@
# Linear Regression
-[Video of Lecture](https://www.uio.no/studier/emner/matnat/fys/FYS-STK3155/h20/forelesningsvideoer/LectureAug21.mp4?vrtx=view-as-webpage)
-
## Introduction
@@ -9,7 +7,7 @@
-Our emphasis throughout this series of lectures (small change)
+Our emphasis throughout this series of lectures
is on understanding the mathematical aspects of
different algorithms used in the fields of data analysis and machine learning.
@@ -229,7 +227,9 @@ How to evaluate which model fits best the data is something we will come back to
## Simple linear regression model using **scikit-learn**
-We start with perhaps our simplest possible example, using **Scikit-Learn** to perform linear regression analysis on a data set produced by us.
+We start with perhaps our simplest possible example, using
+**Scikit-Learn** to perform linear regression analysis on a data set
+produced by us.
What follows is a simple Python code where we have defined a function
$y$ in terms of the variable $x$. Both are defined as vectors with $100$ entries.
@@ -366,7 +366,8 @@ We can modify easily the above Python code and plot the relative error instead
import numpy as np
import matplotlib.pyplot as plt
from sklearn.linear_model import LinearRegression
-
+# Number of data points
+n = 100
x = np.random.rand(100,1)
y = 5*x+0.01*np.random.randn(100,1)
linreg = LinearRegression()
@@ -383,7 +384,8 @@ plt.show()
Depending on the parameter in front of the normal distribution, we may
have a small or larger relative error. Try to play around with
different training data sets and study (graphically) the value of the
-relative error.
+relative error. Note also that **Scikit-Learn** requires a matrix as input for the input values $x$ and $y$. In the above code we have
+solved this by declaring $x$ and $y$ as arrays of dimension $n\times 1$.
As mentioned above, **Scikit-Learn** has an impressive functionality.
We can for example extract the values of $\alpha$ and $\beta$ and
@@ -483,13 +485,14 @@ ways of dealing with outliers.
The Huber cost function is defined as
$$
-H_{\delta}(\boldsymbol{a})=\left\{\begin{array}{cc}\frac{1}{2} \boldsymbol{a}^{2}& \text{for }|\boldsymbol{a}|\leq \delta\\ \delta (|\b\
-m{a}|-\frac{1}{2}\delta ),&\text{otherwise}.\end{array}\right.
+H_{\delta}(\boldsymbol{a})=\left\{\begin{array}{cc}\frac{1}{2} \boldsymbol{a}^{2}& \text{for }|\boldsymbol{a}|\leq \delta\\ \delta (|\boldsymbol{a}|-\frac{1}{2}\delta ),&\text{otherwise}.\end{array}\right.
$$
Here $\boldsymbol{a}=\boldsymbol{y} - \boldsymbol{\tilde{y}}$.
+
+
We will discuss in more
detail these and other functions in the various lectures. We conclude this part with another example. Instead of
a linear $x$-dependence we study now a cubic polynomial and use the polynomial regression analysis tools of scikit-learn.
@@ -735,6 +738,8 @@ X[:,2] = A**(2.0/3.0)
X[:,3] = A**(-1.0/3.0)
X[:,4] = A**(-1.0)
+Note well that we have made life simple here. We perform a fit in terms of the number of nucleons only. A more sophisticated fit can be done by including an explicit dependence on the number of protons and neutrons in the asymmetry and Coulomb terms.
+
With **scikitlearn** we are now ready to use linear regression and fit our data.
clf = skl.LinearRegression().fit(X, Energies)
@@ -749,7 +754,6 @@ print("Mean squared error: %.2f" % mean_squared_error(Energies, fity))
print('Variance score: %.2f' % r2_score(Energies, fity))
# Mean absolute error
print('Mean absolute error: %.2f' % mean_absolute_error(Energies, fity))
-print(clf.coef_, clf.intercept_)
Masses['Eapprox'] = fity
# Generate a plot comparing the experimental with the fitted values values.
@@ -1850,6 +1854,89 @@ print('R2 score is {}'.format(r2))
plt.scatter(Y_test, y_test_predict)
plt.show()
+## Splitting our Data in Training and Test data
+
+
+It is normal in essentially all Machine Learning studies to split the
+data in a training set and a test set (sometimes also an additional
+validation set). **Scikit-Learn** has an own function for this. There
+is no explicit recipe for how much data should be included as training
+data and say test data. An accepted rule of thumb is to use
+approximately $2/3$ to $4/5$ of the data as training data. We will
+postpone a discussion of this splitting to the end of these notes and
+our discussion of the so-called **bias-variance** tradeoff. Here we
+limit ourselves to repeat the above equation of state fitting example
+but now splitting the data into a training set and a test set.
+
+Let us study some examples. The first code here takes a simple
+one-dimensional second-order polynomial and we fit it to a
+second-order polynomial. Depending on the strength of the added noise,
+the various measures like the $R2$ score or the mean-squared error,
+the fit becomes better or worse.
+
+import os
+import numpy as np
+import pandas as pd
+import matplotlib.pyplot as plt
+from sklearn.model_selection import train_test_split
+
+
+def R2(y_data, y_model):
+ return 1 - np.sum((y_data - y_model) ** 2) / np.sum((y_data - np.mean(y_data)) ** 2)
+def MSE(y_data,y_model):
+ n = np.size(y_model)
+ return np.sum((y_data-y_model)**2)/n
+
+x = np.random.rand(100)
+y = 2.0+5*x*x+0.1*np.random.randn(100)
+
+
+# The design matrix now as function of a given polynomial
+X = np.zeros((len(x),3))
+X[:,0] = 1.0
+X[:,1] = x
+X[:,2] = x**2
+# We split the data in test and training data
+X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)
+# matrix inversion to find beta
+beta = np.linalg.inv(X_train.T @ X_train) @ X_train.T @ y_train
+print(beta)
+# and then make the prediction
+ytilde = X_train @ beta
+print("Training R2")
+print(R2(y_train,ytilde))
+print("Training MSE")
+print(MSE(y_train,ytilde))
+ypredict = X_test @ beta
+print("Test R2")
+print(R2(y_test,ypredict))
+print("Test MSE")
+print(MSE(y_test,ypredict))
+
+Alternatively, you could write your own test-train splitting function as shown here.
+
+# equivalently in numpy
+def train_test_split_numpy(inputs, labels, train_size, test_size):
+ n_inputs = len(inputs)
+ inputs_shuffled = inputs.copy()
+ labels_shuffled = labels.copy()
+
+ np.random.shuffle(inputs_shuffled)
+ np.random.shuffle(labels_shuffled)
+
+ train_end = int(n_inputs*train_size)
+ X_train, X_test = inputs_shuffled[:train_end], inputs_shuffled[train_end:]
+ Y_train, Y_test = labels_shuffled[:train_end], labels_shuffled[train_end:]
+
+ return X_train, X_test, Y_train, Y_test
+
+But since **scikit-learn** has its own function for doing this and since
+it interfaces easily with **tensorflow** and other libraries, we
+normally recommend using the latter functionality.
+
+
+
+
## Reducing the number of degrees of freedom, overarching view
Many Machine Learning problems involve thousands or even millions of
@@ -1873,6 +1960,7 @@ is one of the most used tools in data modeling, compression and
visualization.
+
Before we proceed however, we will discuss how to preprocess our
data. Till now and in connection with our previous examples we have
not met so many cases where we are too sensitive to the scaling of our
@@ -1880,6 +1968,15 @@ data. Normally the data may need a rescaling and/or may be sensitive
to extreme values. Scaling the data renders our inputs much more
suitable for the algorithms we want to employ.
+For data sets gathered for real world applications, it is rather normal that
+different features have very different units and
+numerical scales. For example, a data set detailing health habits may include
+features such as **age** in the range $0-80$, and **caloric intake** of order $2000$.
+Many machine learning methods sensitive to the scales of the features and may perform poorly if they
+are very different scales. Therefore, it is typical to scale
+the features in a way to avoid such outlier values.
+
+
**Scikit-Learn** has several functions which allow us to rescale the
data, normally resulting in much better results in terms of various
accuracy scores. The **StandardScaler** function in **Scikit-Learn**
@@ -1909,100 +2006,124 @@ outliers, and might often lead to trouble for other scaling
techniques.
-### Simple preprocessing examples, Franke function and regression
+Many features are often scaled using standardization to improve
+performance. In **Scikit-Learn** this is given by the **StandardScaler**
+function as discussed above. It is easy however to write your own.
+Mathematically, this involves subtracting the mean and divide by the
+standard deviation over the data set, for each feature:
+
+$$
+x_j^{(i)} \rightarrow \frac{x_j^{(i)} - \overline{x}_j}{\sigma(x_j)},
+$$
+
+where $\overline{x}_j$ and $\sigma(x_j)$ are the mean and standard
+deviation, respectively, of the feature $x_j$. This ensures that each
+feature has zero mean and unit standard deviation. For data sets
+where we do not have the standard deviation or don't wish to calculate
+it, it is then common to simply set it to one.
+
+
+
+Let us consider the following vanilla example where we use both
+**Scikit-Learn** and write our own function as well. We produce a
+simple test design matrix with random numbers. Each column could then
+represent a specific feature whose mean value is subracted.
-# Common imports
-import os
-import numpy as np
-import pandas as pd
-import matplotlib.pyplot as plt
import sklearn.linear_model as skl
from sklearn.metrics import mean_squared_error
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import MinMaxScaler, StandardScaler, Normalizer
+import numpy as np
+import pandas as pd
+from IPython.display import display
+np.random.seed(100)
+# setting up a 10 x 5 matrix
+rows = 10
+cols = 5
+X = np.random.randn(rows,cols)
+XPandas = pd.DataFrame(X)
+display(XPandas)
+print(XPandas.mean())
+print(XPandas.std())
+XPandas = (XPandas -XPandas.mean())
+display(XPandas)
+# This option does not include the standard deviation
+scaler = StandardScaler(with_std=False)
+scaler.fit(X)
+Xscaled = scaler.transform(X)
+display(XPandas-Xscaled)
-# Where to save the figures and data files
-PROJECT_ROOT_DIR = "Results"
-FIGURE_ID = "Results/FigureFiles"
-DATA_ID = "DataFiles/"
-
-if not os.path.exists(PROJECT_ROOT_DIR):
- os.mkdir(PROJECT_ROOT_DIR)
-
-if not os.path.exists(FIGURE_ID):
- os.makedirs(FIGURE_ID)
-
-if not os.path.exists(DATA_ID):
- os.makedirs(DATA_ID)
-
-def image_path(fig_id):
- return os.path.join(FIGURE_ID, fig_id)
-
-def data_path(dat_id):
- return os.path.join(DATA_ID, dat_id)
-
-def save_fig(fig_id):
- plt.savefig(image_path(fig_id) + ".png", format='png')
+Small exercise: perform the standard scaling by including the standard deviation and compare with what Scikit-Learn gives.
-def FrankeFunction(x,y):
- term1 = 0.75*np.exp(-(0.25*(9*x-2)**2) - 0.25*((9*y-2)**2))
- term2 = 0.75*np.exp(-((9*x+1)**2)/49.0 - 0.1*(9*y+1))
- term3 = 0.5*np.exp(-(9*x-7)**2/4.0 - 0.25*((9*y-3)**2))
- term4 = -0.2*np.exp(-(9*x-4)**2 - (9*y-7)**2)
- return term1 + term2 + term3 + term4
+
+Another commonly used scaling method is min-max scaling. This is very
+useful for when we want the features to lie in a certain interval. To
+scale the feature $x_j$ to the interval $[a, b]$, we can apply the
+transformation
+
+$$
+x_j^{(i)} \rightarrow (b-a)\frac{x_j^{(i)} - \min(x_j)}{\max(x_j) - \min(x_j)} - a
+$$
+
+where $\min(x_j)$ and $\max(x_j)$ return the minimum and maximum value of $x_j$ over the data set, respectively.
-def create_X(x, y, n ):
- if len(x.shape) > 1:
- x = np.ravel(x)
- y = np.ravel(y)
-
- N = len(x)
- l = int((n+1)*(n+2)/2) # Number of elements in beta
- X = np.ones((N,l))
-
- for i in range(1,n+1):
- q = int((i)*(i+1)/2)
- for k in range(i+1):
- X[:,q+k] = (x**(i-k))*(y**k)
-
- return X
-# Making meshgrid of datapoints and compute Franke's function
-n = 5
-N = 1000
-x = np.sort(np.random.uniform(0, 1, N))
-y = np.sort(np.random.uniform(0, 1, N))
-z = FrankeFunction(x, y)
-X = create_X(x, y, n=n)
-# split in training and test data
-X_train, X_test, y_train, y_test = train_test_split(X,z,test_size=0.2)
+## Testing the Means Squared Error as function of Complexity
-clf = skl.LinearRegression().fit(X_train, y_train)
+Before we proceed with a more detailed analysis of the so-called
+Bias-Variance tradeoff, we present here an example of the relation
+between model complexity and the mean squared error for the triaining
+data and the test data.
-# The mean squared error and R2 score
-print("MSE before scaling: {:.2f}".format(mean_squared_error(clf.predict(X_test), y_test)))
-print("R2 score before scaling {:.2f}".format(clf.score(X_test,y_test)))
+The results here tell us clearly that for the data not included in the
+training, there is an optimal model as function of the complexity of
+ourmodel (here in terms of the polynomial degree of the model).
+The results here will vary as function of model complexity and the amount od data used for training.
+
+
+Our data is defined by $x\in [-3,3]$ with a total of for example $100$ data points.
+
+import matplotlib.pyplot as plt
+import numpy as np
+from sklearn.linear_model import LinearRegression, Ridge, Lasso
+from sklearn.preprocessing import PolynomialFeatures
+from sklearn.model_selection import train_test_split
+from sklearn.pipeline import make_pipeline
+
+
+np.random.seed(2018)
+n = 100
+maxdegree = 14
+# Make data set.
+x = np.linspace(-3, 3, n).reshape(-1, 1)
+y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)
+TestError = np.zeros(maxdegree)
+TrainError = np.zeros(maxdegree)
+polydegree = np.zeros(maxdegree)
+x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2)
scaler = StandardScaler()
-scaler.fit(X_train)
-X_train_scaled = scaler.transform(X_train)
-X_test_scaled = scaler.transform(X_test)
+scaler.fit(x_train)
+x_train_scaled = scaler.transform(x_train)
+x_test_scaled = scaler.transform(x_test)
-print("Feature min values before scaling:\n {}".format(X_train.min(axis=0)))
-print("Feature max values before scaling:\n {}".format(X_train.max(axis=0)))
+for degree in range(maxdegree):
+ model = make_pipeline(PolynomialFeatures(degree=degree), LinearRegression(fit_intercept=False))
+ clf = model.fit(x_train_scaled,y_train)
+ y_fit = clf.predict(x_train_scaled)
+ y_pred = clf.predict(x_test_scaled)
+ polydegree[degree] = degree
+ TestError[degree] = np.mean( np.mean((y_test - y_pred)**2) )
+ TrainError[degree] = np.mean( np.mean((y_train - y_fit)**2) )
-print("Feature min values after scaling:\n {}".format(X_train_scaled.min(axis=0)))
-print("Feature max values after scaling:\n {}".format(X_train_scaled.max(axis=0)))
-
-clf = skl.LinearRegression().fit(X_train_scaled, y_train)
-
-
-print("MSE after scaling: {:.2f}".format(mean_squared_error(clf.predict(X_test_scaled), y_test)))
-print("R2 score for scaled data: {:.2f}".format(clf.score(X_test_scaled,y_test)))
+plt.plot(polydegree, TestError, label='Test Error')
+plt.plot(polydegree, TrainError, label='Train Error')
+plt.legend()
+plt.show()
## Exercises
diff --git a/doc/LectureNotes/_build/jupyter_execute/chapter1_11_0.png b/doc/LectureNotes/_build/jupyter_execute/chapter1_11_0.png
index d56e1d6d0..8585b7491 100644
Binary files a/doc/LectureNotes/_build/jupyter_execute/chapter1_11_0.png and b/doc/LectureNotes/_build/jupyter_execute/chapter1_11_0.png differ
diff --git a/doc/LectureNotes/_build/jupyter_execute/chapter1_13_1.png b/doc/LectureNotes/_build/jupyter_execute/chapter1_13_1.png
index 3fd1831e1..b145ce82a 100644
Binary files a/doc/LectureNotes/_build/jupyter_execute/chapter1_13_1.png and b/doc/LectureNotes/_build/jupyter_execute/chapter1_13_1.png differ
diff --git a/doc/LectureNotes/_build/jupyter_execute/chapter1_214_0.png b/doc/LectureNotes/_build/jupyter_execute/chapter1_214_0.png
new file mode 100644
index 000000000..cbac4fd70
Binary files /dev/null and b/doc/LectureNotes/_build/jupyter_execute/chapter1_214_0.png differ
diff --git a/doc/LectureNotes/_build/jupyter_execute/chapter1_27_0.png b/doc/LectureNotes/_build/jupyter_execute/chapter1_27_0.png
index ab7dda310..bef273ce3 100644
Binary files a/doc/LectureNotes/_build/jupyter_execute/chapter1_27_0.png and b/doc/LectureNotes/_build/jupyter_execute/chapter1_27_0.png differ
diff --git a/doc/LectureNotes/_build/jupyter_execute/chapter1_3_0.png b/doc/LectureNotes/_build/jupyter_execute/chapter1_3_0.png
index 4ad4c8690..0be2e475d 100644
Binary files a/doc/LectureNotes/_build/jupyter_execute/chapter1_3_0.png and b/doc/LectureNotes/_build/jupyter_execute/chapter1_3_0.png differ
diff --git a/doc/LectureNotes/_build/jupyter_execute/chapter1_61_11.png b/doc/LectureNotes/_build/jupyter_execute/chapter1_61_11.png
new file mode 100644
index 000000000..916bfd378
Binary files /dev/null and b/doc/LectureNotes/_build/jupyter_execute/chapter1_61_11.png differ
diff --git a/doc/LectureNotes/_build/jupyter_execute/chapter2.ipynb b/doc/LectureNotes/_build/jupyter_execute/chapter2.ipynb
index 9e68b7c50..1ced6d808 100644
--- a/doc/LectureNotes/_build/jupyter_execute/chapter2.ipynb
+++ b/doc/LectureNotes/_build/jupyter_execute/chapter2.ipynb
@@ -565,6 +565,179 @@
"example\n",
"\n",
"\n",
+ "## Code for SVD and Inversion of Matrices\n",
+ "\n",
+ "How do we use the SVD to invert a matrix $\\boldsymbol{X}^\\boldsymbol{X}$ which is singular or near singular?\n",
+ "The simple answer is to use the linear algebra function for the pseudoinverse, that is"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "metadata": {
+ "collapsed": false,
+ "editable": true
+ },
+ "outputs": [],
+ "source": [
+ "#Ainv = np.linlag.pinv(A)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Let us first look at a matrix which does not causes problems and write our own function where we just use the SVD."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "metadata": {
+ "collapsed": false,
+ "editable": true
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "[[1 2 3]\n",
+ " [2 4 5]\n",
+ " [3 5 6]]\n",
+ "test U\n",
+ "[[ 2.22044605e-16 -7.77156117e-16 -5.55111512e-16]\n",
+ " [-7.77156117e-16 0.00000000e+00 -1.11022302e-16]\n",
+ " [-5.55111512e-16 -1.11022302e-16 0.00000000e+00]]\n",
+ "test VT\n",
+ "[[ 1.11022302e-16 -2.22044605e-16 1.38777878e-16]\n",
+ " [-2.22044605e-16 -1.11022302e-16 -1.11022302e-16]\n",
+ " [ 1.38777878e-16 -1.11022302e-16 0.00000000e+00]]\n",
+ "[[0. 0. 0.]\n",
+ " [0. 0. 0.]\n",
+ " [0. 0. 0.]]\n"
+ ]
+ }
+ ],
+ "source": [
+ "import numpy as np\n",
+ "# SVD inversion\n",
+ "def SVDinv(A):\n",
+ " ''' Takes as input a numpy matrix A and returns inv(A) based on singular value decomposition (SVD).\n",
+ " SVD is numerically more stable than the inversion algorithms provided by\n",
+ " numpy and scipy.linalg at the cost of being slower.\n",
+ " '''\n",
+ " U, s, VT = np.linalg.svd(A)\n",
+ " print('test U')\n",
+ " print( (np.transpose(U) @ U - U @np.transpose(U)))\n",
+ " print('test VT')\n",
+ " print( (np.transpose(VT) @ VT - VT @np.transpose(VT)))\n",
+ "\n",
+ "\n",
+ " D = np.zeros((len(U),len(VT)))\n",
+ " D = np.diag(s)\n",
+ " UT = np.transpose(U); V = np.transpose(VT); invD = np.linalg.inv(D)\n",
+ " return np.matmul(V,np.matmul(invD,UT))\n",
+ "\n",
+ "\n",
+ "#X = np.array([ [1.0, -1.0, 2.0], [1.0, 0.0, 1.0], [1.0, 2.0, -1.0], [1.0, 1.0, 0.0] ])\n",
+ "# Non-singular square matrix\n",
+ "X = np.array( [ [1,2,3],[2,4,5],[3,5,6]])\n",
+ "print(X)\n",
+ "A = np.transpose(X) @ X\n",
+ "# Brute force inversion\n",
+ "B = np.linalg.pinv(A) # here we could use np.linalg.inv(A), try it!\n",
+ "C = SVDinv(A)\n",
+ "print(np.abs(B-C))"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Although our matrix to invert $\\boldsymbol{X}^T\\boldsymbol{X}$ is a square matrix, our matrix may be singular. \n",
+ "\n",
+ "The pseudoinverse is the generalization of the matrix inverse for square matrices to\n",
+ "rectangular matrices where the number of rows and columns are not equal.\n",
+ "\n",
+ "It is also called the the Moore-Penrose Inverse after two independent discoverers of the method or the Generalized Inverse.\n",
+ "It is used for the calculation of the inverse for singular or near singular matrices and for rectangular matrices.\n",
+ "\n",
+ "Using the SVD we can obtain the pseudoinverse of a matrix $\\boldsymbol{A}$ (labeled here as $\\boldsymbol{A}_{\\mathrm{PI}}$"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "$$\n",
+ "\\boldsymbol{A}_{\\mathrm{PI}}= \\boldsymbol{V}\\boldsymbol{D}_{\\mathrm{PI}}\\boldsymbol{U}^T,\n",
+ "$$"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "where $\\boldsymbol{D}_{\\mathrm{PI}}$ can be calculated by creating a diagonal matrix from $\\boldsymbol{Sigma}$ where we only keep the singular values (the non-zero values). The following code computes the pseudoinvers of the matrix based on the SVD."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "metadata": {
+ "collapsed": false,
+ "editable": true
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "[[0.3 0.4]\n",
+ " [0.5 0.6]\n",
+ " [0.7 0.8]\n",
+ " [0.9 1. ]]\n",
+ "[[-13. -6. 1. 8. ]\n",
+ " [ 11.5 5.5 -0.5 -6.5]]\n",
+ "[[0. 0. 0. 0.]\n",
+ " [0. 0. 0. 0.]]\n"
+ ]
+ }
+ ],
+ "source": [
+ "import numpy as np\n",
+ "# SVD inversion\n",
+ "def SVDinv(A):\n",
+ " U, s, VT = np.linalg.svd(A)\n",
+ " # reciprocals of singular values of s\n",
+ " d = 1.0 / s\n",
+ " # create m x n D matrix\n",
+ " D = np.zeros(A.shape)\n",
+ " # populate D with n x n diagonal matrix\n",
+ " D[:A.shape[1], :A.shape[1]] = np.diag(d)\n",
+ " UT = np.transpose(U)\n",
+ " V = np.transpose(VT)\n",
+ " return np.matmul(V,np.matmul(D.T,UT))\n",
+ "\n",
+ "\n",
+ "A = np.array([ [0.3, 0.4], [0.5, 0.6], [0.7, 0.8],[0.9, 1.0]])\n",
+ "print(A)\n",
+ "# Brute force inversion of super-collinear matrix\n",
+ "B = np.linalg.pinv(A)\n",
+ "print(B)\n",
+ "# Compare our own algorithm with pinv\n",
+ "C = SVDinv(A)\n",
+ "print(np.abs(C-B))"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "As you can see from these examples, our own decomposition based on the SVD agrees the pseudoinverse algorithm provided by **Numpy**.\n",
+ "\n",
+ "\n",
"\n",
"\n",
"\n",
@@ -1287,7 +1460,7 @@
},
{
"cell_type": "code",
- "execution_count": 2,
+ "execution_count": 5,
"metadata": {
"collapsed": false,
"editable": true
@@ -1297,10 +1470,10 @@
"name": "stdout",
"output_type": "stream",
"text": [
- "-0.033037772005753835\n",
- "3.7371165871823337\n",
- "[[ 1.22443803 3.75195757]\n",
- " [ 3.75195757 12.47441766]]\n"
+ "0.059680303635344434\n",
+ "4.260227742627244\n",
+ "[[0.89704131 2.55795935]\n",
+ " [2.55795935 8.24340751]]\n"
]
}
],
@@ -1330,7 +1503,7 @@
},
{
"cell_type": "code",
- "execution_count": 3,
+ "execution_count": 6,
"metadata": {
"collapsed": false,
"editable": true
@@ -1340,10 +1513,10 @@
"name": "stdout",
"output_type": "stream",
"text": [
- "0.08665060086846632\n",
- "1.6796265324852733\n",
- "[[1. 0.6183694]\n",
- " [0.6183694 1. ]]\n"
+ "0.08362077210702115\n",
+ "1.8429949116841184\n",
+ "[[1. 0.6996584]\n",
+ " [0.6996584 1. ]]\n"
]
}
],
@@ -1388,7 +1561,7 @@
},
{
"cell_type": "code",
- "execution_count": 4,
+ "execution_count": 7,
"metadata": {
"collapsed": false,
"editable": true
@@ -1398,30 +1571,30 @@
"name": "stdout",
"output_type": "stream",
"text": [
- "[[ 0.68002363 0.95517094]\n",
- " [-0.53545715 0.03652792]\n",
- " [ 1.33886902 4.84251485]\n",
- " [ 0.20375701 -0.16861772]\n",
- " [-2.04455272 -5.89742056]\n",
- " [ 1.17449733 2.3728892 ]\n",
- " [ 0.03019554 0.07581375]\n",
- " [ 0.60949193 1.95628168]\n",
- " [ 0.0530533 1.07839924]\n",
- " [-1.5098779 -5.2515593 ]]\n",
+ "[[-1.25981533 -3.84976606]\n",
+ " [-0.52570079 -0.62262506]\n",
+ " [ 0.51707172 3.46973015]\n",
+ " [-0.47265243 -1.87030083]\n",
+ " [ 0.73734906 2.46183815]\n",
+ " [ 0.4918372 2.05531364]\n",
+ " [ 0.3893239 0.47862383]\n",
+ " [-0.41237437 -0.85058354]\n",
+ " [ 0.77622336 0.54697204]\n",
+ " [-0.24126232 -1.81920231]]\n",
" 0 1\n",
- "0 0.680024 0.955171\n",
- "1 -0.535457 0.036528\n",
- "2 1.338869 4.842515\n",
- "3 0.203757 -0.168618\n",
- "4 -2.044553 -5.897421\n",
- "5 1.174497 2.372889\n",
- "6 0.030196 0.075814\n",
- "7 0.609492 1.956282\n",
- "8 0.053053 1.078399\n",
- "9 -1.509878 -5.251559\n",
+ "0 -1.259815 -3.849766\n",
+ "1 -0.525701 -0.622625\n",
+ "2 0.517072 3.469730\n",
+ "3 -0.472652 -1.870301\n",
+ "4 0.737349 2.461838\n",
+ "5 0.491837 2.055314\n",
+ "6 0.389324 0.478624\n",
+ "7 -0.412374 -0.850584\n",
+ "8 0.776223 0.546972\n",
+ "9 -0.241262 -1.819202\n",
" 0 1\n",
- "0 1.000000 0.959247\n",
- "1 0.959247 1.000000\n"
+ "0 1.000000 0.881323\n",
+ "1 0.881323 1.000000\n"
]
}
],
@@ -1451,7 +1624,7 @@
},
{
"cell_type": "code",
- "execution_count": 5,
+ "execution_count": 8,
"metadata": {
"collapsed": false,
"editable": true
@@ -1463,37 +1636,37 @@
"text": [
" 0 1 2 3 4 5 6 7 \\\n",
"0 0.0 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000 \n",
- "1 0.0 0.073902 0.077303 0.072279 0.077386 0.082866 0.063526 0.068073 \n",
- "2 0.0 0.077303 0.081480 0.074665 0.080336 0.086465 0.064929 0.069872 \n",
- "3 0.0 0.072279 0.074665 0.075754 0.080381 0.085289 0.069635 0.074099 \n",
- "4 0.0 0.077386 0.080336 0.080381 0.085597 0.091159 0.073333 0.078279 \n",
- "5 0.0 0.082866 0.086465 0.085289 0.091159 0.097452 0.077212 0.082687 \n",
- "6 0.0 0.063526 0.064929 0.069635 0.073333 0.077212 0.066047 0.069859 \n",
- "7 0.0 0.068073 0.069872 0.074099 0.078279 0.082687 0.069859 0.074096 \n",
- "8 0.0 0.073035 0.075289 0.078937 0.083657 0.088660 0.073966 0.078674 \n",
- "9 0.0 0.078451 0.081227 0.084181 0.089504 0.095173 0.078388 0.083618 \n",
- "10 0.0 0.055085 0.055774 0.062327 0.065204 0.068185 0.060520 0.063670 \n",
- "11 0.0 0.058892 0.059865 0.066249 0.069510 0.072910 0.064000 0.067504 \n",
- "12 0.0 0.063055 0.064354 0.070517 0.074207 0.078076 0.067769 0.071667 \n",
- "13 0.0 0.067610 0.069282 0.075163 0.079333 0.083728 0.071855 0.076189 \n",
- "14 0.0 0.072598 0.074696 0.080223 0.084931 0.089916 0.076283 0.081103 \n",
+ "1 0.0 0.082272 0.078892 0.081102 0.080738 0.080182 0.072012 0.072246 \n",
+ "2 0.0 0.078892 0.076822 0.075975 0.076166 0.076251 0.066705 0.067202 \n",
+ "3 0.0 0.081102 0.075975 0.085259 0.084103 0.082616 0.078750 0.078649 \n",
+ "4 0.0 0.080738 0.076166 0.084103 0.083246 0.082091 0.077338 0.077407 \n",
+ "5 0.0 0.080182 0.076251 0.082616 0.082091 0.081309 0.075567 0.075818 \n",
+ "6 0.0 0.072012 0.066705 0.078750 0.077338 0.075567 0.074693 0.074434 \n",
+ "7 0.0 0.072246 0.067202 0.078649 0.077407 0.075818 0.074434 0.074285 \n",
+ "8 0.0 0.072468 0.067718 0.078474 0.077416 0.076026 0.074077 0.074044 \n",
+ "9 0.0 0.072661 0.068244 0.078197 0.077339 0.076169 0.073591 0.073681 \n",
+ "10 0.0 0.063190 0.058247 0.070949 0.069542 0.067794 0.068593 0.068293 \n",
+ "11 0.0 0.063523 0.058719 0.071156 0.069853 0.068213 0.068717 0.068491 \n",
+ "12 0.0 0.063880 0.059225 0.071359 0.070166 0.068640 0.068821 0.068673 \n",
+ "13 0.0 0.064257 0.059765 0.071549 0.070473 0.069071 0.068896 0.068828 \n",
+ "14 0.0 0.064647 0.060335 0.071714 0.070764 0.069496 0.068928 0.068945 \n",
"\n",
" 8 9 10 11 12 13 14 \n",
"0 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000 \n",
- "1 0.073035 0.078451 0.055085 0.058892 0.063055 0.067610 0.072598 \n",
- "2 0.075289 0.081227 0.055774 0.059865 0.064354 0.069282 0.074696 \n",
- "3 0.078937 0.084181 0.062327 0.066249 0.070517 0.075163 0.080223 \n",
- "4 0.083657 0.089504 0.065204 0.069510 0.074207 0.079333 0.084931 \n",
- "5 0.088660 0.095173 0.068185 0.072910 0.078076 0.083728 0.089916 \n",
- "6 0.073966 0.078388 0.060520 0.064000 0.067769 0.071855 0.076283 \n",
- "7 0.078674 0.083618 0.063670 0.067504 0.071667 0.076189 0.081103 \n",
- "8 0.083775 0.089300 0.067043 0.071268 0.075865 0.080871 0.086325 \n",
- "9 0.089300 0.095472 0.070653 0.075309 0.080387 0.085928 0.091979 \n",
- "10 0.067043 0.070653 0.056484 0.059455 0.062659 0.066116 0.069848 \n",
- "11 0.071268 0.075309 0.059455 0.062731 0.066272 0.070103 0.074248 \n",
- "12 0.075865 0.080387 0.062659 0.066272 0.070188 0.074432 0.079037 \n",
- "13 0.080871 0.085928 0.066116 0.070103 0.074432 0.079137 0.084251 \n",
- "14 0.086325 0.091979 0.069848 0.074248 0.079037 0.084251 0.089931 \n"
+ "1 0.072468 0.072661 0.063190 0.063523 0.063880 0.064257 0.064647 \n",
+ "2 0.067718 0.068244 0.058247 0.058719 0.059225 0.059765 0.060335 \n",
+ "3 0.078474 0.078197 0.070949 0.071156 0.071359 0.071549 0.071714 \n",
+ "4 0.077416 0.077339 0.069542 0.069853 0.070166 0.070473 0.070764 \n",
+ "5 0.076026 0.076169 0.067794 0.068213 0.068640 0.069071 0.069496 \n",
+ "6 0.074077 0.073591 0.068593 0.068717 0.068821 0.068896 0.068928 \n",
+ "7 0.074044 0.073681 0.068293 0.068491 0.068673 0.068828 0.068945 \n",
+ "8 0.073926 0.073695 0.067889 0.068165 0.068427 0.068667 0.068873 \n",
+ "9 0.073695 0.073607 0.067355 0.067711 0.068056 0.068385 0.068686 \n",
+ "10 0.067889 0.067355 0.063925 0.064013 0.064075 0.064101 0.064078 \n",
+ "11 0.068165 0.067711 0.064013 0.064156 0.064273 0.064357 0.064393 \n",
+ "12 0.068427 0.068056 0.064075 0.064273 0.064449 0.064591 0.064689 \n",
+ "13 0.068667 0.068385 0.064101 0.064357 0.064591 0.064795 0.064958 \n",
+ "14 0.068873 0.068686 0.064078 0.064393 0.064689 0.064958 0.065188 \n"
]
}
],
@@ -1855,571 +2028,8 @@
"\n",
"\n",
"\n",
- "## Ridge and LASSO Regression\n",
"\n",
- "Let us remind ourselves about the expression for the standard Mean Squared Error (MSE) which we used to define our cost function and the equations for the ordinary least squares (OLS) method, that is \n",
- "our optimization problem is"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "$$\n",
- "{\\displaystyle \\min_{\\boldsymbol{\\beta}\\in {\\mathbb{R}}^{p}}}\\frac{1}{n}\\left\\{\\left(\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta}\\right)^T\\left(\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta}\\right)\\right\\}.\n",
- "$$"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "or we can state it as"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "$$\n",
- "{\\displaystyle \\min_{\\boldsymbol{\\beta}\\in\n",
- "{\\mathbb{R}}^{p}}}\\frac{1}{n}\\sum_{i=0}^{n-1}\\left(y_i-\\tilde{y}_i\\right)^2=\\frac{1}{n}\\vert\\vert \\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta}\\vert\\vert_2^2,\n",
- "$$"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "where we have used the definition of a norm-2 vector, that is"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "$$\n",
- "\\vert\\vert \\boldsymbol{x}\\vert\\vert_2 = \\sqrt{\\sum_i x_i^2}.\n",
- "$$"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "By minimizing the above equation with respect to the parameters\n",
- "$\\boldsymbol{\\beta}$ we could then obtain an analytical expression for the\n",
- "parameters $\\boldsymbol{\\beta}$. We can add a regularization parameter $\\lambda$ by\n",
- "defining a new cost function to be optimized, that is"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "$$\n",
- "{\\displaystyle \\min_{\\boldsymbol{\\beta}\\in\n",
- "{\\mathbb{R}}^{p}}}\\frac{1}{n}\\vert\\vert \\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta}\\vert\\vert_2^2+\\lambda\\vert\\vert \\boldsymbol{\\beta}\\vert\\vert_2^2\n",
- "$$"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "which leads to the Ridge regression minimization problem where we\n",
- "require that $\\vert\\vert \\boldsymbol{\\beta}\\vert\\vert_2^2\\le t$, where $t$ is\n",
- "a finite number larger than zero. By defining"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "$$\n",
- "C(\\boldsymbol{X},\\boldsymbol{\\beta})=\\frac{1}{n}\\vert\\vert \\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta}\\vert\\vert_2^2+\\lambda\\vert\\vert \\boldsymbol{\\beta}\\vert\\vert_1,\n",
- "$$"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "we have a new optimization equation"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "$$\n",
- "{\\displaystyle \\min_{\\boldsymbol{\\beta}\\in\n",
- "{\\mathbb{R}}^{p}}}\\frac{1}{n}\\vert\\vert \\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta}\\vert\\vert_2^2+\\lambda\\vert\\vert \\boldsymbol{\\beta}\\vert\\vert_1\n",
- "$$"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "which leads to Lasso regression. Lasso stands for least absolute shrinkage and selection operator. \n",
- "\n",
- "Here we have defined the norm-1 as"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "$$\n",
- "\\vert\\vert \\boldsymbol{x}\\vert\\vert_1 = \\sum_i \\vert x_i\\vert.\n",
- "$$"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Using the matrix-vector expression for Ridge regression and dropping the parameter $1/n$ in front of the standard means squared error equation, we have"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "$$\n",
- "C(\\boldsymbol{X},\\boldsymbol{\\beta})=\\left\\{(\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta})^T(\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta})\\right\\}+\\lambda\\boldsymbol{\\beta}^T\\boldsymbol{\\beta},\n",
- "$$"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "and \n",
- "taking the derivatives with respect to $\\boldsymbol{\\beta}$ we obtain then\n",
- "a slightly modified matrix inversion problem which for finite values\n",
- "of $\\lambda$ does not suffer from singularity problems. We obtain\n",
- "the optimal parameters"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "$$\n",
- "\\hat{\\boldsymbol{\\beta}}_{\\mathrm{Ridge}} = \\left(\\boldsymbol{X}^T\\boldsymbol{X}+\\lambda\\boldsymbol{I}\\right)^{-1}\\boldsymbol{X}^T\\boldsymbol{y},\n",
- "$$"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "with $\\boldsymbol{I}$ being a $p\\times p$ identity matrix with the constraint that"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "$$\n",
- "\\sum_{i=0}^{p-1} \\beta_i^2 \\leq t,\n",
- "$$"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "with $t$ a finite positive number. \n",
- "\n",
- "When we compare this with the ordinary least squares result we have"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "$$\n",
- "\\hat{\\boldsymbol{\\beta}}_{\\mathrm{OLS}} = \\left(\\boldsymbol{X}^T\\boldsymbol{X}\\right)^{-1}\\boldsymbol{X}^T\\boldsymbol{y},\n",
- "$$"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "which can lead to singular matrices. However, with the SVD, we can always compute the inverse of the matrix $\\boldsymbol{X}^T\\boldsymbol{X}$.\n",
- "\n",
- "\n",
- "We see that Ridge regression is nothing but the standard OLS with a\n",
- "modified diagonal term added to $\\boldsymbol{X}^T\\boldsymbol{X}$. The consequences, in\n",
- "particular for our discussion of the bias-variance tradeoff are rather\n",
- "interesting. We will see that for specific values of $\\lambda$, we may\n",
- "even reduce the variance of the optimal parameters $\\boldsymbol{\\beta}$. These topics and other related ones, will be discussed after the more linear algebra oriented analysis here.\n",
- "\n",
- "Using our insights about the SVD of the design matrix $\\boldsymbol{X}$ \n",
- "We have already analyzed the OLS solutions in terms of the eigenvectors (the columns) of the right singular value matrix $\\boldsymbol{U}$ as"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "$$\n",
- "\\tilde{\\boldsymbol{y}}_{\\mathrm{OLS}}=\\boldsymbol{X}\\boldsymbol{\\beta} =\\boldsymbol{U}\\boldsymbol{U}^T\\boldsymbol{y}.\n",
- "$$"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "For Ridge regression this becomes"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "$$\n",
- "\\tilde{\\boldsymbol{y}}_{\\mathrm{Ridge}}=\\boldsymbol{X}\\boldsymbol{\\beta}_{\\mathrm{Ridge}} = \\boldsymbol{U\\Sigma V^T}\\left(\\boldsymbol{V}\\boldsymbol{\\Sigma}^2\\boldsymbol{V}^T+\\lambda\\boldsymbol{I} \\right)^{-1}(\\boldsymbol{U\\Sigma V^T})^T\\boldsymbol{y}=\\sum_{j=0}^{p-1}\\boldsymbol{u}_j\\boldsymbol{u}_j^T\\frac{\\sigma_j^2}{\\sigma_j^2+\\lambda}\\boldsymbol{y},\n",
- "$$"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "with the vectors $\\boldsymbol{u}_j$ being the columns of $\\boldsymbol{U}$ from the SVD of the matrix $\\boldsymbol{X}$. \n",
- "\n",
- "\n",
- "Since $\\lambda \\geq 0$, it means that compared to OLS, we have"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "$$\n",
- "\\frac{\\sigma_j^2}{\\sigma_j^2+\\lambda} \\leq 1.\n",
- "$$"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Ridge regression finds the coordinates of $\\boldsymbol{y}$ with respect to the\n",
- "orthonormal basis $\\boldsymbol{U}$, it then shrinks the coordinates by\n",
- "$\\frac{\\sigma_j^2}{\\sigma_j^2+\\lambda}$. Recall that the SVD has\n",
- "eigenvalues ordered in a descending way, that is $\\sigma_i \\geq\n",
- "\\sigma_{i+1}$.\n",
- "\n",
- "For small eigenvalues $\\sigma_i$ it means that their contributions become less important, a fact which can be used to reduce the number of degrees of freedom. More about this when we have covered the material on a statistical interpretation of various linear regression methods.\n",
- "\n",
- "\n",
- "\n",
- "For the sake of simplicity, let us assume that the design matrix is orthonormal, that is"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "$$\n",
- "\\boldsymbol{X}^T\\boldsymbol{X}=(\\boldsymbol{X}^T\\boldsymbol{X})^{-1} =\\boldsymbol{I}.\n",
- "$$"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "In this case the standard OLS results in"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "$$\n",
- "\\boldsymbol{\\beta}^{\\mathrm{OLS}} = \\boldsymbol{X}^T\\boldsymbol{y}=\\sum_{i=0}^{p-1}\\boldsymbol{u}_j\\boldsymbol{u}_j^T\\boldsymbol{y},\n",
- "$$"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "and"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "$$\n",
- "\\boldsymbol{\\beta}^{\\mathrm{Ridge}} = \\left(\\boldsymbol{I}+\\lambda\\boldsymbol{I}\\right)^{-1}\\boldsymbol{X}^T\\boldsymbol{y}=\\left(1+\\lambda\\right)^{-1}\\boldsymbol{\\beta}^{\\mathrm{OLS}},\n",
- "$$"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "that is the Ridge estimator scales the OLS estimator by the inverse of a factor $1+\\lambda$, and\n",
- "the Ridge estimator converges to zero when the hyperparameter goes to\n",
- "infinity.\n",
- "\n",
- "We will come back to more interpreations after we have gone through some of the statistical analysis part. \n",
- "\n",
- "For more discussions of Ridge and Lasso regression, [Wessel van Wieringen's](https://arxiv.org/abs/1509.09169) article is highly recommended.\n",
- "Similarly, [Mehta et al's article](https://arxiv.org/abs/1803.08823) is also recommended.\n",
- "\n",
- "\n",
- "Using the matrix-vector expression for Lasso regression and dropping the parameter $1/n$ in front of the standard means squared error equation, we have the following **cost** function"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "$$\n",
- "C(\\boldsymbol{X},\\boldsymbol{\\beta})=\\left\\{(\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta})^T(\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta})\\right\\}+\\lambda\\vert\\vert\\boldsymbol{\\beta}\\vert\\vert_1,\n",
- "$$"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Taking the derivative with respect to $\\boldsymbol{\\beta}$ and recalling that the derivative of the absolute value is (we drop the boldfaced vector symbol for simplicty)"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "$$\n",
- "\\frac{d \\vert \\beta\\vert}{d \\boldsymbol{\\beta}}=\\mathrm{sgn}(\\boldsymbol{\\beta})=\\left\\{\\begin{array}{cc} 1 & \\beta > 0 \\\\ 0 & \\beta =0\\\\-1 & \\beta < 0, \\end{array}\\right.\n",
- "$$"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "we have that the derivative of the cost function is"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "$$\n",
- "\\frac{\\partial C(\\boldsymbol{X},\\boldsymbol{\\beta})}{\\partial \\boldsymbol{\\beta}}=-2\\boldsymbol{X}^T(\\boldsymbol{y}-\\boldsymbol{X}\\boldsymbol{\\beta})+\\lambda sgn(\\boldsymbol{\\beta})=0,\n",
- "$$"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "and reordering we have"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "$$\n",
- "\\boldsymbol{X}^T\\boldsymbol{X}\\boldsymbol{\\beta})+\\lambda sgn(\\boldsymbol{\\beta})=2\\boldsymbol{X}^T(\\boldsymbol{y}.\n",
- "$$"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "This equation does not lead to a nice analytical equation as in either Ridge regression or ordinary least squares. This equation can however be solved by using standard convex optimization algorithms using for example the Python package [CVXOPT](https://cvxopt.org/). We will discuss this later. \n",
- "\n",
- "## Code for SVD and Inversion of Matrices\n",
- "\n",
- "How do we use the SVD to invert a matrix $\\boldsymbol{X}^\\boldsymbol{X}$ which is singular or near singular?\n",
- "The simple answer is to use the linear algebra function for the pseudoinverse, that is"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 6,
- "metadata": {
- "collapsed": false,
- "editable": true
- },
- "outputs": [],
- "source": [
- "#Ainv = np.linlag.pinv(A)"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Let us first look at a matrix which does not causes problems and write our own function where we just use the SVD."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 7,
- "metadata": {
- "collapsed": false,
- "editable": true
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "[[1 2 3]\n",
- " [2 4 5]\n",
- " [3 5 6]]\n",
- "test U\n",
- "[[ 2.22044605e-16 -7.77156117e-16 -5.55111512e-16]\n",
- " [-7.77156117e-16 0.00000000e+00 -1.11022302e-16]\n",
- " [-5.55111512e-16 -1.11022302e-16 0.00000000e+00]]\n",
- "test VT\n",
- "[[ 1.11022302e-16 -2.22044605e-16 1.38777878e-16]\n",
- " [-2.22044605e-16 -1.11022302e-16 -1.11022302e-16]\n",
- " [ 1.38777878e-16 -1.11022302e-16 0.00000000e+00]]\n",
- "[[2.35367281e-12 1.70885528e-12 3.20632410e-13]\n",
- " [2.17248441e-12 1.46016532e-12 2.00728323e-13]\n",
- " [6.95443703e-13 4.13891144e-13 2.13162821e-14]]\n"
- ]
- }
- ],
- "source": [
- "import numpy as np\n",
- "# SVD inversion\n",
- "def SVDinv(A):\n",
- " ''' Takes as input a numpy matrix A and returns inv(A) based on singular value decomposition (SVD).\n",
- " SVD is numerically more stable than the inversion algorithms provided by\n",
- " numpy and scipy.linalg at the cost of being slower.\n",
- " '''\n",
- " U, s, VT = np.linalg.svd(A)\n",
- " print('test U')\n",
- " print( (np.transpose(U) @ U - U @np.transpose(U)))\n",
- " print('test VT')\n",
- " print( (np.transpose(VT) @ VT - VT @np.transpose(VT)))\n",
- "\n",
- "\n",
- " D = np.zeros((len(U),len(VT)))\n",
- " D = np.diag(s)\n",
- " UT = np.transpose(U); V = np.transpose(VT); invD = np.linalg.inv(D)\n",
- " return np.matmul(V,np.matmul(invD,UT))\n",
- "\n",
- "\n",
- "#X = np.array([ [1.0, -1.0, 2.0], [1.0, 0.0, 1.0], [1.0, 2.0, -1.0], [1.0, 1.0, 0.0] ])\n",
- "# Non-singular square matrix\n",
- "X = np.array( [ [1,2,3],[2,4,5],[3,5,6]])\n",
- "print(X)\n",
- "A = np.transpose(X) @ X\n",
- "# Brute force inversion\n",
- "B = np.linalg.inv(A) # here we could use np.linalg.pinv(A)\n",
- "C = SVDinv(A)\n",
- "print(np.abs(B-C))"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Although our matrix to invert $\\boldsymbol{X}^T\\boldsymbol{X}$ is a square matrix, our matrix may be singular. \n",
- "\n",
- "The pseudoinverse is the generalization of the matrix inverse for square matrices to\n",
- "rectangular matrices where the number of rows and columns are not equal.\n",
- "\n",
- "It is also called the the Moore-Penrose Inverse after two independent discoverers of the method or the Generalized Inverse.\n",
- "It is used for the calculation of the inverse for singular or near singular matrices and for rectangular matrices.\n",
- "\n",
- "Using the SVD we can obtain the pseudoinverse of a matrix $\\boldsymbol{A}$ (labeled here as $\\boldsymbol{A}_{\\mathrm{PI}}$"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "$$\n",
- "\\boldsymbol{A}_{\\mathrm{PI}}= \\boldsymbol{V}\\boldsymbol{D}_{\\mathrm{PI}}\\boldsymbol{U}^T,\n",
- "$$"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "where $\\boldsymbol{D}_{\\mathrm{PI}}$ can be calculated by creating a diagonal matrix from $\\boldsymbol{Sigma}$ where we only keep the singular values (the non-zero values). The following code computes the pseudoinvers of the matrix based on the SVD."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 8,
- "metadata": {
- "collapsed": false,
- "editable": true
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "[[0.3 0.4]\n",
- " [0.5 0.6]\n",
- " [0.7 0.8]\n",
- " [0.9 1. ]]\n",
- "[[-13. -6. 1. 8. ]\n",
- " [ 11.5 5.5 -0.5 -6.5]]\n",
- "[[0. 0. 0. 0.]\n",
- " [0. 0. 0. 0.]]\n"
- ]
- }
- ],
- "source": [
- "import numpy as np\n",
- "# SVD inversion\n",
- "def SVDinv(A):\n",
- " U, s, VT = np.linalg.svd(A)\n",
- " # reciprocals of singular values of s\n",
- " d = 1.0 / s\n",
- " # create m x n D matrix\n",
- " D = np.zeros(A.shape)\n",
- " # populate D with n x n diagonal matrix\n",
- " D[:A.shape[1], :A.shape[1]] = np.diag(d)\n",
- " UT = np.transpose(U)\n",
- " V = np.transpose(VT)\n",
- " return np.matmul(V,np.matmul(D.T,UT))\n",
- "\n",
- "\n",
- "A = np.array([ [0.3, 0.4], [0.5, 0.6], [0.7, 0.8],[0.9, 1.0]])\n",
- "print(A)\n",
- "# Brute force inversion of super-collinear matrix\n",
- "B = np.linalg.pinv(A)\n",
- "print(B)\n",
- "# Compare our own algorithm with pinv\n",
- "C = SVDinv(A)\n",
- "print(np.abs(C-B))"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "As you can see from this example, our own decomposition based on the SVD agrees the pseudoinverse algorithm provided by **Numpy**.\n",
- "\n",
- "\n",
- "\n",
- "## Deriving the Ridge Regression Equations\n",
+ "## Ridge and Lasso Regression\n",
"\n",
"Let us remind ourselves about the expression for the standard Mean Squared Error (MSE) which we used to define our cost function and the equations for the ordinary least squares (OLS) method, that is \n",
"our optimization problem is"
@@ -3240,7 +2850,8 @@
"\n",
"Here we set up the OLS, Ridge and Lasso functionality in order to study the above example. Note that here we have opted for a set of values of $\\lambda$, meaning that we need to perform a search in order to find the optimal values.\n",
"\n",
- "First we study and compare the OLS and Ridge results. The next code compares all three methods."
+ "First we study and compare the OLS and Ridge results. The next code compares all three methods.\n",
+ "We select values of the hyperparameter $\\lambda\\in [10^{-4},10^4]$ and compute the predicted values for ordinary least squares and Ridge regression."
]
},
{
@@ -3255,7 +2866,14 @@
"name": "stdout",
"output_type": "stream",
"text": [
- "[2. 2.]\n",
+ "[2. 2.]"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "\n",
"Training MSE for OLS\n",
"3.0\n"
]
@@ -3269,7 +2887,7 @@
},
"metadata": {
"filenames": {
- "image/png": "/Users/mhjensen/Teaching/MachineLearning/doc/LectureNotes/_build/jupyter_execute/chapter2_287_1.png"
+ "image/png": "/Users/mhjensen/Teaching/MachineLearning/doc/LectureNotes/_build/jupyter_execute/chapter2_245_2.png"
},
"needs_background": "light"
},
@@ -3334,7 +2952,13 @@
"cell_type": "markdown",
"metadata": {},
"source": [
- "We see here that we reach a plateau. What is actually happening?"
+ "We see here that we reach a plateau for the Ridge results. Writing out the coefficients $\\boldsymbol{\\beta}$, we that they are getting smaller and smaller and our error stabilizes since the predicted values of $\\tilde{\\boldsymbol{y}}$ approach zero.\n",
+ "\n",
+ "This happens also for Lasso regression, as seen from the next code\n",
+ "output. The difference is that Lasso shrinks the values of $\\beta$ to\n",
+ "zero at a much earlier stage and the results flatten out. We see that\n",
+ "Lasso gives also an excellent fit for small values of $\\lambda$ and\n",
+ "shows rthe best performance of the three regression methods."
]
},
{
@@ -3563,7 +3187,7 @@
},
"metadata": {
"filenames": {
- "image/png": "/Users/mhjensen/Teaching/MachineLearning/doc/LectureNotes/_build/jupyter_execute/chapter2_289_1.png"
+ "image/png": "/Users/mhjensen/Teaching/MachineLearning/doc/LectureNotes/_build/jupyter_execute/chapter2_247_1.png"
},
"needs_background": "light"
},
@@ -3633,7 +3257,15 @@
"cell_type": "markdown",
"metadata": {},
"source": [
- "Another Example, now with a polynomial fit."
+ "We bring then back our exponential function example and study all\n",
+ "three regression methods. Depending on the level of noise, we note\n",
+ "that for small values of the hyperparameter $\\lambda$ all three\n",
+ "methods produce the same mean squared error. Again, Lasso shrinks the\n",
+ "parameter values to zero much earlier than Ridge regression and the\n",
+ "Lasso results flatten out much earlier since all $\\beta_j=0$ (check\n",
+ "this by printing the values). This case is an example of where OLS\n",
+ "performs best. Lasso and Ridge reproduce the OLS results for a limited\n",
+ "set of $\\lambda$ values."
]
},
{
@@ -3664,7 +3296,7 @@
},
"metadata": {
"filenames": {
- "image/png": "/Users/mhjensen/Teaching/MachineLearning/doc/LectureNotes/_build/jupyter_execute/chapter2_291_1.png"
+ "image/png": "/Users/mhjensen/Teaching/MachineLearning/doc/LectureNotes/_build/jupyter_execute/chapter2_249_1.png"
},
"needs_background": "light"
},
@@ -3752,6 +3384,325 @@
"plt.show()"
]
},
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Both these example send a clear message. The addition of a\n",
+ "shrinkage/regularization term implies that we need to perform a search\n",
+ "for the optimal values of $\\lambda$. We will see this throughout these\n",
+ "series of lectures.\n",
+ "\n",
+ "\n",
+ "As a small addendum, we note that you can also solve this problem using the convex optimization package [CVXOPT](https://cvxopt.org/examples/mlbook/l1regls.html). This requires, in addition to having installed **CVXOPT**, you need to download the file *l1regl.py*.\n",
+ "The following code example solves the simpler problem we discussed above, where we have added the latter python file."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 12,
+ "metadata": {
+ "collapsed": false,
+ "editable": true
+ },
+ "outputs": [],
+ "source": [
+ "from cvxopt import matrix, spdiag, mul, div, sqrt, normal, setseed\n",
+ "from cvxopt import blas, lapack, solvers, sparse, spmatrix\n",
+ "import math\n",
+ "\n",
+ "try:\n",
+ " import mosek\n",
+ " import sys\n",
+ " __MOSEK = True\n",
+ "except: __MOSEK = False\n",
+ "\n",
+ "if __MOSEK:\n",
+ "\n",
+ " def l1regls_mosek(A, b):\n",
+ " \"\"\"\n",
+ "\n",
+ " Returns the solution of l1-norm regularized least-squares problem\n",
+ "\n",
+ " minimize || A*x - b ||_2^2 + e'*u\n",
+ "\n",
+ " subject to -u <= x <= u\n",
+ "\n",
+ " \"\"\"\n",
+ "\n",
+ " m, n = A.size\n",
+ "\n",
+ " env = mosek.Env()\n",
+ " task = env.Task(0,0)\n",
+ " task.set_Stream(mosek.streamtype.log, lambda x: sys.stdout.write(x))\n",
+ "\n",
+ " task.appendvars( 2*n) # number of variables\n",
+ " task.appendcons( 2*n) # number of constraints\n",
+ "\n",
+ " # input quadratic objective\n",
+ " Q = matrix(0.0, (n,n)) \n",
+ " blas.syrk(A, Q, alpha = 2.0, trans='T')\n",
+ "\n",
+ " I = []\n",
+ " for i in range(n):\n",
+ " I.extend(range(i,n))\n",
+ "\n",
+ " J = []\n",
+ " for i in range(n):\n",
+ " J.extend((n-i)*[i])\n",
+ "\n",
+ " task.putqobj(I, J, list(Q[matrix(I) + matrix(J)*n]))\n",
+ " task.putclist(range(2*n), list(-2*A.T*b) + n*[1.0]) # setup linear objective\n",
+ "\n",
+ " # input constraint matrix row by row\n",
+ " for i in range(n):\n",
+ " task.putarow( i, [i, n+i], [1.0, -1.0])\n",
+ " task.putarow( n+i, [i, n+i], [1.0, 1.0])\n",
+ "\n",
+ " # setup bounds on constraints\n",
+ " task.putboundslice(mosek.accmode.con,\n",
+ " 0, n, n*[mosek.boundkey.up], n*[0.0], n*[0.0])\n",
+ " task.putboundslice(mosek.accmode.con,\n",
+ " n, 2*n, n*[mosek.boundkey.lo], n*[0.0], n*[0.0])\n",
+ "\n",
+ " # setup variable bounds\n",
+ " task.putboundslice(mosek.accmode.var,\n",
+ " 0, 2*n, 2*n*[mosek.boundkey.fr], 2*n*[0.0], 2*n*[0.0])\n",
+ "\n",
+ " # optimize the task\n",
+ " task.putobjsense(mosek.objsense.minimize)\n",
+ " task.optimize()\n",
+ " task.solutionsummary(mosek.streamtype.log)\n",
+ " x = n*[0.0]\n",
+ " task.getsolutionslice(mosek.soltype.itr, mosek.solitem.xx, 0, n, x)\n",
+ "\n",
+ " return matrix(x)\n",
+ "\n",
+ " def l1regls_mosek2(A, b):\n",
+ " \"\"\"\n",
+ "\n",
+ " Returns the solution of l1-norm regularized least-squares problem\n",
+ "\n",
+ " minimize w'*w + e'*u\n",
+ "\n",
+ " subject to -u <= x <= u\n",
+ "\n",
+ " A*x - w = b\n",
+ "\n",
+ " \"\"\"\n",
+ "\n",
+ " m, n = A.size\n",
+ "\n",
+ " env = mosek.Env()\n",
+ " task = env.Task(0,0)\n",
+ " task.set_Stream(mosek.streamtype.log, lambda x: sys.stdout.write(x))\n",
+ "\n",
+ " task.appendvars(2*n + m) # number of variables\n",
+ " task.appendcons(2*n + m) # number of constraints\n",
+ "\n",
+ " # input quadratic objective\n",
+ " task.putqobj(range(2*n,2*n+m), range(2*n,2*n+m), m*[2.0])\n",
+ "\n",
+ " task.putclist(range(2*n+m), n*[0.0] + n*[1.0] + m*[0.0]) # setup linear objective\n",
+ "\n",
+ " # input constraint matrix row by row\n",
+ " for i in range(n):\n",
+ " task.putarow( i, [i, n+i], [1.0, -1.0])\n",
+ " task.putarow( n+i, [i, n+i], [1.0, 1.0])\n",
+ "\n",
+ " for i in range(m):\n",
+ " task.putarow( 2*n+i, range(n) + [2*n+i], list(A[i,:]) + [-1.0])\n",
+ "\n",
+ " # setup bounds on constraints\n",
+ " task.putboundslice(mosek.accmode.con,\n",
+ " 0, n, n*[mosek.boundkey.up], n*[0.0], n*[0.0])\n",
+ " task.putboundslice(mosek.accmode.con,\n",
+ " n, 2*n, n*[mosek.boundkey.lo], n*[0.0], n*[0.0])\n",
+ " task.putboundslice(mosek.accmode.con,\n",
+ " 2*n, 2*n+m, m*[mosek.boundkey.fx], list(b), list(b))\n",
+ "\n",
+ " # setup variable bounds\n",
+ " task.putboundslice(mosek.accmode.var, 0, 2*n+m, (2*n+m)*[mosek.boundkey.fr], \n",
+ " (2*n+m)*[0.0], (2*n+m)*[0.0])\n",
+ "\n",
+ " # optimize the task\n",
+ " task.putobjsense(mosek.objsense.minimize)\n",
+ " task.optimize()\n",
+ " task.solutionsummary(mosek.streamtype.log)\n",
+ " x = n*[0.0]\n",
+ " task.getsolutionslice(mosek.soltype.itr, mosek.solitem.xx, 0, n, x)\n",
+ "\n",
+ " return matrix(x)\n",
+ "\n",
+ "def l1regls(A, b):\n",
+ " \"\"\"\n",
+ " \n",
+ " Returns the solution of l1-norm regularized least-squares problem\n",
+ " \n",
+ " minimize || A*x - b ||_2^2 + || x ||_1.\n",
+ "\n",
+ " \"\"\"\n",
+ "\n",
+ " m, n = A.size\n",
+ " q = matrix(1.0, (2*n,1))\n",
+ " q[:n] = -2.0 * A.T * b\n",
+ "\n",
+ " def P(u, v, alpha = 1.0, beta = 0.0 ):\n",
+ " \"\"\"\n",
+ " v := alpha * 2.0 * [ A'*A, 0; 0, 0 ] * u + beta * v \n",
+ " \"\"\"\n",
+ " v *= beta\n",
+ " v[:n] += alpha * 2.0 * A.T * (A * u[:n])\n",
+ "\n",
+ "\n",
+ " def G(u, v, alpha=1.0, beta=0.0, trans='N'):\n",
+ " \"\"\"\n",
+ " v := alpha*[I, -I; -I, -I] * u + beta * v (trans = 'N' or 'T')\n",
+ " \"\"\"\n",
+ "\n",
+ " v *= beta\n",
+ " v[:n] += alpha*(u[:n] - u[n:])\n",
+ " v[n:] += alpha*(-u[:n] - u[n:])\n",
+ "\n",
+ " h = matrix(0.0, (2*n,1))\n",
+ "\n",
+ "\n",
+ " # Customized solver for the KKT system \n",
+ " #\n",
+ " # [ 2.0*A'*A 0 I -I ] [x[:n] ] [bx[:n] ]\n",
+ " # [ 0 0 -I -I ] [x[n:] ] = [bx[n:] ].\n",
+ " # [ I -I -D1^-1 0 ] [zl[:n]] [bzl[:n]]\n",
+ " # [ -I -I 0 -D2^-1 ] [zl[n:]] [bzl[n:]]\n",
+ " #\n",
+ " # where D1 = W['di'][:n]**2, D2 = W['di'][:n]**2.\n",
+ " # \n",
+ " # We first eliminate zl and x[n:]:\n",
+ " #\n",
+ " # ( 2*A'*A + 4*D1*D2*(D1+D2)^-1 ) * x[:n] = \n",
+ " # bx[:n] - (D2-D1)*(D1+D2)^-1 * bx[n:] + \n",
+ " # D1 * ( I + (D2-D1)*(D1+D2)^-1 ) * bzl[:n] - \n",
+ " # D2 * ( I - (D2-D1)*(D1+D2)^-1 ) * bzl[n:] \n",
+ " #\n",
+ " # x[n:] = (D1+D2)^-1 * ( bx[n:] - D1*bzl[:n] - D2*bzl[n:] ) \n",
+ " # - (D2-D1)*(D1+D2)^-1 * x[:n] \n",
+ " #\n",
+ " # zl[:n] = D1 * ( x[:n] - x[n:] - bzl[:n] )\n",
+ " # zl[n:] = D2 * (-x[:n] - x[n:] - bzl[n:] ).\n",
+ " #\n",
+ " # The first equation has the form\n",
+ " #\n",
+ " # (A'*A + D)*x[:n] = rhs\n",
+ " #\n",
+ " # and is equivalent to\n",
+ " #\n",
+ " # [ D A' ] [ x:n] ] = [ rhs ]\n",
+ " # [ A -I ] [ v ] [ 0 ].\n",
+ " #\n",
+ " # It can be solved as \n",
+ " #\n",
+ " # ( A*D^-1*A' + I ) * v = A * D^-1 * rhs\n",
+ " # x[:n] = D^-1 * ( rhs - A'*v ).\n",
+ "\n",
+ " S = matrix(0.0, (m,m))\n",
+ " Asc = matrix(0.0, (m,n))\n",
+ " v = matrix(0.0, (m,1))\n",
+ "\n",
+ " def Fkkt(W):\n",
+ "\n",
+ " # Factor \n",
+ " #\n",
+ " # S = A*D^-1*A' + I \n",
+ " #\n",
+ " # where D = 2*D1*D2*(D1+D2)^-1, D1 = d[:n]**-2, D2 = d[n:]**-2.\n",
+ "\n",
+ " d1, d2 = W['di'][:n]**2, W['di'][n:]**2\n",
+ "\n",
+ " # ds is square root of diagonal of D\n",
+ " ds = math.sqrt(2.0) * div( mul( W['di'][:n], W['di'][n:]), \n",
+ " sqrt(d1+d2) )\n",
+ " d3 = div(d2 - d1, d1 + d2)\n",
+ " \n",
+ " # Asc = A*diag(d)^-1/2\n",
+ " Asc = A * spdiag(ds**-1)\n",
+ "\n",
+ " # S = I + A * D^-1 * A'\n",
+ " blas.syrk(Asc, S)\n",
+ " S[::m+1] += 1.0 \n",
+ " lapack.potrf(S)\n",
+ "\n",
+ " def g(x, y, z):\n",
+ "\n",
+ " x[:n] = 0.5 * ( x[:n] - mul(d3, x[n:]) + \n",
+ " mul(d1, z[:n] + mul(d3, z[:n])) - mul(d2, z[n:] - \n",
+ " mul(d3, z[n:])) )\n",
+ " x[:n] = div( x[:n], ds) \n",
+ "\n",
+ " # Solve\n",
+ " #\n",
+ " # S * v = 0.5 * A * D^-1 * ( bx[:n] - \n",
+ " # (D2-D1)*(D1+D2)^-1 * bx[n:] + \n",
+ " # D1 * ( I + (D2-D1)*(D1+D2)^-1 ) * bzl[:n] - \n",
+ " # D2 * ( I - (D2-D1)*(D1+D2)^-1 ) * bzl[n:] )\n",
+ " \n",
+ " blas.gemv(Asc, x, v)\n",
+ " lapack.potrs(S, v)\n",
+ " \n",
+ " # x[:n] = D^-1 * ( rhs - A'*v ).\n",
+ " blas.gemv(Asc, v, x, alpha=-1.0, beta=1.0, trans='T')\n",
+ " x[:n] = div(x[:n], ds)\n",
+ "\n",
+ " # x[n:] = (D1+D2)^-1 * ( bx[n:] - D1*bzl[:n] - D2*bzl[n:] ) \n",
+ " # - (D2-D1)*(D1+D2)^-1 * x[:n] \n",
+ " x[n:] = div( x[n:] - mul(d1, z[:n]) - mul(d2, z[n:]), d1+d2 )\\\n",
+ " - mul( d3, x[:n] )\n",
+ " \n",
+ " # zl[:n] = D1^1/2 * ( x[:n] - x[n:] - bzl[:n] )\n",
+ " # zl[n:] = D2^1/2 * ( -x[:n] - x[n:] - bzl[n:] ).\n",
+ " z[:n] = mul( W['di'][:n], x[:n] - x[n:] - z[:n] ) \n",
+ " z[n:] = mul( W['di'][n:], -x[:n] - x[n:] - z[n:] ) \n",
+ "\n",
+ " return g\n",
+ "\n",
+ " return solvers.coneqp(P, q, G, h, kktsolver = Fkkt)['x'][:n]"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Then we call the above functions and solve the problem, as done here"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 13,
+ "metadata": {
+ "collapsed": false,
+ "editable": true
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ " pcost dcost gap pres dres\n",
+ " 0: -2.7070e+01 -1.6544e+01 2e+01 6e+00 2e-01\n",
+ " 1: -2.2433e+01 -2.4632e+01 2e+00 9e-16 1e-16\n",
+ " 2: -2.4140e+01 -2.4332e+01 2e-01 1e-15 3e-16\n",
+ " 3: -2.4200e+01 -2.4203e+01 2e-03 5e-16 2e-15\n",
+ " 4: -2.4201e+01 -2.4201e+01 2e-05 9e-16 6e-16\n",
+ "Optimal solution found.\n"
+ ]
+ }
+ ],
+ "source": [
+ "from cvxopt import matrix, normal\n",
+ "\n",
+ "X = matrix( [ [ 2, 0, 1], [0, 1, 3]])\n",
+ "y = matrix( [4, 2, 3])\n",
+ "x = l1regls(X,y)"
+ ]
+ },
{
"cell_type": "markdown",
"metadata": {},
@@ -4446,7 +4397,7 @@
},
{
"cell_type": "code",
- "execution_count": 12,
+ "execution_count": 14,
"metadata": {
"collapsed": false,
"editable": true
@@ -4460,7 +4411,13 @@
"Test MSE OLS\n",
"0.958228616652075\n",
"0.001 [ 1.0170259 0.27852549 -1.40702 1.03193199 0. ]\n",
- "0.001 [ 1.034342 -0.18063928 -0. 0. 0. ]\n",
+ "0.001 [ 1.034342 -0.18063928 -0. 0. 0. ]\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
"0.021544346900318832 [ 1.01825571 0.26412372 -1.37186301 1.00890601 0. ]\n",
"0.021544346900318832 [ 0.92280994 -0. -0. -0. 0. ]\n",
"0.46415888336127775 [ 1.0344707 0.07160764 -0.89928965 0.69843037 0. ]\n",
@@ -4478,7 +4435,7 @@
},
"metadata": {
"filenames": {
- "image/png": "/Users/mhjensen/Teaching/MachineLearning/doc/LectureNotes/_build/jupyter_execute/chapter2_351_1.png"
+ "image/png": "/Users/mhjensen/Teaching/MachineLearning/doc/LectureNotes/_build/jupyter_execute/chapter2_313_2.png"
},
"needs_background": "light"
},
diff --git a/doc/LectureNotes/_build/jupyter_execute/chapter2.py b/doc/LectureNotes/_build/jupyter_execute/chapter2.py
index 9dfb2e212..99539ec90 100644
--- a/doc/LectureNotes/_build/jupyter_execute/chapter2.py
+++ b/doc/LectureNotes/_build/jupyter_execute/chapter2.py
@@ -342,6 +342,88 @@ resize the matrices and set up a diagonal matrix as done in the above
example
+## Code for SVD and Inversion of Matrices
+
+How do we use the SVD to invert a matrix $\boldsymbol{X}^\boldsymbol{X}$ which is singular or near singular?
+The simple answer is to use the linear algebra function for the pseudoinverse, that is
+
+#Ainv = np.linlag.pinv(A)
+
+Let us first look at a matrix which does not causes problems and write our own function where we just use the SVD.
+
+import numpy as np
+# SVD inversion
+def SVDinv(A):
+ ''' Takes as input a numpy matrix A and returns inv(A) based on singular value decomposition (SVD).
+ SVD is numerically more stable than the inversion algorithms provided by
+ numpy and scipy.linalg at the cost of being slower.
+ '''
+ U, s, VT = np.linalg.svd(A)
+ print('test U')
+ print( (np.transpose(U) @ U - U @np.transpose(U)))
+ print('test VT')
+ print( (np.transpose(VT) @ VT - VT @np.transpose(VT)))
+
+
+ D = np.zeros((len(U),len(VT)))
+ D = np.diag(s)
+ UT = np.transpose(U); V = np.transpose(VT); invD = np.linalg.inv(D)
+ return np.matmul(V,np.matmul(invD,UT))
+
+
+#X = np.array([ [1.0, -1.0, 2.0], [1.0, 0.0, 1.0], [1.0, 2.0, -1.0], [1.0, 1.0, 0.0] ])
+# Non-singular square matrix
+X = np.array( [ [1,2,3],[2,4,5],[3,5,6]])
+print(X)
+A = np.transpose(X) @ X
+# Brute force inversion
+B = np.linalg.pinv(A) # here we could use np.linalg.inv(A), try it!
+C = SVDinv(A)
+print(np.abs(B-C))
+
+Although our matrix to invert $\boldsymbol{X}^T\boldsymbol{X}$ is a square matrix, our matrix may be singular.
+
+The pseudoinverse is the generalization of the matrix inverse for square matrices to
+rectangular matrices where the number of rows and columns are not equal.
+
+It is also called the the Moore-Penrose Inverse after two independent discoverers of the method or the Generalized Inverse.
+It is used for the calculation of the inverse for singular or near singular matrices and for rectangular matrices.
+
+Using the SVD we can obtain the pseudoinverse of a matrix $\boldsymbol{A}$ (labeled here as $\boldsymbol{A}_{\mathrm{PI}}$
+
+$$
+\boldsymbol{A}_{\mathrm{PI}}= \boldsymbol{V}\boldsymbol{D}_{\mathrm{PI}}\boldsymbol{U}^T,
+$$
+
+where $\boldsymbol{D}_{\mathrm{PI}}$ can be calculated by creating a diagonal matrix from $\boldsymbol{Sigma}$ where we only keep the singular values (the non-zero values). The following code computes the pseudoinvers of the matrix based on the SVD.
+
+import numpy as np
+# SVD inversion
+def SVDinv(A):
+ U, s, VT = np.linalg.svd(A)
+ # reciprocals of singular values of s
+ d = 1.0 / s
+ # create m x n D matrix
+ D = np.zeros(A.shape)
+ # populate D with n x n diagonal matrix
+ D[:A.shape[1], :A.shape[1]] = np.diag(d)
+ UT = np.transpose(U)
+ V = np.transpose(VT)
+ return np.matmul(V,np.matmul(D.T,UT))
+
+
+A = np.array([ [0.3, 0.4], [0.5, 0.6], [0.7, 0.8],[0.9, 1.0]])
+print(A)
+# Brute force inversion of super-collinear matrix
+B = np.linalg.pinv(A)
+print(B)
+# Compare our own algorithm with pinv
+C = SVDinv(A)
+print(np.abs(C-B))
+
+As you can see from these examples, our own decomposition based on the SVD agrees the pseudoinverse algorithm provided by **Numpy**.
+
+
@@ -979,270 +1061,8 @@ values and the column vectors of $\boldsymbol{V}$.
-## Ridge and LASSO Regression
-Let us remind ourselves about the expression for the standard Mean Squared Error (MSE) which we used to define our cost function and the equations for the ordinary least squares (OLS) method, that is
-our optimization problem is
-
-$$
-{\displaystyle \min_{\boldsymbol{\beta}\in {\mathbb{R}}^{p}}}\frac{1}{n}\left\{\left(\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\right)^T\left(\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\right)\right\}.
-$$
-
-or we can state it as
-
-$$
-{\displaystyle \min_{\boldsymbol{\beta}\in
-{\mathbb{R}}^{p}}}\frac{1}{n}\sum_{i=0}^{n-1}\left(y_i-\tilde{y}_i\right)^2=\frac{1}{n}\vert\vert \boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\vert\vert_2^2,
-$$
-
-where we have used the definition of a norm-2 vector, that is
-
-$$
-\vert\vert \boldsymbol{x}\vert\vert_2 = \sqrt{\sum_i x_i^2}.
-$$
-
-By minimizing the above equation with respect to the parameters
-$\boldsymbol{\beta}$ we could then obtain an analytical expression for the
-parameters $\boldsymbol{\beta}$. We can add a regularization parameter $\lambda$ by
-defining a new cost function to be optimized, that is
-
-$$
-{\displaystyle \min_{\boldsymbol{\beta}\in
-{\mathbb{R}}^{p}}}\frac{1}{n}\vert\vert \boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\vert\vert_2^2+\lambda\vert\vert \boldsymbol{\beta}\vert\vert_2^2
-$$
-
-which leads to the Ridge regression minimization problem where we
-require that $\vert\vert \boldsymbol{\beta}\vert\vert_2^2\le t$, where $t$ is
-a finite number larger than zero. By defining
-
-$$
-C(\boldsymbol{X},\boldsymbol{\beta})=\frac{1}{n}\vert\vert \boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\vert\vert_2^2+\lambda\vert\vert \boldsymbol{\beta}\vert\vert_1,
-$$
-
-we have a new optimization equation
-
-$$
-{\displaystyle \min_{\boldsymbol{\beta}\in
-{\mathbb{R}}^{p}}}\frac{1}{n}\vert\vert \boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta}\vert\vert_2^2+\lambda\vert\vert \boldsymbol{\beta}\vert\vert_1
-$$
-
-which leads to Lasso regression. Lasso stands for least absolute shrinkage and selection operator.
-
-Here we have defined the norm-1 as
-
-$$
-\vert\vert \boldsymbol{x}\vert\vert_1 = \sum_i \vert x_i\vert.
-$$
-
-Using the matrix-vector expression for Ridge regression and dropping the parameter $1/n$ in front of the standard means squared error equation, we have
-
-$$
-C(\boldsymbol{X},\boldsymbol{\beta})=\left\{(\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta})^T(\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta})\right\}+\lambda\boldsymbol{\beta}^T\boldsymbol{\beta},
-$$
-
-and
-taking the derivatives with respect to $\boldsymbol{\beta}$ we obtain then
-a slightly modified matrix inversion problem which for finite values
-of $\lambda$ does not suffer from singularity problems. We obtain
-the optimal parameters
-
-$$
-\hat{\boldsymbol{\beta}}_{\mathrm{Ridge}} = \left(\boldsymbol{X}^T\boldsymbol{X}+\lambda\boldsymbol{I}\right)^{-1}\boldsymbol{X}^T\boldsymbol{y},
-$$
-
-with $\boldsymbol{I}$ being a $p\times p$ identity matrix with the constraint that
-
-$$
-\sum_{i=0}^{p-1} \beta_i^2 \leq t,
-$$
-
-with $t$ a finite positive number.
-
-When we compare this with the ordinary least squares result we have
-
-$$
-\hat{\boldsymbol{\beta}}_{\mathrm{OLS}} = \left(\boldsymbol{X}^T\boldsymbol{X}\right)^{-1}\boldsymbol{X}^T\boldsymbol{y},
-$$
-
-which can lead to singular matrices. However, with the SVD, we can always compute the inverse of the matrix $\boldsymbol{X}^T\boldsymbol{X}$.
-
-
-We see that Ridge regression is nothing but the standard OLS with a
-modified diagonal term added to $\boldsymbol{X}^T\boldsymbol{X}$. The consequences, in
-particular for our discussion of the bias-variance tradeoff are rather
-interesting. We will see that for specific values of $\lambda$, we may
-even reduce the variance of the optimal parameters $\boldsymbol{\beta}$. These topics and other related ones, will be discussed after the more linear algebra oriented analysis here.
-
-Using our insights about the SVD of the design matrix $\boldsymbol{X}$
-We have already analyzed the OLS solutions in terms of the eigenvectors (the columns) of the right singular value matrix $\boldsymbol{U}$ as
-
-$$
-\tilde{\boldsymbol{y}}_{\mathrm{OLS}}=\boldsymbol{X}\boldsymbol{\beta} =\boldsymbol{U}\boldsymbol{U}^T\boldsymbol{y}.
-$$
-
-For Ridge regression this becomes
-
-$$
-\tilde{\boldsymbol{y}}_{\mathrm{Ridge}}=\boldsymbol{X}\boldsymbol{\beta}_{\mathrm{Ridge}} = \boldsymbol{U\Sigma V^T}\left(\boldsymbol{V}\boldsymbol{\Sigma}^2\boldsymbol{V}^T+\lambda\boldsymbol{I} \right)^{-1}(\boldsymbol{U\Sigma V^T})^T\boldsymbol{y}=\sum_{j=0}^{p-1}\boldsymbol{u}_j\boldsymbol{u}_j^T\frac{\sigma_j^2}{\sigma_j^2+\lambda}\boldsymbol{y},
-$$
-
-with the vectors $\boldsymbol{u}_j$ being the columns of $\boldsymbol{U}$ from the SVD of the matrix $\boldsymbol{X}$.
-
-
-Since $\lambda \geq 0$, it means that compared to OLS, we have
-
-$$
-\frac{\sigma_j^2}{\sigma_j^2+\lambda} \leq 1.
-$$
-
-Ridge regression finds the coordinates of $\boldsymbol{y}$ with respect to the
-orthonormal basis $\boldsymbol{U}$, it then shrinks the coordinates by
-$\frac{\sigma_j^2}{\sigma_j^2+\lambda}$. Recall that the SVD has
-eigenvalues ordered in a descending way, that is $\sigma_i \geq
-\sigma_{i+1}$.
-
-For small eigenvalues $\sigma_i$ it means that their contributions become less important, a fact which can be used to reduce the number of degrees of freedom. More about this when we have covered the material on a statistical interpretation of various linear regression methods.
-
-
-
-For the sake of simplicity, let us assume that the design matrix is orthonormal, that is
-
-$$
-\boldsymbol{X}^T\boldsymbol{X}=(\boldsymbol{X}^T\boldsymbol{X})^{-1} =\boldsymbol{I}.
-$$
-
-In this case the standard OLS results in
-
-$$
-\boldsymbol{\beta}^{\mathrm{OLS}} = \boldsymbol{X}^T\boldsymbol{y}=\sum_{i=0}^{p-1}\boldsymbol{u}_j\boldsymbol{u}_j^T\boldsymbol{y},
-$$
-
-and
-
-$$
-\boldsymbol{\beta}^{\mathrm{Ridge}} = \left(\boldsymbol{I}+\lambda\boldsymbol{I}\right)^{-1}\boldsymbol{X}^T\boldsymbol{y}=\left(1+\lambda\right)^{-1}\boldsymbol{\beta}^{\mathrm{OLS}},
-$$
-
-that is the Ridge estimator scales the OLS estimator by the inverse of a factor $1+\lambda$, and
-the Ridge estimator converges to zero when the hyperparameter goes to
-infinity.
-
-We will come back to more interpreations after we have gone through some of the statistical analysis part.
-
-For more discussions of Ridge and Lasso regression, [Wessel van Wieringen's](https://arxiv.org/abs/1509.09169) article is highly recommended.
-Similarly, [Mehta et al's article](https://arxiv.org/abs/1803.08823) is also recommended.
-
-
-Using the matrix-vector expression for Lasso regression and dropping the parameter $1/n$ in front of the standard means squared error equation, we have the following **cost** function
-
-$$
-C(\boldsymbol{X},\boldsymbol{\beta})=\left\{(\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta})^T(\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta})\right\}+\lambda\vert\vert\boldsymbol{\beta}\vert\vert_1,
-$$
-
-Taking the derivative with respect to $\boldsymbol{\beta}$ and recalling that the derivative of the absolute value is (we drop the boldfaced vector symbol for simplicty)
-
-$$
-\frac{d \vert \beta\vert}{d \boldsymbol{\beta}}=\mathrm{sgn}(\boldsymbol{\beta})=\left\{\begin{array}{cc} 1 & \beta > 0 \\ 0 & \beta =0\\-1 & \beta < 0, \end{array}\right.
-$$
-
-we have that the derivative of the cost function is
-
-$$
-\frac{\partial C(\boldsymbol{X},\boldsymbol{\beta})}{\partial \boldsymbol{\beta}}=-2\boldsymbol{X}^T(\boldsymbol{y}-\boldsymbol{X}\boldsymbol{\beta})+\lambda sgn(\boldsymbol{\beta})=0,
-$$
-
-and reordering we have
-
-$$
-\boldsymbol{X}^T\boldsymbol{X}\boldsymbol{\beta})+\lambda sgn(\boldsymbol{\beta})=2\boldsymbol{X}^T(\boldsymbol{y}.
-$$
-
-This equation does not lead to a nice analytical equation as in either Ridge regression or ordinary least squares. This equation can however be solved by using standard convex optimization algorithms using for example the Python package [CVXOPT](https://cvxopt.org/). We will discuss this later.
-
-## Code for SVD and Inversion of Matrices
-
-How do we use the SVD to invert a matrix $\boldsymbol{X}^\boldsymbol{X}$ which is singular or near singular?
-The simple answer is to use the linear algebra function for the pseudoinverse, that is
-
-#Ainv = np.linlag.pinv(A)
-
-Let us first look at a matrix which does not causes problems and write our own function where we just use the SVD.
-
-import numpy as np
-# SVD inversion
-def SVDinv(A):
- ''' Takes as input a numpy matrix A and returns inv(A) based on singular value decomposition (SVD).
- SVD is numerically more stable than the inversion algorithms provided by
- numpy and scipy.linalg at the cost of being slower.
- '''
- U, s, VT = np.linalg.svd(A)
- print('test U')
- print( (np.transpose(U) @ U - U @np.transpose(U)))
- print('test VT')
- print( (np.transpose(VT) @ VT - VT @np.transpose(VT)))
-
-
- D = np.zeros((len(U),len(VT)))
- D = np.diag(s)
- UT = np.transpose(U); V = np.transpose(VT); invD = np.linalg.inv(D)
- return np.matmul(V,np.matmul(invD,UT))
-
-
-#X = np.array([ [1.0, -1.0, 2.0], [1.0, 0.0, 1.0], [1.0, 2.0, -1.0], [1.0, 1.0, 0.0] ])
-# Non-singular square matrix
-X = np.array( [ [1,2,3],[2,4,5],[3,5,6]])
-print(X)
-A = np.transpose(X) @ X
-# Brute force inversion
-B = np.linalg.inv(A) # here we could use np.linalg.pinv(A)
-C = SVDinv(A)
-print(np.abs(B-C))
-
-Although our matrix to invert $\boldsymbol{X}^T\boldsymbol{X}$ is a square matrix, our matrix may be singular.
-
-The pseudoinverse is the generalization of the matrix inverse for square matrices to
-rectangular matrices where the number of rows and columns are not equal.
-
-It is also called the the Moore-Penrose Inverse after two independent discoverers of the method or the Generalized Inverse.
-It is used for the calculation of the inverse for singular or near singular matrices and for rectangular matrices.
-
-Using the SVD we can obtain the pseudoinverse of a matrix $\boldsymbol{A}$ (labeled here as $\boldsymbol{A}_{\mathrm{PI}}$
-
-$$
-\boldsymbol{A}_{\mathrm{PI}}= \boldsymbol{V}\boldsymbol{D}_{\mathrm{PI}}\boldsymbol{U}^T,
-$$
-
-where $\boldsymbol{D}_{\mathrm{PI}}$ can be calculated by creating a diagonal matrix from $\boldsymbol{Sigma}$ where we only keep the singular values (the non-zero values). The following code computes the pseudoinvers of the matrix based on the SVD.
-
-import numpy as np
-# SVD inversion
-def SVDinv(A):
- U, s, VT = np.linalg.svd(A)
- # reciprocals of singular values of s
- d = 1.0 / s
- # create m x n D matrix
- D = np.zeros(A.shape)
- # populate D with n x n diagonal matrix
- D[:A.shape[1], :A.shape[1]] = np.diag(d)
- UT = np.transpose(U)
- V = np.transpose(VT)
- return np.matmul(V,np.matmul(D.T,UT))
-
-
-A = np.array([ [0.3, 0.4], [0.5, 0.6], [0.7, 0.8],[0.9, 1.0]])
-print(A)
-# Brute force inversion of super-collinear matrix
-B = np.linalg.pinv(A)
-print(B)
-# Compare our own algorithm with pinv
-C = SVDinv(A)
-print(np.abs(C-B))
-
-As you can see from this example, our own decomposition based on the SVD agrees the pseudoinverse algorithm provided by **Numpy**.
-
-
-
-## Deriving the Ridge Regression Equations
+## Ridge and Lasso Regression
Let us remind ourselves about the expression for the standard Mean Squared Error (MSE) which we used to define our cost function and the equations for the ordinary least squares (OLS) method, that is
our optimization problem is
@@ -1614,6 +1434,7 @@ Using the constraint on $\beta_0$ and $\beta_1$ we can then find the optimal val
Here we set up the OLS, Ridge and Lasso functionality in order to study the above example. Note that here we have opted for a set of values of $\lambda$, meaning that we need to perform a search in order to find the optimal values.
First we study and compare the OLS and Ridge results. The next code compares all three methods.
+We select values of the hyperparameter $\lambda\in [10^{-4},10^4]$ and compute the predicted values for ordinary least squares and Ridge regression.
%matplotlib inline
@@ -1667,7 +1488,13 @@ plt.ylabel('MSE')
plt.legend()
plt.show()
-We see here that we reach a plateau. What is actually happening?
+We see here that we reach a plateau for the Ridge results. Writing out the coefficients $\boldsymbol{\beta}$, we that they are getting smaller and smaller and our error stabilizes since the predicted values of $\tilde{\boldsymbol{y}}$ approach zero.
+
+This happens also for Lasso regression, as seen from the next code
+output. The difference is that Lasso shrinks the values of $\beta$ to
+zero at a much earlier stage and the results flatten out. We see that
+Lasso gives also an excellent fit for small values of $\lambda$ and
+shows rthe best performance of the three regression methods.
import os
import numpy as np
@@ -1726,7 +1553,15 @@ plt.ylabel('MSE')
plt.legend()
plt.show()
-Another Example, now with a polynomial fit.
+We bring then back our exponential function example and study all
+three regression methods. Depending on the level of noise, we note
+that for small values of the hyperparameter $\lambda$ all three
+methods produce the same mean squared error. Again, Lasso shrinks the
+parameter values to zero much earlier than Ridge regression and the
+Lasso results flatten out much earlier since all $\beta_j=0$ (check
+this by printing the values). This case is an example of where OLS
+performs best. Lasso and Ridge reproduce the OLS results for a limited
+set of $\lambda$ values.
import os
import numpy as np
@@ -1807,6 +1642,281 @@ plt.ylabel('MSE')
plt.legend()
plt.show()
+Both these example send a clear message. The addition of a
+shrinkage/regularization term implies that we need to perform a search
+for the optimal values of $\lambda$. We will see this throughout these
+series of lectures.
+
+
+As a small addendum, we note that you can also solve this problem using the convex optimization package [CVXOPT](https://cvxopt.org/examples/mlbook/l1regls.html). This requires, in addition to having installed **CVXOPT**, you need to download the file *l1regl.py*.
+The following code example solves the simpler problem we discussed above, where we have added the latter python file.
+
+from cvxopt import matrix, spdiag, mul, div, sqrt, normal, setseed
+from cvxopt import blas, lapack, solvers, sparse, spmatrix
+import math
+
+try:
+ import mosek
+ import sys
+ __MOSEK = True
+except: __MOSEK = False
+
+if __MOSEK:
+
+ def l1regls_mosek(A, b):
+ """
+
+ Returns the solution of l1-norm regularized least-squares problem
+
+ minimize || A*x - b ||_2^2 + e'*u
+
+ subject to -u <= x <= u
+
+ """
+
+ m, n = A.size
+
+ env = mosek.Env()
+ task = env.Task(0,0)
+ task.set_Stream(mosek.streamtype.log, lambda x: sys.stdout.write(x))
+
+ task.appendvars( 2*n) # number of variables
+ task.appendcons( 2*n) # number of constraints
+
+ # input quadratic objective
+ Q = matrix(0.0, (n,n))
+ blas.syrk(A, Q, alpha = 2.0, trans='T')
+
+ I = []
+ for i in range(n):
+ I.extend(range(i,n))
+
+ J = []
+ for i in range(n):
+ J.extend((n-i)*[i])
+
+ task.putqobj(I, J, list(Q[matrix(I) + matrix(J)*n]))
+ task.putclist(range(2*n), list(-2*A.T*b) + n*[1.0]) # setup linear objective
+
+ # input constraint matrix row by row
+ for i in range(n):
+ task.putarow( i, [i, n+i], [1.0, -1.0])
+ task.putarow( n+i, [i, n+i], [1.0, 1.0])
+
+ # setup bounds on constraints
+ task.putboundslice(mosek.accmode.con,
+ 0, n, n*[mosek.boundkey.up], n*[0.0], n*[0.0])
+ task.putboundslice(mosek.accmode.con,
+ n, 2*n, n*[mosek.boundkey.lo], n*[0.0], n*[0.0])
+
+ # setup variable bounds
+ task.putboundslice(mosek.accmode.var,
+ 0, 2*n, 2*n*[mosek.boundkey.fr], 2*n*[0.0], 2*n*[0.0])
+
+ # optimize the task
+ task.putobjsense(mosek.objsense.minimize)
+ task.optimize()
+ task.solutionsummary(mosek.streamtype.log)
+ x = n*[0.0]
+ task.getsolutionslice(mosek.soltype.itr, mosek.solitem.xx, 0, n, x)
+
+ return matrix(x)
+
+ def l1regls_mosek2(A, b):
+ """
+
+ Returns the solution of l1-norm regularized least-squares problem
+
+ minimize w'*w + e'*u
+
+ subject to -u <= x <= u
+
+ A*x - w = b
+
+ """
+
+ m, n = A.size
+
+ env = mosek.Env()
+ task = env.Task(0,0)
+ task.set_Stream(mosek.streamtype.log, lambda x: sys.stdout.write(x))
+
+ task.appendvars(2*n + m) # number of variables
+ task.appendcons(2*n + m) # number of constraints
+
+ # input quadratic objective
+ task.putqobj(range(2*n,2*n+m), range(2*n,2*n+m), m*[2.0])
+
+ task.putclist(range(2*n+m), n*[0.0] + n*[1.0] + m*[0.0]) # setup linear objective
+
+ # input constraint matrix row by row
+ for i in range(n):
+ task.putarow( i, [i, n+i], [1.0, -1.0])
+ task.putarow( n+i, [i, n+i], [1.0, 1.0])
+
+ for i in range(m):
+ task.putarow( 2*n+i, range(n) + [2*n+i], list(A[i,:]) + [-1.0])
+
+ # setup bounds on constraints
+ task.putboundslice(mosek.accmode.con,
+ 0, n, n*[mosek.boundkey.up], n*[0.0], n*[0.0])
+ task.putboundslice(mosek.accmode.con,
+ n, 2*n, n*[mosek.boundkey.lo], n*[0.0], n*[0.0])
+ task.putboundslice(mosek.accmode.con,
+ 2*n, 2*n+m, m*[mosek.boundkey.fx], list(b), list(b))
+
+ # setup variable bounds
+ task.putboundslice(mosek.accmode.var, 0, 2*n+m, (2*n+m)*[mosek.boundkey.fr],
+ (2*n+m)*[0.0], (2*n+m)*[0.0])
+
+ # optimize the task
+ task.putobjsense(mosek.objsense.minimize)
+ task.optimize()
+ task.solutionsummary(mosek.streamtype.log)
+ x = n*[0.0]
+ task.getsolutionslice(mosek.soltype.itr, mosek.solitem.xx, 0, n, x)
+
+ return matrix(x)
+
+def l1regls(A, b):
+ """
+
+ Returns the solution of l1-norm regularized least-squares problem
+
+ minimize || A*x - b ||_2^2 + || x ||_1.
+
+ """
+
+ m, n = A.size
+ q = matrix(1.0, (2*n,1))
+ q[:n] = -2.0 * A.T * b
+
+ def P(u, v, alpha = 1.0, beta = 0.0 ):
+ """
+ v := alpha * 2.0 * [ A'*A, 0; 0, 0 ] * u + beta * v
+ """
+ v *= beta
+ v[:n] += alpha * 2.0 * A.T * (A * u[:n])
+
+
+ def G(u, v, alpha=1.0, beta=0.0, trans='N'):
+ """
+ v := alpha*[I, -I; -I, -I] * u + beta * v (trans = 'N' or 'T')
+ """
+
+ v *= beta
+ v[:n] += alpha*(u[:n] - u[n:])
+ v[n:] += alpha*(-u[:n] - u[n:])
+
+ h = matrix(0.0, (2*n,1))
+
+
+ # Customized solver for the KKT system
+ #
+ # [ 2.0*A'*A 0 I -I ] [x[:n] ] [bx[:n] ]
+ # [ 0 0 -I -I ] [x[n:] ] = [bx[n:] ].
+ # [ I -I -D1^-1 0 ] [zl[:n]] [bzl[:n]]
+ # [ -I -I 0 -D2^-1 ] [zl[n:]] [bzl[n:]]
+ #
+ # where D1 = W['di'][:n]**2, D2 = W['di'][:n]**2.
+ #
+ # We first eliminate zl and x[n:]:
+ #
+ # ( 2*A'*A + 4*D1*D2*(D1+D2)^-1 ) * x[:n] =
+ # bx[:n] - (D2-D1)*(D1+D2)^-1 * bx[n:] +
+ # D1 * ( I + (D2-D1)*(D1+D2)^-1 ) * bzl[:n] -
+ # D2 * ( I - (D2-D1)*(D1+D2)^-1 ) * bzl[n:]
+ #
+ # x[n:] = (D1+D2)^-1 * ( bx[n:] - D1*bzl[:n] - D2*bzl[n:] )
+ # - (D2-D1)*(D1+D2)^-1 * x[:n]
+ #
+ # zl[:n] = D1 * ( x[:n] - x[n:] - bzl[:n] )
+ # zl[n:] = D2 * (-x[:n] - x[n:] - bzl[n:] ).
+ #
+ # The first equation has the form
+ #
+ # (A'*A + D)*x[:n] = rhs
+ #
+ # and is equivalent to
+ #
+ # [ D A' ] [ x:n] ] = [ rhs ]
+ # [ A -I ] [ v ] [ 0 ].
+ #
+ # It can be solved as
+ #
+ # ( A*D^-1*A' + I ) * v = A * D^-1 * rhs
+ # x[:n] = D^-1 * ( rhs - A'*v ).
+
+ S = matrix(0.0, (m,m))
+ Asc = matrix(0.0, (m,n))
+ v = matrix(0.0, (m,1))
+
+ def Fkkt(W):
+
+ # Factor
+ #
+ # S = A*D^-1*A' + I
+ #
+ # where D = 2*D1*D2*(D1+D2)^-1, D1 = d[:n]**-2, D2 = d[n:]**-2.
+
+ d1, d2 = W['di'][:n]**2, W['di'][n:]**2
+
+ # ds is square root of diagonal of D
+ ds = math.sqrt(2.0) * div( mul( W['di'][:n], W['di'][n:]),
+ sqrt(d1+d2) )
+ d3 = div(d2 - d1, d1 + d2)
+
+ # Asc = A*diag(d)^-1/2
+ Asc = A * spdiag(ds**-1)
+
+ # S = I + A * D^-1 * A'
+ blas.syrk(Asc, S)
+ S[::m+1] += 1.0
+ lapack.potrf(S)
+
+ def g(x, y, z):
+
+ x[:n] = 0.5 * ( x[:n] - mul(d3, x[n:]) +
+ mul(d1, z[:n] + mul(d3, z[:n])) - mul(d2, z[n:] -
+ mul(d3, z[n:])) )
+ x[:n] = div( x[:n], ds)
+
+ # Solve
+ #
+ # S * v = 0.5 * A * D^-1 * ( bx[:n] -
+ # (D2-D1)*(D1+D2)^-1 * bx[n:] +
+ # D1 * ( I + (D2-D1)*(D1+D2)^-1 ) * bzl[:n] -
+ # D2 * ( I - (D2-D1)*(D1+D2)^-1 ) * bzl[n:] )
+
+ blas.gemv(Asc, x, v)
+ lapack.potrs(S, v)
+
+ # x[:n] = D^-1 * ( rhs - A'*v ).
+ blas.gemv(Asc, v, x, alpha=-1.0, beta=1.0, trans='T')
+ x[:n] = div(x[:n], ds)
+
+ # x[n:] = (D1+D2)^-1 * ( bx[n:] - D1*bzl[:n] - D2*bzl[n:] )
+ # - (D2-D1)*(D1+D2)^-1 * x[:n]
+ x[n:] = div( x[n:] - mul(d1, z[:n]) - mul(d2, z[n:]), d1+d2 )\
+ - mul( d3, x[:n] )
+
+ # zl[:n] = D1^1/2 * ( x[:n] - x[n:] - bzl[:n] )
+ # zl[n:] = D2^1/2 * ( -x[:n] - x[n:] - bzl[n:] ).
+ z[:n] = mul( W['di'][:n], x[:n] - x[n:] - z[:n] )
+ z[n:] = mul( W['di'][n:], -x[:n] - x[n:] - z[n:] )
+
+ return g
+
+ return solvers.coneqp(P, q, G, h, kktsolver = Fkkt)['x'][:n]
+
+Then we call the above functions and solve the problem, as done here
+
+from cvxopt import matrix, normal
+
+X = matrix( [ [ 2, 0, 1], [0, 1, 3]])
+y = matrix( [4, 2, 3])
+x = l1regls(X,y)
+
## Linking the regression analysis with a statistical interpretation
We will now couple the discussions of ordinary least squares, Ridge
diff --git a/doc/LectureNotes/_build/jupyter_execute/chapter2_245_2.png b/doc/LectureNotes/_build/jupyter_execute/chapter2_245_2.png
new file mode 100644
index 000000000..dc3a3ce3e
Binary files /dev/null and b/doc/LectureNotes/_build/jupyter_execute/chapter2_245_2.png differ
diff --git a/doc/LectureNotes/_build/jupyter_execute/chapter2_247_1.png b/doc/LectureNotes/_build/jupyter_execute/chapter2_247_1.png
new file mode 100644
index 000000000..ac2df9e69
Binary files /dev/null and b/doc/LectureNotes/_build/jupyter_execute/chapter2_247_1.png differ
diff --git a/doc/LectureNotes/_build/jupyter_execute/chapter2_249_1.png b/doc/LectureNotes/_build/jupyter_execute/chapter2_249_1.png
new file mode 100644
index 000000000..9666ad291
Binary files /dev/null and b/doc/LectureNotes/_build/jupyter_execute/chapter2_249_1.png differ
diff --git a/doc/LectureNotes/_build/jupyter_execute/chapter2_313_2.png b/doc/LectureNotes/_build/jupyter_execute/chapter2_313_2.png
new file mode 100644
index 000000000..07584f6bb
Binary files /dev/null and b/doc/LectureNotes/_build/jupyter_execute/chapter2_313_2.png differ
diff --git a/doc/LectureNotes/_build/jupyter_execute/chapter3.ipynb b/doc/LectureNotes/_build/jupyter_execute/chapter3.ipynb
index 11daf9011..c7de9b6cd 100644
--- a/doc/LectureNotes/_build/jupyter_execute/chapter3.ipynb
+++ b/doc/LectureNotes/_build/jupyter_execute/chapter3.ipynb
@@ -397,10 +397,10 @@
"name": "stdout",
"output_type": "stream",
"text": [
- "Runtime: 0.139462 sec\n",
+ "Runtime: 0.136687 sec\n",
"Jackknife Statistics :\n",
"original bias std. error\n",
- " 100.065 100.055 0.149894\n"
+ " 99.9196 99.9096 0.149076\n"
]
}
],
@@ -544,10 +544,10 @@
"name": "stdout",
"output_type": "stream",
"text": [
- "Runtime: 2.40859 sec\n",
+ "Runtime: 1.78082 sec\n",
"Bootstrap Statistics :\n",
"original bias std. error\n",
- " 100.148 14.7733 100.151 0.147722\n"
+ " 99.9769 14.9969 99.9765 0.149832\n"
]
},
{
@@ -567,7 +567,7 @@
},
{
"data": {
- "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAD6CAYAAACs/ECRAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAATaUlEQVR4nO3df6zdd33f8eeLhNCSdjg/bi1ju3U0PCpUiSRcgVk7RvFAJGnraCsR0ba4mTvvR1bBOqm406YNwR/OtI0m2pTVwnRO1QZCCotLM0ZmQG21JuUGUpPEgThpMttz4tsQzErUtXTv/XE+Lifede65955zf/jzfEhH5/P9fD/f7/l8fI7P635/nlQVkqR+vWKlOyBJWlkGgSR1ziCQpM4ZBJLUOYNAkjpnEEhS5+YNgiSvT/Lw0ONbSd6f5NIk9yd5oj1f0tonye1JjiY5nOTqyQ9DkrRYWch1BEkuAE4AbwFuAb5RVXuT7AEuqaoPJLkW+Dng2tbutqp6y8ut9/LLL68tW7YscgiS1KeHHnroj6pqaqnruXCB7bcDT1bVM0l2AG9v9QeALwIfAHYAd9YgYR5Isi7Jhqo6ea6VbtmyhZmZmQV3XpJ6luSZcaxnoccI3gvc1crrh77cnwXWt/JG4NjQMsdbnSRpFRo5CJJcBPwU8Mmz57W//hd0r4oku5PMJJmZnZ1dyKKSpDFayBbBNcCXq+q5Nv1ckg0A7flUqz8BbB5ablOre4mq2ldV01U1PTW15F1ckqRFWkgQ3Mh3dwsBHAR2tvJO4N6h+pva2UPbgNMvd3xAkrSyRjpYnORi4J3APxiq3gvcnWQX8AxwQ6u/j8EZQ0eBF4Gbx9ZbSdLYjRQEVfVt4LKz6p5ncBbR2W2LwamlkqQ1wCuLJalzBoEkdc4gkKTOLfTKYmlN27Lnt+asf3rvdcvcE2n1cItAkjpnEEhS5wwCSeqcQSBJnTMIJKlzBoEkdc4gkKTOGQSS1DmDQJI655XFEl5xrL65RSBJnTMIJKlzBoEkdc4gkKTOGQSS1DnPGtJ56VxnAY1zPZ5RpPOFWwSS1DmDQJI6N1IQJFmX5J4kjyc5kuStSS5Ncn+SJ9rzJa1tktye5GiSw0munuwQJElLMeoWwW3AZ6vqh4E3AkeAPcChqtoKHGrTANcAW9tjN3DHWHssSRqreYMgyWuAtwH7AarqT6vqm8AO4EBrdgC4vpV3AHfWwAPAuiQbxtxvSdKYjLJFcAUwC/xKkq8k+WiSi4H1VXWytXkWWN/KG4FjQ8sfb3WSpFVolCC4ELgauKOqrgK+zXd3AwFQVQXUQl44ye4kM0lmZmdnF7KoJGmMRgmC48DxqnqwTd/DIBieO7PLpz2favNPAJuHlt/U6l6iqvZV1XRVTU9NTS22/5KkJZo3CKrqWeBYkte3qu3AY8BBYGer2wnc28oHgZva2UPbgNNDu5AkSavMqFcW/xzwa0kuAp4CbmYQIncn2QU8A9zQ2t4HXAscBV5sbSVJq9RIQVBVDwPTc8zaPkfbAm5ZWrckScvFK4slqXPedE5r2rhuLif1zC0CSeqcQSBJnTMIJKlzBoEkdc4gkKTOGQSS1DmDQJI6ZxBIUue8oEwas3Nd5Pb03uuWuSfSaNwikKTOGQSS1Dl3DUmL5H2OdL5wi0CSOmcQSFLnDAJJ6pzHCLQmuD9emhy3CCSpcwaBJHXOIJCkzhkEktS5kYIgydNJvprk4SQzre7SJPcneaI9X9Lqk+T2JEeTHE5y9SQHIElamoVsEfx4VV1ZVdNteg9wqKq2AofaNMA1wNb22A3cMa7OSpLGbym7hnYAB1r5AHD9UP2dNfAAsC7JhiW8jiRpgkYNggI+l+ShJLtb3fqqOtnKzwLrW3kjcGxo2eOt7iWS7E4yk2RmdnZ2EV2XJI3DqBeU/VhVnUjyA8D9SR4fnllVlaQW8sJVtQ/YBzA9Pb2gZaW1yN8p0Go10hZBVZ1oz6eATwNvBp47s8unPZ9qzU8Am4cW39TqJEmr0LxBkOTiJN9/pgy8C3gEOAjsbM12Ave28kHgpnb20Dbg9NAuJEnSKjPKrqH1wKeTnGn/61X12SRfAu5Osgt4Brihtb8PuBY4CrwI3Dz2XkuSxmbeIKiqp4A3zlH/PLB9jvoCbhlL7yRJE+fdR6UV5kFkrTSDQKuKt5uWlp/3GpKkzhkEktQ5g0CSOmcQSFLnDAJJ6pxBIEmdMwgkqXMGgSR1ziCQpM4ZBJLUOYNAkjpnEEhS5wwCSeqcQSBJnTMIJKlz/h6BtEr5gzVaLm4RSFLnDAJJ6pxBIEmdGzkIklyQ5CtJPtOmr0jyYJKjST6R5KJW/6o2fbTN3zKhvkuSxmAhWwTvA44MTd8KfKSqXge8AOxq9buAF1r9R1o7SdIqNVIQJNkEXAd8tE0HeAdwT2tyALi+lXe0adr87a29JGkVGnWL4JeAXwD+b5u+DPhmVX2nTR8HNrbyRuAYQJt/urWXJK1C8wZBkp8ATlXVQ+N84SS7k8wkmZmdnR3nqiVJCzDKFsGPAj+V5Gng4wx2Cd0GrEty5oK0TcCJVj4BbAZo818DPH/2SqtqX1VNV9X01NTUkgYhSVq8eYOgqn6xqjZV1RbgvcDnq+pvA18Afro12wnc28oH2zRt/uerqsbaa0nS2CzlOoIPAD+f5CiDYwD7W/1+4LJW//PAnqV1UZI0SQu611BVfRH4Yis/Bbx5jjZ/ArxnDH2TJC0DryyWpM5591GtiHPdWVPS8nOLQJI6ZxBIUucMAknqnEEgSZ0zCCSpcwaBJHXOIJCkzhkEktQ5g0CSOueVxdIac66rsp/ee90y90TnC7cIJKlzBoEkdc4gkKTOeYxAE+VdRqXVzy0CSeqcQSBJnTMIJKlzBoEkdc4gkKTOGQSS1Ll5gyDJ9yT5/SR/kOTRJB9s9VckeTDJ0SSfSHJRq39Vmz7a5m+Z8BgkSUswyhbB/wHeUVVvBK4E3p1kG3Ar8JGqeh3wArCrtd8FvNDqP9LaSZJWqXmDoAb+uE2+sj0KeAdwT6s/AFzfyjvaNG3+9iQZV4clSeM10jGCJBckeRg4BdwPPAl8s6q+05ocBza28kbgGECbfxq4bIx9liSN0UhBUFV/XlVXApuANwM/vNQXTrI7yUySmdnZ2aWuTpK0SAs6a6iqvgl8AXgrsC7JmXsVbQJOtPIJYDNAm/8a4Pk51rWvqqaranpqampxvZckLdkoZw1NJVnXyt8LvBM4wiAQfro12wnc28oH2zRt/uerqsbYZ0nSGI1y99ENwIEkFzAIjrur6jNJHgM+nuTDwFeA/a39fuBXkxwFvgG8dwL9liSNybxBUFWHgavmqH+KwfGCs+v/BHjPWHonSZo4f49AY+HvDkhrl7eYkKTOGQSS1DmDQJI6ZxBIUuc8WCydJ851wP7pvdctc0+01rhFIEmdMwgkqXMGgSR1ziCQpM4ZBJLUOYNAkjpnEEhS5wwCSeqcQSBJnTMIJKlzBoEkdc57DWlB/AGatcd7EGk+bhFIUucMAknqnEEgSZ0zCCSpc/MGQZLNSb6Q5LEkjyZ5X6u/NMn9SZ5oz5e0+iS5PcnRJIeTXD3pQUiSFm+ULYLvAP+sqt4AbANuSfIGYA9wqKq2AofaNMA1wNb22A3cMfZeS5LGZt4gqKqTVfXlVv7fwBFgI7ADONCaHQCub+UdwJ018ACwLsmGcXdckjQeCzpGkGQLcBXwILC+qk62Wc8C61t5I3BsaLHjre7sde1OMpNkZnZ2dqH9liSNychBkOT7gN8A3l9V3xqeV1UF1EJeuKr2VdV0VU1PTU0tZFFJ0hiNFARJXskgBH6tqj7Vqp87s8unPZ9q9SeAzUOLb2p1kqRVaJSzhgLsB45U1b8fmnUQ2NnKO4F7h+pvamcPbQNOD+1CkiStMqPca+hHgb8LfDXJw63unwN7gbuT7AKeAW5o8+4DrgWOAi8CN4+zw5Kk8Zo3CKrqd4GcY/b2OdoXcMsS+yVJWiZeWSxJnfM21JqTt5s+/3l7ap3hFoEkdc4gkKTOGQSS1DmDQJI6ZxBIUucMAknqnEEgSZ0zCCSpcwaBJHXOIJCkzhkEktQ5g0CSOmcQSFLnDAJJ6pxBIEmdMwgkqXP+ME3H/PEZzcUfrOmPWwSS1Dm3CDrgX/6SXs68QZDkY8BPAKeq6kda3aXAJ4AtwNPADVX1QpIAtwHXAi8CP1NVX55M1yUtp5f7g8LdRmvbKLuG/jPw7rPq9gCHqmorcKhNA1wDbG2P3cAd4+mmJGlS5g2Cqvpt4BtnVe8ADrTyAeD6ofo7a+ABYF2SDWPqqyRpAhZ7sHh9VZ1s5WeB9a28ETg21O54q5MkrVJLPmuoqgqohS6XZHeSmSQzs7OzS+2GJGmRFhsEz53Z5dOeT7X6E8DmoXabWt3/p6r2VdV0VU1PTU0tshuSpKVabBAcBHa28k7g3qH6mzKwDTg9tAtJkrQKjXL66F3A24HLkxwH/hWwF7g7yS7gGeCG1vw+BqeOHmVw+ujNE+izJGmM5g2CqrrxHLO2z9G2gFuW2ilJ0vLxFhOS1DmDQJI6572GziPeU0jSYrhFIEmdMwgkqXPuGpK0ZP6YzdrmFoEkdc4gkKTOGQSS1DmPEaxBniYqaZzcIpCkzhkEktQ5g0CSOmcQSFLnDAJJ6pxnDa1inh2ktW6hn2GvRF4ZBsEq4Be+pJXkriFJ6pxbBJJWDW9etzIMAkmrngExWe4akqTOGQSS1LmJ7BpK8m7gNuAC4KNVtXcSr7PWeHaQNF7uMhqPsQdBkguA/wi8EzgOfCnJwap6bNyvtVr5hS+tLANiYSaxRfBm4GhVPQWQ5OPADuC8CwK/8KXzQ+/BMYkg2AgcG5o+DrxlAq8DLPwN9Mtb6tdC//+PMyBWc9is2OmjSXYDu9vkHyf52ljXfysAlwN/NM71rqDzaSzgeFaz82ksMIHxtO+XlVrX8Hh+aBx9mEQQnAA2D01vanUvUVX7gH0TeP2/kGSmqqYn+RrL5XwaCzie1ex8Ggs4nlFM4vTRLwFbk1yR5CLgvcDBCbyOJGkMxr5FUFXfSfJPgP/G4PTRj1XVo+N+HUnSeEzkGEFV3QfcN4l1L9BEdz0ts/NpLOB4VrPzaSzgeOaVqhr3OiVJa4i3mJCkzq3JIEjyviSPJHk0yftb3RuT/F6Sryb5zSR/6RzLrktyT5LHkxxJ8tZl7fzcfVrKeP5pW+6RJHcl+Z5l7fygDx9LcirJI0N1lya5P8kT7fmSVp8ktyc5muRwkqvPsc43tbEfbe2zVseT5NVJfqt95h5Nsmy3XJnEezO0noPD610OE/qsXZRkX5Kvt/fob63x8dzY/u8cTvLZJJfP25GqWlMP4EeAR4BXMzjG8d+B1zE4W+mvtzZ/D/jQOZY/APxsK18ErFur42Fw8d4fAt/bpu8GfmYFxvA24GrgkaG6fwPsaeU9wK2tfC3wX4EA24AHz7HO32/z09pfs1bH097bHx/6zP3Oco1nEu9Na/s3gV8fXu9aHQ/wQeDDrfwK4PK1Op72HXLqzBjauv71vP1YzjdxTP9w7wH2D03/S+AXgNN895jHZuCxOZZ9TfvizEqPY0zjOXMV96XtA/AZ4F0rNI4tZ32YvwZsaOUNwNda+ZeBG+dqN1S3AXh8aPpG4JfX6njmWPdtwN9fq2MBvg/4XeANLHMQTGg8x4CLl3sckxgP8EpglsGFZgH+E7B7vj6sxV1DjwB/LcllSV7NICU3A48yuKcRDL5cN8+x7BUM/pF+JclXknw0ycXL0emXsejxVNUJ4N8C/xM4CZyuqs8tS6/nt76qTrbys8D6Vp7rFiQbz1p2Y6t/uTbLbSnj+QtJ1gE/CRyaQB9HtdSxfAj4d8CLE+vhwix6PO39APhQki8n+WSS9aysRY+nqv4M+EfAV4H/xSCs98/3gmsuCKrqCHAr8Dngs8DDwJ8z2H3yj5M8BHw/8KdzLH4hg82wO6rqKuDbDDa9VsxSxtP2He5gEHCvBS5O8neWp+ejq8GfKufN6WmLHU+SC4G7gNur3ZRxpS10LEmuBP5yVX16Yp1agkW8NxcyuPvB/6iqq4HfY/DH1aqwiPfnlQyC4CoG3wmHgV+cb7k1FwQAVbW/qt5UVW8DXgC+XlWPV9W7qupNDP6zPTnHoseB41X1YJu+h0EwrKgljOdvAH9YVbPtL4FPAX91+Xr+sp5LsgGgPZ9q9aPcguREq3+5NsttKeM5Yx/wRFX90qQ6OaKljOWtwHSSpxnsHvorSb440d7ObynjeZ7Bls2n2vQnWfnvhKWM50qAqnqyhcjdjPCdsCaDIMkPtOcfpB20Gqp7BfAvGOwbe4mqehY4luT1rWo7q+D22IsdD4NdQtvaWSlhMJ4jy9PreR0EdrbyTuDeofqb2hkQ2xjszjo5vGCb/laSbW1cNw0tv1IWPR6AJB9mcIzq/cvQ1/ks5b25o6peW1VbgB9j8EfL25en2+e0lPEU8JvA21vVavhOWMpn7QTwhiRTbfqdjPKdsFIHSJZ4cOV3GLxZfwBsb3XvA77eHnv57oHW1wL3DS17JTDDYJPpvwCXrPHxfBB4nMGxhl8FXrUC/b+LwTGKP2Ow1bULuIzBfvAnGJwJdWlrGwY/XPQkg/2Y00PreXioPN3G9CTwH1jGA/zjHg+Dv9yq/Yd8uD1+di2O5ax1b2H5zxqaxGfth4Dfbt8Jh4AfXOPj+Yfts3aYQchdNl8/vLJYkjq3JncNSZLGxyCQpM4ZBJLUOYNAkjpnEEhS5wwCSeqcQSBJnTMIJKlz/w96PpEQZR0rUwAAAABJRU5ErkJggg==\n",
+ "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX8AAAD4CAYAAAAEhuazAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/Il7ecAAAACXBIWXMAAAsTAAALEwEAmpwYAAATiUlEQVR4nO3df6zdd33f8eeLhEAJFOeHaxnbraPhUVWTSMwVhMEYTQYiSYujrUSwH3Ezd560dErGpNZMmzYEk5xpKxB18moROqelSdMUFjdkjMwEtWgk5QbSkF8QJ01mW0l8myZhJeoK7Xt/nI/JiXede+4951zfm8/zIV2d7/dzvt/ved9jn9f53M/5fL8nVYUkqS+vONkFSJKWn+EvSR0y/CWpQ4a/JHXI8JekDp16sgsAOPvss2vz5s0nuwxJWlXuvvvuP6mqtUvZd0WE/+bNm5mdnT3ZZUjSqpLk8aXu67CPJHXI8JekDhn+ktQhw1+SOmT4S1KHDH9J6pDhL0kdMvwlqUOGvyR1aEWc4Sv1bPOuL8zb/tjuS5a5EvXEnr8kdWjB8E/ypiT3DP18N8nVSc5McnuSh9vtGW37JLk2ycEk9ybZOv1fQ5K0GAuGf1V9u6rOrapzgbcAzwOfB3YBB6pqC3CgrQNcBGxpPzuBPVOoW5I0hsUO+1wIPFJVjwPbgH2tfR9waVveBlxfA3cCa5Ksn0SxkqTJWGz4fxC4oS2vq6on2vKTwLq2vAE4NLTP4db2Ikl2JplNMjs3N7fIMiRJ4xh5tk+S04D3Ax85/r6qqiS1mAeuqr3AXoCZmZlF7SutZM7e0WqwmJ7/RcA3quqptv7UseGcdnu0tR8BNg3tt7G1SZJWiMWE/4d4YcgHYD+wvS1vB24Zar+8zfo5H3huaHhIkrQCjDTsk+R04D3APx1q3g3clGQH8DhwWWu/DbgYOMhgZtAVE6tWkjQRI4V/VX0POOu4tqcZzP45ftsCrpxIdZKkqfAMX0nqkOEvSR0y/CWpQ17VU1qiE83nl1YDw19aJr5ZaCVx2EeSOmT4S1KHDH9J6pDhL0kdMvwlqUOGvyR1yPCXpA4Z/pLUIU/yklYovxFM02T4Sy/Bs3L1cuWwjyR1yPCXpA4Z/pLUIcf8pVXGD4I1Cfb8JalDI4V/kjVJbk7yUJIHk7w9yZlJbk/ycLs9o22bJNcmOZjk3iRbp/srSJIWa9Se/6eAL1bVTwJvBh4EdgEHqmoLcKCtA1wEbGk/O4E9E61YkjS2BcM/yeuBdwHXAVTVX1TVs8A2YF/bbB9waVveBlxfA3cCa5Ksn3DdkqQxjNLzPweYA349yTeTfDrJ6cC6qnqibfMksK4tbwAODe1/uLW9SJKdSWaTzM7NzS39N5AkLdoo4X8qsBXYU1XnAd/jhSEeAKqqgFrMA1fV3qqaqaqZtWvXLmZXSdKYRgn/w8Dhqrqrrd/M4M3gqWPDOe32aLv/CLBpaP+NrU2StEIsGP5V9SRwKMmbWtOFwAPAfmB7a9sO3NKW9wOXt1k/5wPPDQ0PSZJWgFFP8vrnwGeTnAY8ClzB4I3jpiQ7gMeBy9q2twEXAweB59u2kqQVZKTwr6p7gJl57rpwnm0LuHK8siRJ0+QZvpLUIcNfkjrkhd0k/NIW9ceevyR1yPCXpA4Z/pLUIcNfkjpk+EtSh5ztI71M+PWOWgx7/pLUIcNfkjpk+EtShwx/SeqQ4S9JHTL8JalDhr8kdch5/uqKV++UBuz5S1KHDH9J6pDhL0kdGin8kzyW5FtJ7kky29rOTHJ7kofb7RmtPUmuTXIwyb1Jtk7zF5AkLd5iev4/XVXnVtVMW98FHKiqLcCBtg5wEbCl/ewE9kyqWEnSZIwz7LMN2NeW9wGXDrVfXwN3AmuSrB/jcSRJEzZq+BfwpSR3J9nZ2tZV1RNt+UlgXVveABwa2vdwa3uRJDuTzCaZnZubW0LpkqSlGnWe/zur6kiSHwNuT/LQ8J1VVUlqMQ9cVXuBvQAzMzOL2leSNJ6Rev5VdaTdHgU+D7wVeOrYcE67Pdo2PwJsGtp9Y2uTJK0QC4Z/ktOTvO7YMvBe4D5gP7C9bbYduKUt7wcub7N+zgeeGxoekiStAKMM+6wDPp/k2Pa/VVVfTPJ14KYkO4DHgcva9rcBFwMHgeeBKyZetSRpLAuGf1U9Crx5nvangQvnaS/gyolUJ0maCs/wlaQOGf6S1CHDX5I6ZPhLUocMf0nqkOEvSR0y/CWpQ4a/JHXI8JekDhn+ktQhw1+SOmT4S1KHRv0yF0mr1OZdX5i3/bHdlyxzJVpJ7PlLUocMf0nqkOEvSR0y/CWpQ4a/JHXI8JekDhn+ktShkcM/ySlJvpnk1rZ+TpK7khxM8ttJTmvtr2rrB9v9m6dUuyRpiRbT878KeHBo/RrgE1X1RuAZYEdr3wE809o/0baTJK0gI53hm2QjcAnw74EPJwlwAfD32yb7gH8H7AG2tWWAm4FfTZKqqsmVLb20E53VKmlg1J7/J4FfAv6qrZ8FPFtVP2jrh4ENbXkDcAig3f9c2/5FkuxMMptkdm5ubmnVS5KWZMHwT/IzwNGqunuSD1xVe6tqpqpm1q5dO8lDS5IWMMqwzzuA9ye5GHg18KPAp4A1SU5tvfuNwJG2/RFgE3A4yanA64GnJ165JGnJFuz5V9VHqmpjVW0GPgh8uar+AXAH8HNts+3ALW15f1un3f9lx/slaWUZZ57/LzP48PcggzH961r7dcBZrf3DwK7xSpQkTdqirudfVV8BvtKWHwXeOs82fw58YAK1SZKmxDN8JalDhr8kdcjwl6QO+R2+Uqf8bt++2fOXpA4Z/pLUIcNfkjpk+EtShwx/SeqQ4S9JHTL8JalDhr8kdcjwl6QOGf6S1CEv76BVzS9ql5bGnr8kdcjwl6QOOeyjVcHhHWmy7PlLUocWDP8kr07yh0n+KMn9ST7a2s9JcleSg0l+O8lprf1Vbf1gu3/zlH8HSdIijdLz/7/ABVX1ZuBc4H1JzgeuAT5RVW8EngF2tO13AM+09k+07SRJK8iC4V8Df9ZWX9l+CrgAuLm17wMubcvb2jrt/guTZFIFS5LGN9KYf5JTktwDHAVuBx4Bnq2qH7RNDgMb2vIG4BBAu/854KwJ1ixJGtNI4V9Vf1lV5wIbgbcCPznuAyfZmWQ2yezc3Ny4h5MkLcKiZvtU1bPAHcDbgTVJjk0V3QgcactHgE0A7f7XA0/Pc6y9VTVTVTNr165dWvWSpCVZcJ5/krXA96vq2SQ/AryHwYe4dwA/B9wIbAduabvsb+tfa/d/uapqCrVLmoITnVPx2O5LlrkSTdMoJ3mtB/YlOYXBXwo3VdWtSR4AbkzyceCbwHVt++uA30hyEPhT4INTqFuSNIYFw7+q7gXOm6f9UQbj/8e3/znwgYlUJ0maCs/wlaQOGf6S1CHDX5I6ZPhLUocMf0nqkOEvSR0y/CWpQ4a/JHXI8JekDhn+ktQhw1+SOmT4S1KHDH9J6tAol3SWpBNe5x+81v9qZM9fkjpk+EtShwx/SeqQ4S9JHTL8JalDhr8kdWjB8E+yKckdSR5Icn+Sq1r7mUluT/Jwuz2jtSfJtUkOJrk3ydZp/xKSpMUZZZ7/D4B/WVXfSPI64O4ktwM/Dxyoqt1JdgG7gF8GLgK2tJ+3AXvarfRDLzVnXNL0Ldjzr6onquobbfn/AA8CG4BtwL622T7g0ra8Dbi+Bu4E1iRZP+nCJUlLt6gx/ySbgfOAu4B1VfVEu+tJYF1b3gAcGtrtcGs7/lg7k8wmmZ2bm1ts3ZKkMYwc/kleC/wucHVVfXf4vqoqoBbzwFW1t6pmqmpm7dq1i9lVkjSmkcI/ySsZBP9nq+pzrfmpY8M57fZoaz8CbBrafWNrkyStEKPM9glwHfBgVf3K0F37ge1teTtwy1D75W3Wz/nAc0PDQ5KkFWCU2T7vAP4R8K0k97S2fwXsBm5KsgN4HLis3XcbcDFwEHgeuGKSBUuSxrdg+FfVV4Gc4O4L59m+gCvHrEuSNEWe4StJHTL8JalDhr8kdcjwl6QOGf6S1CHDX5I6ZPhLUocMf0nqkOEvSR0a5fIOkvSSTvTlPI/tvmSZK9Go7PlLUocMf0nqkOEvSR0y/CWpQ4a/JHXI8JekDjnVU9LUOAV05bLnL0kdMvwlqUMO+2iqTvRnv6STa8Gef5LPJDma5L6htjOT3J7k4XZ7RmtPkmuTHExyb5Kt0yxekrQ0owz7/Ffgfce17QIOVNUW4EBbB7gI2NJ+dgJ7JlOmJGmSFgz/qvp94E+Pa94G7GvL+4BLh9qvr4E7gTVJ1k+oVknShCz1A991VfVEW34SWNeWNwCHhrY73Nr+P0l2JplNMjs3N7fEMiRJSzH2bJ+qKqCWsN/eqpqpqpm1a9eOW4YkaRGWGv5PHRvOabdHW/sRYNPQdhtbmyRpBVnqVM/9wHZgd7u9Zaj9F5PcCLwNeG5oeEgvY07plFaXBcM/yQ3Au4GzkxwG/i2D0L8pyQ7gceCytvltwMXAQeB54Iop1CxJGtOC4V9VHzrBXRfOs20BV45blCRpujzDV9Ky84JvJ5/X9pGkDhn+ktQhh320KM7q0TQ5HLR87PlLUocMf0nqkOEvSR0y/CWpQ4a/JHXI8JekDhn+ktQh5/lrXs7nl17e7PlLUofs+XfM3r3UL8Nf0oq32I6Kl4NYmMM+ktQhw1+SOmT4S1KHHPPvgB/sSjqePX9J6tBUev5J3gd8CjgF+HRV7Z7G4+jF7OFLA84OWtjEwz/JKcB/Bt4DHAa+nmR/VT0w6ceSpEno8c1iGj3/twIHq+pRgCQ3AtuA7sN/sV9RZ09eWl2W8po9WW8k0wj/DcChofXDwNuO3yjJTmBnW/2zJN8+wfHOBv5kohUuj5HrzjVTrmRxXvbP9wpj3ctrInVP8jU74rFOVPdPLPVxT9psn6raC+xdaLsks1U1swwlTZR1Ly/rXl7WvbymUfc0ZvscATYNrW9sbZKkFWIa4f91YEuSc5KcBnwQ2D+Fx5EkLdHEh32q6gdJfhH4Hwymen6mqu4f45ALDg2tUNa9vKx7eVn38pp43amqSR9TkrTCeYavJHXI8JekDp3U8E9yVZL7ktyf5OrW9uYkX0vyrSS/l+RH59lvU5I7kjzQ9r1qNdQ9tP8pSb6Z5NZlK5rx6k6yJsnNSR5K8mCSt6+Suv9F2+++JDckefUU6/xMkqNJ7htqOzPJ7UkebrdntPYkuTbJwST3Jtl6gmO+pf2OB9v2Wel1J3lNki+0/yv3J5nK5V2m8XwPHWf/8HFXet1JTkuyN8l32vP+9xYspKpOyg/wN4D7gNcw+OD5fwJvZDBb6G+3bf4x8LF59l0PbG3LrwO+A/zUSq976BgfBn4LuHU1PN/tvn3AL7Tl04A1K71uBicc/jHwI239JuDnp1jru4CtwH1Dbf8B2NWWdwHXtOWLgf8OBDgfuOsEx/zDdn/a9het9Lrbv9VPD/1f+YPVUPfQMf5ue33eN+map/j/5KPAx9vyK4CzF6xjGr/ciE/AB4Drhtb/DfBLwHO88EH0JuCBEY51C/Ce1VA3g/MeDgAXsLzhv+S6gdczCNGspv8nvHC2+ZkM3jhuBd475Xo3H/ei/jawvi2vB77dln8N+NB82w21rQceGlr/EPBrK73ueY79KeCfrIa6gdcCXwV+iimF/5TqPgScvpgaTuawz33A30pyVpLXMHiH2wTcz+BaQDB44W86wf4AJNkMnAfcNb1SX2Tcuj/JILz+asp1Hm+cus8B5oBfb8NVn05y+nIUzRh1V9UR4D8C/xt4Aniuqr60LFW/YF1VPdGWnwTWteX5LoOy4bh9N7T2l9pmWsap+4eSrAF+lkGHZzmMW/fHgP8EPD+1Cue35LrbcwzwsSTfSPI7SdaxgJMW/lX1IHAN8CXgi8A9wF8y+BP+nyW5m8GQzl+c6BhJXgv8LnB1VX132jXDeHUn+RngaFXdvRy1Dhvz+T6VwZ+pe6rqPOB7DP40nboxn+8zGLxBnAO8ATg9yT9cjrrnU4Mu2qqbW73UupOcCtwAXFvtQo/LabF1JzkX+GtV9fmpFTWCJTzfpzIYUfhfVbUV+BqDTs9LOqkf+FbVdVX1lqp6F/AM8J2qeqiq3ltVb2HwH+eR+fZN8koGwf/Zqvrc8lU9Vt3vAN6f5DHgRuCCJL+5Cuo+DByuqmN/Xd3M4M1gWYxR998B/riq5qrq+8DngL+5XHU3TyVZD9Buj7b2US6DcqS1v9Q20zJO3cfsBR6uqk9Oq8h5jFP324GZ9vr8KvDXk3xlqtW+YJy6n2bwl8qxHPwdRnh9nuzZPj/Wbn+c9iHLUNsrgH8N/Jd59gtwHfBgVf3K8lX8w8dfUt1V9ZGq2lhVmxlc9uLLVbVsPdEx6n4SOJTkTa3pQpbxEt1LrZvBcM/5bfZJGNT94PJU/UP7ge1teTuDz6eOtV/eZnOcz2BI6onhHdv6d5Oc3+q/fGj/FVs3QJKPM/is6OplqHXYOM/3nqp6Q3t9vpNBJ+Pdy1P2WHUX8HvAsVpHe31O6wONET/0+INW5B8BF7a2qxjM3vkOsJsXPtR7A3BbW34ngz+L7mUwDHAPcPFKr/u4Y7ybZfzAd9y6gXOB2fac/zfgjFVS90eBhxh8dvAbwKumWOcNDD5b+D6Dv5Z2AGcxGO9+mMFMpTPbtmHwpUePAN8CZoaOc8/Q8kyr/RHgV4/9niu5bga902LwRnvs9fkLK73u4469menN9pnG/5OfAH6/vT4PAD++UB1e3kGSOuQZvpLUIcNfkjpk+EtShwx/SeqQ4S9JHTL8JalDhr8kdej/AXo4dAdKce1YAAAAAElFTkSuQmCC\n",
"text/plain": [
"
"
]
diff --git a/doc/LectureNotes/_build/jupyter_execute/chapter3_25_2.png b/doc/LectureNotes/_build/jupyter_execute/chapter3_25_2.png
index 98e9e3a97..b77a1febb 100644
Binary files a/doc/LectureNotes/_build/jupyter_execute/chapter3_25_2.png and b/doc/LectureNotes/_build/jupyter_execute/chapter3_25_2.png differ
diff --git a/doc/LectureNotes/_build/jupyter_execute/chapter4.ipynb b/doc/LectureNotes/_build/jupyter_execute/chapter4.ipynb
index 06422f723..20eae65ba 100644
--- a/doc/LectureNotes/_build/jupyter_execute/chapter4.ipynb
+++ b/doc/LectureNotes/_build/jupyter_execute/chapter4.ipynb
@@ -149,15 +149,148 @@
},
"outputs": [
{
- "ename": "FileNotFoundError",
- "evalue": "[Errno 2] No such file or directory: 'DataFiles/chddata.csv'",
- "output_type": "error",
- "traceback": [
- "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
- "\u001b[0;31mFileNotFoundError\u001b[0m Traceback (most recent call last)",
- "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 38\u001b[0m \u001b[0mplt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msavefig\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mimage_path\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfig_id\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m\".png\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mformat\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'png'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 39\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 40\u001b[0;31m \u001b[0minfile\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata_path\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"chddata.csv\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'r'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 41\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 42\u001b[0m \u001b[0;31m# Read the chd data as csv file and organize the data into arrays with age group, age, and chd\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
- "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: 'DataFiles/chddata.csv'"
- ]
+ "data": {
+ "text/html": [
+ "