-In project 1 when using Ridge and Lasso regression, we end up +In project 1, when using Ridge and Lasso regression, we end up searching for the optimal parameter \( \lambda \) which minimizes our selected scores (MSE or \( R2 \) values for example). The brute force -approach, as discussed in the code here for Ridge regression consists +approach, as discussed in the code here for Ridge regression, consists in evaluating the MSE as function of different \( \lambda \) values. +Based on these calculations, one tries then to determine the value of the hyperparameter \( \lambda \) +which results in optimal scores (for example the smallest MSE or an \( R2=1 \)). +
+ + +
import numpy as np
+import matplotlib.pyplot as plt
+from sklearn.model_selection import KFold
+from sklearn.linear_model import Ridge
+from sklearn.model_selection import cross_val_score
+from sklearn.preprocessing import PolynomialFeatures
+
+# A seed just to ensure that the random numbers are the same for every run.
+np.random.seed(3155)
+# Generate the data.
+n = 100
+x = np.linspace(-3, 3, n).reshape(-1, 1)
+y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)
+# Decide degree on polynomial to fit
+poly = PolynomialFeatures(degree = 10)
+
+# Decide which values of lambda to use
+nlambdas = 500
+lambdas = np.logspace(-3, 5, nlambdas)
+# Initialize a KFold instance
+k = 5
+kfold = KFold(n_splits = k)
+estimated_mse_sklearn = np.zeros(nlambdas)
+i = 0
+for lmb in lambdas:
+ ridge = Ridge(alpha = lmb)
+ estimated_mse_folds = cross_val_score(ridge, x, y, scoring='neg_mean_squared_error', cv=kfold)
+ estimated_mse_sklearn[i] = np.mean(-estimated_mse_folds)
+ i += 1
+plt.figure()
+plt.plot(np.log10(lambdas), estimated_mse_sklearn, label = 'cross_val_score')
+plt.xlabel('log10(lambda)')
+plt.ylabel('MSE')
+plt.legend()
+plt.show()
+
+We see from this plot that the optimal MSE occurs for a value of \( \lambda\in [10,100] \).
+In order to nail down the best value of \( \lambda \), we could in turn narrow down the search area.
+
+
+
+
-An alternative is to use the so-called grid search functionality included with the library Scikit-Learn, as demonstrated for the same example here.
+An alternative is to use the so-called grid search functionality
+included with the library Scikit-Learn, as demonstrated for the same
+example here.
+
+
+
+
+Grid Search
import numpy as np
+import matplotlib.pyplot as plt
+from sklearn.model_selection import KFold
+from sklearn.linear_model import Ridge
+from sklearn.model_selection import cross_val_score
+from sklearn.preprocessing import PolynomialFeatures
+
+# A seed just to ensure that the random numbers are the same for every run.
+np.random.seed(3155)
+# Generate the data.
+n = 100
+x = np.linspace(-3, 3, n).reshape(-1, 1)
+y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)
+# Decide degree on polynomial to fit
+poly = PolynomialFeatures(degree = 10)
+
+# Decide which values of lambda to use
+nlambdas = 500
+lambdas = np.logspace(-3, 5, nlambdas)
+# Initialize a KFold instance
+k = 5
+kfold = KFold(n_splits = k)
+estimated_mse_sklearn = np.zeros(nlambdas)
+i = 0
+for lmb in lambdas:
+ ridge = Ridge(alpha = lmb)
+ estimated_mse_folds = cross_val_score(ridge, x, y, scoring='neg_mean_squared_error', cv=kfold)
+ estimated_mse_sklearn[i] = np.mean(-estimated_mse_folds)
+ i += 1
+plt.figure()
+plt.plot(np.log10(lambdas), estimated_mse_sklearn, label = 'cross_val_score')
+plt.xlabel('log10(lambda)')
+plt.ylabel('MSE')
+plt.legend()
+plt.show()
+
-In project 1 when using Ridge and Lasso regression, we end up +In project 1, when using Ridge and Lasso regression, we end up searching for the optimal parameter \( \lambda \) which minimizes our selected scores (MSE or \( R2 \) values for example). The brute force -approach, as discussed in the code here for Ridge regression consists +approach, as discussed in the code here for Ridge regression, consists in evaluating the MSE as function of different \( \lambda \) values. +Based on these calculations, one tries then to determine the value of the hyperparameter \( \lambda \) +which results in optimal scores (for example the smallest MSE or an \( R2=1 \)). +
+ + +
import numpy as np
+import matplotlib.pyplot as plt
+from sklearn.model_selection import KFold
+from sklearn.linear_model import Ridge
+from sklearn.model_selection import cross_val_score
+from sklearn.preprocessing import PolynomialFeatures
+
+# A seed just to ensure that the random numbers are the same for every run.
+np.random.seed(3155)
+# Generate the data.
+n = 100
+x = np.linspace(-3, 3, n).reshape(-1, 1)
+y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)
+# Decide degree on polynomial to fit
+poly = PolynomialFeatures(degree = 10)
+
+# Decide which values of lambda to use
+nlambdas = 500
+lambdas = np.logspace(-3, 5, nlambdas)
+# Initialize a KFold instance
+k = 5
+kfold = KFold(n_splits = k)
+estimated_mse_sklearn = np.zeros(nlambdas)
+i = 0
+for lmb in lambdas:
+ ridge = Ridge(alpha = lmb)
+ estimated_mse_folds = cross_val_score(ridge, x, y, scoring='neg_mean_squared_error', cv=kfold)
+ estimated_mse_sklearn[i] = np.mean(-estimated_mse_folds)
+ i += 1
+plt.figure()
+plt.plot(np.log10(lambdas), estimated_mse_sklearn, label = 'cross_val_score')
+plt.xlabel('log10(lambda)')
+plt.ylabel('MSE')
+plt.legend()
+plt.show()
++We see from this plot that the optimal MSE occurs for a value of \( \lambda\in [10,100] \). +In order to nail down the best value of \( \lambda \), we could in turn narrow down the search area.
-An alternative is to use the so-called grid search functionality included with the library Scikit-Learn, as demonstrated for the same example here.
+
+
+An alternative is to use the so-called grid search functionality +included with the library Scikit-Learn, as demonstrated for the same +example here. + +
+ + +
import numpy as np
+import matplotlib.pyplot as plt
+from sklearn.model_selection import KFold
+from sklearn.linear_model import Ridge
+from sklearn.model_selection import cross_val_score
+from sklearn.preprocessing import PolynomialFeatures
+
+# A seed just to ensure that the random numbers are the same for every run.
+np.random.seed(3155)
+# Generate the data.
+n = 100
+x = np.linspace(-3, 3, n).reshape(-1, 1)
+y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)
+# Decide degree on polynomial to fit
+poly = PolynomialFeatures(degree = 10)
+
+# Decide which values of lambda to use
+nlambdas = 500
+lambdas = np.logspace(-3, 5, nlambdas)
+# Initialize a KFold instance
+k = 5
+kfold = KFold(n_splits = k)
+estimated_mse_sklearn = np.zeros(nlambdas)
+i = 0
+for lmb in lambdas:
+ ridge = Ridge(alpha = lmb)
+ estimated_mse_folds = cross_val_score(ridge, x, y, scoring='neg_mean_squared_error', cv=kfold)
+ estimated_mse_sklearn[i] = np.mean(-estimated_mse_folds)
+ i += 1
+plt.figure()
+plt.plot(np.log10(lambdas), estimated_mse_sklearn, label = 'cross_val_score')
+plt.xlabel('log10(lambda)')
+plt.ylabel('MSE')
+plt.legend()
+plt.show()
+
diff --git a/doc/pub/week39/html/week39.html b/doc/pub/week39/html/week39.html
index 8e2873538..0b5b084ca 100644
--- a/doc/pub/week39/html/week39.html
+++ b/doc/pub/week39/html/week39.html
@@ -72,6 +72,7 @@ div { text-align: justify; text-justify: inter-word; }
2,
None,
'searching-for-optimal-regularization-parameters-lambda'),
+ ('Grid Search', 2, None, 'grid-search'),
('Optimization, the central part of any Machine Learning '
'algortithm',
2,
@@ -272,15 +273,105 @@ For a good discussion on gradient methods, see Goodfellow et al section 4.3-4.5
-In project 1 when using Ridge and Lasso regression, we end up +In project 1, when using Ridge and Lasso regression, we end up searching for the optimal parameter \( \lambda \) which minimizes our selected scores (MSE or \( R2 \) values for example). The brute force -approach, as discussed in the code here for Ridge regression consists +approach, as discussed in the code here for Ridge regression, consists in evaluating the MSE as function of different \( \lambda \) values. +Based on these calculations, one tries then to determine the value of the hyperparameter \( \lambda \) +which results in optimal scores (for example the smallest MSE or an \( R2=1 \)). +
+ + +
import numpy as np
+import matplotlib.pyplot as plt
+from sklearn.model_selection import KFold
+from sklearn.linear_model import Ridge
+from sklearn.model_selection import cross_val_score
+from sklearn.preprocessing import PolynomialFeatures
+
+# A seed just to ensure that the random numbers are the same for every run.
+np.random.seed(3155)
+# Generate the data.
+n = 100
+x = np.linspace(-3, 3, n).reshape(-1, 1)
+y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)
+# Decide degree on polynomial to fit
+poly = PolynomialFeatures(degree = 10)
+
+# Decide which values of lambda to use
+nlambdas = 500
+lambdas = np.logspace(-3, 5, nlambdas)
+# Initialize a KFold instance
+k = 5
+kfold = KFold(n_splits = k)
+estimated_mse_sklearn = np.zeros(nlambdas)
+i = 0
+for lmb in lambdas:
+ ridge = Ridge(alpha = lmb)
+ estimated_mse_folds = cross_val_score(ridge, x, y, scoring='neg_mean_squared_error', cv=kfold)
+ estimated_mse_sklearn[i] = np.mean(-estimated_mse_folds)
+ i += 1
+plt.figure()
+plt.plot(np.log10(lambdas), estimated_mse_sklearn, label = 'cross_val_score')
+plt.xlabel('log10(lambda)')
+plt.ylabel('MSE')
+plt.legend()
+plt.show()
++We see from this plot that the optimal MSE occurs for a value of \( \lambda\in [10,100] \). +In order to nail down the best value of \( \lambda \), we could in turn narrow down the search area.
-An alternative is to use the so-called grid search functionality included with the library Scikit-Learn, as demonstrated for the same example here.
+
+
+An alternative is to use the so-called grid search functionality +included with the library Scikit-Learn, as demonstrated for the same +example here. + +
+ + +
import numpy as np
+import matplotlib.pyplot as plt
+from sklearn.model_selection import KFold
+from sklearn.linear_model import Ridge
+from sklearn.model_selection import cross_val_score
+from sklearn.preprocessing import PolynomialFeatures
+
+# A seed just to ensure that the random numbers are the same for every run.
+np.random.seed(3155)
+# Generate the data.
+n = 100
+x = np.linspace(-3, 3, n).reshape(-1, 1)
+y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)
+# Decide degree on polynomial to fit
+poly = PolynomialFeatures(degree = 10)
+
+# Decide which values of lambda to use
+nlambdas = 500
+lambdas = np.logspace(-3, 5, nlambdas)
+# Initialize a KFold instance
+k = 5
+kfold = KFold(n_splits = k)
+estimated_mse_sklearn = np.zeros(nlambdas)
+i = 0
+for lmb in lambdas:
+ ridge = Ridge(alpha = lmb)
+ estimated_mse_folds = cross_val_score(ridge, x, y, scoring='neg_mean_squared_error', cv=kfold)
+ estimated_mse_sklearn[i] = np.mean(-estimated_mse_folds)
+ i += 1
+plt.figure()
+plt.plot(np.log10(lambdas), estimated_mse_sklearn, label = 'cross_val_score')
+plt.xlabel('log10(lambda)')
+plt.ylabel('MSE')
+plt.legend()
+plt.show()
+
diff --git a/doc/pub/week39/ipynb/ipynb-week39-src.tar.gz b/doc/pub/week39/ipynb/ipynb-week39-src.tar.gz
index f85447f81..9b8edb6ef 100644
Binary files a/doc/pub/week39/ipynb/ipynb-week39-src.tar.gz and b/doc/pub/week39/ipynb/ipynb-week39-src.tar.gz differ
diff --git a/doc/pub/week39/ipynb/week39.ipynb b/doc/pub/week39/ipynb/week39.ipynb
index 01746bceb..55531bbb4 100644
--- a/doc/pub/week39/ipynb/week39.ipynb
+++ b/doc/pub/week39/ipynb/week39.ipynb
@@ -34,19 +34,128 @@
"\n",
"## Searching for Optimal Regularization Parameters $\\lambda$\n",
"\n",
- "In project 1 when using Ridge and Lasso regression, we end up\n",
+ "In project 1, when using Ridge and Lasso regression, we end up\n",
"searching for the optimal parameter $\\lambda$ which minimizes our\n",
"selected scores (MSE or $R2$ values for example). The brute force\n",
- "approach, as discussed in the code here for Ridge regression consists\n",
+ "approach, as discussed in the code here for Ridge regression, consists\n",
"in evaluating the MSE as function of different $\\lambda$ values.\n",
+ "Based on these calculations, one tries then to determine the value of the hyperparameter $\\lambda$\n",
+ "which results in optimal scores (for example the smallest MSE or an $R2=1$)."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": false,
+ "editable": true
+ },
+ "outputs": [],
+ "source": [
+ "%matplotlib inline\n",
"\n",
- "An alternative is to use the so-called grid search functionality included with the library **Scikit-Learn**, as demonstrated for the same example here.\n",
- "\n",
- "\n",
+ "import numpy as np\n",
+ "import matplotlib.pyplot as plt\n",
+ "from sklearn.model_selection import KFold\n",
+ "from sklearn.linear_model import Ridge\n",
+ "from sklearn.model_selection import cross_val_score\n",
+ "from sklearn.preprocessing import PolynomialFeatures\n",
+ "\n",
+ "# A seed just to ensure that the random numbers are the same for every run.\n",
+ "np.random.seed(3155)\n",
+ "# Generate the data.\n",
+ "n = 100\n",
+ "x = np.linspace(-3, 3, n).reshape(-1, 1)\n",
+ "y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)\n",
+ "# Decide degree on polynomial to fit\n",
+ "poly = PolynomialFeatures(degree = 10)\n",
+ "\n",
+ "# Decide which values of lambda to use\n",
+ "nlambdas = 500\n",
+ "lambdas = np.logspace(-3, 5, nlambdas)\n",
+ "# Initialize a KFold instance\n",
+ "k = 5\n",
+ "kfold = KFold(n_splits = k)\n",
+ "estimated_mse_sklearn = np.zeros(nlambdas)\n",
+ "i = 0\n",
+ "for lmb in lambdas:\n",
+ " ridge = Ridge(alpha = lmb)\n",
+ " estimated_mse_folds = cross_val_score(ridge, x, y, scoring='neg_mean_squared_error', cv=kfold)\n",
+ " estimated_mse_sklearn[i] = np.mean(-estimated_mse_folds)\n",
+ " i += 1\n",
+ "plt.figure()\n",
+ "plt.plot(np.log10(lambdas), estimated_mse_sklearn, label = 'cross_val_score')\n",
+ "plt.xlabel('log10(lambda)')\n",
+ "plt.ylabel('MSE')\n",
+ "plt.legend()\n",
+ "plt.show()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "We see from this plot that the optimal MSE occurs for a value of $\\lambda\\in [10,100]$.\n",
+ "In order to nail down the best value of $\\lambda$, we could in turn narrow down the search area.\n",
+ "\n",
+ "## Grid Search\n",
"\n",
"\n",
+ "An alternative is to use the so-called grid search functionality\n",
+ "included with the library **Scikit-Learn**, as demonstrated for the same\n",
+ "example here."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": false,
+ "editable": true
+ },
+ "outputs": [],
+ "source": [
+ "import numpy as np\n",
+ "import matplotlib.pyplot as plt\n",
+ "from sklearn.model_selection import KFold\n",
+ "from sklearn.linear_model import Ridge\n",
+ "from sklearn.model_selection import cross_val_score\n",
+ "from sklearn.preprocessing import PolynomialFeatures\n",
"\n",
+ "# A seed just to ensure that the random numbers are the same for every run.\n",
+ "np.random.seed(3155)\n",
+ "# Generate the data.\n",
+ "n = 100\n",
+ "x = np.linspace(-3, 3, n).reshape(-1, 1)\n",
+ "y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)\n",
+ "# Decide degree on polynomial to fit\n",
+ "poly = PolynomialFeatures(degree = 10)\n",
"\n",
+ "# Decide which values of lambda to use\n",
+ "nlambdas = 500\n",
+ "lambdas = np.logspace(-3, 5, nlambdas)\n",
+ "# Initialize a KFold instance\n",
+ "k = 5\n",
+ "kfold = KFold(n_splits = k)\n",
+ "estimated_mse_sklearn = np.zeros(nlambdas)\n",
+ "i = 0\n",
+ "for lmb in lambdas:\n",
+ " ridge = Ridge(alpha = lmb)\n",
+ " estimated_mse_folds = cross_val_score(ridge, x, y, scoring='neg_mean_squared_error', cv=kfold)\n",
+ " estimated_mse_sklearn[i] = np.mean(-estimated_mse_folds)\n",
+ " i += 1\n",
+ "plt.figure()\n",
+ "plt.plot(np.log10(lambdas), estimated_mse_sklearn, label = 'cross_val_score')\n",
+ "plt.xlabel('log10(lambda)')\n",
+ "plt.ylabel('MSE')\n",
+ "plt.legend()\n",
+ "plt.show()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
"## Optimization, the central part of any Machine Learning algortithm\n",
"\n",
"Almost every problem in machine learning and data science starts with\n",
@@ -810,8 +919,6 @@
},
"outputs": [],
"source": [
- "%matplotlib inline\n",
- "\n",
"import numpy as np\n",
"import numpy.linalg as la\n",
"\n",
diff --git a/doc/src/week39/week39.do.txt b/doc/src/week39/week39.do.txt
index 7a6448ada..8fc273586 100644
--- a/doc/src/week39/week39.do.txt
+++ b/doc/src/week39/week39.do.txt
@@ -23,13 +23,99 @@ For a good discussion on gradient methods, see Goodfellow et al section 4.3-4.5
!split
===== Searching for Optimal Regularization Parameters $\lambda$ =====
-In project 1 when using Ridge and Lasso regression, we end up
+In project 1, when using Ridge and Lasso regression, we end up
searching for the optimal parameter $\lambda$ which minimizes our
selected scores (MSE or $R2$ values for example). The brute force
-approach, as discussed in the code here for Ridge regression consists
+approach, as discussed in the code here for Ridge regression, consists
in evaluating the MSE as function of different $\lambda$ values.
+Based on these calculations, one tries then to determine the value of the hyperparameter $\lambda$
+which results in optimal scores (for example the smallest MSE or an $R2=1$).
+!bc pycod
+import numpy as np
+import matplotlib.pyplot as plt
+from sklearn.model_selection import KFold
+from sklearn.linear_model import Ridge
+from sklearn.model_selection import cross_val_score
+from sklearn.preprocessing import PolynomialFeatures
-An alternative is to use the so-called grid search functionality included with the library _Scikit-Learn_, as demonstrated for the same example here.
+# A seed just to ensure that the random numbers are the same for every run.
+np.random.seed(3155)
+# Generate the data.
+n = 100
+x = np.linspace(-3, 3, n).reshape(-1, 1)
+y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)
+# Decide degree on polynomial to fit
+poly = PolynomialFeatures(degree = 10)
+
+# Decide which values of lambda to use
+nlambdas = 500
+lambdas = np.logspace(-3, 5, nlambdas)
+# Initialize a KFold instance
+k = 5
+kfold = KFold(n_splits = k)
+estimated_mse_sklearn = np.zeros(nlambdas)
+i = 0
+for lmb in lambdas:
+ ridge = Ridge(alpha = lmb)
+ estimated_mse_folds = cross_val_score(ridge, x, y, scoring='neg_mean_squared_error', cv=kfold)
+ estimated_mse_sklearn[i] = np.mean(-estimated_mse_folds)
+ i += 1
+plt.figure()
+plt.plot(np.log10(lambdas), estimated_mse_sklearn, label = 'cross_val_score')
+plt.xlabel('log10(lambda)')
+plt.ylabel('MSE')
+plt.legend()
+plt.show()
+!ec
+
+We see from this plot that the optimal MSE occurs for a value of $\lambda\in [10,100]$.
+In order to nail down the best value of $\lambda$, we could in turn narrow down the search area.
+
+!split
+===== Grid Search =====
+
+
+An alternative is to use the so-called grid search functionality
+included with the library _Scikit-Learn_, as demonstrated for the same
+example here.
+
+!bc pycod
+import numpy as np
+import matplotlib.pyplot as plt
+from sklearn.model_selection import KFold
+from sklearn.linear_model import Ridge
+from sklearn.model_selection import cross_val_score
+from sklearn.preprocessing import PolynomialFeatures
+
+# A seed just to ensure that the random numbers are the same for every run.
+np.random.seed(3155)
+# Generate the data.
+n = 100
+x = np.linspace(-3, 3, n).reshape(-1, 1)
+y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)
+# Decide degree on polynomial to fit
+poly = PolynomialFeatures(degree = 10)
+
+# Decide which values of lambda to use
+nlambdas = 500
+lambdas = np.logspace(-3, 5, nlambdas)
+# Initialize a KFold instance
+k = 5
+kfold = KFold(n_splits = k)
+estimated_mse_sklearn = np.zeros(nlambdas)
+i = 0
+for lmb in lambdas:
+ ridge = Ridge(alpha = lmb)
+ estimated_mse_folds = cross_val_score(ridge, x, y, scoring='neg_mean_squared_error', cv=kfold)
+ estimated_mse_sklearn[i] = np.mean(-estimated_mse_folds)
+ i += 1
+plt.figure()
+plt.plot(np.log10(lambdas), estimated_mse_sklearn, label = 'cross_val_score')
+plt.xlabel('log10(lambda)')
+plt.ylabel('MSE')
+plt.legend()
+plt.show()
+!ec
@@ -1290,3 +1376,5 @@ plt.show()
_Challenge_: try to write a similar code for a Logistic Regression case.
+
+