update resampling chapter
This commit is contained in:
Morten Hjorth-Jensen
@@ -919,7 +919,7 @@ for polydegree in range(1, Maxpolydegree):
|
||||
trainingerror[polydegree] = 0.0
|
||||
for samples in range(trials):
|
||||
x_train, x_test, y_train, y_test = train_test_split(X, Energies, test_size=0.2)
|
||||
model = LinearRegression(fit_intercept=True).fit(x_train, y_train)
|
||||
model = LinearRegression(fit_intercept=False).fit(x_train, y_train)
|
||||
ypred = model.predict(x_train)
|
||||
ytilde = model.predict(x_test)
|
||||
testerror[polydegree] += mean_squared_error(y_test, ytilde)
|
||||
@@ -1156,7 +1156,7 @@ for polydegree in range(1, Maxpolydegree):
|
||||
polynomial[polydegree] = polydegree
|
||||
for degree in range(polydegree):
|
||||
X[:,degree] = Density**(degree/3.0)
|
||||
OLS = LinearRegression()
|
||||
OLS = LinearRegression(fit_intercept=False)
|
||||
# loop over trials in order to estimate the expectation value of the MSE
|
||||
estimated_mse_folds = cross_val_score(OLS, X, Energies, scoring='neg_mean_squared_error', cv=kfold)
|
||||
#[:, np.newaxis]
|
||||
@@ -1170,46 +1170,7 @@ plt.show()
|
||||
|
||||
!ec
|
||||
|
||||
|
||||
!bc pycod
|
||||
import numpy as np
|
||||
import matplotlib.pyplot as plt
|
||||
from sklearn.model_selection import KFold
|
||||
from sklearn.linear_model import Ridge
|
||||
from sklearn.model_selection import cross_val_score
|
||||
from sklearn.preprocessing import PolynomialFeatures
|
||||
|
||||
# A seed just to ensure that the random numbers are the same for every run.
|
||||
np.random.seed(3155)
|
||||
# Generate the data.
|
||||
n = 100
|
||||
x = np.linspace(-3, 3, n).reshape(-1, 1)
|
||||
y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)
|
||||
# Decide degree on polynomial to fit
|
||||
poly = PolynomialFeatures(degree = 10)
|
||||
|
||||
# Decide which values of lambda to use
|
||||
nlambdas = 500
|
||||
lambdas = np.logspace(-3, 5, nlambdas)
|
||||
# Initialize a KFold instance
|
||||
k = 5
|
||||
kfold = KFold(n_splits = k)
|
||||
estimated_mse_sklearn = np.zeros(nlambdas)
|
||||
i = 0
|
||||
for lmb in lambdas:
|
||||
ridge = Ridge(alpha = lmb)
|
||||
estimated_mse_folds = cross_val_score(ridge, x, y, scoring='neg_mean_squared_error', cv=kfold)
|
||||
estimated_mse_sklearn[i] = np.mean(-estimated_mse_folds)
|
||||
i += 1
|
||||
plt.figure()
|
||||
plt.plot(np.log10(lambdas), estimated_mse_sklearn, label = 'cross_val_score')
|
||||
plt.xlabel('log10(lambda)')
|
||||
plt.ylabel('MSE')
|
||||
plt.legend()
|
||||
plt.show()
|
||||
|
||||
|
||||
!ec
|
||||
Note that we have kept the intercept in the first column of design matrix $\bm{X}$. When we call the corresponding _Scikit-Learn_ function we need thus to set the intercept to _False_. Libraries like _Scikit-Learn_ normally scale the design matrix and does not fit intercept. See the discussions below.
|
||||
|
||||
===== More on Rescaling data =====
|
||||
|
||||
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
|
After Width: | Height: | Size: 15 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 11 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 13 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 20 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 31 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 30 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 20 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 193 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 71 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 28 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 11 KiB After Width: | Height: | Size: 11 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 16 KiB After Width: | Height: | Size: 16 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 12 KiB After Width: | Height: | Size: 12 KiB |
@@ -1253,7 +1253,7 @@
|
||||
" trainingerror[polydegree] = 0.0\n",
|
||||
" for samples in range(trials):\n",
|
||||
" x_train, x_test, y_train, y_test = train_test_split(X, Energies, test_size=0.2)\n",
|
||||
" model = LinearRegression(fit_intercept=True).fit(x_train, y_train)\n",
|
||||
" model = LinearRegression(fit_intercept=False).fit(x_train, y_train)\n",
|
||||
" ypred = model.predict(x_train)\n",
|
||||
" ytilde = model.predict(x_test)\n",
|
||||
" testerror[polydegree] += mean_squared_error(y_test, ytilde)\n",
|
||||
@@ -1535,7 +1535,7 @@
|
||||
" polynomial[polydegree] = polydegree\n",
|
||||
" for degree in range(polydegree):\n",
|
||||
" X[:,degree] = Density**(degree/3.0)\n",
|
||||
" OLS = LinearRegression()\n",
|
||||
" OLS = LinearRegression(fit_intercept=False)\n",
|
||||
"# loop over trials in order to estimate the expectation value of the MSE\n",
|
||||
" estimated_mse_folds = cross_val_score(OLS, X, Energies, scoring='neg_mean_squared_error', cv=kfold)\n",
|
||||
"#[:, np.newaxis]\n",
|
||||
@@ -1548,56 +1548,12 @@
|
||||
"plt.show()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"editable": true
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import numpy as np\n",
|
||||
"import matplotlib.pyplot as plt\n",
|
||||
"from sklearn.model_selection import KFold\n",
|
||||
"from sklearn.linear_model import Ridge\n",
|
||||
"from sklearn.model_selection import cross_val_score\n",
|
||||
"from sklearn.preprocessing import PolynomialFeatures\n",
|
||||
"\n",
|
||||
"# A seed just to ensure that the random numbers are the same for every run.\n",
|
||||
"np.random.seed(3155)\n",
|
||||
"# Generate the data.\n",
|
||||
"n = 100\n",
|
||||
"x = np.linspace(-3, 3, n).reshape(-1, 1)\n",
|
||||
"y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)\n",
|
||||
"# Decide degree on polynomial to fit\n",
|
||||
"poly = PolynomialFeatures(degree = 10)\n",
|
||||
"\n",
|
||||
"# Decide which values of lambda to use\n",
|
||||
"nlambdas = 500\n",
|
||||
"lambdas = np.logspace(-3, 5, nlambdas)\n",
|
||||
"# Initialize a KFold instance\n",
|
||||
"k = 5\n",
|
||||
"kfold = KFold(n_splits = k)\n",
|
||||
"estimated_mse_sklearn = np.zeros(nlambdas)\n",
|
||||
"i = 0\n",
|
||||
"for lmb in lambdas:\n",
|
||||
" ridge = Ridge(alpha = lmb)\n",
|
||||
" estimated_mse_folds = cross_val_score(ridge, x, y, scoring='neg_mean_squared_error', cv=kfold)\n",
|
||||
" estimated_mse_sklearn[i] = np.mean(-estimated_mse_folds)\n",
|
||||
" i += 1\n",
|
||||
"plt.figure()\n",
|
||||
"plt.plot(np.log10(lambdas), estimated_mse_sklearn, label = 'cross_val_score')\n",
|
||||
"plt.xlabel('log10(lambda)')\n",
|
||||
"plt.ylabel('MSE')\n",
|
||||
"plt.legend()\n",
|
||||
"plt.show()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Note that we have kept the intercept in the first column of design matrix $\\boldsymbol{X}$. When we call the corresponding **Scikit-Learn** function we need thus to set the intercept to **False**. Libraries like **Scikit-Learn** normally scale the design matrix and does not fit intercept. See the discussions below.\n",
|
||||
"\n",
|
||||
"## More on Rescaling data\n",
|
||||
"\n",
|
||||
"We end this chapter by adding some words on scaling and how to deal with the intercept for regression cases.\n",
|
||||
|
||||
@@ -640,10 +640,10 @@ number <span class="math notranslate nohighlight">\(i\)</span> is left out. Usin
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell_output docutils container">
|
||||
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Runtime: 0.139992 sec
|
||||
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Runtime: 0.14109 sec
|
||||
Jackknife Statistics :
|
||||
original bias std. error
|
||||
99.9142 99.9042 0.148517
|
||||
100.203 100.193 0.149917
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
@@ -862,7 +862,7 @@ theorem.</p>
|
||||
<div class="cell_output docutils container">
|
||||
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Bootstrap Statistics :
|
||||
original bias std. error
|
||||
100.028 14.8941 100.026 0.148225
|
||||
99.9348 15.1379 99.9341 0.151076
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
@@ -1064,14 +1064,15 @@ Error: 0.32149601703519126
|
||||
Bias^2: 0.3123314713548606
|
||||
Var: 0.009164545680330616
|
||||
0.32149601703519126 >= 0.3123314713548606 + 0.009164545680330616 = 0.3214960170351912
|
||||
Polynomial degree: 1
|
||||
Polynomial degree:
|
||||
</pre></div>
|
||||
</div>
|
||||
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span> 1
|
||||
Error: 0.08426840630693411
|
||||
Bias^2: 0.07968918676726028
|
||||
Var: 0.004579219539673833
|
||||
0.08426840630693411 >= 0.07968918676726028 + 0.004579219539673833 = 0.08426840630693411
|
||||
</pre></div>
|
||||
</div>
|
||||
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Polynomial degree: 2
|
||||
Polynomial degree: 2
|
||||
Error: 0.10398646080125035
|
||||
Bias^2: 0.10077114273548986
|
||||
Var: 0.0032153180657605086
|
||||
@@ -1120,26 +1121,28 @@ Error: 0.021592704588043153
|
||||
Bias^2: 0.010516485576652981
|
||||
Var: 0.011076219011390184
|
||||
0.021592704588043153 >= 0.010516485576652981 + 0.011076219011390184 = 0.021592704588043167
|
||||
Polynomial degree: 11
|
||||
</pre></div>
|
||||
</div>
|
||||
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Polynomial degree: 11
|
||||
Error: 0.07160048164228314
|
||||
Bias^2: 0.01443680008897583
|
||||
Var: 0.0571636815533073
|
||||
0.07160048164228314 >= 0.01443680008897583 + 0.0571636815533073 = 0.07160048164228312
|
||||
</pre></div>
|
||||
</div>
|
||||
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Polynomial degree: 12
|
||||
Polynomial degree: 12
|
||||
Error: 0.1154777721897675
|
||||
Bias^2: 0.01628578269590588
|
||||
Var: 0.09919198949386163
|
||||
0.1154777721897675 >= 0.01628578269590588 + 0.09919198949386163 = 0.11547777218976751
|
||||
Polynomial degree: 13
|
||||
</pre></div>
|
||||
</div>
|
||||
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Polynomial degree: 13
|
||||
Error: 0.22842468702166951
|
||||
Bias^2: 0.01975416527163567
|
||||
Var: 0.20867052175003387
|
||||
0.22842468702166951 >= 0.01975416527163567 + 0.20867052175003387 = 0.22842468702166954
|
||||
</pre></div>
|
||||
</div>
|
||||
<img alt="_images/chapter3_62_5.png" src="_images/chapter3_62_5.png" />
|
||||
<img alt="_images/chapter3_62_6.png" src="_images/chapter3_62_6.png" />
|
||||
</div>
|
||||
</div>
|
||||
<p>The bias-variance tradeoff summarizes the fundamental tension in
|
||||
@@ -1329,7 +1332,7 @@ training data.
|
||||
<span class="n">trainingerror</span><span class="p">[</span><span class="n">polydegree</span><span class="p">]</span> <span class="o">=</span> <span class="mf">0.0</span>
|
||||
<span class="k">for</span> <span class="n">samples</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">trials</span><span class="p">):</span>
|
||||
<span class="n">x_train</span><span class="p">,</span> <span class="n">x_test</span><span class="p">,</span> <span class="n">y_train</span><span class="p">,</span> <span class="n">y_test</span> <span class="o">=</span> <span class="n">train_test_split</span><span class="p">(</span><span class="n">X</span><span class="p">,</span> <span class="n">Energies</span><span class="p">,</span> <span class="n">test_size</span><span class="o">=</span><span class="mf">0.2</span><span class="p">)</span>
|
||||
<span class="n">model</span> <span class="o">=</span> <span class="n">LinearRegression</span><span class="p">(</span><span class="n">fit_intercept</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">x_train</span><span class="p">,</span> <span class="n">y_train</span><span class="p">)</span>
|
||||
<span class="n">model</span> <span class="o">=</span> <span class="n">LinearRegression</span><span class="p">(</span><span class="n">fit_intercept</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">x_train</span><span class="p">,</span> <span class="n">y_train</span><span class="p">)</span>
|
||||
<span class="n">ypred</span> <span class="o">=</span> <span class="n">model</span><span class="o">.</span><span class="n">predict</span><span class="p">(</span><span class="n">x_train</span><span class="p">)</span>
|
||||
<span class="n">ytilde</span> <span class="o">=</span> <span class="n">model</span><span class="o">.</span><span class="n">predict</span><span class="p">(</span><span class="n">x_test</span><span class="p">)</span>
|
||||
<span class="n">testerror</span><span class="p">[</span><span class="n">polydegree</span><span class="p">]</span> <span class="o">+=</span> <span class="n">mean_squared_error</span><span class="p">(</span><span class="n">y_test</span><span class="p">,</span> <span class="n">ytilde</span><span class="p">)</span>
|
||||
@@ -1360,12 +1363,12 @@ Mean squared error on test data: 123711.53703498
|
||||
Degree of polynomial: 3
|
||||
Mean squared error on training data: 9011.85263220
|
||||
Mean squared error on test data: 10913.84780262
|
||||
</pre></div>
|
||||
</div>
|
||||
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Degree of polynomial: 4
|
||||
Degree of polynomial: 4
|
||||
Mean squared error on training data: 303.47610036
|
||||
Mean squared error on test data: 426.30787294
|
||||
Degree of polynomial: 5
|
||||
</pre></div>
|
||||
</div>
|
||||
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Degree of polynomial: 5
|
||||
Mean squared error on training data: 3.80354994
|
||||
Mean squared error on test data: 5.98822371
|
||||
Degree of polynomial: 6
|
||||
@@ -1385,80 +1388,80 @@ Mean squared error on test data: 0.08576932
|
||||
Degree of polynomial: 10
|
||||
Mean squared error on training data: 0.02511518
|
||||
Mean squared error on test data: 1.20015436
|
||||
Degree of polynomial: 11
|
||||
Mean squared error on training data: 0.01640891
|
||||
Mean squared error on test data: 1.35533773
|
||||
</pre></div>
|
||||
</div>
|
||||
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Degree of polynomial: 12
|
||||
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Degree of polynomial: 11
|
||||
Mean squared error on training data: 0.01640891
|
||||
Mean squared error on test data: 1.35533774
|
||||
Degree of polynomial: 12
|
||||
Mean squared error on training data: 0.00813803
|
||||
Mean squared error on test data: 0.17446471
|
||||
Degree of polynomial: 13
|
||||
Mean squared error on training data: 0.00759119
|
||||
Mean squared error on test data: 1.08131003
|
||||
Mean squared error on test data: 1.08131001
|
||||
Degree of polynomial: 14
|
||||
Mean squared error on training data: 0.00472199
|
||||
Mean squared error on test data: 0.81333793
|
||||
Mean squared error on test data: 0.81333802
|
||||
</pre></div>
|
||||
</div>
|
||||
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Degree of polynomial: 15
|
||||
Mean squared error on training data: 0.00410478
|
||||
Mean squared error on test data: 92.09145189
|
||||
Mean squared error on test data: 92.09160813
|
||||
Degree of polynomial: 16
|
||||
Mean squared error on training data: 0.00315593
|
||||
Mean squared error on test data: 234.39716546
|
||||
Mean squared error on test data: 234.40530431
|
||||
Degree of polynomial: 17
|
||||
Mean squared error on training data: 0.00242998
|
||||
Mean squared error on test data: 1271.05295709
|
||||
Mean squared error on training data: 0.00242999
|
||||
Mean squared error on test data: 1270.94936405
|
||||
</pre></div>
|
||||
</div>
|
||||
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Degree of polynomial: 18
|
||||
Mean squared error on training data: 0.00228740
|
||||
Mean squared error on test data: 108.42208194
|
||||
Mean squared error on training data: 0.00228741
|
||||
Mean squared error on test data: 108.11945731
|
||||
Degree of polynomial: 19
|
||||
Mean squared error on training data: 0.00156372
|
||||
Mean squared error on test data: 1388.41078073
|
||||
Mean squared error on test data: 1376.61081005
|
||||
Degree of polynomial: 20
|
||||
Mean squared error on training data: 0.00137982
|
||||
Mean squared error on test data: 1761.43341615
|
||||
Mean squared error on training data: 0.00137945
|
||||
Mean squared error on test data: 1931.97211078
|
||||
Degree of polynomial: 21
|
||||
Mean squared error on training data: 0.00118678
|
||||
Mean squared error on test data: 14496.70992192
|
||||
</pre></div>
|
||||
</div>
|
||||
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Degree of polynomial: 21
|
||||
Mean squared error on training data: 0.00118170
|
||||
Mean squared error on test data: 15061.31603087
|
||||
Degree of polynomial: 22
|
||||
Mean squared error on training data: 0.00092354
|
||||
Mean squared error on test data: 890.63488525
|
||||
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Degree of polynomial: 22
|
||||
Mean squared error on training data: 0.00092686
|
||||
Mean squared error on test data: 873.95463048
|
||||
Degree of polynomial: 23
|
||||
Mean squared error on training data: 0.00085887
|
||||
Mean squared error on test data: 5483.16796929
|
||||
Mean squared error on training data: 0.00085890
|
||||
Mean squared error on test data: 5535.20053452
|
||||
Degree of polynomial: 24
|
||||
Mean squared error on training data: 0.00084714
|
||||
Mean squared error on test data: 1289.22422186
|
||||
</pre></div>
|
||||
</div>
|
||||
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Degree of polynomial: 24
|
||||
Mean squared error on training data: 0.00084589
|
||||
Mean squared error on test data: 1695.57143061
|
||||
Degree of polynomial: 25
|
||||
Mean squared error on training data: 0.00078806
|
||||
Mean squared error on test data: 131343.30655001
|
||||
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Degree of polynomial: 25
|
||||
Mean squared error on training data: 0.00079022
|
||||
Mean squared error on test data: 136582.88824397
|
||||
Degree of polynomial: 26
|
||||
Mean squared error on training data: 0.00076916
|
||||
Mean squared error on test data: 17709.14370264
|
||||
Mean squared error on training data: 0.00076923
|
||||
Mean squared error on test data: 18194.23521766
|
||||
Degree of polynomial: 27
|
||||
Mean squared error on training data: 0.00069302
|
||||
Mean squared error on test data: 2579.13493762
|
||||
</pre></div>
|
||||
</div>
|
||||
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Degree of polynomial: 27
|
||||
Mean squared error on training data: 0.00068970
|
||||
Mean squared error on test data: 2975.38903780
|
||||
Degree of polynomial: 28
|
||||
Mean squared error on training data: 0.00062588
|
||||
Mean squared error on test data: 3848.64522721
|
||||
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Degree of polynomial: 28
|
||||
Mean squared error on training data: 0.00062728
|
||||
Mean squared error on test data: 3984.82493809
|
||||
Degree of polynomial: 29
|
||||
Mean squared error on training data: 0.00060728
|
||||
Mean squared error on test data: 2988.64001211
|
||||
Mean squared error on training data: 0.00060724
|
||||
Mean squared error on test data: 3204.07047448
|
||||
</pre></div>
|
||||
</div>
|
||||
<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span><ipython-input-7-8dc29df57a8c>:73: RuntimeWarning: divide by zero encountered in log10
|
||||
<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span><ipython-input-7-40a38ad763f1>:73: RuntimeWarning: divide by zero encountered in log10
|
||||
plt.plot(polynomial, np.log10(trainingerror), label='Training Error')
|
||||
<ipython-input-7-8dc29df57a8c>:74: RuntimeWarning: divide by zero encountered in log10
|
||||
<ipython-input-7-40a38ad763f1>:74: RuntimeWarning: divide by zero encountered in log10
|
||||
plt.plot(polynomial, np.log10(testerror), label='Test Error')
|
||||
</pre></div>
|
||||
</div>
|
||||
@@ -1677,7 +1680,7 @@ cross-validation (LOOCV).</p>
|
||||
<span class="n">polynomial</span><span class="p">[</span><span class="n">polydegree</span><span class="p">]</span> <span class="o">=</span> <span class="n">polydegree</span>
|
||||
<span class="k">for</span> <span class="n">degree</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">polydegree</span><span class="p">):</span>
|
||||
<span class="n">X</span><span class="p">[:,</span><span class="n">degree</span><span class="p">]</span> <span class="o">=</span> <span class="n">Density</span><span class="o">**</span><span class="p">(</span><span class="n">degree</span><span class="o">/</span><span class="mf">3.0</span><span class="p">)</span>
|
||||
<span class="n">OLS</span> <span class="o">=</span> <span class="n">LinearRegression</span><span class="p">()</span>
|
||||
<span class="n">OLS</span> <span class="o">=</span> <span class="n">LinearRegression</span><span class="p">(</span><span class="n">fit_intercept</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
|
||||
<span class="c1"># loop over trials in order to estimate the expectation value of the MSE</span>
|
||||
<span class="n">estimated_mse_folds</span> <span class="o">=</span> <span class="n">cross_val_score</span><span class="p">(</span><span class="n">OLS</span><span class="p">,</span> <span class="n">X</span><span class="p">,</span> <span class="n">Energies</span><span class="p">,</span> <span class="n">scoring</span><span class="o">=</span><span class="s1">'neg_mean_squared_error'</span><span class="p">,</span> <span class="n">cv</span><span class="o">=</span><span class="n">kfold</span><span class="p">)</span>
|
||||
<span class="c1">#[:, np.newaxis]</span>
|
||||
@@ -1692,57 +1695,14 @@ cross-validation (LOOCV).</p>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell_output docutils container">
|
||||
<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span><ipython-input-9-49b0ef2e51e2>:63: RuntimeWarning: divide by zero encountered in log10
|
||||
<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span><ipython-input-9-6e75736fdab1>:63: RuntimeWarning: divide by zero encountered in log10
|
||||
plt.plot(polynomial, np.log10(estimated_mse_sklearn), label='Test Error')
|
||||
</pre></div>
|
||||
</div>
|
||||
<img alt="_images/chapter3_73_1.png" src="_images/chapter3_73_1.png" />
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell docutils container">
|
||||
<div class="cell_input docutils container">
|
||||
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
|
||||
<span class="kn">import</span> <span class="nn">matplotlib.pyplot</span> <span class="k">as</span> <span class="nn">plt</span>
|
||||
<span class="kn">from</span> <span class="nn">sklearn.model_selection</span> <span class="kn">import</span> <span class="n">KFold</span>
|
||||
<span class="kn">from</span> <span class="nn">sklearn.linear_model</span> <span class="kn">import</span> <span class="n">Ridge</span>
|
||||
<span class="kn">from</span> <span class="nn">sklearn.model_selection</span> <span class="kn">import</span> <span class="n">cross_val_score</span>
|
||||
<span class="kn">from</span> <span class="nn">sklearn.preprocessing</span> <span class="kn">import</span> <span class="n">PolynomialFeatures</span>
|
||||
|
||||
<span class="c1"># A seed just to ensure that the random numbers are the same for every run.</span>
|
||||
<span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">seed</span><span class="p">(</span><span class="mi">3155</span><span class="p">)</span>
|
||||
<span class="c1"># Generate the data.</span>
|
||||
<span class="n">n</span> <span class="o">=</span> <span class="mi">100</span>
|
||||
<span class="n">x</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">linspace</span><span class="p">(</span><span class="o">-</span><span class="mi">3</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="n">n</span><span class="p">)</span><span class="o">.</span><span class="n">reshape</span><span class="p">(</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">)</span>
|
||||
<span class="n">y</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">exp</span><span class="p">(</span><span class="o">-</span><span class="n">x</span><span class="o">**</span><span class="mi">2</span><span class="p">)</span> <span class="o">+</span> <span class="mf">1.5</span> <span class="o">*</span> <span class="n">np</span><span class="o">.</span><span class="n">exp</span><span class="p">(</span><span class="o">-</span><span class="p">(</span><span class="n">x</span><span class="o">-</span><span class="mi">2</span><span class="p">)</span><span class="o">**</span><span class="mi">2</span><span class="p">)</span><span class="o">+</span> <span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">normal</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span> <span class="mf">0.1</span><span class="p">,</span> <span class="n">x</span><span class="o">.</span><span class="n">shape</span><span class="p">)</span>
|
||||
<span class="c1"># Decide degree on polynomial to fit</span>
|
||||
<span class="n">poly</span> <span class="o">=</span> <span class="n">PolynomialFeatures</span><span class="p">(</span><span class="n">degree</span> <span class="o">=</span> <span class="mi">10</span><span class="p">)</span>
|
||||
|
||||
<span class="c1"># Decide which values of lambda to use</span>
|
||||
<span class="n">nlambdas</span> <span class="o">=</span> <span class="mi">500</span>
|
||||
<span class="n">lambdas</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">logspace</span><span class="p">(</span><span class="o">-</span><span class="mi">3</span><span class="p">,</span> <span class="mi">5</span><span class="p">,</span> <span class="n">nlambdas</span><span class="p">)</span>
|
||||
<span class="c1"># Initialize a KFold instance</span>
|
||||
<span class="n">k</span> <span class="o">=</span> <span class="mi">5</span>
|
||||
<span class="n">kfold</span> <span class="o">=</span> <span class="n">KFold</span><span class="p">(</span><span class="n">n_splits</span> <span class="o">=</span> <span class="n">k</span><span class="p">)</span>
|
||||
<span class="n">estimated_mse_sklearn</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros</span><span class="p">(</span><span class="n">nlambdas</span><span class="p">)</span>
|
||||
<span class="n">i</span> <span class="o">=</span> <span class="mi">0</span>
|
||||
<span class="k">for</span> <span class="n">lmb</span> <span class="ow">in</span> <span class="n">lambdas</span><span class="p">:</span>
|
||||
<span class="n">ridge</span> <span class="o">=</span> <span class="n">Ridge</span><span class="p">(</span><span class="n">alpha</span> <span class="o">=</span> <span class="n">lmb</span><span class="p">)</span>
|
||||
<span class="n">estimated_mse_folds</span> <span class="o">=</span> <span class="n">cross_val_score</span><span class="p">(</span><span class="n">ridge</span><span class="p">,</span> <span class="n">x</span><span class="p">,</span> <span class="n">y</span><span class="p">,</span> <span class="n">scoring</span><span class="o">=</span><span class="s1">'neg_mean_squared_error'</span><span class="p">,</span> <span class="n">cv</span><span class="o">=</span><span class="n">kfold</span><span class="p">)</span>
|
||||
<span class="n">estimated_mse_sklearn</span><span class="p">[</span><span class="n">i</span><span class="p">]</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="o">-</span><span class="n">estimated_mse_folds</span><span class="p">)</span>
|
||||
<span class="n">i</span> <span class="o">+=</span> <span class="mi">1</span>
|
||||
<span class="n">plt</span><span class="o">.</span><span class="n">figure</span><span class="p">()</span>
|
||||
<span class="n">plt</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">log10</span><span class="p">(</span><span class="n">lambdas</span><span class="p">),</span> <span class="n">estimated_mse_sklearn</span><span class="p">,</span> <span class="n">label</span> <span class="o">=</span> <span class="s1">'cross_val_score'</span><span class="p">)</span>
|
||||
<span class="n">plt</span><span class="o">.</span><span class="n">xlabel</span><span class="p">(</span><span class="s1">'log10(lambda)'</span><span class="p">)</span>
|
||||
<span class="n">plt</span><span class="o">.</span><span class="n">ylabel</span><span class="p">(</span><span class="s1">'MSE'</span><span class="p">)</span>
|
||||
<span class="n">plt</span><span class="o">.</span><span class="n">legend</span><span class="p">()</span>
|
||||
<span class="n">plt</span><span class="o">.</span><span class="n">show</span><span class="p">()</span>
|
||||
</pre></div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell_output docutils container">
|
||||
<img alt="_images/chapter3_74_0.png" src="_images/chapter3_74_0.png" />
|
||||
</div>
|
||||
</div>
|
||||
<p>Note that we have kept the intercept in the first column of design matrix <span class="math notranslate nohighlight">\(\boldsymbol{X}\)</span>. When we call the corresponding <strong>Scikit-Learn</strong> function we need thus to set the intercept to <strong>False</strong>. Libraries like <strong>Scikit-Learn</strong> normally scale the design matrix and does not fit intercept. See the discussions below.</p>
|
||||
</div>
|
||||
<div class="section" id="more-on-rescaling-data">
|
||||
<h2><span class="section-number">5.6. </span>More on Rescaling data<a class="headerlink" href="#more-on-rescaling-data" title="Permalink to this headline">¶</a></h2>
|
||||
@@ -2020,7 +1980,7 @@ MSE with Sklearn intercept
|
||||
0.004113634617443135
|
||||
</pre></div>
|
||||
</div>
|
||||
<img alt="_images/chapter3_110_1.png" src="_images/chapter3_110_1.png" />
|
||||
<img alt="_images/chapter3_109_1.png" src="_images/chapter3_109_1.png" />
|
||||
</div>
|
||||
</div>
|
||||
<p>The intercept is the value of our output/target variable
|
||||
@@ -2215,7 +2175,7 @@ MSE values for Scikit-Learn Ridge implementation
|
||||
0.26409315307910036
|
||||
</pre></div>
|
||||
</div>
|
||||
<img alt="_images/chapter3_118_1.png" src="_images/chapter3_118_1.png" />
|
||||
<img alt="_images/chapter3_117_1.png" src="_images/chapter3_117_1.png" />
|
||||
</div>
|
||||
</div>
|
||||
<p>The results here agree when we force <strong>Scikit-Learn</strong>’s Ridge function to include the first column in our design matrix.
|
||||
@@ -2420,7 +2380,7 @@ MSE values for Scikit-Learn Ridge implementation
|
||||
0.002381316302584886
|
||||
</pre></div>
|
||||
</div>
|
||||
<img alt="_images/chapter3_120_1.png" src="_images/chapter3_120_1.png" />
|
||||
<img alt="_images/chapter3_119_1.png" src="_images/chapter3_119_1.png" />
|
||||
</div>
|
||||
</div>
|
||||
<p>We see here, when compared to the code which includes explicitely the
|
||||
@@ -2634,11 +2594,11 @@ linear system as an equation would reduce this down to
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell_output docutils container">
|
||||
<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span><ipython-input-22-6f7a6bd7d79f>:7: UserWarning: FixedFormatter should only be used together with FixedLocator
|
||||
<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span><ipython-input-21-6f7a6bd7d79f>:7: UserWarning: FixedFormatter should only be used together with FixedLocator
|
||||
cb.ax.set_yticklabels(cb.ax.get_yticklabels(), fontsize=18)
|
||||
</pre></div>
|
||||
</div>
|
||||
<img alt="_images/chapter3_151_1.png" src="_images/chapter3_151_1.png" />
|
||||
<img alt="_images/chapter3_150_1.png" src="_images/chapter3_150_1.png" />
|
||||
</div>
|
||||
</div>
|
||||
<p>It is interesting to note that OLS
|
||||
@@ -2778,11 +2738,11 @@ with the form utilized in linear regression, viz.</p>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell_output docutils container">
|
||||
<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span><ipython-input-27-5dd54edf2138>:7: UserWarning: FixedFormatter should only be used together with FixedLocator
|
||||
<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span><ipython-input-26-5dd54edf2138>:7: UserWarning: FixedFormatter should only be used together with FixedLocator
|
||||
cb.ax.set_yticklabels(cb.ax.get_yticklabels(), fontsize=18)
|
||||
</pre></div>
|
||||
</div>
|
||||
<img alt="_images/chapter3_169_1.png" src="_images/chapter3_169_1.png" />
|
||||
<img alt="_images/chapter3_168_1.png" src="_images/chapter3_168_1.png" />
|
||||
</div>
|
||||
</div>
|
||||
<p>The results agree perfectly with our previous discussion where we used our own code.</p>
|
||||
@@ -2826,11 +2786,11 @@ K</p>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell_output docutils container">
|
||||
<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span><ipython-input-28-fe5b9d300cc0>:10: UserWarning: FixedFormatter should only be used together with FixedLocator
|
||||
<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span><ipython-input-27-fe5b9d300cc0>:10: UserWarning: FixedFormatter should only be used together with FixedLocator
|
||||
cb.ax.set_yticklabels(cb.ax.get_yticklabels(), fontsize=18)
|
||||
</pre></div>
|
||||
</div>
|
||||
<img alt="_images/chapter3_172_1.png" src="_images/chapter3_172_1.png" />
|
||||
<img alt="_images/chapter3_171_1.png" src="_images/chapter3_171_1.png" />
|
||||
</div>
|
||||
</div>
|
||||
<p>In the <strong>Least Absolute Shrinkage and Selection Operator</strong> (LASSO)-method we get a third cost function.</p>
|
||||
@@ -2861,11 +2821,11 @@ K</p>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell_output docutils container">
|
||||
<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span><ipython-input-29-25845e8df859>:9: UserWarning: FixedFormatter should only be used together with FixedLocator
|
||||
<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span><ipython-input-28-25845e8df859>:9: UserWarning: FixedFormatter should only be used together with FixedLocator
|
||||
cb.ax.set_yticklabels(cb.ax.get_yticklabels(), fontsize=18)
|
||||
</pre></div>
|
||||
</div>
|
||||
<img alt="_images/chapter3_176_1.png" src="_images/chapter3_176_1.png" />
|
||||
<img alt="_images/chapter3_175_1.png" src="_images/chapter3_175_1.png" />
|
||||
</div>
|
||||
</div>
|
||||
<p>It is quite striking how LASSO breaks the symmetry of the coupling
|
||||
@@ -2920,43 +2880,43 @@ constant as opposed to ridge and OLS. We get a sparse solution with
|
||||
<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>/Users/mhjensen/opt/anaconda3/lib/python3.8/site-packages/sklearn/linear_model/_coordinate_descent.py:529: ConvergenceWarning: Objective did not converge. You might want to increase the number of iterations. Duality gap: 3.924197515789051, tolerance: 1.796796
|
||||
model = cd_fast.enet_coordinate_descent(
|
||||
|
||||
10%|█ | 1/10 [00:00<00:04, 2.06it/s]
|
||||
10%|█ | 1/10 [00:00<00:04, 2.01it/s]
|
||||
</pre></div>
|
||||
</div>
|
||||
<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span> 20%|██ | 2/10 [00:00<00:03, 2.39it/s]
|
||||
<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span> 20%|██ | 2/10 [00:00<00:03, 2.26it/s]
|
||||
</pre></div>
|
||||
</div>
|
||||
<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span> 30%|███ | 3/10 [00:00<00:02, 3.03it/s]
|
||||
<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span> 30%|███ | 3/10 [00:00<00:02, 2.75it/s]
|
||||
</pre></div>
|
||||
</div>
|
||||
<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span> 40%|████ | 4/10 [00:00<00:01, 3.75it/s]
|
||||
<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span> 40%|████ | 4/10 [00:01<00:01, 3.39it/s]
|
||||
</pre></div>
|
||||
</div>
|
||||
<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span> 50%|█████ | 5/10 [00:01<00:01, 4.51it/s]
|
||||
<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span> 50%|█████ | 5/10 [00:01<00:01, 4.00it/s]
|
||||
</pre></div>
|
||||
</div>
|
||||
<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span> 60%|██████ | 6/10 [00:01<00:00, 5.17it/s]
|
||||
<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span> 60%|██████ | 6/10 [00:01<00:00, 4.67it/s]
|
||||
</pre></div>
|
||||
</div>
|
||||
<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span> 70%|███████ | 7/10 [00:01<00:00, 5.94it/s]
|
||||
<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span> 70%|███████ | 7/10 [00:01<00:00, 5.36it/s]
|
||||
</pre></div>
|
||||
</div>
|
||||
<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span> 80%|████████ | 8/10 [00:01<00:00, 6.59it/s]
|
||||
<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span> 80%|████████ | 8/10 [00:01<00:00, 6.06it/s]
|
||||
</pre></div>
|
||||
</div>
|
||||
<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span> 90%|█████████ | 9/10 [00:01<00:00, 7.10it/s]
|
||||
<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span> 90%|█████████ | 9/10 [00:01<00:00, 6.05it/s]
|
||||
</pre></div>
|
||||
</div>
|
||||
<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>100%|██████████| 10/10 [00:01<00:00, 7.07it/s]
|
||||
<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>100%|██████████| 10/10 [00:02<00:00, 4.64it/s]
|
||||
</pre></div>
|
||||
</div>
|
||||
<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>100%|██████████| 10/10 [00:01<00:00, 5.83it/s]
|
||||
<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>100%|██████████| 10/10 [00:02<00:00, 4.68it/s]
|
||||
</pre></div>
|
||||
</div>
|
||||
<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>
|
||||
</pre></div>
|
||||
</div>
|
||||
<img alt="_images/chapter3_178_13.png" src="_images/chapter3_178_13.png" />
|
||||
<img alt="_images/chapter3_177_13.png" src="_images/chapter3_177_13.png" />
|
||||
</div>
|
||||
</div>
|
||||
<p>We see that LASSO reaches a good solution for low
|
||||
@@ -3005,7 +2965,7 @@ testing set that is close to the accuracy of the training set.</p>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell_output docutils container">
|
||||
<img alt="_images/chapter3_180_0.png" src="_images/chapter3_180_0.png" />
|
||||
<img alt="_images/chapter3_179_0.png" src="_images/chapter3_179_0.png" />
|
||||
</div>
|
||||
</div>
|
||||
<p>From the above figure we can see that LASSO with <span class="math notranslate nohighlight">\(\lambda = 10^{-2}\)</span>
|
||||
@@ -3097,7 +3057,7 @@ which polynomial fits the data best.</p>
|
||||
</div>
|
||||
</div>
|
||||
<div class="cell_output docutils container">
|
||||
<img alt="_images/chapter3_184_0.png" src="_images/chapter3_184_0.png" />
|
||||
<img alt="_images/chapter3_183_0.png" src="_images/chapter3_183_0.png" />
|
||||
</div>
|
||||
</div>
|
||||
<div class="section" id="exercise-ordinary-least-square-ols-on-the-franke-function">
|
||||
@@ -3254,7 +3214,7 @@ Python program using</p>
|
||||
<div class="cell_output docutils container">
|
||||
<div class="output traceback highlight-ipythontb notranslate"><div class="highlight"><pre><span></span><span class="gt">---------------------------------------------------------------------------</span>
|
||||
<span class="ne">NameError</span><span class="g g-Whitespace"> </span>Traceback (most recent call last)
|
||||
<span class="o"><</span><span class="n">ipython</span><span class="o">-</span><span class="nb">input</span><span class="o">-</span><span class="mi">33</span><span class="o">-</span><span class="n">d985fb40c43d</span><span class="o">></span> <span class="ow">in</span> <span class="o"><</span><span class="n">module</span><span class="o">></span>
|
||||
<span class="o"><</span><span class="n">ipython</span><span class="o">-</span><span class="nb">input</span><span class="o">-</span><span class="mi">32</span><span class="o">-</span><span class="n">d985fb40c43d</span><span class="o">></span> <span class="ow">in</span> <span class="o"><</span><span class="n">module</span><span class="o">></span>
|
||||
<span class="ne">----> </span><span class="mi">1</span> <span class="n">scipy</span><span class="o">.</span><span class="n">misc</span><span class="o">.</span><span class="n">imread</span>
|
||||
|
||||
<span class="ne">NameError</span>: name 'scipy' is not defined
|
||||
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -887,7 +887,7 @@ for polydegree in range(1, Maxpolydegree):
|
||||
trainingerror[polydegree] = 0.0
|
||||
for samples in range(trials):
|
||||
x_train, x_test, y_train, y_test = train_test_split(X, Energies, test_size=0.2)
|
||||
model = LinearRegression(fit_intercept=True).fit(x_train, y_train)
|
||||
model = LinearRegression(fit_intercept=False).fit(x_train, y_train)
|
||||
ypred = model.predict(x_train)
|
||||
ytilde = model.predict(x_test)
|
||||
testerror[polydegree] += mean_squared_error(y_test, ytilde)
|
||||
@@ -1119,7 +1119,7 @@ for polydegree in range(1, Maxpolydegree):
|
||||
polynomial[polydegree] = polydegree
|
||||
for degree in range(polydegree):
|
||||
X[:,degree] = Density**(degree/3.0)
|
||||
OLS = LinearRegression()
|
||||
OLS = LinearRegression(fit_intercept=False)
|
||||
# loop over trials in order to estimate the expectation value of the MSE
|
||||
estimated_mse_folds = cross_val_score(OLS, X, Energies, scoring='neg_mean_squared_error', cv=kfold)
|
||||
#[:, np.newaxis]
|
||||
@@ -1131,41 +1131,7 @@ plt.ylabel('log10[MSE]')
|
||||
plt.legend()
|
||||
plt.show()
|
||||
|
||||
import numpy as np
|
||||
import matplotlib.pyplot as plt
|
||||
from sklearn.model_selection import KFold
|
||||
from sklearn.linear_model import Ridge
|
||||
from sklearn.model_selection import cross_val_score
|
||||
from sklearn.preprocessing import PolynomialFeatures
|
||||
|
||||
# A seed just to ensure that the random numbers are the same for every run.
|
||||
np.random.seed(3155)
|
||||
# Generate the data.
|
||||
n = 100
|
||||
x = np.linspace(-3, 3, n).reshape(-1, 1)
|
||||
y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)
|
||||
# Decide degree on polynomial to fit
|
||||
poly = PolynomialFeatures(degree = 10)
|
||||
|
||||
# Decide which values of lambda to use
|
||||
nlambdas = 500
|
||||
lambdas = np.logspace(-3, 5, nlambdas)
|
||||
# Initialize a KFold instance
|
||||
k = 5
|
||||
kfold = KFold(n_splits = k)
|
||||
estimated_mse_sklearn = np.zeros(nlambdas)
|
||||
i = 0
|
||||
for lmb in lambdas:
|
||||
ridge = Ridge(alpha = lmb)
|
||||
estimated_mse_folds = cross_val_score(ridge, x, y, scoring='neg_mean_squared_error', cv=kfold)
|
||||
estimated_mse_sklearn[i] = np.mean(-estimated_mse_folds)
|
||||
i += 1
|
||||
plt.figure()
|
||||
plt.plot(np.log10(lambdas), estimated_mse_sklearn, label = 'cross_val_score')
|
||||
plt.xlabel('log10(lambda)')
|
||||
plt.ylabel('MSE')
|
||||
plt.legend()
|
||||
plt.show()
|
||||
Note that we have kept the intercept in the first column of design matrix $\boldsymbol{X}$. When we call the corresponding **Scikit-Learn** function we need thus to set the intercept to **False**. Libraries like **Scikit-Learn** normally scale the design matrix and does not fit intercept. See the discussions below.
|
||||
|
||||
## More on Rescaling data
|
||||
|
||||
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 15 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 11 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 13 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 20 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 31 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 30 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 20 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 193 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 71 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 28 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 11 KiB After Width: | Height: | Size: 11 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 16 KiB After Width: | Height: | Size: 16 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 12 KiB After Width: | Height: | Size: 12 KiB |
@@ -1253,7 +1253,7 @@
|
||||
" trainingerror[polydegree] = 0.0\n",
|
||||
" for samples in range(trials):\n",
|
||||
" x_train, x_test, y_train, y_test = train_test_split(X, Energies, test_size=0.2)\n",
|
||||
" model = LinearRegression(fit_intercept=True).fit(x_train, y_train)\n",
|
||||
" model = LinearRegression(fit_intercept=False).fit(x_train, y_train)\n",
|
||||
" ypred = model.predict(x_train)\n",
|
||||
" ytilde = model.predict(x_test)\n",
|
||||
" testerror[polydegree] += mean_squared_error(y_test, ytilde)\n",
|
||||
@@ -1535,7 +1535,7 @@
|
||||
" polynomial[polydegree] = polydegree\n",
|
||||
" for degree in range(polydegree):\n",
|
||||
" X[:,degree] = Density**(degree/3.0)\n",
|
||||
" OLS = LinearRegression()\n",
|
||||
" OLS = LinearRegression(fit_intercept=False)\n",
|
||||
"# loop over trials in order to estimate the expectation value of the MSE\n",
|
||||
" estimated_mse_folds = cross_val_score(OLS, X, Energies, scoring='neg_mean_squared_error', cv=kfold)\n",
|
||||
"#[:, np.newaxis]\n",
|
||||
@@ -1548,56 +1548,12 @@
|
||||
"plt.show()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {
|
||||
"collapsed": false,
|
||||
"editable": true
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import numpy as np\n",
|
||||
"import matplotlib.pyplot as plt\n",
|
||||
"from sklearn.model_selection import KFold\n",
|
||||
"from sklearn.linear_model import Ridge\n",
|
||||
"from sklearn.model_selection import cross_val_score\n",
|
||||
"from sklearn.preprocessing import PolynomialFeatures\n",
|
||||
"\n",
|
||||
"# A seed just to ensure that the random numbers are the same for every run.\n",
|
||||
"np.random.seed(3155)\n",
|
||||
"# Generate the data.\n",
|
||||
"n = 100\n",
|
||||
"x = np.linspace(-3, 3, n).reshape(-1, 1)\n",
|
||||
"y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)\n",
|
||||
"# Decide degree on polynomial to fit\n",
|
||||
"poly = PolynomialFeatures(degree = 10)\n",
|
||||
"\n",
|
||||
"# Decide which values of lambda to use\n",
|
||||
"nlambdas = 500\n",
|
||||
"lambdas = np.logspace(-3, 5, nlambdas)\n",
|
||||
"# Initialize a KFold instance\n",
|
||||
"k = 5\n",
|
||||
"kfold = KFold(n_splits = k)\n",
|
||||
"estimated_mse_sklearn = np.zeros(nlambdas)\n",
|
||||
"i = 0\n",
|
||||
"for lmb in lambdas:\n",
|
||||
" ridge = Ridge(alpha = lmb)\n",
|
||||
" estimated_mse_folds = cross_val_score(ridge, x, y, scoring='neg_mean_squared_error', cv=kfold)\n",
|
||||
" estimated_mse_sklearn[i] = np.mean(-estimated_mse_folds)\n",
|
||||
" i += 1\n",
|
||||
"plt.figure()\n",
|
||||
"plt.plot(np.log10(lambdas), estimated_mse_sklearn, label = 'cross_val_score')\n",
|
||||
"plt.xlabel('log10(lambda)')\n",
|
||||
"plt.ylabel('MSE')\n",
|
||||
"plt.legend()\n",
|
||||
"plt.show()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Note that we have kept the intercept in the first column of design matrix $\\boldsymbol{X}$. When we call the corresponding **Scikit-Learn** function we need thus to set the intercept to **False**. Libraries like **Scikit-Learn** normally scale the design matrix and does not fit intercept. See the discussions below.\n",
|
||||
"\n",
|
||||
"## More on Rescaling data\n",
|
||||
"\n",
|
||||
"We end this chapter by adding some words on scaling and how to deal with the intercept for regression cases.\n",
|
||||
|
||||
File diff suppressed because one or more lines are too long
Reference in New Issue
Block a user