lars/FYS-STK4155
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update chapter 3

Browse Source
This commit is contained in:
Morten Hjorth-Jensen
2021-09-17 08:29:24 +02:00
parent 4b8b3f6da5
commit 5371b6a4ce
27 changed files with 552 additions and 98 deletions
Download Patch File Download Diff File Expand all files Collapse all files
doc/BookChapters/chapter3.do.txt
+58
View File
@@ -504,6 +504,63 @@ finite $m$, it is not always possible to find a closed form /analytic expression
$\tilde{p}(x)$.
The following code starts with a Gaussian distribution with mean value
$\mu =100$ and variance $\sigma=15$. We use this to generate the data
used in the bootstrap analysis. The bootstrap analysis returns a data
set after a given number of bootstrap operations (as many as we have
data points). This data set consists of estimated mean values for each
bootstrap operation. The histogram generated by the bootstrap method
shows that the distribution for these mean values is also a Gaussian,
centered around the mean value $\mu=100$ but with standard deviation
$\sigma/\sqrt{n}$, where $n$ is the number of bootstrap samples (in
this case the same as the number of original data points). The value
of the standard deviation is what we expect from the central limit
theorem.
!bc pycod
import numpy as np
from time import time
from scipy.stats import norm
import matplotlib.pyplot as plt
# Returns mean of bootstrap samples
# Bootstrap algorithm
def bootstrap(data, datapoints):
t = np.zeros(datapoints)
n = len(data)
# non-parametric bootstrap
for i in range(datapoints):
t[i] = np.mean(data[np.random.randint(0,n,n)])
# analysis
print("Bootstrap Statistics :")
print("original bias std. error")
print("%8g %8g %14g %15g" % (np.mean(data), np.std(data),np.mean(t),np.std(t)))
return t
# We set the mean value to 100 and the standard deviation to 15
mu, sigma = 100, 15
datapoints = 10000
# We generate random numbers according to the normal distribution
x = mu + sigma*np.random.randn(datapoints)
# bootstrap returns the data sample
t = bootstrap(x, datapoints)
!ec
We see that our new variance and from that the standard deviation, agrees with the central limit theorem.
We plot then the histogram together with a best fit for the data set.
!bc pycod
# the histogram of the bootstrapped data (normalized data if density = True)
n, binsboot, patches = plt.hist(t, 50, density=True, facecolor='red', alpha=0.75)
# add a 'best fit' line
y = norm.pdf(binsboot, np.mean(t), np.std(t))
lt = plt.plot(binsboot, y, 'b', linewidth=1)
plt.xlabel('x')
plt.ylabel('Probability')
plt.grid(True)
plt.show()
!ec
@@ -1479,3 +1536,4 @@ and discuss the applicability of these regression methods to the type
of data presented here (either the terrain data we propose or other data sets).
doc/LectureNotes/_build/.doctrees/chapter3.doctree
BIN
View File
Binary file not shown.
doc/LectureNotes/_build/.doctrees/environment.pickle
BIN
View File
Binary file not shown.
doc/LectureNotes/_build/html/_images/chapter3_47_0.png
BIN
View File
Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 14 KiB

After

Width:  |  Height:  |  Size: 12 KiB

doc/LectureNotes/_build/html/_images/chapter3_61_1.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 23 KiB

doc/LectureNotes/_build/html/_images/chapter3_62_6.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 16 KiB

doc/LectureNotes/_build/html/_images/chapter3_64_1.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 37 KiB

doc/LectureNotes/_build/html/_images/chapter3_65_11.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 16 KiB

doc/LectureNotes/_build/html/_images/chapter3_71_0.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 14 KiB

doc/LectureNotes/_build/html/_images/chapter3_73_1.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 12 KiB

doc/LectureNotes/_build/html/_images/chapter3_74_0.png
BIN
View File
Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 32 KiB

After

Width:  |  Height:  |  Size: 10 KiB

doc/LectureNotes/_build/html/_images/chapter3_78_0.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 32 KiB

doc/LectureNotes/_build/html/_sources/chapter3.ipynb
+82 -3
View File
@@ -735,11 +735,92 @@
"$\\tilde{p}(x)$.\n",
"\n",
"\n",
"The following code starts with a Gaussian distribution with mean value\n",
"$\\mu =100$ and variance $\\sigma=15$. We use this to generate the data\n",
"used in the bootstrap analysis. The bootstrap analysis returns a data\n",
"set after a given number of bootstrap operations (as many as we have\n",
"data points). This data set consists of estimated mean values for each\n",
"bootstrap operation. The histogram generated by the bootstrap method\n",
"shows that the distribution for these mean values is also a Gaussian,\n",
"centered around the mean value $\\mu=100$ but with standard deviation\n",
"$\\sigma/\\sqrt{n}$, where $n$ is the number of bootstrap samples (in\n",
"this case the same as the number of original data points). The value\n",
"of the standard deviation is what we expect from the central limit\n",
"theorem."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false,
"editable": true
},
"outputs": [],
"source": [
"%matplotlib inline\n",
"\n",
"import numpy as np\n",
"from time import time\n",
"from scipy.stats import norm\n",
"import matplotlib.pyplot as plt\n",
"\n",
"# Returns mean of bootstrap samples \n",
"# Bootstrap algorithm\n",
"def bootstrap(data, datapoints):\n",
" t = np.zeros(datapoints)\n",
" n = len(data)\n",
" # non-parametric bootstrap \n",
" for i in range(datapoints):\n",
" t[i] = np.mean(data[np.random.randint(0,n,n)])\n",
" # analysis \n",
" print(\"Bootstrap Statistics :\")\n",
" print(\"original bias std. error\")\n",
" print(\"%8g %8g %14g %15g\" % (np.mean(data), np.std(data),np.mean(t),np.std(t)))\n",
" return t\n",
"\n",
"# We set the mean value to 100 and the standard deviation to 15\n",
"mu, sigma = 100, 15\n",
"datapoints = 10000\n",
"# We generate random numbers according to the normal distribution\n",
"x = mu + sigma*np.random.randn(datapoints)\n",
"# bootstrap returns the data sample \n",
"t = bootstrap(x, datapoints)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"We see that our new variance and from that the standard deviation, agrees with the central limit theorem.\n",
"\n",
"\n",
"We plot then the histogram together with a best fit for the data set."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false,
"editable": true
},
"outputs": [],
"source": [
"# the histogram of the bootstrapped data (normalized data if density = True)\n",
"n, binsboot, patches = plt.hist(t, 50, density=True, facecolor='red', alpha=0.75)\n",
"# add a 'best fit' line \n",
"y = norm.pdf(binsboot, np.mean(t), np.std(t))\n",
"lt = plt.plot(binsboot, y, 'b', linewidth=1)\n",
"plt.xlabel('x')\n",
"plt.ylabel('Probability')\n",
"plt.grid(True)\n",
"plt.show()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## The bias-variance tradeoff\n",
"\n",
"\n",
@@ -871,8 +952,6 @@
},
"outputs": [],
"source": [
"%matplotlib inline\n",
"\n",
"import matplotlib.pyplot as plt\n",
"import numpy as np\n",
"from sklearn.linear_model import LinearRegression, Ridge, Lasso\n",
doc/LectureNotes/_build/html/chapter3.html
+109 -34
View File
@@ -630,10 +630,10 @@ number <span class="math notranslate nohighlight">\(i\)</span> is left out. Usin
</div>
</div>
<div class="cell_output docutils container">
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Runtime: 0.141725 sec
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Runtime: 0.13859 sec
Jackknife Statistics :
original bias std. error
100.148 100.138 0.1492
99.9054 99.8954 0.149328
</pre></div>
</div>
</div>
@@ -804,6 +804,79 @@ correlations.</p>
<p>The theorem is satisfied by a large class of PDFs. Note however that for a
finite <span class="math notranslate nohighlight">\(m\)</span>, it is not always possible to find a closed form /analytic expression for
<span class="math notranslate nohighlight">\(\tilde{p}(x)\)</span>.</p>
<p>The following code starts with a Gaussian distribution with mean value
<span class="math notranslate nohighlight">\(\mu =100\)</span> and variance <span class="math notranslate nohighlight">\(\sigma=15\)</span>. We use this to generate the data
used in the bootstrap analysis. The bootstrap analysis returns a data
set after a given number of bootstrap operations (as many as we have
data points). This data set consists of estimated mean values for each
bootstrap operation. The histogram generated by the bootstrap method
shows that the distribution for these mean values is also a Gaussian,
centered around the mean value <span class="math notranslate nohighlight">\(\mu=100\)</span> but with standard deviation
<span class="math notranslate nohighlight">\(\sigma/\sqrt{n}\)</span>, where <span class="math notranslate nohighlight">\(n\)</span> is the number of bootstrap samples (in
this case the same as the number of original data points). The value
of the standard deviation is what we expect from the central limit
theorem.</p>
<div class="cell docutils container">
<div class="cell_input docutils container">
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="o">%</span><span class="k">matplotlib</span> inline
<span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
<span class="kn">from</span> <span class="nn">time</span> <span class="kn">import</span> <span class="n">time</span>
<span class="kn">from</span> <span class="nn">scipy.stats</span> <span class="kn">import</span> <span class="n">norm</span>
<span class="kn">import</span> <span class="nn">matplotlib.pyplot</span> <span class="k">as</span> <span class="nn">plt</span>
<span class="c1"># Returns mean of bootstrap samples </span>
<span class="c1"># Bootstrap algorithm</span>
<span class="k">def</span> <span class="nf">bootstrap</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="n">datapoints</span><span class="p">):</span>
<span class="n">t</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros</span><span class="p">(</span><span class="n">datapoints</span><span class="p">)</span>
<span class="n">n</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">data</span><span class="p">)</span>
<span class="c1"># non-parametric bootstrap </span>
<span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">datapoints</span><span class="p">):</span>
<span class="n">t</span><span class="p">[</span><span class="n">i</span><span class="p">]</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">data</span><span class="p">[</span><span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">randint</span><span class="p">(</span><span class="mi">0</span><span class="p">,</span><span class="n">n</span><span class="p">,</span><span class="n">n</span><span class="p">)])</span>
<span class="c1"># analysis </span>
<span class="nb">print</span><span class="p">(</span><span class="s2">&quot;Bootstrap Statistics :&quot;</span><span class="p">)</span>
<span class="nb">print</span><span class="p">(</span><span class="s2">&quot;original bias std. error&quot;</span><span class="p">)</span>
<span class="nb">print</span><span class="p">(</span><span class="s2">&quot;</span><span class="si">%8g</span><span class="s2"> </span><span class="si">%8g</span><span class="s2"> </span><span class="si">%14g</span><span class="s2"> </span><span class="si">%15g</span><span class="s2">&quot;</span> <span class="o">%</span> <span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">data</span><span class="p">),</span> <span class="n">np</span><span class="o">.</span><span class="n">std</span><span class="p">(</span><span class="n">data</span><span class="p">),</span><span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">t</span><span class="p">),</span><span class="n">np</span><span class="o">.</span><span class="n">std</span><span class="p">(</span><span class="n">t</span><span class="p">)))</span>
<span class="k">return</span> <span class="n">t</span>
<span class="c1"># We set the mean value to 100 and the standard deviation to 15</span>
<span class="n">mu</span><span class="p">,</span> <span class="n">sigma</span> <span class="o">=</span> <span class="mi">100</span><span class="p">,</span> <span class="mi">15</span>
<span class="n">datapoints</span> <span class="o">=</span> <span class="mi">10000</span>
<span class="c1"># We generate random numbers according to the normal distribution</span>
<span class="n">x</span> <span class="o">=</span> <span class="n">mu</span> <span class="o">+</span> <span class="n">sigma</span><span class="o">*</span><span class="n">np</span><span class="o">.</span><span class="n">random</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="n">datapoints</span><span class="p">)</span>
<span class="c1"># bootstrap returns the data sample </span>
<span class="n">t</span> <span class="o">=</span> <span class="n">bootstrap</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="n">datapoints</span><span class="p">)</span>
</pre></div>
</div>
</div>
<div class="cell_output docutils container">
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Bootstrap Statistics :
original bias std. error
99.7442 15.0015 99.7465 0.148127
</pre></div>
</div>
</div>
</div>
<p>We see that our new variance and from that the standard deviation, agrees with the central limit theorem.</p>
<p>We plot then the histogram together with a best fit for the data set.</p>
<div class="cell docutils container">
<div class="cell_input docutils container">
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="c1"># the histogram of the bootstrapped data (normalized data if density = True)</span>
<span class="n">n</span><span class="p">,</span> <span class="n">binsboot</span><span class="p">,</span> <span class="n">patches</span> <span class="o">=</span> <span class="n">plt</span><span class="o">.</span><span class="n">hist</span><span class="p">(</span><span class="n">t</span><span class="p">,</span> <span class="mi">50</span><span class="p">,</span> <span class="n">density</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">facecolor</span><span class="o">=</span><span class="s1">&#39;red&#39;</span><span class="p">,</span> <span class="n">alpha</span><span class="o">=</span><span class="mf">0.75</span><span class="p">)</span>
<span class="c1"># add a &#39;best fit&#39; line </span>
<span class="n">y</span> <span class="o">=</span> <span class="n">norm</span><span class="o">.</span><span class="n">pdf</span><span class="p">(</span><span class="n">binsboot</span><span class="p">,</span> <span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">t</span><span class="p">),</span> <span class="n">np</span><span class="o">.</span><span class="n">std</span><span class="p">(</span><span class="n">t</span><span class="p">))</span>
<span class="n">lt</span> <span class="o">=</span> <span class="n">plt</span><span class="o">.</span><span class="n">plot</span><span class="p">(</span><span class="n">binsboot</span><span class="p">,</span> <span class="n">y</span><span class="p">,</span> <span class="s1">&#39;b&#39;</span><span class="p">,</span> <span class="n">linewidth</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
<span class="n">plt</span><span class="o">.</span><span class="n">xlabel</span><span class="p">(</span><span class="s1">&#39;x&#39;</span><span class="p">)</span>
<span class="n">plt</span><span class="o">.</span><span class="n">ylabel</span><span class="p">(</span><span class="s1">&#39;Probability&#39;</span><span class="p">)</span>
<span class="n">plt</span><span class="o">.</span><span class="n">grid</span><span class="p">(</span><span class="kc">True</span><span class="p">)</span>
<span class="n">plt</span><span class="o">.</span><span class="n">show</span><span class="p">()</span>
</pre></div>
</div>
</div>
<div class="cell_output docutils container">
<img alt="_images/chapter3_47_0.png" src="_images/chapter3_47_0.png" />
</div>
</div>
</div>
</div>
<div class="section" id="the-bias-variance-tradeoff">
@@ -857,9 +930,7 @@ We use a more compact notation in terms of the expectation value</p>
<p>that is the rewriting in terms of the so-called bias, the variance of the model <span class="math notranslate nohighlight">\(\boldsymbol{\tilde{y}}\)</span> and the variance of <span class="math notranslate nohighlight">\(\boldsymbol{\epsilon}\)</span>.</p>
<div class="cell docutils container">
<div class="cell_input docutils container">
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="o">%</span><span class="k">matplotlib</span> inline
<span class="kn">import</span> <span class="nn">matplotlib.pyplot</span> <span class="k">as</span> <span class="nn">plt</span>
<div class="highlight-ipython3 notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">matplotlib.pyplot</span> <span class="k">as</span> <span class="nn">plt</span>
<span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
<span class="kn">from</span> <span class="nn">sklearn.linear_model</span> <span class="kn">import</span> <span class="n">LinearRegression</span><span class="p">,</span> <span class="n">Ridge</span><span class="p">,</span> <span class="n">Lasso</span>
<span class="kn">from</span> <span class="nn">sklearn.preprocessing</span> <span class="kn">import</span> <span class="n">PolynomialFeatures</span>
@@ -923,7 +994,7 @@ Var: 0.0010479246398391328
0.01312157412031145 &gt;= 0.012073649480472317 + 0.0010479246398391328 = 0.01312157412031145
</pre></div>
</div>
<img alt="_images/chapter3_57_1.png" src="_images/chapter3_57_1.png" />
<img alt="_images/chapter3_61_1.png" src="_images/chapter3_61_1.png" />
</div>
</div>
<div class="cell docutils container">
@@ -983,14 +1054,15 @@ Error: 0.32149601703519126
Bias^2: 0.3123314713548606
Var: 0.009164545680330616
0.32149601703519126 &gt;= 0.3123314713548606 + 0.009164545680330616 = 0.3214960170351912
Polynomial degree: 1
Polynomial degree:
</pre></div>
</div>
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span> 1
Error: 0.08426840630693411
Bias^2: 0.07968918676726028
Var: 0.004579219539673833
0.08426840630693411 &gt;= 0.07968918676726028 + 0.004579219539673833 = 0.08426840630693411
</pre></div>
</div>
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Polynomial degree: 2
Polynomial degree: 2
Error: 0.10398646080125035
Bias^2: 0.10077114273548986
Var: 0.0032153180657605086
@@ -1010,26 +1082,26 @@ Error: 0.05227921801205707
Bias^2: 0.048187277304303125
Var: 0.004091940707753964
0.05227921801205707 &gt;= 0.048187277304303125 + 0.004091940707753964 = 0.05227921801205709
Polynomial degree: 6
</pre></div>
</div>
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Polynomial degree: 6
Error: 0.03781367141738898
Bias^2: 0.03365768507152761
Var: 0.004155986345861379
0.03781367141738898 &gt;= 0.03365768507152761 + 0.004155986345861379 = 0.03781367141738899
</pre></div>
</div>
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Polynomial degree: 7
Polynomial degree: 7
Error: 0.027609773491022498
Bias^2: 0.02299949826036597
Var: 0.004610275230656537
0.027609773491022498 &gt;= 0.02299949826036597 + 0.004610275230656537 = 0.027609773491022505
Polynomial degree: 8
</pre></div>
</div>
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Polynomial degree: 8
Error: 0.017355848195591973
Bias^2: 0.010331721306655588
Var: 0.007024126888936384
0.017355848195591973 &gt;= 0.010331721306655588 + 0.007024126888936384 = 0.017355848195591973
</pre></div>
</div>
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Polynomial degree: 9
Polynomial degree: 9
Error: 0.026605727637189085
Bias^2: 0.010018312644140933
Var: 0.016587414993048166
@@ -1044,21 +1116,24 @@ Error: 0.07160048164228314
Bias^2: 0.01443680008897583
Var: 0.0571636815533073
0.07160048164228314 &gt;= 0.01443680008897583 + 0.0571636815533073 = 0.07160048164228312
Polynomial degree: 12
</pre></div>
</div>
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Polynomial degree: 12
Error: 0.1154777721897675
Bias^2: 0.01628578269590588
Var: 0.09919198949386163
0.1154777721897675 &gt;= 0.01628578269590588 + 0.09919198949386163 = 0.11547777218976751
Polynomial degree:
</pre></div>
</div>
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Polynomial degree: 13
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span> 13
Error: 0.22842468702166951
Bias^2: 0.01975416527163567
Var: 0.20867052175003387
0.22842468702166951 &gt;= 0.01975416527163567 + 0.20867052175003387 = 0.22842468702166954
</pre></div>
</div>
<img alt="_images/chapter3_58_5.png" src="_images/chapter3_58_5.png" />
<img alt="_images/chapter3_62_6.png" src="_images/chapter3_62_6.png" />
</div>
</div>
<p>The bias-variance tradeoff summarizes the fundamental tension in
@@ -1183,7 +1258,7 @@ set, the higher, the less likely the model generalizes correctly from the
training data.
</pre></div>
</div>
<img alt="_images/chapter3_60_1.png" src="_images/chapter3_60_1.png" />
<img alt="_images/chapter3_64_1.png" src="_images/chapter3_64_1.png" />
</div>
</div>
<div class="cell docutils container">
@@ -1290,11 +1365,11 @@ Mean squared error on test data: 5.98822371
Degree of polynomial: 6
Mean squared error on training data: 3.66204648
Mean squared error on test data: 8.14812206
Degree of polynomial: 7
Mean squared error on training data: 0.47075725
</pre></div>
</div>
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Degree of polynomial: 7
Mean squared error on training data: 0.47075725
Mean squared error on test data: 2.00607783
<div class="output stream highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>Mean squared error on test data: 2.00607783
Degree of polynomial: 8
Mean squared error on training data: 0.04912436
Mean squared error on test data: 0.21596432
@@ -1377,13 +1452,13 @@ Mean squared error on training data: 0.00060728
Mean squared error on test data: 2988.64001211
</pre></div>
</div>
<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>&lt;ipython-input-5-8dc29df57a8c&gt;:73: RuntimeWarning: divide by zero encountered in log10
<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>&lt;ipython-input-7-8dc29df57a8c&gt;:73: RuntimeWarning: divide by zero encountered in log10
plt.plot(polynomial, np.log10(trainingerror), label=&#39;Training Error&#39;)
&lt;ipython-input-5-8dc29df57a8c&gt;:74: RuntimeWarning: divide by zero encountered in log10
&lt;ipython-input-7-8dc29df57a8c&gt;:74: RuntimeWarning: divide by zero encountered in log10
plt.plot(polynomial, np.log10(testerror), label=&#39;Test Error&#39;)
</pre></div>
</div>
<img alt="_images/chapter3_61_11.png" src="_images/chapter3_61_11.png" />
<img alt="_images/chapter3_65_11.png" src="_images/chapter3_65_11.png" />
</div>
</div>
</div>
@@ -1536,7 +1611,7 @@ cross-validation (LOOCV).</p>
</div>
</div>
<div class="cell_output docutils container">
<img alt="_images/chapter3_67_0.png" src="_images/chapter3_67_0.png" />
<img alt="_images/chapter3_71_0.png" src="_images/chapter3_71_0.png" />
</div>
</div>
<p>More examples of the application of cross-validation follow here.</p>
@@ -1613,11 +1688,11 @@ cross-validation (LOOCV).</p>
</div>
</div>
<div class="cell_output docutils container">
<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>&lt;ipython-input-7-49b0ef2e51e2&gt;:63: RuntimeWarning: divide by zero encountered in log10
<div class="output stderr highlight-myst-ansi notranslate"><div class="highlight"><pre><span></span>&lt;ipython-input-9-49b0ef2e51e2&gt;:63: RuntimeWarning: divide by zero encountered in log10
plt.plot(polynomial, np.log10(estimated_mse_sklearn), label=&#39;Test Error&#39;)
</pre></div>
</div>
<img alt="_images/chapter3_69_1.png" src="_images/chapter3_69_1.png" />
<img alt="_images/chapter3_73_1.png" src="_images/chapter3_73_1.png" />
</div>
</div>
<div class="cell docutils container">
@@ -1661,7 +1736,7 @@ cross-validation (LOOCV).</p>
</div>
</div>
<div class="cell_output docutils container">
<img alt="_images/chapter3_70_0.png" src="_images/chapter3_70_0.png" />
<img alt="_images/chapter3_74_0.png" src="_images/chapter3_74_0.png" />
</div>
</div>
</div>
@@ -1750,7 +1825,7 @@ which polynomial fits the data best.</p>
</div>
</div>
<div class="cell_output docutils container">
<img alt="_images/chapter3_74_0.png" src="_images/chapter3_74_0.png" />
<img alt="_images/chapter3_78_0.png" src="_images/chapter3_78_0.png" />
</div>
</div>
<div class="section" id="exercise-ordinary-least-square-ols-on-the-franke-function">
@@ -1907,7 +1982,7 @@ Python program using</p>
<div class="cell_output docutils container">
<div class="output traceback highlight-ipythontb notranslate"><div class="highlight"><pre><span></span><span class="gt">---------------------------------------------------------------------------</span>
<span class="ne">NameError</span><span class="g g-Whitespace"> </span>Traceback (most recent call last)
<span class="o">&lt;</span><span class="n">ipython</span><span class="o">-</span><span class="nb">input</span><span class="o">-</span><span class="mi">10</span><span class="o">-</span><span class="n">d985fb40c43d</span><span class="o">&gt;</span> <span class="ow">in</span> <span class="o">&lt;</span><span class="n">module</span><span class="o">&gt;</span>
<span class="o">&lt;</span><span class="n">ipython</span><span class="o">-</span><span class="nb">input</span><span class="o">-</span><span class="mi">12</span><span class="o">-</span><span class="n">d985fb40c43d</span><span class="o">&gt;</span> <span class="ow">in</span> <span class="o">&lt;</span><span class="n">module</span><span class="o">&gt;</span>
<span class="ne">----&gt; </span><span class="mi">1</span> <span class="n">scipy</span><span class="o">.</span><span class="n">misc</span><span class="o">.</span><span class="n">imread</span>
<span class="ne">NameError</span>: name &#39;scipy&#39; is not defined
doc/LectureNotes/_build/html/searchindex.js
+1 -1
View File
File diff suppressed because one or more lines are too long
doc/LectureNotes/_build/jupyter_execute/chapter3.ipynb
+169 -55
View File
File diff suppressed because one or more lines are too long
doc/LectureNotes/_build/jupyter_execute/chapter3.py
+51 -2
View File
@@ -504,10 +504,61 @@ finite $m$, it is not always possible to find a closed form /analytic expression
$\tilde{p}(x)$.
The following code starts with a Gaussian distribution with mean value
$\mu =100$ and variance $\sigma=15$. We use this to generate the data
used in the bootstrap analysis. The bootstrap analysis returns a data
set after a given number of bootstrap operations (as many as we have
data points). This data set consists of estimated mean values for each
bootstrap operation. The histogram generated by the bootstrap method
shows that the distribution for these mean values is also a Gaussian,
centered around the mean value $\mu=100$ but with standard deviation
$\sigma/\sqrt{n}$, where $n$ is the number of bootstrap samples (in
this case the same as the number of original data points). The value
of the standard deviation is what we expect from the central limit
theorem.
%matplotlib inline
import numpy as np
from time import time
from scipy.stats import norm
import matplotlib.pyplot as plt
# Returns mean of bootstrap samples
# Bootstrap algorithm
def bootstrap(data, datapoints):
t = np.zeros(datapoints)
n = len(data)
# non-parametric bootstrap
for i in range(datapoints):
t[i] = np.mean(data[np.random.randint(0,n,n)])
# analysis
print("Bootstrap Statistics :")
print("original bias std. error")
print("%8g %8g %14g %15g" % (np.mean(data), np.std(data),np.mean(t),np.std(t)))
return t
# We set the mean value to 100 and the standard deviation to 15
mu, sigma = 100, 15
datapoints = 10000
# We generate random numbers according to the normal distribution
x = mu + sigma*np.random.randn(datapoints)
# bootstrap returns the data sample
t = bootstrap(x, datapoints)
We see that our new variance and from that the standard deviation, agrees with the central limit theorem.
We plot then the histogram together with a best fit for the data set.
# the histogram of the bootstrapped data (normalized data if density = True)
n, binsboot, patches = plt.hist(t, 50, density=True, facecolor='red', alpha=0.75)
# add a 'best fit' line
y = norm.pdf(binsboot, np.mean(t), np.std(t))
lt = plt.plot(binsboot, y, 'b', linewidth=1)
plt.xlabel('x')
plt.ylabel('Probability')
plt.grid(True)
plt.show()
## The bias-variance tradeoff
@@ -570,8 +621,6 @@ $$
that is the rewriting in terms of the so-called bias, the variance of the model $\boldsymbol{\tilde{y}}$ and the variance of $\boldsymbol{\epsilon}$.
%matplotlib inline
import matplotlib.pyplot as plt
import numpy as np
from sklearn.linear_model import LinearRegression, Ridge, Lasso
doc/LectureNotes/_build/jupyter_execute/chapter3_47_0.png
BIN
View File
Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 14 KiB

After

Width:  |  Height:  |  Size: 12 KiB

doc/LectureNotes/_build/jupyter_execute/chapter3_61_1.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 23 KiB

doc/LectureNotes/_build/jupyter_execute/chapter3_62_6.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 16 KiB

doc/LectureNotes/_build/jupyter_execute/chapter3_64_1.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 37 KiB

doc/LectureNotes/_build/jupyter_execute/chapter3_65_11.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 16 KiB

doc/LectureNotes/_build/jupyter_execute/chapter3_71_0.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 14 KiB

doc/LectureNotes/_build/jupyter_execute/chapter3_73_1.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 12 KiB

doc/LectureNotes/_build/jupyter_execute/chapter3_74_0.png
BIN
View File
Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 32 KiB

After

Width:  |  Height:  |  Size: 10 KiB

doc/LectureNotes/_build/jupyter_execute/chapter3_78_0.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 32 KiB

doc/LectureNotes/chapter3.ipynb
+82 -3
View File
@@ -735,11 +735,92 @@
"$\\tilde{p}(x)$.\n",
"\n",
"\n",
"The following code starts with a Gaussian distribution with mean value\n",
"$\\mu =100$ and variance $\\sigma=15$. We use this to generate the data\n",
"used in the bootstrap analysis. The bootstrap analysis returns a data\n",
"set after a given number of bootstrap operations (as many as we have\n",
"data points). This data set consists of estimated mean values for each\n",
"bootstrap operation. The histogram generated by the bootstrap method\n",
"shows that the distribution for these mean values is also a Gaussian,\n",
"centered around the mean value $\\mu=100$ but with standard deviation\n",
"$\\sigma/\\sqrt{n}$, where $n$ is the number of bootstrap samples (in\n",
"this case the same as the number of original data points). The value\n",
"of the standard deviation is what we expect from the central limit\n",
"theorem."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false,
"editable": true
},
"outputs": [],
"source": [
"%matplotlib inline\n",
"\n",
"import numpy as np\n",
"from time import time\n",
"from scipy.stats import norm\n",
"import matplotlib.pyplot as plt\n",
"\n",
"# Returns mean of bootstrap samples \n",
"# Bootstrap algorithm\n",
"def bootstrap(data, datapoints):\n",
" t = np.zeros(datapoints)\n",
" n = len(data)\n",
" # non-parametric bootstrap \n",
" for i in range(datapoints):\n",
" t[i] = np.mean(data[np.random.randint(0,n,n)])\n",
" # analysis \n",
" print(\"Bootstrap Statistics :\")\n",
" print(\"original bias std. error\")\n",
" print(\"%8g %8g %14g %15g\" % (np.mean(data), np.std(data),np.mean(t),np.std(t)))\n",
" return t\n",
"\n",
"# We set the mean value to 100 and the standard deviation to 15\n",
"mu, sigma = 100, 15\n",
"datapoints = 10000\n",
"# We generate random numbers according to the normal distribution\n",
"x = mu + sigma*np.random.randn(datapoints)\n",
"# bootstrap returns the data sample \n",
"t = bootstrap(x, datapoints)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"We see that our new variance and from that the standard deviation, agrees with the central limit theorem.\n",
"\n",
"\n",
"We plot then the histogram together with a best fit for the data set."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"collapsed": false,
"editable": true
},
"outputs": [],
"source": [
"# the histogram of the bootstrapped data (normalized data if density = True)\n",
"n, binsboot, patches = plt.hist(t, 50, density=True, facecolor='red', alpha=0.75)\n",
"# add a 'best fit' line \n",
"y = norm.pdf(binsboot, np.mean(t), np.std(t))\n",
"lt = plt.plot(binsboot, y, 'b', linewidth=1)\n",
"plt.xlabel('x')\n",
"plt.ylabel('Probability')\n",
"plt.grid(True)\n",
"plt.show()"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## The bias-variance tradeoff\n",
"\n",
"\n",
@@ -871,8 +952,6 @@
},
"outputs": [],
"source": [
"%matplotlib inline\n",
"\n",
"import matplotlib.pyplot as plt\n",
"import numpy as np\n",
"from sklearn.linear_model import LinearRegression, Ridge, Lasso\n",