lars/FYS-STK4155
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update week 45

Browse Source
This commit is contained in:
mhjensen
2021-11-10 16:37:21 +01:00
parent 902ce81b35
commit 0e5076dfbf
79 changed files with 5458 additions and 7367 deletions
Download Patch File Download Diff File Expand all files Collapse all files
doc/pub/week45/html/._week45-bs066.html
+108 -84
View File
@@ -101,81 +101,74 @@ Automatically generated HTML file from DocOnce source
('An Overview of Ensemble Methods', 2, None, '___sec37'),
('Bagging', 2, None, '___sec38'),
('More bagging', 2, None, '___sec39'),
('Simple Voting Example, head or tail', 2, None, '___sec40'),
('Using the Voting Classifier', 2, None, '___sec41'),
('Please, not the moons again! Voting and Bagging',
2,
None,
'___sec42'),
('Bagging Examples', 2, None, '___sec43'),
('Making your own Bootstrap: Changing the Level of the Decision '
'Tree',
2,
None,
'___sec44'),
('Why Voting?', 2, None, '___sec45'),
('Tossing coins', 2, None, '___sec46'),
('Standard imports first', 2, None, '___sec47'),
('Simple Voting Example, head or tail', 2, None, '___sec48'),
('Using the Voting Classifier', 2, None, '___sec49'),
('Voting and Bagging', 2, None, '___sec50'),
('Random forests', 2, None, '___sec51'),
('Random Forest Algorithm', 2, None, '___sec52'),
'___sec40'),
('Why Voting?', 2, None, '___sec41'),
('Tossing coins', 2, None, '___sec42'),
('Standard imports first', 2, None, '___sec43'),
('Simple Voting Example, head or tail', 2, None, '___sec44'),
('Using the Voting Classifier', 2, None, '___sec45'),
('Voting and Bagging', 2, None, '___sec46'),
('Random forests', 2, None, '___sec47'),
('Random Forest Algorithm', 2, None, '___sec48'),
('Random Forests Compared with other Methods on the Cancer Data',
2,
None,
'___sec53'),
'___sec49'),
('Compare Bagging on Trees with Random Forests',
2,
None,
'___sec54'),
("Boosting, a Bird's Eye View", 2, None, '___sec55'),
'___sec50'),
("Boosting, a Bird's Eye View", 2, None, '___sec51'),
('What is boosting? Additive Modelling/Iterative Fitting',
2,
None,
'___sec56'),
'___sec52'),
('Iterative Fitting, Regression and Squared-error Cost Function',
2,
None,
'___sec57'),
'___sec53'),
('Squared-Error Example and Iterative Fitting',
2,
None,
'___sec58'),
'___sec54'),
('Iterative Fitting, Classification and AdaBoost',
2,
None,
'___sec59'),
('Adaptive Boosting, AdaBoost', 2, None, '___sec60'),
('Building up AdaBoost', 2, None, '___sec61'),
'___sec55'),
('Adaptive Boosting, AdaBoost', 2, None, '___sec56'),
('Building up AdaBoost', 2, None, '___sec57'),
('Adaptive boosting: AdaBoost, Basic Algorithm',
2,
None,
'___sec62'),
('Basic Steps of AdaBoost', 2, None, '___sec63'),
('AdaBoost Examples', 2, None, '___sec64'),
'___sec58'),
('Basic Steps of AdaBoost', 2, None, '___sec59'),
('AdaBoost Examples', 2, None, '___sec60'),
('Gradient boosting: Basics with Steepest Descent/Functional '
'Gradient Descent',
2,
None,
'___sec65'),
'___sec61'),
('The Squared-Error again! Steepest Descent',
2,
None,
'___sec66'),
('Steepest Descent Example', 2, None, '___sec67'),
('Gradient Boosting, algorithm', 2, None, '___sec68'),
'___sec62'),
('Steepest Descent Example', 2, None, '___sec63'),
('Gradient Boosting, algorithm', 2, None, '___sec64'),
('Gradient Boosting, Examples of Regression',
2,
None,
'___sec69'),
'___sec65'),
('Gradient Boosting, Classification Example',
2,
None,
'___sec70'),
('XGBoost: Extreme Gradient Boosting', 2, None, '___sec71'),
('Regression Case', 2, None, '___sec72'),
('Xgboost on the Cancer Data', 2, None, '___sec73')]}
'___sec66'),
('XGBoost: Extreme Gradient Boosting', 2, None, '___sec67'),
('Regression Case', 2, None, '___sec68'),
('Xgboost on the Cancer Data', 2, None, '___sec69')]}
end of tocinfo -->
<body>
@@ -253,40 +246,36 @@ MathJax.Hub.Config({
<!-- navigation toc: --> <li><a href="._week45-bs038.html#___sec37" style="font-size: 80%;">An Overview of Ensemble Methods</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs039.html#___sec38" style="font-size: 80%;">Bagging</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs040.html#___sec39" style="font-size: 80%;">More bagging</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs041.html#___sec40" style="font-size: 80%;">Simple Voting Example, head or tail</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs042.html#___sec41" style="font-size: 80%;">Using the Voting Classifier</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs043.html#___sec42" style="font-size: 80%;">Please, not the moons again! Voting and Bagging</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs044.html#___sec43" style="font-size: 80%;">Bagging Examples</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs045.html#___sec44" style="font-size: 80%;">Making your own Bootstrap: Changing the Level of the Decision Tree</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs046.html#___sec45" style="font-size: 80%;">Why Voting?</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs047.html#___sec46" style="font-size: 80%;">Tossing coins</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs048.html#___sec47" style="font-size: 80%;">Standard imports first</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs049.html#___sec48" style="font-size: 80%;">Simple Voting Example, head or tail</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs050.html#___sec49" style="font-size: 80%;">Using the Voting Classifier</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs051.html#___sec50" style="font-size: 80%;">Voting and Bagging</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs052.html#___sec51" style="font-size: 80%;">Random forests</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs053.html#___sec52" style="font-size: 80%;">Random Forest Algorithm</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs054.html#___sec53" style="font-size: 80%;">Random Forests Compared with other Methods on the Cancer Data</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs055.html#___sec54" style="font-size: 80%;">Compare Bagging on Trees with Random Forests</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs056.html#___sec55" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs057.html#___sec56" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs058.html#___sec57" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs059.html#___sec58" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs060.html#___sec59" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs061.html#___sec60" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs062.html#___sec61" style="font-size: 80%;">Building up AdaBoost</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs063.html#___sec62" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs064.html#___sec63" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs065.html#___sec64" style="font-size: 80%;">AdaBoost Examples</a></li>
<!-- navigation toc: --> <li><a href="#___sec65" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs067.html#___sec66" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs068.html#___sec67" style="font-size: 80%;">Steepest Descent Example</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs069.html#___sec68" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs070.html#___sec69" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs071.html#___sec70" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs072.html#___sec71" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs073.html#___sec72" style="font-size: 80%;">Regression Case</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs074.html#___sec73" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs041.html#___sec40" style="font-size: 80%;">Making your own Bootstrap: Changing the Level of the Decision Tree</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs042.html#___sec41" style="font-size: 80%;">Why Voting?</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs043.html#___sec42" style="font-size: 80%;">Tossing coins</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs044.html#___sec43" style="font-size: 80%;">Standard imports first</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs045.html#___sec44" style="font-size: 80%;">Simple Voting Example, head or tail</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs046.html#___sec45" style="font-size: 80%;">Using the Voting Classifier</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs047.html#___sec46" style="font-size: 80%;">Voting and Bagging</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs048.html#___sec47" style="font-size: 80%;">Random forests</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs049.html#___sec48" style="font-size: 80%;">Random Forest Algorithm</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs050.html#___sec49" style="font-size: 80%;">Random Forests Compared with other Methods on the Cancer Data</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs051.html#___sec50" style="font-size: 80%;">Compare Bagging on Trees with Random Forests</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs052.html#___sec51" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs053.html#___sec52" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs054.html#___sec53" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs055.html#___sec54" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs056.html#___sec55" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs057.html#___sec56" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs058.html#___sec57" style="font-size: 80%;">Building up AdaBoost</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs059.html#___sec58" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs060.html#___sec59" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs061.html#___sec60" style="font-size: 80%;">AdaBoost Examples</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs062.html#___sec61" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs063.html#___sec62" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs064.html#___sec63" style="font-size: 80%;">Steepest Descent Example</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs065.html#___sec64" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
<!-- navigation toc: --> <li><a href="#___sec65" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs067.html#___sec66" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs068.html#___sec67" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs069.html#___sec68" style="font-size: 80%;">Regression Case</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs070.html#___sec69" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
</ul>
</li>
@@ -302,18 +291,57 @@ MathJax.Hub.Config({
<a name="part0066"></a>
<!-- !split -->
<h2 id="___sec65" class="anchor">Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent </h2>
<h2 id="___sec65" class="anchor">Gradient Boosting, Examples of Regression </h2>
<p>
Gradient boosting is again a similar technique to Adaptive boosting,
it combines so-called weak classifiers or regressors into a strong
method via a series of iterations.
<p>
In order to understand the method, let us illustrate its basics by
bringing back the essential steps in linear regression, where our cost
function was the least squares function.
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.model_selection</span> <span style="color: #008000; font-weight: bold">import</span> train_test_split
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.ensemble</span> <span style="color: #008000; font-weight: bold">import</span> GradientBoostingRegressor
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.preprocessing</span> <span style="color: #008000; font-weight: bold">import</span> StandardScaler
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">scikitplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">skplt</span>
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.metrics</span> <span style="color: #008000; font-weight: bold">import</span> mean_squared_error
n <span style="color: #666666">=</span> <span style="color: #666666">100</span>
maxdegree <span style="color: #666666">=</span> <span style="color: #666666">6</span>
<span style="color: #408080; font-style: italic"># Make data set.</span>
x <span style="color: #666666">=</span> np<span style="color: #666666">.</span>linspace(<span style="color: #666666">-3</span>, <span style="color: #666666">3</span>, n)<span style="color: #666666">.</span>reshape(<span style="color: #666666">-1</span>, <span style="color: #666666">1</span>)
y <span style="color: #666666">=</span> np<span style="color: #666666">.</span>exp(<span style="color: #666666">-</span>x<span style="color: #666666">**2</span>) <span style="color: #666666">+</span> <span style="color: #666666">1.5</span> <span style="color: #666666">*</span> np<span style="color: #666666">.</span>exp(<span style="color: #666666">-</span>(x<span style="color: #666666">-2</span>)<span style="color: #666666">**2</span>)<span style="color: #666666">+</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>normal(<span style="color: #666666">0</span>, <span style="color: #666666">0.1</span>, x<span style="color: #666666">.</span>shape)
error <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros(maxdegree)
bias <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros(maxdegree)
variance <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros(maxdegree)
polydegree <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros(maxdegree)
X_train, X_test, y_train, y_test <span style="color: #666666">=</span> train_test_split(x, y, test_size<span style="color: #666666">=0.2</span>)
scaler <span style="color: #666666">=</span> StandardScaler()
scaler<span style="color: #666666">.</span>fit(X_train)
X_train_scaled <span style="color: #666666">=</span> scaler<span style="color: #666666">.</span>transform(X_train)
X_test_scaled <span style="color: #666666">=</span> scaler<span style="color: #666666">.</span>transform(X_test)
<span style="color: #008000; font-weight: bold">for</span> degree <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #666666">1</span>,maxdegree):
model <span style="color: #666666">=</span> GradientBoostingRegressor(max_depth<span style="color: #666666">=</span>degree, n_estimators<span style="color: #666666">=100</span>, learning_rate<span style="color: #666666">=1.0</span>)
model<span style="color: #666666">.</span>fit(X_train_scaled,y_train)
y_pred <span style="color: #666666">=</span> model<span style="color: #666666">.</span>predict(X_test_scaled)
polydegree[degree] <span style="color: #666666">=</span> degree
error[degree] <span style="color: #666666">=</span> np<span style="color: #666666">.</span>mean( np<span style="color: #666666">.</span>mean((y_test <span style="color: #666666">-</span> y_pred)<span style="color: #666666">**2</span>) )
bias[degree] <span style="color: #666666">=</span> np<span style="color: #666666">.</span>mean( (y_test <span style="color: #666666">-</span> np<span style="color: #666666">.</span>mean(y_pred))<span style="color: #666666">**2</span> )
variance[degree] <span style="color: #666666">=</span> np<span style="color: #666666">.</span>mean( np<span style="color: #666666">.</span>var(y_pred) )
<span style="color: #008000">print</span>(<span style="color: #BA2121">&#39;Max depth:&#39;</span>, degree)
<span style="color: #008000">print</span>(<span style="color: #BA2121">&#39;Error:&#39;</span>, error[degree])
<span style="color: #008000">print</span>(<span style="color: #BA2121">&#39;Bias^2:&#39;</span>, bias[degree])
<span style="color: #008000">print</span>(<span style="color: #BA2121">&#39;Var:&#39;</span>, variance[degree])
<span style="color: #008000">print</span>(<span style="color: #BA2121">&#39;</span><span style="color: #BB6688; font-weight: bold">{}</span><span style="color: #BA2121"> &gt;= </span><span style="color: #BB6688; font-weight: bold">{}</span><span style="color: #BA2121"> + </span><span style="color: #BB6688; font-weight: bold">{}</span><span style="color: #BA2121"> = </span><span style="color: #BB6688; font-weight: bold">{}</span><span style="color: #BA2121">&#39;</span><span style="color: #666666">.</span>format(error[degree], bias[degree], variance[degree], bias[degree]<span style="color: #666666">+</span>variance[degree]))
plt<span style="color: #666666">.</span>xlim(<span style="color: #666666">1</span>,maxdegree<span style="color: #666666">-1</span>)
plt<span style="color: #666666">.</span>plot(polydegree, error, label<span style="color: #666666">=</span><span style="color: #BA2121">&#39;Error&#39;</span>)
plt<span style="color: #666666">.</span>plot(polydegree, bias, label<span style="color: #666666">=</span><span style="color: #BA2121">&#39;bias&#39;</span>)
plt<span style="color: #666666">.</span>plot(polydegree, variance, label<span style="color: #666666">=</span><span style="color: #BA2121">&#39;Variance&#39;</span>)
plt<span style="color: #666666">.</span>legend()
save_fig(<span style="color: #BA2121">&quot;gdregression&quot;</span>)
plt<span style="color: #666666">.</span>show()
</pre></div>
<p>
<p>
<!-- navigation buttons at the bottom of the page -->
@@ -334,10 +362,6 @@ function was the least squares function.
<li><a href="._week45-bs068.html">69</a></li>
<li><a href="._week45-bs069.html">70</a></li>
<li><a href="._week45-bs070.html">71</a></li>
<li><a href="._week45-bs071.html">72</a></li>
<li><a href="._week45-bs072.html">73</a></li>
<li><a href="._week45-bs073.html">74</a></li>
<li><a href="._week45-bs074.html">75</a></li>
<li><a href="._week45-bs067.html">&raquo;</a></li>
</ul>
<!-- ------------------- end of main content --------------- -->