lars/FYS-STK4155
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update week 45

Browse Source
This commit is contained in:
mhjensen
2021-11-10 16:37:21 +01:00
parent 902ce81b35
commit 0e5076dfbf
79 changed files with 5458 additions and 7367 deletions
Download Patch File Download Diff File Expand all files Collapse all files
doc/pub/week45/html/._week45-bs067.html
+99 -101
View File
@@ -101,81 +101,74 @@ Automatically generated HTML file from DocOnce source
('An Overview of Ensemble Methods', 2, None, '___sec37'),
('Bagging', 2, None, '___sec38'),
('More bagging', 2, None, '___sec39'),
('Simple Voting Example, head or tail', 2, None, '___sec40'),
('Using the Voting Classifier', 2, None, '___sec41'),
('Please, not the moons again! Voting and Bagging',
2,
None,
'___sec42'),
('Bagging Examples', 2, None, '___sec43'),
('Making your own Bootstrap: Changing the Level of the Decision '
'Tree',
2,
None,
'___sec44'),
('Why Voting?', 2, None, '___sec45'),
('Tossing coins', 2, None, '___sec46'),
('Standard imports first', 2, None, '___sec47'),
('Simple Voting Example, head or tail', 2, None, '___sec48'),
('Using the Voting Classifier', 2, None, '___sec49'),
('Voting and Bagging', 2, None, '___sec50'),
('Random forests', 2, None, '___sec51'),
('Random Forest Algorithm', 2, None, '___sec52'),
'___sec40'),
('Why Voting?', 2, None, '___sec41'),
('Tossing coins', 2, None, '___sec42'),
('Standard imports first', 2, None, '___sec43'),
('Simple Voting Example, head or tail', 2, None, '___sec44'),
('Using the Voting Classifier', 2, None, '___sec45'),
('Voting and Bagging', 2, None, '___sec46'),
('Random forests', 2, None, '___sec47'),
('Random Forest Algorithm', 2, None, '___sec48'),
('Random Forests Compared with other Methods on the Cancer Data',
2,
None,
'___sec53'),
'___sec49'),
('Compare Bagging on Trees with Random Forests',
2,
None,
'___sec54'),
("Boosting, a Bird's Eye View", 2, None, '___sec55'),
'___sec50'),
("Boosting, a Bird's Eye View", 2, None, '___sec51'),
('What is boosting? Additive Modelling/Iterative Fitting',
2,
None,
'___sec56'),
'___sec52'),
('Iterative Fitting, Regression and Squared-error Cost Function',
2,
None,
'___sec57'),
'___sec53'),
('Squared-Error Example and Iterative Fitting',
2,
None,
'___sec58'),
'___sec54'),
('Iterative Fitting, Classification and AdaBoost',
2,
None,
'___sec59'),
('Adaptive Boosting, AdaBoost', 2, None, '___sec60'),
('Building up AdaBoost', 2, None, '___sec61'),
'___sec55'),
('Adaptive Boosting, AdaBoost', 2, None, '___sec56'),
('Building up AdaBoost', 2, None, '___sec57'),
('Adaptive boosting: AdaBoost, Basic Algorithm',
2,
None,
'___sec62'),
('Basic Steps of AdaBoost', 2, None, '___sec63'),
('AdaBoost Examples', 2, None, '___sec64'),
'___sec58'),
('Basic Steps of AdaBoost', 2, None, '___sec59'),
('AdaBoost Examples', 2, None, '___sec60'),
('Gradient boosting: Basics with Steepest Descent/Functional '
'Gradient Descent',
2,
None,
'___sec65'),
'___sec61'),
('The Squared-Error again! Steepest Descent',
2,
None,
'___sec66'),
('Steepest Descent Example', 2, None, '___sec67'),
('Gradient Boosting, algorithm', 2, None, '___sec68'),
'___sec62'),
('Steepest Descent Example', 2, None, '___sec63'),
('Gradient Boosting, algorithm', 2, None, '___sec64'),
('Gradient Boosting, Examples of Regression',
2,
None,
'___sec69'),
'___sec65'),
('Gradient Boosting, Classification Example',
2,
None,
'___sec70'),
('XGBoost: Extreme Gradient Boosting', 2, None, '___sec71'),
('Regression Case', 2, None, '___sec72'),
('Xgboost on the Cancer Data', 2, None, '___sec73')]}
'___sec66'),
('XGBoost: Extreme Gradient Boosting', 2, None, '___sec67'),
('Regression Case', 2, None, '___sec68'),
('Xgboost on the Cancer Data', 2, None, '___sec69')]}
end of tocinfo -->
<body>
@@ -253,40 +246,36 @@ MathJax.Hub.Config({
<!-- navigation toc: --> <li><a href="._week45-bs038.html#___sec37" style="font-size: 80%;">An Overview of Ensemble Methods</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs039.html#___sec38" style="font-size: 80%;">Bagging</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs040.html#___sec39" style="font-size: 80%;">More bagging</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs041.html#___sec40" style="font-size: 80%;">Simple Voting Example, head or tail</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs042.html#___sec41" style="font-size: 80%;">Using the Voting Classifier</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs043.html#___sec42" style="font-size: 80%;">Please, not the moons again! Voting and Bagging</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs044.html#___sec43" style="font-size: 80%;">Bagging Examples</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs045.html#___sec44" style="font-size: 80%;">Making your own Bootstrap: Changing the Level of the Decision Tree</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs046.html#___sec45" style="font-size: 80%;">Why Voting?</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs047.html#___sec46" style="font-size: 80%;">Tossing coins</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs048.html#___sec47" style="font-size: 80%;">Standard imports first</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs049.html#___sec48" style="font-size: 80%;">Simple Voting Example, head or tail</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs050.html#___sec49" style="font-size: 80%;">Using the Voting Classifier</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs051.html#___sec50" style="font-size: 80%;">Voting and Bagging</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs052.html#___sec51" style="font-size: 80%;">Random forests</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs053.html#___sec52" style="font-size: 80%;">Random Forest Algorithm</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs054.html#___sec53" style="font-size: 80%;">Random Forests Compared with other Methods on the Cancer Data</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs055.html#___sec54" style="font-size: 80%;">Compare Bagging on Trees with Random Forests</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs056.html#___sec55" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs057.html#___sec56" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs058.html#___sec57" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs059.html#___sec58" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs060.html#___sec59" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs061.html#___sec60" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs062.html#___sec61" style="font-size: 80%;">Building up AdaBoost</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs063.html#___sec62" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs064.html#___sec63" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs065.html#___sec64" style="font-size: 80%;">AdaBoost Examples</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs066.html#___sec65" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent</a></li>
<!-- navigation toc: --> <li><a href="#___sec66" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs068.html#___sec67" style="font-size: 80%;">Steepest Descent Example</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs069.html#___sec68" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs070.html#___sec69" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs071.html#___sec70" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs072.html#___sec71" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs073.html#___sec72" style="font-size: 80%;">Regression Case</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs074.html#___sec73" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs041.html#___sec40" style="font-size: 80%;">Making your own Bootstrap: Changing the Level of the Decision Tree</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs042.html#___sec41" style="font-size: 80%;">Why Voting?</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs043.html#___sec42" style="font-size: 80%;">Tossing coins</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs044.html#___sec43" style="font-size: 80%;">Standard imports first</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs045.html#___sec44" style="font-size: 80%;">Simple Voting Example, head or tail</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs046.html#___sec45" style="font-size: 80%;">Using the Voting Classifier</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs047.html#___sec46" style="font-size: 80%;">Voting and Bagging</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs048.html#___sec47" style="font-size: 80%;">Random forests</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs049.html#___sec48" style="font-size: 80%;">Random Forest Algorithm</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs050.html#___sec49" style="font-size: 80%;">Random Forests Compared with other Methods on the Cancer Data</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs051.html#___sec50" style="font-size: 80%;">Compare Bagging on Trees with Random Forests</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs052.html#___sec51" style="font-size: 80%;">Boosting, a Bird's Eye View</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs053.html#___sec52" style="font-size: 80%;">What is boosting? Additive Modelling/Iterative Fitting</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs054.html#___sec53" style="font-size: 80%;">Iterative Fitting, Regression and Squared-error Cost Function</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs055.html#___sec54" style="font-size: 80%;">Squared-Error Example and Iterative Fitting</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs056.html#___sec55" style="font-size: 80%;">Iterative Fitting, Classification and AdaBoost</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs057.html#___sec56" style="font-size: 80%;">Adaptive Boosting, AdaBoost</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs058.html#___sec57" style="font-size: 80%;">Building up AdaBoost</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs059.html#___sec58" style="font-size: 80%;">Adaptive boosting: AdaBoost, Basic Algorithm</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs060.html#___sec59" style="font-size: 80%;">Basic Steps of AdaBoost</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs061.html#___sec60" style="font-size: 80%;">AdaBoost Examples</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs062.html#___sec61" style="font-size: 80%;">Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs063.html#___sec62" style="font-size: 80%;">The Squared-Error again! Steepest Descent</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs064.html#___sec63" style="font-size: 80%;">Steepest Descent Example</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs065.html#___sec64" style="font-size: 80%;">Gradient Boosting, algorithm</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs066.html#___sec65" style="font-size: 80%;">Gradient Boosting, Examples of Regression</a></li>
<!-- navigation toc: --> <li><a href="#___sec66" style="font-size: 80%;">Gradient Boosting, Classification Example</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs068.html#___sec67" style="font-size: 80%;">XGBoost: Extreme Gradient Boosting</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs069.html#___sec68" style="font-size: 80%;">Regression Case</a></li>
<!-- navigation toc: --> <li><a href="._week45-bs070.html#___sec69" style="font-size: 80%;">Xgboost on the Cancer Data</a></li>
</ul>
</li>
@@ -302,38 +291,51 @@ MathJax.Hub.Config({
<a name="part0067"></a>
<!-- !split -->
<h2 id="___sec66" class="anchor">The Squared-Error again! Steepest Descent </h2>
<h2 id="___sec66" class="anchor">Gradient Boosting, Classification Example </h2>
<p>
We start again with our cost function \( {\cal C}(\boldsymbol{y}m\boldsymbol{f})=\sum_{i=0}^{n-1}{\cal L}(y_i, f(x_i)) \) where we want to minimize
This means that for every iteration, we need to optimize
$$
(\hat{\boldsymbol{f}}) = \mathrm{argmin}_{\boldsymbol{f}}\hspace{0.1cm} \sum_{i=0}^{n-1}(y_i-f(x_i))^2.
$$
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><span></span><span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.model_selection</span> <span style="color: #008000; font-weight: bold">import</span> train_test_split
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.datasets</span> <span style="color: #008000; font-weight: bold">import</span> load_breast_cancer
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">scikitplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">skplt</span>
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.ensemble</span> <span style="color: #008000; font-weight: bold">import</span> GradientBoostingClassifier
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.model_selection</span> <span style="color: #008000; font-weight: bold">import</span> cross_validate
<p>
We define a real function \( h_m(x) \) that defines our final function \( f_M(x) \) as
$$
f_M(x) = \sum_{m=0}^M h_m(x).
$$
<span style="color: #408080; font-style: italic"># Load the data</span>
cancer <span style="color: #666666">=</span> load_breast_cancer()
<p>
In the steepest decent approach we approximate \( h_m(x) = -\rho_m g_m(x) \), where \( \rho_m \) is a scalar and \( g_m(x) \) the gradient defined as
$$
g_m(x_i) = \left[ \frac{\partial {\cal L}(y_i, f(x_i))}{\partial f(x_i)}\right]_{f(x_i)=f_{m-1}(x_i)}.
$$
X_train, X_test, y_train, y_test <span style="color: #666666">=</span> train_test_split(cancer<span style="color: #666666">.</span>data,cancer<span style="color: #666666">.</span>target,random_state<span style="color: #666666">=0</span>)
<span style="color: #008000">print</span>(X_train<span style="color: #666666">.</span>shape)
<span style="color: #008000">print</span>(X_test<span style="color: #666666">.</span>shape)
<span style="color: #408080; font-style: italic">#now scale the data</span>
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.preprocessing</span> <span style="color: #008000; font-weight: bold">import</span> StandardScaler
scaler <span style="color: #666666">=</span> StandardScaler()
scaler<span style="color: #666666">.</span>fit(X_train)
X_train_scaled <span style="color: #666666">=</span> scaler<span style="color: #666666">.</span>transform(X_train)
X_test_scaled <span style="color: #666666">=</span> scaler<span style="color: #666666">.</span>transform(X_test)
<p>
With the new gradient we can update \( f_m(x) = f_{m-1}(x) -\rho_m g_m(x) \). Using the above squared-error function we see that
the gradient is \( g_m(x_i) = -2(y_i-f(x_i)) \).
<p>
Choosing \( f_0(x)=0 \) we obtain \( g_m(x) = -2y_i \) and inserting this into the minimization problem for the cost function we have
$$
(\rho_1) = \mathrm{argmin}_{\rho}\hspace{0.1cm} \sum_{i=0}^{n-1}(y_i+2\rho y_i)^2.
$$
gd_clf <span style="color: #666666">=</span> GradientBoostingClassifier(max_depth<span style="color: #666666">=3</span>, n_estimators<span style="color: #666666">=100</span>, learning_rate<span style="color: #666666">=1.0</span>)
gd_clf<span style="color: #666666">.</span>fit(X_train_scaled, y_train)
<span style="color: #408080; font-style: italic">#Cross validation</span>
accuracy <span style="color: #666666">=</span> cross_validate(gd_clf,X_test_scaled,y_test,cv<span style="color: #666666">=10</span>)[<span style="color: #BA2121">&#39;test_score&#39;</span>]
<span style="color: #008000">print</span>(accuracy)
<span style="color: #008000">print</span>(<span style="color: #BA2121">&quot;Test set accuracy with Random Forests and scaled data: </span><span style="color: #BB6688; font-weight: bold">{:.2f}</span><span style="color: #BA2121">&quot;</span><span style="color: #666666">.</span>format(gd_clf<span style="color: #666666">.</span>score(X_test_scaled,y_test)))
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">scikitplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">skplt</span>
y_pred <span style="color: #666666">=</span> gd_clf<span style="color: #666666">.</span>predict(X_test_scaled)
skplt<span style="color: #666666">.</span>metrics<span style="color: #666666">.</span>plot_confusion_matrix(y_test, y_pred, normalize<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">True</span>)
save_fig(<span style="color: #BA2121">&quot;gdclassiffierconfusion&quot;</span>)
plt<span style="color: #666666">.</span>show()
y_probas <span style="color: #666666">=</span> gd_clf<span style="color: #666666">.</span>predict_proba(X_test_scaled)
skplt<span style="color: #666666">.</span>metrics<span style="color: #666666">.</span>plot_roc(y_test, y_probas)
save_fig(<span style="color: #BA2121">&quot;gdclassiffierroc&quot;</span>)
plt<span style="color: #666666">.</span>show()
skplt<span style="color: #666666">.</span>metrics<span style="color: #666666">.</span>plot_cumulative_gain(y_test, y_probas)
save_fig(<span style="color: #BA2121">&quot;gdclassiffiercgain&quot;</span>)
plt<span style="color: #666666">.</span>show()
</pre></div>
<p>
<p>
<!-- navigation buttons at the bottom of the page -->
@@ -353,10 +355,6 @@ $$
<li><a href="._week45-bs068.html">69</a></li>
<li><a href="._week45-bs069.html">70</a></li>
<li><a href="._week45-bs070.html">71</a></li>
<li><a href="._week45-bs071.html">72</a></li>
<li><a href="._week45-bs072.html">73</a></li>
<li><a href="._week45-bs073.html">74</a></li>
<li><a href="._week45-bs074.html">75</a></li>
<li><a href="._week45-bs068.html">&raquo;</a></li>
</ul>
<!-- ------------------- end of main content --------------- -->