Morten Hjorth-Jensen
1100 lines
66 KiB
HTML
1100 lines
66 KiB
HTML
<!--
|
|
HTML file automatically generated from DocOnce source
|
|
(https://github.com/doconce/doconce/)
|
|
doconce format html week45.do.txt --pygments_html_style=default --html_style=bloodish --html_links_in_new_window --html_output=week45 --no_mako
|
|
-->
|
|
<html>
|
|
<head>
|
|
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
|
|
<meta name="generator" content="DocOnce: https://github.com/doconce/doconce/" />
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
|
<meta name="description" content="Week 45: Decisions Trees, Random Forests, Bagging and Boosting">
|
|
<title>Week 45: Decisions Trees, Random Forests, Bagging and Boosting</title>
|
|
<style type="text/css">
|
|
/* bloodish style */
|
|
body {
|
|
font-family: Helvetica, Verdana, Arial, Sans-serif;
|
|
color: #404040;
|
|
background: #ffffff;
|
|
}
|
|
h1 { font-size: 1.8em; color: #8A0808; }
|
|
h2 { font-size: 1.6em; color: #8A0808; }
|
|
h3 { font-size: 1.4em; color: #8A0808; }
|
|
h4 { font-size: 1.2em; color: #8A0808; }
|
|
a { color: #8A0808; text-decoration:none; }
|
|
tt { font-family: "Courier New", Courier; }
|
|
p { text-indent: 0px; }
|
|
hr { border: 0; width: 80%; border-bottom: 1px solid #aaa}
|
|
p.caption { width: 80%; font-style: normal; text-align: left; }
|
|
hr.figure { border: 0; width: 80%; border-bottom: 1px solid #aaa; }div.highlight {
|
|
border: 1px solid #cfcfcf;
|
|
border-radius: 2px;
|
|
line-height: 1.21429em;
|
|
}
|
|
div.cell {
|
|
width: 100%;
|
|
padding: 5px 5px 5px 0;
|
|
margin: 0;
|
|
outline: none;
|
|
}
|
|
div.input {
|
|
page-break-inside: avoid;
|
|
box-orient: horizontal;
|
|
box-align: stretch;
|
|
display: flex;
|
|
flex-direction: row;
|
|
align-items: stretch;
|
|
}
|
|
div.inner_cell {
|
|
box-orient: vertical;
|
|
box-align: stretch;
|
|
display: flex;
|
|
flex-direction: column;
|
|
align-items: stretch;
|
|
box-flex: 1;
|
|
flex: 1;
|
|
}
|
|
div.input_area {
|
|
border: 1px solid #cfcfcf;
|
|
border-radius: 4px;
|
|
background: #f7f7f7;
|
|
line-height: 1.21429em;
|
|
}
|
|
div.input_area > div.highlight {
|
|
margin: .4em;
|
|
border: none;
|
|
padding: 0;
|
|
background-color: transparent;
|
|
}
|
|
div.output_wrapper {
|
|
position: relative;
|
|
box-orient: vertical;
|
|
box-align: stretch;
|
|
display: flex;
|
|
flex-direction: column;
|
|
align-items: stretch;
|
|
}
|
|
.output {
|
|
box-orient: vertical;
|
|
box-align: stretch;
|
|
display: flex;
|
|
flex-direction: column;
|
|
align-items: stretch;
|
|
}
|
|
div.output_area {
|
|
padding: 0;
|
|
page-break-inside: avoid;
|
|
box-orient: horizontal;
|
|
box-align: stretch;
|
|
display: flex;
|
|
flex-direction: row;
|
|
align-items: stretch;
|
|
}
|
|
div.output_subarea {
|
|
padding: .4em .4em 0 .4em;
|
|
box-flex: 1;
|
|
flex: 1;
|
|
}
|
|
div.output_text {
|
|
text-align: left;
|
|
color: #000;
|
|
line-height: 1.21429em;
|
|
}
|
|
.alert-text-small { font-size: 80%; }
|
|
.alert-text-large { font-size: 130%; }
|
|
.alert-text-normal { font-size: 90%; }
|
|
.alert {
|
|
padding:8px 35px 8px 14px; margin-bottom:18px;
|
|
text-shadow:0 1px 0 rgba(255,255,255,0.5);
|
|
border:1px solid #bababa;
|
|
border-radius: 4px;
|
|
-webkit-border-radius: 4px;
|
|
-moz-border-radius: 4px;
|
|
color: #555;
|
|
background-color: #f8f8f8;
|
|
background-position: 10px 5px;
|
|
background-repeat: no-repeat;
|
|
background-size: 38px;
|
|
padding-left: 55px;
|
|
width: 75%;
|
|
}
|
|
.alert-block {padding-top:14px; padding-bottom:14px}
|
|
.alert-block > p, .alert-block > ul {margin-bottom:1em}
|
|
.alert li {margin-top: 1em}
|
|
.alert-block p+p {margin-top:5px}
|
|
.alert-notice { background-image: url(https://cdn.rawgit.com/doconce/doconce/master/bundled/html_images/small_gray_notice.png); }
|
|
.alert-summary { background-image:url(https://cdn.rawgit.com/doconce/doconce/master/bundled/html_images/small_gray_summary.png); }
|
|
.alert-warning { background-image: url(https://cdn.rawgit.com/doconce/doconce/master/bundled/html_images/small_gray_warning.png); }
|
|
.alert-question {background-image:url(https://cdn.rawgit.com/doconce/doconce/master/bundled/html_images/small_gray_question.png); }
|
|
div { text-align: justify; text-justify: inter-word; }
|
|
.tab {
|
|
padding-left: 1.5em;
|
|
}
|
|
div.toc p,a {
|
|
line-height: 1.3;
|
|
margin-top: 1.1;
|
|
margin-bottom: 1.1;
|
|
}
|
|
</style>
|
|
</head>
|
|
|
|
<!-- tocinfo
|
|
{'highest level': 2,
|
|
'sections': [('Overview of week 45', 2, None, 'overview-of-week-45'),
|
|
('Brief code reminder from last wekk',
|
|
2,
|
|
None,
|
|
'brief-code-reminder-from-last-wekk'),
|
|
("Boosting, a Bird's Eye View",
|
|
2,
|
|
None,
|
|
'boosting-a-bird-s-eye-view'),
|
|
('What is boosting? Additive Modelling/Iterative Fitting',
|
|
2,
|
|
None,
|
|
'what-is-boosting-additive-modelling-iterative-fitting'),
|
|
('Iterative Fitting, Regression and Squared-error Cost Function',
|
|
2,
|
|
None,
|
|
'iterative-fitting-regression-and-squared-error-cost-function'),
|
|
('Squared-Error Example and Iterative Fitting',
|
|
2,
|
|
None,
|
|
'squared-error-example-and-iterative-fitting'),
|
|
('Iterative Fitting, Classification and AdaBoost',
|
|
2,
|
|
None,
|
|
'iterative-fitting-classification-and-adaboost'),
|
|
('Adaptive Boosting, AdaBoost',
|
|
2,
|
|
None,
|
|
'adaptive-boosting-adaboost'),
|
|
('Building up AdaBoost', 2, None, 'building-up-adaboost'),
|
|
('Adaptive boosting: AdaBoost, Basic Algorithm',
|
|
2,
|
|
None,
|
|
'adaptive-boosting-adaboost-basic-algorithm'),
|
|
('Basic Steps of AdaBoost', 2, None, 'basic-steps-of-adaboost'),
|
|
('AdaBoost Examples', 2, None, 'adaboost-examples'),
|
|
('Gradient boosting: Basics with Steepest Descent/Functional '
|
|
'Gradient Descent',
|
|
2,
|
|
None,
|
|
'gradient-boosting-basics-with-steepest-descent-functional-gradient-descent'),
|
|
('The Squared-Error again! Steepest Descent',
|
|
2,
|
|
None,
|
|
'the-squared-error-again-steepest-descent'),
|
|
('Steepest Descent Example', 2, None, 'steepest-descent-example'),
|
|
('Gradient Boosting, algorithm',
|
|
2,
|
|
None,
|
|
'gradient-boosting-algorithm'),
|
|
('Gradient Boosting, Examples of Regression',
|
|
2,
|
|
None,
|
|
'gradient-boosting-examples-of-regression'),
|
|
('Gradient Boosting, Classification Example',
|
|
2,
|
|
None,
|
|
'gradient-boosting-classification-example'),
|
|
('XGBoost: Extreme Gradient Boosting',
|
|
2,
|
|
None,
|
|
'xgboost-extreme-gradient-boosting'),
|
|
('Regression Case', 2, None, 'regression-case'),
|
|
('Xgboost on the Cancer Data',
|
|
2,
|
|
None,
|
|
'xgboost-on-the-cancer-data')]}
|
|
end of tocinfo -->
|
|
|
|
<body>
|
|
|
|
|
|
|
|
<script type="text/x-mathjax-config">
|
|
MathJax.Hub.Config({
|
|
TeX: {
|
|
equationNumbers: { autoNumber: "AMS" },
|
|
extensions: ["AMSmath.js", "AMSsymbols.js", "autobold.js", "color.js"]
|
|
}
|
|
});
|
|
</script>
|
|
<script type="text/javascript" async
|
|
src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
|
|
</script>
|
|
|
|
|
|
<!-- ------------------- main content ---------------------- -->
|
|
<center>
|
|
<h1>Week 45: Decisions Trees, Random Forests, Bagging and Boosting</h1>
|
|
</center> <!-- document title -->
|
|
|
|
<!-- author(s): Morten Hjorth-Jensen -->
|
|
<center>
|
|
<b>Morten Hjorth-Jensen</b> [1, 2]
|
|
</center>
|
|
<!-- institution(s) -->
|
|
<center>
|
|
[1] <b>Department of Physics, University of Oslo</b>
|
|
</center>
|
|
<center>
|
|
[2] <b>Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University</b>
|
|
</center>
|
|
<br>
|
|
<center>
|
|
<h4>Nov 6, 2022</h4>
|
|
</center> <!-- date -->
|
|
<br>
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="overview-of-week-45">Overview of week 45 </h2>
|
|
|
|
<ul>
|
|
<li> Thursday: Boosting methods, froma AdaBoost to Gradient boosting</li>
|
|
<li> Friday: Gradient boosting and discussion of Decision trees and ensemble methods. Wrapping up trees and start discussing Support Vector Machines</li>
|
|
</ul>
|
|
<div class="alert alert-block alert-block alert-text-normal">
|
|
<b>Videos</b>
|
|
<p>
|
|
<ol>
|
|
<li> <a href="https://www.youtube.com/watch?v=RmajweUFKvM&ab_channel=Simplilearn" target="_blank">Video on Decision trees</a></li>
|
|
<li> <a href="https://www.youtube.com/watch?v=wPqtzj5VZus&ab_channel=H2O.ai" target="_blank">Video on boosting methods by Hastie</a>.</li>
|
|
<li> <a href="https://www.youtube.com/watch?v=LsK-xG1cLYA" target="_blank">Video on AdaBoost</a></li>
|
|
<li> <a href="https://www.youtube.com/watch?v=3CC4N4z3GJc" target="_blank">Video on Gradient boost, part 1, parts 2-4 follows</a></li>
|
|
</ol>
|
|
</div>
|
|
|
|
|
|
<div class="alert alert-block alert-block alert-text-normal">
|
|
<b>Reading</b>
|
|
<p>
|
|
<ol>
|
|
<li> <a href="https://github.com/CompPhysics/MachineLearning/blob/master/doc/Textbooks/elementsstat.pdf" target="_blank">Hastie et al, chapter 10.1-10.10</a></li>
|
|
</ol>
|
|
</div>
|
|
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="brief-code-reminder-from-last-wekk">Brief code reminder from last wekk </h2>
|
|
|
|
|
|
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
|
<div class="cell border-box-sizing code_cell rendered">
|
|
<div class="input">
|
|
<div class="inner_cell">
|
|
<div class="input_area">
|
|
<div class="highlight" style="background: #f8f8f8">
|
|
<pre style="line-height: 125%;"><span style="color: #666666">%</span>matplotlib inline
|
|
|
|
<span style="color: #408080; font-style: italic"># Common imports</span>
|
|
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">IPython.display</span> <span style="color: #008000; font-weight: bold">import</span> Image
|
|
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">pydot</span> <span style="color: #008000; font-weight: bold">import</span> graph_from_dot_data
|
|
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">pandas</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">pd</span>
|
|
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
|
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
|
|
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.model_selection</span> <span style="color: #008000; font-weight: bold">import</span> train_test_split
|
|
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.tree</span> <span style="color: #008000; font-weight: bold">import</span> export_graphviz
|
|
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.preprocessing</span> <span style="color: #008000; font-weight: bold">import</span> StandardScaler, OneHotEncoder
|
|
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.compose</span> <span style="color: #008000; font-weight: bold">import</span> ColumnTransformer
|
|
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">pydot</span> <span style="color: #008000; font-weight: bold">import</span> graph_from_dot_data
|
|
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.datasets</span> <span style="color: #008000; font-weight: bold">import</span> load_breast_cancer
|
|
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.svm</span> <span style="color: #008000; font-weight: bold">import</span> SVC
|
|
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.linear_model</span> <span style="color: #008000; font-weight: bold">import</span> LogisticRegression
|
|
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.ensemble</span> <span style="color: #008000; font-weight: bold">import</span> RandomForestClassifier
|
|
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.preprocessing</span> <span style="color: #008000; font-weight: bold">import</span> LabelEncoder
|
|
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.model_selection</span> <span style="color: #008000; font-weight: bold">import</span> cross_validate
|
|
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">scikitplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">skplt</span>
|
|
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.preprocessing</span> <span style="color: #008000; font-weight: bold">import</span> StandardScaler
|
|
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">os</span>
|
|
|
|
<span style="color: #408080; font-style: italic"># Where to save the figures and data files</span>
|
|
PROJECT_ROOT_DIR <span style="color: #666666">=</span> <span style="color: #BA2121">"Results"</span>
|
|
FIGURE_ID <span style="color: #666666">=</span> <span style="color: #BA2121">"Results/FigureFiles"</span>
|
|
DATA_ID <span style="color: #666666">=</span> <span style="color: #BA2121">"DataFiles/"</span>
|
|
|
|
<span style="color: #008000; font-weight: bold">if</span> <span style="color: #AA22FF; font-weight: bold">not</span> os<span style="color: #666666">.</span>path<span style="color: #666666">.</span>exists(PROJECT_ROOT_DIR):
|
|
os<span style="color: #666666">.</span>mkdir(PROJECT_ROOT_DIR)
|
|
|
|
<span style="color: #008000; font-weight: bold">if</span> <span style="color: #AA22FF; font-weight: bold">not</span> os<span style="color: #666666">.</span>path<span style="color: #666666">.</span>exists(FIGURE_ID):
|
|
os<span style="color: #666666">.</span>makedirs(FIGURE_ID)
|
|
|
|
<span style="color: #008000; font-weight: bold">if</span> <span style="color: #AA22FF; font-weight: bold">not</span> os<span style="color: #666666">.</span>path<span style="color: #666666">.</span>exists(DATA_ID):
|
|
os<span style="color: #666666">.</span>makedirs(DATA_ID)
|
|
|
|
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">image_path</span>(fig_id):
|
|
<span style="color: #008000; font-weight: bold">return</span> os<span style="color: #666666">.</span>path<span style="color: #666666">.</span>join(FIGURE_ID, fig_id)
|
|
|
|
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">data_path</span>(dat_id):
|
|
<span style="color: #008000; font-weight: bold">return</span> os<span style="color: #666666">.</span>path<span style="color: #666666">.</span>join(DATA_ID, dat_id)
|
|
|
|
<span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">save_fig</span>(fig_id):
|
|
plt<span style="color: #666666">.</span>savefig(image_path(fig_id) <span style="color: #666666">+</span> <span style="color: #BA2121">".png"</span>, <span style="color: #008000">format</span><span style="color: #666666">=</span><span style="color: #BA2121">'png'</span>)
|
|
|
|
<span style="color: #408080; font-style: italic"># Load the cancer data</span>
|
|
cancer <span style="color: #666666">=</span> load_breast_cancer()
|
|
|
|
X_train, X_test, y_train, y_test <span style="color: #666666">=</span> train_test_split(cancer<span style="color: #666666">.</span>data,cancer<span style="color: #666666">.</span>target,random_state<span style="color: #666666">=0</span>)
|
|
<span style="color: #008000">print</span>(X_train<span style="color: #666666">.</span>shape)
|
|
<span style="color: #008000">print</span>(X_test<span style="color: #666666">.</span>shape)
|
|
<span style="color: #408080; font-style: italic">#Scale the data</span>
|
|
scaler <span style="color: #666666">=</span> StandardScaler()
|
|
scaler<span style="color: #666666">.</span>fit(X_train)
|
|
X_train_scaled <span style="color: #666666">=</span> scaler<span style="color: #666666">.</span>transform(X_train)
|
|
X_test_scaled <span style="color: #666666">=</span> scaler<span style="color: #666666">.</span>transform(X_test)
|
|
<span style="color: #408080; font-style: italic">#define methods</span>
|
|
<span style="color: #408080; font-style: italic"># Logistic Regression</span>
|
|
logreg <span style="color: #666666">=</span> LogisticRegression(solver<span style="color: #666666">=</span><span style="color: #BA2121">'lbfgs'</span>)
|
|
logreg<span style="color: #666666">.</span>fit(X_train_scaled, y_train)
|
|
<span style="color: #008000">print</span>(<span style="color: #BA2121">"Test set accuracy Logistic Regression with scaled data: </span><span style="color: #BB6688; font-weight: bold">{:.2f}</span><span style="color: #BA2121">"</span><span style="color: #666666">.</span>format(logreg<span style="color: #666666">.</span>score(X_test_scaled,y_test)))
|
|
<span style="color: #408080; font-style: italic"># Decision Trees</span>
|
|
deep_tree_clf <span style="color: #666666">=</span> DecisionTreeClassifier(max_depth<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">None</span>)
|
|
deep_tree_clf<span style="color: #666666">.</span>fit(X_train_scaled, y_train)
|
|
<span style="color: #008000">print</span>(<span style="color: #BA2121">"Test set accuracy with Decision Trees and scaled data: </span><span style="color: #BB6688; font-weight: bold">{:.2f}</span><span style="color: #BA2121">"</span><span style="color: #666666">.</span>format(deep_tree_clf<span style="color: #666666">.</span>score(X_test_scaled,y_test)))
|
|
<span style="color: #408080; font-style: italic"># Support Vector Machine</span>
|
|
svm <span style="color: #666666">=</span> SVC(gamma<span style="color: #666666">=</span><span style="color: #BA2121">'auto'</span>, C<span style="color: #666666">=100</span>)
|
|
svm<span style="color: #666666">.</span>fit(X_train_scaled, y_train)
|
|
<span style="color: #008000">print</span>(<span style="color: #BA2121">"Test set accuracy SVM with scaled data: </span><span style="color: #BB6688; font-weight: bold">{:.2f}</span><span style="color: #BA2121">"</span><span style="color: #666666">.</span>format(logreg<span style="color: #666666">.</span>score(X_test_scaled,y_test)))
|
|
<span style="color: #408080; font-style: italic"># Random forests</span>
|
|
<span style="color: #408080; font-style: italic">#Instantiate the model with 500 trees and entropy as splitting criteria</span>
|
|
Random_Forest_model <span style="color: #666666">=</span> RandomForestClassifier(n_estimators<span style="color: #666666">=500</span>,criterion<span style="color: #666666">=</span><span style="color: #BA2121">"entropy"</span>)
|
|
Random_Forest_model<span style="color: #666666">.</span>fit(X_train_scaled, y_train)
|
|
<span style="color: #008000">print</span>(<span style="color: #BA2121">"Test set accuracy with Random Forests and scaled data: </span><span style="color: #BB6688; font-weight: bold">{:.2f}</span><span style="color: #BA2121">"</span><span style="color: #666666">.</span>format(Random_Forest_model<span style="color: #666666">.</span>score(X_test_scaled,y_test)))
|
|
|
|
|
|
|
|
y_pred <span style="color: #666666">=</span> Random_Forest_model<span style="color: #666666">.</span>predict(X_test_scaled)
|
|
skplt<span style="color: #666666">.</span>metrics<span style="color: #666666">.</span>plot_confusion_matrix(y_test, y_pred, normalize<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">True</span>)
|
|
plt<span style="color: #666666">.</span>show()
|
|
y_probas <span style="color: #666666">=</span> Random_Forest_model<span style="color: #666666">.</span>predict_proba(X_test_scaled)
|
|
skplt<span style="color: #666666">.</span>metrics<span style="color: #666666">.</span>plot_roc(y_test, y_probas)
|
|
plt<span style="color: #666666">.</span>show()
|
|
skplt<span style="color: #666666">.</span>metrics<span style="color: #666666">.</span>plot_cumulative_gain(y_test, y_probas)
|
|
plt<span style="color: #666666">.</span>show()
|
|
</pre>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<div class="output_wrapper">
|
|
<div class="output">
|
|
<div class="output_area">
|
|
<div class="output_subarea output_stream output_stdout output_text">
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="boosting-a-bird-s-eye-view">Boosting, a Bird's Eye View </h2>
|
|
|
|
<p>The basic idea is to combine weak classifiers in order to create a good
|
|
classifier. With a weak classifier we often intend a classifier which
|
|
produces results which are only slightly better than we would get by
|
|
random guesses.
|
|
</p>
|
|
|
|
<p>This is done by applying in an iterative way a weak (or a standard
|
|
classifier like decision trees) to modify the data. In each iteration
|
|
we emphasize those observations which are misclassified by weighting
|
|
them with a factor.
|
|
</p>
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="what-is-boosting-additive-modelling-iterative-fitting">What is boosting? Additive Modelling/Iterative Fitting </h2>
|
|
|
|
<p>Boosting is a way of fitting an additive expansion in a set of
|
|
elementary basis functions like for example some simple polynomials.
|
|
Assume for example that we have a function
|
|
</p>
|
|
$$
|
|
f_M(x) = \sum_{i=1}^M \beta_m b(x;\gamma_m),
|
|
$$
|
|
|
|
<p>where \( \beta_m \) are the expansion parameters to be determined in a
|
|
minimization process and \( b(x;\gamma_m) \) are some simple functions of
|
|
the multivariable parameter \( x \) which is characterized by the
|
|
parameters \( \gamma_m \).
|
|
</p>
|
|
|
|
<p>As an example, consider the Sigmoid function we used in logistic
|
|
regression. In that case, we can translate the function
|
|
\( b(x;\gamma_m) \) into the Sigmoid function
|
|
</p>
|
|
|
|
$$
|
|
\sigma(t) = \frac{1}{1+\exp{(-t)}},
|
|
$$
|
|
|
|
<p>where \( t=\gamma_0+\gamma_1 x \) and the parameters \( \gamma_0 \) and
|
|
\( \gamma_1 \) were determined by the Logistic Regression fitting
|
|
algorithm.
|
|
</p>
|
|
|
|
<p>As another example, consider the cost function we defined for linear regression</p>
|
|
$$
|
|
C(\boldsymbol{y},\boldsymbol{f}) = \frac{1}{n} \sum_{i=0}^{n-1}(y_i-f(x_i))^2.
|
|
$$
|
|
|
|
<p>In this case the function \( f(x) \) was replaced by the design matrix
|
|
\( \boldsymbol{X} \) and the unknown linear regression parameters \( \boldsymbol{\beta} \),
|
|
that is \( \boldsymbol{f}=\boldsymbol{X}\boldsymbol{\beta} \). In linear regression we can
|
|
simply invert a matrix and obtain the parameters \( \beta \) by
|
|
</p>
|
|
|
|
$$
|
|
\boldsymbol{\beta}=\left(\boldsymbol{X}^T\boldsymbol{X}\right)^{-1}\boldsymbol{X}^T\boldsymbol{y}.
|
|
$$
|
|
|
|
<p>In iterative fitting or additive modeling, we minimize the cost function with respect to the parameters \( \beta_m \) and \( \gamma_m \).</p>
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="iterative-fitting-regression-and-squared-error-cost-function">Iterative Fitting, Regression and Squared-error Cost Function </h2>
|
|
|
|
<p>The way we proceed is as follows (here we specialize to the squared-error cost function)</p>
|
|
|
|
<ol>
|
|
<li> Establish a cost function, here \( {\cal C}(\boldsymbol{y},\boldsymbol{f}) = \frac{1}{n} \sum_{i=0}^{n-1}(y_i-f_M(x_i))^2 \) with \( f_M(x) = \sum_{i=1}^M \beta_m b(x;\gamma_m) \).</li>
|
|
<li> Initialize with a guess \( f_0(x) \). It could be one or even zero or some random numbers.</li>
|
|
<li> For \( m=1:M \)
|
|
<ol type="a"></li>
|
|
<li> minimize \( \sum_{i=0}^{n-1}(y_i-f_{m-1}(x_i)-\beta b(x;\gamma))^2 \) wrt \( \gamma \) and \( \beta \)</li>
|
|
<li> This gives the optimal values \( \beta_m \) and \( \gamma_m \)</li>
|
|
<li> Determine then the new values \( f_m(x)=f_{m-1}(x) +\beta_m b(x;\gamma_m) \)</li>
|
|
</ol>
|
|
</ol>
|
|
<p>We could use any of the algorithms we have discussed till now. If we
|
|
use trees, \( \gamma \) parameterizes the split variables and split points
|
|
at the internal nodes, and the predictions at the terminal nodes.
|
|
</p>
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="squared-error-example-and-iterative-fitting">Squared-Error Example and Iterative Fitting </h2>
|
|
|
|
<p>To better understand what happens, let us develop the steps for the iterative fitting using the above squared error function.</p>
|
|
|
|
<p>For simplicity we assume also that our functions \( b(x;\gamma)=1+\gamma x \). </p>
|
|
|
|
<p>This means that for every iteration \( m \), we need to optimize</p>
|
|
|
|
$$
|
|
(\beta_m,\gamma_m) = \mathrm{argmin}_{\beta,\lambda}\hspace{0.1cm} \sum_{i=0}^{n-1}(y_i-f_{m-1}(x_i)-\beta b(x;\gamma))^2=\sum_{i=0}^{n-1}(y_i-f_{m-1}(x_i)-\beta(1+\gamma x_i))^2.
|
|
$$
|
|
|
|
<p>We start our iteration by simply setting \( f_0(x)=0 \).
|
|
Taking the derivatives with respect to \( \beta \) and \( \gamma \) we obtain
|
|
</p>
|
|
$$
|
|
\frac{\partial {\cal C}}{\partial \beta} = -2\sum_{i}(1+\gamma x_i)(y_i-\beta(1+\gamma x_i))=0,
|
|
$$
|
|
|
|
<p>and</p>
|
|
$$
|
|
\frac{\partial {\cal C}}{\partial \gamma} =-2\sum_{i}\beta x_i(y_i-\beta(1+\gamma x_i))=0.
|
|
$$
|
|
|
|
<p>We can then rewrite these equations as (defining \( \boldsymbol{w}=\boldsymbol{e}+\gamma \boldsymbol{x}) \) with \( \boldsymbol{e} \) being the unit vector)</p>
|
|
$$
|
|
\gamma \boldsymbol{w}^T(\boldsymbol{y}-\beta\gamma \boldsymbol{w})=0,
|
|
$$
|
|
|
|
<p>which gives us \( \beta = \boldsymbol{w}^T\boldsymbol{y}/(\boldsymbol{w}^T\boldsymbol{w}) \). Similarly we have </p>
|
|
$$
|
|
\beta\gamma \boldsymbol{x}^T(\boldsymbol{y}-\beta(1+\gamma \boldsymbol{x}))=0,
|
|
$$
|
|
|
|
<p>which leads to \( \gamma =(\boldsymbol{x}^T\boldsymbol{y}-\beta\boldsymbol{x}^T\boldsymbol{e})/(\beta\boldsymbol{x}^T\boldsymbol{x}) \). Inserting
|
|
for \( \beta \) gives us an equation for \( \gamma \). This is a non-linear equation in the unknown \( \gamma \) and has to be solved numerically.
|
|
</p>
|
|
|
|
<p>The solution to these two equations gives us in turn \( \beta_1 \) and \( \gamma_1 \) leading to the new expression for \( f_1(x) \) as
|
|
\( f_1(x) = \beta_1(1+\gamma_1x) \). Doing this \( M \) times results in our final estimate for the function \( f \).
|
|
</p>
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="iterative-fitting-classification-and-adaboost">Iterative Fitting, Classification and AdaBoost </h2>
|
|
|
|
<p>Let us consider a binary classification problem with two outcomes \( y_i \in \{-1,1\} \) and \( i=0,1,2,\dots,n-1 \) as our set of
|
|
observations. We define a classification function \( G(x) \) which produces a prediction taking one or the other of the two values
|
|
\( \{-1,1\} \).
|
|
</p>
|
|
|
|
<p>The error rate of the training sample is then</p>
|
|
|
|
$$
|
|
\mathrm{\overline{err}}=\frac{1}{n} \sum_{i=0}^{n-1} I(y_i\ne G(x_i)).
|
|
$$
|
|
|
|
<p>The iterative procedure starts with defining a weak classifier whose
|
|
error rate is barely better than random guessing. The iterative
|
|
procedure in boosting is to sequentially apply a weak
|
|
classification algorithm to repeatedly modified versions of the data
|
|
producing a sequence of weak classifiers \( G_m(x) \).
|
|
</p>
|
|
|
|
<p>Here we will express our function \( f(x) \) in terms of \( G(x) \). That is</p>
|
|
$$
|
|
f_M(x) = \sum_{i=1}^M \beta_m b(x;\gamma_m),
|
|
$$
|
|
|
|
<p>will be a function of </p>
|
|
$$
|
|
G_M(x) = \mathrm{sign} \sum_{i=1}^M \alpha_m G_m(x).
|
|
$$
|
|
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="adaptive-boosting-adaboost">Adaptive Boosting, AdaBoost </h2>
|
|
|
|
<p>In our iterative procedure we define thus</p>
|
|
$$
|
|
f_m(x) = f_{m-1}(x)+\beta_mG_m(x).
|
|
$$
|
|
|
|
<p>The simplest possible cost function which leads (also simple from a computational point of view) to the AdaBoost algorithm is the
|
|
exponential cost/loss function defined as
|
|
</p>
|
|
$$
|
|
C(\boldsymbol{y},\boldsymbol{f}) = \sum_{i=0}^{n-1}\exp{(-y_i(f_{m-1}(x_i)+\beta G(x_i))}.
|
|
$$
|
|
|
|
<p>We optimize \( \beta \) and \( G \) for each value of \( m=1:M \) as we did in the regression case.
|
|
This is normally done in two steps. Let us however first rewrite the cost function as
|
|
</p>
|
|
|
|
$$
|
|
C(\boldsymbol{y},\boldsymbol{f}) = \sum_{i=0}^{n-1}w_i^{m}\exp{(-y_i\beta G(x_i))},
|
|
$$
|
|
|
|
<p>where we have defined \( w_i^m= \exp{(-y_if_{m-1}(x_i))} \).</p>
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="building-up-adaboost">Building up AdaBoost </h2>
|
|
|
|
<p>First, for any \( \beta > 0 \), we optimize \( G \) by setting</p>
|
|
$$
|
|
G_m(x) = \mathrm{sign} \sum_{i=0}^{n-1} w_i^m I(y_i \ne G_(x_i)),
|
|
$$
|
|
|
|
<p>which is the classifier that minimizes the weighted error rate in predicting \( y \).</p>
|
|
|
|
<p>We can do this by rewriting</p>
|
|
$$
|
|
\exp{-(\beta)}\sum_{y_i=G(x_i)}w_i^m+\exp{(\beta)}\sum_{y_i\ne G(x_i)}w_i^m,
|
|
$$
|
|
|
|
<p>which can be rewritten as</p>
|
|
$$
|
|
(\exp{(\beta)}-\exp{-(\beta)})\sum_{i=0}^{n-1}w_i^mI(y_i\ne G(x_i))+\exp{(-\beta)}\sum_{i=0}^{n-1}w_i^m=0,
|
|
$$
|
|
|
|
<p>which leads to</p>
|
|
$$
|
|
\beta_m = \frac{1}{2}\log{\frac{1-\mathrm{\overline{err}}}{\mathrm{\overline{err}}}},
|
|
$$
|
|
|
|
<p>where we have redefined the error as </p>
|
|
$$
|
|
\mathrm{\overline{err}}_m=\frac{1}{n}\frac{\sum_{i=0}^{n-1}w_i^mI(y_i\ne G(x_i)}{\sum_{i=0}^{n-1}w_i^m},
|
|
$$
|
|
|
|
<p>which leads to an update of</p>
|
|
$$
|
|
f_m(x) = f_{m-1}(x) +\beta_m G_m(x).
|
|
$$
|
|
|
|
<p>This leads to the new weights</p>
|
|
$$
|
|
w_i^{m+1} = w_i^m \exp{(-y_i\beta_m G_m(x_i))}
|
|
$$
|
|
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="adaptive-boosting-adaboost-basic-algorithm">Adaptive boosting: AdaBoost, Basic Algorithm </h2>
|
|
|
|
<p>The algorithm here is rather straightforward. Assume that our weak
|
|
classifier is a decision tree and we consider a binary set of outputs
|
|
with \( y_i \in \{-1,1\} \) and \( i=0,1,2,\dots,n-1 \) as our set of
|
|
observations. Our design matrix is given in terms of the
|
|
feature/predictor vectors
|
|
\( \boldsymbol{X}=[\boldsymbol{x}_0\boldsymbol{x}_1\dots\boldsymbol{x}_{p-1}] \). Finally, we define also a
|
|
classifier determined by our data via a function \( G(x) \). This function tells us how well we are able to classify our outputs/targets \( \boldsymbol{y} \).
|
|
</p>
|
|
|
|
<p>We have already defined the misclassification error \( \mathrm{err} \) as</p>
|
|
$$
|
|
\mathrm{err}=\frac{1}{n}\sum_{i=0}^{n-1}I(y_i\ne G(x_i)),
|
|
$$
|
|
|
|
<p>where the function \( I() \) is one if we misclassify and zero if we classify correctly. </p>
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="basic-steps-of-adaboost">Basic Steps of AdaBoost </h2>
|
|
|
|
<p>With the above definitions we are now ready to set up the algorithm for AdaBoost.
|
|
The basic idea is to set up weights which will be used to scale the correctly classified and the misclassified cases.
|
|
</p>
|
|
<ol>
|
|
<li> We start by initializing all weights to \( w_i = 1/n \), with \( i=0,1,2,\dots n-1 \). It is easy to see that we must have \( \sum_{i=0}^{n-1}w_i = 1 \).</li>
|
|
<li> We rewrite the misclassification error as</li>
|
|
</ol>
|
|
$$
|
|
\mathrm{\overline{err}}_m=\frac{\sum_{i=0}^{n-1}w_i^m I(y_i\ne G(x_i))}{\sum_{i=0}^{n-1}w_i},
|
|
$$
|
|
|
|
<ol>
|
|
<li> Then we start looping over all attempts at classifying, namely we start an iterative process for \( m=1:M \), where \( M \) is the final number of classifications. Our given classifier could for example be a plain decision tree.
|
|
<ol type="a"></li>
|
|
<li> Fit then a given classifier to the training set using the weights \( w_i \).</li>
|
|
<li> Compute then \( \mathrm{err} \) and figure out which events are classified properly and which are classified wrongly.</li>
|
|
<li> Define a quantity \( \alpha_{m} = \log{(1-\mathrm{\overline{err}}_m)/\mathrm{\overline{err}}_m} \)</li>
|
|
<li> Set the new weights to \( w_i = w_i\times \exp{(\alpha_m I(y_i\ne G(x_i)} \).</li>
|
|
</ol>
|
|
<li> Compute the new classifier \( G(x)= \sum_{i=0}^{n-1}\alpha_m I(y_i\ne G(x_i) \).</li>
|
|
</ol>
|
|
<p>For the iterations with \( m \le 2 \) the weights are modified
|
|
individually at each steps. The observations which were misclassified
|
|
at iteration \( m-1 \) have a weight which is larger than those which were
|
|
classified properly. As this proceeds, the observations which were
|
|
difficult to classifiy correctly are given a larger influence. Each
|
|
new classification step \( m \) is then forced to concentrate on those
|
|
observations that are missed in the previous iterations.
|
|
</p>
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="adaboost-examples">AdaBoost Examples </h2>
|
|
|
|
<p>Using <b>Scikit-Learn</b> it is easy to apply the adaptive boosting algorithm, as done here.</p>
|
|
|
|
|
|
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
|
<div class="cell border-box-sizing code_cell rendered">
|
|
<div class="input">
|
|
<div class="inner_cell">
|
|
<div class="input_area">
|
|
<div class="highlight" style="background: #f8f8f8">
|
|
<pre style="line-height: 125%;"><span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.ensemble</span> <span style="color: #008000; font-weight: bold">import</span> AdaBoostClassifier
|
|
|
|
ada_clf <span style="color: #666666">=</span> AdaBoostClassifier(
|
|
DecisionTreeClassifier(max_depth<span style="color: #666666">=1</span>), n_estimators<span style="color: #666666">=200</span>,
|
|
algorithm<span style="color: #666666">=</span><span style="color: #BA2121">"SAMME.R"</span>, learning_rate<span style="color: #666666">=0.5</span>, random_state<span style="color: #666666">=42</span>)
|
|
ada_clf<span style="color: #666666">.</span>fit(X_train, y_train)
|
|
|
|
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.ensemble</span> <span style="color: #008000; font-weight: bold">import</span> AdaBoostClassifier
|
|
|
|
ada_clf <span style="color: #666666">=</span> AdaBoostClassifier(
|
|
DecisionTreeClassifier(max_depth<span style="color: #666666">=1</span>), n_estimators<span style="color: #666666">=200</span>,
|
|
algorithm<span style="color: #666666">=</span><span style="color: #BA2121">"SAMME.R"</span>, learning_rate<span style="color: #666666">=0.5</span>, random_state<span style="color: #666666">=42</span>)
|
|
ada_clf<span style="color: #666666">.</span>fit(X_train_scaled, y_train)
|
|
y_pred <span style="color: #666666">=</span> ada_clf<span style="color: #666666">.</span>predict(X_test_scaled)
|
|
skplt<span style="color: #666666">.</span>metrics<span style="color: #666666">.</span>plot_confusion_matrix(y_test, y_pred, normalize<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">True</span>)
|
|
plt<span style="color: #666666">.</span>show()
|
|
y_probas <span style="color: #666666">=</span> ada_clf<span style="color: #666666">.</span>predict_proba(X_test_scaled)
|
|
skplt<span style="color: #666666">.</span>metrics<span style="color: #666666">.</span>plot_roc(y_test, y_probas)
|
|
plt<span style="color: #666666">.</span>show()
|
|
skplt<span style="color: #666666">.</span>metrics<span style="color: #666666">.</span>plot_cumulative_gain(y_test, y_probas)
|
|
plt<span style="color: #666666">.</span>show()
|
|
</pre>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<div class="output_wrapper">
|
|
<div class="output">
|
|
<div class="output_area">
|
|
<div class="output_subarea output_stream output_stdout output_text">
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="gradient-boosting-basics-with-steepest-descent-functional-gradient-descent">Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent </h2>
|
|
|
|
<p>Gradient boosting is again a similar technique to Adaptive boosting,
|
|
it combines so-called weak classifiers or regressors into a strong
|
|
method via a series of iterations.
|
|
</p>
|
|
|
|
<p>In order to understand the method, let us illustrate its basics by
|
|
bringing back the essential steps in linear regression, where our cost
|
|
function was the least squares function.
|
|
</p>
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="the-squared-error-again-steepest-descent">The Squared-Error again! Steepest Descent </h2>
|
|
|
|
<p>We start again with our cost function \( {\cal C}(\boldsymbol{y}m\boldsymbol{f})=\sum_{i=0}^{n-1}{\cal L}(y_i, f(x_i)) \) where we want to minimize
|
|
This means that for every iteration, we need to optimize
|
|
</p>
|
|
|
|
$$
|
|
(\hat{\boldsymbol{f}}) = \mathrm{argmin}_{\boldsymbol{f}}\hspace{0.1cm} \sum_{i=0}^{n-1}(y_i-f(x_i))^2.
|
|
$$
|
|
|
|
<p>We define a real function \( h_m(x) \) that defines our final function \( f_M(x) \) as</p>
|
|
$$
|
|
f_M(x) = \sum_{m=0}^M h_m(x).
|
|
$$
|
|
|
|
<p>In the steepest decent approach we approximate \( h_m(x) = -\rho_m g_m(x) \), where \( \rho_m \) is a scalar and \( g_m(x) \) the gradient defined as</p>
|
|
$$
|
|
g_m(x_i) = \left[ \frac{\partial {\cal L}(y_i, f(x_i))}{\partial f(x_i)}\right]_{f(x_i)=f_{m-1}(x_i)}.
|
|
$$
|
|
|
|
<p>With the new gradient we can update \( f_m(x) = f_{m-1}(x) -\rho_m g_m(x) \). Using the above squared-error function we see that
|
|
the gradient is \( g_m(x_i) = -2(y_i-f(x_i)) \).
|
|
</p>
|
|
|
|
<p>Choosing \( f_0(x)=0 \) we obtain \( g_m(x) = -2y_i \) and inserting this into the minimization problem for the cost function we have</p>
|
|
$$
|
|
(\rho_1) = \mathrm{argmin}_{\rho}\hspace{0.1cm} \sum_{i=0}^{n-1}(y_i+2\rho y_i)^2.
|
|
$$
|
|
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="steepest-descent-example">Steepest Descent Example </h2>
|
|
|
|
<p>Optimizing with respect to \( \rho \) we obtain (taking the derivative) that \( \rho_1 = -1/2 \). We have then that</p>
|
|
$$
|
|
f_1(x) = f_{0}(x) -\rho_1 g_1(x)=-y_i.
|
|
$$
|
|
|
|
<p>We can then proceed and compute</p>
|
|
$$
|
|
g_2(x_i) = \left[ \frac{\partial {\cal L}(y_i, f(x_i))}{\partial f(x_i)}\right]_{f(x_i)=f_{1}(x_i)=y_i}=-4y_i,
|
|
$$
|
|
|
|
<p>and find a new value for \( \rho_2=-1/2 \) and continue till we have reached \( m=M \). We can modify the steepest descent method, or steepest boosting, by introducing what is called <b>gradient boosting</b>. </p>
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="gradient-boosting-algorithm">Gradient Boosting, algorithm </h2>
|
|
|
|
<p>Steepest descent is however not much used, since it only optimizes \( f \) at a fixed set of \( n \) points,
|
|
so we do not learn a function that can generalize. However, we can modify the algorithm by
|
|
fitting a weak learner to approximate the negative gradient signal.
|
|
</p>
|
|
|
|
<p>Suppose we have a cost function \( C(f)=\sum_{i=0}^{n-1}L(y_i, f(x_i)) \) where \( y_i \) is our target and \( f(x_i) \) the function which is meant to model \( y_i \). The above cost function could be our standard squared-error function</p>
|
|
$$
|
|
C(\boldsymbol{y},\boldsymbol{f})=\sum_{i=0}^{n-1}(y_i-f(x_i))^2.
|
|
$$
|
|
|
|
<p>The way we proceed in an iterative fashion is to</p>
|
|
<ol>
|
|
<li> Initialize our estimate \( f_0(x) \).</li>
|
|
<li> For \( m=1:M \), we
|
|
<ol type="a"></li>
|
|
<li> compute the negative gradient vector \( \boldsymbol{u}_m = -\partial C(\boldsymbol{y},\boldsymbol{f})/\partial \boldsymbol{f}(x) \) at \( f(x) = f_{m-1}(x) \);</li>
|
|
<li> fit the so-called base-learner to the negative gradient \( h_m(u_m,x) \);</li>
|
|
<li> update the estimate \( f_m(x) = f_{m-1}(x)+h_m(u_m,x) \);</li>
|
|
</ol>
|
|
<li> The final estimate is then \( f_M(x) = \sum_{m=1}^M h_m(u_m,x) \).</li>
|
|
</ol>
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="gradient-boosting-examples-of-regression">Gradient Boosting, Examples of Regression </h2>
|
|
|
|
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
|
<div class="cell border-box-sizing code_cell rendered">
|
|
<div class="input">
|
|
<div class="inner_cell">
|
|
<div class="input_area">
|
|
<div class="highlight" style="background: #f8f8f8">
|
|
<pre style="line-height: 125%;"><span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
|
|
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
|
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.model_selection</span> <span style="color: #008000; font-weight: bold">import</span> train_test_split
|
|
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.ensemble</span> <span style="color: #008000; font-weight: bold">import</span> GradientBoostingRegressor
|
|
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">scikitplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">skplt</span>
|
|
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.metrics</span> <span style="color: #008000; font-weight: bold">import</span> mean_squared_error
|
|
|
|
n <span style="color: #666666">=</span> <span style="color: #666666">100</span>
|
|
maxdegree <span style="color: #666666">=</span> <span style="color: #666666">6</span>
|
|
|
|
<span style="color: #408080; font-style: italic"># Make data set.</span>
|
|
x <span style="color: #666666">=</span> np<span style="color: #666666">.</span>linspace(<span style="color: #666666">-3</span>, <span style="color: #666666">3</span>, n)<span style="color: #666666">.</span>reshape(<span style="color: #666666">-1</span>, <span style="color: #666666">1</span>)
|
|
y <span style="color: #666666">=</span> np<span style="color: #666666">.</span>exp(<span style="color: #666666">-</span>x<span style="color: #666666">**2</span>) <span style="color: #666666">+</span> <span style="color: #666666">1.5</span> <span style="color: #666666">*</span> np<span style="color: #666666">.</span>exp(<span style="color: #666666">-</span>(x<span style="color: #666666">-2</span>)<span style="color: #666666">**2</span>)<span style="color: #666666">+</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>normal(<span style="color: #666666">0</span>, <span style="color: #666666">0.1</span>, x<span style="color: #666666">.</span>shape)
|
|
|
|
error <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros(maxdegree)
|
|
bias <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros(maxdegree)
|
|
variance <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros(maxdegree)
|
|
polydegree <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros(maxdegree)
|
|
X_train, X_test, y_train, y_test <span style="color: #666666">=</span> train_test_split(x, y, test_size<span style="color: #666666">=0.2</span>)
|
|
|
|
<span style="color: #008000; font-weight: bold">for</span> degree <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(<span style="color: #666666">1</span>,maxdegree):
|
|
model <span style="color: #666666">=</span> GradientBoostingRegressor(max_depth<span style="color: #666666">=</span>degree, n_estimators<span style="color: #666666">=100</span>, learning_rate<span style="color: #666666">=1.0</span>)
|
|
model<span style="color: #666666">.</span>fit(X_train,y_train)
|
|
y_pred <span style="color: #666666">=</span> model<span style="color: #666666">.</span>predict(X_test)
|
|
polydegree[degree] <span style="color: #666666">=</span> degree
|
|
error[degree] <span style="color: #666666">=</span> np<span style="color: #666666">.</span>mean( np<span style="color: #666666">.</span>mean((y_test <span style="color: #666666">-</span> y_pred)<span style="color: #666666">**2</span>) )
|
|
bias[degree] <span style="color: #666666">=</span> np<span style="color: #666666">.</span>mean( (y_test <span style="color: #666666">-</span> np<span style="color: #666666">.</span>mean(y_pred))<span style="color: #666666">**2</span> )
|
|
variance[degree] <span style="color: #666666">=</span> np<span style="color: #666666">.</span>mean( np<span style="color: #666666">.</span>var(y_pred) )
|
|
<span style="color: #008000">print</span>(<span style="color: #BA2121">'Max depth:'</span>, degree)
|
|
<span style="color: #008000">print</span>(<span style="color: #BA2121">'Error:'</span>, error[degree])
|
|
<span style="color: #008000">print</span>(<span style="color: #BA2121">'Bias^2:'</span>, bias[degree])
|
|
<span style="color: #008000">print</span>(<span style="color: #BA2121">'Var:'</span>, variance[degree])
|
|
<span style="color: #008000">print</span>(<span style="color: #BA2121">'</span><span style="color: #BB6688; font-weight: bold">{}</span><span style="color: #BA2121"> >= </span><span style="color: #BB6688; font-weight: bold">{}</span><span style="color: #BA2121"> + </span><span style="color: #BB6688; font-weight: bold">{}</span><span style="color: #BA2121"> = </span><span style="color: #BB6688; font-weight: bold">{}</span><span style="color: #BA2121">'</span><span style="color: #666666">.</span>format(error[degree], bias[degree], variance[degree], bias[degree]<span style="color: #666666">+</span>variance[degree]))
|
|
|
|
plt<span style="color: #666666">.</span>xlim(<span style="color: #666666">1</span>,maxdegree<span style="color: #666666">-1</span>)
|
|
plt<span style="color: #666666">.</span>plot(polydegree, error, label<span style="color: #666666">=</span><span style="color: #BA2121">'Error'</span>)
|
|
plt<span style="color: #666666">.</span>plot(polydegree, bias, label<span style="color: #666666">=</span><span style="color: #BA2121">'bias'</span>)
|
|
plt<span style="color: #666666">.</span>plot(polydegree, variance, label<span style="color: #666666">=</span><span style="color: #BA2121">'Variance'</span>)
|
|
plt<span style="color: #666666">.</span>legend()
|
|
save_fig(<span style="color: #BA2121">"gdregression"</span>)
|
|
plt<span style="color: #666666">.</span>show()
|
|
</pre>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<div class="output_wrapper">
|
|
<div class="output">
|
|
<div class="output_area">
|
|
<div class="output_subarea output_stream output_stdout output_text">
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="gradient-boosting-classification-example">Gradient Boosting, Classification Example </h2>
|
|
|
|
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
|
<div class="cell border-box-sizing code_cell rendered">
|
|
<div class="input">
|
|
<div class="inner_cell">
|
|
<div class="input_area">
|
|
<div class="highlight" style="background: #f8f8f8">
|
|
<pre style="line-height: 125%;"><span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
|
|
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
|
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.model_selection</span> <span style="color: #008000; font-weight: bold">import</span> train_test_split
|
|
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.datasets</span> <span style="color: #008000; font-weight: bold">import</span> load_breast_cancer
|
|
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">scikitplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">skplt</span>
|
|
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.ensemble</span> <span style="color: #008000; font-weight: bold">import</span> GradientBoostingClassifier
|
|
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.model_selection</span> <span style="color: #008000; font-weight: bold">import</span> cross_validate
|
|
|
|
<span style="color: #408080; font-style: italic"># Load the data</span>
|
|
cancer <span style="color: #666666">=</span> load_breast_cancer()
|
|
|
|
X_train, X_test, y_train, y_test <span style="color: #666666">=</span> train_test_split(cancer<span style="color: #666666">.</span>data,cancer<span style="color: #666666">.</span>target,random_state<span style="color: #666666">=0</span>)
|
|
<span style="color: #008000">print</span>(X_train<span style="color: #666666">.</span>shape)
|
|
<span style="color: #008000">print</span>(X_test<span style="color: #666666">.</span>shape)
|
|
<span style="color: #408080; font-style: italic">#now scale the data</span>
|
|
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.preprocessing</span> <span style="color: #008000; font-weight: bold">import</span> StandardScaler
|
|
scaler <span style="color: #666666">=</span> StandardScaler()
|
|
scaler<span style="color: #666666">.</span>fit(X_train)
|
|
X_train_scaled <span style="color: #666666">=</span> scaler<span style="color: #666666">.</span>transform(X_train)
|
|
X_test_scaled <span style="color: #666666">=</span> scaler<span style="color: #666666">.</span>transform(X_test)
|
|
|
|
gd_clf <span style="color: #666666">=</span> GradientBoostingClassifier(max_depth<span style="color: #666666">=3</span>, n_estimators<span style="color: #666666">=100</span>, learning_rate<span style="color: #666666">=1.0</span>)
|
|
gd_clf<span style="color: #666666">.</span>fit(X_train_scaled, y_train)
|
|
<span style="color: #408080; font-style: italic">#Cross validation</span>
|
|
accuracy <span style="color: #666666">=</span> cross_validate(gd_clf,X_test_scaled,y_test,cv<span style="color: #666666">=10</span>)[<span style="color: #BA2121">'test_score'</span>]
|
|
<span style="color: #008000">print</span>(accuracy)
|
|
<span style="color: #008000">print</span>(<span style="color: #BA2121">"Test set accuracy with Random Forests and scaled data: </span><span style="color: #BB6688; font-weight: bold">{:.2f}</span><span style="color: #BA2121">"</span><span style="color: #666666">.</span>format(gd_clf<span style="color: #666666">.</span>score(X_test_scaled,y_test)))
|
|
|
|
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">scikitplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">skplt</span>
|
|
y_pred <span style="color: #666666">=</span> gd_clf<span style="color: #666666">.</span>predict(X_test_scaled)
|
|
skplt<span style="color: #666666">.</span>metrics<span style="color: #666666">.</span>plot_confusion_matrix(y_test, y_pred, normalize<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">True</span>)
|
|
save_fig(<span style="color: #BA2121">"gdclassiffierconfusion"</span>)
|
|
plt<span style="color: #666666">.</span>show()
|
|
y_probas <span style="color: #666666">=</span> gd_clf<span style="color: #666666">.</span>predict_proba(X_test_scaled)
|
|
skplt<span style="color: #666666">.</span>metrics<span style="color: #666666">.</span>plot_roc(y_test, y_probas)
|
|
save_fig(<span style="color: #BA2121">"gdclassiffierroc"</span>)
|
|
plt<span style="color: #666666">.</span>show()
|
|
skplt<span style="color: #666666">.</span>metrics<span style="color: #666666">.</span>plot_cumulative_gain(y_test, y_probas)
|
|
save_fig(<span style="color: #BA2121">"gdclassiffiercgain"</span>)
|
|
plt<span style="color: #666666">.</span>show()
|
|
</pre>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<div class="output_wrapper">
|
|
<div class="output">
|
|
<div class="output_area">
|
|
<div class="output_subarea output_stream output_stdout output_text">
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="xgboost-extreme-gradient-boosting">XGBoost: Extreme Gradient Boosting </h2>
|
|
|
|
<p><a href="https://github.com/dmlc/xgboost" target="_blank">XGBoost</a> or Extreme Gradient
|
|
Boosting, is an optimized distributed gradient boosting library
|
|
designed to be highly efficient, flexible and portable. It implements
|
|
machine learning algorithms under the Gradient Boosting
|
|
framework. XGBoost provides a parallel tree boosting that solve many
|
|
data science problems in a fast and accurate way. See the <a href="https://arxiv.org/abs/1603.02754" target="_blank">article by Chen and Guestrin</a>.
|
|
</p>
|
|
|
|
<p>The authors design and build a highly scalable end-to-end tree
|
|
boosting system. It has a theoretically justified weighted quantile
|
|
sketch for efficient proposal calculation. It introduces a novel sparsity-aware algorithm for parallel tree learning and an effective cache-aware block structure for out-of-core tree learning.
|
|
</p>
|
|
|
|
<p>It is now the algorithm which wins essentially all ML competitions!!!</p>
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="regression-case">Regression Case </h2>
|
|
|
|
|
|
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
|
<div class="cell border-box-sizing code_cell rendered">
|
|
<div class="input">
|
|
<div class="inner_cell">
|
|
<div class="input_area">
|
|
<div class="highlight" style="background: #f8f8f8">
|
|
<pre style="line-height: 125%;"><span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
|
|
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
|
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.model_selection</span> <span style="color: #008000; font-weight: bold">import</span> train_test_split
|
|
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">xgboost</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">xgb</span>
|
|
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">scikitplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">skplt</span>
|
|
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.metrics</span> <span style="color: #008000; font-weight: bold">import</span> mean_squared_error
|
|
|
|
n <span style="color: #666666">=</span> <span style="color: #666666">100</span>
|
|
maxdegree <span style="color: #666666">=</span> <span style="color: #666666">6</span>
|
|
|
|
<span style="color: #408080; font-style: italic"># Make data set.</span>
|
|
x <span style="color: #666666">=</span> np<span style="color: #666666">.</span>linspace(<span style="color: #666666">-3</span>, <span style="color: #666666">3</span>, n)<span style="color: #666666">.</span>reshape(<span style="color: #666666">-1</span>, <span style="color: #666666">1</span>)
|
|
y <span style="color: #666666">=</span> np<span style="color: #666666">.</span>exp(<span style="color: #666666">-</span>x<span style="color: #666666">**2</span>) <span style="color: #666666">+</span> <span style="color: #666666">1.5</span> <span style="color: #666666">*</span> np<span style="color: #666666">.</span>exp(<span style="color: #666666">-</span>(x<span style="color: #666666">-2</span>)<span style="color: #666666">**2</span>)<span style="color: #666666">+</span> np<span style="color: #666666">.</span>random<span style="color: #666666">.</span>normal(<span style="color: #666666">0</span>, <span style="color: #666666">0.1</span>, x<span style="color: #666666">.</span>shape)
|
|
|
|
error <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros(maxdegree)
|
|
bias <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros(maxdegree)
|
|
variance <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros(maxdegree)
|
|
polydegree <span style="color: #666666">=</span> np<span style="color: #666666">.</span>zeros(maxdegree)
|
|
X_train, X_test, y_train, y_test <span style="color: #666666">=</span> train_test_split(x, y, test_size<span style="color: #666666">=0.2</span>)
|
|
|
|
<span style="color: #008000; font-weight: bold">for</span> degree <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">range</span>(maxdegree):
|
|
model <span style="color: #666666">=</span> xgb<span style="color: #666666">.</span>XGBRegressor(objective <span style="color: #666666">=</span><span style="color: #BA2121">'reg:squarederror'</span>, colsaobjective <span style="color: #666666">=</span><span style="color: #BA2121">'reg:squarederror'</span>, colsample_bytree <span style="color: #666666">=</span> <span style="color: #666666">0.3</span>, learning_rate <span style="color: #666666">=</span> <span style="color: #666666">0.1</span>,max_depth <span style="color: #666666">=</span> degree, alpha <span style="color: #666666">=</span> <span style="color: #666666">10</span>, n_estimators <span style="color: #666666">=</span> <span style="color: #666666">200</span>)
|
|
|
|
model<span style="color: #666666">.</span>fit(X_train,y_train)
|
|
y_pred <span style="color: #666666">=</span> model<span style="color: #666666">.</span>predict(X_test)
|
|
polydegree[degree] <span style="color: #666666">=</span> degree
|
|
error[degree] <span style="color: #666666">=</span> np<span style="color: #666666">.</span>mean( np<span style="color: #666666">.</span>mean((y_test <span style="color: #666666">-</span> y_pred)<span style="color: #666666">**2</span>) )
|
|
bias[degree] <span style="color: #666666">=</span> np<span style="color: #666666">.</span>mean( (y_test <span style="color: #666666">-</span> np<span style="color: #666666">.</span>mean(y_pred))<span style="color: #666666">**2</span> )
|
|
variance[degree] <span style="color: #666666">=</span> np<span style="color: #666666">.</span>mean( np<span style="color: #666666">.</span>var(y_pred) )
|
|
<span style="color: #008000">print</span>(<span style="color: #BA2121">'Max depth:'</span>, degree)
|
|
<span style="color: #008000">print</span>(<span style="color: #BA2121">'Error:'</span>, error[degree])
|
|
<span style="color: #008000">print</span>(<span style="color: #BA2121">'Bias^2:'</span>, bias[degree])
|
|
<span style="color: #008000">print</span>(<span style="color: #BA2121">'Var:'</span>, variance[degree])
|
|
<span style="color: #008000">print</span>(<span style="color: #BA2121">'</span><span style="color: #BB6688; font-weight: bold">{}</span><span style="color: #BA2121"> >= </span><span style="color: #BB6688; font-weight: bold">{}</span><span style="color: #BA2121"> + </span><span style="color: #BB6688; font-weight: bold">{}</span><span style="color: #BA2121"> = </span><span style="color: #BB6688; font-weight: bold">{}</span><span style="color: #BA2121">'</span><span style="color: #666666">.</span>format(error[degree], bias[degree], variance[degree], bias[degree]<span style="color: #666666">+</span>variance[degree]))
|
|
|
|
plt<span style="color: #666666">.</span>xlim(<span style="color: #666666">1</span>,maxdegree<span style="color: #666666">-1</span>)
|
|
plt<span style="color: #666666">.</span>plot(polydegree, error, label<span style="color: #666666">=</span><span style="color: #BA2121">'Error'</span>)
|
|
plt<span style="color: #666666">.</span>plot(polydegree, bias, label<span style="color: #666666">=</span><span style="color: #BA2121">'bias'</span>)
|
|
plt<span style="color: #666666">.</span>plot(polydegree, variance, label<span style="color: #666666">=</span><span style="color: #BA2121">'Variance'</span>)
|
|
plt<span style="color: #666666">.</span>legend()
|
|
plt<span style="color: #666666">.</span>show()
|
|
</pre>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<div class="output_wrapper">
|
|
<div class="output">
|
|
<div class="output_area">
|
|
<div class="output_subarea output_stream output_stdout output_text">
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
|
|
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
|
<h2 id="xgboost-on-the-cancer-data">Xgboost on the Cancer Data </h2>
|
|
|
|
<p>As you will see from the confusion matrix below, XGBoots does an excellent job on the Wisconsin cancer data and outperforms essentially all agorithms we have discussed till now. </p>
|
|
|
|
<!-- code=python (!bc pycod) typeset with pygments style "default" -->
|
|
<div class="cell border-box-sizing code_cell rendered">
|
|
<div class="input">
|
|
<div class="inner_cell">
|
|
<div class="input_area">
|
|
<div class="highlight" style="background: #f8f8f8">
|
|
<pre style="line-height: 125%;"><span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">matplotlib.pyplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">plt</span>
|
|
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">numpy</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">np</span>
|
|
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.model_selection</span> <span style="color: #008000; font-weight: bold">import</span> train_test_split
|
|
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.datasets</span> <span style="color: #008000; font-weight: bold">import</span> load_breast_cancer
|
|
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.preprocessing</span> <span style="color: #008000; font-weight: bold">import</span> LabelEncoder
|
|
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.model_selection</span> <span style="color: #008000; font-weight: bold">import</span> cross_validate
|
|
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">scikitplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">skplt</span>
|
|
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">xgboost</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">xgb</span>
|
|
<span style="color: #408080; font-style: italic"># Load the data</span>
|
|
cancer <span style="color: #666666">=</span> load_breast_cancer()
|
|
|
|
X_train, X_test, y_train, y_test <span style="color: #666666">=</span> train_test_split(cancer<span style="color: #666666">.</span>data,cancer<span style="color: #666666">.</span>target,random_state<span style="color: #666666">=0</span>)
|
|
<span style="color: #008000">print</span>(X_train<span style="color: #666666">.</span>shape)
|
|
<span style="color: #008000">print</span>(X_test<span style="color: #666666">.</span>shape)
|
|
<span style="color: #408080; font-style: italic">#now scale the data</span>
|
|
<span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">sklearn.preprocessing</span> <span style="color: #008000; font-weight: bold">import</span> StandardScaler
|
|
scaler <span style="color: #666666">=</span> StandardScaler()
|
|
scaler<span style="color: #666666">.</span>fit(X_train)
|
|
X_train_scaled <span style="color: #666666">=</span> scaler<span style="color: #666666">.</span>transform(X_train)
|
|
X_test_scaled <span style="color: #666666">=</span> scaler<span style="color: #666666">.</span>transform(X_test)
|
|
|
|
xg_clf <span style="color: #666666">=</span> xgb<span style="color: #666666">.</span>XGBClassifier()
|
|
xg_clf<span style="color: #666666">.</span>fit(X_train_scaled,y_train)
|
|
|
|
y_test <span style="color: #666666">=</span> xg_clf<span style="color: #666666">.</span>predict(X_test_scaled)
|
|
|
|
<span style="color: #008000">print</span>(<span style="color: #BA2121">"Test set accuracy with Random Forests and scaled data: </span><span style="color: #BB6688; font-weight: bold">{:.2f}</span><span style="color: #BA2121">"</span><span style="color: #666666">.</span>format(xg_clf<span style="color: #666666">.</span>score(X_test_scaled,y_test)))
|
|
|
|
<span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">scikitplot</span> <span style="color: #008000; font-weight: bold">as</span> <span style="color: #0000FF; font-weight: bold">skplt</span>
|
|
y_pred <span style="color: #666666">=</span> xg_clf<span style="color: #666666">.</span>predict(X_test_scaled)
|
|
skplt<span style="color: #666666">.</span>metrics<span style="color: #666666">.</span>plot_confusion_matrix(y_test, y_pred, normalize<span style="color: #666666">=</span><span style="color: #008000; font-weight: bold">True</span>)
|
|
save_fig(<span style="color: #BA2121">"xdclassiffierconfusion"</span>)
|
|
plt<span style="color: #666666">.</span>show()
|
|
y_probas <span style="color: #666666">=</span> xg_clf<span style="color: #666666">.</span>predict_proba(X_test_scaled)
|
|
skplt<span style="color: #666666">.</span>metrics<span style="color: #666666">.</span>plot_roc(y_test, y_probas)
|
|
save_fig(<span style="color: #BA2121">"xdclassiffierroc"</span>)
|
|
plt<span style="color: #666666">.</span>show()
|
|
skplt<span style="color: #666666">.</span>metrics<span style="color: #666666">.</span>plot_cumulative_gain(y_test, y_probas)
|
|
save_fig(<span style="color: #BA2121">"gdclassiffiercgain"</span>)
|
|
plt<span style="color: #666666">.</span>show()
|
|
|
|
|
|
xgb<span style="color: #666666">.</span>plot_tree(xg_clf,num_trees<span style="color: #666666">=0</span>)
|
|
plt<span style="color: #666666">.</span>rcParams[<span style="color: #BA2121">'figure.figsize'</span>] <span style="color: #666666">=</span> [<span style="color: #666666">50</span>, <span style="color: #666666">10</span>]
|
|
save_fig(<span style="color: #BA2121">"xgtree"</span>)
|
|
plt<span style="color: #666666">.</span>show()
|
|
|
|
xgb<span style="color: #666666">.</span>plot_importance(xg_clf)
|
|
plt<span style="color: #666666">.</span>rcParams[<span style="color: #BA2121">'figure.figsize'</span>] <span style="color: #666666">=</span> [<span style="color: #666666">5</span>, <span style="color: #666666">5</span>]
|
|
save_fig(<span style="color: #BA2121">"xgparams"</span>)
|
|
plt<span style="color: #666666">.</span>show()
|
|
</pre>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
<div class="output_wrapper">
|
|
<div class="output">
|
|
<div class="output_area">
|
|
<div class="output_subarea output_stream output_stdout output_text">
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
|
|
<!-- ------------------- end of main content --------------- -->
|
|
<center style="font-size:80%">
|
|
<!-- copyright --> © 1999-2022, Morten Hjorth-Jensen. Released under CC Attribution-NonCommercial 4.0 license
|
|
</center>
|
|
</body>
|
|
</html>
|
|
|