update week 45
This commit is contained in:
mhjensen
@@ -121,81 +121,74 @@ div { text-align: justify; text-justify: inter-word; }
|
||||
('An Overview of Ensemble Methods', 2, None, '___sec37'),
|
||||
('Bagging', 2, None, '___sec38'),
|
||||
('More bagging', 2, None, '___sec39'),
|
||||
('Simple Voting Example, head or tail', 2, None, '___sec40'),
|
||||
('Using the Voting Classifier', 2, None, '___sec41'),
|
||||
('Please, not the moons again! Voting and Bagging',
|
||||
2,
|
||||
None,
|
||||
'___sec42'),
|
||||
('Bagging Examples', 2, None, '___sec43'),
|
||||
('Making your own Bootstrap: Changing the Level of the Decision '
|
||||
'Tree',
|
||||
2,
|
||||
None,
|
||||
'___sec44'),
|
||||
('Why Voting?', 2, None, '___sec45'),
|
||||
('Tossing coins', 2, None, '___sec46'),
|
||||
('Standard imports first', 2, None, '___sec47'),
|
||||
('Simple Voting Example, head or tail', 2, None, '___sec48'),
|
||||
('Using the Voting Classifier', 2, None, '___sec49'),
|
||||
('Voting and Bagging', 2, None, '___sec50'),
|
||||
('Random forests', 2, None, '___sec51'),
|
||||
('Random Forest Algorithm', 2, None, '___sec52'),
|
||||
'___sec40'),
|
||||
('Why Voting?', 2, None, '___sec41'),
|
||||
('Tossing coins', 2, None, '___sec42'),
|
||||
('Standard imports first', 2, None, '___sec43'),
|
||||
('Simple Voting Example, head or tail', 2, None, '___sec44'),
|
||||
('Using the Voting Classifier', 2, None, '___sec45'),
|
||||
('Voting and Bagging', 2, None, '___sec46'),
|
||||
('Random forests', 2, None, '___sec47'),
|
||||
('Random Forest Algorithm', 2, None, '___sec48'),
|
||||
('Random Forests Compared with other Methods on the Cancer Data',
|
||||
2,
|
||||
None,
|
||||
'___sec53'),
|
||||
'___sec49'),
|
||||
('Compare Bagging on Trees with Random Forests',
|
||||
2,
|
||||
None,
|
||||
'___sec54'),
|
||||
("Boosting, a Bird's Eye View", 2, None, '___sec55'),
|
||||
'___sec50'),
|
||||
("Boosting, a Bird's Eye View", 2, None, '___sec51'),
|
||||
('What is boosting? Additive Modelling/Iterative Fitting',
|
||||
2,
|
||||
None,
|
||||
'___sec56'),
|
||||
'___sec52'),
|
||||
('Iterative Fitting, Regression and Squared-error Cost Function',
|
||||
2,
|
||||
None,
|
||||
'___sec57'),
|
||||
'___sec53'),
|
||||
('Squared-Error Example and Iterative Fitting',
|
||||
2,
|
||||
None,
|
||||
'___sec58'),
|
||||
'___sec54'),
|
||||
('Iterative Fitting, Classification and AdaBoost',
|
||||
2,
|
||||
None,
|
||||
'___sec59'),
|
||||
('Adaptive Boosting, AdaBoost', 2, None, '___sec60'),
|
||||
('Building up AdaBoost', 2, None, '___sec61'),
|
||||
'___sec55'),
|
||||
('Adaptive Boosting, AdaBoost', 2, None, '___sec56'),
|
||||
('Building up AdaBoost', 2, None, '___sec57'),
|
||||
('Adaptive boosting: AdaBoost, Basic Algorithm',
|
||||
2,
|
||||
None,
|
||||
'___sec62'),
|
||||
('Basic Steps of AdaBoost', 2, None, '___sec63'),
|
||||
('AdaBoost Examples', 2, None, '___sec64'),
|
||||
'___sec58'),
|
||||
('Basic Steps of AdaBoost', 2, None, '___sec59'),
|
||||
('AdaBoost Examples', 2, None, '___sec60'),
|
||||
('Gradient boosting: Basics with Steepest Descent/Functional '
|
||||
'Gradient Descent',
|
||||
2,
|
||||
None,
|
||||
'___sec65'),
|
||||
'___sec61'),
|
||||
('The Squared-Error again! Steepest Descent',
|
||||
2,
|
||||
None,
|
||||
'___sec66'),
|
||||
('Steepest Descent Example', 2, None, '___sec67'),
|
||||
('Gradient Boosting, algorithm', 2, None, '___sec68'),
|
||||
'___sec62'),
|
||||
('Steepest Descent Example', 2, None, '___sec63'),
|
||||
('Gradient Boosting, algorithm', 2, None, '___sec64'),
|
||||
('Gradient Boosting, Examples of Regression',
|
||||
2,
|
||||
None,
|
||||
'___sec69'),
|
||||
'___sec65'),
|
||||
('Gradient Boosting, Classification Example',
|
||||
2,
|
||||
None,
|
||||
'___sec70'),
|
||||
('XGBoost: Extreme Gradient Boosting', 2, None, '___sec71'),
|
||||
('Regression Case', 2, None, '___sec72'),
|
||||
('Xgboost on the Cancer Data', 2, None, '___sec73')]}
|
||||
'___sec66'),
|
||||
('XGBoost: Extreme Gradient Boosting', 2, None, '___sec67'),
|
||||
('Regression Case', 2, None, '___sec68'),
|
||||
('Xgboost on the Cancer Data', 2, None, '___sec69')]}
|
||||
end of tocinfo -->
|
||||
|
||||
<body>
|
||||
@@ -1534,200 +1527,7 @@ predictor, averaged over all \( B \) trees.
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec40">Simple Voting Example, head or tail </h2>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="line-height: 125%"><span></span>heads_proba = <span style="color: #B452CD">0.51</span>
|
||||
coin_tosses = (np.random.rand(<span style="color: #B452CD">10000</span>, <span style="color: #B452CD">10</span>) < heads_proba).astype(np.int32)
|
||||
cumulative_heads_ratio = np.cumsum(coin_tosses, axis=<span style="color: #B452CD">0</span>) / np.arange(<span style="color: #B452CD">1</span>, <span style="color: #B452CD">10001</span>).reshape(-<span style="color: #B452CD">1</span>, <span style="color: #B452CD">1</span>)
|
||||
plt.figure(figsize=(<span style="color: #B452CD">8</span>,<span style="color: #B452CD">3.5</span>))
|
||||
plt.plot(cumulative_heads_ratio)
|
||||
plt.plot([<span style="color: #B452CD">0</span>, <span style="color: #B452CD">10000</span>], [<span style="color: #B452CD">0.51</span>, <span style="color: #B452CD">0.51</span>], <span style="color: #CD5555">"k--"</span>, linewidth=<span style="color: #B452CD">2</span>, label=<span style="color: #CD5555">"51%"</span>)
|
||||
plt.plot([<span style="color: #B452CD">0</span>, <span style="color: #B452CD">10000</span>], [<span style="color: #B452CD">0.5</span>, <span style="color: #B452CD">0.5</span>], <span style="color: #CD5555">"k-"</span>, label=<span style="color: #CD5555">"50%"</span>)
|
||||
plt.xlabel(<span style="color: #CD5555">"Number of coin tosses"</span>)
|
||||
plt.ylabel(<span style="color: #CD5555">"Heads ratio"</span>)
|
||||
plt.legend(loc=<span style="color: #CD5555">"lower right"</span>)
|
||||
plt.axis([<span style="color: #B452CD">0</span>, <span style="color: #B452CD">10000</span>, <span style="color: #B452CD">0.42</span>, <span style="color: #B452CD">0.58</span>])
|
||||
save_fig(<span style="color: #CD5555">"votingsimple"</span>)
|
||||
plt.show()
|
||||
</pre></div>
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec41">Using the Voting Classifier </h2>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="line-height: 125%"><span></span><span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.model_selection</span> <span style="color: #8B008B; font-weight: bold">import</span> train_test_split
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.datasets</span> <span style="color: #8B008B; font-weight: bold">import</span> make_moons
|
||||
|
||||
X, y = make_moons(n_samples=<span style="color: #B452CD">500</span>, noise=<span style="color: #B452CD">0.30</span>, random_state=<span style="color: #B452CD">42</span>)
|
||||
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=<span style="color: #B452CD">42</span>)
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.ensemble</span> <span style="color: #8B008B; font-weight: bold">import</span> RandomForestClassifier
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.ensemble</span> <span style="color: #8B008B; font-weight: bold">import</span> VotingClassifier
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.linear_model</span> <span style="color: #8B008B; font-weight: bold">import</span> LogisticRegression
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.svm</span> <span style="color: #8B008B; font-weight: bold">import</span> SVC
|
||||
|
||||
log_clf = LogisticRegression(solver=<span style="color: #CD5555">"liblinear"</span>, random_state=<span style="color: #B452CD">42</span>)
|
||||
rnd_clf = RandomForestClassifier(n_estimators=<span style="color: #B452CD">10</span>, random_state=<span style="color: #B452CD">42</span>)
|
||||
svm_clf = SVC(gamma=<span style="color: #CD5555">"auto"</span>, random_state=<span style="color: #B452CD">42</span>)
|
||||
|
||||
voting_clf = VotingClassifier(
|
||||
estimators=[(<span style="color: #CD5555">'lr'</span>, log_clf), (<span style="color: #CD5555">'rf'</span>, rnd_clf), (<span style="color: #CD5555">'svc'</span>, svm_clf)],
|
||||
voting=<span style="color: #CD5555">'hard'</span>)
|
||||
|
||||
voting_clf.fit(X_train, y_train)
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.metrics</span> <span style="color: #8B008B; font-weight: bold">import</span> accuracy_score
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> clf <span style="color: #8B008B">in</span> (log_clf, rnd_clf, svm_clf, voting_clf):
|
||||
clf.fit(X_train, y_train)
|
||||
y_pred = clf.predict(X_test)
|
||||
<span style="color: #658b00">print</span>(clf.<span style="color: #00688B">__class__</span>.<span style="color: #00688B">__name__</span>, accuracy_score(y_test, y_pred))
|
||||
|
||||
log_clf = LogisticRegression(solver=<span style="color: #CD5555">"liblinear"</span>, random_state=<span style="color: #B452CD">42</span>)
|
||||
rnd_clf = RandomForestClassifier(n_estimators=<span style="color: #B452CD">10</span>, random_state=<span style="color: #B452CD">42</span>)
|
||||
svm_clf = SVC(gamma=<span style="color: #CD5555">"auto"</span>, probability=<span style="color: #8B008B; font-weight: bold">True</span>, random_state=<span style="color: #B452CD">42</span>)
|
||||
|
||||
voting_clf = VotingClassifier(
|
||||
estimators=[(<span style="color: #CD5555">'lr'</span>, log_clf), (<span style="color: #CD5555">'rf'</span>, rnd_clf), (<span style="color: #CD5555">'svc'</span>, svm_clf)],
|
||||
voting=<span style="color: #CD5555">'soft'</span>)
|
||||
voting_clf.fit(X_train, y_train)
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.metrics</span> <span style="color: #8B008B; font-weight: bold">import</span> accuracy_score
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> clf <span style="color: #8B008B">in</span> (log_clf, rnd_clf, svm_clf, voting_clf):
|
||||
clf.fit(X_train, y_train)
|
||||
y_pred = clf.predict(X_test)
|
||||
<span style="color: #658b00">print</span>(clf.<span style="color: #00688B">__class__</span>.<span style="color: #00688B">__name__</span>, accuracy_score(y_test, y_pred))
|
||||
</pre></div>
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec42">Please, not the moons again! Voting and Bagging </h2>
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="line-height: 125%"><span></span><span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.model_selection</span> <span style="color: #8B008B; font-weight: bold">import</span> train_test_split
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.datasets</span> <span style="color: #8B008B; font-weight: bold">import</span> make_moons
|
||||
|
||||
X, y = make_moons(n_samples=<span style="color: #B452CD">500</span>, noise=<span style="color: #B452CD">0.30</span>, random_state=<span style="color: #B452CD">42</span>)
|
||||
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=<span style="color: #B452CD">42</span>)
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.ensemble</span> <span style="color: #8B008B; font-weight: bold">import</span> RandomForestClassifier
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.ensemble</span> <span style="color: #8B008B; font-weight: bold">import</span> VotingClassifier
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.linear_model</span> <span style="color: #8B008B; font-weight: bold">import</span> LogisticRegression
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.svm</span> <span style="color: #8B008B; font-weight: bold">import</span> SVC
|
||||
|
||||
log_clf = LogisticRegression(random_state=<span style="color: #B452CD">42</span>)
|
||||
rnd_clf = RandomForestClassifier(random_state=<span style="color: #B452CD">42</span>)
|
||||
svm_clf = SVC(random_state=<span style="color: #B452CD">42</span>)
|
||||
|
||||
voting_clf = VotingClassifier(
|
||||
estimators=[(<span style="color: #CD5555">'lr'</span>, log_clf), (<span style="color: #CD5555">'rf'</span>, rnd_clf), (<span style="color: #CD5555">'svc'</span>, svm_clf)],
|
||||
voting=<span style="color: #CD5555">'hard'</span>)
|
||||
voting_clf.fit(X_train, y_train)
|
||||
</pre></div>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="line-height: 125%"><span></span><span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.metrics</span> <span style="color: #8B008B; font-weight: bold">import</span> accuracy_score
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> clf <span style="color: #8B008B">in</span> (log_clf, rnd_clf, svm_clf, voting_clf):
|
||||
clf.fit(X_train, y_train)
|
||||
y_pred = clf.predict(X_test)
|
||||
<span style="color: #658b00">print</span>(clf.<span style="color: #00688B">__class__</span>.<span style="color: #00688B">__name__</span>, accuracy_score(y_test, y_pred))
|
||||
</pre></div>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="line-height: 125%"><span></span>log_clf = LogisticRegression(random_state=<span style="color: #B452CD">42</span>)
|
||||
rnd_clf = RandomForestClassifier(random_state=<span style="color: #B452CD">42</span>)
|
||||
svm_clf = SVC(probability=<span style="color: #8B008B; font-weight: bold">True</span>, random_state=<span style="color: #B452CD">42</span>)
|
||||
|
||||
voting_clf = VotingClassifier(
|
||||
estimators=[(<span style="color: #CD5555">'lr'</span>, log_clf), (<span style="color: #CD5555">'rf'</span>, rnd_clf), (<span style="color: #CD5555">'svc'</span>, svm_clf)],
|
||||
voting=<span style="color: #CD5555">'soft'</span>)
|
||||
voting_clf.fit(X_train, y_train)
|
||||
</pre></div>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="line-height: 125%"><span></span><span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.metrics</span> <span style="color: #8B008B; font-weight: bold">import</span> accuracy_score
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> clf <span style="color: #8B008B">in</span> (log_clf, rnd_clf, svm_clf, voting_clf):
|
||||
clf.fit(X_train, y_train)
|
||||
y_pred = clf.predict(X_test)
|
||||
<span style="color: #658b00">print</span>(clf.<span style="color: #00688B">__class__</span>.<span style="color: #00688B">__name__</span>, accuracy_score(y_test, y_pred))
|
||||
</pre></div>
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec43">Bagging Examples </h2>
|
||||
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="line-height: 125%"><span></span><span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.ensemble</span> <span style="color: #8B008B; font-weight: bold">import</span> BaggingClassifier
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.tree</span> <span style="color: #8B008B; font-weight: bold">import</span> DecisionTreeClassifier
|
||||
|
||||
bag_clf = BaggingClassifier(
|
||||
DecisionTreeClassifier(random_state=<span style="color: #B452CD">42</span>), n_estimators=<span style="color: #B452CD">500</span>,
|
||||
max_samples=<span style="color: #B452CD">100</span>, bootstrap=<span style="color: #8B008B; font-weight: bold">True</span>, n_jobs=-<span style="color: #B452CD">1</span>, random_state=<span style="color: #B452CD">42</span>)
|
||||
bag_clf.fit(X_train, y_train)
|
||||
y_pred = bag_clf.predict(X_test)
|
||||
</pre></div>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="line-height: 125%"><span></span><span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.metrics</span> <span style="color: #8B008B; font-weight: bold">import</span> accuracy_score
|
||||
<span style="color: #658b00">print</span>(accuracy_score(y_test, y_pred))
|
||||
</pre></div>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="line-height: 125%"><span></span>tree_clf = DecisionTreeClassifier(random_state=<span style="color: #B452CD">42</span>)
|
||||
tree_clf.fit(X_train, y_train)
|
||||
y_pred_tree = tree_clf.predict(X_test)
|
||||
<span style="color: #658b00">print</span>(accuracy_score(y_test, y_pred_tree))
|
||||
</pre></div>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
<div class="highlight" style="background: #eeeedd"><pre style="line-height: 125%"><span></span><span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">matplotlib.colors</span> <span style="color: #8B008B; font-weight: bold">import</span> ListedColormap
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">def</span> <span style="color: #008b45">plot_decision_boundary</span>(clf, X, y, axes=[-<span style="color: #B452CD">1.5</span>, <span style="color: #B452CD">2.5</span>, -<span style="color: #B452CD">1</span>, <span style="color: #B452CD">1.5</span>], alpha=<span style="color: #B452CD">0.5</span>, contour=<span style="color: #8B008B; font-weight: bold">True</span>):
|
||||
x1s = np.linspace(axes[<span style="color: #B452CD">0</span>], axes[<span style="color: #B452CD">1</span>], <span style="color: #B452CD">100</span>)
|
||||
x2s = np.linspace(axes[<span style="color: #B452CD">2</span>], axes[<span style="color: #B452CD">3</span>], <span style="color: #B452CD">100</span>)
|
||||
x1, x2 = np.meshgrid(x1s, x2s)
|
||||
X_new = np.c_[x1.ravel(), x2.ravel()]
|
||||
y_pred = clf.predict(X_new).reshape(x1.shape)
|
||||
custom_cmap = ListedColormap([<span style="color: #CD5555">'#fafab0'</span>,<span style="color: #CD5555">'#9898ff'</span>,<span style="color: #CD5555">'#a0faa0'</span>])
|
||||
plt.contourf(x1, x2, y_pred, alpha=<span style="color: #B452CD">0.3</span>, cmap=custom_cmap)
|
||||
<span style="color: #8B008B; font-weight: bold">if</span> contour:
|
||||
custom_cmap2 = ListedColormap([<span style="color: #CD5555">'#7d7d58'</span>,<span style="color: #CD5555">'#4c4c7f'</span>,<span style="color: #CD5555">'#507d50'</span>])
|
||||
plt.contour(x1, x2, y_pred, cmap=custom_cmap2, alpha=<span style="color: #B452CD">0.8</span>)
|
||||
plt.plot(X[:, <span style="color: #B452CD">0</span>][y==<span style="color: #B452CD">0</span>], X[:, <span style="color: #B452CD">1</span>][y==<span style="color: #B452CD">0</span>], <span style="color: #CD5555">"yo"</span>, alpha=alpha)
|
||||
plt.plot(X[:, <span style="color: #B452CD">0</span>][y==<span style="color: #B452CD">1</span>], X[:, <span style="color: #B452CD">1</span>][y==<span style="color: #B452CD">1</span>], <span style="color: #CD5555">"bs"</span>, alpha=alpha)
|
||||
plt.axis(axes)
|
||||
plt.xlabel(<span style="color: #CD5555">r"$x_1$"</span>, fontsize=<span style="color: #B452CD">18</span>)
|
||||
plt.ylabel(<span style="color: #CD5555">r"$x_2$"</span>, fontsize=<span style="color: #B452CD">18</span>, rotation=<span style="color: #B452CD">0</span>)
|
||||
plt.figure(figsize=(<span style="color: #B452CD">11</span>,<span style="color: #B452CD">4</span>))
|
||||
plt.subplot(<span style="color: #B452CD">121</span>)
|
||||
plot_decision_boundary(tree_clf, X, y)
|
||||
plt.title(<span style="color: #CD5555">"Decision Tree"</span>, fontsize=<span style="color: #B452CD">14</span>)
|
||||
plt.subplot(<span style="color: #B452CD">122</span>)
|
||||
plot_decision_boundary(bag_clf, X, y)
|
||||
plt.title(<span style="color: #CD5555">"Decision Trees with Bagging"</span>, fontsize=<span style="color: #B452CD">14</span>)
|
||||
save_fig(<span style="color: #CD5555">"baggingtree"</span>)
|
||||
plt.show()
|
||||
</pre></div>
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec44">Making your own Bootstrap: Changing the Level of the Decision Tree </h2>
|
||||
<h2 id="___sec40">Making your own Bootstrap: Changing the Level of the Decision Tree </h2>
|
||||
|
||||
<p>
|
||||
Let us bring up our good old boostrap example from the linear regression lectures. We change the linerar regression algorithm with
|
||||
@@ -1742,9 +1542,9 @@ a decision tree wth different depths and perform a bootstrap aggregate (in this
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.utils</span> <span style="color: #8B008B; font-weight: bold">import</span> resample
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.tree</span> <span style="color: #8B008B; font-weight: bold">import</span> DecisionTreeRegressor
|
||||
|
||||
n = <span style="color: #B452CD">100</span>
|
||||
n = <span style="color: #B452CD">1000</span>
|
||||
n_boostraps = <span style="color: #B452CD">100</span>
|
||||
maxdepth = <span style="color: #B452CD">8</span>
|
||||
maxdepth = <span style="color: #B452CD">10</span>
|
||||
|
||||
<span style="color: #228B22"># Make data set.</span>
|
||||
x = np.linspace(-<span style="color: #B452CD">3</span>, <span style="color: #B452CD">3</span>, n).reshape(-<span style="color: #B452CD">1</span>, <span style="color: #B452CD">1</span>)
|
||||
@@ -1755,23 +1555,17 @@ variance = np.zeros(maxdepth)
|
||||
polydegree = np.zeros(maxdepth)
|
||||
X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=<span style="color: #B452CD">0.2</span>)
|
||||
|
||||
<span style="color: #8B008B; font-weight: bold">from</span> <span style="color: #008b45; text-decoration: underline">sklearn.preprocessing</span> <span style="color: #8B008B; font-weight: bold">import</span> StandardScaler
|
||||
scaler = StandardScaler()
|
||||
scaler.fit(X_train)
|
||||
X_train_scaled = scaler.transform(X_train)
|
||||
X_test_scaled = scaler.transform(X_test)
|
||||
|
||||
<span style="color: #228B22"># we produce a simple tree first as benchmark</span>
|
||||
<span style="color: #228B22"># we produce a simple tree first as benchmark, no scaling</span>
|
||||
simpletree = DecisionTreeRegressor(max_depth=<span style="color: #B452CD">3</span>)
|
||||
simpletree.fit(X_train_scaled, y_train)
|
||||
simpleprediction = simpletree.predict(X_test_scaled)
|
||||
simpletree.fit(X_train, y_train)
|
||||
simpleprediction = simpletree.predict(X_test)
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> degree <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(<span style="color: #B452CD">1</span>,maxdepth):
|
||||
model = DecisionTreeRegressor(max_depth=degree)
|
||||
y_pred = np.empty((y_test.shape[<span style="color: #B452CD">0</span>], n_boostraps))
|
||||
<span style="color: #8B008B; font-weight: bold">for</span> i <span style="color: #8B008B">in</span> <span style="color: #658b00">range</span>(n_boostraps):
|
||||
x_, y_ = resample(X_train_scaled, y_train)
|
||||
x_, y_ = resample(X_train, y_train)
|
||||
model.fit(x_, y_)
|
||||
y_pred[:, i] = model.predict(X_test_scaled)<span style="color: #228B22">#.ravel()</span>
|
||||
y_pred[:, i] = model.predict(X_test)<span style="color: #228B22">#.ravel()</span>
|
||||
|
||||
polydegree[degree] = degree
|
||||
error[degree] = np.mean( np.mean((y_test - y_pred)**<span style="color: #B452CD">2</span>, axis=<span style="color: #B452CD">1</span>, keepdims=<span style="color: #8B008B; font-weight: bold">True</span>) )
|
||||
@@ -1783,7 +1577,7 @@ simpleprediction = simpletree.predict(X_test_scaled)
|
||||
<span style="color: #658b00">print</span>(<span style="color: #CD5555">'Var:'</span>, variance[degree])
|
||||
<span style="color: #658b00">print</span>(<span style="color: #CD5555">'{} >= {} + {} = {}'</span>.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree]))
|
||||
|
||||
mse_simpletree= np.mean( np.mean((y_test - simpleprediction)**<span style="color: #B452CD">2</span>)
|
||||
mse_simpletree= np.mean( np.mean((y_test - simpleprediction)**<span style="color: #B452CD">2</span>))
|
||||
<span style="color: #658b00">print</span>(mse_simpletree)
|
||||
plt.xlim(<span style="color: #B452CD">1</span>,maxdepth)
|
||||
plt.plot(polydegree, error, label=<span style="color: #CD5555">'MSE'</span>)
|
||||
@@ -1796,7 +1590,7 @@ plt.show()
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec45">Why Voting? </h2>
|
||||
<h2 id="___sec41">Why Voting? </h2>
|
||||
|
||||
<p>
|
||||
The idea behind boosting, and voting as well can be phrased as follows:
|
||||
@@ -1819,7 +1613,7 @@ Decision trees play an important role as our weak classifier. They serve as the
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec46">Tossing coins </h2>
|
||||
<h2 id="___sec42">Tossing coins </h2>
|
||||
|
||||
<p>
|
||||
The simplest case is a so-called voting ensemble. To illustrate this,
|
||||
@@ -1849,7 +1643,7 @@ numbers kicking in.
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec47">Standard imports first </h2>
|
||||
<h2 id="___sec43">Standard imports first </h2>
|
||||
|
||||
<p>
|
||||
|
||||
@@ -1895,7 +1689,7 @@ DATA_ID = <span style="color: #CD5555">"DataFiles/"</span>
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec48">Simple Voting Example, head or tail </h2>
|
||||
<h2 id="___sec44">Simple Voting Example, head or tail </h2>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
@@ -1925,7 +1719,7 @@ plt.show()
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec49">Using the Voting Classifier </h2>
|
||||
<h2 id="___sec45">Using the Voting Classifier </h2>
|
||||
|
||||
<p>
|
||||
We can use the voting classifier on other data sets, here the exciting binary case of two distinct objects using the make moons functionality of <b>Scikit-Learn</b>.
|
||||
@@ -1978,7 +1772,7 @@ voting_clf.fit(X_train, y_train)
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec50">Voting and Bagging </h2>
|
||||
<h2 id="___sec46">Voting and Bagging </h2>
|
||||
|
||||
<p>
|
||||
|
||||
@@ -2037,7 +1831,7 @@ voting_clf.fit(X_train, y_train)
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec51">Random forests </h2>
|
||||
<h2 id="___sec47">Random forests </h2>
|
||||
|
||||
<p>
|
||||
Random forests provide an improvement over bagged trees by way of a
|
||||
@@ -2081,7 +1875,7 @@ this setting.
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec52">Random Forest Algorithm </h2>
|
||||
<h2 id="___sec48">Random Forest Algorithm </h2>
|
||||
The algorithm described here can be applied to both classification and regression problems.
|
||||
|
||||
<p>
|
||||
@@ -2107,7 +1901,7 @@ We will grow of forest of say \( B \) trees.
|
||||
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec53">Random Forests Compared with other Methods on the Cancer Data </h2>
|
||||
<h2 id="___sec49">Random Forests Compared with other Methods on the Cancer Data </h2>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
@@ -2191,7 +1985,7 @@ discrimination threshold is varied. It plots the true positive rate against the
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec54">Compare Bagging on Trees with Random Forests </h2>
|
||||
<h2 id="___sec50">Compare Bagging on Trees with Random Forests </h2>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
@@ -2213,7 +2007,7 @@ np.sum(y_pred == y_pred_rf) / <span style="color: #658b00">len</span>(y_pred)
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec55">Boosting, a Bird's Eye View </h2>
|
||||
<h2 id="___sec51">Boosting, a Bird's Eye View </h2>
|
||||
|
||||
<p>
|
||||
The basic idea is to combine weak classifiers in order to create a good
|
||||
@@ -2230,7 +2024,7 @@ them with a factor.
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec56">What is boosting? Additive Modelling/Iterative Fitting </h2>
|
||||
<h2 id="___sec52">What is boosting? Additive Modelling/Iterative Fitting </h2>
|
||||
|
||||
<p>
|
||||
Boosting is a way of fitting an additive expansion in a set of
|
||||
@@ -2282,7 +2076,7 @@ In iterative fitting or additive modeling, we minimize the cost function with re
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec57">Iterative Fitting, Regression and Squared-error Cost Function </h2>
|
||||
<h2 id="___sec53">Iterative Fitting, Regression and Squared-error Cost Function </h2>
|
||||
|
||||
<p>
|
||||
The way we proceed is as follows (here we specialize to the squared-error cost function)
|
||||
@@ -2307,7 +2101,7 @@ at the internal nodes, and the predictions at the terminal nodes.
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec58">Squared-Error Example and Iterative Fitting </h2>
|
||||
<h2 id="___sec54">Squared-Error Example and Iterative Fitting </h2>
|
||||
|
||||
<p>
|
||||
To better understand what happens, let us develop the steps for the iterative fitting using the above squared error function.
|
||||
@@ -2355,7 +2149,7 @@ The solution to these two equations gives us in turn \( \beta_1 \) and \( \gamma
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec59">Iterative Fitting, Classification and AdaBoost </h2>
|
||||
<h2 id="___sec55">Iterative Fitting, Classification and AdaBoost </h2>
|
||||
|
||||
<p>
|
||||
Let us consider a binary classification problem with two outcomes \( y_i \in \{-1,1\} \) and \( i=0,1,2,\dots,n-1 \) as our set of
|
||||
@@ -2390,7 +2184,7 @@ $$
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec60">Adaptive Boosting, AdaBoost </h2>
|
||||
<h2 id="___sec56">Adaptive Boosting, AdaBoost </h2>
|
||||
|
||||
<p>
|
||||
In our iterative procedure we define thus
|
||||
@@ -2418,7 +2212,7 @@ where we have defined \( w_i^m= \exp{(-y_if_{m-1}(x_i))} \).
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec61">Building up AdaBoost </h2>
|
||||
<h2 id="___sec57">Building up AdaBoost </h2>
|
||||
|
||||
<p>
|
||||
First, for any \( \beta > 0 \), we optimize \( G \) by setting
|
||||
@@ -2462,7 +2256,7 @@ $$
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec62">Adaptive boosting: AdaBoost, Basic Algorithm </h2>
|
||||
<h2 id="___sec58">Adaptive boosting: AdaBoost, Basic Algorithm </h2>
|
||||
|
||||
<p>
|
||||
The algorithm here is rather straightforward. Assume that our weak
|
||||
@@ -2484,7 +2278,7 @@ where the function \( I() \) is one if we misclassify and zero if we classify co
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec63">Basic Steps of AdaBoost </h2>
|
||||
<h2 id="___sec59">Basic Steps of AdaBoost </h2>
|
||||
|
||||
<p>
|
||||
With the above definitions we are now ready to set up the algorithm for AdaBoost.
|
||||
@@ -2524,7 +2318,7 @@ observations that are missed in the previous iterations.
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec64">AdaBoost Examples </h2>
|
||||
<h2 id="___sec60">AdaBoost Examples </h2>
|
||||
|
||||
<p>
|
||||
Using <b>Scikit-Learn</b> it is easy to apply the adaptive boosting algorithm, as done here.
|
||||
@@ -2557,7 +2351,7 @@ plt.show()
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec65">Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent </h2>
|
||||
<h2 id="___sec61">Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent </h2>
|
||||
|
||||
<p>
|
||||
Gradient boosting is again a similar technique to Adaptive boosting,
|
||||
@@ -2572,7 +2366,7 @@ function was the least squares function.
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec66">The Squared-Error again! Steepest Descent </h2>
|
||||
<h2 id="___sec62">The Squared-Error again! Steepest Descent </h2>
|
||||
|
||||
<p>
|
||||
We start again with our cost function \( {\cal C}(\boldsymbol{y}m\boldsymbol{f})=\sum_{i=0}^{n-1}{\cal L}(y_i, f(x_i)) \) where we want to minimize
|
||||
@@ -2607,7 +2401,7 @@ $$
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec67">Steepest Descent Example </h2>
|
||||
<h2 id="___sec63">Steepest Descent Example </h2>
|
||||
|
||||
<p>
|
||||
Optimizing with respect to \( \rho \) we obtain (taking the derivative) that \( \rho_1 = -1/2 \). We have then that
|
||||
@@ -2625,7 +2419,7 @@ and find a new value for \( \rho_2=-1/2 \) and continue till we have reached \(
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec68">Gradient Boosting, algorithm </h2>
|
||||
<h2 id="___sec64">Gradient Boosting, algorithm </h2>
|
||||
|
||||
<p>
|
||||
Steepest descent is however not much used, since it only optimizes \( f \) at a fixed set of \( n \) points,
|
||||
@@ -2656,7 +2450,7 @@ The way we proceed in an iterative fashion is to
|
||||
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec69">Gradient Boosting, Examples of Regression </h2>
|
||||
<h2 id="___sec65">Gradient Boosting, Examples of Regression </h2>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
@@ -2710,7 +2504,7 @@ plt.show()
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec70">Gradient Boosting, Classification Example </h2>
|
||||
<h2 id="___sec66">Gradient Boosting, Classification Example </h2>
|
||||
<p>
|
||||
|
||||
<!-- code=python (!bc pycod) typeset with pygments style "perldoc" -->
|
||||
@@ -2758,7 +2552,7 @@ plt.show()
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec71">XGBoost: Extreme Gradient Boosting </h2>
|
||||
<h2 id="___sec67">XGBoost: Extreme Gradient Boosting </h2>
|
||||
|
||||
<p>
|
||||
<a href="https://github.com/dmlc/xgboost" target="_blank">XGBoost</a> or Extreme Gradient
|
||||
@@ -2779,7 +2573,7 @@ It is now the algorithm which wins essentially all ML competitions!!!
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec72">Regression Case </h2>
|
||||
<h2 id="___sec68">Regression Case </h2>
|
||||
|
||||
<p>
|
||||
|
||||
@@ -2834,7 +2628,7 @@ plt.show()
|
||||
<p>
|
||||
<!-- !split --><br><br><br><br><br><br><br><br><br><br>
|
||||
|
||||
<h2 id="___sec73">Xgboost on the Cancer Data </h2>
|
||||
<h2 id="___sec69">Xgboost on the Cancer Data </h2>
|
||||
|
||||
<p>
|
||||
As you will see from the confusion matrix below, XGBoots does an excellent job on the Wisconsin cancer data and outperforms essentially all agorithms we have discussed till now.
|
||||
|
||||
Reference in New Issue
Block a user