From 3c7efbf586fdfbb745e957f1011270cb945ac276 Mon Sep 17 00:00:00 2001 From: Morten Hjorth-Jensen Date: Sun, 6 Nov 2022 10:22:04 +0100 Subject: [PATCH] added code --- doc/pub/week45/html/._week45-bs000.html | 45 ++- doc/pub/week45/html/._week45-bs001.html | 45 ++- doc/pub/week45/html/._week45-bs002.html | 168 +++++++-- doc/pub/week45/html/._week45-bs003.html | 96 ++--- doc/pub/week45/html/._week45-bs004.html | 105 ++++-- doc/pub/week45/html/._week45-bs005.html | 98 ++--- doc/pub/week45/html/._week45-bs006.html | 108 +++--- doc/pub/week45/html/._week45-bs007.html | 86 +++-- doc/pub/week45/html/._week45-bs008.html | 87 ++--- doc/pub/week45/html/._week45-bs009.html | 93 +++-- doc/pub/week45/html/._week45-bs010.html | 83 ++--- doc/pub/week45/html/._week45-bs011.html | 119 +++--- doc/pub/week45/html/._week45-bs012.html | 98 +++-- doc/pub/week45/html/._week45-bs013.html | 76 ++-- doc/pub/week45/html/._week45-bs014.html | 72 ++-- doc/pub/week45/html/._week45-bs015.html | 73 ++-- doc/pub/week45/html/._week45-bs016.html | 126 +++---- doc/pub/week45/html/._week45-bs017.html | 111 +++--- doc/pub/week45/html/._week45-bs018.html | 117 ++++-- doc/pub/week45/html/._week45-bs019.html | 119 ++---- doc/pub/week45/html/._week45-bs020.html | 124 +++---- doc/pub/week45/html/._week45-bs021.html | 368 +++++-------------- doc/pub/week45/html/week45-bs.html | 45 ++- doc/pub/week45/html/week45-reveal.html | 116 ++++++ doc/pub/week45/html/week45-solarized.html | 120 ++++++ doc/pub/week45/html/week45.html | 120 ++++++ doc/pub/week45/ipynb/ipynb-week45-src.tar.gz | Bin 192 -> 192 bytes doc/pub/week45/ipynb/week45.ipynb | 298 ++++++++++----- doc/src/week45/week45.do.txt | 100 +++++ 29 files changed, 1885 insertions(+), 1331 deletions(-) diff --git a/doc/pub/week45/html/._week45-bs000.html b/doc/pub/week45/html/._week45-bs000.html index fb002f8e6..817a4baea 100644 --- a/doc/pub/week45/html/._week45-bs000.html +++ b/doc/pub/week45/html/._week45-bs000.html @@ -37,6 +37,10 @@ doconce format html week45.do.txt --html_style=bootstrap --pygments_html_style=d
  • Overview of week 45
    • -
  • Boosting, a Bird's Eye View
    • -
  • What is boosting? Additive Modelling/Iterative Fitting
    • -
  • Iterative Fitting, Regression and Squared-error Cost Function
    • -
  • Squared-Error Example and Iterative Fitting
    • -
  • Iterative Fitting, Classification and AdaBoost
    • -
  • Adaptive Boosting, AdaBoost
    • -
  • Building up AdaBoost
    • -
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • -
  • Basic Steps of AdaBoost
    • -
  • AdaBoost Examples
    • -
  • Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent
    • -
  • The Squared-Error again! Steepest Descent
    • -
  • Steepest Descent Example
    • -
  • Gradient Boosting, algorithm
    • -
  • Gradient Boosting, Examples of Regression
    • -
  • Gradient Boosting, Classification Example
    • -
  • XGBoost: Extreme Gradient Boosting
    • -
  • Regression Case
    • -
  • Xgboost on the Cancer Data
    • +
  • Brief code reminder from last wekk
    • +
  • Boosting, a Bird's Eye View
    • +
  • What is boosting? Additive Modelling/Iterative Fitting
    • +
  • Iterative Fitting, Regression and Squared-error Cost Function
    • +
  • Squared-Error Example and Iterative Fitting
    • +
  • Iterative Fitting, Classification and AdaBoost
    • +
  • Adaptive Boosting, AdaBoost
    • +
  • Building up AdaBoost
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • AdaBoost Examples
    • +
  • Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent
    • +
  • The Squared-Error again! Steepest Descent
    • +
  • Steepest Descent Example
    • +
  • Gradient Boosting, algorithm
    • +
  • Gradient Boosting, Examples of Regression
    • +
  • Gradient Boosting, Classification Example
    • +
  • XGBoost: Extreme Gradient Boosting
    • +
  • Regression Case
    • +
  • Xgboost on the Cancer Data
    • @@ -207,7 +212,7 @@ MathJax.Hub.Config({
  • 9
  • 10
  • ...
    • -
  • 21
    • +
  • 22
  • »
    • diff --git a/doc/pub/week45/html/._week45-bs001.html b/doc/pub/week45/html/._week45-bs001.html index 669ff2010..0fb9f6bc0 100644 --- a/doc/pub/week45/html/._week45-bs001.html +++ b/doc/pub/week45/html/._week45-bs001.html @@ -37,6 +37,10 @@ doconce format html week45.do.txt --html_style=bootstrap --pygments_html_style=d
  • Overview of week 45
    • -
  • Boosting, a Bird's Eye View
    • -
  • What is boosting? Additive Modelling/Iterative Fitting
    • -
  • Iterative Fitting, Regression and Squared-error Cost Function
    • -
  • Squared-Error Example and Iterative Fitting
    • -
  • Iterative Fitting, Classification and AdaBoost
    • -
  • Adaptive Boosting, AdaBoost
    • -
  • Building up AdaBoost
    • -
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • -
  • Basic Steps of AdaBoost
    • -
  • AdaBoost Examples
    • -
  • Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent
    • -
  • The Squared-Error again! Steepest Descent
    • -
  • Steepest Descent Example
    • -
  • Gradient Boosting, algorithm
    • -
  • Gradient Boosting, Examples of Regression
    • -
  • Gradient Boosting, Classification Example
    • -
  • XGBoost: Extreme Gradient Boosting
    • -
  • Regression Case
    • -
  • Xgboost on the Cancer Data
    • +
  • Brief code reminder from last wekk
    • +
  • Boosting, a Bird's Eye View
    • +
  • What is boosting? Additive Modelling/Iterative Fitting
    • +
  • Iterative Fitting, Regression and Squared-error Cost Function
    • +
  • Squared-Error Example and Iterative Fitting
    • +
  • Iterative Fitting, Classification and AdaBoost
    • +
  • Adaptive Boosting, AdaBoost
    • +
  • Building up AdaBoost
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • AdaBoost Examples
    • +
  • Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent
    • +
  • The Squared-Error again! Steepest Descent
    • +
  • Steepest Descent Example
    • +
  • Gradient Boosting, algorithm
    • +
  • Gradient Boosting, Examples of Regression
    • +
  • Gradient Boosting, Classification Example
    • +
  • XGBoost: Extreme Gradient Boosting
    • +
  • Regression Case
    • +
  • Xgboost on the Cancer Data
    • @@ -207,7 +212,7 @@ MathJax.Hub.Config({
  • 10
  • 11
  • ...
    • -
  • 21
    • +
  • 22
  • »
    • diff --git a/doc/pub/week45/html/._week45-bs002.html b/doc/pub/week45/html/._week45-bs002.html index be2cef016..3462f8c93 100644 --- a/doc/pub/week45/html/._week45-bs002.html +++ b/doc/pub/week45/html/._week45-bs002.html @@ -37,6 +37,10 @@ doconce format html week45.do.txt --html_style=bootstrap --pygments_html_style=d
  • Overview of week 45
    • -
  • Boosting, a Bird's Eye View
    • -
  • What is boosting? Additive Modelling/Iterative Fitting
    • -
  • Iterative Fitting, Regression and Squared-error Cost Function
    • -
  • Squared-Error Example and Iterative Fitting
    • -
  • Iterative Fitting, Classification and AdaBoost
    • -
  • Adaptive Boosting, AdaBoost
    • -
  • Building up AdaBoost
    • -
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • -
  • Basic Steps of AdaBoost
    • -
  • AdaBoost Examples
    • -
  • Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent
    • -
  • The Squared-Error again! Steepest Descent
    • -
  • Steepest Descent Example
    • -
  • Gradient Boosting, algorithm
    • -
  • Gradient Boosting, Examples of Regression
    • -
  • Gradient Boosting, Classification Example
    • -
  • XGBoost: Extreme Gradient Boosting
    • -
  • Regression Case
    • -
  • Xgboost on the Cancer Data
    • +
  • Brief code reminder from last wekk
    • +
  • Boosting, a Bird's Eye View
    • +
  • What is boosting? Additive Modelling/Iterative Fitting
    • +
  • Iterative Fitting, Regression and Squared-error Cost Function
    • +
  • Squared-Error Example and Iterative Fitting
    • +
  • Iterative Fitting, Classification and AdaBoost
    • +
  • Adaptive Boosting, AdaBoost
    • +
  • Building up AdaBoost
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • AdaBoost Examples
    • +
  • Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent
    • +
  • The Squared-Error again! Steepest Descent
    • +
  • Steepest Descent Example
    • +
  • Gradient Boosting, algorithm
    • +
  • Gradient Boosting, Examples of Regression
    • +
  • Gradient Boosting, Classification Example
    • +
  • XGBoost: Extreme Gradient Boosting
    • +
  • Regression Case
    • +
  • Xgboost on the Cancer Data
    • @@ -164,19 +169,120 @@ MathJax.Hub.Config({

     

     

     

    -

    Boosting, a Bird's Eye View

    +

    Brief code reminder from last wekk

    -

    The basic idea is to combine weak classifiers in order to create a good -classifier. With a weak classifier we often intend a classifier which -produces results which are only slightly better than we would get by -random guesses. -

    -

    This is done by applying in an iterative way a weak (or a standard -classifier like decision trees) to modify the data. In each iteration -we emphasize those observations which are misclassified by weighting -them with a factor. -

    + +
    +
    +
    +
    +
    + 
    # Common imports
+from IPython.display import Image 
+from pydot import graph_from_dot_data
+import pandas as pd
+import numpy as np
+import matplotlib.pyplot as plt
+from sklearn.tree import DecisionTreeClassifier
+from sklearn.tree import DecisionTreeRegressor
+from sklearn.model_selection import train_test_split
+from sklearn.tree import export_graphviz
+from sklearn.preprocessing import StandardScaler, OneHotEncoder
+from sklearn.compose import ColumnTransformer
+from IPython.display import Image 
+from pydot import graph_from_dot_data
+from sklearn.datasets import load_breast_cancer
+from sklearn.svm import SVC
+from sklearn.linear_model import LogisticRegression
+from sklearn.ensemble import BaggingClassifier
+
+import os
+
+# Where to save the figures and data files
+PROJECT_ROOT_DIR = "Results"
+FIGURE_ID = "Results/FigureFiles"
+DATA_ID = "DataFiles/"
+
+if not os.path.exists(PROJECT_ROOT_DIR):
+    os.mkdir(PROJECT_ROOT_DIR)
+
+if not os.path.exists(FIGURE_ID):
+    os.makedirs(FIGURE_ID)
+
+if not os.path.exists(DATA_ID):
+    os.makedirs(DATA_ID)
+
+def image_path(fig_id):
+    return os.path.join(FIGURE_ID, fig_id)
+
+def data_path(dat_id):
+    return os.path.join(DATA_ID, dat_id)
+
+def save_fig(fig_id):
+    plt.savefig(image_path(fig_id) + ".png", format='png')
+
+# Load the cancer data
+cancer = load_breast_cancer()
+
+X_train, X_test, y_train, y_test = train_test_split(cancer.data,cancer.target,random_state=0)
+print(X_train.shape)
+print(X_test.shape)
+#Scale the data
+from sklearn.preprocessing import StandardScaler
+scaler = StandardScaler()
+scaler.fit(X_train)
+X_train_scaled = scaler.transform(X_train)
+X_test_scaled = scaler.transform(X_test)
+#define methods
+# Logistic Regression
+logreg.fit(X_train_scaled, y_train)
+print("Test set accuracy Logistic Regression with scaled data: {:.2f}".format(logreg.score(X_test_scaled,y_test)))
+# Support Vector Machine
+svm.fit(X_train_scaled, y_train)
+print("Test set accuracy SVM with scaled data: {:.2f}".format(logreg.score(X_test_scaled,y_test)))
+# Decision Trees
+deep_tree_clf.fit(X_train_scaled, y_train)
+print("Test set accuracy with Decision Trees and scaled data: {:.2f}".format(deep_tree_clf.score(X_test_scaled,y_test)))
+
+
+from sklearn.ensemble import RandomForestClassifier
+from sklearn.preprocessing import LabelEncoder
+from sklearn.model_selection import cross_validate
+# Data set not specificied
+#Instantiate the model with 500 trees and entropy as splitting criteria
+Random_Forest_model = RandomForestClassifier(n_estimators=500,criterion="entropy")
+Random_Forest_model.fit(X_train_scaled, y_train)
+#Cross validation
+accuracy = cross_validate(Random_Forest_model,X_test_scaled,y_test,cv=10)['test_score']
+print(accuracy)
+print("Test set accuracy with Random Forests and scaled data: {:.2f}".format(Random_Forest_model.score(X_test_scaled,y_test)))
+
+
+import scikitplot as skplt
+y_pred = Random_Forest_model.predict(X_test_scaled)
+skplt.metrics.plot_confusion_matrix(y_test, y_pred, normalize=True)
+plt.show()
+y_probas = Random_Forest_model.predict_proba(X_test_scaled)
+skplt.metrics.plot_roc(y_test, y_probas)
+plt.show()
+skplt.metrics.plot_cumulative_gain(y_test, y_probas)
+plt.show()
+
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +

    @@ -195,7 +301,7 @@ them with a factor.

  • 11
  • 12
  • ...
    • -
  • 21
    • +
  • 22
  • »
    • diff --git a/doc/pub/week45/html/._week45-bs003.html b/doc/pub/week45/html/._week45-bs003.html index aae25dce3..34860ce24 100644 --- a/doc/pub/week45/html/._week45-bs003.html +++ b/doc/pub/week45/html/._week45-bs003.html @@ -37,6 +37,10 @@ doconce format html week45.do.txt --html_style=bootstrap --pygments_html_style=d
  • Overview of week 45
    • -
  • Boosting, a Bird's Eye View
    • -
  • What is boosting? Additive Modelling/Iterative Fitting
    • -
  • Iterative Fitting, Regression and Squared-error Cost Function
    • -
  • Squared-Error Example and Iterative Fitting
    • -
  • Iterative Fitting, Classification and AdaBoost
    • -
  • Adaptive Boosting, AdaBoost
    • -
  • Building up AdaBoost
    • -
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • -
  • Basic Steps of AdaBoost
    • -
  • AdaBoost Examples
    • -
  • Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent
    • -
  • The Squared-Error again! Steepest Descent
    • -
  • Steepest Descent Example
    • -
  • Gradient Boosting, algorithm
    • -
  • Gradient Boosting, Examples of Regression
    • -
  • Gradient Boosting, Classification Example
    • -
  • XGBoost: Extreme Gradient Boosting
    • -
  • Regression Case
    • -
  • Xgboost on the Cancer Data
    • +
  • Brief code reminder from last wekk
    • +
  • Boosting, a Bird's Eye View
    • +
  • What is boosting? Additive Modelling/Iterative Fitting
    • +
  • Iterative Fitting, Regression and Squared-error Cost Function
    • +
  • Squared-Error Example and Iterative Fitting
    • +
  • Iterative Fitting, Classification and AdaBoost
    • +
  • Adaptive Boosting, AdaBoost
    • +
  • Building up AdaBoost
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • AdaBoost Examples
    • +
  • Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent
    • +
  • The Squared-Error again! Steepest Descent
    • +
  • Steepest Descent Example
    • +
  • Gradient Boosting, algorithm
    • +
  • Gradient Boosting, Examples of Regression
    • +
  • Gradient Boosting, Classification Example
    • +
  • XGBoost: Extreme Gradient Boosting
    • +
  • Regression Case
    • +
  • Xgboost on the Cancer Data
    • @@ -164,53 +169,20 @@ MathJax.Hub.Config({

     

     

     

    -

    What is boosting? Additive Modelling/Iterative Fitting

    +

    Boosting, a Bird's Eye View

    -

    Boosting is a way of fitting an additive expansion in a set of -elementary basis functions like for example some simple polynomials. -Assume for example that we have a function -

    -$$ -f_M(x) = \sum_{i=1}^M \beta_m b(x;\gamma_m), -$$ - -

    where \( \beta_m \) are the expansion parameters to be determined in a -minimization process and \( b(x;\gamma_m) \) are some simple functions of -the multivariable parameter \( x \) which is characterized by the -parameters \( \gamma_m \). +

    The basic idea is to combine weak classifiers in order to create a good +classifier. With a weak classifier we often intend a classifier which +produces results which are only slightly better than we would get by +random guesses.

    -

    As an example, consider the Sigmoid function we used in logistic -regression. In that case, we can translate the function -\( b(x;\gamma_m) \) into the Sigmoid function +

    This is done by applying in an iterative way a weak (or a standard +classifier like decision trees) to modify the data. In each iteration +we emphasize those observations which are misclassified by weighting +them with a factor.

    -$$ -\sigma(t) = \frac{1}{1+\exp{(-t)}}, -$$ - -

    where \( t=\gamma_0+\gamma_1 x \) and the parameters \( \gamma_0 \) and -\( \gamma_1 \) were determined by the Logistic Regression fitting -algorithm. -

    - -

    As another example, consider the cost function we defined for linear regression

    -$$ -C(\boldsymbol{y},\boldsymbol{f}) = \frac{1}{n} \sum_{i=0}^{n-1}(y_i-f(x_i))^2. -$$ - -

    In this case the function \( f(x) \) was replaced by the design matrix -\( \boldsymbol{X} \) and the unknown linear regression parameters \( \boldsymbol{\beta} \), -that is \( \boldsymbol{f}=\boldsymbol{X}\boldsymbol{\beta} \). In linear regression we can -simply invert a matrix and obtain the parameters \( \beta \) by -

    - -$$ -\boldsymbol{\beta}=\left(\boldsymbol{X}^T\boldsymbol{X}\right)^{-1}\boldsymbol{X}^T\boldsymbol{y}. -$$ - -

    In iterative fitting or additive modeling, we minimize the cost function with respect to the parameters \( \beta_m \) and \( \gamma_m \).

    -

    diff --git a/doc/pub/week45/html/._week45-bs004.html b/doc/pub/week45/html/._week45-bs004.html index bd5f65968..525e357a6 100644 --- a/doc/pub/week45/html/._week45-bs004.html +++ b/doc/pub/week45/html/._week45-bs004.html @@ -37,6 +37,10 @@ doconce format html week45.do.txt --html_style=bootstrap --pygments_html_style=d
  • Overview of week 45
    • -
  • Boosting, a Bird's Eye View
    • -
  • What is boosting? Additive Modelling/Iterative Fitting
    • -
  • Iterative Fitting, Regression and Squared-error Cost Function
    • -
  • Squared-Error Example and Iterative Fitting
    • -
  • Iterative Fitting, Classification and AdaBoost
    • -
  • Adaptive Boosting, AdaBoost
    • -
  • Building up AdaBoost
    • -
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • -
  • Basic Steps of AdaBoost
    • -
  • AdaBoost Examples
    • -
  • Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent
    • -
  • The Squared-Error again! Steepest Descent
    • -
  • Steepest Descent Example
    • -
  • Gradient Boosting, algorithm
    • -
  • Gradient Boosting, Examples of Regression
    • -
  • Gradient Boosting, Classification Example
    • -
  • XGBoost: Extreme Gradient Boosting
    • -
  • Regression Case
    • -
  • Xgboost on the Cancer Data
    • +
  • Brief code reminder from last wekk
    • +
  • Boosting, a Bird's Eye View
    • +
  • What is boosting? Additive Modelling/Iterative Fitting
    • +
  • Iterative Fitting, Regression and Squared-error Cost Function
    • +
  • Squared-Error Example and Iterative Fitting
    • +
  • Iterative Fitting, Classification and AdaBoost
    • +
  • Adaptive Boosting, AdaBoost
    • +
  • Building up AdaBoost
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • AdaBoost Examples
    • +
  • Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent
    • +
  • The Squared-Error again! Steepest Descent
    • +
  • Steepest Descent Example
    • +
  • Gradient Boosting, algorithm
    • +
  • Gradient Boosting, Examples of Regression
    • +
  • Gradient Boosting, Classification Example
    • +
  • XGBoost: Extreme Gradient Boosting
    • +
  • Regression Case
    • +
  • Xgboost on the Cancer Data
    • @@ -164,24 +169,52 @@ MathJax.Hub.Config({

     

     

     

    -

    Iterative Fitting, Regression and Squared-error Cost Function

    +

    What is boosting? Additive Modelling/Iterative Fitting

    -

    The way we proceed is as follows (here we specialize to the squared-error cost function)

    - -
      -
    1. Establish a cost function, here \( {\cal C}(\boldsymbol{y},\boldsymbol{f}) = \frac{1}{n} \sum_{i=0}^{n-1}(y_i-f_M(x_i))^2 \) with \( f_M(x) = \sum_{i=1}^M \beta_m b(x;\gamma_m) \).
      2. -
    2. Initialize with a guess \( f_0(x) \). It could be one or even zero or some random numbers.
      3. -
    3. For \( m=1:M \) -
        -
      1. minimize \( \sum_{i=0}^{n-1}(y_i-f_{m-1}(x_i)-\beta b(x;\gamma))^2 \) wrt \( \gamma \) and \( \beta \)
        2. -
      2. This gives the optimal values \( \beta_m \) and \( \gamma_m \)
        3. -
      3. Determine then the new values \( f_m(x)=f_{m-1}(x) +\beta_m b(x;\gamma_m) \)
        4. -
      -
    -

    We could use any of the algorithms we have discussed till now. If we -use trees, \( \gamma \) parameterizes the split variables and split points -at the internal nodes, and the predictions at the terminal nodes. +

    Boosting is a way of fitting an additive expansion in a set of +elementary basis functions like for example some simple polynomials. +Assume for example that we have a function

    +$$ +f_M(x) = \sum_{i=1}^M \beta_m b(x;\gamma_m), +$$ + +

    where \( \beta_m \) are the expansion parameters to be determined in a +minimization process and \( b(x;\gamma_m) \) are some simple functions of +the multivariable parameter \( x \) which is characterized by the +parameters \( \gamma_m \). +

    + +

    As an example, consider the Sigmoid function we used in logistic +regression. In that case, we can translate the function +\( b(x;\gamma_m) \) into the Sigmoid function +

    + +$$ +\sigma(t) = \frac{1}{1+\exp{(-t)}}, +$$ + +

    where \( t=\gamma_0+\gamma_1 x \) and the parameters \( \gamma_0 \) and +\( \gamma_1 \) were determined by the Logistic Regression fitting +algorithm. +

    + +

    As another example, consider the cost function we defined for linear regression

    +$$ +C(\boldsymbol{y},\boldsymbol{f}) = \frac{1}{n} \sum_{i=0}^{n-1}(y_i-f(x_i))^2. +$$ + +

    In this case the function \( f(x) \) was replaced by the design matrix +\( \boldsymbol{X} \) and the unknown linear regression parameters \( \boldsymbol{\beta} \), +that is \( \boldsymbol{f}=\boldsymbol{X}\boldsymbol{\beta} \). In linear regression we can +simply invert a matrix and obtain the parameters \( \beta \) by +

    + +$$ +\boldsymbol{\beta}=\left(\boldsymbol{X}^T\boldsymbol{X}\right)^{-1}\boldsymbol{X}^T\boldsymbol{y}. +$$ + +

    In iterative fitting or additive modeling, we minimize the cost function with respect to the parameters \( \beta_m \) and \( \gamma_m \).

    @@ -202,7 +235,7 @@ at the internal nodes, and the predictions at the terminal nodes.

  • 13
  • 14
  • ...
    • -
  • 21
    • +
  • 22
  • »
    • diff --git a/doc/pub/week45/html/._week45-bs005.html b/doc/pub/week45/html/._week45-bs005.html index 54074b9cc..93e27fdc5 100644 --- a/doc/pub/week45/html/._week45-bs005.html +++ b/doc/pub/week45/html/._week45-bs005.html @@ -37,6 +37,10 @@ doconce format html week45.do.txt --html_style=bootstrap --pygments_html_style=d
  • Overview of week 45
    • -
  • Boosting, a Bird's Eye View
    • -
  • What is boosting? Additive Modelling/Iterative Fitting
    • -
  • Iterative Fitting, Regression and Squared-error Cost Function
    • -
  • Squared-Error Example and Iterative Fitting
    • -
  • Iterative Fitting, Classification and AdaBoost
    • -
  • Adaptive Boosting, AdaBoost
    • -
  • Building up AdaBoost
    • -
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • -
  • Basic Steps of AdaBoost
    • -
  • AdaBoost Examples
    • -
  • Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent
    • -
  • The Squared-Error again! Steepest Descent
    • -
  • Steepest Descent Example
    • -
  • Gradient Boosting, algorithm
    • -
  • Gradient Boosting, Examples of Regression
    • -
  • Gradient Boosting, Classification Example
    • -
  • XGBoost: Extreme Gradient Boosting
    • -
  • Regression Case
    • -
  • Xgboost on the Cancer Data
    • +
  • Brief code reminder from last wekk
    • +
  • Boosting, a Bird's Eye View
    • +
  • What is boosting? Additive Modelling/Iterative Fitting
    • +
  • Iterative Fitting, Regression and Squared-error Cost Function
    • +
  • Squared-Error Example and Iterative Fitting
    • +
  • Iterative Fitting, Classification and AdaBoost
    • +
  • Adaptive Boosting, AdaBoost
    • +
  • Building up AdaBoost
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • AdaBoost Examples
    • +
  • Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent
    • +
  • The Squared-Error again! Steepest Descent
    • +
  • Steepest Descent Example
    • +
  • Gradient Boosting, algorithm
    • +
  • Gradient Boosting, Examples of Regression
    • +
  • Gradient Boosting, Classification Example
    • +
  • XGBoost: Extreme Gradient Boosting
    • +
  • Regression Case
    • +
  • Xgboost on the Cancer Data
    • @@ -164,46 +169,23 @@ MathJax.Hub.Config({

     

     

     

    -

    Squared-Error Example and Iterative Fitting

    +

    Iterative Fitting, Regression and Squared-error Cost Function

    -

    To better understand what happens, let us develop the steps for the iterative fitting using the above squared error function.

    +

    The way we proceed is as follows (here we specialize to the squared-error cost function)

    -

    For simplicity we assume also that our functions \( b(x;\gamma)=1+\gamma x \).

    - -

    This means that for every iteration \( m \), we need to optimize

    - -$$ -(\beta_m,\gamma_m) = \mathrm{argmin}_{\beta,\lambda}\hspace{0.1cm} \sum_{i=0}^{n-1}(y_i-f_{m-1}(x_i)-\beta b(x;\gamma))^2=\sum_{i=0}^{n-1}(y_i-f_{m-1}(x_i)-\beta(1+\gamma x_i))^2. -$$ - -

    We start our iteration by simply setting \( f_0(x)=0 \). -Taking the derivatives with respect to \( \beta \) and \( \gamma \) we obtain -

    -$$ -\frac{\partial {\cal C}}{\partial \beta} = -2\sum_{i}(1+\gamma x_i)(y_i-\beta(1+\gamma x_i))=0, -$$ - -

    and

    -$$ -\frac{\partial {\cal C}}{\partial \gamma} =-2\sum_{i}\beta x_i(y_i-\beta(1+\gamma x_i))=0. -$$ - -

    We can then rewrite these equations as (defining \( \boldsymbol{w}=\boldsymbol{e}+\gamma \boldsymbol{x}) \) with \( \boldsymbol{e} \) being the unit vector)

    -$$ -\gamma \boldsymbol{w}^T(\boldsymbol{y}-\beta\gamma \boldsymbol{w})=0, -$$ - -

    which gives us \( \beta = \boldsymbol{w}^T\boldsymbol{y}/(\boldsymbol{w}^T\boldsymbol{w}) \). Similarly we have

    -$$ -\beta\gamma \boldsymbol{x}^T(\boldsymbol{y}-\beta(1+\gamma \boldsymbol{x}))=0, -$$ - -

    which leads to \( \gamma =(\boldsymbol{x}^T\boldsymbol{y}-\beta\boldsymbol{x}^T\boldsymbol{e})/(\beta\boldsymbol{x}^T\boldsymbol{x}) \). Inserting -for \( \beta \) gives us an equation for \( \gamma \). This is a non-linear equation in the unknown \( \gamma \) and has to be solved numerically. -

    - -

    The solution to these two equations gives us in turn \( \beta_1 \) and \( \gamma_1 \) leading to the new expression for \( f_1(x) \) as -\( f_1(x) = \beta_1(1+\gamma_1x) \). Doing this \( M \) times results in our final estimate for the function \( f \). +

      +
    1. Establish a cost function, here \( {\cal C}(\boldsymbol{y},\boldsymbol{f}) = \frac{1}{n} \sum_{i=0}^{n-1}(y_i-f_M(x_i))^2 \) with \( f_M(x) = \sum_{i=1}^M \beta_m b(x;\gamma_m) \).
      2. +
    2. Initialize with a guess \( f_0(x) \). It could be one or even zero or some random numbers.
      3. +
    3. For \( m=1:M \) +
        +
      1. minimize \( \sum_{i=0}^{n-1}(y_i-f_{m-1}(x_i)-\beta b(x;\gamma))^2 \) wrt \( \gamma \) and \( \beta \)
        2. +
      2. This gives the optimal values \( \beta_m \) and \( \gamma_m \)
        3. +
      3. Determine then the new values \( f_m(x)=f_{m-1}(x) +\beta_m b(x;\gamma_m) \)
        4. +
      +
    +

    We could use any of the algorithms we have discussed till now. If we +use trees, \( \gamma \) parameterizes the split variables and split points +at the internal nodes, and the predictions at the terminal nodes.

    @@ -226,7 +208,7 @@ for \( \beta \) gives us an equation for \( \gamma \). This is a non-linear equa

  • 14
  • 15
  • ...
    • -
  • 21
    • +
  • 22
  • »
    • diff --git a/doc/pub/week45/html/._week45-bs006.html b/doc/pub/week45/html/._week45-bs006.html index 3ee2445d1..1dc7094a6 100644 --- a/doc/pub/week45/html/._week45-bs006.html +++ b/doc/pub/week45/html/._week45-bs006.html @@ -37,6 +37,10 @@ doconce format html week45.do.txt --html_style=bootstrap --pygments_html_style=d
  • Overview of week 45
    • -
  • Boosting, a Bird's Eye View
    • -
  • What is boosting? Additive Modelling/Iterative Fitting
    • -
  • Iterative Fitting, Regression and Squared-error Cost Function
    • -
  • Squared-Error Example and Iterative Fitting
    • -
  • Iterative Fitting, Classification and AdaBoost
    • -
  • Adaptive Boosting, AdaBoost
    • -
  • Building up AdaBoost
    • -
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • -
  • Basic Steps of AdaBoost
    • -
  • AdaBoost Examples
    • -
  • Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent
    • -
  • The Squared-Error again! Steepest Descent
    • -
  • Steepest Descent Example
    • -
  • Gradient Boosting, algorithm
    • -
  • Gradient Boosting, Examples of Regression
    • -
  • Gradient Boosting, Classification Example
    • -
  • XGBoost: Extreme Gradient Boosting
    • -
  • Regression Case
    • -
  • Xgboost on the Cancer Data
    • +
  • Brief code reminder from last wekk
    • +
  • Boosting, a Bird's Eye View
    • +
  • What is boosting? Additive Modelling/Iterative Fitting
    • +
  • Iterative Fitting, Regression and Squared-error Cost Function
    • +
  • Squared-Error Example and Iterative Fitting
    • +
  • Iterative Fitting, Classification and AdaBoost
    • +
  • Adaptive Boosting, AdaBoost
    • +
  • Building up AdaBoost
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • AdaBoost Examples
    • +
  • Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent
    • +
  • The Squared-Error again! Steepest Descent
    • +
  • Steepest Descent Example
    • +
  • Gradient Boosting, algorithm
    • +
  • Gradient Boosting, Examples of Regression
    • +
  • Gradient Boosting, Classification Example
    • +
  • XGBoost: Extreme Gradient Boosting
    • +
  • Regression Case
    • +
  • Xgboost on the Cancer Data
    • @@ -164,37 +169,48 @@ MathJax.Hub.Config({

     

     

     

    -

    Iterative Fitting, Classification and AdaBoost

    +

    Squared-Error Example and Iterative Fitting

    -

    Let us consider a binary classification problem with two outcomes \( y_i \in \{-1,1\} \) and \( i=0,1,2,\dots,n-1 \) as our set of -observations. We define a classification function \( G(x) \) which produces a prediction taking one or the other of the two values -\( \{-1,1\} \). +

    To better understand what happens, let us develop the steps for the iterative fitting using the above squared error function.

    + +

    For simplicity we assume also that our functions \( b(x;\gamma)=1+\gamma x \).

    + +

    This means that for every iteration \( m \), we need to optimize

    + +$$ +(\beta_m,\gamma_m) = \mathrm{argmin}_{\beta,\lambda}\hspace{0.1cm} \sum_{i=0}^{n-1}(y_i-f_{m-1}(x_i)-\beta b(x;\gamma))^2=\sum_{i=0}^{n-1}(y_i-f_{m-1}(x_i)-\beta(1+\gamma x_i))^2. +$$ + +

    We start our iteration by simply setting \( f_0(x)=0 \). +Taking the derivatives with respect to \( \beta \) and \( \gamma \) we obtain +

    +$$ +\frac{\partial {\cal C}}{\partial \beta} = -2\sum_{i}(1+\gamma x_i)(y_i-\beta(1+\gamma x_i))=0, +$$ + +

    and

    +$$ +\frac{\partial {\cal C}}{\partial \gamma} =-2\sum_{i}\beta x_i(y_i-\beta(1+\gamma x_i))=0. +$$ + +

    We can then rewrite these equations as (defining \( \boldsymbol{w}=\boldsymbol{e}+\gamma \boldsymbol{x}) \) with \( \boldsymbol{e} \) being the unit vector)

    +$$ +\gamma \boldsymbol{w}^T(\boldsymbol{y}-\beta\gamma \boldsymbol{w})=0, +$$ + +

    which gives us \( \beta = \boldsymbol{w}^T\boldsymbol{y}/(\boldsymbol{w}^T\boldsymbol{w}) \). Similarly we have

    +$$ +\beta\gamma \boldsymbol{x}^T(\boldsymbol{y}-\beta(1+\gamma \boldsymbol{x}))=0, +$$ + +

    which leads to \( \gamma =(\boldsymbol{x}^T\boldsymbol{y}-\beta\boldsymbol{x}^T\boldsymbol{e})/(\beta\boldsymbol{x}^T\boldsymbol{x}) \). Inserting +for \( \beta \) gives us an equation for \( \gamma \). This is a non-linear equation in the unknown \( \gamma \) and has to be solved numerically.

    -

    The error rate of the training sample is then

    - -$$ -\mathrm{\overline{err}}=\frac{1}{n} \sum_{i=0}^{n-1} I(y_i\ne G(x_i)). -$$ - -

    The iterative procedure starts with defining a weak classifier whose -error rate is barely better than random guessing. The iterative -procedure in boosting is to sequentially apply a weak -classification algorithm to repeatedly modified versions of the data -producing a sequence of weak classifiers \( G_m(x) \). +

    The solution to these two equations gives us in turn \( \beta_1 \) and \( \gamma_1 \) leading to the new expression for \( f_1(x) \) as +\( f_1(x) = \beta_1(1+\gamma_1x) \). Doing this \( M \) times results in our final estimate for the function \( f \).

    -

    Here we will express our function \( f(x) \) in terms of \( G(x) \). That is

    -$$ -f_M(x) = \sum_{i=1}^M \beta_m b(x;\gamma_m), -$$ - -

    will be a function of

    -$$ -G_M(x) = \mathrm{sign} \sum_{i=1}^M \alpha_m G_m(x). -$$ - -

    diff --git a/doc/pub/week45/html/._week45-bs007.html b/doc/pub/week45/html/._week45-bs007.html index 0b17341b8..6ecf6e879 100644 --- a/doc/pub/week45/html/._week45-bs007.html +++ b/doc/pub/week45/html/._week45-bs007.html @@ -37,6 +37,10 @@ doconce format html week45.do.txt --html_style=bootstrap --pygments_html_style=d
  • Overview of week 45
    • -
  • Boosting, a Bird's Eye View
    • -
  • What is boosting? Additive Modelling/Iterative Fitting
    • -
  • Iterative Fitting, Regression and Squared-error Cost Function
    • -
  • Squared-Error Example and Iterative Fitting
    • -
  • Iterative Fitting, Classification and AdaBoost
    • -
  • Adaptive Boosting, AdaBoost
    • -
  • Building up AdaBoost
    • -
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • -
  • Basic Steps of AdaBoost
    • -
  • AdaBoost Examples
    • -
  • Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent
    • -
  • The Squared-Error again! Steepest Descent
    • -
  • Steepest Descent Example
    • -
  • Gradient Boosting, algorithm
    • -
  • Gradient Boosting, Examples of Regression
    • -
  • Gradient Boosting, Classification Example
    • -
  • XGBoost: Extreme Gradient Boosting
    • -
  • Regression Case
    • -
  • Xgboost on the Cancer Data
    • +
  • Brief code reminder from last wekk
    • +
  • Boosting, a Bird's Eye View
    • +
  • What is boosting? Additive Modelling/Iterative Fitting
    • +
  • Iterative Fitting, Regression and Squared-error Cost Function
    • +
  • Squared-Error Example and Iterative Fitting
    • +
  • Iterative Fitting, Classification and AdaBoost
    • +
  • Adaptive Boosting, AdaBoost
    • +
  • Building up AdaBoost
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • AdaBoost Examples
    • +
  • Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent
    • +
  • The Squared-Error again! Steepest Descent
    • +
  • Steepest Descent Example
    • +
  • Gradient Boosting, algorithm
    • +
  • Gradient Boosting, Examples of Regression
    • +
  • Gradient Boosting, Classification Example
    • +
  • XGBoost: Extreme Gradient Boosting
    • +
  • Regression Case
    • +
  • Xgboost on the Cancer Data
    • @@ -164,29 +169,36 @@ MathJax.Hub.Config({

     

     

     

    -

    Adaptive Boosting, AdaBoost

    +

    Iterative Fitting, Classification and AdaBoost

    -

    In our iterative procedure we define thus

    -$$ -f_m(x) = f_{m-1}(x)+\beta_mG_m(x). -$$ - -

    The simplest possible cost function which leads (also simple from a computational point of view) to the AdaBoost algorithm is the -exponential cost/loss function defined as -

    -$$ -C(\boldsymbol{y},\boldsymbol{f}) = \sum_{i=0}^{n-1}\exp{(-y_i(f_{m-1}(x_i)+\beta G(x_i))}. -$$ - -

    We optimize \( \beta \) and \( G \) for each value of \( m=1:M \) as we did in the regression case. -This is normally done in two steps. Let us however first rewrite the cost function as +

    Let us consider a binary classification problem with two outcomes \( y_i \in \{-1,1\} \) and \( i=0,1,2,\dots,n-1 \) as our set of +observations. We define a classification function \( G(x) \) which produces a prediction taking one or the other of the two values +\( \{-1,1\} \).

    +

    The error rate of the training sample is then

    + $$ -C(\boldsymbol{y},\boldsymbol{f}) = \sum_{i=0}^{n-1}w_i^{m}\exp{(-y_i\beta G(x_i))}, +\mathrm{\overline{err}}=\frac{1}{n} \sum_{i=0}^{n-1} I(y_i\ne G(x_i)). +$$ + +

    The iterative procedure starts with defining a weak classifier whose +error rate is barely better than random guessing. The iterative +procedure in boosting is to sequentially apply a weak +classification algorithm to repeatedly modified versions of the data +producing a sequence of weak classifiers \( G_m(x) \). +

    + +

    Here we will express our function \( f(x) \) in terms of \( G(x) \). That is

    +$$ +f_M(x) = \sum_{i=1}^M \beta_m b(x;\gamma_m), +$$ + +

    will be a function of

    +$$ +G_M(x) = \mathrm{sign} \sum_{i=1}^M \alpha_m G_m(x). $$ -

    where we have defined \( w_i^m= \exp{(-y_if_{m-1}(x_i))} \).

    @@ -210,7 +222,7 @@ $$

  • 16
  • 17
  • ...
    • -
  • 21
    • +
  • 22
  • »
    • diff --git a/doc/pub/week45/html/._week45-bs008.html b/doc/pub/week45/html/._week45-bs008.html index e3ac6adca..78009d5cc 100644 --- a/doc/pub/week45/html/._week45-bs008.html +++ b/doc/pub/week45/html/._week45-bs008.html @@ -37,6 +37,10 @@ doconce format html week45.do.txt --html_style=bootstrap --pygments_html_style=d
  • Overview of week 45
    • -
  • Boosting, a Bird's Eye View
    • -
  • What is boosting? Additive Modelling/Iterative Fitting
    • -
  • Iterative Fitting, Regression and Squared-error Cost Function
    • -
  • Squared-Error Example and Iterative Fitting
    • -
  • Iterative Fitting, Classification and AdaBoost
    • -
  • Adaptive Boosting, AdaBoost
    • -
  • Building up AdaBoost
    • -
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • -
  • Basic Steps of AdaBoost
    • -
  • AdaBoost Examples
    • -
  • Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent
    • -
  • The Squared-Error again! Steepest Descent
    • -
  • Steepest Descent Example
    • -
  • Gradient Boosting, algorithm
    • -
  • Gradient Boosting, Examples of Regression
    • -
  • Gradient Boosting, Classification Example
    • -
  • XGBoost: Extreme Gradient Boosting
    • -
  • Regression Case
    • -
  • Xgboost on the Cancer Data
    • +
  • Brief code reminder from last wekk
    • +
  • Boosting, a Bird's Eye View
    • +
  • What is boosting? Additive Modelling/Iterative Fitting
    • +
  • Iterative Fitting, Regression and Squared-error Cost Function
    • +
  • Squared-Error Example and Iterative Fitting
    • +
  • Iterative Fitting, Classification and AdaBoost
    • +
  • Adaptive Boosting, AdaBoost
    • +
  • Building up AdaBoost
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • AdaBoost Examples
    • +
  • Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent
    • +
  • The Squared-Error again! Steepest Descent
    • +
  • Steepest Descent Example
    • +
  • Gradient Boosting, algorithm
    • +
  • Gradient Boosting, Examples of Regression
    • +
  • Gradient Boosting, Classification Example
    • +
  • XGBoost: Extreme Gradient Boosting
    • +
  • Regression Case
    • +
  • Xgboost on the Cancer Data
    • @@ -164,45 +169,29 @@ MathJax.Hub.Config({

     

     

     

    -

    Building up AdaBoost

    +

    Adaptive Boosting, AdaBoost

    -

    First, for any \( \beta > 0 \), we optimize \( G \) by setting

    +

    In our iterative procedure we define thus

    $$ -G_m(x) = \mathrm{sign} \sum_{i=0}^{n-1} w_i^m I(y_i \ne G_(x_i)), +f_m(x) = f_{m-1}(x)+\beta_mG_m(x). $$ -

    which is the classifier that minimizes the weighted error rate in predicting \( y \).

    - -

    We can do this by rewriting

    +

    The simplest possible cost function which leads (also simple from a computational point of view) to the AdaBoost algorithm is the +exponential cost/loss function defined as +

    $$ -\exp{-(\beta)}\sum_{y_i=G(x_i)}w_i^m+\exp{(\beta)}\sum_{y_i\ne G(x_i)}w_i^m, +C(\boldsymbol{y},\boldsymbol{f}) = \sum_{i=0}^{n-1}\exp{(-y_i(f_{m-1}(x_i)+\beta G(x_i))}. $$ -

    which can be rewritten as

    +

    We optimize \( \beta \) and \( G \) for each value of \( m=1:M \) as we did in the regression case. +This is normally done in two steps. Let us however first rewrite the cost function as +

    + $$ -(\exp{(\beta)}-\exp{-(\beta)})\sum_{i=0}^{n-1}w_i^mI(y_i\ne G(x_i))+\exp{(-\beta)}\sum_{i=0}^{n-1}w_i^m=0, -$$ - -

    which leads to

    -$$ -\beta_m = \frac{1}{2}\log{\frac{1-\mathrm{\overline{err}}}{\mathrm{\overline{err}}}}, -$$ - -

    where we have redefined the error as

    -$$ -\mathrm{\overline{err}}_m=\frac{1}{n}\frac{\sum_{i=0}^{n-1}w_i^mI(y_i\ne G(x_i)}{\sum_{i=0}^{n-1}w_i^m}, -$$ - -

    which leads to an update of

    -$$ -f_m(x) = f_{m-1}(x) +\beta_m G_m(x). -$$ - -

    This leads to the new weights

    -$$ -w_i^{m+1} = w_i^m \exp{(-y_i\beta_m G_m(x_i))} +C(\boldsymbol{y},\boldsymbol{f}) = \sum_{i=0}^{n-1}w_i^{m}\exp{(-y_i\beta G(x_i))}, $$ +

    where we have defined \( w_i^m= \exp{(-y_if_{m-1}(x_i))} \).

    @@ -227,7 +216,7 @@ $$

  • 17
  • 18
  • ...
    • -
  • 21
    • +
  • 22
  • »
    • diff --git a/doc/pub/week45/html/._week45-bs009.html b/doc/pub/week45/html/._week45-bs009.html index e986e422d..03afa6a26 100644 --- a/doc/pub/week45/html/._week45-bs009.html +++ b/doc/pub/week45/html/._week45-bs009.html @@ -37,6 +37,10 @@ doconce format html week45.do.txt --html_style=bootstrap --pygments_html_style=d
  • Overview of week 45
    • -
  • Boosting, a Bird's Eye View
    • -
  • What is boosting? Additive Modelling/Iterative Fitting
    • -
  • Iterative Fitting, Regression and Squared-error Cost Function
    • -
  • Squared-Error Example and Iterative Fitting
    • -
  • Iterative Fitting, Classification and AdaBoost
    • -
  • Adaptive Boosting, AdaBoost
    • -
  • Building up AdaBoost
    • -
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • -
  • Basic Steps of AdaBoost
    • -
  • AdaBoost Examples
    • -
  • Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent
    • -
  • The Squared-Error again! Steepest Descent
    • -
  • Steepest Descent Example
    • -
  • Gradient Boosting, algorithm
    • -
  • Gradient Boosting, Examples of Regression
    • -
  • Gradient Boosting, Classification Example
    • -
  • XGBoost: Extreme Gradient Boosting
    • -
  • Regression Case
    • -
  • Xgboost on the Cancer Data
    • +
  • Brief code reminder from last wekk
    • +
  • Boosting, a Bird's Eye View
    • +
  • What is boosting? Additive Modelling/Iterative Fitting
    • +
  • Iterative Fitting, Regression and Squared-error Cost Function
    • +
  • Squared-Error Example and Iterative Fitting
    • +
  • Iterative Fitting, Classification and AdaBoost
    • +
  • Adaptive Boosting, AdaBoost
    • +
  • Building up AdaBoost
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • AdaBoost Examples
    • +
  • Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent
    • +
  • The Squared-Error again! Steepest Descent
    • +
  • Steepest Descent Example
    • +
  • Gradient Boosting, algorithm
    • +
  • Gradient Boosting, Examples of Regression
    • +
  • Gradient Boosting, Classification Example
    • +
  • XGBoost: Extreme Gradient Boosting
    • +
  • Regression Case
    • +
  • Xgboost on the Cancer Data
    • @@ -164,23 +169,45 @@ MathJax.Hub.Config({

     

     

     

    -

    Adaptive boosting: AdaBoost, Basic Algorithm

    +

    Building up AdaBoost

    -

    The algorithm here is rather straightforward. Assume that our weak -classifier is a decision tree and we consider a binary set of outputs -with \( y_i \in \{-1,1\} \) and \( i=0,1,2,\dots,n-1 \) as our set of -observations. Our design matrix is given in terms of the -feature/predictor vectors -\( \boldsymbol{X}=[\boldsymbol{x}_0\boldsymbol{x}_1\dots\boldsymbol{x}_{p-1}] \). Finally, we define also a -classifier determined by our data via a function \( G(x) \). This function tells us how well we are able to classify our outputs/targets \( \boldsymbol{y} \). -

    - -

    We have already defined the misclassification error \( \mathrm{err} \) as

    +

    First, for any \( \beta > 0 \), we optimize \( G \) by setting

    $$ -\mathrm{err}=\frac{1}{n}\sum_{i=0}^{n-1}I(y_i\ne G(x_i)), +G_m(x) = \mathrm{sign} \sum_{i=0}^{n-1} w_i^m I(y_i \ne G_(x_i)), +$$ + +

    which is the classifier that minimizes the weighted error rate in predicting \( y \).

    + +

    We can do this by rewriting

    +$$ +\exp{-(\beta)}\sum_{y_i=G(x_i)}w_i^m+\exp{(\beta)}\sum_{y_i\ne G(x_i)}w_i^m, +$$ + +

    which can be rewritten as

    +$$ +(\exp{(\beta)}-\exp{-(\beta)})\sum_{i=0}^{n-1}w_i^mI(y_i\ne G(x_i))+\exp{(-\beta)}\sum_{i=0}^{n-1}w_i^m=0, +$$ + +

    which leads to

    +$$ +\beta_m = \frac{1}{2}\log{\frac{1-\mathrm{\overline{err}}}{\mathrm{\overline{err}}}}, +$$ + +

    where we have redefined the error as

    +$$ +\mathrm{\overline{err}}_m=\frac{1}{n}\frac{\sum_{i=0}^{n-1}w_i^mI(y_i\ne G(x_i)}{\sum_{i=0}^{n-1}w_i^m}, +$$ + +

    which leads to an update of

    +$$ +f_m(x) = f_{m-1}(x) +\beta_m G_m(x). +$$ + +

    This leads to the new weights

    +$$ +w_i^{m+1} = w_i^m \exp{(-y_i\beta_m G_m(x_i))} $$ -

    where the function \( I() \) is one if we misclassify and zero if we classify correctly.

    @@ -206,7 +233,7 @@ $$

  • 18
  • 19
  • ...
    • -
  • 21
    • +
  • 22
  • »
    • diff --git a/doc/pub/week45/html/._week45-bs010.html b/doc/pub/week45/html/._week45-bs010.html index 413953c52..485370ad7 100644 --- a/doc/pub/week45/html/._week45-bs010.html +++ b/doc/pub/week45/html/._week45-bs010.html @@ -37,6 +37,10 @@ doconce format html week45.do.txt --html_style=bootstrap --pygments_html_style=d
  • Overview of week 45
    • -
  • Boosting, a Bird's Eye View
    • -
  • What is boosting? Additive Modelling/Iterative Fitting
    • -
  • Iterative Fitting, Regression and Squared-error Cost Function
    • -
  • Squared-Error Example and Iterative Fitting
    • -
  • Iterative Fitting, Classification and AdaBoost
    • -
  • Adaptive Boosting, AdaBoost
    • -
  • Building up AdaBoost
    • -
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • -
  • Basic Steps of AdaBoost
    • -
  • AdaBoost Examples
    • -
  • Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent
    • -
  • The Squared-Error again! Steepest Descent
    • -
  • Steepest Descent Example
    • -
  • Gradient Boosting, algorithm
    • -
  • Gradient Boosting, Examples of Regression
    • -
  • Gradient Boosting, Classification Example
    • -
  • XGBoost: Extreme Gradient Boosting
    • -
  • Regression Case
    • -
  • Xgboost on the Cancer Data
    • +
  • Brief code reminder from last wekk
    • +
  • Boosting, a Bird's Eye View
    • +
  • What is boosting? Additive Modelling/Iterative Fitting
    • +
  • Iterative Fitting, Regression and Squared-error Cost Function
    • +
  • Squared-Error Example and Iterative Fitting
    • +
  • Iterative Fitting, Classification and AdaBoost
    • +
  • Adaptive Boosting, AdaBoost
    • +
  • Building up AdaBoost
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • AdaBoost Examples
    • +
  • Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent
    • +
  • The Squared-Error again! Steepest Descent
    • +
  • Steepest Descent Example
    • +
  • Gradient Boosting, algorithm
    • +
  • Gradient Boosting, Examples of Regression
    • +
  • Gradient Boosting, Classification Example
    • +
  • XGBoost: Extreme Gradient Boosting
    • +
  • Regression Case
    • +
  • Xgboost on the Cancer Data
    • @@ -164,37 +169,23 @@ MathJax.Hub.Config({

     

     

     

    -

    Basic Steps of AdaBoost

    +

    Adaptive boosting: AdaBoost, Basic Algorithm

    -

    With the above definitions we are now ready to set up the algorithm for AdaBoost. -The basic idea is to set up weights which will be used to scale the correctly classified and the misclassified cases. +

    The algorithm here is rather straightforward. Assume that our weak +classifier is a decision tree and we consider a binary set of outputs +with \( y_i \in \{-1,1\} \) and \( i=0,1,2,\dots,n-1 \) as our set of +observations. Our design matrix is given in terms of the +feature/predictor vectors +\( \boldsymbol{X}=[\boldsymbol{x}_0\boldsymbol{x}_1\dots\boldsymbol{x}_{p-1}] \). Finally, we define also a +classifier determined by our data via a function \( G(x) \). This function tells us how well we are able to classify our outputs/targets \( \boldsymbol{y} \).

    -
      -
    1. We start by initializing all weights to \( w_i = 1/n \), with \( i=0,1,2,\dots n-1 \). It is easy to see that we must have \( \sum_{i=0}^{n-1}w_i = 1 \).
      2. -
    2. We rewrite the misclassification error as
      3. -
    + +

    We have already defined the misclassification error \( \mathrm{err} \) as

    $$ -\mathrm{\overline{err}}_m=\frac{\sum_{i=0}^{n-1}w_i^m I(y_i\ne G(x_i))}{\sum_{i=0}^{n-1}w_i}, +\mathrm{err}=\frac{1}{n}\sum_{i=0}^{n-1}I(y_i\ne G(x_i)), $$ -
      -
    1. Then we start looping over all attempts at classifying, namely we start an iterative process for \( m=1:M \), where \( M \) is the final number of classifications. Our given classifier could for example be a plain decision tree. -
        -
      1. Fit then a given classifier to the training set using the weights \( w_i \).
        2. -
      2. Compute then \( \mathrm{err} \) and figure out which events are classified properly and which are classified wrongly.
        3. -
      3. Define a quantity \( \alpha_{m} = \log{(1-\mathrm{\overline{err}}_m)/\mathrm{\overline{err}}_m} \)
        4. -
      4. Set the new weights to \( w_i = w_i\times \exp{(\alpha_m I(y_i\ne G(x_i)} \).
        5. -
      -
    2. Compute the new classifier \( G(x)= \sum_{i=0}^{n-1}\alpha_m I(y_i\ne G(x_i) \).
      3. -
    -

    For the iterations with \( m \le 2 \) the weights are modified -individually at each steps. The observations which were misclassified -at iteration \( m-1 \) have a weight which is larger than those which were -classified properly. As this proceeds, the observations which were -difficult to classifiy correctly are given a larger influence. Each -new classification step \( m \) is then forced to concentrate on those -observations that are missed in the previous iterations. -

    +

    where the function \( I() \) is one if we misclassify and zero if we classify correctly.

    @@ -221,7 +212,7 @@ observations that are missed in the previous iterations.

  • 19
  • 20
  • ...
    • -
  • 21
    • +
  • 22
  • »
    • diff --git a/doc/pub/week45/html/._week45-bs011.html b/doc/pub/week45/html/._week45-bs011.html index 2d6cfc8a7..42de926b5 100644 --- a/doc/pub/week45/html/._week45-bs011.html +++ b/doc/pub/week45/html/._week45-bs011.html @@ -37,6 +37,10 @@ doconce format html week45.do.txt --html_style=bootstrap --pygments_html_style=d
  • Overview of week 45
    • -
  • Boosting, a Bird's Eye View
    • -
  • What is boosting? Additive Modelling/Iterative Fitting
    • -
  • Iterative Fitting, Regression and Squared-error Cost Function
    • -
  • Squared-Error Example and Iterative Fitting
    • -
  • Iterative Fitting, Classification and AdaBoost
    • -
  • Adaptive Boosting, AdaBoost
    • -
  • Building up AdaBoost
    • -
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • -
  • Basic Steps of AdaBoost
    • -
  • AdaBoost Examples
    • -
  • Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent
    • -
  • The Squared-Error again! Steepest Descent
    • -
  • Steepest Descent Example
    • -
  • Gradient Boosting, algorithm
    • -
  • Gradient Boosting, Examples of Regression
    • -
  • Gradient Boosting, Classification Example
    • -
  • XGBoost: Extreme Gradient Boosting
    • -
  • Regression Case
    • -
  • Xgboost on the Cancer Data
    • +
  • Brief code reminder from last wekk
    • +
  • Boosting, a Bird's Eye View
    • +
  • What is boosting? Additive Modelling/Iterative Fitting
    • +
  • Iterative Fitting, Regression and Squared-error Cost Function
    • +
  • Squared-Error Example and Iterative Fitting
    • +
  • Iterative Fitting, Classification and AdaBoost
    • +
  • Adaptive Boosting, AdaBoost
    • +
  • Building up AdaBoost
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • AdaBoost Examples
    • +
  • Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent
    • +
  • The Squared-Error again! Steepest Descent
    • +
  • Steepest Descent Example
    • +
  • Gradient Boosting, algorithm
    • +
  • Gradient Boosting, Examples of Regression
    • +
  • Gradient Boosting, Classification Example
    • +
  • XGBoost: Extreme Gradient Boosting
    • +
  • Regression Case
    • +
  • Xgboost on the Cancer Data
    • @@ -164,53 +169,37 @@ MathJax.Hub.Config({

     

     

     

    -

    AdaBoost Examples

    +

    Basic Steps of AdaBoost

    -

    Using Scikit-Learn it is easy to apply the adaptive boosting algorithm, as done here.

    - - - -
    -
    -
    -
    -
    - 
    from sklearn.ensemble import AdaBoostClassifier
-
-ada_clf = AdaBoostClassifier(
-    DecisionTreeClassifier(max_depth=1), n_estimators=200,
-    algorithm="SAMME.R", learning_rate=0.5, random_state=42)
-ada_clf.fit(X_train, y_train)
-
-from sklearn.ensemble import AdaBoostClassifier
-
-ada_clf = AdaBoostClassifier(
-    DecisionTreeClassifier(max_depth=1), n_estimators=200,
-    algorithm="SAMME.R", learning_rate=0.5, random_state=42)
-ada_clf.fit(X_train_scaled, y_train)
-y_pred = ada_clf.predict(X_test_scaled)
-skplt.metrics.plot_confusion_matrix(y_test, y_pred, normalize=True)
-plt.show()
-y_probas = ada_clf.predict_proba(X_test_scaled)
-skplt.metrics.plot_roc(y_test, y_probas)
-plt.show()
-skplt.metrics.plot_cumulative_gain(y_test, y_probas)
-plt.show()
-
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    +

    With the above definitions we are now ready to set up the algorithm for AdaBoost. +The basic idea is to set up weights which will be used to scale the correctly classified and the misclassified cases. +

    +
      +
    1. We start by initializing all weights to \( w_i = 1/n \), with \( i=0,1,2,\dots n-1 \). It is easy to see that we must have \( \sum_{i=0}^{n-1}w_i = 1 \).
      2. +
    2. We rewrite the misclassification error as
      3. +
    +$$ +\mathrm{\overline{err}}_m=\frac{\sum_{i=0}^{n-1}w_i^m I(y_i\ne G(x_i))}{\sum_{i=0}^{n-1}w_i}, +$$ +
      +
    1. Then we start looping over all attempts at classifying, namely we start an iterative process for \( m=1:M \), where \( M \) is the final number of classifications. Our given classifier could for example be a plain decision tree. +
        +
      1. Fit then a given classifier to the training set using the weights \( w_i \).
        2. +
      2. Compute then \( \mathrm{err} \) and figure out which events are classified properly and which are classified wrongly.
        3. +
      3. Define a quantity \( \alpha_{m} = \log{(1-\mathrm{\overline{err}}_m)/\mathrm{\overline{err}}_m} \)
        4. +
      4. Set the new weights to \( w_i = w_i\times \exp{(\alpha_m I(y_i\ne G(x_i)} \).
        5. +
      +
    2. Compute the new classifier \( G(x)= \sum_{i=0}^{n-1}\alpha_m I(y_i\ne G(x_i) \).
      3. +
    +

    For the iterations with \( m \le 2 \) the weights are modified +individually at each steps. The observations which were misclassified +at iteration \( m-1 \) have a weight which is larger than those which were +classified properly. As this proceeds, the observations which were +difficult to classifiy correctly are given a larger influence. Each +new classification step \( m \) is then forced to concentrate on those +observations that are missed in the previous iterations. +

    @@ -236,6 +225,8 @@ plt.show()

  • 19
  • 20
  • 21
    • +
  • ...
    • +
  • 22
  • »
    • diff --git a/doc/pub/week45/html/._week45-bs012.html b/doc/pub/week45/html/._week45-bs012.html index b164b5da8..3cc30719e 100644 --- a/doc/pub/week45/html/._week45-bs012.html +++ b/doc/pub/week45/html/._week45-bs012.html @@ -37,6 +37,10 @@ doconce format html week45.do.txt --html_style=bootstrap --pygments_html_style=d
  • Overview of week 45
    • -
  • Boosting, a Bird's Eye View
    • -
  • What is boosting? Additive Modelling/Iterative Fitting
    • -
  • Iterative Fitting, Regression and Squared-error Cost Function
    • -
  • Squared-Error Example and Iterative Fitting
    • -
  • Iterative Fitting, Classification and AdaBoost
    • -
  • Adaptive Boosting, AdaBoost
    • -
  • Building up AdaBoost
    • -
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • -
  • Basic Steps of AdaBoost
    • -
  • AdaBoost Examples
    • -
  • Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent
    • -
  • The Squared-Error again! Steepest Descent
    • -
  • Steepest Descent Example
    • -
  • Gradient Boosting, algorithm
    • -
  • Gradient Boosting, Examples of Regression
    • -
  • Gradient Boosting, Classification Example
    • -
  • XGBoost: Extreme Gradient Boosting
    • -
  • Regression Case
    • -
  • Xgboost on the Cancer Data
    • +
  • Brief code reminder from last wekk
    • +
  • Boosting, a Bird's Eye View
    • +
  • What is boosting? Additive Modelling/Iterative Fitting
    • +
  • Iterative Fitting, Regression and Squared-error Cost Function
    • +
  • Squared-Error Example and Iterative Fitting
    • +
  • Iterative Fitting, Classification and AdaBoost
    • +
  • Adaptive Boosting, AdaBoost
    • +
  • Building up AdaBoost
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • AdaBoost Examples
    • +
  • Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent
    • +
  • The Squared-Error again! Steepest Descent
    • +
  • Steepest Descent Example
    • +
  • Gradient Boosting, algorithm
    • +
  • Gradient Boosting, Examples of Regression
    • +
  • Gradient Boosting, Classification Example
    • +
  • XGBoost: Extreme Gradient Boosting
    • +
  • Regression Case
    • +
  • Xgboost on the Cancer Data
    • @@ -164,17 +169,53 @@ MathJax.Hub.Config({

     

     

     

    -

    Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent

    +

    AdaBoost Examples

    -

    Gradient boosting is again a similar technique to Adaptive boosting, -it combines so-called weak classifiers or regressors into a strong -method via a series of iterations. -

    +

    Using Scikit-Learn it is easy to apply the adaptive boosting algorithm, as done here.

    + + + +
    +
    +
    +
    +
    + 
    from sklearn.ensemble import AdaBoostClassifier
+
+ada_clf = AdaBoostClassifier(
+    DecisionTreeClassifier(max_depth=1), n_estimators=200,
+    algorithm="SAMME.R", learning_rate=0.5, random_state=42)
+ada_clf.fit(X_train, y_train)
+
+from sklearn.ensemble import AdaBoostClassifier
+
+ada_clf = AdaBoostClassifier(
+    DecisionTreeClassifier(max_depth=1), n_estimators=200,
+    algorithm="SAMME.R", learning_rate=0.5, random_state=42)
+ada_clf.fit(X_train_scaled, y_train)
+y_pred = ada_clf.predict(X_test_scaled)
+skplt.metrics.plot_confusion_matrix(y_test, y_pred, normalize=True)
+plt.show()
+y_probas = ada_clf.predict_proba(X_test_scaled)
+skplt.metrics.plot_roc(y_test, y_probas)
+plt.show()
+skplt.metrics.plot_cumulative_gain(y_test, y_probas)
+plt.show()
+
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    -

    In order to understand the method, let us illustrate its basics by -bringing back the essential steps in linear regression, where our cost -function was the least squares function. -

    @@ -199,6 +240,7 @@ function was the least squares function.

  • 19
  • 20
  • 21
    • +
  • 22
  • »
    • diff --git a/doc/pub/week45/html/._week45-bs013.html b/doc/pub/week45/html/._week45-bs013.html index e698bd7a0..d6c3e240a 100644 --- a/doc/pub/week45/html/._week45-bs013.html +++ b/doc/pub/week45/html/._week45-bs013.html @@ -37,6 +37,10 @@ doconce format html week45.do.txt --html_style=bootstrap --pygments_html_style=d
  • Overview of week 45
    • -
  • Boosting, a Bird's Eye View
    • -
  • What is boosting? Additive Modelling/Iterative Fitting
    • -
  • Iterative Fitting, Regression and Squared-error Cost Function
    • -
  • Squared-Error Example and Iterative Fitting
    • -
  • Iterative Fitting, Classification and AdaBoost
    • -
  • Adaptive Boosting, AdaBoost
    • -
  • Building up AdaBoost
    • -
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • -
  • Basic Steps of AdaBoost
    • -
  • AdaBoost Examples
    • -
  • Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent
    • -
  • The Squared-Error again! Steepest Descent
    • -
  • Steepest Descent Example
    • -
  • Gradient Boosting, algorithm
    • -
  • Gradient Boosting, Examples of Regression
    • -
  • Gradient Boosting, Classification Example
    • -
  • XGBoost: Extreme Gradient Boosting
    • -
  • Regression Case
    • -
  • Xgboost on the Cancer Data
    • +
  • Brief code reminder from last wekk
    • +
  • Boosting, a Bird's Eye View
    • +
  • What is boosting? Additive Modelling/Iterative Fitting
    • +
  • Iterative Fitting, Regression and Squared-error Cost Function
    • +
  • Squared-Error Example and Iterative Fitting
    • +
  • Iterative Fitting, Classification and AdaBoost
    • +
  • Adaptive Boosting, AdaBoost
    • +
  • Building up AdaBoost
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • AdaBoost Examples
    • +
  • Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent
    • +
  • The Squared-Error again! Steepest Descent
    • +
  • Steepest Descent Example
    • +
  • Gradient Boosting, algorithm
    • +
  • Gradient Boosting, Examples of Regression
    • +
  • Gradient Boosting, Classification Example
    • +
  • XGBoost: Extreme Gradient Boosting
    • +
  • Regression Case
    • +
  • Xgboost on the Cancer Data
    • @@ -164,36 +169,18 @@ MathJax.Hub.Config({

     

     

     

    -

    The Squared-Error again! Steepest Descent

    +

    Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent

    -

    We start again with our cost function \( {\cal C}(\boldsymbol{y}m\boldsymbol{f})=\sum_{i=0}^{n-1}{\cal L}(y_i, f(x_i)) \) where we want to minimize -This means that for every iteration, we need to optimize +

    Gradient boosting is again a similar technique to Adaptive boosting, +it combines so-called weak classifiers or regressors into a strong +method via a series of iterations.

    -$$ -(\hat{\boldsymbol{f}}) = \mathrm{argmin}_{\boldsymbol{f}}\hspace{0.1cm} \sum_{i=0}^{n-1}(y_i-f(x_i))^2. -$$ - -

    We define a real function \( h_m(x) \) that defines our final function \( f_M(x) \) as

    -$$ -f_M(x) = \sum_{m=0}^M h_m(x). -$$ - -

    In the steepest decent approach we approximate \( h_m(x) = -\rho_m g_m(x) \), where \( \rho_m \) is a scalar and \( g_m(x) \) the gradient defined as

    -$$ -g_m(x_i) = \left[ \frac{\partial {\cal L}(y_i, f(x_i))}{\partial f(x_i)}\right]_{f(x_i)=f_{m-1}(x_i)}. -$$ - -

    With the new gradient we can update \( f_m(x) = f_{m-1}(x) -\rho_m g_m(x) \). Using the above squared-error function we see that -the gradient is \( g_m(x_i) = -2(y_i-f(x_i)) \). +

    In order to understand the method, let us illustrate its basics by +bringing back the essential steps in linear regression, where our cost +function was the least squares function.

    -

    Choosing \( f_0(x)=0 \) we obtain \( g_m(x) = -2y_i \) and inserting this into the minimization problem for the cost function we have

    -$$ -(\rho_1) = \mathrm{argmin}_{\rho}\hspace{0.1cm} \sum_{i=0}^{n-1}(y_i+2\rho y_i)^2. -$$ - -

    diff --git a/doc/pub/week45/html/._week45-bs014.html b/doc/pub/week45/html/._week45-bs014.html index 9f33fddf0..2d6004163 100644 --- a/doc/pub/week45/html/._week45-bs014.html +++ b/doc/pub/week45/html/._week45-bs014.html @@ -37,6 +37,10 @@ doconce format html week45.do.txt --html_style=bootstrap --pygments_html_style=d
  • Overview of week 45
    • -
  • Boosting, a Bird's Eye View
    • -
  • What is boosting? Additive Modelling/Iterative Fitting
    • -
  • Iterative Fitting, Regression and Squared-error Cost Function
    • -
  • Squared-Error Example and Iterative Fitting
    • -
  • Iterative Fitting, Classification and AdaBoost
    • -
  • Adaptive Boosting, AdaBoost
    • -
  • Building up AdaBoost
    • -
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • -
  • Basic Steps of AdaBoost
    • -
  • AdaBoost Examples
    • -
  • Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent
    • -
  • The Squared-Error again! Steepest Descent
    • -
  • Steepest Descent Example
    • -
  • Gradient Boosting, algorithm
    • -
  • Gradient Boosting, Examples of Regression
    • -
  • Gradient Boosting, Classification Example
    • -
  • XGBoost: Extreme Gradient Boosting
    • -
  • Regression Case
    • -
  • Xgboost on the Cancer Data
    • +
  • Brief code reminder from last wekk
    • +
  • Boosting, a Bird's Eye View
    • +
  • What is boosting? Additive Modelling/Iterative Fitting
    • +
  • Iterative Fitting, Regression and Squared-error Cost Function
    • +
  • Squared-Error Example and Iterative Fitting
    • +
  • Iterative Fitting, Classification and AdaBoost
    • +
  • Adaptive Boosting, AdaBoost
    • +
  • Building up AdaBoost
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • AdaBoost Examples
    • +
  • Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent
    • +
  • The Squared-Error again! Steepest Descent
    • +
  • Steepest Descent Example
    • +
  • Gradient Boosting, algorithm
    • +
  • Gradient Boosting, Examples of Regression
    • +
  • Gradient Boosting, Classification Example
    • +
  • XGBoost: Extreme Gradient Boosting
    • +
  • Regression Case
    • +
  • Xgboost on the Cancer Data
    • @@ -164,19 +169,35 @@ MathJax.Hub.Config({

     

     

     

    -

    Steepest Descent Example

    +

    The Squared-Error again! Steepest Descent

    + +

    We start again with our cost function \( {\cal C}(\boldsymbol{y}m\boldsymbol{f})=\sum_{i=0}^{n-1}{\cal L}(y_i, f(x_i)) \) where we want to minimize +This means that for every iteration, we need to optimize +

    -

    Optimizing with respect to \( \rho \) we obtain (taking the derivative) that \( \rho_1 = -1/2 \). We have then that

    $$ -f_1(x) = f_{0}(x) -\rho_1 g_1(x)=-y_i. +(\hat{\boldsymbol{f}}) = \mathrm{argmin}_{\boldsymbol{f}}\hspace{0.1cm} \sum_{i=0}^{n-1}(y_i-f(x_i))^2. $$ -

    We can then proceed and compute

    +

    We define a real function \( h_m(x) \) that defines our final function \( f_M(x) \) as

    $$ -g_2(x_i) = \left[ \frac{\partial {\cal L}(y_i, f(x_i))}{\partial f(x_i)}\right]_{f(x_i)=f_{1}(x_i)=y_i}=-4y_i, +f_M(x) = \sum_{m=0}^M h_m(x). +$$ + +

    In the steepest decent approach we approximate \( h_m(x) = -\rho_m g_m(x) \), where \( \rho_m \) is a scalar and \( g_m(x) \) the gradient defined as

    +$$ +g_m(x_i) = \left[ \frac{\partial {\cal L}(y_i, f(x_i))}{\partial f(x_i)}\right]_{f(x_i)=f_{m-1}(x_i)}. +$$ + +

    With the new gradient we can update \( f_m(x) = f_{m-1}(x) -\rho_m g_m(x) \). Using the above squared-error function we see that +the gradient is \( g_m(x_i) = -2(y_i-f(x_i)) \). +

    + +

    Choosing \( f_0(x)=0 \) we obtain \( g_m(x) = -2y_i \) and inserting this into the minimization problem for the cost function we have

    +$$ +(\rho_1) = \mathrm{argmin}_{\rho}\hspace{0.1cm} \sum_{i=0}^{n-1}(y_i+2\rho y_i)^2. $$ -

    and find a new value for \( \rho_2=-1/2 \) and continue till we have reached \( m=M \). We can modify the steepest descent method, or steepest boosting, by introducing what is called gradient boosting.

    @@ -199,6 +220,7 @@ $$

  • 19
  • 20
  • 21
    • +
  • 22
  • »
    • diff --git a/doc/pub/week45/html/._week45-bs015.html b/doc/pub/week45/html/._week45-bs015.html index 20d1bb18a..4b3b33aeb 100644 --- a/doc/pub/week45/html/._week45-bs015.html +++ b/doc/pub/week45/html/._week45-bs015.html @@ -37,6 +37,10 @@ doconce format html week45.do.txt --html_style=bootstrap --pygments_html_style=d
  • Overview of week 45
    • -
  • Boosting, a Bird's Eye View
    • -
  • What is boosting? Additive Modelling/Iterative Fitting
    • -
  • Iterative Fitting, Regression and Squared-error Cost Function
    • -
  • Squared-Error Example and Iterative Fitting
    • -
  • Iterative Fitting, Classification and AdaBoost
    • -
  • Adaptive Boosting, AdaBoost
    • -
  • Building up AdaBoost
    • -
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • -
  • Basic Steps of AdaBoost
    • -
  • AdaBoost Examples
    • -
  • Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent
    • -
  • The Squared-Error again! Steepest Descent
    • -
  • Steepest Descent Example
    • -
  • Gradient Boosting, algorithm
    • -
  • Gradient Boosting, Examples of Regression
    • -
  • Gradient Boosting, Classification Example
    • -
  • XGBoost: Extreme Gradient Boosting
    • -
  • Regression Case
    • -
  • Xgboost on the Cancer Data
    • +
  • Brief code reminder from last wekk
    • +
  • Boosting, a Bird's Eye View
    • +
  • What is boosting? Additive Modelling/Iterative Fitting
    • +
  • Iterative Fitting, Regression and Squared-error Cost Function
    • +
  • Squared-Error Example and Iterative Fitting
    • +
  • Iterative Fitting, Classification and AdaBoost
    • +
  • Adaptive Boosting, AdaBoost
    • +
  • Building up AdaBoost
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • AdaBoost Examples
    • +
  • Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent
    • +
  • The Squared-Error again! Steepest Descent
    • +
  • Steepest Descent Example
    • +
  • Gradient Boosting, algorithm
    • +
  • Gradient Boosting, Examples of Regression
    • +
  • Gradient Boosting, Classification Example
    • +
  • XGBoost: Extreme Gradient Boosting
    • +
  • Regression Case
    • +
  • Xgboost on the Cancer Data
    • @@ -164,29 +169,20 @@ MathJax.Hub.Config({

     

     

     

    -

    Gradient Boosting, algorithm

    +

    Steepest Descent Example

    -

    Steepest descent is however not much used, since it only optimizes \( f \) at a fixed set of \( n \) points, -so we do not learn a function that can generalize. However, we can modify the algorithm by -fitting a weak learner to approximate the negative gradient signal. -

    - -

    Suppose we have a cost function \( C(f)=\sum_{i=0}^{n-1}L(y_i, f(x_i)) \) where \( y_i \) is our target and \( f(x_i) \) the function which is meant to model \( y_i \). The above cost function could be our standard squared-error function

    +

    Optimizing with respect to \( \rho \) we obtain (taking the derivative) that \( \rho_1 = -1/2 \). We have then that

    $$ -C(\boldsymbol{y},\boldsymbol{f})=\sum_{i=0}^{n-1}(y_i-f(x_i))^2. +f_1(x) = f_{0}(x) -\rho_1 g_1(x)=-y_i. $$ -

    The way we proceed in an iterative fashion is to

    -
      -
    1. Initialize our estimate \( f_0(x) \).
      2. -
    2. For \( m=1:M \), we -
        -
      1. compute the negative gradient vector \( \boldsymbol{u}_m = -\partial C(\boldsymbol{y},\boldsymbol{f})/\partial \boldsymbol{f}(x) \) at \( f(x) = f_{m-1}(x) \);
        2. -
      2. fit the so-called base-learner to the negative gradient \( h_m(u_m,x) \);
        3. -
      3. update the estimate \( f_m(x) = f_{m-1}(x)+h_m(u_m,x) \);
        4. -
      -
    3. The final estimate is then \( f_M(x) = \sum_{m=1}^M h_m(u_m,x) \).
      4. -
    +

    We can then proceed and compute

    +$$ +g_2(x_i) = \left[ \frac{\partial {\cal L}(y_i, f(x_i))}{\partial f(x_i)}\right]_{f(x_i)=f_{1}(x_i)=y_i}=-4y_i, +$$ + +

    and find a new value for \( \rho_2=-1/2 \) and continue till we have reached \( m=M \). We can modify the steepest descent method, or steepest boosting, by introducing what is called gradient boosting.

    +

    diff --git a/doc/pub/week45/html/._week45-bs016.html b/doc/pub/week45/html/._week45-bs016.html index 2c87de64f..53c864ee1 100644 --- a/doc/pub/week45/html/._week45-bs016.html +++ b/doc/pub/week45/html/._week45-bs016.html @@ -37,6 +37,10 @@ doconce format html week45.do.txt --html_style=bootstrap --pygments_html_style=d
  • Overview of week 45
    • -
  • Boosting, a Bird's Eye View
    • -
  • What is boosting? Additive Modelling/Iterative Fitting
    • -
  • Iterative Fitting, Regression and Squared-error Cost Function
    • -
  • Squared-Error Example and Iterative Fitting
    • -
  • Iterative Fitting, Classification and AdaBoost
    • -
  • Adaptive Boosting, AdaBoost
    • -
  • Building up AdaBoost
    • -
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • -
  • Basic Steps of AdaBoost
    • -
  • AdaBoost Examples
    • -
  • Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent
    • -
  • The Squared-Error again! Steepest Descent
    • -
  • Steepest Descent Example
    • -
  • Gradient Boosting, algorithm
    • -
  • Gradient Boosting, Examples of Regression
    • -
  • Gradient Boosting, Classification Example
    • -
  • XGBoost: Extreme Gradient Boosting
    • -
  • Regression Case
    • -
  • Xgboost on the Cancer Data
    • +
  • Brief code reminder from last wekk
    • +
  • Boosting, a Bird's Eye View
    • +
  • What is boosting? Additive Modelling/Iterative Fitting
    • +
  • Iterative Fitting, Regression and Squared-error Cost Function
    • +
  • Squared-Error Example and Iterative Fitting
    • +
  • Iterative Fitting, Classification and AdaBoost
    • +
  • Adaptive Boosting, AdaBoost
    • +
  • Building up AdaBoost
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • AdaBoost Examples
    • +
  • Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent
    • +
  • The Squared-Error again! Steepest Descent
    • +
  • Steepest Descent Example
    • +
  • Gradient Boosting, algorithm
    • +
  • Gradient Boosting, Examples of Regression
    • +
  • Gradient Boosting, Classification Example
    • +
  • XGBoost: Extreme Gradient Boosting
    • +
  • Regression Case
    • +
  • Xgboost on the Cancer Data
    • @@ -164,71 +169,29 @@ MathJax.Hub.Config({

     

     

     

    -

    Gradient Boosting, Examples of Regression

    +

    Gradient Boosting, algorithm

    - -
    -
    -
    -
    -
    - 
    import matplotlib.pyplot as plt
-import numpy as np
-from sklearn.model_selection import train_test_split
-from sklearn.ensemble import GradientBoostingRegressor
-import scikitplot as skplt
-from sklearn.metrics import mean_squared_error
-
-n = 100
-maxdegree = 6
-
-# Make data set.
-x = np.linspace(-3, 3, n).reshape(-1, 1)
-y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)
-
-error = np.zeros(maxdegree)
-bias = np.zeros(maxdegree)
-variance = np.zeros(maxdegree)
-polydegree = np.zeros(maxdegree)
-X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.2)
-
-for degree in range(1,maxdegree):
-    model = GradientBoostingRegressor(max_depth=degree, n_estimators=100, learning_rate=1.0)  
-    model.fit(X_train,y_train)
-    y_pred = model.predict(X_test)
-    polydegree[degree] = degree
-    error[degree] = np.mean( np.mean((y_test - y_pred)**2) )
-    bias[degree] = np.mean( (y_test - np.mean(y_pred))**2 )
-    variance[degree] = np.mean( np.var(y_pred) )
-    print('Max depth:', degree)
-    print('Error:', error[degree])
-    print('Bias^2:', bias[degree])
-    print('Var:', variance[degree])
-    print('{} >= {} + {} = {}'.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree]))
-
-plt.xlim(1,maxdegree-1)
-plt.plot(polydegree, error, label='Error')
-plt.plot(polydegree, bias, label='bias')
-plt.plot(polydegree, variance, label='Variance')
-plt.legend()
-save_fig("gdregression")
-plt.show()
-
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    +

    Steepest descent is however not much used, since it only optimizes \( f \) at a fixed set of \( n \) points, +so we do not learn a function that can generalize. However, we can modify the algorithm by +fitting a weak learner to approximate the negative gradient signal. +

    +

    Suppose we have a cost function \( C(f)=\sum_{i=0}^{n-1}L(y_i, f(x_i)) \) where \( y_i \) is our target and \( f(x_i) \) the function which is meant to model \( y_i \). The above cost function could be our standard squared-error function

    +$$ +C(\boldsymbol{y},\boldsymbol{f})=\sum_{i=0}^{n-1}(y_i-f(x_i))^2. +$$ +

    The way we proceed in an iterative fashion is to

    +
      +
    1. Initialize our estimate \( f_0(x) \).
      2. +
    2. For \( m=1:M \), we +
        +
      1. compute the negative gradient vector \( \boldsymbol{u}_m = -\partial C(\boldsymbol{y},\boldsymbol{f})/\partial \boldsymbol{f}(x) \) at \( f(x) = f_{m-1}(x) \);
        2. +
      2. fit the so-called base-learner to the negative gradient \( h_m(u_m,x) \);
        3. +
      3. update the estimate \( f_m(x) = f_{m-1}(x)+h_m(u_m,x) \);
        4. +
      +
    3. The final estimate is then \( f_M(x) = \sum_{m=1}^M h_m(u_m,x) \).
      4. +

    diff --git a/doc/pub/week45/html/._week45-bs017.html b/doc/pub/week45/html/._week45-bs017.html index 6215df638..68a0e7d2d 100644 --- a/doc/pub/week45/html/._week45-bs017.html +++ b/doc/pub/week45/html/._week45-bs017.html @@ -37,6 +37,10 @@ doconce format html week45.do.txt --html_style=bootstrap --pygments_html_style=d
  • Overview of week 45
    • -
  • Boosting, a Bird's Eye View
    • -
  • What is boosting? Additive Modelling/Iterative Fitting
    • -
  • Iterative Fitting, Regression and Squared-error Cost Function
    • -
  • Squared-Error Example and Iterative Fitting
    • -
  • Iterative Fitting, Classification and AdaBoost
    • -
  • Adaptive Boosting, AdaBoost
    • -
  • Building up AdaBoost
    • -
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • -
  • Basic Steps of AdaBoost
    • -
  • AdaBoost Examples
    • -
  • Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent
    • -
  • The Squared-Error again! Steepest Descent
    • -
  • Steepest Descent Example
    • -
  • Gradient Boosting, algorithm
    • -
  • Gradient Boosting, Examples of Regression
    • -
  • Gradient Boosting, Classification Example
    • -
  • XGBoost: Extreme Gradient Boosting
    • -
  • Regression Case
    • -
  • Xgboost on the Cancer Data
    • +
  • Brief code reminder from last wekk
    • +
  • Boosting, a Bird's Eye View
    • +
  • What is boosting? Additive Modelling/Iterative Fitting
    • +
  • Iterative Fitting, Regression and Squared-error Cost Function
    • +
  • Squared-Error Example and Iterative Fitting
    • +
  • Iterative Fitting, Classification and AdaBoost
    • +
  • Adaptive Boosting, AdaBoost
    • +
  • Building up AdaBoost
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • AdaBoost Examples
    • +
  • Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent
    • +
  • The Squared-Error again! Steepest Descent
    • +
  • Steepest Descent Example
    • +
  • Gradient Boosting, algorithm
    • +
  • Gradient Boosting, Examples of Regression
    • +
  • Gradient Boosting, Classification Example
    • +
  • XGBoost: Extreme Gradient Boosting
    • +
  • Regression Case
    • +
  • Xgboost on the Cancer Data
    • @@ -164,7 +169,7 @@ MathJax.Hub.Config({

     

     

     

    -

    Gradient Boosting, Classification Example

    +

    Gradient Boosting, Examples of Regression

    @@ -174,43 +179,44 @@ MathJax.Hub.Config({ 
    import matplotlib.pyplot as plt
 import numpy as np
-from sklearn.model_selection import  train_test_split 
-from sklearn.datasets import load_breast_cancer
+from sklearn.model_selection import train_test_split
+from sklearn.ensemble import GradientBoostingRegressor
 import scikitplot as skplt
-from sklearn.ensemble import GradientBoostingClassifier
-from sklearn.model_selection import cross_validate
+from sklearn.metrics import mean_squared_error
 
-# Load the data
-cancer = load_breast_cancer()
+n = 100
+maxdegree = 6
 
-X_train, X_test, y_train, y_test = train_test_split(cancer.data,cancer.target,random_state=0)
-print(X_train.shape)
-print(X_test.shape)
-#now scale the data
-from sklearn.preprocessing import StandardScaler
-scaler = StandardScaler()
-scaler.fit(X_train)
-X_train_scaled = scaler.transform(X_train)
-X_test_scaled = scaler.transform(X_test)
+# Make data set.
+x = np.linspace(-3, 3, n).reshape(-1, 1)
+y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)
 
-gd_clf = GradientBoostingClassifier(max_depth=3, n_estimators=100, learning_rate=1.0)  
-gd_clf.fit(X_train_scaled, y_train)
-#Cross validation
-accuracy = cross_validate(gd_clf,X_test_scaled,y_test,cv=10)['test_score']
-print(accuracy)
-print("Test set accuracy with Random Forests and scaled data: {:.2f}".format(gd_clf.score(X_test_scaled,y_test)))
+error = np.zeros(maxdegree)
+bias = np.zeros(maxdegree)
+variance = np.zeros(maxdegree)
+polydegree = np.zeros(maxdegree)
+X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.2)
 
-import scikitplot as skplt
-y_pred = gd_clf.predict(X_test_scaled)
-skplt.metrics.plot_confusion_matrix(y_test, y_pred, normalize=True)
-save_fig("gdclassiffierconfusion")
-plt.show()
-y_probas = gd_clf.predict_proba(X_test_scaled)
-skplt.metrics.plot_roc(y_test, y_probas)
-save_fig("gdclassiffierroc")
-plt.show()
-skplt.metrics.plot_cumulative_gain(y_test, y_probas)
-save_fig("gdclassiffiercgain")
+for degree in range(1,maxdegree):
+    model = GradientBoostingRegressor(max_depth=degree, n_estimators=100, learning_rate=1.0)  
+    model.fit(X_train,y_train)
+    y_pred = model.predict(X_test)
+    polydegree[degree] = degree
+    error[degree] = np.mean( np.mean((y_test - y_pred)**2) )
+    bias[degree] = np.mean( (y_test - np.mean(y_pred))**2 )
+    variance[degree] = np.mean( np.var(y_pred) )
+    print('Max depth:', degree)
+    print('Error:', error[degree])
+    print('Bias^2:', bias[degree])
+    print('Var:', variance[degree])
+    print('{} >= {} + {} = {}'.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree]))
+
+plt.xlim(1,maxdegree-1)
+plt.plot(polydegree, error, label='Error')
+plt.plot(polydegree, bias, label='bias')
+plt.plot(polydegree, variance, label='Variance')
+plt.legend()
+save_fig("gdregression")
 plt.show()
    @@ -246,6 +252,7 @@ plt.show()
  • 19
  • 20
  • 21
    • +
  • 22
  • »
    • diff --git a/doc/pub/week45/html/._week45-bs018.html b/doc/pub/week45/html/._week45-bs018.html index 4ddc59c93..7d12fd09c 100644 --- a/doc/pub/week45/html/._week45-bs018.html +++ b/doc/pub/week45/html/._week45-bs018.html @@ -37,6 +37,10 @@ doconce format html week45.do.txt --html_style=bootstrap --pygments_html_style=d
  • Overview of week 45
    • -
  • Boosting, a Bird's Eye View
    • -
  • What is boosting? Additive Modelling/Iterative Fitting
    • -
  • Iterative Fitting, Regression and Squared-error Cost Function
    • -
  • Squared-Error Example and Iterative Fitting
    • -
  • Iterative Fitting, Classification and AdaBoost
    • -
  • Adaptive Boosting, AdaBoost
    • -
  • Building up AdaBoost
    • -
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • -
  • Basic Steps of AdaBoost
    • -
  • AdaBoost Examples
    • -
  • Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent
    • -
  • The Squared-Error again! Steepest Descent
    • -
  • Steepest Descent Example
    • -
  • Gradient Boosting, algorithm
    • -
  • Gradient Boosting, Examples of Regression
    • -
  • Gradient Boosting, Classification Example
    • -
  • XGBoost: Extreme Gradient Boosting
    • -
  • Regression Case
    • -
  • Xgboost on the Cancer Data
    • +
  • Brief code reminder from last wekk
    • +
  • Boosting, a Bird's Eye View
    • +
  • What is boosting? Additive Modelling/Iterative Fitting
    • +
  • Iterative Fitting, Regression and Squared-error Cost Function
    • +
  • Squared-Error Example and Iterative Fitting
    • +
  • Iterative Fitting, Classification and AdaBoost
    • +
  • Adaptive Boosting, AdaBoost
    • +
  • Building up AdaBoost
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • AdaBoost Examples
    • +
  • Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent
    • +
  • The Squared-Error again! Steepest Descent
    • +
  • Steepest Descent Example
    • +
  • Gradient Boosting, algorithm
    • +
  • Gradient Boosting, Examples of Regression
    • +
  • Gradient Boosting, Classification Example
    • +
  • XGBoost: Extreme Gradient Boosting
    • +
  • Regression Case
    • +
  • Xgboost on the Cancer Data
    • @@ -164,22 +169,69 @@ MathJax.Hub.Config({

     

     

     

    -

    XGBoost: Extreme Gradient Boosting

    +

    Gradient Boosting, Classification Example

    -

    XGBoost or Extreme Gradient -Boosting, is an optimized distributed gradient boosting library -designed to be highly efficient, flexible and portable. It implements -machine learning algorithms under the Gradient Boosting -framework. XGBoost provides a parallel tree boosting that solve many -data science problems in a fast and accurate way. See the article by Chen and Guestrin. -

    + +
    +
    +
    +
    +
    + 
    import matplotlib.pyplot as plt
+import numpy as np
+from sklearn.model_selection import  train_test_split 
+from sklearn.datasets import load_breast_cancer
+import scikitplot as skplt
+from sklearn.ensemble import GradientBoostingClassifier
+from sklearn.model_selection import cross_validate
 
-
    The authors design and build a highly scalable end-to-end tree
-boosting system. It has  a theoretically justified weighted quantile
-sketch for efficient proposal calculation. It introduces a novel sparsity-aware algorithm for parallel tree learning and an effective cache-aware block structure for out-of-core tree learning.
-
    
+# Load the data
+cancer = load_breast_cancer()
+
+X_train, X_test, y_train, y_test = train_test_split(cancer.data,cancer.target,random_state=0)
+print(X_train.shape)
+print(X_test.shape)
+#now scale the data
+from sklearn.preprocessing import StandardScaler
+scaler = StandardScaler()
+scaler.fit(X_train)
+X_train_scaled = scaler.transform(X_train)
+X_test_scaled = scaler.transform(X_test)
+
+gd_clf = GradientBoostingClassifier(max_depth=3, n_estimators=100, learning_rate=1.0)  
+gd_clf.fit(X_train_scaled, y_train)
+#Cross validation
+accuracy = cross_validate(gd_clf,X_test_scaled,y_test,cv=10)['test_score']
+print(accuracy)
+print("Test set accuracy with Random Forests and scaled data: {:.2f}".format(gd_clf.score(X_test_scaled,y_test)))
+
+import scikitplot as skplt
+y_pred = gd_clf.predict(X_test_scaled)
+skplt.metrics.plot_confusion_matrix(y_test, y_pred, normalize=True)
+save_fig("gdclassiffierconfusion")
+plt.show()
+y_probas = gd_clf.predict_proba(X_test_scaled)
+skplt.metrics.plot_roc(y_test, y_probas)
+save_fig("gdclassiffierroc")
+plt.show()
+skplt.metrics.plot_cumulative_gain(y_test, y_probas)
+save_fig("gdclassiffiercgain")
+plt.show()
+
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    -

    It is now the algorithm which wins essentially all ML competitions!!!

    @@ -198,6 +250,7 @@ sketch for efficient proposal calculation. It introduces a novel sparsity-aware

  • 19
  • 20
  • 21
    • +
  • 22
  • »
    • diff --git a/doc/pub/week45/html/._week45-bs019.html b/doc/pub/week45/html/._week45-bs019.html index 48190cd86..0cde3b833 100644 --- a/doc/pub/week45/html/._week45-bs019.html +++ b/doc/pub/week45/html/._week45-bs019.html @@ -37,6 +37,10 @@ doconce format html week45.do.txt --html_style=bootstrap --pygments_html_style=d
  • Overview of week 45
    • -
  • Boosting, a Bird's Eye View
    • -
  • What is boosting? Additive Modelling/Iterative Fitting
    • -
  • Iterative Fitting, Regression and Squared-error Cost Function
    • -
  • Squared-Error Example and Iterative Fitting
    • -
  • Iterative Fitting, Classification and AdaBoost
    • -
  • Adaptive Boosting, AdaBoost
    • -
  • Building up AdaBoost
    • -
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • -
  • Basic Steps of AdaBoost
    • -
  • AdaBoost Examples
    • -
  • Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent
    • -
  • The Squared-Error again! Steepest Descent
    • -
  • Steepest Descent Example
    • -
  • Gradient Boosting, algorithm
    • -
  • Gradient Boosting, Examples of Regression
    • -
  • Gradient Boosting, Classification Example
    • -
  • XGBoost: Extreme Gradient Boosting
    • -
  • Regression Case
    • -
  • Xgboost on the Cancer Data
    • +
  • Brief code reminder from last wekk
    • +
  • Boosting, a Bird's Eye View
    • +
  • What is boosting? Additive Modelling/Iterative Fitting
    • +
  • Iterative Fitting, Regression and Squared-error Cost Function
    • +
  • Squared-Error Example and Iterative Fitting
    • +
  • Iterative Fitting, Classification and AdaBoost
    • +
  • Adaptive Boosting, AdaBoost
    • +
  • Building up AdaBoost
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • AdaBoost Examples
    • +
  • Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent
    • +
  • The Squared-Error again! Steepest Descent
    • +
  • Steepest Descent Example
    • +
  • Gradient Boosting, algorithm
    • +
  • Gradient Boosting, Examples of Regression
    • +
  • Gradient Boosting, Classification Example
    • +
  • XGBoost: Extreme Gradient Boosting
    • +
  • Regression Case
    • +
  • Xgboost on the Cancer Data
    • @@ -164,71 +169,22 @@ MathJax.Hub.Config({

     

     

     

    -

    Regression Case

    +

    XGBoost: Extreme Gradient Boosting

    +

    XGBoost or Extreme Gradient +Boosting, is an optimized distributed gradient boosting library +designed to be highly efficient, flexible and portable. It implements +machine learning algorithms under the Gradient Boosting +framework. XGBoost provides a parallel tree boosting that solve many +data science problems in a fast and accurate way. See the article by Chen and Guestrin. +

    - -
    -
    -
    -
    -
    - 
    import matplotlib.pyplot as plt
-import numpy as np
-from sklearn.model_selection import train_test_split
-import xgboost as xgb
-import scikitplot as skplt
-from sklearn.metrics import mean_squared_error
-
-n = 100
-maxdegree = 6
-
-# Make data set.
-x = np.linspace(-3, 3, n).reshape(-1, 1)
-y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)
-
-error = np.zeros(maxdegree)
-bias = np.zeros(maxdegree)
-variance = np.zeros(maxdegree)
-polydegree = np.zeros(maxdegree)
-X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.2)
-
-for degree in range(maxdegree):
-    model =  xgb.XGBRegressor(objective ='reg:squarederror', colsaobjective ='reg:squarederror', colsample_bytree = 0.3, learning_rate = 0.1,max_depth = degree, alpha = 10, n_estimators = 200)
-
-    model.fit(X_train,y_train)
-    y_pred = model.predict(X_test)
-    polydegree[degree] = degree
-    error[degree] = np.mean( np.mean((y_test - y_pred)**2) )
-    bias[degree] = np.mean( (y_test - np.mean(y_pred))**2 )
-    variance[degree] = np.mean( np.var(y_pred) )
-    print('Max depth:', degree)
-    print('Error:', error[degree])
-    print('Bias^2:', bias[degree])
-    print('Var:', variance[degree])
-    print('{} >= {} + {} = {}'.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree]))
-
-plt.xlim(1,maxdegree-1)
-plt.plot(polydegree, error, label='Error')
-plt.plot(polydegree, bias, label='bias')
-plt.plot(polydegree, variance, label='Variance')
-plt.legend()
-plt.show()
-
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    +

    The authors design and build a highly scalable end-to-end tree +boosting system. It has a theoretically justified weighted quantile +sketch for efficient proposal calculation. It introduces a novel sparsity-aware algorithm for parallel tree learning and an effective cache-aware block structure for out-of-core tree learning. +

    +

    It is now the algorithm which wins essentially all ML competitions!!!

    @@ -246,6 +202,7 @@ plt.show()

  • 19
  • 20
  • 21
    • +
  • 22
  • »
    • diff --git a/doc/pub/week45/html/._week45-bs020.html b/doc/pub/week45/html/._week45-bs020.html index 453dd02f0..562c04cf5 100644 --- a/doc/pub/week45/html/._week45-bs020.html +++ b/doc/pub/week45/html/._week45-bs020.html @@ -37,6 +37,10 @@ doconce format html week45.do.txt --html_style=bootstrap --pygments_html_style=d
  • Overview of week 45
    • -
  • Boosting, a Bird's Eye View
    • -
  • What is boosting? Additive Modelling/Iterative Fitting
    • -
  • Iterative Fitting, Regression and Squared-error Cost Function
    • -
  • Squared-Error Example and Iterative Fitting
    • -
  • Iterative Fitting, Classification and AdaBoost
    • -
  • Adaptive Boosting, AdaBoost
    • -
  • Building up AdaBoost
    • -
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • -
  • Basic Steps of AdaBoost
    • -
  • AdaBoost Examples
    • -
  • Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent
    • -
  • The Squared-Error again! Steepest Descent
    • -
  • Steepest Descent Example
    • -
  • Gradient Boosting, algorithm
    • -
  • Gradient Boosting, Examples of Regression
    • -
  • Gradient Boosting, Classification Example
    • -
  • XGBoost: Extreme Gradient Boosting
    • -
  • Regression Case
    • -
  • Xgboost on the Cancer Data
    • +
  • Brief code reminder from last wekk
    • +
  • Boosting, a Bird's Eye View
    • +
  • What is boosting? Additive Modelling/Iterative Fitting
    • +
  • Iterative Fitting, Regression and Squared-error Cost Function
    • +
  • Squared-Error Example and Iterative Fitting
    • +
  • Iterative Fitting, Classification and AdaBoost
    • +
  • Adaptive Boosting, AdaBoost
    • +
  • Building up AdaBoost
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • AdaBoost Examples
    • +
  • Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent
    • +
  • The Squared-Error again! Steepest Descent
    • +
  • Steepest Descent Example
    • +
  • Gradient Boosting, algorithm
    • +
  • Gradient Boosting, Examples of Regression
    • +
  • Gradient Boosting, Classification Example
    • +
  • XGBoost: Extreme Gradient Boosting
    • +
  • Regression Case
    • +
  • Xgboost on the Cancer Data
    • @@ -164,9 +169,8 @@ MathJax.Hub.Config({

     

     

     

    -

    Xgboost on the Cancer Data

    +

    Regression Case

    -

    As you will see from the confusion matrix below, XGBoots does an excellent job on the Wisconsin cancer data and outperforms essentially all agorithms we have discussed till now.

    @@ -176,54 +180,44 @@ MathJax.Hub.Config({ 
    import matplotlib.pyplot as plt
 import numpy as np
-from sklearn.model_selection import  train_test_split 
-from sklearn.datasets import load_breast_cancer
-from sklearn.preprocessing import LabelEncoder
-from sklearn.model_selection import cross_validate
-import scikitplot as skplt
+from sklearn.model_selection import train_test_split
 import xgboost as xgb
-# Load the data
-cancer = load_breast_cancer()
-
-X_train, X_test, y_train, y_test = train_test_split(cancer.data,cancer.target,random_state=0)
-print(X_train.shape)
-print(X_test.shape)
-#now scale the data
-from sklearn.preprocessing import StandardScaler
-scaler = StandardScaler()
-scaler.fit(X_train)
-X_train_scaled = scaler.transform(X_train)
-X_test_scaled = scaler.transform(X_test)
-
-xg_clf = xgb.XGBClassifier()
-xg_clf.fit(X_train_scaled,y_train)
-
-y_test = xg_clf.predict(X_test_scaled)
-
-print("Test set accuracy with Random Forests and scaled data: {:.2f}".format(xg_clf.score(X_test_scaled,y_test)))
-
 import scikitplot as skplt
-y_pred = xg_clf.predict(X_test_scaled)
-skplt.metrics.plot_confusion_matrix(y_test, y_pred, normalize=True)
-save_fig("xdclassiffierconfusion")
-plt.show()
-y_probas = xg_clf.predict_proba(X_test_scaled)
-skplt.metrics.plot_roc(y_test, y_probas)
-save_fig("xdclassiffierroc")
-plt.show()
-skplt.metrics.plot_cumulative_gain(y_test, y_probas)
-save_fig("gdclassiffiercgain")
-plt.show()
+from sklearn.metrics import mean_squared_error
 
+n = 100
+maxdegree = 6
 
-xgb.plot_tree(xg_clf,num_trees=0)
-plt.rcParams['figure.figsize'] = [50, 10]
-save_fig("xgtree")
-plt.show()
+# Make data set.
+x = np.linspace(-3, 3, n).reshape(-1, 1)
+y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)
 
-xgb.plot_importance(xg_clf)
-plt.rcParams['figure.figsize'] = [5, 5]
-save_fig("xgparams")
+error = np.zeros(maxdegree)
+bias = np.zeros(maxdegree)
+variance = np.zeros(maxdegree)
+polydegree = np.zeros(maxdegree)
+X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.2)
+
+for degree in range(maxdegree):
+    model =  xgb.XGBRegressor(objective ='reg:squarederror', colsaobjective ='reg:squarederror', colsample_bytree = 0.3, learning_rate = 0.1,max_depth = degree, alpha = 10, n_estimators = 200)
+
+    model.fit(X_train,y_train)
+    y_pred = model.predict(X_test)
+    polydegree[degree] = degree
+    error[degree] = np.mean( np.mean((y_test - y_pred)**2) )
+    bias[degree] = np.mean( (y_test - np.mean(y_pred))**2 )
+    variance[degree] = np.mean( np.var(y_pred) )
+    print('Max depth:', degree)
+    print('Error:', error[degree])
+    print('Bias^2:', bias[degree])
+    print('Var:', variance[degree])
+    print('{} >= {} + {} = {}'.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree]))
+
+plt.xlim(1,maxdegree-1)
+plt.plot(polydegree, error, label='Error')
+plt.plot(polydegree, bias, label='bias')
+plt.plot(polydegree, variance, label='Variance')
+plt.legend()
 plt.show()
    @@ -256,6 +250,8 @@ plt.show()
  • 19
  • 20
  • 21
    • +
  • 22
    • +
  • »
    • diff --git a/doc/pub/week45/html/._week45-bs021.html b/doc/pub/week45/html/._week45-bs021.html index 446666575..c56ba7632 100644 --- a/doc/pub/week45/html/._week45-bs021.html +++ b/doc/pub/week45/html/._week45-bs021.html @@ -37,175 +37,10 @@ doconce format html week45.do.txt --html_style=bootstrap --pygments_html_style=d
  • Overview of week 45
    • -
  • Decision trees, overarching aims
    • -
  • Basics of a tree
    • -
  • A Sketch of a Tree, Regression problem
    • -
  • A Sketch of a Tree, Classification problem
    • -
  • A typical Decision Tree with its pertinent Jargon, Classification Problem
    • -
  • General Features
    • -
  • How do we set it up?
    • -
  • Decision trees and Regression
    • -
  • Building a tree, regression
    • -
  • A top-down approach, recursive binary splitting
    • -
  • Making a tree
    • -
  • Pruning the tree
    • -
  • Cost complexity pruning
    • -
  • Schematic Regression Procedure
    • -
  • A Classification Tree
    • -
  • Growing a classification tree
    • -
  • Classification tree, how to split nodes
    • -
  • Gini Index (or Coefficient or Impurity)
    • -
  • Why binary splits?
    • -
  • Computing a Tree using the Gini Index
    • -
  • The Table
    • -
  • Computing the various Gini Indices
    • -
  • Computing the various Gini Indices, Hours slept
    • -
  • Computing the various Gini Indices, Hours studied
    • -
  • A possible code using Scikit-Learn
    • -
  • Visualizing Trees, More examples
    • -
  • Visualizing the Tree, The Moons
    • -
  • Other ways of visualizing the trees
    • -
  • Printing out as text
    • -
  • Algorithms for Setting up Decision Trees
    • -
  • The CART algorithm for Classification
    • -
  • The CART algorithm for Regression
    • -
  • Computing the Gini index
    • -
  • Simple Python Code to read in Data and perform Classification
    • -
  • Computing the Gini Factor
    • -
  • Entropy and the ID3 algorithm
    • -
  • Cancer Data again now with Decision Trees and other Methods
    • -
  • Another example, the moons again
    • -
  • Playing around with regions
    • -
  • Regression trees
    • -
  • Final regressor code
    • -
  • Pros and cons of trees, pros
    • -
  • Disadvantages
    • -
  • Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods
    • -
  • An Overview of Ensemble Methods
    • -
  • Bagging
    • -
  • More bagging
    • -
  • Making your own Bootstrap: Changing the Level of the Decision Tree
    • -
  • Why Voting?
    • -
  • Tossing coins
    • -
  • Standard imports first
    • -
  • Simple Voting Example, head or tail
    • -
  • Using the Voting Classifier
    • -
  • Voting and Bagging
    • -
  • Random forests
    • -
  • Random Forest Algorithm
    • -
  • Random Forests Compared with other Methods on the Cancer Data
    • -
  • Compare Bagging on Trees with Random Forests
    • -
  • Boosting, a Bird's Eye View
    • -
  • What is boosting? Additive Modelling/Iterative Fitting
    • -
  • Iterative Fitting, Regression and Squared-error Cost Function
    • -
  • Squared-Error Example and Iterative Fitting
    • -
  • Iterative Fitting, Classification and AdaBoost
    • -
  • Adaptive Boosting, AdaBoost
    • -
  • Building up AdaBoost
    • -
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • -
  • Basic Steps of AdaBoost
    • -
  • AdaBoost Examples
    • -
  • Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent
    • -
  • The Squared-Error again! Steepest Descent
    • -
  • Steepest Descent Example
    • -
  • Gradient Boosting, algorithm
    • -
  • Gradient Boosting, Examples of Regression
    • -
  • Gradient Boosting, Classification Example
    • -
  • XGBoost: Extreme Gradient Boosting
    • -
  • Regression Case
    • -
  • Xgboost on the Cancer Data
    • +
  • Brief code reminder from last wekk
    • +
  • Boosting, a Bird's Eye View
    • +
  • What is boosting? Additive Modelling/Iterative Fitting
    • +
  • Iterative Fitting, Regression and Squared-error Cost Function
    • +
  • Squared-Error Example and Iterative Fitting
    • +
  • Iterative Fitting, Classification and AdaBoost
    • +
  • Adaptive Boosting, AdaBoost
    • +
  • Building up AdaBoost
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • AdaBoost Examples
    • +
  • Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent
    • +
  • The Squared-Error again! Steepest Descent
    • +
  • Steepest Descent Example
    • +
  • Gradient Boosting, algorithm
    • +
  • Gradient Boosting, Examples of Regression
    • +
  • Gradient Boosting, Classification Example
    • +
  • XGBoost: Extreme Gradient Boosting
    • +
  • Regression Case
    • +
  • Xgboost on the Cancer Data
    • @@ -391,23 +169,83 @@ MathJax.Hub.Config({

     

     

     

    -

    Computing a Tree using the Gini Index

    +

    Xgboost on the Cancer Data

    + +

    As you will see from the confusion matrix below, XGBoots does an excellent job on the Wisconsin cancer data and outperforms essentially all agorithms we have discussed till now.

    + + +
    +
    +
    +
    +
    + 
    import matplotlib.pyplot as plt
+import numpy as np
+from sklearn.model_selection import  train_test_split 
+from sklearn.datasets import load_breast_cancer
+from sklearn.preprocessing import LabelEncoder
+from sklearn.model_selection import cross_validate
+import scikitplot as skplt
+import xgboost as xgb
+# Load the data
+cancer = load_breast_cancer()
+
+X_train, X_test, y_train, y_test = train_test_split(cancer.data,cancer.target,random_state=0)
+print(X_train.shape)
+print(X_test.shape)
+#now scale the data
+from sklearn.preprocessing import StandardScaler
+scaler = StandardScaler()
+scaler.fit(X_train)
+X_train_scaled = scaler.transform(X_train)
+X_test_scaled = scaler.transform(X_test)
+
+xg_clf = xgb.XGBClassifier()
+xg_clf.fit(X_train_scaled,y_train)
+
+y_test = xg_clf.predict(X_test_scaled)
+
+print("Test set accuracy with Random Forests and scaled data: {:.2f}".format(xg_clf.score(X_test_scaled,y_test)))
+
+import scikitplot as skplt
+y_pred = xg_clf.predict(X_test_scaled)
+skplt.metrics.plot_confusion_matrix(y_test, y_pred, normalize=True)
+save_fig("xdclassiffierconfusion")
+plt.show()
+y_probas = xg_clf.predict_proba(X_test_scaled)
+skplt.metrics.plot_roc(y_test, y_probas)
+save_fig("xdclassiffierroc")
+plt.show()
+skplt.metrics.plot_cumulative_gain(y_test, y_probas)
+save_fig("gdclassiffiercgain")
+plt.show()
+
+
+xgb.plot_tree(xg_clf,num_trees=0)
+plt.rcParams['figure.figsize'] = [50, 10]
+save_fig("xgtree")
+plt.show()
+
+xgb.plot_importance(xg_clf)
+plt.rcParams['figure.figsize'] = [5, 5]
+save_fig("xgparams")
+plt.show()
+
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    -

    Consider the following example with attributes/features and two -possible outcomes (classes) for each attribute. Assume we wish to find some -correlations between the average grade of a student as function of the -number of hours studied and hours slept. We want also to correlate the -grade in a given course with the general trend, whether the students -recently has gotten grades below average or above. -

    -

    We have three features/attributes

    -
      -
    1. Trend of average grades before present course, classified as either below or above the average grade of the whole class
      2. -
    2. The number of hours studies, classified again as either higher (more than 3 hours per day) or lower . Here we have used a standard for one \( ECTS \) which is scaled to 25-30 hours of work for a semester which lasts 18 weeks, with 15 weeks of lectures and 3 weeks for exams, assuming a total of 30 ECTS per semester.
      3. -
    3. The number of hours slept as high for more than \( 8 \) hours and below for less than 8 hours of sleep, classified again as either high or low
      4. -
    4. The final grade whether it is above or below average
      5. -

    diff --git a/doc/pub/week45/html/week45-bs.html b/doc/pub/week45/html/week45-bs.html index fb002f8e6..817a4baea 100644 --- a/doc/pub/week45/html/week45-bs.html +++ b/doc/pub/week45/html/week45-bs.html @@ -37,6 +37,10 @@ doconce format html week45.do.txt --html_style=bootstrap --pygments_html_style=d
  • Overview of week 45
    • -
  • Boosting, a Bird's Eye View
    • -
  • What is boosting? Additive Modelling/Iterative Fitting
    • -
  • Iterative Fitting, Regression and Squared-error Cost Function
    • -
  • Squared-Error Example and Iterative Fitting
    • -
  • Iterative Fitting, Classification and AdaBoost
    • -
  • Adaptive Boosting, AdaBoost
    • -
  • Building up AdaBoost
    • -
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • -
  • Basic Steps of AdaBoost
    • -
  • AdaBoost Examples
    • -
  • Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent
    • -
  • The Squared-Error again! Steepest Descent
    • -
  • Steepest Descent Example
    • -
  • Gradient Boosting, algorithm
    • -
  • Gradient Boosting, Examples of Regression
    • -
  • Gradient Boosting, Classification Example
    • -
  • XGBoost: Extreme Gradient Boosting
    • -
  • Regression Case
    • -
  • Xgboost on the Cancer Data
    • +
  • Brief code reminder from last wekk
    • +
  • Boosting, a Bird's Eye View
    • +
  • What is boosting? Additive Modelling/Iterative Fitting
    • +
  • Iterative Fitting, Regression and Squared-error Cost Function
    • +
  • Squared-Error Example and Iterative Fitting
    • +
  • Iterative Fitting, Classification and AdaBoost
    • +
  • Adaptive Boosting, AdaBoost
    • +
  • Building up AdaBoost
    • +
  • Adaptive boosting: AdaBoost, Basic Algorithm
    • +
  • Basic Steps of AdaBoost
    • +
  • AdaBoost Examples
    • +
  • Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent
    • +
  • The Squared-Error again! Steepest Descent
    • +
  • Steepest Descent Example
    • +
  • Gradient Boosting, algorithm
    • +
  • Gradient Boosting, Examples of Regression
    • +
  • Gradient Boosting, Classification Example
    • +
  • XGBoost: Extreme Gradient Boosting
    • +
  • Regression Case
    • +
  • Xgboost on the Cancer Data
    • @@ -207,7 +212,7 @@ MathJax.Hub.Config({
  • 9
  • 10
  • ...
    • -
  • 21
    • +
  • 22
  • »
    • diff --git a/doc/pub/week45/html/week45-reveal.html b/doc/pub/week45/html/week45-reveal.html index a3579a832..3f8253074 100644 --- a/doc/pub/week45/html/week45-reveal.html +++ b/doc/pub/week45/html/week45-reveal.html @@ -221,6 +221,122 @@ MathJax.Hub.Config({ +
    +

    Brief code reminder from last wekk

    + + + +
    +
    +
    +
    +
    + 
    # Common imports
+from IPython.display import Image 
+from pydot import graph_from_dot_data
+import pandas as pd
+import numpy as np
+import matplotlib.pyplot as plt
+from sklearn.tree import DecisionTreeClassifier
+from sklearn.tree import DecisionTreeRegressor
+from sklearn.model_selection import train_test_split
+from sklearn.tree import export_graphviz
+from sklearn.preprocessing import StandardScaler, OneHotEncoder
+from sklearn.compose import ColumnTransformer
+from IPython.display import Image 
+from pydot import graph_from_dot_data
+from sklearn.datasets import load_breast_cancer
+from sklearn.svm import SVC
+from sklearn.linear_model import LogisticRegression
+from sklearn.ensemble import BaggingClassifier
+
+import os
+
+# Where to save the figures and data files
+PROJECT_ROOT_DIR = "Results"
+FIGURE_ID = "Results/FigureFiles"
+DATA_ID = "DataFiles/"
+
+if not os.path.exists(PROJECT_ROOT_DIR):
+    os.mkdir(PROJECT_ROOT_DIR)
+
+if not os.path.exists(FIGURE_ID):
+    os.makedirs(FIGURE_ID)
+
+if not os.path.exists(DATA_ID):
+    os.makedirs(DATA_ID)
+
+def image_path(fig_id):
+    return os.path.join(FIGURE_ID, fig_id)
+
+def data_path(dat_id):
+    return os.path.join(DATA_ID, dat_id)
+
+def save_fig(fig_id):
+    plt.savefig(image_path(fig_id) + ".png", format='png')
+
+# Load the cancer data
+cancer = load_breast_cancer()
+
+X_train, X_test, y_train, y_test = train_test_split(cancer.data,cancer.target,random_state=0)
+print(X_train.shape)
+print(X_test.shape)
+#Scale the data
+from sklearn.preprocessing import StandardScaler
+scaler = StandardScaler()
+scaler.fit(X_train)
+X_train_scaled = scaler.transform(X_train)
+X_test_scaled = scaler.transform(X_test)
+#define methods
+# Logistic Regression
+logreg.fit(X_train_scaled, y_train)
+print("Test set accuracy Logistic Regression with scaled data: {:.2f}".format(logreg.score(X_test_scaled,y_test)))
+# Support Vector Machine
+svm.fit(X_train_scaled, y_train)
+print("Test set accuracy SVM with scaled data: {:.2f}".format(logreg.score(X_test_scaled,y_test)))
+# Decision Trees
+deep_tree_clf.fit(X_train_scaled, y_train)
+print("Test set accuracy with Decision Trees and scaled data: {:.2f}".format(deep_tree_clf.score(X_test_scaled,y_test)))
+
+
+from sklearn.ensemble import RandomForestClassifier
+from sklearn.preprocessing import LabelEncoder
+from sklearn.model_selection import cross_validate
+# Data set not specificied
+#Instantiate the model with 500 trees and entropy as splitting criteria
+Random_Forest_model = RandomForestClassifier(n_estimators=500,criterion="entropy")
+Random_Forest_model.fit(X_train_scaled, y_train)
+#Cross validation
+accuracy = cross_validate(Random_Forest_model,X_test_scaled,y_test,cv=10)['test_score']
+print(accuracy)
+print("Test set accuracy with Random Forests and scaled data: {:.2f}".format(Random_Forest_model.score(X_test_scaled,y_test)))
+
+
+import scikitplot as skplt
+y_pred = Random_Forest_model.predict(X_test_scaled)
+skplt.metrics.plot_confusion_matrix(y_test, y_pred, normalize=True)
+plt.show()
+y_probas = Random_Forest_model.predict_proba(X_test_scaled)
+skplt.metrics.plot_roc(y_test, y_probas)
+plt.show()
+skplt.metrics.plot_cumulative_gain(y_test, y_probas)
+plt.show()
+
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +

    Boosting, a Bird's Eye View

    diff --git a/doc/pub/week45/html/week45-solarized.html b/doc/pub/week45/html/week45-solarized.html index 6681afd70..227923c39 100644 --- a/doc/pub/week45/html/week45-solarized.html +++ b/doc/pub/week45/html/week45-solarized.html @@ -64,6 +64,10 @@ div.toc p,a {









    +

    Brief code reminder from last wekk

    + + + +
    +
    +
    +
    +
    + 
    # Common imports
+from IPython.display import Image 
+from pydot import graph_from_dot_data
+import pandas as pd
+import numpy as np
+import matplotlib.pyplot as plt
+from sklearn.tree import DecisionTreeClassifier
+from sklearn.tree import DecisionTreeRegressor
+from sklearn.model_selection import train_test_split
+from sklearn.tree import export_graphviz
+from sklearn.preprocessing import StandardScaler, OneHotEncoder
+from sklearn.compose import ColumnTransformer
+from IPython.display import Image 
+from pydot import graph_from_dot_data
+from sklearn.datasets import load_breast_cancer
+from sklearn.svm import SVC
+from sklearn.linear_model import LogisticRegression
+from sklearn.ensemble import BaggingClassifier
+
+import os
+
+# Where to save the figures and data files
+PROJECT_ROOT_DIR = "Results"
+FIGURE_ID = "Results/FigureFiles"
+DATA_ID = "DataFiles/"
+
+if not os.path.exists(PROJECT_ROOT_DIR):
+    os.mkdir(PROJECT_ROOT_DIR)
+
+if not os.path.exists(FIGURE_ID):
+    os.makedirs(FIGURE_ID)
+
+if not os.path.exists(DATA_ID):
+    os.makedirs(DATA_ID)
+
+def image_path(fig_id):
+    return os.path.join(FIGURE_ID, fig_id)
+
+def data_path(dat_id):
+    return os.path.join(DATA_ID, dat_id)
+
+def save_fig(fig_id):
+    plt.savefig(image_path(fig_id) + ".png", format='png')
+
+# Load the cancer data
+cancer = load_breast_cancer()
+
+X_train, X_test, y_train, y_test = train_test_split(cancer.data,cancer.target,random_state=0)
+print(X_train.shape)
+print(X_test.shape)
+#Scale the data
+from sklearn.preprocessing import StandardScaler
+scaler = StandardScaler()
+scaler.fit(X_train)
+X_train_scaled = scaler.transform(X_train)
+X_test_scaled = scaler.transform(X_test)
+#define methods
+# Logistic Regression
+logreg.fit(X_train_scaled, y_train)
+print("Test set accuracy Logistic Regression with scaled data: {:.2f}".format(logreg.score(X_test_scaled,y_test)))
+# Support Vector Machine
+svm.fit(X_train_scaled, y_train)
+print("Test set accuracy SVM with scaled data: {:.2f}".format(logreg.score(X_test_scaled,y_test)))
+# Decision Trees
+deep_tree_clf.fit(X_train_scaled, y_train)
+print("Test set accuracy with Decision Trees and scaled data: {:.2f}".format(deep_tree_clf.score(X_test_scaled,y_test)))
+
+
+from sklearn.ensemble import RandomForestClassifier
+from sklearn.preprocessing import LabelEncoder
+from sklearn.model_selection import cross_validate
+# Data set not specificied
+#Instantiate the model with 500 trees and entropy as splitting criteria
+Random_Forest_model = RandomForestClassifier(n_estimators=500,criterion="entropy")
+Random_Forest_model.fit(X_train_scaled, y_train)
+#Cross validation
+accuracy = cross_validate(Random_Forest_model,X_test_scaled,y_test,cv=10)['test_score']
+print(accuracy)
+print("Test set accuracy with Random Forests and scaled data: {:.2f}".format(Random_Forest_model.score(X_test_scaled,y_test)))
+
+
+import scikitplot as skplt
+y_pred = Random_Forest_model.predict(X_test_scaled)
+skplt.metrics.plot_confusion_matrix(y_test, y_pred, normalize=True)
+plt.show()
+y_probas = Random_Forest_model.predict_proba(X_test_scaled)
+skplt.metrics.plot_roc(y_test, y_probas)
+plt.show()
+skplt.metrics.plot_cumulative_gain(y_test, y_probas)
+plt.show()
+
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    + +









    Boosting, a Bird's Eye View

    diff --git a/doc/pub/week45/html/week45.html b/doc/pub/week45/html/week45.html index 8439d8a2b..cb9dfef72 100644 --- a/doc/pub/week45/html/week45.html +++ b/doc/pub/week45/html/week45.html @@ -141,6 +141,10 @@ div.toc p,a {









    +

    Brief code reminder from last wekk

    + + + +
    +
    +
    +
    +
    + 
    # Common imports
+from IPython.display import Image 
+from pydot import graph_from_dot_data
+import pandas as pd
+import numpy as np
+import matplotlib.pyplot as plt
+from sklearn.tree import DecisionTreeClassifier
+from sklearn.tree import DecisionTreeRegressor
+from sklearn.model_selection import train_test_split
+from sklearn.tree import export_graphviz
+from sklearn.preprocessing import StandardScaler, OneHotEncoder
+from sklearn.compose import ColumnTransformer
+from IPython.display import Image 
+from pydot import graph_from_dot_data
+from sklearn.datasets import load_breast_cancer
+from sklearn.svm import SVC
+from sklearn.linear_model import LogisticRegression
+from sklearn.ensemble import BaggingClassifier
+
+import os
+
+# Where to save the figures and data files
+PROJECT_ROOT_DIR = "Results"
+FIGURE_ID = "Results/FigureFiles"
+DATA_ID = "DataFiles/"
+
+if not os.path.exists(PROJECT_ROOT_DIR):
+    os.mkdir(PROJECT_ROOT_DIR)
+
+if not os.path.exists(FIGURE_ID):
+    os.makedirs(FIGURE_ID)
+
+if not os.path.exists(DATA_ID):
+    os.makedirs(DATA_ID)
+
+def image_path(fig_id):
+    return os.path.join(FIGURE_ID, fig_id)
+
+def data_path(dat_id):
+    return os.path.join(DATA_ID, dat_id)
+
+def save_fig(fig_id):
+    plt.savefig(image_path(fig_id) + ".png", format='png')
+
+# Load the cancer data
+cancer = load_breast_cancer()
+
+X_train, X_test, y_train, y_test = train_test_split(cancer.data,cancer.target,random_state=0)
+print(X_train.shape)
+print(X_test.shape)
+#Scale the data
+from sklearn.preprocessing import StandardScaler
+scaler = StandardScaler()
+scaler.fit(X_train)
+X_train_scaled = scaler.transform(X_train)
+X_test_scaled = scaler.transform(X_test)
+#define methods
+# Logistic Regression
+logreg.fit(X_train_scaled, y_train)
+print("Test set accuracy Logistic Regression with scaled data: {:.2f}".format(logreg.score(X_test_scaled,y_test)))
+# Support Vector Machine
+svm.fit(X_train_scaled, y_train)
+print("Test set accuracy SVM with scaled data: {:.2f}".format(logreg.score(X_test_scaled,y_test)))
+# Decision Trees
+deep_tree_clf.fit(X_train_scaled, y_train)
+print("Test set accuracy with Decision Trees and scaled data: {:.2f}".format(deep_tree_clf.score(X_test_scaled,y_test)))
+
+
+from sklearn.ensemble import RandomForestClassifier
+from sklearn.preprocessing import LabelEncoder
+from sklearn.model_selection import cross_validate
+# Data set not specificied
+#Instantiate the model with 500 trees and entropy as splitting criteria
+Random_Forest_model = RandomForestClassifier(n_estimators=500,criterion="entropy")
+Random_Forest_model.fit(X_train_scaled, y_train)
+#Cross validation
+accuracy = cross_validate(Random_Forest_model,X_test_scaled,y_test,cv=10)['test_score']
+print(accuracy)
+print("Test set accuracy with Random Forests and scaled data: {:.2f}".format(Random_Forest_model.score(X_test_scaled,y_test)))
+
+
+import scikitplot as skplt
+y_pred = Random_Forest_model.predict(X_test_scaled)
+skplt.metrics.plot_confusion_matrix(y_test, y_pred, normalize=True)
+plt.show()
+y_probas = Random_Forest_model.predict_proba(X_test_scaled)
+skplt.metrics.plot_roc(y_test, y_probas)
+plt.show()
+skplt.metrics.plot_cumulative_gain(y_test, y_probas)
+plt.show()
+
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    +
    + +









    Boosting, a Bird's Eye View

    diff --git a/doc/pub/week45/ipynb/ipynb-week45-src.tar.gz b/doc/pub/week45/ipynb/ipynb-week45-src.tar.gz index ad944419b1132faeb0406a230d3685f421889005..84265531fd2ed28b595614461185bc9f9a00b837 100644 GIT binary patch literal 192 zcmV;x06+g9iwFRcd}m_-1MSaC3c@fD2H>uHia9|^nn%}yE?fvAULd8ZjkQTlQna_X z573q3rihSl^E1pa%p5kW^=^~+yN?!w5aN`=m>FkNqI9k&7&E|>5X>{4rYRsy7{L*s z1q(qL5B%B`#rW^vQ_EM&YlY@jTD-zV-l_pT@BO2mk