From ea69981166aa93349fd652dcf273c20b0a83ef6c Mon Sep 17 00:00:00 2001 From: Morten Hjorth-Jensen Date: Thu, 11 Nov 2021 09:54:06 +0100 Subject: [PATCH] added files --- doc/pub/week45/html/._week45-bs078.html | 502 ++++++++++++++ doc/pub/week45/ipynb/week45.ipynb | 859 +++++++----------------- 2 files changed, 733 insertions(+), 628 deletions(-) create mode 100644 doc/pub/week45/html/._week45-bs078.html diff --git a/doc/pub/week45/html/._week45-bs078.html b/doc/pub/week45/html/._week45-bs078.html new file mode 100644 index 000000000..83e55f6a5 --- /dev/null +++ b/doc/pub/week45/html/._week45-bs078.html @@ -0,0 +1,502 @@ + + + + + + + +Week 45: Decisions Trees, Random Forests, Bagging and Boosting + + + + + + + + + + + + + + + + + + +
+
+ + Week 45: Decisions Trees, Random Forests, Bagging and Boosting +
+
+ +
+
+ +
+

 

 

 

+ + +

Xgboost on the Cancer Data

+ +

As you will see from the confusion matrix below, XGBoots does an excellent job on the Wisconsin cancer data and outperforms essentially all agorithms we have discussed till now.

+ + +
+
+
+
+
+ 
import matplotlib.pyplot as plt
+import numpy as np
+from sklearn.model_selection import  train_test_split 
+from sklearn.datasets import load_breast_cancer
+from sklearn.preprocessing import LabelEncoder
+from sklearn.model_selection import cross_validate
+import scikitplot as skplt
+import xgboost as xgb
+# Load the data
+cancer = load_breast_cancer()
+
+X_train, X_test, y_train, y_test = train_test_split(cancer.data,cancer.target,random_state=0)
+print(X_train.shape)
+print(X_test.shape)
+#now scale the data
+from sklearn.preprocessing import StandardScaler
+scaler = StandardScaler()
+scaler.fit(X_train)
+X_train_scaled = scaler.transform(X_train)
+X_test_scaled = scaler.transform(X_test)
+
+xg_clf = xgb.XGBClassifier()
+xg_clf.fit(X_train_scaled,y_train)
+
+y_test = xg_clf.predict(X_test_scaled)
+
+print("Test set accuracy with Random Forests and scaled data: {:.2f}".format(xg_clf.score(X_test_scaled,y_test)))
+
+import scikitplot as skplt
+y_pred = xg_clf.predict(X_test_scaled)
+skplt.metrics.plot_confusion_matrix(y_test, y_pred, normalize=True)
+save_fig("xdclassiffierconfusion")
+plt.show()
+y_probas = xg_clf.predict_proba(X_test_scaled)
+skplt.metrics.plot_roc(y_test, y_probas)
+save_fig("xdclassiffierroc")
+plt.show()
+skplt.metrics.plot_cumulative_gain(y_test, y_probas)
+save_fig("gdclassiffiercgain")
+plt.show()
+
+
+xgb.plot_tree(xg_clf,num_trees=0)
+plt.rcParams['figure.figsize'] = [50, 10]
+save_fig("xgtree")
+plt.show()
+
+xgb.plot_importance(xg_clf)
+plt.rcParams['figure.figsize'] = [5, 5]
+save_fig("xgparams")
+plt.show()
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ + +

+ +

+ +
+ + + + +
+ +
+ + + diff --git a/doc/pub/week45/ipynb/week45.ipynb b/doc/pub/week45/ipynb/week45.ipynb index 4677328cf..18443a8f1 100644 --- a/doc/pub/week45/ipynb/week45.ipynb +++ b/doc/pub/week45/ipynb/week45.ipynb @@ -3,9 +3,7 @@ { "cell_type": "markdown", "id": "6038c455", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "\n", @@ -15,9 +13,7 @@ { "cell_type": "markdown", "id": "85e8fd68", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "# Week 45: Decisions Trees, Random Forests, Bagging and Boosting\n", "**Morten Hjorth-Jensen**, Department of Physics, University of Oslo and Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University\n", @@ -30,9 +26,7 @@ { "cell_type": "markdown", "id": "f8028a3d", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Overview of week 45\n", "\n", @@ -54,9 +48,7 @@ { "cell_type": "markdown", "id": "87e9cace", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Decision trees, overarching aims\n", "\n", @@ -85,9 +77,7 @@ { "cell_type": "markdown", "id": "7cbc9228", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Basics of a tree\n", "\n", @@ -105,9 +95,7 @@ { "cell_type": "markdown", "id": "8d8cad8e", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## A Sketch of a Tree, Regression problem\n", "\n", @@ -117,9 +105,7 @@ { "cell_type": "markdown", "id": "1910a78f", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## A Sketch of a Tree, Classification problem\n", "\n", @@ -129,9 +115,7 @@ { "cell_type": "markdown", "id": "b8ad32fa", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## A typical Decision Tree with its pertinent Jargon, Classification Problem\n", "\n", @@ -147,9 +131,7 @@ { "cell_type": "markdown", "id": "b4f17c84", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## General Features\n", "\n", @@ -170,9 +152,7 @@ { "cell_type": "markdown", "id": "9d2b7455", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## How do we set it up?\n", "\n", @@ -193,9 +173,7 @@ { "cell_type": "markdown", "id": "f9c47829", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Decision trees and Regression" ] @@ -204,10 +182,7 @@ "cell_type": "code", "execution_count": 1, "id": "a9b40c02", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "%matplotlib inline\n", @@ -305,9 +280,7 @@ { "cell_type": "markdown", "id": "46f5b18a", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Building a tree, regression\n", "\n", @@ -327,9 +300,7 @@ { "cell_type": "markdown", "id": "9016ab06", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\sum_{j=1}^J\\sum_{i\\in R_j}(y_i-\\overline{y}_{R_j})^2,\n", @@ -339,9 +310,7 @@ { "cell_type": "markdown", "id": "3c493861", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "where $\\overline{y}_{R_j}$ is the mean response for the training observations \n", "within box $j$." @@ -350,9 +319,7 @@ { "cell_type": "markdown", "id": "5b4203d6", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## A top-down approach, recursive binary splitting\n", "\n", @@ -372,9 +339,7 @@ { "cell_type": "markdown", "id": "38afe033", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Making a tree\n", "\n", @@ -385,9 +350,7 @@ { "cell_type": "markdown", "id": "5ee51e92", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\left\\{X\\vert x_j < s\\right\\},\n", @@ -397,9 +360,7 @@ { "cell_type": "markdown", "id": "d44d4233", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "and" ] @@ -407,9 +368,7 @@ { "cell_type": "markdown", "id": "42b18eb4", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\left\\{X\\vert x_j \\geq s\\right\\},\n", @@ -419,9 +378,7 @@ { "cell_type": "markdown", "id": "40ea65e5", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "so that we obtain the lowest MSE, that is" ] @@ -429,9 +386,7 @@ { "cell_type": "markdown", "id": "5765dd31", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\sum_{i:x_i\\in R_j}(y_i-\\overline{y}_{R_1})^2+\\sum_{i:x_i\\in R_2}(y_i-\\overline{y}_{R_2})^2,\n", @@ -441,9 +396,7 @@ { "cell_type": "markdown", "id": "c441521b", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "which we want to minimize by considering all predictors\n", "$x_1,x_2,\\dots,x_p$. We consider also all possible values of $s$ for\n", @@ -474,9 +427,7 @@ { "cell_type": "markdown", "id": "4600886b", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Pruning the tree\n", "\n", @@ -498,9 +449,7 @@ { "cell_type": "markdown", "id": "2c95bc5f", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Cost complexity pruning\n", "\n", @@ -510,9 +459,7 @@ { "cell_type": "markdown", "id": "b37ec50f", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\sum_{m=1}^{\\overline{T}}\\sum_{i:x_i\\in R_m}(y_i-\\overline{y}_{R_m})^2+\\alpha\\overline{T},\n", @@ -522,9 +469,7 @@ { "cell_type": "markdown", "id": "1cad8e0d", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "is as small as possible. Here $\\overline{T}$ is \n", "the number of terminal nodes of the tree $T$ , $R_m$ is the\n", @@ -550,9 +495,7 @@ { "cell_type": "markdown", "id": "cc370b93", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Schematic Regression Procedure\n", "\n", @@ -576,9 +519,7 @@ { "cell_type": "markdown", "id": "fa639bd0", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## A Classification Tree\n", "\n", @@ -599,9 +540,7 @@ { "cell_type": "markdown", "id": "c4334771", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Growing a classification tree\n", "\n", @@ -626,9 +565,7 @@ { "cell_type": "markdown", "id": "65e09794", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Classification tree, how to split nodes\n", "\n", @@ -645,9 +582,7 @@ { "cell_type": "markdown", "id": "2351ab6b", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "p_{mk} = \\frac{1}{N_m}\\sum_{i\\in R_m}I(y_i=k).\n", @@ -657,9 +592,7 @@ { "cell_type": "markdown", "id": "827a3f8d", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "We let $p_{mk}$ represent the majority class of observations in region\n", "$m$. The three most common ways of splitting a node are given by\n", @@ -670,9 +603,7 @@ { "cell_type": "markdown", "id": "35f85c1c", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\frac{1}{N_m}\\sum_{i\\in R_m}I(y_i\\ne k) = 1-p_{mk}.\n", @@ -682,9 +613,7 @@ { "cell_type": "markdown", "id": "b1909c9f", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "* Gini index $g$" ] @@ -692,9 +621,7 @@ { "cell_type": "markdown", "id": "4c72e5c9", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "g = \\sum_{k\\ne k'} p_{mk}p_{mk'}=\\sum_{k=1}^K p_{mk}(1-p_{mk}).\n", @@ -704,9 +631,7 @@ { "cell_type": "markdown", "id": "9354027b", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "* Information entropy or just entropy $s$" ] @@ -714,9 +639,7 @@ { "cell_type": "markdown", "id": "cd7b6f3e", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "s = -\\sum_{k=1}^K p_{mk}\\log{p_{mk}}.\n", @@ -726,9 +649,7 @@ { "cell_type": "markdown", "id": "9fe8b307", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Gini Index (or Coefficient or Impurity)\n", "\n", @@ -748,9 +669,7 @@ { "cell_type": "markdown", "id": "a39f2477", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Why binary splits?\n", "\n", @@ -763,9 +682,7 @@ { "cell_type": "markdown", "id": "e3274b4e", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Computing a Tree using the Gini Index\n", "\n", @@ -789,9 +706,7 @@ { "cell_type": "markdown", "id": "6a3183bd", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## The Table\n", "\n", @@ -817,9 +732,7 @@ { "cell_type": "markdown", "id": "e5d4eb28", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Computing the various Gini Indices\n", "\n", @@ -834,9 +747,7 @@ { "cell_type": "markdown", "id": "98bac75b", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Computing the various Gini Indices, Hours slept\n", "\n", @@ -848,9 +759,7 @@ { "cell_type": "markdown", "id": "263f4079", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Computing the various Gini Indices, Hours studied\n", "\n", @@ -864,22 +773,29 @@ { "cell_type": "markdown", "id": "16de86aa", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## A possible code using Scikit-Learn" ] }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 1, "id": "6206aec1", - "metadata": { - "collapsed": false, - "editable": true - }, - "outputs": [], + "metadata": {}, + "outputs": [ + { + "ename": "ModuleNotFoundError", + "evalue": "No module named 'pydot'", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mModuleNotFoundError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 9\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0msklearn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcompose\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mColumnTransformer\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 10\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mIPython\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdisplay\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mImage\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 11\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mpydot\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mgraph_from_dot_data\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 12\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 13\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'pydot'" + ] + } + ], "source": [ "# Common imports\n", "import numpy as np\n", @@ -953,9 +869,7 @@ { "cell_type": "markdown", "id": "280ae557", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Visualizing Trees, More examples" ] @@ -964,10 +878,7 @@ "cell_type": "code", "execution_count": 3, "id": "6c8124ac", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "import os\n", @@ -1008,9 +919,7 @@ { "cell_type": "markdown", "id": "8a5e6e58", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Visualizing the Tree, The Moons" ] @@ -1019,10 +928,7 @@ "cell_type": "code", "execution_count": 4, "id": "7fc84427", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "# Common imports\n", @@ -1054,9 +960,7 @@ { "cell_type": "markdown", "id": "51420499", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Other ways of visualizing the trees\n", "\n", @@ -1067,10 +971,7 @@ "cell_type": "code", "execution_count": 5, "id": "d01a0b57", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "from sklearn.datasets import load_iris\n", @@ -1085,9 +986,7 @@ { "cell_type": "markdown", "id": "0d5f2779", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Printing out as text\n", "\n", @@ -1099,10 +998,7 @@ "cell_type": "code", "execution_count": 6, "id": "18b6d8ae", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "from sklearn.datasets import load_iris\n", @@ -1118,9 +1014,7 @@ { "cell_type": "markdown", "id": "17129339", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Algorithms for Setting up Decision Trees\n", "\n", @@ -1138,9 +1032,7 @@ { "cell_type": "markdown", "id": "08011876", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## The CART algorithm for Classification\n", "\n", @@ -1155,9 +1047,7 @@ { "cell_type": "markdown", "id": "624b8153", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "C(k,t_k) = \\frac{m_{\\mathrm{left}}}{m}G_{\\mathrm{left}}+ \\frac{m_{\\mathrm{right}}}{m}G_{\\mathrm{right}},\n", @@ -1167,9 +1057,7 @@ { "cell_type": "markdown", "id": "353969b6", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "where $G_{\\mathrm{left/right}}$ measures the impurity of the left/right subset and $m_{\\mathrm{left/right}}$\n", " is the number of instances in the left/right subset\n", @@ -1184,9 +1072,7 @@ { "cell_type": "markdown", "id": "bcdb3558", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## The CART algorithm for Regression\n", "\n", @@ -1197,9 +1083,7 @@ { "cell_type": "markdown", "id": "661abdb2", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "C(k,t_k) = \\frac{m_{\\mathrm{left}}}{m}\\mathrm{MSE}_{\\mathrm{left}}+ \\frac{m_{\\mathrm{right}}}{m}\\mathrm{MSE}_{\\mathrm{right}}.\n", @@ -1209,9 +1093,7 @@ { "cell_type": "markdown", "id": "b1baa1b7", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "Here the MSE for a specific node is defined as" ] @@ -1219,9 +1101,7 @@ { "cell_type": "markdown", "id": "5aeb1ed8", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\mathrm{MSE}_{\\mathrm{node}}=\\frac{1}{m_\\mathrm{node}}\\sum_{i\\in \\mathrm{node}}(\\overline{y}_{\\mathrm{node}}-y_i)^2,\n", @@ -1231,9 +1111,7 @@ { "cell_type": "markdown", "id": "d7f66eb1", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "with" ] @@ -1241,9 +1119,7 @@ { "cell_type": "markdown", "id": "6bcb07d8", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\overline{y}_{\\mathrm{node}}=\\frac{1}{m_\\mathrm{node}}\\sum_{i\\in \\mathrm{node}}y_i,\n", @@ -1253,9 +1129,7 @@ { "cell_type": "markdown", "id": "a5cd6be2", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "the mean value of all observations in a specific node.\n", "\n", @@ -1266,9 +1140,7 @@ { "cell_type": "markdown", "id": "f7c1aca8", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Computing the Gini index\n", "\n", @@ -1310,9 +1182,7 @@ { "cell_type": "markdown", "id": "dd90d405", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Simple Python Code to read in Data and perform Classification" ] @@ -1321,10 +1191,7 @@ "cell_type": "code", "execution_count": 7, "id": "055dfb75", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "# Common imports\n", @@ -1399,9 +1266,7 @@ { "cell_type": "markdown", "id": "8fa370be", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Computing the Gini Factor\n", "\n", @@ -1417,10 +1282,7 @@ "cell_type": "code", "execution_count": 8, "id": "2eae920c", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "# Split a dataset based on an attribute and an attribute value\n", @@ -1488,9 +1350,7 @@ { "cell_type": "markdown", "id": "76e6b68a", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Entropy and the ID3 algorithm\n", "\n", @@ -1527,9 +1387,7 @@ { "cell_type": "markdown", "id": "c3ef1baf", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Cancer Data again now with Decision Trees and other Methods" ] @@ -1538,10 +1396,7 @@ "cell_type": "code", "execution_count": 9, "id": "7637aca8", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "import matplotlib.pyplot as plt\n", @@ -1590,9 +1445,7 @@ { "cell_type": "markdown", "id": "e2a20f84", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Another example, the moons again" ] @@ -1601,10 +1454,7 @@ "cell_type": "code", "execution_count": 10, "id": "6dc739e6", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "from __future__ import division, print_function, unicode_literals\n", @@ -1676,9 +1526,7 @@ { "cell_type": "markdown", "id": "ffd5aca0", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Playing around with regions" ] @@ -1687,10 +1535,7 @@ "cell_type": "code", "execution_count": 11, "id": "a535e0a3", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "np.random.seed(6)\n", @@ -1718,9 +1563,7 @@ { "cell_type": "markdown", "id": "c43dd40f", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Regression trees" ] @@ -1729,10 +1572,7 @@ "cell_type": "code", "execution_count": 12, "id": "2f92a4bc", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "# Quadratic training set + noise\n", @@ -1747,10 +1587,7 @@ "cell_type": "code", "execution_count": 13, "id": "786d574c", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "from sklearn.tree import DecisionTreeRegressor\n", @@ -1762,9 +1599,7 @@ { "cell_type": "markdown", "id": "4667ad89", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Final regressor code" ] @@ -1773,10 +1608,7 @@ "cell_type": "code", "execution_count": 14, "id": "1a164143", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "from sklearn.tree import DecisionTreeRegressor\n", @@ -1823,10 +1655,7 @@ "cell_type": "code", "execution_count": 15, "id": "39ad2aed", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "tree_reg1 = DecisionTreeRegressor(random_state=42)\n", @@ -1862,9 +1691,7 @@ { "cell_type": "markdown", "id": "5b0a0240", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Pros and cons of trees, pros\n", "\n", @@ -1886,9 +1713,7 @@ { "cell_type": "markdown", "id": "f05ef6f9", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Disadvantages\n", "\n", @@ -1914,9 +1739,7 @@ { "cell_type": "markdown", "id": "50ea5d1d", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods\n", "\n", @@ -1945,9 +1768,7 @@ { "cell_type": "markdown", "id": "3dfb29fa", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## An Overview of Ensemble Methods\n", "\n", @@ -1961,9 +1782,7 @@ { "cell_type": "markdown", "id": "9505410d", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Bagging\n", "\n", @@ -1983,9 +1802,7 @@ { "cell_type": "markdown", "id": "807639f2", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## More bagging\n", "\n", @@ -2015,9 +1832,7 @@ { "cell_type": "markdown", "id": "a3763486", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Making your own Bootstrap: Changing the Level of the Decision Tree\n", "\n", @@ -2029,10 +1844,7 @@ "cell_type": "code", "execution_count": 16, "id": "e9d87bdd", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "\n", @@ -2092,9 +1904,7 @@ { "cell_type": "markdown", "id": "daf49858", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Why Voting?\n", "\n", @@ -2116,9 +1926,7 @@ { "cell_type": "markdown", "id": "6a146f2d", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Tossing coins\n", "\n", @@ -2147,9 +1955,7 @@ { "cell_type": "markdown", "id": "42892d97", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Standard imports first" ] @@ -2158,10 +1964,7 @@ "cell_type": "code", "execution_count": 17, "id": "231bdf38", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "# Common imports\n", @@ -2206,9 +2009,7 @@ { "cell_type": "markdown", "id": "094cf1cb", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Simple Voting Example, head or tail" ] @@ -2217,10 +2018,7 @@ "cell_type": "code", "execution_count": 18, "id": "48790785", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "\n", @@ -2251,9 +2049,7 @@ { "cell_type": "markdown", "id": "d5339f7b", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Using the Voting Classifier\n", "\n", @@ -2264,10 +2060,7 @@ "cell_type": "code", "execution_count": 19, "id": "6a1080ec", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "from sklearn.model_selection import train_test_split\n", @@ -2317,9 +2110,7 @@ { "cell_type": "markdown", "id": "47d98581", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Voting and Bagging" ] @@ -2328,10 +2119,7 @@ "cell_type": "code", "execution_count": 20, "id": "c7fd3f00", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "from sklearn.model_selection import train_test_split\n", @@ -2358,10 +2146,7 @@ "cell_type": "code", "execution_count": 21, "id": "c9f7b269", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "from sklearn.metrics import accuracy_score\n", @@ -2376,10 +2161,7 @@ "cell_type": "code", "execution_count": 22, "id": "b45c2311", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "log_clf = LogisticRegression(random_state=42)\n", @@ -2396,10 +2178,7 @@ "cell_type": "code", "execution_count": 23, "id": "f16cc6d0", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "from sklearn.metrics import accuracy_score\n", @@ -2413,9 +2192,7 @@ { "cell_type": "markdown", "id": "4018fbab", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Random forests\n", "\n", @@ -2436,9 +2213,7 @@ { "cell_type": "markdown", "id": "d2e3facd", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "m\\approx \\sqrt{p}.\n", @@ -2448,9 +2223,7 @@ { "cell_type": "markdown", "id": "7a73c7de", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "In building a random forest, at\n", "each split in the tree, the algorithm is not even allowed to consider\n", @@ -2473,9 +2246,7 @@ { "cell_type": "markdown", "id": "761888c6", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Random Forest Algorithm\n", "The algorithm described here can be applied to both classification and regression problems.\n", @@ -2499,9 +2270,7 @@ { "cell_type": "markdown", "id": "4a3c676f", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Random Forests Compared with other Methods on the Cancer Data" ] @@ -2510,10 +2279,7 @@ "cell_type": "code", "execution_count": 24, "id": "0839a696", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "import matplotlib.pyplot as plt\n", @@ -2587,9 +2353,7 @@ { "cell_type": "markdown", "id": "57e88525", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "Recall that the cumulative gains curve shows the percentage of the\n", "overall number of cases in a given category *gained* by targeting a\n", @@ -2603,9 +2367,7 @@ { "cell_type": "markdown", "id": "4515344d", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Compare Bagging on Trees with Random Forests" ] @@ -2614,10 +2376,7 @@ "cell_type": "code", "execution_count": 25, "id": "d80e2f55", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "bag_clf = BaggingClassifier(\n", @@ -2629,10 +2388,7 @@ "cell_type": "code", "execution_count": 26, "id": "27457485", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "bag_clf.fit(X_train, y_train)\n", @@ -2647,9 +2403,7 @@ { "cell_type": "markdown", "id": "e25dd4ec", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Boosting, a Bird's Eye View\n", "\n", @@ -2667,9 +2421,7 @@ { "cell_type": "markdown", "id": "92dafde6", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## What is boosting? Additive Modelling/Iterative Fitting\n", "\n", @@ -2681,9 +2433,7 @@ { "cell_type": "markdown", "id": "bc20c85e", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "f_M(x) = \\sum_{i=1}^M \\beta_m b(x;\\gamma_m),\n", @@ -2693,9 +2443,7 @@ { "cell_type": "markdown", "id": "4ba594a4", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "where $\\beta_m$ are the expansion parameters to be determined in a\n", "minimization process and $b(x;\\gamma_m)$ are some simple functions of\n", @@ -2710,9 +2458,7 @@ { "cell_type": "markdown", "id": "8593117f", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\sigma(t) = \\frac{1}{1+\\exp{(-t)}},\n", @@ -2722,9 +2468,7 @@ { "cell_type": "markdown", "id": "cc0781ac", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "where $t=\\gamma_0+\\gamma_1 x$ and the parameters $\\gamma_0$ and\n", "$\\gamma_1$ were determined by the Logistic Regression fitting\n", @@ -2736,9 +2480,7 @@ { "cell_type": "markdown", "id": "ce1a2a56", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "C(\\boldsymbol{y},\\boldsymbol{f}) = \\frac{1}{n} \\sum_{i=0}^{n-1}(y_i-f(x_i))^2.\n", @@ -2748,9 +2490,7 @@ { "cell_type": "markdown", "id": "93ab835c", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "In this case the function $f(x)$ was replaced by the design matrix\n", "$\\boldsymbol{X}$ and the unknown linear regression parameters $\\boldsymbol{\\beta}$,\n", @@ -2761,9 +2501,7 @@ { "cell_type": "markdown", "id": "d9a0cc62", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\boldsymbol{\\beta}=\\left(\\boldsymbol{X}^T\\boldsymbol{X}\\right)^{-1}\\boldsymbol{X}^T\\boldsymbol{y}.\n", @@ -2773,9 +2511,7 @@ { "cell_type": "markdown", "id": "a4e87a0b", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "In iterative fitting or additive modeling, we minimize the cost function with respect to the parameters $\\beta_m$ and $\\gamma_m$." ] @@ -2783,9 +2519,7 @@ { "cell_type": "markdown", "id": "4cd671b2", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Iterative Fitting, Regression and Squared-error Cost Function\n", "\n", @@ -2811,9 +2545,7 @@ { "cell_type": "markdown", "id": "f9e4e4c5", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Squared-Error Example and Iterative Fitting\n", "\n", @@ -2827,9 +2559,7 @@ { "cell_type": "markdown", "id": "98051368", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "(\\beta_m,\\gamma_m) = \\mathrm{argmin}_{\\beta,\\lambda}\\hspace{0.1cm} \\sum_{i=0}^{n-1}(y_i-f_{m-1}(x_i)-\\beta b(x;\\gamma))^2=\\sum_{i=0}^{n-1}(y_i-f_{m-1}(x_i)-\\beta(1+\\gamma x_i))^2.\n", @@ -2839,9 +2569,7 @@ { "cell_type": "markdown", "id": "b89afedd", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "We start our iteration by simply setting $f_0(x)=0$. \n", "Taking the derivatives with respect to $\\beta$ and $\\gamma$ we obtain" @@ -2850,9 +2578,7 @@ { "cell_type": "markdown", "id": "4ce53503", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\frac{\\partial {\\cal C}}{\\partial \\beta} = -2\\sum_{i}(1+\\gamma x_i)(y_i-\\beta(1+\\gamma x_i))=0,\n", @@ -2862,9 +2588,7 @@ { "cell_type": "markdown", "id": "bfd354cd", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "and" ] @@ -2872,9 +2596,7 @@ { "cell_type": "markdown", "id": "1cdd2ab0", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\frac{\\partial {\\cal C}}{\\partial \\gamma} =-2\\sum_{i}\\beta x_i(y_i-\\beta(1+\\gamma x_i))=0.\n", @@ -2884,9 +2606,7 @@ { "cell_type": "markdown", "id": "abc6bca0", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "We can then rewrite these equations as (defining $\\boldsymbol{w}=\\boldsymbol{e}+\\gamma \\boldsymbol{x})$ with $\\boldsymbol{e}$ being the unit vector)" ] @@ -2894,9 +2614,7 @@ { "cell_type": "markdown", "id": "15ab72b0", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\gamma \\boldsymbol{w}^T(\\boldsymbol{y}-\\beta\\gamma \\boldsymbol{w})=0,\n", @@ -2906,9 +2624,7 @@ { "cell_type": "markdown", "id": "30076526", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "which gives us $\\beta = \\boldsymbol{w}^T\\boldsymbol{y}/(\\boldsymbol{w}^T\\boldsymbol{w})$. Similarly we have" ] @@ -2916,9 +2632,7 @@ { "cell_type": "markdown", "id": "036983c0", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\beta\\gamma \\boldsymbol{x}^T(\\boldsymbol{y}-\\beta(1+\\gamma \\boldsymbol{x}))=0,\n", @@ -2928,9 +2642,7 @@ { "cell_type": "markdown", "id": "01855f77", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "which leads to $\\gamma =(\\boldsymbol{x}^T\\boldsymbol{y}-\\beta\\boldsymbol{x}^T\\boldsymbol{e})/(\\beta\\boldsymbol{x}^T\\boldsymbol{x})$. Inserting\n", "for $\\beta$ gives us an equation for $\\gamma$. This is a non-linear equation in the unknown $\\gamma$ and has to be solved numerically. \n", @@ -2942,9 +2654,7 @@ { "cell_type": "markdown", "id": "ea3b23fd", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Iterative Fitting, Classification and AdaBoost\n", "\n", @@ -2958,9 +2668,7 @@ { "cell_type": "markdown", "id": "785ad3fb", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\mathrm{\\overline{err}}=\\frac{1}{n} \\sum_{i=0}^{n-1} I(y_i\\ne G(x_i)).\n", @@ -2970,9 +2678,7 @@ { "cell_type": "markdown", "id": "bcdd75ca", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "The iterative procedure starts with defining a weak classifier whose\n", "error rate is barely better than random guessing. The iterative\n", @@ -2986,9 +2692,7 @@ { "cell_type": "markdown", "id": "4066ea94", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "f_M(x) = \\sum_{i=1}^M \\beta_m b(x;\\gamma_m),\n", @@ -2998,9 +2702,7 @@ { "cell_type": "markdown", "id": "2bd9f2bd", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "will be a function of" ] @@ -3008,9 +2710,7 @@ { "cell_type": "markdown", "id": "4e0d1bcd", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "G_M(x) = \\mathrm{sign} \\sum_{i=1}^M \\alpha_m G_m(x).\n", @@ -3020,9 +2720,7 @@ { "cell_type": "markdown", "id": "c843142c", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Adaptive Boosting, AdaBoost\n", "\n", @@ -3032,9 +2730,7 @@ { "cell_type": "markdown", "id": "fa9764ef", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "f_m(x) = f_{m-1}(x)+\\beta_mG_m(x).\n", @@ -3044,9 +2740,7 @@ { "cell_type": "markdown", "id": "08271d96", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "The simplest possible cost function which leads (also simple from a computational point of view) to the AdaBoost algorithm is the\n", "exponential cost/loss function defined as" @@ -3055,9 +2749,7 @@ { "cell_type": "markdown", "id": "730500b4", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "C(\\boldsymbol{y},\\boldsymbol{f}) = \\sum_{i=0}^{n-1}\\exp{(-y_i(f_{m-1}(x_i)+\\beta G(x_i))}.\n", @@ -3067,9 +2759,7 @@ { "cell_type": "markdown", "id": "a5cfcea5", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "We optimize $\\beta$ and $G$ for each value of $m=1:M$ as we did in the regression case.\n", "This is normally done in two steps. Let us however first rewrite the cost function as" @@ -3078,9 +2768,7 @@ { "cell_type": "markdown", "id": "f120a894", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "C(\\boldsymbol{y},\\boldsymbol{f}) = \\sum_{i=0}^{n-1}w_i^{m}\\exp{(-y_i\\beta G(x_i))},\n", @@ -3090,9 +2778,7 @@ { "cell_type": "markdown", "id": "b9b9773e", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "where we have defined $w_i^m= \\exp{(-y_if_{m-1}(x_i))}$." ] @@ -3100,9 +2786,7 @@ { "cell_type": "markdown", "id": "945e7c76", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Building up AdaBoost\n", "\n", @@ -3112,9 +2796,7 @@ { "cell_type": "markdown", "id": "02655c91", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "G_m(x) = \\mathrm{sign} \\sum_{i=0}^{n-1} w_i^m I(y_i \\ne G_(x_i)),\n", @@ -3124,9 +2806,7 @@ { "cell_type": "markdown", "id": "a75a0f9b", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "which is the classifier that minimizes the weighted error rate in predicting $y$.\n", "\n", @@ -3136,9 +2816,7 @@ { "cell_type": "markdown", "id": "ae35403a", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\exp{-(\\beta)}\\sum_{y_i=G(x_i)}w_i^m+\\exp{(\\beta)}\\sum_{y_i\\ne G(x_i)}w_i^m,\n", @@ -3148,9 +2826,7 @@ { "cell_type": "markdown", "id": "091c1c6f", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "which can be rewritten as" ] @@ -3158,9 +2834,7 @@ { "cell_type": "markdown", "id": "30eb3053", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "(\\exp{(\\beta)}-\\exp{-(\\beta)})\\sum_{i=0}^{n-1}w_i^mI(y_i\\ne G(x_i))+\\exp{(-\\beta)}\\sum_{i=0}^{n-1}w_i^m=0,\n", @@ -3170,9 +2844,7 @@ { "cell_type": "markdown", "id": "f6a6d37d", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "which leads to" ] @@ -3180,9 +2852,7 @@ { "cell_type": "markdown", "id": "37e33765", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\beta_m = \\frac{1}{2}\\log{\\frac{1-\\mathrm{\\overline{err}}}{\\mathrm{\\overline{err}}}},\n", @@ -3192,9 +2862,7 @@ { "cell_type": "markdown", "id": "fffc2eed", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "where we have redefined the error as" ] @@ -3202,9 +2870,7 @@ { "cell_type": "markdown", "id": "f5a1e2cc", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\mathrm{\\overline{err}}_m=\\frac{1}{n}\\frac{\\sum_{i=0}^{n-1}w_i^mI(y_i\\ne G(x_i)}{\\sum_{i=0}^{n-1}w_i^m},\n", @@ -3214,9 +2880,7 @@ { "cell_type": "markdown", "id": "922eb174", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "which leads to an update of" ] @@ -3224,9 +2888,7 @@ { "cell_type": "markdown", "id": "e20016de", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "f_m(x) = f_{m-1}(x) +\\beta_m G_m(x).\n", @@ -3236,9 +2898,7 @@ { "cell_type": "markdown", "id": "2c6ddb1a", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "This leads to the new weights" ] @@ -3246,9 +2906,7 @@ { "cell_type": "markdown", "id": "d10fc0fe", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "w_i^{m+1} = w_i^m \\exp{(-y_i\\beta_m G_m(x_i))}\n", @@ -3258,9 +2916,7 @@ { "cell_type": "markdown", "id": "da5a926c", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Adaptive boosting: AdaBoost, Basic Algorithm\n", "\n", @@ -3278,9 +2934,7 @@ { "cell_type": "markdown", "id": "d5985efa", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\mathrm{err}=\\frac{1}{n}\\sum_{i=0}^{n-1}I(y_i\\ne G(x_i)),\n", @@ -3290,9 +2944,7 @@ { "cell_type": "markdown", "id": "19bb8280", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "where the function $I()$ is one if we misclassify and zero if we classify correctly." ] @@ -3300,9 +2952,7 @@ { "cell_type": "markdown", "id": "7608b386", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Basic Steps of AdaBoost\n", "\n", @@ -3316,9 +2966,7 @@ { "cell_type": "markdown", "id": "0ae121e3", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "\\mathrm{\\overline{err}}_m=\\frac{\\sum_{i=0}^{n-1}w_i^m I(y_i\\ne G(x_i))}{\\sum_{i=0}^{n-1}w_i},\n", @@ -3328,9 +2976,7 @@ { "cell_type": "markdown", "id": "6a21a16e", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "1. Then we start looping over all attempts at classifying, namely we start an iterative process for $m=1:M$, where $M$ is the final number of classifications. Our given classifier could for example be a plain decision tree.\n", "\n", @@ -3356,9 +3002,7 @@ { "cell_type": "markdown", "id": "5a06c2c0", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## AdaBoost Examples\n", "\n", @@ -3369,10 +3013,7 @@ "cell_type": "code", "execution_count": 27, "id": "5879f377", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "from sklearn.ensemble import AdaBoostClassifier\n", @@ -3401,9 +3042,7 @@ { "cell_type": "markdown", "id": "e9279b9f", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Gradient boosting: Basics with Steepest Descent/Functional Gradient Descent\n", "\n", @@ -3419,9 +3058,7 @@ { "cell_type": "markdown", "id": "89199bb6", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## The Squared-Error again! Steepest Descent\n", "\n", @@ -3432,9 +3069,7 @@ { "cell_type": "markdown", "id": "d1d4dff1", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "(\\hat{\\boldsymbol{f}}) = \\mathrm{argmin}_{\\boldsymbol{f}}\\hspace{0.1cm} \\sum_{i=0}^{n-1}(y_i-f(x_i))^2.\n", @@ -3444,9 +3079,7 @@ { "cell_type": "markdown", "id": "15d22a20", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "We define a real function $h_m(x)$ that defines our final function $f_M(x)$ as" ] @@ -3454,9 +3087,7 @@ { "cell_type": "markdown", "id": "0a210fee", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "f_M(x) = \\sum_{m=0}^M h_m(x).\n", @@ -3466,9 +3097,7 @@ { "cell_type": "markdown", "id": "5cf2a6a3", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "In the steepest decent approach we approximate $h_m(x) = -\\rho_m g_m(x)$, where $\\rho_m$ is a scalar and $g_m(x)$ the gradient defined as" ] @@ -3476,9 +3105,7 @@ { "cell_type": "markdown", "id": "ace744f9", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "g_m(x_i) = \\left[ \\frac{\\partial {\\cal L}(y_i, f(x_i))}{\\partial f(x_i)}\\right]_{f(x_i)=f_{m-1}(x_i)}.\n", @@ -3488,9 +3115,7 @@ { "cell_type": "markdown", "id": "5b5c897f", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "With the new gradient we can update $f_m(x) = f_{m-1}(x) -\\rho_m g_m(x)$. Using the above squared-error function we see that\n", "the gradient is $g_m(x_i) = -2(y_i-f(x_i))$.\n", @@ -3501,9 +3126,7 @@ { "cell_type": "markdown", "id": "5562a17c", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "(\\rho_1) = \\mathrm{argmin}_{\\rho}\\hspace{0.1cm} \\sum_{i=0}^{n-1}(y_i+2\\rho y_i)^2.\n", @@ -3513,9 +3136,7 @@ { "cell_type": "markdown", "id": "a37be653", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Steepest Descent Example\n", "\n", @@ -3525,9 +3146,7 @@ { "cell_type": "markdown", "id": "ad73dbb2", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "f_1(x) = f_{0}(x) -\\rho_1 g_1(x)=-y_i.\n", @@ -3537,9 +3156,7 @@ { "cell_type": "markdown", "id": "126d482b", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "We can then proceed and compute" ] @@ -3547,9 +3164,7 @@ { "cell_type": "markdown", "id": "4d415cc8", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "g_2(x_i) = \\left[ \\frac{\\partial {\\cal L}(y_i, f(x_i))}{\\partial f(x_i)}\\right]_{f(x_i)=f_{1}(x_i)=y_i}=-4y_i,\n", @@ -3559,9 +3174,7 @@ { "cell_type": "markdown", "id": "a1ef6ad9", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "and find a new value for $\\rho_2=-1/2$ and continue till we have reached $m=M$. We can modify the steepest descent method, or steepest boosting, by introducing what is called **gradient boosting**." ] @@ -3569,9 +3182,7 @@ { "cell_type": "markdown", "id": "8961dc18", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Gradient Boosting, algorithm\n", "\n", @@ -3585,9 +3196,7 @@ { "cell_type": "markdown", "id": "769a885a", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "$$\n", "C(\\boldsymbol{y},\\boldsymbol{f})=\\sum_{i=0}^{n-1}(y_i-f(x_i))^2.\n", @@ -3597,9 +3206,7 @@ { "cell_type": "markdown", "id": "94f43097", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "The way we proceed in an iterative fashion is to\n", "1. Initialize our estimate $f_0(x)$.\n", @@ -3618,9 +3225,7 @@ { "cell_type": "markdown", "id": "f7fb4f86", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Gradient Boosting, Examples of Regression" ] @@ -3629,10 +3234,7 @@ "cell_type": "code", "execution_count": 28, "id": "8b3f2c0c", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "import matplotlib.pyplot as plt\n", @@ -3686,9 +3288,7 @@ { "cell_type": "markdown", "id": "bf4be4e9", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Gradient Boosting, Classification Example" ] @@ -3697,10 +3297,7 @@ "cell_type": "code", "execution_count": 29, "id": "bee1829a", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "import matplotlib.pyplot as plt\n", @@ -3748,9 +3345,7 @@ { "cell_type": "markdown", "id": "ac9eb4bb", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## XGBoost: Extreme Gradient Boosting\n", "\n", @@ -3771,9 +3366,7 @@ { "cell_type": "markdown", "id": "fbe2011a", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Regression Case" ] @@ -3782,10 +3375,7 @@ "cell_type": "code", "execution_count": 30, "id": "878c0380", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "import matplotlib.pyplot as plt\n", @@ -3839,9 +3429,7 @@ { "cell_type": "markdown", "id": "7cf64b38", - "metadata": { - "editable": true - }, + "metadata": {}, "source": [ "## Xgboost on the Cancer Data\n", "\n", @@ -3852,10 +3440,7 @@ "cell_type": "code", "execution_count": 31, "id": "65a3e655", - "metadata": { - "collapsed": false, - "editable": true - }, + "metadata": {}, "outputs": [], "source": [ "\n", @@ -3913,7 +3498,25 @@ ] } ], - "metadata": {}, + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.8" + } + }, "nbformat": 4, "nbformat_minor": 5 }