-
Simple Python Code to read in Data and perform Classification
- - - -# Common imports
-import numpy as np
-import pandas as pd
-import matplotlib.pyplot as plt
-from sklearn.tree import DecisionTreeClassifier
-from sklearn.model_selection import train_test_split
-from sklearn.tree import export_graphviz
-from sklearn.preprocessing import StandardScaler, OneHotEncoder
-from sklearn.compose import ColumnTransformer
-from IPython.display import Image
-from pydot import graph_from_dot_data
-import os
-
-# Where to save the figures and data files
-PROJECT_ROOT_DIR = "Results"
-FIGURE_ID = "Results/FigureFiles"
-DATA_ID = "DataFiles/"
-
-if not os.path.exists(PROJECT_ROOT_DIR):
- os.mkdir(PROJECT_ROOT_DIR)
-
-if not os.path.exists(FIGURE_ID):
- os.makedirs(FIGURE_ID)
-
-if not os.path.exists(DATA_ID):
- os.makedirs(DATA_ID)
-
-def image_path(fig_id):
- return os.path.join(FIGURE_ID, fig_id)
-
-def data_path(dat_id):
- return os.path.join(DATA_ID, dat_id)
-
-def save_fig(fig_id):
- plt.savefig(image_path(fig_id) + ".png", format='png')
-
-infile = open(data_path("rideclass.csv"),'r')
-
-# Read the experimental data with Pandas
-from IPython.display import display
-ridedata = pd.read_csv(infile,names = ('Outlook','Temperature','Humidity','Wind','Ride'))
-ridedata = pd.DataFrame(ridedata)
-
-# Features and targets
-X = ridedata.loc[:, ridedata.columns != 'Ride'].values
-y = ridedata.loc[:, ridedata.columns == 'Ride'].values
-
-# Create the encoder.
-encoder = OneHotEncoder(handle_unknown="ignore")
-# Assume for simplicity all features are categorical.
-encoder.fit(X)
-# Apply the encoder.
-X = encoder.transform(X)
-print(X)
-# Then do a Classification tree
-tree_clf = DecisionTreeClassifier(max_depth=2)
-tree_clf.fit(X, y)
-print("Train set accuracy with Decision Tree: {:.2f}".format(tree_clf.score(X,y)))
-#transfer to a decision tree graph
-export_graphviz(
- tree_clf,
- out_file="DataFiles/ride.dot",
- rounded=True,
- filled=True
-)
-cmd = 'dot -Tpng DataFiles/cancer.dot -o DataFiles/cancer.png'
-os.system(cmd)
-
-Computing a Tree using the Gini Index
+Consider the following example with attributes/features and two +possible outcomes (classes) for each attribute. Assume we wish to find some +correlations between the average grade of a student as function of the +number of hours studied and hours slept. We want also to correlate the +grade in a given course with the general trend, whether the students +recently has gotten grades below average or above. +
+We have three features/attributes
+-
+
- Trend of average grades before present course, classified as either below or above the average grade of the whole class +
- The number of hours studies, classified again as either higher (more than 3 hours per day) or lower . Here we have used a standard for one \( ECTS \) which is scaled to 25-30 hours of work for a semester which lasts 18 weeks, with 15 weeks of lectures and 3 weeks for exams, assuming a total of 30 ECTS per semester. +
- The number of hours slept as high for more than \( 8 \) hours and below for less than 8 hours of sleep, classified again as either high or low +
- The final grade whether it is above or below average +
-
diff --git a/doc/pub/week44/html/._week44-bs030.html b/doc/pub/week44/html/._week44-bs030.html
index 19732afd8..1db5e6abf 100644
--- a/doc/pub/week44/html/._week44-bs030.html
+++ b/doc/pub/week44/html/._week44-bs030.html
@@ -115,67 +115,6 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d
2,
None,
'the-cart-algorithm-for-regression'),
- ('Computing the Gini index', 2, None, 'computing-the-gini-index'),
- ('Simple Python Code to read in Data and perform Classification',
- 2,
- None,
- 'simple-python-code-to-read-in-data-and-perform-classification'),
- ('Computing the Gini Factor',
- 2,
- None,
- 'computing-the-gini-factor'),
- ('Entropy and the ID3 algorithm',
- 2,
- None,
- 'entropy-and-the-id3-algorithm'),
- ('Cancer Data again now with Decision Trees and other Methods',
- 2,
- None,
- 'cancer-data-again-now-with-decision-trees-and-other-methods'),
- ('Another example, the moons again',
- 2,
- None,
- 'another-example-the-moons-again'),
- ('Playing around with regions',
- 2,
- None,
- 'playing-around-with-regions'),
- ('Regression trees', 2, None, 'regression-trees'),
- ('Final regressor code', 2, None, 'final-regressor-code'),
- ('Pros and cons of trees, pros',
- 2,
- None,
- 'pros-and-cons-of-trees-pros'),
- ('Disadvantages', 2, None, 'disadvantages'),
- ('Ensemble Methods: From a Single Tree to Many Trees and Extreme '
- 'Boosting, Meet the Jungle of Methods',
- 2,
- None,
- 'ensemble-methods-from-a-single-tree-to-many-trees-and-extreme-boosting-meet-the-jungle-of-methods'),
- ('An Overview of Ensemble Methods',
- 2,
- None,
- 'an-overview-of-ensemble-methods'),
- ('Bagging', 2, None, 'bagging'),
- ('More bagging', 2, None, 'more-bagging'),
- ('Simple Voting Example, head or tail',
- 2,
- None,
- 'simple-voting-example-head-or-tail'),
- ('Using the Voting Classifier',
- 2,
- None,
- 'using-the-voting-classifier'),
- ('Please, not the moons again! Voting and Bagging',
- 2,
- None,
- 'please-not-the-moons-again-voting-and-bagging'),
- ('Bagging Examples', 2, None, 'bagging-examples'),
- ('Making your own Bootstrap: Changing the Level of the Decision '
- 'Tree',
- 2,
- None,
- 'making-your-own-bootstrap-changing-the-level-of-the-decision-tree'),
('Why binary splits?', 2, None, 'why-binary-splits'),
('Computing a Tree using the Gini Index',
2,
@@ -250,6 +189,70 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d
'playing-around-with-regions'),
('Regression trees', 2, None, 'regression-trees'),
('Final regressor code', 2, None, 'final-regressor-code'),
+ ('Example: Computing the Gini index',
+ 2,
+ None,
+ 'example-computing-the-gini-index'),
+ ('Simple Python Code to read in Data and perform Classification',
+ 2,
+ None,
+ 'simple-python-code-to-read-in-data-and-perform-classification'),
+ ('Computing the Gini Factor',
+ 2,
+ None,
+ 'computing-the-gini-factor'),
+ ('Entropy and the ID3 algorithm',
+ 2,
+ None,
+ 'entropy-and-the-id3-algorithm'),
+ ('Cancer Data again now with Decision Trees and other Methods',
+ 2,
+ None,
+ 'cancer-data-again-now-with-decision-trees-and-other-methods'),
+ ('Another example, the moons again',
+ 2,
+ None,
+ 'another-example-the-moons-again'),
+ ('Playing around with regions',
+ 2,
+ None,
+ 'playing-around-with-regions'),
+ ('Regression trees', 2, None, 'regression-trees'),
+ ('Final regressor code', 2, None, 'final-regressor-code'),
+ ('Pros and cons of trees, pros',
+ 2,
+ None,
+ 'pros-and-cons-of-trees-pros'),
+ ('Disadvantages', 2, None, 'disadvantages'),
+ ('Ensemble Methods: From a Single Tree to Many Trees and Extreme '
+ 'Boosting, Meet the Jungle of Methods',
+ 2,
+ None,
+ 'ensemble-methods-from-a-single-tree-to-many-trees-and-extreme-boosting-meet-the-jungle-of-methods'),
+ ('An Overview of Ensemble Methods',
+ 2,
+ None,
+ 'an-overview-of-ensemble-methods'),
+ ('Bagging', 2, None, 'bagging'),
+ ('More bagging', 2, None, 'more-bagging'),
+ ('Simple Voting Example, head or tail',
+ 2,
+ None,
+ 'simple-voting-example-head-or-tail'),
+ ('Using the Voting Classifier',
+ 2,
+ None,
+ 'using-the-voting-classifier'),
+ ('Please, not the moons again! Voting and Bagging',
+ 2,
+ None,
+ 'please-not-the-moons-again-voting-and-bagging'),
+ ('Bagging Examples', 2, None, 'bagging-examples'),
+ ('Making your own Bootstrap: Changing the Level of the Decision '
+ 'Tree',
+ 2,
+ None,
+ 'making-your-own-bootstrap-changing-the-level-of-the-decision-tree'),
('Why Voting?', 2, None, 'why-voting'),
('Tossing coins', 2, None, 'tossing-coins'),
('Standard imports first', 2, None, 'standard-imports-first'),
@@ -327,55 +330,55 @@ MathJax.Hub.Config({
- Growing a classification tree
- Classification tree, how to split nodes
- Visualizing the Tree, Classification -
- Visualizing the Tree, The Moons -
- Other ways of visualizing the trees -
- Printing out as text -
- Algorithms for Setting up Decision Trees -
- The CART algorithm for Classification -
- The CART algorithm for Regression -
- Computing the Gini index -
- Simple Python Code to read in Data and perform Classification -
- Computing the Gini Factor -
- Entropy and the ID3 algorithm -
- Cancer Data again now with Decision Trees and other Methods -
- Another example, the moons again -
- Playing around with regions -
- Regression trees -
- Final regressor code -
- Pros and cons of trees, pros -
- Disadvantages -
- Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods -
- An Overview of Ensemble Methods -
- Bagging -
- More bagging +
- Visualizing the Tree, The Moons +
- Other ways of visualizing the trees +
- Printing out as text +
- Algorithms for Setting up Decision Trees +
- The CART algorithm for Classification +
- The CART algorithm for Regression +
- Why binary splits? +
- Computing a Tree using the Gini Index +
- The Table +
- Computing the various Gini Indices +
- Computing the various Gini Indices, Hours slept +
- Computing the various Gini Indices, Hours studied +
- A possible code using Scikit-Learn +
- Visualizing Trees, More examples +
- Visualizing the Tree, The Moons +
- Other ways of visualizing the trees +
- Printing out as text +
- Algorithms for Setting up Decision Trees +
- The CART algorithm for Classification +
- The CART algorithm for Regression +
- Computing the Gini index +
- Simple Python Code to read in Data and perform Classification +
- Computing the Gini Factor +
- Entropy and the ID3 algorithm +
- Cancer Data again now with Decision Trees and other Methods +
- Another example, the moons again +
- Playing around with regions +
- Regression trees +
- Final regressor code +
- Example: Computing the Gini index +
- Simple Python Code to read in Data and perform Classification +
- Computing the Gini Factor +
- Entropy and the ID3 algorithm +
- Cancer Data again now with Decision Trees and other Methods +
- Another example, the moons again +
- Playing around with regions +
- Regression trees +
- Final regressor code +
- Pros and cons of trees, pros +
- Disadvantages +
- Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods +
- An Overview of Ensemble Methods +
- Bagging +
- More bagging
- Simple Voting Example, head or tail
- Using the Voting Classifier -
- Please, not the moons again! Voting and Bagging -
- Bagging Examples -
- Making your own Bootstrap: Changing the Level of the Decision Tree -
- Why binary splits? -
- Computing a Tree using the Gini Index -
- The Table -
- Computing the various Gini Indices -
- Computing the various Gini Indices, Hours slept -
- Computing the various Gini Indices, Hours studied -
- A possible code using Scikit-Learn -
- Visualizing Trees, More examples -
- Visualizing the Tree, The Moons -
- Other ways of visualizing the trees -
- Printing out as text -
- Algorithms for Setting up Decision Trees -
- The CART algorithm for Classification -
- The CART algorithm for Regression -
- Computing the Gini index -
- Simple Python Code to read in Data and perform Classification -
- Computing the Gini Factor -
- Entropy and the ID3 algorithm -
- Cancer Data again now with Decision Trees and other Methods -
- Another example, the moons again -
- Playing around with regions -
- Regression trees -
- Final regressor code +
- Please, not the moons again! Voting and Bagging +
- Bagging Examples +
- Making your own Bootstrap: Changing the Level of the Decision Tree
- Why Voting?
- Tossing coins
- Standard imports first @@ -397,98 +400,29 @@ MathJax.Hub.Config({
- Growing a classification tree
- Classification tree, how to split nodes
- Visualizing the Tree, Classification -
- Visualizing the Tree, The Moons -
- Other ways of visualizing the trees -
- Printing out as text -
- Algorithms for Setting up Decision Trees -
- The CART algorithm for Classification -
- The CART algorithm for Regression -
- Computing the Gini index -
- Simple Python Code to read in Data and perform Classification -
- Computing the Gini Factor -
- Entropy and the ID3 algorithm -
- Cancer Data again now with Decision Trees and other Methods -
- Another example, the moons again -
- Playing around with regions -
- Regression trees -
- Final regressor code -
- Pros and cons of trees, pros -
- Disadvantages -
- Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods -
- An Overview of Ensemble Methods -
- Bagging -
- More bagging +
- Visualizing the Tree, The Moons +
- Other ways of visualizing the trees +
- Printing out as text +
- Algorithms for Setting up Decision Trees +
- The CART algorithm for Classification +
- The CART algorithm for Regression +
- Why binary splits? +
- Computing a Tree using the Gini Index +
- The Table +
- Computing the various Gini Indices +
- Computing the various Gini Indices, Hours slept +
- Computing the various Gini Indices, Hours studied +
- A possible code using Scikit-Learn +
- Visualizing Trees, More examples +
- Visualizing the Tree, The Moons +
- Other ways of visualizing the trees +
- Printing out as text +
- Algorithms for Setting up Decision Trees +
- The CART algorithm for Classification +
- The CART algorithm for Regression +
- Computing the Gini index +
- Simple Python Code to read in Data and perform Classification +
- Computing the Gini Factor +
- Entropy and the ID3 algorithm +
- Cancer Data again now with Decision Trees and other Methods +
- Another example, the moons again +
- Playing around with regions +
- Regression trees +
- Final regressor code +
- Example: Computing the Gini index +
- Simple Python Code to read in Data and perform Classification +
- Computing the Gini Factor +
- Entropy and the ID3 algorithm +
- Cancer Data again now with Decision Trees and other Methods +
- Another example, the moons again +
- Playing around with regions +
- Regression trees +
- Final regressor code +
- Pros and cons of trees, pros +
- Disadvantages +
- Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods +
- An Overview of Ensemble Methods +
- Bagging +
- More bagging
- Simple Voting Example, head or tail
- Using the Voting Classifier -
- Please, not the moons again! Voting and Bagging -
- Bagging Examples -
- Making your own Bootstrap: Changing the Level of the Decision Tree -
- Why binary splits? -
- Computing a Tree using the Gini Index -
- The Table -
- Computing the various Gini Indices -
- Computing the various Gini Indices, Hours slept -
- Computing the various Gini Indices, Hours studied -
- A possible code using Scikit-Learn -
- Visualizing Trees, More examples -
- Visualizing the Tree, The Moons -
- Other ways of visualizing the trees -
- Printing out as text -
- Algorithms for Setting up Decision Trees -
- The CART algorithm for Classification -
- The CART algorithm for Regression -
- Computing the Gini index -
- Simple Python Code to read in Data and perform Classification -
- Computing the Gini Factor -
- Entropy and the ID3 algorithm -
- Cancer Data again now with Decision Trees and other Methods -
- Another example, the moons again -
- Playing around with regions -
- Regression trees -
- Final regressor code +
- Please, not the moons again! Voting and Bagging +
- Bagging Examples +
- Making your own Bootstrap: Changing the Level of the Decision Tree
- Why Voting?
- Tossing coins
- Standard imports first @@ -397,37 +400,19 @@ MathJax.Hub.Config({
- Each instance attribute is evaluated using a statistical test to determine how well it alone classifies the training examples. -
- The best attribute is selected and used as the test at the root node of the tree. -
- A descendant of the root node is then created for each possible value of this attribute. -
- Training examples are sorted to the appropriate descendant node. -
- The entire process is then repeated using the training examples associated with each descendant node to select the best attribute to test at that point in the tree. -
- This forms a greedy search for an acceptable decision tree, in which the algorithm never backtracks to reconsider earlier choices. -
- Growing a classification tree
- Classification tree, how to split nodes
- Visualizing the Tree, Classification -
- Visualizing the Tree, The Moons -
- Other ways of visualizing the trees -
- Printing out as text -
- Algorithms for Setting up Decision Trees -
- The CART algorithm for Classification -
- The CART algorithm for Regression -
- Computing the Gini index -
- Simple Python Code to read in Data and perform Classification -
- Computing the Gini Factor -
- Entropy and the ID3 algorithm -
- Cancer Data again now with Decision Trees and other Methods -
- Another example, the moons again -
- Playing around with regions -
- Regression trees -
- Final regressor code -
- Pros and cons of trees, pros -
- Disadvantages -
- Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods -
- An Overview of Ensemble Methods -
- Bagging -
- More bagging +
- Visualizing the Tree, The Moons +
- Other ways of visualizing the trees +
- Printing out as text +
- Algorithms for Setting up Decision Trees +
- The CART algorithm for Classification +
- The CART algorithm for Regression +
- Why binary splits? +
- Computing a Tree using the Gini Index +
- The Table +
- Computing the various Gini Indices +
- Computing the various Gini Indices, Hours slept +
- Computing the various Gini Indices, Hours studied +
- A possible code using Scikit-Learn +
- Visualizing Trees, More examples +
- Visualizing the Tree, The Moons +
- Other ways of visualizing the trees +
- Printing out as text +
- Algorithms for Setting up Decision Trees +
- The CART algorithm for Classification +
- The CART algorithm for Regression +
- Computing the Gini index +
- Simple Python Code to read in Data and perform Classification +
- Computing the Gini Factor +
- Entropy and the ID3 algorithm +
- Cancer Data again now with Decision Trees and other Methods +
- Another example, the moons again +
- Playing around with regions +
- Regression trees +
- Final regressor code +
- Example: Computing the Gini index +
- Simple Python Code to read in Data and perform Classification +
- Computing the Gini Factor +
- Entropy and the ID3 algorithm +
- Cancer Data again now with Decision Trees and other Methods +
- Another example, the moons again +
- Playing around with regions +
- Regression trees +
- Final regressor code +
- Pros and cons of trees, pros +
- Disadvantages +
- Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods +
- An Overview of Ensemble Methods +
- Bagging +
- More bagging
- Simple Voting Example, head or tail
- Using the Voting Classifier -
- Please, not the moons again! Voting and Bagging -
- Bagging Examples -
- Making your own Bootstrap: Changing the Level of the Decision Tree -
- Why binary splits? -
- Computing a Tree using the Gini Index -
- The Table -
- Computing the various Gini Indices -
- Computing the various Gini Indices, Hours slept -
- Computing the various Gini Indices, Hours studied -
- A possible code using Scikit-Learn -
- Visualizing Trees, More examples -
- Visualizing the Tree, The Moons -
- Other ways of visualizing the trees -
- Printing out as text -
- Algorithms for Setting up Decision Trees -
- The CART algorithm for Classification -
- The CART algorithm for Regression -
- Computing the Gini index -
- Simple Python Code to read in Data and perform Classification -
- Computing the Gini Factor -
- Entropy and the ID3 algorithm -
- Cancer Data again now with Decision Trees and other Methods -
- Another example, the moons again -
- Playing around with regions -
- Regression trees -
- Final regressor code +
- Please, not the moons again! Voting and Bagging +
- Bagging Examples +
- Making your own Bootstrap: Changing the Level of the Decision Tree
- Why Voting?
- Tossing coins
- Standard imports first @@ -397,68 +400,13 @@ MathJax.Hub.Config({
- Growing a classification tree
- Classification tree, how to split nodes
- Visualizing the Tree, Classification -
- Visualizing the Tree, The Moons -
- Other ways of visualizing the trees -
- Printing out as text -
- Algorithms for Setting up Decision Trees -
- The CART algorithm for Classification -
- The CART algorithm for Regression -
- Computing the Gini index -
- Simple Python Code to read in Data and perform Classification -
- Computing the Gini Factor -
- Entropy and the ID3 algorithm -
- Cancer Data again now with Decision Trees and other Methods -
- Another example, the moons again -
- Playing around with regions -
- Regression trees -
- Final regressor code -
- Pros and cons of trees, pros -
- Disadvantages -
- Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods -
- An Overview of Ensemble Methods -
- Bagging -
- More bagging +
- Visualizing the Tree, The Moons +
- Other ways of visualizing the trees +
- Printing out as text +
- Algorithms for Setting up Decision Trees +
- The CART algorithm for Classification +
- The CART algorithm for Regression +
- Why binary splits? +
- Computing a Tree using the Gini Index +
- The Table +
- Computing the various Gini Indices +
- Computing the various Gini Indices, Hours slept +
- Computing the various Gini Indices, Hours studied +
- A possible code using Scikit-Learn +
- Visualizing Trees, More examples +
- Visualizing the Tree, The Moons +
- Other ways of visualizing the trees +
- Printing out as text +
- Algorithms for Setting up Decision Trees +
- The CART algorithm for Classification +
- The CART algorithm for Regression +
- Computing the Gini index +
- Simple Python Code to read in Data and perform Classification +
- Computing the Gini Factor +
- Entropy and the ID3 algorithm +
- Cancer Data again now with Decision Trees and other Methods +
- Another example, the moons again +
- Playing around with regions +
- Regression trees +
- Final regressor code +
- Example: Computing the Gini index +
- Simple Python Code to read in Data and perform Classification +
- Computing the Gini Factor +
- Entropy and the ID3 algorithm +
- Cancer Data again now with Decision Trees and other Methods +
- Another example, the moons again +
- Playing around with regions +
- Regression trees +
- Final regressor code +
- Pros and cons of trees, pros +
- Disadvantages +
- Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods +
- An Overview of Ensemble Methods +
- Bagging +
- More bagging
- Simple Voting Example, head or tail
- Using the Voting Classifier -
- Please, not the moons again! Voting and Bagging -
- Bagging Examples -
- Making your own Bootstrap: Changing the Level of the Decision Tree -
- Why binary splits? -
- Computing a Tree using the Gini Index -
- The Table -
- Computing the various Gini Indices -
- Computing the various Gini Indices, Hours slept -
- Computing the various Gini Indices, Hours studied -
- A possible code using Scikit-Learn -
- Visualizing Trees, More examples -
- Visualizing the Tree, The Moons -
- Other ways of visualizing the trees -
- Printing out as text -
- Algorithms for Setting up Decision Trees -
- The CART algorithm for Classification -
- The CART algorithm for Regression -
- Computing the Gini index -
- Simple Python Code to read in Data and perform Classification -
- Computing the Gini Factor -
- Entropy and the ID3 algorithm -
- Cancer Data again now with Decision Trees and other Methods -
- Another example, the moons again -
- Playing around with regions -
- Regression trees -
- Final regressor code +
- Please, not the moons again! Voting and Bagging +
- Bagging Examples +
- Making your own Bootstrap: Changing the Level of the Decision Tree
- Why Voting?
- Tossing coins
- Standard imports first @@ -397,94 +400,18 @@ MathJax.Hub.Config({
- Growing a classification tree
- Classification tree, how to split nodes
- Visualizing the Tree, Classification -
- Visualizing the Tree, The Moons -
- Other ways of visualizing the trees -
- Printing out as text -
- Algorithms for Setting up Decision Trees -
- The CART algorithm for Classification -
- The CART algorithm for Regression -
- Computing the Gini index -
- Simple Python Code to read in Data and perform Classification -
- Computing the Gini Factor -
- Entropy and the ID3 algorithm -
- Cancer Data again now with Decision Trees and other Methods -
- Another example, the moons again -
- Playing around with regions -
- Regression trees -
- Final regressor code -
- Pros and cons of trees, pros -
- Disadvantages -
- Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods -
- An Overview of Ensemble Methods -
- Bagging -
- More bagging +
- Visualizing the Tree, The Moons +
- Other ways of visualizing the trees +
- Printing out as text +
- Algorithms for Setting up Decision Trees +
- The CART algorithm for Classification +
- The CART algorithm for Regression +
- Why binary splits? +
- Computing a Tree using the Gini Index +
- The Table +
- Computing the various Gini Indices +
- Computing the various Gini Indices, Hours slept +
- Computing the various Gini Indices, Hours studied +
- A possible code using Scikit-Learn +
- Visualizing Trees, More examples +
- Visualizing the Tree, The Moons +
- Other ways of visualizing the trees +
- Printing out as text +
- Algorithms for Setting up Decision Trees +
- The CART algorithm for Classification +
- The CART algorithm for Regression +
- Computing the Gini index +
- Simple Python Code to read in Data and perform Classification +
- Computing the Gini Factor +
- Entropy and the ID3 algorithm +
- Cancer Data again now with Decision Trees and other Methods +
- Another example, the moons again +
- Playing around with regions +
- Regression trees +
- Final regressor code +
- Example: Computing the Gini index +
- Simple Python Code to read in Data and perform Classification +
- Computing the Gini Factor +
- Entropy and the ID3 algorithm +
- Cancer Data again now with Decision Trees and other Methods +
- Another example, the moons again +
- Playing around with regions +
- Regression trees +
- Final regressor code +
- Pros and cons of trees, pros +
- Disadvantages +
- Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods +
- An Overview of Ensemble Methods +
- Bagging +
- More bagging
- Simple Voting Example, head or tail
- Using the Voting Classifier -
- Please, not the moons again! Voting and Bagging -
- Bagging Examples -
- Making your own Bootstrap: Changing the Level of the Decision Tree -
- Why binary splits? -
- Computing a Tree using the Gini Index -
- The Table -
- Computing the various Gini Indices -
- Computing the various Gini Indices, Hours slept -
- Computing the various Gini Indices, Hours studied -
- A possible code using Scikit-Learn -
- Visualizing Trees, More examples -
- Visualizing the Tree, The Moons -
- Other ways of visualizing the trees -
- Printing out as text -
- Algorithms for Setting up Decision Trees -
- The CART algorithm for Classification -
- The CART algorithm for Regression -
- Computing the Gini index -
- Simple Python Code to read in Data and perform Classification -
- Computing the Gini Factor -
- Entropy and the ID3 algorithm -
- Cancer Data again now with Decision Trees and other Methods -
- Another example, the moons again -
- Playing around with regions -
- Regression trees -
- Final regressor code +
- Please, not the moons again! Voting and Bagging +
- Bagging Examples +
- Making your own Bootstrap: Changing the Level of the Decision Tree
- Why Voting?
- Tossing coins
- Standard imports first @@ -397,7 +400,8 @@ MathJax.Hub.Config({
- Growing a classification tree
- Classification tree, how to split nodes
- Visualizing the Tree, Classification -
- Visualizing the Tree, The Moons -
- Other ways of visualizing the trees -
- Printing out as text -
- Algorithms for Setting up Decision Trees -
- The CART algorithm for Classification -
- The CART algorithm for Regression -
- Computing the Gini index -
- Simple Python Code to read in Data and perform Classification -
- Computing the Gini Factor -
- Entropy and the ID3 algorithm -
- Cancer Data again now with Decision Trees and other Methods -
- Another example, the moons again -
- Playing around with regions -
- Regression trees -
- Final regressor code -
- Pros and cons of trees, pros -
- Disadvantages -
- Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods -
- An Overview of Ensemble Methods -
- Bagging -
- More bagging +
- Visualizing the Tree, The Moons +
- Other ways of visualizing the trees +
- Printing out as text +
- Algorithms for Setting up Decision Trees +
- The CART algorithm for Classification +
- The CART algorithm for Regression +
- Why binary splits? +
- Computing a Tree using the Gini Index +
- The Table +
- Computing the various Gini Indices +
- Computing the various Gini Indices, Hours slept +
- Computing the various Gini Indices, Hours studied +
- A possible code using Scikit-Learn +
- Visualizing Trees, More examples +
- Visualizing the Tree, The Moons +
- Other ways of visualizing the trees +
- Printing out as text +
- Algorithms for Setting up Decision Trees +
- The CART algorithm for Classification +
- The CART algorithm for Regression +
- Computing the Gini index +
- Simple Python Code to read in Data and perform Classification +
- Computing the Gini Factor +
- Entropy and the ID3 algorithm +
- Cancer Data again now with Decision Trees and other Methods +
- Another example, the moons again +
- Playing around with regions +
- Regression trees +
- Final regressor code +
- Example: Computing the Gini index +
- Simple Python Code to read in Data and perform Classification +
- Computing the Gini Factor +
- Entropy and the ID3 algorithm +
- Cancer Data again now with Decision Trees and other Methods +
- Another example, the moons again +
- Playing around with regions +
- Regression trees +
- Final regressor code +
- Pros and cons of trees, pros +
- Disadvantages +
- Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods +
- An Overview of Ensemble Methods +
- Bagging +
- More bagging
- Simple Voting Example, head or tail
- Using the Voting Classifier -
- Please, not the moons again! Voting and Bagging -
- Bagging Examples -
- Making your own Bootstrap: Changing the Level of the Decision Tree -
- Why binary splits? -
- Computing a Tree using the Gini Index -
- The Table -
- Computing the various Gini Indices -
- Computing the various Gini Indices, Hours slept -
- Computing the various Gini Indices, Hours studied -
- A possible code using Scikit-Learn -
- Visualizing Trees, More examples -
- Visualizing the Tree, The Moons -
- Other ways of visualizing the trees -
- Printing out as text -
- Algorithms for Setting up Decision Trees -
- The CART algorithm for Classification -
- The CART algorithm for Regression -
- Computing the Gini index -
- Simple Python Code to read in Data and perform Classification -
- Computing the Gini Factor -
- Entropy and the ID3 algorithm -
- Cancer Data again now with Decision Trees and other Methods -
- Another example, the moons again -
- Playing around with regions -
- Regression trees -
- Final regressor code +
- Please, not the moons again! Voting and Bagging +
- Bagging Examples +
- Making your own Bootstrap: Changing the Level of the Decision Tree
- Why Voting?
- Tossing coins
- Standard imports first @@ -397,7 +400,7 @@ MathJax.Hub.Config({
- Growing a classification tree
- Classification tree, how to split nodes
- Visualizing the Tree, Classification -
- Visualizing the Tree, The Moons -
- Other ways of visualizing the trees -
- Printing out as text -
- Algorithms for Setting up Decision Trees -
- The CART algorithm for Classification -
- The CART algorithm for Regression -
- Computing the Gini index -
- Simple Python Code to read in Data and perform Classification -
- Computing the Gini Factor -
- Entropy and the ID3 algorithm -
- Cancer Data again now with Decision Trees and other Methods -
- Another example, the moons again -
- Playing around with regions -
- Regression trees -
- Final regressor code -
- Pros and cons of trees, pros -
- Disadvantages -
- Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods -
- An Overview of Ensemble Methods -
- Bagging -
- More bagging +
- Visualizing the Tree, The Moons +
- Other ways of visualizing the trees +
- Printing out as text +
- Algorithms for Setting up Decision Trees +
- The CART algorithm for Classification +
- The CART algorithm for Regression +
- Why binary splits? +
- Computing a Tree using the Gini Index +
- The Table +
- Computing the various Gini Indices +
- Computing the various Gini Indices, Hours slept +
- Computing the various Gini Indices, Hours studied +
- A possible code using Scikit-Learn +
- Visualizing Trees, More examples +
- Visualizing the Tree, The Moons +
- Other ways of visualizing the trees +
- Printing out as text +
- Algorithms for Setting up Decision Trees +
- The CART algorithm for Classification +
- The CART algorithm for Regression +
- Computing the Gini index +
- Simple Python Code to read in Data and perform Classification +
- Computing the Gini Factor +
- Entropy and the ID3 algorithm +
- Cancer Data again now with Decision Trees and other Methods +
- Another example, the moons again +
- Playing around with regions +
- Regression trees +
- Final regressor code +
- Example: Computing the Gini index +
- Simple Python Code to read in Data and perform Classification +
- Computing the Gini Factor +
- Entropy and the ID3 algorithm +
- Cancer Data again now with Decision Trees and other Methods +
- Another example, the moons again +
- Playing around with regions +
- Regression trees +
- Final regressor code +
- Pros and cons of trees, pros +
- Disadvantages +
- Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods +
- An Overview of Ensemble Methods +
- Bagging +
- More bagging
- Simple Voting Example, head or tail
- Using the Voting Classifier -
- Please, not the moons again! Voting and Bagging -
- Bagging Examples -
- Making your own Bootstrap: Changing the Level of the Decision Tree -
- Why binary splits? -
- Computing a Tree using the Gini Index -
- The Table -
- Computing the various Gini Indices -
- Computing the various Gini Indices, Hours slept -
- Computing the various Gini Indices, Hours studied -
- A possible code using Scikit-Learn -
- Visualizing Trees, More examples -
- Visualizing the Tree, The Moons -
- Other ways of visualizing the trees -
- Printing out as text -
- Algorithms for Setting up Decision Trees -
- The CART algorithm for Classification -
- The CART algorithm for Regression -
- Computing the Gini index -
- Simple Python Code to read in Data and perform Classification -
- Computing the Gini Factor -
- Entropy and the ID3 algorithm -
- Cancer Data again now with Decision Trees and other Methods -
- Another example, the moons again -
- Playing around with regions -
- Regression trees -
- Final regressor code +
- Please, not the moons again! Voting and Bagging +
- Bagging Examples +
- Making your own Bootstrap: Changing the Level of the Decision Tree
- Why Voting?
- Tossing coins
- Standard imports first @@ -397,7 +400,7 @@ MathJax.Hub.Config({
- Growing a classification tree
- Classification tree, how to split nodes
- Visualizing the Tree, Classification -
- Visualizing the Tree, The Moons -
- Other ways of visualizing the trees -
- Printing out as text -
- Algorithms for Setting up Decision Trees -
- The CART algorithm for Classification -
- The CART algorithm for Regression -
- Computing the Gini index -
- Simple Python Code to read in Data and perform Classification -
- Computing the Gini Factor -
- Entropy and the ID3 algorithm -
- Cancer Data again now with Decision Trees and other Methods -
- Another example, the moons again -
- Playing around with regions -
- Regression trees -
- Final regressor code -
- Pros and cons of trees, pros -
- Disadvantages -
- Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods -
- An Overview of Ensemble Methods -
- Bagging -
- More bagging +
- Visualizing the Tree, The Moons +
- Other ways of visualizing the trees +
- Printing out as text +
- Algorithms for Setting up Decision Trees +
- The CART algorithm for Classification +
- The CART algorithm for Regression +
- Why binary splits? +
- Computing a Tree using the Gini Index +
- The Table +
- Computing the various Gini Indices +
- Computing the various Gini Indices, Hours slept +
- Computing the various Gini Indices, Hours studied +
- A possible code using Scikit-Learn +
- Visualizing Trees, More examples +
- Visualizing the Tree, The Moons +
- Other ways of visualizing the trees +
- Printing out as text +
- Algorithms for Setting up Decision Trees +
- The CART algorithm for Classification +
- The CART algorithm for Regression +
- Computing the Gini index +
- Simple Python Code to read in Data and perform Classification +
- Computing the Gini Factor +
- Entropy and the ID3 algorithm +
- Cancer Data again now with Decision Trees and other Methods +
- Another example, the moons again +
- Playing around with regions +
- Regression trees +
- Final regressor code +
- Example: Computing the Gini index +
- Simple Python Code to read in Data and perform Classification +
- Computing the Gini Factor +
- Entropy and the ID3 algorithm +
- Cancer Data again now with Decision Trees and other Methods +
- Another example, the moons again +
- Playing around with regions +
- Regression trees +
- Final regressor code +
- Pros and cons of trees, pros +
- Disadvantages +
- Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods +
- An Overview of Ensemble Methods +
- Bagging +
- More bagging
- Simple Voting Example, head or tail
- Using the Voting Classifier -
- Please, not the moons again! Voting and Bagging -
- Bagging Examples -
- Making your own Bootstrap: Changing the Level of the Decision Tree -
- Why binary splits? -
- Computing a Tree using the Gini Index -
- The Table -
- Computing the various Gini Indices -
- Computing the various Gini Indices, Hours slept -
- Computing the various Gini Indices, Hours studied -
- A possible code using Scikit-Learn -
- Visualizing Trees, More examples -
- Visualizing the Tree, The Moons -
- Other ways of visualizing the trees -
- Printing out as text -
- Algorithms for Setting up Decision Trees -
- The CART algorithm for Classification -
- The CART algorithm for Regression -
- Computing the Gini index -
- Simple Python Code to read in Data and perform Classification -
- Computing the Gini Factor -
- Entropy and the ID3 algorithm -
- Cancer Data again now with Decision Trees and other Methods -
- Another example, the moons again -
- Playing around with regions -
- Regression trees -
- Final regressor code +
- Please, not the moons again! Voting and Bagging +
- Bagging Examples +
- Making your own Bootstrap: Changing the Level of the Decision Tree
- Why Voting?
- Tossing coins
- Standard imports first @@ -397,17 +400,40 @@ MathJax.Hub.Config({
- White box, easy to interpret model. Some people believe that decision trees more closely mirror human decision-making than do the regression and classification approaches discussed earlier (think of support vector machines) -
- Trees are very easy to explain to people. In fact, they are even easier to explain than linear regression! -
- No feature normalization needed -
- Tree models can handle both continuous and categorical data (Classification and Regression Trees) -
- Can model nonlinear relationships -
- Can model interactions between the different descriptive features -
- Trees can be displayed graphically, and are easily interpreted even by a non-expert (especially if they are small) -
- Growing a classification tree
- Classification tree, how to split nodes
- Visualizing the Tree, Classification -
- Visualizing the Tree, The Moons -
- Other ways of visualizing the trees -
- Printing out as text -
- Algorithms for Setting up Decision Trees -
- The CART algorithm for Classification -
- The CART algorithm for Regression -
- Computing the Gini index -
- Simple Python Code to read in Data and perform Classification -
- Computing the Gini Factor -
- Entropy and the ID3 algorithm -
- Cancer Data again now with Decision Trees and other Methods -
- Another example, the moons again -
- Playing around with regions -
- Regression trees -
- Final regressor code -
- Pros and cons of trees, pros -
- Disadvantages -
- Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods -
- An Overview of Ensemble Methods -
- Bagging -
- More bagging +
- Visualizing the Tree, The Moons +
- Other ways of visualizing the trees +
- Printing out as text +
- Algorithms for Setting up Decision Trees +
- The CART algorithm for Classification +
- The CART algorithm for Regression +
- Why binary splits? +
- Computing a Tree using the Gini Index +
- The Table +
- Computing the various Gini Indices +
- Computing the various Gini Indices, Hours slept +
- Computing the various Gini Indices, Hours studied +
- A possible code using Scikit-Learn +
- Visualizing Trees, More examples +
- Visualizing the Tree, The Moons +
- Other ways of visualizing the trees +
- Printing out as text +
- Algorithms for Setting up Decision Trees +
- The CART algorithm for Classification +
- The CART algorithm for Regression +
- Computing the Gini index +
- Simple Python Code to read in Data and perform Classification +
- Computing the Gini Factor +
- Entropy and the ID3 algorithm +
- Cancer Data again now with Decision Trees and other Methods +
- Another example, the moons again +
- Playing around with regions +
- Regression trees +
- Final regressor code +
- Example: Computing the Gini index +
- Simple Python Code to read in Data and perform Classification +
- Computing the Gini Factor +
- Entropy and the ID3 algorithm +
- Cancer Data again now with Decision Trees and other Methods +
- Another example, the moons again +
- Playing around with regions +
- Regression trees +
- Final regressor code +
- Pros and cons of trees, pros +
- Disadvantages +
- Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods +
- An Overview of Ensemble Methods +
- Bagging +
- More bagging
- Simple Voting Example, head or tail
- Using the Voting Classifier -
- Please, not the moons again! Voting and Bagging -
- Bagging Examples -
- Making your own Bootstrap: Changing the Level of the Decision Tree -
- Why binary splits? -
- Computing a Tree using the Gini Index -
- The Table -
- Computing the various Gini Indices -
- Computing the various Gini Indices, Hours slept -
- Computing the various Gini Indices, Hours studied -
- A possible code using Scikit-Learn -
- Visualizing Trees, More examples -
- Visualizing the Tree, The Moons -
- Other ways of visualizing the trees -
- Printing out as text -
- Algorithms for Setting up Decision Trees -
- The CART algorithm for Classification -
- The CART algorithm for Regression -
- Computing the Gini index -
- Simple Python Code to read in Data and perform Classification -
- Computing the Gini Factor -
- Entropy and the ID3 algorithm -
- Cancer Data again now with Decision Trees and other Methods -
- Another example, the moons again -
- Playing around with regions -
- Regression trees -
- Final regressor code +
- Please, not the moons again! Voting and Bagging +
- Bagging Examples +
- Making your own Bootstrap: Changing the Level of the Decision Tree
- Why Voting?
- Tossing coins
- Standard imports first @@ -397,22 +400,43 @@ MathJax.Hub.Config({
- Unfortunately, trees generally do not have the same level of predictive accuracy as some of the other regression and classification approaches -
- If continuous features are used the tree may become quite large and hence less interpretable -
- Decision trees are prone to overfit the training data and hence do not well generalize the data if no stopping criteria or improvements like pruning, boosting or bagging are implemented -
- Small changes in the data may lead to a completely different tree. This issue can be addressed by using ensemble methods like bagging, boosting or random forests -
- Unbalanced datasets where some target feature values occur much more frequently than others may lead to biased trees since the frequently occurring feature values are preferred over the less frequently occurring ones. -
- If the number of features is relatively large (high dimensional) and the number of instances is relatively low, the tree might overfit the data -
- Features with many levels may be preferred over features with less levels since for them it is more easy to split the dataset such that the sub datasets only contain pure target feature values. This issue can be addressed by preferring for instance the information gain ratio as splitting criteria over information gain -
- Growing a classification tree
- Classification tree, how to split nodes
- Visualizing the Tree, Classification -
- Visualizing the Tree, The Moons -
- Other ways of visualizing the trees -
- Printing out as text -
- Algorithms for Setting up Decision Trees -
- The CART algorithm for Classification -
- The CART algorithm for Regression -
- Computing the Gini index -
- Simple Python Code to read in Data and perform Classification -
- Computing the Gini Factor -
- Entropy and the ID3 algorithm -
- Cancer Data again now with Decision Trees and other Methods -
- Another example, the moons again -
- Playing around with regions -
- Regression trees -
- Final regressor code -
- Pros and cons of trees, pros -
- Disadvantages -
- Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods -
- An Overview of Ensemble Methods -
- Bagging -
- More bagging +
- Visualizing the Tree, The Moons +
- Other ways of visualizing the trees +
- Printing out as text +
- Algorithms for Setting up Decision Trees +
- The CART algorithm for Classification +
- The CART algorithm for Regression +
- Why binary splits? +
- Computing a Tree using the Gini Index +
- The Table +
- Computing the various Gini Indices +
- Computing the various Gini Indices, Hours slept +
- Computing the various Gini Indices, Hours studied +
- A possible code using Scikit-Learn +
- Visualizing Trees, More examples +
- Visualizing the Tree, The Moons +
- Other ways of visualizing the trees +
- Printing out as text +
- Algorithms for Setting up Decision Trees +
- The CART algorithm for Classification +
- The CART algorithm for Regression +
- Computing the Gini index +
- Simple Python Code to read in Data and perform Classification +
- Computing the Gini Factor +
- Entropy and the ID3 algorithm +
- Cancer Data again now with Decision Trees and other Methods +
- Another example, the moons again +
- Playing around with regions +
- Regression trees +
- Final regressor code +
- Example: Computing the Gini index +
- Simple Python Code to read in Data and perform Classification +
- Computing the Gini Factor +
- Entropy and the ID3 algorithm +
- Cancer Data again now with Decision Trees and other Methods +
- Another example, the moons again +
- Playing around with regions +
- Regression trees +
- Final regressor code +
- Pros and cons of trees, pros +
- Disadvantages +
- Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods +
- An Overview of Ensemble Methods +
- Bagging +
- More bagging
- Simple Voting Example, head or tail
- Using the Voting Classifier -
- Please, not the moons again! Voting and Bagging -
- Bagging Examples -
- Making your own Bootstrap: Changing the Level of the Decision Tree -
- Why binary splits? -
- Computing a Tree using the Gini Index -
- The Table -
- Computing the various Gini Indices -
- Computing the various Gini Indices, Hours slept -
- Computing the various Gini Indices, Hours studied -
- A possible code using Scikit-Learn -
- Visualizing Trees, More examples -
- Visualizing the Tree, The Moons -
- Other ways of visualizing the trees -
- Printing out as text -
- Algorithms for Setting up Decision Trees -
- The CART algorithm for Classification -
- The CART algorithm for Regression -
- Computing the Gini index -
- Simple Python Code to read in Data and perform Classification -
- Computing the Gini Factor -
- Entropy and the ID3 algorithm -
- Cancer Data again now with Decision Trees and other Methods -
- Another example, the moons again -
- Playing around with regions -
- Regression trees -
- Final regressor code +
- Please, not the moons again! Voting and Bagging +
- Bagging Examples +
- Making your own Bootstrap: Changing the Level of the Decision Tree
- Why Voting?
- Tossing coins
- Standard imports first @@ -397,28 +400,18 @@ MathJax.Hub.Config({
- Voting classifiers -
- Bagging and Pasting -
- Random forests -
- Boosting methods, from adaptive to Extreme Gradient Boosting (XGBoost) +
- The CART (Classification And Regression Tree) algorithm for both classification and regression +
- The ID3 algorithm based on the computation of the information gain for classification
- Growing a classification tree
- Classification tree, how to split nodes
- Visualizing the Tree, Classification -
- Visualizing the Tree, The Moons -
- Other ways of visualizing the trees -
- Printing out as text -
- Algorithms for Setting up Decision Trees -
- The CART algorithm for Classification -
- The CART algorithm for Regression -
- Computing the Gini index -
- Simple Python Code to read in Data and perform Classification -
- Computing the Gini Factor -
- Entropy and the ID3 algorithm -
- Cancer Data again now with Decision Trees and other Methods -
- Another example, the moons again -
- Playing around with regions -
- Regression trees -
- Final regressor code -
- Pros and cons of trees, pros -
- Disadvantages -
- Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods -
- An Overview of Ensemble Methods -
- Bagging -
- More bagging +
- Visualizing the Tree, The Moons +
- Other ways of visualizing the trees +
- Printing out as text +
- Algorithms for Setting up Decision Trees +
- The CART algorithm for Classification +
- The CART algorithm for Regression +
- Why binary splits? +
- Computing a Tree using the Gini Index +
- The Table +
- Computing the various Gini Indices +
- Computing the various Gini Indices, Hours slept +
- Computing the various Gini Indices, Hours studied +
- A possible code using Scikit-Learn +
- Visualizing Trees, More examples +
- Visualizing the Tree, The Moons +
- Other ways of visualizing the trees +
- Printing out as text +
- Algorithms for Setting up Decision Trees +
- The CART algorithm for Classification +
- The CART algorithm for Regression +
- Computing the Gini index +
- Simple Python Code to read in Data and perform Classification +
- Computing the Gini Factor +
- Entropy and the ID3 algorithm +
- Cancer Data again now with Decision Trees and other Methods +
- Another example, the moons again +
- Playing around with regions +
- Regression trees +
- Final regressor code +
- Example: Computing the Gini index +
- Simple Python Code to read in Data and perform Classification +
- Computing the Gini Factor +
- Entropy and the ID3 algorithm +
- Cancer Data again now with Decision Trees and other Methods +
- Another example, the moons again +
- Playing around with regions +
- Regression trees +
- Final regressor code +
- Pros and cons of trees, pros +
- Disadvantages +
- Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods +
- An Overview of Ensemble Methods +
- Bagging +
- More bagging
- Simple Voting Example, head or tail
- Using the Voting Classifier -
- Please, not the moons again! Voting and Bagging -
- Bagging Examples -
- Making your own Bootstrap: Changing the Level of the Decision Tree -
- Why binary splits? -
- Computing a Tree using the Gini Index -
- The Table -
- Computing the various Gini Indices -
- Computing the various Gini Indices, Hours slept -
- Computing the various Gini Indices, Hours studied -
- A possible code using Scikit-Learn -
- Visualizing Trees, More examples -
- Visualizing the Tree, The Moons -
- Other ways of visualizing the trees -
- Printing out as text -
- Algorithms for Setting up Decision Trees -
- The CART algorithm for Classification -
- The CART algorithm for Regression -
- Computing the Gini index -
- Simple Python Code to read in Data and perform Classification -
- Computing the Gini Factor -
- Entropy and the ID3 algorithm -
- Cancer Data again now with Decision Trees and other Methods -
- Another example, the moons again -
- Playing around with regions -
- Regression trees -
- Final regressor code +
- Please, not the moons again! Voting and Bagging +
- Bagging Examples +
- Making your own Bootstrap: Changing the Level of the Decision Tree
- Why Voting?
- Tossing coins
- Standard imports first @@ -397,13 +400,30 @@ MathJax.Hub.Config({
- Growing a classification tree
- Classification tree, how to split nodes
- Visualizing the Tree, Classification -
- Visualizing the Tree, The Moons -
- Other ways of visualizing the trees -
- Printing out as text -
- Algorithms for Setting up Decision Trees -
- The CART algorithm for Classification -
- The CART algorithm for Regression -
- Computing the Gini index -
- Simple Python Code to read in Data and perform Classification -
- Computing the Gini Factor -
- Entropy and the ID3 algorithm -
- Cancer Data again now with Decision Trees and other Methods -
- Another example, the moons again -
- Playing around with regions -
- Regression trees -
- Final regressor code -
- Pros and cons of trees, pros -
- Disadvantages -
- Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods -
- An Overview of Ensemble Methods -
- Bagging -
- More bagging +
- Visualizing the Tree, The Moons +
- Other ways of visualizing the trees +
- Printing out as text +
- Algorithms for Setting up Decision Trees +
- The CART algorithm for Classification +
- The CART algorithm for Regression +
- Why binary splits? +
- Computing a Tree using the Gini Index +
- The Table +
- Computing the various Gini Indices +
- Computing the various Gini Indices, Hours slept +
- Computing the various Gini Indices, Hours studied +
- A possible code using Scikit-Learn +
- Visualizing Trees, More examples +
- Visualizing the Tree, The Moons +
- Other ways of visualizing the trees +
- Printing out as text +
- Algorithms for Setting up Decision Trees +
- The CART algorithm for Classification +
- The CART algorithm for Regression +
- Computing the Gini index +
- Simple Python Code to read in Data and perform Classification +
- Computing the Gini Factor +
- Entropy and the ID3 algorithm +
- Cancer Data again now with Decision Trees and other Methods +
- Another example, the moons again +
- Playing around with regions +
- Regression trees +
- Final regressor code +
- Example: Computing the Gini index +
- Simple Python Code to read in Data and perform Classification +
- Computing the Gini Factor +
- Entropy and the ID3 algorithm +
- Cancer Data again now with Decision Trees and other Methods +
- Another example, the moons again +
- Playing around with regions +
- Regression trees +
- Final regressor code +
- Pros and cons of trees, pros +
- Disadvantages +
- Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods +
- An Overview of Ensemble Methods +
- Bagging +
- More bagging
- Simple Voting Example, head or tail
- Using the Voting Classifier -
- Please, not the moons again! Voting and Bagging -
- Bagging Examples -
- Making your own Bootstrap: Changing the Level of the Decision Tree -
- Why binary splits? -
- Computing a Tree using the Gini Index -
- The Table -
- Computing the various Gini Indices -
- Computing the various Gini Indices, Hours slept -
- Computing the various Gini Indices, Hours studied -
- A possible code using Scikit-Learn -
- Visualizing Trees, More examples -
- Visualizing the Tree, The Moons -
- Other ways of visualizing the trees -
- Printing out as text -
- Algorithms for Setting up Decision Trees -
- The CART algorithm for Classification -
- The CART algorithm for Regression -
- Computing the Gini index -
- Simple Python Code to read in Data and perform Classification -
- Computing the Gini Factor -
- Entropy and the ID3 algorithm -
- Cancer Data again now with Decision Trees and other Methods -
- Another example, the moons again -
- Playing around with regions -
- Regression trees -
- Final regressor code +
- Please, not the moons again! Voting and Bagging +
- Bagging Examples +
- Making your own Bootstrap: Changing the Level of the Decision Tree
- Why Voting?
- Tossing coins
- Standard imports first @@ -397,20 +400,29 @@ MathJax.Hub.Config({
- Growing a classification tree
- Classification tree, how to split nodes
- Visualizing the Tree, Classification -
- Visualizing the Tree, The Moons -
- Other ways of visualizing the trees -
- Printing out as text -
- Algorithms for Setting up Decision Trees -
- The CART algorithm for Classification -
- The CART algorithm for Regression -
- Computing the Gini index -
- Simple Python Code to read in Data and perform Classification -
- Computing the Gini Factor -
- Entropy and the ID3 algorithm -
- Cancer Data again now with Decision Trees and other Methods -
- Another example, the moons again -
- Playing around with regions -
- Regression trees -
- Final regressor code -
- Pros and cons of trees, pros -
- Disadvantages -
- Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods -
- An Overview of Ensemble Methods -
- Bagging -
- More bagging +
- Visualizing the Tree, The Moons +
- Other ways of visualizing the trees +
- Printing out as text +
- Algorithms for Setting up Decision Trees +
- The CART algorithm for Classification +
- The CART algorithm for Regression +
- Why binary splits? +
- Computing a Tree using the Gini Index +
- The Table +
- Computing the various Gini Indices +
- Computing the various Gini Indices, Hours slept +
- Computing the various Gini Indices, Hours studied +
- A possible code using Scikit-Learn +
- Visualizing Trees, More examples +
- Visualizing the Tree, The Moons +
- Other ways of visualizing the trees +
- Printing out as text +
- Algorithms for Setting up Decision Trees +
- The CART algorithm for Classification +
- The CART algorithm for Regression +
- Computing the Gini index +
- Simple Python Code to read in Data and perform Classification +
- Computing the Gini Factor +
- Entropy and the ID3 algorithm +
- Cancer Data again now with Decision Trees and other Methods +
- Another example, the moons again +
- Playing around with regions +
- Regression trees +
- Final regressor code +
- Example: Computing the Gini index +
- Simple Python Code to read in Data and perform Classification +
- Computing the Gini Factor +
- Entropy and the ID3 algorithm +
- Cancer Data again now with Decision Trees and other Methods +
- Another example, the moons again +
- Playing around with regions +
- Regression trees +
- Final regressor code +
- Pros and cons of trees, pros +
- Disadvantages +
- Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods +
- An Overview of Ensemble Methods +
- Bagging +
- More bagging
- Simple Voting Example, head or tail
- Using the Voting Classifier -
- Please, not the moons again! Voting and Bagging -
- Bagging Examples -
- Making your own Bootstrap: Changing the Level of the Decision Tree -
- Why binary splits? -
- Computing a Tree using the Gini Index -
- The Table -
- Computing the various Gini Indices -
- Computing the various Gini Indices, Hours slept -
- Computing the various Gini Indices, Hours studied -
- A possible code using Scikit-Learn -
- Visualizing Trees, More examples -
- Visualizing the Tree, The Moons -
- Other ways of visualizing the trees -
- Printing out as text -
- Algorithms for Setting up Decision Trees -
- The CART algorithm for Classification -
- The CART algorithm for Regression -
- Computing the Gini index -
- Simple Python Code to read in Data and perform Classification -
- Computing the Gini Factor -
- Entropy and the ID3 algorithm -
- Cancer Data again now with Decision Trees and other Methods -
- Another example, the moons again -
- Playing around with regions -
- Regression trees -
- Final regressor code +
- Please, not the moons again! Voting and Bagging +
- Bagging Examples +
- Making your own Bootstrap: Changing the Level of the Decision Tree
- Why Voting?
- Tossing coins
- Standard imports first @@ -397,31 +400,46 @@ MathJax.Hub.Config({
- Growing a classification tree
- Classification tree, how to split nodes
- Visualizing the Tree, Classification -
- Visualizing the Tree, The Moons -
- Other ways of visualizing the trees -
- Printing out as text -
- Algorithms for Setting up Decision Trees -
- The CART algorithm for Classification -
- The CART algorithm for Regression -
- Computing the Gini index -
- Simple Python Code to read in Data and perform Classification -
- Computing the Gini Factor -
- Entropy and the ID3 algorithm -
- Cancer Data again now with Decision Trees and other Methods -
- Another example, the moons again -
- Playing around with regions -
- Regression trees -
- Final regressor code -
- Pros and cons of trees, pros -
- Disadvantages -
- Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods -
- An Overview of Ensemble Methods -
- Bagging -
- More bagging +
- Visualizing the Tree, The Moons +
- Other ways of visualizing the trees +
- Printing out as text +
- Algorithms for Setting up Decision Trees +
- The CART algorithm for Classification +
- The CART algorithm for Regression +
- Why binary splits? +
- Computing a Tree using the Gini Index +
- The Table +
- Computing the various Gini Indices +
- Computing the various Gini Indices, Hours slept +
- Computing the various Gini Indices, Hours studied +
- A possible code using Scikit-Learn +
- Visualizing Trees, More examples +
- Visualizing the Tree, The Moons +
- Other ways of visualizing the trees +
- Printing out as text +
- Algorithms for Setting up Decision Trees +
- The CART algorithm for Classification +
- The CART algorithm for Regression +
- Computing the Gini index +
- Simple Python Code to read in Data and perform Classification +
- Computing the Gini Factor +
- Entropy and the ID3 algorithm +
- Cancer Data again now with Decision Trees and other Methods +
- Another example, the moons again +
- Playing around with regions +
- Regression trees +
- Final regressor code +
- Example: Computing the Gini index +
- Simple Python Code to read in Data and perform Classification +
- Computing the Gini Factor +
- Entropy and the ID3 algorithm +
- Cancer Data again now with Decision Trees and other Methods +
- Another example, the moons again +
- Playing around with regions +
- Regression trees +
- Final regressor code +
- Pros and cons of trees, pros +
- Disadvantages +
- Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods +
- An Overview of Ensemble Methods +
- Bagging +
- More bagging
- Simple Voting Example, head or tail
- Using the Voting Classifier -
- Please, not the moons again! Voting and Bagging -
- Bagging Examples -
- Making your own Bootstrap: Changing the Level of the Decision Tree -
- Why binary splits? -
- Computing a Tree using the Gini Index -
- The Table -
- Computing the various Gini Indices -
- Computing the various Gini Indices, Hours slept -
- Computing the various Gini Indices, Hours studied -
- A possible code using Scikit-Learn -
- Visualizing Trees, More examples -
- Visualizing the Tree, The Moons -
- Other ways of visualizing the trees -
- Printing out as text -
- Algorithms for Setting up Decision Trees -
- The CART algorithm for Classification -
- The CART algorithm for Regression -
- Computing the Gini index -
- Simple Python Code to read in Data and perform Classification -
- Computing the Gini Factor -
- Entropy and the ID3 algorithm -
- Cancer Data again now with Decision Trees and other Methods -
- Another example, the moons again -
- Playing around with regions -
- Regression trees -
- Final regressor code +
- Please, not the moons again! Voting and Bagging +
- Bagging Examples +
- Making your own Bootstrap: Changing the Level of the Decision Tree
- Why Voting?
- Tossing coins
- Standard imports first @@ -397,7 +400,8 @@ MathJax.Hub.Config({
- Growing a classification tree
- Classification tree, how to split nodes
- Visualizing the Tree, Classification -
- Visualizing the Tree, The Moons -
- Other ways of visualizing the trees -
- Printing out as text -
- Algorithms for Setting up Decision Trees -
- The CART algorithm for Classification -
- The CART algorithm for Regression -
- Computing the Gini index -
- Simple Python Code to read in Data and perform Classification -
- Computing the Gini Factor -
- Entropy and the ID3 algorithm -
- Cancer Data again now with Decision Trees and other Methods -
- Another example, the moons again -
- Playing around with regions -
- Regression trees -
- Final regressor code -
- Pros and cons of trees, pros -
- Disadvantages -
- Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods -
- An Overview of Ensemble Methods -
- Bagging -
- More bagging +
- Visualizing the Tree, The Moons +
- Other ways of visualizing the trees +
- Printing out as text +
- Algorithms for Setting up Decision Trees +
- The CART algorithm for Classification +
- The CART algorithm for Regression +
- Why binary splits? +
- Computing a Tree using the Gini Index +
- The Table +
- Computing the various Gini Indices +
- Computing the various Gini Indices, Hours slept +
- Computing the various Gini Indices, Hours studied +
- A possible code using Scikit-Learn +
- Visualizing Trees, More examples +
- Visualizing the Tree, The Moons +
- Other ways of visualizing the trees +
- Printing out as text +
- Algorithms for Setting up Decision Trees +
- The CART algorithm for Classification +
- The CART algorithm for Regression +
- Computing the Gini index +
- Simple Python Code to read in Data and perform Classification +
- Computing the Gini Factor +
- Entropy and the ID3 algorithm +
- Cancer Data again now with Decision Trees and other Methods +
- Another example, the moons again +
- Playing around with regions +
- Regression trees +
- Final regressor code +
- Example: Computing the Gini index +
- Simple Python Code to read in Data and perform Classification +
- Computing the Gini Factor +
- Entropy and the ID3 algorithm +
- Cancer Data again now with Decision Trees and other Methods +
- Another example, the moons again +
- Playing around with regions +
- Regression trees +
- Final regressor code +
- Pros and cons of trees, pros +
- Disadvantages +
- Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods +
- An Overview of Ensemble Methods +
- Bagging +
- More bagging
- Simple Voting Example, head or tail
- Using the Voting Classifier -
- Please, not the moons again! Voting and Bagging -
- Bagging Examples -
- Making your own Bootstrap: Changing the Level of the Decision Tree -
- Why binary splits? -
- Computing a Tree using the Gini Index -
- The Table -
- Computing the various Gini Indices -
- Computing the various Gini Indices, Hours slept -
- Computing the various Gini Indices, Hours studied -
- A possible code using Scikit-Learn -
- Visualizing Trees, More examples -
- Visualizing the Tree, The Moons -
- Other ways of visualizing the trees -
- Printing out as text -
- Algorithms for Setting up Decision Trees -
- The CART algorithm for Classification -
- The CART algorithm for Regression -
- Computing the Gini index -
- Simple Python Code to read in Data and perform Classification -
- Computing the Gini Factor -
- Entropy and the ID3 algorithm -
- Cancer Data again now with Decision Trees and other Methods -
- Another example, the moons again -
- Playing around with regions -
- Regression trees -
- Final regressor code +
- Please, not the moons again! Voting and Bagging +
- Bagging Examples +
- Making your own Bootstrap: Changing the Level of the Decision Tree
- Why Voting?
- Tossing coins
- Standard imports first @@ -397,7 +400,16 @@ MathJax.Hub.Config({
- Growing a classification tree
- Classification tree, how to split nodes
- Visualizing the Tree, Classification -
- Visualizing the Tree, The Moons -
- Other ways of visualizing the trees -
- Printing out as text -
- Algorithms for Setting up Decision Trees -
- The CART algorithm for Classification -
- The CART algorithm for Regression -
- Computing the Gini index -
- Simple Python Code to read in Data and perform Classification -
- Computing the Gini Factor -
- Entropy and the ID3 algorithm -
- Cancer Data again now with Decision Trees and other Methods -
- Another example, the moons again -
- Playing around with regions -
- Regression trees -
- Final regressor code -
- Pros and cons of trees, pros -
- Disadvantages -
- Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods -
- An Overview of Ensemble Methods -
- Bagging -
- More bagging +
- Visualizing the Tree, The Moons +
- Other ways of visualizing the trees +
- Printing out as text +
- Algorithms for Setting up Decision Trees +
- The CART algorithm for Classification +
- The CART algorithm for Regression +
- Why binary splits? +
- Computing a Tree using the Gini Index +
- The Table +
- Computing the various Gini Indices +
- Computing the various Gini Indices, Hours slept +
- Computing the various Gini Indices, Hours studied +
- A possible code using Scikit-Learn +
- Visualizing Trees, More examples +
- Visualizing the Tree, The Moons +
- Other ways of visualizing the trees +
- Printing out as text +
- Algorithms for Setting up Decision Trees +
- The CART algorithm for Classification +
- The CART algorithm for Regression +
- Computing the Gini index +
- Simple Python Code to read in Data and perform Classification +
- Computing the Gini Factor +
- Entropy and the ID3 algorithm +
- Cancer Data again now with Decision Trees and other Methods +
- Another example, the moons again +
- Playing around with regions +
- Regression trees +
- Final regressor code +
- Example: Computing the Gini index +
- Simple Python Code to read in Data and perform Classification +
- Computing the Gini Factor +
- Entropy and the ID3 algorithm +
- Cancer Data again now with Decision Trees and other Methods +
- Another example, the moons again +
- Playing around with regions +
- Regression trees +
- Final regressor code +
- Pros and cons of trees, pros +
- Disadvantages +
- Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods +
- An Overview of Ensemble Methods +
- Bagging +
- More bagging
- Simple Voting Example, head or tail
- Using the Voting Classifier -
- Please, not the moons again! Voting and Bagging -
- Bagging Examples -
- Making your own Bootstrap: Changing the Level of the Decision Tree -
- Why binary splits? -
- Computing a Tree using the Gini Index -
- The Table -
- Computing the various Gini Indices -
- Computing the various Gini Indices, Hours slept -
- Computing the various Gini Indices, Hours studied -
- A possible code using Scikit-Learn -
- Visualizing Trees, More examples -
- Visualizing the Tree, The Moons -
- Other ways of visualizing the trees -
- Printing out as text -
- Algorithms for Setting up Decision Trees -
- The CART algorithm for Classification -
- The CART algorithm for Regression -
- Computing the Gini index -
- Simple Python Code to read in Data and perform Classification -
- Computing the Gini Factor -
- Entropy and the ID3 algorithm -
- Cancer Data again now with Decision Trees and other Methods -
- Another example, the moons again -
- Playing around with regions -
- Regression trees -
- Final regressor code +
- Please, not the moons again! Voting and Bagging +
- Bagging Examples +
- Making your own Bootstrap: Changing the Level of the Decision Tree
- Why Voting?
- Tossing coins
- Standard imports first @@ -397,125 +400,37 @@ MathJax.Hub.Config({
- Each instance attribute is evaluated using a statistical test to determine how well it alone classifies the training examples. +
- The best attribute is selected and used as the test at the root node of the tree. +
- A descendant of the root node is then created for each possible value of this attribute. +
- Training examples are sorted to the appropriate descendant node. +
- The entire process is then repeated using the training examples associated with each descendant node to select the best attribute to test at that point in the tree. +
- This forms a greedy search for an acceptable decision tree, in which the algorithm never backtracks to reconsider earlier choices. +
- Growing a classification tree
- Classification tree, how to split nodes
- Visualizing the Tree, Classification -
- Visualizing the Tree, The Moons -
- Other ways of visualizing the trees -
- Printing out as text -
- Algorithms for Setting up Decision Trees -
- The CART algorithm for Classification -
- The CART algorithm for Regression -
- Computing the Gini index -
- Simple Python Code to read in Data and perform Classification -
- Computing the Gini Factor -
- Entropy and the ID3 algorithm -
- Cancer Data again now with Decision Trees and other Methods -
- Another example, the moons again -
- Playing around with regions -
- Regression trees -
- Final regressor code -
- Pros and cons of trees, pros -
- Disadvantages -
- Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods -
- An Overview of Ensemble Methods -
- Bagging -
- More bagging +
- Visualizing the Tree, The Moons +
- Other ways of visualizing the trees +
- Printing out as text +
- Algorithms for Setting up Decision Trees +
- The CART algorithm for Classification +
- The CART algorithm for Regression +
- Why binary splits? +
- Computing a Tree using the Gini Index +
- The Table +
- Computing the various Gini Indices +
- Computing the various Gini Indices, Hours slept +
- Computing the various Gini Indices, Hours studied +
- A possible code using Scikit-Learn +
- Visualizing Trees, More examples +
- Visualizing the Tree, The Moons +
- Other ways of visualizing the trees +
- Printing out as text +
- Algorithms for Setting up Decision Trees +
- The CART algorithm for Classification +
- The CART algorithm for Regression +
- Computing the Gini index +
- Simple Python Code to read in Data and perform Classification +
- Computing the Gini Factor +
- Entropy and the ID3 algorithm +
- Cancer Data again now with Decision Trees and other Methods +
- Another example, the moons again +
- Playing around with regions +
- Regression trees +
- Final regressor code +
- Example: Computing the Gini index +
- Simple Python Code to read in Data and perform Classification +
- Computing the Gini Factor +
- Entropy and the ID3 algorithm +
- Cancer Data again now with Decision Trees and other Methods +
- Another example, the moons again +
- Playing around with regions +
- Regression trees +
- Final regressor code +
- Pros and cons of trees, pros +
- Disadvantages +
- Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods +
- An Overview of Ensemble Methods +
- Bagging +
- More bagging
- Simple Voting Example, head or tail
- Using the Voting Classifier -
- Please, not the moons again! Voting and Bagging -
- Bagging Examples -
- Making your own Bootstrap: Changing the Level of the Decision Tree -
- Why binary splits? -
- Computing a Tree using the Gini Index -
- The Table -
- Computing the various Gini Indices -
- Computing the various Gini Indices, Hours slept -
- Computing the various Gini Indices, Hours studied -
- A possible code using Scikit-Learn -
- Visualizing Trees, More examples -
- Visualizing the Tree, The Moons -
- Other ways of visualizing the trees -
- Printing out as text -
- Algorithms for Setting up Decision Trees -
- The CART algorithm for Classification -
- The CART algorithm for Regression -
- Computing the Gini index -
- Simple Python Code to read in Data and perform Classification -
- Computing the Gini Factor -
- Entropy and the ID3 algorithm -
- Cancer Data again now with Decision Trees and other Methods -
- Another example, the moons again -
- Playing around with regions -
- Regression trees -
- Final regressor code +
- Please, not the moons again! Voting and Bagging +
- Bagging Examples +
- Making your own Bootstrap: Changing the Level of the Decision Tree
- Why Voting?
- Tossing coins
- Standard imports first @@ -397,8 +400,7 @@ MathJax.Hub.Config({
- Growing a classification tree
- Classification tree, how to split nodes
- Visualizing the Tree, Classification -
- Visualizing the Tree, The Moons -
- Other ways of visualizing the trees -
- Printing out as text -
- Algorithms for Setting up Decision Trees -
- The CART algorithm for Classification -
- The CART algorithm for Regression -
- Computing the Gini index -
- Simple Python Code to read in Data and perform Classification -
- Computing the Gini Factor -
- Entropy and the ID3 algorithm -
- Cancer Data again now with Decision Trees and other Methods -
- Another example, the moons again -
- Playing around with regions -
- Regression trees -
- Final regressor code -
- Pros and cons of trees, pros -
- Disadvantages -
- Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods -
- An Overview of Ensemble Methods -
- Bagging -
- More bagging +
- Visualizing the Tree, The Moons +
- Other ways of visualizing the trees +
- Printing out as text +
- Algorithms for Setting up Decision Trees +
- The CART algorithm for Classification +
- The CART algorithm for Regression +
- Why binary splits? +
- Computing a Tree using the Gini Index +
- The Table +
- Computing the various Gini Indices +
- Computing the various Gini Indices, Hours slept +
- Computing the various Gini Indices, Hours studied +
- A possible code using Scikit-Learn +
- Visualizing Trees, More examples +
- Visualizing the Tree, The Moons +
- Other ways of visualizing the trees +
- Printing out as text +
- Algorithms for Setting up Decision Trees +
- The CART algorithm for Classification +
- The CART algorithm for Regression +
- Computing the Gini index +
- Simple Python Code to read in Data and perform Classification +
- Computing the Gini Factor +
- Entropy and the ID3 algorithm +
- Cancer Data again now with Decision Trees and other Methods +
- Another example, the moons again +
- Playing around with regions +
- Regression trees +
- Final regressor code +
- Example: Computing the Gini index +
- Simple Python Code to read in Data and perform Classification +
- Computing the Gini Factor +
- Entropy and the ID3 algorithm +
- Cancer Data again now with Decision Trees and other Methods +
- Another example, the moons again +
- Playing around with regions +
- Regression trees +
- Final regressor code +
- Pros and cons of trees, pros +
- Disadvantages +
- Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods +
- An Overview of Ensemble Methods +
- Bagging +
- More bagging
- Simple Voting Example, head or tail
- Using the Voting Classifier -
- Please, not the moons again! Voting and Bagging -
- Bagging Examples -
- Making your own Bootstrap: Changing the Level of the Decision Tree -
- Why binary splits? -
- Computing a Tree using the Gini Index -
- The Table -
- Computing the various Gini Indices -
- Computing the various Gini Indices, Hours slept -
- Computing the various Gini Indices, Hours studied -
- A possible code using Scikit-Learn -
- Visualizing Trees, More examples -
- Visualizing the Tree, The Moons -
- Other ways of visualizing the trees -
- Printing out as text -
- Algorithms for Setting up Decision Trees -
- The CART algorithm for Classification -
- The CART algorithm for Regression -
- Computing the Gini index -
- Simple Python Code to read in Data and perform Classification -
- Computing the Gini Factor -
- Entropy and the ID3 algorithm -
- Cancer Data again now with Decision Trees and other Methods -
- Another example, the moons again -
- Playing around with regions -
- Regression trees -
- Final regressor code +
- Please, not the moons again! Voting and Bagging +
- Bagging Examples +
- Making your own Bootstrap: Changing the Level of the Decision Tree
- Why Voting?
- Tossing coins
- Standard imports first @@ -397,11 +400,7 @@ MathJax.Hub.Config({
- Growing a classification tree
- Classification tree, how to split nodes
- Visualizing the Tree, Classification -
- Visualizing the Tree, The Moons -
- Other ways of visualizing the trees -
- Printing out as text -
- Algorithms for Setting up Decision Trees -
- The CART algorithm for Classification -
- The CART algorithm for Regression -
- Computing the Gini index -
- Simple Python Code to read in Data and perform Classification -
- Computing the Gini Factor -
- Entropy and the ID3 algorithm -
- Cancer Data again now with Decision Trees and other Methods -
- Another example, the moons again -
- Playing around with regions -
- Regression trees -
- Final regressor code -
- Pros and cons of trees, pros -
- Disadvantages -
- Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods -
- An Overview of Ensemble Methods -
- Bagging -
- More bagging +
- Visualizing the Tree, The Moons +
- Other ways of visualizing the trees +
- Printing out as text +
- Algorithms for Setting up Decision Trees +
- The CART algorithm for Classification +
- The CART algorithm for Regression +
- Why binary splits? +
- Computing a Tree using the Gini Index +
- The Table +
- Computing the various Gini Indices +
- Computing the various Gini Indices, Hours slept +
- Computing the various Gini Indices, Hours studied +
- A possible code using Scikit-Learn +
- Visualizing Trees, More examples +
- Visualizing the Tree, The Moons +
- Other ways of visualizing the trees +
- Printing out as text +
- Algorithms for Setting up Decision Trees +
- The CART algorithm for Classification +
- The CART algorithm for Regression +
- Computing the Gini index +
- Simple Python Code to read in Data and perform Classification +
- Computing the Gini Factor +
- Entropy and the ID3 algorithm +
- Cancer Data again now with Decision Trees and other Methods +
- Another example, the moons again +
- Playing around with regions +
- Regression trees +
- Final regressor code +
- Example: Computing the Gini index +
- Simple Python Code to read in Data and perform Classification +
- Computing the Gini Factor +
- Entropy and the ID3 algorithm +
- Cancer Data again now with Decision Trees and other Methods +
- Another example, the moons again +
- Playing around with regions +
- Regression trees +
- Final regressor code +
- Pros and cons of trees, pros +
- Disadvantages +
- Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods +
- An Overview of Ensemble Methods +
- Bagging +
- More bagging
- Simple Voting Example, head or tail
- Using the Voting Classifier -
- Please, not the moons again! Voting and Bagging -
- Bagging Examples -
- Making your own Bootstrap: Changing the Level of the Decision Tree -
- Why binary splits? -
- Computing a Tree using the Gini Index -
- The Table -
- Computing the various Gini Indices -
- Computing the various Gini Indices, Hours slept -
- Computing the various Gini Indices, Hours studied -
- A possible code using Scikit-Learn -
- Visualizing Trees, More examples -
- Visualizing the Tree, The Moons -
- Other ways of visualizing the trees -
- Printing out as text -
- Algorithms for Setting up Decision Trees -
- The CART algorithm for Classification -
- The CART algorithm for Regression -
- Computing the Gini index -
- Simple Python Code to read in Data and perform Classification -
- Computing the Gini Factor -
- Entropy and the ID3 algorithm -
- Cancer Data again now with Decision Trees and other Methods -
- Another example, the moons again -
- Playing around with regions -
- Regression trees -
- Final regressor code +
- Please, not the moons again! Voting and Bagging +
- Bagging Examples +
- Making your own Bootstrap: Changing the Level of the Decision Tree
- Why Voting?
- Tossing coins
- Standard imports first @@ -397,13 +400,49 @@ MathJax.Hub.Config({
- Growing a classification tree
- Classification tree, how to split nodes
- Visualizing the Tree, Classification -
- Visualizing the Tree, The Moons -
- Other ways of visualizing the trees -
- Printing out as text -
- Algorithms for Setting up Decision Trees -
- The CART algorithm for Classification -
- The CART algorithm for Regression -
- Computing the Gini index -
- Simple Python Code to read in Data and perform Classification -
- Computing the Gini Factor -
- Entropy and the ID3 algorithm -
- Cancer Data again now with Decision Trees and other Methods -
- Another example, the moons again -
- Playing around with regions -
- Regression trees -
- Final regressor code -
- Pros and cons of trees, pros -
- Disadvantages -
- Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods -
- An Overview of Ensemble Methods -
- Bagging -
- More bagging +
- Visualizing the Tree, The Moons +
- Other ways of visualizing the trees +
- Printing out as text +
- Algorithms for Setting up Decision Trees +
- The CART algorithm for Classification +
- The CART algorithm for Regression +
- Why binary splits? +
- Computing a Tree using the Gini Index +
- The Table +
- Computing the various Gini Indices +
- Computing the various Gini Indices, Hours slept +
- Computing the various Gini Indices, Hours studied +
- A possible code using Scikit-Learn +
- Visualizing Trees, More examples +
- Visualizing the Tree, The Moons +
- Other ways of visualizing the trees +
- Printing out as text +
- Algorithms for Setting up Decision Trees +
- The CART algorithm for Classification +
- The CART algorithm for Regression +
- Computing the Gini index +
- Simple Python Code to read in Data and perform Classification +
- Computing the Gini Factor +
- Entropy and the ID3 algorithm +
- Cancer Data again now with Decision Trees and other Methods +
- Another example, the moons again +
- Playing around with regions +
- Regression trees +
- Final regressor code +
- Example: Computing the Gini index +
- Simple Python Code to read in Data and perform Classification +
- Computing the Gini Factor +
- Entropy and the ID3 algorithm +
- Cancer Data again now with Decision Trees and other Methods +
- Another example, the moons again +
- Playing around with regions +
- Regression trees +
- Final regressor code +
- Pros and cons of trees, pros +
- Disadvantages +
- Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods +
- An Overview of Ensemble Methods +
- Bagging +
- More bagging
- Simple Voting Example, head or tail
- Using the Voting Classifier -
- Please, not the moons again! Voting and Bagging -
- Bagging Examples -
- Making your own Bootstrap: Changing the Level of the Decision Tree -
- Why binary splits? -
- Computing a Tree using the Gini Index -
- The Table -
- Computing the various Gini Indices -
- Computing the various Gini Indices, Hours slept -
- Computing the various Gini Indices, Hours studied -
- A possible code using Scikit-Learn -
- Visualizing Trees, More examples -
- Visualizing the Tree, The Moons -
- Other ways of visualizing the trees -
- Printing out as text -
- Algorithms for Setting up Decision Trees -
- The CART algorithm for Classification -
- The CART algorithm for Regression -
- Computing the Gini index -
- Simple Python Code to read in Data and perform Classification -
- Computing the Gini Factor -
- Entropy and the ID3 algorithm -
- Cancer Data again now with Decision Trees and other Methods -
- Another example, the moons again -
- Playing around with regions -
- Regression trees -
- Final regressor code +
- Please, not the moons again! Voting and Bagging +
- Bagging Examples +
- Making your own Bootstrap: Changing the Level of the Decision Tree
- Why Voting?
- Tossing coins
- Standard imports first @@ -397,23 +400,59 @@ MathJax.Hub.Config({
- Trend of average grades before present course, classified as either below or above the average grade of the whole class -
- The number of hours studies, classified again as either higher (more than 3 hours per day) or lower . Here we have used a standard for one \( ECTS \) which is scaled to 25-30 hours of work for a semester which lasts 18 weeks, with 15 weeks of lectures and 3 weeks for exams, assuming a total of 30 ECTS per semester. -
- The number of hours slept as high for more than \( 8 \) hours and below for less than 8 hours of sleep, classified again as either high or low -
- The final grade whether it is above or below average -
- Growing a classification tree
- Classification tree, how to split nodes
- Visualizing the Tree, Classification -
- Visualizing the Tree, The Moons -
- Other ways of visualizing the trees -
- Printing out as text -
- Algorithms for Setting up Decision Trees -
- The CART algorithm for Classification -
- The CART algorithm for Regression -
- Computing the Gini index -
- Simple Python Code to read in Data and perform Classification -
- Computing the Gini Factor -
- Entropy and the ID3 algorithm -
- Cancer Data again now with Decision Trees and other Methods -
- Another example, the moons again -
- Playing around with regions -
- Regression trees -
- Final regressor code -
- Pros and cons of trees, pros -
- Disadvantages -
- Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods -
- An Overview of Ensemble Methods -
- Bagging -
- More bagging +
- Visualizing the Tree, The Moons +
- Other ways of visualizing the trees +
- Printing out as text +
- Algorithms for Setting up Decision Trees +
- The CART algorithm for Classification +
- The CART algorithm for Regression +
- Why binary splits? +
- Computing a Tree using the Gini Index +
- The Table +
- Computing the various Gini Indices +
- Computing the various Gini Indices, Hours slept +
- Computing the various Gini Indices, Hours studied +
- A possible code using Scikit-Learn +
- Visualizing Trees, More examples +
- Visualizing the Tree, The Moons +
- Other ways of visualizing the trees +
- Printing out as text +
- Algorithms for Setting up Decision Trees +
- The CART algorithm for Classification +
- The CART algorithm for Regression +
- Computing the Gini index +
- Simple Python Code to read in Data and perform Classification +
- Computing the Gini Factor +
- Entropy and the ID3 algorithm +
- Cancer Data again now with Decision Trees and other Methods +
- Another example, the moons again +
- Playing around with regions +
- Regression trees +
- Final regressor code +
- Example: Computing the Gini index +
- Simple Python Code to read in Data and perform Classification +
- Computing the Gini Factor +
- Entropy and the ID3 algorithm +
- Cancer Data again now with Decision Trees and other Methods +
- Another example, the moons again +
- Playing around with regions +
- Regression trees +
- Final regressor code +
- Pros and cons of trees, pros +
- Disadvantages +
- Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods +
- An Overview of Ensemble Methods +
- Bagging +
- More bagging
- Simple Voting Example, head or tail
- Using the Voting Classifier -
- Please, not the moons again! Voting and Bagging -
- Bagging Examples -
- Making your own Bootstrap: Changing the Level of the Decision Tree -
- Why binary splits? -
- Computing a Tree using the Gini Index -
- The Table -
- Computing the various Gini Indices -
- Computing the various Gini Indices, Hours slept -
- Computing the various Gini Indices, Hours studied -
- A possible code using Scikit-Learn -
- Visualizing Trees, More examples -
- Visualizing the Tree, The Moons -
- Other ways of visualizing the trees -
- Printing out as text -
- Algorithms for Setting up Decision Trees -
- The CART algorithm for Classification -
- The CART algorithm for Regression -
- Computing the Gini index -
- Simple Python Code to read in Data and perform Classification -
- Computing the Gini Factor -
- Entropy and the ID3 algorithm -
- Cancer Data again now with Decision Trees and other Methods -
- Another example, the moons again -
- Playing around with regions -
- Regression trees -
- Final regressor code +
- Please, not the moons again! Voting and Bagging +
- Bagging Examples +
- Making your own Bootstrap: Changing the Level of the Decision Tree
- Why Voting?
- Tossing coins
- Standard imports first @@ -397,29 +400,114 @@ MathJax.Hub.Config({
- Growing a classification tree
- Classification tree, how to split nodes
- Visualizing the Tree, Classification -
- Visualizing the Tree, The Moons -
- Other ways of visualizing the trees -
- Printing out as text -
- Algorithms for Setting up Decision Trees -
- The CART algorithm for Classification -
- The CART algorithm for Regression -
- Computing the Gini index -
- Simple Python Code to read in Data and perform Classification -
- Computing the Gini Factor -
- Entropy and the ID3 algorithm -
- Cancer Data again now with Decision Trees and other Methods -
- Another example, the moons again -
- Playing around with regions -
- Regression trees -
- Final regressor code -
- Pros and cons of trees, pros -
- Disadvantages -
- Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods -
- An Overview of Ensemble Methods -
- Bagging -
- More bagging +
- Visualizing the Tree, The Moons +
- Other ways of visualizing the trees +
- Printing out as text +
- Algorithms for Setting up Decision Trees +
- The CART algorithm for Classification +
- The CART algorithm for Regression +
- Why binary splits? +
- Computing a Tree using the Gini Index +
- The Table +
- Computing the various Gini Indices +
- Computing the various Gini Indices, Hours slept +
- Computing the various Gini Indices, Hours studied +
- A possible code using Scikit-Learn +
- Visualizing Trees, More examples +
- Visualizing the Tree, The Moons +
- Other ways of visualizing the trees +
- Printing out as text +
- Algorithms for Setting up Decision Trees +
- The CART algorithm for Classification +
- The CART algorithm for Regression +
- Computing the Gini index +
- Simple Python Code to read in Data and perform Classification +
- Computing the Gini Factor +
- Entropy and the ID3 algorithm +
- Cancer Data again now with Decision Trees and other Methods +
- Another example, the moons again +
- Playing around with regions +
- Regression trees +
- Final regressor code +
- Example: Computing the Gini index +
- Simple Python Code to read in Data and perform Classification +
- Computing the Gini Factor +
- Entropy and the ID3 algorithm +
- Cancer Data again now with Decision Trees and other Methods +
- Another example, the moons again +
- Playing around with regions +
- Regression trees +
- Final regressor code +
- Pros and cons of trees, pros +
- Disadvantages +
- Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods +
- An Overview of Ensemble Methods +
- Bagging +
- More bagging
- Simple Voting Example, head or tail
- Using the Voting Classifier -
- Please, not the moons again! Voting and Bagging -
- Bagging Examples -
- Making your own Bootstrap: Changing the Level of the Decision Tree -
- Why binary splits? -
- Computing a Tree using the Gini Index -
- The Table -
- Computing the various Gini Indices -
- Computing the various Gini Indices, Hours slept -
- Computing the various Gini Indices, Hours studied -
- A possible code using Scikit-Learn -
- Visualizing Trees, More examples -
- Visualizing the Tree, The Moons -
- Other ways of visualizing the trees -
- Printing out as text -
- Algorithms for Setting up Decision Trees -
- The CART algorithm for Classification -
- The CART algorithm for Regression -
- Computing the Gini index -
- Simple Python Code to read in Data and perform Classification -
- Computing the Gini Factor -
- Entropy and the ID3 algorithm -
- Cancer Data again now with Decision Trees and other Methods -
- Another example, the moons again -
- Playing around with regions -
- Regression trees -
- Final regressor code +
- Please, not the moons again! Voting and Bagging +
- Bagging Examples +
- Making your own Bootstrap: Changing the Level of the Decision Tree
- Why Voting?
- Tossing coins
- Standard imports first @@ -397,19 +400,46 @@ MathJax.Hub.Config({
- Growing a classification tree
- Classification tree, how to split nodes
- Visualizing the Tree, Classification -
- Visualizing the Tree, The Moons -
- Other ways of visualizing the trees -
- Printing out as text -
- Algorithms for Setting up Decision Trees -
- The CART algorithm for Classification -
- The CART algorithm for Regression -
- Computing the Gini index -
- Simple Python Code to read in Data and perform Classification -
- Computing the Gini Factor -
- Entropy and the ID3 algorithm -
- Cancer Data again now with Decision Trees and other Methods -
- Another example, the moons again -
- Playing around with regions -
- Regression trees -
- Final regressor code -
- Pros and cons of trees, pros -
- Disadvantages -
- Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods -
- An Overview of Ensemble Methods -
- Bagging -
- More bagging +
- Visualizing the Tree, The Moons +
- Other ways of visualizing the trees +
- Printing out as text +
- Algorithms for Setting up Decision Trees +
- The CART algorithm for Classification +
- The CART algorithm for Regression +
- Why binary splits? +
- Computing a Tree using the Gini Index +
- The Table +
- Computing the various Gini Indices +
- Computing the various Gini Indices, Hours slept +
- Computing the various Gini Indices, Hours studied +
- A possible code using Scikit-Learn +
- Visualizing Trees, More examples +
- Visualizing the Tree, The Moons +
- Other ways of visualizing the trees +
- Printing out as text +
- Algorithms for Setting up Decision Trees +
- The CART algorithm for Classification +
- The CART algorithm for Regression +
- Computing the Gini index +
- Simple Python Code to read in Data and perform Classification +
- Computing the Gini Factor +
- Entropy and the ID3 algorithm +
- Cancer Data again now with Decision Trees and other Methods +
- Another example, the moons again +
- Playing around with regions +
- Regression trees +
- Final regressor code +
- Example: Computing the Gini index +
- Simple Python Code to read in Data and perform Classification +
- Computing the Gini Factor +
- Entropy and the ID3 algorithm +
- Cancer Data again now with Decision Trees and other Methods +
- Another example, the moons again +
- Playing around with regions +
- Regression trees +
- Final regressor code +
- Pros and cons of trees, pros +
- Disadvantages +
- Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods +
- An Overview of Ensemble Methods +
- Bagging +
- More bagging
- Simple Voting Example, head or tail
- Using the Voting Classifier -
- Please, not the moons again! Voting and Bagging -
- Bagging Examples -
- Making your own Bootstrap: Changing the Level of the Decision Tree -
- Why binary splits? -
- Computing a Tree using the Gini Index -
- The Table -
- Computing the various Gini Indices -
- Computing the various Gini Indices, Hours slept -
- Computing the various Gini Indices, Hours studied -
- A possible code using Scikit-Learn -
- Visualizing Trees, More examples -
- Visualizing the Tree, The Moons -
- Other ways of visualizing the trees -
- Printing out as text -
- Algorithms for Setting up Decision Trees -
- The CART algorithm for Classification -
- The CART algorithm for Regression -
- Computing the Gini index -
- Simple Python Code to read in Data and perform Classification -
- Computing the Gini Factor -
- Entropy and the ID3 algorithm -
- Cancer Data again now with Decision Trees and other Methods -
- Another example, the moons again -
- Playing around with regions -
- Regression trees -
- Final regressor code +
- Please, not the moons again! Voting and Bagging +
- Bagging Examples +
- Making your own Bootstrap: Changing the Level of the Decision Tree
- Why Voting?
- Tossing coins
- Standard imports first @@ -397,13 +400,95 @@ MathJax.Hub.Config({
- Growing a classification tree
- Classification tree, how to split nodes
- Visualizing the Tree, Classification -
- Visualizing the Tree, The Moons -
- Other ways of visualizing the trees -
- Printing out as text -
- Algorithms for Setting up Decision Trees -
- The CART algorithm for Classification -
- The CART algorithm for Regression -
- Computing the Gini index -
- Simple Python Code to read in Data and perform Classification -
- Computing the Gini Factor -
- Entropy and the ID3 algorithm -
- Cancer Data again now with Decision Trees and other Methods -
- Another example, the moons again -
- Playing around with regions -
- Regression trees -
- Final regressor code -
- Pros and cons of trees, pros -
- Disadvantages -
- Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods -
- An Overview of Ensemble Methods -
- Bagging -
- More bagging +
- Visualizing the Tree, The Moons +
- Other ways of visualizing the trees +
- Printing out as text +
- Algorithms for Setting up Decision Trees +
- The CART algorithm for Classification +
- The CART algorithm for Regression +
- Why binary splits? +
- Computing a Tree using the Gini Index +
- The Table +
- Computing the various Gini Indices +
- Computing the various Gini Indices, Hours slept +
- Computing the various Gini Indices, Hours studied +
- A possible code using Scikit-Learn +
- Visualizing Trees, More examples +
- Visualizing the Tree, The Moons +
- Other ways of visualizing the trees +
- Printing out as text +
- Algorithms for Setting up Decision Trees +
- The CART algorithm for Classification +
- The CART algorithm for Regression +
- Computing the Gini index +
- Simple Python Code to read in Data and perform Classification +
- Computing the Gini Factor +
- Entropy and the ID3 algorithm +
- Cancer Data again now with Decision Trees and other Methods +
- Another example, the moons again +
- Playing around with regions +
- Regression trees +
- Final regressor code +
- Example: Computing the Gini index +
- Simple Python Code to read in Data and perform Classification +
- Computing the Gini Factor +
- Entropy and the ID3 algorithm +
- Cancer Data again now with Decision Trees and other Methods +
- Another example, the moons again +
- Playing around with regions +
- Regression trees +
- Final regressor code +
- Pros and cons of trees, pros +
- Disadvantages +
- Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods +
- An Overview of Ensemble Methods +
- Bagging +
- More bagging
- Simple Voting Example, head or tail
- Using the Voting Classifier -
- Please, not the moons again! Voting and Bagging -
- Bagging Examples -
- Making your own Bootstrap: Changing the Level of the Decision Tree -
- Why binary splits? -
- Computing a Tree using the Gini Index -
- The Table -
- Computing the various Gini Indices -
- Computing the various Gini Indices, Hours slept -
- Computing the various Gini Indices, Hours studied -
- A possible code using Scikit-Learn -
- Visualizing Trees, More examples -
- Visualizing the Tree, The Moons -
- Other ways of visualizing the trees -
- Printing out as text -
- Algorithms for Setting up Decision Trees -
- The CART algorithm for Classification -
- The CART algorithm for Regression -
- Computing the Gini index -
- Simple Python Code to read in Data and perform Classification -
- Computing the Gini Factor -
- Entropy and the ID3 algorithm -
- Cancer Data again now with Decision Trees and other Methods -
- Another example, the moons again -
- Playing around with regions -
- Regression trees -
- Final regressor code +
- Please, not the moons again! Voting and Bagging +
- Bagging Examples +
- Making your own Bootstrap: Changing the Level of the Decision Tree
- Why Voting?
- Tossing coins
- Standard imports first @@ -397,18 +400,99 @@ MathJax.Hub.Config({
- Growing a classification tree
- Classification tree, how to split nodes
- Visualizing the Tree, Classification -
- Visualizing the Tree, The Moons -
- Other ways of visualizing the trees -
- Printing out as text -
- Algorithms for Setting up Decision Trees -
- The CART algorithm for Classification -
- The CART algorithm for Regression -
- Computing the Gini index -
- Simple Python Code to read in Data and perform Classification -
- Computing the Gini Factor -
- Entropy and the ID3 algorithm -
- Cancer Data again now with Decision Trees and other Methods -
- Another example, the moons again -
- Playing around with regions -
- Regression trees -
- Final regressor code -
- Pros and cons of trees, pros -
- Disadvantages -
- Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods -
- An Overview of Ensemble Methods -
- Bagging -
- More bagging +
- Visualizing the Tree, The Moons +
- Other ways of visualizing the trees +
- Printing out as text +
- Algorithms for Setting up Decision Trees +
- The CART algorithm for Classification +
- The CART algorithm for Regression +
- Why binary splits? +
- Computing a Tree using the Gini Index +
- The Table +
- Computing the various Gini Indices +
- Computing the various Gini Indices, Hours slept +
- Computing the various Gini Indices, Hours studied +
- A possible code using Scikit-Learn +
- Visualizing Trees, More examples +
- Visualizing the Tree, The Moons +
- Other ways of visualizing the trees +
- Printing out as text +
- Algorithms for Setting up Decision Trees +
- The CART algorithm for Classification +
- The CART algorithm for Regression +
- Computing the Gini index +
- Simple Python Code to read in Data and perform Classification +
- Computing the Gini Factor +
- Entropy and the ID3 algorithm +
- Cancer Data again now with Decision Trees and other Methods +
- Another example, the moons again +
- Playing around with regions +
- Regression trees +
- Final regressor code +
- Example: Computing the Gini index +
- Simple Python Code to read in Data and perform Classification +
- Computing the Gini Factor +
- Entropy and the ID3 algorithm +
- Cancer Data again now with Decision Trees and other Methods +
- Another example, the moons again +
- Playing around with regions +
- Regression trees +
- Final regressor code +
- Pros and cons of trees, pros +
- Disadvantages +
- Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods +
- An Overview of Ensemble Methods +
- Bagging +
- More bagging
- Simple Voting Example, head or tail
- Using the Voting Classifier -
- Please, not the moons again! Voting and Bagging -
- Bagging Examples -
- Making your own Bootstrap: Changing the Level of the Decision Tree -
- Why binary splits? -
- Computing a Tree using the Gini Index -
- The Table -
- Computing the various Gini Indices -
- Computing the various Gini Indices, Hours slept -
- Computing the various Gini Indices, Hours studied -
- A possible code using Scikit-Learn -
- Visualizing Trees, More examples -
- Visualizing the Tree, The Moons -
- Other ways of visualizing the trees -
- Printing out as text -
- Algorithms for Setting up Decision Trees -
- The CART algorithm for Classification -
- The CART algorithm for Regression -
- Computing the Gini index -
- Simple Python Code to read in Data and perform Classification -
- Computing the Gini Factor -
- Entropy and the ID3 algorithm -
- Cancer Data again now with Decision Trees and other Methods -
- Another example, the moons again -
- Playing around with regions -
- Regression trees -
- Final regressor code +
- Please, not the moons again! Voting and Bagging +
- Bagging Examples +
- Making your own Bootstrap: Changing the Level of the Decision Tree
- Why Voting?
- Tossing coins
- Standard imports first @@ -397,97 +400,37 @@ MathJax.Hub.Config({
- Each instance attribute is evaluated using a statistical test to determine how well it alone classifies the training examples. +
- The best attribute is selected and used as the test at the root node of the tree. +
- A descendant of the root node is then created for each possible value of this attribute. +
- Training examples are sorted to the appropriate descendant node. +
- The entire process is then repeated using the training examples associated with each descendant node to select the best attribute to test at that point in the tree. +
- This forms a greedy search for an acceptable decision tree, in which the algorithm never backtracks to reconsider earlier choices. +
- Growing a classification tree
- Classification tree, how to split nodes
- Visualizing the Tree, Classification -
- Visualizing the Tree, The Moons -
- Other ways of visualizing the trees -
- Printing out as text -
- Algorithms for Setting up Decision Trees -
- The CART algorithm for Classification -
- The CART algorithm for Regression -
- Computing the Gini index -
- Simple Python Code to read in Data and perform Classification -
- Computing the Gini Factor -
- Entropy and the ID3 algorithm -
- Cancer Data again now with Decision Trees and other Methods -
- Another example, the moons again -
- Playing around with regions -
- Regression trees -
- Final regressor code -
- Pros and cons of trees, pros -
- Disadvantages -
- Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods -
- An Overview of Ensemble Methods -
- Bagging -
- More bagging +
- Visualizing the Tree, The Moons +
- Other ways of visualizing the trees +
- Printing out as text +
- Algorithms for Setting up Decision Trees +
- The CART algorithm for Classification +
- The CART algorithm for Regression +
- Why binary splits? +
- Computing a Tree using the Gini Index +
- The Table +
- Computing the various Gini Indices +
- Computing the various Gini Indices, Hours slept +
- Computing the various Gini Indices, Hours studied +
- A possible code using Scikit-Learn +
- Visualizing Trees, More examples +
- Visualizing the Tree, The Moons +
- Other ways of visualizing the trees +
- Printing out as text +
- Algorithms for Setting up Decision Trees +
- The CART algorithm for Classification +
- The CART algorithm for Regression +
- Computing the Gini index +
- Simple Python Code to read in Data and perform Classification +
- Computing the Gini Factor +
- Entropy and the ID3 algorithm +
- Cancer Data again now with Decision Trees and other Methods +
- Another example, the moons again +
- Playing around with regions +
- Regression trees +
- Final regressor code +
- Example: Computing the Gini index +
- Simple Python Code to read in Data and perform Classification +
- Computing the Gini Factor +
- Entropy and the ID3 algorithm +
- Cancer Data again now with Decision Trees and other Methods +
- Another example, the moons again +
- Playing around with regions +
- Regression trees +
- Final regressor code +
- Pros and cons of trees, pros +
- Disadvantages +
- Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods +
- An Overview of Ensemble Methods +
- Bagging +
- More bagging
- Simple Voting Example, head or tail
- Using the Voting Classifier -
- Please, not the moons again! Voting and Bagging -
- Bagging Examples -
- Making your own Bootstrap: Changing the Level of the Decision Tree -
- Why binary splits? -
- Computing a Tree using the Gini Index -
- The Table -
- Computing the various Gini Indices -
- Computing the various Gini Indices, Hours slept -
- Computing the various Gini Indices, Hours studied -
- A possible code using Scikit-Learn -
- Visualizing Trees, More examples -
- Visualizing the Tree, The Moons -
- Other ways of visualizing the trees -
- Printing out as text -
- Algorithms for Setting up Decision Trees -
- The CART algorithm for Classification -
- The CART algorithm for Regression -
- Computing the Gini index -
- Simple Python Code to read in Data and perform Classification -
- Computing the Gini Factor -
- Entropy and the ID3 algorithm -
- Cancer Data again now with Decision Trees and other Methods -
- Another example, the moons again -
- Playing around with regions -
- Regression trees -
- Final regressor code +
- Please, not the moons again! Voting and Bagging +
- Bagging Examples +
- Making your own Bootstrap: Changing the Level of the Decision Tree
- Why Voting?
- Tossing coins
- Standard imports first @@ -397,7 +400,7 @@ MathJax.Hub.Config({
- Growing a classification tree
- Classification tree, how to split nodes
- Visualizing the Tree, Classification -
- Visualizing the Tree, The Moons -
- Other ways of visualizing the trees -
- Printing out as text -
- Algorithms for Setting up Decision Trees -
- The CART algorithm for Classification -
- The CART algorithm for Regression -
- Computing the Gini index -
- Simple Python Code to read in Data and perform Classification -
- Computing the Gini Factor -
- Entropy and the ID3 algorithm -
- Cancer Data again now with Decision Trees and other Methods -
- Another example, the moons again -
- Playing around with regions -
- Regression trees -
- Final regressor code -
- Pros and cons of trees, pros -
- Disadvantages -
- Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods -
- An Overview of Ensemble Methods -
- Bagging -
- More bagging +
- Visualizing the Tree, The Moons +
- Other ways of visualizing the trees +
- Printing out as text +
- Algorithms for Setting up Decision Trees +
- The CART algorithm for Classification +
- The CART algorithm for Regression +
- Why binary splits? +
- Computing a Tree using the Gini Index +
- The Table +
- Computing the various Gini Indices +
- Computing the various Gini Indices, Hours slept +
- Computing the various Gini Indices, Hours studied +
- A possible code using Scikit-Learn +
- Visualizing Trees, More examples +
- Visualizing the Tree, The Moons +
- Other ways of visualizing the trees +
- Printing out as text +
- Algorithms for Setting up Decision Trees +
- The CART algorithm for Classification +
- The CART algorithm for Regression +
- Computing the Gini index +
- Simple Python Code to read in Data and perform Classification +
- Computing the Gini Factor +
- Entropy and the ID3 algorithm +
- Cancer Data again now with Decision Trees and other Methods +
- Another example, the moons again +
- Playing around with regions +
- Regression trees +
- Final regressor code +
- Example: Computing the Gini index +
- Simple Python Code to read in Data and perform Classification +
- Computing the Gini Factor +
- Entropy and the ID3 algorithm +
- Cancer Data again now with Decision Trees and other Methods +
- Another example, the moons again +
- Playing around with regions +
- Regression trees +
- Final regressor code +
- Pros and cons of trees, pros +
- Disadvantages +
- Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods +
- An Overview of Ensemble Methods +
- Bagging +
- More bagging
- Simple Voting Example, head or tail
- Using the Voting Classifier -
- Please, not the moons again! Voting and Bagging -
- Bagging Examples -
- Making your own Bootstrap: Changing the Level of the Decision Tree -
- Why binary splits? -
- Computing a Tree using the Gini Index -
- The Table -
- Computing the various Gini Indices -
- Computing the various Gini Indices, Hours slept -
- Computing the various Gini Indices, Hours studied -
- A possible code using Scikit-Learn -
- Visualizing Trees, More examples -
- Visualizing the Tree, The Moons -
- Other ways of visualizing the trees -
- Printing out as text -
- Algorithms for Setting up Decision Trees -
- The CART algorithm for Classification -
- The CART algorithm for Regression -
- Computing the Gini index -
- Simple Python Code to read in Data and perform Classification -
- Computing the Gini Factor -
- Entropy and the ID3 algorithm -
- Cancer Data again now with Decision Trees and other Methods -
- Another example, the moons again -
- Playing around with regions -
- Regression trees -
- Final regressor code +
- Please, not the moons again! Voting and Bagging +
- Bagging Examples +
- Making your own Bootstrap: Changing the Level of the Decision Tree
- Why Voting?
- Tossing coins
- Standard imports first @@ -397,7 +400,7 @@ MathJax.Hub.Config({
- Growing a classification tree
- Classification tree, how to split nodes
- Visualizing the Tree, Classification -
- Visualizing the Tree, The Moons -
- Other ways of visualizing the trees -
- Printing out as text -
- Algorithms for Setting up Decision Trees -
- The CART algorithm for Classification -
- The CART algorithm for Regression -
- Computing the Gini index -
- Simple Python Code to read in Data and perform Classification -
- Computing the Gini Factor -
- Entropy and the ID3 algorithm -
- Cancer Data again now with Decision Trees and other Methods -
- Another example, the moons again -
- Playing around with regions -
- Regression trees -
- Final regressor code -
- Pros and cons of trees, pros -
- Disadvantages -
- Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods -
- An Overview of Ensemble Methods -
- Bagging -
- More bagging +
- Visualizing the Tree, The Moons +
- Other ways of visualizing the trees +
- Printing out as text +
- Algorithms for Setting up Decision Trees +
- The CART algorithm for Classification +
- The CART algorithm for Regression +
- Why binary splits? +
- Computing a Tree using the Gini Index +
- The Table +
- Computing the various Gini Indices +
- Computing the various Gini Indices, Hours slept +
- Computing the various Gini Indices, Hours studied +
- A possible code using Scikit-Learn +
- Visualizing Trees, More examples +
- Visualizing the Tree, The Moons +
- Other ways of visualizing the trees +
- Printing out as text +
- Algorithms for Setting up Decision Trees +
- The CART algorithm for Classification +
- The CART algorithm for Regression +
- Computing the Gini index +
- Simple Python Code to read in Data and perform Classification +
- Computing the Gini Factor +
- Entropy and the ID3 algorithm +
- Cancer Data again now with Decision Trees and other Methods +
- Another example, the moons again +
- Playing around with regions +
- Regression trees +
- Final regressor code +
- Example: Computing the Gini index +
- Simple Python Code to read in Data and perform Classification +
- Computing the Gini Factor +
- Entropy and the ID3 algorithm +
- Cancer Data again now with Decision Trees and other Methods +
- Another example, the moons again +
- Playing around with regions +
- Regression trees +
- Final regressor code +
- Pros and cons of trees, pros +
- Disadvantages +
- Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods +
- An Overview of Ensemble Methods +
- Bagging +
- More bagging
- Simple Voting Example, head or tail
- Using the Voting Classifier -
- Please, not the moons again! Voting and Bagging -
- Bagging Examples -
- Making your own Bootstrap: Changing the Level of the Decision Tree -
- Why binary splits? -
- Computing a Tree using the Gini Index -
- The Table -
- Computing the various Gini Indices -
- Computing the various Gini Indices, Hours slept -
- Computing the various Gini Indices, Hours studied -
- A possible code using Scikit-Learn -
- Visualizing Trees, More examples -
- Visualizing the Tree, The Moons -
- Other ways of visualizing the trees -
- Printing out as text -
- Algorithms for Setting up Decision Trees -
- The CART algorithm for Classification -
- The CART algorithm for Regression -
- Computing the Gini index -
- Simple Python Code to read in Data and perform Classification -
- Computing the Gini Factor -
- Entropy and the ID3 algorithm -
- Cancer Data again now with Decision Trees and other Methods -
- Another example, the moons again -
- Playing around with regions -
- Regression trees -
- Final regressor code +
- Please, not the moons again! Voting and Bagging +
- Bagging Examples +
- Making your own Bootstrap: Changing the Level of the Decision Tree
- Why Voting?
- Tossing coins
- Standard imports first @@ -397,10 +400,7 @@ MathJax.Hub.Config({
- Growing a classification tree
- Classification tree, how to split nodes
- Visualizing the Tree, Classification -
- Visualizing the Tree, The Moons -
- Other ways of visualizing the trees -
- Printing out as text -
- Algorithms for Setting up Decision Trees -
- The CART algorithm for Classification -
- The CART algorithm for Regression -
- Computing the Gini index -
- Simple Python Code to read in Data and perform Classification -
- Computing the Gini Factor -
- Entropy and the ID3 algorithm -
- Cancer Data again now with Decision Trees and other Methods -
- Another example, the moons again -
- Playing around with regions -
- Regression trees -
- Final regressor code -
- Pros and cons of trees, pros -
- Disadvantages -
- Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods -
- An Overview of Ensemble Methods -
- Bagging -
- More bagging +
- Visualizing the Tree, The Moons +
- Other ways of visualizing the trees +
- Printing out as text +
- Algorithms for Setting up Decision Trees +
- The CART algorithm for Classification +
- The CART algorithm for Regression +
- Why binary splits? +
- Computing a Tree using the Gini Index +
- The Table +
- Computing the various Gini Indices +
- Computing the various Gini Indices, Hours slept +
- Computing the various Gini Indices, Hours studied +
- A possible code using Scikit-Learn +
- Visualizing Trees, More examples +
- Visualizing the Tree, The Moons +
- Other ways of visualizing the trees +
- Printing out as text +
- Algorithms for Setting up Decision Trees +
- The CART algorithm for Classification +
- The CART algorithm for Regression +
- Computing the Gini index +
- Simple Python Code to read in Data and perform Classification +
- Computing the Gini Factor +
- Entropy and the ID3 algorithm +
- Cancer Data again now with Decision Trees and other Methods +
- Another example, the moons again +
- Playing around with regions +
- Regression trees +
- Final regressor code +
- Example: Computing the Gini index +
- Simple Python Code to read in Data and perform Classification +
- Computing the Gini Factor +
- Entropy and the ID3 algorithm +
- Cancer Data again now with Decision Trees and other Methods +
- Another example, the moons again +
- Playing around with regions +
- Regression trees +
- Final regressor code +
- Pros and cons of trees, pros +
- Disadvantages +
- Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods +
- An Overview of Ensemble Methods +
- Bagging +
- More bagging
- Simple Voting Example, head or tail
- Using the Voting Classifier -
- Please, not the moons again! Voting and Bagging -
- Bagging Examples -
- Making your own Bootstrap: Changing the Level of the Decision Tree -
- Why binary splits? -
- Computing a Tree using the Gini Index -
- The Table -
- Computing the various Gini Indices -
- Computing the various Gini Indices, Hours slept -
- Computing the various Gini Indices, Hours studied -
- A possible code using Scikit-Learn -
- Visualizing Trees, More examples -
- Visualizing the Tree, The Moons -
- Other ways of visualizing the trees -
- Printing out as text -
- Algorithms for Setting up Decision Trees -
- The CART algorithm for Classification -
- The CART algorithm for Regression -
- Computing the Gini index -
- Simple Python Code to read in Data and perform Classification -
- Computing the Gini Factor -
- Entropy and the ID3 algorithm -
- Cancer Data again now with Decision Trees and other Methods -
- Another example, the moons again -
- Playing around with regions -
- Regression trees -
- Final regressor code +
- Please, not the moons again! Voting and Bagging +
- Bagging Examples +
- Making your own Bootstrap: Changing the Level of the Decision Tree
- Why Voting?
- Tossing coins
- Standard imports first @@ -397,12 +400,7 @@ MathJax.Hub.Config({
- Growing a classification tree
- Classification tree, how to split nodes
- Visualizing the Tree, Classification -
- Visualizing the Tree, The Moons -
- Other ways of visualizing the trees -
- Printing out as text -
- Algorithms for Setting up Decision Trees -
- The CART algorithm for Classification -
- The CART algorithm for Regression -
- Computing the Gini index -
- Simple Python Code to read in Data and perform Classification -
- Computing the Gini Factor -
- Entropy and the ID3 algorithm -
- Cancer Data again now with Decision Trees and other Methods -
- Another example, the moons again -
- Playing around with regions -
- Regression trees -
- Final regressor code -
- Pros and cons of trees, pros -
- Disadvantages -
- Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods -
- An Overview of Ensemble Methods -
- Bagging -
- More bagging +
- Visualizing the Tree, The Moons +
- Other ways of visualizing the trees +
- Printing out as text +
- Algorithms for Setting up Decision Trees +
- The CART algorithm for Classification +
- The CART algorithm for Regression +
- Why binary splits? +
- Computing a Tree using the Gini Index +
- The Table +
- Computing the various Gini Indices +
- Computing the various Gini Indices, Hours slept +
- Computing the various Gini Indices, Hours studied +
- A possible code using Scikit-Learn +
- Visualizing Trees, More examples +
- Visualizing the Tree, The Moons +
- Other ways of visualizing the trees +
- Printing out as text +
- Algorithms for Setting up Decision Trees +
- The CART algorithm for Classification +
- The CART algorithm for Regression +
- Computing the Gini index +
- Simple Python Code to read in Data and perform Classification +
- Computing the Gini Factor +
- Entropy and the ID3 algorithm +
- Cancer Data again now with Decision Trees and other Methods +
- Another example, the moons again +
- Playing around with regions +
- Regression trees +
- Final regressor code +
- Example: Computing the Gini index +
- Simple Python Code to read in Data and perform Classification +
- Computing the Gini Factor +
- Entropy and the ID3 algorithm +
- Cancer Data again now with Decision Trees and other Methods +
- Another example, the moons again +
- Playing around with regions +
- Regression trees +
- Final regressor code +
- Pros and cons of trees, pros +
- Disadvantages +
- Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods +
- An Overview of Ensemble Methods +
- Bagging +
- More bagging
- Simple Voting Example, head or tail
- Using the Voting Classifier -
- Please, not the moons again! Voting and Bagging -
- Bagging Examples -
- Making your own Bootstrap: Changing the Level of the Decision Tree -
- Why binary splits? -
- Computing a Tree using the Gini Index -
- The Table -
- Computing the various Gini Indices -
- Computing the various Gini Indices, Hours slept -
- Computing the various Gini Indices, Hours studied -
- A possible code using Scikit-Learn -
- Visualizing Trees, More examples -
- Visualizing the Tree, The Moons -
- Other ways of visualizing the trees -
- Printing out as text -
- Algorithms for Setting up Decision Trees -
- The CART algorithm for Classification -
- The CART algorithm for Regression -
- Computing the Gini index -
- Simple Python Code to read in Data and perform Classification -
- Computing the Gini Factor -
- Entropy and the ID3 algorithm -
- Cancer Data again now with Decision Trees and other Methods -
- Another example, the moons again -
- Playing around with regions -
- Regression trees -
- Final regressor code +
- Please, not the moons again! Voting and Bagging +
- Bagging Examples +
- Making your own Bootstrap: Changing the Level of the Decision Tree
- Why Voting?
- Tossing coins
- Standard imports first @@ -397,18 +400,114 @@ MathJax.Hub.Config({
- The CART (Classification And Regression Tree) algorithm for both classification and regression -
- The ID3 algorithm based on the computation of the information gain for classification -
- Growing a classification tree
- Classification tree, how to split nodes
- Visualizing the Tree, Classification -
- Visualizing the Tree, The Moons -
- Other ways of visualizing the trees -
- Printing out as text -
- Algorithms for Setting up Decision Trees -
- The CART algorithm for Classification -
- The CART algorithm for Regression -
- Computing the Gini index -
- Simple Python Code to read in Data and perform Classification -
- Computing the Gini Factor -
- Entropy and the ID3 algorithm -
- Cancer Data again now with Decision Trees and other Methods -
- Another example, the moons again -
- Playing around with regions -
- Regression trees -
- Final regressor code -
- Pros and cons of trees, pros -
- Disadvantages -
- Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods -
- An Overview of Ensemble Methods -
- Bagging -
- More bagging +
- Visualizing the Tree, The Moons +
- Other ways of visualizing the trees +
- Printing out as text +
- Algorithms for Setting up Decision Trees +
- The CART algorithm for Classification +
- The CART algorithm for Regression +
- Why binary splits? +
- Computing a Tree using the Gini Index +
- The Table +
- Computing the various Gini Indices +
- Computing the various Gini Indices, Hours slept +
- Computing the various Gini Indices, Hours studied +
- A possible code using Scikit-Learn +
- Visualizing Trees, More examples +
- Visualizing the Tree, The Moons +
- Other ways of visualizing the trees +
- Printing out as text +
- Algorithms for Setting up Decision Trees +
- The CART algorithm for Classification +
- The CART algorithm for Regression +
- Computing the Gini index +
- Simple Python Code to read in Data and perform Classification +
- Computing the Gini Factor +
- Entropy and the ID3 algorithm +
- Cancer Data again now with Decision Trees and other Methods +
- Another example, the moons again +
- Playing around with regions +
- Regression trees +
- Final regressor code +
- Example: Computing the Gini index +
- Simple Python Code to read in Data and perform Classification +
- Computing the Gini Factor +
- Entropy and the ID3 algorithm +
- Cancer Data again now with Decision Trees and other Methods +
- Another example, the moons again +
- Playing around with regions +
- Regression trees +
- Final regressor code +
- Pros and cons of trees, pros +
- Disadvantages +
- Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods +
- An Overview of Ensemble Methods +
- Bagging +
- More bagging
- Simple Voting Example, head or tail
- Using the Voting Classifier -
- Please, not the moons again! Voting and Bagging -
- Bagging Examples -
- Making your own Bootstrap: Changing the Level of the Decision Tree -
- Why binary splits? -
- Computing a Tree using the Gini Index -
- The Table -
- Computing the various Gini Indices -
- Computing the various Gini Indices, Hours slept -
- Computing the various Gini Indices, Hours studied -
- A possible code using Scikit-Learn -
- Visualizing Trees, More examples -
- Visualizing the Tree, The Moons -
- Other ways of visualizing the trees -
- Printing out as text -
- Algorithms for Setting up Decision Trees -
- The CART algorithm for Classification -
- The CART algorithm for Regression -
- Computing the Gini index -
- Simple Python Code to read in Data and perform Classification -
- Computing the Gini Factor -
- Entropy and the ID3 algorithm -
- Cancer Data again now with Decision Trees and other Methods -
- Another example, the moons again -
- Playing around with regions -
- Regression trees -
- Final regressor code +
- Please, not the moons again! Voting and Bagging +
- Bagging Examples +
- Making your own Bootstrap: Changing the Level of the Decision Tree
- Why Voting?
- Tossing coins
- Standard imports first @@ -397,31 +400,17 @@ MathJax.Hub.Config({
- White box, easy to interpret model. Some people believe that decision trees more closely mirror human decision-making than do the regression and classification approaches discussed earlier (think of support vector machines) +
- Trees are very easy to explain to people. In fact, they are even easier to explain than linear regression! +
- No feature normalization needed +
- Tree models can handle both continuous and categorical data (Classification and Regression Trees) +
- Can model nonlinear relationships +
- Can model interactions between the different descriptive features +
- Trees can be displayed graphically, and are easily interpreted even by a non-expert (especially if they are small) +
- Growing a classification tree
- Classification tree, how to split nodes
- Visualizing the Tree, Classification -
- Visualizing the Tree, The Moons -
- Other ways of visualizing the trees -
- Printing out as text -
- Algorithms for Setting up Decision Trees -
- The CART algorithm for Classification -
- The CART algorithm for Regression -
- Computing the Gini index -
- Simple Python Code to read in Data and perform Classification -
- Computing the Gini Factor -
- Entropy and the ID3 algorithm -
- Cancer Data again now with Decision Trees and other Methods -
- Another example, the moons again -
- Playing around with regions -
- Regression trees -
- Final regressor code -
- Pros and cons of trees, pros -
- Disadvantages -
- Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods -
- An Overview of Ensemble Methods -
- Bagging -
- More bagging +
- Visualizing the Tree, The Moons +
- Other ways of visualizing the trees +
- Printing out as text +
- Algorithms for Setting up Decision Trees +
- The CART algorithm for Classification +
- The CART algorithm for Regression +
- Why binary splits? +
- Computing a Tree using the Gini Index +
- The Table +
- Computing the various Gini Indices +
- Computing the various Gini Indices, Hours slept +
- Computing the various Gini Indices, Hours studied +
- A possible code using Scikit-Learn +
- Visualizing Trees, More examples +
- Visualizing the Tree, The Moons +
- Other ways of visualizing the trees +
- Printing out as text +
- Algorithms for Setting up Decision Trees +
- The CART algorithm for Classification +
- The CART algorithm for Regression +
- Computing the Gini index +
- Simple Python Code to read in Data and perform Classification +
- Computing the Gini Factor +
- Entropy and the ID3 algorithm +
- Cancer Data again now with Decision Trees and other Methods +
- Another example, the moons again +
- Playing around with regions +
- Regression trees +
- Final regressor code +
- Example: Computing the Gini index +
- Simple Python Code to read in Data and perform Classification +
- Computing the Gini Factor +
- Entropy and the ID3 algorithm +
- Cancer Data again now with Decision Trees and other Methods +
- Another example, the moons again +
- Playing around with regions +
- Regression trees +
- Final regressor code +
- Pros and cons of trees, pros +
- Disadvantages +
- Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods +
- An Overview of Ensemble Methods +
- Bagging +
- More bagging
- Simple Voting Example, head or tail
- Using the Voting Classifier -
- Please, not the moons again! Voting and Bagging -
- Bagging Examples -
- Making your own Bootstrap: Changing the Level of the Decision Tree -
- Why binary splits? -
- Computing a Tree using the Gini Index -
- The Table -
- Computing the various Gini Indices -
- Computing the various Gini Indices, Hours slept -
- Computing the various Gini Indices, Hours studied -
- A possible code using Scikit-Learn -
- Visualizing Trees, More examples -
- Visualizing the Tree, The Moons -
- Other ways of visualizing the trees -
- Printing out as text -
- Algorithms for Setting up Decision Trees -
- The CART algorithm for Classification -
- The CART algorithm for Regression -
- Computing the Gini index -
- Simple Python Code to read in Data and perform Classification -
- Computing the Gini Factor -
- Entropy and the ID3 algorithm -
- Cancer Data again now with Decision Trees and other Methods -
- Another example, the moons again -
- Playing around with regions -
- Regression trees -
- Final regressor code +
- Please, not the moons again! Voting and Bagging +
- Bagging Examples +
- Making your own Bootstrap: Changing the Level of the Decision Tree
- Why Voting?
- Tossing coins
- Standard imports first @@ -397,29 +400,20 @@ MathJax.Hub.Config({
- Unfortunately, trees generally do not have the same level of predictive accuracy as some of the other regression and classification approaches +
- If continuous features are used the tree may become quite large and hence less interpretable +
- Decision trees are prone to overfit the training data and hence do not well generalize the data if no stopping criteria or improvements like pruning, boosting or bagging are implemented +
- Small changes in the data may lead to a completely different tree. This issue can be addressed by using ensemble methods like bagging, boosting or random forests +
- Unbalanced datasets where some target feature values occur much more frequently than others may lead to biased trees since the frequently occurring feature values are preferred over the less frequently occurring ones. +
- If the number of features is relatively large (high dimensional) and the number of instances is relatively low, the tree might overfit the data +
- Features with many levels may be preferred over features with less levels since for them it is more easy to split the dataset such that the sub datasets only contain pure target feature values. This issue can be addressed by preferring for instance the information gain ratio as splitting criteria over information gain +
- Growing a classification tree
- Classification tree, how to split nodes
- Visualizing the Tree, Classification -
- Visualizing the Tree, The Moons -
- Other ways of visualizing the trees -
- Printing out as text -
- Algorithms for Setting up Decision Trees -
- The CART algorithm for Classification -
- The CART algorithm for Regression -
- Computing the Gini index -
- Simple Python Code to read in Data and perform Classification -
- Computing the Gini Factor -
- Entropy and the ID3 algorithm -
- Cancer Data again now with Decision Trees and other Methods -
- Another example, the moons again -
- Playing around with regions -
- Regression trees -
- Final regressor code -
- Pros and cons of trees, pros -
- Disadvantages -
- Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods -
- An Overview of Ensemble Methods -
- Bagging -
- More bagging +
- Visualizing the Tree, The Moons +
- Other ways of visualizing the trees +
- Printing out as text +
- Algorithms for Setting up Decision Trees +
- The CART algorithm for Classification +
- The CART algorithm for Regression +
- Why binary splits? +
- Computing a Tree using the Gini Index +
- The Table +
- Computing the various Gini Indices +
- Computing the various Gini Indices, Hours slept +
- Computing the various Gini Indices, Hours studied +
- A possible code using Scikit-Learn +
- Visualizing Trees, More examples +
- Visualizing the Tree, The Moons +
- Other ways of visualizing the trees +
- Printing out as text +
- Algorithms for Setting up Decision Trees +
- The CART algorithm for Classification +
- The CART algorithm for Regression +
- Computing the Gini index +
- Simple Python Code to read in Data and perform Classification +
- Computing the Gini Factor +
- Entropy and the ID3 algorithm +
- Cancer Data again now with Decision Trees and other Methods +
- Another example, the moons again +
- Playing around with regions +
- Regression trees +
- Final regressor code +
- Example: Computing the Gini index +
- Simple Python Code to read in Data and perform Classification +
- Computing the Gini Factor +
- Entropy and the ID3 algorithm +
- Cancer Data again now with Decision Trees and other Methods +
- Another example, the moons again +
- Playing around with regions +
- Regression trees +
- Final regressor code +
- Pros and cons of trees, pros +
- Disadvantages +
- Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods +
- An Overview of Ensemble Methods +
- Bagging +
- More bagging
- Simple Voting Example, head or tail
- Using the Voting Classifier -
- Please, not the moons again! Voting and Bagging -
- Bagging Examples -
- Making your own Bootstrap: Changing the Level of the Decision Tree -
- Why binary splits? -
- Computing a Tree using the Gini Index -
- The Table -
- Computing the various Gini Indices -
- Computing the various Gini Indices, Hours slept -
- Computing the various Gini Indices, Hours studied -
- A possible code using Scikit-Learn -
- Visualizing Trees, More examples -
- Visualizing the Tree, The Moons -
- Other ways of visualizing the trees -
- Printing out as text -
- Algorithms for Setting up Decision Trees -
- The CART algorithm for Classification -
- The CART algorithm for Regression -
- Computing the Gini index -
- Simple Python Code to read in Data and perform Classification -
- Computing the Gini Factor -
- Entropy and the ID3 algorithm -
- Cancer Data again now with Decision Trees and other Methods -
- Another example, the moons again -
- Playing around with regions -
- Regression trees -
- Final regressor code +
- Please, not the moons again! Voting and Bagging +
- Bagging Examples +
- Making your own Bootstrap: Changing the Level of the Decision Tree
- Why Voting?
- Tossing coins
- Standard imports first @@ -397,46 +400,28 @@ MathJax.Hub.Config({
- Voting classifiers +
- Bagging and Pasting +
- Random forests +
- Boosting methods, from adaptive to Extreme Gradient Boosting (XGBoost) +
- Growing a classification tree
- Classification tree, how to split nodes
- Visualizing the Tree, Classification -
- Visualizing the Tree, The Moons -
- Other ways of visualizing the trees -
- Printing out as text -
- Algorithms for Setting up Decision Trees -
- The CART algorithm for Classification -
- The CART algorithm for Regression -
- Computing the Gini index -
- Simple Python Code to read in Data and perform Classification -
- Computing the Gini Factor -
- Entropy and the ID3 algorithm -
- Cancer Data again now with Decision Trees and other Methods -
- Another example, the moons again -
- Playing around with regions -
- Regression trees -
- Final regressor code -
- Pros and cons of trees, pros -
- Disadvantages -
- Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods -
- An Overview of Ensemble Methods -
- Bagging -
- More bagging +
- Visualizing the Tree, The Moons +
- Other ways of visualizing the trees +
- Printing out as text +
- Algorithms for Setting up Decision Trees +
- The CART algorithm for Classification +
- The CART algorithm for Regression +
- Why binary splits? +
- Computing a Tree using the Gini Index +
- The Table +
- Computing the various Gini Indices +
- Computing the various Gini Indices, Hours slept +
- Computing the various Gini Indices, Hours studied +
- A possible code using Scikit-Learn +
- Visualizing Trees, More examples +
- Visualizing the Tree, The Moons +
- Other ways of visualizing the trees +
- Printing out as text +
- Algorithms for Setting up Decision Trees +
- The CART algorithm for Classification +
- The CART algorithm for Regression +
- Computing the Gini index +
- Simple Python Code to read in Data and perform Classification +
- Computing the Gini Factor +
- Entropy and the ID3 algorithm +
- Cancer Data again now with Decision Trees and other Methods +
- Another example, the moons again +
- Playing around with regions +
- Regression trees +
- Final regressor code +
- Example: Computing the Gini index +
- Simple Python Code to read in Data and perform Classification +
- Computing the Gini Factor +
- Entropy and the ID3 algorithm +
- Cancer Data again now with Decision Trees and other Methods +
- Another example, the moons again +
- Playing around with regions +
- Regression trees +
- Final regressor code +
- Pros and cons of trees, pros +
- Disadvantages +
- Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods +
- An Overview of Ensemble Methods +
- Bagging +
- More bagging
- Simple Voting Example, head or tail
- Using the Voting Classifier -
- Please, not the moons again! Voting and Bagging -
- Bagging Examples -
- Making your own Bootstrap: Changing the Level of the Decision Tree -
- Why binary splits? -
- Computing a Tree using the Gini Index -
- The Table -
- Computing the various Gini Indices -
- Computing the various Gini Indices, Hours slept -
- Computing the various Gini Indices, Hours studied -
- A possible code using Scikit-Learn -
- Visualizing Trees, More examples -
- Visualizing the Tree, The Moons -
- Other ways of visualizing the trees -
- Printing out as text -
- Algorithms for Setting up Decision Trees -
- The CART algorithm for Classification -
- The CART algorithm for Regression -
- Computing the Gini index -
- Simple Python Code to read in Data and perform Classification -
- Computing the Gini Factor -
- Entropy and the ID3 algorithm -
- Cancer Data again now with Decision Trees and other Methods -
- Another example, the moons again -
- Playing around with regions -
- Regression trees -
- Final regressor code +
- Please, not the moons again! Voting and Bagging +
- Bagging Examples +
- Making your own Bootstrap: Changing the Level of the Decision Tree
- Why Voting?
- Tossing coins
- Standard imports first @@ -397,97 +400,13 @@ MathJax.Hub.Config({
- Growing a classification tree
- Classification tree, how to split nodes
- Visualizing the Tree, Classification -
- Visualizing the Tree, The Moons -
- Other ways of visualizing the trees -
- Printing out as text -
- Algorithms for Setting up Decision Trees -
- The CART algorithm for Classification -
- The CART algorithm for Regression -
- Computing the Gini index -
- Simple Python Code to read in Data and perform Classification -
- Computing the Gini Factor -
- Entropy and the ID3 algorithm -
- Cancer Data again now with Decision Trees and other Methods -
- Another example, the moons again -
- Playing around with regions -
- Regression trees -
- Final regressor code -
- Pros and cons of trees, pros -
- Disadvantages -
- Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods -
- An Overview of Ensemble Methods -
- Bagging -
- More bagging +
- Visualizing the Tree, The Moons +
- Other ways of visualizing the trees +
- Printing out as text +
- Algorithms for Setting up Decision Trees +
- The CART algorithm for Classification +
- The CART algorithm for Regression +
- Why binary splits? +
- Computing a Tree using the Gini Index +
- The Table +
- Computing the various Gini Indices +
- Computing the various Gini Indices, Hours slept +
- Computing the various Gini Indices, Hours studied +
- A possible code using Scikit-Learn +
- Visualizing Trees, More examples +
- Visualizing the Tree, The Moons +
- Other ways of visualizing the trees +
- Printing out as text +
- Algorithms for Setting up Decision Trees +
- The CART algorithm for Classification +
- The CART algorithm for Regression +
- Computing the Gini index +
- Simple Python Code to read in Data and perform Classification +
- Computing the Gini Factor +
- Entropy and the ID3 algorithm +
- Cancer Data again now with Decision Trees and other Methods +
- Another example, the moons again +
- Playing around with regions +
- Regression trees +
- Final regressor code +
- Example: Computing the Gini index +
- Simple Python Code to read in Data and perform Classification +
- Computing the Gini Factor +
- Entropy and the ID3 algorithm +
- Cancer Data again now with Decision Trees and other Methods +
- Another example, the moons again +
- Playing around with regions +
- Regression trees +
- Final regressor code +
- Pros and cons of trees, pros +
- Disadvantages +
- Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods +
- An Overview of Ensemble Methods +
- Bagging +
- More bagging
- Simple Voting Example, head or tail
- Using the Voting Classifier -
- Please, not the moons again! Voting and Bagging -
- Bagging Examples -
- Making your own Bootstrap: Changing the Level of the Decision Tree -
- Why binary splits? -
- Computing a Tree using the Gini Index -
- The Table -
- Computing the various Gini Indices -
- Computing the various Gini Indices, Hours slept -
- Computing the various Gini Indices, Hours studied -
- A possible code using Scikit-Learn -
- Visualizing Trees, More examples -
- Visualizing the Tree, The Moons -
- Other ways of visualizing the trees -
- Printing out as text -
- Algorithms for Setting up Decision Trees -
- The CART algorithm for Classification -
- The CART algorithm for Regression -
- Computing the Gini index -
- Simple Python Code to read in Data and perform Classification -
- Computing the Gini Factor -
- Entropy and the ID3 algorithm -
- Cancer Data again now with Decision Trees and other Methods -
- Another example, the moons again -
- Playing around with regions -
- Regression trees -
- Final regressor code +
- Please, not the moons again! Voting and Bagging +
- Bagging Examples +
- Making your own Bootstrap: Changing the Level of the Decision Tree
- Why Voting?
- Tossing coins
- Standard imports first @@ -415,7 +418,7 @@ MathJax.Hub.Config({
- Trend of average grades before present course, classified as either below or above the average grade of the whole class + +
- The number of hours studies, classified again as either higher (more than 3 hours per day) or lower . Here we have used a standard for one \( ECTS \) which is scaled to 25-30 hours of work for a semester which lasts 18 weeks, with 15 weeks of lectures and 3 weeks for exams, assuming a total of 30 ECTS per semester. +
- The number of hours slept as high for more than \( 8 \) hours and below for less than 8 hours of sleep, classified again as either high or low +
- The final grade whether it is above or below average +
- The CART (Classification And Regression Tree) algorithm for both classification and regression +
- The ID3 algorithm based on the computation of the information gain for classification +
- Each instance attribute is evaluated using a statistical test to determine how well it alone classifies the training examples. +
- The best attribute is selected and used as the test at the root node of the tree. +
- A descendant of the root node is then created for each possible value of this attribute. +
- Training examples are sorted to the appropriate descendant node. +
- The entire process is then repeated using the training examples associated with each descendant node to select the best attribute to test at that point in the tree. +
- This forms a greedy search for an acceptable decision tree, in which the algorithm never backtracks to reconsider earlier choices. +
- Trend of average grades before present course, classified as either below or above the average grade of the whole class - -
- The number of hours studies, classified again as either higher (more than 3 hours per day) or lower . Here we have used a standard for one \( ECTS \) which is scaled to 25-30 hours of work for a semester which lasts 18 weeks, with 15 weeks of lectures and 3 weeks for exams, assuming a total of 30 ECTS per semester. -
- The number of hours slept as high for more than \( 8 \) hours and below for less than 8 hours of sleep, classified again as either high or low -
- The final grade whether it is above or below average -
- The CART (Classification And Regression Tree) algorithm for both classification and regression -
- The ID3 algorithm based on the computation of the information gain for classification -
- Each instance attribute is evaluated using a statistical test to determine how well it alone classifies the training examples. -
- The best attribute is selected and used as the test at the root node of the tree. -
- A descendant of the root node is then created for each possible value of this attribute. -
- Training examples are sorted to the appropriate descendant node. -
- The entire process is then repeated using the training examples associated with each descendant node to select the best attribute to test at that point in the tree. -
- This forms a greedy search for an acceptable decision tree, in which the algorithm never backtracks to reconsider earlier choices. -
- The CART (Classification And Regression Tree) algorithm for both classification and regression +
- The ID3 algorithm based on the computation of the information gain for classification +
- Trend of average grades before present course, classified as either below or above the average grade of the whole class +
- The number of hours studies, classified again as either higher (more than 3 hours per day) or lower . Here we have used a standard for one \( ECTS \) which is scaled to 25-30 hours of work for a semester which lasts 18 weeks, with 15 weeks of lectures and 3 weeks for exams, assuming a total of 30 ECTS per semester. +
- The number of hours slept as high for more than \( 8 \) hours and below for less than 8 hours of sleep, classified again as either high or low +
- The final grade whether it is above or below average +
- Each instance attribute is evaluated using a statistical test to determine how well it alone classifies the training examples. +
- The best attribute is selected and used as the test at the root node of the tree. +
- A descendant of the root node is then created for each possible value of this attribute. +
- Training examples are sorted to the appropriate descendant node. +
- The entire process is then repeated using the training examples associated with each descendant node to select the best attribute to test at that point in the tree. +
- This forms a greedy search for an acceptable decision tree, in which the algorithm never backtracks to reconsider earlier choices. +
- Trend of average grades before present course, classified as either below or above the average grade of the whole class -
- The number of hours studies, classified again as either higher (more than 3 hours per day) or lower . Here we have used a standard for one \( ECTS \) which is scaled to 25-30 hours of work for a semester which lasts 18 weeks, with 15 weeks of lectures and 3 weeks for exams, assuming a total of 30 ECTS per semester. -
- The number of hours slept as high for more than \( 8 \) hours and below for less than 8 hours of sleep, classified again as either high or low -
- The final grade whether it is above or below average -
- The CART (Classification And Regression Tree) algorithm for both classification and regression -
- The ID3 algorithm based on the computation of the information gain for classification -
- Each instance attribute is evaluated using a statistical test to determine how well it alone classifies the training examples. -
- The best attribute is selected and used as the test at the root node of the tree. -
- A descendant of the root node is then created for each possible value of this attribute. -
- Training examples are sorted to the appropriate descendant node. -
- The entire process is then repeated using the training examples associated with each descendant node to select the best attribute to test at that point in the tree. -
- This forms a greedy search for an acceptable decision tree, in which the algorithm never backtracks to reconsider earlier choices. -
- The CART (Classification And Regression Tree) algorithm for both classification and regression +
- The ID3 algorithm based on the computation of the information gain for classification +
- Trend of average grades before present course, classified as either below or above the average grade of the whole class +
- The number of hours studies, classified again as either higher (more than 3 hours per day) or lower . Here we have used a standard for one \( ECTS \) which is scaled to 25-30 hours of work for a semester which lasts 18 weeks, with 15 weeks of lectures and 3 weeks for exams, assuming a total of 30 ECTS per semester. +
- The number of hours slept as high for more than \( 8 \) hours and below for less than 8 hours of sleep, classified again as either high or low +
- The final grade whether it is above or below average +
- Each instance attribute is evaluated using a statistical test to determine how well it alone classifies the training examples. +
- The best attribute is selected and used as the test at the root node of the tree. +
- A descendant of the root node is then created for each possible value of this attribute. +
- Training examples are sorted to the appropriate descendant node. +
- The entire process is then repeated using the training examples associated with each descendant node to select the best attribute to test at that point in the tree. +
- This forms a greedy search for an acceptable decision tree, in which the algorithm never backtracks to reconsider earlier choices. +
- Trend of average grades before present course, classified as either below or above the average grade of the whole class -
- The number of hours studies, classified again as either higher (more than 3 hours per day) or lower . Here we have used a standard for one \( ECTS \) which is scaled to 25-30 hours of work for a semester which lasts 18 weeks, with 15 weeks of lectures and 3 weeks for exams, assuming a total of 30 ECTS per semester. -
- The number of hours slept as high for more than \( 8 \) hours and below for less than 8 hours of sleep, classified again as either high or low -
- The final grade whether it is above or below average -
- The CART (Classification And Regression Tree) algorithm for both classification and regression -
- The ID3 algorithm based on the computation of the information gain for classification -
- Each instance attribute is evaluated using a statistical test to determine how well it alone classifies the training examples. -
- The best attribute is selected and used as the test at the root node of the tree. -
- A descendant of the root node is then created for each possible value of this attribute. -
- Training examples are sorted to the appropriate descendant node. -
- The entire process is then repeated using the training examples associated with each descendant node to select the best attribute to test at that point in the tree. -
- This forms a greedy search for an acceptable decision tree, in which the algorithm never backtracks to reconsider earlier choices. -
-
Computing the Gini Factor
- -The above functions (gini, entropy and misclassification error) are -important components of the so-called CART algorithm. We will discuss -this algorithm below after we have discussed the information gain -algorithm ID3. -
- -In the example here we have converted all our attributes into numerical values \( 0,1,2 \) etc.
- - - -# Split a dataset based on an attribute and an attribute value
-def test_split(index, value, dataset):
- left, right = list(), list()
- for row in dataset:
- if row[index] < value:
- left.append(row)
- else:
- right.append(row)
- return left, right
-
-# Calculate the Gini index for a split dataset
-def gini_index(groups, classes):
- # count all samples at split point
- n_instances = float(sum([len(group) for group in groups]))
- # sum weighted Gini index for each group
- gini = 0.0
- for group in groups:
- size = float(len(group))
- # avoid divide by zero
- if size == 0:
- continue
- score = 0.0
- # score the group based on the score for each class
- for class_val in classes:
- p = [row[-1] for row in group].count(class_val) / size
- score += p * p
- # weight the group score by its relative size
- gini += (1.0 - score) * (size / n_instances)
- return gini
-
-# Select the best split point for a dataset
-def get_split(dataset):
- class_values = list(set(row[-1] for row in dataset))
- b_index, b_value, b_score, b_groups = 999, 999, 999, None
- for index in range(len(dataset[0])-1):
- for row in dataset:
- groups = test_split(index, row[index], dataset)
- gini = gini_index(groups, class_values)
- print('X%d < %.3f Gini=%.3f' % ((index+1), row[index], gini))
- if gini < b_score:
- b_index, b_value, b_score, b_groups = index, row[index], gini, groups
- return {'index':b_index, 'value':b_value, 'groups':b_groups}
-
-dataset = [[0,0,0,0,0],
- [0,0,0,1,1],
- [1,0,0,0,1],
- [2,1,0,0,1],
- [2,2,1,0,1],
- [2,2,1,1,0],
- [1,2,1,1,1],
- [0,1,0,0,0],
- [0,2,1,0,1],
- [2,1,1,0,1],
- [0,1,1,1,1],
- [1,1,0,1,1],
- [1,0,1,0,1],
- [2,1,0,1,0]]
-
-split = get_split(dataset)
-print('Split: [X%d < %.3f]' % ((split['index']+1), split['value']))
-
-The Table
+| Grade Trend | Hours slept | Hours Studied | Grade |
| Above | Low | High | Above |
| Below | High | Low | Below |
| Above | Low | High | Above |
| Above | High | High | Above |
| Below | Low | High | Below |
| Above | Low | Low | Below |
| Below | High | High | Below |
| Below | Low | High | Below |
| Above | Low | Low | Below |
| Above | High | High | Above |
diff --git a/doc/pub/week44/html/._week44-bs031.html b/doc/pub/week44/html/._week44-bs031.html index e11aa27ec..bda69607f 100644 --- a/doc/pub/week44/html/._week44-bs031.html +++ b/doc/pub/week44/html/._week44-bs031.html @@ -115,67 +115,6 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'the-cart-algorithm-for-regression'), - ('Computing the Gini index', 2, None, 'computing-the-gini-index'), - ('Simple Python Code to read in Data and perform Classification', - 2, - None, - 'simple-python-code-to-read-in-data-and-perform-classification'), - ('Computing the Gini Factor', - 2, - None, - 'computing-the-gini-factor'), - ('Entropy and the ID3 algorithm', - 2, - None, - 'entropy-and-the-id3-algorithm'), - ('Cancer Data again now with Decision Trees and other Methods', - 2, - None, - 'cancer-data-again-now-with-decision-trees-and-other-methods'), - ('Another example, the moons again', - 2, - None, - 'another-example-the-moons-again'), - ('Playing around with regions', - 2, - None, - 'playing-around-with-regions'), - ('Regression trees', 2, None, 'regression-trees'), - ('Final regressor code', 2, None, 'final-regressor-code'), - ('Pros and cons of trees, pros', - 2, - None, - 'pros-and-cons-of-trees-pros'), - ('Disadvantages', 2, None, 'disadvantages'), - ('Ensemble Methods: From a Single Tree to Many Trees and Extreme ' - 'Boosting, Meet the Jungle of Methods', - 2, - None, - 'ensemble-methods-from-a-single-tree-to-many-trees-and-extreme-boosting-meet-the-jungle-of-methods'), - ('An Overview of Ensemble Methods', - 2, - None, - 'an-overview-of-ensemble-methods'), - ('Bagging', 2, None, 'bagging'), - ('More bagging', 2, None, 'more-bagging'), - ('Simple Voting Example, head or tail', - 2, - None, - 'simple-voting-example-head-or-tail'), - ('Using the Voting Classifier', - 2, - None, - 'using-the-voting-classifier'), - ('Please, not the moons again! Voting and Bagging', - 2, - None, - 'please-not-the-moons-again-voting-and-bagging'), - ('Bagging Examples', 2, None, 'bagging-examples'), - ('Making your own Bootstrap: Changing the Level of the Decision ' - 'Tree', - 2, - None, - 'making-your-own-bootstrap-changing-the-level-of-the-decision-tree'), ('Why binary splits?', 2, None, 'why-binary-splits'), ('Computing a Tree using the Gini Index', 2, @@ -250,6 +189,70 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d 'playing-around-with-regions'), ('Regression trees', 2, None, 'regression-trees'), ('Final regressor code', 2, None, 'final-regressor-code'), + ('Example: Computing the Gini index', + 2, + None, + 'example-computing-the-gini-index'), + ('Simple Python Code to read in Data and perform Classification', + 2, + None, + 'simple-python-code-to-read-in-data-and-perform-classification'), + ('Computing the Gini Factor', + 2, + None, + 'computing-the-gini-factor'), + ('Entropy and the ID3 algorithm', + 2, + None, + 'entropy-and-the-id3-algorithm'), + ('Cancer Data again now with Decision Trees and other Methods', + 2, + None, + 'cancer-data-again-now-with-decision-trees-and-other-methods'), + ('Another example, the moons again', + 2, + None, + 'another-example-the-moons-again'), + ('Playing around with regions', + 2, + None, + 'playing-around-with-regions'), + ('Regression trees', 2, None, 'regression-trees'), + ('Final regressor code', 2, None, 'final-regressor-code'), + ('Pros and cons of trees, pros', + 2, + None, + 'pros-and-cons-of-trees-pros'), + ('Disadvantages', 2, None, 'disadvantages'), + ('Ensemble Methods: From a Single Tree to Many Trees and Extreme ' + 'Boosting, Meet the Jungle of Methods', + 2, + None, + 'ensemble-methods-from-a-single-tree-to-many-trees-and-extreme-boosting-meet-the-jungle-of-methods'), + ('An Overview of Ensemble Methods', + 2, + None, + 'an-overview-of-ensemble-methods'), + ('Bagging', 2, None, 'bagging'), + ('More bagging', 2, None, 'more-bagging'), + ('Simple Voting Example, head or tail', + 2, + None, + 'simple-voting-example-head-or-tail'), + ('Using the Voting Classifier', + 2, + None, + 'using-the-voting-classifier'), + ('Please, not the moons again! Voting and Bagging', + 2, + None, + 'please-not-the-moons-again-voting-and-bagging'), + ('Bagging Examples', 2, None, 'bagging-examples'), + ('Making your own Bootstrap: Changing the Level of the Decision ' + 'Tree', + 2, + None, + 'making-your-own-bootstrap-changing-the-level-of-the-decision-tree'), ('Why Voting?', 2, None, 'why-voting'), ('Tossing coins', 2, None, 'tossing-coins'), ('Standard imports first', 2, None, 'standard-imports-first'), @@ -327,55 +330,55 @@ MathJax.Hub.Config({
-
Entropy and the ID3 algorithm
+Computing the various Gini Indices
-The ID3 algorithm learns decision trees by constructing -them in a top down way, beginning with the question which attribute should be tested at the root of the tree? +
In computations we will translate all classes into numbers. Being +these binary classes, they can easily be split into ones and zeros.
--
-
The ID3 algorithm selects which attribute to test at each node in the -tree. -
+See handwritten notes for Thursday November 11
+We would like to select the attribute that is most useful for classifying -examples. -
- -What is a good quantitative measure of the worth of an attribute?
- -Information gain measures how well a given attribute separates the -training examples according to their target classification. -
- -The ID3 algorithm uses this information gain measure to select among the candidate -attributes at each step while growing the tree. -
diff --git a/doc/pub/week44/html/._week44-bs032.html b/doc/pub/week44/html/._week44-bs032.html index de7edf322..dffea248a 100644 --- a/doc/pub/week44/html/._week44-bs032.html +++ b/doc/pub/week44/html/._week44-bs032.html @@ -115,67 +115,6 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'the-cart-algorithm-for-regression'), - ('Computing the Gini index', 2, None, 'computing-the-gini-index'), - ('Simple Python Code to read in Data and perform Classification', - 2, - None, - 'simple-python-code-to-read-in-data-and-perform-classification'), - ('Computing the Gini Factor', - 2, - None, - 'computing-the-gini-factor'), - ('Entropy and the ID3 algorithm', - 2, - None, - 'entropy-and-the-id3-algorithm'), - ('Cancer Data again now with Decision Trees and other Methods', - 2, - None, - 'cancer-data-again-now-with-decision-trees-and-other-methods'), - ('Another example, the moons again', - 2, - None, - 'another-example-the-moons-again'), - ('Playing around with regions', - 2, - None, - 'playing-around-with-regions'), - ('Regression trees', 2, None, 'regression-trees'), - ('Final regressor code', 2, None, 'final-regressor-code'), - ('Pros and cons of trees, pros', - 2, - None, - 'pros-and-cons-of-trees-pros'), - ('Disadvantages', 2, None, 'disadvantages'), - ('Ensemble Methods: From a Single Tree to Many Trees and Extreme ' - 'Boosting, Meet the Jungle of Methods', - 2, - None, - 'ensemble-methods-from-a-single-tree-to-many-trees-and-extreme-boosting-meet-the-jungle-of-methods'), - ('An Overview of Ensemble Methods', - 2, - None, - 'an-overview-of-ensemble-methods'), - ('Bagging', 2, None, 'bagging'), - ('More bagging', 2, None, 'more-bagging'), - ('Simple Voting Example, head or tail', - 2, - None, - 'simple-voting-example-head-or-tail'), - ('Using the Voting Classifier', - 2, - None, - 'using-the-voting-classifier'), - ('Please, not the moons again! Voting and Bagging', - 2, - None, - 'please-not-the-moons-again-voting-and-bagging'), - ('Bagging Examples', 2, None, 'bagging-examples'), - ('Making your own Bootstrap: Changing the Level of the Decision ' - 'Tree', - 2, - None, - 'making-your-own-bootstrap-changing-the-level-of-the-decision-tree'), ('Why binary splits?', 2, None, 'why-binary-splits'), ('Computing a Tree using the Gini Index', 2, @@ -250,6 +189,70 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d 'playing-around-with-regions'), ('Regression trees', 2, None, 'regression-trees'), ('Final regressor code', 2, None, 'final-regressor-code'), + ('Example: Computing the Gini index', + 2, + None, + 'example-computing-the-gini-index'), + ('Simple Python Code to read in Data and perform Classification', + 2, + None, + 'simple-python-code-to-read-in-data-and-perform-classification'), + ('Computing the Gini Factor', + 2, + None, + 'computing-the-gini-factor'), + ('Entropy and the ID3 algorithm', + 2, + None, + 'entropy-and-the-id3-algorithm'), + ('Cancer Data again now with Decision Trees and other Methods', + 2, + None, + 'cancer-data-again-now-with-decision-trees-and-other-methods'), + ('Another example, the moons again', + 2, + None, + 'another-example-the-moons-again'), + ('Playing around with regions', + 2, + None, + 'playing-around-with-regions'), + ('Regression trees', 2, None, 'regression-trees'), + ('Final regressor code', 2, None, 'final-regressor-code'), + ('Pros and cons of trees, pros', + 2, + None, + 'pros-and-cons-of-trees-pros'), + ('Disadvantages', 2, None, 'disadvantages'), + ('Ensemble Methods: From a Single Tree to Many Trees and Extreme ' + 'Boosting, Meet the Jungle of Methods', + 2, + None, + 'ensemble-methods-from-a-single-tree-to-many-trees-and-extreme-boosting-meet-the-jungle-of-methods'), + ('An Overview of Ensemble Methods', + 2, + None, + 'an-overview-of-ensemble-methods'), + ('Bagging', 2, None, 'bagging'), + ('More bagging', 2, None, 'more-bagging'), + ('Simple Voting Example, head or tail', + 2, + None, + 'simple-voting-example-head-or-tail'), + ('Using the Voting Classifier', + 2, + None, + 'using-the-voting-classifier'), + ('Please, not the moons again! Voting and Bagging', + 2, + None, + 'please-not-the-moons-again-voting-and-bagging'), + ('Bagging Examples', 2, None, 'bagging-examples'), + ('Making your own Bootstrap: Changing the Level of the Decision ' + 'Tree', + 2, + None, + 'making-your-own-bootstrap-changing-the-level-of-the-decision-tree'), ('Why Voting?', 2, None, 'why-voting'), ('Tossing coins', 2, None, 'tossing-coins'), ('Standard imports first', 2, None, 'standard-imports-first'), @@ -327,55 +330,55 @@ MathJax.Hub.Config({
-
Cancer Data again now with Decision Trees and other Methods
+Computing the various Gini Indices, Hours slept
- -import matplotlib.pyplot as plt
-import numpy as np
-from sklearn.model_selection import train_test_split
-from sklearn.datasets import load_breast_cancer
-from sklearn.svm import SVC
-from sklearn.linear_model import LogisticRegression
-from sklearn.tree import DecisionTreeClassifier
-
-# Load the data
-cancer = load_breast_cancer()
-
-X_train, X_test, y_train, y_test = train_test_split(cancer.data,cancer.target,random_state=0)
-print(X_train.shape)
-print(X_test.shape)
-# Logistic Regression
-logreg = LogisticRegression(solver='lbfgs')
-logreg.fit(X_train, y_train)
-print("Test set accuracy with Logistic Regression: {:.2f}".format(logreg.score(X_test,y_test)))
-# Support vector machine
-svm = SVC(gamma='auto', C=100)
-svm.fit(X_train, y_train)
-print("Test set accuracy with SVM: {:.2f}".format(svm.score(X_test,y_test)))
-# Decision Trees
-deep_tree_clf = DecisionTreeClassifier(max_depth=None)
-deep_tree_clf.fit(X_train, y_train)
-print("Test set accuracy with Decision Trees: {:.2f}".format(deep_tree_clf.score(X_test,y_test)))
-#now scale the data
-from sklearn.preprocessing import StandardScaler
-scaler = StandardScaler()
-scaler.fit(X_train)
-X_train_scaled = scaler.transform(X_train)
-X_test_scaled = scaler.transform(X_test)
-# Logistic Regression
-logreg.fit(X_train_scaled, y_train)
-print("Test set accuracy Logistic Regression with scaled data: {:.2f}".format(logreg.score(X_test_scaled,y_test)))
-# Support Vector Machine
-svm.fit(X_train_scaled, y_train)
-print("Test set accuracy SVM with scaled data: {:.2f}".format(logreg.score(X_test_scaled,y_test)))
-# Decision Trees
-deep_tree_clf.fit(X_train_scaled, y_train)
-print("Test set accuracy with Decision Trees and scaled data: {:.2f}".format(deep_tree_clf.score(X_test_scaled,y_test)))
-
+See handwritten notes for Thursday November 3
-
Another example, the moons again
+Computing the various Gini Indices, Hours studied
- -from __future__ import division, print_function, unicode_literals
-
-# Common imports
-import numpy as np
-import os
-
-# to make this notebook's output stable across runs
-np.random.seed(42)
-
-# To plot pretty figures
-import matplotlib
-import matplotlib.pyplot as plt
-from matplotlib.colors import ListedColormap
-plt.rcParams['axes.labelsize'] = 14
-plt.rcParams['xtick.labelsize'] = 12
-plt.rcParams['ytick.labelsize'] = 12
-
-
-from sklearn.svm import SVC
-from sklearn import datasets
-from sklearn.tree import DecisionTreeClassifier
-from sklearn.datasets import make_moons
-from sklearn.tree import export_graphviz
-
-Xm, ym = make_moons(n_samples=100, noise=0.25, random_state=53)
-
-deep_tree_clf1 = DecisionTreeClassifier(random_state=42)
-deep_tree_clf2 = DecisionTreeClassifier(min_samples_leaf=4, random_state=42)
-deep_tree_clf1.fit(Xm, ym)
-deep_tree_clf2.fit(Xm, ym)
-
-
-def plot_decision_boundary(clf, X, y, axes=[0, 7.5, 0, 3], iris=True, legend=False, plot_training=True):
- x1s = np.linspace(axes[0], axes[1], 100)
- x2s = np.linspace(axes[2], axes[3], 100)
- x1, x2 = np.meshgrid(x1s, x2s)
- X_new = np.c_[x1.ravel(), x2.ravel()]
- y_pred = clf.predict(X_new).reshape(x1.shape)
- custom_cmap = ListedColormap(['#fafab0','#9898ff','#a0faa0'])
- plt.contourf(x1, x2, y_pred, alpha=0.3, cmap=custom_cmap)
- if not iris:
- custom_cmap2 = ListedColormap(['#7d7d58','#4c4c7f','#507d50'])
- plt.contour(x1, x2, y_pred, cmap=custom_cmap2, alpha=0.8)
- if plot_training:
- plt.plot(X[:, 0][y==0], X[:, 1][y==0], "yo", label="Iris-Setosa")
- plt.plot(X[:, 0][y==1], X[:, 1][y==1], "bs", label="Iris-Versicolor")
- plt.plot(X[:, 0][y==2], X[:, 1][y==2], "g^", label="Iris-Virginica")
- plt.axis(axes)
- if iris:
- plt.xlabel("Petal length", fontsize=14)
- plt.ylabel("Petal width", fontsize=14)
- else:
- plt.xlabel(r"$x_1$", fontsize=18)
- plt.ylabel(r"$x_2$", fontsize=18, rotation=0)
- if legend:
- plt.legend(loc="lower right", fontsize=14)
-plt.figure(figsize=(11, 4))
-plt.subplot(121)
-plot_decision_boundary(deep_tree_clf1, Xm, ym, axes=[-1.5, 2.5, -1, 1.5], iris=False)
-plt.title("No restrictions", fontsize=16)
-plt.subplot(122)
-plot_decision_boundary(deep_tree_clf2, Xm, ym, axes=[-1.5, 2.5, -1, 1.5], iris=False)
-plt.title("min_samples_leaf = {}".format(deep_tree_clf2.min_samples_leaf), fontsize=14)
-plt.show()
-
+See handwritten notes for Thursday November 3
For final tree, see the above handwritten notes
+
-
diff --git a/doc/pub/week44/html/._week44-bs034.html b/doc/pub/week44/html/._week44-bs034.html
index 815237ab5..adcbc345a 100644
--- a/doc/pub/week44/html/._week44-bs034.html
+++ b/doc/pub/week44/html/._week44-bs034.html
@@ -115,67 +115,6 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d
2,
None,
'the-cart-algorithm-for-regression'),
- ('Computing the Gini index', 2, None, 'computing-the-gini-index'),
- ('Simple Python Code to read in Data and perform Classification',
- 2,
- None,
- 'simple-python-code-to-read-in-data-and-perform-classification'),
- ('Computing the Gini Factor',
- 2,
- None,
- 'computing-the-gini-factor'),
- ('Entropy and the ID3 algorithm',
- 2,
- None,
- 'entropy-and-the-id3-algorithm'),
- ('Cancer Data again now with Decision Trees and other Methods',
- 2,
- None,
- 'cancer-data-again-now-with-decision-trees-and-other-methods'),
- ('Another example, the moons again',
- 2,
- None,
- 'another-example-the-moons-again'),
- ('Playing around with regions',
- 2,
- None,
- 'playing-around-with-regions'),
- ('Regression trees', 2, None, 'regression-trees'),
- ('Final regressor code', 2, None, 'final-regressor-code'),
- ('Pros and cons of trees, pros',
- 2,
- None,
- 'pros-and-cons-of-trees-pros'),
- ('Disadvantages', 2, None, 'disadvantages'),
- ('Ensemble Methods: From a Single Tree to Many Trees and Extreme '
- 'Boosting, Meet the Jungle of Methods',
- 2,
- None,
- 'ensemble-methods-from-a-single-tree-to-many-trees-and-extreme-boosting-meet-the-jungle-of-methods'),
- ('An Overview of Ensemble Methods',
- 2,
- None,
- 'an-overview-of-ensemble-methods'),
- ('Bagging', 2, None, 'bagging'),
- ('More bagging', 2, None, 'more-bagging'),
- ('Simple Voting Example, head or tail',
- 2,
- None,
- 'simple-voting-example-head-or-tail'),
- ('Using the Voting Classifier',
- 2,
- None,
- 'using-the-voting-classifier'),
- ('Please, not the moons again! Voting and Bagging',
- 2,
- None,
- 'please-not-the-moons-again-voting-and-bagging'),
- ('Bagging Examples', 2, None, 'bagging-examples'),
- ('Making your own Bootstrap: Changing the Level of the Decision '
- 'Tree',
- 2,
- None,
- 'making-your-own-bootstrap-changing-the-level-of-the-decision-tree'),
('Why binary splits?', 2, None, 'why-binary-splits'),
('Computing a Tree using the Gini Index',
2,
@@ -250,6 +189,70 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d
'playing-around-with-regions'),
('Regression trees', 2, None, 'regression-trees'),
('Final regressor code', 2, None, 'final-regressor-code'),
+ ('Example: Computing the Gini index',
+ 2,
+ None,
+ 'example-computing-the-gini-index'),
+ ('Simple Python Code to read in Data and perform Classification',
+ 2,
+ None,
+ 'simple-python-code-to-read-in-data-and-perform-classification'),
+ ('Computing the Gini Factor',
+ 2,
+ None,
+ 'computing-the-gini-factor'),
+ ('Entropy and the ID3 algorithm',
+ 2,
+ None,
+ 'entropy-and-the-id3-algorithm'),
+ ('Cancer Data again now with Decision Trees and other Methods',
+ 2,
+ None,
+ 'cancer-data-again-now-with-decision-trees-and-other-methods'),
+ ('Another example, the moons again',
+ 2,
+ None,
+ 'another-example-the-moons-again'),
+ ('Playing around with regions',
+ 2,
+ None,
+ 'playing-around-with-regions'),
+ ('Regression trees', 2, None, 'regression-trees'),
+ ('Final regressor code', 2, None, 'final-regressor-code'),
+ ('Pros and cons of trees, pros',
+ 2,
+ None,
+ 'pros-and-cons-of-trees-pros'),
+ ('Disadvantages', 2, None, 'disadvantages'),
+ ('Ensemble Methods: From a Single Tree to Many Trees and Extreme '
+ 'Boosting, Meet the Jungle of Methods',
+ 2,
+ None,
+ 'ensemble-methods-from-a-single-tree-to-many-trees-and-extreme-boosting-meet-the-jungle-of-methods'),
+ ('An Overview of Ensemble Methods',
+ 2,
+ None,
+ 'an-overview-of-ensemble-methods'),
+ ('Bagging', 2, None, 'bagging'),
+ ('More bagging', 2, None, 'more-bagging'),
+ ('Simple Voting Example, head or tail',
+ 2,
+ None,
+ 'simple-voting-example-head-or-tail'),
+ ('Using the Voting Classifier',
+ 2,
+ None,
+ 'using-the-voting-classifier'),
+ ('Please, not the moons again! Voting and Bagging',
+ 2,
+ None,
+ 'please-not-the-moons-again-voting-and-bagging'),
+ ('Bagging Examples', 2, None, 'bagging-examples'),
+ ('Making your own Bootstrap: Changing the Level of the Decision '
+ 'Tree',
+ 2,
+ None,
+ 'making-your-own-bootstrap-changing-the-level-of-the-decision-tree'),
('Why Voting?', 2, None, 'why-voting'),
('Tossing coins', 2, None, 'tossing-coins'),
('Standard imports first', 2, None, 'standard-imports-first'),
@@ -327,55 +330,55 @@ MathJax.Hub.Config({
-
Playing around with regions
+A possible code using Scikit-Learn
+np.random.seed(6)
-Xs = np.random.rand(100, 2) - 0.5
-ys = (Xs[:, 0] > 0).astype(np.float32) * 2
+ # Common imports
+import numpy as np
+import pandas as pd
+import matplotlib.pyplot as plt
+from sklearn.tree import DecisionTreeClassifier
+from sklearn.model_selection import train_test_split
+from sklearn.tree import export_graphviz
+from sklearn.preprocessing import StandardScaler, OneHotEncoder
+from sklearn.compose import ColumnTransformer
+from IPython.display import Image
+from pydot import graph_from_dot_data
+import os
-angle = np.pi/4
-rotation_matrix = np.array([[np.cos(angle), -np.sin(angle)], [np.sin(angle), np.cos(angle)]])
-Xsr = Xs.dot(rotation_matrix)
+# Where to save the figures and data files
+PROJECT_ROOT_DIR = "Results"
+FIGURE_ID = "Results/FigureFiles"
+DATA_ID = "DataFiles/"
-tree_clf_s = DecisionTreeClassifier(random_state=42)
-tree_clf_s.fit(Xs, ys)
-tree_clf_sr = DecisionTreeClassifier(random_state=42)
-tree_clf_sr.fit(Xsr, ys)
+if not os.path.exists(PROJECT_ROOT_DIR):
+ os.mkdir(PROJECT_ROOT_DIR)
-plt.figure(figsize=(11, 4))
-plt.subplot(121)
-plot_decision_boundary(tree_clf_s, Xs, ys, axes=[-0.7, 0.7, -0.7, 0.7], iris=False)
-plt.subplot(122)
-plot_decision_boundary(tree_clf_sr, Xsr, ys, axes=[-0.7, 0.7, -0.7, 0.7], iris=False)
+if not os.path.exists(FIGURE_ID):
+ os.makedirs(FIGURE_ID)
-plt.show()
+if not os.path.exists(DATA_ID):
+ os.makedirs(DATA_ID)
+
+def image_path(fig_id):
+ return os.path.join(FIGURE_ID, fig_id)
+
+def data_path(dat_id):
+ return os.path.join(DATA_ID, dat_id)
+
+def save_fig(fig_id):
+ plt.savefig(image_path(fig_id) + ".png", format='png')
+
+infile = open(data_path("grades.csv"),'r')
+
+# Read the experimental data with Pandas
+from IPython.display import display
+grades = pd.read_csv(infile,names = ('Trend','Sleep','Studied','Grade'))
+grades = pd.DataFrame(grades)
+
+# Features and targets
+X = grades.loc[:, grades.columns != 'Grade'].values
+y = grades.loc[:, grades.columns == 'Grade'].values
+
+# Create the encoder.
+encoder = OneHotEncoder(handle_unknown="ignore")
+# Assume for simplicity all features are categorical.
+encoder.fit(X)
+# Apply the encoder.
+X = encoder.transform(X)
+print(X)
+# Then do a Classification tree
+tree_clf = DecisionTreeClassifier(max_depth=2)
+tree_clf.fit(X, y)
+print("Train set accuracy with Decision Tree: {:.2f}".format(tree_clf.score(X,y)))
+#transfer to a decision tree graph
+export_graphviz(
+ tree_clf,
+ out_file="DataFiles/grade.dot",
+ rounded=True,
+ filled=True
+)
+cmd = 'dot -Tpng DataFiles/grade.dot -o DataFiles/grades.png'
+os.system(cmd)
-
Regression trees
+Visualizing Trees, More examples
# Quadratic training set + noise
-np.random.seed(42)
-m = 200
-X = np.random.rand(m, 1)
-y = 4 * (X - 0.5) ** 2
-y = y + np.random.randn(m, 1) / 10
-
-from sklearn.tree import DecisionTreeRegressor
+ import os
+from sklearn.datasets import load_breast_cancer
+from sklearn.tree import DecisionTreeClassifier
+from sklearn.model_selection import train_test_split
+from sklearn.metrics import confusion_matrix
+from sklearn.tree import export_graphviz
-tree_reg = DecisionTreeRegressor(max_depth=2, random_state=42)
-tree_reg.fit(X, y)
+from IPython.display import Image
+from pydot import graph_from_dot_data
+import pandas as pd
+import numpy as np
+
+
+cancer = load_breast_cancer()
+X = pd.DataFrame(cancer.data, columns=cancer.feature_names)
+print(X)
+y = pd.Categorical.from_codes(cancer.target, cancer.target_names)
+y = pd.get_dummies(y)
+print(y)
+X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=1)
+tree_clf = DecisionTreeClassifier(max_depth=5)
+tree_clf.fit(X_train, y_train)
+
+export_graphviz(
+ tree_clf,
+ out_file="DataFiles/cancer.dot",
+ feature_names=cancer.feature_names,
+ class_names=cancer.target_names,
+ rounded=True,
+ filled=True
+)
+cmd = 'dot -Tpng DataFiles/cancer.dot -o DataFiles/cancer.png'
+os.system(cmd)
-
Final regressor code
+Visualizing the Tree, The Moons
from sklearn.tree import DecisionTreeRegressor
+ # Common imports
+import numpy as np
+from sklearn.model_selection import train_test_split
+from sklearn.tree import DecisionTreeClassifier
+from sklearn.datasets import make_moons
+from sklearn.tree import export_graphviz
+from pydot import graph_from_dot_data
+import pandas as pd
+import os
-tree_reg1 = DecisionTreeRegressor(random_state=42, max_depth=2)
-tree_reg2 = DecisionTreeRegressor(random_state=42, max_depth=3)
-tree_reg1.fit(X, y)
-tree_reg2.fit(X, y)
+np.random.seed(42)
+X, y = make_moons(n_samples=100, noise=0.25, random_state=53)
+X_train, X_test, y_train, y_test = train_test_split(X,y,random_state=0)
+tree_clf = DecisionTreeClassifier(max_depth=5)
+tree_clf.fit(X_train, y_train)
-def plot_regression_predictions(tree_reg, X, y, axes=[0, 1, -0.2, 1], ylabel="$y$"):
- x1 = np.linspace(axes[0], axes[1], 500).reshape(-1, 1)
- y_pred = tree_reg.predict(x1)
- plt.axis(axes)
- plt.xlabel("$x_1$", fontsize=18)
- if ylabel:
- plt.ylabel(ylabel, fontsize=18, rotation=0)
- plt.plot(X, y, "b.")
- plt.plot(x1, y_pred, "r.-", linewidth=2, label=r"$\hat{y}$")
-
-plt.figure(figsize=(11, 4))
-plt.subplot(121)
-plot_regression_predictions(tree_reg1, X, y)
-for split, style in ((0.1973, "k-"), (0.0917, "k--"), (0.7718, "k--")):
- plt.plot([split, split], [-0.2, 1], style, linewidth=2)
-plt.text(0.21, 0.65, "Depth=0", fontsize=15)
-plt.text(0.01, 0.2, "Depth=1", fontsize=13)
-plt.text(0.65, 0.8, "Depth=1", fontsize=13)
-plt.legend(loc="upper center", fontsize=18)
-plt.title("max_depth=2", fontsize=14)
-
-plt.subplot(122)
-plot_regression_predictions(tree_reg2, X, y, ylabel=None)
-for split, style in ((0.1973, "k-"), (0.0917, "k--"), (0.7718, "k--")):
- plt.plot([split, split], [-0.2, 1], style, linewidth=2)
-for split in (0.0458, 0.1298, 0.2873, 0.9040):
- plt.plot([split, split], [-0.2, 1], "k:", linewidth=1)
-plt.text(0.3, 0.5, "Depth=2", fontsize=13)
-plt.title("max_depth=3", fontsize=14)
-
-plt.show()
-
-tree_reg1 = DecisionTreeRegressor(random_state=42)
-tree_reg2 = DecisionTreeRegressor(random_state=42, min_samples_leaf=10)
-tree_reg1.fit(X, y)
-tree_reg2.fit(X, y)
-
-x1 = np.linspace(0, 1, 500).reshape(-1, 1)
-y_pred1 = tree_reg1.predict(x1)
-y_pred2 = tree_reg2.predict(x1)
-
-plt.figure(figsize=(11, 4))
-
-plt.subplot(121)
-plt.plot(X, y, "b.")
-plt.plot(x1, y_pred1, "r.-", linewidth=2, label=r"$\hat{y}$")
-plt.axis([0, 1, -0.2, 1.1])
-plt.xlabel("$x_1$", fontsize=18)
-plt.ylabel("$y$", fontsize=18, rotation=0)
-plt.legend(loc="upper center", fontsize=18)
-plt.title("No restrictions", fontsize=14)
-
-plt.subplot(122)
-plt.plot(X, y, "b.")
-plt.plot(x1, y_pred2, "r.-", linewidth=2, label=r"$\hat{y}$")
-plt.axis([0, 1, -0.2, 1.1])
-plt.xlabel("$x_1$", fontsize=18)
-plt.title("min_samples_leaf={}".format(tree_reg2.min_samples_leaf), fontsize=14)
-
-plt.show()
+export_graphviz(
+ tree_clf,
+ out_file="DataFiles/moons.dot",
+ rounded=True,
+ filled=True
+)
+cmd = 'dot -Tpng DataFiles/moons.dot -o DataFiles/moons.png'
+os.system(cmd)
-
Pros and cons of trees, pros
+Other ways of visualizing the trees
+ +Scikit-Learn has also another way to visualize the trees which is very useful, here with the Iris data.
+ + + +from sklearn.datasets import load_iris
+from sklearn import tree
+X, y = load_iris(return_X_y=True)
+tree_clf = tree.DecisionTreeClassifier()
+tree_clf = tree_clf.fit(X, y)
+# and then plot the tree
+tree.plot_tree(tree_clf)
+
+-
-
-
diff --git a/doc/pub/week44/html/._week44-bs038.html b/doc/pub/week44/html/._week44-bs038.html
index e48cdf42c..b22fc9962 100644
--- a/doc/pub/week44/html/._week44-bs038.html
+++ b/doc/pub/week44/html/._week44-bs038.html
@@ -115,67 +115,6 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d
2,
None,
'the-cart-algorithm-for-regression'),
- ('Computing the Gini index', 2, None, 'computing-the-gini-index'),
- ('Simple Python Code to read in Data and perform Classification',
- 2,
- None,
- 'simple-python-code-to-read-in-data-and-perform-classification'),
- ('Computing the Gini Factor',
- 2,
- None,
- 'computing-the-gini-factor'),
- ('Entropy and the ID3 algorithm',
- 2,
- None,
- 'entropy-and-the-id3-algorithm'),
- ('Cancer Data again now with Decision Trees and other Methods',
- 2,
- None,
- 'cancer-data-again-now-with-decision-trees-and-other-methods'),
- ('Another example, the moons again',
- 2,
- None,
- 'another-example-the-moons-again'),
- ('Playing around with regions',
- 2,
- None,
- 'playing-around-with-regions'),
- ('Regression trees', 2, None, 'regression-trees'),
- ('Final regressor code', 2, None, 'final-regressor-code'),
- ('Pros and cons of trees, pros',
- 2,
- None,
- 'pros-and-cons-of-trees-pros'),
- ('Disadvantages', 2, None, 'disadvantages'),
- ('Ensemble Methods: From a Single Tree to Many Trees and Extreme '
- 'Boosting, Meet the Jungle of Methods',
- 2,
- None,
- 'ensemble-methods-from-a-single-tree-to-many-trees-and-extreme-boosting-meet-the-jungle-of-methods'),
- ('An Overview of Ensemble Methods',
- 2,
- None,
- 'an-overview-of-ensemble-methods'),
- ('Bagging', 2, None, 'bagging'),
- ('More bagging', 2, None, 'more-bagging'),
- ('Simple Voting Example, head or tail',
- 2,
- None,
- 'simple-voting-example-head-or-tail'),
- ('Using the Voting Classifier',
- 2,
- None,
- 'using-the-voting-classifier'),
- ('Please, not the moons again! Voting and Bagging',
- 2,
- None,
- 'please-not-the-moons-again-voting-and-bagging'),
- ('Bagging Examples', 2, None, 'bagging-examples'),
- ('Making your own Bootstrap: Changing the Level of the Decision '
- 'Tree',
- 2,
- None,
- 'making-your-own-bootstrap-changing-the-level-of-the-decision-tree'),
('Why binary splits?', 2, None, 'why-binary-splits'),
('Computing a Tree using the Gini Index',
2,
@@ -250,6 +189,70 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d
'playing-around-with-regions'),
('Regression trees', 2, None, 'regression-trees'),
('Final regressor code', 2, None, 'final-regressor-code'),
+ ('Example: Computing the Gini index',
+ 2,
+ None,
+ 'example-computing-the-gini-index'),
+ ('Simple Python Code to read in Data and perform Classification',
+ 2,
+ None,
+ 'simple-python-code-to-read-in-data-and-perform-classification'),
+ ('Computing the Gini Factor',
+ 2,
+ None,
+ 'computing-the-gini-factor'),
+ ('Entropy and the ID3 algorithm',
+ 2,
+ None,
+ 'entropy-and-the-id3-algorithm'),
+ ('Cancer Data again now with Decision Trees and other Methods',
+ 2,
+ None,
+ 'cancer-data-again-now-with-decision-trees-and-other-methods'),
+ ('Another example, the moons again',
+ 2,
+ None,
+ 'another-example-the-moons-again'),
+ ('Playing around with regions',
+ 2,
+ None,
+ 'playing-around-with-regions'),
+ ('Regression trees', 2, None, 'regression-trees'),
+ ('Final regressor code', 2, None, 'final-regressor-code'),
+ ('Pros and cons of trees, pros',
+ 2,
+ None,
+ 'pros-and-cons-of-trees-pros'),
+ ('Disadvantages', 2, None, 'disadvantages'),
+ ('Ensemble Methods: From a Single Tree to Many Trees and Extreme '
+ 'Boosting, Meet the Jungle of Methods',
+ 2,
+ None,
+ 'ensemble-methods-from-a-single-tree-to-many-trees-and-extreme-boosting-meet-the-jungle-of-methods'),
+ ('An Overview of Ensemble Methods',
+ 2,
+ None,
+ 'an-overview-of-ensemble-methods'),
+ ('Bagging', 2, None, 'bagging'),
+ ('More bagging', 2, None, 'more-bagging'),
+ ('Simple Voting Example, head or tail',
+ 2,
+ None,
+ 'simple-voting-example-head-or-tail'),
+ ('Using the Voting Classifier',
+ 2,
+ None,
+ 'using-the-voting-classifier'),
+ ('Please, not the moons again! Voting and Bagging',
+ 2,
+ None,
+ 'please-not-the-moons-again-voting-and-bagging'),
+ ('Bagging Examples', 2, None, 'bagging-examples'),
+ ('Making your own Bootstrap: Changing the Level of the Decision '
+ 'Tree',
+ 2,
+ None,
+ 'making-your-own-bootstrap-changing-the-level-of-the-decision-tree'),
('Why Voting?', 2, None, 'why-voting'),
('Tossing coins', 2, None, 'tossing-coins'),
('Standard imports first', 2, None, 'standard-imports-first'),
@@ -327,55 +330,55 @@ MathJax.Hub.Config({
-
Disadvantages
+Printing out as text
--
-
However, by aggregating many decision trees, using methods like -bagging, random forests, and boosting, the predictive performance of -trees can be substantially improved. +
Alternatively, the tree can also be exported in textual format with the function exporttext. +This method doesn’t require the installation of external libraries and is more compact:
+ + +from sklearn.datasets import load_iris
+from sklearn.tree import DecisionTreeClassifier
+from sklearn.tree import export_text
+iris = load_iris()
+decision_tree = DecisionTreeClassifier(random_state=0, max_depth=2)
+decision_tree = decision_tree.fit(iris.data, iris.target)
+r = export_text(decision_tree, feature_names=iris['feature_names'])
+print(r)
+
+
-
diff --git a/doc/pub/week44/html/._week44-bs039.html b/doc/pub/week44/html/._week44-bs039.html
index 5e5a6e986..910c304ab 100644
--- a/doc/pub/week44/html/._week44-bs039.html
+++ b/doc/pub/week44/html/._week44-bs039.html
@@ -115,67 +115,6 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d
2,
None,
'the-cart-algorithm-for-regression'),
- ('Computing the Gini index', 2, None, 'computing-the-gini-index'),
- ('Simple Python Code to read in Data and perform Classification',
- 2,
- None,
- 'simple-python-code-to-read-in-data-and-perform-classification'),
- ('Computing the Gini Factor',
- 2,
- None,
- 'computing-the-gini-factor'),
- ('Entropy and the ID3 algorithm',
- 2,
- None,
- 'entropy-and-the-id3-algorithm'),
- ('Cancer Data again now with Decision Trees and other Methods',
- 2,
- None,
- 'cancer-data-again-now-with-decision-trees-and-other-methods'),
- ('Another example, the moons again',
- 2,
- None,
- 'another-example-the-moons-again'),
- ('Playing around with regions',
- 2,
- None,
- 'playing-around-with-regions'),
- ('Regression trees', 2, None, 'regression-trees'),
- ('Final regressor code', 2, None, 'final-regressor-code'),
- ('Pros and cons of trees, pros',
- 2,
- None,
- 'pros-and-cons-of-trees-pros'),
- ('Disadvantages', 2, None, 'disadvantages'),
- ('Ensemble Methods: From a Single Tree to Many Trees and Extreme '
- 'Boosting, Meet the Jungle of Methods',
- 2,
- None,
- 'ensemble-methods-from-a-single-tree-to-many-trees-and-extreme-boosting-meet-the-jungle-of-methods'),
- ('An Overview of Ensemble Methods',
- 2,
- None,
- 'an-overview-of-ensemble-methods'),
- ('Bagging', 2, None, 'bagging'),
- ('More bagging', 2, None, 'more-bagging'),
- ('Simple Voting Example, head or tail',
- 2,
- None,
- 'simple-voting-example-head-or-tail'),
- ('Using the Voting Classifier',
- 2,
- None,
- 'using-the-voting-classifier'),
- ('Please, not the moons again! Voting and Bagging',
- 2,
- None,
- 'please-not-the-moons-again-voting-and-bagging'),
- ('Bagging Examples', 2, None, 'bagging-examples'),
- ('Making your own Bootstrap: Changing the Level of the Decision '
- 'Tree',
- 2,
- None,
- 'making-your-own-bootstrap-changing-the-level-of-the-decision-tree'),
('Why binary splits?', 2, None, 'why-binary-splits'),
('Computing a Tree using the Gini Index',
2,
@@ -250,6 +189,70 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d
'playing-around-with-regions'),
('Regression trees', 2, None, 'regression-trees'),
('Final regressor code', 2, None, 'final-regressor-code'),
+ ('Example: Computing the Gini index',
+ 2,
+ None,
+ 'example-computing-the-gini-index'),
+ ('Simple Python Code to read in Data and perform Classification',
+ 2,
+ None,
+ 'simple-python-code-to-read-in-data-and-perform-classification'),
+ ('Computing the Gini Factor',
+ 2,
+ None,
+ 'computing-the-gini-factor'),
+ ('Entropy and the ID3 algorithm',
+ 2,
+ None,
+ 'entropy-and-the-id3-algorithm'),
+ ('Cancer Data again now with Decision Trees and other Methods',
+ 2,
+ None,
+ 'cancer-data-again-now-with-decision-trees-and-other-methods'),
+ ('Another example, the moons again',
+ 2,
+ None,
+ 'another-example-the-moons-again'),
+ ('Playing around with regions',
+ 2,
+ None,
+ 'playing-around-with-regions'),
+ ('Regression trees', 2, None, 'regression-trees'),
+ ('Final regressor code', 2, None, 'final-regressor-code'),
+ ('Pros and cons of trees, pros',
+ 2,
+ None,
+ 'pros-and-cons-of-trees-pros'),
+ ('Disadvantages', 2, None, 'disadvantages'),
+ ('Ensemble Methods: From a Single Tree to Many Trees and Extreme '
+ 'Boosting, Meet the Jungle of Methods',
+ 2,
+ None,
+ 'ensemble-methods-from-a-single-tree-to-many-trees-and-extreme-boosting-meet-the-jungle-of-methods'),
+ ('An Overview of Ensemble Methods',
+ 2,
+ None,
+ 'an-overview-of-ensemble-methods'),
+ ('Bagging', 2, None, 'bagging'),
+ ('More bagging', 2, None, 'more-bagging'),
+ ('Simple Voting Example, head or tail',
+ 2,
+ None,
+ 'simple-voting-example-head-or-tail'),
+ ('Using the Voting Classifier',
+ 2,
+ None,
+ 'using-the-voting-classifier'),
+ ('Please, not the moons again! Voting and Bagging',
+ 2,
+ None,
+ 'please-not-the-moons-again-voting-and-bagging'),
+ ('Bagging Examples', 2, None, 'bagging-examples'),
+ ('Making your own Bootstrap: Changing the Level of the Decision '
+ 'Tree',
+ 2,
+ None,
+ 'making-your-own-bootstrap-changing-the-level-of-the-decision-tree'),
('Why Voting?', 2, None, 'why-voting'),
('Tossing coins', 2, None, 'tossing-coins'),
('Standard imports first', 2, None, 'standard-imports-first'),
@@ -327,55 +330,55 @@ MathJax.Hub.Config({
-
Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods
- -As stated above and seen in many of the examples discussed here about -a single decision tree, we often end up overfitting our training -data. This normally means that we have a high variance. Can we reduce -the variance of a statistical learning method? -
- -This leads us to a set of different methods that can combine different -machine learning algorithms or just use one of them to construct -forests and jungles of trees, homogeneous ones or heterogenous -ones. These methods are recognized by different names which we will -try to explain here. These are -
+Algorithms for Setting up Decision Trees
+Two algorithms stand out in the set up of decision trees:
-
-
We discuss these methods here.
+We discuss both algorithms with applications here. The popular library +Scikit-Learn uses the CART algorithm. For classification problems +you can use either the gini index or the entropy to split a tree +in two branches. +
diff --git a/doc/pub/week44/html/._week44-bs040.html b/doc/pub/week44/html/._week44-bs040.html index 4ee3da16d..222597ab8 100644 --- a/doc/pub/week44/html/._week44-bs040.html +++ b/doc/pub/week44/html/._week44-bs040.html @@ -115,67 +115,6 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'the-cart-algorithm-for-regression'), - ('Computing the Gini index', 2, None, 'computing-the-gini-index'), - ('Simple Python Code to read in Data and perform Classification', - 2, - None, - 'simple-python-code-to-read-in-data-and-perform-classification'), - ('Computing the Gini Factor', - 2, - None, - 'computing-the-gini-factor'), - ('Entropy and the ID3 algorithm', - 2, - None, - 'entropy-and-the-id3-algorithm'), - ('Cancer Data again now with Decision Trees and other Methods', - 2, - None, - 'cancer-data-again-now-with-decision-trees-and-other-methods'), - ('Another example, the moons again', - 2, - None, - 'another-example-the-moons-again'), - ('Playing around with regions', - 2, - None, - 'playing-around-with-regions'), - ('Regression trees', 2, None, 'regression-trees'), - ('Final regressor code', 2, None, 'final-regressor-code'), - ('Pros and cons of trees, pros', - 2, - None, - 'pros-and-cons-of-trees-pros'), - ('Disadvantages', 2, None, 'disadvantages'), - ('Ensemble Methods: From a Single Tree to Many Trees and Extreme ' - 'Boosting, Meet the Jungle of Methods', - 2, - None, - 'ensemble-methods-from-a-single-tree-to-many-trees-and-extreme-boosting-meet-the-jungle-of-methods'), - ('An Overview of Ensemble Methods', - 2, - None, - 'an-overview-of-ensemble-methods'), - ('Bagging', 2, None, 'bagging'), - ('More bagging', 2, None, 'more-bagging'), - ('Simple Voting Example, head or tail', - 2, - None, - 'simple-voting-example-head-or-tail'), - ('Using the Voting Classifier', - 2, - None, - 'using-the-voting-classifier'), - ('Please, not the moons again! Voting and Bagging', - 2, - None, - 'please-not-the-moons-again-voting-and-bagging'), - ('Bagging Examples', 2, None, 'bagging-examples'), - ('Making your own Bootstrap: Changing the Level of the Decision ' - 'Tree', - 2, - None, - 'making-your-own-bootstrap-changing-the-level-of-the-decision-tree'), ('Why binary splits?', 2, None, 'why-binary-splits'), ('Computing a Tree using the Gini Index', 2, @@ -250,6 +189,70 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d 'playing-around-with-regions'), ('Regression trees', 2, None, 'regression-trees'), ('Final regressor code', 2, None, 'final-regressor-code'), + ('Example: Computing the Gini index', + 2, + None, + 'example-computing-the-gini-index'), + ('Simple Python Code to read in Data and perform Classification', + 2, + None, + 'simple-python-code-to-read-in-data-and-perform-classification'), + ('Computing the Gini Factor', + 2, + None, + 'computing-the-gini-factor'), + ('Entropy and the ID3 algorithm', + 2, + None, + 'entropy-and-the-id3-algorithm'), + ('Cancer Data again now with Decision Trees and other Methods', + 2, + None, + 'cancer-data-again-now-with-decision-trees-and-other-methods'), + ('Another example, the moons again', + 2, + None, + 'another-example-the-moons-again'), + ('Playing around with regions', + 2, + None, + 'playing-around-with-regions'), + ('Regression trees', 2, None, 'regression-trees'), + ('Final regressor code', 2, None, 'final-regressor-code'), + ('Pros and cons of trees, pros', + 2, + None, + 'pros-and-cons-of-trees-pros'), + ('Disadvantages', 2, None, 'disadvantages'), + ('Ensemble Methods: From a Single Tree to Many Trees and Extreme ' + 'Boosting, Meet the Jungle of Methods', + 2, + None, + 'ensemble-methods-from-a-single-tree-to-many-trees-and-extreme-boosting-meet-the-jungle-of-methods'), + ('An Overview of Ensemble Methods', + 2, + None, + 'an-overview-of-ensemble-methods'), + ('Bagging', 2, None, 'bagging'), + ('More bagging', 2, None, 'more-bagging'), + ('Simple Voting Example, head or tail', + 2, + None, + 'simple-voting-example-head-or-tail'), + ('Using the Voting Classifier', + 2, + None, + 'using-the-voting-classifier'), + ('Please, not the moons again! Voting and Bagging', + 2, + None, + 'please-not-the-moons-again-voting-and-bagging'), + ('Bagging Examples', 2, None, 'bagging-examples'), + ('Making your own Bootstrap: Changing the Level of the Decision ' + 'Tree', + 2, + None, + 'making-your-own-bootstrap-changing-the-level-of-the-decision-tree'), ('Why Voting?', 2, None, 'why-voting'), ('Tossing coins', 2, None, 'tossing-coins'), ('Standard imports first', 2, None, 'standard-imports-first'), @@ -327,55 +330,55 @@ MathJax.Hub.Config({
-
An Overview of Ensemble Methods
+The CART algorithm for Classification
--
+
For classification, the CART algorithm splits the data set in two subsets using a single feature \( k \) and a threshold \( t_k \). +This could be for example a threshold set by a number below a certain circumference of a malign tumor. +
+ +How do we find these two quantities? +We search for the pair \( (k,t_k) \) that produces the purest subset using for example the gini factor \( G \). +The cost function it tries to minimize is then +
+$$ +C(k,t_k) = \frac{m_{\mathrm{left}}}{m}G_{\mathrm{left}}+ \frac{m_{\mathrm{right}}}{m}G_{\mathrm{right}}, +$$ + +where \( G_{\mathrm{left/right}} \) measures the impurity of the left/right subset and \( m_{\mathrm{left/right}} \) + is the number of instances in the left/right subset +
+ +Once it has successfully split the training set in two, it splits the subsets using the same logic, then the subsubsets +and so on, recursively. It stops recursing once it reaches the maximum depth (defined by the +\( max\_depth \) hyperparameter), or if it cannot find a split that will reduce impurity. A few other +hyperparameters control additional stopping conditions such as the \( min\_samples\_split \), +\( min\_samples\_leaf \), \( min\_weight\_fraction\_leaf \), and \( max\_leaf\_nodes \). +
diff --git a/doc/pub/week44/html/._week44-bs041.html b/doc/pub/week44/html/._week44-bs041.html index ad70501f7..9602d9ae7 100644 --- a/doc/pub/week44/html/._week44-bs041.html +++ b/doc/pub/week44/html/._week44-bs041.html @@ -115,67 +115,6 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'the-cart-algorithm-for-regression'), - ('Computing the Gini index', 2, None, 'computing-the-gini-index'), - ('Simple Python Code to read in Data and perform Classification', - 2, - None, - 'simple-python-code-to-read-in-data-and-perform-classification'), - ('Computing the Gini Factor', - 2, - None, - 'computing-the-gini-factor'), - ('Entropy and the ID3 algorithm', - 2, - None, - 'entropy-and-the-id3-algorithm'), - ('Cancer Data again now with Decision Trees and other Methods', - 2, - None, - 'cancer-data-again-now-with-decision-trees-and-other-methods'), - ('Another example, the moons again', - 2, - None, - 'another-example-the-moons-again'), - ('Playing around with regions', - 2, - None, - 'playing-around-with-regions'), - ('Regression trees', 2, None, 'regression-trees'), - ('Final regressor code', 2, None, 'final-regressor-code'), - ('Pros and cons of trees, pros', - 2, - None, - 'pros-and-cons-of-trees-pros'), - ('Disadvantages', 2, None, 'disadvantages'), - ('Ensemble Methods: From a Single Tree to Many Trees and Extreme ' - 'Boosting, Meet the Jungle of Methods', - 2, - None, - 'ensemble-methods-from-a-single-tree-to-many-trees-and-extreme-boosting-meet-the-jungle-of-methods'), - ('An Overview of Ensemble Methods', - 2, - None, - 'an-overview-of-ensemble-methods'), - ('Bagging', 2, None, 'bagging'), - ('More bagging', 2, None, 'more-bagging'), - ('Simple Voting Example, head or tail', - 2, - None, - 'simple-voting-example-head-or-tail'), - ('Using the Voting Classifier', - 2, - None, - 'using-the-voting-classifier'), - ('Please, not the moons again! Voting and Bagging', - 2, - None, - 'please-not-the-moons-again-voting-and-bagging'), - ('Bagging Examples', 2, None, 'bagging-examples'), - ('Making your own Bootstrap: Changing the Level of the Decision ' - 'Tree', - 2, - None, - 'making-your-own-bootstrap-changing-the-level-of-the-decision-tree'), ('Why binary splits?', 2, None, 'why-binary-splits'), ('Computing a Tree using the Gini Index', 2, @@ -250,6 +189,70 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d 'playing-around-with-regions'), ('Regression trees', 2, None, 'regression-trees'), ('Final regressor code', 2, None, 'final-regressor-code'), + ('Example: Computing the Gini index', + 2, + None, + 'example-computing-the-gini-index'), + ('Simple Python Code to read in Data and perform Classification', + 2, + None, + 'simple-python-code-to-read-in-data-and-perform-classification'), + ('Computing the Gini Factor', + 2, + None, + 'computing-the-gini-factor'), + ('Entropy and the ID3 algorithm', + 2, + None, + 'entropy-and-the-id3-algorithm'), + ('Cancer Data again now with Decision Trees and other Methods', + 2, + None, + 'cancer-data-again-now-with-decision-trees-and-other-methods'), + ('Another example, the moons again', + 2, + None, + 'another-example-the-moons-again'), + ('Playing around with regions', + 2, + None, + 'playing-around-with-regions'), + ('Regression trees', 2, None, 'regression-trees'), + ('Final regressor code', 2, None, 'final-regressor-code'), + ('Pros and cons of trees, pros', + 2, + None, + 'pros-and-cons-of-trees-pros'), + ('Disadvantages', 2, None, 'disadvantages'), + ('Ensemble Methods: From a Single Tree to Many Trees and Extreme ' + 'Boosting, Meet the Jungle of Methods', + 2, + None, + 'ensemble-methods-from-a-single-tree-to-many-trees-and-extreme-boosting-meet-the-jungle-of-methods'), + ('An Overview of Ensemble Methods', + 2, + None, + 'an-overview-of-ensemble-methods'), + ('Bagging', 2, None, 'bagging'), + ('More bagging', 2, None, 'more-bagging'), + ('Simple Voting Example, head or tail', + 2, + None, + 'simple-voting-example-head-or-tail'), + ('Using the Voting Classifier', + 2, + None, + 'using-the-voting-classifier'), + ('Please, not the moons again! Voting and Bagging', + 2, + None, + 'please-not-the-moons-again-voting-and-bagging'), + ('Bagging Examples', 2, None, 'bagging-examples'), + ('Making your own Bootstrap: Changing the Level of the Decision ' + 'Tree', + 2, + None, + 'making-your-own-bootstrap-changing-the-level-of-the-decision-tree'), ('Why Voting?', 2, None, 'why-voting'), ('Tossing coins', 2, None, 'tossing-coins'), ('Standard imports first', 2, None, 'standard-imports-first'), @@ -327,55 +330,55 @@ MathJax.Hub.Config({
-
Bagging
+The CART algorithm for Regression
-The plain decision trees suffer from high -variance. This means that if we split the training data into two parts -at random, and fit a decision tree to both halves, the results that we -get could be quite different. In contrast, a procedure with low -variance will yield similar results if applied repeatedly to distinct -data sets; linear regression tends to have low variance, if the ratio -of \( n \) to \( p \) is moderately large. +
The CART algorithm for regression works is similar to the one for classification except that instead of trying to split the +training set in a way that minimizes say the gini or entropy impurity, it now tries to split the training set in a way that minimizes our well-known mean-squared error (MSE). The cost function is now
+$$ +C(k,t_k) = \frac{m_{\mathrm{left}}}{m}\mathrm{MSE}_{\mathrm{left}}+ \frac{m_{\mathrm{right}}}{m}\mathrm{MSE}_{\mathrm{right}}. +$$ -Bootstrap aggregation, or just bagging, is a -general-purpose procedure for reducing the variance of a statistical -learning method. +
Here the MSE for a specific node is defined as
+$$ +\mathrm{MSE}_{\mathrm{node}}=\frac{1}{m_\mathrm{node}}\sum_{i\in \mathrm{node}}(\overline{y}_{\mathrm{node}}-y_i)^2, +$$ + +with
+$$ +\overline{y}_{\mathrm{node}}=\frac{1}{m_\mathrm{node}}\sum_{i\in \mathrm{node}}y_i, +$$ + +the mean value of all observations in a specific node.
+ +Without any regularization, the regression task for decision trees, +just like for classification tasks, is prone to overfitting.
diff --git a/doc/pub/week44/html/._week44-bs042.html b/doc/pub/week44/html/._week44-bs042.html index f733f2a2f..c9fa3ad16 100644 --- a/doc/pub/week44/html/._week44-bs042.html +++ b/doc/pub/week44/html/._week44-bs042.html @@ -115,67 +115,6 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'the-cart-algorithm-for-regression'), - ('Computing the Gini index', 2, None, 'computing-the-gini-index'), - ('Simple Python Code to read in Data and perform Classification', - 2, - None, - 'simple-python-code-to-read-in-data-and-perform-classification'), - ('Computing the Gini Factor', - 2, - None, - 'computing-the-gini-factor'), - ('Entropy and the ID3 algorithm', - 2, - None, - 'entropy-and-the-id3-algorithm'), - ('Cancer Data again now with Decision Trees and other Methods', - 2, - None, - 'cancer-data-again-now-with-decision-trees-and-other-methods'), - ('Another example, the moons again', - 2, - None, - 'another-example-the-moons-again'), - ('Playing around with regions', - 2, - None, - 'playing-around-with-regions'), - ('Regression trees', 2, None, 'regression-trees'), - ('Final regressor code', 2, None, 'final-regressor-code'), - ('Pros and cons of trees, pros', - 2, - None, - 'pros-and-cons-of-trees-pros'), - ('Disadvantages', 2, None, 'disadvantages'), - ('Ensemble Methods: From a Single Tree to Many Trees and Extreme ' - 'Boosting, Meet the Jungle of Methods', - 2, - None, - 'ensemble-methods-from-a-single-tree-to-many-trees-and-extreme-boosting-meet-the-jungle-of-methods'), - ('An Overview of Ensemble Methods', - 2, - None, - 'an-overview-of-ensemble-methods'), - ('Bagging', 2, None, 'bagging'), - ('More bagging', 2, None, 'more-bagging'), - ('Simple Voting Example, head or tail', - 2, - None, - 'simple-voting-example-head-or-tail'), - ('Using the Voting Classifier', - 2, - None, - 'using-the-voting-classifier'), - ('Please, not the moons again! Voting and Bagging', - 2, - None, - 'please-not-the-moons-again-voting-and-bagging'), - ('Bagging Examples', 2, None, 'bagging-examples'), - ('Making your own Bootstrap: Changing the Level of the Decision ' - 'Tree', - 2, - None, - 'making-your-own-bootstrap-changing-the-level-of-the-decision-tree'), ('Why binary splits?', 2, None, 'why-binary-splits'), ('Computing a Tree using the Gini Index', 2, @@ -250,6 +189,70 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d 'playing-around-with-regions'), ('Regression trees', 2, None, 'regression-trees'), ('Final regressor code', 2, None, 'final-regressor-code'), + ('Example: Computing the Gini index', + 2, + None, + 'example-computing-the-gini-index'), + ('Simple Python Code to read in Data and perform Classification', + 2, + None, + 'simple-python-code-to-read-in-data-and-perform-classification'), + ('Computing the Gini Factor', + 2, + None, + 'computing-the-gini-factor'), + ('Entropy and the ID3 algorithm', + 2, + None, + 'entropy-and-the-id3-algorithm'), + ('Cancer Data again now with Decision Trees and other Methods', + 2, + None, + 'cancer-data-again-now-with-decision-trees-and-other-methods'), + ('Another example, the moons again', + 2, + None, + 'another-example-the-moons-again'), + ('Playing around with regions', + 2, + None, + 'playing-around-with-regions'), + ('Regression trees', 2, None, 'regression-trees'), + ('Final regressor code', 2, None, 'final-regressor-code'), + ('Pros and cons of trees, pros', + 2, + None, + 'pros-and-cons-of-trees-pros'), + ('Disadvantages', 2, None, 'disadvantages'), + ('Ensemble Methods: From a Single Tree to Many Trees and Extreme ' + 'Boosting, Meet the Jungle of Methods', + 2, + None, + 'ensemble-methods-from-a-single-tree-to-many-trees-and-extreme-boosting-meet-the-jungle-of-methods'), + ('An Overview of Ensemble Methods', + 2, + None, + 'an-overview-of-ensemble-methods'), + ('Bagging', 2, None, 'bagging'), + ('More bagging', 2, None, 'more-bagging'), + ('Simple Voting Example, head or tail', + 2, + None, + 'simple-voting-example-head-or-tail'), + ('Using the Voting Classifier', + 2, + None, + 'using-the-voting-classifier'), + ('Please, not the moons again! Voting and Bagging', + 2, + None, + 'please-not-the-moons-again-voting-and-bagging'), + ('Bagging Examples', 2, None, 'bagging-examples'), + ('Making your own Bootstrap: Changing the Level of the Decision ' + 'Tree', + 2, + None, + 'making-your-own-bootstrap-changing-the-level-of-the-decision-tree'), ('Why Voting?', 2, None, 'why-voting'), ('Tossing coins', 2, None, 'tossing-coins'), ('Standard imports first', 2, None, 'standard-imports-first'), @@ -327,55 +330,55 @@ MathJax.Hub.Config({
-
More bagging
+Computing the Gini index
-Bagging typically results in improved accuracy -over prediction using a single tree. Unfortunately, however, it can be -difficult to interpret the resulting model. Recall that one of the -advantages of decision trees is the attractive and easily interpreted -diagram that results. +
The example we will look at is a classical one in many Machine +Learning applications. Based on various meteorological features, we +have several so-called attributes which decide whether we at the end +will do some outdoor activity like skiing, going for a bike ride etc +etc. The table here contains the feautures outlook, temperature, +humidity and wind. The target or output is whether we ride +(True=1) or whether we do something else that day (False=0). The +attributes for each feature are then sunny, overcast and rain for the +outlook, hot, cold and mild for temperature, high and normal for +humidity and weak and strong for wind.
-However, when we bag a large number of trees, it is no longer -possible to represent the resulting statistical learning procedure -using a single tree, and it is no longer clear which variables are -most important to the procedure. Thus, bagging improves prediction -accuracy at the expense of interpretability. Although the collection -of bagged trees is much more difficult to interpret than a single -tree, one can obtain an overall summary of the importance of each -predictor using the MSE (for bagging regression trees) or the Gini -index (for bagging classification trees). In the case of bagging -regression trees, we can record the total amount that the MSE is -decreased due to splits over a given predictor, averaged over all \( B \) possible -trees. A large value indicates an important predictor. Similarly, in -the context of bagging classification trees, we can add up the total -amount that the Gini index is decreased by splits over a given -predictor, averaged over all \( B \) trees. -
+The table here summarizes the various attributes and
+| Day | Outlook | Temperature | Humidity | Wind | Ride |
| 1 | Sunny | Hot | High | Weak | 0 |
| 2 | Sunny | Hot | High | Strong | 1 |
| 3 | Overcast | Hot | High | Weak | 1 |
| 4 | Rain | Mild | High | Weak | 1 |
| 5 | Rain | Cool | Normal | Weak | 1 |
| 6 | Rain | Cool | Normal | Strong | 0 |
| 7 | Overcast | Cool | Normal | Strong | 1 |
| 8 | Sunny | Mild | High | Weak | 0 |
| 9 | Sunny | Cool | Normal | Weak | 1 |
| 10 | Rain | Mild | Normal | Weak | 1 |
| 11 | Sunny | Mild | Normal | Strong | 1 |
| 12 | Overcast | Mild | High | Strong | 1 |
| 13 | Overcast | Hot | Normal | Weak | 1 |
| 14 | Rain | Mild | High | Strong | 0 |
diff --git a/doc/pub/week44/html/._week44-bs043.html b/doc/pub/week44/html/._week44-bs043.html index 32d9461b8..480ec6af5 100644 --- a/doc/pub/week44/html/._week44-bs043.html +++ b/doc/pub/week44/html/._week44-bs043.html @@ -115,67 +115,6 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'the-cart-algorithm-for-regression'), - ('Computing the Gini index', 2, None, 'computing-the-gini-index'), - ('Simple Python Code to read in Data and perform Classification', - 2, - None, - 'simple-python-code-to-read-in-data-and-perform-classification'), - ('Computing the Gini Factor', - 2, - None, - 'computing-the-gini-factor'), - ('Entropy and the ID3 algorithm', - 2, - None, - 'entropy-and-the-id3-algorithm'), - ('Cancer Data again now with Decision Trees and other Methods', - 2, - None, - 'cancer-data-again-now-with-decision-trees-and-other-methods'), - ('Another example, the moons again', - 2, - None, - 'another-example-the-moons-again'), - ('Playing around with regions', - 2, - None, - 'playing-around-with-regions'), - ('Regression trees', 2, None, 'regression-trees'), - ('Final regressor code', 2, None, 'final-regressor-code'), - ('Pros and cons of trees, pros', - 2, - None, - 'pros-and-cons-of-trees-pros'), - ('Disadvantages', 2, None, 'disadvantages'), - ('Ensemble Methods: From a Single Tree to Many Trees and Extreme ' - 'Boosting, Meet the Jungle of Methods', - 2, - None, - 'ensemble-methods-from-a-single-tree-to-many-trees-and-extreme-boosting-meet-the-jungle-of-methods'), - ('An Overview of Ensemble Methods', - 2, - None, - 'an-overview-of-ensemble-methods'), - ('Bagging', 2, None, 'bagging'), - ('More bagging', 2, None, 'more-bagging'), - ('Simple Voting Example, head or tail', - 2, - None, - 'simple-voting-example-head-or-tail'), - ('Using the Voting Classifier', - 2, - None, - 'using-the-voting-classifier'), - ('Please, not the moons again! Voting and Bagging', - 2, - None, - 'please-not-the-moons-again-voting-and-bagging'), - ('Bagging Examples', 2, None, 'bagging-examples'), - ('Making your own Bootstrap: Changing the Level of the Decision ' - 'Tree', - 2, - None, - 'making-your-own-bootstrap-changing-the-level-of-the-decision-tree'), ('Why binary splits?', 2, None, 'why-binary-splits'), ('Computing a Tree using the Gini Index', 2, @@ -250,6 +189,70 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d 'playing-around-with-regions'), ('Regression trees', 2, None, 'regression-trees'), ('Final regressor code', 2, None, 'final-regressor-code'), + ('Example: Computing the Gini index', + 2, + None, + 'example-computing-the-gini-index'), + ('Simple Python Code to read in Data and perform Classification', + 2, + None, + 'simple-python-code-to-read-in-data-and-perform-classification'), + ('Computing the Gini Factor', + 2, + None, + 'computing-the-gini-factor'), + ('Entropy and the ID3 algorithm', + 2, + None, + 'entropy-and-the-id3-algorithm'), + ('Cancer Data again now with Decision Trees and other Methods', + 2, + None, + 'cancer-data-again-now-with-decision-trees-and-other-methods'), + ('Another example, the moons again', + 2, + None, + 'another-example-the-moons-again'), + ('Playing around with regions', + 2, + None, + 'playing-around-with-regions'), + ('Regression trees', 2, None, 'regression-trees'), + ('Final regressor code', 2, None, 'final-regressor-code'), + ('Pros and cons of trees, pros', + 2, + None, + 'pros-and-cons-of-trees-pros'), + ('Disadvantages', 2, None, 'disadvantages'), + ('Ensemble Methods: From a Single Tree to Many Trees and Extreme ' + 'Boosting, Meet the Jungle of Methods', + 2, + None, + 'ensemble-methods-from-a-single-tree-to-many-trees-and-extreme-boosting-meet-the-jungle-of-methods'), + ('An Overview of Ensemble Methods', + 2, + None, + 'an-overview-of-ensemble-methods'), + ('Bagging', 2, None, 'bagging'), + ('More bagging', 2, None, 'more-bagging'), + ('Simple Voting Example, head or tail', + 2, + None, + 'simple-voting-example-head-or-tail'), + ('Using the Voting Classifier', + 2, + None, + 'using-the-voting-classifier'), + ('Please, not the moons again! Voting and Bagging', + 2, + None, + 'please-not-the-moons-again-voting-and-bagging'), + ('Bagging Examples', 2, None, 'bagging-examples'), + ('Making your own Bootstrap: Changing the Level of the Decision ' + 'Tree', + 2, + None, + 'making-your-own-bootstrap-changing-the-level-of-the-decision-tree'), ('Why Voting?', 2, None, 'why-voting'), ('Tossing coins', 2, None, 'tossing-coins'), ('Standard imports first', 2, None, 'standard-imports-first'), @@ -327,55 +330,55 @@ MathJax.Hub.Config({
-
Simple Voting Example, head or tail
+Simple Python Code to read in Data and perform Classification
+heads_proba = 0.51
-coin_tosses = (np.random.rand(10000, 10) < heads_proba).astype(np.int32)
-cumulative_heads_ratio = np.cumsum(coin_tosses, axis=0) / np.arange(1, 10001).reshape(-1, 1)
-plt.figure(figsize=(8,3.5))
-plt.plot(cumulative_heads_ratio)
-plt.plot([0, 10000], [0.51, 0.51], "k--", linewidth=2, label="51%")
-plt.plot([0, 10000], [0.5, 0.5], "k-", label="50%")
-plt.xlabel("Number of coin tosses")
-plt.ylabel("Heads ratio")
-plt.legend(loc="lower right")
-plt.axis([0, 10000, 0.42, 0.58])
-save_fig("votingsimple")
-plt.show()
+ # Common imports
+import numpy as np
+import pandas as pd
+import matplotlib.pyplot as plt
+from sklearn.tree import DecisionTreeClassifier
+from sklearn.model_selection import train_test_split
+from sklearn.tree import export_graphviz
+from sklearn.preprocessing import StandardScaler, OneHotEncoder
+from sklearn.compose import ColumnTransformer
+from IPython.display import Image
+from pydot import graph_from_dot_data
+import os
+
+# Where to save the figures and data files
+PROJECT_ROOT_DIR = "Results"
+FIGURE_ID = "Results/FigureFiles"
+DATA_ID = "DataFiles/"
+
+if not os.path.exists(PROJECT_ROOT_DIR):
+ os.mkdir(PROJECT_ROOT_DIR)
+
+if not os.path.exists(FIGURE_ID):
+ os.makedirs(FIGURE_ID)
+
+if not os.path.exists(DATA_ID):
+ os.makedirs(DATA_ID)
+
+def image_path(fig_id):
+ return os.path.join(FIGURE_ID, fig_id)
+
+def data_path(dat_id):
+ return os.path.join(DATA_ID, dat_id)
+
+def save_fig(fig_id):
+ plt.savefig(image_path(fig_id) + ".png", format='png')
+
+infile = open(data_path("rideclass.csv"),'r')
+
+# Read the experimental data with Pandas
+from IPython.display import display
+ridedata = pd.read_csv(infile,names = ('Outlook','Temperature','Humidity','Wind','Ride'))
+ridedata = pd.DataFrame(ridedata)
+
+# Features and targets
+X = ridedata.loc[:, ridedata.columns != 'Ride'].values
+y = ridedata.loc[:, ridedata.columns == 'Ride'].values
+
+# Create the encoder.
+encoder = OneHotEncoder(handle_unknown="ignore")
+# Assume for simplicity all features are categorical.
+encoder.fit(X)
+# Apply the encoder.
+X = encoder.transform(X)
+print(X)
+# Then do a Classification tree
+tree_clf = DecisionTreeClassifier(max_depth=2)
+tree_clf.fit(X, y)
+print("Train set accuracy with Decision Tree: {:.2f}".format(tree_clf.score(X,y)))
+#transfer to a decision tree graph
+export_graphviz(
+ tree_clf,
+ out_file="DataFiles/ride.dot",
+ rounded=True,
+ filled=True
+)
+cmd = 'dot -Tpng DataFiles/cancer.dot -o DataFiles/cancer.png'
+os.system(cmd)
-
Using the Voting Classifier
+Computing the Gini Factor
+ +The above functions (gini, entropy and misclassification error) are +important components of the so-called CART algorithm. We will discuss +this algorithm below after we have discussed the information gain +algorithm ID3. +
+ +In the example here we have converted all our attributes into numerical values \( 0,1,2 \) etc.
+from sklearn.model_selection import train_test_split
-from sklearn.datasets import make_moons
+ # Split a dataset based on an attribute and an attribute value
+def test_split(index, value, dataset):
+ left, right = list(), list()
+ for row in dataset:
+ if row[index] < value:
+ left.append(row)
+ else:
+ right.append(row)
+ return left, right
+
+# Calculate the Gini index for a split dataset
+def gini_index(groups, classes):
+ # count all samples at split point
+ n_instances = float(sum([len(group) for group in groups]))
+ # sum weighted Gini index for each group
+ gini = 0.0
+ for group in groups:
+ size = float(len(group))
+ # avoid divide by zero
+ if size == 0:
+ continue
+ score = 0.0
+ # score the group based on the score for each class
+ for class_val in classes:
+ p = [row[-1] for row in group].count(class_val) / size
+ score += p * p
+ # weight the group score by its relative size
+ gini += (1.0 - score) * (size / n_instances)
+ return gini
-X, y = make_moons(n_samples=500, noise=0.30, random_state=42)
-X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42)
+# Select the best split point for a dataset
+def get_split(dataset):
+ class_values = list(set(row[-1] for row in dataset))
+ b_index, b_value, b_score, b_groups = 999, 999, 999, None
+ for index in range(len(dataset[0])-1):
+ for row in dataset:
+ groups = test_split(index, row[index], dataset)
+ gini = gini_index(groups, class_values)
+ print('X%d < %.3f Gini=%.3f' % ((index+1), row[index], gini))
+ if gini < b_score:
+ b_index, b_value, b_score, b_groups = index, row[index], gini, groups
+ return {'index':b_index, 'value':b_value, 'groups':b_groups}
+
+dataset = [[0,0,0,0,0],
+ [0,0,0,1,1],
+ [1,0,0,0,1],
+ [2,1,0,0,1],
+ [2,2,1,0,1],
+ [2,2,1,1,0],
+ [1,2,1,1,1],
+ [0,1,0,0,0],
+ [0,2,1,0,1],
+ [2,1,1,0,1],
+ [0,1,1,1,1],
+ [1,1,0,1,1],
+ [1,0,1,0,1],
+ [2,1,0,1,0]]
-from sklearn.ensemble import RandomForestClassifier
-from sklearn.ensemble import VotingClassifier
-from sklearn.linear_model import LogisticRegression
-from sklearn.svm import SVC
-
-log_clf = LogisticRegression(solver="liblinear", random_state=42)
-rnd_clf = RandomForestClassifier(n_estimators=10, random_state=42)
-svm_clf = SVC(gamma="auto", random_state=42)
-
-voting_clf = VotingClassifier(
- estimators=[('lr', log_clf), ('rf', rnd_clf), ('svc', svm_clf)],
- voting='hard')
-
-voting_clf.fit(X_train, y_train)
-
-from sklearn.metrics import accuracy_score
-
-for clf in (log_clf, rnd_clf, svm_clf, voting_clf):
- clf.fit(X_train, y_train)
- y_pred = clf.predict(X_test)
- print(clf.__class__.__name__, accuracy_score(y_test, y_pred))
-
-log_clf = LogisticRegression(solver="liblinear", random_state=42)
-rnd_clf = RandomForestClassifier(n_estimators=10, random_state=42)
-svm_clf = SVC(gamma="auto", probability=True, random_state=42)
-
-voting_clf = VotingClassifier(
- estimators=[('lr', log_clf), ('rf', rnd_clf), ('svc', svm_clf)],
- voting='soft')
-voting_clf.fit(X_train, y_train)
-
-from sklearn.metrics import accuracy_score
-
-for clf in (log_clf, rnd_clf, svm_clf, voting_clf):
- clf.fit(X_train, y_train)
- y_pred = clf.predict(X_test)
- print(clf.__class__.__name__, accuracy_score(y_test, y_pred))
+split = get_split(dataset)
+print('Split: [X%d < %.3f]' % ((split['index']+1), split['value']))
-
Please, not the moons again! Voting and Bagging
+Entropy and the ID3 algorithm
+The ID3 algorithm learns decision trees by constructing +them in a top down way, beginning with the question which attribute should be tested at the root of the tree? +
- -from sklearn.model_selection import train_test_split
-from sklearn.datasets import make_moons
+
+
+The ID3 algorithm selects which attribute to test at each node in the
+tree.
+
-X, y = make_moons(n_samples=500, noise=0.30, random_state=42)
-X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42)
-from sklearn.ensemble import RandomForestClassifier
-from sklearn.ensemble import VotingClassifier
-from sklearn.linear_model import LogisticRegression
-from sklearn.svm import SVC
+We would like to select the attribute that is most useful for classifying
+examples.
+
-log_clf = LogisticRegression(random_state=42)
-rnd_clf = RandomForestClassifier(random_state=42)
-svm_clf = SVC(random_state=42)
+What is a good quantitative measure of the worth of an attribute?
-voting_clf = VotingClassifier(
- estimators=[('lr', log_clf), ('rf', rnd_clf), ('svc', svm_clf)],
- voting='hard')
-voting_clf.fit(X_train, y_train)
-
-from sklearn.metrics import accuracy_score
-
-for clf in (log_clf, rnd_clf, svm_clf, voting_clf):
- clf.fit(X_train, y_train)
- y_pred = clf.predict(X_test)
- print(clf.__class__.__name__, accuracy_score(y_test, y_pred))
-
-log_clf = LogisticRegression(random_state=42)
-rnd_clf = RandomForestClassifier(random_state=42)
-svm_clf = SVC(probability=True, random_state=42)
-
-voting_clf = VotingClassifier(
- estimators=[('lr', log_clf), ('rf', rnd_clf), ('svc', svm_clf)],
- voting='soft')
-voting_clf.fit(X_train, y_train)
-
-from sklearn.metrics import accuracy_score
-
-for clf in (log_clf, rnd_clf, svm_clf, voting_clf):
- clf.fit(X_train, y_train)
- y_pred = clf.predict(X_test)
- print(clf.__class__.__name__, accuracy_score(y_test, y_pred))
-
-Information gain measures how well a given attribute separates the +training examples according to their target classification. +
+The ID3 algorithm uses this information gain measure to select among the candidate +attributes at each step while growing the tree. +
diff --git a/doc/pub/week44/html/._week44-bs046.html b/doc/pub/week44/html/._week44-bs046.html index b2317f60c..c297e5d6f 100644 --- a/doc/pub/week44/html/._week44-bs046.html +++ b/doc/pub/week44/html/._week44-bs046.html @@ -115,67 +115,6 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'the-cart-algorithm-for-regression'), - ('Computing the Gini index', 2, None, 'computing-the-gini-index'), - ('Simple Python Code to read in Data and perform Classification', - 2, - None, - 'simple-python-code-to-read-in-data-and-perform-classification'), - ('Computing the Gini Factor', - 2, - None, - 'computing-the-gini-factor'), - ('Entropy and the ID3 algorithm', - 2, - None, - 'entropy-and-the-id3-algorithm'), - ('Cancer Data again now with Decision Trees and other Methods', - 2, - None, - 'cancer-data-again-now-with-decision-trees-and-other-methods'), - ('Another example, the moons again', - 2, - None, - 'another-example-the-moons-again'), - ('Playing around with regions', - 2, - None, - 'playing-around-with-regions'), - ('Regression trees', 2, None, 'regression-trees'), - ('Final regressor code', 2, None, 'final-regressor-code'), - ('Pros and cons of trees, pros', - 2, - None, - 'pros-and-cons-of-trees-pros'), - ('Disadvantages', 2, None, 'disadvantages'), - ('Ensemble Methods: From a Single Tree to Many Trees and Extreme ' - 'Boosting, Meet the Jungle of Methods', - 2, - None, - 'ensemble-methods-from-a-single-tree-to-many-trees-and-extreme-boosting-meet-the-jungle-of-methods'), - ('An Overview of Ensemble Methods', - 2, - None, - 'an-overview-of-ensemble-methods'), - ('Bagging', 2, None, 'bagging'), - ('More bagging', 2, None, 'more-bagging'), - ('Simple Voting Example, head or tail', - 2, - None, - 'simple-voting-example-head-or-tail'), - ('Using the Voting Classifier', - 2, - None, - 'using-the-voting-classifier'), - ('Please, not the moons again! Voting and Bagging', - 2, - None, - 'please-not-the-moons-again-voting-and-bagging'), - ('Bagging Examples', 2, None, 'bagging-examples'), - ('Making your own Bootstrap: Changing the Level of the Decision ' - 'Tree', - 2, - None, - 'making-your-own-bootstrap-changing-the-level-of-the-decision-tree'), ('Why binary splits?', 2, None, 'why-binary-splits'), ('Computing a Tree using the Gini Index', 2, @@ -250,6 +189,70 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d 'playing-around-with-regions'), ('Regression trees', 2, None, 'regression-trees'), ('Final regressor code', 2, None, 'final-regressor-code'), + ('Example: Computing the Gini index', + 2, + None, + 'example-computing-the-gini-index'), + ('Simple Python Code to read in Data and perform Classification', + 2, + None, + 'simple-python-code-to-read-in-data-and-perform-classification'), + ('Computing the Gini Factor', + 2, + None, + 'computing-the-gini-factor'), + ('Entropy and the ID3 algorithm', + 2, + None, + 'entropy-and-the-id3-algorithm'), + ('Cancer Data again now with Decision Trees and other Methods', + 2, + None, + 'cancer-data-again-now-with-decision-trees-and-other-methods'), + ('Another example, the moons again', + 2, + None, + 'another-example-the-moons-again'), + ('Playing around with regions', + 2, + None, + 'playing-around-with-regions'), + ('Regression trees', 2, None, 'regression-trees'), + ('Final regressor code', 2, None, 'final-regressor-code'), + ('Pros and cons of trees, pros', + 2, + None, + 'pros-and-cons-of-trees-pros'), + ('Disadvantages', 2, None, 'disadvantages'), + ('Ensemble Methods: From a Single Tree to Many Trees and Extreme ' + 'Boosting, Meet the Jungle of Methods', + 2, + None, + 'ensemble-methods-from-a-single-tree-to-many-trees-and-extreme-boosting-meet-the-jungle-of-methods'), + ('An Overview of Ensemble Methods', + 2, + None, + 'an-overview-of-ensemble-methods'), + ('Bagging', 2, None, 'bagging'), + ('More bagging', 2, None, 'more-bagging'), + ('Simple Voting Example, head or tail', + 2, + None, + 'simple-voting-example-head-or-tail'), + ('Using the Voting Classifier', + 2, + None, + 'using-the-voting-classifier'), + ('Please, not the moons again! Voting and Bagging', + 2, + None, + 'please-not-the-moons-again-voting-and-bagging'), + ('Bagging Examples', 2, None, 'bagging-examples'), + ('Making your own Bootstrap: Changing the Level of the Decision ' + 'Tree', + 2, + None, + 'making-your-own-bootstrap-changing-the-level-of-the-decision-tree'), ('Why Voting?', 2, None, 'why-voting'), ('Tossing coins', 2, None, 'tossing-coins'), ('Standard imports first', 2, None, 'standard-imports-first'), @@ -327,55 +330,55 @@ MathJax.Hub.Config({
-
Bagging Examples
- +Cancer Data again now with Decision Trees and other Methods
from sklearn.ensemble import BaggingClassifier
+ import matplotlib.pyplot as plt
+import numpy as np
+from sklearn.model_selection import train_test_split
+from sklearn.datasets import load_breast_cancer
+from sklearn.svm import SVC
+from sklearn.linear_model import LogisticRegression
from sklearn.tree import DecisionTreeClassifier
-bag_clf = BaggingClassifier(
- DecisionTreeClassifier(random_state=42), n_estimators=500,
- max_samples=100, bootstrap=True, n_jobs=-1, random_state=42)
-bag_clf.fit(X_train, y_train)
-y_pred = bag_clf.predict(X_test)
-
-from sklearn.metrics import accuracy_score
-print(accuracy_score(y_test, y_pred))
-
-tree_clf = DecisionTreeClassifier(random_state=42)
-tree_clf.fit(X_train, y_train)
-y_pred_tree = tree_clf.predict(X_test)
-print(accuracy_score(y_test, y_pred_tree))
-
-from matplotlib.colors import ListedColormap
+# Load the data
+cancer = load_breast_cancer()
-def plot_decision_boundary(clf, X, y, axes=[-1.5, 2.5, -1, 1.5], alpha=0.5, contour=True):
- x1s = np.linspace(axes[0], axes[1], 100)
- x2s = np.linspace(axes[2], axes[3], 100)
- x1, x2 = np.meshgrid(x1s, x2s)
- X_new = np.c_[x1.ravel(), x2.ravel()]
- y_pred = clf.predict(X_new).reshape(x1.shape)
- custom_cmap = ListedColormap(['#fafab0','#9898ff','#a0faa0'])
- plt.contourf(x1, x2, y_pred, alpha=0.3, cmap=custom_cmap)
- if contour:
- custom_cmap2 = ListedColormap(['#7d7d58','#4c4c7f','#507d50'])
- plt.contour(x1, x2, y_pred, cmap=custom_cmap2, alpha=0.8)
- plt.plot(X[:, 0][y==0], X[:, 1][y==0], "yo", alpha=alpha)
- plt.plot(X[:, 0][y==1], X[:, 1][y==1], "bs", alpha=alpha)
- plt.axis(axes)
- plt.xlabel(r"$x_1$", fontsize=18)
- plt.ylabel(r"$x_2$", fontsize=18, rotation=0)
-plt.figure(figsize=(11,4))
-plt.subplot(121)
-plot_decision_boundary(tree_clf, X, y)
-plt.title("Decision Tree", fontsize=14)
-plt.subplot(122)
-plot_decision_boundary(bag_clf, X, y)
-plt.title("Decision Trees with Bagging", fontsize=14)
-save_fig("baggingtree")
-plt.show()
+X_train, X_test, y_train, y_test = train_test_split(cancer.data,cancer.target,random_state=0)
+print(X_train.shape)
+print(X_test.shape)
+# Logistic Regression
+logreg = LogisticRegression(solver='lbfgs')
+logreg.fit(X_train, y_train)
+print("Test set accuracy with Logistic Regression: {:.2f}".format(logreg.score(X_test,y_test)))
+# Support vector machine
+svm = SVC(gamma='auto', C=100)
+svm.fit(X_train, y_train)
+print("Test set accuracy with SVM: {:.2f}".format(svm.score(X_test,y_test)))
+# Decision Trees
+deep_tree_clf = DecisionTreeClassifier(max_depth=None)
+deep_tree_clf.fit(X_train, y_train)
+print("Test set accuracy with Decision Trees: {:.2f}".format(deep_tree_clf.score(X_test,y_test)))
+#now scale the data
+from sklearn.preprocessing import StandardScaler
+scaler = StandardScaler()
+scaler.fit(X_train)
+X_train_scaled = scaler.transform(X_train)
+X_test_scaled = scaler.transform(X_test)
+# Logistic Regression
+logreg.fit(X_train_scaled, y_train)
+print("Test set accuracy Logistic Regression with scaled data: {:.2f}".format(logreg.score(X_test_scaled,y_test)))
+# Support Vector Machine
+svm.fit(X_train_scaled, y_train)
+print("Test set accuracy SVM with scaled data: {:.2f}".format(logreg.score(X_test_scaled,y_test)))
+# Decision Trees
+deep_tree_clf.fit(X_train_scaled, y_train)
+print("Test set accuracy with Decision Trees and scaled data: {:.2f}".format(deep_tree_clf.score(X_test_scaled,y_test)))
-
Making your own Bootstrap: Changing the Level of the Decision Tree
- -Let us bring up our good old boostrap example from the linear regression lectures. We change the linerar regression algorithm with -a decision tree wth different depths and perform a bootstrap aggregate (in this case we perform as many bootstraps as data points \( n \)). -
+Another example, the moons again
import matplotlib.pyplot as plt
+ from __future__ import division, print_function, unicode_literals
+
+# Common imports
import numpy as np
-from sklearn.model_selection import train_test_split
-from sklearn.pipeline import make_pipeline
-from sklearn.utils import resample
-from sklearn.tree import DecisionTreeRegressor
+import os
-n = 100
-n_boostraps = 100
-maxdepth = 8
+# to make this notebook's output stable across runs
+np.random.seed(42)
-# Make data set.
-x = np.linspace(-3, 3, n).reshape(-1, 1)
-y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)
-error = np.zeros(maxdepth)
-bias = np.zeros(maxdepth)
-variance = np.zeros(maxdepth)
-polydegree = np.zeros(maxdepth)
-X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.2)
+# To plot pretty figures
+import matplotlib
+import matplotlib.pyplot as plt
+from matplotlib.colors import ListedColormap
+plt.rcParams['axes.labelsize'] = 14
+plt.rcParams['xtick.labelsize'] = 12
+plt.rcParams['ytick.labelsize'] = 12
-from sklearn.preprocessing import StandardScaler
-scaler = StandardScaler()
-scaler.fit(X_train)
-X_train_scaled = scaler.transform(X_train)
-X_test_scaled = scaler.transform(X_test)
-# we produce a simple tree first as benchmark
-simpletree = DecisionTreeRegressor(max_depth=3)
-simpletree.fit(X_train_scaled, y_train)
-simpleprediction = simpletree.predict(X_test_scaled)
-for degree in range(1,maxdepth):
- model = DecisionTreeRegressor(max_depth=degree)
- y_pred = np.empty((y_test.shape[0], n_boostraps))
- for i in range(n_boostraps):
- x_, y_ = resample(X_train_scaled, y_train)
- model.fit(x_, y_)
- y_pred[:, i] = model.predict(X_test_scaled)#.ravel()
+from sklearn.svm import SVC
+from sklearn import datasets
+from sklearn.tree import DecisionTreeClassifier
+from sklearn.datasets import make_moons
+from sklearn.tree import export_graphviz
- polydegree[degree] = degree
- error[degree] = np.mean( np.mean((y_test - y_pred)**2, axis=1, keepdims=True) )
- bias[degree] = np.mean( (y_test - np.mean(y_pred, axis=1, keepdims=True))**2 )
- variance[degree] = np.mean( np.var(y_pred, axis=1, keepdims=True) )
- print('Polynomial degree:', degree)
- print('Error:', error[degree])
- print('Bias^2:', bias[degree])
- print('Var:', variance[degree])
- print('{} >= {} + {} = {}'.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree]))
-
-mse_simpletree= np.mean( np.mean((y_test - simpleprediction)**2)
-print(mse_simpletree)
-plt.xlim(1,maxdepth)
-plt.plot(polydegree, error, label='MSE')
-plt.plot(polydegree, bias, label='bias')
-plt.plot(polydegree, variance, label='Variance')
-plt.legend()
-save_fig("baggingboot")
+Xm, ym = make_moons(n_samples=100, noise=0.25, random_state=53)
+
+deep_tree_clf1 = DecisionTreeClassifier(random_state=42)
+deep_tree_clf2 = DecisionTreeClassifier(min_samples_leaf=4, random_state=42)
+deep_tree_clf1.fit(Xm, ym)
+deep_tree_clf2.fit(Xm, ym)
+
+
+def plot_decision_boundary(clf, X, y, axes=[0, 7.5, 0, 3], iris=True, legend=False, plot_training=True):
+ x1s = np.linspace(axes[0], axes[1], 100)
+ x2s = np.linspace(axes[2], axes[3], 100)
+ x1, x2 = np.meshgrid(x1s, x2s)
+ X_new = np.c_[x1.ravel(), x2.ravel()]
+ y_pred = clf.predict(X_new).reshape(x1.shape)
+ custom_cmap = ListedColormap(['#fafab0','#9898ff','#a0faa0'])
+ plt.contourf(x1, x2, y_pred, alpha=0.3, cmap=custom_cmap)
+ if not iris:
+ custom_cmap2 = ListedColormap(['#7d7d58','#4c4c7f','#507d50'])
+ plt.contour(x1, x2, y_pred, cmap=custom_cmap2, alpha=0.8)
+ if plot_training:
+ plt.plot(X[:, 0][y==0], X[:, 1][y==0], "yo", label="Iris-Setosa")
+ plt.plot(X[:, 0][y==1], X[:, 1][y==1], "bs", label="Iris-Versicolor")
+ plt.plot(X[:, 0][y==2], X[:, 1][y==2], "g^", label="Iris-Virginica")
+ plt.axis(axes)
+ if iris:
+ plt.xlabel("Petal length", fontsize=14)
+ plt.ylabel("Petal width", fontsize=14)
+ else:
+ plt.xlabel(r"$x_1$", fontsize=18)
+ plt.ylabel(r"$x_2$", fontsize=18, rotation=0)
+ if legend:
+ plt.legend(loc="lower right", fontsize=14)
+plt.figure(figsize=(11, 4))
+plt.subplot(121)
+plot_decision_boundary(deep_tree_clf1, Xm, ym, axes=[-1.5, 2.5, -1, 1.5], iris=False)
+plt.title("No restrictions", fontsize=16)
+plt.subplot(122)
+plot_decision_boundary(deep_tree_clf2, Xm, ym, axes=[-1.5, 2.5, -1, 1.5], iris=False)
+plt.title("min_samples_leaf = {}".format(deep_tree_clf2.min_samples_leaf), fontsize=14)
plt.show()
-
Why binary splits?
+Playing around with regions
+ + +np.random.seed(6)
+Xs = np.random.rand(100, 2) - 0.5
+ys = (Xs[:, 0] > 0).astype(np.float32) * 2
+
+angle = np.pi/4
+rotation_matrix = np.array([[np.cos(angle), -np.sin(angle)], [np.sin(angle), np.cos(angle)]])
+Xsr = Xs.dot(rotation_matrix)
+
+tree_clf_s = DecisionTreeClassifier(random_state=42)
+tree_clf_s.fit(Xs, ys)
+tree_clf_sr = DecisionTreeClassifier(random_state=42)
+tree_clf_sr.fit(Xsr, ys)
+
+plt.figure(figsize=(11, 4))
+plt.subplot(121)
+plot_decision_boundary(tree_clf_s, Xs, ys, axes=[-0.7, 0.7, -0.7, 0.7], iris=False)
+plt.subplot(122)
+plot_decision_boundary(tree_clf_sr, Xsr, ys, axes=[-0.7, 0.7, -0.7, 0.7], iris=False)
+
+plt.show()
+
+It is custom to split to a tree uising binary splits. The reason is -that multiway splits fragment the data too quickly, leaving -insufficient data at the next level down. Multiway splits can be -achieved by a series of binary split and this is normally preferred. -
diff --git a/doc/pub/week44/html/._week44-bs049.html b/doc/pub/week44/html/._week44-bs049.html index 70ed4f4a1..49b18213e 100644 --- a/doc/pub/week44/html/._week44-bs049.html +++ b/doc/pub/week44/html/._week44-bs049.html @@ -115,67 +115,6 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'the-cart-algorithm-for-regression'), - ('Computing the Gini index', 2, None, 'computing-the-gini-index'), - ('Simple Python Code to read in Data and perform Classification', - 2, - None, - 'simple-python-code-to-read-in-data-and-perform-classification'), - ('Computing the Gini Factor', - 2, - None, - 'computing-the-gini-factor'), - ('Entropy and the ID3 algorithm', - 2, - None, - 'entropy-and-the-id3-algorithm'), - ('Cancer Data again now with Decision Trees and other Methods', - 2, - None, - 'cancer-data-again-now-with-decision-trees-and-other-methods'), - ('Another example, the moons again', - 2, - None, - 'another-example-the-moons-again'), - ('Playing around with regions', - 2, - None, - 'playing-around-with-regions'), - ('Regression trees', 2, None, 'regression-trees'), - ('Final regressor code', 2, None, 'final-regressor-code'), - ('Pros and cons of trees, pros', - 2, - None, - 'pros-and-cons-of-trees-pros'), - ('Disadvantages', 2, None, 'disadvantages'), - ('Ensemble Methods: From a Single Tree to Many Trees and Extreme ' - 'Boosting, Meet the Jungle of Methods', - 2, - None, - 'ensemble-methods-from-a-single-tree-to-many-trees-and-extreme-boosting-meet-the-jungle-of-methods'), - ('An Overview of Ensemble Methods', - 2, - None, - 'an-overview-of-ensemble-methods'), - ('Bagging', 2, None, 'bagging'), - ('More bagging', 2, None, 'more-bagging'), - ('Simple Voting Example, head or tail', - 2, - None, - 'simple-voting-example-head-or-tail'), - ('Using the Voting Classifier', - 2, - None, - 'using-the-voting-classifier'), - ('Please, not the moons again! Voting and Bagging', - 2, - None, - 'please-not-the-moons-again-voting-and-bagging'), - ('Bagging Examples', 2, None, 'bagging-examples'), - ('Making your own Bootstrap: Changing the Level of the Decision ' - 'Tree', - 2, - None, - 'making-your-own-bootstrap-changing-the-level-of-the-decision-tree'), ('Why binary splits?', 2, None, 'why-binary-splits'), ('Computing a Tree using the Gini Index', 2, @@ -250,6 +189,70 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d 'playing-around-with-regions'), ('Regression trees', 2, None, 'regression-trees'), ('Final regressor code', 2, None, 'final-regressor-code'), + ('Example: Computing the Gini index', + 2, + None, + 'example-computing-the-gini-index'), + ('Simple Python Code to read in Data and perform Classification', + 2, + None, + 'simple-python-code-to-read-in-data-and-perform-classification'), + ('Computing the Gini Factor', + 2, + None, + 'computing-the-gini-factor'), + ('Entropy and the ID3 algorithm', + 2, + None, + 'entropy-and-the-id3-algorithm'), + ('Cancer Data again now with Decision Trees and other Methods', + 2, + None, + 'cancer-data-again-now-with-decision-trees-and-other-methods'), + ('Another example, the moons again', + 2, + None, + 'another-example-the-moons-again'), + ('Playing around with regions', + 2, + None, + 'playing-around-with-regions'), + ('Regression trees', 2, None, 'regression-trees'), + ('Final regressor code', 2, None, 'final-regressor-code'), + ('Pros and cons of trees, pros', + 2, + None, + 'pros-and-cons-of-trees-pros'), + ('Disadvantages', 2, None, 'disadvantages'), + ('Ensemble Methods: From a Single Tree to Many Trees and Extreme ' + 'Boosting, Meet the Jungle of Methods', + 2, + None, + 'ensemble-methods-from-a-single-tree-to-many-trees-and-extreme-boosting-meet-the-jungle-of-methods'), + ('An Overview of Ensemble Methods', + 2, + None, + 'an-overview-of-ensemble-methods'), + ('Bagging', 2, None, 'bagging'), + ('More bagging', 2, None, 'more-bagging'), + ('Simple Voting Example, head or tail', + 2, + None, + 'simple-voting-example-head-or-tail'), + ('Using the Voting Classifier', + 2, + None, + 'using-the-voting-classifier'), + ('Please, not the moons again! Voting and Bagging', + 2, + None, + 'please-not-the-moons-again-voting-and-bagging'), + ('Bagging Examples', 2, None, 'bagging-examples'), + ('Making your own Bootstrap: Changing the Level of the Decision ' + 'Tree', + 2, + None, + 'making-your-own-bootstrap-changing-the-level-of-the-decision-tree'), ('Why Voting?', 2, None, 'why-voting'), ('Tossing coins', 2, None, 'tossing-coins'), ('Standard imports first', 2, None, 'standard-imports-first'), @@ -327,55 +330,55 @@ MathJax.Hub.Config({
-
Computing a Tree using the Gini Index
+Regression trees
+ + +# Quadratic training set + noise
+np.random.seed(42)
+m = 200
+X = np.random.rand(m, 1)
+y = 4 * (X - 0.5) ** 2
+y = y + np.random.randn(m, 1) / 10
+
+from sklearn.tree import DecisionTreeRegressor
+
+tree_reg = DecisionTreeRegressor(max_depth=2, random_state=42)
+tree_reg.fit(X, y)
+
+Consider the following example with attributes/features and two -possible outcomes (classes) for each attribute. Assume we wish to find some -correlations between the average grade of a student as function of the -number of hours studied and hours slept. We want also to correlate the -grade in a given course with the general trend, whether the students -recently has gotten grades below average or above. -
-We have three features/attributes
--
-
-
diff --git a/doc/pub/week44/html/._week44-bs050.html b/doc/pub/week44/html/._week44-bs050.html
index c3dd07d40..197d51e62 100644
--- a/doc/pub/week44/html/._week44-bs050.html
+++ b/doc/pub/week44/html/._week44-bs050.html
@@ -115,67 +115,6 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d
2,
None,
'the-cart-algorithm-for-regression'),
- ('Computing the Gini index', 2, None, 'computing-the-gini-index'),
- ('Simple Python Code to read in Data and perform Classification',
- 2,
- None,
- 'simple-python-code-to-read-in-data-and-perform-classification'),
- ('Computing the Gini Factor',
- 2,
- None,
- 'computing-the-gini-factor'),
- ('Entropy and the ID3 algorithm',
- 2,
- None,
- 'entropy-and-the-id3-algorithm'),
- ('Cancer Data again now with Decision Trees and other Methods',
- 2,
- None,
- 'cancer-data-again-now-with-decision-trees-and-other-methods'),
- ('Another example, the moons again',
- 2,
- None,
- 'another-example-the-moons-again'),
- ('Playing around with regions',
- 2,
- None,
- 'playing-around-with-regions'),
- ('Regression trees', 2, None, 'regression-trees'),
- ('Final regressor code', 2, None, 'final-regressor-code'),
- ('Pros and cons of trees, pros',
- 2,
- None,
- 'pros-and-cons-of-trees-pros'),
- ('Disadvantages', 2, None, 'disadvantages'),
- ('Ensemble Methods: From a Single Tree to Many Trees and Extreme '
- 'Boosting, Meet the Jungle of Methods',
- 2,
- None,
- 'ensemble-methods-from-a-single-tree-to-many-trees-and-extreme-boosting-meet-the-jungle-of-methods'),
- ('An Overview of Ensemble Methods',
- 2,
- None,
- 'an-overview-of-ensemble-methods'),
- ('Bagging', 2, None, 'bagging'),
- ('More bagging', 2, None, 'more-bagging'),
- ('Simple Voting Example, head or tail',
- 2,
- None,
- 'simple-voting-example-head-or-tail'),
- ('Using the Voting Classifier',
- 2,
- None,
- 'using-the-voting-classifier'),
- ('Please, not the moons again! Voting and Bagging',
- 2,
- None,
- 'please-not-the-moons-again-voting-and-bagging'),
- ('Bagging Examples', 2, None, 'bagging-examples'),
- ('Making your own Bootstrap: Changing the Level of the Decision '
- 'Tree',
- 2,
- None,
- 'making-your-own-bootstrap-changing-the-level-of-the-decision-tree'),
('Why binary splits?', 2, None, 'why-binary-splits'),
('Computing a Tree using the Gini Index',
2,
@@ -250,6 +189,70 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d
'playing-around-with-regions'),
('Regression trees', 2, None, 'regression-trees'),
('Final regressor code', 2, None, 'final-regressor-code'),
+ ('Example: Computing the Gini index',
+ 2,
+ None,
+ 'example-computing-the-gini-index'),
+ ('Simple Python Code to read in Data and perform Classification',
+ 2,
+ None,
+ 'simple-python-code-to-read-in-data-and-perform-classification'),
+ ('Computing the Gini Factor',
+ 2,
+ None,
+ 'computing-the-gini-factor'),
+ ('Entropy and the ID3 algorithm',
+ 2,
+ None,
+ 'entropy-and-the-id3-algorithm'),
+ ('Cancer Data again now with Decision Trees and other Methods',
+ 2,
+ None,
+ 'cancer-data-again-now-with-decision-trees-and-other-methods'),
+ ('Another example, the moons again',
+ 2,
+ None,
+ 'another-example-the-moons-again'),
+ ('Playing around with regions',
+ 2,
+ None,
+ 'playing-around-with-regions'),
+ ('Regression trees', 2, None, 'regression-trees'),
+ ('Final regressor code', 2, None, 'final-regressor-code'),
+ ('Pros and cons of trees, pros',
+ 2,
+ None,
+ 'pros-and-cons-of-trees-pros'),
+ ('Disadvantages', 2, None, 'disadvantages'),
+ ('Ensemble Methods: From a Single Tree to Many Trees and Extreme '
+ 'Boosting, Meet the Jungle of Methods',
+ 2,
+ None,
+ 'ensemble-methods-from-a-single-tree-to-many-trees-and-extreme-boosting-meet-the-jungle-of-methods'),
+ ('An Overview of Ensemble Methods',
+ 2,
+ None,
+ 'an-overview-of-ensemble-methods'),
+ ('Bagging', 2, None, 'bagging'),
+ ('More bagging', 2, None, 'more-bagging'),
+ ('Simple Voting Example, head or tail',
+ 2,
+ None,
+ 'simple-voting-example-head-or-tail'),
+ ('Using the Voting Classifier',
+ 2,
+ None,
+ 'using-the-voting-classifier'),
+ ('Please, not the moons again! Voting and Bagging',
+ 2,
+ None,
+ 'please-not-the-moons-again-voting-and-bagging'),
+ ('Bagging Examples', 2, None, 'bagging-examples'),
+ ('Making your own Bootstrap: Changing the Level of the Decision '
+ 'Tree',
+ 2,
+ None,
+ 'making-your-own-bootstrap-changing-the-level-of-the-decision-tree'),
('Why Voting?', 2, None, 'why-voting'),
('Tossing coins', 2, None, 'tossing-coins'),
('Standard imports first', 2, None, 'standard-imports-first'),
@@ -327,55 +330,55 @@ MathJax.Hub.Config({
-
The Table
+Final regressor code
+ + +from sklearn.tree import DecisionTreeRegressor
+
+tree_reg1 = DecisionTreeRegressor(random_state=42, max_depth=2)
+tree_reg2 = DecisionTreeRegressor(random_state=42, max_depth=3)
+tree_reg1.fit(X, y)
+tree_reg2.fit(X, y)
+
+def plot_regression_predictions(tree_reg, X, y, axes=[0, 1, -0.2, 1], ylabel="$y$"):
+ x1 = np.linspace(axes[0], axes[1], 500).reshape(-1, 1)
+ y_pred = tree_reg.predict(x1)
+ plt.axis(axes)
+ plt.xlabel("$x_1$", fontsize=18)
+ if ylabel:
+ plt.ylabel(ylabel, fontsize=18, rotation=0)
+ plt.plot(X, y, "b.")
+ plt.plot(x1, y_pred, "r.-", linewidth=2, label=r"$\hat{y}$")
+
+plt.figure(figsize=(11, 4))
+plt.subplot(121)
+plot_regression_predictions(tree_reg1, X, y)
+for split, style in ((0.1973, "k-"), (0.0917, "k--"), (0.7718, "k--")):
+ plt.plot([split, split], [-0.2, 1], style, linewidth=2)
+plt.text(0.21, 0.65, "Depth=0", fontsize=15)
+plt.text(0.01, 0.2, "Depth=1", fontsize=13)
+plt.text(0.65, 0.8, "Depth=1", fontsize=13)
+plt.legend(loc="upper center", fontsize=18)
+plt.title("max_depth=2", fontsize=14)
+
+plt.subplot(122)
+plot_regression_predictions(tree_reg2, X, y, ylabel=None)
+for split, style in ((0.1973, "k-"), (0.0917, "k--"), (0.7718, "k--")):
+ plt.plot([split, split], [-0.2, 1], style, linewidth=2)
+for split in (0.0458, 0.1298, 0.2873, 0.9040):
+ plt.plot([split, split], [-0.2, 1], "k:", linewidth=1)
+plt.text(0.3, 0.5, "Depth=2", fontsize=13)
+plt.title("max_depth=3", fontsize=14)
+
+plt.show()
+
+tree_reg1 = DecisionTreeRegressor(random_state=42)
+tree_reg2 = DecisionTreeRegressor(random_state=42, min_samples_leaf=10)
+tree_reg1.fit(X, y)
+tree_reg2.fit(X, y)
+
+x1 = np.linspace(0, 1, 500).reshape(-1, 1)
+y_pred1 = tree_reg1.predict(x1)
+y_pred2 = tree_reg2.predict(x1)
+
+plt.figure(figsize=(11, 4))
+
+plt.subplot(121)
+plt.plot(X, y, "b.")
+plt.plot(x1, y_pred1, "r.-", linewidth=2, label=r"$\hat{y}$")
+plt.axis([0, 1, -0.2, 1.1])
+plt.xlabel("$x_1$", fontsize=18)
+plt.ylabel("$y$", fontsize=18, rotation=0)
+plt.legend(loc="upper center", fontsize=18)
+plt.title("No restrictions", fontsize=14)
+
+plt.subplot(122)
+plt.plot(X, y, "b.")
+plt.plot(x1, y_pred2, "r.-", linewidth=2, label=r"$\hat{y}$")
+plt.axis([0, 1, -0.2, 1.1])
+plt.xlabel("$x_1$", fontsize=18)
+plt.title("min_samples_leaf={}".format(tree_reg2.min_samples_leaf), fontsize=14)
+
+plt.show()
+
+| Grade Trend | Hours slept | Hours Studied | Grade |
| Above | Low | High | Above |
| Below | High | Low | Below |
| Above | Low | High | Above |
| Above | High | High | Above |
| Below | Low | High | Below |
| Above | Low | Low | Below |
| Below | High | High | Below |
| Below | Low | High | Below |
| Above | Low | Low | Below |
| Above | High | High | Above |
diff --git a/doc/pub/week44/html/._week44-bs051.html b/doc/pub/week44/html/._week44-bs051.html index 1db44b1e0..f415b2bb8 100644 --- a/doc/pub/week44/html/._week44-bs051.html +++ b/doc/pub/week44/html/._week44-bs051.html @@ -115,67 +115,6 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'the-cart-algorithm-for-regression'), - ('Computing the Gini index', 2, None, 'computing-the-gini-index'), - ('Simple Python Code to read in Data and perform Classification', - 2, - None, - 'simple-python-code-to-read-in-data-and-perform-classification'), - ('Computing the Gini Factor', - 2, - None, - 'computing-the-gini-factor'), - ('Entropy and the ID3 algorithm', - 2, - None, - 'entropy-and-the-id3-algorithm'), - ('Cancer Data again now with Decision Trees and other Methods', - 2, - None, - 'cancer-data-again-now-with-decision-trees-and-other-methods'), - ('Another example, the moons again', - 2, - None, - 'another-example-the-moons-again'), - ('Playing around with regions', - 2, - None, - 'playing-around-with-regions'), - ('Regression trees', 2, None, 'regression-trees'), - ('Final regressor code', 2, None, 'final-regressor-code'), - ('Pros and cons of trees, pros', - 2, - None, - 'pros-and-cons-of-trees-pros'), - ('Disadvantages', 2, None, 'disadvantages'), - ('Ensemble Methods: From a Single Tree to Many Trees and Extreme ' - 'Boosting, Meet the Jungle of Methods', - 2, - None, - 'ensemble-methods-from-a-single-tree-to-many-trees-and-extreme-boosting-meet-the-jungle-of-methods'), - ('An Overview of Ensemble Methods', - 2, - None, - 'an-overview-of-ensemble-methods'), - ('Bagging', 2, None, 'bagging'), - ('More bagging', 2, None, 'more-bagging'), - ('Simple Voting Example, head or tail', - 2, - None, - 'simple-voting-example-head-or-tail'), - ('Using the Voting Classifier', - 2, - None, - 'using-the-voting-classifier'), - ('Please, not the moons again! Voting and Bagging', - 2, - None, - 'please-not-the-moons-again-voting-and-bagging'), - ('Bagging Examples', 2, None, 'bagging-examples'), - ('Making your own Bootstrap: Changing the Level of the Decision ' - 'Tree', - 2, - None, - 'making-your-own-bootstrap-changing-the-level-of-the-decision-tree'), ('Why binary splits?', 2, None, 'why-binary-splits'), ('Computing a Tree using the Gini Index', 2, @@ -250,6 +189,70 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d 'playing-around-with-regions'), ('Regression trees', 2, None, 'regression-trees'), ('Final regressor code', 2, None, 'final-regressor-code'), + ('Example: Computing the Gini index', + 2, + None, + 'example-computing-the-gini-index'), + ('Simple Python Code to read in Data and perform Classification', + 2, + None, + 'simple-python-code-to-read-in-data-and-perform-classification'), + ('Computing the Gini Factor', + 2, + None, + 'computing-the-gini-factor'), + ('Entropy and the ID3 algorithm', + 2, + None, + 'entropy-and-the-id3-algorithm'), + ('Cancer Data again now with Decision Trees and other Methods', + 2, + None, + 'cancer-data-again-now-with-decision-trees-and-other-methods'), + ('Another example, the moons again', + 2, + None, + 'another-example-the-moons-again'), + ('Playing around with regions', + 2, + None, + 'playing-around-with-regions'), + ('Regression trees', 2, None, 'regression-trees'), + ('Final regressor code', 2, None, 'final-regressor-code'), + ('Pros and cons of trees, pros', + 2, + None, + 'pros-and-cons-of-trees-pros'), + ('Disadvantages', 2, None, 'disadvantages'), + ('Ensemble Methods: From a Single Tree to Many Trees and Extreme ' + 'Boosting, Meet the Jungle of Methods', + 2, + None, + 'ensemble-methods-from-a-single-tree-to-many-trees-and-extreme-boosting-meet-the-jungle-of-methods'), + ('An Overview of Ensemble Methods', + 2, + None, + 'an-overview-of-ensemble-methods'), + ('Bagging', 2, None, 'bagging'), + ('More bagging', 2, None, 'more-bagging'), + ('Simple Voting Example, head or tail', + 2, + None, + 'simple-voting-example-head-or-tail'), + ('Using the Voting Classifier', + 2, + None, + 'using-the-voting-classifier'), + ('Please, not the moons again! Voting and Bagging', + 2, + None, + 'please-not-the-moons-again-voting-and-bagging'), + ('Bagging Examples', 2, None, 'bagging-examples'), + ('Making your own Bootstrap: Changing the Level of the Decision ' + 'Tree', + 2, + None, + 'making-your-own-bootstrap-changing-the-level-of-the-decision-tree'), ('Why Voting?', 2, None, 'why-voting'), ('Tossing coins', 2, None, 'tossing-coins'), ('Standard imports first', 2, None, 'standard-imports-first'), @@ -327,55 +330,55 @@ MathJax.Hub.Config({
-
Computing the various Gini Indices
+Example: Computing the Gini index
-In computations we will translate all classes into numbers. Being -these binary classes, they can easily be split into ones and zeros. +
The example we will look at is a classical one in many Machine +Learning applications. Based on various meteorological features, we +have several so-called attributes which decide whether we at the end +will do some outdoor activity like skiing, going for a bike ride etc +etc. The table here contains the feautures outlook, temperature, +humidity and wind. The target or output is whether we ride +(True=1) or whether we do something else that day (False=0). The +attributes for each feature are then sunny, overcast and rain for the +outlook, hot, cold and mild for temperature, high and normal for +humidity and weak and strong for wind.
-See handwritten notes for Thursday November 11
-The table here summarizes the various attributes and
+| Day | Outlook | Temperature | Humidity | Wind | Ride |
| 1 | Sunny | Hot | High | Weak | 0 |
| 2 | Sunny | Hot | High | Strong | 1 |
| 3 | Overcast | Hot | High | Weak | 1 |
| 4 | Rain | Mild | High | Weak | 1 |
| 5 | Rain | Cool | Normal | Weak | 1 |
| 6 | Rain | Cool | Normal | Strong | 0 |
| 7 | Overcast | Cool | Normal | Strong | 1 |
| 8 | Sunny | Mild | High | Weak | 0 |
| 9 | Sunny | Cool | Normal | Weak | 1 |
| 10 | Rain | Mild | Normal | Weak | 1 |
| 11 | Sunny | Mild | Normal | Strong | 1 |
| 12 | Overcast | Mild | High | Strong | 1 |
| 13 | Overcast | Hot | Normal | Weak | 1 |
| 14 | Rain | Mild | High | Strong | 0 |
diff --git a/doc/pub/week44/html/._week44-bs052.html b/doc/pub/week44/html/._week44-bs052.html index c1d4b6484..8076cd7a0 100644 --- a/doc/pub/week44/html/._week44-bs052.html +++ b/doc/pub/week44/html/._week44-bs052.html @@ -115,67 +115,6 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'the-cart-algorithm-for-regression'), - ('Computing the Gini index', 2, None, 'computing-the-gini-index'), - ('Simple Python Code to read in Data and perform Classification', - 2, - None, - 'simple-python-code-to-read-in-data-and-perform-classification'), - ('Computing the Gini Factor', - 2, - None, - 'computing-the-gini-factor'), - ('Entropy and the ID3 algorithm', - 2, - None, - 'entropy-and-the-id3-algorithm'), - ('Cancer Data again now with Decision Trees and other Methods', - 2, - None, - 'cancer-data-again-now-with-decision-trees-and-other-methods'), - ('Another example, the moons again', - 2, - None, - 'another-example-the-moons-again'), - ('Playing around with regions', - 2, - None, - 'playing-around-with-regions'), - ('Regression trees', 2, None, 'regression-trees'), - ('Final regressor code', 2, None, 'final-regressor-code'), - ('Pros and cons of trees, pros', - 2, - None, - 'pros-and-cons-of-trees-pros'), - ('Disadvantages', 2, None, 'disadvantages'), - ('Ensemble Methods: From a Single Tree to Many Trees and Extreme ' - 'Boosting, Meet the Jungle of Methods', - 2, - None, - 'ensemble-methods-from-a-single-tree-to-many-trees-and-extreme-boosting-meet-the-jungle-of-methods'), - ('An Overview of Ensemble Methods', - 2, - None, - 'an-overview-of-ensemble-methods'), - ('Bagging', 2, None, 'bagging'), - ('More bagging', 2, None, 'more-bagging'), - ('Simple Voting Example, head or tail', - 2, - None, - 'simple-voting-example-head-or-tail'), - ('Using the Voting Classifier', - 2, - None, - 'using-the-voting-classifier'), - ('Please, not the moons again! Voting and Bagging', - 2, - None, - 'please-not-the-moons-again-voting-and-bagging'), - ('Bagging Examples', 2, None, 'bagging-examples'), - ('Making your own Bootstrap: Changing the Level of the Decision ' - 'Tree', - 2, - None, - 'making-your-own-bootstrap-changing-the-level-of-the-decision-tree'), ('Why binary splits?', 2, None, 'why-binary-splits'), ('Computing a Tree using the Gini Index', 2, @@ -250,6 +189,70 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d 'playing-around-with-regions'), ('Regression trees', 2, None, 'regression-trees'), ('Final regressor code', 2, None, 'final-regressor-code'), + ('Example: Computing the Gini index', + 2, + None, + 'example-computing-the-gini-index'), + ('Simple Python Code to read in Data and perform Classification', + 2, + None, + 'simple-python-code-to-read-in-data-and-perform-classification'), + ('Computing the Gini Factor', + 2, + None, + 'computing-the-gini-factor'), + ('Entropy and the ID3 algorithm', + 2, + None, + 'entropy-and-the-id3-algorithm'), + ('Cancer Data again now with Decision Trees and other Methods', + 2, + None, + 'cancer-data-again-now-with-decision-trees-and-other-methods'), + ('Another example, the moons again', + 2, + None, + 'another-example-the-moons-again'), + ('Playing around with regions', + 2, + None, + 'playing-around-with-regions'), + ('Regression trees', 2, None, 'regression-trees'), + ('Final regressor code', 2, None, 'final-regressor-code'), + ('Pros and cons of trees, pros', + 2, + None, + 'pros-and-cons-of-trees-pros'), + ('Disadvantages', 2, None, 'disadvantages'), + ('Ensemble Methods: From a Single Tree to Many Trees and Extreme ' + 'Boosting, Meet the Jungle of Methods', + 2, + None, + 'ensemble-methods-from-a-single-tree-to-many-trees-and-extreme-boosting-meet-the-jungle-of-methods'), + ('An Overview of Ensemble Methods', + 2, + None, + 'an-overview-of-ensemble-methods'), + ('Bagging', 2, None, 'bagging'), + ('More bagging', 2, None, 'more-bagging'), + ('Simple Voting Example, head or tail', + 2, + None, + 'simple-voting-example-head-or-tail'), + ('Using the Voting Classifier', + 2, + None, + 'using-the-voting-classifier'), + ('Please, not the moons again! Voting and Bagging', + 2, + None, + 'please-not-the-moons-again-voting-and-bagging'), + ('Bagging Examples', 2, None, 'bagging-examples'), + ('Making your own Bootstrap: Changing the Level of the Decision ' + 'Tree', + 2, + None, + 'making-your-own-bootstrap-changing-the-level-of-the-decision-tree'), ('Why Voting?', 2, None, 'why-voting'), ('Tossing coins', 2, None, 'tossing-coins'), ('Standard imports first', 2, None, 'standard-imports-first'), @@ -327,55 +330,55 @@ MathJax.Hub.Config({
-
Computing the various Gini Indices, Hours slept
+Simple Python Code to read in Data and perform Classification
-See handwritten notes for Thursday November 11
+ + +# Common imports
+import numpy as np
+import pandas as pd
+import matplotlib.pyplot as plt
+from sklearn.tree import DecisionTreeClassifier
+from sklearn.model_selection import train_test_split
+from sklearn.tree import export_graphviz
+from sklearn.preprocessing import StandardScaler, OneHotEncoder
+from sklearn.compose import ColumnTransformer
+from IPython.display import Image
+from pydot import graph_from_dot_data
+import os
+
+# Where to save the figures and data files
+PROJECT_ROOT_DIR = "Results"
+FIGURE_ID = "Results/FigureFiles"
+DATA_ID = "DataFiles/"
+
+if not os.path.exists(PROJECT_ROOT_DIR):
+ os.mkdir(PROJECT_ROOT_DIR)
+
+if not os.path.exists(FIGURE_ID):
+ os.makedirs(FIGURE_ID)
+
+if not os.path.exists(DATA_ID):
+ os.makedirs(DATA_ID)
+
+def image_path(fig_id):
+ return os.path.join(FIGURE_ID, fig_id)
+
+def data_path(dat_id):
+ return os.path.join(DATA_ID, dat_id)
+
+def save_fig(fig_id):
+ plt.savefig(image_path(fig_id) + ".png", format='png')
+
+infile = open(data_path("rideclass.csv"),'r')
+
+# Read the experimental data with Pandas
+from IPython.display import display
+ridedata = pd.read_csv(infile,names = ('Outlook','Temperature','Humidity','Wind','Ride'))
+ridedata = pd.DataFrame(ridedata)
+
+# Features and targets
+X = ridedata.loc[:, ridedata.columns != 'Ride'].values
+y = ridedata.loc[:, ridedata.columns == 'Ride'].values
+
+# Create the encoder.
+encoder = OneHotEncoder(handle_unknown="ignore")
+# Assume for simplicity all features are categorical.
+encoder.fit(X)
+# Apply the encoder.
+X = encoder.transform(X)
+print(X)
+# Then do a Classification tree
+tree_clf = DecisionTreeClassifier(max_depth=2)
+tree_clf.fit(X, y)
+print("Train set accuracy with Decision Tree: {:.2f}".format(tree_clf.score(X,y)))
+#transfer to a decision tree graph
+export_graphviz(
+ tree_clf,
+ out_file="DataFiles/ride.dot",
+ rounded=True,
+ filled=True
+)
+cmd = 'dot -Tpng DataFiles/cancer.dot -o DataFiles/cancer.png'
+os.system(cmd)
+
-
Computing the various Gini Indices, Hours studied
+Computing the Gini Factor
-See handwritten notes for Thursday November 11
+The above functions (gini, entropy and misclassification error) are +important components of the so-called CART algorithm. We will discuss +this algorithm below after we have discussed the information gain +algorithm ID3. +
+ +In the example here we have converted all our attributes into numerical values \( 0,1,2 \) etc.
+ + + +# Split a dataset based on an attribute and an attribute value
+def test_split(index, value, dataset):
+ left, right = list(), list()
+ for row in dataset:
+ if row[index] < value:
+ left.append(row)
+ else:
+ right.append(row)
+ return left, right
+
+# Calculate the Gini index for a split dataset
+def gini_index(groups, classes):
+ # count all samples at split point
+ n_instances = float(sum([len(group) for group in groups]))
+ # sum weighted Gini index for each group
+ gini = 0.0
+ for group in groups:
+ size = float(len(group))
+ # avoid divide by zero
+ if size == 0:
+ continue
+ score = 0.0
+ # score the group based on the score for each class
+ for class_val in classes:
+ p = [row[-1] for row in group].count(class_val) / size
+ score += p * p
+ # weight the group score by its relative size
+ gini += (1.0 - score) * (size / n_instances)
+ return gini
+
+# Select the best split point for a dataset
+def get_split(dataset):
+ class_values = list(set(row[-1] for row in dataset))
+ b_index, b_value, b_score, b_groups = 999, 999, 999, None
+ for index in range(len(dataset[0])-1):
+ for row in dataset:
+ groups = test_split(index, row[index], dataset)
+ gini = gini_index(groups, class_values)
+ print('X%d < %.3f Gini=%.3f' % ((index+1), row[index], gini))
+ if gini < b_score:
+ b_index, b_value, b_score, b_groups = index, row[index], gini, groups
+ return {'index':b_index, 'value':b_value, 'groups':b_groups}
+
+dataset = [[0,0,0,0,0],
+ [0,0,0,1,1],
+ [1,0,0,0,1],
+ [2,1,0,0,1],
+ [2,2,1,0,1],
+ [2,2,1,1,0],
+ [1,2,1,1,1],
+ [0,1,0,0,0],
+ [0,2,1,0,1],
+ [2,1,1,0,1],
+ [0,1,1,1,1],
+ [1,1,0,1,1],
+ [1,0,1,0,1],
+ [2,1,0,1,0]]
+
+split = get_split(dataset)
+print('Split: [X%d < %.3f]' % ((split['index']+1), split['value']))
+
For final tree, see the above handwritten notes
-
-
diff --git a/doc/pub/week44/html/._week44-bs054.html b/doc/pub/week44/html/._week44-bs054.html
index d4fca77eb..0b5643607 100644
--- a/doc/pub/week44/html/._week44-bs054.html
+++ b/doc/pub/week44/html/._week44-bs054.html
@@ -115,67 +115,6 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d
2,
None,
'the-cart-algorithm-for-regression'),
- ('Computing the Gini index', 2, None, 'computing-the-gini-index'),
- ('Simple Python Code to read in Data and perform Classification',
- 2,
- None,
- 'simple-python-code-to-read-in-data-and-perform-classification'),
- ('Computing the Gini Factor',
- 2,
- None,
- 'computing-the-gini-factor'),
- ('Entropy and the ID3 algorithm',
- 2,
- None,
- 'entropy-and-the-id3-algorithm'),
- ('Cancer Data again now with Decision Trees and other Methods',
- 2,
- None,
- 'cancer-data-again-now-with-decision-trees-and-other-methods'),
- ('Another example, the moons again',
- 2,
- None,
- 'another-example-the-moons-again'),
- ('Playing around with regions',
- 2,
- None,
- 'playing-around-with-regions'),
- ('Regression trees', 2, None, 'regression-trees'),
- ('Final regressor code', 2, None, 'final-regressor-code'),
- ('Pros and cons of trees, pros',
- 2,
- None,
- 'pros-and-cons-of-trees-pros'),
- ('Disadvantages', 2, None, 'disadvantages'),
- ('Ensemble Methods: From a Single Tree to Many Trees and Extreme '
- 'Boosting, Meet the Jungle of Methods',
- 2,
- None,
- 'ensemble-methods-from-a-single-tree-to-many-trees-and-extreme-boosting-meet-the-jungle-of-methods'),
- ('An Overview of Ensemble Methods',
- 2,
- None,
- 'an-overview-of-ensemble-methods'),
- ('Bagging', 2, None, 'bagging'),
- ('More bagging', 2, None, 'more-bagging'),
- ('Simple Voting Example, head or tail',
- 2,
- None,
- 'simple-voting-example-head-or-tail'),
- ('Using the Voting Classifier',
- 2,
- None,
- 'using-the-voting-classifier'),
- ('Please, not the moons again! Voting and Bagging',
- 2,
- None,
- 'please-not-the-moons-again-voting-and-bagging'),
- ('Bagging Examples', 2, None, 'bagging-examples'),
- ('Making your own Bootstrap: Changing the Level of the Decision '
- 'Tree',
- 2,
- None,
- 'making-your-own-bootstrap-changing-the-level-of-the-decision-tree'),
('Why binary splits?', 2, None, 'why-binary-splits'),
('Computing a Tree using the Gini Index',
2,
@@ -250,6 +189,70 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d
'playing-around-with-regions'),
('Regression trees', 2, None, 'regression-trees'),
('Final regressor code', 2, None, 'final-regressor-code'),
+ ('Example: Computing the Gini index',
+ 2,
+ None,
+ 'example-computing-the-gini-index'),
+ ('Simple Python Code to read in Data and perform Classification',
+ 2,
+ None,
+ 'simple-python-code-to-read-in-data-and-perform-classification'),
+ ('Computing the Gini Factor',
+ 2,
+ None,
+ 'computing-the-gini-factor'),
+ ('Entropy and the ID3 algorithm',
+ 2,
+ None,
+ 'entropy-and-the-id3-algorithm'),
+ ('Cancer Data again now with Decision Trees and other Methods',
+ 2,
+ None,
+ 'cancer-data-again-now-with-decision-trees-and-other-methods'),
+ ('Another example, the moons again',
+ 2,
+ None,
+ 'another-example-the-moons-again'),
+ ('Playing around with regions',
+ 2,
+ None,
+ 'playing-around-with-regions'),
+ ('Regression trees', 2, None, 'regression-trees'),
+ ('Final regressor code', 2, None, 'final-regressor-code'),
+ ('Pros and cons of trees, pros',
+ 2,
+ None,
+ 'pros-and-cons-of-trees-pros'),
+ ('Disadvantages', 2, None, 'disadvantages'),
+ ('Ensemble Methods: From a Single Tree to Many Trees and Extreme '
+ 'Boosting, Meet the Jungle of Methods',
+ 2,
+ None,
+ 'ensemble-methods-from-a-single-tree-to-many-trees-and-extreme-boosting-meet-the-jungle-of-methods'),
+ ('An Overview of Ensemble Methods',
+ 2,
+ None,
+ 'an-overview-of-ensemble-methods'),
+ ('Bagging', 2, None, 'bagging'),
+ ('More bagging', 2, None, 'more-bagging'),
+ ('Simple Voting Example, head or tail',
+ 2,
+ None,
+ 'simple-voting-example-head-or-tail'),
+ ('Using the Voting Classifier',
+ 2,
+ None,
+ 'using-the-voting-classifier'),
+ ('Please, not the moons again! Voting and Bagging',
+ 2,
+ None,
+ 'please-not-the-moons-again-voting-and-bagging'),
+ ('Bagging Examples', 2, None, 'bagging-examples'),
+ ('Making your own Bootstrap: Changing the Level of the Decision '
+ 'Tree',
+ 2,
+ None,
+ 'making-your-own-bootstrap-changing-the-level-of-the-decision-tree'),
('Why Voting?', 2, None, 'why-voting'),
('Tossing coins', 2, None, 'tossing-coins'),
('Standard imports first', 2, None, 'standard-imports-first'),
@@ -327,55 +330,55 @@ MathJax.Hub.Config({
-
A possible code using Scikit-Learn
+Entropy and the ID3 algorithm
+The ID3 algorithm learns decision trees by constructing +them in a top down way, beginning with the question which attribute should be tested at the root of the tree? +
- -# Common imports
-import numpy as np
-import pandas as pd
-import matplotlib.pyplot as plt
-from sklearn.tree import DecisionTreeClassifier
-from sklearn.model_selection import train_test_split
-from sklearn.tree import export_graphviz
-from sklearn.preprocessing import StandardScaler, OneHotEncoder
-from sklearn.compose import ColumnTransformer
-from IPython.display import Image
-from pydot import graph_from_dot_data
-import os
+
+
+The ID3 algorithm selects which attribute to test at each node in the
+tree.
+
-# Where to save the figures and data files
-PROJECT_ROOT_DIR = "Results"
-FIGURE_ID = "Results/FigureFiles"
-DATA_ID = "DataFiles/"
+We would like to select the attribute that is most useful for classifying
+examples.
+
-if not os.path.exists(PROJECT_ROOT_DIR):
- os.mkdir(PROJECT_ROOT_DIR)
+What is a good quantitative measure of the worth of an attribute?
-if not os.path.exists(FIGURE_ID):
- os.makedirs(FIGURE_ID)
-
-if not os.path.exists(DATA_ID):
- os.makedirs(DATA_ID)
-
-def image_path(fig_id):
- return os.path.join(FIGURE_ID, fig_id)
-
-def data_path(dat_id):
- return os.path.join(DATA_ID, dat_id)
-
-def save_fig(fig_id):
- plt.savefig(image_path(fig_id) + ".png", format='png')
-
-infile = open(data_path("grades.csv"),'r')
-
-# Read the experimental data with Pandas
-from IPython.display import display
-grades = pd.read_csv(infile,names = ('Trend','Sleep','Studied','Grade'))
-grades = pd.DataFrame(grades)
-
-# Features and targets
-X = grades.loc[:, grades.columns != 'Grade'].values
-y = grades.loc[:, grades.columns == 'Grade'].values
-
-# Create the encoder.
-encoder = OneHotEncoder(handle_unknown="ignore")
-# Assume for simplicity all features are categorical.
-encoder.fit(X)
-# Apply the encoder.
-X = encoder.transform(X)
-print(X)
-# Then do a Classification tree
-tree_clf = DecisionTreeClassifier(max_depth=2)
-tree_clf.fit(X, y)
-print("Train set accuracy with Decision Tree: {:.2f}".format(tree_clf.score(X,y)))
-#transfer to a decision tree graph
-export_graphviz(
- tree_clf,
- out_file="DataFiles/grade.dot",
- rounded=True,
- filled=True
-)
-cmd = 'dot -Tpng DataFiles/grade.dot -o DataFiles/grades.png'
-os.system(cmd)
-
-Information gain measures how well a given attribute separates the +training examples according to their target classification. +
+The ID3 algorithm uses this information gain measure to select among the candidate +attributes at each step while growing the tree. +
diff --git a/doc/pub/week44/html/._week44-bs055.html b/doc/pub/week44/html/._week44-bs055.html index 861521fed..9004c4481 100644 --- a/doc/pub/week44/html/._week44-bs055.html +++ b/doc/pub/week44/html/._week44-bs055.html @@ -115,67 +115,6 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'the-cart-algorithm-for-regression'), - ('Computing the Gini index', 2, None, 'computing-the-gini-index'), - ('Simple Python Code to read in Data and perform Classification', - 2, - None, - 'simple-python-code-to-read-in-data-and-perform-classification'), - ('Computing the Gini Factor', - 2, - None, - 'computing-the-gini-factor'), - ('Entropy and the ID3 algorithm', - 2, - None, - 'entropy-and-the-id3-algorithm'), - ('Cancer Data again now with Decision Trees and other Methods', - 2, - None, - 'cancer-data-again-now-with-decision-trees-and-other-methods'), - ('Another example, the moons again', - 2, - None, - 'another-example-the-moons-again'), - ('Playing around with regions', - 2, - None, - 'playing-around-with-regions'), - ('Regression trees', 2, None, 'regression-trees'), - ('Final regressor code', 2, None, 'final-regressor-code'), - ('Pros and cons of trees, pros', - 2, - None, - 'pros-and-cons-of-trees-pros'), - ('Disadvantages', 2, None, 'disadvantages'), - ('Ensemble Methods: From a Single Tree to Many Trees and Extreme ' - 'Boosting, Meet the Jungle of Methods', - 2, - None, - 'ensemble-methods-from-a-single-tree-to-many-trees-and-extreme-boosting-meet-the-jungle-of-methods'), - ('An Overview of Ensemble Methods', - 2, - None, - 'an-overview-of-ensemble-methods'), - ('Bagging', 2, None, 'bagging'), - ('More bagging', 2, None, 'more-bagging'), - ('Simple Voting Example, head or tail', - 2, - None, - 'simple-voting-example-head-or-tail'), - ('Using the Voting Classifier', - 2, - None, - 'using-the-voting-classifier'), - ('Please, not the moons again! Voting and Bagging', - 2, - None, - 'please-not-the-moons-again-voting-and-bagging'), - ('Bagging Examples', 2, None, 'bagging-examples'), - ('Making your own Bootstrap: Changing the Level of the Decision ' - 'Tree', - 2, - None, - 'making-your-own-bootstrap-changing-the-level-of-the-decision-tree'), ('Why binary splits?', 2, None, 'why-binary-splits'), ('Computing a Tree using the Gini Index', 2, @@ -250,6 +189,70 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d 'playing-around-with-regions'), ('Regression trees', 2, None, 'regression-trees'), ('Final regressor code', 2, None, 'final-regressor-code'), + ('Example: Computing the Gini index', + 2, + None, + 'example-computing-the-gini-index'), + ('Simple Python Code to read in Data and perform Classification', + 2, + None, + 'simple-python-code-to-read-in-data-and-perform-classification'), + ('Computing the Gini Factor', + 2, + None, + 'computing-the-gini-factor'), + ('Entropy and the ID3 algorithm', + 2, + None, + 'entropy-and-the-id3-algorithm'), + ('Cancer Data again now with Decision Trees and other Methods', + 2, + None, + 'cancer-data-again-now-with-decision-trees-and-other-methods'), + ('Another example, the moons again', + 2, + None, + 'another-example-the-moons-again'), + ('Playing around with regions', + 2, + None, + 'playing-around-with-regions'), + ('Regression trees', 2, None, 'regression-trees'), + ('Final regressor code', 2, None, 'final-regressor-code'), + ('Pros and cons of trees, pros', + 2, + None, + 'pros-and-cons-of-trees-pros'), + ('Disadvantages', 2, None, 'disadvantages'), + ('Ensemble Methods: From a Single Tree to Many Trees and Extreme ' + 'Boosting, Meet the Jungle of Methods', + 2, + None, + 'ensemble-methods-from-a-single-tree-to-many-trees-and-extreme-boosting-meet-the-jungle-of-methods'), + ('An Overview of Ensemble Methods', + 2, + None, + 'an-overview-of-ensemble-methods'), + ('Bagging', 2, None, 'bagging'), + ('More bagging', 2, None, 'more-bagging'), + ('Simple Voting Example, head or tail', + 2, + None, + 'simple-voting-example-head-or-tail'), + ('Using the Voting Classifier', + 2, + None, + 'using-the-voting-classifier'), + ('Please, not the moons again! Voting and Bagging', + 2, + None, + 'please-not-the-moons-again-voting-and-bagging'), + ('Bagging Examples', 2, None, 'bagging-examples'), + ('Making your own Bootstrap: Changing the Level of the Decision ' + 'Tree', + 2, + None, + 'making-your-own-bootstrap-changing-the-level-of-the-decision-tree'), ('Why Voting?', 2, None, 'why-voting'), ('Tossing coins', 2, None, 'tossing-coins'), ('Standard imports first', 2, None, 'standard-imports-first'), @@ -327,55 +330,55 @@ MathJax.Hub.Config({
-
Visualizing Trees, More examples
+Cancer Data again now with Decision Trees and other Methods
import os
-from sklearn.datasets import load_breast_cancer
-from sklearn.tree import DecisionTreeClassifier
-from sklearn.model_selection import train_test_split
-from sklearn.metrics import confusion_matrix
-from sklearn.tree import export_graphviz
-
-from IPython.display import Image
-from pydot import graph_from_dot_data
-import pandas as pd
+ import matplotlib.pyplot as plt
import numpy as np
+from sklearn.model_selection import train_test_split
+from sklearn.datasets import load_breast_cancer
+from sklearn.svm import SVC
+from sklearn.linear_model import LogisticRegression
+from sklearn.tree import DecisionTreeClassifier
-
+# Load the data
cancer = load_breast_cancer()
-X = pd.DataFrame(cancer.data, columns=cancer.feature_names)
-print(X)
-y = pd.Categorical.from_codes(cancer.target, cancer.target_names)
-y = pd.get_dummies(y)
-print(y)
-X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=1)
-tree_clf = DecisionTreeClassifier(max_depth=5)
-tree_clf.fit(X_train, y_train)
-export_graphviz(
- tree_clf,
- out_file="DataFiles/cancer.dot",
- feature_names=cancer.feature_names,
- class_names=cancer.target_names,
- rounded=True,
- filled=True
-)
-cmd = 'dot -Tpng DataFiles/cancer.dot -o DataFiles/cancer.png'
-os.system(cmd)
+X_train, X_test, y_train, y_test = train_test_split(cancer.data,cancer.target,random_state=0)
+print(X_train.shape)
+print(X_test.shape)
+# Logistic Regression
+logreg = LogisticRegression(solver='lbfgs')
+logreg.fit(X_train, y_train)
+print("Test set accuracy with Logistic Regression: {:.2f}".format(logreg.score(X_test,y_test)))
+# Support vector machine
+svm = SVC(gamma='auto', C=100)
+svm.fit(X_train, y_train)
+print("Test set accuracy with SVM: {:.2f}".format(svm.score(X_test,y_test)))
+# Decision Trees
+deep_tree_clf = DecisionTreeClassifier(max_depth=None)
+deep_tree_clf.fit(X_train, y_train)
+print("Test set accuracy with Decision Trees: {:.2f}".format(deep_tree_clf.score(X_test,y_test)))
+#now scale the data
+from sklearn.preprocessing import StandardScaler
+scaler = StandardScaler()
+scaler.fit(X_train)
+X_train_scaled = scaler.transform(X_train)
+X_test_scaled = scaler.transform(X_test)
+# Logistic Regression
+logreg.fit(X_train_scaled, y_train)
+print("Test set accuracy Logistic Regression with scaled data: {:.2f}".format(logreg.score(X_test_scaled,y_test)))
+# Support Vector Machine
+svm.fit(X_train_scaled, y_train)
+print("Test set accuracy SVM with scaled data: {:.2f}".format(logreg.score(X_test_scaled,y_test)))
+# Decision Trees
+deep_tree_clf.fit(X_train_scaled, y_train)
+print("Test set accuracy with Decision Trees and scaled data: {:.2f}".format(deep_tree_clf.score(X_test_scaled,y_test)))
-
Visualizing the Tree, The Moons
+Another example, the moons again
# Common imports
+ from __future__ import division, print_function, unicode_literals
+
+# Common imports
import numpy as np
-from sklearn.model_selection import train_test_split
+import os
+
+# to make this notebook's output stable across runs
+np.random.seed(42)
+
+# To plot pretty figures
+import matplotlib
+import matplotlib.pyplot as plt
+from matplotlib.colors import ListedColormap
+plt.rcParams['axes.labelsize'] = 14
+plt.rcParams['xtick.labelsize'] = 12
+plt.rcParams['ytick.labelsize'] = 12
+
+
+from sklearn.svm import SVC
+from sklearn import datasets
from sklearn.tree import DecisionTreeClassifier
from sklearn.datasets import make_moons
from sklearn.tree import export_graphviz
-from pydot import graph_from_dot_data
-import pandas as pd
-import os
-np.random.seed(42)
-X, y = make_moons(n_samples=100, noise=0.25, random_state=53)
-X_train, X_test, y_train, y_test = train_test_split(X,y,random_state=0)
-tree_clf = DecisionTreeClassifier(max_depth=5)
-tree_clf.fit(X_train, y_train)
+Xm, ym = make_moons(n_samples=100, noise=0.25, random_state=53)
-export_graphviz(
- tree_clf,
- out_file="DataFiles/moons.dot",
- rounded=True,
- filled=True
-)
-cmd = 'dot -Tpng DataFiles/moons.dot -o DataFiles/moons.png'
-os.system(cmd)
+deep_tree_clf1 = DecisionTreeClassifier(random_state=42)
+deep_tree_clf2 = DecisionTreeClassifier(min_samples_leaf=4, random_state=42)
+deep_tree_clf1.fit(Xm, ym)
+deep_tree_clf2.fit(Xm, ym)
+
+
+def plot_decision_boundary(clf, X, y, axes=[0, 7.5, 0, 3], iris=True, legend=False, plot_training=True):
+ x1s = np.linspace(axes[0], axes[1], 100)
+ x2s = np.linspace(axes[2], axes[3], 100)
+ x1, x2 = np.meshgrid(x1s, x2s)
+ X_new = np.c_[x1.ravel(), x2.ravel()]
+ y_pred = clf.predict(X_new).reshape(x1.shape)
+ custom_cmap = ListedColormap(['#fafab0','#9898ff','#a0faa0'])
+ plt.contourf(x1, x2, y_pred, alpha=0.3, cmap=custom_cmap)
+ if not iris:
+ custom_cmap2 = ListedColormap(['#7d7d58','#4c4c7f','#507d50'])
+ plt.contour(x1, x2, y_pred, cmap=custom_cmap2, alpha=0.8)
+ if plot_training:
+ plt.plot(X[:, 0][y==0], X[:, 1][y==0], "yo", label="Iris-Setosa")
+ plt.plot(X[:, 0][y==1], X[:, 1][y==1], "bs", label="Iris-Versicolor")
+ plt.plot(X[:, 0][y==2], X[:, 1][y==2], "g^", label="Iris-Virginica")
+ plt.axis(axes)
+ if iris:
+ plt.xlabel("Petal length", fontsize=14)
+ plt.ylabel("Petal width", fontsize=14)
+ else:
+ plt.xlabel(r"$x_1$", fontsize=18)
+ plt.ylabel(r"$x_2$", fontsize=18, rotation=0)
+ if legend:
+ plt.legend(loc="lower right", fontsize=14)
+plt.figure(figsize=(11, 4))
+plt.subplot(121)
+plot_decision_boundary(deep_tree_clf1, Xm, ym, axes=[-1.5, 2.5, -1, 1.5], iris=False)
+plt.title("No restrictions", fontsize=16)
+plt.subplot(122)
+plot_decision_boundary(deep_tree_clf2, Xm, ym, axes=[-1.5, 2.5, -1, 1.5], iris=False)
+plt.title("min_samples_leaf = {}".format(deep_tree_clf2.min_samples_leaf), fontsize=14)
+plt.show()
-
Other ways of visualizing the trees
- -Scikit-Learn has also another way to visualize the trees which is very useful, here with the Iris data.
- +Playing around with regions
from sklearn.datasets import load_iris
-from sklearn import tree
-X, y = load_iris(return_X_y=True)
-tree_clf = tree.DecisionTreeClassifier()
-tree_clf = tree_clf.fit(X, y)
-# and then plot the tree
-tree.plot_tree(tree_clf)
+ np.random.seed(6)
+Xs = np.random.rand(100, 2) - 0.5
+ys = (Xs[:, 0] > 0).astype(np.float32) * 2
+
+angle = np.pi/4
+rotation_matrix = np.array([[np.cos(angle), -np.sin(angle)], [np.sin(angle), np.cos(angle)]])
+Xsr = Xs.dot(rotation_matrix)
+
+tree_clf_s = DecisionTreeClassifier(random_state=42)
+tree_clf_s.fit(Xs, ys)
+tree_clf_sr = DecisionTreeClassifier(random_state=42)
+tree_clf_sr.fit(Xsr, ys)
+
+plt.figure(figsize=(11, 4))
+plt.subplot(121)
+plot_decision_boundary(tree_clf_s, Xs, ys, axes=[-0.7, 0.7, -0.7, 0.7], iris=False)
+plt.subplot(122)
+plot_decision_boundary(tree_clf_sr, Xsr, ys, axes=[-0.7, 0.7, -0.7, 0.7], iris=False)
+
+plt.show()
-
Printing out as text
- -Alternatively, the tree can also be exported in textual format with the function exporttext. -This method doesn’t require the installation of external libraries and is more compact: -
- +Regression trees
from sklearn.datasets import load_iris
-from sklearn.tree import DecisionTreeClassifier
-from sklearn.tree import export_text
-iris = load_iris()
-decision_tree = DecisionTreeClassifier(random_state=0, max_depth=2)
-decision_tree = decision_tree.fit(iris.data, iris.target)
-r = export_text(decision_tree, feature_names=iris['feature_names'])
-print(r)
+ # Quadratic training set + noise
+np.random.seed(42)
+m = 200
+X = np.random.rand(m, 1)
+y = 4 * (X - 0.5) ** 2
+y = y + np.random.randn(m, 1) / 10
+
+from sklearn.tree import DecisionTreeRegressor
+
+tree_reg = DecisionTreeRegressor(max_depth=2, random_state=42)
+tree_reg.fit(X, y)
-
Algorithms for Setting up Decision Trees
+Final regressor code
+ + +from sklearn.tree import DecisionTreeRegressor
+
+tree_reg1 = DecisionTreeRegressor(random_state=42, max_depth=2)
+tree_reg2 = DecisionTreeRegressor(random_state=42, max_depth=3)
+tree_reg1.fit(X, y)
+tree_reg2.fit(X, y)
+
+def plot_regression_predictions(tree_reg, X, y, axes=[0, 1, -0.2, 1], ylabel="$y$"):
+ x1 = np.linspace(axes[0], axes[1], 500).reshape(-1, 1)
+ y_pred = tree_reg.predict(x1)
+ plt.axis(axes)
+ plt.xlabel("$x_1$", fontsize=18)
+ if ylabel:
+ plt.ylabel(ylabel, fontsize=18, rotation=0)
+ plt.plot(X, y, "b.")
+ plt.plot(x1, y_pred, "r.-", linewidth=2, label=r"$\hat{y}$")
+
+plt.figure(figsize=(11, 4))
+plt.subplot(121)
+plot_regression_predictions(tree_reg1, X, y)
+for split, style in ((0.1973, "k-"), (0.0917, "k--"), (0.7718, "k--")):
+ plt.plot([split, split], [-0.2, 1], style, linewidth=2)
+plt.text(0.21, 0.65, "Depth=0", fontsize=15)
+plt.text(0.01, 0.2, "Depth=1", fontsize=13)
+plt.text(0.65, 0.8, "Depth=1", fontsize=13)
+plt.legend(loc="upper center", fontsize=18)
+plt.title("max_depth=2", fontsize=14)
+
+plt.subplot(122)
+plot_regression_predictions(tree_reg2, X, y, ylabel=None)
+for split, style in ((0.1973, "k-"), (0.0917, "k--"), (0.7718, "k--")):
+ plt.plot([split, split], [-0.2, 1], style, linewidth=2)
+for split in (0.0458, 0.1298, 0.2873, 0.9040):
+ plt.plot([split, split], [-0.2, 1], "k:", linewidth=1)
+plt.text(0.3, 0.5, "Depth=2", fontsize=13)
+plt.title("max_depth=3", fontsize=14)
+
+plt.show()
+
+tree_reg1 = DecisionTreeRegressor(random_state=42)
+tree_reg2 = DecisionTreeRegressor(random_state=42, min_samples_leaf=10)
+tree_reg1.fit(X, y)
+tree_reg2.fit(X, y)
+
+x1 = np.linspace(0, 1, 500).reshape(-1, 1)
+y_pred1 = tree_reg1.predict(x1)
+y_pred2 = tree_reg2.predict(x1)
+
+plt.figure(figsize=(11, 4))
+
+plt.subplot(121)
+plt.plot(X, y, "b.")
+plt.plot(x1, y_pred1, "r.-", linewidth=2, label=r"$\hat{y}$")
+plt.axis([0, 1, -0.2, 1.1])
+plt.xlabel("$x_1$", fontsize=18)
+plt.ylabel("$y$", fontsize=18, rotation=0)
+plt.legend(loc="upper center", fontsize=18)
+plt.title("No restrictions", fontsize=14)
+
+plt.subplot(122)
+plt.plot(X, y, "b.")
+plt.plot(x1, y_pred2, "r.-", linewidth=2, label=r"$\hat{y}$")
+plt.axis([0, 1, -0.2, 1.1])
+plt.xlabel("$x_1$", fontsize=18)
+plt.title("min_samples_leaf={}".format(tree_reg2.min_samples_leaf), fontsize=14)
+
+plt.show()
+
+Two algorithms stand out in the set up of decision trees:
--
-
We discuss both algorithms with applications here. The popular library -Scikit-Learn uses the CART algorithm. For classification problems -you can use either the gini index or the entropy to split a tree -in two branches. -
diff --git a/doc/pub/week44/html/._week44-bs060.html b/doc/pub/week44/html/._week44-bs060.html index d81d6f94c..fd98dc4e9 100644 --- a/doc/pub/week44/html/._week44-bs060.html +++ b/doc/pub/week44/html/._week44-bs060.html @@ -115,67 +115,6 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'the-cart-algorithm-for-regression'), - ('Computing the Gini index', 2, None, 'computing-the-gini-index'), - ('Simple Python Code to read in Data and perform Classification', - 2, - None, - 'simple-python-code-to-read-in-data-and-perform-classification'), - ('Computing the Gini Factor', - 2, - None, - 'computing-the-gini-factor'), - ('Entropy and the ID3 algorithm', - 2, - None, - 'entropy-and-the-id3-algorithm'), - ('Cancer Data again now with Decision Trees and other Methods', - 2, - None, - 'cancer-data-again-now-with-decision-trees-and-other-methods'), - ('Another example, the moons again', - 2, - None, - 'another-example-the-moons-again'), - ('Playing around with regions', - 2, - None, - 'playing-around-with-regions'), - ('Regression trees', 2, None, 'regression-trees'), - ('Final regressor code', 2, None, 'final-regressor-code'), - ('Pros and cons of trees, pros', - 2, - None, - 'pros-and-cons-of-trees-pros'), - ('Disadvantages', 2, None, 'disadvantages'), - ('Ensemble Methods: From a Single Tree to Many Trees and Extreme ' - 'Boosting, Meet the Jungle of Methods', - 2, - None, - 'ensemble-methods-from-a-single-tree-to-many-trees-and-extreme-boosting-meet-the-jungle-of-methods'), - ('An Overview of Ensemble Methods', - 2, - None, - 'an-overview-of-ensemble-methods'), - ('Bagging', 2, None, 'bagging'), - ('More bagging', 2, None, 'more-bagging'), - ('Simple Voting Example, head or tail', - 2, - None, - 'simple-voting-example-head-or-tail'), - ('Using the Voting Classifier', - 2, - None, - 'using-the-voting-classifier'), - ('Please, not the moons again! Voting and Bagging', - 2, - None, - 'please-not-the-moons-again-voting-and-bagging'), - ('Bagging Examples', 2, None, 'bagging-examples'), - ('Making your own Bootstrap: Changing the Level of the Decision ' - 'Tree', - 2, - None, - 'making-your-own-bootstrap-changing-the-level-of-the-decision-tree'), ('Why binary splits?', 2, None, 'why-binary-splits'), ('Computing a Tree using the Gini Index', 2, @@ -250,6 +189,70 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d 'playing-around-with-regions'), ('Regression trees', 2, None, 'regression-trees'), ('Final regressor code', 2, None, 'final-regressor-code'), + ('Example: Computing the Gini index', + 2, + None, + 'example-computing-the-gini-index'), + ('Simple Python Code to read in Data and perform Classification', + 2, + None, + 'simple-python-code-to-read-in-data-and-perform-classification'), + ('Computing the Gini Factor', + 2, + None, + 'computing-the-gini-factor'), + ('Entropy and the ID3 algorithm', + 2, + None, + 'entropy-and-the-id3-algorithm'), + ('Cancer Data again now with Decision Trees and other Methods', + 2, + None, + 'cancer-data-again-now-with-decision-trees-and-other-methods'), + ('Another example, the moons again', + 2, + None, + 'another-example-the-moons-again'), + ('Playing around with regions', + 2, + None, + 'playing-around-with-regions'), + ('Regression trees', 2, None, 'regression-trees'), + ('Final regressor code', 2, None, 'final-regressor-code'), + ('Pros and cons of trees, pros', + 2, + None, + 'pros-and-cons-of-trees-pros'), + ('Disadvantages', 2, None, 'disadvantages'), + ('Ensemble Methods: From a Single Tree to Many Trees and Extreme ' + 'Boosting, Meet the Jungle of Methods', + 2, + None, + 'ensemble-methods-from-a-single-tree-to-many-trees-and-extreme-boosting-meet-the-jungle-of-methods'), + ('An Overview of Ensemble Methods', + 2, + None, + 'an-overview-of-ensemble-methods'), + ('Bagging', 2, None, 'bagging'), + ('More bagging', 2, None, 'more-bagging'), + ('Simple Voting Example, head or tail', + 2, + None, + 'simple-voting-example-head-or-tail'), + ('Using the Voting Classifier', + 2, + None, + 'using-the-voting-classifier'), + ('Please, not the moons again! Voting and Bagging', + 2, + None, + 'please-not-the-moons-again-voting-and-bagging'), + ('Bagging Examples', 2, None, 'bagging-examples'), + ('Making your own Bootstrap: Changing the Level of the Decision ' + 'Tree', + 2, + None, + 'making-your-own-bootstrap-changing-the-level-of-the-decision-tree'), ('Why Voting?', 2, None, 'why-voting'), ('Tossing coins', 2, None, 'tossing-coins'), ('Standard imports first', 2, None, 'standard-imports-first'), @@ -327,55 +330,55 @@ MathJax.Hub.Config({
-
The CART algorithm for Classification
- -For classification, the CART algorithm splits the data set in two subsets using a single feature \( k \) and a threshold \( t_k \). -This could be for example a threshold set by a number below a certain circumference of a malign tumor. -
- -How do we find these two quantities? -We search for the pair \( (k,t_k) \) that produces the purest subset using for example the gini factor \( G \). -The cost function it tries to minimize is then -
-$$ -C(k,t_k) = \frac{m_{\mathrm{left}}}{m}G_{\mathrm{left}}+ \frac{m_{\mathrm{right}}}{m}G_{\mathrm{right}}, -$$ - -where \( G_{\mathrm{left/right}} \) measures the impurity of the left/right subset and \( m_{\mathrm{left/right}} \) - is the number of instances in the left/right subset -
- -Once it has successfully split the training set in two, it splits the subsets using the same logic, then the subsubsets -and so on, recursively. It stops recursing once it reaches the maximum depth (defined by the -\( max\_depth \) hyperparameter), or if it cannot find a split that will reduce impurity. A few other -hyperparameters control additional stopping conditions such as the \( min\_samples\_split \), -\( min\_samples\_leaf \), \( min\_weight\_fraction\_leaf \), and \( max\_leaf\_nodes \). -
+Pros and cons of trees, pros
+-
+
-
diff --git a/doc/pub/week44/html/._week44-bs061.html b/doc/pub/week44/html/._week44-bs061.html
index b849267d5..43cbf096c 100644
--- a/doc/pub/week44/html/._week44-bs061.html
+++ b/doc/pub/week44/html/._week44-bs061.html
@@ -115,67 +115,6 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d
2,
None,
'the-cart-algorithm-for-regression'),
- ('Computing the Gini index', 2, None, 'computing-the-gini-index'),
- ('Simple Python Code to read in Data and perform Classification',
- 2,
- None,
- 'simple-python-code-to-read-in-data-and-perform-classification'),
- ('Computing the Gini Factor',
- 2,
- None,
- 'computing-the-gini-factor'),
- ('Entropy and the ID3 algorithm',
- 2,
- None,
- 'entropy-and-the-id3-algorithm'),
- ('Cancer Data again now with Decision Trees and other Methods',
- 2,
- None,
- 'cancer-data-again-now-with-decision-trees-and-other-methods'),
- ('Another example, the moons again',
- 2,
- None,
- 'another-example-the-moons-again'),
- ('Playing around with regions',
- 2,
- None,
- 'playing-around-with-regions'),
- ('Regression trees', 2, None, 'regression-trees'),
- ('Final regressor code', 2, None, 'final-regressor-code'),
- ('Pros and cons of trees, pros',
- 2,
- None,
- 'pros-and-cons-of-trees-pros'),
- ('Disadvantages', 2, None, 'disadvantages'),
- ('Ensemble Methods: From a Single Tree to Many Trees and Extreme '
- 'Boosting, Meet the Jungle of Methods',
- 2,
- None,
- 'ensemble-methods-from-a-single-tree-to-many-trees-and-extreme-boosting-meet-the-jungle-of-methods'),
- ('An Overview of Ensemble Methods',
- 2,
- None,
- 'an-overview-of-ensemble-methods'),
- ('Bagging', 2, None, 'bagging'),
- ('More bagging', 2, None, 'more-bagging'),
- ('Simple Voting Example, head or tail',
- 2,
- None,
- 'simple-voting-example-head-or-tail'),
- ('Using the Voting Classifier',
- 2,
- None,
- 'using-the-voting-classifier'),
- ('Please, not the moons again! Voting and Bagging',
- 2,
- None,
- 'please-not-the-moons-again-voting-and-bagging'),
- ('Bagging Examples', 2, None, 'bagging-examples'),
- ('Making your own Bootstrap: Changing the Level of the Decision '
- 'Tree',
- 2,
- None,
- 'making-your-own-bootstrap-changing-the-level-of-the-decision-tree'),
('Why binary splits?', 2, None, 'why-binary-splits'),
('Computing a Tree using the Gini Index',
2,
@@ -250,6 +189,70 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d
'playing-around-with-regions'),
('Regression trees', 2, None, 'regression-trees'),
('Final regressor code', 2, None, 'final-regressor-code'),
+ ('Example: Computing the Gini index',
+ 2,
+ None,
+ 'example-computing-the-gini-index'),
+ ('Simple Python Code to read in Data and perform Classification',
+ 2,
+ None,
+ 'simple-python-code-to-read-in-data-and-perform-classification'),
+ ('Computing the Gini Factor',
+ 2,
+ None,
+ 'computing-the-gini-factor'),
+ ('Entropy and the ID3 algorithm',
+ 2,
+ None,
+ 'entropy-and-the-id3-algorithm'),
+ ('Cancer Data again now with Decision Trees and other Methods',
+ 2,
+ None,
+ 'cancer-data-again-now-with-decision-trees-and-other-methods'),
+ ('Another example, the moons again',
+ 2,
+ None,
+ 'another-example-the-moons-again'),
+ ('Playing around with regions',
+ 2,
+ None,
+ 'playing-around-with-regions'),
+ ('Regression trees', 2, None, 'regression-trees'),
+ ('Final regressor code', 2, None, 'final-regressor-code'),
+ ('Pros and cons of trees, pros',
+ 2,
+ None,
+ 'pros-and-cons-of-trees-pros'),
+ ('Disadvantages', 2, None, 'disadvantages'),
+ ('Ensemble Methods: From a Single Tree to Many Trees and Extreme '
+ 'Boosting, Meet the Jungle of Methods',
+ 2,
+ None,
+ 'ensemble-methods-from-a-single-tree-to-many-trees-and-extreme-boosting-meet-the-jungle-of-methods'),
+ ('An Overview of Ensemble Methods',
+ 2,
+ None,
+ 'an-overview-of-ensemble-methods'),
+ ('Bagging', 2, None, 'bagging'),
+ ('More bagging', 2, None, 'more-bagging'),
+ ('Simple Voting Example, head or tail',
+ 2,
+ None,
+ 'simple-voting-example-head-or-tail'),
+ ('Using the Voting Classifier',
+ 2,
+ None,
+ 'using-the-voting-classifier'),
+ ('Please, not the moons again! Voting and Bagging',
+ 2,
+ None,
+ 'please-not-the-moons-again-voting-and-bagging'),
+ ('Bagging Examples', 2, None, 'bagging-examples'),
+ ('Making your own Bootstrap: Changing the Level of the Decision '
+ 'Tree',
+ 2,
+ None,
+ 'making-your-own-bootstrap-changing-the-level-of-the-decision-tree'),
('Why Voting?', 2, None, 'why-voting'),
('Tossing coins', 2, None, 'tossing-coins'),
('Standard imports first', 2, None, 'standard-imports-first'),
@@ -327,55 +330,55 @@ MathJax.Hub.Config({
-
The CART algorithm for Regression
+Disadvantages
-The CART algorithm for regression works is similar to the one for classification except that instead of trying to split the -training set in a way that minimizes say the gini or entropy impurity, it now tries to split the training set in a way that minimizes our well-known mean-squared error (MSE). The cost function is now -
-$$ -C(k,t_k) = \frac{m_{\mathrm{left}}}{m}\mathrm{MSE}_{\mathrm{left}}+ \frac{m_{\mathrm{right}}}{m}\mathrm{MSE}_{\mathrm{right}}. -$$ - -Here the MSE for a specific node is defined as
-$$ -\mathrm{MSE}_{\mathrm{node}}=\frac{1}{m_\mathrm{node}}\sum_{i\in \mathrm{node}}(\overline{y}_{\mathrm{node}}-y_i)^2, -$$ - -with
-$$ -\overline{y}_{\mathrm{node}}=\frac{1}{m_\mathrm{node}}\sum_{i\in \mathrm{node}}y_i, -$$ - -the mean value of all observations in a specific node.
- -Without any regularization, the regression task for decision trees, -just like for classification tasks, is prone to overfitting. +
-
+
However, by aggregating many decision trees, using methods like +bagging, random forests, and boosting, the predictive performance of +trees can be substantially improved.
diff --git a/doc/pub/week44/html/._week44-bs062.html b/doc/pub/week44/html/._week44-bs062.html index b07a22d25..57d63b163 100644 --- a/doc/pub/week44/html/._week44-bs062.html +++ b/doc/pub/week44/html/._week44-bs062.html @@ -115,67 +115,6 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'the-cart-algorithm-for-regression'), - ('Computing the Gini index', 2, None, 'computing-the-gini-index'), - ('Simple Python Code to read in Data and perform Classification', - 2, - None, - 'simple-python-code-to-read-in-data-and-perform-classification'), - ('Computing the Gini Factor', - 2, - None, - 'computing-the-gini-factor'), - ('Entropy and the ID3 algorithm', - 2, - None, - 'entropy-and-the-id3-algorithm'), - ('Cancer Data again now with Decision Trees and other Methods', - 2, - None, - 'cancer-data-again-now-with-decision-trees-and-other-methods'), - ('Another example, the moons again', - 2, - None, - 'another-example-the-moons-again'), - ('Playing around with regions', - 2, - None, - 'playing-around-with-regions'), - ('Regression trees', 2, None, 'regression-trees'), - ('Final regressor code', 2, None, 'final-regressor-code'), - ('Pros and cons of trees, pros', - 2, - None, - 'pros-and-cons-of-trees-pros'), - ('Disadvantages', 2, None, 'disadvantages'), - ('Ensemble Methods: From a Single Tree to Many Trees and Extreme ' - 'Boosting, Meet the Jungle of Methods', - 2, - None, - 'ensemble-methods-from-a-single-tree-to-many-trees-and-extreme-boosting-meet-the-jungle-of-methods'), - ('An Overview of Ensemble Methods', - 2, - None, - 'an-overview-of-ensemble-methods'), - ('Bagging', 2, None, 'bagging'), - ('More bagging', 2, None, 'more-bagging'), - ('Simple Voting Example, head or tail', - 2, - None, - 'simple-voting-example-head-or-tail'), - ('Using the Voting Classifier', - 2, - None, - 'using-the-voting-classifier'), - ('Please, not the moons again! Voting and Bagging', - 2, - None, - 'please-not-the-moons-again-voting-and-bagging'), - ('Bagging Examples', 2, None, 'bagging-examples'), - ('Making your own Bootstrap: Changing the Level of the Decision ' - 'Tree', - 2, - None, - 'making-your-own-bootstrap-changing-the-level-of-the-decision-tree'), ('Why binary splits?', 2, None, 'why-binary-splits'), ('Computing a Tree using the Gini Index', 2, @@ -250,6 +189,70 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d 'playing-around-with-regions'), ('Regression trees', 2, None, 'regression-trees'), ('Final regressor code', 2, None, 'final-regressor-code'), + ('Example: Computing the Gini index', + 2, + None, + 'example-computing-the-gini-index'), + ('Simple Python Code to read in Data and perform Classification', + 2, + None, + 'simple-python-code-to-read-in-data-and-perform-classification'), + ('Computing the Gini Factor', + 2, + None, + 'computing-the-gini-factor'), + ('Entropy and the ID3 algorithm', + 2, + None, + 'entropy-and-the-id3-algorithm'), + ('Cancer Data again now with Decision Trees and other Methods', + 2, + None, + 'cancer-data-again-now-with-decision-trees-and-other-methods'), + ('Another example, the moons again', + 2, + None, + 'another-example-the-moons-again'), + ('Playing around with regions', + 2, + None, + 'playing-around-with-regions'), + ('Regression trees', 2, None, 'regression-trees'), + ('Final regressor code', 2, None, 'final-regressor-code'), + ('Pros and cons of trees, pros', + 2, + None, + 'pros-and-cons-of-trees-pros'), + ('Disadvantages', 2, None, 'disadvantages'), + ('Ensemble Methods: From a Single Tree to Many Trees and Extreme ' + 'Boosting, Meet the Jungle of Methods', + 2, + None, + 'ensemble-methods-from-a-single-tree-to-many-trees-and-extreme-boosting-meet-the-jungle-of-methods'), + ('An Overview of Ensemble Methods', + 2, + None, + 'an-overview-of-ensemble-methods'), + ('Bagging', 2, None, 'bagging'), + ('More bagging', 2, None, 'more-bagging'), + ('Simple Voting Example, head or tail', + 2, + None, + 'simple-voting-example-head-or-tail'), + ('Using the Voting Classifier', + 2, + None, + 'using-the-voting-classifier'), + ('Please, not the moons again! Voting and Bagging', + 2, + None, + 'please-not-the-moons-again-voting-and-bagging'), + ('Bagging Examples', 2, None, 'bagging-examples'), + ('Making your own Bootstrap: Changing the Level of the Decision ' + 'Tree', + 2, + None, + 'making-your-own-bootstrap-changing-the-level-of-the-decision-tree'), ('Why Voting?', 2, None, 'why-voting'), ('Tossing coins', 2, None, 'tossing-coins'), ('Standard imports first', 2, None, 'standard-imports-first'), @@ -327,55 +330,55 @@ MathJax.Hub.Config({
-
Computing the Gini index
+Ensemble Methods: From a Single Tree to Many Trees and Extreme Boosting, Meet the Jungle of Methods
-The example we will look at is a classical one in many Machine -Learning applications. Based on various meteorological features, we -have several so-called attributes which decide whether we at the end -will do some outdoor activity like skiing, going for a bike ride etc -etc. The table here contains the feautures outlook, temperature, -humidity and wind. The target or output is whether we ride -(True=1) or whether we do something else that day (False=0). The -attributes for each feature are then sunny, overcast and rain for the -outlook, hot, cold and mild for temperature, high and normal for -humidity and weak and strong for wind. +
As stated above and seen in many of the examples discussed here about +a single decision tree, we often end up overfitting our training +data. This normally means that we have a high variance. Can we reduce +the variance of a statistical learning method?
-The table here summarizes the various attributes and
-| Day | Outlook | Temperature | Humidity | Wind | Ride |
| 1 | Sunny | Hot | High | Weak | 0 |
| 2 | Sunny | Hot | High | Strong | 1 |
| 3 | Overcast | Hot | High | Weak | 1 |
| 4 | Rain | Mild | High | Weak | 1 |
| 5 | Rain | Cool | Normal | Weak | 1 |
| 6 | Rain | Cool | Normal | Strong | 0 |
| 7 | Overcast | Cool | Normal | Strong | 1 |
| 8 | Sunny | Mild | High | Weak | 0 |
| 9 | Sunny | Cool | Normal | Weak | 1 |
| 10 | Rain | Mild | Normal | Weak | 1 |
| 11 | Sunny | Mild | Normal | Strong | 1 |
| 12 | Overcast | Mild | High | Strong | 1 |
| 13 | Overcast | Hot | Normal | Weak | 1 |
| 14 | Rain | Mild | High | Strong | 0 |
This leads us to a set of different methods that can combine different +machine learning algorithms or just use one of them to construct +forests and jungles of trees, homogeneous ones or heterogenous +ones. These methods are recognized by different names which we will +try to explain here. These are +
+ +-
+
We discuss these methods here.
diff --git a/doc/pub/week44/html/._week44-bs063.html b/doc/pub/week44/html/._week44-bs063.html index b4b1b6cc5..b0680d6ea 100644 --- a/doc/pub/week44/html/._week44-bs063.html +++ b/doc/pub/week44/html/._week44-bs063.html @@ -115,67 +115,6 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'the-cart-algorithm-for-regression'), - ('Computing the Gini index', 2, None, 'computing-the-gini-index'), - ('Simple Python Code to read in Data and perform Classification', - 2, - None, - 'simple-python-code-to-read-in-data-and-perform-classification'), - ('Computing the Gini Factor', - 2, - None, - 'computing-the-gini-factor'), - ('Entropy and the ID3 algorithm', - 2, - None, - 'entropy-and-the-id3-algorithm'), - ('Cancer Data again now with Decision Trees and other Methods', - 2, - None, - 'cancer-data-again-now-with-decision-trees-and-other-methods'), - ('Another example, the moons again', - 2, - None, - 'another-example-the-moons-again'), - ('Playing around with regions', - 2, - None, - 'playing-around-with-regions'), - ('Regression trees', 2, None, 'regression-trees'), - ('Final regressor code', 2, None, 'final-regressor-code'), - ('Pros and cons of trees, pros', - 2, - None, - 'pros-and-cons-of-trees-pros'), - ('Disadvantages', 2, None, 'disadvantages'), - ('Ensemble Methods: From a Single Tree to Many Trees and Extreme ' - 'Boosting, Meet the Jungle of Methods', - 2, - None, - 'ensemble-methods-from-a-single-tree-to-many-trees-and-extreme-boosting-meet-the-jungle-of-methods'), - ('An Overview of Ensemble Methods', - 2, - None, - 'an-overview-of-ensemble-methods'), - ('Bagging', 2, None, 'bagging'), - ('More bagging', 2, None, 'more-bagging'), - ('Simple Voting Example, head or tail', - 2, - None, - 'simple-voting-example-head-or-tail'), - ('Using the Voting Classifier', - 2, - None, - 'using-the-voting-classifier'), - ('Please, not the moons again! Voting and Bagging', - 2, - None, - 'please-not-the-moons-again-voting-and-bagging'), - ('Bagging Examples', 2, None, 'bagging-examples'), - ('Making your own Bootstrap: Changing the Level of the Decision ' - 'Tree', - 2, - None, - 'making-your-own-bootstrap-changing-the-level-of-the-decision-tree'), ('Why binary splits?', 2, None, 'why-binary-splits'), ('Computing a Tree using the Gini Index', 2, @@ -250,6 +189,70 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d 'playing-around-with-regions'), ('Regression trees', 2, None, 'regression-trees'), ('Final regressor code', 2, None, 'final-regressor-code'), + ('Example: Computing the Gini index', + 2, + None, + 'example-computing-the-gini-index'), + ('Simple Python Code to read in Data and perform Classification', + 2, + None, + 'simple-python-code-to-read-in-data-and-perform-classification'), + ('Computing the Gini Factor', + 2, + None, + 'computing-the-gini-factor'), + ('Entropy and the ID3 algorithm', + 2, + None, + 'entropy-and-the-id3-algorithm'), + ('Cancer Data again now with Decision Trees and other Methods', + 2, + None, + 'cancer-data-again-now-with-decision-trees-and-other-methods'), + ('Another example, the moons again', + 2, + None, + 'another-example-the-moons-again'), + ('Playing around with regions', + 2, + None, + 'playing-around-with-regions'), + ('Regression trees', 2, None, 'regression-trees'), + ('Final regressor code', 2, None, 'final-regressor-code'), + ('Pros and cons of trees, pros', + 2, + None, + 'pros-and-cons-of-trees-pros'), + ('Disadvantages', 2, None, 'disadvantages'), + ('Ensemble Methods: From a Single Tree to Many Trees and Extreme ' + 'Boosting, Meet the Jungle of Methods', + 2, + None, + 'ensemble-methods-from-a-single-tree-to-many-trees-and-extreme-boosting-meet-the-jungle-of-methods'), + ('An Overview of Ensemble Methods', + 2, + None, + 'an-overview-of-ensemble-methods'), + ('Bagging', 2, None, 'bagging'), + ('More bagging', 2, None, 'more-bagging'), + ('Simple Voting Example, head or tail', + 2, + None, + 'simple-voting-example-head-or-tail'), + ('Using the Voting Classifier', + 2, + None, + 'using-the-voting-classifier'), + ('Please, not the moons again! Voting and Bagging', + 2, + None, + 'please-not-the-moons-again-voting-and-bagging'), + ('Bagging Examples', 2, None, 'bagging-examples'), + ('Making your own Bootstrap: Changing the Level of the Decision ' + 'Tree', + 2, + None, + 'making-your-own-bootstrap-changing-the-level-of-the-decision-tree'), ('Why Voting?', 2, None, 'why-voting'), ('Tossing coins', 2, None, 'tossing-coins'), ('Standard imports first', 2, None, 'standard-imports-first'), @@ -327,55 +330,55 @@ MathJax.Hub.Config({
-
Simple Python Code to read in Data and perform Classification
- - - -# Common imports
-import numpy as np
-import pandas as pd
-import matplotlib.pyplot as plt
-from sklearn.tree import DecisionTreeClassifier
-from sklearn.model_selection import train_test_split
-from sklearn.tree import export_graphviz
-from sklearn.preprocessing import StandardScaler, OneHotEncoder
-from sklearn.compose import ColumnTransformer
-from IPython.display import Image
-from pydot import graph_from_dot_data
-import os
-
-# Where to save the figures and data files
-PROJECT_ROOT_DIR = "Results"
-FIGURE_ID = "Results/FigureFiles"
-DATA_ID = "DataFiles/"
-
-if not os.path.exists(PROJECT_ROOT_DIR):
- os.mkdir(PROJECT_ROOT_DIR)
-
-if not os.path.exists(FIGURE_ID):
- os.makedirs(FIGURE_ID)
-
-if not os.path.exists(DATA_ID):
- os.makedirs(DATA_ID)
-
-def image_path(fig_id):
- return os.path.join(FIGURE_ID, fig_id)
-
-def data_path(dat_id):
- return os.path.join(DATA_ID, dat_id)
-
-def save_fig(fig_id):
- plt.savefig(image_path(fig_id) + ".png", format='png')
-
-infile = open(data_path("rideclass.csv"),'r')
-
-# Read the experimental data with Pandas
-from IPython.display import display
-ridedata = pd.read_csv(infile,names = ('Outlook','Temperature','Humidity','Wind','Ride'))
-ridedata = pd.DataFrame(ridedata)
-
-# Features and targets
-X = ridedata.loc[:, ridedata.columns != 'Ride'].values
-y = ridedata.loc[:, ridedata.columns == 'Ride'].values
-
-# Create the encoder.
-encoder = OneHotEncoder(handle_unknown="ignore")
-# Assume for simplicity all features are categorical.
-encoder.fit(X)
-# Apply the encoder.
-X = encoder.transform(X)
-print(X)
-# Then do a Classification tree
-tree_clf = DecisionTreeClassifier(max_depth=2)
-tree_clf.fit(X, y)
-print("Train set accuracy with Decision Tree: {:.2f}".format(tree_clf.score(X,y)))
-#transfer to a decision tree graph
-export_graphviz(
- tree_clf,
- out_file="DataFiles/ride.dot",
- rounded=True,
- filled=True
-)
-cmd = 'dot -Tpng DataFiles/cancer.dot -o DataFiles/cancer.png'
-os.system(cmd)
-
-An Overview of Ensemble Methods
++
diff --git a/doc/pub/week44/html/week44-bs.html b/doc/pub/week44/html/week44-bs.html index 6872062e3..e030ff96d 100644 --- a/doc/pub/week44/html/week44-bs.html +++ b/doc/pub/week44/html/week44-bs.html @@ -115,67 +115,6 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d 2, None, 'the-cart-algorithm-for-regression'), - ('Computing the Gini index', 2, None, 'computing-the-gini-index'), - ('Simple Python Code to read in Data and perform Classification', - 2, - None, - 'simple-python-code-to-read-in-data-and-perform-classification'), - ('Computing the Gini Factor', - 2, - None, - 'computing-the-gini-factor'), - ('Entropy and the ID3 algorithm', - 2, - None, - 'entropy-and-the-id3-algorithm'), - ('Cancer Data again now with Decision Trees and other Methods', - 2, - None, - 'cancer-data-again-now-with-decision-trees-and-other-methods'), - ('Another example, the moons again', - 2, - None, - 'another-example-the-moons-again'), - ('Playing around with regions', - 2, - None, - 'playing-around-with-regions'), - ('Regression trees', 2, None, 'regression-trees'), - ('Final regressor code', 2, None, 'final-regressor-code'), - ('Pros and cons of trees, pros', - 2, - None, - 'pros-and-cons-of-trees-pros'), - ('Disadvantages', 2, None, 'disadvantages'), - ('Ensemble Methods: From a Single Tree to Many Trees and Extreme ' - 'Boosting, Meet the Jungle of Methods', - 2, - None, - 'ensemble-methods-from-a-single-tree-to-many-trees-and-extreme-boosting-meet-the-jungle-of-methods'), - ('An Overview of Ensemble Methods', - 2, - None, - 'an-overview-of-ensemble-methods'), - ('Bagging', 2, None, 'bagging'), - ('More bagging', 2, None, 'more-bagging'), - ('Simple Voting Example, head or tail', - 2, - None, - 'simple-voting-example-head-or-tail'), - ('Using the Voting Classifier', - 2, - None, - 'using-the-voting-classifier'), - ('Please, not the moons again! Voting and Bagging', - 2, - None, - 'please-not-the-moons-again-voting-and-bagging'), - ('Bagging Examples', 2, None, 'bagging-examples'), - ('Making your own Bootstrap: Changing the Level of the Decision ' - 'Tree', - 2, - None, - 'making-your-own-bootstrap-changing-the-level-of-the-decision-tree'), ('Why binary splits?', 2, None, 'why-binary-splits'), ('Computing a Tree using the Gini Index', 2, @@ -250,6 +189,70 @@ doconce format html week44.do.txt --html_style=bootstrap --pygments_html_style=d 'playing-around-with-regions'), ('Regression trees', 2, None, 'regression-trees'), ('Final regressor code', 2, None, 'final-regressor-code'), + ('Example: Computing the Gini index', + 2, + None, + 'example-computing-the-gini-index'), + ('Simple Python Code to read in Data and perform Classification', + 2, + None, + 'simple-python-code-to-read-in-data-and-perform-classification'), + ('Computing the Gini Factor', + 2, + None, + 'computing-the-gini-factor'), + ('Entropy and the ID3 algorithm', + 2, + None, + 'entropy-and-the-id3-algorithm'), + ('Cancer Data again now with Decision Trees and other Methods', + 2, + None, + 'cancer-data-again-now-with-decision-trees-and-other-methods'), + ('Another example, the moons again', + 2, + None, + 'another-example-the-moons-again'), + ('Playing around with regions', + 2, + None, + 'playing-around-with-regions'), + ('Regression trees', 2, None, 'regression-trees'), + ('Final regressor code', 2, None, 'final-regressor-code'), + ('Pros and cons of trees, pros', + 2, + None, + 'pros-and-cons-of-trees-pros'), + ('Disadvantages', 2, None, 'disadvantages'), + ('Ensemble Methods: From a Single Tree to Many Trees and Extreme ' + 'Boosting, Meet the Jungle of Methods', + 2, + None, + 'ensemble-methods-from-a-single-tree-to-many-trees-and-extreme-boosting-meet-the-jungle-of-methods'), + ('An Overview of Ensemble Methods', + 2, + None, + 'an-overview-of-ensemble-methods'), + ('Bagging', 2, None, 'bagging'), + ('More bagging', 2, None, 'more-bagging'), + ('Simple Voting Example, head or tail', + 2, + None, + 'simple-voting-example-head-or-tail'), + ('Using the Voting Classifier', + 2, + None, + 'using-the-voting-classifier'), + ('Please, not the moons again! Voting and Bagging', + 2, + None, + 'please-not-the-moons-again-voting-and-bagging'), + ('Bagging Examples', 2, None, 'bagging-examples'), + ('Making your own Bootstrap: Changing the Level of the Decision ' + 'Tree', + 2, + None, + 'making-your-own-bootstrap-changing-the-level-of-the-decision-tree'), ('Why Voting?', 2, None, 'why-voting'), ('Tossing coins', 2, None, 'tossing-coins'), ('Standard imports first', 2, None, 'standard-imports-first'), @@ -327,55 +330,55 @@ MathJax.Hub.Config({
Oct 30, 2022
+Nov 1, 2022
diff --git a/doc/pub/week44/html/week44-reveal.html b/doc/pub/week44/html/week44-reveal.html index 6261ee517..b2de4daa2 100644 --- a/doc/pub/week44/html/week44-reveal.html +++ b/doc/pub/week44/html/week44-reveal.html @@ -184,7 +184,7 @@ MathJax.Hub.Config({
Oct 30, 2022
+Nov 1, 2022
@@ -1118,6 +1118,446 @@ just like for classification tasks, is prone to overfitting. +
Why binary splits?
+ +It is custom to split to a tree uising binary splits. The reason is +that multiway splits fragment the data too quickly, leaving +insufficient data at the next level down. Multiway splits can be +achieved by a series of binary split and this is normally preferred. +
+Computing a Tree using the Gini Index
+ +Consider the following example with attributes/features and two +possible outcomes (classes) for each attribute. Assume we wish to find some +correlations between the average grade of a student as function of the +number of hours studied and hours slept. We want also to correlate the +grade in a given course with the general trend, whether the students +recently has gotten grades below average or above. +
+ +We have three features/attributes
+-
+
The Table
+ +| Grade Trend | Hours slept | Hours Studied | Grade |
|---|---|---|---|
| Above | Low | High | Above |
| Below | High | Low | Below |
| Above | Low | High | Above |
| Above | High | High | Above |
| Below | Low | High | Below |
| Above | Low | Low | Below |
| Below | High | High | Below |
| Below | Low | High | Below |
| Above | Low | Low | Below |
| Above | High | High | Above |
Computing the various Gini Indices
+ +In computations we will translate all classes into numbers. Being +these binary classes, they can easily be split into ones and zeros. +
+ ++
See handwritten notes for Thursday November 11
+Computing the various Gini Indices, Hours slept
+ ++
See handwritten notes for Thursday November 3
+Computing the various Gini Indices, Hours studied
+ ++
See handwritten notes for Thursday November 3
+For final tree, see the above handwritten notes
+A possible code using Scikit-Learn
+ + + +# Common imports
+import numpy as np
+import pandas as pd
+import matplotlib.pyplot as plt
+from sklearn.tree import DecisionTreeClassifier
+from sklearn.model_selection import train_test_split
+from sklearn.tree import export_graphviz
+from sklearn.preprocessing import StandardScaler, OneHotEncoder
+from sklearn.compose import ColumnTransformer
+from IPython.display import Image
+from pydot import graph_from_dot_data
+import os
+
+# Where to save the figures and data files
+PROJECT_ROOT_DIR = "Results"
+FIGURE_ID = "Results/FigureFiles"
+DATA_ID = "DataFiles/"
+
+if not os.path.exists(PROJECT_ROOT_DIR):
+ os.mkdir(PROJECT_ROOT_DIR)
+
+if not os.path.exists(FIGURE_ID):
+ os.makedirs(FIGURE_ID)
+
+if not os.path.exists(DATA_ID):
+ os.makedirs(DATA_ID)
+
+def image_path(fig_id):
+ return os.path.join(FIGURE_ID, fig_id)
+
+def data_path(dat_id):
+ return os.path.join(DATA_ID, dat_id)
+
+def save_fig(fig_id):
+ plt.savefig(image_path(fig_id) + ".png", format='png')
+
+infile = open(data_path("grades.csv"),'r')
+
+# Read the experimental data with Pandas
+from IPython.display import display
+grades = pd.read_csv(infile,names = ('Trend','Sleep','Studied','Grade'))
+grades = pd.DataFrame(grades)
+
+# Features and targets
+X = grades.loc[:, grades.columns != 'Grade'].values
+y = grades.loc[:, grades.columns == 'Grade'].values
+
+# Create the encoder.
+encoder = OneHotEncoder(handle_unknown="ignore")
+# Assume for simplicity all features are categorical.
+encoder.fit(X)
+# Apply the encoder.
+X = encoder.transform(X)
+print(X)
+# Then do a Classification tree
+tree_clf = DecisionTreeClassifier(max_depth=2)
+tree_clf.fit(X, y)
+print("Train set accuracy with Decision Tree: {:.2f}".format(tree_clf.score(X,y)))
+#transfer to a decision tree graph
+export_graphviz(
+ tree_clf,
+ out_file="DataFiles/grade.dot",
+ rounded=True,
+ filled=True
+)
+cmd = 'dot -Tpng DataFiles/grade.dot -o DataFiles/grades.png'
+os.system(cmd)
+
+Visualizing Trees, More examples
+ + +import os
+from sklearn.datasets import load_breast_cancer
+from sklearn.tree import DecisionTreeClassifier
+from sklearn.model_selection import train_test_split
+from sklearn.metrics import confusion_matrix
+from sklearn.tree import export_graphviz
+
+from IPython.display import Image
+from pydot import graph_from_dot_data
+import pandas as pd
+import numpy as np
+
+
+cancer = load_breast_cancer()
+X = pd.DataFrame(cancer.data, columns=cancer.feature_names)
+print(X)
+y = pd.Categorical.from_codes(cancer.target, cancer.target_names)
+y = pd.get_dummies(y)
+print(y)
+X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=1)
+tree_clf = DecisionTreeClassifier(max_depth=5)
+tree_clf.fit(X_train, y_train)
+
+export_graphviz(
+ tree_clf,
+ out_file="DataFiles/cancer.dot",
+ feature_names=cancer.feature_names,
+ class_names=cancer.target_names,
+ rounded=True,
+ filled=True
+)
+cmd = 'dot -Tpng DataFiles/cancer.dot -o DataFiles/cancer.png'
+os.system(cmd)
+
+Visualizing the Tree, The Moons
+ + +# Common imports
+import numpy as np
+from sklearn.model_selection import train_test_split
+from sklearn.tree import DecisionTreeClassifier
+from sklearn.datasets import make_moons
+from sklearn.tree import export_graphviz
+from pydot import graph_from_dot_data
+import pandas as pd
+import os
+
+np.random.seed(42)
+X, y = make_moons(n_samples=100, noise=0.25, random_state=53)
+X_train, X_test, y_train, y_test = train_test_split(X,y,random_state=0)
+tree_clf = DecisionTreeClassifier(max_depth=5)
+tree_clf.fit(X_train, y_train)
+
+export_graphviz(
+ tree_clf,
+ out_file="DataFiles/moons.dot",
+ rounded=True,
+ filled=True
+)
+cmd = 'dot -Tpng DataFiles/moons.dot -o DataFiles/moons.png'
+os.system(cmd)
+
+Other ways of visualizing the trees
+ +Scikit-Learn has also another way to visualize the trees which is very useful, here with the Iris data.
+ + + +from sklearn.datasets import load_iris
+from sklearn import tree
+X, y = load_iris(return_X_y=True)
+tree_clf = tree.DecisionTreeClassifier()
+tree_clf = tree_clf.fit(X, y)
+# and then plot the tree
+tree.plot_tree(tree_clf)
+
+Printing out as text
+ +Alternatively, the tree can also be exported in textual format with the function exporttext. +This method doesn’t require the installation of external libraries and is more compact: +
+ + + +from sklearn.datasets import load_iris
+from sklearn.tree import DecisionTreeClassifier
+from sklearn.tree import export_text
+iris = load_iris()
+decision_tree = DecisionTreeClassifier(random_state=0, max_depth=2)
+decision_tree = decision_tree.fit(iris.data, iris.target)
+r = export_text(decision_tree, feature_names=iris['feature_names'])
+print(r)
+
+Algorithms for Setting up Decision Trees
+ +Two algorithms stand out in the set up of decision trees:
+-
+
+
We discuss both algorithms with applications here. The popular library +Scikit-Learn uses the CART algorithm. For classification problems +you can use either the gini index or the entropy to split a tree +in two branches. +
+The CART algorithm for Classification
+ +For classification, the CART algorithm splits the data set in two subsets using a single feature \( k \) and a threshold \( t_k \). +This could be for example a threshold set by a number below a certain circumference of a malign tumor. +
+ +How do we find these two quantities? +We search for the pair \( (k,t_k) \) that produces the purest subset using for example the gini factor \( G \). +The cost function it tries to minimize is then +
+
+$$
+C(k,t_k) = \frac{m_{\mathrm{left}}}{m}G_{\mathrm{left}}+ \frac{m_{\mathrm{right}}}{m}G_{\mathrm{right}},
+$$
+
+
+
where \( G_{\mathrm{left/right}} \) measures the impurity of the left/right subset and \( m_{\mathrm{left/right}} \) + is the number of instances in the left/right subset +
+ +Once it has successfully split the training set in two, it splits the subsets using the same logic, then the subsubsets +and so on, recursively. It stops recursing once it reaches the maximum depth (defined by the +\( max\_depth \) hyperparameter), or if it cannot find a split that will reduce impurity. A few other +hyperparameters control additional stopping conditions such as the \( min\_samples\_split \), +\( min\_samples\_leaf \), \( min\_weight\_fraction\_leaf \), and \( max\_leaf\_nodes \). +
+The CART algorithm for Regression
+ +The CART algorithm for regression works is similar to the one for classification except that instead of trying to split the +training set in a way that minimizes say the gini or entropy impurity, it now tries to split the training set in a way that minimizes our well-known mean-squared error (MSE). The cost function is now +
+
+$$
+C(k,t_k) = \frac{m_{\mathrm{left}}}{m}\mathrm{MSE}_{\mathrm{left}}+ \frac{m_{\mathrm{right}}}{m}\mathrm{MSE}_{\mathrm{right}}.
+$$
+
+
+
Here the MSE for a specific node is defined as
+
+$$
+\mathrm{MSE}_{\mathrm{node}}=\frac{1}{m_\mathrm{node}}\sum_{i\in \mathrm{node}}(\overline{y}_{\mathrm{node}}-y_i)^2,
+$$
+
+
+
with
+
+$$
+\overline{y}_{\mathrm{node}}=\frac{1}{m_\mathrm{node}}\sum_{i\in \mathrm{node}}y_i,
+$$
+
+
+
the mean value of all observations in a specific node.
+ +Without any regularization, the regression task for decision trees, +just like for classification tasks, is prone to overfitting. +
+Computing the Gini index
@@ -1743,6 +2183,631 @@ plt.show()Example: Computing the Gini index
+ +The example we will look at is a classical one in many Machine +Learning applications. Based on various meteorological features, we +have several so-called attributes which decide whether we at the end +will do some outdoor activity like skiing, going for a bike ride etc +etc. The table here contains the feautures outlook, temperature, +humidity and wind. The target or output is whether we ride +(True=1) or whether we do something else that day (False=0). The +attributes for each feature are then sunny, overcast and rain for the +outlook, hot, cold and mild for temperature, high and normal for +humidity and weak and strong for wind. +
+ +The table here summarizes the various attributes and
+| Day | Outlook | Temperature | Humidity | Wind | Ride |
|---|---|---|---|---|---|
| 1 | Sunny | Hot | High | Weak | 0 |
| 2 | Sunny | Hot | High | Strong | 1 |
| 3 | Overcast | Hot | High | Weak | 1 |
| 4 | Rain | Mild | High | Weak | 1 |
| 5 | Rain | Cool | Normal | Weak | 1 |
| 6 | Rain | Cool | Normal | Strong | 0 |
| 7 | Overcast | Cool | Normal | Strong | 1 |
| 8 | Sunny | Mild | High | Weak | 0 |
| 9 | Sunny | Cool | Normal | Weak | 1 |
| 10 | Rain | Mild | Normal | Weak | 1 |
| 11 | Sunny | Mild | Normal | Strong | 1 |
| 12 | Overcast | Mild | High | Strong | 1 |
| 13 | Overcast | Hot | Normal | Weak | 1 |
| 14 | Rain | Mild | High | Strong | 0 |
Simple Python Code to read in Data and perform Classification
+ + + +# Common imports
+import numpy as np
+import pandas as pd
+import matplotlib.pyplot as plt
+from sklearn.tree import DecisionTreeClassifier
+from sklearn.model_selection import train_test_split
+from sklearn.tree import export_graphviz
+from sklearn.preprocessing import StandardScaler, OneHotEncoder
+from sklearn.compose import ColumnTransformer
+from IPython.display import Image
+from pydot import graph_from_dot_data
+import os
+
+# Where to save the figures and data files
+PROJECT_ROOT_DIR = "Results"
+FIGURE_ID = "Results/FigureFiles"
+DATA_ID = "DataFiles/"
+
+if not os.path.exists(PROJECT_ROOT_DIR):
+ os.mkdir(PROJECT_ROOT_DIR)
+
+if not os.path.exists(FIGURE_ID):
+ os.makedirs(FIGURE_ID)
+
+if not os.path.exists(DATA_ID):
+ os.makedirs(DATA_ID)
+
+def image_path(fig_id):
+ return os.path.join(FIGURE_ID, fig_id)
+
+def data_path(dat_id):
+ return os.path.join(DATA_ID, dat_id)
+
+def save_fig(fig_id):
+ plt.savefig(image_path(fig_id) + ".png", format='png')
+
+infile = open(data_path("rideclass.csv"),'r')
+
+# Read the experimental data with Pandas
+from IPython.display import display
+ridedata = pd.read_csv(infile,names = ('Outlook','Temperature','Humidity','Wind','Ride'))
+ridedata = pd.DataFrame(ridedata)
+
+# Features and targets
+X = ridedata.loc[:, ridedata.columns != 'Ride'].values
+y = ridedata.loc[:, ridedata.columns == 'Ride'].values
+
+# Create the encoder.
+encoder = OneHotEncoder(handle_unknown="ignore")
+# Assume for simplicity all features are categorical.
+encoder.fit(X)
+# Apply the encoder.
+X = encoder.transform(X)
+print(X)
+# Then do a Classification tree
+tree_clf = DecisionTreeClassifier(max_depth=2)
+tree_clf.fit(X, y)
+print("Train set accuracy with Decision Tree: {:.2f}".format(tree_clf.score(X,y)))
+#transfer to a decision tree graph
+export_graphviz(
+ tree_clf,
+ out_file="DataFiles/ride.dot",
+ rounded=True,
+ filled=True
+)
+cmd = 'dot -Tpng DataFiles/cancer.dot -o DataFiles/cancer.png'
+os.system(cmd)
+
+Computing the Gini Factor
+ +The above functions (gini, entropy and misclassification error) are +important components of the so-called CART algorithm. We will discuss +this algorithm below after we have discussed the information gain +algorithm ID3. +
+ +In the example here we have converted all our attributes into numerical values \( 0,1,2 \) etc.
+ + + +# Split a dataset based on an attribute and an attribute value
+def test_split(index, value, dataset):
+ left, right = list(), list()
+ for row in dataset:
+ if row[index] < value:
+ left.append(row)
+ else:
+ right.append(row)
+ return left, right
+
+# Calculate the Gini index for a split dataset
+def gini_index(groups, classes):
+ # count all samples at split point
+ n_instances = float(sum([len(group) for group in groups]))
+ # sum weighted Gini index for each group
+ gini = 0.0
+ for group in groups:
+ size = float(len(group))
+ # avoid divide by zero
+ if size == 0:
+ continue
+ score = 0.0
+ # score the group based on the score for each class
+ for class_val in classes:
+ p = [row[-1] for row in group].count(class_val) / size
+ score += p * p
+ # weight the group score by its relative size
+ gini += (1.0 - score) * (size / n_instances)
+ return gini
+
+# Select the best split point for a dataset
+def get_split(dataset):
+ class_values = list(set(row[-1] for row in dataset))
+ b_index, b_value, b_score, b_groups = 999, 999, 999, None
+ for index in range(len(dataset[0])-1):
+ for row in dataset:
+ groups = test_split(index, row[index], dataset)
+ gini = gini_index(groups, class_values)
+ print('X%d < %.3f Gini=%.3f' % ((index+1), row[index], gini))
+ if gini < b_score:
+ b_index, b_value, b_score, b_groups = index, row[index], gini, groups
+ return {'index':b_index, 'value':b_value, 'groups':b_groups}
+
+dataset = [[0,0,0,0,0],
+ [0,0,0,1,1],
+ [1,0,0,0,1],
+ [2,1,0,0,1],
+ [2,2,1,0,1],
+ [2,2,1,1,0],
+ [1,2,1,1,1],
+ [0,1,0,0,0],
+ [0,2,1,0,1],
+ [2,1,1,0,1],
+ [0,1,1,1,1],
+ [1,1,0,1,1],
+ [1,0,1,0,1],
+ [2,1,0,1,0]]
+
+split = get_split(dataset)
+print('Split: [X%d < %.3f]' % ((split['index']+1), split['value']))
+
+Entropy and the ID3 algorithm
+ +The ID3 algorithm learns decision trees by constructing +them in a top down way, beginning with the question which attribute should be tested at the root of the tree? +
+ +-
+
+
The ID3 algorithm selects which attribute to test at each node in the +tree. +
+ +We would like to select the attribute that is most useful for classifying +examples. +
+ +What is a good quantitative measure of the worth of an attribute?
+ +Information gain measures how well a given attribute separates the +training examples according to their target classification. +
+ +The ID3 algorithm uses this information gain measure to select among the candidate +attributes at each step while growing the tree. +
+Cancer Data again now with Decision Trees and other Methods
+ + +import matplotlib.pyplot as plt
+import numpy as np
+from sklearn.model_selection import train_test_split
+from sklearn.datasets import load_breast_cancer
+from sklearn.svm import SVC
+from sklearn.linear_model import LogisticRegression
+from sklearn.tree import DecisionTreeClassifier
+
+# Load the data
+cancer = load_breast_cancer()
+
+X_train, X_test, y_train, y_test = train_test_split(cancer.data,cancer.target,random_state=0)
+print(X_train.shape)
+print(X_test.shape)
+# Logistic Regression
+logreg = LogisticRegression(solver='lbfgs')
+logreg.fit(X_train, y_train)
+print("Test set accuracy with Logistic Regression: {:.2f}".format(logreg.score(X_test,y_test)))
+# Support vector machine
+svm = SVC(gamma='auto', C=100)
+svm.fit(X_train, y_train)
+print("Test set accuracy with SVM: {:.2f}".format(svm.score(X_test,y_test)))
+# Decision Trees
+deep_tree_clf = DecisionTreeClassifier(max_depth=None)
+deep_tree_clf.fit(X_train, y_train)
+print("Test set accuracy with Decision Trees: {:.2f}".format(deep_tree_clf.score(X_test,y_test)))
+#now scale the data
+from sklearn.preprocessing import StandardScaler
+scaler = StandardScaler()
+scaler.fit(X_train)
+X_train_scaled = scaler.transform(X_train)
+X_test_scaled = scaler.transform(X_test)
+# Logistic Regression
+logreg.fit(X_train_scaled, y_train)
+print("Test set accuracy Logistic Regression with scaled data: {:.2f}".format(logreg.score(X_test_scaled,y_test)))
+# Support Vector Machine
+svm.fit(X_train_scaled, y_train)
+print("Test set accuracy SVM with scaled data: {:.2f}".format(logreg.score(X_test_scaled,y_test)))
+# Decision Trees
+deep_tree_clf.fit(X_train_scaled, y_train)
+print("Test set accuracy with Decision Trees and scaled data: {:.2f}".format(deep_tree_clf.score(X_test_scaled,y_test)))
+
+Another example, the moons again
+ + +from __future__ import division, print_function, unicode_literals
+
+# Common imports
+import numpy as np
+import os
+
+# to make this notebook's output stable across runs
+np.random.seed(42)
+
+# To plot pretty figures
+import matplotlib
+import matplotlib.pyplot as plt
+from matplotlib.colors import ListedColormap
+plt.rcParams['axes.labelsize'] = 14
+plt.rcParams['xtick.labelsize'] = 12
+plt.rcParams['ytick.labelsize'] = 12
+
+
+from sklearn.svm import SVC
+from sklearn import datasets
+from sklearn.tree import DecisionTreeClassifier
+from sklearn.datasets import make_moons
+from sklearn.tree import export_graphviz
+
+Xm, ym = make_moons(n_samples=100, noise=0.25, random_state=53)
+
+deep_tree_clf1 = DecisionTreeClassifier(random_state=42)
+deep_tree_clf2 = DecisionTreeClassifier(min_samples_leaf=4, random_state=42)
+deep_tree_clf1.fit(Xm, ym)
+deep_tree_clf2.fit(Xm, ym)
+
+
+def plot_decision_boundary(clf, X, y, axes=[0, 7.5, 0, 3], iris=True, legend=False, plot_training=True):
+ x1s = np.linspace(axes[0], axes[1], 100)
+ x2s = np.linspace(axes[2], axes[3], 100)
+ x1, x2 = np.meshgrid(x1s, x2s)
+ X_new = np.c_[x1.ravel(), x2.ravel()]
+ y_pred = clf.predict(X_new).reshape(x1.shape)
+ custom_cmap = ListedColormap(['#fafab0','#9898ff','#a0faa0'])
+ plt.contourf(x1, x2, y_pred, alpha=0.3, cmap=custom_cmap)
+ if not iris:
+ custom_cmap2 = ListedColormap(['#7d7d58','#4c4c7f','#507d50'])
+ plt.contour(x1, x2, y_pred, cmap=custom_cmap2, alpha=0.8)
+ if plot_training:
+ plt.plot(X[:, 0][y==0], X[:, 1][y==0], "yo", label="Iris-Setosa")
+ plt.plot(X[:, 0][y==1], X[:, 1][y==1], "bs", label="Iris-Versicolor")
+ plt.plot(X[:, 0][y==2], X[:, 1][y==2], "g^", label="Iris-Virginica")
+ plt.axis(axes)
+ if iris:
+ plt.xlabel("Petal length", fontsize=14)
+ plt.ylabel("Petal width", fontsize=14)
+ else:
+ plt.xlabel(r"$x_1$", fontsize=18)
+ plt.ylabel(r"$x_2$", fontsize=18, rotation=0)
+ if legend:
+ plt.legend(loc="lower right", fontsize=14)
+plt.figure(figsize=(11, 4))
+plt.subplot(121)
+plot_decision_boundary(deep_tree_clf1, Xm, ym, axes=[-1.5, 2.5, -1, 1.5], iris=False)
+plt.title("No restrictions", fontsize=16)
+plt.subplot(122)
+plot_decision_boundary(deep_tree_clf2, Xm, ym, axes=[-1.5, 2.5, -1, 1.5], iris=False)
+plt.title("min_samples_leaf = {}".format(deep_tree_clf2.min_samples_leaf), fontsize=14)
+plt.show()
+
+Playing around with regions
+ + +np.random.seed(6)
+Xs = np.random.rand(100, 2) - 0.5
+ys = (Xs[:, 0] > 0).astype(np.float32) * 2
+
+angle = np.pi/4
+rotation_matrix = np.array([[np.cos(angle), -np.sin(angle)], [np.sin(angle), np.cos(angle)]])
+Xsr = Xs.dot(rotation_matrix)
+
+tree_clf_s = DecisionTreeClassifier(random_state=42)
+tree_clf_s.fit(Xs, ys)
+tree_clf_sr = DecisionTreeClassifier(random_state=42)
+tree_clf_sr.fit(Xsr, ys)
+
+plt.figure(figsize=(11, 4))
+plt.subplot(121)
+plot_decision_boundary(tree_clf_s, Xs, ys, axes=[-0.7, 0.7, -0.7, 0.7], iris=False)
+plt.subplot(122)
+plot_decision_boundary(tree_clf_sr, Xsr, ys, axes=[-0.7, 0.7, -0.7, 0.7], iris=False)
+
+plt.show()
+
+Regression trees
+ + +# Quadratic training set + noise
+np.random.seed(42)
+m = 200
+X = np.random.rand(m, 1)
+y = 4 * (X - 0.5) ** 2
+y = y + np.random.randn(m, 1) / 10
+
+from sklearn.tree import DecisionTreeRegressor
+
+tree_reg = DecisionTreeRegressor(max_depth=2, random_state=42)
+tree_reg.fit(X, y)
+
+Final regressor code
+ + +from sklearn.tree import DecisionTreeRegressor
+
+tree_reg1 = DecisionTreeRegressor(random_state=42, max_depth=2)
+tree_reg2 = DecisionTreeRegressor(random_state=42, max_depth=3)
+tree_reg1.fit(X, y)
+tree_reg2.fit(X, y)
+
+def plot_regression_predictions(tree_reg, X, y, axes=[0, 1, -0.2, 1], ylabel="$y$"):
+ x1 = np.linspace(axes[0], axes[1], 500).reshape(-1, 1)
+ y_pred = tree_reg.predict(x1)
+ plt.axis(axes)
+ plt.xlabel("$x_1$", fontsize=18)
+ if ylabel:
+ plt.ylabel(ylabel, fontsize=18, rotation=0)
+ plt.plot(X, y, "b.")
+ plt.plot(x1, y_pred, "r.-", linewidth=2, label=r"$\hat{y}$")
+
+plt.figure(figsize=(11, 4))
+plt.subplot(121)
+plot_regression_predictions(tree_reg1, X, y)
+for split, style in ((0.1973, "k-"), (0.0917, "k--"), (0.7718, "k--")):
+ plt.plot([split, split], [-0.2, 1], style, linewidth=2)
+plt.text(0.21, 0.65, "Depth=0", fontsize=15)
+plt.text(0.01, 0.2, "Depth=1", fontsize=13)
+plt.text(0.65, 0.8, "Depth=1", fontsize=13)
+plt.legend(loc="upper center", fontsize=18)
+plt.title("max_depth=2", fontsize=14)
+
+plt.subplot(122)
+plot_regression_predictions(tree_reg2, X, y, ylabel=None)
+for split, style in ((0.1973, "k-"), (0.0917, "k--"), (0.7718, "k--")):
+ plt.plot([split, split], [-0.2, 1], style, linewidth=2)
+for split in (0.0458, 0.1298, 0.2873, 0.9040):
+ plt.plot([split, split], [-0.2, 1], "k:", linewidth=1)
+plt.text(0.3, 0.5, "Depth=2", fontsize=13)
+plt.title("max_depth=3", fontsize=14)
+
+plt.show()
+
+tree_reg1 = DecisionTreeRegressor(random_state=42)
+tree_reg2 = DecisionTreeRegressor(random_state=42, min_samples_leaf=10)
+tree_reg1.fit(X, y)
+tree_reg2.fit(X, y)
+
+x1 = np.linspace(0, 1, 500).reshape(-1, 1)
+y_pred1 = tree_reg1.predict(x1)
+y_pred2 = tree_reg2.predict(x1)
+
+plt.figure(figsize=(11, 4))
+
+plt.subplot(121)
+plt.plot(X, y, "b.")
+plt.plot(x1, y_pred1, "r.-", linewidth=2, label=r"$\hat{y}$")
+plt.axis([0, 1, -0.2, 1.1])
+plt.xlabel("$x_1$", fontsize=18)
+plt.ylabel("$y$", fontsize=18, rotation=0)
+plt.legend(loc="upper center", fontsize=18)
+plt.title("No restrictions", fontsize=14)
+
+plt.subplot(122)
+plt.plot(X, y, "b.")
+plt.plot(x1, y_pred2, "r.-", linewidth=2, label=r"$\hat{y}$")
+plt.axis([0, 1, -0.2, 1.1])
+plt.xlabel("$x_1$", fontsize=18)
+plt.title("min_samples_leaf={}".format(tree_reg2.min_samples_leaf), fontsize=14)
+
+plt.show()
+
+Pros and cons of trees, pros
@@ -2295,1071 +3360,6 @@ plt.show()Why binary splits?
- -It is custom to split to a tree uising binary splits. The reason is -that multiway splits fragment the data too quickly, leaving -insufficient data at the next level down. Multiway splits can be -achieved by a series of binary split and this is normally preferred. -
-Computing a Tree using the Gini Index
- -Consider the following example with attributes/features and two -possible outcomes (classes) for each attribute. Assume we wish to find some -correlations between the average grade of a student as function of the -number of hours studied and hours slept. We want also to correlate the -grade in a given course with the general trend, whether the students -recently has gotten grades below average or above. -
- -We have three features/attributes
--
-
The Table
- -| Grade Trend | Hours slept | Hours Studied | Grade |
|---|---|---|---|
| Above | Low | High | Above |
| Below | High | Low | Below |
| Above | Low | High | Above |
| Above | High | High | Above |
| Below | Low | High | Below |
| Above | Low | Low | Below |
| Below | High | High | Below |
| Below | Low | High | Below |
| Above | Low | Low | Below |
| Above | High | High | Above |
Computing the various Gini Indices
- -In computations we will translate all classes into numbers. Being -these binary classes, they can easily be split into ones and zeros. -
- --
See handwritten notes for Thursday November 11
-Computing the various Gini Indices, Hours slept
- --
See handwritten notes for Thursday November 11
-Computing the various Gini Indices, Hours studied
- --
See handwritten notes for Thursday November 11
-For final tree, see the above handwritten notes
-A possible code using Scikit-Learn
- - - -# Common imports
-import numpy as np
-import pandas as pd
-import matplotlib.pyplot as plt
-from sklearn.tree import DecisionTreeClassifier
-from sklearn.model_selection import train_test_split
-from sklearn.tree import export_graphviz
-from sklearn.preprocessing import StandardScaler, OneHotEncoder
-from sklearn.compose import ColumnTransformer
-from IPython.display import Image
-from pydot import graph_from_dot_data
-import os
-
-# Where to save the figures and data files
-PROJECT_ROOT_DIR = "Results"
-FIGURE_ID = "Results/FigureFiles"
-DATA_ID = "DataFiles/"
-
-if not os.path.exists(PROJECT_ROOT_DIR):
- os.mkdir(PROJECT_ROOT_DIR)
-
-if not os.path.exists(FIGURE_ID):
- os.makedirs(FIGURE_ID)
-
-if not os.path.exists(DATA_ID):
- os.makedirs(DATA_ID)
-
-def image_path(fig_id):
- return os.path.join(FIGURE_ID, fig_id)
-
-def data_path(dat_id):
- return os.path.join(DATA_ID, dat_id)
-
-def save_fig(fig_id):
- plt.savefig(image_path(fig_id) + ".png", format='png')
-
-infile = open(data_path("grades.csv"),'r')
-
-# Read the experimental data with Pandas
-from IPython.display import display
-grades = pd.read_csv(infile,names = ('Trend','Sleep','Studied','Grade'))
-grades = pd.DataFrame(grades)
-
-# Features and targets
-X = grades.loc[:, grades.columns != 'Grade'].values
-y = grades.loc[:, grades.columns == 'Grade'].values
-
-# Create the encoder.
-encoder = OneHotEncoder(handle_unknown="ignore")
-# Assume for simplicity all features are categorical.
-encoder.fit(X)
-# Apply the encoder.
-X = encoder.transform(X)
-print(X)
-# Then do a Classification tree
-tree_clf = DecisionTreeClassifier(max_depth=2)
-tree_clf.fit(X, y)
-print("Train set accuracy with Decision Tree: {:.2f}".format(tree_clf.score(X,y)))
-#transfer to a decision tree graph
-export_graphviz(
- tree_clf,
- out_file="DataFiles/grade.dot",
- rounded=True,
- filled=True
-)
-cmd = 'dot -Tpng DataFiles/grade.dot -o DataFiles/grades.png'
-os.system(cmd)
-
-Visualizing Trees, More examples
- - -import os
-from sklearn.datasets import load_breast_cancer
-from sklearn.tree import DecisionTreeClassifier
-from sklearn.model_selection import train_test_split
-from sklearn.metrics import confusion_matrix
-from sklearn.tree import export_graphviz
-
-from IPython.display import Image
-from pydot import graph_from_dot_data
-import pandas as pd
-import numpy as np
-
-
-cancer = load_breast_cancer()
-X = pd.DataFrame(cancer.data, columns=cancer.feature_names)
-print(X)
-y = pd.Categorical.from_codes(cancer.target, cancer.target_names)
-y = pd.get_dummies(y)
-print(y)
-X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=1)
-tree_clf = DecisionTreeClassifier(max_depth=5)
-tree_clf.fit(X_train, y_train)
-
-export_graphviz(
- tree_clf,
- out_file="DataFiles/cancer.dot",
- feature_names=cancer.feature_names,
- class_names=cancer.target_names,
- rounded=True,
- filled=True
-)
-cmd = 'dot -Tpng DataFiles/cancer.dot -o DataFiles/cancer.png'
-os.system(cmd)
-
-Visualizing the Tree, The Moons
- - -# Common imports
-import numpy as np
-from sklearn.model_selection import train_test_split
-from sklearn.tree import DecisionTreeClassifier
-from sklearn.datasets import make_moons
-from sklearn.tree import export_graphviz
-from pydot import graph_from_dot_data
-import pandas as pd
-import os
-
-np.random.seed(42)
-X, y = make_moons(n_samples=100, noise=0.25, random_state=53)
-X_train, X_test, y_train, y_test = train_test_split(X,y,random_state=0)
-tree_clf = DecisionTreeClassifier(max_depth=5)
-tree_clf.fit(X_train, y_train)
-
-export_graphviz(
- tree_clf,
- out_file="DataFiles/moons.dot",
- rounded=True,
- filled=True
-)
-cmd = 'dot -Tpng DataFiles/moons.dot -o DataFiles/moons.png'
-os.system(cmd)
-
-Other ways of visualizing the trees
- -Scikit-Learn has also another way to visualize the trees which is very useful, here with the Iris data.
- - - -from sklearn.datasets import load_iris
-from sklearn import tree
-X, y = load_iris(return_X_y=True)
-tree_clf = tree.DecisionTreeClassifier()
-tree_clf = tree_clf.fit(X, y)
-# and then plot the tree
-tree.plot_tree(tree_clf)
-
-Printing out as text
- -Alternatively, the tree can also be exported in textual format with the function exporttext. -This method doesn’t require the installation of external libraries and is more compact: -
- - - -from sklearn.datasets import load_iris
-from sklearn.tree import DecisionTreeClassifier
-from sklearn.tree import export_text
-iris = load_iris()
-decision_tree = DecisionTreeClassifier(random_state=0, max_depth=2)
-decision_tree = decision_tree.fit(iris.data, iris.target)
-r = export_text(decision_tree, feature_names=iris['feature_names'])
-print(r)
-
-Algorithms for Setting up Decision Trees
- -Two algorithms stand out in the set up of decision trees:
--
-
-
We discuss both algorithms with applications here. The popular library -Scikit-Learn uses the CART algorithm. For classification problems -you can use either the gini index or the entropy to split a tree -in two branches. -
-The CART algorithm for Classification
- -For classification, the CART algorithm splits the data set in two subsets using a single feature \( k \) and a threshold \( t_k \). -This could be for example a threshold set by a number below a certain circumference of a malign tumor. -
- -How do we find these two quantities? -We search for the pair \( (k,t_k) \) that produces the purest subset using for example the gini factor \( G \). -The cost function it tries to minimize is then -
-
-$$
-C(k,t_k) = \frac{m_{\mathrm{left}}}{m}G_{\mathrm{left}}+ \frac{m_{\mathrm{right}}}{m}G_{\mathrm{right}},
-$$
-
-
-
where \( G_{\mathrm{left/right}} \) measures the impurity of the left/right subset and \( m_{\mathrm{left/right}} \) - is the number of instances in the left/right subset -
- -Once it has successfully split the training set in two, it splits the subsets using the same logic, then the subsubsets -and so on, recursively. It stops recursing once it reaches the maximum depth (defined by the -\( max\_depth \) hyperparameter), or if it cannot find a split that will reduce impurity. A few other -hyperparameters control additional stopping conditions such as the \( min\_samples\_split \), -\( min\_samples\_leaf \), \( min\_weight\_fraction\_leaf \), and \( max\_leaf\_nodes \). -
-The CART algorithm for Regression
- -The CART algorithm for regression works is similar to the one for classification except that instead of trying to split the -training set in a way that minimizes say the gini or entropy impurity, it now tries to split the training set in a way that minimizes our well-known mean-squared error (MSE). The cost function is now -
-
-$$
-C(k,t_k) = \frac{m_{\mathrm{left}}}{m}\mathrm{MSE}_{\mathrm{left}}+ \frac{m_{\mathrm{right}}}{m}\mathrm{MSE}_{\mathrm{right}}.
-$$
-
-
-
Here the MSE for a specific node is defined as
-
-$$
-\mathrm{MSE}_{\mathrm{node}}=\frac{1}{m_\mathrm{node}}\sum_{i\in \mathrm{node}}(\overline{y}_{\mathrm{node}}-y_i)^2,
-$$
-
-
-
with
-
-$$
-\overline{y}_{\mathrm{node}}=\frac{1}{m_\mathrm{node}}\sum_{i\in \mathrm{node}}y_i,
-$$
-
-
-
the mean value of all observations in a specific node.
- -Without any regularization, the regression task for decision trees, -just like for classification tasks, is prone to overfitting. -
-Computing the Gini index
- -The example we will look at is a classical one in many Machine -Learning applications. Based on various meteorological features, we -have several so-called attributes which decide whether we at the end -will do some outdoor activity like skiing, going for a bike ride etc -etc. The table here contains the feautures outlook, temperature, -humidity and wind. The target or output is whether we ride -(True=1) or whether we do something else that day (False=0). The -attributes for each feature are then sunny, overcast and rain for the -outlook, hot, cold and mild for temperature, high and normal for -humidity and weak and strong for wind. -
- -The table here summarizes the various attributes and
-| Day | Outlook | Temperature | Humidity | Wind | Ride |
|---|---|---|---|---|---|
| 1 | Sunny | Hot | High | Weak | 0 |
| 2 | Sunny | Hot | High | Strong | 1 |
| 3 | Overcast | Hot | High | Weak | 1 |
| 4 | Rain | Mild | High | Weak | 1 |
| 5 | Rain | Cool | Normal | Weak | 1 |
| 6 | Rain | Cool | Normal | Strong | 0 |
| 7 | Overcast | Cool | Normal | Strong | 1 |
| 8 | Sunny | Mild | High | Weak | 0 |
| 9 | Sunny | Cool | Normal | Weak | 1 |
| 10 | Rain | Mild | Normal | Weak | 1 |
| 11 | Sunny | Mild | Normal | Strong | 1 |
| 12 | Overcast | Mild | High | Strong | 1 |
| 13 | Overcast | Hot | Normal | Weak | 1 |
| 14 | Rain | Mild | High | Strong | 0 |
Simple Python Code to read in Data and perform Classification
- - - -# Common imports
-import numpy as np
-import pandas as pd
-import matplotlib.pyplot as plt
-from sklearn.tree import DecisionTreeClassifier
-from sklearn.model_selection import train_test_split
-from sklearn.tree import export_graphviz
-from sklearn.preprocessing import StandardScaler, OneHotEncoder
-from sklearn.compose import ColumnTransformer
-from IPython.display import Image
-from pydot import graph_from_dot_data
-import os
-
-# Where to save the figures and data files
-PROJECT_ROOT_DIR = "Results"
-FIGURE_ID = "Results/FigureFiles"
-DATA_ID = "DataFiles/"
-
-if not os.path.exists(PROJECT_ROOT_DIR):
- os.mkdir(PROJECT_ROOT_DIR)
-
-if not os.path.exists(FIGURE_ID):
- os.makedirs(FIGURE_ID)
-
-if not os.path.exists(DATA_ID):
- os.makedirs(DATA_ID)
-
-def image_path(fig_id):
- return os.path.join(FIGURE_ID, fig_id)
-
-def data_path(dat_id):
- return os.path.join(DATA_ID, dat_id)
-
-def save_fig(fig_id):
- plt.savefig(image_path(fig_id) + ".png", format='png')
-
-infile = open(data_path("rideclass.csv"),'r')
-
-# Read the experimental data with Pandas
-from IPython.display import display
-ridedata = pd.read_csv(infile,names = ('Outlook','Temperature','Humidity','Wind','Ride'))
-ridedata = pd.DataFrame(ridedata)
-
-# Features and targets
-X = ridedata.loc[:, ridedata.columns != 'Ride'].values
-y = ridedata.loc[:, ridedata.columns == 'Ride'].values
-
-# Create the encoder.
-encoder = OneHotEncoder(handle_unknown="ignore")
-# Assume for simplicity all features are categorical.
-encoder.fit(X)
-# Apply the encoder.
-X = encoder.transform(X)
-print(X)
-# Then do a Classification tree
-tree_clf = DecisionTreeClassifier(max_depth=2)
-tree_clf.fit(X, y)
-print("Train set accuracy with Decision Tree: {:.2f}".format(tree_clf.score(X,y)))
-#transfer to a decision tree graph
-export_graphviz(
- tree_clf,
- out_file="DataFiles/ride.dot",
- rounded=True,
- filled=True
-)
-cmd = 'dot -Tpng DataFiles/cancer.dot -o DataFiles/cancer.png'
-os.system(cmd)
-
-Computing the Gini Factor
- -The above functions (gini, entropy and misclassification error) are -important components of the so-called CART algorithm. We will discuss -this algorithm below after we have discussed the information gain -algorithm ID3. -
- -In the example here we have converted all our attributes into numerical values \( 0,1,2 \) etc.
- - - -# Split a dataset based on an attribute and an attribute value
-def test_split(index, value, dataset):
- left, right = list(), list()
- for row in dataset:
- if row[index] < value:
- left.append(row)
- else:
- right.append(row)
- return left, right
-
-# Calculate the Gini index for a split dataset
-def gini_index(groups, classes):
- # count all samples at split point
- n_instances = float(sum([len(group) for group in groups]))
- # sum weighted Gini index for each group
- gini = 0.0
- for group in groups:
- size = float(len(group))
- # avoid divide by zero
- if size == 0:
- continue
- score = 0.0
- # score the group based on the score for each class
- for class_val in classes:
- p = [row[-1] for row in group].count(class_val) / size
- score += p * p
- # weight the group score by its relative size
- gini += (1.0 - score) * (size / n_instances)
- return gini
-
-# Select the best split point for a dataset
-def get_split(dataset):
- class_values = list(set(row[-1] for row in dataset))
- b_index, b_value, b_score, b_groups = 999, 999, 999, None
- for index in range(len(dataset[0])-1):
- for row in dataset:
- groups = test_split(index, row[index], dataset)
- gini = gini_index(groups, class_values)
- print('X%d < %.3f Gini=%.3f' % ((index+1), row[index], gini))
- if gini < b_score:
- b_index, b_value, b_score, b_groups = index, row[index], gini, groups
- return {'index':b_index, 'value':b_value, 'groups':b_groups}
-
-dataset = [[0,0,0,0,0],
- [0,0,0,1,1],
- [1,0,0,0,1],
- [2,1,0,0,1],
- [2,2,1,0,1],
- [2,2,1,1,0],
- [1,2,1,1,1],
- [0,1,0,0,0],
- [0,2,1,0,1],
- [2,1,1,0,1],
- [0,1,1,1,1],
- [1,1,0,1,1],
- [1,0,1,0,1],
- [2,1,0,1,0]]
-
-split = get_split(dataset)
-print('Split: [X%d < %.3f]' % ((split['index']+1), split['value']))
-
-Entropy and the ID3 algorithm
- -The ID3 algorithm learns decision trees by constructing -them in a top down way, beginning with the question which attribute should be tested at the root of the tree? -
- --
-
-
The ID3 algorithm selects which attribute to test at each node in the -tree. -
- -We would like to select the attribute that is most useful for classifying -examples. -
- -What is a good quantitative measure of the worth of an attribute?
- -Information gain measures how well a given attribute separates the -training examples according to their target classification. -
- -The ID3 algorithm uses this information gain measure to select among the candidate -attributes at each step while growing the tree. -
-Cancer Data again now with Decision Trees and other Methods
- - -import matplotlib.pyplot as plt
-import numpy as np
-from sklearn.model_selection import train_test_split
-from sklearn.datasets import load_breast_cancer
-from sklearn.svm import SVC
-from sklearn.linear_model import LogisticRegression
-from sklearn.tree import DecisionTreeClassifier
-
-# Load the data
-cancer = load_breast_cancer()
-
-X_train, X_test, y_train, y_test = train_test_split(cancer.data,cancer.target,random_state=0)
-print(X_train.shape)
-print(X_test.shape)
-# Logistic Regression
-logreg = LogisticRegression(solver='lbfgs')
-logreg.fit(X_train, y_train)
-print("Test set accuracy with Logistic Regression: {:.2f}".format(logreg.score(X_test,y_test)))
-# Support vector machine
-svm = SVC(gamma='auto', C=100)
-svm.fit(X_train, y_train)
-print("Test set accuracy with SVM: {:.2f}".format(svm.score(X_test,y_test)))
-# Decision Trees
-deep_tree_clf = DecisionTreeClassifier(max_depth=None)
-deep_tree_clf.fit(X_train, y_train)
-print("Test set accuracy with Decision Trees: {:.2f}".format(deep_tree_clf.score(X_test,y_test)))
-#now scale the data
-from sklearn.preprocessing import StandardScaler
-scaler = StandardScaler()
-scaler.fit(X_train)
-X_train_scaled = scaler.transform(X_train)
-X_test_scaled = scaler.transform(X_test)
-# Logistic Regression
-logreg.fit(X_train_scaled, y_train)
-print("Test set accuracy Logistic Regression with scaled data: {:.2f}".format(logreg.score(X_test_scaled,y_test)))
-# Support Vector Machine
-svm.fit(X_train_scaled, y_train)
-print("Test set accuracy SVM with scaled data: {:.2f}".format(logreg.score(X_test_scaled,y_test)))
-# Decision Trees
-deep_tree_clf.fit(X_train_scaled, y_train)
-print("Test set accuracy with Decision Trees and scaled data: {:.2f}".format(deep_tree_clf.score(X_test_scaled,y_test)))
-
-Another example, the moons again
- - -from __future__ import division, print_function, unicode_literals
-
-# Common imports
-import numpy as np
-import os
-
-# to make this notebook's output stable across runs
-np.random.seed(42)
-
-# To plot pretty figures
-import matplotlib
-import matplotlib.pyplot as plt
-from matplotlib.colors import ListedColormap
-plt.rcParams['axes.labelsize'] = 14
-plt.rcParams['xtick.labelsize'] = 12
-plt.rcParams['ytick.labelsize'] = 12
-
-
-from sklearn.svm import SVC
-from sklearn import datasets
-from sklearn.tree import DecisionTreeClassifier
-from sklearn.datasets import make_moons
-from sklearn.tree import export_graphviz
-
-Xm, ym = make_moons(n_samples=100, noise=0.25, random_state=53)
-
-deep_tree_clf1 = DecisionTreeClassifier(random_state=42)
-deep_tree_clf2 = DecisionTreeClassifier(min_samples_leaf=4, random_state=42)
-deep_tree_clf1.fit(Xm, ym)
-deep_tree_clf2.fit(Xm, ym)
-
-
-def plot_decision_boundary(clf, X, y, axes=[0, 7.5, 0, 3], iris=True, legend=False, plot_training=True):
- x1s = np.linspace(axes[0], axes[1], 100)
- x2s = np.linspace(axes[2], axes[3], 100)
- x1, x2 = np.meshgrid(x1s, x2s)
- X_new = np.c_[x1.ravel(), x2.ravel()]
- y_pred = clf.predict(X_new).reshape(x1.shape)
- custom_cmap = ListedColormap(['#fafab0','#9898ff','#a0faa0'])
- plt.contourf(x1, x2, y_pred, alpha=0.3, cmap=custom_cmap)
- if not iris:
- custom_cmap2 = ListedColormap(['#7d7d58','#4c4c7f','#507d50'])
- plt.contour(x1, x2, y_pred, cmap=custom_cmap2, alpha=0.8)
- if plot_training:
- plt.plot(X[:, 0][y==0], X[:, 1][y==0], "yo", label="Iris-Setosa")
- plt.plot(X[:, 0][y==1], X[:, 1][y==1], "bs", label="Iris-Versicolor")
- plt.plot(X[:, 0][y==2], X[:, 1][y==2], "g^", label="Iris-Virginica")
- plt.axis(axes)
- if iris:
- plt.xlabel("Petal length", fontsize=14)
- plt.ylabel("Petal width", fontsize=14)
- else:
- plt.xlabel(r"$x_1$", fontsize=18)
- plt.ylabel(r"$x_2$", fontsize=18, rotation=0)
- if legend:
- plt.legend(loc="lower right", fontsize=14)
-plt.figure(figsize=(11, 4))
-plt.subplot(121)
-plot_decision_boundary(deep_tree_clf1, Xm, ym, axes=[-1.5, 2.5, -1, 1.5], iris=False)
-plt.title("No restrictions", fontsize=16)
-plt.subplot(122)
-plot_decision_boundary(deep_tree_clf2, Xm, ym, axes=[-1.5, 2.5, -1, 1.5], iris=False)
-plt.title("min_samples_leaf = {}".format(deep_tree_clf2.min_samples_leaf), fontsize=14)
-plt.show()
-
-Playing around with regions
- - -np.random.seed(6)
-Xs = np.random.rand(100, 2) - 0.5
-ys = (Xs[:, 0] > 0).astype(np.float32) * 2
-
-angle = np.pi/4
-rotation_matrix = np.array([[np.cos(angle), -np.sin(angle)], [np.sin(angle), np.cos(angle)]])
-Xsr = Xs.dot(rotation_matrix)
-
-tree_clf_s = DecisionTreeClassifier(random_state=42)
-tree_clf_s.fit(Xs, ys)
-tree_clf_sr = DecisionTreeClassifier(random_state=42)
-tree_clf_sr.fit(Xsr, ys)
-
-plt.figure(figsize=(11, 4))
-plt.subplot(121)
-plot_decision_boundary(tree_clf_s, Xs, ys, axes=[-0.7, 0.7, -0.7, 0.7], iris=False)
-plt.subplot(122)
-plot_decision_boundary(tree_clf_sr, Xsr, ys, axes=[-0.7, 0.7, -0.7, 0.7], iris=False)
-
-plt.show()
-
-Regression trees
- - -# Quadratic training set + noise
-np.random.seed(42)
-m = 200
-X = np.random.rand(m, 1)
-y = 4 * (X - 0.5) ** 2
-y = y + np.random.randn(m, 1) / 10
-
-from sklearn.tree import DecisionTreeRegressor
-
-tree_reg = DecisionTreeRegressor(max_depth=2, random_state=42)
-tree_reg.fit(X, y)
-
-Final regressor code
- - -from sklearn.tree import DecisionTreeRegressor
-
-tree_reg1 = DecisionTreeRegressor(random_state=42, max_depth=2)
-tree_reg2 = DecisionTreeRegressor(random_state=42, max_depth=3)
-tree_reg1.fit(X, y)
-tree_reg2.fit(X, y)
-
-def plot_regression_predictions(tree_reg, X, y, axes=[0, 1, -0.2, 1], ylabel="$y$"):
- x1 = np.linspace(axes[0], axes[1], 500).reshape(-1, 1)
- y_pred = tree_reg.predict(x1)
- plt.axis(axes)
- plt.xlabel("$x_1$", fontsize=18)
- if ylabel:
- plt.ylabel(ylabel, fontsize=18, rotation=0)
- plt.plot(X, y, "b.")
- plt.plot(x1, y_pred, "r.-", linewidth=2, label=r"$\hat{y}$")
-
-plt.figure(figsize=(11, 4))
-plt.subplot(121)
-plot_regression_predictions(tree_reg1, X, y)
-for split, style in ((0.1973, "k-"), (0.0917, "k--"), (0.7718, "k--")):
- plt.plot([split, split], [-0.2, 1], style, linewidth=2)
-plt.text(0.21, 0.65, "Depth=0", fontsize=15)
-plt.text(0.01, 0.2, "Depth=1", fontsize=13)
-plt.text(0.65, 0.8, "Depth=1", fontsize=13)
-plt.legend(loc="upper center", fontsize=18)
-plt.title("max_depth=2", fontsize=14)
-
-plt.subplot(122)
-plot_regression_predictions(tree_reg2, X, y, ylabel=None)
-for split, style in ((0.1973, "k-"), (0.0917, "k--"), (0.7718, "k--")):
- plt.plot([split, split], [-0.2, 1], style, linewidth=2)
-for split in (0.0458, 0.1298, 0.2873, 0.9040):
- plt.plot([split, split], [-0.2, 1], "k:", linewidth=1)
-plt.text(0.3, 0.5, "Depth=2", fontsize=13)
-plt.title("max_depth=3", fontsize=14)
-
-plt.show()
-
-tree_reg1 = DecisionTreeRegressor(random_state=42)
-tree_reg2 = DecisionTreeRegressor(random_state=42, min_samples_leaf=10)
-tree_reg1.fit(X, y)
-tree_reg2.fit(X, y)
-
-x1 = np.linspace(0, 1, 500).reshape(-1, 1)
-y_pred1 = tree_reg1.predict(x1)
-y_pred2 = tree_reg2.predict(x1)
-
-plt.figure(figsize=(11, 4))
-
-plt.subplot(121)
-plt.plot(X, y, "b.")
-plt.plot(x1, y_pred1, "r.-", linewidth=2, label=r"$\hat{y}$")
-plt.axis([0, 1, -0.2, 1.1])
-plt.xlabel("$x_1$", fontsize=18)
-plt.ylabel("$y$", fontsize=18, rotation=0)
-plt.legend(loc="upper center", fontsize=18)
-plt.title("No restrictions", fontsize=14)
-
-plt.subplot(122)
-plt.plot(X, y, "b.")
-plt.plot(x1, y_pred2, "r.-", linewidth=2, label=r"$\hat{y}$")
-plt.axis([0, 1, -0.2, 1.1])
-plt.xlabel("$x_1$", fontsize=18)
-plt.title("min_samples_leaf={}".format(tree_reg2.min_samples_leaf), fontsize=14)
-
-plt.show()
-
-Why Voting?
diff --git a/doc/pub/week44/html/week44-solarized.html b/doc/pub/week44/html/week44-solarized.html index 4b522b2d0..b13c240e3 100644 --- a/doc/pub/week44/html/week44-solarized.html +++ b/doc/pub/week44/html/week44-solarized.html @@ -142,67 +142,6 @@ div.toc p,a { 2, None, 'the-cart-algorithm-for-regression'), - ('Computing the Gini index', 2, None, 'computing-the-gini-index'), - ('Simple Python Code to read in Data and perform Classification', - 2, - None, - 'simple-python-code-to-read-in-data-and-perform-classification'), - ('Computing the Gini Factor', - 2, - None, - 'computing-the-gini-factor'), - ('Entropy and the ID3 algorithm', - 2, - None, - 'entropy-and-the-id3-algorithm'), - ('Cancer Data again now with Decision Trees and other Methods', - 2, - None, - 'cancer-data-again-now-with-decision-trees-and-other-methods'), - ('Another example, the moons again', - 2, - None, - 'another-example-the-moons-again'), - ('Playing around with regions', - 2, - None, - 'playing-around-with-regions'), - ('Regression trees', 2, None, 'regression-trees'), - ('Final regressor code', 2, None, 'final-regressor-code'), - ('Pros and cons of trees, pros', - 2, - None, - 'pros-and-cons-of-trees-pros'), - ('Disadvantages', 2, None, 'disadvantages'), - ('Ensemble Methods: From a Single Tree to Many Trees and Extreme ' - 'Boosting, Meet the Jungle of Methods', - 2, - None, - 'ensemble-methods-from-a-single-tree-to-many-trees-and-extreme-boosting-meet-the-jungle-of-methods'), - ('An Overview of Ensemble Methods', - 2, - None, - 'an-overview-of-ensemble-methods'), - ('Bagging', 2, None, 'bagging'), - ('More bagging', 2, None, 'more-bagging'), - ('Simple Voting Example, head or tail', - 2, - None, - 'simple-voting-example-head-or-tail'), - ('Using the Voting Classifier', - 2, - None, - 'using-the-voting-classifier'), - ('Please, not the moons again! Voting and Bagging', - 2, - None, - 'please-not-the-moons-again-voting-and-bagging'), - ('Bagging Examples', 2, None, 'bagging-examples'), - ('Making your own Bootstrap: Changing the Level of the Decision ' - 'Tree', - 2, - None, - 'making-your-own-bootstrap-changing-the-level-of-the-decision-tree'), ('Why binary splits?', 2, None, 'why-binary-splits'), ('Computing a Tree using the Gini Index', 2, @@ -277,6 +216,70 @@ div.toc p,a { 'playing-around-with-regions'), ('Regression trees', 2, None, 'regression-trees'), ('Final regressor code', 2, None, 'final-regressor-code'), + ('Example: Computing the Gini index', + 2, + None, + 'example-computing-the-gini-index'), + ('Simple Python Code to read in Data and perform Classification', + 2, + None, + 'simple-python-code-to-read-in-data-and-perform-classification'), + ('Computing the Gini Factor', + 2, + None, + 'computing-the-gini-factor'), + ('Entropy and the ID3 algorithm', + 2, + None, + 'entropy-and-the-id3-algorithm'), + ('Cancer Data again now with Decision Trees and other Methods', + 2, + None, + 'cancer-data-again-now-with-decision-trees-and-other-methods'), + ('Another example, the moons again', + 2, + None, + 'another-example-the-moons-again'), + ('Playing around with regions', + 2, + None, + 'playing-around-with-regions'), + ('Regression trees', 2, None, 'regression-trees'), + ('Final regressor code', 2, None, 'final-regressor-code'), + ('Pros and cons of trees, pros', + 2, + None, + 'pros-and-cons-of-trees-pros'), + ('Disadvantages', 2, None, 'disadvantages'), + ('Ensemble Methods: From a Single Tree to Many Trees and Extreme ' + 'Boosting, Meet the Jungle of Methods', + 2, + None, + 'ensemble-methods-from-a-single-tree-to-many-trees-and-extreme-boosting-meet-the-jungle-of-methods'), + ('An Overview of Ensemble Methods', + 2, + None, + 'an-overview-of-ensemble-methods'), + ('Bagging', 2, None, 'bagging'), + ('More bagging', 2, None, 'more-bagging'), + ('Simple Voting Example, head or tail', + 2, + None, + 'simple-voting-example-head-or-tail'), + ('Using the Voting Classifier', + 2, + None, + 'using-the-voting-classifier'), + ('Please, not the moons again! Voting and Bagging', + 2, + None, + 'please-not-the-moons-again-voting-and-bagging'), + ('Bagging Examples', 2, None, 'bagging-examples'), + ('Making your own Bootstrap: Changing the Level of the Decision ' + 'Tree', + 2, + None, + 'making-your-own-bootstrap-changing-the-level-of-the-decision-tree'), ('Why Voting?', 2, None, 'why-voting'), ('Tossing coins', 2, None, 'tossing-coins'), ('Standard imports first', 2, None, 'standard-imports-first'), @@ -336,7 +339,7 @@ MathJax.Hub.Config({Oct 30, 2022
+Nov 1, 2022
@@ -1141,6 +1144,429 @@ r = export_text(decision_tree, feature_names=iris[&
+
Algorithms for Setting up Decision Trees
+ +Two algorithms stand out in the set up of decision trees:
+-
+
We discuss both algorithms with applications here. The popular library +Scikit-Learn uses the CART algorithm. For classification problems +you can use either the gini index or the entropy to split a tree +in two branches. +
+ ++
The CART algorithm for Classification
+ +For classification, the CART algorithm splits the data set in two subsets using a single feature \( k \) and a threshold \( t_k \). +This could be for example a threshold set by a number below a certain circumference of a malign tumor. +
+ +How do we find these two quantities? +We search for the pair \( (k,t_k) \) that produces the purest subset using for example the gini factor \( G \). +The cost function it tries to minimize is then +
+$$ +C(k,t_k) = \frac{m_{\mathrm{left}}}{m}G_{\mathrm{left}}+ \frac{m_{\mathrm{right}}}{m}G_{\mathrm{right}}, +$$ + +where \( G_{\mathrm{left/right}} \) measures the impurity of the left/right subset and \( m_{\mathrm{left/right}} \) + is the number of instances in the left/right subset +
+ +Once it has successfully split the training set in two, it splits the subsets using the same logic, then the subsubsets +and so on, recursively. It stops recursing once it reaches the maximum depth (defined by the +\( max\_depth \) hyperparameter), or if it cannot find a split that will reduce impurity. A few other +hyperparameters control additional stopping conditions such as the \( min\_samples\_split \), +\( min\_samples\_leaf \), \( min\_weight\_fraction\_leaf \), and \( max\_leaf\_nodes \). +
+ ++
The CART algorithm for Regression
+ +The CART algorithm for regression works is similar to the one for classification except that instead of trying to split the +training set in a way that minimizes say the gini or entropy impurity, it now tries to split the training set in a way that minimizes our well-known mean-squared error (MSE). The cost function is now +
+$$ +C(k,t_k) = \frac{m_{\mathrm{left}}}{m}\mathrm{MSE}_{\mathrm{left}}+ \frac{m_{\mathrm{right}}}{m}\mathrm{MSE}_{\mathrm{right}}. +$$ + +Here the MSE for a specific node is defined as
+$$ +\mathrm{MSE}_{\mathrm{node}}=\frac{1}{m_\mathrm{node}}\sum_{i\in \mathrm{node}}(\overline{y}_{\mathrm{node}}-y_i)^2, +$$ + +with
+$$ +\overline{y}_{\mathrm{node}}=\frac{1}{m_\mathrm{node}}\sum_{i\in \mathrm{node}}y_i, +$$ + +the mean value of all observations in a specific node.
+ +Without any regularization, the regression task for decision trees, +just like for classification tasks, is prone to overfitting. +
+ ++
Why binary splits?
+ +It is custom to split to a tree uising binary splits. The reason is +that multiway splits fragment the data too quickly, leaving +insufficient data at the next level down. Multiway splits can be +achieved by a series of binary split and this is normally preferred. +
+ ++
Computing a Tree using the Gini Index
+ +Consider the following example with attributes/features and two +possible outcomes (classes) for each attribute. Assume we wish to find some +correlations between the average grade of a student as function of the +number of hours studied and hours slept. We want also to correlate the +grade in a given course with the general trend, whether the students +recently has gotten grades below average or above. +
+ +We have three features/attributes
+-
+
+
The Table
+ +| Grade Trend | Hours slept | Hours Studied | Grade |
|---|---|---|---|
| Above | Low | High | Above |
| Below | High | Low | Below |
| Above | Low | High | Above |
| Above | High | High | Above |
| Below | Low | High | Below |
| Above | Low | Low | Below |
| Below | High | High | Below |
| Below | Low | High | Below |
| Above | Low | Low | Below |
| Above | High | High | Above |
+
Computing the various Gini Indices
+ +In computations we will translate all classes into numbers. Being +these binary classes, they can easily be split into ones and zeros. +
+ ++
See handwritten notes for Thursday November 11
++
Computing the various Gini Indices, Hours slept
+ ++
See handwritten notes for Thursday November 3
++
Computing the various Gini Indices, Hours studied
+ ++
See handwritten notes for Thursday November 3
+For final tree, see the above handwritten notes
+ ++
A possible code using Scikit-Learn
+ + + +# Common imports
+import numpy as np
+import pandas as pd
+import matplotlib.pyplot as plt
+from sklearn.tree import DecisionTreeClassifier
+from sklearn.model_selection import train_test_split
+from sklearn.tree import export_graphviz
+from sklearn.preprocessing import StandardScaler, OneHotEncoder
+from sklearn.compose import ColumnTransformer
+from IPython.display import Image
+from pydot import graph_from_dot_data
+import os
+
+# Where to save the figures and data files
+PROJECT_ROOT_DIR = "Results"
+FIGURE_ID = "Results/FigureFiles"
+DATA_ID = "DataFiles/"
+
+if not os.path.exists(PROJECT_ROOT_DIR):
+ os.mkdir(PROJECT_ROOT_DIR)
+
+if not os.path.exists(FIGURE_ID):
+ os.makedirs(FIGURE_ID)
+
+if not os.path.exists(DATA_ID):
+ os.makedirs(DATA_ID)
+
+def image_path(fig_id):
+ return os.path.join(FIGURE_ID, fig_id)
+
+def data_path(dat_id):
+ return os.path.join(DATA_ID, dat_id)
+
+def save_fig(fig_id):
+ plt.savefig(image_path(fig_id) + ".png", format='png')
+
+infile = open(data_path("grades.csv"),'r')
+
+# Read the experimental data with Pandas
+from IPython.display import display
+grades = pd.read_csv(infile,names = ('Trend','Sleep','Studied','Grade'))
+grades = pd.DataFrame(grades)
+
+# Features and targets
+X = grades.loc[:, grades.columns != 'Grade'].values
+y = grades.loc[:, grades.columns == 'Grade'].values
+
+# Create the encoder.
+encoder = OneHotEncoder(handle_unknown="ignore")
+# Assume for simplicity all features are categorical.
+encoder.fit(X)
+# Apply the encoder.
+X = encoder.transform(X)
+print(X)
+# Then do a Classification tree
+tree_clf = DecisionTreeClassifier(max_depth=2)
+tree_clf.fit(X, y)
+print("Train set accuracy with Decision Tree: {:.2f}".format(tree_clf.score(X,y)))
+#transfer to a decision tree graph
+export_graphviz(
+ tree_clf,
+ out_file="DataFiles/grade.dot",
+ rounded=True,
+ filled=True
+)
+cmd = 'dot -Tpng DataFiles/grade.dot -o DataFiles/grades.png'
+os.system(cmd)
+
++
Visualizing Trees, More examples
+ + +import os
+from sklearn.datasets import load_breast_cancer
+from sklearn.tree import DecisionTreeClassifier
+from sklearn.model_selection import train_test_split
+from sklearn.metrics import confusion_matrix
+from sklearn.tree import export_graphviz
+
+from IPython.display import Image
+from pydot import graph_from_dot_data
+import pandas as pd
+import numpy as np
+
+
+cancer = load_breast_cancer()
+X = pd.DataFrame(cancer.data, columns=cancer.feature_names)
+print(X)
+y = pd.Categorical.from_codes(cancer.target, cancer.target_names)
+y = pd.get_dummies(y)
+print(y)
+X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=1)
+tree_clf = DecisionTreeClassifier(max_depth=5)
+tree_clf.fit(X_train, y_train)
+
+export_graphviz(
+ tree_clf,
+ out_file="DataFiles/cancer.dot",
+ feature_names=cancer.feature_names,
+ class_names=cancer.target_names,
+ rounded=True,
+ filled=True
+)
+cmd = 'dot -Tpng DataFiles/cancer.dot -o DataFiles/cancer.png'
+os.system(cmd)
+
++
Visualizing the Tree, The Moons
+ + +# Common imports
+import numpy as np
+from sklearn.model_selection import train_test_split
+from sklearn.tree import DecisionTreeClassifier
+from sklearn.datasets import make_moons
+from sklearn.tree import export_graphviz
+from pydot import graph_from_dot_data
+import pandas as pd
+import os
+
+np.random.seed(42)
+X, y = make_moons(n_samples=100, noise=0.25, random_state=53)
+X_train, X_test, y_train, y_test = train_test_split(X,y,random_state=0)
+tree_clf = DecisionTreeClassifier(max_depth=5)
+tree_clf.fit(X_train, y_train)
+
+export_graphviz(
+ tree_clf,
+ out_file="DataFiles/moons.dot",
+ rounded=True,
+ filled=True
+)
+cmd = 'dot -Tpng DataFiles/moons.dot -o DataFiles/moons.png'
+os.system(cmd)
+
++
Other ways of visualizing the trees
+ +Scikit-Learn has also another way to visualize the trees which is very useful, here with the Iris data.
+ + + +from sklearn.datasets import load_iris
+from sklearn import tree
+X, y = load_iris(return_X_y=True)
+tree_clf = tree.DecisionTreeClassifier()
+tree_clf = tree_clf.fit(X, y)
+# and then plot the tree
+tree.plot_tree(tree_clf)
+
++
Printing out as text
+ +Alternatively, the tree can also be exported in textual format with the function exporttext. +This method doesn’t require the installation of external libraries and is more compact: +
+ + + +from sklearn.datasets import load_iris
+from sklearn.tree import DecisionTreeClassifier
+from sklearn.tree import export_text
+iris = load_iris()
+decision_tree = DecisionTreeClassifier(random_state=0, max_depth=2)
+decision_tree = decision_tree.fit(iris.data, iris.target)
+r = export_text(decision_tree, feature_names=iris['feature_names'])
+print(r)
+
+Algorithms for Setting up Decision Trees
@@ -1829,6 +2255,628 @@ plt.show()+
Example: Computing the Gini index
+ +The example we will look at is a classical one in many Machine +Learning applications. Based on various meteorological features, we +have several so-called attributes which decide whether we at the end +will do some outdoor activity like skiing, going for a bike ride etc +etc. The table here contains the feautures outlook, temperature, +humidity and wind. The target or output is whether we ride +(True=1) or whether we do something else that day (False=0). The +attributes for each feature are then sunny, overcast and rain for the +outlook, hot, cold and mild for temperature, high and normal for +humidity and weak and strong for wind. +
+ +The table here summarizes the various attributes and
+| Day | Outlook | Temperature | Humidity | Wind | Ride |
|---|---|---|---|---|---|
| 1 | Sunny | Hot | High | Weak | 0 |
| 2 | Sunny | Hot | High | Strong | 1 |
| 3 | Overcast | Hot | High | Weak | 1 |
| 4 | Rain | Mild | High | Weak | 1 |
| 5 | Rain | Cool | Normal | Weak | 1 |
| 6 | Rain | Cool | Normal | Strong | 0 |
| 7 | Overcast | Cool | Normal | Strong | 1 |
| 8 | Sunny | Mild | High | Weak | 0 |
| 9 | Sunny | Cool | Normal | Weak | 1 |
| 10 | Rain | Mild | Normal | Weak | 1 |
| 11 | Sunny | Mild | Normal | Strong | 1 |
| 12 | Overcast | Mild | High | Strong | 1 |
| 13 | Overcast | Hot | Normal | Weak | 1 |
| 14 | Rain | Mild | High | Strong | 0 |
+
Simple Python Code to read in Data and perform Classification
+ + + +# Common imports
+import numpy as np
+import pandas as pd
+import matplotlib.pyplot as plt
+from sklearn.tree import DecisionTreeClassifier
+from sklearn.model_selection import train_test_split
+from sklearn.tree import export_graphviz
+from sklearn.preprocessing import StandardScaler, OneHotEncoder
+from sklearn.compose import ColumnTransformer
+from IPython.display import Image
+from pydot import graph_from_dot_data
+import os
+
+# Where to save the figures and data files
+PROJECT_ROOT_DIR = "Results"
+FIGURE_ID = "Results/FigureFiles"
+DATA_ID = "DataFiles/"
+
+if not os.path.exists(PROJECT_ROOT_DIR):
+ os.mkdir(PROJECT_ROOT_DIR)
+
+if not os.path.exists(FIGURE_ID):
+ os.makedirs(FIGURE_ID)
+
+if not os.path.exists(DATA_ID):
+ os.makedirs(DATA_ID)
+
+def image_path(fig_id):
+ return os.path.join(FIGURE_ID, fig_id)
+
+def data_path(dat_id):
+ return os.path.join(DATA_ID, dat_id)
+
+def save_fig(fig_id):
+ plt.savefig(image_path(fig_id) + ".png", format='png')
+
+infile = open(data_path("rideclass.csv"),'r')
+
+# Read the experimental data with Pandas
+from IPython.display import display
+ridedata = pd.read_csv(infile,names = ('Outlook','Temperature','Humidity','Wind','Ride'))
+ridedata = pd.DataFrame(ridedata)
+
+# Features and targets
+X = ridedata.loc[:, ridedata.columns != 'Ride'].values
+y = ridedata.loc[:, ridedata.columns == 'Ride'].values
+
+# Create the encoder.
+encoder = OneHotEncoder(handle_unknown="ignore")
+# Assume for simplicity all features are categorical.
+encoder.fit(X)
+# Apply the encoder.
+X = encoder.transform(X)
+print(X)
+# Then do a Classification tree
+tree_clf = DecisionTreeClassifier(max_depth=2)
+tree_clf.fit(X, y)
+print("Train set accuracy with Decision Tree: {:.2f}".format(tree_clf.score(X,y)))
+#transfer to a decision tree graph
+export_graphviz(
+ tree_clf,
+ out_file="DataFiles/ride.dot",
+ rounded=True,
+ filled=True
+)
+cmd = 'dot -Tpng DataFiles/cancer.dot -o DataFiles/cancer.png'
+os.system(cmd)
+
++
Computing the Gini Factor
+ +The above functions (gini, entropy and misclassification error) are +important components of the so-called CART algorithm. We will discuss +this algorithm below after we have discussed the information gain +algorithm ID3. +
+ +In the example here we have converted all our attributes into numerical values \( 0,1,2 \) etc.
+ + + +# Split a dataset based on an attribute and an attribute value
+def test_split(index, value, dataset):
+ left, right = list(), list()
+ for row in dataset:
+ if row[index] < value:
+ left.append(row)
+ else:
+ right.append(row)
+ return left, right
+
+# Calculate the Gini index for a split dataset
+def gini_index(groups, classes):
+ # count all samples at split point
+ n_instances = float(sum([len(group) for group in groups]))
+ # sum weighted Gini index for each group
+ gini = 0.0
+ for group in groups:
+ size = float(len(group))
+ # avoid divide by zero
+ if size == 0:
+ continue
+ score = 0.0
+ # score the group based on the score for each class
+ for class_val in classes:
+ p = [row[-1] for row in group].count(class_val) / size
+ score += p * p
+ # weight the group score by its relative size
+ gini += (1.0 - score) * (size / n_instances)
+ return gini
+
+# Select the best split point for a dataset
+def get_split(dataset):
+ class_values = list(set(row[-1] for row in dataset))
+ b_index, b_value, b_score, b_groups = 999, 999, 999, None
+ for index in range(len(dataset[0])-1):
+ for row in dataset:
+ groups = test_split(index, row[index], dataset)
+ gini = gini_index(groups, class_values)
+ print('X%d < %.3f Gini=%.3f' % ((index+1), row[index], gini))
+ if gini < b_score:
+ b_index, b_value, b_score, b_groups = index, row[index], gini, groups
+ return {'index':b_index, 'value':b_value, 'groups':b_groups}
+
+dataset = [[0,0,0,0,0],
+ [0,0,0,1,1],
+ [1,0,0,0,1],
+ [2,1,0,0,1],
+ [2,2,1,0,1],
+ [2,2,1,1,0],
+ [1,2,1,1,1],
+ [0,1,0,0,0],
+ [0,2,1,0,1],
+ [2,1,1,0,1],
+ [0,1,1,1,1],
+ [1,1,0,1,1],
+ [1,0,1,0,1],
+ [2,1,0,1,0]]
+
+split = get_split(dataset)
+print('Split: [X%d < %.3f]' % ((split['index']+1), split['value']))
+
++
Entropy and the ID3 algorithm
+ +The ID3 algorithm learns decision trees by constructing +them in a top down way, beginning with the question which attribute should be tested at the root of the tree? +
+ +-
+
The ID3 algorithm selects which attribute to test at each node in the +tree. +
+ +We would like to select the attribute that is most useful for classifying +examples. +
+ +What is a good quantitative measure of the worth of an attribute?
+ +Information gain measures how well a given attribute separates the +training examples according to their target classification. +
+ +The ID3 algorithm uses this information gain measure to select among the candidate +attributes at each step while growing the tree. +
+ ++
Cancer Data again now with Decision Trees and other Methods
+ + +import matplotlib.pyplot as plt
+import numpy as np
+from sklearn.model_selection import train_test_split
+from sklearn.datasets import load_breast_cancer
+from sklearn.svm import SVC
+from sklearn.linear_model import LogisticRegression
+from sklearn.tree import DecisionTreeClassifier
+
+# Load the data
+cancer = load_breast_cancer()
+
+X_train, X_test, y_train, y_test = train_test_split(cancer.data,cancer.target,random_state=0)
+print(X_train.shape)
+print(X_test.shape)
+# Logistic Regression
+logreg = LogisticRegression(solver='lbfgs')
+logreg.fit(X_train, y_train)
+print("Test set accuracy with Logistic Regression: {:.2f}".format(logreg.score(X_test,y_test)))
+# Support vector machine
+svm = SVC(gamma='auto', C=100)
+svm.fit(X_train, y_train)
+print("Test set accuracy with SVM: {:.2f}".format(svm.score(X_test,y_test)))
+# Decision Trees
+deep_tree_clf = DecisionTreeClassifier(max_depth=None)
+deep_tree_clf.fit(X_train, y_train)
+print("Test set accuracy with Decision Trees: {:.2f}".format(deep_tree_clf.score(X_test,y_test)))
+#now scale the data
+from sklearn.preprocessing import StandardScaler
+scaler = StandardScaler()
+scaler.fit(X_train)
+X_train_scaled = scaler.transform(X_train)
+X_test_scaled = scaler.transform(X_test)
+# Logistic Regression
+logreg.fit(X_train_scaled, y_train)
+print("Test set accuracy Logistic Regression with scaled data: {:.2f}".format(logreg.score(X_test_scaled,y_test)))
+# Support Vector Machine
+svm.fit(X_train_scaled, y_train)
+print("Test set accuracy SVM with scaled data: {:.2f}".format(logreg.score(X_test_scaled,y_test)))
+# Decision Trees
+deep_tree_clf.fit(X_train_scaled, y_train)
+print("Test set accuracy with Decision Trees and scaled data: {:.2f}".format(deep_tree_clf.score(X_test_scaled,y_test)))
+
++
Another example, the moons again
+ + +from __future__ import division, print_function, unicode_literals
+
+# Common imports
+import numpy as np
+import os
+
+# to make this notebook's output stable across runs
+np.random.seed(42)
+
+# To plot pretty figures
+import matplotlib
+import matplotlib.pyplot as plt
+from matplotlib.colors import ListedColormap
+plt.rcParams['axes.labelsize'] = 14
+plt.rcParams['xtick.labelsize'] = 12
+plt.rcParams['ytick.labelsize'] = 12
+
+
+from sklearn.svm import SVC
+from sklearn import datasets
+from sklearn.tree import DecisionTreeClassifier
+from sklearn.datasets import make_moons
+from sklearn.tree import export_graphviz
+
+Xm, ym = make_moons(n_samples=100, noise=0.25, random_state=53)
+
+deep_tree_clf1 = DecisionTreeClassifier(random_state=42)
+deep_tree_clf2 = DecisionTreeClassifier(min_samples_leaf=4, random_state=42)
+deep_tree_clf1.fit(Xm, ym)
+deep_tree_clf2.fit(Xm, ym)
+
+
+def plot_decision_boundary(clf, X, y, axes=[0, 7.5, 0, 3], iris=True, legend=False, plot_training=True):
+ x1s = np.linspace(axes[0], axes[1], 100)
+ x2s = np.linspace(axes[2], axes[3], 100)
+ x1, x2 = np.meshgrid(x1s, x2s)
+ X_new = np.c_[x1.ravel(), x2.ravel()]
+ y_pred = clf.predict(X_new).reshape(x1.shape)
+ custom_cmap = ListedColormap(['#fafab0','#9898ff','#a0faa0'])
+ plt.contourf(x1, x2, y_pred, alpha=0.3, cmap=custom_cmap)
+ if not iris:
+ custom_cmap2 = ListedColormap(['#7d7d58','#4c4c7f','#507d50'])
+ plt.contour(x1, x2, y_pred, cmap=custom_cmap2, alpha=0.8)
+ if plot_training:
+ plt.plot(X[:, 0][y==0], X[:, 1][y==0], "yo", label="Iris-Setosa")
+ plt.plot(X[:, 0][y==1], X[:, 1][y==1], "bs", label="Iris-Versicolor")
+ plt.plot(X[:, 0][y==2], X[:, 1][y==2], "g^", label="Iris-Virginica")
+ plt.axis(axes)
+ if iris:
+ plt.xlabel("Petal length", fontsize=14)
+ plt.ylabel("Petal width", fontsize=14)
+ else:
+ plt.xlabel(r"$x_1$", fontsize=18)
+ plt.ylabel(r"$x_2$", fontsize=18, rotation=0)
+ if legend:
+ plt.legend(loc="lower right", fontsize=14)
+plt.figure(figsize=(11, 4))
+plt.subplot(121)
+plot_decision_boundary(deep_tree_clf1, Xm, ym, axes=[-1.5, 2.5, -1, 1.5], iris=False)
+plt.title("No restrictions", fontsize=16)
+plt.subplot(122)
+plot_decision_boundary(deep_tree_clf2, Xm, ym, axes=[-1.5, 2.5, -1, 1.5], iris=False)
+plt.title("min_samples_leaf = {}".format(deep_tree_clf2.min_samples_leaf), fontsize=14)
+plt.show()
+
++
Playing around with regions
+ + +np.random.seed(6)
+Xs = np.random.rand(100, 2) - 0.5
+ys = (Xs[:, 0] > 0).astype(np.float32) * 2
+
+angle = np.pi/4
+rotation_matrix = np.array([[np.cos(angle), -np.sin(angle)], [np.sin(angle), np.cos(angle)]])
+Xsr = Xs.dot(rotation_matrix)
+
+tree_clf_s = DecisionTreeClassifier(random_state=42)
+tree_clf_s.fit(Xs, ys)
+tree_clf_sr = DecisionTreeClassifier(random_state=42)
+tree_clf_sr.fit(Xsr, ys)
+
+plt.figure(figsize=(11, 4))
+plt.subplot(121)
+plot_decision_boundary(tree_clf_s, Xs, ys, axes=[-0.7, 0.7, -0.7, 0.7], iris=False)
+plt.subplot(122)
+plot_decision_boundary(tree_clf_sr, Xsr, ys, axes=[-0.7, 0.7, -0.7, 0.7], iris=False)
+
+plt.show()
+
++
Regression trees
+ + +# Quadratic training set + noise
+np.random.seed(42)
+m = 200
+X = np.random.rand(m, 1)
+y = 4 * (X - 0.5) ** 2
+y = y + np.random.randn(m, 1) / 10
+
+from sklearn.tree import DecisionTreeRegressor
+
+tree_reg = DecisionTreeRegressor(max_depth=2, random_state=42)
+tree_reg.fit(X, y)
+
++
Final regressor code
+ + +from sklearn.tree import DecisionTreeRegressor
+
+tree_reg1 = DecisionTreeRegressor(random_state=42, max_depth=2)
+tree_reg2 = DecisionTreeRegressor(random_state=42, max_depth=3)
+tree_reg1.fit(X, y)
+tree_reg2.fit(X, y)
+
+def plot_regression_predictions(tree_reg, X, y, axes=[0, 1, -0.2, 1], ylabel="$y$"):
+ x1 = np.linspace(axes[0], axes[1], 500).reshape(-1, 1)
+ y_pred = tree_reg.predict(x1)
+ plt.axis(axes)
+ plt.xlabel("$x_1$", fontsize=18)
+ if ylabel:
+ plt.ylabel(ylabel, fontsize=18, rotation=0)
+ plt.plot(X, y, "b.")
+ plt.plot(x1, y_pred, "r.-", linewidth=2, label=r"$\hat{y}$")
+
+plt.figure(figsize=(11, 4))
+plt.subplot(121)
+plot_regression_predictions(tree_reg1, X, y)
+for split, style in ((0.1973, "k-"), (0.0917, "k--"), (0.7718, "k--")):
+ plt.plot([split, split], [-0.2, 1], style, linewidth=2)
+plt.text(0.21, 0.65, "Depth=0", fontsize=15)
+plt.text(0.01, 0.2, "Depth=1", fontsize=13)
+plt.text(0.65, 0.8, "Depth=1", fontsize=13)
+plt.legend(loc="upper center", fontsize=18)
+plt.title("max_depth=2", fontsize=14)
+
+plt.subplot(122)
+plot_regression_predictions(tree_reg2, X, y, ylabel=None)
+for split, style in ((0.1973, "k-"), (0.0917, "k--"), (0.7718, "k--")):
+ plt.plot([split, split], [-0.2, 1], style, linewidth=2)
+for split in (0.0458, 0.1298, 0.2873, 0.9040):
+ plt.plot([split, split], [-0.2, 1], "k:", linewidth=1)
+plt.text(0.3, 0.5, "Depth=2", fontsize=13)
+plt.title("max_depth=3", fontsize=14)
+
+plt.show()
+
+tree_reg1 = DecisionTreeRegressor(random_state=42)
+tree_reg2 = DecisionTreeRegressor(random_state=42, min_samples_leaf=10)
+tree_reg1.fit(X, y)
+tree_reg2.fit(X, y)
+
+x1 = np.linspace(0, 1, 500).reshape(-1, 1)
+y_pred1 = tree_reg1.predict(x1)
+y_pred2 = tree_reg2.predict(x1)
+
+plt.figure(figsize=(11, 4))
+
+plt.subplot(121)
+plt.plot(X, y, "b.")
+plt.plot(x1, y_pred1, "r.-", linewidth=2, label=r"$\hat{y}$")
+plt.axis([0, 1, -0.2, 1.1])
+plt.xlabel("$x_1$", fontsize=18)
+plt.ylabel("$y$", fontsize=18, rotation=0)
+plt.legend(loc="upper center", fontsize=18)
+plt.title("No restrictions", fontsize=14)
+
+plt.subplot(122)
+plt.plot(X, y, "b.")
+plt.plot(x1, y_pred2, "r.-", linewidth=2, label=r"$\hat{y}$")
+plt.axis([0, 1, -0.2, 1.1])
+plt.xlabel("$x_1$", fontsize=18)
+plt.title("min_samples_leaf={}".format(tree_reg2.min_samples_leaf), fontsize=14)
+
+plt.show()
+
+Pros and cons of trees, pros
@@ -2372,1051 +3420,6 @@ plt.show()-
Why binary splits?
- -It is custom to split to a tree uising binary splits. The reason is -that multiway splits fragment the data too quickly, leaving -insufficient data at the next level down. Multiway splits can be -achieved by a series of binary split and this is normally preferred. -
- --
Computing a Tree using the Gini Index
- -Consider the following example with attributes/features and two -possible outcomes (classes) for each attribute. Assume we wish to find some -correlations between the average grade of a student as function of the -number of hours studied and hours slept. We want also to correlate the -grade in a given course with the general trend, whether the students -recently has gotten grades below average or above. -
- -We have three features/attributes
--
-
-
The Table
- -| Grade Trend | Hours slept | Hours Studied | Grade |
|---|---|---|---|
| Above | Low | High | Above |
| Below | High | Low | Below |
| Above | Low | High | Above |
| Above | High | High | Above |
| Below | Low | High | Below |
| Above | Low | Low | Below |
| Below | High | High | Below |
| Below | Low | High | Below |
| Above | Low | Low | Below |
| Above | High | High | Above |
-
Computing the various Gini Indices
- -In computations we will translate all classes into numbers. Being -these binary classes, they can easily be split into ones and zeros. -
- --
See handwritten notes for Thursday November 11
--
Computing the various Gini Indices, Hours slept
- --
See handwritten notes for Thursday November 11
--
Computing the various Gini Indices, Hours studied
- --
See handwritten notes for Thursday November 11
-For final tree, see the above handwritten notes
- --
A possible code using Scikit-Learn
- - - -# Common imports
-import numpy as np
-import pandas as pd
-import matplotlib.pyplot as plt
-from sklearn.tree import DecisionTreeClassifier
-from sklearn.model_selection import train_test_split
-from sklearn.tree import export_graphviz
-from sklearn.preprocessing import StandardScaler, OneHotEncoder
-from sklearn.compose import ColumnTransformer
-from IPython.display import Image
-from pydot import graph_from_dot_data
-import os
-
-# Where to save the figures and data files
-PROJECT_ROOT_DIR = "Results"
-FIGURE_ID = "Results/FigureFiles"
-DATA_ID = "DataFiles/"
-
-if not os.path.exists(PROJECT_ROOT_DIR):
- os.mkdir(PROJECT_ROOT_DIR)
-
-if not os.path.exists(FIGURE_ID):
- os.makedirs(FIGURE_ID)
-
-if not os.path.exists(DATA_ID):
- os.makedirs(DATA_ID)
-
-def image_path(fig_id):
- return os.path.join(FIGURE_ID, fig_id)
-
-def data_path(dat_id):
- return os.path.join(DATA_ID, dat_id)
-
-def save_fig(fig_id):
- plt.savefig(image_path(fig_id) + ".png", format='png')
-
-infile = open(data_path("grades.csv"),'r')
-
-# Read the experimental data with Pandas
-from IPython.display import display
-grades = pd.read_csv(infile,names = ('Trend','Sleep','Studied','Grade'))
-grades = pd.DataFrame(grades)
-
-# Features and targets
-X = grades.loc[:, grades.columns != 'Grade'].values
-y = grades.loc[:, grades.columns == 'Grade'].values
-
-# Create the encoder.
-encoder = OneHotEncoder(handle_unknown="ignore")
-# Assume for simplicity all features are categorical.
-encoder.fit(X)
-# Apply the encoder.
-X = encoder.transform(X)
-print(X)
-# Then do a Classification tree
-tree_clf = DecisionTreeClassifier(max_depth=2)
-tree_clf.fit(X, y)
-print("Train set accuracy with Decision Tree: {:.2f}".format(tree_clf.score(X,y)))
-#transfer to a decision tree graph
-export_graphviz(
- tree_clf,
- out_file="DataFiles/grade.dot",
- rounded=True,
- filled=True
-)
-cmd = 'dot -Tpng DataFiles/grade.dot -o DataFiles/grades.png'
-os.system(cmd)
-
--
Visualizing Trees, More examples
- - -import os
-from sklearn.datasets import load_breast_cancer
-from sklearn.tree import DecisionTreeClassifier
-from sklearn.model_selection import train_test_split
-from sklearn.metrics import confusion_matrix
-from sklearn.tree import export_graphviz
-
-from IPython.display import Image
-from pydot import graph_from_dot_data
-import pandas as pd
-import numpy as np
-
-
-cancer = load_breast_cancer()
-X = pd.DataFrame(cancer.data, columns=cancer.feature_names)
-print(X)
-y = pd.Categorical.from_codes(cancer.target, cancer.target_names)
-y = pd.get_dummies(y)
-print(y)
-X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=1)
-tree_clf = DecisionTreeClassifier(max_depth=5)
-tree_clf.fit(X_train, y_train)
-
-export_graphviz(
- tree_clf,
- out_file="DataFiles/cancer.dot",
- feature_names=cancer.feature_names,
- class_names=cancer.target_names,
- rounded=True,
- filled=True
-)
-cmd = 'dot -Tpng DataFiles/cancer.dot -o DataFiles/cancer.png'
-os.system(cmd)
-
--
Visualizing the Tree, The Moons
- - -# Common imports
-import numpy as np
-from sklearn.model_selection import train_test_split
-from sklearn.tree import DecisionTreeClassifier
-from sklearn.datasets import make_moons
-from sklearn.tree import export_graphviz
-from pydot import graph_from_dot_data
-import pandas as pd
-import os
-
-np.random.seed(42)
-X, y = make_moons(n_samples=100, noise=0.25, random_state=53)
-X_train, X_test, y_train, y_test = train_test_split(X,y,random_state=0)
-tree_clf = DecisionTreeClassifier(max_depth=5)
-tree_clf.fit(X_train, y_train)
-
-export_graphviz(
- tree_clf,
- out_file="DataFiles/moons.dot",
- rounded=True,
- filled=True
-)
-cmd = 'dot -Tpng DataFiles/moons.dot -o DataFiles/moons.png'
-os.system(cmd)
-
--
Other ways of visualizing the trees
- -Scikit-Learn has also another way to visualize the trees which is very useful, here with the Iris data.
- - - -from sklearn.datasets import load_iris
-from sklearn import tree
-X, y = load_iris(return_X_y=True)
-tree_clf = tree.DecisionTreeClassifier()
-tree_clf = tree_clf.fit(X, y)
-# and then plot the tree
-tree.plot_tree(tree_clf)
-
--
Printing out as text
- -Alternatively, the tree can also be exported in textual format with the function exporttext. -This method doesn’t require the installation of external libraries and is more compact: -
- - - -from sklearn.datasets import load_iris
-from sklearn.tree import DecisionTreeClassifier
-from sklearn.tree import export_text
-iris = load_iris()
-decision_tree = DecisionTreeClassifier(random_state=0, max_depth=2)
-decision_tree = decision_tree.fit(iris.data, iris.target)
-r = export_text(decision_tree, feature_names=iris['feature_names'])
-print(r)
-
--
Algorithms for Setting up Decision Trees
- -Two algorithms stand out in the set up of decision trees:
--
-
We discuss both algorithms with applications here. The popular library -Scikit-Learn uses the CART algorithm. For classification problems -you can use either the gini index or the entropy to split a tree -in two branches. -
- --
The CART algorithm for Classification
- -For classification, the CART algorithm splits the data set in two subsets using a single feature \( k \) and a threshold \( t_k \). -This could be for example a threshold set by a number below a certain circumference of a malign tumor. -
- -How do we find these two quantities? -We search for the pair \( (k,t_k) \) that produces the purest subset using for example the gini factor \( G \). -The cost function it tries to minimize is then -
-$$ -C(k,t_k) = \frac{m_{\mathrm{left}}}{m}G_{\mathrm{left}}+ \frac{m_{\mathrm{right}}}{m}G_{\mathrm{right}}, -$$ - -where \( G_{\mathrm{left/right}} \) measures the impurity of the left/right subset and \( m_{\mathrm{left/right}} \) - is the number of instances in the left/right subset -
- -Once it has successfully split the training set in two, it splits the subsets using the same logic, then the subsubsets -and so on, recursively. It stops recursing once it reaches the maximum depth (defined by the -\( max\_depth \) hyperparameter), or if it cannot find a split that will reduce impurity. A few other -hyperparameters control additional stopping conditions such as the \( min\_samples\_split \), -\( min\_samples\_leaf \), \( min\_weight\_fraction\_leaf \), and \( max\_leaf\_nodes \). -
- --
The CART algorithm for Regression
- -The CART algorithm for regression works is similar to the one for classification except that instead of trying to split the -training set in a way that minimizes say the gini or entropy impurity, it now tries to split the training set in a way that minimizes our well-known mean-squared error (MSE). The cost function is now -
-$$ -C(k,t_k) = \frac{m_{\mathrm{left}}}{m}\mathrm{MSE}_{\mathrm{left}}+ \frac{m_{\mathrm{right}}}{m}\mathrm{MSE}_{\mathrm{right}}. -$$ - -Here the MSE for a specific node is defined as
-$$ -\mathrm{MSE}_{\mathrm{node}}=\frac{1}{m_\mathrm{node}}\sum_{i\in \mathrm{node}}(\overline{y}_{\mathrm{node}}-y_i)^2, -$$ - -with
-$$ -\overline{y}_{\mathrm{node}}=\frac{1}{m_\mathrm{node}}\sum_{i\in \mathrm{node}}y_i, -$$ - -the mean value of all observations in a specific node.
- -Without any regularization, the regression task for decision trees, -just like for classification tasks, is prone to overfitting. -
- --
Computing the Gini index
- -The example we will look at is a classical one in many Machine -Learning applications. Based on various meteorological features, we -have several so-called attributes which decide whether we at the end -will do some outdoor activity like skiing, going for a bike ride etc -etc. The table here contains the feautures outlook, temperature, -humidity and wind. The target or output is whether we ride -(True=1) or whether we do something else that day (False=0). The -attributes for each feature are then sunny, overcast and rain for the -outlook, hot, cold and mild for temperature, high and normal for -humidity and weak and strong for wind. -
- -The table here summarizes the various attributes and
-| Day | Outlook | Temperature | Humidity | Wind | Ride |
|---|---|---|---|---|---|
| 1 | Sunny | Hot | High | Weak | 0 |
| 2 | Sunny | Hot | High | Strong | 1 |
| 3 | Overcast | Hot | High | Weak | 1 |
| 4 | Rain | Mild | High | Weak | 1 |
| 5 | Rain | Cool | Normal | Weak | 1 |
| 6 | Rain | Cool | Normal | Strong | 0 |
| 7 | Overcast | Cool | Normal | Strong | 1 |
| 8 | Sunny | Mild | High | Weak | 0 |
| 9 | Sunny | Cool | Normal | Weak | 1 |
| 10 | Rain | Mild | Normal | Weak | 1 |
| 11 | Sunny | Mild | Normal | Strong | 1 |
| 12 | Overcast | Mild | High | Strong | 1 |
| 13 | Overcast | Hot | Normal | Weak | 1 |
| 14 | Rain | Mild | High | Strong | 0 |
-
Simple Python Code to read in Data and perform Classification
- - - -# Common imports
-import numpy as np
-import pandas as pd
-import matplotlib.pyplot as plt
-from sklearn.tree import DecisionTreeClassifier
-from sklearn.model_selection import train_test_split
-from sklearn.tree import export_graphviz
-from sklearn.preprocessing import StandardScaler, OneHotEncoder
-from sklearn.compose import ColumnTransformer
-from IPython.display import Image
-from pydot import graph_from_dot_data
-import os
-
-# Where to save the figures and data files
-PROJECT_ROOT_DIR = "Results"
-FIGURE_ID = "Results/FigureFiles"
-DATA_ID = "DataFiles/"
-
-if not os.path.exists(PROJECT_ROOT_DIR):
- os.mkdir(PROJECT_ROOT_DIR)
-
-if not os.path.exists(FIGURE_ID):
- os.makedirs(FIGURE_ID)
-
-if not os.path.exists(DATA_ID):
- os.makedirs(DATA_ID)
-
-def image_path(fig_id):
- return os.path.join(FIGURE_ID, fig_id)
-
-def data_path(dat_id):
- return os.path.join(DATA_ID, dat_id)
-
-def save_fig(fig_id):
- plt.savefig(image_path(fig_id) + ".png", format='png')
-
-infile = open(data_path("rideclass.csv"),'r')
-
-# Read the experimental data with Pandas
-from IPython.display import display
-ridedata = pd.read_csv(infile,names = ('Outlook','Temperature','Humidity','Wind','Ride'))
-ridedata = pd.DataFrame(ridedata)
-
-# Features and targets
-X = ridedata.loc[:, ridedata.columns != 'Ride'].values
-y = ridedata.loc[:, ridedata.columns == 'Ride'].values
-
-# Create the encoder.
-encoder = OneHotEncoder(handle_unknown="ignore")
-# Assume for simplicity all features are categorical.
-encoder.fit(X)
-# Apply the encoder.
-X = encoder.transform(X)
-print(X)
-# Then do a Classification tree
-tree_clf = DecisionTreeClassifier(max_depth=2)
-tree_clf.fit(X, y)
-print("Train set accuracy with Decision Tree: {:.2f}".format(tree_clf.score(X,y)))
-#transfer to a decision tree graph
-export_graphviz(
- tree_clf,
- out_file="DataFiles/ride.dot",
- rounded=True,
- filled=True
-)
-cmd = 'dot -Tpng DataFiles/cancer.dot -o DataFiles/cancer.png'
-os.system(cmd)
-
--
Computing the Gini Factor
- -The above functions (gini, entropy and misclassification error) are -important components of the so-called CART algorithm. We will discuss -this algorithm below after we have discussed the information gain -algorithm ID3. -
- -In the example here we have converted all our attributes into numerical values \( 0,1,2 \) etc.
- - - -# Split a dataset based on an attribute and an attribute value
-def test_split(index, value, dataset):
- left, right = list(), list()
- for row in dataset:
- if row[index] < value:
- left.append(row)
- else:
- right.append(row)
- return left, right
-
-# Calculate the Gini index for a split dataset
-def gini_index(groups, classes):
- # count all samples at split point
- n_instances = float(sum([len(group) for group in groups]))
- # sum weighted Gini index for each group
- gini = 0.0
- for group in groups:
- size = float(len(group))
- # avoid divide by zero
- if size == 0:
- continue
- score = 0.0
- # score the group based on the score for each class
- for class_val in classes:
- p = [row[-1] for row in group].count(class_val) / size
- score += p * p
- # weight the group score by its relative size
- gini += (1.0 - score) * (size / n_instances)
- return gini
-
-# Select the best split point for a dataset
-def get_split(dataset):
- class_values = list(set(row[-1] for row in dataset))
- b_index, b_value, b_score, b_groups = 999, 999, 999, None
- for index in range(len(dataset[0])-1):
- for row in dataset:
- groups = test_split(index, row[index], dataset)
- gini = gini_index(groups, class_values)
- print('X%d < %.3f Gini=%.3f' % ((index+1), row[index], gini))
- if gini < b_score:
- b_index, b_value, b_score, b_groups = index, row[index], gini, groups
- return {'index':b_index, 'value':b_value, 'groups':b_groups}
-
-dataset = [[0,0,0,0,0],
- [0,0,0,1,1],
- [1,0,0,0,1],
- [2,1,0,0,1],
- [2,2,1,0,1],
- [2,2,1,1,0],
- [1,2,1,1,1],
- [0,1,0,0,0],
- [0,2,1,0,1],
- [2,1,1,0,1],
- [0,1,1,1,1],
- [1,1,0,1,1],
- [1,0,1,0,1],
- [2,1,0,1,0]]
-
-split = get_split(dataset)
-print('Split: [X%d < %.3f]' % ((split['index']+1), split['value']))
-
--
Entropy and the ID3 algorithm
- -The ID3 algorithm learns decision trees by constructing -them in a top down way, beginning with the question which attribute should be tested at the root of the tree? -
- --
-
The ID3 algorithm selects which attribute to test at each node in the -tree. -
- -We would like to select the attribute that is most useful for classifying -examples. -
- -What is a good quantitative measure of the worth of an attribute?
- -Information gain measures how well a given attribute separates the -training examples according to their target classification. -
- -The ID3 algorithm uses this information gain measure to select among the candidate -attributes at each step while growing the tree. -
- --
Cancer Data again now with Decision Trees and other Methods
- - -import matplotlib.pyplot as plt
-import numpy as np
-from sklearn.model_selection import train_test_split
-from sklearn.datasets import load_breast_cancer
-from sklearn.svm import SVC
-from sklearn.linear_model import LogisticRegression
-from sklearn.tree import DecisionTreeClassifier
-
-# Load the data
-cancer = load_breast_cancer()
-
-X_train, X_test, y_train, y_test = train_test_split(cancer.data,cancer.target,random_state=0)
-print(X_train.shape)
-print(X_test.shape)
-# Logistic Regression
-logreg = LogisticRegression(solver='lbfgs')
-logreg.fit(X_train, y_train)
-print("Test set accuracy with Logistic Regression: {:.2f}".format(logreg.score(X_test,y_test)))
-# Support vector machine
-svm = SVC(gamma='auto', C=100)
-svm.fit(X_train, y_train)
-print("Test set accuracy with SVM: {:.2f}".format(svm.score(X_test,y_test)))
-# Decision Trees
-deep_tree_clf = DecisionTreeClassifier(max_depth=None)
-deep_tree_clf.fit(X_train, y_train)
-print("Test set accuracy with Decision Trees: {:.2f}".format(deep_tree_clf.score(X_test,y_test)))
-#now scale the data
-from sklearn.preprocessing import StandardScaler
-scaler = StandardScaler()
-scaler.fit(X_train)
-X_train_scaled = scaler.transform(X_train)
-X_test_scaled = scaler.transform(X_test)
-# Logistic Regression
-logreg.fit(X_train_scaled, y_train)
-print("Test set accuracy Logistic Regression with scaled data: {:.2f}".format(logreg.score(X_test_scaled,y_test)))
-# Support Vector Machine
-svm.fit(X_train_scaled, y_train)
-print("Test set accuracy SVM with scaled data: {:.2f}".format(logreg.score(X_test_scaled,y_test)))
-# Decision Trees
-deep_tree_clf.fit(X_train_scaled, y_train)
-print("Test set accuracy with Decision Trees and scaled data: {:.2f}".format(deep_tree_clf.score(X_test_scaled,y_test)))
-
--
Another example, the moons again
- - -from __future__ import division, print_function, unicode_literals
-
-# Common imports
-import numpy as np
-import os
-
-# to make this notebook's output stable across runs
-np.random.seed(42)
-
-# To plot pretty figures
-import matplotlib
-import matplotlib.pyplot as plt
-from matplotlib.colors import ListedColormap
-plt.rcParams['axes.labelsize'] = 14
-plt.rcParams['xtick.labelsize'] = 12
-plt.rcParams['ytick.labelsize'] = 12
-
-
-from sklearn.svm import SVC
-from sklearn import datasets
-from sklearn.tree import DecisionTreeClassifier
-from sklearn.datasets import make_moons
-from sklearn.tree import export_graphviz
-
-Xm, ym = make_moons(n_samples=100, noise=0.25, random_state=53)
-
-deep_tree_clf1 = DecisionTreeClassifier(random_state=42)
-deep_tree_clf2 = DecisionTreeClassifier(min_samples_leaf=4, random_state=42)
-deep_tree_clf1.fit(Xm, ym)
-deep_tree_clf2.fit(Xm, ym)
-
-
-def plot_decision_boundary(clf, X, y, axes=[0, 7.5, 0, 3], iris=True, legend=False, plot_training=True):
- x1s = np.linspace(axes[0], axes[1], 100)
- x2s = np.linspace(axes[2], axes[3], 100)
- x1, x2 = np.meshgrid(x1s, x2s)
- X_new = np.c_[x1.ravel(), x2.ravel()]
- y_pred = clf.predict(X_new).reshape(x1.shape)
- custom_cmap = ListedColormap(['#fafab0','#9898ff','#a0faa0'])
- plt.contourf(x1, x2, y_pred, alpha=0.3, cmap=custom_cmap)
- if not iris:
- custom_cmap2 = ListedColormap(['#7d7d58','#4c4c7f','#507d50'])
- plt.contour(x1, x2, y_pred, cmap=custom_cmap2, alpha=0.8)
- if plot_training:
- plt.plot(X[:, 0][y==0], X[:, 1][y==0], "yo", label="Iris-Setosa")
- plt.plot(X[:, 0][y==1], X[:, 1][y==1], "bs", label="Iris-Versicolor")
- plt.plot(X[:, 0][y==2], X[:, 1][y==2], "g^", label="Iris-Virginica")
- plt.axis(axes)
- if iris:
- plt.xlabel("Petal length", fontsize=14)
- plt.ylabel("Petal width", fontsize=14)
- else:
- plt.xlabel(r"$x_1$", fontsize=18)
- plt.ylabel(r"$x_2$", fontsize=18, rotation=0)
- if legend:
- plt.legend(loc="lower right", fontsize=14)
-plt.figure(figsize=(11, 4))
-plt.subplot(121)
-plot_decision_boundary(deep_tree_clf1, Xm, ym, axes=[-1.5, 2.5, -1, 1.5], iris=False)
-plt.title("No restrictions", fontsize=16)
-plt.subplot(122)
-plot_decision_boundary(deep_tree_clf2, Xm, ym, axes=[-1.5, 2.5, -1, 1.5], iris=False)
-plt.title("min_samples_leaf = {}".format(deep_tree_clf2.min_samples_leaf), fontsize=14)
-plt.show()
-
--
Playing around with regions
- - -np.random.seed(6)
-Xs = np.random.rand(100, 2) - 0.5
-ys = (Xs[:, 0] > 0).astype(np.float32) * 2
-
-angle = np.pi/4
-rotation_matrix = np.array([[np.cos(angle), -np.sin(angle)], [np.sin(angle), np.cos(angle)]])
-Xsr = Xs.dot(rotation_matrix)
-
-tree_clf_s = DecisionTreeClassifier(random_state=42)
-tree_clf_s.fit(Xs, ys)
-tree_clf_sr = DecisionTreeClassifier(random_state=42)
-tree_clf_sr.fit(Xsr, ys)
-
-plt.figure(figsize=(11, 4))
-plt.subplot(121)
-plot_decision_boundary(tree_clf_s, Xs, ys, axes=[-0.7, 0.7, -0.7, 0.7], iris=False)
-plt.subplot(122)
-plot_decision_boundary(tree_clf_sr, Xsr, ys, axes=[-0.7, 0.7, -0.7, 0.7], iris=False)
-
-plt.show()
-
--
Regression trees
- - -# Quadratic training set + noise
-np.random.seed(42)
-m = 200
-X = np.random.rand(m, 1)
-y = 4 * (X - 0.5) ** 2
-y = y + np.random.randn(m, 1) / 10
-
-from sklearn.tree import DecisionTreeRegressor
-
-tree_reg = DecisionTreeRegressor(max_depth=2, random_state=42)
-tree_reg.fit(X, y)
-
--
Final regressor code
- - -from sklearn.tree import DecisionTreeRegressor
-
-tree_reg1 = DecisionTreeRegressor(random_state=42, max_depth=2)
-tree_reg2 = DecisionTreeRegressor(random_state=42, max_depth=3)
-tree_reg1.fit(X, y)
-tree_reg2.fit(X, y)
-
-def plot_regression_predictions(tree_reg, X, y, axes=[0, 1, -0.2, 1], ylabel="$y$"):
- x1 = np.linspace(axes[0], axes[1], 500).reshape(-1, 1)
- y_pred = tree_reg.predict(x1)
- plt.axis(axes)
- plt.xlabel("$x_1$", fontsize=18)
- if ylabel:
- plt.ylabel(ylabel, fontsize=18, rotation=0)
- plt.plot(X, y, "b.")
- plt.plot(x1, y_pred, "r.-", linewidth=2, label=r"$\hat{y}$")
-
-plt.figure(figsize=(11, 4))
-plt.subplot(121)
-plot_regression_predictions(tree_reg1, X, y)
-for split, style in ((0.1973, "k-"), (0.0917, "k--"), (0.7718, "k--")):
- plt.plot([split, split], [-0.2, 1], style, linewidth=2)
-plt.text(0.21, 0.65, "Depth=0", fontsize=15)
-plt.text(0.01, 0.2, "Depth=1", fontsize=13)
-plt.text(0.65, 0.8, "Depth=1", fontsize=13)
-plt.legend(loc="upper center", fontsize=18)
-plt.title("max_depth=2", fontsize=14)
-
-plt.subplot(122)
-plot_regression_predictions(tree_reg2, X, y, ylabel=None)
-for split, style in ((0.1973, "k-"), (0.0917, "k--"), (0.7718, "k--")):
- plt.plot([split, split], [-0.2, 1], style, linewidth=2)
-for split in (0.0458, 0.1298, 0.2873, 0.9040):
- plt.plot([split, split], [-0.2, 1], "k:", linewidth=1)
-plt.text(0.3, 0.5, "Depth=2", fontsize=13)
-plt.title("max_depth=3", fontsize=14)
-
-plt.show()
-
-tree_reg1 = DecisionTreeRegressor(random_state=42)
-tree_reg2 = DecisionTreeRegressor(random_state=42, min_samples_leaf=10)
-tree_reg1.fit(X, y)
-tree_reg2.fit(X, y)
-
-x1 = np.linspace(0, 1, 500).reshape(-1, 1)
-y_pred1 = tree_reg1.predict(x1)
-y_pred2 = tree_reg2.predict(x1)
-
-plt.figure(figsize=(11, 4))
-
-plt.subplot(121)
-plt.plot(X, y, "b.")
-plt.plot(x1, y_pred1, "r.-", linewidth=2, label=r"$\hat{y}$")
-plt.axis([0, 1, -0.2, 1.1])
-plt.xlabel("$x_1$", fontsize=18)
-plt.ylabel("$y$", fontsize=18, rotation=0)
-plt.legend(loc="upper center", fontsize=18)
-plt.title("No restrictions", fontsize=14)
-
-plt.subplot(122)
-plt.plot(X, y, "b.")
-plt.plot(x1, y_pred2, "r.-", linewidth=2, label=r"$\hat{y}$")
-plt.axis([0, 1, -0.2, 1.1])
-plt.xlabel("$x_1$", fontsize=18)
-plt.title("min_samples_leaf={}".format(tree_reg2.min_samples_leaf), fontsize=14)
-
-plt.show()
-
-Why Voting?
diff --git a/doc/pub/week44/html/week44.html b/doc/pub/week44/html/week44.html index c910b1e24..470142c96 100644 --- a/doc/pub/week44/html/week44.html +++ b/doc/pub/week44/html/week44.html @@ -219,67 +219,6 @@ div.toc p,a { 2, None, 'the-cart-algorithm-for-regression'), - ('Computing the Gini index', 2, None, 'computing-the-gini-index'), - ('Simple Python Code to read in Data and perform Classification', - 2, - None, - 'simple-python-code-to-read-in-data-and-perform-classification'), - ('Computing the Gini Factor', - 2, - None, - 'computing-the-gini-factor'), - ('Entropy and the ID3 algorithm', - 2, - None, - 'entropy-and-the-id3-algorithm'), - ('Cancer Data again now with Decision Trees and other Methods', - 2, - None, - 'cancer-data-again-now-with-decision-trees-and-other-methods'), - ('Another example, the moons again', - 2, - None, - 'another-example-the-moons-again'), - ('Playing around with regions', - 2, - None, - 'playing-around-with-regions'), - ('Regression trees', 2, None, 'regression-trees'), - ('Final regressor code', 2, None, 'final-regressor-code'), - ('Pros and cons of trees, pros', - 2, - None, - 'pros-and-cons-of-trees-pros'), - ('Disadvantages', 2, None, 'disadvantages'), - ('Ensemble Methods: From a Single Tree to Many Trees and Extreme ' - 'Boosting, Meet the Jungle of Methods', - 2, - None, - 'ensemble-methods-from-a-single-tree-to-many-trees-and-extreme-boosting-meet-the-jungle-of-methods'), - ('An Overview of Ensemble Methods', - 2, - None, - 'an-overview-of-ensemble-methods'), - ('Bagging', 2, None, 'bagging'), - ('More bagging', 2, None, 'more-bagging'), - ('Simple Voting Example, head or tail', - 2, - None, - 'simple-voting-example-head-or-tail'), - ('Using the Voting Classifier', - 2, - None, - 'using-the-voting-classifier'), - ('Please, not the moons again! Voting and Bagging', - 2, - None, - 'please-not-the-moons-again-voting-and-bagging'), - ('Bagging Examples', 2, None, 'bagging-examples'), - ('Making your own Bootstrap: Changing the Level of the Decision ' - 'Tree', - 2, - None, - 'making-your-own-bootstrap-changing-the-level-of-the-decision-tree'), ('Why binary splits?', 2, None, 'why-binary-splits'), ('Computing a Tree using the Gini Index', 2, @@ -354,6 +293,70 @@ div.toc p,a { 'playing-around-with-regions'), ('Regression trees', 2, None, 'regression-trees'), ('Final regressor code', 2, None, 'final-regressor-code'), + ('Example: Computing the Gini index', + 2, + None, + 'example-computing-the-gini-index'), + ('Simple Python Code to read in Data and perform Classification', + 2, + None, + 'simple-python-code-to-read-in-data-and-perform-classification'), + ('Computing the Gini Factor', + 2, + None, + 'computing-the-gini-factor'), + ('Entropy and the ID3 algorithm', + 2, + None, + 'entropy-and-the-id3-algorithm'), + ('Cancer Data again now with Decision Trees and other Methods', + 2, + None, + 'cancer-data-again-now-with-decision-trees-and-other-methods'), + ('Another example, the moons again', + 2, + None, + 'another-example-the-moons-again'), + ('Playing around with regions', + 2, + None, + 'playing-around-with-regions'), + ('Regression trees', 2, None, 'regression-trees'), + ('Final regressor code', 2, None, 'final-regressor-code'), + ('Pros and cons of trees, pros', + 2, + None, + 'pros-and-cons-of-trees-pros'), + ('Disadvantages', 2, None, 'disadvantages'), + ('Ensemble Methods: From a Single Tree to Many Trees and Extreme ' + 'Boosting, Meet the Jungle of Methods', + 2, + None, + 'ensemble-methods-from-a-single-tree-to-many-trees-and-extreme-boosting-meet-the-jungle-of-methods'), + ('An Overview of Ensemble Methods', + 2, + None, + 'an-overview-of-ensemble-methods'), + ('Bagging', 2, None, 'bagging'), + ('More bagging', 2, None, 'more-bagging'), + ('Simple Voting Example, head or tail', + 2, + None, + 'simple-voting-example-head-or-tail'), + ('Using the Voting Classifier', + 2, + None, + 'using-the-voting-classifier'), + ('Please, not the moons again! Voting and Bagging', + 2, + None, + 'please-not-the-moons-again-voting-and-bagging'), + ('Bagging Examples', 2, None, 'bagging-examples'), + ('Making your own Bootstrap: Changing the Level of the Decision ' + 'Tree', + 2, + None, + 'making-your-own-bootstrap-changing-the-level-of-the-decision-tree'), ('Why Voting?', 2, None, 'why-voting'), ('Tossing coins', 2, None, 'tossing-coins'), ('Standard imports first', 2, None, 'standard-imports-first'), @@ -413,7 +416,7 @@ MathJax.Hub.Config({Oct 30, 2022
+Nov 1, 2022
@@ -1218,6 +1221,429 @@ r = export_text(decision_tree, feature_names
+
Algorithms for Setting up Decision Trees
+ +Two algorithms stand out in the set up of decision trees:
+-
+
We discuss both algorithms with applications here. The popular library +Scikit-Learn uses the CART algorithm. For classification problems +you can use either the gini index or the entropy to split a tree +in two branches. +
+ ++
The CART algorithm for Classification
+ +For classification, the CART algorithm splits the data set in two subsets using a single feature \( k \) and a threshold \( t_k \). +This could be for example a threshold set by a number below a certain circumference of a malign tumor. +
+ +How do we find these two quantities? +We search for the pair \( (k,t_k) \) that produces the purest subset using for example the gini factor \( G \). +The cost function it tries to minimize is then +
+$$ +C(k,t_k) = \frac{m_{\mathrm{left}}}{m}G_{\mathrm{left}}+ \frac{m_{\mathrm{right}}}{m}G_{\mathrm{right}}, +$$ + +where \( G_{\mathrm{left/right}} \) measures the impurity of the left/right subset and \( m_{\mathrm{left/right}} \) + is the number of instances in the left/right subset +
+ +Once it has successfully split the training set in two, it splits the subsets using the same logic, then the subsubsets +and so on, recursively. It stops recursing once it reaches the maximum depth (defined by the +\( max\_depth \) hyperparameter), or if it cannot find a split that will reduce impurity. A few other +hyperparameters control additional stopping conditions such as the \( min\_samples\_split \), +\( min\_samples\_leaf \), \( min\_weight\_fraction\_leaf \), and \( max\_leaf\_nodes \). +
+ ++
The CART algorithm for Regression
+ +The CART algorithm for regression works is similar to the one for classification except that instead of trying to split the +training set in a way that minimizes say the gini or entropy impurity, it now tries to split the training set in a way that minimizes our well-known mean-squared error (MSE). The cost function is now +
+$$ +C(k,t_k) = \frac{m_{\mathrm{left}}}{m}\mathrm{MSE}_{\mathrm{left}}+ \frac{m_{\mathrm{right}}}{m}\mathrm{MSE}_{\mathrm{right}}. +$$ + +Here the MSE for a specific node is defined as
+$$ +\mathrm{MSE}_{\mathrm{node}}=\frac{1}{m_\mathrm{node}}\sum_{i\in \mathrm{node}}(\overline{y}_{\mathrm{node}}-y_i)^2, +$$ + +with
+$$ +\overline{y}_{\mathrm{node}}=\frac{1}{m_\mathrm{node}}\sum_{i\in \mathrm{node}}y_i, +$$ + +the mean value of all observations in a specific node.
+ +Without any regularization, the regression task for decision trees, +just like for classification tasks, is prone to overfitting. +
+ ++
Why binary splits?
+ +It is custom to split to a tree uising binary splits. The reason is +that multiway splits fragment the data too quickly, leaving +insufficient data at the next level down. Multiway splits can be +achieved by a series of binary split and this is normally preferred. +
+ ++
Computing a Tree using the Gini Index
+ +Consider the following example with attributes/features and two +possible outcomes (classes) for each attribute. Assume we wish to find some +correlations between the average grade of a student as function of the +number of hours studied and hours slept. We want also to correlate the +grade in a given course with the general trend, whether the students +recently has gotten grades below average or above. +
+ +We have three features/attributes
+-
+
+
The Table
+ +| Grade Trend | Hours slept | Hours Studied | Grade |
|---|---|---|---|
| Above | Low | High | Above |
| Below | High | Low | Below |
| Above | Low | High | Above |
| Above | High | High | Above |
| Below | Low | High | Below |
| Above | Low | Low | Below |
| Below | High | High | Below |
| Below | Low | High | Below |
| Above | Low | Low | Below |
| Above | High | High | Above |
+
Computing the various Gini Indices
+ +In computations we will translate all classes into numbers. Being +these binary classes, they can easily be split into ones and zeros. +
+ ++
See handwritten notes for Thursday November 11
++
Computing the various Gini Indices, Hours slept
+ ++
See handwritten notes for Thursday November 3
++
Computing the various Gini Indices, Hours studied
+ ++
See handwritten notes for Thursday November 3
+For final tree, see the above handwritten notes
+ ++
A possible code using Scikit-Learn
+ + + +# Common imports
+import numpy as np
+import pandas as pd
+import matplotlib.pyplot as plt
+from sklearn.tree import DecisionTreeClassifier
+from sklearn.model_selection import train_test_split
+from sklearn.tree import export_graphviz
+from sklearn.preprocessing import StandardScaler, OneHotEncoder
+from sklearn.compose import ColumnTransformer
+from IPython.display import Image
+from pydot import graph_from_dot_data
+import os
+
+# Where to save the figures and data files
+PROJECT_ROOT_DIR = "Results"
+FIGURE_ID = "Results/FigureFiles"
+DATA_ID = "DataFiles/"
+
+if not os.path.exists(PROJECT_ROOT_DIR):
+ os.mkdir(PROJECT_ROOT_DIR)
+
+if not os.path.exists(FIGURE_ID):
+ os.makedirs(FIGURE_ID)
+
+if not os.path.exists(DATA_ID):
+ os.makedirs(DATA_ID)
+
+def image_path(fig_id):
+ return os.path.join(FIGURE_ID, fig_id)
+
+def data_path(dat_id):
+ return os.path.join(DATA_ID, dat_id)
+
+def save_fig(fig_id):
+ plt.savefig(image_path(fig_id) + ".png", format='png')
+
+infile = open(data_path("grades.csv"),'r')
+
+# Read the experimental data with Pandas
+from IPython.display import display
+grades = pd.read_csv(infile,names = ('Trend','Sleep','Studied','Grade'))
+grades = pd.DataFrame(grades)
+
+# Features and targets
+X = grades.loc[:, grades.columns != 'Grade'].values
+y = grades.loc[:, grades.columns == 'Grade'].values
+
+# Create the encoder.
+encoder = OneHotEncoder(handle_unknown="ignore")
+# Assume for simplicity all features are categorical.
+encoder.fit(X)
+# Apply the encoder.
+X = encoder.transform(X)
+print(X)
+# Then do a Classification tree
+tree_clf = DecisionTreeClassifier(max_depth=2)
+tree_clf.fit(X, y)
+print("Train set accuracy with Decision Tree: {:.2f}".format(tree_clf.score(X,y)))
+#transfer to a decision tree graph
+export_graphviz(
+ tree_clf,
+ out_file="DataFiles/grade.dot",
+ rounded=True,
+ filled=True
+)
+cmd = 'dot -Tpng DataFiles/grade.dot -o DataFiles/grades.png'
+os.system(cmd)
+
++
Visualizing Trees, More examples
+ + +import os
+from sklearn.datasets import load_breast_cancer
+from sklearn.tree import DecisionTreeClassifier
+from sklearn.model_selection import train_test_split
+from sklearn.metrics import confusion_matrix
+from sklearn.tree import export_graphviz
+
+from IPython.display import Image
+from pydot import graph_from_dot_data
+import pandas as pd
+import numpy as np
+
+
+cancer = load_breast_cancer()
+X = pd.DataFrame(cancer.data, columns=cancer.feature_names)
+print(X)
+y = pd.Categorical.from_codes(cancer.target, cancer.target_names)
+y = pd.get_dummies(y)
+print(y)
+X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=1)
+tree_clf = DecisionTreeClassifier(max_depth=5)
+tree_clf.fit(X_train, y_train)
+
+export_graphviz(
+ tree_clf,
+ out_file="DataFiles/cancer.dot",
+ feature_names=cancer.feature_names,
+ class_names=cancer.target_names,
+ rounded=True,
+ filled=True
+)
+cmd = 'dot -Tpng DataFiles/cancer.dot -o DataFiles/cancer.png'
+os.system(cmd)
+
++
Visualizing the Tree, The Moons
+ + +# Common imports
+import numpy as np
+from sklearn.model_selection import train_test_split
+from sklearn.tree import DecisionTreeClassifier
+from sklearn.datasets import make_moons
+from sklearn.tree import export_graphviz
+from pydot import graph_from_dot_data
+import pandas as pd
+import os
+
+np.random.seed(42)
+X, y = make_moons(n_samples=100, noise=0.25, random_state=53)
+X_train, X_test, y_train, y_test = train_test_split(X,y,random_state=0)
+tree_clf = DecisionTreeClassifier(max_depth=5)
+tree_clf.fit(X_train, y_train)
+
+export_graphviz(
+ tree_clf,
+ out_file="DataFiles/moons.dot",
+ rounded=True,
+ filled=True
+)
+cmd = 'dot -Tpng DataFiles/moons.dot -o DataFiles/moons.png'
+os.system(cmd)
+
++
Other ways of visualizing the trees
+ +Scikit-Learn has also another way to visualize the trees which is very useful, here with the Iris data.
+ + + +from sklearn.datasets import load_iris
+from sklearn import tree
+X, y = load_iris(return_X_y=True)
+tree_clf = tree.DecisionTreeClassifier()
+tree_clf = tree_clf.fit(X, y)
+# and then plot the tree
+tree.plot_tree(tree_clf)
+
++
Printing out as text
+ +Alternatively, the tree can also be exported in textual format with the function exporttext. +This method doesn’t require the installation of external libraries and is more compact: +
+ + + +from sklearn.datasets import load_iris
+from sklearn.tree import DecisionTreeClassifier
+from sklearn.tree import export_text
+iris = load_iris()
+decision_tree = DecisionTreeClassifier(random_state=0, max_depth=2)
+decision_tree = decision_tree.fit(iris.data, iris.target)
+r = export_text(decision_tree, feature_names=iris['feature_names'])
+print(r)
+
+Algorithms for Setting up Decision Trees
@@ -1906,6 +2332,628 @@ plt.show()+
Example: Computing the Gini index
+ +The example we will look at is a classical one in many Machine +Learning applications. Based on various meteorological features, we +have several so-called attributes which decide whether we at the end +will do some outdoor activity like skiing, going for a bike ride etc +etc. The table here contains the feautures outlook, temperature, +humidity and wind. The target or output is whether we ride +(True=1) or whether we do something else that day (False=0). The +attributes for each feature are then sunny, overcast and rain for the +outlook, hot, cold and mild for temperature, high and normal for +humidity and weak and strong for wind. +
+ +The table here summarizes the various attributes and
+| Day | Outlook | Temperature | Humidity | Wind | Ride |
|---|---|---|---|---|---|
| 1 | Sunny | Hot | High | Weak | 0 |
| 2 | Sunny | Hot | High | Strong | 1 |
| 3 | Overcast | Hot | High | Weak | 1 |
| 4 | Rain | Mild | High | Weak | 1 |
| 5 | Rain | Cool | Normal | Weak | 1 |
| 6 | Rain | Cool | Normal | Strong | 0 |
| 7 | Overcast | Cool | Normal | Strong | 1 |
| 8 | Sunny | Mild | High | Weak | 0 |
| 9 | Sunny | Cool | Normal | Weak | 1 |
| 10 | Rain | Mild | Normal | Weak | 1 |
| 11 | Sunny | Mild | Normal | Strong | 1 |
| 12 | Overcast | Mild | High | Strong | 1 |
| 13 | Overcast | Hot | Normal | Weak | 1 |
| 14 | Rain | Mild | High | Strong | 0 |
+
Simple Python Code to read in Data and perform Classification
+ + + +# Common imports
+import numpy as np
+import pandas as pd
+import matplotlib.pyplot as plt
+from sklearn.tree import DecisionTreeClassifier
+from sklearn.model_selection import train_test_split
+from sklearn.tree import export_graphviz
+from sklearn.preprocessing import StandardScaler, OneHotEncoder
+from sklearn.compose import ColumnTransformer
+from IPython.display import Image
+from pydot import graph_from_dot_data
+import os
+
+# Where to save the figures and data files
+PROJECT_ROOT_DIR = "Results"
+FIGURE_ID = "Results/FigureFiles"
+DATA_ID = "DataFiles/"
+
+if not os.path.exists(PROJECT_ROOT_DIR):
+ os.mkdir(PROJECT_ROOT_DIR)
+
+if not os.path.exists(FIGURE_ID):
+ os.makedirs(FIGURE_ID)
+
+if not os.path.exists(DATA_ID):
+ os.makedirs(DATA_ID)
+
+def image_path(fig_id):
+ return os.path.join(FIGURE_ID, fig_id)
+
+def data_path(dat_id):
+ return os.path.join(DATA_ID, dat_id)
+
+def save_fig(fig_id):
+ plt.savefig(image_path(fig_id) + ".png", format='png')
+
+infile = open(data_path("rideclass.csv"),'r')
+
+# Read the experimental data with Pandas
+from IPython.display import display
+ridedata = pd.read_csv(infile,names = ('Outlook','Temperature','Humidity','Wind','Ride'))
+ridedata = pd.DataFrame(ridedata)
+
+# Features and targets
+X = ridedata.loc[:, ridedata.columns != 'Ride'].values
+y = ridedata.loc[:, ridedata.columns == 'Ride'].values
+
+# Create the encoder.
+encoder = OneHotEncoder(handle_unknown="ignore")
+# Assume for simplicity all features are categorical.
+encoder.fit(X)
+# Apply the encoder.
+X = encoder.transform(X)
+print(X)
+# Then do a Classification tree
+tree_clf = DecisionTreeClassifier(max_depth=2)
+tree_clf.fit(X, y)
+print("Train set accuracy with Decision Tree: {:.2f}".format(tree_clf.score(X,y)))
+#transfer to a decision tree graph
+export_graphviz(
+ tree_clf,
+ out_file="DataFiles/ride.dot",
+ rounded=True,
+ filled=True
+)
+cmd = 'dot -Tpng DataFiles/cancer.dot -o DataFiles/cancer.png'
+os.system(cmd)
+
++
Computing the Gini Factor
+ +The above functions (gini, entropy and misclassification error) are +important components of the so-called CART algorithm. We will discuss +this algorithm below after we have discussed the information gain +algorithm ID3. +
+ +In the example here we have converted all our attributes into numerical values \( 0,1,2 \) etc.
+ + + +# Split a dataset based on an attribute and an attribute value
+def test_split(index, value, dataset):
+ left, right = list(), list()
+ for row in dataset:
+ if row[index] < value:
+ left.append(row)
+ else:
+ right.append(row)
+ return left, right
+
+# Calculate the Gini index for a split dataset
+def gini_index(groups, classes):
+ # count all samples at split point
+ n_instances = float(sum([len(group) for group in groups]))
+ # sum weighted Gini index for each group
+ gini = 0.0
+ for group in groups:
+ size = float(len(group))
+ # avoid divide by zero
+ if size == 0:
+ continue
+ score = 0.0
+ # score the group based on the score for each class
+ for class_val in classes:
+ p = [row[-1] for row in group].count(class_val) / size
+ score += p * p
+ # weight the group score by its relative size
+ gini += (1.0 - score) * (size / n_instances)
+ return gini
+
+# Select the best split point for a dataset
+def get_split(dataset):
+ class_values = list(set(row[-1] for row in dataset))
+ b_index, b_value, b_score, b_groups = 999, 999, 999, None
+ for index in range(len(dataset[0])-1):
+ for row in dataset:
+ groups = test_split(index, row[index], dataset)
+ gini = gini_index(groups, class_values)
+ print('X%d < %.3f Gini=%.3f' % ((index+1), row[index], gini))
+ if gini < b_score:
+ b_index, b_value, b_score, b_groups = index, row[index], gini, groups
+ return {'index':b_index, 'value':b_value, 'groups':b_groups}
+
+dataset = [[0,0,0,0,0],
+ [0,0,0,1,1],
+ [1,0,0,0,1],
+ [2,1,0,0,1],
+ [2,2,1,0,1],
+ [2,2,1,1,0],
+ [1,2,1,1,1],
+ [0,1,0,0,0],
+ [0,2,1,0,1],
+ [2,1,1,0,1],
+ [0,1,1,1,1],
+ [1,1,0,1,1],
+ [1,0,1,0,1],
+ [2,1,0,1,0]]
+
+split = get_split(dataset)
+print('Split: [X%d < %.3f]' % ((split['index']+1), split['value']))
+
++
Entropy and the ID3 algorithm
+ +The ID3 algorithm learns decision trees by constructing +them in a top down way, beginning with the question which attribute should be tested at the root of the tree? +
+ +-
+
The ID3 algorithm selects which attribute to test at each node in the +tree. +
+ +We would like to select the attribute that is most useful for classifying +examples. +
+ +What is a good quantitative measure of the worth of an attribute?
+ +Information gain measures how well a given attribute separates the +training examples according to their target classification. +
+ +The ID3 algorithm uses this information gain measure to select among the candidate +attributes at each step while growing the tree. +
+ ++
Cancer Data again now with Decision Trees and other Methods
+ + +import matplotlib.pyplot as plt
+import numpy as np
+from sklearn.model_selection import train_test_split
+from sklearn.datasets import load_breast_cancer
+from sklearn.svm import SVC
+from sklearn.linear_model import LogisticRegression
+from sklearn.tree import DecisionTreeClassifier
+
+# Load the data
+cancer = load_breast_cancer()
+
+X_train, X_test, y_train, y_test = train_test_split(cancer.data,cancer.target,random_state=0)
+print(X_train.shape)
+print(X_test.shape)
+# Logistic Regression
+logreg = LogisticRegression(solver='lbfgs')
+logreg.fit(X_train, y_train)
+print("Test set accuracy with Logistic Regression: {:.2f}".format(logreg.score(X_test,y_test)))
+# Support vector machine
+svm = SVC(gamma='auto', C=100)
+svm.fit(X_train, y_train)
+print("Test set accuracy with SVM: {:.2f}".format(svm.score(X_test,y_test)))
+# Decision Trees
+deep_tree_clf = DecisionTreeClassifier(max_depth=None)
+deep_tree_clf.fit(X_train, y_train)
+print("Test set accuracy with Decision Trees: {:.2f}".format(deep_tree_clf.score(X_test,y_test)))
+#now scale the data
+from sklearn.preprocessing import StandardScaler
+scaler = StandardScaler()
+scaler.fit(X_train)
+X_train_scaled = scaler.transform(X_train)
+X_test_scaled = scaler.transform(X_test)
+# Logistic Regression
+logreg.fit(X_train_scaled, y_train)
+print("Test set accuracy Logistic Regression with scaled data: {:.2f}".format(logreg.score(X_test_scaled,y_test)))
+# Support Vector Machine
+svm.fit(X_train_scaled, y_train)
+print("Test set accuracy SVM with scaled data: {:.2f}".format(logreg.score(X_test_scaled,y_test)))
+# Decision Trees
+deep_tree_clf.fit(X_train_scaled, y_train)
+print("Test set accuracy with Decision Trees and scaled data: {:.2f}".format(deep_tree_clf.score(X_test_scaled,y_test)))
+
++
Another example, the moons again
+ + +from __future__ import division, print_function, unicode_literals
+
+# Common imports
+import numpy as np
+import os
+
+# to make this notebook's output stable across runs
+np.random.seed(42)
+
+# To plot pretty figures
+import matplotlib
+import matplotlib.pyplot as plt
+from matplotlib.colors import ListedColormap
+plt.rcParams['axes.labelsize'] = 14
+plt.rcParams['xtick.labelsize'] = 12
+plt.rcParams['ytick.labelsize'] = 12
+
+
+from sklearn.svm import SVC
+from sklearn import datasets
+from sklearn.tree import DecisionTreeClassifier
+from sklearn.datasets import make_moons
+from sklearn.tree import export_graphviz
+
+Xm, ym = make_moons(n_samples=100, noise=0.25, random_state=53)
+
+deep_tree_clf1 = DecisionTreeClassifier(random_state=42)
+deep_tree_clf2 = DecisionTreeClassifier(min_samples_leaf=4, random_state=42)
+deep_tree_clf1.fit(Xm, ym)
+deep_tree_clf2.fit(Xm, ym)
+
+
+def plot_decision_boundary(clf, X, y, axes=[0, 7.5, 0, 3], iris=True, legend=False, plot_training=True):
+ x1s = np.linspace(axes[0], axes[1], 100)
+ x2s = np.linspace(axes[2], axes[3], 100)
+ x1, x2 = np.meshgrid(x1s, x2s)
+ X_new = np.c_[x1.ravel(), x2.ravel()]
+ y_pred = clf.predict(X_new).reshape(x1.shape)
+ custom_cmap = ListedColormap(['#fafab0','#9898ff','#a0faa0'])
+ plt.contourf(x1, x2, y_pred, alpha=0.3, cmap=custom_cmap)
+ if not iris:
+ custom_cmap2 = ListedColormap(['#7d7d58','#4c4c7f','#507d50'])
+ plt.contour(x1, x2, y_pred, cmap=custom_cmap2, alpha=0.8)
+ if plot_training:
+ plt.plot(X[:, 0][y==0], X[:, 1][y==0], "yo", label="Iris-Setosa")
+ plt.plot(X[:, 0][y==1], X[:, 1][y==1], "bs", label="Iris-Versicolor")
+ plt.plot(X[:, 0][y==2], X[:, 1][y==2], "g^", label="Iris-Virginica")
+ plt.axis(axes)
+ if iris:
+ plt.xlabel("Petal length", fontsize=14)
+ plt.ylabel("Petal width", fontsize=14)
+ else:
+ plt.xlabel(r"$x_1$", fontsize=18)
+ plt.ylabel(r"$x_2$", fontsize=18, rotation=0)
+ if legend:
+ plt.legend(loc="lower right", fontsize=14)
+plt.figure(figsize=(11, 4))
+plt.subplot(121)
+plot_decision_boundary(deep_tree_clf1, Xm, ym, axes=[-1.5, 2.5, -1, 1.5], iris=False)
+plt.title("No restrictions", fontsize=16)
+plt.subplot(122)
+plot_decision_boundary(deep_tree_clf2, Xm, ym, axes=[-1.5, 2.5, -1, 1.5], iris=False)
+plt.title("min_samples_leaf = {}".format(deep_tree_clf2.min_samples_leaf), fontsize=14)
+plt.show()
+
++
Playing around with regions
+ + +np.random.seed(6)
+Xs = np.random.rand(100, 2) - 0.5
+ys = (Xs[:, 0] > 0).astype(np.float32) * 2
+
+angle = np.pi/4
+rotation_matrix = np.array([[np.cos(angle), -np.sin(angle)], [np.sin(angle), np.cos(angle)]])
+Xsr = Xs.dot(rotation_matrix)
+
+tree_clf_s = DecisionTreeClassifier(random_state=42)
+tree_clf_s.fit(Xs, ys)
+tree_clf_sr = DecisionTreeClassifier(random_state=42)
+tree_clf_sr.fit(Xsr, ys)
+
+plt.figure(figsize=(11, 4))
+plt.subplot(121)
+plot_decision_boundary(tree_clf_s, Xs, ys, axes=[-0.7, 0.7, -0.7, 0.7], iris=False)
+plt.subplot(122)
+plot_decision_boundary(tree_clf_sr, Xsr, ys, axes=[-0.7, 0.7, -0.7, 0.7], iris=False)
+
+plt.show()
+
++
Regression trees
+ + +# Quadratic training set + noise
+np.random.seed(42)
+m = 200
+X = np.random.rand(m, 1)
+y = 4 * (X - 0.5) ** 2
+y = y + np.random.randn(m, 1) / 10
+
+from sklearn.tree import DecisionTreeRegressor
+
+tree_reg = DecisionTreeRegressor(max_depth=2, random_state=42)
+tree_reg.fit(X, y)
+
++
Final regressor code
+ + +from sklearn.tree import DecisionTreeRegressor
+
+tree_reg1 = DecisionTreeRegressor(random_state=42, max_depth=2)
+tree_reg2 = DecisionTreeRegressor(random_state=42, max_depth=3)
+tree_reg1.fit(X, y)
+tree_reg2.fit(X, y)
+
+def plot_regression_predictions(tree_reg, X, y, axes=[0, 1, -0.2, 1], ylabel="$y$"):
+ x1 = np.linspace(axes[0], axes[1], 500).reshape(-1, 1)
+ y_pred = tree_reg.predict(x1)
+ plt.axis(axes)
+ plt.xlabel("$x_1$", fontsize=18)
+ if ylabel:
+ plt.ylabel(ylabel, fontsize=18, rotation=0)
+ plt.plot(X, y, "b.")
+ plt.plot(x1, y_pred, "r.-", linewidth=2, label=r"$\hat{y}$")
+
+plt.figure(figsize=(11, 4))
+plt.subplot(121)
+plot_regression_predictions(tree_reg1, X, y)
+for split, style in ((0.1973, "k-"), (0.0917, "k--"), (0.7718, "k--")):
+ plt.plot([split, split], [-0.2, 1], style, linewidth=2)
+plt.text(0.21, 0.65, "Depth=0", fontsize=15)
+plt.text(0.01, 0.2, "Depth=1", fontsize=13)
+plt.text(0.65, 0.8, "Depth=1", fontsize=13)
+plt.legend(loc="upper center", fontsize=18)
+plt.title("max_depth=2", fontsize=14)
+
+plt.subplot(122)
+plot_regression_predictions(tree_reg2, X, y, ylabel=None)
+for split, style in ((0.1973, "k-"), (0.0917, "k--"), (0.7718, "k--")):
+ plt.plot([split, split], [-0.2, 1], style, linewidth=2)
+for split in (0.0458, 0.1298, 0.2873, 0.9040):
+ plt.plot([split, split], [-0.2, 1], "k:", linewidth=1)
+plt.text(0.3, 0.5, "Depth=2", fontsize=13)
+plt.title("max_depth=3", fontsize=14)
+
+plt.show()
+
+tree_reg1 = DecisionTreeRegressor(random_state=42)
+tree_reg2 = DecisionTreeRegressor(random_state=42, min_samples_leaf=10)
+tree_reg1.fit(X, y)
+tree_reg2.fit(X, y)
+
+x1 = np.linspace(0, 1, 500).reshape(-1, 1)
+y_pred1 = tree_reg1.predict(x1)
+y_pred2 = tree_reg2.predict(x1)
+
+plt.figure(figsize=(11, 4))
+
+plt.subplot(121)
+plt.plot(X, y, "b.")
+plt.plot(x1, y_pred1, "r.-", linewidth=2, label=r"$\hat{y}$")
+plt.axis([0, 1, -0.2, 1.1])
+plt.xlabel("$x_1$", fontsize=18)
+plt.ylabel("$y$", fontsize=18, rotation=0)
+plt.legend(loc="upper center", fontsize=18)
+plt.title("No restrictions", fontsize=14)
+
+plt.subplot(122)
+plt.plot(X, y, "b.")
+plt.plot(x1, y_pred2, "r.-", linewidth=2, label=r"$\hat{y}$")
+plt.axis([0, 1, -0.2, 1.1])
+plt.xlabel("$x_1$", fontsize=18)
+plt.title("min_samples_leaf={}".format(tree_reg2.min_samples_leaf), fontsize=14)
+
+plt.show()
+
+Pros and cons of trees, pros
@@ -2449,1051 +3497,6 @@ plt.show()-
Why binary splits?
- -It is custom to split to a tree uising binary splits. The reason is -that multiway splits fragment the data too quickly, leaving -insufficient data at the next level down. Multiway splits can be -achieved by a series of binary split and this is normally preferred. -
- --
Computing a Tree using the Gini Index
- -Consider the following example with attributes/features and two -possible outcomes (classes) for each attribute. Assume we wish to find some -correlations between the average grade of a student as function of the -number of hours studied and hours slept. We want also to correlate the -grade in a given course with the general trend, whether the students -recently has gotten grades below average or above. -
- -We have three features/attributes
--
-
-
The Table
- -| Grade Trend | Hours slept | Hours Studied | Grade |
|---|---|---|---|
| Above | Low | High | Above |
| Below | High | Low | Below |
| Above | Low | High | Above |
| Above | High | High | Above |
| Below | Low | High | Below |
| Above | Low | Low | Below |
| Below | High | High | Below |
| Below | Low | High | Below |
| Above | Low | Low | Below |
| Above | High | High | Above |
-
Computing the various Gini Indices
- -In computations we will translate all classes into numbers. Being -these binary classes, they can easily be split into ones and zeros. -
- --
See handwritten notes for Thursday November 11
--
Computing the various Gini Indices, Hours slept
- --
See handwritten notes for Thursday November 11
--
Computing the various Gini Indices, Hours studied
- --
See handwritten notes for Thursday November 11
-For final tree, see the above handwritten notes
- --
A possible code using Scikit-Learn
- - - -# Common imports
-import numpy as np
-import pandas as pd
-import matplotlib.pyplot as plt
-from sklearn.tree import DecisionTreeClassifier
-from sklearn.model_selection import train_test_split
-from sklearn.tree import export_graphviz
-from sklearn.preprocessing import StandardScaler, OneHotEncoder
-from sklearn.compose import ColumnTransformer
-from IPython.display import Image
-from pydot import graph_from_dot_data
-import os
-
-# Where to save the figures and data files
-PROJECT_ROOT_DIR = "Results"
-FIGURE_ID = "Results/FigureFiles"
-DATA_ID = "DataFiles/"
-
-if not os.path.exists(PROJECT_ROOT_DIR):
- os.mkdir(PROJECT_ROOT_DIR)
-
-if not os.path.exists(FIGURE_ID):
- os.makedirs(FIGURE_ID)
-
-if not os.path.exists(DATA_ID):
- os.makedirs(DATA_ID)
-
-def image_path(fig_id):
- return os.path.join(FIGURE_ID, fig_id)
-
-def data_path(dat_id):
- return os.path.join(DATA_ID, dat_id)
-
-def save_fig(fig_id):
- plt.savefig(image_path(fig_id) + ".png", format='png')
-
-infile = open(data_path("grades.csv"),'r')
-
-# Read the experimental data with Pandas
-from IPython.display import display
-grades = pd.read_csv(infile,names = ('Trend','Sleep','Studied','Grade'))
-grades = pd.DataFrame(grades)
-
-# Features and targets
-X = grades.loc[:, grades.columns != 'Grade'].values
-y = grades.loc[:, grades.columns == 'Grade'].values
-
-# Create the encoder.
-encoder = OneHotEncoder(handle_unknown="ignore")
-# Assume for simplicity all features are categorical.
-encoder.fit(X)
-# Apply the encoder.
-X = encoder.transform(X)
-print(X)
-# Then do a Classification tree
-tree_clf = DecisionTreeClassifier(max_depth=2)
-tree_clf.fit(X, y)
-print("Train set accuracy with Decision Tree: {:.2f}".format(tree_clf.score(X,y)))
-#transfer to a decision tree graph
-export_graphviz(
- tree_clf,
- out_file="DataFiles/grade.dot",
- rounded=True,
- filled=True
-)
-cmd = 'dot -Tpng DataFiles/grade.dot -o DataFiles/grades.png'
-os.system(cmd)
-
--
Visualizing Trees, More examples
- - -import os
-from sklearn.datasets import load_breast_cancer
-from sklearn.tree import DecisionTreeClassifier
-from sklearn.model_selection import train_test_split
-from sklearn.metrics import confusion_matrix
-from sklearn.tree import export_graphviz
-
-from IPython.display import Image
-from pydot import graph_from_dot_data
-import pandas as pd
-import numpy as np
-
-
-cancer = load_breast_cancer()
-X = pd.DataFrame(cancer.data, columns=cancer.feature_names)
-print(X)
-y = pd.Categorical.from_codes(cancer.target, cancer.target_names)
-y = pd.get_dummies(y)
-print(y)
-X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=1)
-tree_clf = DecisionTreeClassifier(max_depth=5)
-tree_clf.fit(X_train, y_train)
-
-export_graphviz(
- tree_clf,
- out_file="DataFiles/cancer.dot",
- feature_names=cancer.feature_names,
- class_names=cancer.target_names,
- rounded=True,
- filled=True
-)
-cmd = 'dot -Tpng DataFiles/cancer.dot -o DataFiles/cancer.png'
-os.system(cmd)
-
--
Visualizing the Tree, The Moons
- - -# Common imports
-import numpy as np
-from sklearn.model_selection import train_test_split
-from sklearn.tree import DecisionTreeClassifier
-from sklearn.datasets import make_moons
-from sklearn.tree import export_graphviz
-from pydot import graph_from_dot_data
-import pandas as pd
-import os
-
-np.random.seed(42)
-X, y = make_moons(n_samples=100, noise=0.25, random_state=53)
-X_train, X_test, y_train, y_test = train_test_split(X,y,random_state=0)
-tree_clf = DecisionTreeClassifier(max_depth=5)
-tree_clf.fit(X_train, y_train)
-
-export_graphviz(
- tree_clf,
- out_file="DataFiles/moons.dot",
- rounded=True,
- filled=True
-)
-cmd = 'dot -Tpng DataFiles/moons.dot -o DataFiles/moons.png'
-os.system(cmd)
-
--
Other ways of visualizing the trees
- -Scikit-Learn has also another way to visualize the trees which is very useful, here with the Iris data.
- - - -from sklearn.datasets import load_iris
-from sklearn import tree
-X, y = load_iris(return_X_y=True)
-tree_clf = tree.DecisionTreeClassifier()
-tree_clf = tree_clf.fit(X, y)
-# and then plot the tree
-tree.plot_tree(tree_clf)
-
--
Printing out as text
- -Alternatively, the tree can also be exported in textual format with the function exporttext. -This method doesn’t require the installation of external libraries and is more compact: -
- - - -from sklearn.datasets import load_iris
-from sklearn.tree import DecisionTreeClassifier
-from sklearn.tree import export_text
-iris = load_iris()
-decision_tree = DecisionTreeClassifier(random_state=0, max_depth=2)
-decision_tree = decision_tree.fit(iris.data, iris.target)
-r = export_text(decision_tree, feature_names=iris['feature_names'])
-print(r)
-
--
Algorithms for Setting up Decision Trees
- -Two algorithms stand out in the set up of decision trees:
--
-
We discuss both algorithms with applications here. The popular library -Scikit-Learn uses the CART algorithm. For classification problems -you can use either the gini index or the entropy to split a tree -in two branches. -
- --
The CART algorithm for Classification
- -For classification, the CART algorithm splits the data set in two subsets using a single feature \( k \) and a threshold \( t_k \). -This could be for example a threshold set by a number below a certain circumference of a malign tumor. -
- -How do we find these two quantities? -We search for the pair \( (k,t_k) \) that produces the purest subset using for example the gini factor \( G \). -The cost function it tries to minimize is then -
-$$ -C(k,t_k) = \frac{m_{\mathrm{left}}}{m}G_{\mathrm{left}}+ \frac{m_{\mathrm{right}}}{m}G_{\mathrm{right}}, -$$ - -where \( G_{\mathrm{left/right}} \) measures the impurity of the left/right subset and \( m_{\mathrm{left/right}} \) - is the number of instances in the left/right subset -
- -Once it has successfully split the training set in two, it splits the subsets using the same logic, then the subsubsets -and so on, recursively. It stops recursing once it reaches the maximum depth (defined by the -\( max\_depth \) hyperparameter), or if it cannot find a split that will reduce impurity. A few other -hyperparameters control additional stopping conditions such as the \( min\_samples\_split \), -\( min\_samples\_leaf \), \( min\_weight\_fraction\_leaf \), and \( max\_leaf\_nodes \). -
- --
The CART algorithm for Regression
- -The CART algorithm for regression works is similar to the one for classification except that instead of trying to split the -training set in a way that minimizes say the gini or entropy impurity, it now tries to split the training set in a way that minimizes our well-known mean-squared error (MSE). The cost function is now -
-$$ -C(k,t_k) = \frac{m_{\mathrm{left}}}{m}\mathrm{MSE}_{\mathrm{left}}+ \frac{m_{\mathrm{right}}}{m}\mathrm{MSE}_{\mathrm{right}}. -$$ - -Here the MSE for a specific node is defined as
-$$ -\mathrm{MSE}_{\mathrm{node}}=\frac{1}{m_\mathrm{node}}\sum_{i\in \mathrm{node}}(\overline{y}_{\mathrm{node}}-y_i)^2, -$$ - -with
-$$ -\overline{y}_{\mathrm{node}}=\frac{1}{m_\mathrm{node}}\sum_{i\in \mathrm{node}}y_i, -$$ - -the mean value of all observations in a specific node.
- -Without any regularization, the regression task for decision trees, -just like for classification tasks, is prone to overfitting. -
- --
Computing the Gini index
- -The example we will look at is a classical one in many Machine -Learning applications. Based on various meteorological features, we -have several so-called attributes which decide whether we at the end -will do some outdoor activity like skiing, going for a bike ride etc -etc. The table here contains the feautures outlook, temperature, -humidity and wind. The target or output is whether we ride -(True=1) or whether we do something else that day (False=0). The -attributes for each feature are then sunny, overcast and rain for the -outlook, hot, cold and mild for temperature, high and normal for -humidity and weak and strong for wind. -
- -The table here summarizes the various attributes and
-| Day | Outlook | Temperature | Humidity | Wind | Ride |
|---|---|---|---|---|---|
| 1 | Sunny | Hot | High | Weak | 0 |
| 2 | Sunny | Hot | High | Strong | 1 |
| 3 | Overcast | Hot | High | Weak | 1 |
| 4 | Rain | Mild | High | Weak | 1 |
| 5 | Rain | Cool | Normal | Weak | 1 |
| 6 | Rain | Cool | Normal | Strong | 0 |
| 7 | Overcast | Cool | Normal | Strong | 1 |
| 8 | Sunny | Mild | High | Weak | 0 |
| 9 | Sunny | Cool | Normal | Weak | 1 |
| 10 | Rain | Mild | Normal | Weak | 1 |
| 11 | Sunny | Mild | Normal | Strong | 1 |
| 12 | Overcast | Mild | High | Strong | 1 |
| 13 | Overcast | Hot | Normal | Weak | 1 |
| 14 | Rain | Mild | High | Strong | 0 |
-
Simple Python Code to read in Data and perform Classification
- - - -# Common imports
-import numpy as np
-import pandas as pd
-import matplotlib.pyplot as plt
-from sklearn.tree import DecisionTreeClassifier
-from sklearn.model_selection import train_test_split
-from sklearn.tree import export_graphviz
-from sklearn.preprocessing import StandardScaler, OneHotEncoder
-from sklearn.compose import ColumnTransformer
-from IPython.display import Image
-from pydot import graph_from_dot_data
-import os
-
-# Where to save the figures and data files
-PROJECT_ROOT_DIR = "Results"
-FIGURE_ID = "Results/FigureFiles"
-DATA_ID = "DataFiles/"
-
-if not os.path.exists(PROJECT_ROOT_DIR):
- os.mkdir(PROJECT_ROOT_DIR)
-
-if not os.path.exists(FIGURE_ID):
- os.makedirs(FIGURE_ID)
-
-if not os.path.exists(DATA_ID):
- os.makedirs(DATA_ID)
-
-def image_path(fig_id):
- return os.path.join(FIGURE_ID, fig_id)
-
-def data_path(dat_id):
- return os.path.join(DATA_ID, dat_id)
-
-def save_fig(fig_id):
- plt.savefig(image_path(fig_id) + ".png", format='png')
-
-infile = open(data_path("rideclass.csv"),'r')
-
-# Read the experimental data with Pandas
-from IPython.display import display
-ridedata = pd.read_csv(infile,names = ('Outlook','Temperature','Humidity','Wind','Ride'))
-ridedata = pd.DataFrame(ridedata)
-
-# Features and targets
-X = ridedata.loc[:, ridedata.columns != 'Ride'].values
-y = ridedata.loc[:, ridedata.columns == 'Ride'].values
-
-# Create the encoder.
-encoder = OneHotEncoder(handle_unknown="ignore")
-# Assume for simplicity all features are categorical.
-encoder.fit(X)
-# Apply the encoder.
-X = encoder.transform(X)
-print(X)
-# Then do a Classification tree
-tree_clf = DecisionTreeClassifier(max_depth=2)
-tree_clf.fit(X, y)
-print("Train set accuracy with Decision Tree: {:.2f}".format(tree_clf.score(X,y)))
-#transfer to a decision tree graph
-export_graphviz(
- tree_clf,
- out_file="DataFiles/ride.dot",
- rounded=True,
- filled=True
-)
-cmd = 'dot -Tpng DataFiles/cancer.dot -o DataFiles/cancer.png'
-os.system(cmd)
-
--
Computing the Gini Factor
- -The above functions (gini, entropy and misclassification error) are -important components of the so-called CART algorithm. We will discuss -this algorithm below after we have discussed the information gain -algorithm ID3. -
- -In the example here we have converted all our attributes into numerical values \( 0,1,2 \) etc.
- - - -# Split a dataset based on an attribute and an attribute value
-def test_split(index, value, dataset):
- left, right = list(), list()
- for row in dataset:
- if row[index] < value:
- left.append(row)
- else:
- right.append(row)
- return left, right
-
-# Calculate the Gini index for a split dataset
-def gini_index(groups, classes):
- # count all samples at split point
- n_instances = float(sum([len(group) for group in groups]))
- # sum weighted Gini index for each group
- gini = 0.0
- for group in groups:
- size = float(len(group))
- # avoid divide by zero
- if size == 0:
- continue
- score = 0.0
- # score the group based on the score for each class
- for class_val in classes:
- p = [row[-1] for row in group].count(class_val) / size
- score += p * p
- # weight the group score by its relative size
- gini += (1.0 - score) * (size / n_instances)
- return gini
-
-# Select the best split point for a dataset
-def get_split(dataset):
- class_values = list(set(row[-1] for row in dataset))
- b_index, b_value, b_score, b_groups = 999, 999, 999, None
- for index in range(len(dataset[0])-1):
- for row in dataset:
- groups = test_split(index, row[index], dataset)
- gini = gini_index(groups, class_values)
- print('X%d < %.3f Gini=%.3f' % ((index+1), row[index], gini))
- if gini < b_score:
- b_index, b_value, b_score, b_groups = index, row[index], gini, groups
- return {'index':b_index, 'value':b_value, 'groups':b_groups}
-
-dataset = [[0,0,0,0,0],
- [0,0,0,1,1],
- [1,0,0,0,1],
- [2,1,0,0,1],
- [2,2,1,0,1],
- [2,2,1,1,0],
- [1,2,1,1,1],
- [0,1,0,0,0],
- [0,2,1,0,1],
- [2,1,1,0,1],
- [0,1,1,1,1],
- [1,1,0,1,1],
- [1,0,1,0,1],
- [2,1,0,1,0]]
-
-split = get_split(dataset)
-print('Split: [X%d < %.3f]' % ((split['index']+1), split['value']))
-
--
Entropy and the ID3 algorithm
- -The ID3 algorithm learns decision trees by constructing -them in a top down way, beginning with the question which attribute should be tested at the root of the tree? -
- --
-
The ID3 algorithm selects which attribute to test at each node in the -tree. -
- -We would like to select the attribute that is most useful for classifying -examples. -
- -What is a good quantitative measure of the worth of an attribute?
- -Information gain measures how well a given attribute separates the -training examples according to their target classification. -
- -The ID3 algorithm uses this information gain measure to select among the candidate -attributes at each step while growing the tree. -
- --
Cancer Data again now with Decision Trees and other Methods
- - -import matplotlib.pyplot as plt
-import numpy as np
-from sklearn.model_selection import train_test_split
-from sklearn.datasets import load_breast_cancer
-from sklearn.svm import SVC
-from sklearn.linear_model import LogisticRegression
-from sklearn.tree import DecisionTreeClassifier
-
-# Load the data
-cancer = load_breast_cancer()
-
-X_train, X_test, y_train, y_test = train_test_split(cancer.data,cancer.target,random_state=0)
-print(X_train.shape)
-print(X_test.shape)
-# Logistic Regression
-logreg = LogisticRegression(solver='lbfgs')
-logreg.fit(X_train, y_train)
-print("Test set accuracy with Logistic Regression: {:.2f}".format(logreg.score(X_test,y_test)))
-# Support vector machine
-svm = SVC(gamma='auto', C=100)
-svm.fit(X_train, y_train)
-print("Test set accuracy with SVM: {:.2f}".format(svm.score(X_test,y_test)))
-# Decision Trees
-deep_tree_clf = DecisionTreeClassifier(max_depth=None)
-deep_tree_clf.fit(X_train, y_train)
-print("Test set accuracy with Decision Trees: {:.2f}".format(deep_tree_clf.score(X_test,y_test)))
-#now scale the data
-from sklearn.preprocessing import StandardScaler
-scaler = StandardScaler()
-scaler.fit(X_train)
-X_train_scaled = scaler.transform(X_train)
-X_test_scaled = scaler.transform(X_test)
-# Logistic Regression
-logreg.fit(X_train_scaled, y_train)
-print("Test set accuracy Logistic Regression with scaled data: {:.2f}".format(logreg.score(X_test_scaled,y_test)))
-# Support Vector Machine
-svm.fit(X_train_scaled, y_train)
-print("Test set accuracy SVM with scaled data: {:.2f}".format(logreg.score(X_test_scaled,y_test)))
-# Decision Trees
-deep_tree_clf.fit(X_train_scaled, y_train)
-print("Test set accuracy with Decision Trees and scaled data: {:.2f}".format(deep_tree_clf.score(X_test_scaled,y_test)))
-
--
Another example, the moons again
- - -from __future__ import division, print_function, unicode_literals
-
-# Common imports
-import numpy as np
-import os
-
-# to make this notebook's output stable across runs
-np.random.seed(42)
-
-# To plot pretty figures
-import matplotlib
-import matplotlib.pyplot as plt
-from matplotlib.colors import ListedColormap
-plt.rcParams['axes.labelsize'] = 14
-plt.rcParams['xtick.labelsize'] = 12
-plt.rcParams['ytick.labelsize'] = 12
-
-
-from sklearn.svm import SVC
-from sklearn import datasets
-from sklearn.tree import DecisionTreeClassifier
-from sklearn.datasets import make_moons
-from sklearn.tree import export_graphviz
-
-Xm, ym = make_moons(n_samples=100, noise=0.25, random_state=53)
-
-deep_tree_clf1 = DecisionTreeClassifier(random_state=42)
-deep_tree_clf2 = DecisionTreeClassifier(min_samples_leaf=4, random_state=42)
-deep_tree_clf1.fit(Xm, ym)
-deep_tree_clf2.fit(Xm, ym)
-
-
-def plot_decision_boundary(clf, X, y, axes=[0, 7.5, 0, 3], iris=True, legend=False, plot_training=True):
- x1s = np.linspace(axes[0], axes[1], 100)
- x2s = np.linspace(axes[2], axes[3], 100)
- x1, x2 = np.meshgrid(x1s, x2s)
- X_new = np.c_[x1.ravel(), x2.ravel()]
- y_pred = clf.predict(X_new).reshape(x1.shape)
- custom_cmap = ListedColormap(['#fafab0','#9898ff','#a0faa0'])
- plt.contourf(x1, x2, y_pred, alpha=0.3, cmap=custom_cmap)
- if not iris:
- custom_cmap2 = ListedColormap(['#7d7d58','#4c4c7f','#507d50'])
- plt.contour(x1, x2, y_pred, cmap=custom_cmap2, alpha=0.8)
- if plot_training:
- plt.plot(X[:, 0][y==0], X[:, 1][y==0], "yo", label="Iris-Setosa")
- plt.plot(X[:, 0][y==1], X[:, 1][y==1], "bs", label="Iris-Versicolor")
- plt.plot(X[:, 0][y==2], X[:, 1][y==2], "g^", label="Iris-Virginica")
- plt.axis(axes)
- if iris:
- plt.xlabel("Petal length", fontsize=14)
- plt.ylabel("Petal width", fontsize=14)
- else:
- plt.xlabel(r"$x_1$", fontsize=18)
- plt.ylabel(r"$x_2$", fontsize=18, rotation=0)
- if legend:
- plt.legend(loc="lower right", fontsize=14)
-plt.figure(figsize=(11, 4))
-plt.subplot(121)
-plot_decision_boundary(deep_tree_clf1, Xm, ym, axes=[-1.5, 2.5, -1, 1.5], iris=False)
-plt.title("No restrictions", fontsize=16)
-plt.subplot(122)
-plot_decision_boundary(deep_tree_clf2, Xm, ym, axes=[-1.5, 2.5, -1, 1.5], iris=False)
-plt.title("min_samples_leaf = {}".format(deep_tree_clf2.min_samples_leaf), fontsize=14)
-plt.show()
-
--
Playing around with regions
- - -np.random.seed(6)
-Xs = np.random.rand(100, 2) - 0.5
-ys = (Xs[:, 0] > 0).astype(np.float32) * 2
-
-angle = np.pi/4
-rotation_matrix = np.array([[np.cos(angle), -np.sin(angle)], [np.sin(angle), np.cos(angle)]])
-Xsr = Xs.dot(rotation_matrix)
-
-tree_clf_s = DecisionTreeClassifier(random_state=42)
-tree_clf_s.fit(Xs, ys)
-tree_clf_sr = DecisionTreeClassifier(random_state=42)
-tree_clf_sr.fit(Xsr, ys)
-
-plt.figure(figsize=(11, 4))
-plt.subplot(121)
-plot_decision_boundary(tree_clf_s, Xs, ys, axes=[-0.7, 0.7, -0.7, 0.7], iris=False)
-plt.subplot(122)
-plot_decision_boundary(tree_clf_sr, Xsr, ys, axes=[-0.7, 0.7, -0.7, 0.7], iris=False)
-
-plt.show()
-
--
Regression trees
- - -# Quadratic training set + noise
-np.random.seed(42)
-m = 200
-X = np.random.rand(m, 1)
-y = 4 * (X - 0.5) ** 2
-y = y + np.random.randn(m, 1) / 10
-
-from sklearn.tree import DecisionTreeRegressor
-
-tree_reg = DecisionTreeRegressor(max_depth=2, random_state=42)
-tree_reg.fit(X, y)
-
--
Final regressor code
- - -from sklearn.tree import DecisionTreeRegressor
-
-tree_reg1 = DecisionTreeRegressor(random_state=42, max_depth=2)
-tree_reg2 = DecisionTreeRegressor(random_state=42, max_depth=3)
-tree_reg1.fit(X, y)
-tree_reg2.fit(X, y)
-
-def plot_regression_predictions(tree_reg, X, y, axes=[0, 1, -0.2, 1], ylabel="$y$"):
- x1 = np.linspace(axes[0], axes[1], 500).reshape(-1, 1)
- y_pred = tree_reg.predict(x1)
- plt.axis(axes)
- plt.xlabel("$x_1$", fontsize=18)
- if ylabel:
- plt.ylabel(ylabel, fontsize=18, rotation=0)
- plt.plot(X, y, "b.")
- plt.plot(x1, y_pred, "r.-", linewidth=2, label=r"$\hat{y}$")
-
-plt.figure(figsize=(11, 4))
-plt.subplot(121)
-plot_regression_predictions(tree_reg1, X, y)
-for split, style in ((0.1973, "k-"), (0.0917, "k--"), (0.7718, "k--")):
- plt.plot([split, split], [-0.2, 1], style, linewidth=2)
-plt.text(0.21, 0.65, "Depth=0", fontsize=15)
-plt.text(0.01, 0.2, "Depth=1", fontsize=13)
-plt.text(0.65, 0.8, "Depth=1", fontsize=13)
-plt.legend(loc="upper center", fontsize=18)
-plt.title("max_depth=2", fontsize=14)
-
-plt.subplot(122)
-plot_regression_predictions(tree_reg2, X, y, ylabel=None)
-for split, style in ((0.1973, "k-"), (0.0917, "k--"), (0.7718, "k--")):
- plt.plot([split, split], [-0.2, 1], style, linewidth=2)
-for split in (0.0458, 0.1298, 0.2873, 0.9040):
- plt.plot([split, split], [-0.2, 1], "k:", linewidth=1)
-plt.text(0.3, 0.5, "Depth=2", fontsize=13)
-plt.title("max_depth=3", fontsize=14)
-
-plt.show()
-
-tree_reg1 = DecisionTreeRegressor(random_state=42)
-tree_reg2 = DecisionTreeRegressor(random_state=42, min_samples_leaf=10)
-tree_reg1.fit(X, y)
-tree_reg2.fit(X, y)
-
-x1 = np.linspace(0, 1, 500).reshape(-1, 1)
-y_pred1 = tree_reg1.predict(x1)
-y_pred2 = tree_reg2.predict(x1)
-
-plt.figure(figsize=(11, 4))
-
-plt.subplot(121)
-plt.plot(X, y, "b.")
-plt.plot(x1, y_pred1, "r.-", linewidth=2, label=r"$\hat{y}$")
-plt.axis([0, 1, -0.2, 1.1])
-plt.xlabel("$x_1$", fontsize=18)
-plt.ylabel("$y$", fontsize=18, rotation=0)
-plt.legend(loc="upper center", fontsize=18)
-plt.title("No restrictions", fontsize=14)
-
-plt.subplot(122)
-plt.plot(X, y, "b.")
-plt.plot(x1, y_pred2, "r.-", linewidth=2, label=r"$\hat{y}$")
-plt.axis([0, 1, -0.2, 1.1])
-plt.xlabel("$x_1$", fontsize=18)
-plt.title("min_samples_leaf={}".format(tree_reg2.min_samples_leaf), fontsize=14)
-
-plt.show()
-
-Why Voting?
diff --git a/doc/pub/week44/ipynb/ipynb-week44-src.tar.gz b/doc/pub/week44/ipynb/ipynb-week44-src.tar.gz index e6d4dc6dc..e970ba306 100644 Binary files a/doc/pub/week44/ipynb/ipynb-week44-src.tar.gz and b/doc/pub/week44/ipynb/ipynb-week44-src.tar.gz differ diff --git a/doc/pub/week44/ipynb/week44.ipynb b/doc/pub/week44/ipynb/week44.ipynb index 01991c29f..a733d3255 100644 --- a/doc/pub/week44/ipynb/week44.ipynb +++ b/doc/pub/week44/ipynb/week44.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "markdown", - "id": "4d5623b6", + "id": "fa87e930", "metadata": { "editable": true }, @@ -14,7 +14,7 @@ }, { "cell_type": "markdown", - "id": "340a8f1e", + "id": "7c19be06", "metadata": { "editable": true }, @@ -22,14 +22,14 @@ "# Week 44: Decision Trees, Ensemble methods and Random Forests\n", "**Morten Hjorth-Jensen**, Department of Physics, University of Oslo and Department of Physics and Astronomy and National Superconducting Cyclotron Laboratory, Michigan State University\n", "\n", - "Date: **Oct 30, 2022**\n", + "Date: **Nov 1, 2022**\n", "\n", "Copyright 1999-2022, Morten Hjorth-Jensen. Released under CC Attribution-NonCommercial 4.0 license" ] }, { "cell_type": "markdown", - "id": "8b14c3b1", + "id": "450a14f3", "metadata": { "editable": true }, @@ -51,7 +51,7 @@ }, { "cell_type": "markdown", - "id": "14a30e42", + "id": "01c31a03", "metadata": { "editable": true }, @@ -69,7 +69,7 @@ }, { "cell_type": "markdown", - "id": "18145384", + "id": "aa0dfe31", "metadata": { "editable": true }, @@ -83,7 +83,7 @@ { "cell_type": "code", "execution_count": 1, - "id": "ae670413", + "id": "50f5b259", "metadata": { "collapsed": false, "editable": true @@ -199,7 +199,7 @@ }, { "cell_type": "markdown", - "id": "9b203089", + "id": "802b34a2", "metadata": { "editable": true }, @@ -230,7 +230,7 @@ }, { "cell_type": "markdown", - "id": "365eb0c2", + "id": "ca2b6f9f", "metadata": { "editable": true }, @@ -250,7 +250,7 @@ }, { "cell_type": "markdown", - "id": "37d92ace", + "id": "e295e14b", "metadata": { "editable": true }, @@ -262,7 +262,7 @@ }, { "cell_type": "markdown", - "id": "b80fe1a6", + "id": "754e64ac", "metadata": { "editable": true }, @@ -274,7 +274,7 @@ }, { "cell_type": "markdown", - "id": "5180c9f2", + "id": "f32c0b1e", "metadata": { "editable": true }, @@ -292,7 +292,7 @@ }, { "cell_type": "markdown", - "id": "94920b43", + "id": "57d6e619", "metadata": { "editable": true }, @@ -315,7 +315,7 @@ }, { "cell_type": "markdown", - "id": "0af26b3e", + "id": "4ece4236", "metadata": { "editable": true }, @@ -338,7 +338,7 @@ }, { "cell_type": "markdown", - "id": "7fc7e166", + "id": "8a6bbf85", "metadata": { "editable": true }, @@ -349,7 +349,7 @@ { "cell_type": "code", "execution_count": 2, - "id": "e03afff6", + "id": "26539eb1", "metadata": { "collapsed": false, "editable": true @@ -448,7 +448,7 @@ }, { "cell_type": "markdown", - "id": "9accc99b", + "id": "09e84faf", "metadata": { "editable": true }, @@ -470,7 +470,7 @@ }, { "cell_type": "markdown", - "id": "990fa257", + "id": "a1873a0a", "metadata": { "editable": true }, @@ -482,7 +482,7 @@ }, { "cell_type": "markdown", - "id": "bbd1331d", + "id": "d743d151", "metadata": { "editable": true }, @@ -493,7 +493,7 @@ }, { "cell_type": "markdown", - "id": "6b050d29", + "id": "c27a33dd", "metadata": { "editable": true }, @@ -515,7 +515,7 @@ }, { "cell_type": "markdown", - "id": "30ffc652", + "id": "480ccaab", "metadata": { "editable": true }, @@ -528,7 +528,7 @@ }, { "cell_type": "markdown", - "id": "43572752", + "id": "d09275e9", "metadata": { "editable": true }, @@ -540,7 +540,7 @@ }, { "cell_type": "markdown", - "id": "d3140374", + "id": "9485a265", "metadata": { "editable": true }, @@ -550,7 +550,7 @@ }, { "cell_type": "markdown", - "id": "0af3c67c", + "id": "ed23e8a8", "metadata": { "editable": true }, @@ -562,7 +562,7 @@ }, { "cell_type": "markdown", - "id": "911ec911", + "id": "f56f7a06", "metadata": { "editable": true }, @@ -572,7 +572,7 @@ }, { "cell_type": "markdown", - "id": "cbe922b7", + "id": "5c177479", "metadata": { "editable": true }, @@ -584,7 +584,7 @@ }, { "cell_type": "markdown", - "id": "b3d8df30", + "id": "4e400721", "metadata": { "editable": true }, @@ -617,7 +617,7 @@ }, { "cell_type": "markdown", - "id": "d098a7e8", + "id": "d2af1aa0", "metadata": { "editable": true }, @@ -641,7 +641,7 @@ }, { "cell_type": "markdown", - "id": "5b8f5179", + "id": "3892efeb", "metadata": { "editable": true }, @@ -653,7 +653,7 @@ }, { "cell_type": "markdown", - "id": "da837fc5", + "id": "af171665", "metadata": { "editable": true }, @@ -665,7 +665,7 @@ }, { "cell_type": "markdown", - "id": "5688a52e", + "id": "de6423ae", "metadata": { "editable": true }, @@ -693,7 +693,7 @@ }, { "cell_type": "markdown", - "id": "46af1b61", + "id": "68161041", "metadata": { "editable": true }, @@ -719,7 +719,7 @@ }, { "cell_type": "markdown", - "id": "87752e4d", + "id": "e6f75a5c", "metadata": { "editable": true }, @@ -742,7 +742,7 @@ }, { "cell_type": "markdown", - "id": "6dc6e377", + "id": "788ab5b5", "metadata": { "editable": true }, @@ -769,7 +769,7 @@ }, { "cell_type": "markdown", - "id": "0d874e20", + "id": "c08468fb", "metadata": { "editable": true }, @@ -788,7 +788,7 @@ }, { "cell_type": "markdown", - "id": "1ef78162", + "id": "49bead9b", "metadata": { "editable": true }, @@ -800,7 +800,7 @@ }, { "cell_type": "markdown", - "id": "31c9b77a", + "id": "d7b2ca40", "metadata": { "editable": true }, @@ -813,7 +813,7 @@ }, { "cell_type": "markdown", - "id": "326c934e", + "id": "eaa5f880", "metadata": { "editable": true }, @@ -825,7 +825,7 @@ }, { "cell_type": "markdown", - "id": "c39029bc", + "id": "00f9e5df", "metadata": { "editable": true }, @@ -835,7 +835,7 @@ }, { "cell_type": "markdown", - "id": "ba23b61e", + "id": "214dab84", "metadata": { "editable": true }, @@ -847,7 +847,7 @@ }, { "cell_type": "markdown", - "id": "06461c13", + "id": "30af7c11", "metadata": { "editable": true }, @@ -857,7 +857,7 @@ }, { "cell_type": "markdown", - "id": "c29d1044", + "id": "31c64e65", "metadata": { "editable": true }, @@ -869,7 +869,7 @@ }, { "cell_type": "markdown", - "id": "ec98cbb1", + "id": "5fc5e7c4", "metadata": { "editable": true }, @@ -880,7 +880,7 @@ { "cell_type": "code", "execution_count": 3, - "id": "c14691da", + "id": "b54ed5c5", "metadata": { "collapsed": false, "editable": true @@ -924,7 +924,7 @@ }, { "cell_type": "markdown", - "id": "b95d139b", + "id": "50d2c8ee", "metadata": { "editable": true }, @@ -935,7 +935,7 @@ { "cell_type": "code", "execution_count": 4, - "id": "c35cf440", + "id": "1cc07431", "metadata": { "collapsed": false, "editable": true @@ -970,7 +970,7 @@ }, { "cell_type": "markdown", - "id": "ffac4a98", + "id": "9079f77b", "metadata": { "editable": true }, @@ -983,7 +983,7 @@ { "cell_type": "code", "execution_count": 5, - "id": "c3f42544", + "id": "288805dd", "metadata": { "collapsed": false, "editable": true @@ -1001,7 +1001,7 @@ }, { "cell_type": "markdown", - "id": "a492d5c8", + "id": "a7d74d78", "metadata": { "editable": true }, @@ -1015,7 +1015,7 @@ { "cell_type": "code", "execution_count": 6, - "id": "a86512f4", + "id": "d7557fd6", "metadata": { "collapsed": false, "editable": true @@ -1034,7 +1034,7 @@ }, { "cell_type": "markdown", - "id": "8b4ffffd", + "id": "a31c70ec", "metadata": { "editable": true }, @@ -1054,7 +1054,7 @@ }, { "cell_type": "markdown", - "id": "d4942d00", + "id": "5547ae43", "metadata": { "editable": true }, @@ -1071,7 +1071,7 @@ }, { "cell_type": "markdown", - "id": "421d32b8", + "id": "aaaf660e", "metadata": { "editable": true }, @@ -1083,7 +1083,7 @@ }, { "cell_type": "markdown", - "id": "20a9f367", + "id": "34ee170c", "metadata": { "editable": true }, @@ -1100,7 +1100,7 @@ }, { "cell_type": "markdown", - "id": "03c4b84e", + "id": "566366c5", "metadata": { "editable": true }, @@ -1113,7 +1113,7 @@ }, { "cell_type": "markdown", - "id": "00dc3a7a", + "id": "620717c7", "metadata": { "editable": true }, @@ -1125,7 +1125,7 @@ }, { "cell_type": "markdown", - "id": "4c10aa3e", + "id": "7f42b173", "metadata": { "editable": true }, @@ -1135,7 +1135,7 @@ }, { "cell_type": "markdown", - "id": "d0c27e95", + "id": "cb8c819f", "metadata": { "editable": true }, @@ -1147,7 +1147,7 @@ }, { "cell_type": "markdown", - "id": "8486a79f", + "id": "ba04694b", "metadata": { "editable": true }, @@ -1157,7 +1157,7 @@ }, { "cell_type": "markdown", - "id": "f53fb661", + "id": "fef57383", "metadata": { "editable": true }, @@ -1169,7 +1169,7 @@ }, { "cell_type": "markdown", - "id": "b3ef600a", + "id": "1558631d", "metadata": { "editable": true }, @@ -1182,1134 +1182,7 @@ }, { "cell_type": "markdown", - "id": "2187470d", - "metadata": { - "editable": true - }, - "source": [ - "## Computing the Gini index\n", - "\n", - "The example we will look at is a classical one in many Machine\n", - "Learning applications. Based on various meteorological features, we\n", - "have several so-called attributes which decide whether we at the end\n", - "will do some outdoor activity like skiing, going for a bike ride etc\n", - "etc. The table here contains the feautures **outlook**, **temperature**,\n", - "**humidity** and **wind**. The target or output is whether we ride\n", - "(True=1) or whether we do something else that day (False=0). The\n", - "attributes for each feature are then sunny, overcast and rain for the\n", - "outlook, hot, cold and mild for temperature, high and normal for\n", - "humidity and weak and strong for wind.\n", - "\n", - "The table here summarizes the various attributes and\n", - "| Day | Outlook | Temperature | Humidity | Wind | Ride |
|---|---|---|---|---|---|
| 1 | Sunny | Hot | High | Weak | 0 |
| 2 | Sunny | Hot | High | Strong | 1 |
| 3 | Overcast | Hot | High | Weak | 1 |
| 4 | Rain | Mild | High | Weak | 1 |
| 5 | Rain | Cool | Normal | Weak | 1 |
| 6 | Rain | Cool | Normal | Strong | 0 |
| 7 | Overcast | Cool | Normal | Strong | 1 |
| 8 | Sunny | Mild | High | Weak | 0 |
| 9 | Sunny | Cool | Normal | Weak | 1 |
| 10 | Rain | Mild | Normal | Weak | 1 |
| 11 | Sunny | Mild | Normal | Strong | 1 |
| 12 | Overcast | Mild | High | Strong | 1 |
| 13 | Overcast | Hot | Normal | Weak | 1 |
| 14 | Rain | Mild | High | Strong | 0 |
![](\"DataFiles/ensembleoverview.png\")
Figure 1:
\n", - "" - ] - }, - { - "cell_type": "markdown", - "id": "bfbd6dd9", - "metadata": { - "editable": true - }, - "source": [ - "## Bagging\n", - "\n", - "The **plain** decision trees suffer from high\n", - "variance. This means that if we split the training data into two parts\n", - "at random, and fit a decision tree to both halves, the results that we\n", - "get could be quite different. In contrast, a procedure with low\n", - "variance will yield similar results if applied repeatedly to distinct\n", - "data sets; linear regression tends to have low variance, if the ratio\n", - "of $n$ to $p$ is moderately large. \n", - "\n", - "**Bootstrap aggregation**, or just **bagging**, is a\n", - "general-purpose procedure for reducing the variance of a statistical\n", - "learning method." - ] - }, - { - "cell_type": "markdown", - "id": "6d7a9006", - "metadata": { - "editable": true - }, - "source": [ - "## More bagging\n", - "\n", - "Bagging typically results in improved accuracy\n", - "over prediction using a single tree. Unfortunately, however, it can be\n", - "difficult to interpret the resulting model. Recall that one of the\n", - "advantages of decision trees is the attractive and easily interpreted\n", - "diagram that results.\n", - "\n", - "However, when we bag a large number of trees, it is no longer\n", - "possible to represent the resulting statistical learning procedure\n", - "using a single tree, and it is no longer clear which variables are\n", - "most important to the procedure. Thus, bagging improves prediction\n", - "accuracy at the expense of interpretability. Although the collection\n", - "of bagged trees is much more difficult to interpret than a single\n", - "tree, one can obtain an overall summary of the importance of each\n", - "predictor using the MSE (for bagging regression trees) or the Gini\n", - "index (for bagging classification trees). In the case of bagging\n", - "regression trees, we can record the total amount that the MSE is\n", - "decreased due to splits over a given predictor, averaged over all $B$ possible\n", - "trees. A large value indicates an important predictor. Similarly, in\n", - "the context of bagging classification trees, we can add up the total\n", - "amount that the Gini index is decreased by splits over a given\n", - "predictor, averaged over all $B$ trees." - ] - }, - { - "cell_type": "markdown", - "id": "7be3f691", - "metadata": { - "editable": true - }, - "source": [ - "## Simple Voting Example, head or tail" - ] - }, - { - "cell_type": "code", - "execution_count": 16, - "id": "e306db03", - "metadata": { - "collapsed": false, - "editable": true - }, - "outputs": [], - "source": [ - "heads_proba = 0.51\n", - "coin_tosses = (np.random.rand(10000, 10) < heads_proba).astype(np.int32)\n", - "cumulative_heads_ratio = np.cumsum(coin_tosses, axis=0) / np.arange(1, 10001).reshape(-1, 1)\n", - "plt.figure(figsize=(8,3.5))\n", - "plt.plot(cumulative_heads_ratio)\n", - "plt.plot([0, 10000], [0.51, 0.51], \"k--\", linewidth=2, label=\"51%\")\n", - "plt.plot([0, 10000], [0.5, 0.5], \"k-\", label=\"50%\")\n", - "plt.xlabel(\"Number of coin tosses\")\n", - "plt.ylabel(\"Heads ratio\")\n", - "plt.legend(loc=\"lower right\")\n", - "plt.axis([0, 10000, 0.42, 0.58])\n", - "save_fig(\"votingsimple\")\n", - "plt.show()" - ] - }, - { - "cell_type": "markdown", - "id": "443de822", - "metadata": { - "editable": true - }, - "source": [ - "## Using the Voting Classifier" - ] - }, - { - "cell_type": "code", - "execution_count": 17, - "id": "662a7680", - "metadata": { - "collapsed": false, - "editable": true - }, - "outputs": [], - "source": [ - "from sklearn.model_selection import train_test_split\n", - "from sklearn.datasets import make_moons\n", - "\n", - "X, y = make_moons(n_samples=500, noise=0.30, random_state=42)\n", - "X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42)\n", - "\n", - "from sklearn.ensemble import RandomForestClassifier\n", - "from sklearn.ensemble import VotingClassifier\n", - "from sklearn.linear_model import LogisticRegression\n", - "from sklearn.svm import SVC\n", - "\n", - "log_clf = LogisticRegression(solver=\"liblinear\", random_state=42)\n", - "rnd_clf = RandomForestClassifier(n_estimators=10, random_state=42)\n", - "svm_clf = SVC(gamma=\"auto\", random_state=42)\n", - "\n", - "voting_clf = VotingClassifier(\n", - " estimators=[('lr', log_clf), ('rf', rnd_clf), ('svc', svm_clf)],\n", - " voting='hard')\n", - "\n", - "voting_clf.fit(X_train, y_train)\n", - "\n", - "from sklearn.metrics import accuracy_score\n", - "\n", - "for clf in (log_clf, rnd_clf, svm_clf, voting_clf):\n", - " clf.fit(X_train, y_train)\n", - " y_pred = clf.predict(X_test)\n", - " print(clf.__class__.__name__, accuracy_score(y_test, y_pred))\n", - "\n", - "log_clf = LogisticRegression(solver=\"liblinear\", random_state=42)\n", - "rnd_clf = RandomForestClassifier(n_estimators=10, random_state=42)\n", - "svm_clf = SVC(gamma=\"auto\", probability=True, random_state=42)\n", - "\n", - "voting_clf = VotingClassifier(\n", - " estimators=[('lr', log_clf), ('rf', rnd_clf), ('svc', svm_clf)],\n", - " voting='soft')\n", - "voting_clf.fit(X_train, y_train)\n", - "\n", - "from sklearn.metrics import accuracy_score\n", - "\n", - "for clf in (log_clf, rnd_clf, svm_clf, voting_clf):\n", - " clf.fit(X_train, y_train)\n", - " y_pred = clf.predict(X_test)\n", - " print(clf.__class__.__name__, accuracy_score(y_test, y_pred))" - ] - }, - { - "cell_type": "markdown", - "id": "eb5b7a17", - "metadata": { - "editable": true - }, - "source": [ - "## Please, not the moons again! Voting and Bagging" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "id": "d7a1c436", - "metadata": { - "collapsed": false, - "editable": true - }, - "outputs": [], - "source": [ - "from sklearn.model_selection import train_test_split\n", - "from sklearn.datasets import make_moons\n", - "\n", - "X, y = make_moons(n_samples=500, noise=0.30, random_state=42)\n", - "X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42)\n", - "from sklearn.ensemble import RandomForestClassifier\n", - "from sklearn.ensemble import VotingClassifier\n", - "from sklearn.linear_model import LogisticRegression\n", - "from sklearn.svm import SVC\n", - "\n", - "log_clf = LogisticRegression(random_state=42)\n", - "rnd_clf = RandomForestClassifier(random_state=42)\n", - "svm_clf = SVC(random_state=42)\n", - "\n", - "voting_clf = VotingClassifier(\n", - " estimators=[('lr', log_clf), ('rf', rnd_clf), ('svc', svm_clf)],\n", - " voting='hard')\n", - "voting_clf.fit(X_train, y_train)" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "id": "19ee5bed", - "metadata": { - "collapsed": false, - "editable": true - }, - "outputs": [], - "source": [ - "from sklearn.metrics import accuracy_score\n", - "\n", - "for clf in (log_clf, rnd_clf, svm_clf, voting_clf):\n", - " clf.fit(X_train, y_train)\n", - " y_pred = clf.predict(X_test)\n", - " print(clf.__class__.__name__, accuracy_score(y_test, y_pred))" - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "id": "52fe9487", - "metadata": { - "collapsed": false, - "editable": true - }, - "outputs": [], - "source": [ - "log_clf = LogisticRegression(random_state=42)\n", - "rnd_clf = RandomForestClassifier(random_state=42)\n", - "svm_clf = SVC(probability=True, random_state=42)\n", - "\n", - "voting_clf = VotingClassifier(\n", - " estimators=[('lr', log_clf), ('rf', rnd_clf), ('svc', svm_clf)],\n", - " voting='soft')\n", - "voting_clf.fit(X_train, y_train)" - ] - }, - { - "cell_type": "code", - "execution_count": 21, - "id": "aa58e6f2", - "metadata": { - "collapsed": false, - "editable": true - }, - "outputs": [], - "source": [ - "from sklearn.metrics import accuracy_score\n", - "\n", - "for clf in (log_clf, rnd_clf, svm_clf, voting_clf):\n", - " clf.fit(X_train, y_train)\n", - " y_pred = clf.predict(X_test)\n", - " print(clf.__class__.__name__, accuracy_score(y_test, y_pred))" - ] - }, - { - "cell_type": "markdown", - "id": "e28f8931", - "metadata": { - "editable": true - }, - "source": [ - "## Bagging Examples" - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "id": "6db78fad", - "metadata": { - "collapsed": false, - "editable": true - }, - "outputs": [], - "source": [ - "from sklearn.ensemble import BaggingClassifier\n", - "from sklearn.tree import DecisionTreeClassifier\n", - "\n", - "bag_clf = BaggingClassifier(\n", - " DecisionTreeClassifier(random_state=42), n_estimators=500,\n", - " max_samples=100, bootstrap=True, n_jobs=-1, random_state=42)\n", - "bag_clf.fit(X_train, y_train)\n", - "y_pred = bag_clf.predict(X_test)" - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "id": "c968bde2", - "metadata": { - "collapsed": false, - "editable": true - }, - "outputs": [], - "source": [ - "from sklearn.metrics import accuracy_score\n", - "print(accuracy_score(y_test, y_pred))" - ] - }, - { - "cell_type": "code", - "execution_count": 24, - "id": "57ef3f0d", - "metadata": { - "collapsed": false, - "editable": true - }, - "outputs": [], - "source": [ - "tree_clf = DecisionTreeClassifier(random_state=42)\n", - "tree_clf.fit(X_train, y_train)\n", - "y_pred_tree = tree_clf.predict(X_test)\n", - "print(accuracy_score(y_test, y_pred_tree))" - ] - }, - { - "cell_type": "code", - "execution_count": 25, - "id": "2516a60f", - "metadata": { - "collapsed": false, - "editable": true - }, - "outputs": [], - "source": [ - "from matplotlib.colors import ListedColormap\n", - "\n", - "def plot_decision_boundary(clf, X, y, axes=[-1.5, 2.5, -1, 1.5], alpha=0.5, contour=True):\n", - " x1s = np.linspace(axes[0], axes[1], 100)\n", - " x2s = np.linspace(axes[2], axes[3], 100)\n", - " x1, x2 = np.meshgrid(x1s, x2s)\n", - " X_new = np.c_[x1.ravel(), x2.ravel()]\n", - " y_pred = clf.predict(X_new).reshape(x1.shape)\n", - " custom_cmap = ListedColormap(['#fafab0','#9898ff','#a0faa0'])\n", - " plt.contourf(x1, x2, y_pred, alpha=0.3, cmap=custom_cmap)\n", - " if contour:\n", - " custom_cmap2 = ListedColormap(['#7d7d58','#4c4c7f','#507d50'])\n", - " plt.contour(x1, x2, y_pred, cmap=custom_cmap2, alpha=0.8)\n", - " plt.plot(X[:, 0][y==0], X[:, 1][y==0], \"yo\", alpha=alpha)\n", - " plt.plot(X[:, 0][y==1], X[:, 1][y==1], \"bs\", alpha=alpha)\n", - " plt.axis(axes)\n", - " plt.xlabel(r\"$x_1$\", fontsize=18)\n", - " plt.ylabel(r\"$x_2$\", fontsize=18, rotation=0)\n", - "plt.figure(figsize=(11,4))\n", - "plt.subplot(121)\n", - "plot_decision_boundary(tree_clf, X, y)\n", - "plt.title(\"Decision Tree\", fontsize=14)\n", - "plt.subplot(122)\n", - "plot_decision_boundary(bag_clf, X, y)\n", - "plt.title(\"Decision Trees with Bagging\", fontsize=14)\n", - "save_fig(\"baggingtree\")\n", - "plt.show()" - ] - }, - { - "cell_type": "markdown", - "id": "94238f26", - "metadata": { - "editable": true - }, - "source": [ - "## Making your own Bootstrap: Changing the Level of the Decision Tree\n", - "\n", - "Let us bring up our good old boostrap example from the linear regression lectures. We change the linerar regression algorithm with\n", - "a decision tree wth different depths and perform a bootstrap aggregate (in this case we perform as many bootstraps as data points $n$)." - ] - }, - { - "cell_type": "code", - "execution_count": 26, - "id": "fdc56e2f", - "metadata": { - "collapsed": false, - "editable": true - }, - "outputs": [], - "source": [ - "\n", - "import matplotlib.pyplot as plt\n", - "import numpy as np\n", - "from sklearn.model_selection import train_test_split\n", - "from sklearn.pipeline import make_pipeline\n", - "from sklearn.utils import resample\n", - "from sklearn.tree import DecisionTreeRegressor\n", - "\n", - "n = 100\n", - "n_boostraps = 100\n", - "maxdepth = 8\n", - "\n", - "# Make data set.\n", - "x = np.linspace(-3, 3, n).reshape(-1, 1)\n", - "y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)\n", - "error = np.zeros(maxdepth)\n", - "bias = np.zeros(maxdepth)\n", - "variance = np.zeros(maxdepth)\n", - "polydegree = np.zeros(maxdepth)\n", - "X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.2)\n", - "\n", - "from sklearn.preprocessing import StandardScaler\n", - "scaler = StandardScaler()\n", - "scaler.fit(X_train)\n", - "X_train_scaled = scaler.transform(X_train)\n", - "X_test_scaled = scaler.transform(X_test)\n", - "\n", - "# we produce a simple tree first as benchmark\n", - "simpletree = DecisionTreeRegressor(max_depth=3) \n", - "simpletree.fit(X_train_scaled, y_train)\n", - "simpleprediction = simpletree.predict(X_test_scaled)\n", - "for degree in range(1,maxdepth):\n", - " model = DecisionTreeRegressor(max_depth=degree) \n", - " y_pred = np.empty((y_test.shape[0], n_boostraps))\n", - " for i in range(n_boostraps):\n", - " x_, y_ = resample(X_train_scaled, y_train)\n", - " model.fit(x_, y_)\n", - " y_pred[:, i] = model.predict(X_test_scaled)#.ravel()\n", - "\n", - " polydegree[degree] = degree\n", - " error[degree] = np.mean( np.mean((y_test - y_pred)**2, axis=1, keepdims=True) )\n", - " bias[degree] = np.mean( (y_test - np.mean(y_pred, axis=1, keepdims=True))**2 )\n", - " variance[degree] = np.mean( np.var(y_pred, axis=1, keepdims=True) )\n", - " print('Polynomial degree:', degree)\n", - " print('Error:', error[degree])\n", - " print('Bias^2:', bias[degree])\n", - " print('Var:', variance[degree])\n", - " print('{} >= {} + {} = {}'.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree]))\n", - " \n", - "mse_simpletree= np.mean( np.mean((y_test - simpleprediction)**2)\n", - "print(mse_simpletree)\n", - "plt.xlim(1,maxdepth)\n", - "plt.plot(polydegree, error, label='MSE')\n", - "plt.plot(polydegree, bias, label='bias')\n", - "plt.plot(polydegree, variance, label='Variance')\n", - "plt.legend()\n", - "save_fig(\"baggingboot\")\n", - "plt.show()" - ] - }, - { - "cell_type": "markdown", - "id": "a2152251", + "id": "e5e0c59a", "metadata": { "editable": true }, @@ -2324,7 +1197,7 @@ }, { "cell_type": "markdown", - "id": "2580d159", + "id": "1a9bfe29", "metadata": { "editable": true }, @@ -2350,7 +1223,7 @@ }, { "cell_type": "markdown", - "id": "91785abe", + "id": "2cf605f0", "metadata": { "editable": true }, @@ -2378,7 +1251,7 @@ }, { "cell_type": "markdown", - "id": "aff0725e", + "id": "15f5ba4a", "metadata": { "editable": true }, @@ -2395,7 +1268,7 @@ }, { "cell_type": "markdown", - "id": "e009576a", + "id": "845b8dd1", "metadata": { "editable": true }, @@ -2404,12 +1277,12 @@ "\n", "**Gini index for hour slept.**\n", "\n", - "See handwritten notes for Thursday November 11" + "See handwritten notes for Thursday November 3" ] }, { "cell_type": "markdown", - "id": "4716a25a", + "id": "110fd1bb", "metadata": { "editable": true }, @@ -2418,14 +1291,14 @@ "\n", "**Gini index for hour studied.**\n", "\n", - "See handwritten notes for Thursday November 11\n", + "See handwritten notes for Thursday November 3\n", "\n", "For final tree, see the above handwritten notes" ] }, { "cell_type": "markdown", - "id": "000bbbec", + "id": "eb2e935b", "metadata": { "editable": true }, @@ -2435,8 +1308,8 @@ }, { "cell_type": "code", - "execution_count": 27, - "id": "c954f1a3", + "execution_count": 7, + "id": "6fcbfb29", "metadata": { "collapsed": false, "editable": true @@ -2514,7 +1387,7 @@ }, { "cell_type": "markdown", - "id": "27d64625", + "id": "fc49e714", "metadata": { "editable": true }, @@ -2524,8 +1397,8 @@ }, { "cell_type": "code", - "execution_count": 28, - "id": "fadc5906", + "execution_count": 8, + "id": "22b06167", "metadata": { "collapsed": false, "editable": true @@ -2569,7 +1442,7 @@ }, { "cell_type": "markdown", - "id": "75ac2805", + "id": "23945e25", "metadata": { "editable": true }, @@ -2579,8 +1452,8 @@ }, { "cell_type": "code", - "execution_count": 29, - "id": "56db7126", + "execution_count": 9, + "id": "5067a505", "metadata": { "collapsed": false, "editable": true @@ -2615,7 +1488,7 @@ }, { "cell_type": "markdown", - "id": "996da175", + "id": "bbfb8d35", "metadata": { "editable": true }, @@ -2627,8 +1500,8 @@ }, { "cell_type": "code", - "execution_count": 30, - "id": "514adf4c", + "execution_count": 10, + "id": "ee1b4bcb", "metadata": { "collapsed": false, "editable": true @@ -2646,7 +1519,7 @@ }, { "cell_type": "markdown", - "id": "d3cf9e06", + "id": "69e5c13e", "metadata": { "editable": true }, @@ -2659,8 +1532,8 @@ }, { "cell_type": "code", - "execution_count": 31, - "id": "00ccdb8f", + "execution_count": 11, + "id": "82b62d44", "metadata": { "collapsed": false, "editable": true @@ -2679,7 +1552,7 @@ }, { "cell_type": "markdown", - "id": "4eef5c8e", + "id": "c1cc4b18", "metadata": { "editable": true }, @@ -2699,7 +1572,7 @@ }, { "cell_type": "markdown", - "id": "cf9cabd2", + "id": "5166bff6", "metadata": { "editable": true }, @@ -2716,7 +1589,7 @@ }, { "cell_type": "markdown", - "id": "e1285ba7", + "id": "c7e8061d", "metadata": { "editable": true }, @@ -2728,7 +1601,7 @@ }, { "cell_type": "markdown", - "id": "f9cca74c", + "id": "6c655cfc", "metadata": { "editable": true }, @@ -2745,7 +1618,7 @@ }, { "cell_type": "markdown", - "id": "98b6a486", + "id": "3be73cc7", "metadata": { "editable": true }, @@ -2758,7 +1631,7 @@ }, { "cell_type": "markdown", - "id": "eb8e00a0", + "id": "81fa9b19", "metadata": { "editable": true }, @@ -2770,7 +1643,7 @@ }, { "cell_type": "markdown", - "id": "87074a7c", + "id": "c09eea17", "metadata": { "editable": true }, @@ -2780,7 +1653,7 @@ }, { "cell_type": "markdown", - "id": "0c6b96c0", + "id": "a43ee22b", "metadata": { "editable": true }, @@ -2792,7 +1665,7 @@ }, { "cell_type": "markdown", - "id": "d164bb2f", + "id": "974ce892", "metadata": { "editable": true }, @@ -2802,7 +1675,7 @@ }, { "cell_type": "markdown", - "id": "44bb90b8", + "id": "cc0a986e", "metadata": { "editable": true }, @@ -2814,7 +1687,7 @@ }, { "cell_type": "markdown", - "id": "3e4d8236", + "id": "3b8fcb4b", "metadata": { "editable": true }, @@ -2827,7 +1700,7 @@ }, { "cell_type": "markdown", - "id": "819fe171", + "id": "10e4a51a", "metadata": { "editable": true }, @@ -2871,7 +1744,7 @@ }, { "cell_type": "markdown", - "id": "5e60797d", + "id": "bee5f51e", "metadata": { "editable": true }, @@ -2881,8 +1754,8 @@ }, { "cell_type": "code", - "execution_count": 32, - "id": "ec7983c3", + "execution_count": 12, + "id": "84bd9fe3", "metadata": { "collapsed": false, "editable": true @@ -2960,7 +1833,7 @@ }, { "cell_type": "markdown", - "id": "634a9a06", + "id": "7708b736", "metadata": { "editable": true }, @@ -2977,8 +1850,8 @@ }, { "cell_type": "code", - "execution_count": 33, - "id": "1e596f67", + "execution_count": 13, + "id": "80f20db8", "metadata": { "collapsed": false, "editable": true @@ -3049,7 +1922,7 @@ }, { "cell_type": "markdown", - "id": "3bbe605d", + "id": "3f8fdb99", "metadata": { "editable": true }, @@ -3088,7 +1961,7 @@ }, { "cell_type": "markdown", - "id": "b4f333a4", + "id": "8787f239", "metadata": { "editable": true }, @@ -3098,8 +1971,8 @@ }, { "cell_type": "code", - "execution_count": 34, - "id": "73f52dc4", + "execution_count": 14, + "id": "33615a2f", "metadata": { "collapsed": false, "editable": true @@ -3151,7 +2024,7 @@ }, { "cell_type": "markdown", - "id": "d2093489", + "id": "86af16b5", "metadata": { "editable": true }, @@ -3161,8 +2034,8 @@ }, { "cell_type": "code", - "execution_count": 35, - "id": "46d0e993", + "execution_count": 15, + "id": "9156dfa5", "metadata": { "collapsed": false, "editable": true @@ -3237,7 +2110,7 @@ }, { "cell_type": "markdown", - "id": "bf265e4c", + "id": "acb03e2d", "metadata": { "editable": true }, @@ -3247,8 +2120,8 @@ }, { "cell_type": "code", - "execution_count": 36, - "id": "da8e27bb", + "execution_count": 16, + "id": "52dd18ad", "metadata": { "collapsed": false, "editable": true @@ -3279,7 +2152,7 @@ }, { "cell_type": "markdown", - "id": "ab124d4a", + "id": "64016383", "metadata": { "editable": true }, @@ -3289,8 +2162,8 @@ }, { "cell_type": "code", - "execution_count": 37, - "id": "9e26ed73", + "execution_count": 17, + "id": "79b5bba9", "metadata": { "collapsed": false, "editable": true @@ -3307,8 +2180,8 @@ }, { "cell_type": "code", - "execution_count": 38, - "id": "fc64f4fc", + "execution_count": 18, + "id": "31f137ed", "metadata": { "collapsed": false, "editable": true @@ -3323,7 +2196,7 @@ }, { "cell_type": "markdown", - "id": "b3d48f29", + "id": "6c038990", "metadata": { "editable": true }, @@ -3333,8 +2206,8 @@ }, { "cell_type": "code", - "execution_count": 39, - "id": "c8093883", + "execution_count": 19, + "id": "fadbc304", "metadata": { "collapsed": false, "editable": true @@ -3383,8 +2256,8 @@ }, { "cell_type": "code", - "execution_count": 40, - "id": "cc8dfee9", + "execution_count": 20, + "id": "ac6be3cf", "metadata": { "collapsed": false, "editable": true @@ -3423,7 +2296,1134 @@ }, { "cell_type": "markdown", - "id": "20e8260b", + "id": "602021c7", + "metadata": { + "editable": true + }, + "source": [ + "## Example: Computing the Gini index\n", + "\n", + "The example we will look at is a classical one in many Machine\n", + "Learning applications. Based on various meteorological features, we\n", + "have several so-called attributes which decide whether we at the end\n", + "will do some outdoor activity like skiing, going for a bike ride etc\n", + "etc. The table here contains the feautures **outlook**, **temperature**,\n", + "**humidity** and **wind**. The target or output is whether we ride\n", + "(True=1) or whether we do something else that day (False=0). The\n", + "attributes for each feature are then sunny, overcast and rain for the\n", + "outlook, hot, cold and mild for temperature, high and normal for\n", + "humidity and weak and strong for wind.\n", + "\n", + "The table here summarizes the various attributes and\n", + "| Day | Outlook | Temperature | Humidity | Wind | Ride |
|---|---|---|---|---|---|
| 1 | Sunny | Hot | High | Weak | 0 |
| 2 | Sunny | Hot | High | Strong | 1 |
| 3 | Overcast | Hot | High | Weak | 1 |
| 4 | Rain | Mild | High | Weak | 1 |
| 5 | Rain | Cool | Normal | Weak | 1 |
| 6 | Rain | Cool | Normal | Strong | 0 |
| 7 | Overcast | Cool | Normal | Strong | 1 |
| 8 | Sunny | Mild | High | Weak | 0 |
| 9 | Sunny | Cool | Normal | Weak | 1 |
| 10 | Rain | Mild | Normal | Weak | 1 |
| 11 | Sunny | Mild | Normal | Strong | 1 |
| 12 | Overcast | Mild | High | Strong | 1 |
| 13 | Overcast | Hot | Normal | Weak | 1 |
| 14 | Rain | Mild | High | Strong | 0 |
Figure 1:
\n", + "" + ] + }, + { + "cell_type": "markdown", + "id": "2922c038", + "metadata": { + "editable": true + }, + "source": [ + "## Bagging\n", + "\n", + "The **plain** decision trees suffer from high\n", + "variance. This means that if we split the training data into two parts\n", + "at random, and fit a decision tree to both halves, the results that we\n", + "get could be quite different. In contrast, a procedure with low\n", + "variance will yield similar results if applied repeatedly to distinct\n", + "data sets; linear regression tends to have low variance, if the ratio\n", + "of $n$ to $p$ is moderately large. \n", + "\n", + "**Bootstrap aggregation**, or just **bagging**, is a\n", + "general-purpose procedure for reducing the variance of a statistical\n", + "learning method." + ] + }, + { + "cell_type": "markdown", + "id": "c1be9c10", + "metadata": { + "editable": true + }, + "source": [ + "## More bagging\n", + "\n", + "Bagging typically results in improved accuracy\n", + "over prediction using a single tree. Unfortunately, however, it can be\n", + "difficult to interpret the resulting model. Recall that one of the\n", + "advantages of decision trees is the attractive and easily interpreted\n", + "diagram that results.\n", + "\n", + "However, when we bag a large number of trees, it is no longer\n", + "possible to represent the resulting statistical learning procedure\n", + "using a single tree, and it is no longer clear which variables are\n", + "most important to the procedure. Thus, bagging improves prediction\n", + "accuracy at the expense of interpretability. Although the collection\n", + "of bagged trees is much more difficult to interpret than a single\n", + "tree, one can obtain an overall summary of the importance of each\n", + "predictor using the MSE (for bagging regression trees) or the Gini\n", + "index (for bagging classification trees). In the case of bagging\n", + "regression trees, we can record the total amount that the MSE is\n", + "decreased due to splits over a given predictor, averaged over all $B$ possible\n", + "trees. A large value indicates an important predictor. Similarly, in\n", + "the context of bagging classification trees, we can add up the total\n", + "amount that the Gini index is decreased by splits over a given\n", + "predictor, averaged over all $B$ trees." + ] + }, + { + "cell_type": "markdown", + "id": "b2e06e1a", + "metadata": { + "editable": true + }, + "source": [ + "## Simple Voting Example, head or tail" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "id": "f8ec4475", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "heads_proba = 0.51\n", + "coin_tosses = (np.random.rand(10000, 10) < heads_proba).astype(np.int32)\n", + "cumulative_heads_ratio = np.cumsum(coin_tosses, axis=0) / np.arange(1, 10001).reshape(-1, 1)\n", + "plt.figure(figsize=(8,3.5))\n", + "plt.plot(cumulative_heads_ratio)\n", + "plt.plot([0, 10000], [0.51, 0.51], \"k--\", linewidth=2, label=\"51%\")\n", + "plt.plot([0, 10000], [0.5, 0.5], \"k-\", label=\"50%\")\n", + "plt.xlabel(\"Number of coin tosses\")\n", + "plt.ylabel(\"Heads ratio\")\n", + "plt.legend(loc=\"lower right\")\n", + "plt.axis([0, 10000, 0.42, 0.58])\n", + "save_fig(\"votingsimple\")\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "1158700d", + "metadata": { + "editable": true + }, + "source": [ + "## Using the Voting Classifier" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "aa6c99a5", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "from sklearn.model_selection import train_test_split\n", + "from sklearn.datasets import make_moons\n", + "\n", + "X, y = make_moons(n_samples=500, noise=0.30, random_state=42)\n", + "X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42)\n", + "\n", + "from sklearn.ensemble import RandomForestClassifier\n", + "from sklearn.ensemble import VotingClassifier\n", + "from sklearn.linear_model import LogisticRegression\n", + "from sklearn.svm import SVC\n", + "\n", + "log_clf = LogisticRegression(solver=\"liblinear\", random_state=42)\n", + "rnd_clf = RandomForestClassifier(n_estimators=10, random_state=42)\n", + "svm_clf = SVC(gamma=\"auto\", random_state=42)\n", + "\n", + "voting_clf = VotingClassifier(\n", + " estimators=[('lr', log_clf), ('rf', rnd_clf), ('svc', svm_clf)],\n", + " voting='hard')\n", + "\n", + "voting_clf.fit(X_train, y_train)\n", + "\n", + "from sklearn.metrics import accuracy_score\n", + "\n", + "for clf in (log_clf, rnd_clf, svm_clf, voting_clf):\n", + " clf.fit(X_train, y_train)\n", + " y_pred = clf.predict(X_test)\n", + " print(clf.__class__.__name__, accuracy_score(y_test, y_pred))\n", + "\n", + "log_clf = LogisticRegression(solver=\"liblinear\", random_state=42)\n", + "rnd_clf = RandomForestClassifier(n_estimators=10, random_state=42)\n", + "svm_clf = SVC(gamma=\"auto\", probability=True, random_state=42)\n", + "\n", + "voting_clf = VotingClassifier(\n", + " estimators=[('lr', log_clf), ('rf', rnd_clf), ('svc', svm_clf)],\n", + " voting='soft')\n", + "voting_clf.fit(X_train, y_train)\n", + "\n", + "from sklearn.metrics import accuracy_score\n", + "\n", + "for clf in (log_clf, rnd_clf, svm_clf, voting_clf):\n", + " clf.fit(X_train, y_train)\n", + " y_pred = clf.predict(X_test)\n", + " print(clf.__class__.__name__, accuracy_score(y_test, y_pred))" + ] + }, + { + "cell_type": "markdown", + "id": "b3786e72", + "metadata": { + "editable": true + }, + "source": [ + "## Please, not the moons again! Voting and Bagging" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "id": "5e4d4b54", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "from sklearn.model_selection import train_test_split\n", + "from sklearn.datasets import make_moons\n", + "\n", + "X, y = make_moons(n_samples=500, noise=0.30, random_state=42)\n", + "X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42)\n", + "from sklearn.ensemble import RandomForestClassifier\n", + "from sklearn.ensemble import VotingClassifier\n", + "from sklearn.linear_model import LogisticRegression\n", + "from sklearn.svm import SVC\n", + "\n", + "log_clf = LogisticRegression(random_state=42)\n", + "rnd_clf = RandomForestClassifier(random_state=42)\n", + "svm_clf = SVC(random_state=42)\n", + "\n", + "voting_clf = VotingClassifier(\n", + " estimators=[('lr', log_clf), ('rf', rnd_clf), ('svc', svm_clf)],\n", + " voting='hard')\n", + "voting_clf.fit(X_train, y_train)" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "id": "003eafd1", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "from sklearn.metrics import accuracy_score\n", + "\n", + "for clf in (log_clf, rnd_clf, svm_clf, voting_clf):\n", + " clf.fit(X_train, y_train)\n", + " y_pred = clf.predict(X_test)\n", + " print(clf.__class__.__name__, accuracy_score(y_test, y_pred))" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "id": "1450e5e1", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "log_clf = LogisticRegression(random_state=42)\n", + "rnd_clf = RandomForestClassifier(random_state=42)\n", + "svm_clf = SVC(probability=True, random_state=42)\n", + "\n", + "voting_clf = VotingClassifier(\n", + " estimators=[('lr', log_clf), ('rf', rnd_clf), ('svc', svm_clf)],\n", + " voting='soft')\n", + "voting_clf.fit(X_train, y_train)" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "90334ede", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "from sklearn.metrics import accuracy_score\n", + "\n", + "for clf in (log_clf, rnd_clf, svm_clf, voting_clf):\n", + " clf.fit(X_train, y_train)\n", + " y_pred = clf.predict(X_test)\n", + " print(clf.__class__.__name__, accuracy_score(y_test, y_pred))" + ] + }, + { + "cell_type": "markdown", + "id": "9a4eed1c", + "metadata": { + "editable": true + }, + "source": [ + "## Bagging Examples" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "id": "e3917951", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "from sklearn.ensemble import BaggingClassifier\n", + "from sklearn.tree import DecisionTreeClassifier\n", + "\n", + "bag_clf = BaggingClassifier(\n", + " DecisionTreeClassifier(random_state=42), n_estimators=500,\n", + " max_samples=100, bootstrap=True, n_jobs=-1, random_state=42)\n", + "bag_clf.fit(X_train, y_train)\n", + "y_pred = bag_clf.predict(X_test)" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "id": "6971fd95", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "from sklearn.metrics import accuracy_score\n", + "print(accuracy_score(y_test, y_pred))" + ] + }, + { + "cell_type": "code", + "execution_count": 38, + "id": "b85f5a4b", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "tree_clf = DecisionTreeClassifier(random_state=42)\n", + "tree_clf.fit(X_train, y_train)\n", + "y_pred_tree = tree_clf.predict(X_test)\n", + "print(accuracy_score(y_test, y_pred_tree))" + ] + }, + { + "cell_type": "code", + "execution_count": 39, + "id": "1b0ebc28", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "from matplotlib.colors import ListedColormap\n", + "\n", + "def plot_decision_boundary(clf, X, y, axes=[-1.5, 2.5, -1, 1.5], alpha=0.5, contour=True):\n", + " x1s = np.linspace(axes[0], axes[1], 100)\n", + " x2s = np.linspace(axes[2], axes[3], 100)\n", + " x1, x2 = np.meshgrid(x1s, x2s)\n", + " X_new = np.c_[x1.ravel(), x2.ravel()]\n", + " y_pred = clf.predict(X_new).reshape(x1.shape)\n", + " custom_cmap = ListedColormap(['#fafab0','#9898ff','#a0faa0'])\n", + " plt.contourf(x1, x2, y_pred, alpha=0.3, cmap=custom_cmap)\n", + " if contour:\n", + " custom_cmap2 = ListedColormap(['#7d7d58','#4c4c7f','#507d50'])\n", + " plt.contour(x1, x2, y_pred, cmap=custom_cmap2, alpha=0.8)\n", + " plt.plot(X[:, 0][y==0], X[:, 1][y==0], \"yo\", alpha=alpha)\n", + " plt.plot(X[:, 0][y==1], X[:, 1][y==1], \"bs\", alpha=alpha)\n", + " plt.axis(axes)\n", + " plt.xlabel(r\"$x_1$\", fontsize=18)\n", + " plt.ylabel(r\"$x_2$\", fontsize=18, rotation=0)\n", + "plt.figure(figsize=(11,4))\n", + "plt.subplot(121)\n", + "plot_decision_boundary(tree_clf, X, y)\n", + "plt.title(\"Decision Tree\", fontsize=14)\n", + "plt.subplot(122)\n", + "plot_decision_boundary(bag_clf, X, y)\n", + "plt.title(\"Decision Trees with Bagging\", fontsize=14)\n", + "save_fig(\"baggingtree\")\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "13e78e8f", + "metadata": { + "editable": true + }, + "source": [ + "## Making your own Bootstrap: Changing the Level of the Decision Tree\n", + "\n", + "Let us bring up our good old boostrap example from the linear regression lectures. We change the linerar regression algorithm with\n", + "a decision tree wth different depths and perform a bootstrap aggregate (in this case we perform as many bootstraps as data points $n$)." + ] + }, + { + "cell_type": "code", + "execution_count": 40, + "id": "95166266", + "metadata": { + "collapsed": false, + "editable": true + }, + "outputs": [], + "source": [ + "\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "from sklearn.model_selection import train_test_split\n", + "from sklearn.pipeline import make_pipeline\n", + "from sklearn.utils import resample\n", + "from sklearn.tree import DecisionTreeRegressor\n", + "\n", + "n = 100\n", + "n_boostraps = 100\n", + "maxdepth = 8\n", + "\n", + "# Make data set.\n", + "x = np.linspace(-3, 3, n).reshape(-1, 1)\n", + "y = np.exp(-x**2) + 1.5 * np.exp(-(x-2)**2)+ np.random.normal(0, 0.1, x.shape)\n", + "error = np.zeros(maxdepth)\n", + "bias = np.zeros(maxdepth)\n", + "variance = np.zeros(maxdepth)\n", + "polydegree = np.zeros(maxdepth)\n", + "X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.2)\n", + "\n", + "from sklearn.preprocessing import StandardScaler\n", + "scaler = StandardScaler()\n", + "scaler.fit(X_train)\n", + "X_train_scaled = scaler.transform(X_train)\n", + "X_test_scaled = scaler.transform(X_test)\n", + "\n", + "# we produce a simple tree first as benchmark\n", + "simpletree = DecisionTreeRegressor(max_depth=3) \n", + "simpletree.fit(X_train_scaled, y_train)\n", + "simpleprediction = simpletree.predict(X_test_scaled)\n", + "for degree in range(1,maxdepth):\n", + " model = DecisionTreeRegressor(max_depth=degree) \n", + " y_pred = np.empty((y_test.shape[0], n_boostraps))\n", + " for i in range(n_boostraps):\n", + " x_, y_ = resample(X_train_scaled, y_train)\n", + " model.fit(x_, y_)\n", + " y_pred[:, i] = model.predict(X_test_scaled)#.ravel()\n", + "\n", + " polydegree[degree] = degree\n", + " error[degree] = np.mean( np.mean((y_test - y_pred)**2, axis=1, keepdims=True) )\n", + " bias[degree] = np.mean( (y_test - np.mean(y_pred, axis=1, keepdims=True))**2 )\n", + " variance[degree] = np.mean( np.var(y_pred, axis=1, keepdims=True) )\n", + " print('Polynomial degree:', degree)\n", + " print('Error:', error[degree])\n", + " print('Bias^2:', bias[degree])\n", + " print('Var:', variance[degree])\n", + " print('{} >= {} + {} = {}'.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree]))\n", + " \n", + "mse_simpletree= np.mean( np.mean((y_test - simpleprediction)**2)\n", + "print(mse_simpletree)\n", + "plt.xlim(1,maxdepth)\n", + "plt.plot(polydegree, error, label='MSE')\n", + "plt.plot(polydegree, bias, label='bias')\n", + "plt.plot(polydegree, variance, label='Variance')\n", + "plt.legend()\n", + "save_fig(\"baggingboot\")\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "8f6edeac", "metadata": { "editable": true }, @@ -3447,7 +3447,7 @@ }, { "cell_type": "markdown", - "id": "ed634a04", + "id": "a84df029", "metadata": { "editable": true }, @@ -3478,7 +3478,7 @@ }, { "cell_type": "markdown", - "id": "830be8d4", + "id": "55598ae3", "metadata": { "editable": true }, @@ -3489,7 +3489,7 @@ { "cell_type": "code", "execution_count": 41, - "id": "a5bba637", + "id": "f12a3a95", "metadata": { "collapsed": false, "editable": true @@ -3537,7 +3537,7 @@ }, { "cell_type": "markdown", - "id": "bf430c99", + "id": "a8343dbb", "metadata": { "editable": true }, @@ -3548,7 +3548,7 @@ { "cell_type": "code", "execution_count": 42, - "id": "9e83b5a3", + "id": "04b07a81", "metadata": { "collapsed": false, "editable": true @@ -3582,7 +3582,7 @@ }, { "cell_type": "markdown", - "id": "19597a5f", + "id": "da599ec1", "metadata": { "editable": true }, @@ -3595,7 +3595,7 @@ { "cell_type": "code", "execution_count": 43, - "id": "a4665db0", + "id": "6aa6d10a", "metadata": { "collapsed": false, "editable": true @@ -3648,7 +3648,7 @@ }, { "cell_type": "markdown", - "id": "9703ef22", + "id": "eb83dba0", "metadata": { "editable": true }, @@ -3659,7 +3659,7 @@ { "cell_type": "code", "execution_count": 44, - "id": "c2113ac3", + "id": "5fd35cb6", "metadata": { "collapsed": false, "editable": true @@ -3689,7 +3689,7 @@ { "cell_type": "code", "execution_count": 45, - "id": "05f14b77", + "id": "bf2ee8f1", "metadata": { "collapsed": false, "editable": true @@ -3707,7 +3707,7 @@ { "cell_type": "code", "execution_count": 46, - "id": "617f544f", + "id": "8113c552", "metadata": { "collapsed": false, "editable": true @@ -3727,7 +3727,7 @@ { "cell_type": "code", "execution_count": 47, - "id": "f7d7b367", + "id": "8ac8fd4c", "metadata": { "collapsed": false, "editable": true @@ -3744,7 +3744,7 @@ }, { "cell_type": "markdown", - "id": "d8612bf4", + "id": "860525e5", "metadata": { "editable": true }, @@ -3767,7 +3767,7 @@ }, { "cell_type": "markdown", - "id": "b04c8bf3", + "id": "a3c8cd7d", "metadata": { "editable": true }, @@ -3779,7 +3779,7 @@ }, { "cell_type": "markdown", - "id": "f3633859", + "id": "110ced5a", "metadata": { "editable": true }, @@ -3804,7 +3804,7 @@ }, { "cell_type": "markdown", - "id": "6506fbb6", + "id": "63e88bb3", "metadata": { "editable": true }, @@ -3830,7 +3830,7 @@ }, { "cell_type": "markdown", - "id": "0e4baaf3", + "id": "d579fe83", "metadata": { "editable": true }, @@ -3841,7 +3841,7 @@ { "cell_type": "code", "execution_count": 48, - "id": "53f80b1d", + "id": "b2704a99", "metadata": { "collapsed": false, "editable": true @@ -3918,7 +3918,7 @@ }, { "cell_type": "markdown", - "id": "1a0c5686", + "id": "18851285", "metadata": { "editable": true }, @@ -3934,7 +3934,7 @@ }, { "cell_type": "markdown", - "id": "cc4e628f", + "id": "7b477771", "metadata": { "editable": true }, @@ -3945,7 +3945,7 @@ { "cell_type": "code", "execution_count": 49, - "id": "224dc41f", + "id": "e0433ae0", "metadata": { "collapsed": false, "editable": true @@ -3960,7 +3960,7 @@ { "cell_type": "code", "execution_count": 50, - "id": "abc5aee5", + "id": "1d445962", "metadata": { "collapsed": false, "editable": true diff --git a/doc/src/week44/week44.do.txt b/doc/src/week44/week44.do.txt index d8255821f..b6a94b973 100644 --- a/doc/src/week44/week44.do.txt +++ b/doc/src/week44/week44.do.txt @@ -663,6 +663,319 @@ print(r) !ec +!split +===== Algorithms for Setting up Decision Trees ===== + +Two algorithms stand out in the set up of decision trees: +o The CART (Classification And Regression Tree) algorithm for both classification and regression +o The ID3 algorithm based on the computation of the information gain for classification + +We discuss both algorithms with applications here. The popular library +_Scikit-Learn_ uses the CART algorithm. For classification problems +you can use either the _gini_ index or the _entropy_ to split a tree +in two branches. + +!split +===== The CART algorithm for Classification ===== + +For classification, the CART algorithm splits the data set in two subsets using a single feature $k$ and a threshold $t_k$. +This could be for example a threshold set by a number below a certain circumference of a malign tumor. + +How do we find these two quantities? +We search for the pair $(k,t_k)$ that produces the purest subset using for example the _gini_ factor $G$. +The cost function it tries to minimize is then +!bt +\[ +C(k,t_k) = \frac{m_{\mathrm{left}}}{m}G_{\mathrm{left}}+ \frac{m_{\mathrm{right}}}{m}G_{\mathrm{right}}, +\] +!et +where $G_{\mathrm{left/right}}$ measures the impurity of the left/right subset and $m_{\mathrm{left/right}}$ + is the number of instances in the left/right subset + +Once it has successfully split the training set in two, it splits the subsets using the same logic, then the subsubsets +and so on, recursively. It stops recursing once it reaches the maximum depth (defined by the +$max\_depth$ hyperparameter), or if it cannot find a split that will reduce impurity. A few other +hyperparameters control additional stopping conditions such as the $min\_samples\_split$, +$min\_samples\_leaf$, $min\_weight\_fraction\_leaf$, and $max\_leaf\_nodes$. + +!split +===== The CART algorithm for Regression ===== + +The CART algorithm for regression works is similar to the one for classification except that instead of trying to split the +training set in a way that minimizes say the _gini_ or _entropy_ impurity, it now tries to split the training set in a way that minimizes our well-known mean-squared error (MSE). The cost function is now +!bt +\[ +C(k,t_k) = \frac{m_{\mathrm{left}}}{m}\mathrm{MSE}_{\mathrm{left}}+ \frac{m_{\mathrm{right}}}{m}\mathrm{MSE}_{\mathrm{right}}. +\] +!et +Here the MSE for a specific node is defined as +!bt +\[ +\mathrm{MSE}_{\mathrm{node}}=\frac{1}{m_\mathrm{node}}\sum_{i\in \mathrm{node}}(\overline{y}_{\mathrm{node}}-y_i)^2, +\] +!et +with +!bt +\[ +\overline{y}_{\mathrm{node}}=\frac{1}{m_\mathrm{node}}\sum_{i\in \mathrm{node}}y_i, +\] +!et +the mean value of all observations in a specific node. + +Without any regularization, the regression task for decision trees, +just like for classification tasks, is prone to overfitting. + +!split +===== Why binary splits? ===== + +It is custom to split to a tree uising binary splits. The reason is +that multiway splits fragment the data too quickly, leaving +insufficient data at the next level down. Multiway splits can be +achieved by a series of binary split and this is normally preferred. + + +!split +===== Computing a Tree using the Gini Index ===== + +Consider the following example with attributes/features and two +possible outcomes (classes) for each attribute. Assume we wish to find some +correlations between the average grade of a student as function of the +number of hours studied and hours slept. We want also to correlate the +grade in a given course with the general trend, whether the students +recently has gotten grades below average or above. + +We have three features/attributes +o Trend of average grades before present course, classified as either below or above the average grade of the whole class +o The number of hours studies, classified again as either higher (more than 3 hours per day) or lower . Here we have used a standard for one $ECTS$ which is scaled to 25-30 hours of work for a semester which lasts 18 weeks, with 15 weeks of lectures and 3 weeks for exams, assuming a total of 30 ECTS per semester. +o The number of hours slept as high for more than $8$ hours and below for less than 8 hours of sleep, classified again as either high or low +o The final grade whether it is above or below average + + +!split +===== The Table ===== + +|---------------------------------------------------| +| Grade Trend | Hours slept | Hours Studied | Grade | +|---------------------------------------------------| +| Above | Low | High | Above | +| Below | High | Low | Below | +| Above | Low | High | Above | +| Above | High | High | Above | +| Below | Low | High | Below | +| Above | Low | Low | Below | +| Below | High | High | Below | +| Below | Low | High | Below | +| Above | Low | Low | Below | +| Above | High | High | Above | +|---------------------------------------------------| + + +!split +===== Computing the various Gini Indices ===== + +In computations we will translate all classes into numbers. Being +these binary classes, they can easily be split into ones and zeros. + +!bblock Gini index for Average trend +See handwritten notes for Thursday November 11 +!eblock + + +!split +===== Computing the various Gini Indices, Hours slept ===== + + +!bblock Gini index for hour slept +See handwritten notes for Thursday November 3 +!eblock + + +!split +===== Computing the various Gini Indices, Hours studied ===== + + +!bblock Gini index for hour studied +See handwritten notes for Thursday November 3 +!eblock + +For final tree, see the above handwritten notes + + +!split +===== A possible code using Scikit-Learn ===== + +!bc pycod +# Common imports +import numpy as np +import pandas as pd +import matplotlib.pyplot as plt +from sklearn.tree import DecisionTreeClassifier +from sklearn.model_selection import train_test_split +from sklearn.tree import export_graphviz +from sklearn.preprocessing import StandardScaler, OneHotEncoder +from sklearn.compose import ColumnTransformer +from IPython.display import Image +from pydot import graph_from_dot_data +import os + +# Where to save the figures and data files +PROJECT_ROOT_DIR = "Results" +FIGURE_ID = "Results/FigureFiles" +DATA_ID = "DataFiles/" + +if not os.path.exists(PROJECT_ROOT_DIR): + os.mkdir(PROJECT_ROOT_DIR) + +if not os.path.exists(FIGURE_ID): + os.makedirs(FIGURE_ID) + +if not os.path.exists(DATA_ID): + os.makedirs(DATA_ID) + +def image_path(fig_id): + return os.path.join(FIGURE_ID, fig_id) + +def data_path(dat_id): + return os.path.join(DATA_ID, dat_id) + +def save_fig(fig_id): + plt.savefig(image_path(fig_id) + ".png", format='png') + +infile = open(data_path("grades.csv"),'r') + +# Read the experimental data with Pandas +from IPython.display import display +grades = pd.read_csv(infile,names = ('Trend','Sleep','Studied','Grade')) +grades = pd.DataFrame(grades) + +# Features and targets +X = grades.loc[:, grades.columns != 'Grade'].values +y = grades.loc[:, grades.columns == 'Grade'].values + +# Create the encoder. +encoder = OneHotEncoder(handle_unknown="ignore") +# Assume for simplicity all features are categorical. +encoder.fit(X) +# Apply the encoder. +X = encoder.transform(X) +print(X) +# Then do a Classification tree +tree_clf = DecisionTreeClassifier(max_depth=2) +tree_clf.fit(X, y) +print("Train set accuracy with Decision Tree: {:.2f}".format(tree_clf.score(X,y))) +#transfer to a decision tree graph +export_graphviz( + tree_clf, + out_file="DataFiles/grade.dot", + rounded=True, + filled=True +) +cmd = 'dot -Tpng DataFiles/grade.dot -o DataFiles/grades.png' +os.system(cmd) +!ec + + + +!split +===== Visualizing Trees, More examples ===== +!bc pycod +import os +from sklearn.datasets import load_breast_cancer +from sklearn.tree import DecisionTreeClassifier +from sklearn.model_selection import train_test_split +from sklearn.metrics import confusion_matrix +from sklearn.tree import export_graphviz + +from IPython.display import Image +from pydot import graph_from_dot_data +import pandas as pd +import numpy as np + + +cancer = load_breast_cancer() +X = pd.DataFrame(cancer.data, columns=cancer.feature_names) +print(X) +y = pd.Categorical.from_codes(cancer.target, cancer.target_names) +y = pd.get_dummies(y) +print(y) +X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=1) +tree_clf = DecisionTreeClassifier(max_depth=5) +tree_clf.fit(X_train, y_train) + +export_graphviz( + tree_clf, + out_file="DataFiles/cancer.dot", + feature_names=cancer.feature_names, + class_names=cancer.target_names, + rounded=True, + filled=True +) +cmd = 'dot -Tpng DataFiles/cancer.dot -o DataFiles/cancer.png' +os.system(cmd) +!ec + +!split +===== Visualizing the Tree, The Moons ===== +!bc pycod +# Common imports +import numpy as np +from sklearn.model_selection import train_test_split +from sklearn.tree import DecisionTreeClassifier +from sklearn.datasets import make_moons +from sklearn.tree import export_graphviz +from pydot import graph_from_dot_data +import pandas as pd +import os + +np.random.seed(42) +X, y = make_moons(n_samples=100, noise=0.25, random_state=53) +X_train, X_test, y_train, y_test = train_test_split(X,y,random_state=0) +tree_clf = DecisionTreeClassifier(max_depth=5) +tree_clf.fit(X_train, y_train) + +export_graphviz( + tree_clf, + out_file="DataFiles/moons.dot", + rounded=True, + filled=True +) +cmd = 'dot -Tpng DataFiles/moons.dot -o DataFiles/moons.png' +os.system(cmd) +!ec + +!split +===== Other ways of visualizing the trees ===== + +_Scikit-Learn_ has also another way to visualize the trees which is very useful, here with the Iris data. + +!bc pycod +from sklearn.datasets import load_iris +from sklearn import tree +X, y = load_iris(return_X_y=True) +tree_clf = tree.DecisionTreeClassifier() +tree_clf = tree_clf.fit(X, y) +# and then plot the tree +tree.plot_tree(tree_clf) +!ec + +!split +===== Printing out as text ===== + +Alternatively, the tree can also be exported in textual format with the function exporttext. +This method doesn’t require the installation of external libraries and is more compact: + +!bc pycod +from sklearn.datasets import load_iris +from sklearn.tree import DecisionTreeClassifier +from sklearn.tree import export_text +iris = load_iris() +decision_tree = DecisionTreeClassifier(random_state=0, max_depth=2) +decision_tree = decision_tree.fit(iris.data, iris.target) +r = export_text(decision_tree, feature_names=iris['feature_names']) +print(r) +!ec + + !split ===== Algorithms for Setting up Decision Trees ===== @@ -1174,6 +1487,456 @@ plt.show() + + +!split +===== Example: Computing the Gini index ===== + +The example we will look at is a classical one in many Machine +Learning applications. Based on various meteorological features, we +have several so-called attributes which decide whether we at the end +will do some outdoor activity like skiing, going for a bike ride etc +etc. The table here contains the feautures _outlook_, _temperature_, +_humidity_ and _wind_. The target or output is whether we ride +(True=1) or whether we do something else that day (False=0). The +attributes for each feature are then sunny, overcast and rain for the +outlook, hot, cold and mild for temperature, high and normal for +humidity and weak and strong for wind. + +The table here summarizes the various attributes and +|-------------------------------------------| +|Day| Outlook |Temperature | Humidity | Wind | Ride| +|-------------------------------------------| +|1 | Sunny | Hot | High | Weak | 0 | +|2 | Sunny | Hot | High | Strong | 1 | +|3 | Overcast | Hot | High | Weak | 1 | +|4 | Rain | Mild | High | Weak | 1 | +|5 | Rain | Cool | Normal | Weak | 1 | +|6 | Rain | Cool | Normal | Strong | 0 | +|7 | Overcast | Cool | Normal | Strong | 1 | +|8 | Sunny | Mild | High | Weak | 0 | +|9 | Sunny | Cool | Normal | Weak | 1 | +|10 | Rain | Mild | Normal | Weak | 1 | +|11 | Sunny | Mild | Normal | Strong | 1 | +|12 | Overcast | Mild | High | Strong | 1 | +|13 | Overcast | Hot | Normal | Weak | 1 | +|14 | Rain | Mild | High | Strong | 0 | +|-------------------------------------------| + + +!split +===== Simple Python Code to read in Data and perform Classification ===== + +!bc pycod +# Common imports +import numpy as np +import pandas as pd +import matplotlib.pyplot as plt +from sklearn.tree import DecisionTreeClassifier +from sklearn.model_selection import train_test_split +from sklearn.tree import export_graphviz +from sklearn.preprocessing import StandardScaler, OneHotEncoder +from sklearn.compose import ColumnTransformer +from IPython.display import Image +from pydot import graph_from_dot_data +import os + +# Where to save the figures and data files +PROJECT_ROOT_DIR = "Results" +FIGURE_ID = "Results/FigureFiles" +DATA_ID = "DataFiles/" + +if not os.path.exists(PROJECT_ROOT_DIR): + os.mkdir(PROJECT_ROOT_DIR) + +if not os.path.exists(FIGURE_ID): + os.makedirs(FIGURE_ID) + +if not os.path.exists(DATA_ID): + os.makedirs(DATA_ID) + +def image_path(fig_id): + return os.path.join(FIGURE_ID, fig_id) + +def data_path(dat_id): + return os.path.join(DATA_ID, dat_id) + +def save_fig(fig_id): + plt.savefig(image_path(fig_id) + ".png", format='png') + +infile = open(data_path("rideclass.csv"),'r') + +# Read the experimental data with Pandas +from IPython.display import display +ridedata = pd.read_csv(infile,names = ('Outlook','Temperature','Humidity','Wind','Ride')) +ridedata = pd.DataFrame(ridedata) + +# Features and targets +X = ridedata.loc[:, ridedata.columns != 'Ride'].values +y = ridedata.loc[:, ridedata.columns == 'Ride'].values + +# Create the encoder. +encoder = OneHotEncoder(handle_unknown="ignore") +# Assume for simplicity all features are categorical. +encoder.fit(X) +# Apply the encoder. +X = encoder.transform(X) +print(X) +# Then do a Classification tree +tree_clf = DecisionTreeClassifier(max_depth=2) +tree_clf.fit(X, y) +print("Train set accuracy with Decision Tree: {:.2f}".format(tree_clf.score(X,y))) +#transfer to a decision tree graph +export_graphviz( + tree_clf, + out_file="DataFiles/ride.dot", + rounded=True, + filled=True +) +cmd = 'dot -Tpng DataFiles/cancer.dot -o DataFiles/cancer.png' +os.system(cmd) + + +!ec + +!split +===== Computing the Gini Factor ===== + +The above functions (gini, entropy and misclassification error) are +important components of the so-called CART algorithm. We will discuss +this algorithm below after we have discussed the information gain +algorithm ID3. + +In the example here we have converted all our attributes into numerical values $0,1,2$ etc. + +!bc pycod +# Split a dataset based on an attribute and an attribute value +def test_split(index, value, dataset): + left, right = list(), list() + for row in dataset: + if row[index] < value: + left.append(row) + else: + right.append(row) + return left, right + +# Calculate the Gini index for a split dataset +def gini_index(groups, classes): + # count all samples at split point + n_instances = float(sum([len(group) for group in groups])) + # sum weighted Gini index for each group + gini = 0.0 + for group in groups: + size = float(len(group)) + # avoid divide by zero + if size == 0: + continue + score = 0.0 + # score the group based on the score for each class + for class_val in classes: + p = [row[-1] for row in group].count(class_val) / size + score += p * p + # weight the group score by its relative size + gini += (1.0 - score) * (size / n_instances) + return gini + +# Select the best split point for a dataset +def get_split(dataset): + class_values = list(set(row[-1] for row in dataset)) + b_index, b_value, b_score, b_groups = 999, 999, 999, None + for index in range(len(dataset[0])-1): + for row in dataset: + groups = test_split(index, row[index], dataset) + gini = gini_index(groups, class_values) + print('X%d < %.3f Gini=%.3f' % ((index+1), row[index], gini)) + if gini < b_score: + b_index, b_value, b_score, b_groups = index, row[index], gini, groups + return {'index':b_index, 'value':b_value, 'groups':b_groups} + +dataset = [[0,0,0,0,0], + [0,0,0,1,1], + [1,0,0,0,1], + [2,1,0,0,1], + [2,2,1,0,1], + [2,2,1,1,0], + [1,2,1,1,1], + [0,1,0,0,0], + [0,2,1,0,1], + [2,1,1,0,1], + [0,1,1,1,1], + [1,1,0,1,1], + [1,0,1,0,1], + [2,1,0,1,0]] + +split = get_split(dataset) +print('Split: [X%d < %.3f]' % ((split['index']+1), split['value'])) +!ec + +!split +===== Entropy and the ID3 algorithm ===== + +The ID3 algorithm learns decision trees by constructing +them in a top down way, beginning with the question _which attribute should be tested at the root of the tree_? + +o Each instance attribute is evaluated using a statistical test to determine how well it alone classifies the training examples. +o The best attribute is selected and used as the test at the root node of the tree. +o A descendant of the root node is then created for each possible value of this attribute. +o Training examples are sorted to the appropriate descendant node. +o The entire process is then repeated using the training examples associated with each descendant node to select the best attribute to test at that point in the tree. +o This forms a greedy search for an acceptable decision tree, in which the algorithm never backtracks to reconsider earlier choices. + +The ID3 algorithm selects which attribute to test at each node in the +tree. + +We would like to select the attribute that is most useful for classifying +examples. + +What is a good quantitative measure of the worth of an attribute? + +Information gain measures how well a given attribute separates the +training examples according to their target classification. + +The ID3 algorithm uses this information gain measure to select among the candidate +attributes at each step while growing the tree. + + +!split +===== Cancer Data again now with Decision Trees and other Methods ===== +!bc pycod +import matplotlib.pyplot as plt +import numpy as np +from sklearn.model_selection import train_test_split +from sklearn.datasets import load_breast_cancer +from sklearn.svm import SVC +from sklearn.linear_model import LogisticRegression +from sklearn.tree import DecisionTreeClassifier + +# Load the data +cancer = load_breast_cancer() + +X_train, X_test, y_train, y_test = train_test_split(cancer.data,cancer.target,random_state=0) +print(X_train.shape) +print(X_test.shape) +# Logistic Regression +logreg = LogisticRegression(solver='lbfgs') +logreg.fit(X_train, y_train) +print("Test set accuracy with Logistic Regression: {:.2f}".format(logreg.score(X_test,y_test))) +# Support vector machine +svm = SVC(gamma='auto', C=100) +svm.fit(X_train, y_train) +print("Test set accuracy with SVM: {:.2f}".format(svm.score(X_test,y_test))) +# Decision Trees +deep_tree_clf = DecisionTreeClassifier(max_depth=None) +deep_tree_clf.fit(X_train, y_train) +print("Test set accuracy with Decision Trees: {:.2f}".format(deep_tree_clf.score(X_test,y_test))) +#now scale the data +from sklearn.preprocessing import StandardScaler +scaler = StandardScaler() +scaler.fit(X_train) +X_train_scaled = scaler.transform(X_train) +X_test_scaled = scaler.transform(X_test) +# Logistic Regression +logreg.fit(X_train_scaled, y_train) +print("Test set accuracy Logistic Regression with scaled data: {:.2f}".format(logreg.score(X_test_scaled,y_test))) +# Support Vector Machine +svm.fit(X_train_scaled, y_train) +print("Test set accuracy SVM with scaled data: {:.2f}".format(logreg.score(X_test_scaled,y_test))) +# Decision Trees +deep_tree_clf.fit(X_train_scaled, y_train) +print("Test set accuracy with Decision Trees and scaled data: {:.2f}".format(deep_tree_clf.score(X_test_scaled,y_test))) + +!ec + + +!split +===== Another example, the moons again ===== +!bc pycod +from __future__ import division, print_function, unicode_literals + +# Common imports +import numpy as np +import os + +# to make this notebook's output stable across runs +np.random.seed(42) + +# To plot pretty figures +import matplotlib +import matplotlib.pyplot as plt +from matplotlib.colors import ListedColormap +plt.rcParams['axes.labelsize'] = 14 +plt.rcParams['xtick.labelsize'] = 12 +plt.rcParams['ytick.labelsize'] = 12 + + +from sklearn.svm import SVC +from sklearn import datasets +from sklearn.tree import DecisionTreeClassifier +from sklearn.datasets import make_moons +from sklearn.tree import export_graphviz + +Xm, ym = make_moons(n_samples=100, noise=0.25, random_state=53) + +deep_tree_clf1 = DecisionTreeClassifier(random_state=42) +deep_tree_clf2 = DecisionTreeClassifier(min_samples_leaf=4, random_state=42) +deep_tree_clf1.fit(Xm, ym) +deep_tree_clf2.fit(Xm, ym) + + +def plot_decision_boundary(clf, X, y, axes=[0, 7.5, 0, 3], iris=True, legend=False, plot_training=True): + x1s = np.linspace(axes[0], axes[1], 100) + x2s = np.linspace(axes[2], axes[3], 100) + x1, x2 = np.meshgrid(x1s, x2s) + X_new = np.c_[x1.ravel(), x2.ravel()] + y_pred = clf.predict(X_new).reshape(x1.shape) + custom_cmap = ListedColormap(['#fafab0','#9898ff','#a0faa0']) + plt.contourf(x1, x2, y_pred, alpha=0.3, cmap=custom_cmap) + if not iris: + custom_cmap2 = ListedColormap(['#7d7d58','#4c4c7f','#507d50']) + plt.contour(x1, x2, y_pred, cmap=custom_cmap2, alpha=0.8) + if plot_training: + plt.plot(X[:, 0][y==0], X[:, 1][y==0], "yo", label="Iris-Setosa") + plt.plot(X[:, 0][y==1], X[:, 1][y==1], "bs", label="Iris-Versicolor") + plt.plot(X[:, 0][y==2], X[:, 1][y==2], "g^", label="Iris-Virginica") + plt.axis(axes) + if iris: + plt.xlabel("Petal length", fontsize=14) + plt.ylabel("Petal width", fontsize=14) + else: + plt.xlabel(r"$x_1$", fontsize=18) + plt.ylabel(r"$x_2$", fontsize=18, rotation=0) + if legend: + plt.legend(loc="lower right", fontsize=14) +plt.figure(figsize=(11, 4)) +plt.subplot(121) +plot_decision_boundary(deep_tree_clf1, Xm, ym, axes=[-1.5, 2.5, -1, 1.5], iris=False) +plt.title("No restrictions", fontsize=16) +plt.subplot(122) +plot_decision_boundary(deep_tree_clf2, Xm, ym, axes=[-1.5, 2.5, -1, 1.5], iris=False) +plt.title("min_samples_leaf = {}".format(deep_tree_clf2.min_samples_leaf), fontsize=14) +plt.show() + +!ec + +!split +===== Playing around with regions ===== +!bc pycod +np.random.seed(6) +Xs = np.random.rand(100, 2) - 0.5 +ys = (Xs[:, 0] > 0).astype(np.float32) * 2 + +angle = np.pi/4 +rotation_matrix = np.array([[np.cos(angle), -np.sin(angle)], [np.sin(angle), np.cos(angle)]]) +Xsr = Xs.dot(rotation_matrix) + +tree_clf_s = DecisionTreeClassifier(random_state=42) +tree_clf_s.fit(Xs, ys) +tree_clf_sr = DecisionTreeClassifier(random_state=42) +tree_clf_sr.fit(Xsr, ys) + +plt.figure(figsize=(11, 4)) +plt.subplot(121) +plot_decision_boundary(tree_clf_s, Xs, ys, axes=[-0.7, 0.7, -0.7, 0.7], iris=False) +plt.subplot(122) +plot_decision_boundary(tree_clf_sr, Xsr, ys, axes=[-0.7, 0.7, -0.7, 0.7], iris=False) + +plt.show() +!ec + +!split +===== Regression trees ===== +!bc pycod +# Quadratic training set + noise +np.random.seed(42) +m = 200 +X = np.random.rand(m, 1) +y = 4 * (X - 0.5) ** 2 +y = y + np.random.randn(m, 1) / 10 +!ec + +!bc pycod +from sklearn.tree import DecisionTreeRegressor + +tree_reg = DecisionTreeRegressor(max_depth=2, random_state=42) +tree_reg.fit(X, y) +!ec + +!split +===== Final regressor code ===== +!bc pycod +from sklearn.tree import DecisionTreeRegressor + +tree_reg1 = DecisionTreeRegressor(random_state=42, max_depth=2) +tree_reg2 = DecisionTreeRegressor(random_state=42, max_depth=3) +tree_reg1.fit(X, y) +tree_reg2.fit(X, y) + +def plot_regression_predictions(tree_reg, X, y, axes=[0, 1, -0.2, 1], ylabel="$y$"): + x1 = np.linspace(axes[0], axes[1], 500).reshape(-1, 1) + y_pred = tree_reg.predict(x1) + plt.axis(axes) + plt.xlabel("$x_1$", fontsize=18) + if ylabel: + plt.ylabel(ylabel, fontsize=18, rotation=0) + plt.plot(X, y, "b.") + plt.plot(x1, y_pred, "r.-", linewidth=2, label=r"$\hat{y}$") + +plt.figure(figsize=(11, 4)) +plt.subplot(121) +plot_regression_predictions(tree_reg1, X, y) +for split, style in ((0.1973, "k-"), (0.0917, "k--"), (0.7718, "k--")): + plt.plot([split, split], [-0.2, 1], style, linewidth=2) +plt.text(0.21, 0.65, "Depth=0", fontsize=15) +plt.text(0.01, 0.2, "Depth=1", fontsize=13) +plt.text(0.65, 0.8, "Depth=1", fontsize=13) +plt.legend(loc="upper center", fontsize=18) +plt.title("max_depth=2", fontsize=14) + +plt.subplot(122) +plot_regression_predictions(tree_reg2, X, y, ylabel=None) +for split, style in ((0.1973, "k-"), (0.0917, "k--"), (0.7718, "k--")): + plt.plot([split, split], [-0.2, 1], style, linewidth=2) +for split in (0.0458, 0.1298, 0.2873, 0.9040): + plt.plot([split, split], [-0.2, 1], "k:", linewidth=1) +plt.text(0.3, 0.5, "Depth=2", fontsize=13) +plt.title("max_depth=3", fontsize=14) + +plt.show() +!ec + +!bc pycod +tree_reg1 = DecisionTreeRegressor(random_state=42) +tree_reg2 = DecisionTreeRegressor(random_state=42, min_samples_leaf=10) +tree_reg1.fit(X, y) +tree_reg2.fit(X, y) + +x1 = np.linspace(0, 1, 500).reshape(-1, 1) +y_pred1 = tree_reg1.predict(x1) +y_pred2 = tree_reg2.predict(x1) + +plt.figure(figsize=(11, 4)) + +plt.subplot(121) +plt.plot(X, y, "b.") +plt.plot(x1, y_pred1, "r.-", linewidth=2, label=r"$\hat{y}$") +plt.axis([0, 1, -0.2, 1.1]) +plt.xlabel("$x_1$", fontsize=18) +plt.ylabel("$y$", fontsize=18, rotation=0) +plt.legend(loc="upper center", fontsize=18) +plt.title("No restrictions", fontsize=14) + +plt.subplot(122) +plt.plot(X, y, "b.") +plt.plot(x1, y_pred2, "r.-", linewidth=2, label=r"$\hat{y}$") +plt.axis([0, 1, -0.2, 1.1]) +plt.xlabel("$x_1$", fontsize=18) +plt.title("min_samples_leaf={}".format(tree_reg2.min_samples_leaf), fontsize=14) + +plt.show() +!ec + + + !split ===== Pros and cons of trees, pros ===== @@ -1520,768 +2283,6 @@ plt.show() !ec -!split -===== Why binary splits? ===== - -It is custom to split to a tree uising binary splits. The reason is -that multiway splits fragment the data too quickly, leaving -insufficient data at the next level down. Multiway splits can be -achieved by a series of binary split and this is normally preferred. - - -!split -===== Computing a Tree using the Gini Index ===== - -Consider the following example with attributes/features and two -possible outcomes (classes) for each attribute. Assume we wish to find some -correlations between the average grade of a student as function of the -number of hours studied and hours slept. We want also to correlate the -grade in a given course with the general trend, whether the students -recently has gotten grades below average or above. - -We have three features/attributes -o Trend of average grades before present course, classified as either below or above the average grade of the whole class -o The number of hours studies, classified again as either higher (more than 3 hours per day) or lower . Here we have used a standard for one $ECTS$ which is scaled to 25-30 hours of work for a semester which lasts 18 weeks, with 15 weeks of lectures and 3 weeks for exams, assuming a total of 30 ECTS per semester. -o The number of hours slept as high for more than $8$ hours and below for less than 8 hours of sleep, classified again as either high or low -o The final grade whether it is above or below average - - -!split -===== The Table ===== - -|---------------------------------------------------| -| Grade Trend | Hours slept | Hours Studied | Grade | -|---------------------------------------------------| -| Above | Low | High | Above | -| Below | High | Low | Below | -| Above | Low | High | Above | -| Above | High | High | Above | -| Below | Low | High | Below | -| Above | Low | Low | Below | -| Below | High | High | Below | -| Below | Low | High | Below | -| Above | Low | Low | Below | -| Above | High | High | Above | -|---------------------------------------------------| - - -!split -===== Computing the various Gini Indices ===== - -In computations we will translate all classes into numbers. Being -these binary classes, they can easily be split into ones and zeros. - -!bblock Gini index for Average trend -See handwritten notes for Thursday November 11 -!eblock - - -!split -===== Computing the various Gini Indices, Hours slept ===== - - -!bblock Gini index for hour slept -See handwritten notes for Thursday November 11 -!eblock - - -!split -===== Computing the various Gini Indices, Hours studied ===== - - -!bblock Gini index for hour studied -See handwritten notes for Thursday November 11 -!eblock - -For final tree, see the above handwritten notes - - -!split -===== A possible code using Scikit-Learn ===== - -!bc pycod -# Common imports -import numpy as np -import pandas as pd -import matplotlib.pyplot as plt -from sklearn.tree import DecisionTreeClassifier -from sklearn.model_selection import train_test_split -from sklearn.tree import export_graphviz -from sklearn.preprocessing import StandardScaler, OneHotEncoder -from sklearn.compose import ColumnTransformer -from IPython.display import Image -from pydot import graph_from_dot_data -import os - -# Where to save the figures and data files -PROJECT_ROOT_DIR = "Results" -FIGURE_ID = "Results/FigureFiles" -DATA_ID = "DataFiles/" - -if not os.path.exists(PROJECT_ROOT_DIR): - os.mkdir(PROJECT_ROOT_DIR) - -if not os.path.exists(FIGURE_ID): - os.makedirs(FIGURE_ID) - -if not os.path.exists(DATA_ID): - os.makedirs(DATA_ID) - -def image_path(fig_id): - return os.path.join(FIGURE_ID, fig_id) - -def data_path(dat_id): - return os.path.join(DATA_ID, dat_id) - -def save_fig(fig_id): - plt.savefig(image_path(fig_id) + ".png", format='png') - -infile = open(data_path("grades.csv"),'r') - -# Read the experimental data with Pandas -from IPython.display import display -grades = pd.read_csv(infile,names = ('Trend','Sleep','Studied','Grade')) -grades = pd.DataFrame(grades) - -# Features and targets -X = grades.loc[:, grades.columns != 'Grade'].values -y = grades.loc[:, grades.columns == 'Grade'].values - -# Create the encoder. -encoder = OneHotEncoder(handle_unknown="ignore") -# Assume for simplicity all features are categorical. -encoder.fit(X) -# Apply the encoder. -X = encoder.transform(X) -print(X) -# Then do a Classification tree -tree_clf = DecisionTreeClassifier(max_depth=2) -tree_clf.fit(X, y) -print("Train set accuracy with Decision Tree: {:.2f}".format(tree_clf.score(X,y))) -#transfer to a decision tree graph -export_graphviz( - tree_clf, - out_file="DataFiles/grade.dot", - rounded=True, - filled=True -) -cmd = 'dot -Tpng DataFiles/grade.dot -o DataFiles/grades.png' -os.system(cmd) -!ec - - - -!split -===== Visualizing Trees, More examples ===== -!bc pycod -import os -from sklearn.datasets import load_breast_cancer -from sklearn.tree import DecisionTreeClassifier -from sklearn.model_selection import train_test_split -from sklearn.metrics import confusion_matrix -from sklearn.tree import export_graphviz - -from IPython.display import Image -from pydot import graph_from_dot_data -import pandas as pd -import numpy as np - - -cancer = load_breast_cancer() -X = pd.DataFrame(cancer.data, columns=cancer.feature_names) -print(X) -y = pd.Categorical.from_codes(cancer.target, cancer.target_names) -y = pd.get_dummies(y) -print(y) -X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=1) -tree_clf = DecisionTreeClassifier(max_depth=5) -tree_clf.fit(X_train, y_train) - -export_graphviz( - tree_clf, - out_file="DataFiles/cancer.dot", - feature_names=cancer.feature_names, - class_names=cancer.target_names, - rounded=True, - filled=True -) -cmd = 'dot -Tpng DataFiles/cancer.dot -o DataFiles/cancer.png' -os.system(cmd) -!ec - -!split -===== Visualizing the Tree, The Moons ===== -!bc pycod -# Common imports -import numpy as np -from sklearn.model_selection import train_test_split -from sklearn.tree import DecisionTreeClassifier -from sklearn.datasets import make_moons -from sklearn.tree import export_graphviz -from pydot import graph_from_dot_data -import pandas as pd -import os - -np.random.seed(42) -X, y = make_moons(n_samples=100, noise=0.25, random_state=53) -X_train, X_test, y_train, y_test = train_test_split(X,y,random_state=0) -tree_clf = DecisionTreeClassifier(max_depth=5) -tree_clf.fit(X_train, y_train) - -export_graphviz( - tree_clf, - out_file="DataFiles/moons.dot", - rounded=True, - filled=True -) -cmd = 'dot -Tpng DataFiles/moons.dot -o DataFiles/moons.png' -os.system(cmd) -!ec - -!split -===== Other ways of visualizing the trees ===== - -_Scikit-Learn_ has also another way to visualize the trees which is very useful, here with the Iris data. - -!bc pycod -from sklearn.datasets import load_iris -from sklearn import tree -X, y = load_iris(return_X_y=True) -tree_clf = tree.DecisionTreeClassifier() -tree_clf = tree_clf.fit(X, y) -# and then plot the tree -tree.plot_tree(tree_clf) -!ec - -!split -===== Printing out as text ===== - -Alternatively, the tree can also be exported in textual format with the function exporttext. -This method doesn’t require the installation of external libraries and is more compact: - -!bc pycod -from sklearn.datasets import load_iris -from sklearn.tree import DecisionTreeClassifier -from sklearn.tree import export_text -iris = load_iris() -decision_tree = DecisionTreeClassifier(random_state=0, max_depth=2) -decision_tree = decision_tree.fit(iris.data, iris.target) -r = export_text(decision_tree, feature_names=iris['feature_names']) -print(r) -!ec - - -!split -===== Algorithms for Setting up Decision Trees ===== - -Two algorithms stand out in the set up of decision trees: -o The CART (Classification And Regression Tree) algorithm for both classification and regression -o The ID3 algorithm based on the computation of the information gain for classification - -We discuss both algorithms with applications here. The popular library -_Scikit-Learn_ uses the CART algorithm. For classification problems -you can use either the _gini_ index or the _entropy_ to split a tree -in two branches. - -!split -===== The CART algorithm for Classification ===== - -For classification, the CART algorithm splits the data set in two subsets using a single feature $k$ and a threshold $t_k$. -This could be for example a threshold set by a number below a certain circumference of a malign tumor. - -How do we find these two quantities? -We search for the pair $(k,t_k)$ that produces the purest subset using for example the _gini_ factor $G$. -The cost function it tries to minimize is then -!bt -\[ -C(k,t_k) = \frac{m_{\mathrm{left}}}{m}G_{\mathrm{left}}+ \frac{m_{\mathrm{right}}}{m}G_{\mathrm{right}}, -\] -!et -where $G_{\mathrm{left/right}}$ measures the impurity of the left/right subset and $m_{\mathrm{left/right}}$ - is the number of instances in the left/right subset - -Once it has successfully split the training set in two, it splits the subsets using the same logic, then the subsubsets -and so on, recursively. It stops recursing once it reaches the maximum depth (defined by the -$max\_depth$ hyperparameter), or if it cannot find a split that will reduce impurity. A few other -hyperparameters control additional stopping conditions such as the $min\_samples\_split$, -$min\_samples\_leaf$, $min\_weight\_fraction\_leaf$, and $max\_leaf\_nodes$. - -!split -===== The CART algorithm for Regression ===== - -The CART algorithm for regression works is similar to the one for classification except that instead of trying to split the -training set in a way that minimizes say the _gini_ or _entropy_ impurity, it now tries to split the training set in a way that minimizes our well-known mean-squared error (MSE). The cost function is now -!bt -\[ -C(k,t_k) = \frac{m_{\mathrm{left}}}{m}\mathrm{MSE}_{\mathrm{left}}+ \frac{m_{\mathrm{right}}}{m}\mathrm{MSE}_{\mathrm{right}}. -\] -!et -Here the MSE for a specific node is defined as -!bt -\[ -\mathrm{MSE}_{\mathrm{node}}=\frac{1}{m_\mathrm{node}}\sum_{i\in \mathrm{node}}(\overline{y}_{\mathrm{node}}-y_i)^2, -\] -!et -with -!bt -\[ -\overline{y}_{\mathrm{node}}=\frac{1}{m_\mathrm{node}}\sum_{i\in \mathrm{node}}y_i, -\] -!et -the mean value of all observations in a specific node. - -Without any regularization, the regression task for decision trees, -just like for classification tasks, is prone to overfitting. - - -!split -===== Computing the Gini index ===== - -The example we will look at is a classical one in many Machine -Learning applications. Based on various meteorological features, we -have several so-called attributes which decide whether we at the end -will do some outdoor activity like skiing, going for a bike ride etc -etc. The table here contains the feautures _outlook_, _temperature_, -_humidity_ and _wind_. The target or output is whether we ride -(True=1) or whether we do something else that day (False=0). The -attributes for each feature are then sunny, overcast and rain for the -outlook, hot, cold and mild for temperature, high and normal for -humidity and weak and strong for wind. - -The table here summarizes the various attributes and -|-------------------------------------------| -|Day| Outlook |Temperature | Humidity | Wind | Ride| -|-------------------------------------------| -|1 | Sunny | Hot | High | Weak | 0 | -|2 | Sunny | Hot | High | Strong | 1 | -|3 | Overcast | Hot | High | Weak | 1 | -|4 | Rain | Mild | High | Weak | 1 | -|5 | Rain | Cool | Normal | Weak | 1 | -|6 | Rain | Cool | Normal | Strong | 0 | -|7 | Overcast | Cool | Normal | Strong | 1 | -|8 | Sunny | Mild | High | Weak | 0 | -|9 | Sunny | Cool | Normal | Weak | 1 | -|10 | Rain | Mild | Normal | Weak | 1 | -|11 | Sunny | Mild | Normal | Strong | 1 | -|12 | Overcast | Mild | High | Strong | 1 | -|13 | Overcast | Hot | Normal | Weak | 1 | -|14 | Rain | Mild | High | Strong | 0 | -|-------------------------------------------| - - -!split -===== Simple Python Code to read in Data and perform Classification ===== - -!bc pycod -# Common imports -import numpy as np -import pandas as pd -import matplotlib.pyplot as plt -from sklearn.tree import DecisionTreeClassifier -from sklearn.model_selection import train_test_split -from sklearn.tree import export_graphviz -from sklearn.preprocessing import StandardScaler, OneHotEncoder -from sklearn.compose import ColumnTransformer -from IPython.display import Image -from pydot import graph_from_dot_data -import os - -# Where to save the figures and data files -PROJECT_ROOT_DIR = "Results" -FIGURE_ID = "Results/FigureFiles" -DATA_ID = "DataFiles/" - -if not os.path.exists(PROJECT_ROOT_DIR): - os.mkdir(PROJECT_ROOT_DIR) - -if not os.path.exists(FIGURE_ID): - os.makedirs(FIGURE_ID) - -if not os.path.exists(DATA_ID): - os.makedirs(DATA_ID) - -def image_path(fig_id): - return os.path.join(FIGURE_ID, fig_id) - -def data_path(dat_id): - return os.path.join(DATA_ID, dat_id) - -def save_fig(fig_id): - plt.savefig(image_path(fig_id) + ".png", format='png') - -infile = open(data_path("rideclass.csv"),'r') - -# Read the experimental data with Pandas -from IPython.display import display -ridedata = pd.read_csv(infile,names = ('Outlook','Temperature','Humidity','Wind','Ride')) -ridedata = pd.DataFrame(ridedata) - -# Features and targets -X = ridedata.loc[:, ridedata.columns != 'Ride'].values -y = ridedata.loc[:, ridedata.columns == 'Ride'].values - -# Create the encoder. -encoder = OneHotEncoder(handle_unknown="ignore") -# Assume for simplicity all features are categorical. -encoder.fit(X) -# Apply the encoder. -X = encoder.transform(X) -print(X) -# Then do a Classification tree -tree_clf = DecisionTreeClassifier(max_depth=2) -tree_clf.fit(X, y) -print("Train set accuracy with Decision Tree: {:.2f}".format(tree_clf.score(X,y))) -#transfer to a decision tree graph -export_graphviz( - tree_clf, - out_file="DataFiles/ride.dot", - rounded=True, - filled=True -) -cmd = 'dot -Tpng DataFiles/cancer.dot -o DataFiles/cancer.png' -os.system(cmd) - - -!ec - -!split -===== Computing the Gini Factor ===== - -The above functions (gini, entropy and misclassification error) are -important components of the so-called CART algorithm. We will discuss -this algorithm below after we have discussed the information gain -algorithm ID3. - -In the example here we have converted all our attributes into numerical values $0,1,2$ etc. - -!bc pycod -# Split a dataset based on an attribute and an attribute value -def test_split(index, value, dataset): - left, right = list(), list() - for row in dataset: - if row[index] < value: - left.append(row) - else: - right.append(row) - return left, right - -# Calculate the Gini index for a split dataset -def gini_index(groups, classes): - # count all samples at split point - n_instances = float(sum([len(group) for group in groups])) - # sum weighted Gini index for each group - gini = 0.0 - for group in groups: - size = float(len(group)) - # avoid divide by zero - if size == 0: - continue - score = 0.0 - # score the group based on the score for each class - for class_val in classes: - p = [row[-1] for row in group].count(class_val) / size - score += p * p - # weight the group score by its relative size - gini += (1.0 - score) * (size / n_instances) - return gini - -# Select the best split point for a dataset -def get_split(dataset): - class_values = list(set(row[-1] for row in dataset)) - b_index, b_value, b_score, b_groups = 999, 999, 999, None - for index in range(len(dataset[0])-1): - for row in dataset: - groups = test_split(index, row[index], dataset) - gini = gini_index(groups, class_values) - print('X%d < %.3f Gini=%.3f' % ((index+1), row[index], gini)) - if gini < b_score: - b_index, b_value, b_score, b_groups = index, row[index], gini, groups - return {'index':b_index, 'value':b_value, 'groups':b_groups} - -dataset = [[0,0,0,0,0], - [0,0,0,1,1], - [1,0,0,0,1], - [2,1,0,0,1], - [2,2,1,0,1], - [2,2,1,1,0], - [1,2,1,1,1], - [0,1,0,0,0], - [0,2,1,0,1], - [2,1,1,0,1], - [0,1,1,1,1], - [1,1,0,1,1], - [1,0,1,0,1], - [2,1,0,1,0]] - -split = get_split(dataset) -print('Split: [X%d < %.3f]' % ((split['index']+1), split['value'])) -!ec - -!split -===== Entropy and the ID3 algorithm ===== - -The ID3 algorithm learns decision trees by constructing -them in a top down way, beginning with the question _which attribute should be tested at the root of the tree_? - -o Each instance attribute is evaluated using a statistical test to determine how well it alone classifies the training examples. -o The best attribute is selected and used as the test at the root node of the tree. -o A descendant of the root node is then created for each possible value of this attribute. -o Training examples are sorted to the appropriate descendant node. -o The entire process is then repeated using the training examples associated with each descendant node to select the best attribute to test at that point in the tree. -o This forms a greedy search for an acceptable decision tree, in which the algorithm never backtracks to reconsider earlier choices. - -The ID3 algorithm selects which attribute to test at each node in the -tree. - -We would like to select the attribute that is most useful for classifying -examples. - -What is a good quantitative measure of the worth of an attribute? - -Information gain measures how well a given attribute separates the -training examples according to their target classification. - -The ID3 algorithm uses this information gain measure to select among the candidate -attributes at each step while growing the tree. - - -!split -===== Cancer Data again now with Decision Trees and other Methods ===== -!bc pycod -import matplotlib.pyplot as plt -import numpy as np -from sklearn.model_selection import train_test_split -from sklearn.datasets import load_breast_cancer -from sklearn.svm import SVC -from sklearn.linear_model import LogisticRegression -from sklearn.tree import DecisionTreeClassifier - -# Load the data -cancer = load_breast_cancer() - -X_train, X_test, y_train, y_test = train_test_split(cancer.data,cancer.target,random_state=0) -print(X_train.shape) -print(X_test.shape) -# Logistic Regression -logreg = LogisticRegression(solver='lbfgs') -logreg.fit(X_train, y_train) -print("Test set accuracy with Logistic Regression: {:.2f}".format(logreg.score(X_test,y_test))) -# Support vector machine -svm = SVC(gamma='auto', C=100) -svm.fit(X_train, y_train) -print("Test set accuracy with SVM: {:.2f}".format(svm.score(X_test,y_test))) -# Decision Trees -deep_tree_clf = DecisionTreeClassifier(max_depth=None) -deep_tree_clf.fit(X_train, y_train) -print("Test set accuracy with Decision Trees: {:.2f}".format(deep_tree_clf.score(X_test,y_test))) -#now scale the data -from sklearn.preprocessing import StandardScaler -scaler = StandardScaler() -scaler.fit(X_train) -X_train_scaled = scaler.transform(X_train) -X_test_scaled = scaler.transform(X_test) -# Logistic Regression -logreg.fit(X_train_scaled, y_train) -print("Test set accuracy Logistic Regression with scaled data: {:.2f}".format(logreg.score(X_test_scaled,y_test))) -# Support Vector Machine -svm.fit(X_train_scaled, y_train) -print("Test set accuracy SVM with scaled data: {:.2f}".format(logreg.score(X_test_scaled,y_test))) -# Decision Trees -deep_tree_clf.fit(X_train_scaled, y_train) -print("Test set accuracy with Decision Trees and scaled data: {:.2f}".format(deep_tree_clf.score(X_test_scaled,y_test))) - -!ec - - -!split -===== Another example, the moons again ===== -!bc pycod -from __future__ import division, print_function, unicode_literals - -# Common imports -import numpy as np -import os - -# to make this notebook's output stable across runs -np.random.seed(42) - -# To plot pretty figures -import matplotlib -import matplotlib.pyplot as plt -from matplotlib.colors import ListedColormap -plt.rcParams['axes.labelsize'] = 14 -plt.rcParams['xtick.labelsize'] = 12 -plt.rcParams['ytick.labelsize'] = 12 - - -from sklearn.svm import SVC -from sklearn import datasets -from sklearn.tree import DecisionTreeClassifier -from sklearn.datasets import make_moons -from sklearn.tree import export_graphviz - -Xm, ym = make_moons(n_samples=100, noise=0.25, random_state=53) - -deep_tree_clf1 = DecisionTreeClassifier(random_state=42) -deep_tree_clf2 = DecisionTreeClassifier(min_samples_leaf=4, random_state=42) -deep_tree_clf1.fit(Xm, ym) -deep_tree_clf2.fit(Xm, ym) - - -def plot_decision_boundary(clf, X, y, axes=[0, 7.5, 0, 3], iris=True, legend=False, plot_training=True): - x1s = np.linspace(axes[0], axes[1], 100) - x2s = np.linspace(axes[2], axes[3], 100) - x1, x2 = np.meshgrid(x1s, x2s) - X_new = np.c_[x1.ravel(), x2.ravel()] - y_pred = clf.predict(X_new).reshape(x1.shape) - custom_cmap = ListedColormap(['#fafab0','#9898ff','#a0faa0']) - plt.contourf(x1, x2, y_pred, alpha=0.3, cmap=custom_cmap) - if not iris: - custom_cmap2 = ListedColormap(['#7d7d58','#4c4c7f','#507d50']) - plt.contour(x1, x2, y_pred, cmap=custom_cmap2, alpha=0.8) - if plot_training: - plt.plot(X[:, 0][y==0], X[:, 1][y==0], "yo", label="Iris-Setosa") - plt.plot(X[:, 0][y==1], X[:, 1][y==1], "bs", label="Iris-Versicolor") - plt.plot(X[:, 0][y==2], X[:, 1][y==2], "g^", label="Iris-Virginica") - plt.axis(axes) - if iris: - plt.xlabel("Petal length", fontsize=14) - plt.ylabel("Petal width", fontsize=14) - else: - plt.xlabel(r"$x_1$", fontsize=18) - plt.ylabel(r"$x_2$", fontsize=18, rotation=0) - if legend: - plt.legend(loc="lower right", fontsize=14) -plt.figure(figsize=(11, 4)) -plt.subplot(121) -plot_decision_boundary(deep_tree_clf1, Xm, ym, axes=[-1.5, 2.5, -1, 1.5], iris=False) -plt.title("No restrictions", fontsize=16) -plt.subplot(122) -plot_decision_boundary(deep_tree_clf2, Xm, ym, axes=[-1.5, 2.5, -1, 1.5], iris=False) -plt.title("min_samples_leaf = {}".format(deep_tree_clf2.min_samples_leaf), fontsize=14) -plt.show() - -!ec - -!split -===== Playing around with regions ===== -!bc pycod -np.random.seed(6) -Xs = np.random.rand(100, 2) - 0.5 -ys = (Xs[:, 0] > 0).astype(np.float32) * 2 - -angle = np.pi/4 -rotation_matrix = np.array([[np.cos(angle), -np.sin(angle)], [np.sin(angle), np.cos(angle)]]) -Xsr = Xs.dot(rotation_matrix) - -tree_clf_s = DecisionTreeClassifier(random_state=42) -tree_clf_s.fit(Xs, ys) -tree_clf_sr = DecisionTreeClassifier(random_state=42) -tree_clf_sr.fit(Xsr, ys) - -plt.figure(figsize=(11, 4)) -plt.subplot(121) -plot_decision_boundary(tree_clf_s, Xs, ys, axes=[-0.7, 0.7, -0.7, 0.7], iris=False) -plt.subplot(122) -plot_decision_boundary(tree_clf_sr, Xsr, ys, axes=[-0.7, 0.7, -0.7, 0.7], iris=False) - -plt.show() -!ec - -!split -===== Regression trees ===== -!bc pycod -# Quadratic training set + noise -np.random.seed(42) -m = 200 -X = np.random.rand(m, 1) -y = 4 * (X - 0.5) ** 2 -y = y + np.random.randn(m, 1) / 10 -!ec - -!bc pycod -from sklearn.tree import DecisionTreeRegressor - -tree_reg = DecisionTreeRegressor(max_depth=2, random_state=42) -tree_reg.fit(X, y) -!ec - -!split -===== Final regressor code ===== -!bc pycod -from sklearn.tree import DecisionTreeRegressor - -tree_reg1 = DecisionTreeRegressor(random_state=42, max_depth=2) -tree_reg2 = DecisionTreeRegressor(random_state=42, max_depth=3) -tree_reg1.fit(X, y) -tree_reg2.fit(X, y) - -def plot_regression_predictions(tree_reg, X, y, axes=[0, 1, -0.2, 1], ylabel="$y$"): - x1 = np.linspace(axes[0], axes[1], 500).reshape(-1, 1) - y_pred = tree_reg.predict(x1) - plt.axis(axes) - plt.xlabel("$x_1$", fontsize=18) - if ylabel: - plt.ylabel(ylabel, fontsize=18, rotation=0) - plt.plot(X, y, "b.") - plt.plot(x1, y_pred, "r.-", linewidth=2, label=r"$\hat{y}$") - -plt.figure(figsize=(11, 4)) -plt.subplot(121) -plot_regression_predictions(tree_reg1, X, y) -for split, style in ((0.1973, "k-"), (0.0917, "k--"), (0.7718, "k--")): - plt.plot([split, split], [-0.2, 1], style, linewidth=2) -plt.text(0.21, 0.65, "Depth=0", fontsize=15) -plt.text(0.01, 0.2, "Depth=1", fontsize=13) -plt.text(0.65, 0.8, "Depth=1", fontsize=13) -plt.legend(loc="upper center", fontsize=18) -plt.title("max_depth=2", fontsize=14) - -plt.subplot(122) -plot_regression_predictions(tree_reg2, X, y, ylabel=None) -for split, style in ((0.1973, "k-"), (0.0917, "k--"), (0.7718, "k--")): - plt.plot([split, split], [-0.2, 1], style, linewidth=2) -for split in (0.0458, 0.1298, 0.2873, 0.9040): - plt.plot([split, split], [-0.2, 1], "k:", linewidth=1) -plt.text(0.3, 0.5, "Depth=2", fontsize=13) -plt.title("max_depth=3", fontsize=14) - -plt.show() -!ec - -!bc pycod -tree_reg1 = DecisionTreeRegressor(random_state=42) -tree_reg2 = DecisionTreeRegressor(random_state=42, min_samples_leaf=10) -tree_reg1.fit(X, y) -tree_reg2.fit(X, y) - -x1 = np.linspace(0, 1, 500).reshape(-1, 1) -y_pred1 = tree_reg1.predict(x1) -y_pred2 = tree_reg2.predict(x1) - -plt.figure(figsize=(11, 4)) - -plt.subplot(121) -plt.plot(X, y, "b.") -plt.plot(x1, y_pred1, "r.-", linewidth=2, label=r"$\hat{y}$") -plt.axis([0, 1, -0.2, 1.1]) -plt.xlabel("$x_1$", fontsize=18) -plt.ylabel("$y$", fontsize=18, rotation=0) -plt.legend(loc="upper center", fontsize=18) -plt.title("No restrictions", fontsize=14) - -plt.subplot(122) -plt.plot(X, y, "b.") -plt.plot(x1, y_pred2, "r.-", linewidth=2, label=r"$\hat{y}$") -plt.axis([0, 1, -0.2, 1.1]) -plt.xlabel("$x_1$", fontsize=18) -plt.title("min_samples_leaf={}".format(tree_reg2.min_samples_leaf), fontsize=14) - -plt.show() -!ec - - -