update week 45
This commit is contained in:
mhjensen
Binary file not shown.
|
Before Width: | Height: | Size: 41 KiB After Width: | Height: | Size: 39 KiB |
Binary file not shown.
@@ -1457,269 +1457,8 @@
|
||||
"amount that the Gini index is decreased by splits over a given\n",
|
||||
"predictor, averaged over all $B$ trees.\n",
|
||||
"\n",
|
||||
"## Simple Voting Example, head or tail"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 15,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"heads_proba = 0.51\n",
|
||||
"coin_tosses = (np.random.rand(10000, 10) < heads_proba).astype(np.int32)\n",
|
||||
"cumulative_heads_ratio = np.cumsum(coin_tosses, axis=0) / np.arange(1, 10001).reshape(-1, 1)\n",
|
||||
"plt.figure(figsize=(8,3.5))\n",
|
||||
"plt.plot(cumulative_heads_ratio)\n",
|
||||
"plt.plot([0, 10000], [0.51, 0.51], \"k--\", linewidth=2, label=\"51%\")\n",
|
||||
"plt.plot([0, 10000], [0.5, 0.5], \"k-\", label=\"50%\")\n",
|
||||
"plt.xlabel(\"Number of coin tosses\")\n",
|
||||
"plt.ylabel(\"Heads ratio\")\n",
|
||||
"plt.legend(loc=\"lower right\")\n",
|
||||
"plt.axis([0, 10000, 0.42, 0.58])\n",
|
||||
"save_fig(\"votingsimple\")\n",
|
||||
"plt.show()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Using the Voting Classifier"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 16,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from sklearn.model_selection import train_test_split\n",
|
||||
"from sklearn.datasets import make_moons\n",
|
||||
"\n",
|
||||
"X, y = make_moons(n_samples=500, noise=0.30, random_state=42)\n",
|
||||
"X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42)\n",
|
||||
"\n",
|
||||
"from sklearn.ensemble import RandomForestClassifier\n",
|
||||
"from sklearn.ensemble import VotingClassifier\n",
|
||||
"from sklearn.linear_model import LogisticRegression\n",
|
||||
"from sklearn.svm import SVC\n",
|
||||
"\n",
|
||||
"log_clf = LogisticRegression(solver=\"liblinear\", random_state=42)\n",
|
||||
"rnd_clf = RandomForestClassifier(n_estimators=10, random_state=42)\n",
|
||||
"svm_clf = SVC(gamma=\"auto\", random_state=42)\n",
|
||||
"\n",
|
||||
"voting_clf = VotingClassifier(\n",
|
||||
" estimators=[('lr', log_clf), ('rf', rnd_clf), ('svc', svm_clf)],\n",
|
||||
" voting='hard')\n",
|
||||
"\n",
|
||||
"voting_clf.fit(X_train, y_train)\n",
|
||||
"\n",
|
||||
"from sklearn.metrics import accuracy_score\n",
|
||||
"\n",
|
||||
"for clf in (log_clf, rnd_clf, svm_clf, voting_clf):\n",
|
||||
" clf.fit(X_train, y_train)\n",
|
||||
" y_pred = clf.predict(X_test)\n",
|
||||
" print(clf.__class__.__name__, accuracy_score(y_test, y_pred))\n",
|
||||
"\n",
|
||||
"log_clf = LogisticRegression(solver=\"liblinear\", random_state=42)\n",
|
||||
"rnd_clf = RandomForestClassifier(n_estimators=10, random_state=42)\n",
|
||||
"svm_clf = SVC(gamma=\"auto\", probability=True, random_state=42)\n",
|
||||
"\n",
|
||||
"voting_clf = VotingClassifier(\n",
|
||||
" estimators=[('lr', log_clf), ('rf', rnd_clf), ('svc', svm_clf)],\n",
|
||||
" voting='soft')\n",
|
||||
"voting_clf.fit(X_train, y_train)\n",
|
||||
"\n",
|
||||
"from sklearn.metrics import accuracy_score\n",
|
||||
"\n",
|
||||
"for clf in (log_clf, rnd_clf, svm_clf, voting_clf):\n",
|
||||
" clf.fit(X_train, y_train)\n",
|
||||
" y_pred = clf.predict(X_test)\n",
|
||||
" print(clf.__class__.__name__, accuracy_score(y_test, y_pred))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Please, not the moons again! Voting and Bagging"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 17,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from sklearn.model_selection import train_test_split\n",
|
||||
"from sklearn.datasets import make_moons\n",
|
||||
"\n",
|
||||
"X, y = make_moons(n_samples=500, noise=0.30, random_state=42)\n",
|
||||
"X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42)\n",
|
||||
"from sklearn.ensemble import RandomForestClassifier\n",
|
||||
"from sklearn.ensemble import VotingClassifier\n",
|
||||
"from sklearn.linear_model import LogisticRegression\n",
|
||||
"from sklearn.svm import SVC\n",
|
||||
"\n",
|
||||
"log_clf = LogisticRegression(random_state=42)\n",
|
||||
"rnd_clf = RandomForestClassifier(random_state=42)\n",
|
||||
"svm_clf = SVC(random_state=42)\n",
|
||||
"\n",
|
||||
"voting_clf = VotingClassifier(\n",
|
||||
" estimators=[('lr', log_clf), ('rf', rnd_clf), ('svc', svm_clf)],\n",
|
||||
" voting='hard')\n",
|
||||
"voting_clf.fit(X_train, y_train)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 18,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from sklearn.metrics import accuracy_score\n",
|
||||
"\n",
|
||||
"for clf in (log_clf, rnd_clf, svm_clf, voting_clf):\n",
|
||||
" clf.fit(X_train, y_train)\n",
|
||||
" y_pred = clf.predict(X_test)\n",
|
||||
" print(clf.__class__.__name__, accuracy_score(y_test, y_pred))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 19,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"log_clf = LogisticRegression(random_state=42)\n",
|
||||
"rnd_clf = RandomForestClassifier(random_state=42)\n",
|
||||
"svm_clf = SVC(probability=True, random_state=42)\n",
|
||||
"\n",
|
||||
"voting_clf = VotingClassifier(\n",
|
||||
" estimators=[('lr', log_clf), ('rf', rnd_clf), ('svc', svm_clf)],\n",
|
||||
" voting='soft')\n",
|
||||
"voting_clf.fit(X_train, y_train)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 20,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from sklearn.metrics import accuracy_score\n",
|
||||
"\n",
|
||||
"for clf in (log_clf, rnd_clf, svm_clf, voting_clf):\n",
|
||||
" clf.fit(X_train, y_train)\n",
|
||||
" y_pred = clf.predict(X_test)\n",
|
||||
" print(clf.__class__.__name__, accuracy_score(y_test, y_pred))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Bagging Examples"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 21,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from sklearn.ensemble import BaggingClassifier\n",
|
||||
"from sklearn.tree import DecisionTreeClassifier\n",
|
||||
"\n",
|
||||
"bag_clf = BaggingClassifier(\n",
|
||||
" DecisionTreeClassifier(random_state=42), n_estimators=500,\n",
|
||||
" max_samples=100, bootstrap=True, n_jobs=-1, random_state=42)\n",
|
||||
"bag_clf.fit(X_train, y_train)\n",
|
||||
"y_pred = bag_clf.predict(X_test)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 22,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from sklearn.metrics import accuracy_score\n",
|
||||
"print(accuracy_score(y_test, y_pred))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 23,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"tree_clf = DecisionTreeClassifier(random_state=42)\n",
|
||||
"tree_clf.fit(X_train, y_train)\n",
|
||||
"y_pred_tree = tree_clf.predict(X_test)\n",
|
||||
"print(accuracy_score(y_test, y_pred_tree))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 24,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from matplotlib.colors import ListedColormap\n",
|
||||
"\n",
|
||||
"def plot_decision_boundary(clf, X, y, axes=[-1.5, 2.5, -1, 1.5], alpha=0.5, contour=True):\n",
|
||||
" x1s = np.linspace(axes[0], axes[1], 100)\n",
|
||||
" x2s = np.linspace(axes[2], axes[3], 100)\n",
|
||||
" x1, x2 = np.meshgrid(x1s, x2s)\n",
|
||||
" X_new = np.c_[x1.ravel(), x2.ravel()]\n",
|
||||
" y_pred = clf.predict(X_new).reshape(x1.shape)\n",
|
||||
" custom_cmap = ListedColormap(['#fafab0','#9898ff','#a0faa0'])\n",
|
||||
" plt.contourf(x1, x2, y_pred, alpha=0.3, cmap=custom_cmap)\n",
|
||||
" if contour:\n",
|
||||
" custom_cmap2 = ListedColormap(['#7d7d58','#4c4c7f','#507d50'])\n",
|
||||
" plt.contour(x1, x2, y_pred, cmap=custom_cmap2, alpha=0.8)\n",
|
||||
" plt.plot(X[:, 0][y==0], X[:, 1][y==0], \"yo\", alpha=alpha)\n",
|
||||
" plt.plot(X[:, 0][y==1], X[:, 1][y==1], \"bs\", alpha=alpha)\n",
|
||||
" plt.axis(axes)\n",
|
||||
" plt.xlabel(r\"$x_1$\", fontsize=18)\n",
|
||||
" plt.ylabel(r\"$x_2$\", fontsize=18, rotation=0)\n",
|
||||
"plt.figure(figsize=(11,4))\n",
|
||||
"plt.subplot(121)\n",
|
||||
"plot_decision_boundary(tree_clf, X, y)\n",
|
||||
"plt.title(\"Decision Tree\", fontsize=14)\n",
|
||||
"plt.subplot(122)\n",
|
||||
"plot_decision_boundary(bag_clf, X, y)\n",
|
||||
"plt.title(\"Decision Trees with Bagging\", fontsize=14)\n",
|
||||
"save_fig(\"baggingtree\")\n",
|
||||
"plt.show()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Making your own Bootstrap: Changing the Level of the Decision Tree\n",
|
||||
"\n",
|
||||
"Let us bring up our good old boostrap example from the linear regression lectures. We change the linerar regression algorithm with\n",
|
||||
@@ -1728,7 +1467,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 25,
|
||||
"execution_count": 15,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
@@ -1742,9 +1481,9 @@
|
||||
"from sklearn.utils import resample\n",
|
||||
"from sklearn.tree import DecisionTreeRegressor\n",
|
||||
"\n",
|
||||
"n = 100\n",
|
||||
"n = 1000\n",
|
||||
"n_boostraps = 100\n",
|
||||
"maxdepth = 8\n",
|
||||
"maxdepth = 10\n",
|
||||
"\n",
|
||||
"# Make data set.\n",
|
||||
"x = np.linspace(-3, 3, n).reshape(-1, 1)\n",
|
||||
@@ -1755,23 +1494,17 @@
|
||||
"polydegree = np.zeros(maxdepth)\n",
|
||||
"X_train, X_test, y_train, y_test = train_test_split(x, y, test_size=0.2)\n",
|
||||
"\n",
|
||||
"from sklearn.preprocessing import StandardScaler\n",
|
||||
"scaler = StandardScaler()\n",
|
||||
"scaler.fit(X_train)\n",
|
||||
"X_train_scaled = scaler.transform(X_train)\n",
|
||||
"X_test_scaled = scaler.transform(X_test)\n",
|
||||
"\n",
|
||||
"# we produce a simple tree first as benchmark\n",
|
||||
"# we produce a simple tree first as benchmark, no scaling\n",
|
||||
"simpletree = DecisionTreeRegressor(max_depth=3) \n",
|
||||
"simpletree.fit(X_train_scaled, y_train)\n",
|
||||
"simpleprediction = simpletree.predict(X_test_scaled)\n",
|
||||
"simpletree.fit(X_train, y_train)\n",
|
||||
"simpleprediction = simpletree.predict(X_test)\n",
|
||||
"for degree in range(1,maxdepth):\n",
|
||||
" model = DecisionTreeRegressor(max_depth=degree) \n",
|
||||
" y_pred = np.empty((y_test.shape[0], n_boostraps))\n",
|
||||
" for i in range(n_boostraps):\n",
|
||||
" x_, y_ = resample(X_train_scaled, y_train)\n",
|
||||
" x_, y_ = resample(X_train, y_train)\n",
|
||||
" model.fit(x_, y_)\n",
|
||||
" y_pred[:, i] = model.predict(X_test_scaled)#.ravel()\n",
|
||||
" y_pred[:, i] = model.predict(X_test)#.ravel()\n",
|
||||
"\n",
|
||||
" polydegree[degree] = degree\n",
|
||||
" error[degree] = np.mean( np.mean((y_test - y_pred)**2, axis=1, keepdims=True) )\n",
|
||||
@@ -1783,7 +1516,7 @@
|
||||
" print('Var:', variance[degree])\n",
|
||||
" print('{} >= {} + {} = {}'.format(error[degree], bias[degree], variance[degree], bias[degree]+variance[degree]))\n",
|
||||
" \n",
|
||||
"mse_simpletree= np.mean( np.mean((y_test - simpleprediction)**2)\n",
|
||||
"mse_simpletree= np.mean( np.mean((y_test - simpleprediction)**2))\n",
|
||||
"print(mse_simpletree)\n",
|
||||
"plt.xlim(1,maxdepth)\n",
|
||||
"plt.plot(polydegree, error, label='MSE')\n",
|
||||
@@ -1842,7 +1575,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 26,
|
||||
"execution_count": 16,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
@@ -1896,7 +1629,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 27,
|
||||
"execution_count": 17,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
@@ -1938,7 +1671,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 28,
|
||||
"execution_count": 18,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
@@ -1997,7 +1730,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 29,
|
||||
"execution_count": 19,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
@@ -2025,7 +1758,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 30,
|
||||
"execution_count": 20,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
@@ -2041,7 +1774,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 31,
|
||||
"execution_count": 21,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
@@ -2059,7 +1792,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 32,
|
||||
"execution_count": 22,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
@@ -2149,7 +1882,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 33,
|
||||
"execution_count": 23,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
@@ -2241,7 +1974,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 34,
|
||||
"execution_count": 24,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
@@ -2254,7 +1987,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 35,
|
||||
"execution_count": 25,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
@@ -2798,7 +2531,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 36,
|
||||
"execution_count": 26,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
@@ -2988,7 +2721,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 37,
|
||||
"execution_count": 27,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
@@ -3051,7 +2784,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 38,
|
||||
"execution_count": 28,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
@@ -3124,7 +2857,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 39,
|
||||
"execution_count": 29,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
@@ -3189,7 +2922,7 @@
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 40,
|
||||
"execution_count": 30,
|
||||
"metadata": {
|
||||
"collapsed": false
|
||||
},
|
||||
|
||||
Reference in New Issue
Block a user