{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Chapter 9: Classification and Regression Trees\n",
    "\n",
    "> (c) 2019 Galit Shmueli, Peter C. Bruce, Peter Gedeck \n",
    ">\n",
    "> Code included in\n",
    ">\n",
    "> _Data Mining for Business Analytics: Concepts, Techniques, and Applications in Python_ (First Edition) \n",
    "> Galit Shmueli, Peter C. Bruce, Peter Gedeck, and Nitin R. Patel. 2019.\n",
    "\n",
    "## Import required packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-05T23:03:50.053106Z",
     "iopub.status.busy": "2022-02-05T23:03:50.052584Z",
     "iopub.status.idle": "2022-02-05T23:03:52.036351Z",
     "shell.execute_reply": "2022-02-05T23:03:52.036728Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "no display found. Using non-interactive Agg backend\n"
     ]
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "from pathlib import Path\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor\n",
    "from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier\n",
    "from sklearn.model_selection import train_test_split, cross_val_score, GridSearchCV\n",
    "import matplotlib.pylab as plt\n",
    "import dmba\n",
    "from dmba import plotDecisionTree, classificationSummary, regressionSummary\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Figure 9.7"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-05T23:03:52.041303Z",
     "iopub.status.busy": "2022-02-05T23:03:52.040891Z",
     "iopub.status.idle": "2022-02-05T23:03:52.136152Z",
     "shell.execute_reply": "2022-02-05T23:03:52.136471Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Classes: Nonowner, Owner\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 23108 characters]\n",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mower_df = dmba.load_data('RidingMowers.csv')\n",
    "\n",
    "classTree = DecisionTreeClassifier(random_state=0, max_depth=1)\n",
    "classTree.fit(mower_df.drop(columns=['Ownership']), mower_df['Ownership'])\n",
    "\n",
    "print(\"Classes: {}\".format(', '.join(classTree.classes_)))\n",
    "plotDecisionTree(classTree, feature_names=mower_df.columns[:2], class_names=classTree.classes_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The order of the `values` vector in the boxes is the same as `classTree.classes_`."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Figure 9.8\n",
    "Grow tree fully"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-05T23:03:52.145329Z",
     "iopub.status.busy": "2022-02-05T23:03:52.144931Z",
     "iopub.status.idle": "2022-02-05T23:03:52.333491Z",
     "shell.execute_reply": "2022-02-05T23:03:52.333831Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Classes: Nonowner, Owner\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 92276 characters]\n",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "classTree = DecisionTreeClassifier(random_state=0)\n",
    "classTree.fit(mower_df.drop(columns=['Ownership']), mower_df['Ownership'])\n",
    "\n",
    "print(\"Classes: {}\".format(', '.join(classTree.classes_)))\n",
    "plotDecisionTree(classTree, feature_names=mower_df.columns[:2], class_names=classTree.classes_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Figure 9.10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-05T23:03:52.340477Z",
     "iopub.status.busy": "2022-02-05T23:03:52.340071Z",
     "iopub.status.idle": "2022-02-05T23:03:53.307989Z",
     "shell.execute_reply": "2022-02-05T23:03:53.308279Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 455844 characters]\n",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bank_df = dmba.load_data('UniversalBank.csv')\n",
    "bank_df = bank_df.drop(columns=['ID', 'ZIP Code'])\n",
    "\n",
    "X = bank_df.drop(columns=['Personal Loan'])\n",
    "y = bank_df['Personal Loan']\n",
    "train_X, valid_X, train_y, valid_y = train_test_split(X, y, test_size=0.4, random_state=1)\n",
    "\n",
    "fullClassTree = DecisionTreeClassifier()\n",
    "fullClassTree.fit(train_X, train_y)\n",
    "\n",
    "plotDecisionTree(fullClassTree, feature_names=train_X.columns)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Table 9.3\n",
    "<!-- <div class='alert alert-danger'>R output for reference - remove once missing metrics are available.\n",
    "```\n",
    "> confusionMatrix(default.ct.point.pred.train, train.df$Personal.Loan)\n",
    "Confusion Matrix and Statistics\n",
    "\n",
    "          Reference\n",
    "Prediction    0    1\n",
    "         0 2696   26\n",
    "         1   13  265\n",
    "                                          \n",
    "               Accuracy : 0.987           \n",
    "                 95% CI : (0.9823, 0.9907)\n",
    "    No Information Rate : 0.903           \n",
    "    P-Value [Acc > NIR] : < 2e-16         \n",
    "                                          \n",
    "                  Kappa : 0.9243          \n",
    " Mcnemar's Test P-Value : 0.05466         \n",
    "                                          \n",
    "            Sensitivity : 0.9952          \n",
    "            Specificity : 0.9107          \n",
    "         Pos Pred Value : 0.9904          \n",
    "         Neg Pred Value : 0.9532          \n",
    "             Prevalence : 0.9030          \n",
    "         Detection Rate : 0.8987          \n",
    "   Detection Prevalence : 0.9073          \n",
    "      Balanced Accuracy : 0.9529          \n",
    "                                          \n",
    "       'Positive' Class : 0               \n",
    "```\n",
    "</div>-->"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-05T23:03:53.310673Z",
     "iopub.status.busy": "2022-02-05T23:03:53.310102Z",
     "iopub.status.idle": "2022-02-05T23:03:53.320053Z",
     "shell.execute_reply": "2022-02-05T23:03:53.320342Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Confusion Matrix (Accuracy 1.0000)\n",
      "\n",
      "       Prediction\n",
      "Actual    0    1\n",
      "     0 2713    0\n",
      "     1    0  287\n",
      "Confusion Matrix (Accuracy 0.9800)\n",
      "\n",
      "       Prediction\n",
      "Actual    0    1\n",
      "     0 1791   16\n",
      "     1   24  169\n"
     ]
    }
   ],
   "source": [
    "classificationSummary(train_y, fullClassTree.predict(train_X))\n",
    "classificationSummary(valid_y, fullClassTree.predict(valid_X))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Table 9.4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-05T23:03:53.323041Z",
     "iopub.status.busy": "2022-02-05T23:03:53.322657Z",
     "iopub.status.idle": "2022-02-05T23:03:53.353825Z",
     "shell.execute_reply": "2022-02-05T23:03:53.353400Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy scores of each fold:  ['0.983', '0.972', '0.992', '0.985', '0.992']\n",
      "Accuracy: 0.985 (+/- 0.015)\n",
      "Accuracy: 0.985 (+/- 0.007)\n"
     ]
    }
   ],
   "source": [
    "# Five-fold cross-validation of the full decision tree classifier\n",
    "treeClassifier = DecisionTreeClassifier()\n",
    "\n",
    "scores = cross_val_score(treeClassifier, train_X, train_y, cv=5)\n",
    "print('Accuracy scores of each fold: ', [f'{acc:.3f}' for acc in scores])\n",
    "print(f'Accuracy: {scores.mean():.3f} (+/- {scores.std() * 2:.3f})')\n",
    "print(f'Accuracy: {scores.mean():.3f} (+/- {scores.std():.3f})')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Figure 9.12"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-05T23:03:53.359134Z",
     "iopub.status.busy": "2022-02-05T23:03:53.358736Z",
     "iopub.status.idle": "2022-02-05T23:03:53.480558Z",
     "shell.execute_reply": "2022-02-05T23:03:53.480240Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 59236 characters]\n",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "smallClassTree = DecisionTreeClassifier(max_depth=30, min_samples_split=20, min_impurity_decrease=0.01)\n",
    "smallClassTree.fit(train_X, train_y)\n",
    "\n",
    "plotDecisionTree(smallClassTree, feature_names=train_X.columns)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Table 9.5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-05T23:03:53.486400Z",
     "iopub.status.busy": "2022-02-05T23:03:53.486018Z",
     "iopub.status.idle": "2022-02-05T23:03:53.496324Z",
     "shell.execute_reply": "2022-02-05T23:03:53.496698Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Confusion Matrix (Accuracy 0.9823)\n",
      "\n",
      "       Prediction\n",
      "Actual    0    1\n",
      "     0 2711    2\n",
      "     1   51  236\n",
      "Confusion Matrix (Accuracy 0.9770)\n",
      "\n",
      "       Prediction\n",
      "Actual    0    1\n",
      "     0 1804    3\n",
      "     1   43  150\n"
     ]
    }
   ],
   "source": [
    "classificationSummary(train_y, smallClassTree.predict(train_X))\n",
    "classificationSummary(valid_y, smallClassTree.predict(valid_X))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Grid search"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Table 9.6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-05T23:03:53.504462Z",
     "iopub.status.busy": "2022-02-05T23:03:53.504063Z",
     "iopub.status.idle": "2022-02-05T23:03:57.010321Z",
     "shell.execute_reply": "2022-02-05T23:03:57.009989Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initial score:  0.9876666666666667\n",
      "Initial parameters:  {'max_depth': 20, 'min_impurity_decrease': 0, 'min_samples_split': 20}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Improved score:  0.9873333333333333\n",
      "Improved parameters:  {'max_depth': 4, 'min_impurity_decrease': 0.0011, 'min_samples_split': 13}\n"
     ]
    }
   ],
   "source": [
    "# Start with an initial guess for parameters\n",
    "param_grid = {\n",
    "    'max_depth': [10, 20, 30, 40], \n",
    "    'min_samples_split': [20, 40, 60, 80, 100], \n",
    "    'min_impurity_decrease': [0, 0.0005, 0.001, 0.005, 0.01], \n",
    "}\n",
    "gridSearch = GridSearchCV(DecisionTreeClassifier(), param_grid, cv=5, n_jobs=-1)\n",
    "gridSearch.fit(train_X, train_y)\n",
    "print('Initial score: ', gridSearch.best_score_)\n",
    "print('Initial parameters: ', gridSearch.best_params_)\n",
    "\n",
    "# Adapt grid based on result from initial grid search\n",
    "param_grid = {\n",
    "    'max_depth': list(range(2, 16)), \n",
    "    'min_samples_split': list(range(10, 22)), \n",
    "    'min_impurity_decrease': [0.0009, 0.001, 0.0011], \n",
    "}\n",
    "gridSearch = GridSearchCV(DecisionTreeClassifier(), param_grid, cv=5, n_jobs=-1)\n",
    "gridSearch.fit(train_X, train_y)\n",
    "print('Improved score: ', gridSearch.best_score_)\n",
    "print('Improved parameters: ', gridSearch.best_params_)\n",
    "\n",
    "bestClassTree = gridSearch.best_estimator_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-05T23:03:57.022405Z",
     "iopub.status.busy": "2022-02-05T23:03:57.014995Z",
     "iopub.status.idle": "2022-02-05T23:03:57.273833Z",
     "shell.execute_reply": "2022-02-05T23:03:57.273473Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 127724 characters]\n",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "plotDecisionTree(bestClassTree, feature_names=train_X.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-05T23:03:57.279718Z",
     "iopub.status.busy": "2022-02-05T23:03:57.279293Z",
     "iopub.status.idle": "2022-02-05T23:03:57.287782Z",
     "shell.execute_reply": "2022-02-05T23:03:57.288164Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Confusion Matrix (Accuracy 0.9867)\n",
      "\n",
      "       Prediction\n",
      "Actual    0    1\n",
      "     0 2708    5\n",
      "     1   35  252\n",
      "Confusion Matrix (Accuracy 0.9815)\n",
      "\n",
      "       Prediction\n",
      "Actual    0    1\n",
      "     0 1801    6\n",
      "     1   31  162\n"
     ]
    }
   ],
   "source": [
    "classificationSummary(train_y, bestClassTree.predict(train_X))\n",
    "classificationSummary(valid_y, bestClassTree.predict(valid_X))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Figure 9.16"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-05T23:03:57.296009Z",
     "iopub.status.busy": "2022-02-05T23:03:57.295623Z",
     "iopub.status.idle": "2022-02-05T23:03:57.490748Z",
     "shell.execute_reply": "2022-02-05T23:03:57.490384Z"
    }
   },
   "outputs": [],
   "source": [
    "fakeScatter = pd.DataFrame([\n",
    "    (30, 19, 1), (35, 5, 1), (40, 25, 1), (45, 13, 1), \n",
    "    (38, 10, 1), (36, 23, 1), (48, 8, 1), (44, 20, 1), (50, 12),\n",
    "    (51, 19, 0), (62, 30, 0), (42, 29, 0), (45, 25, 0), \n",
    "    (55, 10, 0), (52, 27, 0), (58, 6, 0), (53, 11, 0), \n",
    "], columns=['x', 'y', 'category'])\n",
    "ax = fakeScatter.loc[fakeScatter.category==1,:].plot.scatter(x='x', y='y', color='C0')\n",
    "fakeScatter.loc[fakeScatter.category==0,:].plot.scatter(x='x', y='y', ax=ax, color='C1')\n",
    "plt.plot((40, 54), (30, 5), ':', color='grey')\n",
    "ax.set_xlabel('Income ($000s)')\n",
    "ax.set_ylabel('Education')\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-05T23:03:57.508586Z",
     "iopub.status.busy": "2022-02-05T23:03:57.506409Z",
     "iopub.status.idle": "2022-02-05T23:03:57.619516Z",
     "shell.execute_reply": "2022-02-05T23:03:57.619188Z"
    }
   },
   "outputs": [],
   "source": [
    "def basePlot(ax):\n",
    "    mower_df.loc[mower_df.Ownership=='Owner'].plot(x='Income', y='Lot_Size', style='o', \n",
    "                                                   markerfacecolor='C0', markeredgecolor='C0',\n",
    "                                                   ax=ax)\n",
    "    mower_df.loc[mower_df.Ownership=='Nonowner'].plot(x='Income', y='Lot_Size', style='o',\n",
    "                                                      markerfacecolor='none', markeredgecolor='C1',\n",
    "                                                      ax=ax)\n",
    "    ax.legend([\"Owner\", \"Nonowner\"]);\n",
    "    ax.set_xlim(20, 120)\n",
    "    ax.set_ylim(13, 25)\n",
    "    ax.set_xlabel('Income ($000s)')\n",
    "    ax.set_ylabel('Lot Size (000s sqft)')\n",
    "    return ax\n",
    "\n",
    "fig, ax = plt.subplots(figsize=(7, 4))\n",
    "\n",
    "ax = basePlot(ax)\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-05T23:03:57.637074Z",
     "iopub.status.busy": "2022-02-05T23:03:57.629908Z",
     "iopub.status.idle": "2022-02-05T23:03:57.740166Z",
     "shell.execute_reply": "2022-02-05T23:03:57.740447Z"
    }
   },
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(figsize=(7, 4))\n",
    "\n",
    "ax = basePlot(ax)\n",
    "x0 = 59.7\n",
    "ax.plot((x0, x0), (25, 13), color='grey')\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-05T23:03:57.753743Z",
     "iopub.status.busy": "2022-02-05T23:03:57.753318Z",
     "iopub.status.idle": "2022-02-05T23:03:57.830155Z",
     "shell.execute_reply": "2022-02-05T23:03:57.829765Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 23108 characters]\n",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "classTree = DecisionTreeClassifier(random_state=0, max_depth=1)\n",
    "classTree.fit(mower_df.drop(columns=['Ownership']), mower_df['Ownership'])\n",
    "plotDecisionTree(classTree, feature_names=mower_df.columns[:2], class_names=classTree.classes_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-05T23:03:57.839604Z",
     "iopub.status.busy": "2022-02-05T23:03:57.839183Z",
     "iopub.status.idle": "2022-02-05T23:03:57.972762Z",
     "shell.execute_reply": "2022-02-05T23:03:57.972382Z"
    }
   },
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(figsize=(7, 4))\n",
    "\n",
    "ax = basePlot(ax)\n",
    "x0 = 59.7\n",
    "y1 = 21.4\n",
    "ax.plot((x0, x0), (25, 13), color='grey')\n",
    "ax.plot((20, x0), (y1, y1), color='grey')\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-05T23:03:57.988734Z",
     "iopub.status.busy": "2022-02-05T23:03:57.981734Z",
     "iopub.status.idle": "2022-02-05T23:03:58.096734Z",
     "shell.execute_reply": "2022-02-05T23:03:58.096389Z"
    }
   },
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(figsize=(7, 4))\n",
    "\n",
    "ax = basePlot(ax)\n",
    "x0 = 59.7\n",
    "y1 = 21.4\n",
    "y2 = 19.8\n",
    "ax.plot((x0, x0), (25, 13), color='grey')\n",
    "ax.plot((20, x0), (y1, y1), color='grey')\n",
    "ax.plot((x0, 120), (y2, y2), color='grey')\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-05T23:03:58.105163Z",
     "iopub.status.busy": "2022-02-05T23:03:58.104776Z",
     "iopub.status.idle": "2022-02-05T23:03:58.214762Z",
     "shell.execute_reply": "2022-02-05T23:03:58.214432Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 56660 characters]\n",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "classTree = DecisionTreeClassifier(random_state=0, max_depth=2)\n",
    "classTree.fit(mower_df.drop(columns=['Ownership']), mower_df['Ownership'])\n",
    "plotDecisionTree(classTree, feature_names=mower_df.columns[:2], class_names=classTree.classes_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-05T23:03:58.220513Z",
     "iopub.status.busy": "2022-02-05T23:03:58.220111Z",
     "iopub.status.idle": "2022-02-05T23:03:58.346328Z",
     "shell.execute_reply": "2022-02-05T23:03:58.346602Z"
    }
   },
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(figsize=(7, 4))\n",
    "\n",
    "ax = basePlot(ax)\n",
    "x0 = 59.7\n",
    "y1 = 21.4\n",
    "y2 = 19.8\n",
    "x3 = 84.75\n",
    "x4 = 61.5\n",
    "ax.plot((x0, x0), (25, 13), color='grey')\n",
    "ax.plot((20, x0), (y1, y1), color='grey')\n",
    "ax.plot((x0, 120), (y2, y2), color='grey')\n",
    "ax.plot((x3, x3), (13, y2), color='grey')\n",
    "ax.plot((x4, x4), (13, y2), color='grey')\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-05T23:03:58.354162Z",
     "iopub.status.busy": "2022-02-05T23:03:58.353645Z",
     "iopub.status.idle": "2022-02-05T23:03:58.581446Z",
     "shell.execute_reply": "2022-02-05T23:03:58.581778Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 92276 characters]\n",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "classTree = DecisionTreeClassifier(random_state=0)\n",
    "classTree.fit(mower_df.drop(columns=['Ownership']), mower_df['Ownership'])\n",
    "plotDecisionTree(classTree, feature_names=mower_df.columns[:2], class_names=classTree.classes_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Information about full decision tree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-05T23:03:58.590102Z",
     "iopub.status.busy": "2022-02-05T23:03:58.587091Z",
     "iopub.status.idle": "2022-02-05T23:03:59.251181Z",
     "shell.execute_reply": "2022-02-05T23:03:59.251442Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of nodes 85\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 425836 characters]\n",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tree = fullClassTree\n",
    "print('Number of nodes', tree.tree_.node_count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-05T23:03:59.260970Z",
     "iopub.status.busy": "2022-02-05T23:03:59.260320Z",
     "iopub.status.idle": "2022-02-05T23:03:59.263086Z",
     "shell.execute_reply": "2022-02-05T23:03:59.262767Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Number of terminal leaves 43\n",
      "Counter({0: 24, 1: 19})\n"
     ]
    }
   ],
   "source": [
    "estimator = tree\n",
    "# Using those arrays, we can parse the tree structure:\n",
    "\n",
    "n_nodes = estimator.tree_.node_count\n",
    "children_left = estimator.tree_.children_left\n",
    "children_right = estimator.tree_.children_right\n",
    "feature = estimator.tree_.feature\n",
    "threshold = estimator.tree_.threshold\n",
    "value = estimator.tree_.value\n",
    "\n",
    "\n",
    "# The tree structure can be traversed to compute various properties such\n",
    "# as the depth of each node and whether or not it is a leaf.\n",
    "node_depth = np.zeros(shape=n_nodes, dtype=np.int64)\n",
    "is_leaves = np.zeros(shape=n_nodes, dtype=bool)\n",
    "stack = [(0, -1)]  # seed is the root node id and its parent depth\n",
    "while len(stack) > 0:\n",
    "    node_id, parent_depth = stack.pop()\n",
    "    node_depth[node_id] = parent_depth + 1\n",
    "\n",
    "    # If we have a test node\n",
    "    if (children_left[node_id] != children_right[node_id]):\n",
    "        stack.append((children_left[node_id], parent_depth + 1))\n",
    "        stack.append((children_right[node_id], parent_depth + 1))\n",
    "    else:\n",
    "        is_leaves[node_id] = True\n",
    "\n",
    "from collections import Counter\n",
    "nodeClassCounter = Counter()\n",
    "terminal_leaves = 0\n",
    "for i in range(n_nodes):\n",
    "    if is_leaves[i]:\n",
    "        terminal_leaves = terminal_leaves + 1\n",
    "        nodeClassCounter.update([np.argmax(value[i][0])])\n",
    "print()\n",
    "print('Number of terminal leaves', terminal_leaves)\n",
    "print(nodeClassCounter)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Regression Trees : Table 9.7 and Figure 9.14"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-05T23:03:59.268467Z",
     "iopub.status.busy": "2022-02-05T23:03:59.264761Z",
     "iopub.status.idle": "2022-02-05T23:04:00.857698Z",
     "shell.execute_reply": "2022-02-05T23:04:00.857347Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initial parameters:  {'max_depth': 5, 'min_impurity_decrease': 0, 'min_samples_split': 20}\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Improved parameters:  {'max_depth': 6, 'min_impurity_decrease': 0, 'min_samples_split': 16}\n"
     ]
    }
   ],
   "source": [
    "toyotaCorolla_df = dmba.load_data('ToyotaCorolla.csv').iloc[:1000,:]\n",
    "toyotaCorolla_df = toyotaCorolla_df.rename(columns={'Age_08_04': 'Age', 'Quarterly_Tax': 'Tax'})\n",
    "\n",
    "predictors = ['Age', 'KM', 'Fuel_Type', 'HP', 'Met_Color', 'Automatic', 'CC', \n",
    "              'Doors', 'Tax', 'Weight']\n",
    "outcome = 'Price'\n",
    "\n",
    "X = pd.get_dummies(toyotaCorolla_df[predictors], drop_first=True)\n",
    "y = toyotaCorolla_df[outcome]\n",
    "\n",
    "train_X, valid_X, train_y, valid_y = train_test_split(X, y, test_size=0.4, random_state=1)\n",
    "\n",
    "# user grid search to find optimized tree\n",
    "param_grid = {\n",
    "    'max_depth': [5, 10, 15, 20, 25], \n",
    "    'min_impurity_decrease': [0, 0.001, 0.005, 0.01], \n",
    "    'min_samples_split': [10, 20, 30, 40, 50], \n",
    "}\n",
    "gridSearch = GridSearchCV(DecisionTreeRegressor(), param_grid, cv=5, n_jobs=-1)\n",
    "gridSearch.fit(train_X, train_y)\n",
    "print('Initial parameters: ', gridSearch.best_params_)\n",
    "\n",
    "param_grid = {\n",
    "    'max_depth': [3, 4, 5, 6, 7, 8, 9, 10, 11, 12], \n",
    "    'min_impurity_decrease': [0, 0.001, 0.002, 0.003, 0.005, 0.006, 0.007, 0.008], \n",
    "    'min_samples_split': [14, 15, 16, 18, 20, ], \n",
    "}\n",
    "gridSearch = GridSearchCV(DecisionTreeRegressor(), param_grid, cv=5, n_jobs=-1)\n",
    "gridSearch.fit(train_X, train_y)\n",
    "print('Improved parameters: ', gridSearch.best_params_)\n",
    "\n",
    "regTree = gridSearch.best_estimator_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-05T23:04:00.862179Z",
     "iopub.status.busy": "2022-02-05T23:04:00.861787Z",
     "iopub.status.idle": "2022-02-05T23:04:00.868277Z",
     "shell.execute_reply": "2022-02-05T23:04:00.867976Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Regression statistics\n",
      "\n",
      "                      Mean Error (ME) : 0.0000\n",
      "       Root Mean Squared Error (RMSE) : 1058.8202\n",
      "            Mean Absolute Error (MAE) : 767.7203\n",
      "          Mean Percentage Error (MPE) : -0.8074\n",
      "Mean Absolute Percentage Error (MAPE) : 6.8325\n",
      "\n",
      "Regression statistics\n",
      "\n",
      "                      Mean Error (ME) : 60.5241\n",
      "       Root Mean Squared Error (RMSE) : 1554.9146\n",
      "            Mean Absolute Error (MAE) : 1026.3487\n",
      "          Mean Percentage Error (MPE) : -1.3082\n",
      "Mean Absolute Percentage Error (MAPE) : 9.2311\n"
     ]
    }
   ],
   "source": [
    "regressionSummary(train_y, regTree.predict(train_X))\n",
    "regressionSummary(valid_y, regTree.predict(valid_X))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-05T23:04:00.901943Z",
     "iopub.status.busy": "2022-02-05T23:04:00.870072Z",
     "iopub.status.idle": "2022-02-05T23:04:02.167757Z",
     "shell.execute_reply": "2022-02-05T23:04:02.168018Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 535160 characters]\n",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "plotDecisionTree(regTree, feature_names=train_X.columns)\n",
    "plotDecisionTree(regTree, feature_names=train_X.columns, rotate=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Figure 9.15 (no longer used)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-05T23:04:02.173269Z",
     "iopub.status.busy": "2022-02-05T23:04:02.172873Z",
     "iopub.status.idle": "2022-02-05T23:04:03.099555Z",
     "shell.execute_reply": "2022-02-05T23:04:03.099204Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestClassifier(n_estimators=500, random_state=1)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bank_df = dmba.load_data('UniversalBank.csv')\n",
    "bank_df = bank_df.drop(columns=['ID', 'ZIP Code'])\n",
    "\n",
    "X = bank_df.drop(columns=['Personal Loan'])\n",
    "y = bank_df['Personal Loan']\n",
    "train_X, valid_X, train_y, valid_y = train_test_split(X, y, test_size=0.4, random_state=1)\n",
    "\n",
    "rf = RandomForestClassifier(n_estimators=500, random_state=1)\n",
    "rf.fit(train_X, train_y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Variable importance plot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-05T23:04:03.157226Z",
     "iopub.status.busy": "2022-02-05T23:04:03.156843Z",
     "iopub.status.idle": "2022-02-05T23:04:03.280388Z",
     "shell.execute_reply": "2022-02-05T23:04:03.280056Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "               feature  importance       std\n",
      "7   Securities Account    0.003964  0.004998\n",
      "9               Online    0.006394  0.005350\n",
      "10          CreditCard    0.007678  0.007053\n",
      "6             Mortgage    0.034243  0.023469\n",
      "1           Experience    0.035539  0.016061\n",
      "0                  Age    0.036258  0.015858\n",
      "8           CD Account    0.057917  0.043185\n",
      "3               Family    0.111375  0.053146\n",
      "4                CCAvg    0.172105  0.103011\n",
      "5            Education    0.200772  0.101002\n",
      "2               Income    0.333756  0.129227\n"
     ]
    }
   ],
   "source": [
    "importances = rf.feature_importances_\n",
    "std = np.std([tree.feature_importances_ for tree in rf.estimators_], axis=0)\n",
    "\n",
    "df = pd.DataFrame({'feature': train_X.columns, 'importance': importances, 'std': std})\n",
    "df = df.sort_values('importance')\n",
    "print(df)\n",
    "\n",
    "ax = df.plot(kind='barh', xerr='std', x='feature', legend=False)\n",
    "ax.set_ylabel('')\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Confusion matrix and metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-05T23:04:03.285080Z",
     "iopub.status.busy": "2022-02-05T23:04:03.284731Z",
     "iopub.status.idle": "2022-02-05T23:04:03.382525Z",
     "shell.execute_reply": "2022-02-05T23:04:03.382185Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Confusion Matrix (Accuracy 0.9820)\n",
      "\n",
      "       Prediction\n",
      "Actual    0    1\n",
      "     0 1803    4\n",
      "     1   32  161\n"
     ]
    }
   ],
   "source": [
    "classificationSummary(valid_y, rf.predict(valid_X))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Table 9.5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-02-05T23:04:03.386898Z",
     "iopub.status.busy": "2022-02-05T23:04:03.386567Z",
     "iopub.status.idle": "2022-02-05T23:04:03.615077Z",
     "shell.execute_reply": "2022-02-05T23:04:03.614350Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Confusion Matrix (Accuracy 0.9835)\n",
      "\n",
      "       Prediction\n",
      "Actual    0    1\n",
      "     0 1799    8\n",
      "     1   25  168\n"
     ]
    }
   ],
   "source": [
    "boost = GradientBoostingClassifier()\n",
    "boost.fit(train_X, train_y)\n",
    "classificationSummary(valid_y, boost.predict(valid_X))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
