{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Chapter 8: The Naive Bayes Classifier\n",
    "\n",
    "> (c) 2019 Galit Shmueli, Peter C. Bruce, Peter Gedeck \n",
    ">\n",
    "> Code included in\n",
    ">\n",
    "> _Data Mining for Business Analytics: Concepts, Techniques, and Applications in Python_ (First Edition) \n",
    "> Galit Shmueli, Peter C. Bruce, Peter Gedeck, and Nitin R. Patel. 2019.\n",
    "\n",
    "## Import required packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-27T20:03:07.684323Z",
     "iopub.status.busy": "2021-05-27T20:03:07.683817Z",
     "iopub.status.idle": "2021-05-27T20:03:08.485055Z",
     "shell.execute_reply": "2021-05-27T20:03:08.484503Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "no display found. Using non-interactive Agg backend\n"
     ]
    }
   ],
   "source": [
    "from pathlib import Path\n",
    "\n",
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "import matplotlib.pylab as plt\n",
    "\n",
    "import dmba\n",
    "from dmba import classificationSummary, gainsChart\n",
    "\n",
    "%matplotlib inline\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Table 8.4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-27T20:03:08.488861Z",
     "iopub.status.busy": "2021-05-27T20:03:08.488421Z",
     "iopub.status.idle": "2021-05-27T20:03:08.509636Z",
     "shell.execute_reply": "2021-05-27T20:03:08.509192Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>CRS_DEP_TIME</th>\n",
       "      <th>CARRIER</th>\n",
       "      <th>DEP_TIME</th>\n",
       "      <th>DEST</th>\n",
       "      <th>DISTANCE</th>\n",
       "      <th>FL_DATE</th>\n",
       "      <th>FL_NUM</th>\n",
       "      <th>ORIGIN</th>\n",
       "      <th>Weather</th>\n",
       "      <th>DAY_WEEK</th>\n",
       "      <th>DAY_OF_MONTH</th>\n",
       "      <th>TAIL_NUM</th>\n",
       "      <th>Flight Status</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1455</td>\n",
       "      <td>OH</td>\n",
       "      <td>1455</td>\n",
       "      <td>JFK</td>\n",
       "      <td>184</td>\n",
       "      <td>01/01/2004</td>\n",
       "      <td>5935</td>\n",
       "      <td>BWI</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>N940CA</td>\n",
       "      <td>ontime</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1640</td>\n",
       "      <td>DH</td>\n",
       "      <td>1640</td>\n",
       "      <td>JFK</td>\n",
       "      <td>213</td>\n",
       "      <td>01/01/2004</td>\n",
       "      <td>6155</td>\n",
       "      <td>DCA</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>N405FJ</td>\n",
       "      <td>ontime</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1245</td>\n",
       "      <td>DH</td>\n",
       "      <td>1245</td>\n",
       "      <td>LGA</td>\n",
       "      <td>229</td>\n",
       "      <td>01/01/2004</td>\n",
       "      <td>7208</td>\n",
       "      <td>IAD</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>N695BR</td>\n",
       "      <td>ontime</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1715</td>\n",
       "      <td>DH</td>\n",
       "      <td>1709</td>\n",
       "      <td>LGA</td>\n",
       "      <td>229</td>\n",
       "      <td>01/01/2004</td>\n",
       "      <td>7215</td>\n",
       "      <td>IAD</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>N662BR</td>\n",
       "      <td>ontime</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1039</td>\n",
       "      <td>DH</td>\n",
       "      <td>1035</td>\n",
       "      <td>LGA</td>\n",
       "      <td>229</td>\n",
       "      <td>01/01/2004</td>\n",
       "      <td>7792</td>\n",
       "      <td>IAD</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>N698BR</td>\n",
       "      <td>ontime</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   CRS_DEP_TIME CARRIER  DEP_TIME DEST  DISTANCE     FL_DATE  FL_NUM ORIGIN  \\\n",
       "0          1455      OH      1455  JFK       184  01/01/2004    5935    BWI   \n",
       "1          1640      DH      1640  JFK       213  01/01/2004    6155    DCA   \n",
       "2          1245      DH      1245  LGA       229  01/01/2004    7208    IAD   \n",
       "3          1715      DH      1709  LGA       229  01/01/2004    7215    IAD   \n",
       "4          1039      DH      1035  LGA       229  01/01/2004    7792    IAD   \n",
       "\n",
       "   Weather  DAY_WEEK  DAY_OF_MONTH TAIL_NUM Flight Status  \n",
       "0        0         4             1   N940CA        ontime  \n",
       "1        0         4             1   N405FJ        ontime  \n",
       "2        0         4             1   N695BR        ontime  \n",
       "3        0         4             1   N662BR        ontime  \n",
       "4        0         4             1   N698BR        ontime  "
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "delays_df = dmba.load_data('FlightDelays.csv')\n",
    "delays_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-27T20:03:08.514912Z",
     "iopub.status.busy": "2021-05-27T20:03:08.514477Z",
     "iopub.status.idle": "2021-05-27T20:03:08.544592Z",
     "shell.execute_reply": "2021-05-27T20:03:08.544192Z"
    }
   },
   "outputs": [],
   "source": [
    "delays_df = dmba.load_data('FlightDelays.csv')\n",
    "\n",
    "# convert to categorical\n",
    "delays_df.DAY_WEEK = delays_df.DAY_WEEK.astype('category')\n",
    "\n",
    "# create hourly bins departure time \n",
    "delays_df.CRS_DEP_TIME = [round(t / 100) for t in delays_df.CRS_DEP_TIME]\n",
    "delays_df.CRS_DEP_TIME = delays_df.CRS_DEP_TIME.astype('category')\n",
    "\n",
    "predictors = ['DAY_WEEK', 'CRS_DEP_TIME', 'ORIGIN', 'DEST', 'CARRIER']\n",
    "outcome = 'Flight Status'\n",
    "\n",
    "X = pd.get_dummies(delays_df[predictors])\n",
    "y = (delays_df[outcome] == 'delayed').astype(int)\n",
    "classes = ['ontime', 'delayed']\n",
    "\n",
    "# split into training and validation\n",
    "X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.40, random_state=1)\n",
    "\n",
    "# run naive Bayes\n",
    "delays_nb = MultinomialNB(alpha=0.01)\n",
    "delays_nb.fit(X_train, y_train)\n",
    "\n",
    "# predict probabilities\n",
    "predProb_train = delays_nb.predict_proba(X_train)\n",
    "predProb_valid = delays_nb.predict_proba(X_valid)\n",
    "\n",
    "# predict class membership\n",
    "y_valid_pred = delays_nb.predict(X_valid)\n",
    "y_train_pred = delays_nb.predict(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-27T20:03:08.550323Z",
     "iopub.status.busy": "2021-05-27T20:03:08.549856Z",
     "iopub.status.idle": "2021-05-27T20:03:08.580733Z",
     "shell.execute_reply": "2021-05-27T20:03:08.580279Z"
    }
   },
   "outputs": [],
   "source": [
    "delays_df = dmba.load_data('FlightDelays.csv')\n",
    "\n",
    "# convert to categorical\n",
    "delays_df.DAY_WEEK = delays_df.DAY_WEEK.astype('category')\n",
    "delays_df['Flight Status'] = delays_df['Flight Status'].astype('category')\n",
    "\n",
    "# create hourly bins departure time \n",
    "delays_df.CRS_DEP_TIME = [round(t / 100) for t in delays_df.CRS_DEP_TIME]\n",
    "delays_df.CRS_DEP_TIME = delays_df.CRS_DEP_TIME.astype('category')\n",
    "\n",
    "predictors = ['DAY_WEEK', 'CRS_DEP_TIME', 'ORIGIN', 'DEST', 'CARRIER']\n",
    "outcome = 'Flight Status'\n",
    "\n",
    "X = pd.get_dummies(delays_df[predictors])\n",
    "y = delays_df['Flight Status']\n",
    "classes = list(y.cat.categories)\n",
    "\n",
    "# split into training and validation\n",
    "X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.40, random_state=1)\n",
    "\n",
    "# run naive Bayes\n",
    "delays_nb = MultinomialNB(alpha=0.01)\n",
    "delays_nb.fit(X_train, y_train)\n",
    "\n",
    "# predict probabilities\n",
    "predProb_train = delays_nb.predict_proba(X_train)\n",
    "predProb_valid = delays_nb.predict_proba(X_valid)\n",
    "\n",
    "# predict class membership\n",
    "y_valid_pred = delays_nb.predict(X_valid)\n",
    "y_train_pred = delays_nb.predict(X_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Table 8.5\n",
    "First construct a frequency table and then convert it to the propability table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-27T20:03:08.585945Z",
     "iopub.status.busy": "2021-05-27T20:03:08.585512Z",
     "iopub.status.idle": "2021-05-27T20:03:08.654449Z",
     "shell.execute_reply": "2021-05-27T20:03:08.654031Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ontime     0.8023\n",
      "delayed    0.1977\n",
      "Name: Flight Status, dtype: float64\n",
      "\n",
      "DAY_WEEK            1       2       3       4       5      6       7\n",
      "Flight Status                                                       \n",
      "delayed        0.1916  0.1494  0.1149  0.1264  0.1877  0.069  0.1609\n",
      "ontime         0.1246  0.1416  0.1445  0.1794  0.1690  0.136  0.1048\n",
      "\n",
      "CRS_DEP_TIME        6       7       8       9      10      11      12      13  \\\n",
      "Flight Status                                                                   \n",
      "delayed        0.0345  0.0536  0.0651  0.0192  0.0307  0.0115  0.0498  0.0460   \n",
      "ontime         0.0623  0.0633  0.0850  0.0567  0.0519  0.0340  0.0661  0.0746   \n",
      "\n",
      "CRS_DEP_TIME       14      15      16      17      18      19      20      21  \n",
      "Flight Status                                                                  \n",
      "delayed        0.0383  0.2031  0.0728  0.1533  0.0192  0.0996  0.0153  0.0881  \n",
      "ontime         0.0576  0.1171  0.0774  0.1001  0.0349  0.0397  0.0264  0.0529  \n",
      "\n",
      "ORIGIN            BWI     DCA     IAD\n",
      "Flight Status                        \n",
      "delayed        0.0805  0.5211  0.3985\n",
      "ontime         0.0604  0.6478  0.2918\n",
      "\n",
      "DEST              EWR     JFK     LGA\n",
      "Flight Status                        \n",
      "delayed        0.3793  0.1992  0.4215\n",
      "ontime         0.2663  0.1558  0.5779\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CARRIER            CO      DH      DL      MQ      OH      RU      UA      US\n",
      "Flight Status                                                                \n",
      "delayed        0.0575  0.3142  0.0958  0.2222  0.0077  0.2184  0.0153  0.0690\n",
      "ontime         0.0349  0.2295  0.2040  0.1171  0.0104  0.1690  0.0170  0.2181\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# split the original data frame into a train and test using the same random_state\n",
    "train_df, valid_df = train_test_split(delays_df, test_size=0.4, random_state=1)\n",
    "\n",
    "pd.set_option('precision', 4)\n",
    "# probability of flight status\n",
    "print(train_df['Flight Status'].value_counts() / len(train_df))\n",
    "print()\n",
    "\n",
    "for predictor in predictors:\n",
    "    # construct the frequency table\n",
    "    df = train_df[['Flight Status', predictor]]\n",
    "    freqTable = df.pivot_table(index='Flight Status', columns=predictor, aggfunc=len)\n",
    "\n",
    "    # divide each row by the sum of the row to get conditional probabilities\n",
    "    propTable = freqTable.apply(lambda x: x / sum(x), axis=1)\n",
    "    print(propTable)\n",
    "    print()\n",
    "pd.reset_option('precision')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-27T20:03:08.658544Z",
     "iopub.status.busy": "2021-05-27T20:03:08.658126Z",
     "iopub.status.idle": "2021-05-27T20:03:08.661872Z",
     "shell.execute_reply": "2021-05-27T20:03:08.662250Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P_hat_delayed ~  2.0548742506526157e-05\n",
      "P_hat_ontime ~  0.00033326464123921066\n",
      "P(delayed|...) =  0.05807791183301656\n",
      "P(ontime|...) =  0.9419220881669834\n"
     ]
    }
   ],
   "source": [
    "# P(delayed | Carrier = DL, Day_Week = 7, Dep_Time = 10, Dest = LGA, Origin = DCA)\n",
    "P_hat_delayed = 0.0958 * 0.1609 * 0.0307 * 0.4215 * 0.5211 * 0.1977\n",
    "# P(ontime | Carrier = DL, Day_Week = 7, Dep_Time = 10, Dest = LGA, Origin = DCA)\n",
    "P_hat_ontime = 0.2040 * 0.1048 * 0.0519 * 0.5779 * 0.6478 * 0.8023\n",
    "print('P_hat_delayed ~ ', P_hat_delayed)\n",
    "print('P_hat_ontime ~ ', P_hat_ontime)\n",
    "\n",
    "print('P(delayed|...) = ', P_hat_delayed / (P_hat_delayed + P_hat_ontime))\n",
    "print('P(ontime|...) = ', P_hat_ontime / (P_hat_delayed + P_hat_ontime))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Table 8.6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-27T20:03:08.673083Z",
     "iopub.status.busy": "2021-05-27T20:03:08.672644Z",
     "iopub.status.idle": "2021-05-27T20:03:08.676522Z",
     "shell.execute_reply": "2021-05-27T20:03:08.676852Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "      actual predicted         0         1\n",
      "1225  ontime    ontime  0.057989  0.942011\n"
     ]
    }
   ],
   "source": [
    "# Subset a specific set\n",
    "df = pd.concat([pd.DataFrame({'actual': y_valid, 'predicted': y_valid_pred}),\n",
    "                pd.DataFrame(predProb_valid, index=y_valid.index)], axis=1)\n",
    "mask = ((X_valid.CARRIER_DL == 1) & (X_valid.DAY_WEEK_7 == 1) & (X_valid.CRS_DEP_TIME_10 == 1) & \n",
    "        (X_valid.DEST_LGA == 1) & (X_valid.ORIGIN_DCA == 1))\n",
    "\n",
    "print(df[mask])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Table 8.7"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-27T20:03:08.681425Z",
     "iopub.status.busy": "2021-05-27T20:03:08.680967Z",
     "iopub.status.idle": "2021-05-27T20:03:08.694806Z",
     "shell.execute_reply": "2021-05-27T20:03:08.695275Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Confusion Matrix (Accuracy 0.7955)\n",
      "\n",
      "        Prediction\n",
      " Actual delayed  ontime\n",
      "delayed      52     209\n",
      " ontime      61     998\n",
      "\n",
      "Confusion Matrix (Accuracy 0.7821)\n",
      "\n",
      "        Prediction\n",
      " Actual delayed  ontime\n",
      "delayed      26     141\n",
      " ontime      51     663\n"
     ]
    }
   ],
   "source": [
    "classificationSummary(y_train, y_train_pred, class_names=classes) \n",
    "\n",
    "print()\n",
    "\n",
    "classificationSummary(y_valid, y_valid_pred, class_names=classes) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Figure 8.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-27T20:03:08.700877Z",
     "iopub.status.busy": "2021-05-27T20:03:08.700445Z",
     "iopub.status.idle": "2021-05-27T20:03:08.912635Z",
     "shell.execute_reply": "2021-05-27T20:03:08.912261Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 18080 characters]\n",
      "text/plain": [
       "<Figure size 288x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "df = pd.DataFrame({'actual': 1 - y_valid.cat.codes, 'prob': predProb_valid[:, 0]})\n",
    "df = df.sort_values(by=['prob'], ascending=False).reset_index(drop=True)\n",
    "\n",
    "fig, ax = plt.subplots()\n",
    "fig.set_size_inches(4, 4)\n",
    "gainsChart(df.actual, ax=ax)\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-27T20:03:08.916889Z",
     "iopub.status.busy": "2021-05-27T20:03:08.916462Z",
     "iopub.status.idle": "2021-05-27T20:03:08.920294Z",
     "shell.execute_reply": "2021-05-27T20:03:08.919883Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.03831098, 0.02988425, 0.02298965, 0.02528785, 0.03754491,\n",
       "        0.01379685, 0.03218245, 0.00690226, 0.01073259, 0.01303079,\n",
       "        0.00383799, 0.00613619, 0.00230586, 0.00996652, 0.00920046,\n",
       "        0.00766832, 0.04060918, 0.01456292, 0.03065031, 0.00383799,\n",
       "        0.01992539, 0.00307193, 0.01762719, 0.01609505, 0.10419268,\n",
       "        0.07967856, 0.07584823, 0.03984311, 0.08427496, 0.01149866,\n",
       "        0.0628251 , 0.01915932, 0.04443951, 0.00153979, 0.04367344,\n",
       "        0.00307193, 0.01379685],\n",
       "       [0.02492933, 0.02832852, 0.02889505, 0.03588229, 0.033805  ,\n",
       "        0.02719546, 0.0209636 , 0.01246561, 0.01265445, 0.01699787,\n",
       "        0.01133254, 0.01038832, 0.00680028, 0.01322098, 0.01492058,\n",
       "        0.01152139, 0.02341857, 0.01548711, 0.02001938, 0.00698912,\n",
       "        0.00793335, 0.00528953, 0.01057716, 0.01208792, 0.12954902,\n",
       "        0.05835475, 0.05325596, 0.03116118, 0.11557455, 0.00698912,\n",
       "        0.04589103, 0.04079224, 0.02341857, 0.00207917, 0.033805  ,\n",
       "        0.00340108, 0.0436249 ]])"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "np.exp(delays_nb.feature_log_prob_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
