{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Chapter 13: Combining Models: Ensembles and Uplift Modeling\n",
    "\n",
    "> (c) 2019 Galit Shmueli, Peter C. Bruce, Peter Gedeck \n",
    ">\n",
    "> Code included in\n",
    ">\n",
    "> _Data Mining for Business Analytics: Concepts, Techniques, and Applications in Python_ (First Edition) \n",
    "> Galit Shmueli, Peter C. Bruce, Peter Gedeck, and Nitin R. Patel. 2019.\n",
    "\n",
    "## Import required packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-27T20:03:22.339386Z",
     "iopub.status.busy": "2021-05-27T20:03:22.338960Z",
     "iopub.status.idle": "2021-05-27T20:03:23.147070Z",
     "shell.execute_reply": "2021-05-27T20:03:23.147484Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "no display found. Using non-interactive Agg backend\n"
     ]
    }
   ],
   "source": [
    "from pathlib import Path\n",
    "\n",
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.ensemble import AdaBoostClassifier\n",
    "from sklearn.ensemble import BaggingClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "\n",
    "import dmba\n",
    "from dmba import classificationSummary\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Table 13.1, 13.2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-27T20:03:23.151866Z",
     "iopub.status.busy": "2021-05-27T20:03:23.151418Z",
     "iopub.status.idle": "2021-05-27T20:03:23.166312Z",
     "shell.execute_reply": "2021-05-27T20:03:23.165918Z"
    }
   },
   "outputs": [],
   "source": [
    "bank_df = dmba.load_data('UniversalBank.csv')\n",
    "bank_df.drop(columns=['ID', 'ZIP Code'], inplace=True)\n",
    "\n",
    "# split into training and validation\n",
    "X = bank_df.drop(columns=['Personal Loan'])\n",
    "y = bank_df['Personal Loan']\n",
    "X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.40, random_state=3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### single tree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-27T20:03:23.171986Z",
     "iopub.status.busy": "2021-05-27T20:03:23.171517Z",
     "iopub.status.idle": "2021-05-27T20:03:23.183153Z",
     "shell.execute_reply": "2021-05-27T20:03:23.182799Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Confusion Matrix (Accuracy 0.9825)\n",
      "\n",
      "       Prediction\n",
      "Actual    0    1\n",
      "     0 1778   15\n",
      "     1   20  187\n"
     ]
    }
   ],
   "source": [
    "defaultTree = DecisionTreeClassifier(random_state=1)\n",
    "defaultTree.fit(X_train, y_train)\n",
    "\n",
    "classes = defaultTree.classes_\n",
    "classificationSummary(y_valid, defaultTree.predict(X_valid), class_names=defaultTree.classes_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### bagging"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-27T20:03:23.188945Z",
     "iopub.status.busy": "2021-05-27T20:03:23.188475Z",
     "iopub.status.idle": "2021-05-27T20:03:23.524591Z",
     "shell.execute_reply": "2021-05-27T20:03:23.525170Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Confusion Matrix (Accuracy 0.9855)\n",
      "\n",
      "       Prediction\n",
      "Actual    0    1\n",
      "     0 1781   12\n",
      "     1   17  190\n"
     ]
    }
   ],
   "source": [
    "bagging = BaggingClassifier(DecisionTreeClassifier(random_state=1), \n",
    "                            n_estimators=100, random_state=1)\n",
    "bagging.fit(X_train, y_train)\n",
    "\n",
    "classificationSummary(y_valid, bagging.predict(X_valid), class_names=classes)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### boosting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-27T20:03:23.531570Z",
     "iopub.status.busy": "2021-05-27T20:03:23.530135Z",
     "iopub.status.idle": "2021-05-27T20:03:23.542629Z",
     "shell.execute_reply": "2021-05-27T20:03:23.542272Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Confusion Matrix (Accuracy 0.9840)\n",
      "\n",
      "       Prediction\n",
      "Actual    0    1\n",
      "     0 1776   17\n",
      "     1   15  192\n"
     ]
    }
   ],
   "source": [
    "boost = AdaBoostClassifier(DecisionTreeClassifier(random_state=1), n_estimators=100, random_state=1)\n",
    "boost.fit(X_train, y_train)\n",
    "\n",
    "classificationSummary(y_valid, boost.predict(X_valid), class_names=classes)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Table 13.9"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-27T20:03:23.547719Z",
     "iopub.status.busy": "2021-05-27T20:03:23.547237Z",
     "iopub.status.idle": "2021-05-27T20:03:24.378102Z",
     "shell.execute_reply": "2021-05-27T20:03:24.378493Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Confusion Matrix (Accuracy 0.6415)\n",
      "\n",
      "       Prediction\n",
      "Actual    N    Y\n",
      "     N 2018  450\n",
      "     Y  984  548\n"
     ]
    }
   ],
   "source": [
    "voter_df = dmba.load_data('Voter-Persuasion.csv')\n",
    "\n",
    "# Preprocess data frame\n",
    "predictors = ['AGE', 'NH_WHITE', 'COMM_PT', 'H_F1', 'REG_DAYS', \n",
    "              'PR_PELIG', 'E_PELIG', 'POLITICALC', 'MESSAGE_A']\n",
    "outcome = 'MOVED_AD'\n",
    "\n",
    "classes = list(voter_df.MOVED_AD.unique())\n",
    "\n",
    "# Partition the data\n",
    "X = voter_df[predictors]\n",
    "y = voter_df[outcome]\n",
    "X_train, X_valid, y_train, y_valid = train_test_split(X, y, test_size=0.40, random_state=1)\n",
    "\n",
    "# Train a random forest classifier using the training set\n",
    "rfModel = RandomForestClassifier(n_estimators=100, random_state=1)\n",
    "rfModel.fit(X_train, y_train)\n",
    "\n",
    "classificationSummary(y_valid, rfModel.predict(X_valid), class_names=classes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-27T20:03:24.382650Z",
     "iopub.status.busy": "2021-05-27T20:03:24.382163Z",
     "iopub.status.idle": "2021-05-27T20:03:24.531767Z",
     "shell.execute_reply": "2021-05-27T20:03:24.532103Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "      probMessage  probNoMessage  uplift\n",
      "9953         0.77           0.62    0.15\n",
      "3850         0.39           0.39    0.00\n",
      "4962         0.20           0.14    0.06\n",
      "3886         0.86           0.62    0.24\n",
      "5437         0.10           0.28   -0.18\n"
     ]
    }
   ],
   "source": [
    "uplift_df = X_valid.copy()  # Need to create a copy to allow modifying data\n",
    "\n",
    "uplift_df.MESSAGE_A = 1\n",
    "predTreatment = rfModel.predict_proba(uplift_df)\n",
    "uplift_df.MESSAGE_A = 0\n",
    "predControl = rfModel.predict_proba(uplift_df)\n",
    "\n",
    "upliftResult_df = pd.DataFrame({\n",
    "    'probMessage': predTreatment[:,1],\n",
    "    'probNoMessage': predControl[:,1],\n",
    "    'uplift': predTreatment[:,1] - predControl[:,1],\n",
    "    }, index=uplift_df.index)\n",
    "print(upliftResult_df.head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-27T20:03:24.535752Z",
     "iopub.status.busy": "2021-05-27T20:03:24.535293Z",
     "iopub.status.idle": "2021-05-27T20:03:24.538010Z",
     "shell.execute_reply": "2021-05-27T20:03:24.537576Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0.23 0.77]\n",
      " [0.61 0.39]\n",
      " [0.8  0.2 ]\n",
      " ...\n",
      " [0.56 0.44]\n",
      " [0.23 0.77]\n",
      " [0.3  0.7 ]]\n"
     ]
    }
   ],
   "source": [
    "print(predTreatment)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
