{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Chapter 11: Neural nets\n",
    "\n",
    "> (c) 2019 Galit Shmueli, Peter C. Bruce, Peter Gedeck \n",
    ">\n",
    "> Code included in\n",
    ">\n",
    "> _Data Mining for Business Analytics: Concepts, Techniques, and Applications in Python_ (First Edition) \n",
    "> Galit Shmueli, Peter C. Bruce, Peter Gedeck, and Nitin R. Patel. 2019.\n",
    "\n",
    "## Import required packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-27T20:03:12.799932Z",
     "iopub.status.busy": "2021-05-27T20:03:12.799512Z",
     "iopub.status.idle": "2021-05-27T20:03:13.596154Z",
     "shell.execute_reply": "2021-05-27T20:03:13.595728Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "no display found. Using non-interactive Agg backend\n"
     ]
    }
   ],
   "source": [
    "from pathlib import Path\n",
    "\n",
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import dmba\n",
    "from dmba import classificationSummary\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Table 11.2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-27T20:03:13.602248Z",
     "iopub.status.busy": "2021-05-27T20:03:13.601353Z",
     "iopub.status.idle": "2021-05-27T20:03:13.629795Z",
     "shell.execute_reply": "2021-05-27T20:03:13.629380Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Intercepts\n",
      "[array([0.13368045, 4.07247552, 7.00768104]), array([14.30748676])]\n",
      "Weights\n",
      "[array([[ -1.30656481,  -4.20427792, -13.29587332],\n",
      "       [ -0.04399727,  -4.91606924,  -6.03356987]]), array([[ -0.27348313],\n",
      "       [ -9.01211573],\n",
      "       [-17.63504694]])]\n",
      "   Obs.  Fat  Salt Acceptance   dislike      like\n",
      "0     1  0.2   0.9       like  0.000490  0.999510\n",
      "1     2  0.1   0.1    dislike  0.999994  0.000006\n",
      "2     3  0.2   0.4    dislike  0.999741  0.000259\n",
      "3     4  0.2   0.5    dislike  0.997368  0.002632\n",
      "4     5  0.4   0.5       like  0.002133  0.997867\n",
      "5     6  0.3   0.8       like  0.000075  0.999925\n"
     ]
    }
   ],
   "source": [
    "example_df = dmba.load_data('TinyData.csv')\n",
    "\n",
    "predictors = ['Fat', 'Salt']\n",
    "outcome = 'Acceptance'\n",
    "\n",
    "X = example_df[predictors]\n",
    "y = example_df[outcome]\n",
    "classes = sorted(y.unique())\n",
    "\n",
    "clf = MLPClassifier(hidden_layer_sizes=(3), activation='logistic', solver='lbfgs', random_state=1)\n",
    "clf.fit(X, y)\n",
    "clf.predict(X)\n",
    "\n",
    "# Network structure\n",
    "print('Intercepts')\n",
    "print(clf.intercepts_)\n",
    "\n",
    "print('Weights')\n",
    "print(clf.coefs_)\n",
    "\n",
    "# Prediction\n",
    "print(pd.concat([\n",
    "    example_df,\n",
    "    pd.DataFrame(clf.predict_proba(X), columns=classes)\n",
    "], axis=1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-27T20:03:13.635200Z",
     "iopub.status.busy": "2021-05-27T20:03:13.634737Z",
     "iopub.status.idle": "2021-05-27T20:03:13.638574Z",
     "shell.execute_reply": "2021-05-27T20:03:13.638214Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Hidden layer 2 => 3\n",
      " Intercepts:\n",
      "  [0.13368045 4.07247552 7.00768104]\n",
      " Weights:\n",
      "  [ -1.30656481  -4.20427792 -13.29587332]\n",
      "  [-0.04399727 -4.91606924 -6.03356987]\n",
      "\n",
      "Output layer 3 => 1\n",
      " Intercepts:\n",
      "  [14.30748676]\n",
      " Weights:\n",
      "  [-0.27348313]\n",
      "  [-9.01211573]\n",
      "  [-17.63504694]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Pretty print layers\n",
    "for i, (weights, intercepts) in enumerate(zip(clf.coefs_, clf.intercepts_)):\n",
    "    print('Hidden layer' if i == 0 else 'Output layer', '{0[0]} => {0[1]}'.format(weights.shape))\n",
    "    print(' Intercepts:\\n ', intercepts)\n",
    "    print(' Weights:')\n",
    "    for weight in weights:\n",
    "        print(' ', weight)\n",
    "    print()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Table 11.3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-27T20:03:13.644983Z",
     "iopub.status.busy": "2021-05-27T20:03:13.644551Z",
     "iopub.status.idle": "2021-05-27T20:03:13.646810Z",
     "shell.execute_reply": "2021-05-27T20:03:13.647273Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Confusion Matrix (Accuracy 1.0000)\n",
      "\n",
      "        Prediction\n",
      " Actual dislike    like\n",
      "dislike       3       0\n",
      "   like       0       3\n"
     ]
    }
   ],
   "source": [
    "classificationSummary(y, clf.predict(X), class_names=classes)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Table 11.6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-27T20:03:13.652677Z",
     "iopub.status.busy": "2021-05-27T20:03:13.652241Z",
     "iopub.status.idle": "2021-05-27T20:03:13.741990Z",
     "shell.execute_reply": "2021-05-27T20:03:13.742462Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Confusion Matrix (Accuracy 0.8664)\n",
      "\n",
      "       Prediction\n",
      "Actual   0   1   2\n",
      "     0 331   0   1\n",
      "     1   0 180   0\n",
      "     2  30  49   8\n",
      "Confusion Matrix (Accuracy 0.8550)\n",
      "\n",
      "       Prediction\n",
      "Actual   0   1   2\n",
      "     0 218   0   1\n",
      "     1   0 119   0\n",
      "     2  24  33   5\n"
     ]
    }
   ],
   "source": [
    "accidents_df = dmba.load_data('accidentsnn.csv')\n",
    "input_vars = ['ALCHL_I', 'PROFIL_I_R', 'VEH_INVL']\n",
    "\n",
    "accidents_df.SUR_COND = accidents_df.SUR_COND.astype('category')\n",
    "accidents_df.MAX_SEV_IR = accidents_df.MAX_SEV_IR.astype('category')\n",
    "\n",
    "# convert the categorical data into dummy variables\n",
    "# exclude the column for SUR_COND 9 = unknown\n",
    "processed = pd.get_dummies(accidents_df, columns=['SUR_COND']).drop(columns=['SUR_COND_9'])\n",
    "\n",
    "outcome = 'MAX_SEV_IR'\n",
    "predictors = [c for c in processed.columns if c != outcome]\n",
    "\n",
    "# partition data\n",
    "X = processed[predictors]\n",
    "y = processed[outcome]\n",
    "train_X, valid_X, train_y, valid_y = train_test_split(X, y, test_size=0.4, random_state=1)\n",
    "\n",
    "# train neural network with 2 hidden nodes\n",
    "clf = MLPClassifier(hidden_layer_sizes=(2), activation='logistic', solver='lbfgs',\n",
    "                    random_state=1)\n",
    "clf.fit(train_X, train_y.values)\n",
    "\n",
    "# training performance (use idxmax to revert the one-hot-encoding)\n",
    "classificationSummary(train_y, clf.predict(train_X))\n",
    "\n",
    "# validation performance\n",
    "classificationSummary(valid_y, clf.predict(valid_X))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fitting class probabilities separately"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-27T20:03:13.748828Z",
     "iopub.status.busy": "2021-05-27T20:03:13.748363Z",
     "iopub.status.idle": "2021-05-27T20:03:13.916927Z",
     "shell.execute_reply": "2021-05-27T20:03:13.916557Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Confusion Matrix (Accuracy 0.8731)\n",
      "\n",
      "             Prediction\n",
      "      Actual MAX_SEV_IR_0 MAX_SEV_IR_1 MAX_SEV_IR_2\n",
      "MAX_SEV_IR_0          332            0            0\n",
      "MAX_SEV_IR_1            0          170           10\n",
      "MAX_SEV_IR_2           31           35           21\n",
      "Confusion Matrix (Accuracy 0.8675)\n",
      "\n",
      "             Prediction\n",
      "      Actual MAX_SEV_IR_0 MAX_SEV_IR_1 MAX_SEV_IR_2\n",
      "MAX_SEV_IR_0          218            0            1\n",
      "MAX_SEV_IR_1            0          113            6\n",
      "MAX_SEV_IR_2           24           22           16\n"
     ]
    }
   ],
   "source": [
    "accidents_df = dmba.load_data('accidentsnn.csv')\n",
    "input_vars = ['ALCHL_I', 'PROFIL_I_R', 'VEH_INVL']\n",
    "\n",
    "accidents_df.SUR_COND = accidents_df.SUR_COND.astype('category')\n",
    "accidents_df.MAX_SEV_IR = accidents_df.MAX_SEV_IR.astype('category')\n",
    "\n",
    "# convert the categorical data into dummy variables\n",
    "processed = pd.get_dummies(accidents_df)\n",
    "# drop the column for SUR_COND 9 = unknown\n",
    "processed = processed.drop(columns=['SUR_COND_9'])\n",
    "\n",
    "outcome = ['MAX_SEV_IR_0', 'MAX_SEV_IR_1', 'MAX_SEV_IR_2']\n",
    "predictors = [c for c in processed.columns if c not in outcome]\n",
    "classes = sorted(outcome)\n",
    "\n",
    "# partition data\n",
    "X = processed[predictors]\n",
    "y = processed[outcome]\n",
    "train_X, valid_X, train_y, valid_y = train_test_split(X, y, test_size=0.4, random_state=1)\n",
    "\n",
    "# train neural network with 2 hidden nodes\n",
    "clf = MLPClassifier(hidden_layer_sizes=(2), activation='logistic', solver='lbfgs',\n",
    "                    random_state=1, max_iter=500)\n",
    "clf.fit(train_X, train_y)\n",
    "\n",
    "# training performance (use idxmax to revert the one-hot-encoding)\n",
    "trainPrediction = pd.DataFrame(clf.predict(train_X), columns=outcome).idxmax(axis=1)\n",
    "classificationSummary(train_y.idxmax(axis=1), trainPrediction, class_names=classes)\n",
    "\n",
    "# validation performance\n",
    "validPrediction = pd.DataFrame(clf.predict(valid_X), columns=outcome).idxmax(axis=1)\n",
    "classificationSummary(valid_y.idxmax(axis=1), validPrediction, class_names=classes)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Grid search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-27T20:03:13.924695Z",
     "iopub.status.busy": "2021-05-27T20:03:13.923633Z",
     "iopub.status.idle": "2021-05-27T20:03:15.678417Z",
     "shell.execute_reply": "2021-05-27T20:03:15.677914Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best score:  0.8614005602240896\n",
      "Best parameters:  {'hidden_layer_sizes': 3}\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import cross_val_score, train_test_split, GridSearchCV\n",
    "\n",
    "# train neural network with 2 hidden nodes\n",
    "clf = MLPClassifier(hidden_layer_sizes=(2), activation='logistic', solver='lbfgs',\n",
    "                    random_state=1, max_iter=500)\n",
    "clf.fit(train_X, train_y.values)\n",
    "\n",
    "param_grid = {\n",
    "    'hidden_layer_sizes': [(1), (2), (3), (4), (5)], \n",
    "}\n",
    "gridSearch = GridSearchCV(MLPClassifier(activation='logistic', solver='lbfgs', random_state=1, max_iter=1000), \n",
    "                          param_grid, cv=5, n_jobs=-1, return_train_score=True)\n",
    "gridSearch.fit(train_X, train_y)\n",
    "print('Best score: ', gridSearch.best_score_)\n",
    "print('Best parameters: ', gridSearch.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-27T20:03:15.686679Z",
     "iopub.status.busy": "2021-05-27T20:03:15.686258Z",
     "iopub.status.idle": "2021-05-27T20:03:15.689289Z",
     "shell.execute_reply": "2021-05-27T20:03:15.688858Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  param_hidden_layer_sizes  mean_test_score  std_test_score\n",
      "0                        1         0.844762        0.011030\n",
      "1                        2         0.848067        0.009913\n",
      "2                        3         0.861401        0.023566\n",
      "3                        4         0.861387        0.025918\n",
      "4                        5         0.854748        0.023393\n"
     ]
    }
   ],
   "source": [
    "display=['param_hidden_layer_sizes', 'mean_test_score', 'std_test_score']\n",
    "print(pd.DataFrame(gridSearch.cv_results_)[display])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-27T20:03:15.694942Z",
     "iopub.status.busy": "2021-05-27T20:03:15.694532Z",
     "iopub.status.idle": "2021-05-27T20:03:15.836292Z",
     "shell.execute_reply": "2021-05-27T20:03:15.835876Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 12408 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "pd.DataFrame(gridSearch.cv_results_)[display].plot(x='param_hidden_layer_sizes', \n",
    "                                                   y='mean_test_score', yerr='std_test_score', ylim=(0.8, 0.9))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
