{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Chapter 16: Handling Time Series \n",
    "\n",
    "> (c) 2019 Galit Shmueli, Peter C. Bruce, Peter Gedeck \n",
    ">\n",
    "> Code included in\n",
    ">\n",
    "> _Data Mining for Business Analytics: Concepts, Techniques, and Applications in Python_ (First Edition) \n",
    "> Galit Shmueli, Peter C. Bruce, Peter Gedeck, and Nitin R. Patel. 2019.\n",
    "\n",
    "## Import required packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-27T20:03:33.167570Z",
     "iopub.status.busy": "2021-05-27T20:03:33.167135Z",
     "iopub.status.idle": "2021-05-27T20:03:34.041266Z",
     "shell.execute_reply": "2021-05-27T20:03:34.041715Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "no display found. Using non-interactive Agg backend\n"
     ]
    }
   ],
   "source": [
    "from pathlib import Path\n",
    "import datetime\n",
    "\n",
    "import pandas as pd\n",
    "import matplotlib.pylab as plt\n",
    "import statsmodels.formula.api as sm\n",
    "from statsmodels.tsa import tsatools\n",
    "\n",
    "import dmba\n",
    "from dmba import regressionSummary\n",
    "\n",
    "%matplotlib inline\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Figure 16.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-27T20:03:34.047541Z",
     "iopub.status.busy": "2021-05-27T20:03:34.047100Z",
     "iopub.status.idle": "2021-05-27T20:03:34.556307Z",
     "shell.execute_reply": "2021-05-27T20:03:34.556737Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 103440 characters]\n",
      "text/plain": [
       "<Figure size 720x432 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "baregg_df = dmba.load_data('BareggTunnel.csv')\n",
    "\n",
    "# convert the date information to a datetime object\n",
    "baregg_df['Date'] = pd.to_datetime(baregg_df.Day, format='%d %b %Y')\n",
    "vehicles_ts = pd.Series(baregg_df['Number of vehicles'].values, index=baregg_df.Date, name='Vehicles')\n",
    "\n",
    "fig, axes = plt.subplots(nrows=2, ncols=1, figsize=(10,6))\n",
    "vehicles_ts.plot(ax=axes[0])\n",
    "axes[0].set_xlabel('Day')\n",
    "axes[0].set_ylabel('Number of vehicles')\n",
    "vehicles_ts['2004-02-01':'2004-05-31'].plot(ax=axes[1])\n",
    "axes[1].set_xlabel('Day')\n",
    "axes[1].set_ylabel('Number of vehicles')\n",
    "plt.suptitle('')  # Suppress the overall title\n",
    "plt.tight_layout()  # Increase the separation between the plots\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-27T20:03:34.561470Z",
     "iopub.status.busy": "2021-05-27T20:03:34.560870Z",
     "iopub.status.idle": "2021-05-27T20:03:34.568702Z",
     "shell.execute_reply": "2021-05-27T20:03:34.569127Z"
    }
   },
   "outputs": [],
   "source": [
    "Amtrak_df = dmba.load_data('Amtrak.csv')\n",
    "\n",
    "# convert the date information to a datetime object\n",
    "Amtrak_df['Date'] = pd.to_datetime(Amtrak_df.Month, format='%d/%m/%Y')\n",
    "\n",
    "# convert dataframe column to series (name is used to label the data)\n",
    "ridership_ts = pd.Series(Amtrak_df.Ridership.values, index=Amtrak_df.Date, name='Ridership')\n",
    "\n",
    "# define the time series frequency\n",
    "ridership_ts.index = pd.DatetimeIndex(ridership_ts.index, freq=ridership_ts.index.inferred_freq)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Figure 16.2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-27T20:03:34.601944Z",
     "iopub.status.busy": "2021-05-27T20:03:34.596177Z",
     "iopub.status.idle": "2021-05-27T20:03:34.768071Z",
     "shell.execute_reply": "2021-05-27T20:03:34.767699Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 37228 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plot the series\n",
    "ax = ridership_ts.plot()\n",
    "ax.set_xlabel('Time')\n",
    "ax.set_ylabel('Ridership (in 000s)')\n",
    "_ = ax.set_ylim(1300, 2300)\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Figure 16.3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-27T20:03:34.824019Z",
     "iopub.status.busy": "2021-05-27T20:03:34.817100Z",
     "iopub.status.idle": "2021-05-27T20:03:35.261254Z",
     "shell.execute_reply": "2021-05-27T20:03:35.260876Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 67060 characters]\n",
      "text/plain": [
       "<Figure size 720x432 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# create short time series from 1997 to 1999 using a slice \n",
    "ridership_ts_3yrs = ridership_ts['1997':'1999']\n",
    "\n",
    "# create a data frame with additional independent variables from time series\n",
    "# the following command adds a constant term, a trend term and a quadratic trend term\n",
    "ridership_df = tsatools.add_trend(ridership_ts, trend='ctt')\n",
    "# fit a linear regression model to the time series\n",
    "ridership_lm = sm.ols(formula='Ridership ~ trend + trend_squared', data=ridership_df).fit()\n",
    "\n",
    "# shorter and longer time series\n",
    "fig, axes = plt.subplots(nrows=2, ncols=1, figsize=(10,6))\n",
    "ridership_ts_3yrs.plot(ax=axes[0])\n",
    "ridership_ts.plot(ax=axes[1])\n",
    "for ax in axes:\n",
    "    ax.set_xlabel('Time')\n",
    "    ax.set_ylabel('Ridership (in 000s)')\n",
    "    ax.set_ylim(1300, 2300)\n",
    "ridership_lm.predict(ridership_df).plot(ax=axes[1])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Figure 16.4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-27T20:03:35.290758Z",
     "iopub.status.busy": "2021-05-27T20:03:35.269090Z",
     "iopub.status.idle": "2021-05-27T20:03:35.568619Z",
     "shell.execute_reply": "2021-05-27T20:03:35.568182Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 55812 characters]\n",
      "text/plain": [
       "<Figure size 648x504 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "nValid = 36\n",
    "nTrain = len(ridership_ts) - nValid\n",
    "\n",
    "# partition the data\n",
    "train_ts = ridership_ts[:nTrain]\n",
    "valid_ts = ridership_ts[nTrain:]\n",
    "\n",
    "# Generate the naive and seasonal naive forecast\n",
    "naive_pred = pd.Series(train_ts[-1], index=valid_ts.index)\n",
    "last_season = train_ts[-12:]\n",
    "seasonal_pred = pd.Series(pd.concat([last_season]*5)[:len(valid_ts)].values, index=valid_ts.index)\n",
    "\n",
    "\n",
    "\n",
    "# plot forecasts and actual in the training and validation sets\n",
    "ax = train_ts.plot(color='C0', linewidth=0.75, figsize=(9,7))\n",
    "valid_ts.plot(ax=ax, color='C0', linestyle='dashed', linewidth=0.75)\n",
    "# ax.set_xlim('1990', '2006-06-01')\n",
    "ax.set_ylim(1300, 2600)\n",
    "ax.set_xlabel('Time')\n",
    "ax.set_ylabel('Ridership (in 000s)')\n",
    "\n",
    "naive_pred.plot(ax=ax, color='green')\n",
    "seasonal_pred.plot(ax=ax, color='orange')\n",
    "\n",
    "\n",
    "# Determine coordinates for drawing the arrows and lines\n",
    "one_month = pd.Timedelta('31 days')\n",
    "xtrain = (min(train_ts.index), max(train_ts.index) - one_month)\n",
    "xvalid = (min(valid_ts.index) + one_month, max(valid_ts.index) - one_month)\n",
    "xfuture = (max(valid_ts.index) + one_month, max(valid_ts.index) + 20 * one_month)\n",
    "xtv = xtrain[1] + 0.5 * (xvalid[0] - xtrain[1])\n",
    "xvf = xvalid[1] + 0.5 * (xfuture[0] - xvalid[1])\n",
    "ax.set_xlim(min(train_ts.index) - 6 * one_month, max(valid_ts.index) + 24 * one_month)\n",
    "\n",
    "ax.add_line(plt.Line2D(xtrain, (2450, 2450), color='black', linewidth=0.5))\n",
    "ax.add_line(plt.Line2D(xvalid, (2450, 2450), color='black', linewidth=0.5))\n",
    "ax.add_line(plt.Line2D(xfuture, (2450, 2450), color='black', linewidth=0.5))\n",
    "ax.text('1995', 2500, 'Training')\n",
    "ax.text('2001-9', 2500, 'Validation')\n",
    "ax.text('2004-7', 2500, 'Future')\n",
    "ax.axvline(x=xtv, ymin=0, ymax=1, color='black', linewidth=0.5)\n",
    "ax.axvline(x=xvf, ymin=0, ymax=1, color='black', linewidth=0.5)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Table 16.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-27T20:03:35.572564Z",
     "iopub.status.busy": "2021-05-27T20:03:35.572059Z",
     "iopub.status.idle": "2021-05-27T20:03:35.575743Z",
     "shell.execute_reply": "2021-05-27T20:03:35.575313Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Regression statistics\n",
      "\n",
      "                      Mean Error (ME) : -14.7177\n",
      "       Root Mean Squared Error (RMSE) : 142.7551\n",
      "            Mean Absolute Error (MAE) : 115.9234\n",
      "          Mean Percentage Error (MPE) : -1.2750\n",
      "Mean Absolute Percentage Error (MAPE) : 6.0214\n"
     ]
    }
   ],
   "source": [
    "regressionSummary(valid_ts, naive_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-27T20:03:35.579225Z",
     "iopub.status.busy": "2021-05-27T20:03:35.578803Z",
     "iopub.status.idle": "2021-05-27T20:03:35.582720Z",
     "shell.execute_reply": "2021-05-27T20:03:35.582295Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Regression statistics\n",
      "\n",
      "                      Mean Error (ME) : 54.7296\n",
      "       Root Mean Squared Error (RMSE) : 95.6243\n",
      "            Mean Absolute Error (MAE) : 84.0941\n",
      "          Mean Percentage Error (MPE) : 2.6528\n",
      "Mean Absolute Percentage Error (MAPE) : 4.2477\n"
     ]
    }
   ],
   "source": [
    "regressionSummary(valid_ts, seasonal_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-27T20:03:35.587193Z",
     "iopub.status.busy": "2021-05-27T20:03:35.586763Z",
     "iopub.status.idle": "2021-05-27T20:03:35.593032Z",
     "shell.execute_reply": "2021-05-27T20:03:35.592670Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "naive model in training set\n",
      "\n",
      "Regression statistics\n",
      "\n",
      "                      Mean Error (ME) : 2.4509\n",
      "       Root Mean Squared Error (RMSE) : 168.1470\n",
      "            Mean Absolute Error (MAE) : 125.2975\n",
      "          Mean Percentage Error (MPE) : -0.3460\n",
      "Mean Absolute Percentage Error (MAPE) : 7.2714\n",
      "seasonal model in training set\n",
      "\n",
      "Regression statistics\n",
      "\n",
      "                      Mean Error (ME) : 13.9399\n",
      "       Root Mean Squared Error (RMSE) : 99.2656\n",
      "            Mean Absolute Error (MAE) : 82.4920\n",
      "          Mean Percentage Error (MPE) : 0.5851\n",
      "Mean Absolute Percentage Error (MAPE) : 4.7153\n"
     ]
    }
   ],
   "source": [
    "print('naive model in training set')\n",
    "regressionSummary(train_ts[1:], train_ts[:-1])\n",
    "\n",
    "print('seasonal model in training set')\n",
    "regressionSummary(train_ts[12:], train_ts[:-12])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-27T20:03:35.631990Z",
     "iopub.status.busy": "2021-05-27T20:03:35.629593Z",
     "iopub.status.idle": "2021-05-27T20:03:35.871341Z",
     "shell.execute_reply": "2021-05-27T20:03:35.871738Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:xlabel='Date'>"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 53020 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "naive_train_ts = pd.Series(train_ts[:-1].values, index=train_ts[1:].index)\n",
    "ax = train_ts[1:].plot()\n",
    "naive_train_ts.plot(ax=ax)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-27T20:03:35.932565Z",
     "iopub.status.busy": "2021-05-27T20:03:35.898551Z",
     "iopub.status.idle": "2021-05-27T20:03:36.135728Z",
     "shell.execute_reply": "2021-05-27T20:03:36.136096Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:xlabel='Date'>"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 52780 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "seasonal_train_ts = pd.Series(train_ts[:-12].values, index=train_ts[12:].index)\n",
    "ax = train_ts[12:].plot()\n",
    "seasonal_train_ts.plot(ax=ax)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-27T20:03:36.140523Z",
     "iopub.status.busy": "2021-05-27T20:03:36.140103Z",
     "iopub.status.idle": "2021-05-27T20:03:36.548860Z",
     "shell.execute_reply": "2021-05-27T20:03:36.548411Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 48472 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 4 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "from statsmodels.tsa.seasonal import seasonal_decompose\n",
    "result = seasonal_decompose(ridership_ts, model='additive')\n",
    "result.plot()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
