{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Chapter 20: Text mining\n",
    "\n",
    "> (c) 2019 Galit Shmueli, Peter C. Bruce, Peter Gedeck \n",
    ">\n",
    "> Code included in\n",
    ">\n",
    "> _Data Mining for Business Analytics: Concepts, Techniques, and Applications in Python_ (First Edition) \n",
    "> Galit Shmueli, Peter C. Bruce, Peter Gedeck, and Nitin R. Patel. 2019.\n",
    "\n",
    "## Import required packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-27T20:03:51.762646Z",
     "iopub.status.busy": "2021-05-27T20:03:51.762196Z",
     "iopub.status.idle": "2021-05-27T20:03:56.908350Z",
     "shell.execute_reply": "2021-05-27T20:03:56.907752Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "no display found. Using non-interactive Agg backend\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to /home/jovyan/nltk_data...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data]   Unzipping tokenizers/punkt.zip.\n"
     ]
    }
   ],
   "source": [
    "from pathlib import Path\n",
    "\n",
    "from zipfile import ZipFile\n",
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "from sklearn.decomposition import TruncatedSVD\n",
    "from sklearn.preprocessing import Normalizer\n",
    "from sklearn.pipeline import make_pipeline\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "import nltk\n",
    "from nltk import word_tokenize          \n",
    "from nltk.stem.snowball import EnglishStemmer \n",
    "import matplotlib.pylab as plt\n",
    "\n",
    "import dmba\n",
    "from dmba import printTermDocumentMatrix, classificationSummary, liftChart\n",
    "\n",
    "nltk.download('punkt')\n",
    "\n",
    "%matplotlib inline\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Table 20.1 Term-document representation of words in sentences S1-S3\n",
    " "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-27T20:03:56.913713Z",
     "iopub.status.busy": "2021-05-27T20:03:56.913262Z",
     "iopub.status.idle": "2021-05-27T20:03:56.919301Z",
     "shell.execute_reply": "2021-05-27T20:03:56.918792Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "          S1  S2  S3\n",
      "first      1   0   0\n",
      "here       0   0   1\n",
      "is         1   1   1\n",
      "second     0   1   0\n",
      "sentence   1   1   1\n",
      "the        1   0   1\n",
      "third      0   0   1\n",
      "this       1   1   0\n"
     ]
    }
   ],
   "source": [
    "text = ['this is the first sentence.',\n",
    "        'this is a second sentence.',\n",
    "        'the third sentence is here.']\n",
    "\n",
    "# Learn features based on text\n",
    "count_vect = CountVectorizer()\n",
    "counts = count_vect.fit_transform(text)\n",
    "\n",
    "printTermDocumentMatrix(count_vect, counts)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Table 20.2 Term-document representation of words in sentences S1-S4 (Example 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-27T20:03:56.925826Z",
     "iopub.status.busy": "2021-05-27T20:03:56.925357Z",
     "iopub.status.idle": "2021-05-27T20:03:56.928475Z",
     "shell.execute_reply": "2021-05-27T20:03:56.928094Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "           S1  S2  S3  S4\n",
      "all         0   0   0   1\n",
      "first       1   0   0   0\n",
      "forth       0   0   0   1\n",
      "here        0   0   1   0\n",
      "is          1   1   1   0\n",
      "of          0   0   0   1\n",
      "second      0   1   0   0\n",
      "sentence    1   1   1   0\n",
      "sentences   0   0   0   1\n",
      "the         1   0   1   0\n",
      "third       0   0   1   0\n",
      "this        1   1   0   0\n"
     ]
    }
   ],
   "source": [
    "text = ['this is the first     sentence!!',\n",
    "        'this is a second Sentence :)',\n",
    "        'the third sentence, is here ',\n",
    "        'forth of all sentences']\n",
    "\n",
    "# Learn features based on text. Include special characters that are part of a word in the analysis\n",
    "count_vect = CountVectorizer()\n",
    "counts = count_vect.fit_transform(text)\n",
    "\n",
    "printTermDocumentMatrix(count_vect, counts)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Table 20.3 Tokenization of S1-S4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-27T20:03:56.932603Z",
     "iopub.status.busy": "2021-05-27T20:03:56.932146Z",
     "iopub.status.idle": "2021-05-27T20:03:56.937953Z",
     "shell.execute_reply": "2021-05-27T20:03:56.937574Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "            S1  S2  S3  S4\n",
      ":)           0   1   0   0\n",
      "a            0   1   0   0\n",
      "all          0   0   0   1\n",
      "first        1   0   0   0\n",
      "forth        0   0   0   1\n",
      "here         0   0   1   0\n",
      "is           1   1   1   0\n",
      "of           0   0   0   1\n",
      "second       0   1   0   0\n",
      "sentence     0   1   1   0\n",
      "sentence!!   1   0   0   0\n",
      "sentences    0   0   0   1\n",
      "the          1   0   1   0\n",
      "third        0   0   1   0\n",
      "this         1   1   0   0\n"
     ]
    }
   ],
   "source": [
    "text = ['this is the first     sentence!!',\n",
    "        'this is a second Sentence :)',\n",
    "        'the third sentence, is here ',\n",
    "        'forth of all sentences']\n",
    "\n",
    "# Learn features based on text. Include special characters that are part of a word in the analysis\n",
    "count_vect = CountVectorizer(token_pattern='[a-zA-Z!:)]+')\n",
    "counts = count_vect.fit_transform(text)\n",
    "\n",
    "printTermDocumentMatrix(count_vect, counts)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Table 20.4 Stopwords in scitkit-learn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-27T20:03:56.942398Z",
     "iopub.status.busy": "2021-05-27T20:03:56.941877Z",
     "iopub.status.idle": "2021-05-27T20:03:56.946885Z",
     "shell.execute_reply": "2021-05-27T20:03:56.948047Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "First 180 of 318 stopwords\n",
      "a            about        above        across       after        afterwards   \n",
      "again        against      all          almost       alone        along        \n",
      "already      also         although     always       am           among        \n",
      "amongst      amoungst     amount       an           and          another      \n",
      "any          anyhow       anyone       anything     anyway       anywhere     \n",
      "are          around       as           at           back         be           \n",
      "became       because      become       becomes      becoming     been         \n",
      "before       beforehand   behind       being        below        beside       \n",
      "besides      between      beyond       bill         both         bottom       \n",
      "but          by           call         can          cannot       cant         \n",
      "co           con          could        couldnt      cry          de           \n",
      "describe     detail       do           done         down         due          \n",
      "during       each         eg           eight        either       eleven       \n",
      "else         elsewhere    empty        enough       etc          even         \n",
      "ever         every        everyone     everything   everywhere   except       \n",
      "few          fifteen      fifty        fill         find         fire         \n",
      "first        five         for          former       formerly     forty        \n",
      "found        four         from         front        full         further      \n",
      "get          give         go           had          has          hasnt        \n",
      "have         he           hence        her          here         hereafter    \n",
      "hereby       herein       hereupon     hers         herself      him          \n",
      "himself      his          how          however      hundred      i            \n",
      "ie           if           in           inc          indeed       interest     \n",
      "into         is           it           its          itself       keep         \n",
      "last         latter       latterly     least        less         ltd          \n",
      "made         many         may          me           meanwhile    might        \n",
      "mill         mine         more         moreover     most         mostly       \n",
      "move         much         must         my           myself       name         \n",
      "namely       neither      never        nevertheless next         nine         \n",
      "no           nobody       none         noone        nor          not          \n"
     ]
    }
   ],
   "source": [
    "from sklearn.feature_extraction._stop_words import ENGLISH_STOP_WORDS\n",
    "stopWords = list(sorted(ENGLISH_STOP_WORDS))\n",
    "ncolumns = 6; nrows= 30\n",
    "\n",
    "print('First {} of {} stopwords'.format(ncolumns * nrows, len(stopWords)))\n",
    "for i in range(0, len(stopWords[:(ncolumns * nrows)]), ncolumns):\n",
    "    print(''.join(word.ljust(13) for word in stopWords[i:(i+ncolumns)]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Table 20.5 Text reduction of S1-S4 using stemming"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-27T20:03:56.953430Z",
     "iopub.status.busy": "2021-05-27T20:03:56.952922Z",
     "iopub.status.idle": "2021-05-27T20:03:56.966057Z",
     "shell.execute_reply": "2021-05-27T20:03:56.965603Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "         S1  S2  S3  S4\n",
      "forth     0   0   0   1\n",
      "second    0   1   0   0\n",
      "sentenc   1   1   1   1\n"
     ]
    }
   ],
   "source": [
    "from sklearn.feature_extraction._stop_words import ENGLISH_STOP_WORDS\n",
    "text = ['this is the first     sentence!! ',\n",
    "        'this is a second Sentence :)',\n",
    "        'the third sentence, is here ',\n",
    "        'forth of all sentences']\n",
    "\n",
    "# Create a custom tokenizer that will use NLTK for tokenizing and lemmatizing \n",
    "# (removes interpunctuation and stop words)\n",
    "class LemmaTokenizer:\n",
    "    def __init__(self):\n",
    "        self.stemmer = EnglishStemmer()\n",
    "        self.stopWords = set(ENGLISH_STOP_WORDS)\n",
    "\n",
    "    def __call__(self, doc):\n",
    "        return [self.stemmer.stem(t) for t in word_tokenize(doc) \n",
    "                if t.isalpha() and t not in self.stopWords]\n",
    "\n",
    "# Learn features based on text\n",
    "count_vect = CountVectorizer(tokenizer=LemmaTokenizer())\n",
    "counts = count_vect.fit_transform(text)\n",
    "\n",
    "printTermDocumentMatrix(count_vect, counts)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Table 20.6 tf-idf matrix for S1-S4 example (after tokenization)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-27T20:03:56.976369Z",
     "iopub.status.busy": "2021-05-27T20:03:56.975800Z",
     "iopub.status.idle": "2021-05-27T20:03:56.979215Z",
     "shell.execute_reply": "2021-05-27T20:03:56.978704Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                 S1        S2        S3        S4\n",
      "all        0.000000  0.000000  0.000000  2.386294\n",
      "first      2.386294  0.000000  0.000000  0.000000\n",
      "forth      0.000000  0.000000  0.000000  2.386294\n",
      "here       0.000000  0.000000  2.386294  0.000000\n",
      "is         1.287682  1.287682  1.287682  0.000000\n",
      "of         0.000000  0.000000  0.000000  2.386294\n",
      "second     0.000000  2.386294  0.000000  0.000000\n",
      "sentence   1.287682  1.287682  1.287682  0.000000\n",
      "sentences  0.000000  0.000000  0.000000  2.386294\n",
      "the        1.693147  0.000000  1.693147  0.000000\n",
      "third      0.000000  0.000000  2.386294  0.000000\n",
      "this       1.693147  1.693147  0.000000  0.000000\n"
     ]
    }
   ],
   "source": [
    "text = ['this is the first     sentence!!',\n",
    "        'this is a second Sentence :)',\n",
    "        'the third sentence, is here ',\n",
    "        'forth of all sentences']\n",
    "\n",
    "# Apply CountVectorizer and TfidfTransformer sequentially\n",
    "count_vect = CountVectorizer()\n",
    "tfidfTransformer = TfidfTransformer(smooth_idf=False, norm=None)\n",
    "counts = count_vect.fit_transform(text)\n",
    "tfidf = tfidfTransformer.fit_transform(counts)\n",
    "\n",
    "printTermDocumentMatrix(count_vect, tfidf)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Table 20.7 Importing and labeling the records, preprocessing text, and producing concept matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-27T20:03:56.984673Z",
     "iopub.status.busy": "2021-05-27T20:03:56.984214Z",
     "iopub.status.idle": "2021-05-27T20:04:04.885265Z",
     "shell.execute_reply": "2021-05-27T20:04:04.884790Z"
    }
   },
   "outputs": [],
   "source": [
    "# Step 1: import and label records\n",
    "corpus = []\n",
    "label = []\n",
    "with ZipFile(dmba.get_data_file('AutoAndElectronics.zip')) as rawData:\n",
    "    for info in rawData.infolist():\n",
    "        if info.is_dir(): \n",
    "            continue\n",
    "        label.append(1 if 'rec.autos' in info.filename else 0)\n",
    "        corpus.append(rawData.read(info))\n",
    "\n",
    "# Step 2: preprocessing (tokenization, stemming, and stopwords)\n",
    "class LemmaTokenizer(object):\n",
    "    def __init__(self):\n",
    "        self.stemmer = EnglishStemmer()\n",
    "        self.stopWords = set(ENGLISH_STOP_WORDS)\n",
    "    def __call__(self, doc):\n",
    "        return [self.stemmer.stem(t) for t in word_tokenize(doc) \n",
    "                if t.isalpha() and t not in self.stopWords]\n",
    "\n",
    "preprocessor = CountVectorizer(tokenizer=LemmaTokenizer(), encoding='latin1')\n",
    "preprocessedText = preprocessor.fit_transform(corpus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-27T20:04:04.889328Z",
     "iopub.status.busy": "2021-05-27T20:04:04.888821Z",
     "iopub.status.idle": "2021-05-27T20:04:05.133778Z",
     "shell.execute_reply": "2021-05-27T20:04:05.134190Z"
    }
   },
   "outputs": [],
   "source": [
    "\n",
    "# Step 3: TF-IDF and latent semantic analysis\n",
    "tfidfTransformer = TfidfTransformer()\n",
    "tfidf = tfidfTransformer.fit_transform(preprocessedText)\n",
    "\n",
    "# Extract 20 concepts using LSA ()\n",
    "svd = TruncatedSVD(20)\n",
    "normalizer = Normalizer(copy=False)\n",
    "lsa = make_pipeline(svd, normalizer)\n",
    "\n",
    "lsa_tfidf = lsa.fit_transform(tfidf)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Table 20.8 Fitting a predictive model to the autos and electronics discussion data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-27T20:04:05.141333Z",
     "iopub.status.busy": "2021-05-27T20:04:05.139284Z",
     "iopub.status.idle": "2021-05-27T20:04:05.157423Z",
     "shell.execute_reply": "2021-05-27T20:04:05.156939Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Confusion Matrix (Accuracy 0.9563)\n",
      "\n",
      "       Prediction\n",
      "Actual   0   1\n",
      "     0 389   8\n",
      "     1  27 376\n"
     ]
    }
   ],
   "source": [
    "# split dataset into 60% training and 40% test set\n",
    "Xtrain, Xtest, ytrain, ytest = train_test_split(lsa_tfidf, label, test_size=0.4, random_state=42)\n",
    "\n",
    "# run logistic regression model on training\n",
    "logit_reg = LogisticRegression(solver='lbfgs')\n",
    "logit_reg.fit(Xtrain, ytrain)\n",
    "\n",
    "# print confusion matrix and accuracty\n",
    "classificationSummary(ytest, logit_reg.predict(Xtest))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-05-27T20:04:05.173375Z",
     "iopub.status.busy": "2021-05-27T20:04:05.172375Z",
     "iopub.status.idle": "2021-05-27T20:04:05.471242Z",
     "shell.execute_reply": "2021-05-27T20:04:05.472126Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 9760 characters]\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "df = pd.DataFrame(data={'prob': [p[1] for p in logit_reg.predict_proba(Xtest)], 'actual': ytest})\n",
    "df = df.sort_values(by=['prob'], ascending=False).reset_index(drop=True)\n",
    "\n",
    "ax = liftChart(df.actual, labelBars=False)\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
