preprocessing_Training_data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Twitter Sentiment Analysis Data PreProcessing (Training Data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import the dependencies\n",
    "import keras\n",
    "from keras import backend as k\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import re\n",
    "import nltk\n",
    "from nltk import PorterStemmer\n",
    "from nltk.tokenize import word_tokenize\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [],
   "source": [
    "# reading the data\n",
    "train_data = pd.read_csv('data/train.csv', encoding = 'ISO-8859-1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ItemID</th>\n",
       "      <th>Sentiment</th>\n",
       "      <th>SentimentText</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>is so sad for my APL frie...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>I missed the New Moon trail...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>omg its already 7:30 :O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>.. Omgaga. Im sooo  im gunna CRy. I'...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>i think mi bf is cheating on me!!!   ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>or i just worry too much?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>Juuuuuuuuuuuuuuuuussssst Chillin!!</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "      <td>Sunny Again        Work Tomorrow  :-|  ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>handed in my uniform today . i miss you ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>hmmmm.... i wonder how she my number @-)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   ItemID  Sentiment                                      SentimentText\n",
       "0       1          0                       is so sad for my APL frie...\n",
       "1       2          0                     I missed the New Moon trail...\n",
       "2       3          1                            omg its already 7:30 :O\n",
       "3       4          0            .. Omgaga. Im sooo  im gunna CRy. I'...\n",
       "4       5          0           i think mi bf is cheating on me!!!   ...\n",
       "5       6          0                  or i just worry too much?        \n",
       "6       7          1                 Juuuuuuuuuuuuuuuuussssst Chillin!!\n",
       "7       8          0         Sunny Again        Work Tomorrow  :-|  ...\n",
       "8       9          1        handed in my uniform today . i miss you ...\n",
       "9      10          1           hmmmm.... i wonder how she my number @-)"
      ]
     },
     "execution_count": 106,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# showing the structure of the data\n",
    "train_data.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "no of rows in the data set :  99989\n"
     ]
    }
   ],
   "source": [
    "# no of rows in the dataset\n",
    "rows = train_data.count()\n",
    "print('no of rows in the data set : ', rows[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "remove @ from the tweet\n",
      "original text :  @aditya you are a very good boy\n",
      "processed text :   you are a very good boy\n",
      "\n",
      "\n",
      "remove links from the tweets\n",
      "original text:  http://aditya.com is a very good site\n",
      "precessed text:   is a very good site\n",
      "\n",
      "\n",
      "remove hastags from the text\n",
      "original text:  we won #INvsAUS #cricket #bleedblue\n",
      "processed text:  we won  INvsAUS  cricket  bleedblue\n",
      "\n",
      "\n",
      "to remove emoji from the text\n",
      "original text:   hello boy 🙅 \n",
      "processed text :   hello boy  \n",
      "\n",
      "\n",
      "to convert uppercase letters to lower case letters\n",
      "original text:  This Is \n",
      "processed text :  this is \n",
      "\n",
      "\n",
      "to remove punctuations from the text\n",
      "original text:  string. With. Punctuation?\n",
      "processed text:  string With Punctuation\n",
      "\n",
      "\n",
      "to remove repeating characters from the text\n",
      "original text:  I’m in a hurrryyyyyyyyyyyyyyyyy\n",
      "processed text:  I’m in a hurryy\n",
      "\n",
      "\n",
      "to deal with stemming words\n",
      "original text:  running\n",
      "processed text:  run\n"
     ]
    }
   ],
   "source": [
    "# first of all we will remove @ from the text like this\n",
    "\n",
    "text = \"@aditya you are a very good boy\"\n",
    "print('remove @ from the tweet')\n",
    "print('original text : ', text)\n",
    "print('processed text : ',re.sub(r'@[A-Za-z0-9]+','',text))\n",
    "\n",
    "print('\\n')\n",
    "# removing links from the text\n",
    "text= \"http://aditya.com is a very good site\"\n",
    "print('remove links from the tweets')\n",
    "print('original text: ', text)\n",
    "print('precessed text: ',re.sub('https?://[A-Za-z0-9./]+','',text))\n",
    "\n",
    "print('\\n')\n",
    "# removing hash tags and numbers from the text\n",
    "text = \"we won #INvsAUS #cricket #bleedblue\"\n",
    "print('remove hastags from the text')\n",
    "print('original text: ',text)\n",
    "print('processed text: ',re.sub(\"[^a-zA-Z]\", \" \", text))\n",
    "\n",
    "print('\\n')\n",
    "# remove emoji from the text\n",
    "print('to remove emoji from the text')\n",
    "text = \" hello boy 🙅 \"\n",
    "emoji_pattern = re.compile(\"[\"\n",
    "        u\"\\U0001F600-\\U0001F64F\"  # emoticons\n",
    "        u\"\\U0001F300-\\U0001F5FF\"  # symbols & pictographs\n",
    "        u\"\\U0001F680-\\U0001F6FF\"  # transport & map symbols\n",
    "        u\"\\U0001F1E0-\\U0001F1FF\"  # flags (iOS)\n",
    "                           \"]+\", flags=re.UNICODE)\n",
    "\n",
    "print('original text: ',text)\n",
    "print('processed text : ',emoji_pattern.sub(r'', text)) # no emoji\n",
    "\n",
    "print('\\n')\n",
    "# to convert the text to lower case\n",
    "print('to convert uppercase letters to lower case letters')\n",
    "text = \"This Is \"\n",
    "print('original text: ',text)\n",
    "print('processed text : ', text.lower())\n",
    "\n",
    "print('\\n')\n",
    "# remove punctuations from the text\n",
    "print('to remove punctuations from the text')\n",
    "text = \"string. With. Punctuation?\"\n",
    "print('original text: ',text)\n",
    "print('processed text: ',re.sub(r'[^\\w\\s]','',text))\n",
    "\n",
    "print('\\n')\n",
    "# to remove repeating characters \n",
    "text = \"I’m in a hurrryyyyyyyyyyyyyyyyy\"\n",
    "print('to remove repeating characters from the text')\n",
    "print('original text: ', text)\n",
    "print('processed text: ',re.sub(r'(.)\\1+', r'\\1\\1', text))\n",
    "\n",
    "print('\\n')\n",
    "# to deal with stemming words\n",
    "text = \"running\"\n",
    "print('to deal with stemming words')\n",
    "print('original text: ', text)\n",
    "print('processed text: ',PorterStemmer().stem(text))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['It', \"'s\", 'true', 'that', 'the', 'chicken', 'was', 'the', 'best', 'bamboozler', 'in', 'the', 'known', 'multiverse', '.']\n"
     ]
    }
   ],
   "source": [
    "\n",
    "text1 = \"It's true that the chicken was the best bamboozler in the known multiverse.\"\n",
    "tokens = word_tokenize(text1)\n",
    "print(tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "SentimentText = []\n",
    "def preprocess(train_data):\n",
    "    i = 0\n",
    "    while i <rows[0]:\n",
    "        text =train_data['SentimentText'][i]\n",
    "        text = re.sub(r'@[A-Za-z0-9]+','',text)\n",
    "        text = re.sub(\"[^a-zA-Z]\", \" \", text)\n",
    "        text =emoji_pattern.sub(r'', text)\n",
    "        text =text.lower()\n",
    "        text = re.sub(r'[^\\w\\s]','',text)\n",
    "        text = re.sub(r'(.)\\1+', r'\\1\\1', text)\n",
    "        tokens = word_tokenize(text)\n",
    "        new_text =\"\"\n",
    "        for token in tokens:\n",
    "            new_text += PorterStemmer().stem(token)\n",
    "            new_text +=' ' \n",
    "        SentimentText.append(new_text)\n",
    "        \n",
    "        if i%10000 == 0:\n",
    "            print('processed :' , i , 'tweets')\n",
    "        i = i + 1\n",
    "    print('Done PreProcessing the text' )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processed : 0 tweets\n",
      "processed : 10000 tweets\n",
      "processed : 20000 tweets\n",
      "processed : 30000 tweets\n",
      "processed : 40000 tweets\n",
      "processed : 50000 tweets\n",
      "processed : 60000 tweets\n",
      "processed : 70000 tweets\n",
      "processed : 80000 tweets\n",
      "processed : 90000 tweets\n",
      "Done PreProcessing the text\n",
      "time taken :  1.16768874724706  mins\n"
     ]
    }
   ],
   "source": [
    "curr_time = time.time()\n",
    "preprocess(train_data)\n",
    "new_time = time.time()\n",
    "\n",
    "time_taken = (new_time - curr_time)/60\n",
    "print('time taken : ',time_taken, ' mins')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}