NLP_movie+review+analysis_using+TFIDF.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import re\n",
    "from bs4 import BeautifulSoup\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from nltk.corpus import stopwords\n",
    "import nltk\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'C:\\\\Users\\\\LENOVO'"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.getcwd()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#changing the directory \n",
    "os.chdir(\"E:\\\\INSOFE\\\\text mining\\\\movie_reviews_sentiment_analysis\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#reading the data\n",
    "train = pd.read_csv(\"labeledTrainData.tsv\", header = 0, delimiter=\"\\t\",quoting=3)\n",
    "test  = pd.read_csv(\"testData.tsv\",header = 0,delimiter=\"\\t\",quoting = 3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(25000, 3)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.shape\n",
    "#test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['id', 'sentiment', 'review'], dtype=object)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.columns.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# let's make all these preprocessing steps combine into a function\n",
    "\n",
    "def raw_to_preprocessed( raw_review ):\n",
    "    # Function to convert a raw review to a string of words\n",
    "    # The input is a single string (a raw movie review), and \n",
    "    # the output is a single string (a preprocessed movie review)\n",
    "    #\n",
    "    # 1. Remove HTML\n",
    "    review_text = BeautifulSoup(raw_review).get_text() \n",
    "    #\n",
    "    # 2. Remove non-letters        \n",
    "    letters_only = re.sub(\"[^a-zA-Z]\", \" \", review_text) \n",
    "    #\n",
    "    # 3. Convert to lower case, split into individual words\n",
    "    words = letters_only.lower().split()                             \n",
    "    #\n",
    "    # 4. In Python, searching a set is much faster than searching\n",
    "    #   a list, so convert the stop words to a set\n",
    "    stops = set(stopwords.words(\"english\"))                  \n",
    "    # \n",
    "    # 5. Remove stop words\n",
    "    meaningful_words = [w for w in words if not w in stops]   \n",
    "    #\n",
    "    # 6. Join the words back into one string separated by space, \n",
    "    # and return the result.\n",
    "    processed_sentence = \" \".join(meaningful_words)\n",
    "    return(processed_sentence)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def raw_to_prepwithtokenize( raw_review ):\n",
    "    # Function to convert a raw review to a string of words\n",
    "    # The input is a single string (a raw movie review), and \n",
    "    # the output is a single string (a preprocessed movie review)\n",
    "    #\n",
    "    # 1. Remove HTML\n",
    "    review_text = BeautifulSoup(raw_review).get_text() \n",
    "    #\n",
    "    # 2. Remove non-letters        \n",
    "    letters_only = re.sub(\"[^a-zA-Z]\", \" \", review_text) \n",
    "    #\n",
    "    # 3. Convert to lower case, split into individual words\n",
    "    low_case = letters_only.lower()\n",
    "    words = nltk.word_tokenize(low_case)\n",
    "    #\n",
    "    # 4. In Python, searching a set is much faster than searching\n",
    "    #   a list, so convert the stop words to a set\n",
    "    stops = set(stopwords.words(\"english\"))                  \n",
    "    # \n",
    "    # 5. Remove stop words\n",
    "    meaningful_words = [w for w in words if not w in stops]   \n",
    "    #\n",
    "    # 6. Join the words back into one string separated by space, \n",
    "    # and return the result.\n",
    "    processed_sentence = \" \".join(meaningful_words)\n",
    "    return(processed_sentence)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "25000"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "num_reviews = train[\"review\"].size\n",
    "num_reviews"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "E:\\Anaconda3\\lib\\site-packages\\bs4\\__init__.py:181: UserWarning: No parser was explicitly specified, so I'm using the best available HTML parser for this system (\"lxml\"). This usually isn't a problem, but if you run this code on another system, or in a different virtual environment, it may use a different parser and behave differently.\n",
      "\n",
      "The code that caused this warning is on line 193 of the file E:\\Anaconda3\\lib\\runpy.py. To get rid of this warning, change code that looks like this:\n",
      "\n",
      " BeautifulSoup(YOUR_MARKUP})\n",
      "\n",
      "to this:\n",
      "\n",
      " BeautifulSoup(YOUR_MARKUP, \"lxml\")\n",
      "\n",
      "  markup_type=markup_type))\n"
     ]
    }
   ],
   "source": [
    "for i in range (0,3):\n",
    "    raw_to_preprocessed(train[\"review\"][i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "no.of rows reviews processed: 0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "E:\\Anaconda3\\lib\\site-packages\\bs4\\__init__.py:181: UserWarning: No parser was explicitly specified, so I'm using the best available HTML parser for this system (\"lxml\"). This usually isn't a problem, but if you run this code on another system, or in a different virtual environment, it may use a different parser and behave differently.\n",
      "\n",
      "The code that caused this warning is on line 193 of the file E:\\Anaconda3\\lib\\runpy.py. To get rid of this warning, change code that looks like this:\n",
      "\n",
      " BeautifulSoup(YOUR_MARKUP})\n",
      "\n",
      "to this:\n",
      "\n",
      " BeautifulSoup(YOUR_MARKUP, \"lxml\")\n",
      "\n",
      "  markup_type=markup_type))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "no.of rows reviews processed: 10000\n",
      "no.of rows reviews processed: 20000\n"
     ]
    }
   ],
   "source": [
    "# initialize an empty list and hold the clean reviews\n",
    "\n",
    "processed_train_reviews = []\n",
    "for i in range (0,num_reviews):\n",
    "    if i%10000 == 0:\n",
    "        print (\"no.of rows reviews processed:\",i)\n",
    "    processed_train_reviews.append(raw_to_prepwithtokenize(train[\"review\"][i]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'stuff going moment mj started listening music watching odd documentary watched wiz watched moonwalker maybe want get certain insight guy thought really cool eighties maybe make mind whether guilty innocent moonwalker part biography part feature film remember going see cinema originally released subtle messages mj feeling towards press also obvious message drugs bad kay visually impressive course michael jackson unless remotely like mj anyway going hate find boring may call mj egotist consenting making movie mj fans would say made fans true really nice actual feature film bit finally starts minutes excluding smooth criminal sequence joe pesci convincing psychopathic powerful drug lord wants mj dead bad beyond mj overheard plans nah joe pesci character ranted wanted people know supplying drugs etc dunno maybe hates mj music lots cool things like mj turning car robot whole speed demon sequence also director must patience saint came filming kiddy bad sequence usually directors hate working one kid let alone whole bunch performing complex dance scene bottom line movie people like mj one level another think people stay away try give wholesome message ironically mj bestest buddy movie girl michael jackson truly one talented people ever grace planet guilty well attention gave subject hmmm well know people different behind closed doors know fact either extremely nice stupid guy one sickest liars hope latter'"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "processed_train_reviews[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processed test reviews: 0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "E:\\Anaconda3\\lib\\site-packages\\bs4\\__init__.py:181: UserWarning: No parser was explicitly specified, so I'm using the best available HTML parser for this system (\"lxml\"). This usually isn't a problem, but if you run this code on another system, or in a different virtual environment, it may use a different parser and behave differently.\n",
      "\n",
      "The code that caused this warning is on line 193 of the file E:\\Anaconda3\\lib\\runpy.py. To get rid of this warning, change code that looks like this:\n",
      "\n",
      " BeautifulSoup(YOUR_MARKUP})\n",
      "\n",
      "to this:\n",
      "\n",
      " BeautifulSoup(YOUR_MARKUP, \"lxml\")\n",
      "\n",
      "  markup_type=markup_type))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processed test reviews: 5000\n",
      "processed test reviews: 10000\n",
      "processed test reviews: 15000\n",
      "processed test reviews: 20000\n"
     ]
    }
   ],
   "source": [
    "# similarly doing the same to test data\n",
    "processed_test_reviews = []\n",
    "for i in range(0,len(test[\"review\"])):\n",
    "    if(i%5000 == 0):\n",
    "        print(\"processed test reviews:\",i)\n",
    "    processed_test_reviews.append(raw_to_prepwithtokenize(test[\"review\"][i]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#using TFIDF word embedding\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "tfidf_vectorizer = TfidfVectorizer(ngram_range=(1,2),max_features= 1000)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tfidf_train_data = tfidf_vectorizer.fit_transform(processed_train_reviews)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dense_train_data = tfidf_train_data.todense()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(25000, 1000)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dense_train_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tfidf_test_data = tfidf_vectorizer.fit_transform(processed_test_reviews)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dense_test_data = tfidf_test_data.todense()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(25000, 1000)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dense_test_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1000\n",
      "['able', 'absolutely', 'across', 'act', 'acted', 'acting', 'action', 'actor', 'actors', 'actress', 'actual', 'actually', 'add', 'admit', 'age', 'ago', 'agree', 'air', 'alien', 'alive', 'almost', 'alone', 'along', 'already', 'also', 'although', 'always', 'amazing', 'america', 'american', 'among', 'amount', 'animation', 'annoying', 'another', 'anti', 'anyone', 'anything', 'anyway', 'apart', 'apparently', 'appear', 'appears', 'appreciate', 'around', 'art', 'ask', 'atmosphere', 'attempt', 'attempts']\n"
     ]
    }
   ],
   "source": [
    "feature_names = tfidf_vectorizer.get_feature_names()\n",
    "print(len(feature_names))\n",
    "print(feature_names[:50])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "model_random = RandomForestClassifier(n_estimators = 500)\n",
    "model_fit = model_random.fit(dense_train_data,train['sentiment'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pred_test = model_fit.predict(dense_test_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#copy it to dataframe\n",
    "output = pd.DataFrame(data={\"id\":test[\"id\"], \"sentiment\":pred_test})\n",
    "#Use pandas to write the comma-separated output file\n",
    "output.to_csv( \"Bag_of_Words_model_tfidf.csv\", index=False, quoting=3 )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pred_train = model_fit.predict(dense_train_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'             precision    recall  f1-score   support\\n\\n          0       1.00      1.00      1.00     12500\\n          1       1.00      1.00      1.00     12500\\n\\navg / total       1.00      1.00      1.00     25000\\n'"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import accuracy_score,classification_report\n",
    "accur = accuracy_score(train['sentiment'],pred_train)\n",
    "classification_report(train['sentiment'],pred_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accur"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "#using nueral networks\n",
    "import keras"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from keras.models import Sequential\n",
    "from keras.layers import Dense"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = Sequential()\n",
    "model.add(Dense(units=10,kernel_initializer='uniform',activation='relu',input_dim = 1000))\n",
    "#model.add(Dense(units = 10, kernel_initializer='uniform',activation='relu'))\n",
    "model.add(Dense(units = 1,kernel_initializer='uniform',activation='sigmoid'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.compile(optimizer='adam',loss='binary_crossentropy',metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/30\n",
      "25000/25000 [==============================] - 3s - loss: 0.4927 - acc: 0.8114     \n",
      "Epoch 2/30\n",
      "25000/25000 [==============================] - 1s - loss: 0.3262 - acc: 0.8618     \n",
      "Epoch 3/30\n",
      "25000/25000 [==============================] - 1s - loss: 0.3081 - acc: 0.8665     \n",
      "Epoch 4/30\n",
      "25000/25000 [==============================] - 1s - loss: 0.3031 - acc: 0.8697     - ETA: 1s - loss: 0\n",
      "Epoch 5/30\n",
      "25000/25000 [==============================] - 1s - loss: 0.3006 - acc: 0.8703     \n",
      "Epoch 6/30\n",
      "25000/25000 [==============================] - 2s - loss: 0.2994 - acc: 0.8710     \n",
      "Epoch 7/30\n",
      "25000/25000 [==============================] - 2s - loss: 0.2989 - acc: 0.8724     \n",
      "Epoch 8/30\n",
      "25000/25000 [==============================] - 1s - loss: 0.2985 - acc: 0.8719     \n",
      "Epoch 9/30\n",
      "25000/25000 [==============================] - 1s - loss: 0.2980 - acc: 0.8714     \n",
      "Epoch 10/30\n",
      "25000/25000 [==============================] - 1s - loss: 0.2977 - acc: 0.8717     \n",
      "Epoch 11/30\n",
      "25000/25000 [==============================] - 1s - loss: 0.2972 - acc: 0.8719     \n",
      "Epoch 12/30\n",
      "25000/25000 [==============================] - 1s - loss: 0.2968 - acc: 0.8722     \n",
      "Epoch 13/30\n",
      "25000/25000 [==============================] - 1s - loss: 0.2966 - acc: 0.8732     \n",
      "Epoch 14/30\n",
      "25000/25000 [==============================] - 1s - loss: 0.2962 - acc: 0.8720     \n",
      "Epoch 15/30\n",
      "25000/25000 [==============================] - 1s - loss: 0.2957 - acc: 0.8730     \n",
      "Epoch 16/30\n",
      "25000/25000 [==============================] - 1s - loss: 0.2955 - acc: 0.8723     \n",
      "Epoch 17/30\n",
      "25000/25000 [==============================] - 1s - loss: 0.2949 - acc: 0.8724     \n",
      "Epoch 18/30\n",
      "25000/25000 [==============================] - 1s - loss: 0.2944 - acc: 0.8728     \n",
      "Epoch 19/30\n",
      "25000/25000 [==============================] - 1s - loss: 0.2939 - acc: 0.8722     \n",
      "Epoch 20/30\n",
      "25000/25000 [==============================] - 1s - loss: 0.2932 - acc: 0.8731     \n",
      "Epoch 21/30\n",
      "25000/25000 [==============================] - 1s - loss: 0.2925 - acc: 0.8731     - ETA: 0s - loss: 0.2892 \n",
      "Epoch 22/30\n",
      "25000/25000 [==============================] - 2s - loss: 0.2918 - acc: 0.8729     \n",
      "Epoch 23/30\n",
      "25000/25000 [==============================] - 1s - loss: 0.2909 - acc: 0.8739     \n",
      "Epoch 24/30\n",
      "25000/25000 [==============================] - 2s - loss: 0.2898 - acc: 0.8738     \n",
      "Epoch 25/30\n",
      "25000/25000 [==============================] - 2s - loss: 0.2889 - acc: 0.8742     \n",
      "Epoch 26/30\n",
      "25000/25000 [==============================] - 2s - loss: 0.2879 - acc: 0.8748     \n",
      "Epoch 27/30\n",
      "25000/25000 [==============================] - 2s - loss: 0.2866 - acc: 0.8740     \n",
      "Epoch 28/30\n",
      "25000/25000 [==============================] - 2s - loss: 0.2854 - acc: 0.8761     - ET\n",
      "Epoch 29/30\n",
      "25000/25000 [==============================] - 2s - loss: 0.2840 - acc: 0.8756     \n",
      "Epoch 30/30\n",
      "25000/25000 [==============================] - 2s - loss: 0.2824 - acc: 0.8768     \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x1cb88f33cc0>"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(dense_train_data,train[\"sentiment\"],batch_size=32,epochs=30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "23264/25000 [==========================>...] - ETA: 0s"
     ]
    }
   ],
   "source": [
    "pred_test = model.predict_classes(dense_test_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 0, 0, ..., 0, 0, 1], dtype=int64)"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#copy it to dataframe\n",
    "output = pd.DataFrame(data={\"id\":test[\"id\"], \"sentiment\":pred_test[:,0]})\n",
    "#Use pandas to write the comma-separated output file\n",
    "output.to_csv( \"Bag_of_Words_model.csv\", index=False, quoting=3 )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}