Gensim Model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "usual-choir",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "# Matplot\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.metrics import confusion_matrix, classification_report, accuracy_score\n",
    "from sklearn.manifold import TSNE\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Activation, Dense, Dropout, Embedding, Flatten, Conv1D, MaxPooling1D, LSTM\n",
    "from keras import utils\n",
    "from keras.callbacks import ReduceLROnPlateau, EarlyStopping\n",
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "from  nltk.stem import SnowballStemmer\n",
    "import gensim\n",
    "import re\n",
    "import numpy as np\n",
    "import os\n",
    "from collections import Counter\n",
    "import logging\n",
    "import time\n",
    "import pickle\n",
    "import itertools"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "muslim-tradition",
   "metadata": {},
   "outputs": [],
   "source": [
    "DATASET_COLUMNS = [\"target\", \"ids\", \"date\", \"flag\", \"user\", \"text\"]\n",
    "DATASET_ENCODING = \"ISO-8859-1\"\n",
    "TRAIN_SIZE = 0.8\n",
    "\n",
    "# TEXT CLENAING\n",
    "TEXT_CLEANING_RE = \"@\\S+|https?:\\S+|http?:\\S|[^A-Za-z0-9]+\"\n",
    "\n",
    "# WORD2VEC \n",
    "W2V_SIZE = 300\n",
    "W2V_WINDOW = 7\n",
    "W2V_EPOCH = 32\n",
    "W2V_MIN_COUNT = 10\n",
    "\n",
    "# KERAS\n",
    "SEQUENCE_LENGTH = 300\n",
    "EPOCHS = 8\n",
    "BATCH_SIZE = 1024\n",
    "\n",
    "# SENTIMENT\n",
    "POSITIVE = \"POSITIVE\"\n",
    "NEGATIVE = \"NEGATIVE\"\n",
    "NEUTRAL = \"NEUTRAL\"\n",
    "SENTIMENT_THRESHOLDS = (0.4, 0.7)\n",
    "\n",
    "# EXPORT\n",
    "KERAS_MODEL = \"model.h5\"\n",
    "WORD2VEC_MODEL = \"model.w2v\"\n",
    "TOKENIZER_MODEL = \"tokenizer.pkl\"\n",
    "ENCODER_MODEL = \"encoder.pkl\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "featured-method",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('./dataset/tweets.csv', encoding =DATASET_ENCODING , names=DATASET_COLUMNS)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "mature-comparison",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>target</th>\n",
       "      <th>ids</th>\n",
       "      <th>date</th>\n",
       "      <th>flag</th>\n",
       "      <th>user</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>1467810369</td>\n",
       "      <td>Mon Apr 06 22:19:45 PDT 2009</td>\n",
       "      <td>NO_QUERY</td>\n",
       "      <td>_TheSpecialOne_</td>\n",
       "      <td>@switchfoot http://twitpic.com/2y1zl - Awww, t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>1467810672</td>\n",
       "      <td>Mon Apr 06 22:19:49 PDT 2009</td>\n",
       "      <td>NO_QUERY</td>\n",
       "      <td>scotthamilton</td>\n",
       "      <td>is upset that he can't update his Facebook by ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>1467810917</td>\n",
       "      <td>Mon Apr 06 22:19:53 PDT 2009</td>\n",
       "      <td>NO_QUERY</td>\n",
       "      <td>mattycus</td>\n",
       "      <td>@Kenichan I dived many times for the ball. Man...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>1467811184</td>\n",
       "      <td>Mon Apr 06 22:19:57 PDT 2009</td>\n",
       "      <td>NO_QUERY</td>\n",
       "      <td>ElleCTF</td>\n",
       "      <td>my whole body feels itchy and like its on fire</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>1467811193</td>\n",
       "      <td>Mon Apr 06 22:19:57 PDT 2009</td>\n",
       "      <td>NO_QUERY</td>\n",
       "      <td>Karoli</td>\n",
       "      <td>@nationwideclass no, it's not behaving at all....</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   target         ids                          date      flag  \\\n",
       "0       0  1467810369  Mon Apr 06 22:19:45 PDT 2009  NO_QUERY   \n",
       "1       0  1467810672  Mon Apr 06 22:19:49 PDT 2009  NO_QUERY   \n",
       "2       0  1467810917  Mon Apr 06 22:19:53 PDT 2009  NO_QUERY   \n",
       "3       0  1467811184  Mon Apr 06 22:19:57 PDT 2009  NO_QUERY   \n",
       "4       0  1467811193  Mon Apr 06 22:19:57 PDT 2009  NO_QUERY   \n",
       "\n",
       "              user                                               text  \n",
       "0  _TheSpecialOne_  @switchfoot http://twitpic.com/2y1zl - Awww, t...  \n",
       "1    scotthamilton  is upset that he can't update his Facebook by ...  \n",
       "2         mattycus  @Kenichan I dived many times for the ball. Man...  \n",
       "3          ElleCTF    my whole body feels itchy and like its on fire   \n",
       "4           Karoli  @nationwideclass no, it's not behaving at all....  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "contemporary-electric",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset size: 1600000\n"
     ]
    }
   ],
   "source": [
    "print(\"Dataset size:\", len(df))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "offshore-desert",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.utils import shuffle\n",
    "df = shuffle(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "greenhouse-assessment",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df[0:100000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "senior-rugby",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>target</th>\n",
       "      <th>ids</th>\n",
       "      <th>date</th>\n",
       "      <th>flag</th>\n",
       "      <th>user</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>29622</th>\n",
       "      <td>0</td>\n",
       "      <td>1563536351</td>\n",
       "      <td>Sun Apr 19 23:12:53 PDT 2009</td>\n",
       "      <td>NO_QUERY</td>\n",
       "      <td>nadiakovacs</td>\n",
       "      <td>I was dissapointed by The Criminal Intent prem...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>247888</th>\n",
       "      <td>0</td>\n",
       "      <td>1982578044</td>\n",
       "      <td>Sun May 31 11:29:23 PDT 2009</td>\n",
       "      <td>NO_QUERY</td>\n",
       "      <td>LadyProducHer</td>\n",
       "      <td>@ZaraGreen Hey sis remember that bid I put in ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>347108</th>\n",
       "      <td>0</td>\n",
       "      <td>2016361025</td>\n",
       "      <td>Wed Jun 03 06:38:34 PDT 2009</td>\n",
       "      <td>NO_QUERY</td>\n",
       "      <td>Mexi_Flip</td>\n",
       "      <td>Legs feel wobbly  thank you twenty four cycle..</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1515473</th>\n",
       "      <td>4</td>\n",
       "      <td>2175608376</td>\n",
       "      <td>Mon Jun 15 01:28:25 PDT 2009</td>\n",
       "      <td>NO_QUERY</td>\n",
       "      <td>maaabelle</td>\n",
       "      <td>yay i got a kitkat and a marshmallow</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>582697</th>\n",
       "      <td>0</td>\n",
       "      <td>2214704665</td>\n",
       "      <td>Wed Jun 17 17:07:01 PDT 2009</td>\n",
       "      <td>NO_QUERY</td>\n",
       "      <td>julietlandau</td>\n",
       "      <td>Working out only made me grumpier about loss o...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         target         ids                          date      flag  \\\n",
       "29622         0  1563536351  Sun Apr 19 23:12:53 PDT 2009  NO_QUERY   \n",
       "247888        0  1982578044  Sun May 31 11:29:23 PDT 2009  NO_QUERY   \n",
       "347108        0  2016361025  Wed Jun 03 06:38:34 PDT 2009  NO_QUERY   \n",
       "1515473       4  2175608376  Mon Jun 15 01:28:25 PDT 2009  NO_QUERY   \n",
       "582697        0  2214704665  Wed Jun 17 17:07:01 PDT 2009  NO_QUERY   \n",
       "\n",
       "                  user                                               text  \n",
       "29622      nadiakovacs  I was dissapointed by The Criminal Intent prem...  \n",
       "247888   LadyProducHer  @ZaraGreen Hey sis remember that bid I put in ...  \n",
       "347108       Mexi_Flip    Legs feel wobbly  thank you twenty four cycle..  \n",
       "1515473      maaabelle              yay i got a kitkat and a marshmallow   \n",
       "582697    julietlandau  Working out only made me grumpier about loss o...  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "characteristic-township",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(100000, 6)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "mysterious-thread",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "target    0\n",
       "ids       0\n",
       "date      0\n",
       "flag      0\n",
       "user      0\n",
       "text      0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df['target']=='NEGATIVE'].count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "powered-partnership",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "target    0\n",
       "ids       0\n",
       "date      0\n",
       "flag      0\n",
       "user      0\n",
       "text      0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df['target']=='POSITIVE'].count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "second-focus",
   "metadata": {},
   "outputs": [],
   "source": [
    "decode_map = {0: \"NEGATIVE\", 2: \"NEUTRAL\", 4: \"POSITIVE\"}\n",
    "def decode_sentiment(label):\n",
    "    return decode_map[int(label)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "level-greek",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 33.3 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "df.target = df.target.apply(lambda x: decode_sentiment(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "impossible-secret",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 1.0, 'Dataset labels distribuition')"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA7IAAAHiCAYAAADVkfAqAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAiUUlEQVR4nO3df7jtdV3n/ddbjqGloMiPGznksTyVYOkEITM2jYWTVBbe3ZjHLGluZrhz7OrHNJNYXZPNDKNOU5aXP+6LO0s0FclKyLJbwsycCDsaE6IxHhPlCMFRFDGFBN/zx/rsXGz2OWefw49zPtvH47r2tdf6fH991kJdPvl+13dXdwcAAABm8YADPQEAAADYF0IWAACAqQhZAAAApiJkAQAAmIqQBQAAYCpCFgAAgKkIWQDYR1X1wqr6rXWu+5qq+i/7eZz93nbVft5ZVf96PH52Vb39nu5zad9XV9WTx+N1vy9r7OefV9U1e1j+1VX12ao6ZP9mCsBGImQBuF9U1bVV9fmqurWqPl1Vf15VP1pV6/osqqotVdVVtek+nuf9cpwDpbtf393fubf11hvR3X1id7/zXpjXn3X31y8d/9qqesrS8o9190O6+857eiwA5idkAbg/fW93PzTJo5K8OMnzk7z6wE6J/bFRQx+AOQhZAO533X1Ld1+S5JlJzqqqxyVJVX1PVf1VVX2mqq6rqhcubfau8fvT4xLTf1pVX1tV76iqT1bVJ6rq9VX1sJUNqur5VfXxcRb4mqo6bYw/oKrOraoPj20vqqojdnecvb2eqvrtqvq7qrqlqt5VVSeuWuXIqrp0zONPq+pRS9t+w1h285jjD+zmGEdW1VvH2eybq+rPdnc2u6r+ZVX9zZjPy5PU0rIfqap3j8dVVS+tqpvGun9dVY+rqnOSPDvJz4z34PfH+teO9/Svk/x9VW1afeY0yYOq6k3jtb6vqh6/dOyuqscsPf/Hs75V9eSq2jkevy7JVyf5/XH8n1l9pryqHllVl4z3YkdV/Zul/b5w/DN97ZjH1VV18u7/CQIwGyELwAHT3e9JsjPJPx9Df5/kOUkeluR7kjy3qp4+ln3b+P2wcYnp5VkE2ouSPDLJY5Mcn+SFSVJVX5/kx5J8yzgL/NQk1459/HiSpyf5F2PbTyV5xR6OszdvS7I1ydFJ3pfk9auWPzvJf05yZJIrV5ZX1VcluTTJG8a2z0ryyjVCOEl+Oov36qgkxyT52SS9eqWqOjLJ7yT5+XG8Dyd50m7m/Z1ZvN6vy+I9f2aST3b3+WOO/228B9+7tM2zsvhn87DuvmONfZ6R5LeTHDFe11uq6oG7Of6auvuHk3wsizP4D+nu/7bGam/M4v14ZJIzk/zXlX9RMXxfkgvH67okycv3ZQ4AHNyELAAH2vVZRE+6+53dfVV3f7G7/zqLWPkXu9uwu3d096XdfXt370ryK0vr35nk0CQnVNUDu/va7v7wWPb/JPm57t7Z3bdnEb9n7u/lst39G91969K+Hl9Vhy+t8gfd/a6x/OeS/NOqOj7J05Jc292/2d13dPf7sojQM9c4zBeSHJvkUd39hfGd0ruFbJLvTvKB7n5zd38hya8m+bvdTP0LSR6a5BuSVHd/sLtv2MvLfVl3X9fdn9/N8vcuHftXkjwoyal72ec+Ge/dtyZ5fnff1t1XJvn1JD+8tNq7u/sPx3dqX5fk8XffEwCzErIAHGjHJbk5SarqiVX1J1W1q6puSfKjWZxVXFNVHV1VF47Lhz+T5LdW1u/uHUl+MouwvGms98ix6aOS/N64TPfTST6YRfges6+Tr6pDqurF4zLlz+RLZ32X533dyoPu/ux4vY8c83jiyjzGXJ6d5P9Y41C/lGRHkrdX1d9W1bm7mdIjVx2vl58v6+53ZHGm8hVJbqyq86vqsL285DX3tdby7v5ivnTW9N70yCQ3d/etS2MfzeI/SyuW4/1zWVzy7Hu9ABuEkAXggKmqb8kiPt49ht6QxWWgx3f34Un+33zp+51rnX180Rj/pu4+LMkPLa2f7n5Dd39rFsHYSV4yFl2X5Lu6+2FLPw/q7o/v5jh78oNZXE77lCSHJ9my8vKW1jl+6TU/JIsz0NePefzpqnk8pLufu/og44zvT3f31yT53iT/btWltCtuWHW8Wn6+xn5f1t0nJTkxi0uM/8PKot1tsrt9DcvHfkCSzVm81mQRlF+5tO5awb6e41yf5IiqeujS2Fcn+fhe5gbABiFkAbjfVdVhVfW0LL7D+FvdfdVY9NAszrTdVlWnZBGJK3Yl+WKSr1kae2iSz2ZxY6bj8qUIS1V9fVV9R1UdmuS2JJ/P4qxrsgjk81ZuulRVR1XVGXs4zp48NMntST6ZRaT91zXW+e6q+taq+oosvit7RXdfl+StSb6uqn64qh44fr6lqh67egdV9bSqeswI08+M17LWn6L5gyQnVtX3jzOQP57dBOM41hPHd1j/Pov3aWWfN+7De7DspKVj/2QW781fjGVXJvnBcRb79OzhsvE9HX+8d3+e5EVV9aCq+qYkZ+fu300GYIMSsgDcn36/qm7N4kzkz2XxHcp/tbT83yb5T2Od/5jkopUF3f25JOcl+R/jMtxTk/xikm9OcksWAfe7S/s6NIs/8fOJLC4zPTqLGyQlya9lceb37eNYf5HkiXs4zp68NovLWj+e5AP5UrQte0OSX8jikuKTsrh8OOPS2O9Msi2Ls4x/l8VZ40PX2MfWJH+cRbhfnuSVa/391u7+RJJnjNf+ybHd/9jN3A9L8v9lcbOrj471//tY9uosvl/86ap6y262X8vFWdw06lNZfGf1+8f3ZZPkJ7I4m/zpLN6DPe33RUl+fhz/36+x/FlZnP2+PsnvJfmF7r50H+YJwMRq7ftEAAAAwMHJGVkAAACmImQBAACYipAFAABgKkIWAACAqQhZAAAAprLpQE9gfx155JG9ZcuWAz0NAAAA7gPvfe97P9HdR621bNqQ3bJlS7Zv336gpwEAAMB9oKo+urtlLi0GAABgKkIWAACAqQhZAAAApiJkAQAAmIqQBQAAYCpCFgAAgKkIWQAAAKYiZAEAAJiKkAUAAGAqQhYAAICpCFkAAACmImQBAACYipAFAABgKkIWAACAqQhZAAAApiJkAQAAmIqQBQAAYCrrCtmquraqrqqqK6tq+xg7oqouraoPjd8PX1r/BVW1o6quqaqnLo2fNPazo6peVlU1xg+tqjeN8Suqasu9/DoBAADYIPbljOy3d/cTuvvk8fzcJJd199Ykl43nqaoTkmxLcmKS05O8sqoOGdu8Ksk5SbaOn9PH+NlJPtXdj0ny0iQv2f+XBAAAwEa26R5se0aSJ4/HFyR5Z5Lnj/ELu/v2JB+pqh1JTqmqa5Mc1t2XJ0lVvTbJ05O8bWzzwrGvNyd5eVVVd/c9mB8AcIBtOfcPDvQUAFjDtS/+ngM9hXtkvWdkO8nbq+q9VXXOGDumu29IkvH76DF+XJLrlrbdOcaOG49Xj99lm+6+I8ktSR6xehJVdU5Vba+q7bt27Vrn1AEAANhI1ntG9kndfX1VHZ3k0qr6mz2sW2uM9R7G97TNXQe6z09yfpKcfPLJztYCAAB8GVpXyHb39eP3TVX1e0lOSXJjVR3b3TdU1bFJbhqr70xy/NLmm5NcP8Y3rzG+vM3OqtqU5PAkN+/fSzp4uJwK4OA0++VUAPDlbq+XFlfVV1XVQ1ceJ/nOJO9PckmSs8ZqZyW5eDy+JMm2cSfiR2dxU6f3jMuPb62qU8fdip+zapuVfZ2Z5B2+HwsAAMBa1nNG9pgkvzf+Us6mJG/o7j+qqr9MclFVnZ3kY0mekSTdfXVVXZTkA0nuSPK87r5z7Ou5SV6T5MFZ3OTpbWP81UleN24MdXMWdz0GAACAu9lryHb33yZ5/Brjn0xy2m62OS/JeWuMb0/yuDXGb8sIYQAAANiTffk7sgAAAHDACVkAAACmImQBAACYipAFAABgKkIWAACAqQhZAAAApiJkAQAAmIqQBQAAYCpCFgAAgKkIWQAAAKYiZAEAAJiKkAUAAGAqQhYAAICpCFkAAACmImQBAACYipAFAABgKkIWAACAqQhZAAAApiJkAQAAmIqQBQAAYCpCFgAAgKkIWQAAAKYiZAEAAJiKkAUAAGAqQhYAAICpCFkAAACmImQBAACYipAFAABgKkIWAACAqQhZAAAApiJkAQAAmIqQBQAAYCpCFgAAgKkIWQAAAKYiZAEAAJiKkAUAAGAqQhYAAICpCFkAAACmImQBAACYipAFAABgKkIWAACAqQhZAAAApiJkAQAAmIqQBQAAYCpCFgAAgKkIWQAAAKYiZAEAAJiKkAUAAGAqQhYAAICpCFkAAACmImQBAACYipAFAABgKkIWAACAqQhZAAAApiJkAQAAmIqQBQAAYCpCFgAAgKkIWQAAAKYiZAEAAJiKkAUAAGAqQhYAAICpCFkAAACmImQBAACYipAFAABgKkIWAACAqQhZAAAApiJkAQAAmIqQBQAAYCpCFgAAgKkIWQAAAKYiZAEAAJiKkAUAAGAqQhYAAICprDtkq+qQqvqrqnrreH5EVV1aVR8avx++tO4LqmpHVV1TVU9dGj+pqq4ay15WVTXGD62qN43xK6pqy734GgEAANhA9uWM7E8k+eDS83OTXNbdW5NcNp6nqk5Isi3JiUlOT/LKqjpkbPOqJOck2Tp+Th/jZyf5VHc/JslLk7xkv14NAAAAG966QraqNif5niS/vjR8RpILxuMLkjx9afzC7r69uz+SZEeSU6rq2CSHdffl3d1JXrtqm5V9vTnJaStnawEAAGDZes/I/mqSn0nyxaWxY7r7hiQZv48e48cluW5pvZ1j7LjxePX4Xbbp7juS3JLkEet9EQAAAHz52GvIVtXTktzU3e9d5z7XOpPaexjf0zar53JOVW2vqu27du1a53QAAADYSNZzRvZJSb6vqq5NcmGS76iq30py47hcOOP3TWP9nUmOX9p+c5Lrx/jmNcbvsk1VbUpyeJKbV0+ku8/v7pO7++SjjjpqXS8QAACAjWWvIdvdL+juzd29JYubOL2ju38oySVJzhqrnZXk4vH4kiTbxp2IH53FTZ3eMy4/vrWqTh3ff33Oqm1W9nXmOMbdzsgCAADApnuw7YuTXFRVZyf5WJJnJEl3X11VFyX5QJI7kjyvu+8c2zw3yWuSPDjJ28ZPkrw6yeuqakcWZ2K33YN5AQAAsIHtU8h29zuTvHM8/mSS03az3nlJzltjfHuSx60xfltGCAMAAMCe7MvfkQUAAIADTsgCAAAwFSELAADAVIQsAAAAUxGyAAAATEXIAgAAMBUhCwAAwFSELAAAAFMRsgAAAExFyAIAADAVIQsAAMBUhCwAAABTEbIAAABMRcgCAAAwFSELAADAVIQsAAAAUxGyAAAATEXIAgAAMBUhCwAAwFSELAAAAFMRsgAAAExFyAIAADAVIQsAAMBUhCwAAABTEbIAAABMRcgCAAAwFSELAADAVIQsAAAAUxGyAAAATEXIAgAAMBUhCwAAwFSELAAAAFMRsgAAAExFyAIAADAVIQsAAMBUhCwAAABTEbIAAABMRcgCAAAwFSELAADAVIQsAAAAUxGyAAAATEXIAgAAMBUhCwAAwFSELAAAAFMRsgAAAExFyAIAADAVIQsAAMBUhCwAAABTEbIAAABMRcgCAAAwFSELAADAVIQsAAAAUxGyAAAATEXIAgAAMBUhCwAAwFSELAAAAFMRsgAAAExFyAIAADAVIQsAAMBUhCwAAABTEbIAAABMRcgCAAAwFSELAADAVIQsAAAAUxGyAAAATEXIAgAAMBUhCwAAwFSELAAAAFMRsgAAAExFyAIAADAVIQsAAMBUhCwAAABTEbIAAABMRcgCAAAwlb2GbFU9qKreU1X/s6qurqpfHONHVNWlVfWh8fvhS9u8oKp2VNU1VfXUpfGTquqqsexlVVVj/NCqetMYv6KqttwHrxUAAIANYD1nZG9P8h3d/fgkT0hyelWdmuTcJJd199Ykl43nqaoTkmxLcmKS05O8sqoOGft6VZJzkmwdP6eP8bOTfKq7H5PkpUlecs9fGgAAABvRXkO2Fz47nj5w/HSSM5JcMMYvSPL08fiMJBd29+3d/ZEkO5KcUlXHJjmsuy/v7k7y2lXbrOzrzUlOWzlbCwAAAMvW9R3Zqjqkqq5MclOSS7v7iiTHdPcNSTJ+Hz1WPy7JdUub7xxjx43Hq8fvsk1335HkliSPWGMe51TV9qravmvXrnW9QAAAADaWdYVsd9/Z3U9IsjmLs6uP28Pqa51J7T2M72mb1fM4v7tP7u6TjzrqqL3MGgAAgI1on+5a3N2fTvLOLL7beuO4XDjj901jtZ1Jjl/abHOS68f45jXG77JNVW1KcniSm/dlbgAAAHx5WM9di4+qqoeNxw9O8pQkf5PkkiRnjdXOSnLxeHxJkm3jTsSPzuKmTu8Zlx/fWlWnju+/PmfVNiv7OjPJO8b3aAEAAOAuNq1jnWOTXDDuPPyAJBd191ur6vIkF1XV2Uk+luQZSdLdV1fVRUk+kOSOJM/r7jvHvp6b5DVJHpzkbeMnSV6d5HVVtSOLM7Hb7o0XBwAAwMaz15Dt7r9O8k/WGP9kktN2s815Sc5bY3x7krt9v7a7b8sIYQAAANiTffqOLAAAABxoQhYAAICpCFkAAACmImQBAACYipAFAABgKkIWAACAqQhZAAAApiJkAQAAmIqQBQAAYCpCFgAAgKkIWQAAAKYiZAEAAJiKkAUAAGAqQhYAAICpCFkAAACmImQBAACYipAFAABgKkIWAACAqQhZAAAApiJkAQAAmIqQBQAAYCpCFgAAgKkIWQAAAKYiZAEAAJiKkAUAAGAqQhYAAICpCFkAAACmImQBAACYipAFAABgKkIWAACAqQhZAAAApiJkAQAAmIqQBQAAYCpCFgAAgKkIWQAAAKYiZAEAAJiKkAUAAGAqQhYAAICpCFkAAACmImQBAACYipAFAABgKkIWAACAqQhZAAAApiJkAQAAmIqQBQAAYCpCFgAAgKkIWQAAAKYiZAEAAJiKkAUAAGAqQhYAAICpCFkAAACmImQBAACYipAFAABgKkIWAACAqQhZAAAApiJkAQAAmIqQBQAAYCpCFgAAgKkIWQAAAKYiZAEAAJiKkAUAAGAqQhYAAICpCFkAAACmImQBAACYipAFAABgKkIWAACAqQhZAAAApiJkAQAAmIqQBQAAYCpCFgAAgKkIWQAAAKYiZAEAAJiKkAUAAGAqew3Zqjq+qv6kqj5YVVdX1U+M8SOq6tKq+tD4/fClbV5QVTuq6pqqeurS+ElVddVY9rKqqjF+aFW9aYxfUVVb7oPXCgAAwAawnjOydyT56e5+bJJTkzyvqk5Icm6Sy7p7a5LLxvOMZduSnJjk9CSvrKpDxr5eleScJFvHz+lj/Owkn+ruxyR5aZKX3AuvDQAAgA1oryHb3Td09/vG41uTfDDJcUnOSHLBWO2CJE8fj89IcmF3397dH0myI8kpVXVsksO6+/Lu7iSvXbXNyr7enOS0lbO1AAAAsGyfviM7Lvn9J0muSHJMd9+QLGI3ydFjteOSXLe02c4xdtx4vHr8Ltt09x1JbknyiH2ZGwAAAF8e1h2yVfWQJL+T5Ce7+zN7WnWNsd7D+J62WT2Hc6pqe1Vt37Vr196mDAAAwAa0rpCtqgdmEbGv7+7fHcM3jsuFM37fNMZ3Jjl+afPNSa4f45vXGL/LNlW1KcnhSW5ePY/uPr+7T+7uk4866qj1TB0AAIANZj13La4kr07ywe7+laVFlyQ5azw+K8nFS+Pbxp2IH53FTZ3eMy4/vrWqTh37fM6qbVb2dWaSd4zv0QIAAMBdbFrHOk9K8sNJrqqqK8fYzyZ5cZKLqursJB9L8owk6e6rq+qiJB/I4o7Hz+vuO8d2z03ymiQPTvK28ZMsQvl1VbUjizOx2+7ZywIAAGCj2mvIdve7s/Z3WJPktN1sc16S89YY357kcWuM35YRwgAAALAn+3TXYgAAADjQhCwAAABTEbIAAABMRcgCAAAwFSELAADAVIQsAAAAUxGyAAAATEXIAgAAMBUhCwAAwFSELAAAAFMRsgAAAExFyAIAADAVIQsAAMBUhCwAAABTEbIAAABMRcgCAAAwFSELAADAVIQsAAAAUxGyAAAATEXIAgAAMBUhCwAAwFSELAAAAFMRsgAAAExFyAIAADAVIQsAAMBUhCwAAABTEbIAAABMRcgCAAAwFSELAADAVIQsAAAAUxGyAAAATEXIAgAAMBUhCwAAwFSELAAAAFMRsgAAAExFyAIAADAVIQsAAMBUhCwAAABTEbIAAABMRcgCAAAwFSELAADAVIQsAAAAUxGyAAAATEXIAgAAMBUhCwAAwFSELAAAAFMRsgAAAExFyAIAADAVIQsAAMBUhCwAAABTEbIAAABMRcgCAAAwFSELAADAVIQsAAAAUxGyAAAATEXIAgAAMBUhCwAAwFSELAAAAFMRsgAAAExFyAIAADAVIQsAAMBUhCwAAABTEbIAAABMRcgCAAAwFSELAADAVIQsAAAAUxGyAAAATEXIAgAAMBUhCwAAwFSELAAAAFMRsgAAAExFyAIAADAVIQsAAMBUhCwAAABT2WvIVtVvVNVNVfX+pbEjqurSqvrQ+P3wpWUvqKodVXVNVT11afykqrpqLHtZVdUYP7Sq3jTGr6iqLffyawQAAGADWc8Z2dckOX3V2LlJLuvurUkuG89TVSck2ZbkxLHNK6vqkLHNq5Kck2Tr+FnZ59lJPtXdj0ny0iQv2d8XAwAAwMa315Dt7ncluXnV8BlJLhiPL0jy9KXxC7v79u7+SJIdSU6pqmOTHNbdl3d3J3ntqm1W9vXmJKetnK0FAACA1fb3O7LHdPcNSTJ+Hz3Gj0ty3dJ6O8fYcePx6vG7bNPddyS5Jckj1jpoVZ1TVduravuuXbv2c+oAAADM7N6+2dNaZ1J7D+N72ubug93nd/fJ3X3yUUcdtZ9TBAAAYGb7G7I3jsuFM37fNMZ3Jjl+ab3NSa4f45vXGL/LNlW1KcnhufulzAAAAJBk/0P2kiRnjcdnJbl4aXzbuBPxo7O4qdN7xuXHt1bVqeP7r89Ztc3Kvs5M8o7xPVoAAAC4m017W6Gq3pjkyUmOrKqdSX4hyYuTXFRVZyf5WJJnJEl3X11VFyX5QJI7kjyvu+8cu3puFndAfnCSt42fJHl1ktdV1Y4szsRuu1deGQAAABvSXkO2u5+1m0Wn7Wb985Kct8b49iSPW2P8towQBgAAgL25t2/2BAAAAPcpIQsAAMBUhCwAAABTEbIAAABMRcgCAAAwFSELAADAVIQsAAAAUxGyAAAATEXIAgAAMBUhCwAAwFSELAAAAFMRsgAAAExFyAIAADAVIQsAAMBUhCwAAABTEbIAAABMRcgCAAAwFSELAADAVIQsAAAAUxGyAAAATEXIAgAAMBUhCwAAwFSELAAAAFMRsgAAAExFyAIAADAVIQsAAMBUhCwAAABTEbIAAABMRcgCAAAwFSELAADAVIQsAAAAUxGyAAAATEXIAgAAMBUhCwAAwFSELAAAAFMRsgAAAExFyAIAADAVIQsAAMBUhCwAAABTEbIAAABMRcgCAAAwFSELAADAVIQsAAAAUxGyAAAATEXIAgAAMBUhCwAAwFSELAAAAFMRsgAAAExFyAIAADAVIQsAAMBUhCwAAABTEbIAAABMRcgCAAAwFSELAADAVIQsAAAAUxGyAAAATEXIAgAAMBUhCwAAwFSELAAAAFMRsgAAAExFyAIAADAVIQsAAMBUhCwAAABTEbIAAABMRcgCAAAwFSELAADAVIQsAAAAUxGyAAAATEXIAgAAMBUhCwAAwFSELAAAAFMRsgAAAExFyAIAADCVgyZkq+r0qrqmqnZU1bkHej4AAAAcnA6KkK2qQ5K8Isl3JTkhybOq6oQDOysAAAAORgdFyCY5JcmO7v7b7v6HJBcmOeMAzwkAAICD0MESsscluW7p+c4xBgAAAHex6UBPYKg1xvpuK1Wdk+Sc8fSzVXXNfTorYNmRST5xoCcB94Z6yYGeATApn4VsGJN8Fj5qdwsOlpDdmeT4peebk1y/eqXuPj/J+ffXpIAvqart3X3ygZ4HABwoPgvh4HGwXFr8l0m2VtWjq+orkmxLcskBnhMAAAAHoYPijGx331FVP5bk/09ySJLf6O6rD/C0AAAAOAgdFCGbJN39h0n+8EDPA9gtl/UD8OXOZyEcJKr7bvdUAgAAgIPWwfIdWQAAAFgXIQsbRFV1Vf3y0vN/X1UvHI9fWFUfr6orl34eNpadUlXvrKoPVdX7quoPquobV+37f1bVG8fjf7W0j3+oqqvG4xdX1Y9U1cur6slVdfmqfWyqqhur6tiqek1VfWRpP39+X78/AHz5qKo7x+fL+6vqt6vqK8f45qq6eHzmfbiqfm3caDRV9ZVV9frxufb+qnp3VT1kLPtsVX3j0ufWzUufY39cVVvGNl9VVZ+sqsNXzectVfUD43Ny16rP4xPu/3cI5idkYeO4Pcn3V9WRu1n+0u5+wtLPp6vqmCQXJfnZ7t7a3d+c5EVJvnZlo6p6bBb/W/FtVfVV3f2bK/vI4s9kfft4fu7Ssd6VZHNVbVkae0qS93f3DeP5f1iayz+7F14/AKz4/Ph8eVySf0jyo1VVSX43yVu6e2uSr0vykCTnjW1+IsmN3f2NY7uzk3xhZYfdfdXS598l+dLn2FOW1vn7JG9P8vSVsRG135rkrWPoTas+jz9wX7wBsNEJWdg47sjiJhQ/tQ/b/FiSC7r7H8+Idve7u/stS+v8YJLXZfHB/H3r2Wl3fzHJbyd55tLwtiRv3Ie5AcC94c+SPCbJdyS5rbt/M0m6+84sPjP/73HG9tgkH1/ZqLuv6e7b9+N4b8ziM2/F/5nkj7r7c/s5f2ANQhY2llckefbqS5qGn1q6jOlPxtiJSd63l30+M8mbsvhgftY+zOUfP8ir6tAk353kd5aW/9LSfF6/D/sFgHWpqk1JvivJVVl85r13eXl3fybJx7II3d9I8vyquryq/ktVbd3Pw/5RkpOq6hHj+ep/kfvMVZcWP3g/jwNf1oQsbCDjA/m1SX58jcXLlxZ/+1rbV9UVVfXBqvq18fxbkuzq7o8muSzJN1fVw9c5l79M8pCq+vos/k/EX3T3p5ZWWb60+Nnrf5UAsFcPrqork2zPIlRfnaSSrPXnOipJd/eVSb4myS8lOSLJX46v1+yT7v6HLC49PnN83ecJWVzVtGL1pcWf39djAAfR35EF7jW/msVZ1t9cx7pXJ/nmJBcnSXc/sarOTPK0sfxZSb6hqq4dzw9L8n8l+fV1zuXCLP5N9GPjsmIA7j+fH99l/UdVdXUWn2HLY4clOT7Jh5Okuz+bxfdof7eqvpjF1UQf3I/jvzHJz2cRyRd39xf2sj6wj5yRhQ2mu2/O4gZOZ69j9Vck+ZGqWr7Z0sqdHR+Q5BlJvqm7t3T3liRnZN8vL/6hLL6XdMk+bAcA97bLknxlVT0nSarqkCS/nOQ13f25qnrSylVH407GJyT56H4e60+SbE3yvPgXuXCfELKwMf1yktV3L/6pVd/J2dLdf5fFd2BfVFU7xp/BOTPJy5N8W5KPd/fHl/bxriQnVNWx65nEuBPj55K8Y9zJcdkvrZrPV+zH6wSAdenuzuLGS8+oqg8l+V9Jbkvys2OVr03yp1V1VZK/yuKy5N9Za1/rONYXx7aPyOKzc9nq78i6cz/sh1r8dxoAAADm4IwsAAAAUxGyAAAATEXIAgAAMBUhCwAAwFSELAAAAFMRsgAAAExFyAIAADAVIQsAAMBU/jdYqPUay3Y1IQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 1152x576 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "target_cnt = Counter(df.target)\n",
    "\n",
    "plt.figure(figsize=(16,8))\n",
    "plt.bar(target_cnt.keys(), target_cnt.values())\n",
    "plt.title(\"Dataset labels distribuition\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "signal-regression",
   "metadata": {},
   "outputs": [],
   "source": [
    "stop_words = stopwords.words(\"english\")\n",
    "stemmer = SnowballStemmer(\"english\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "descending-trustee",
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess(text, stem=False):\n",
    "    # Remove link,user and special characters\n",
    "    text = re.sub(TEXT_CLEANING_RE, ' ', str(text).lower()).strip()\n",
    "    tokens = []\n",
    "    for token in text.split():\n",
    "        if token not in stop_words:\n",
    "            if stem:\n",
    "                tokens.append(stemmer.stem(token))\n",
    "            else:\n",
    "                tokens.append(token)\n",
    "    return \" \".join(tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "agreed-mistress",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 3.22 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "df.text = df.text.apply(lambda x: preprocess(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "ongoing-emission",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TRAIN size: 80000\n",
      "TEST size: 20000\n"
     ]
    }
   ],
   "source": [
    "df_train, df_test = train_test_split(df, test_size=1-TRAIN_SIZE, random_state=42)\n",
    "print(\"TRAIN size:\", len(df_train))\n",
    "print(\"TEST size:\", len(df_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "fundamental-congo",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 178 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "documents = [_text.split() for _text in df_train.text] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "spare-palestinian",
   "metadata": {},
   "outputs": [],
   "source": [
    "w2v_model = gensim.models.word2vec.Word2Vec(size=W2V_SIZE, \n",
    "                                            window=W2V_WINDOW, \n",
    "                                            min_count=W2V_MIN_COUNT, \n",
    "                                            workers=8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "surprised-crime",
   "metadata": {},
   "outputs": [],
   "source": [
    "w2v_model.build_vocab(documents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "signal-qatar",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Vocab size 5466\n"
     ]
    }
   ],
   "source": [
    "words = w2v_model.wv.vocab.keys()\n",
    "vocab_size = len(words)\n",
    "print(\"Vocab size\", vocab_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "passive-schema",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 16.1 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(14519474, 18420960)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "w2v_model.train(documents, total_examples=len(documents), epochs=W2V_EPOCH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "colored-hybrid",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<ipython-input-25-e8209ce8d2e1>:1: DeprecationWarning: Call to deprecated `most_similar` (Method will be removed in 4.0.0, use self.wv.most_similar() instead).\n",
      "  w2v_model.most_similar(\"peace\")\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[('goodnight', 0.47451621294021606),\n",
       " ('blessings', 0.44418567419052124),\n",
       " ('farrah', 0.43473100662231445),\n",
       " ('jesus', 0.4297372102737427),\n",
       " ('fawcett', 0.41378721594810486),\n",
       " ('twitterverse', 0.39191874861717224),\n",
       " ('gud', 0.387679785490036),\n",
       " ('waves', 0.38381022214889526),\n",
       " ('twitterville', 0.37875837087631226),\n",
       " ('ty', 0.3701634109020233)]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "w2v_model.most_similar(\"peace\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "exclusive-chain",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total words 50056\n",
      "Wall time: 1.24 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "tokenizer = Tokenizer()\n",
    "tokenizer.fit_on_texts(df_train.text)\n",
    "\n",
    "vocab_size = len(tokenizer.word_index) + 1\n",
    "print(\"Total words\", vocab_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "saved-quantity",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 1.57 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "x_train = pad_sequences(tokenizer.texts_to_sequences(df_train.text), maxlen=SEQUENCE_LENGTH)\n",
    "x_test = pad_sequences(tokenizer.texts_to_sequences(df_test.text), maxlen=SEQUENCE_LENGTH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "plain-decimal",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['NEGATIVE', 'POSITIVE', 'NEUTRAL']"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labels = df_train.target.unique().tolist()\n",
    "labels.append(NEUTRAL)\n",
    "labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "authentic-petersburg",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "y_train (80000, 1)\n",
      "y_test (20000, 1)\n"
     ]
    }
   ],
   "source": [
    "encoder = LabelEncoder()\n",
    "encoder.fit(df_train.target.tolist())\n",
    "\n",
    "y_train = encoder.transform(df_train.target.tolist())\n",
    "y_test = encoder.transform(df_test.target.tolist())\n",
    "\n",
    "y_train = y_train.reshape(-1,1)\n",
    "y_test = y_test.reshape(-1,1)\n",
    "\n",
    "print(\"y_train\",y_train.shape)\n",
    "print(\"y_test\",y_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "fossil-empire",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x_train (80000, 300)\n",
      "y_train (80000, 1)\n",
      "\n",
      "x_test (20000, 300)\n",
      "y_test (20000, 1)\n"
     ]
    }
   ],
   "source": [
    "print(\"x_train\", x_train.shape)\n",
    "print(\"y_train\", y_train.shape)\n",
    "print()\n",
    "print(\"x_test\", x_test.shape)\n",
    "print(\"y_test\", y_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "antique-asbestos",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(50056, 300)\n"
     ]
    }
   ],
   "source": [
    "embedding_matrix = np.zeros((vocab_size, W2V_SIZE))\n",
    "for word, i in tokenizer.word_index.items():\n",
    "  if word in w2v_model.wv:\n",
    "    embedding_matrix[i] = w2v_model.wv[word]\n",
    "print(embedding_matrix.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "catholic-playing",
   "metadata": {},
   "outputs": [],
   "source": [
    "embedding_layer = Embedding(vocab_size, W2V_SIZE, weights=[embedding_matrix], input_length=SEQUENCE_LENGTH, trainable=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "intended-doctor",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding (Embedding)        (None, 300, 300)          15016800  \n",
      "_________________________________________________________________\n",
      "dropout (Dropout)            (None, 300, 300)          0         \n",
      "_________________________________________________________________\n",
      "lstm (LSTM)                  (None, 100)               160400    \n",
      "_________________________________________________________________\n",
      "dense (Dense)                (None, 1)                 101       \n",
      "=================================================================\n",
      "Total params: 15,177,301\n",
      "Trainable params: 160,501\n",
      "Non-trainable params: 15,016,800\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model = Sequential()\n",
    "model.add(embedding_layer)\n",
    "model.add(Dropout(0.5))\n",
    "model.add(LSTM(100, dropout=0.2, recurrent_dropout=0.2))\n",
    "model.add(Dense(1, activation='sigmoid'))\n",
    "\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "underlying-papua",
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss='binary_crossentropy',\n",
    "              optimizer=\"adam\",\n",
    "              metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "rational-terrace",
   "metadata": {},
   "outputs": [],
   "source": [
    "callbacks = [ ReduceLROnPlateau(monitor='val_loss', patience=5, cooldown=0),\n",
    "              EarlyStopping(monitor='val_acc', min_delta=1e-4, patience=5)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "correct-paintball",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/8\n",
      " 4/36 [==>...........................] - ETA: 14:12 - loss: 0.6840 - accuracy: 0.5538"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "history = model.fit(x_train, y_train,\n",
    "                    batch_size=2048,\n",
    "                    epochs=EPOCHS,\n",
    "                    validation_split=0.1,\n",
    "                    verbose=1,\n",
    "                    callbacks=callbacks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "graphic-personal",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "score = model.evaluate(x_test, y_test, batch_size=2048)\n",
    "print()\n",
    "print(\"ACCURACY:\",score[1])\n",
    "print(\"LOSS:\",score[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "educated-updating",
   "metadata": {},
   "outputs": [],
   "source": [
    "acc = history.history['acc']\n",
    "val_acc = history.history['val_acc']\n",
    "loss = history.history['loss']\n",
    "val_loss = history.history['val_loss']\n",
    " \n",
    "epochs = range(len(acc))\n",
    " \n",
    "plt.plot(epochs, acc, 'b', label='Training acc')\n",
    "plt.plot(epochs, val_acc, 'r', label='Validation acc')\n",
    "plt.title('Training and validation accuracy')\n",
    "plt.legend()\n",
    " \n",
    "plt.figure()\n",
    " \n",
    "plt.plot(epochs, loss, 'b', label='Training loss')\n",
    "plt.plot(epochs, val_loss, 'r', label='Validation loss')\n",
    "plt.title('Training and validation loss')\n",
    "plt.legend()\n",
    " \n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "noted-slovenia",
   "metadata": {},
   "outputs": [],
   "source": [
    "def decode_sentiment(score, include_neutral=True):\n",
    "    if include_neutral:        \n",
    "        label = NEUTRAL\n",
    "        if score <= SENTIMENT_THRESHOLDS[0]:\n",
    "            label = NEGATIVE\n",
    "        elif score >= SENTIMENT_THRESHOLDS[1]:\n",
    "            label = POSITIVE\n",
    "\n",
    "        return label\n",
    "    else:\n",
    "        return NEGATIVE if score < 0.5 else POSITIVE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "backed-windows",
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict(text, include_neutral=True):\n",
    "    start_at = time.time()\n",
    "    # Tokenize text\n",
    "    x_test = pad_sequences(tokenizer.texts_to_sequences([text]), maxlen=SEQUENCE_LENGTH)\n",
    "    # Predict\n",
    "    score = model.predict([x_test])[0]\n",
    "    # Decode sentiment\n",
    "    label = decode_sentiment(score, include_neutral=include_neutral)\n",
    "\n",
    "    return {\"label\": label, \"score\": float(score),\n",
    "       \"elapsed_time\": time.time()-start_at}  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "tired-shell",
   "metadata": {},
   "outputs": [],
   "source": [
    "predict(\"I love the music\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "scientific-relationship",
   "metadata": {},
   "outputs": [],
   "source": [
    "predict(\"i don't know what i'm doing\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "painful-motion",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "y_pred_1d = []\n",
    "y_test_1d = list(df_test.target)\n",
    "scores = model.predict(x_test, verbose=1, batch_size=8000)\n",
    "y_pred_1d = [decode_sentiment(score, include_neutral=False) for score in scores]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "adjusted-protection",
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_confusion_matrix(cm, classes,\n",
    "                          title='Confusion matrix',\n",
    "                          cmap=plt.cm.Blues):\n",
    "    \"\"\"\n",
    "    This function prints and plots the confusion matrix.\n",
    "    Normalization can be applied by setting `normalize=True`.\n",
    "    \"\"\"\n",
    "\n",
    "    cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n",
    "\n",
    "    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n",
    "    plt.title(title, fontsize=30)\n",
    "    plt.colorbar()\n",
    "    tick_marks = np.arange(len(classes))\n",
    "    plt.xticks(tick_marks, classes, rotation=90, fontsize=22)\n",
    "    plt.yticks(tick_marks, classes, fontsize=22)\n",
    "\n",
    "    fmt = '.2f'\n",
    "    thresh = cm.max() / 2.\n",
    "    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n",
    "        plt.text(j, i, format(cm[i, j], fmt),\n",
    "                 horizontalalignment=\"center\",\n",
    "                 color=\"white\" if cm[i, j] > thresh else \"black\")\n",
    "\n",
    "    plt.ylabel('True label', fontsize=25)\n",
    "    plt.xlabel('Predicted label', fontsize=25)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "global-donna",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "cnf_matrix = confusion_matrix(y_test_1d, y_pred_1d)\n",
    "plt.figure(figsize=(12,12))\n",
    "plot_confusion_matrix(cnf_matrix, classes=df_train.target.unique(), title=\"Confusion matrix\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "convenient-immigration",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(classification_report(y_test_1d, y_pred_1d))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "architectural-mounting",
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save(KERAS_MODEL)\n",
    "w2v_model.save(WORD2VEC_MODEL)\n",
    "pickle.dump(tokenizer, open(TOKENIZER_MODEL, \"wb\"), protocol=0)\n",
    "pickle.dump(encoder, open(ENCODER_MODEL, \"wb\"), protocol=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "placed-algeria",
   "metadata": {},
   "outputs": [],
   "source": [
    "accuracy_score(y_test_1d, y_pred_1d)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "english-liquid",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}