Sentiment Analysis.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Introduction\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " I tried to predict the sentiment of tweets in Anaconda Kernel by using texts and basic Classifier .In this notebook I want to try whether we can outperform these model with a deep learning.\n",
    " \n",
    " \n",
    ".fit a deep learning model with keras\n",
    "\n",
    "\n",
    ".identify and deal with overfitting\n",
    "\n",
    ".use word embeddings\n",
    "\n",
    ".build on a pretrained model\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Set-up of the project"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We'll start by importing some packages."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "# Basic packages\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import re\n",
    "import collections\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "# Packages for data preparation\n",
    "from sklearn.model_selection import train_test_split\n",
    "from nltk.corpus import stopwords\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.utils.np_utils import to_categorical\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "\n",
    "# Packages for modeling\n",
    "from keras import models\n",
    "from keras import layers\n",
    "from keras import regularizers"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Then we set some parameters that will be used throughout the notebook."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "NB_WORDS = 10000  # Parameter indicating the number of words we'll put in the dictionary\n",
    "VAL_SIZE = 1000  # Size of the validation set\n",
    "NB_START_EPOCHS = 20  # Number of epochs we usually start to train with\n",
    "BATCH_SIZE = 512  # Size of the batches used in the mini-batch gradient descent"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We read in the csv with  the tweets data and perform a random shuffle.It's a good prectice to shuffle data before spiliting between a train and test set.That way the sentiment classes ae equally distibuted over the train and test sets.\n",
    "\n",
    "We will keep the text column as input and the airline_sentiment column as the target."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "ename": "FileNotFoundError",
     "evalue": "[Errno 2] File Tweets.csv does not exist: 'Tweets.csv'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-3-13c9c0564b97>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mdf\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mpd\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'Tweets.csv'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      2\u001b[0m \u001b[0mdisplay\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[0mdisplay\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mhead\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\pandas\\io\\parsers.py\u001b[0m in \u001b[0;36mparser_f\u001b[1;34m(filepath_or_buffer, sep, delimiter, header, names, index_col, usecols, squeeze, prefix, mangle_dupe_cols, dtype, engine, converters, true_values, false_values, skipinitialspace, skiprows, skipfooter, nrows, na_values, keep_default_na, na_filter, verbose, skip_blank_lines, parse_dates, infer_datetime_format, keep_date_col, date_parser, dayfirst, cache_dates, iterator, chunksize, compression, thousands, decimal, lineterminator, quotechar, quoting, doublequote, escapechar, comment, encoding, dialect, error_bad_lines, warn_bad_lines, delim_whitespace, low_memory, memory_map, float_precision)\u001b[0m\n\u001b[0;32m    674\u001b[0m         )\n\u001b[0;32m    675\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 676\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0m_read\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfilepath_or_buffer\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkwds\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    677\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    678\u001b[0m     \u001b[0mparser_f\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__name__\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mname\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\pandas\\io\\parsers.py\u001b[0m in \u001b[0;36m_read\u001b[1;34m(filepath_or_buffer, kwds)\u001b[0m\n\u001b[0;32m    446\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    447\u001b[0m     \u001b[1;31m# Create the parser.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 448\u001b[1;33m     \u001b[0mparser\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mTextFileReader\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfp_or_buf\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    449\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    450\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mchunksize\u001b[0m \u001b[1;32mor\u001b[0m \u001b[0miterator\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\pandas\\io\\parsers.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, f, engine, **kwds)\u001b[0m\n\u001b[0;32m    878\u001b[0m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0moptions\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"has_index_names\"\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mkwds\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"has_index_names\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    879\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 880\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_make_engine\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mengine\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    881\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    882\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mclose\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\pandas\\io\\parsers.py\u001b[0m in \u001b[0;36m_make_engine\u001b[1;34m(self, engine)\u001b[0m\n\u001b[0;32m   1112\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m_make_engine\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mengine\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m\"c\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1113\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mengine\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;34m\"c\"\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1114\u001b[1;33m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_engine\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mCParserWrapper\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mf\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0moptions\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1115\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1116\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mengine\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;34m\"python\"\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\pandas\\io\\parsers.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, src, **kwds)\u001b[0m\n\u001b[0;32m   1889\u001b[0m         \u001b[0mkwds\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"usecols\"\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0musecols\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1890\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1891\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_reader\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mparsers\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mTextReader\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msrc\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1892\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0munnamed_cols\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_reader\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0munnamed_cols\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1893\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mpandas\\_libs\\parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader.__cinit__\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;32mpandas\\_libs\\parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader._setup_parser_source\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;31mFileNotFoundError\u001b[0m: [Errno 2] File Tweets.csv does not exist: 'Tweets.csv'"
     ]
    }
   ],
   "source": [
    "df=pd.read_csv('Tweets.csv')\n",
    "display(df.shape)\n",
    "\n",
    "display(df.head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['tweet_id'].is_unique"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df=df.reindex(np.random.permutation(df.index))\n",
    "df=df[['text','airline_sentiment']]\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data preparation "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Data Cleaning\n",
    "\n",
    "The first thing we'll do is removing stopwords.These words do not have any value for predicting the sentiment.Furthermore,as we want build a model that can be used for other airline comapnies as well,we remove the mentions."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip  install nltk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk\n",
    "nltk.download('stopwords')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_stopwords(input_text):\n",
    "        stopwords_list = stopwords.words('english')\n",
    "        # Some words which might indicate a certain sentiment are kept via a whitelist\n",
    "        whitelist = [\"n't\", \"not\", \"no\"]\n",
    "        words = input_text.split() \n",
    "        clean_words = [word for word in words if (word not in stopwords_list or word in whitelist) and len(word) > 1] \n",
    "        return \" \".join(clean_words) \n",
    "    \n",
    "def remove_mentions(input_text):\n",
    "        return re.sub(r'@\\w+', '', input_text)\n",
    "       \n",
    "df.text = df.text.apply(remove_stopwords).apply(remove_mentions)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['airline_sentiment'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sns.countplot(df['airline_sentiment'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train-Test split"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The evaluation of the model performance needs to be done on a separate test set.As such, we can estimate how well the model generalizes.This is done with the train_test_split method of scikit-learn."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train,X_test,y_train,y_test=train_test_split(df.text,df.airline_sentiment,test_size=0.1,random_state=37)\n",
    "print('# Train data samples:',X_train.shape[0])\n",
    "print('#Test data samples:',X_test.shape[0])\n",
    "assert X_train.shape[0]==y_train.shape[0]\n",
    "assert X_test.shape[0]==y_test.shape[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Converting words to numbers"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To use the text as input for a model ,we first need to convert the tweet's words into tokens,which simply means converting the words to integers that refer to an index in a dictionary.Here we will only keep the most frequent words in the tain set.\n",
    "\n",
    "We clean up the text by applying filters and putting the words to lowercase.Words are separated by spaces."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tk = Tokenizer(num_words=NB_WORDS,\n",
    "               filters='!\"#$%&()*+,-./:;<=>?@[\\\\]^_`{|}~\\t\\n',\n",
    "               lower=True,\n",
    "               split=\" \")\n",
    "tk.fit_on_texts(X_train)\n",
    "\n",
    "print('Fitted tokenizer on {} documents'.format(tk.document_count))\n",
    "print('{} words in dictionary'.format(tk.num_words))\n",
    "print('Top 5 most common words are:', collections.Counter(tk.word_counts).most_common(5))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "After having created the dictionary we can convert the text to list of integer indexes.This is done with the text_to_sequences method of the Tokenizer."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_seq = tk.texts_to_sequences(X_train)\n",
    "X_test_seq = tk.texts_to_sequences(X_test)\n",
    "\n",
    "print('\"{}\" is converted into {}'.format(X_train[0], X_train_seq[0]))\n",
    "print('\"{}\" is converted into {}'.format(X_test[0], X_test_seq[0]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "These integers should now be converted into a one-hot encoded features."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def one_hot_seq(seqs, nb_features = NB_WORDS):\n",
    "    ohs = np.zeros((len(seqs), nb_features))\n",
    "    for i, s in enumerate(seqs):\n",
    "        ohs[i, s] = 1.\n",
    "    return ohs\n",
    "\n",
    "X_train_oh = one_hot_seq(X_train_seq)\n",
    "X_test_oh = one_hot_seq(X_test_seq)\n",
    "\n",
    "print('\"{}\" is converted into {}'.format(X_train_seq[0], X_train_oh[0]))\n",
    "print('For this example we have {} features with a value of 1.'.format(X_train_oh[0].sum()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Converting the target classes to numbers"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We need to convert the target classes to numbers as well,which in turn are one-hot-encoded with the to_categorical method in keras."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "le = LabelEncoder()\n",
    "y_train_le = le.fit_transform(y_train)\n",
    "y_test_le = le.transform(y_test)\n",
    "y_train_oh = to_categorical(y_train_le)\n",
    "y_test_oh = to_categorical(y_test_le)\n",
    "\n",
    "print('\"{}\" is converted into {}'.format(y_train[0], y_train_le[0]))\n",
    "print('\"{}\" is converted into {}'.format(y_train_le[0], y_train_oh[0]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Splitting of a validation set"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now that our data is ready,we split of a validation set.This validation set will be used to evaluate the model performance when we tune the parameters of the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_rest, X_valid, y_train_rest, y_valid = train_test_split(X_train_oh, y_train_oh, test_size=0.1, random_state=37)\n",
    "\n",
    "assert X_valid.shape[0] == y_valid.shape[0]\n",
    "assert X_train_rest.shape[0] == y_train_rest.shape[0]\n",
    "\n",
    "print('Shape of validation set:',X_valid.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Deep learning"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "# Baseline Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We start with a model with 2 densely connected layers of 64 hidden elements.The input_shape for the first layer is equal to the number of words we allowed in the dictionary and for which we created one-hot-encoded features.\n",
    "\n",
    "As we need to predict 3 different sentiment classes,the last layer has 3 hiddden elements.The softmax activation function makes sure the three probabilites sum up to 1."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "base_model = models.Sequential()\n",
    "base_model.add(layers.Dense(64, activation='relu', input_shape=(NB_WORDS,)))\n",
    "base_model.add(layers.Dense(64, activation='relu'))\n",
    "base_model.add(layers.Dense(3, activation='softmax'))\n",
    "base_model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Because this project is a multi-class ,single-label prediction,we use categorical_crossentropy as the loss function and softmax as final activation function.We fit the model on the reamining train data and validate on the validation set.We run for predetermined number of epochs and will see when the model starts to overfit."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def deep_model(model):\n",
    "    model.compile(optimizer='rmsprop'\n",
    "                  , loss='categorical_crossentropy'\n",
    "                  , metrics=['accuracy'])\n",
    "    \n",
    "    history = model.fit(X_train_rest\n",
    "                       , y_train_rest\n",
    "                       , epochs=NB_START_EPOCHS\n",
    "                       , batch_size=BATCH_SIZE\n",
    "                       , validation_data=(X_valid, y_valid)\n",
    "                       , verbose=0)\n",
    "    \n",
    "    return history"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "base_history = deep_model(base_model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To evaluate the model performance,we will look at the training and validation loss and accuracy."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def eval_metric(history, metric_name):\n",
    "    metric = history.history[metric_name]\n",
    "    val_metric = history.history['val_' + metric_name]\n",
    "\n",
    "    e = range(1, NB_START_EPOCHS + 1)\n",
    "\n",
    "    plt.plot(e, metric, 'bo', label='Train ' + metric_name)\n",
    "    plt.plot(e, val_metric, 'b', label='Validation ' + metric_name)\n",
    "    plt.legend()\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can see here that the validation loss starts to increase as from epoch 4.\n",
    "The training loss continues to lower,which is normal as the model is trained to fit the train data as good as possible."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "eval_metric(base_history, 'loss')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Just as with validation loss, the validation accuracy peaks at an early epoch.After that,it goes down slightly.So to conclude,we can say that the model starts overfitting as from epoch 4."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "eval_metric(base_history,'accuracy')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Handling overfitting"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now,we can try to do something about the overfitting.These are different options to do that:\n",
    "    \n",
    "    .Option 1:reduce the network's size by removing layers or reducing the number of hidden elements in the layers.\n",
    "        \n",
    "        .Option 2: add regularization,which comes down to adding a cost to the loss function for large weights\n",
    "            \n",
    "            .option 3: adding dropout layers, which will randomly remove certain features by setting them to zero\n",
    "                \n",
    "            \n",
    "            \n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Reducing the network's size"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We reduce the network's size by removing one layer and lowering the number of hidden element in the remaining layer to 32."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "reduced_model = models.Sequential()\n",
    "reduced_model.add(layers.Dense(32, activation='relu', input_shape=(NB_WORDS,)))\n",
    "reduced_model.add(layers.Dense(3, activation='softmax'))\n",
    "reduced_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "reduced_history = deep_model(reduced_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compare_loss_with_baseline(h, model_name):\n",
    "    loss_base_model = base_history.history['val_loss']\n",
    "    loss_model = h.history['val_loss']\n",
    "\n",
    "    e = range(1, NB_START_EPOCHS + 1)\n",
    "\n",
    "    plt.plot(e, loss_base_model, 'bo', label='Validation Loss Baseline Model')\n",
    "    plt.plot(e, loss_model, 'b', label='Validation Loss ' + model_name)\n",
    "    plt.legend()\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We can see that it takes more epochs before the reduced model starts overfitting(around epoch 10).Moreover,the loss increases much slower after that epoch compared to the baseline model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "compare_loss_with_baseline(reduced_history, 'Reduced Model')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Adding regularization\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To address overfitting,we can also add regularization to the model.Let's try with L2 regularization."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "reg_model = models.Sequential()\n",
    "reg_model.add(layers.Dense(64, kernel_regularizer=regularizers.l2(0.001), activation='relu', input_shape=(NB_WORDS,)))\n",
    "reg_model.add(layers.Dense(64, kernel_regularizer=regularizers.l2(0.001), activation='relu'))\n",
    "reg_model.add(layers.Dense(3, activation='softmax'))\n",
    "reg_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "reg_history = deep_model(reg_model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For the regularized model we notice that it starts overfitting earlier than the baseline model.However,the loss increases much slower afterwards."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "compare_loss_with_baseline(reg_history, 'Regularized Model')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Adding dropout layers"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The last option we'll try to add dropout layers."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "drop_model = models.Sequential()\n",
    "drop_model.add(layers.Dense(64, activation='relu', input_shape=(NB_WORDS,)))\n",
    "drop_model.add(layers.Dropout(0.5))\n",
    "drop_model.add(layers.Dense(64, activation='relu'))\n",
    "drop_model.add(layers.Dropout(0.5))\n",
    "drop_model.add(layers.Dense(3, activation='softmax'))\n",
    "drop_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "drop_history = deep_model(drop_model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The model with dropout layers starts overfitting a bit later than the baseline model.The loss also increases slower than the baseline model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "compare_loss_with_baseline(drop_history, 'Dropout Model')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training on the full train data and evaluation on test data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "At first sight the reduced model seems to be the best model for generalization.But let's check that on the test set."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test_model(model, epoch_stop):\n",
    "    model.fit(X_train_oh\n",
    "              , y_train_oh\n",
    "              , epochs=epoch_stop\n",
    "              , batch_size=BATCH_SIZE\n",
    "              , verbose=0)\n",
    "    results = model.evaluate(X_test_oh, y_test_oh)\n",
    "    \n",
    "    return results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "base_results = test_model(base_model, 4)\n",
    "print('/n')\n",
    "print('Test accuracy of baseline model: {0:.2f}%'.format(base_results[1]*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "reduced_results = test_model(reduced_model, 10)\n",
    "print('/n')\n",
    "print('Test accuracy of reduced model: {0:.2f}%'.format(reduced_results[1]*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "reg_results = test_model(reg_model, 5)\n",
    "print('/n')\n",
    "print('Test accuracy of regularized model: {0:.2f}%'.format(reg_results[1]*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "drop_results = test_model(drop_model, 6)\n",
    "print('/n')\n",
    "print('Test accuracy of dropout model: {0:.2f}%'.format(drop_results[1]*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}