TDA_Backtranslation-MovieReviewsData-prototype9.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Textual Data Amplification (TDA) Experiments - Prototype 9\n",
    "\n",
    "## Paraphrase Generation - Backtranslation\n",
    "\n",
    "## Movie Reviews Data\n",
    "\n",
    "## Sentiment polarity prediction task"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Python Basic Packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Python:  3.6.5 (v3.6.5:f59c0932b4, Mar 28 2018, 03:03:55) \n",
      "[GCC 4.2.1 (Apple Inc. build 5666) (dot 3)] True\n",
      "NumPy:  1.16.2 True\n",
      "Pandas:  0.24.2\n",
      "Matplotlib:  3.0.3\n",
      "NLTK:  3.4\n",
      "IPython:  7.4.0\n",
      "skikit-learn 0.20.3\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import re\n",
    "import sys\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib\n",
    "import nltk\n",
    "import IPython\n",
    "import sklearn\n",
    "\n",
    "# Versions of the packages\n",
    "print('Python: ', sys.version, float(str(sys.version).split('.')[0])+float(str(sys.version).split('.')[1])/10  >= 2.6)\n",
    "print('NumPy: ', np.__version__, float(np.__version__.split('.')[0])+float(np.__version__.split('.')[1])/10 >= 1.6 )\n",
    "print('Pandas: ', pd.__version__)\n",
    "print('Matplotlib: ', matplotlib.__version__)\n",
    "print('NLTK: ', nltk.__version__)\n",
    "print('IPython: ', IPython.__version__)\n",
    "print('skikit-learn', sklearn.__version__)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Data\n",
    "\n",
    "#### Movie Review Data\n",
    "http://www.cs.cornell.edu/people/pabo/movie-review-data/\n",
    "\n",
    "#### Download link\n",
    "http://www.cs.cornell.edu/people/pabo/movie-review-data/review_polarity.tar.gz\n",
    "\n",
    "#### Data files structure\n",
    "\n",
    "Create a directory for the data (DATA) then negative (pos) and positive reviews (neg) sub directories.\n",
    "\n",
    "You should also create two augmented data sub-directories (backtranslation_aug_neg and backtranslation_aug_pos) and a cuttoff_model sub-directory which contains CuttOff_LogisticRegression_model.pkl"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Text Data Augmentation Object\n",
    "### Generic Class Definition"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import re\n",
    "from nltk import wordpunct_tokenize\n",
    "from nltk import sent_tokenize\n",
    "from nltk.translate.bleu_score import sentence_bleu, SmoothingFunction\n",
    "from os import listdir\n",
    "import random\n",
    "import timeit\n",
    "import numpy as np\n",
    "from scipy.stats import norm\n",
    "\n",
    "class TextDataAugmentation(object):\n",
    "    \n",
    "    # Constructor \n",
    "    def __init__(self, data_dir, \n",
    "                 neg_rev, pos_rev, aug_neg_rev, aug_pos_rev,\n",
    "                 paraphr_nbr_max=5, paratext_nbr_max=5\n",
    "                ): \n",
    "        self.DATA_DIR = data_dir\n",
    "        self.NEG_REVIEWS = neg_rev\n",
    "        self.POS_REVIEWS = pos_rev\n",
    "        self.AUGMENTED_NEG_REVIEWS = aug_neg_rev\n",
    "        self.AUGMENTED_POS_REVIEWS = aug_pos_rev\n",
    "        self.name = \"TextDataAugmentation\" \n",
    "        self.encoding = 'utf-8'\n",
    "        self.SENTENCE_LENGTH_MAX = 1000\n",
    "        self.PARAPHRASES_NUMBER_MAX = paraphr_nbr_max\n",
    "        self.PARATEXTS_NUMBER_MAX = paratext_nbr_max\n",
    "        self.PARATEXT_ITERATION_LIMIT = 0.50\n",
    "        self.TRACE_RANDOM_KEY = False\n",
    "        self.TRACE_COMBINATIONS = False\n",
    "\n",
    "    # To check if this object is TextDataAugmentation class \n",
    "    def getName(self): \n",
    "        return self.name\n",
    " \n",
    "    # To check if this object is TextDataAugmentation class \n",
    "    def isTextDataAugmentation(self): \n",
    "        return True\n",
    "\n",
    "    # load doc into memory\n",
    "    def load_doc(self, file_path_with_name):\n",
    "        # open the file as read only\n",
    "        with open(file_path_with_name, 'r', encoding=self.encoding) as input_file:\n",
    "            # read all text\n",
    "            text = input_file.read()\n",
    "        return text\n",
    "\n",
    "    # save doc to file\n",
    "    def save_doc(self,file_name_with_path,text):\n",
    "        # open the file as write only\n",
    "        with open(file_name_with_path, 'w', encoding=self.encoding) as ouput_file:\n",
    "            # write text\n",
    "            ouput_file.write(text)\n",
    "\n",
    "    def tokenizeTextString(self, text_string):\n",
    "        return wordpunct_tokenize(text_string.replace('_',' ').replace('(',' ').replace(')',' ').replace(\"'\",' ').replace('\"','').replace('/',' ').replace(\"\\\\\", \" \").replace('  ',' '))\n",
    "\n",
    "    def standardize_text(self, raw_input_text):\n",
    "        raw_input_text = raw_input_text.lower().replace(\" n't\",\"n't\").replace(\" 's\",\"'s\")\n",
    "        return ' '.join(self.tokenizeTextString(raw_input_text.lower()))\n",
    "    \n",
    "    def sentence_length(self, sentence):\n",
    "        return len(sentence.split(\" \"))\n",
    "\n",
    "    def get_heuristic_weigth(self, original_sentence, new_paraphrase):\n",
    "        return 1\n",
    "\n",
    "    def add_new_paraphrase(self, paraphrases, original_sentence, new_paraphrase):\n",
    "        new_paraphrase_dict = dict()\n",
    "        if  new_paraphrase not in paraphrases.keys():\n",
    "            new_paraphrase_dict[new_paraphrase] = self.get_heuristic_weigth(original_sentence,new_paraphrase)\n",
    "            return new_paraphrase_dict\n",
    "        return new_paraphrase_dict\n",
    "\n",
    "    def splitted(self, document):\n",
    "        return '\\n' in document\n",
    "\n",
    "    def sentences_splitting(self, document):\n",
    "        if self.splitted(document):\n",
    "            # returns a list of sentences already splitted\n",
    "            # and marked by a \\n tag\n",
    "            return document.split('\\n')\n",
    "        else:\n",
    "            return sent_tokenize(document)\n",
    "\n",
    "    def evaluate_combinations_number(self, paratext_dict):\n",
    "        combinations_nbr = 1\n",
    "        for sentence_index in paratext_dict.keys():\n",
    "            combinations_nbr *= len(paratext_dict[sentence_index])\n",
    "        return combinations_nbr\n",
    "\n",
    "    # Normalizing the replacement weights list in order to get probabilities \n",
    "    def normalize_weights_list(self, weights_list):\n",
    "        weights_sum = sum(weights_list)\n",
    "        # Avoid division by 0 \n",
    "        if weights_sum == 0:\n",
    "            weights_sum = 1\n",
    "        return [a_weight/weights_sum for a_weight in weights_list]\n",
    "\n",
    "    def get_an_integer_normaly_draw_from_n_range(n):\n",
    "        if n > 0:\n",
    "            mean = n/2.0\n",
    "            std = n/5.0\n",
    "            draw_number = list(norm.ppf(np.random.random(1), loc=mean, scale=std).astype(int))[0]\n",
    "            while not (draw_number >= 1) and (draw_number <= n):\n",
    "                draw_number = list(norm.ppf(np.random.random(1), loc=mean, scale=std).astype(int))[0]\n",
    "        else:\n",
    "            draw_number = 0\n",
    "        return draw_number \n",
    "\n",
    "    # draw a random number of replacements among all possible replacements \n",
    "    # distribution: 'left_skewed', 'right_skewed', 'uniform'\n",
    "    def get_variants_using_random_distribution(self, paratext_dict,distribution='uniform'):\n",
    "        variants_index = [n for n in range(1,len(paratext_dict)+1) if len(paratext_dict[str(n)])>1]\n",
    "        if len(variants_index) > 0:\n",
    "            variants_quantity = self.get_an_integer_normaly_draw_from_n_range(len(variants_index))\n",
    "            if distribution == 'left_skewed':\n",
    "                distribution_weights = [2*n for n in range(1,len(variants_index)+1)]\n",
    "            elif distribution == 'right_skewed':\n",
    "                distribution_weights = [2/n for n in range(1,len(variants_index)+1)]\n",
    "            else: # uniform\n",
    "                distribution_weights = [1 for n in range(1,len(variants_index)+1)]\n",
    "            normalized_distribution_weights = self.normalize_weights_list(distribution_weights)\n",
    "            if self.TRACE_RANDOM_KEY:\n",
    "                print(\"len(variants_index):\",len(variants_index),\"variants_quantity:\",variants_quantity)\n",
    "            if variants_quantity > len(variants_index):\n",
    "                variants_quantity = len(variants_index)\n",
    "                if self.TRACE_RANDOM_KEY:\n",
    "                    print(\"New variants_quantity:\",variants_quantity)\n",
    "            randomized_variants = np.random.choice(variants_index,\\\n",
    "                                                   size=variants_quantity,\\\n",
    "                                                   replace=False,\\\n",
    "                                                   p=normalized_distribution_weights)\n",
    "            if self.TRACE_RANDOM_KEY:\n",
    "                print(\"*** variants_quantity: \", variants_quantity)\n",
    "                print(\"*** randomized_variants: \",randomized_variants)\n",
    "                print(\"distribution_weights: \",distribution_weights)\n",
    "                print(\"normalized_distribution_weights: \",normalized_distribution_weights)\n",
    "                print(\"variants_quantity:\",variants_quantity,\"len(paratext_dict):\",len(paratext_dict),\"len(variants_index)\",len(variants_index))\n",
    "        else:\n",
    "            randomized_variants = []\n",
    "        return list(randomized_variants)\n",
    "\n",
    "    # Choose randomly one variant for a part among all possible variants based on heuristic weights\n",
    "    def get_one_variant_using_heuristic_weights(self, paraphrase_dict,part_index):\n",
    "        # Choose the replacement among all the possible replacements based on heuristic weights\n",
    "        variants = list(range(0,len(paraphrase_dict[str(part_index)])))\n",
    "        variants_heuristic_weights = list(paraphrase_dict[str(part_index)].values())\n",
    "        if len(variants_heuristic_weights)> 1:\n",
    "            a_variant = random.choices(population=variants,\\\n",
    "                                       weights=variants_heuristic_weights,k=1)[0]\n",
    "        else:\n",
    "            a_variant = variants[0]\n",
    "        return a_variant\n",
    "\n",
    "    def generate_sampling_random_key(self, paratext_dict,distribution='uniform'):\n",
    "        random_part_key = \"\"\n",
    "        # Choose randomly the variants based on a distribution\n",
    "        variants_indexes = self.get_variants_using_random_distribution(paratext_dict,distribution=distribution)\n",
    "        if self.TRACE_RANDOM_KEY:\n",
    "            print(\"variants_indexes: \",variants_indexes)\n",
    "        for part_index in range(1,len(paratext_dict)+1):\n",
    "            if part_index in variants_indexes:\n",
    "                if self.TRACE_RANDOM_KEY:\n",
    "                    print(\"part_index found: \",part_index)\n",
    "                # Choose randomly one variant for a precise part among all possible variants based on heuristic weights\n",
    "                a_variant = self.get_one_variant_using_heuristic_weights(paratext_dict,part_index)\n",
    "                if self.TRACE_RANDOM_KEY:\n",
    "                    print(\"a_variant: \",a_variant)\n",
    "                random_part_key = random_part_key + str(part_index) + \"-\" + str(a_variant) + \"_\"\n",
    "            else:\n",
    "                random_part_key = random_part_key + str(part_index) + \"-0\" + \"_\"\n",
    "        return random_part_key[:-1]\n",
    "\n",
    "    def generate_new_unique_key(self, paratext_dict,keys_memory,distribution=\"uniform\"):\n",
    "        random_key = self.generate_sampling_random_key(paratext_dict,distribution)\n",
    "        while random_key in keys_memory:\n",
    "            random_key = self.generate_sampling_random_key(paratext_dict,distribution)\n",
    "        keys_memory.add(random_key)\n",
    "        return (keys_memory,random_key)\n",
    "\n",
    "    def generate_one_random_paratext(self, paratext_dict, random_key):\n",
    "        paratext = []\n",
    "        for index,key in enumerate(random_key.split(\"_\")):\n",
    "            sub_keys = key.split(\"-\")\n",
    "            paraphrase = list(paratext_dict[str(index+1)].keys())[int(sub_keys[1])]\n",
    "            paratext.append(paraphrase)\n",
    "        return paratext\n",
    "\n",
    "    def generate_paraphrases(new_sentence):\n",
    "        return new_sentence\n",
    "    \n",
    "    def create_paratext_dict(self, sentences_list):\n",
    "        sentences_number = len(sentences_list)\n",
    "        paratext_dict = {}\n",
    "        for sentence_index in range(sentences_number):\n",
    "            new_sentence = ' '.join(re.findall(r\"[a-zA-Z'-]+|[.,;!?\\'\\’]+\",sentences_list[sentence_index].lower()))\n",
    "            paraphrases_dict = {new_sentence:1}\n",
    "            if len(new_sentence) > 0:\n",
    "                new_paraphrases_dict = self.generate_paraphrases(new_sentence)\n",
    "                paraphrases_dict.update(new_paraphrases_dict)\n",
    "            paratext_dict[str(sentence_index+1)] = paraphrases_dict\n",
    "        return paratext_dict\n",
    "\n",
    "    def generate_sampling(self, original_document, augmentation_factor):\n",
    "        splitted_document = self.sentences_splitting(original_document)\n",
    "        paratext_dict = self.create_paratext_dict(splitted_document)\n",
    "        print(\"Paratext Dict created for the document\")\n",
    "        combinations_number_max = self.evaluate_combinations_number(paratext_dict)\n",
    "        if augmentation_factor > combinations_number_max:\n",
    "            print(\"*** WARNING! Desired paratexts_sample number: \" + str(augmentation_factor) + \" exceeds Combinations Number Max, which is: \", combinations_number_max)\n",
    "            augmentation_factor = combinations_number_max\n",
    "        if augmentation_factor > self.PARATEXTS_NUMBER_MAX:\n",
    "            print(\"*** WARNING! Desired paratexts_sample number: \" + str(augmentation_factor) + \" exceeds PARATEXTS_NUMBER_MAX, which is: \", PARATEXTS_NUMBER_MAX)\n",
    "            augmentation_factor = self.PARATEXTS_NUMBER_MAX\n",
    "        paratexts_sample = []\n",
    "        keys_memory = set()\n",
    "        while len(paratexts_sample) < augmentation_factor:\n",
    "            if len(keys_memory) > combinations_number_max * self.PARATEXT_ITERATION_LIMIT:\n",
    "                break\n",
    "            keys_memory, random_key = self.generate_new_unique_key(paratext_dict,keys_memory,distribution='uniform')\n",
    "            new_paratext = self.generate_one_random_paratext(paratext_dict,random_key)\n",
    "            paratexts_sample.append(new_paratext)\n",
    "        print(\"Text Data Augmentation done for the document\")\n",
    "        return paratexts_sample\n",
    "\n",
    "    def get_datafile_id(filename):\n",
    "        return filename.split(\"_\")[-2][-3:]\n",
    "        \n",
    "    def augment_text_data(self, augmentation_factor=5):\n",
    "        is_train = True\n",
    "        # WARNING - many lines of code below are data source specific !\n",
    "        for sub_directory in [self.NEG_REVIEWS,self.POS_REVIEWS]:\n",
    "            data_directory = data_dir + sub_directory\n",
    "            for filename in listdir(data_directory):\n",
    "                start_time = timeit.default_timer()\n",
    "                # skip files that do not have the right extension\n",
    "                # WARNING - data source specific\n",
    "                if not filename.endswith(\".txt\"):\n",
    "                    continue\n",
    "                # Processing training set, so skip any reviews in the test set\n",
    "                # which start with 'cv9' - WARNING - data source specific\n",
    "                if is_train and filename.startswith('cv9'):\n",
    "                    continue\n",
    "                # Processing test set, so skip any reviews in the training set\n",
    "                # which not start with 'cv9' - WARNING - data source specific\n",
    "                if not is_train and not filename.startswith('cv9'):\n",
    "                    continue\n",
    "                # create the full path of the file to open\n",
    "                file_path = data_directory + filename\n",
    "                # load the original document\n",
    "                original_text = self.load_doc(file_path)\n",
    "                print(\"Augmenting...\", file_path)\n",
    "                paratexts = self.generate_sampling(original_text,augmentation_factor=augmentation_factor)\n",
    "                for index,paraphrases in enumerate(paratexts):\n",
    "                    new_text = \" \\n\".join(paraphrases)\n",
    "                    new_filename = \"ampl_\" + filename.split(\".\")[0] + \"_\" + str(index)+ '.txt'\n",
    "                    if sub_directory == self.NEG_REVIEWS:\n",
    "                        new_file_path = self.DATA_DIR+self.AUGMENTED_NEG_REVIEWS+new_filename \n",
    "                        self.save_doc(new_file_path,new_text)\n",
    "                    else:\n",
    "                        new_file_path = self.DATA_DIR+self.AUGMENTED_POS_REVIEWS+new_filename\n",
    "                        self.save_doc(new_file_path,new_text)\n",
    "                    if (index % 10) == 0:\n",
    "                        print(\"Saved:\",new_file_path)\n",
    "                        end_time = timeit.default_timer()\n",
    "                        print(\"Elapsed time: {}\".format(end_time - start_time))\n",
    "                        print(\"-----\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Backtranslation TDA Object\n",
    "### Sub Class Definition\n",
    "\n",
    "IMPORTANT: Don't forget to get and replace the Google Translate Developer Key API\n",
    "\n",
    "So this line of code: self.api_key = 'YOUR OWN GOOGLE TRANSLATE DEVELOPER KEY'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.externals import joblib\n",
    "import re\n",
    "from nltk.translate.bleu_score import sentence_bleu, SmoothingFunction\n",
    "import requests\n",
    "from requests.adapters import HTTPAdapter\n",
    "from requests.packages.urllib3.util.retry import Retry\n",
    "import html\n",
    "import urllib\n",
    "import numpy as np\n",
    "\n",
    "\n",
    "# Inherited or Sub class (TextDataAugmentation) \n",
    "class Backtranslation(TextDataAugmentation): \n",
    "\n",
    "    # Constructor \n",
    "    def __init__(self,cutoff_model_filepath,\n",
    "                 data_dir,neg_rev, pos_rev,aug_neg_rev, aug_pos_rev,\n",
    "                 paraphr_nbr_max=5, paratext_nbr_max = 5,): \n",
    "        self.SIMILARITY_LENGTH_MIN = 0.6\n",
    "        self.SIMILARITY_BLEU_MIN = 0.6\n",
    "        self.CUTOFF_MODEL_FILEPATH = cutoff_model_filepath\n",
    "        # Less accurate translation languages were commented\n",
    "        self.Google_Translate_languages = {\n",
    "            #'Afrikaans':'af',\n",
    "            #'Albanian':'sq',\n",
    "            #'Arabic':'ar',\n",
    "            'Belarusian':'be',\n",
    "            'Bulgarian':'bg',\n",
    "            #'Catalan':'ca',\n",
    "            'ChineseSimplified':'zh-CN',\n",
    "            'ChineseTraditional':'zh-TW',\n",
    "            #'Croatian':'hr',\n",
    "            #'Czech':'cs',\n",
    "            'Danish':'da',\n",
    "            'Dutch':'nl',\n",
    "            #'English':'en',\n",
    "            #'Estonian':'et',\n",
    "            #'Filipino':'tl',\n",
    "            #'Finnish':'fi',\n",
    "            'French':'fr',\n",
    "            'Galician':'gl',\n",
    "            'German':'de',\n",
    "            #'Greek':'el',\n",
    "            'Hebrew':'iw',\n",
    "            #'Hindi':'hi',\n",
    "            #'Hungarian':'hu',\n",
    "            #'Icelandic':'is',\n",
    "            #'Indonesian':'id',\n",
    "            #'Irish':'ga',\n",
    "            'Italian':'it',\n",
    "            'Japanese':'ja',\n",
    "            #'Korean':'ko',\n",
    "            #'Latvian':'lv',\n",
    "            #'Lithuanian':'lt',\n",
    "            #'Macedonian':'mk',\n",
    "            #'Malay':'ms',\n",
    "            #'Maltese':'mt',\n",
    "            'Norwegian':'no',\n",
    "            #'Persian':'fa',\n",
    "            'Polish':'pl',\n",
    "            'Portuguese':'pt',\n",
    "            'Romanian':'ro',\n",
    "            'Russian':'ru',\n",
    "            #'Serbian':'sr',\n",
    "            #'Slovak':'sk',\n",
    "            'Slovenian':'sl',\n",
    "            'Spanish':'es',\n",
    "            #'Swahili':'sw',\n",
    "            'Swedish':'sv',\n",
    "            # 'Thai':'th',\n",
    "            'Turkish':'tr',\n",
    "            'Ukrainian':'uk',\n",
    "            'Vietnamese':'vi',\n",
    "            #'Welsh':'cy',\n",
    "            #'Yiddish':'yi',\n",
    "            } \n",
    "        self.language_codes = list(self.Google_Translate_languages.values())\n",
    "        # Translate API key\n",
    "        self.api_key = 'YOUR OWN GOOGLE TRANSLATE DEVELOPER KEY'\n",
    "        self.agent = {\n",
    "            'User-Agent':\"Mozilla/4.0 (\\\n",
    "            compatible;\\\n",
    "            MSIE 6.0;\\\n",
    "            Windows NT 5.1;\\\n",
    "            SV1;\\\n",
    "            .NET CLR 1.1.4322;\\\n",
    "            .NET CLR 2.0.50727;\\\n",
    "            .NET CLR 3.0.04506.30\\\n",
    "            )\"}\n",
    "        self.logreg_model = joblib.load(self.CUTOFF_MODEL_FILEPATH)\n",
    "        self.conversion_langtag_to_int = {'be': 0, 'bg': 1, 'zh+AC0-CN': 2, 'zh+AC0-TW': 3, 'da': 4,\\\n",
    "                                          'nl': 5, 'fr': 6,'gl': 7, 'de': 8,\\\n",
    "                                          'iw': 9, 'it': 10, 'ja': 11, 'no': 12,\\\n",
    "                                          'pl': 13, 'pt': 14, 'ro': 15, 'ru': 16,\\\n",
    "                                          'sl': 17, 'es': 18,'sv': 19, 'tr': 20,\\\n",
    "                                          'uk': 21, 'vi': 22, 'zh-CN': 23, 'zh-TW': 24\n",
    "                                         }\n",
    "        super(Backtranslation, self).__init__(data_dir, \n",
    "                                              neg_rev, pos_rev, \n",
    "                                              aug_neg_rev, aug_pos_rev,\n",
    "                                              paraphr_nbr_max, paratext_nbr_max\n",
    "                                             )\n",
    "\n",
    "    # To check if this object is Backtranslation class \n",
    "    def isBacktranslation(self): \n",
    "        return True\n",
    "\n",
    "    # To get ModelPah \n",
    "    def getModelPath(self): \n",
    "        return self.model_path \n",
    "\n",
    "    # http://docs.python-requests.org/en/master/user/quickstart/#raw-response-content\n",
    "    # Requests is an elegant and simple HTTP library for Python, built for human beings. \n",
    "    # https://www.peterbe.com/plog/best-practice-with-retries-with-requests\n",
    "    def requests_retry_session(self,session=None):\n",
    "        session = session or requests.Session()\n",
    "        retry = Retry(\n",
    "            total = 3,\n",
    "            read = 3,\n",
    "            connect = 3,\n",
    "            backoff_factor = 0.3,\n",
    "            status_forcelist = (500, 502, 504),\n",
    "        )\n",
    "        adapter = HTTPAdapter(max_retries=retry)\n",
    "        session.mount('http://', adapter)\n",
    "        session.mount('https://', adapter)\n",
    "        return session\n",
    "\n",
    "    def translate(self, text, to_language=\"auto\", from_language=\"auto\"):\n",
    "        text = urllib.parse.quote(text)\n",
    "        link = \"http://translate.google.com/m?hl=%s&sl=%s&q=%s&key=%s\" % (to_language, from_language, text, self.api_key)\n",
    "        session = requests.Session()\n",
    "        session.auth = ('user', 'pass')\n",
    "        session.headers.update({'x-test': 'true'})\n",
    "        response = self.requests_retry_session(session=session).get(link)\n",
    "        re_result = re.findall(r'class=\"t0\">(.*?)<', response.text)\n",
    "        if (len(re_result) == 0):\n",
    "            result = \"\"\n",
    "        else:\n",
    "            result = html.unescape(re_result[0])\n",
    "        return (result)\n",
    "\n",
    "    def get_BLEU_similarity(self, sentence1, sentence2):\n",
    "        bleu_score = sentence_bleu([sentence1.split()], sentence2.split(), smoothing_function=SmoothingFunction().method4)\n",
    "        return bleu_score\n",
    "\n",
    "    def get_simple_similarity(self, sentence1, sentence2):\n",
    "        length_sentence1 = len(sentence1.split())\n",
    "        length_sentence2 = len(sentence2.split())\n",
    "        return 1- abs(length_sentence2 - length_sentence1)/length_sentence1\n",
    "\n",
    "    def get_prediction_LogisticRegression_model(self, target_language_tag, source_sentence, backtranslation):\n",
    "        target_language_int = self.conversion_langtag_to_int[target_language_tag]\n",
    "        source_length = len(source_sentence.split())\n",
    "        backtranslation_length = len(backtranslation.split())\n",
    "        bleu_score = self.get_BLEU_similarity(source_sentence,backtranslation)\n",
    "        new_data_array = np.array([target_language_int,source_length,backtranslation_length,bleu_score])\n",
    "        return self.logreg_model.predict(new_data_array.reshape(1, -1))[0]\n",
    "    \n",
    "    def generate_paraphrases(self, source_sentence, source_language_code=\"en\", cut_off_method=\"logistic_model\"):\n",
    "        standardized_source_sentence = self.standardize_text(source_sentence)\n",
    "        paraphrases = {}\n",
    "        for target_language_code in self.language_codes:\n",
    "            if (self.sentence_length(source_sentence) < self.SENTENCE_LENGTH_MAX):\n",
    "                try:\n",
    "                    forward_translation = self.translate(source_sentence, target_language_code, source_language_code)\n",
    "                    back_translation = self.translate(forward_translation, source_language_code, target_language_code)\n",
    "                    standardized_back_translation = self.standardize_text(back_translation)\n",
    "                    if (standardized_source_sentence != standardized_back_translation):\n",
    "                        if cut_off_method == \"logistic_model\":\n",
    "                            if self.get_prediction_LogisticRegression_model(target_language_code,standardized_source_sentence,standardized_back_translation) > 0:\n",
    "                                if len(paraphrases) < self.PARAPHRASES_NUMBER_MAX:\n",
    "                                    paraphrases.update(self.add_new_paraphrase(paraphrases,standardized_source_sentence,standardized_back_translation))\n",
    "                                else:\n",
    "                                    break\n",
    "                        elif cut_off_method == \"bleu_similarity\":\n",
    "                            if (self.get_BLEU_similarity(standardized_source_sentence, standardized_back_translation) >  self.SIMILARITY_BLEU_MIN):\n",
    "                                if len(paraphrases) < self.PARAPHRASES_NUMBER_MAX:\n",
    "                                    paraphrases.update(self.add_new_paraphrase(paraphrases,standardized_source_sentence,standardized_back_translation))\n",
    "                                else:\n",
    "                                    break\n",
    "                        elif cut_off_method == \"length_similarity\":\n",
    "                            if (self.get_simple_similarity(standardized_source_sentence, standardized_back_translation) >  self.SIMILARITY_LENGTH_MIN):\n",
    "                                if len(paraphrases) < self.PARAPHRASES_NUMBER_MAX:\n",
    "                                    paraphrases.update(self.add_new_paraphrase(paraphrases,standardized_source_sentence,standardized_back_translation))\n",
    "                                else:\n",
    "                                    break\n",
    "                except Exception:\n",
    "                    print(\"Google Translate server exception\",target_language_code)\n",
    "                    continue\n",
    "        return paraphrases"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Local data directories setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "data_dir = \"DATA/\"\n",
    "neg_rev = \"neg/\"\n",
    "pos_rev = \"pos/\"\n",
    "aug_neg_rev = \"backtranslation_aug_neg/\"\n",
    "aug_pos_rev = \"backtranslation_aug_pos/\"\n",
    "model_dir = \"DATA/cutoff_model/\"\n",
    "model_file_name = \"CuttOff_LogisticRegression_model.pkl\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Experiment - Creation of TDA Backtranslation object "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "paraphr_nbr_max = 5\n",
    "paratext_nbr_max = 5\n",
    "my_tda = Backtranslation(model_dir+model_file_name,data_dir,\n",
    "                         neg_rev, pos_rev, aug_neg_rev, aug_pos_rev, \n",
    "                         paraphr_nbr_max, paratext_nbr_max)\n",
    "my_tda.isBacktranslation()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "my_tda.isTextDataAugmentation()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'TextDataAugmentation'"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "my_tda.name"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Experiment - Cutoff Metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Simple Length Similarity Cutoff: 0.8\n",
      "BLEU Metric Similarity Cutoff: 0.32756475929865714\n",
      "Logistic Regression Model Cuttoff: 1\n"
     ]
    }
   ],
   "source": [
    "first_sentence = \"this is a first sentence\"\n",
    "second_sentence = \"this is a second test sentence\"\n",
    "print(\"Simple Length Similarity Cutoff:\",my_tda.get_simple_similarity(first_sentence,second_sentence))\n",
    "print(\"BLEU Metric Similarity Cutoff:\",my_tda.get_BLEU_similarity(first_sentence,second_sentence))\n",
    "print(\"Logistic Regression Model Cuttoff:\",my_tda.get_prediction_LogisticRegression_model('fr',first_sentence,second_sentence))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Experiment - sentence translation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'être ou ne pas être telle est la question'"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "original_text = \"to be or not to be that is the question\"\n",
    "my_tda.translate(original_text, to_language=\"fr\", from_language=\"en\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'be or not to be , it is a question': 1,\n",
       " 'be it or not this is the question': 1,\n",
       " 'yes or no that question': 1,\n",
       " 'to be or not to be , that is the question': 1,\n",
       " 'to be or not to be such is the question': 1}"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "my_tda.generate_paraphrases(original_text, source_language_code='en', cut_off_method=\"logistic_model\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Experiment - Text Data Augmentation by extensive paratexts generation using Backtranslation\n",
    "\n",
    "Should be distributed over many CPUs / GPUs to accelerate the processing time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "my_tda.augment_text_data()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "    Augmenting... DATA/neg/cv676_22202.txt\n",
    "    Google Translate server exception ja\n",
    "    Google Translate server exception ru\n",
    "    Google Translate server exception tr\n",
    "    Paratext Dict created for the document\n",
    "    Text Data Augmentation done for the document\n",
    "    Saved: DATA/backtranslation_aug_neg/ampl_cv676_22202_0.txt\n",
    "    Elapsed time: 707.6035030048806\n",
    "    -----\n",
    "    Augmenting... DATA/neg/cv839_22807.txt\n",
    "    Google Translate server exception ru\n",
    "    Paratext Dict created for the document\n",
    "    Text Data Augmentation done for the document\n",
    "    Saved: DATA/backtranslation_aug_neg/ampl_cv839_22807_0.txt\n",
    "    Elapsed time: 406.81371479993686\n",
    "    -----\n",
    "    Augmenting... DATA/neg/cv155_7845.txt\n",
    "    Paratext Dict created for the document\n",
    "    Text Data Augmentation done for the document\n",
    "    Saved: DATA/backtranslation_aug_neg/ampl_cv155_7845_0.txt\n",
    "    Elapsed time: 247.3113233819604\n",
    "    -----\n",
    "    ..."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}