scripts/fine_tuned_BERT.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from transformers import BertTokenizer, BertForMaskedLM\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import pickle\n",
    "import os\n",
    "import argparse\n",
    "import numpy as np\n",
    "from tqdm import tqdm\n",
    "from nltk import ngrams\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\n",
    "\n",
    "\n",
    "from collections import Counter\n",
    "c = Counter()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.environ['CUDA_VISIBLE_DEVICES'] = '0'\n",
    "device = torch.device('cuda:0')\n",
    "# device = torch.device('cpu')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_name = 'bert-base-uncased'\n",
    "tokenizer = BertTokenizer.from_pretrained(model_name)\n",
    "model = BertForMaskedLM.from_pretrained(model_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BertForMaskedLM(\n",
       "  (bert): BertModel(\n",
       "    (embeddings): BertEmbeddings(\n",
       "      (word_embeddings): Embedding(30522, 768, padding_idx=0)\n",
       "      (position_embeddings): Embedding(512, 768)\n",
       "      (token_type_embeddings): Embedding(2, 768)\n",
       "      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "      (dropout): Dropout(p=0.1, inplace=False)\n",
       "    )\n",
       "    (encoder): BertEncoder(\n",
       "      (layer): ModuleList(\n",
       "        (0): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (intermediate_act_fn): GELUActivation()\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (1): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (intermediate_act_fn): GELUActivation()\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (2): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (intermediate_act_fn): GELUActivation()\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (3): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (intermediate_act_fn): GELUActivation()\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (4): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (intermediate_act_fn): GELUActivation()\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (5): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (intermediate_act_fn): GELUActivation()\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (6): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (intermediate_act_fn): GELUActivation()\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (7): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (intermediate_act_fn): GELUActivation()\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (8): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (intermediate_act_fn): GELUActivation()\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (9): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (intermediate_act_fn): GELUActivation()\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (10): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (intermediate_act_fn): GELUActivation()\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "        (11): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (intermediate_act_fn): GELUActivation()\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (cls): BertOnlyMLMHead(\n",
       "    (predictions): BertLMPredictionHead(\n",
       "      (transform): BertPredictionHeadTransform(\n",
       "        (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "        (transform_act_fn): GELUActivation()\n",
       "        (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "      )\n",
       "      (decoder): Linear(in_features=768, out_features=30522, bias=True)\n",
       "    )\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.to(device)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Making a list of negative and positive words"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Give a toxicity and normalcy score "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "class NgramSalienceCalculator():\n",
    "    def __init__(self, tox_corpus, norm_corpus, use_ngrams=False):\n",
    "        ngrams = (1, 3) if use_ngrams else (1, 1)\n",
    "        self.vectorizer = CountVectorizer(ngram_range=ngrams)\n",
    "\n",
    "        tox_count_matrix = self.vectorizer.fit_transform(tox_corpus)\n",
    "        self.tox_vocab = self.vectorizer.vocabulary_\n",
    "        self.tox_counts = np.sum(tox_count_matrix, axis=0)\n",
    "\n",
    "        norm_count_matrix = self.vectorizer.fit_transform(norm_corpus)\n",
    "        self.norm_vocab = self.vectorizer.vocabulary_\n",
    "        self.norm_counts = np.sum(norm_count_matrix, axis=0)\n",
    "\n",
    "    def salience(self, feature, attribute='tox', lmbda=0.5):\n",
    "        assert attribute in ['tox', 'norm']\n",
    "        if feature not in self.tox_vocab:\n",
    "            tox_count = 0.0\n",
    "        else:\n",
    "            tox_count = self.tox_counts[0, self.tox_vocab[feature]]\n",
    "\n",
    "        if feature not in self.norm_vocab:\n",
    "            norm_count = 0.0\n",
    "        else:\n",
    "            norm_count = self.norm_counts[0, self.norm_vocab[feature]]\n",
    "\n",
    "        if attribute == 'tox':\n",
    "            return (tox_count + lmbda) / (norm_count + lmbda)\n",
    "        else:\n",
    "            return (norm_count + lmbda) / (tox_count + lmbda)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "88645\n"
     ]
    }
   ],
   "source": [
    "tox_corpus_path = '../data/train/train_toxic'\n",
    "norm_corpus_path = '../data/train/train_normal'\n",
    "\n",
    "for fn in [tox_corpus_path, norm_corpus_path]:\n",
    "    with open(fn, 'r') as corpus:\n",
    "        for line in corpus.readlines():\n",
    "            for tok in line.strip().split():\n",
    "                c[tok] += 1\n",
    "\n",
    "print(len(c))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "88645\n"
     ]
    }
   ],
   "source": [
    "vocab = {w for w, _ in c.most_common() if _ > 0}  # if we took words with > 1 occurences, vocabulary would be x2 smaller, but we'll survive this size\n",
    "print(len(vocab))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(tox_corpus_path, 'r') as tox_corpus, open(norm_corpus_path, 'r') as norm_corpus:\n",
    "    corpus_tox = [' '.join([w if w in vocab else '<unk>' for w in line.strip().split()]) for line in tox_corpus.readlines()]\n",
    "    corpus_norm = [' '.join([w if w in vocab else '<unk>' for w in line.strip().split()]) for line in norm_corpus.readlines()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "threshold = 4\n",
    "\n",
    "sc = NgramSalienceCalculator(corpus_tox, corpus_norm, False)\n",
    "seen_grams = set()\n",
    "\n",
    "neg_out_name = '../data/vocab/negative_words.txt'\n",
    "pos_out_name = '../data/vocab/positive_words.txt'\n",
    "\n",
    "with open(neg_out_name, 'a') as neg_out, open(pos_out_name, 'a') as pos_out:\n",
    "    for gram in set(sc.tox_vocab.keys()).union(set(sc.norm_vocab.keys())):\n",
    "        if gram not in seen_grams:\n",
    "            seen_grams.add(gram)\n",
    "            toxic_salience = sc.salience(gram, attribute='tox')\n",
    "            polite_salience = sc.salience(gram, attribute='norm')\n",
    "            if toxic_salience > threshold:\n",
    "                neg_out.writelines(f'{gram}\\n')\n",
    "            elif polite_salience > threshold:\n",
    "                pos_out.writelines(f'{gram}\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fine Tuning the BERT Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"../data/train/train_toxic\", 'r') as f:\n",
    "    toxic = f.readlines()\n",
    "\n",
    "with open(\"../data/train/train_normal\", 'r') as f:\n",
    "    normal = f.readlines()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "135390"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(toxic)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "135390"
      ]
     },
     "execution_count": 104,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(normal)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# read a txt in a list, every line a new element\n",
    "def read_txt(path):\n",
    "    with open(path, 'r') as f:\n",
    "        lines = f.readlines()\n",
    "    # remove \\n\n",
    "    lines = [line.strip() for line in lines]\n",
    "    return lines\n",
    "\n",
    "negative = read_txt(\"../data/vocab/negative_words.txt\")\n",
    "positive = read_txt(\"../data/vocab/positive_words.txt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# convert list to pandas\n",
    "toxic_df = pd.DataFrame(toxic, columns=['text'])\n",
    "normal_df = pd.DataFrame(normal, columns=['text'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "device(type='cuda', index=0)"
      ]
     },
     "execution_count": 107,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>just a comment regarding family trusts , they ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>nor do they conform to the notion that our tit...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>yeah , the pers employees and their pensions f...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>why risk our marine parks and sea life for a f...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>not sure what flavor koolaid you drinking but ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>135385</th>\n",
       "      <td>i think you , other like minded individuals ge...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>135386</th>\n",
       "      <td>a lot of people will be wondering what they we...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>135387</th>\n",
       "      <td>they have endured saber rattling by the us for...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>135388</th>\n",
       "      <td>the cbc has been totally silent on reporting a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>135389</th>\n",
       "      <td>its not the police fault .\\n</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>135390 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                     text\n",
       "0       just a comment regarding family trusts , they ...\n",
       "1       nor do they conform to the notion that our tit...\n",
       "2       yeah , the pers employees and their pensions f...\n",
       "3       why risk our marine parks and sea life for a f...\n",
       "4       not sure what flavor koolaid you drinking but ...\n",
       "...                                                   ...\n",
       "135385  i think you , other like minded individuals ge...\n",
       "135386  a lot of people will be wondering what they we...\n",
       "135387  they have endured saber rattling by the us for...\n",
       "135388  the cbc has been totally silent on reporting a...\n",
       "135389                       its not the police fault .\\n\n",
       "\n",
       "[135390 rows x 1 columns]"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "normal_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# tokenize the text\n",
    "\n",
    "def masking(df, word_list):\n",
    "    df['tokenized_text'] = df['text'].apply((lambda x: tokenizer.tokenize(x)))\n",
    "    df['masked_text'] = df['tokenized_text'].apply((lambda x: [word.lower() if word not in word_list else '[MASK]' for word in x]))\n",
    "    df[\"masked_encoded_text\"] = df['masked_text'].apply((lambda x: tokenizer.encode(x,)))\n",
    "    df[\"encoded_text\"] = df['tokenized_text'].apply((lambda x: tokenizer.encode(x)))\n",
    "    return df\n",
    "\n",
    "toxic_df_masked = masking(toxic_df, negative)\n",
    "normal_df_masked = masking(normal_df, positive)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "toxic_df_masked[\"label\"] = 1\n",
    "normal_df_masked[\"label\"] = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>tokenized_text</th>\n",
       "      <th>masked_text</th>\n",
       "      <th>masked_encoded_text</th>\n",
       "      <th>encoded_text</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>just a comment regarding family trusts , they ...</td>\n",
       "      <td>[just, a, comment, regarding, family, trusts, ...</td>\n",
       "      <td>[just, a, comment, regarding, family, trusts, ...</td>\n",
       "      <td>[101, 2074, 1037, 7615, 4953, 2155, 20278, 101...</td>\n",
       "      <td>[101, 2074, 1037, 7615, 4953, 2155, 20278, 101...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>nor do they conform to the notion that our tit...</td>\n",
       "      <td>[nor, do, they, conform, to, the, notion, that...</td>\n",
       "      <td>[nor, do, they, conform, to, the, notion, that...</td>\n",
       "      <td>[101, 4496, 2079, 2027, 23758, 2000, 1996, 936...</td>\n",
       "      <td>[101, 4496, 2079, 2027, 23758, 2000, 1996, 936...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>yeah , the pers employees and their pensions f...</td>\n",
       "      <td>[yeah, ,, the, per, ##s, employees, and, their...</td>\n",
       "      <td>[yeah, ,, the, [MASK], ##s, [MASK], and, their...</td>\n",
       "      <td>[101, 3398, 1010, 1996, 103, 2015, 103, 1998, ...</td>\n",
       "      <td>[101, 3398, 1010, 1996, 2566, 2015, 5126, 1998...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>why risk our marine parks and sea life for a f...</td>\n",
       "      <td>[why, risk, our, marine, parks, and, sea, life...</td>\n",
       "      <td>[why, risk, our, marine, [MASK], and, sea, lif...</td>\n",
       "      <td>[101, 2339, 3891, 2256, 3884, 103, 1998, 2712,...</td>\n",
       "      <td>[101, 2339, 3891, 2256, 3884, 6328, 1998, 2712...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>not sure what flavor koolaid you drinking but ...</td>\n",
       "      <td>[not, sure, what, flavor, ko, ##ola, ##id, you...</td>\n",
       "      <td>[not, sure, what, flavor, [MASK], ##ola, ##id,...</td>\n",
       "      <td>[101, 2025, 2469, 2054, 14894, 103, 6030, 3593...</td>\n",
       "      <td>[101, 2025, 2469, 2054, 14894, 12849, 6030, 35...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>135385</th>\n",
       "      <td>i think you , other like minded individuals ge...</td>\n",
       "      <td>[i, think, you, ,, other, like, minded, indivi...</td>\n",
       "      <td>[i, think, you, ,, other, like, minded, indivi...</td>\n",
       "      <td>[101, 1045, 2228, 2017, 1010, 2060, 2066, 1312...</td>\n",
       "      <td>[101, 1045, 2228, 2017, 1010, 2060, 2066, 1312...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>135386</th>\n",
       "      <td>a lot of people will be wondering what they we...</td>\n",
       "      <td>[a, lot, of, people, will, be, wondering, what...</td>\n",
       "      <td>[a, lot, of, people, will, be, wondering, what...</td>\n",
       "      <td>[101, 1037, 2843, 1997, 2111, 2097, 2022, 6603...</td>\n",
       "      <td>[101, 1037, 2843, 1997, 2111, 2097, 2022, 6603...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>135387</th>\n",
       "      <td>they have endured saber rattling by the us for...</td>\n",
       "      <td>[they, have, endured, saber, rattling, by, the...</td>\n",
       "      <td>[they, have, endured, saber, rattling, by, the...</td>\n",
       "      <td>[101, 2027, 2031, 16753, 25653, 26347, 2011, 1...</td>\n",
       "      <td>[101, 2027, 2031, 16753, 25653, 26347, 2011, 1...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>135388</th>\n",
       "      <td>the cbc has been totally silent on reporting a...</td>\n",
       "      <td>[the, cbc, has, been, totally, silent, on, rep...</td>\n",
       "      <td>[the, cbc, has, been, totally, silent, on, rep...</td>\n",
       "      <td>[101, 1996, 13581, 2038, 2042, 6135, 4333, 200...</td>\n",
       "      <td>[101, 1996, 13581, 2038, 2042, 6135, 4333, 200...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>135389</th>\n",
       "      <td>its not the police fault .\\n</td>\n",
       "      <td>[its, not, the, police, fault, .]</td>\n",
       "      <td>[its, not, the, police, fault, .]</td>\n",
       "      <td>[101, 2049, 2025, 1996, 2610, 6346, 1012, 102]</td>\n",
       "      <td>[101, 2049, 2025, 1996, 2610, 6346, 1012, 102]</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>135390 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                     text  \\\n",
       "0       just a comment regarding family trusts , they ...   \n",
       "1       nor do they conform to the notion that our tit...   \n",
       "2       yeah , the pers employees and their pensions f...   \n",
       "3       why risk our marine parks and sea life for a f...   \n",
       "4       not sure what flavor koolaid you drinking but ...   \n",
       "...                                                   ...   \n",
       "135385  i think you , other like minded individuals ge...   \n",
       "135386  a lot of people will be wondering what they we...   \n",
       "135387  they have endured saber rattling by the us for...   \n",
       "135388  the cbc has been totally silent on reporting a...   \n",
       "135389                       its not the police fault .\\n   \n",
       "\n",
       "                                           tokenized_text  \\\n",
       "0       [just, a, comment, regarding, family, trusts, ...   \n",
       "1       [nor, do, they, conform, to, the, notion, that...   \n",
       "2       [yeah, ,, the, per, ##s, employees, and, their...   \n",
       "3       [why, risk, our, marine, parks, and, sea, life...   \n",
       "4       [not, sure, what, flavor, ko, ##ola, ##id, you...   \n",
       "...                                                   ...   \n",
       "135385  [i, think, you, ,, other, like, minded, indivi...   \n",
       "135386  [a, lot, of, people, will, be, wondering, what...   \n",
       "135387  [they, have, endured, saber, rattling, by, the...   \n",
       "135388  [the, cbc, has, been, totally, silent, on, rep...   \n",
       "135389                  [its, not, the, police, fault, .]   \n",
       "\n",
       "                                              masked_text  \\\n",
       "0       [just, a, comment, regarding, family, trusts, ...   \n",
       "1       [nor, do, they, conform, to, the, notion, that...   \n",
       "2       [yeah, ,, the, [MASK], ##s, [MASK], and, their...   \n",
       "3       [why, risk, our, marine, [MASK], and, sea, lif...   \n",
       "4       [not, sure, what, flavor, [MASK], ##ola, ##id,...   \n",
       "...                                                   ...   \n",
       "135385  [i, think, you, ,, other, like, minded, indivi...   \n",
       "135386  [a, lot, of, people, will, be, wondering, what...   \n",
       "135387  [they, have, endured, saber, rattling, by, the...   \n",
       "135388  [the, cbc, has, been, totally, silent, on, rep...   \n",
       "135389                  [its, not, the, police, fault, .]   \n",
       "\n",
       "                                      masked_encoded_text  \\\n",
       "0       [101, 2074, 1037, 7615, 4953, 2155, 20278, 101...   \n",
       "1       [101, 4496, 2079, 2027, 23758, 2000, 1996, 936...   \n",
       "2       [101, 3398, 1010, 1996, 103, 2015, 103, 1998, ...   \n",
       "3       [101, 2339, 3891, 2256, 3884, 103, 1998, 2712,...   \n",
       "4       [101, 2025, 2469, 2054, 14894, 103, 6030, 3593...   \n",
       "...                                                   ...   \n",
       "135385  [101, 1045, 2228, 2017, 1010, 2060, 2066, 1312...   \n",
       "135386  [101, 1037, 2843, 1997, 2111, 2097, 2022, 6603...   \n",
       "135387  [101, 2027, 2031, 16753, 25653, 26347, 2011, 1...   \n",
       "135388  [101, 1996, 13581, 2038, 2042, 6135, 4333, 200...   \n",
       "135389     [101, 2049, 2025, 1996, 2610, 6346, 1012, 102]   \n",
       "\n",
       "                                             encoded_text  label  \n",
       "0       [101, 2074, 1037, 7615, 4953, 2155, 20278, 101...      0  \n",
       "1       [101, 4496, 2079, 2027, 23758, 2000, 1996, 936...      0  \n",
       "2       [101, 3398, 1010, 1996, 2566, 2015, 5126, 1998...      0  \n",
       "3       [101, 2339, 3891, 2256, 3884, 6328, 1998, 2712...      0  \n",
       "4       [101, 2025, 2469, 2054, 14894, 12849, 6030, 35...      0  \n",
       "...                                                   ...    ...  \n",
       "135385  [101, 1045, 2228, 2017, 1010, 2060, 2066, 1312...      0  \n",
       "135386  [101, 1037, 2843, 1997, 2111, 2097, 2022, 6603...      0  \n",
       "135387  [101, 2027, 2031, 16753, 25653, 26347, 2011, 1...      0  \n",
       "135388  [101, 1996, 13581, 2038, 2042, 6135, 4333, 200...      0  \n",
       "135389     [101, 2049, 2025, 1996, 2610, 6346, 1012, 102]      0  \n",
       "\n",
       "[135390 rows x 6 columns]"
      ]
     },
     "execution_count": 242,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "normal_df_masked"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# concat the two dataframes and shuffle them\n",
    "df = pd.concat([toxic_df_masked, normal_df_masked], ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "72\n"
     ]
    }
   ],
   "source": [
    "# print avg length of the sentences\n",
    "print(df['masked_encoded_text'].apply(len).max())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>tokenized_text</th>\n",
       "      <th>masked_text</th>\n",
       "      <th>masked_encoded_text</th>\n",
       "      <th>encoded_text</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>fuck you , you nixonian twit .\\n</td>\n",
       "      <td>[fuck, you, ,, you, nixon, ##ian, t, ##wi, ##t...</td>\n",
       "      <td>[[MASK], you, ,, you, nixon, ##ian, t, ##wi, #...</td>\n",
       "      <td>[101, 103, 2017, 1010, 2017, 11296, 2937, 1056...</td>\n",
       "      <td>[101, 6616, 2017, 1010, 2017, 11296, 2937, 105...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>just another vicious and trashy antitrump , pr...</td>\n",
       "      <td>[just, another, vicious, and, trash, ##y, anti...</td>\n",
       "      <td>[just, another, vicious, and, [MASK], ##y, ant...</td>\n",
       "      <td>[101, 2074, 2178, 13925, 1998, 103, 2100, 3424...</td>\n",
       "      <td>[101, 2074, 2178, 13925, 1998, 11669, 2100, 34...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>if you want to talk hypocrites , bauer , im up...</td>\n",
       "      <td>[if, you, want, to, talk, h, ##yp, ##oc, ##rit...</td>\n",
       "      <td>[if, you, want, to, talk, h, ##yp, ##oc, ##rit...</td>\n",
       "      <td>[101, 2065, 2017, 2215, 2000, 2831, 1044, 2257...</td>\n",
       "      <td>[101, 2065, 2017, 2215, 2000, 2831, 1044, 2257...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>that you were shocked by the backlash only fur...</td>\n",
       "      <td>[that, you, were, shocked, by, the, backlash, ...</td>\n",
       "      <td>[that, you, were, shocked, by, the, backlash, ...</td>\n",
       "      <td>[101, 2008, 2017, 2020, 7135, 2011, 1996, 2574...</td>\n",
       "      <td>[101, 2008, 2017, 2020, 7135, 2011, 1996, 2574...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>you all suck shapiros cock and lick his mother...</td>\n",
       "      <td>[you, all, suck, shapiro, ##s, cock, and, lick...</td>\n",
       "      <td>[you, all, [MASK], shapiro, ##s, [MASK], and, ...</td>\n",
       "      <td>[101, 2017, 2035, 103, 24630, 2015, 103, 1998,...</td>\n",
       "      <td>[101, 2017, 2035, 11891, 24630, 2015, 10338, 1...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>270775</th>\n",
       "      <td>i think you , other like minded individuals ge...</td>\n",
       "      <td>[i, think, you, ,, other, like, minded, indivi...</td>\n",
       "      <td>[i, think, you, ,, other, like, minded, indivi...</td>\n",
       "      <td>[101, 1045, 2228, 2017, 1010, 2060, 2066, 1312...</td>\n",
       "      <td>[101, 1045, 2228, 2017, 1010, 2060, 2066, 1312...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>270776</th>\n",
       "      <td>a lot of people will be wondering what they we...</td>\n",
       "      <td>[a, lot, of, people, will, be, wondering, what...</td>\n",
       "      <td>[a, lot, of, people, will, be, wondering, what...</td>\n",
       "      <td>[101, 1037, 2843, 1997, 2111, 2097, 2022, 6603...</td>\n",
       "      <td>[101, 1037, 2843, 1997, 2111, 2097, 2022, 6603...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>270777</th>\n",
       "      <td>they have endured saber rattling by the us for...</td>\n",
       "      <td>[they, have, endured, saber, rattling, by, the...</td>\n",
       "      <td>[they, have, endured, saber, rattling, by, the...</td>\n",
       "      <td>[101, 2027, 2031, 16753, 25653, 26347, 2011, 1...</td>\n",
       "      <td>[101, 2027, 2031, 16753, 25653, 26347, 2011, 1...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>270778</th>\n",
       "      <td>the cbc has been totally silent on reporting a...</td>\n",
       "      <td>[the, cbc, has, been, totally, silent, on, rep...</td>\n",
       "      <td>[the, cbc, has, been, totally, silent, on, rep...</td>\n",
       "      <td>[101, 1996, 13581, 2038, 2042, 6135, 4333, 200...</td>\n",
       "      <td>[101, 1996, 13581, 2038, 2042, 6135, 4333, 200...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>270779</th>\n",
       "      <td>its not the police fault .\\n</td>\n",
       "      <td>[its, not, the, police, fault, .]</td>\n",
       "      <td>[its, not, the, police, fault, .]</td>\n",
       "      <td>[101, 2049, 2025, 1996, 2610, 6346, 1012, 102]</td>\n",
       "      <td>[101, 2049, 2025, 1996, 2610, 6346, 1012, 102]</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>270780 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                     text  \\\n",
       "0                        fuck you , you nixonian twit .\\n   \n",
       "1       just another vicious and trashy antitrump , pr...   \n",
       "2       if you want to talk hypocrites , bauer , im up...   \n",
       "3       that you were shocked by the backlash only fur...   \n",
       "4       you all suck shapiros cock and lick his mother...   \n",
       "...                                                   ...   \n",
       "270775  i think you , other like minded individuals ge...   \n",
       "270776  a lot of people will be wondering what they we...   \n",
       "270777  they have endured saber rattling by the us for...   \n",
       "270778  the cbc has been totally silent on reporting a...   \n",
       "270779                       its not the police fault .\\n   \n",
       "\n",
       "                                           tokenized_text  \\\n",
       "0       [fuck, you, ,, you, nixon, ##ian, t, ##wi, ##t...   \n",
       "1       [just, another, vicious, and, trash, ##y, anti...   \n",
       "2       [if, you, want, to, talk, h, ##yp, ##oc, ##rit...   \n",
       "3       [that, you, were, shocked, by, the, backlash, ...   \n",
       "4       [you, all, suck, shapiro, ##s, cock, and, lick...   \n",
       "...                                                   ...   \n",
       "270775  [i, think, you, ,, other, like, minded, indivi...   \n",
       "270776  [a, lot, of, people, will, be, wondering, what...   \n",
       "270777  [they, have, endured, saber, rattling, by, the...   \n",
       "270778  [the, cbc, has, been, totally, silent, on, rep...   \n",
       "270779                  [its, not, the, police, fault, .]   \n",
       "\n",
       "                                              masked_text  \\\n",
       "0       [[MASK], you, ,, you, nixon, ##ian, t, ##wi, #...   \n",
       "1       [just, another, vicious, and, [MASK], ##y, ant...   \n",
       "2       [if, you, want, to, talk, h, ##yp, ##oc, ##rit...   \n",
       "3       [that, you, were, shocked, by, the, backlash, ...   \n",
       "4       [you, all, [MASK], shapiro, ##s, [MASK], and, ...   \n",
       "...                                                   ...   \n",
       "270775  [i, think, you, ,, other, like, minded, indivi...   \n",
       "270776  [a, lot, of, people, will, be, wondering, what...   \n",
       "270777  [they, have, endured, saber, rattling, by, the...   \n",
       "270778  [the, cbc, has, been, totally, silent, on, rep...   \n",
       "270779                  [its, not, the, police, fault, .]   \n",
       "\n",
       "                                      masked_encoded_text  \\\n",
       "0       [101, 103, 2017, 1010, 2017, 11296, 2937, 1056...   \n",
       "1       [101, 2074, 2178, 13925, 1998, 103, 2100, 3424...   \n",
       "2       [101, 2065, 2017, 2215, 2000, 2831, 1044, 2257...   \n",
       "3       [101, 2008, 2017, 2020, 7135, 2011, 1996, 2574...   \n",
       "4       [101, 2017, 2035, 103, 24630, 2015, 103, 1998,...   \n",
       "...                                                   ...   \n",
       "270775  [101, 1045, 2228, 2017, 1010, 2060, 2066, 1312...   \n",
       "270776  [101, 1037, 2843, 1997, 2111, 2097, 2022, 6603...   \n",
       "270777  [101, 2027, 2031, 16753, 25653, 26347, 2011, 1...   \n",
       "270778  [101, 1996, 13581, 2038, 2042, 6135, 4333, 200...   \n",
       "270779     [101, 2049, 2025, 1996, 2610, 6346, 1012, 102]   \n",
       "\n",
       "                                             encoded_text  label  \n",
       "0       [101, 6616, 2017, 1010, 2017, 11296, 2937, 105...      1  \n",
       "1       [101, 2074, 2178, 13925, 1998, 11669, 2100, 34...      1  \n",
       "2       [101, 2065, 2017, 2215, 2000, 2831, 1044, 2257...      1  \n",
       "3       [101, 2008, 2017, 2020, 7135, 2011, 1996, 2574...      1  \n",
       "4       [101, 2017, 2035, 11891, 24630, 2015, 10338, 1...      1  \n",
       "...                                                   ...    ...  \n",
       "270775  [101, 1045, 2228, 2017, 1010, 2060, 2066, 1312...      0  \n",
       "270776  [101, 1037, 2843, 1997, 2111, 2097, 2022, 6603...      0  \n",
       "270777  [101, 2027, 2031, 16753, 25653, 26347, 2011, 1...      0  \n",
       "270778  [101, 1996, 13581, 2038, 2042, 6135, 4333, 200...      0  \n",
       "270779     [101, 2049, 2025, 1996, 2610, 6346, 1012, 102]      0  \n",
       "\n",
       "[270780 rows x 6 columns]"
      ]
     },
     "execution_count": 258,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save the dataframe in pickle file\n",
    "df.to_pickle(\"../data/data.pkl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get the data from pickle file\n",
    "df = pd.read_pickle(\"../data/data.pkl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>tokenized_text</th>\n",
       "      <th>masked_text</th>\n",
       "      <th>masked_encoded_text</th>\n",
       "      <th>encoded_text</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>fuck you , you nixonian twit .\\n</td>\n",
       "      <td>[fuck, you, ,, you, nixon, ##ian, t, ##wi, ##t...</td>\n",
       "      <td>[[MASK], you, ,, you, nixon, ##ian, t, ##wi, #...</td>\n",
       "      <td>[101, 103, 2017, 1010, 2017, 11296, 2937, 1056...</td>\n",
       "      <td>[101, 6616, 2017, 1010, 2017, 11296, 2937, 105...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>just another vicious and trashy antitrump , pr...</td>\n",
       "      <td>[just, another, vicious, and, trash, ##y, anti...</td>\n",
       "      <td>[just, another, vicious, and, [MASK], ##y, ant...</td>\n",
       "      <td>[101, 2074, 2178, 13925, 1998, 103, 2100, 3424...</td>\n",
       "      <td>[101, 2074, 2178, 13925, 1998, 11669, 2100, 34...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>if you want to talk hypocrites , bauer , im up...</td>\n",
       "      <td>[if, you, want, to, talk, h, ##yp, ##oc, ##rit...</td>\n",
       "      <td>[if, you, want, to, talk, h, ##yp, ##oc, ##rit...</td>\n",
       "      <td>[101, 2065, 2017, 2215, 2000, 2831, 1044, 2257...</td>\n",
       "      <td>[101, 2065, 2017, 2215, 2000, 2831, 1044, 2257...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>that you were shocked by the backlash only fur...</td>\n",
       "      <td>[that, you, were, shocked, by, the, backlash, ...</td>\n",
       "      <td>[that, you, were, shocked, by, the, backlash, ...</td>\n",
       "      <td>[101, 2008, 2017, 2020, 7135, 2011, 1996, 2574...</td>\n",
       "      <td>[101, 2008, 2017, 2020, 7135, 2011, 1996, 2574...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>you all suck shapiros cock and lick his mother...</td>\n",
       "      <td>[you, all, suck, shapiro, ##s, cock, and, lick...</td>\n",
       "      <td>[you, all, [MASK], shapiro, ##s, [MASK], and, ...</td>\n",
       "      <td>[101, 2017, 2035, 103, 24630, 2015, 103, 1998,...</td>\n",
       "      <td>[101, 2017, 2035, 11891, 24630, 2015, 10338, 1...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>270775</th>\n",
       "      <td>i think you , other like minded individuals ge...</td>\n",
       "      <td>[i, think, you, ,, other, like, minded, indivi...</td>\n",
       "      <td>[i, think, you, ,, other, like, minded, indivi...</td>\n",
       "      <td>[101, 1045, 2228, 2017, 1010, 2060, 2066, 1312...</td>\n",
       "      <td>[101, 1045, 2228, 2017, 1010, 2060, 2066, 1312...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>270776</th>\n",
       "      <td>a lot of people will be wondering what they we...</td>\n",
       "      <td>[a, lot, of, people, will, be, wondering, what...</td>\n",
       "      <td>[a, lot, of, people, will, be, wondering, what...</td>\n",
       "      <td>[101, 1037, 2843, 1997, 2111, 2097, 2022, 6603...</td>\n",
       "      <td>[101, 1037, 2843, 1997, 2111, 2097, 2022, 6603...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>270777</th>\n",
       "      <td>they have endured saber rattling by the us for...</td>\n",
       "      <td>[they, have, endured, saber, rattling, by, the...</td>\n",
       "      <td>[they, have, endured, saber, rattling, by, the...</td>\n",
       "      <td>[101, 2027, 2031, 16753, 25653, 26347, 2011, 1...</td>\n",
       "      <td>[101, 2027, 2031, 16753, 25653, 26347, 2011, 1...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>270778</th>\n",
       "      <td>the cbc has been totally silent on reporting a...</td>\n",
       "      <td>[the, cbc, has, been, totally, silent, on, rep...</td>\n",
       "      <td>[the, cbc, has, been, totally, silent, on, rep...</td>\n",
       "      <td>[101, 1996, 13581, 2038, 2042, 6135, 4333, 200...</td>\n",
       "      <td>[101, 1996, 13581, 2038, 2042, 6135, 4333, 200...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>270779</th>\n",
       "      <td>its not the police fault .\\n</td>\n",
       "      <td>[its, not, the, police, fault, .]</td>\n",
       "      <td>[its, not, the, police, fault, .]</td>\n",
       "      <td>[101, 2049, 2025, 1996, 2610, 6346, 1012, 102]</td>\n",
       "      <td>[101, 2049, 2025, 1996, 2610, 6346, 1012, 102]</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>270780 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                     text  \\\n",
       "0                        fuck you , you nixonian twit .\\n   \n",
       "1       just another vicious and trashy antitrump , pr...   \n",
       "2       if you want to talk hypocrites , bauer , im up...   \n",
       "3       that you were shocked by the backlash only fur...   \n",
       "4       you all suck shapiros cock and lick his mother...   \n",
       "...                                                   ...   \n",
       "270775  i think you , other like minded individuals ge...   \n",
       "270776  a lot of people will be wondering what they we...   \n",
       "270777  they have endured saber rattling by the us for...   \n",
       "270778  the cbc has been totally silent on reporting a...   \n",
       "270779                       its not the police fault .\\n   \n",
       "\n",
       "                                           tokenized_text  \\\n",
       "0       [fuck, you, ,, you, nixon, ##ian, t, ##wi, ##t...   \n",
       "1       [just, another, vicious, and, trash, ##y, anti...   \n",
       "2       [if, you, want, to, talk, h, ##yp, ##oc, ##rit...   \n",
       "3       [that, you, were, shocked, by, the, backlash, ...   \n",
       "4       [you, all, suck, shapiro, ##s, cock, and, lick...   \n",
       "...                                                   ...   \n",
       "270775  [i, think, you, ,, other, like, minded, indivi...   \n",
       "270776  [a, lot, of, people, will, be, wondering, what...   \n",
       "270777  [they, have, endured, saber, rattling, by, the...   \n",
       "270778  [the, cbc, has, been, totally, silent, on, rep...   \n",
       "270779                  [its, not, the, police, fault, .]   \n",
       "\n",
       "                                              masked_text  \\\n",
       "0       [[MASK], you, ,, you, nixon, ##ian, t, ##wi, #...   \n",
       "1       [just, another, vicious, and, [MASK], ##y, ant...   \n",
       "2       [if, you, want, to, talk, h, ##yp, ##oc, ##rit...   \n",
       "3       [that, you, were, shocked, by, the, backlash, ...   \n",
       "4       [you, all, [MASK], shapiro, ##s, [MASK], and, ...   \n",
       "...                                                   ...   \n",
       "270775  [i, think, you, ,, other, like, minded, indivi...   \n",
       "270776  [a, lot, of, people, will, be, wondering, what...   \n",
       "270777  [they, have, endured, saber, rattling, by, the...   \n",
       "270778  [the, cbc, has, been, totally, silent, on, rep...   \n",
       "270779                  [its, not, the, police, fault, .]   \n",
       "\n",
       "                                      masked_encoded_text  \\\n",
       "0       [101, 103, 2017, 1010, 2017, 11296, 2937, 1056...   \n",
       "1       [101, 2074, 2178, 13925, 1998, 103, 2100, 3424...   \n",
       "2       [101, 2065, 2017, 2215, 2000, 2831, 1044, 2257...   \n",
       "3       [101, 2008, 2017, 2020, 7135, 2011, 1996, 2574...   \n",
       "4       [101, 2017, 2035, 103, 24630, 2015, 103, 1998,...   \n",
       "...                                                   ...   \n",
       "270775  [101, 1045, 2228, 2017, 1010, 2060, 2066, 1312...   \n",
       "270776  [101, 1037, 2843, 1997, 2111, 2097, 2022, 6603...   \n",
       "270777  [101, 2027, 2031, 16753, 25653, 26347, 2011, 1...   \n",
       "270778  [101, 1996, 13581, 2038, 2042, 6135, 4333, 200...   \n",
       "270779     [101, 2049, 2025, 1996, 2610, 6346, 1012, 102]   \n",
       "\n",
       "                                             encoded_text  label  \n",
       "0       [101, 6616, 2017, 1010, 2017, 11296, 2937, 105...      1  \n",
       "1       [101, 2074, 2178, 13925, 1998, 11669, 2100, 34...      1  \n",
       "2       [101, 2065, 2017, 2215, 2000, 2831, 1044, 2257...      1  \n",
       "3       [101, 2008, 2017, 2020, 7135, 2011, 1996, 2574...      1  \n",
       "4       [101, 2017, 2035, 11891, 24630, 2015, 10338, 1...      1  \n",
       "...                                                   ...    ...  \n",
       "270775  [101, 1045, 2228, 2017, 1010, 2060, 2066, 1312...      0  \n",
       "270776  [101, 1037, 2843, 1997, 2111, 2097, 2022, 6603...      0  \n",
       "270777  [101, 2027, 2031, 16753, 25653, 26347, 2011, 1...      0  \n",
       "270778  [101, 1996, 13581, 2038, 2042, 6135, 4333, 200...      0  \n",
       "270779     [101, 2049, 2025, 1996, 2610, 6346, 1012, 102]      0  \n",
       "\n",
       "[270780 rows x 6 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# split the data into train, val and test\n",
    "train, val, test = np.split(df.sample(frac=1), [int(.6*len(df)), int(.8*len(df))])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class BERTFineTuneDataset(torch.utils.data.Dataset):\n",
    "  def __init__(self,df, maxlen = 64):\n",
    "    super(BERTFineTuneDataset, self).__init__()\n",
    "    # pad the encoded text to maxlen\n",
    "    self._encoded_text = torch.nn.utils.rnn.pad_sequence(df['encoded_text'].apply(torch.tensor), batch_first=True, padding_value=0)\n",
    "    df[\"labels\"] = df[\"label\"].apply((lambda x: torch.ones(maxlen).int() if x == 1 else torch.zeros(maxlen).int()))\n",
    "    \n",
    "    self._labels = df['labels'].values\n",
    "\n",
    "    self._maxlen = maxlen\n",
    "  \n",
    "    self._masked_encoded_text = torch.nn.utils.rnn.pad_sequence(df['masked_encoded_text'].apply(torch.tensor), batch_first=True, padding_value=0)\n",
    "    self._attention_mask = torch.nn.utils.rnn.pad_sequence(df['masked_encoded_text'].apply(lambda x: torch.tensor([1 if i != 0 else 0 for i in x])), batch_first=True, padding_value=0)\n",
    "\n",
    "  def __len__(self):\n",
    "    return len(self._labels)\n",
    "  \n",
    "  def __getitem__(self, idx):\n",
    "    # make dict of the data\n",
    "    return {\n",
    "\n",
    "        'encoded_text': self._encoded_text[idx][:self._maxlen],\n",
    "        'masked_encoded_text': self._masked_encoded_text[idx][:self._maxlen],\n",
    "        'attention_mask': self._attention_mask[idx][:self._maxlen],\n",
    "        'labels': self._labels[idx][:self._maxlen]    \n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# dataloader\n",
    "train_dataset = BERTFineTuneDataset(train)\n",
    "val_dataset = BERTFineTuneDataset(val)\n",
    "test_dataset = BERTFineTuneDataset(test)\n",
    "\n",
    "train_dataloader = torch.utils.data.DataLoader(train_dataset, batch_size=32, shuffle=True)\n",
    "val_dataloader = torch.utils.data.DataLoader(val_dataset, batch_size=32, shuffle=True)\n",
    "test_dataloader = torch.utils.data.DataLoader(test_dataset, batch_size=32, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save the dataloader in pickle file\n",
    "with open(\"../data/train_dataloader.pkl\", 'wb') as f:\n",
    "    pickle.dump(train_dataloader, f)\n",
    "\n",
    "with open(\"../data/val_dataloader.pkl\", 'wb') as f:\n",
    "    pickle.dump(val_dataloader, f)\n",
    "\n",
    "with open(\"../data/test_dataloader.pkl\", 'wb') as f:\n",
    "    pickle.dump(test_dataloader, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.026597120194012114\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import pickle\n",
    "\n",
    "# load the finetuned model\n",
    "model = torch.load(\"../bert_detox_ft_4epochs.pth\")\n",
    "\n",
    "#load the val dataloader\n",
    "with open(\"../data/val_dataloader.pkl\", 'rb') as f:\n",
    "    val_dataloader = pickle.load(f)\n",
    "\n",
    "# validation code\n",
    "model.eval()\n",
    "val_loss = 0\n",
    "for batch in val_dataloader:\n",
    "    input_ids = batch['masked_encoded_text'].to(torch.device('cuda'))\n",
    "    attention_mask = batch['attention_mask'].to(torch.device('cuda'))\n",
    "    labels = batch['encoded_text'].to(torch.device('cuda'))\n",
    "    segment_ids = batch['labels'].to(torch.device('cuda'))\n",
    "    outputs = model(input_ids, attention_mask=attention_mask,token_type_ids = segment_ids, labels=labels)\n",
    "    loss = outputs[0]\n",
    "    val_loss += loss.item()\n",
    "\n",
    "print(val_loss/len(val_dataloader))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Mask one word at random - Fine Tuning BERT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "# df[\"masked_random_encoding\"] = df[\"tokenized_text\"].apply(lambda x: [ i[random.randint(1, len(i)-1)] = 103 for i in x]) # 103 is the encoding for [MASK]\n",
    "\n",
    "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')\n",
    "\n",
    "# make a new column for masked random encoding\n",
    "masked_random_encoding = []\n",
    "masked_random = []\n",
    "\n",
    "for i in df[\"tokenized_text\"]:\n",
    "    temp = i.copy()\n",
    "    rand = random.randint(1, len(i)-1)\n",
    "    temp[rand] = \"[MASK]\"\n",
    "    masked_random.append(temp)\n",
    "\n",
    "    masked_random_encoding.append(tokenizer.encode(temp))\n",
    "\n",
    "df[\"masked_random_encoding\"] = masked_random_encoding\n",
    "df[\"masked_random\"] = masked_random\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>tokenized_text</th>\n",
       "      <th>masked_text</th>\n",
       "      <th>masked_encoded_text</th>\n",
       "      <th>encoded_text</th>\n",
       "      <th>label</th>\n",
       "      <th>masked_random_encoding</th>\n",
       "      <th>masked_random</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>fuck you , you nixonian twit .\\n</td>\n",
       "      <td>[fuck, you, ,, you, nixon, ##ian, t, ##wi, ##t...</td>\n",
       "      <td>[[MASK], you, ,, you, nixon, ##ian, t, ##wi, #...</td>\n",
       "      <td>[101, 103, 2017, 1010, 2017, 11296, 2937, 1056...</td>\n",
       "      <td>[101, 6616, 2017, 1010, 2017, 11296, 2937, 105...</td>\n",
       "      <td>1</td>\n",
       "      <td>[101, 6616, 103, 1010, 2017, 11296, 2937, 1056...</td>\n",
       "      <td>[fuck, [MASK], ,, you, nixon, ##ian, t, ##wi, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>just another vicious and trashy antitrump , pr...</td>\n",
       "      <td>[just, another, vicious, and, trash, ##y, anti...</td>\n",
       "      <td>[just, another, vicious, and, [MASK], ##y, ant...</td>\n",
       "      <td>[101, 2074, 2178, 13925, 1998, 103, 2100, 3424...</td>\n",
       "      <td>[101, 2074, 2178, 13925, 1998, 11669, 2100, 34...</td>\n",
       "      <td>1</td>\n",
       "      <td>[101, 2074, 2178, 13925, 1998, 11669, 2100, 34...</td>\n",
       "      <td>[just, another, vicious, and, trash, ##y, anti...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>if you want to talk hypocrites , bauer , im up...</td>\n",
       "      <td>[if, you, want, to, talk, h, ##yp, ##oc, ##rit...</td>\n",
       "      <td>[if, you, want, to, talk, h, ##yp, ##oc, ##rit...</td>\n",
       "      <td>[101, 2065, 2017, 2215, 2000, 2831, 1044, 2257...</td>\n",
       "      <td>[101, 2065, 2017, 2215, 2000, 2831, 1044, 2257...</td>\n",
       "      <td>1</td>\n",
       "      <td>[101, 2065, 2017, 2215, 2000, 2831, 1044, 2257...</td>\n",
       "      <td>[if, you, want, to, talk, h, ##yp, ##oc, [MASK...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>that you were shocked by the backlash only fur...</td>\n",
       "      <td>[that, you, were, shocked, by, the, backlash, ...</td>\n",
       "      <td>[that, you, were, shocked, by, the, backlash, ...</td>\n",
       "      <td>[101, 2008, 2017, 2020, 7135, 2011, 1996, 2574...</td>\n",
       "      <td>[101, 2008, 2017, 2020, 7135, 2011, 1996, 2574...</td>\n",
       "      <td>1</td>\n",
       "      <td>[101, 2008, 2017, 2020, 7135, 2011, 1996, 2574...</td>\n",
       "      <td>[that, you, were, shocked, by, the, backlash, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>you all suck shapiros cock and lick his mother...</td>\n",
       "      <td>[you, all, suck, shapiro, ##s, cock, and, lick...</td>\n",
       "      <td>[you, all, [MASK], shapiro, ##s, [MASK], and, ...</td>\n",
       "      <td>[101, 2017, 2035, 103, 24630, 2015, 103, 1998,...</td>\n",
       "      <td>[101, 2017, 2035, 11891, 24630, 2015, 10338, 1...</td>\n",
       "      <td>1</td>\n",
       "      <td>[101, 2017, 2035, 11891, 24630, 103, 10338, 19...</td>\n",
       "      <td>[you, all, suck, shapiro, [MASK], cock, and, l...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>270775</th>\n",
       "      <td>i think you , other like minded individuals ge...</td>\n",
       "      <td>[i, think, you, ,, other, like, minded, indivi...</td>\n",
       "      <td>[i, think, you, ,, other, like, minded, indivi...</td>\n",
       "      <td>[101, 1045, 2228, 2017, 1010, 2060, 2066, 1312...</td>\n",
       "      <td>[101, 1045, 2228, 2017, 1010, 2060, 2066, 1312...</td>\n",
       "      <td>0</td>\n",
       "      <td>[101, 1045, 2228, 2017, 1010, 2060, 2066, 1312...</td>\n",
       "      <td>[i, think, you, ,, other, like, minded, indivi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>270776</th>\n",
       "      <td>a lot of people will be wondering what they we...</td>\n",
       "      <td>[a, lot, of, people, will, be, wondering, what...</td>\n",
       "      <td>[a, lot, of, people, will, be, wondering, what...</td>\n",
       "      <td>[101, 1037, 2843, 1997, 2111, 2097, 2022, 6603...</td>\n",
       "      <td>[101, 1037, 2843, 1997, 2111, 2097, 2022, 6603...</td>\n",
       "      <td>0</td>\n",
       "      <td>[101, 1037, 2843, 1997, 2111, 2097, 2022, 6603...</td>\n",
       "      <td>[a, lot, of, people, will, be, wondering, what...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>270777</th>\n",
       "      <td>they have endured saber rattling by the us for...</td>\n",
       "      <td>[they, have, endured, saber, rattling, by, the...</td>\n",
       "      <td>[they, have, endured, saber, rattling, by, the...</td>\n",
       "      <td>[101, 2027, 2031, 16753, 25653, 26347, 2011, 1...</td>\n",
       "      <td>[101, 2027, 2031, 16753, 25653, 26347, 2011, 1...</td>\n",
       "      <td>0</td>\n",
       "      <td>[101, 2027, 2031, 16753, 25653, 26347, 2011, 1...</td>\n",
       "      <td>[they, have, endured, saber, rattling, by, the...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>270778</th>\n",
       "      <td>the cbc has been totally silent on reporting a...</td>\n",
       "      <td>[the, cbc, has, been, totally, silent, on, rep...</td>\n",
       "      <td>[the, cbc, has, been, totally, silent, on, rep...</td>\n",
       "      <td>[101, 1996, 13581, 2038, 2042, 6135, 4333, 200...</td>\n",
       "      <td>[101, 1996, 13581, 2038, 2042, 6135, 4333, 200...</td>\n",
       "      <td>0</td>\n",
       "      <td>[101, 1996, 13581, 2038, 2042, 6135, 103, 2006...</td>\n",
       "      <td>[the, cbc, has, been, totally, [MASK], on, rep...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>270779</th>\n",
       "      <td>its not the police fault .\\n</td>\n",
       "      <td>[its, not, the, police, fault, .]</td>\n",
       "      <td>[its, not, the, police, fault, .]</td>\n",
       "      <td>[101, 2049, 2025, 1996, 2610, 6346, 1012, 102]</td>\n",
       "      <td>[101, 2049, 2025, 1996, 2610, 6346, 1012, 102]</td>\n",
       "      <td>0</td>\n",
       "      <td>[101, 2049, 2025, 1996, 2610, 6346, 103, 102]</td>\n",
       "      <td>[its, not, the, police, fault, [MASK]]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>270780 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                     text  \\\n",
       "0                        fuck you , you nixonian twit .\\n   \n",
       "1       just another vicious and trashy antitrump , pr...   \n",
       "2       if you want to talk hypocrites , bauer , im up...   \n",
       "3       that you were shocked by the backlash only fur...   \n",
       "4       you all suck shapiros cock and lick his mother...   \n",
       "...                                                   ...   \n",
       "270775  i think you , other like minded individuals ge...   \n",
       "270776  a lot of people will be wondering what they we...   \n",
       "270777  they have endured saber rattling by the us for...   \n",
       "270778  the cbc has been totally silent on reporting a...   \n",
       "270779                       its not the police fault .\\n   \n",
       "\n",
       "                                           tokenized_text  \\\n",
       "0       [fuck, you, ,, you, nixon, ##ian, t, ##wi, ##t...   \n",
       "1       [just, another, vicious, and, trash, ##y, anti...   \n",
       "2       [if, you, want, to, talk, h, ##yp, ##oc, ##rit...   \n",
       "3       [that, you, were, shocked, by, the, backlash, ...   \n",
       "4       [you, all, suck, shapiro, ##s, cock, and, lick...   \n",
       "...                                                   ...   \n",
       "270775  [i, think, you, ,, other, like, minded, indivi...   \n",
       "270776  [a, lot, of, people, will, be, wondering, what...   \n",
       "270777  [they, have, endured, saber, rattling, by, the...   \n",
       "270778  [the, cbc, has, been, totally, silent, on, rep...   \n",
       "270779                  [its, not, the, police, fault, .]   \n",
       "\n",
       "                                              masked_text  \\\n",
       "0       [[MASK], you, ,, you, nixon, ##ian, t, ##wi, #...   \n",
       "1       [just, another, vicious, and, [MASK], ##y, ant...   \n",
       "2       [if, you, want, to, talk, h, ##yp, ##oc, ##rit...   \n",
       "3       [that, you, were, shocked, by, the, backlash, ...   \n",
       "4       [you, all, [MASK], shapiro, ##s, [MASK], and, ...   \n",
       "...                                                   ...   \n",
       "270775  [i, think, you, ,, other, like, minded, indivi...   \n",
       "270776  [a, lot, of, people, will, be, wondering, what...   \n",
       "270777  [they, have, endured, saber, rattling, by, the...   \n",
       "270778  [the, cbc, has, been, totally, silent, on, rep...   \n",
       "270779                  [its, not, the, police, fault, .]   \n",
       "\n",
       "                                      masked_encoded_text  \\\n",
       "0       [101, 103, 2017, 1010, 2017, 11296, 2937, 1056...   \n",
       "1       [101, 2074, 2178, 13925, 1998, 103, 2100, 3424...   \n",
       "2       [101, 2065, 2017, 2215, 2000, 2831, 1044, 2257...   \n",
       "3       [101, 2008, 2017, 2020, 7135, 2011, 1996, 2574...   \n",
       "4       [101, 2017, 2035, 103, 24630, 2015, 103, 1998,...   \n",
       "...                                                   ...   \n",
       "270775  [101, 1045, 2228, 2017, 1010, 2060, 2066, 1312...   \n",
       "270776  [101, 1037, 2843, 1997, 2111, 2097, 2022, 6603...   \n",
       "270777  [101, 2027, 2031, 16753, 25653, 26347, 2011, 1...   \n",
       "270778  [101, 1996, 13581, 2038, 2042, 6135, 4333, 200...   \n",
       "270779     [101, 2049, 2025, 1996, 2610, 6346, 1012, 102]   \n",
       "\n",
       "                                             encoded_text  label  \\\n",
       "0       [101, 6616, 2017, 1010, 2017, 11296, 2937, 105...      1   \n",
       "1       [101, 2074, 2178, 13925, 1998, 11669, 2100, 34...      1   \n",
       "2       [101, 2065, 2017, 2215, 2000, 2831, 1044, 2257...      1   \n",
       "3       [101, 2008, 2017, 2020, 7135, 2011, 1996, 2574...      1   \n",
       "4       [101, 2017, 2035, 11891, 24630, 2015, 10338, 1...      1   \n",
       "...                                                   ...    ...   \n",
       "270775  [101, 1045, 2228, 2017, 1010, 2060, 2066, 1312...      0   \n",
       "270776  [101, 1037, 2843, 1997, 2111, 2097, 2022, 6603...      0   \n",
       "270777  [101, 2027, 2031, 16753, 25653, 26347, 2011, 1...      0   \n",
       "270778  [101, 1996, 13581, 2038, 2042, 6135, 4333, 200...      0   \n",
       "270779     [101, 2049, 2025, 1996, 2610, 6346, 1012, 102]      0   \n",
       "\n",
       "                                   masked_random_encoding  \\\n",
       "0       [101, 6616, 103, 1010, 2017, 11296, 2937, 1056...   \n",
       "1       [101, 2074, 2178, 13925, 1998, 11669, 2100, 34...   \n",
       "2       [101, 2065, 2017, 2215, 2000, 2831, 1044, 2257...   \n",
       "3       [101, 2008, 2017, 2020, 7135, 2011, 1996, 2574...   \n",
       "4       [101, 2017, 2035, 11891, 24630, 103, 10338, 19...   \n",
       "...                                                   ...   \n",
       "270775  [101, 1045, 2228, 2017, 1010, 2060, 2066, 1312...   \n",
       "270776  [101, 1037, 2843, 1997, 2111, 2097, 2022, 6603...   \n",
       "270777  [101, 2027, 2031, 16753, 25653, 26347, 2011, 1...   \n",
       "270778  [101, 1996, 13581, 2038, 2042, 6135, 103, 2006...   \n",
       "270779      [101, 2049, 2025, 1996, 2610, 6346, 103, 102]   \n",
       "\n",
       "                                            masked_random  \n",
       "0       [fuck, [MASK], ,, you, nixon, ##ian, t, ##wi, ...  \n",
       "1       [just, another, vicious, and, trash, ##y, anti...  \n",
       "2       [if, you, want, to, talk, h, ##yp, ##oc, [MASK...  \n",
       "3       [that, you, were, shocked, by, the, backlash, ...  \n",
       "4       [you, all, suck, shapiro, [MASK], cock, and, l...  \n",
       "...                                                   ...  \n",
       "270775  [i, think, you, ,, other, like, minded, indivi...  \n",
       "270776  [a, lot, of, people, will, be, wondering, what...  \n",
       "270777  [they, have, endured, saber, rattling, by, the...  \n",
       "270778  [the, cbc, has, been, totally, [MASK], on, rep...  \n",
       "270779             [its, not, the, police, fault, [MASK]]  \n",
       "\n",
       "[270780 rows x 8 columns]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save the dataframe in pickle file\n",
    "df.to_pickle(\"../data/data.pkl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# split the data into train, val and test\n",
    "train_rand, val_rand, test_rand = np.split(df.sample(frac=1), [int(.6*len(df)), int(.8*len(df))])\n",
    "\n",
    "# save the data in pickle file\n",
    "with open(\"../data/train_rand.pkl\", 'wb') as f:\n",
    "    pickle.dump(train_rand, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class BERTFineTuneRandDataset(torch.utils.data.Dataset):\n",
    "  def __init__(self,df, maxlen = 64):\n",
    "    super(BERTFineTuneRandDataset, self).__init__()\n",
    "    # pad the encoded text to maxlen\n",
    "    self._encoded_text = torch.nn.utils.rnn.pad_sequence(df['encoded_text'].apply(torch.tensor), batch_first=True, padding_value=0)\n",
    "    df[\"labels\"] = df[\"label\"].apply((lambda x: torch.ones(maxlen).int() if x == 1 else torch.zeros(maxlen).int()))\n",
    "    \n",
    "    self._labels = df['labels'].values\n",
    "\n",
    "    self._maxlen = maxlen\n",
    "  \n",
    "    self._masked_encoded_text = torch.nn.utils.rnn.pad_sequence(df['masked_random_encoding'].apply(torch.tensor), batch_first=True, padding_value=0)\n",
    "    self._attention_mask = torch.nn.utils.rnn.pad_sequence(df['masked_random_encoding'].apply(lambda x: torch.tensor([1 if i != 0 else 0 for i in x])), batch_first=True, padding_value=0)\n",
    "\n",
    "  def __len__(self):\n",
    "    return len(self._labels)\n",
    "  \n",
    "  def __getitem__(self, idx):\n",
    "    # make dict of the data\n",
    "    return {\n",
    "\n",
    "        'encoded_text': self._encoded_text[idx][:self._maxlen],\n",
    "        'masked_random_encoding': self._masked_encoded_text[idx][:self._maxlen],\n",
    "        'attention_mask': self._attention_mask[idx][:self._maxlen],\n",
    "        'labels': self._labels[idx][:self._maxlen]    \n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# dataloader\n",
    "train_dataset_rand = BERTFineTuneRandDataset(train_rand)\n",
    "val_dataset_rand = BERTFineTuneRandDataset(val_rand)\n",
    "test_dataset_rand = BERTFineTuneRandDataset(test_rand)\n",
    "\n",
    "train_dataloader_rand = torch.utils.data.DataLoader(train_dataset_rand, batch_size=32, shuffle=True)\n",
    "val_dataloader_rand = torch.utils.data.DataLoader(val_dataset_rand, batch_size=32, shuffle=True)\n",
    "test_dataloader_rand = torch.utils.data.DataLoader(test_dataset_rand, batch_size=32, shuffle=True)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save the dataloader in pickle file\n",
    "with open(\"../data/train_dataloader_rand.pkl\", 'wb') as f:\n",
    "    pickle.dump(train_dataloader_rand, f)\n",
    "\n",
    "with open(\"../data/val_dataloader_rand.pkl\", 'wb') as f:\n",
    "    pickle.dump(val_dataloader_rand, f)\n",
    "\n",
    "with open(\"../data/test_dataloader_rand.pkl\", 'wb') as f:\n",
    "    pickle.dump(test_dataloader_rand, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertForMaskedLM: ['cls.seq_relationship.weight', 'cls.seq_relationship.bias']\n",
      "- This IS expected if you are initializing BertForMaskedLM from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertForMaskedLM from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Epoch 0: 100%|██████████| 5078/5078 [13:42<00:00,  6.17it/s, loss=0.0372]\n",
      "Epoch 1:  84%|████████▍ | 4282/5078 [11:34<02:09,  6.15it/s, loss=0.0417]"
     ]
    }
   ],
   "source": [
    "#fine tuning bert model\n",
    "\n",
    "import torch\n",
    "from transformers import BertTokenizer, BertForMaskedLM\n",
    "from transformers import AdamW\n",
    "from tqdm import tqdm\n",
    "\n",
    "\n",
    "# load train data\n",
    "with open(\"../data/train_dataloader_rand.pkl\", 'rb') as f:\n",
    "    train_dataloader = pickle.load(f)\n",
    "\n",
    "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')\n",
    "model = BertForMaskedLM.from_pretrained('bert-base-uncased')\n",
    "\n",
    "model.to(torch.device('cuda'))\n",
    "\n",
    "model.train()\n",
    "optimizer = AdamW(model.parameters(), lr=1e-5)\n",
    "\n",
    "epochs = 10\n",
    "\n",
    "for epoch in range(epochs):\n",
    "    loop  = tqdm(train_dataloader, leave=True)\n",
    "    for batch in loop:\n",
    "        optimizer.zero_grad()\n",
    "        input_ids = batch['masked_random_encoding'].to(torch.device('cuda'))\n",
    "        attention_mask = batch['attention_mask'].to(torch.device('cuda'))\n",
    "        labels = batch['encoded_text'].to(torch.device('cuda'))\n",
    "        segment_ids = batch['labels'].to(torch.device('cuda'))\n",
    "        outputs = model(input_ids, attention_mask=attention_mask,token_type_ids = segment_ids, labels=labels)\n",
    "        loss = outputs[0]\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        loop.set_description(f'Epoch {epoch}')\n",
    "        loop.set_postfix(loss=loss.item())\n",
    "    \n",
    "    path = \"./bert_ft_rand_\"+str(epoch+1)+\"epochs.pth\"\n",
    "    torch.save(model, path)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import pickle\n",
    "\n",
    "# load the finetuned model\n",
    "model = torch.load(\"bert_ft_rand_5epochs.pth\")\n",
    "\n",
    "#load the val dataloader\n",
    "with open(\"../data/val_dataloader_rand.pkl\", 'rb') as f:\n",
    "    val_dataloader = pickle.load(f)\n",
    "\n",
    "# validation code\n",
    "model.eval()\n",
    "val_loss = 0\n",
    "for batch in val_dataloader:\n",
    "    input_ids = batch['masked_random_encoding'].to(torch.device('cuda'))\n",
    "    attention_mask = batch['attention_mask'].to(torch.device('cuda'))\n",
    "    labels = batch['encoded_text'].to(torch.device('cuda'))\n",
    "    segment_ids = batch['labels'].to(torch.device('cuda'))\n",
    "    outputs = model(input_ids, attention_mask=attention_mask,token_type_ids = segment_ids, labels=labels)\n",
    "    loss = outputs[0]\n",
    "    val_loss += loss.item()\n",
    "\n",
    "print(val_loss/len(val_dataloader))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.10.8 64-bit",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.8"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "b0fa6594d8f4cbf19f97940f81e996739fb7646882a419484c72d19e05852a7e"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}