film_electra.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import pickle\n",
    "import time\n",
    "import tensorflow as tf\n",
    "\n",
    "# uncomment below line lines to view available cuda memory\n",
    "# t = torch.cuda.get_device_properties(0).total_memory\n",
    "# print(t/(1024*1024*1024))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "jwau8gz-veI4",
    "outputId": "1454ee3b-cd4a-487a-d4c2-8c79f1344909"
   },
   "source": [
    "### Flag to set if data is not ready"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "79rzLzvQmrKf"
   },
   "outputs": [],
   "source": [
    "DATA_READY = True"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Utility functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "6_qWOCsHm55A"
   },
   "outputs": [],
   "source": [
    "def create_dict(keys, val1s, val2s =None, val3s = None, top = False):\n",
    "    temp = {}\n",
    "    if (top):\n",
    "        for key, val in zip(keys, val1s):\n",
    "            temp[str(key)] = val\n",
    "    else:\n",
    "        for key, val1, val2 in zip(keys, val1s, val2s):\n",
    "            temp[str(key)] = {\n",
    "                'text':np.array(val1),\n",
    "                'top':np.array(val3s[str(key)]),\n",
    "                'label':int(val2)\n",
    "            }\n",
    "    return temp\n",
    "\n",
    "# utility function to load file from pickle dump\n",
    "def load_file(filename):\n",
    "    with open(filename, 'rb') as filehandle:\n",
    "        ret = pickle.load(filehandle)\n",
    "        return ret\n",
    "    \n",
    "# utility function to save file as pickle dump\n",
    "def save_file(filename, obj):\n",
    "    with open(filename, 'wb') as filehandle:\n",
    "        pickle.dump(obj, filehandle)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tokenizer class for text data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "XMiLqfAInB0h"
   },
   "outputs": [],
   "source": [
    "from tensorflow.keras.preprocessing.text import Tokenizer\n",
    "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
    "\n",
    "seq_len = 80\n",
    "\n",
    "class TokenizerWrap(Tokenizer):\n",
    "    \"\"\"Wrap the Tokenizer-class from Keras with more functionality.\"\"\"\n",
    "    \n",
    "    def __init__(self, texts, seq_len):\n",
    "        Tokenizer.__init__(self, oov_token = '<unk>')\n",
    "        self.seq_len = seq_len\n",
    "        # Create the vocabulary from the texts.\n",
    "        self.fit_on_texts(texts)\n",
    "        self.index_to_word = dict(zip(self.word_index.values(),\n",
    "                                      self.word_index.keys()))\n",
    "        \n",
    "        self.tokens = self.texts_to_sequences(texts)\n",
    "        \n",
    "        # The number of integer-tokens in each sequence.\n",
    "        self.num_tokens = [len(x) for x in self.tokens]\n",
    "\n",
    "        # Max number of tokens to use in all sequences.\n",
    "        # We will pad / truncate all sequences to this length.\n",
    "        # This is a compromise so we save a lot of memory and\n",
    "        # only have to truncate maybe 5% of all the sequences.\n",
    "        self.max_tokens = np.mean(self.num_tokens) + 2 * np.std(self.num_tokens)\n",
    "        self.max_tokens = int(self.max_tokens)\n",
    "        \n",
    "        self.total_tokens = len(self.index_to_word)\n",
    "\n",
    "    def tokens_to_string(self, tokens):\n",
    "        \"\"\"Convert a list of integer-tokens to a string.\"\"\"\n",
    "\n",
    "        # Create a list of the individual words.\n",
    "        words = [self.index_to_word[token]\n",
    "                 for token in tokens\n",
    "                 if token != 0]\n",
    "        \n",
    "        # Concatenate the words to a single string\n",
    "        # with space between all the words.\n",
    "        text = \" \".join(words)\n",
    "\n",
    "        return text\n",
    "    \n",
    "    def texts_to_tokens(self, text, reverse=False, padding=True):\n",
    "        \"\"\"\n",
    "        Convert a single text-string to tokens with optional\n",
    "        reversal and padding.\n",
    "        \"\"\"\n",
    "\n",
    "        # Convert to tokens. Note that we assume there is only\n",
    "        # a single text-string so we wrap it in a list.\n",
    "        tokens = self.texts_to_sequences(text)\n",
    "\n",
    "        # Pad and truncate sequences to the given length.\n",
    "        if padding:\n",
    "            tokens = pad_sequences(tokens,\n",
    "                                   maxlen=self.seq_len,\n",
    "                                   padding='post',\n",
    "                                   truncating='post')\n",
    "        tokens = np.array(tokens)\n",
    "        return tokens"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Proprcessing class to download required data and build embedding matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "6qi5KOlhnG1X"
   },
   "outputs": [],
   "source": [
    "class Preprocessor():\n",
    "    def __init__(self, data_ready):\n",
    "        self.data_ready = data_ready\n",
    "\n",
    "    def download(self):\n",
    "        url_img_dataset = 'https://www.dropbox.com/s/ofmxf7fxyixdw4a/dataset_image_all.zip?dl=1'\n",
    "        file_img_dataset = 'dataset_image_all.zip'\n",
    "        if not(self.data_ready):\n",
    "            tf.keras.utils.get_file(fname=file_img_dataset, origin=url_img_dataset, extract=True, cache_subdir=os.getcwd())\n",
    "        \n",
    "        url_dataset = 'https://www.dropbox.com/s/n5i5pid134v5rkj/twitter-multi-modal.zip?dl=1'\n",
    "        file_dataset = 'twitter-multi-modal.zip'\n",
    "        if not(self.data_ready):\n",
    "            tf.keras.utils.get_file(fname=file_dataset, origin=url_dataset, extract=True, cache_subdir=os.getcwd())\n",
    "        \n",
    "        url_glove = \"http://nlp.stanford.edu/data/glove.twitter.27B.zip\"\n",
    "        file_glove_zip = 'glove.twitter.27B.zip'\n",
    "        self.file_glove = './glove.twitter.27B.100d.txt'\n",
    "        if not(self.data_ready):\n",
    "            tf.keras.utils.get_file(fname=file_glove_zip, origin=url_glove, extract=True, cache_subdir=os.getcwd())\n",
    "            \n",
    "    def load_embedding(self):\n",
    "        self.embeddings_index = {}\n",
    "        with open(self.file_glove) as f:\n",
    "            for line in f:\n",
    "                word, coefs = line.split(maxsplit=1)\n",
    "                coefs = np.fromstring(coefs, \"f\", sep=\" \")\n",
    "                self.embeddings_index[word] = coefs\n",
    "        \n",
    "        self.embedding_dim = len(self.embeddings_index.get(list(self.embeddings_index.keys())[0]))\n",
    "        self.embedding_vocab = len(self.embeddings_index)\n",
    "        return self.embedding_vocab, self.embedding_dim\n",
    "    \n",
    "    def get_embedding_matrix(self, tokenizer):\n",
    "        num_tokens = tokenizer.total_tokens + 1\n",
    "        hits = 0\n",
    "        misses = 0\n",
    "        miss_list = []\n",
    "\n",
    "        # Prepare embedding matrix\n",
    "        embedding_matrix = np.zeros((num_tokens, self.embedding_dim))\n",
    "        for word, i in tokenizer.word_index.items():\n",
    "            if i == 1:\n",
    "                continue\n",
    "            try:\n",
    "                embedding_vector =  self.embeddings_index.get(word)\n",
    "            except:\n",
    "                embedding_vector = np.random.normal(scale = 0.6, size = (self.embedding_dim, ))\n",
    "            if embedding_vector is not None:\n",
    "                # Words not found in embedding index will be all-zeros.\n",
    "                # This includes the representation for \"padding\" and \"OOV\"\n",
    "                embedding_matrix[i] = embedding_vector\n",
    "                hits += 1\n",
    "            else:\n",
    "                miss_list.append(word)\n",
    "                misses += 1\n",
    "        return embedding_matrix, hits, misses, miss_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "yzN-R8vAnJrG"
   },
   "outputs": [],
   "source": [
    "pp = Preprocessor(DATA_READY)\n",
    "pp.download()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prepare data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "_R6T--WWnO2P"
   },
   "outputs": [],
   "source": [
    "train_data = load_file('./train_data')\n",
    "valid_data = load_file('./valid_data')\n",
    "test_data = load_file('./test_data')\n",
    "image_top = load_file('./image_top')\n",
    "\n",
    "train_text = [str(item['text']) for item in train_data.values()]\n",
    "valid_text = [str(item['text']) for item in valid_data.values()]\n",
    "test_text = [str(item['text']) for item in test_data.values()]\n",
    "top_text = [ \" \".join(item) for item in image_top.values()]\n",
    "\n",
    "train_labels = [int(item['label']) for item in train_data.values()]\n",
    "valid_labels = [int(item['label']) for item in valid_data.values()]\n",
    "test_labels = [int(item['label']) for item in test_data.values()]\n",
    "\n",
    "train_id = [str(item) for item in train_data.keys()]\n",
    "valid_id = [str(item) for item in valid_data.keys()]\n",
    "test_id = [str(item) for item in test_data.keys()]\n",
    "top_id = [ str(item) for item in image_top.keys()]\n",
    "\n",
    "texts = train_text + valid_text + test_text + top_text\n",
    "\n",
    "if not(DATA_READY):\n",
    "    save_file('./train_text', train_text)\n",
    "    save_file('./train_labels', train_labels)\n",
    "    save_file('./train_id', train_id)\n",
    "\n",
    "    save_file('./test_text', test_text)\n",
    "    save_file('./test_labels', test_labels)\n",
    "    save_file('./test_id', test_id)\n",
    "\n",
    "    save_file('./valid_text', valid_text)\n",
    "    save_file('./valid_labels', valid_labels)\n",
    "    save_file('./valid_id', valid_id)\n",
    "\n",
    "    save_file('./image_top', image_top)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load prepared data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "haVWlHWPDJNV",
    "outputId": "b5efb8c3-37c1-44b6-fada-078573aed77c"
   },
   "outputs": [],
   "source": [
    "train_text = load_file(\"/home/ckm/sarcasm_detection/sarcasm/full/train_text\")\n",
    "train_labels = load_file(\"/home/ckm/sarcasm_detection/sarcasm/full/train_labels\")\n",
    "train_id = load_file(\"/home/ckm/sarcasm_detection/sarcasm/full/train_id\")\n",
    "image_top = load_file(\"/home/ckm/sarcasm_detection/sarcasm/full/image_top\")\n",
    "valid_text = load_file(\"/home/ckm/sarcasm_detection/sarcasm/full/valid_text\")\n",
    "valid_labels = load_file(\"/home/ckm/sarcasm_detection/sarcasm/full/valid_labels\")\n",
    "valid_id = load_file(\"/home/ckm/sarcasm_detection/sarcasm/full/valid_id\")\n",
    "test_text = load_file(\"/home/ckm/sarcasm_detection/sarcasm/full/test_text\")\n",
    "test_labels = load_file(\"/home/ckm/sarcasm_detection/sarcasm/full/test_labels\")\n",
    "test_id = load_file(\"/home/ckm/sarcasm_detection/sarcasm/full/test_id\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Build embedding matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "hlv0ZcaxnW0b"
   },
   "outputs": [],
   "source": [
    "embedding_vocab_size, embedding_dim = pp.load_embedding()\n",
    "\n",
    "tokenizer_wrap = TokenizerWrap(texts, seq_len)\n",
    "\n",
    "embedding_matrix, num_not_oov, num_oov, miss_list = pp.get_embedding_matrix(tokenizer_wrap)\n",
    "vocab_size, embed_dim = embedding_matrix.shape\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Import transformer model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "5RxcLuld7kJG",
    "outputId": "d04a0c5a-2b31-4af2-fcc0-2cf2335a3ef9"
   },
   "outputs": [],
   "source": [
    "# !pip install transformers\n",
    "from transformers import ElectraTokenizer, ElectraModel\n",
    "tokenizer_XLM = ElectraTokenizer.from_pretrained('google/electra-small-discriminator')  #tokenizer directly returns a list of ids eg: [0, 23, 244,, 45, 2, ..... 2]    shape = (batch_size, 512)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Image processing pipelines"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from PIL import Image\n",
    "from torchvision import transforms\n",
    "\n",
    "transform_pipe = transforms.Compose([\n",
    "    transforms.Resize(256),\n",
    "    transforms.CenterCrop(224),\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n",
    "])\n",
    "\n",
    "transform_pipe_train = transforms.Compose([\n",
    "    transforms.RandomResizedCrop(224),\n",
    "    transforms.RandomHorizontalFlip(),\n",
    "    transforms.ToTensor(),\n",
    "    transforms.ColorJitter(brightness=0, contrast=0, saturation=0, hue=0),\n",
    "    transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),\n",
    "])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Hbszd0qenZ9Q"
   },
   "outputs": [],
   "source": [
    "import re\n",
    "\n",
    "class TwitterDataset(torch.utils.data.Dataset):\n",
    "    \"\"\"\n",
    "    This is our custom dataset class which will load the images, perform transforms on them,\n",
    "    and load their corresponding labels.\n",
    "    \"\"\"\n",
    "    def __init__(self, img_dir, labels, text, filenames, ocr_text = None, transform=None, tokenizer_wrap = None, tokenizer_bert = None, image_top = None):\n",
    "        \"\"\"\n",
    "        img_dir = dir in which images are located\n",
    "        labels = list containing true (0/1) values\n",
    "        text = list containing all the texts\n",
    "        filenames = list containing the names of images\n",
    "        transform = transformer to preprocess images\n",
    "        \"\"\"\n",
    "        self.img_dir = img_dir\n",
    "        self.labels = labels\n",
    "        self.text = text\n",
    "        self.filenames = filenames\n",
    "        self.transform = transform\n",
    "        self.tokenizer_wrap = tokenizer_wrap\n",
    "        self.image_top = image_top\n",
    "        self.tokenizer_XLM = tokenizer_bert  #assign tokenizer to tokenizer_XLM\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        try:\n",
    "            img_path = os.path.join(\n",
    "                self.img_dir,\n",
    "                \"{}.jpg\".format(self.filenames[idx])\n",
    "            )\n",
    "        except Exception as e: \n",
    "            print(e)\n",
    "                \n",
    "        img = Image.open(img_path)\n",
    "        \n",
    "        if self.transform:\n",
    "            img = self.transform(img)\n",
    "        \n",
    "        sample = {\n",
    "            \"image\": img,\n",
    "        }\n",
    "        text = re.sub(r'(\\s)emoji\\w+', '', self.text[idx])\n",
    "        \n",
    "        token = self.tokenizer_wrap.texts_to_tokens([text])\n",
    "        token.resize(seq_len)\n",
    "        tokens_tensor = torch.tensor(token).long()\n",
    "\n",
    "        # here we don't need to pass the input_ids here. this tokenizer directly returns integer values\n",
    "        # have set max_length = 360 here\n",
    "        indexed_tokens_for_text = self.tokenizer_XLM.encode(text, padding = 'max_length', truncation= True, max_length = 360)  \n",
    "        tokens_tensor_text = torch.tensor(indexed_tokens_for_text)\n",
    "\n",
    "        attribute = self.image_top[int(self.filenames[idx])]\n",
    "        attribute = ' '.join(attribute)\n",
    "        # same for attribute, have set max_length = 13 here\n",
    "        indexed_tokens_for_attribute = self.tokenizer_XLM.encode(attribute, padding = 'max_length', truncation= True, max_length = 13)    \n",
    "        tokens_tensor_attribute = torch.tensor(indexed_tokens_for_attribute)\n",
    "\n",
    "        try:\n",
    "            sample[\"label\"] = self.labels[idx]\n",
    "            sample[\"token\"] = tokens_tensor\n",
    "            sample[\"text\"] = tokens_tensor_text\n",
    "            sample[\"attribute\"] = tokens_tensor_attribute\n",
    "        except Exception as e:\n",
    "            print(e)\n",
    "        \n",
    "        return sample\n",
    "    \n",
    "    def __len__(self):\n",
    "        return len(self.labels)\n",
    "        \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Make Dataloaders"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "WO6xXjWYnp9W"
   },
   "outputs": [],
   "source": [
    "train_data_object = TwitterDataset(\n",
    "    img_dir=\"./dataset_image/\",\n",
    "    labels = train_labels,\n",
    "    text = train_text,\n",
    "    filenames = train_id,\n",
    "    transform=transform_pipe_train,\n",
    "    tokenizer_wrap=tokenizer_wrap,\n",
    "    tokenizer_bert=tokenizer_XLM,  # pass tokenizer_XLM here\n",
    "    image_top = image_top\n",
    ")\n",
    "\n",
    "test_data_object = TwitterDataset(\n",
    "    img_dir=\"./dataset_image/\",\n",
    "    labels = test_labels,\n",
    "    text = test_text,\n",
    "    filenames = test_id,\n",
    "    transform=transform_pipe,\n",
    "    tokenizer_wrap=tokenizer_wrap,\n",
    "    tokenizer_bert=tokenizer_XLM,  # pass tokenizer_XLM here\n",
    "    image_top = image_top\n",
    ")\n",
    "\n",
    "val_data_object = TwitterDataset(\n",
    "    img_dir=\"./dataset_image/\",\n",
    "    labels = valid_labels,\n",
    "    text = valid_text,\n",
    "    filenames = valid_id,\n",
    "    transform=transform_pipe,\n",
    "    tokenizer_wrap=tokenizer_wrap,\n",
    "    tokenizer_bert=tokenizer_XLM,  # pass tokenizer_XLM here\n",
    "    image_top = image_top\n",
    ")\n",
    "\n",
    "BATCH_SIZE = 32\n",
    "\n",
    "train_loader = torch.utils.data.DataLoader(\n",
    "    train_data_object,\n",
    "    batch_size=BATCH_SIZE,\n",
    "    pin_memory=True,\n",
    "    num_workers=8,\n",
    "    shuffle=True,\n",
    "    drop_last=True\n",
    " )\n",
    "\n",
    "test_loader = torch.utils.data.DataLoader(\n",
    "    test_data_object,\n",
    "    batch_size=BATCH_SIZE,\n",
    "    pin_memory=True,\n",
    "    num_workers=8,\n",
    "    shuffle=True,\n",
    "    drop_last=True\n",
    " )\n",
    "\n",
    "val_loader = torch.utils.data.DataLoader(\n",
    "    val_data_object,\n",
    "    batch_size=BATCH_SIZE,\n",
    "    pin_memory=True,\n",
    "    num_workers=8,\n",
    "    shuffle=True,\n",
    "    drop_last=True\n",
    " )\n",
    "\n",
    "dataloaders = {'train': train_loader, 'test': test_loader, 'val': val_loader}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Verify Cuda device is loaded properly "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "WLFM0fbCnhOY",
    "outputId": "9d072910-30cd-4874-d784-b87c3e8a5a1f"
   },
   "outputs": [],
   "source": [
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "print(device)\n",
    "\n",
    "torch.backends.cudnn.benchmark = True\n",
    "torch.backends.cudnn.enabled   = True\n",
    "\n",
    "print(torch.cuda.device_count())\n",
    "print('GPU Allocated')\n",
    "\n",
    "dtype = torch.cuda.FloatTensor"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model Preparation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ResNet layer which will be passed to FiLM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "9rS9VmpMnmr5"
   },
   "outputs": [],
   "source": [
    "from torchvision import models\n",
    "cnn = models.resnet50(pretrained = True)\n",
    "layers = [\n",
    "    cnn.conv1,\n",
    "    cnn.bn1,\n",
    "    cnn.relu,\n",
    "    cnn.maxpool,\n",
    "  ]\n",
    "for i in range(3):\n",
    "    name = 'layer%d' % (i + 1)\n",
    "    layers.append(getattr(cnn, name))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Import FiLM modules"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "CYmwpZgxnsea",
    "outputId": "640781fb-bb79-4275-9070-0fff85668933"
   },
   "outputs": [],
   "source": [
    "!pip install ipdb\n",
    "from vr.models import FiLMedNet\n",
    "from vr.models import FiLMGen\n",
    "\n",
    "from torch.autograd import Variable\n",
    "\n",
    "# utility function required for FiLM\n",
    "def parse_int_list(s):\n",
    "    if s == '': return ()\n",
    "    return tuple(int(n) for n in s.split(','))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Functions to generate FiLM network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "ckG2GStQnue3"
   },
   "outputs": [],
   "source": [
    "def get_program_generator():\n",
    "    kwargs = {\n",
    "          'encoder_vocab_size': vocab_size,\n",
    "          'decoder_vocab_size': vocab_size,\n",
    "          'wordvec_dim': embed_dim,\n",
    "          'hidden_dim': 4096,\n",
    "          'rnn_num_layers': 1,\n",
    "          'rnn_dropout': 0,\n",
    "        }\n",
    "\n",
    "    kwargs['parameter_efficient'] = 1\n",
    "    kwargs['output_batchnorm'] = 0\n",
    "    kwargs['bidirectional'] = 0\n",
    "    kwargs['encoder_type'] = 'gru'\n",
    "    kwargs['decoder_type'] = 'linear'\n",
    "    kwargs['gamma_option'] = 'linear'\n",
    "    kwargs['gamma_baseline'] = 1\n",
    "    kwargs['num_modules'] = 4\n",
    "    kwargs['module_num_layers'] = 1\n",
    "    kwargs['module_dim'] = 128\n",
    "    kwargs['debug_every'] = float('inf')\n",
    "    pg = FiLMGen(**kwargs)\n",
    "    return pg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "tOqBqW15nw1B"
   },
   "outputs": [],
   "source": [
    "def get_execution_engine():\n",
    "    kwargs = {\n",
    "          'vocab': dict(),\n",
    "          'feature_dim': parse_int_list('1024,14,14'),\n",
    "          'stem_batchnorm': 1,\n",
    "          'stem_num_layers': 1,\n",
    "          'module_dim': 128,\n",
    "          'module_residual': 1,\n",
    "          'module_batchnorm': 1,\n",
    "          'classifier_proj_dim': 512,\n",
    "          'classifier_downsample': 'maxpoolfull',\n",
    "          'classifier_fc_layers': parse_int_list('1024'),\n",
    "          'classifier_batchnorm': 1,\n",
    "          'classifier_dropout': 0.0\n",
    "        }\n",
    "    kwargs['num_modules'] = 4\n",
    "    kwargs['stem_kernel_size'] = 3\n",
    "    kwargs['stem_stride'] = 1\n",
    "    kwargs['stem_padding'] = None\n",
    "    kwargs['module_num_layers'] = 1\n",
    "    kwargs['module_batchnorm_affine'] = 0\n",
    "    kwargs['module_dropout'] = 0e-2\n",
    "    kwargs['module_input_proj'] = 1\n",
    "    kwargs['module_kernel_size'] = 3\n",
    "    kwargs['use_gamma'] = 1\n",
    "    kwargs['use_beta'] = 1\n",
    "    kwargs['use_coords'] = 1\n",
    "    kwargs['debug_every'] = float('inf')\n",
    "    kwargs['print_verbose_every'] = 10\n",
    "    kwargs['condition_method'] = 'bn-film'\n",
    "    kwargs['condition_pattern'] = parse_int_list('1,1,1,1')\n",
    "    ee = FiLMedNet(**kwargs)\n",
    "    return ee"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Co-attention layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "HF7YLjO37dLg"
   },
   "outputs": [],
   "source": [
    "import math\n",
    "\n",
    "class Co_attention(torch.nn.Module):\n",
    "    def __init__(self, size_in, size_out):\n",
    "        super().__init__()\n",
    "        self.size_in, self.size_out = size_in, size_out\n",
    "        weights = torch.Tensor(size_out, size_in)\n",
    "        self.weights = torch.nn.Parameter(weights)  # nn.Parameter is a Tensor that's a module parameter.\n",
    "\n",
    "        # initialize weights and biases\n",
    "        torch.nn.init.kaiming_uniform_(self.weights, a=math.sqrt(5)) # weight init\n",
    "        fan_in, _ = torch.nn.init._calculate_fan_in_and_fan_out(self.weights)\n",
    "\n",
    "        self.tanh = torch.nn.Tanh()\n",
    "        self.max_pool = torch.nn.MaxPool2d(kernel_size = (360,1))\n",
    "\n",
    "    def forward(self, H, T):\n",
    "        # H.shape = batch_size, 360, 768   now it will be [batch_size, 360, 1280]\n",
    "        # T.shape = batch_size, 12, 768    now it will be [batch_size, 12, 1280]\n",
    "        T_tmp = T.clone()\n",
    "        T = T.permute(0, 2, 1)\n",
    "        z_1 = torch.matmul(H, self.weights)\n",
    "        z_2 = torch.matmul(z_1, T)\n",
    "        C = self.tanh(z_2)\n",
    "        alpha = self.max_pool(C)\n",
    "        HT = torch.matmul(alpha, T_tmp).resize(BATCH_SIZE, self.size_in)\n",
    "\n",
    "        return HT"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Transformer Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "uuKHCCNX8ZkA"
   },
   "outputs": [],
   "source": [
    "from torchvision import models\n",
    "from transformers import ElectraModel  # xlm model\n",
    "\n",
    "elec_text = ElectraModel.from_pretrained('google/electra-small-discriminator')\n",
    "elec_attribute = ElectraModel.from_pretrained('google/electra-small-discriminator')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Final Model Class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "G1NpkHjR8r19"
   },
   "outputs": [],
   "source": [
    "class FilmModel(torch.nn.Module):\n",
    "  # define model elements\n",
    "    def __init__(self):\n",
    "        super(FilmModel, self).__init__()\n",
    "\n",
    "        self.res50 = torch.nn.Sequential(*layers).cuda()\n",
    "        self.pg = get_program_generator().cuda()\n",
    "        self.ee = get_execution_engine().cuda()\n",
    "        self.pg.encoder_embed.load_state_dict({'weight':torch.tensor(embedding_matrix)})\n",
    "        self.pg.encoder_embed.requires_grad = False\n",
    "        \n",
    "        self.elec_text = elec_text.to(device)    #output of electra model is  [batch_size, seq_len, 256]\n",
    "        self.elec_attribute = elec_attribute.to(device)\n",
    "        self.co_attention = Co_attention(256, 256).cuda()\n",
    "        self.fc = torch.nn.Linear(256+1024+256, 2)\n",
    "        self.sigm = torch.nn.Sigmoid()\n",
    "        \n",
    "    # forward propagate input\n",
    "    def forward(self, X1, X2, X3, X4):\n",
    "        \"\"\"\n",
    "        X1=images\n",
    "        X2=token (tokenizer_wrap)\n",
    "        X3=text (bert)\n",
    "        X4=attribute (bert) \n",
    "        \"\"\"\n",
    "        programs_pred = self.pg(X2).cuda()\n",
    "        image_features = self.res50(images).cuda()\n",
    "        out1 = self.ee(image_features, programs_pred)\n",
    "\n",
    "        elec_text = self.elec_text.embeddings(X3)  \n",
    "        elec_text = self.elec_text.embeddings_project(elec_text)\n",
    "        elec_text = self.elec_text.encoder.layer[0](elec_text)[0]  #torch.Size([batch_size, 360, 256])\n",
    "        elec_attribute = self.elec_attribute(X4).last_hidden_state  #torch.Size([batch_size, 13, 256])\n",
    "        \n",
    "        out2 = self.co_attention(elec_text, elec_attribute)  # need to check dimension for this pls\n",
    "        out3 = elec_text[:, 0, :]    \n",
    "        \n",
    "        out = torch.cat((out1, out2, out3), dim = 1) # this should be torch.Size([batch_size, 256+1024+256]) \n",
    "\n",
    "        out = self.fc(out) # torch.Size([batch_size, 1])\n",
    "        yhat = self.sigm(out)\n",
    "    \n",
    "        return yhat"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Building instance of our model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "IvMdlkvwQ1qf",
    "outputId": "58632718-a102-42d8-b46e-235294ce277f"
   },
   "outputs": [],
   "source": [
    "model = FilmModel()\n",
    "model = model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.pg.encoder_embed.load_state_dict({'weight':torch.tensor(embedding_matrix)})\n",
    "model.pg.encoder_embed.requires_grad = False"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Loading optimizers and Loss functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "khf0F76QAxjK",
    "outputId": "8ad88fa0-0634-4ca1-967a-766ab62dde70"
   },
   "outputs": [],
   "source": [
    "optimizer = torch.optim.Adam(\n",
    "    [\n",
    "        {'params': model.pg.parameters(), 'lr': 3e-4, 'weight_decay': 1e-5},\n",
    "        {'params': model.ee.parameters(), 'lr': 3e-4, 'weight_decay': 1e-5},\n",
    "        {'params': model.res50.parameters(), 'lr': 1e-6},\n",
    "        {'params': model.elec_text.parameters(), 'lr': 1e-6},\n",
    "        {'params': model.elec_attribute.parameters(), 'lr': 1e-6},\n",
    "        {'params': model.fc.parameters()},\n",
    "        {'params': model.co_attention.parameters()}\n",
    "    ], lr = 1e-4, weight_decay = 1e-2\n",
    ")\n",
    "\n",
    "\n",
    "loss_fn = torch.nn.CrossEntropyLoss().cuda()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Runner code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 629
    },
    "id": "-8lO0FcLn0TT",
    "outputId": "4b0537b0-65ed-4103-f6ec-907d0fc007cb"
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.metrics import f1_score\n",
    "from sklearn.metrics import precision_score\n",
    "from sklearn.metrics import recall_score\n",
    "import time\n",
    "\n",
    "num_epochs = 15\n",
    "\n",
    "tick = time.time()\n",
    "\n",
    "print(num_epochs)\n",
    "\n",
    "for epoch in range(0, num_epochs):\n",
    "\n",
    "    print('-'*10)\n",
    "    print('Epoch {}/{}'.format(epoch+1, num_epochs))\n",
    "\n",
    "    for phase in ['train', 'val', 'test']:\n",
    "        running_loss = 0.0\n",
    "        running_corrects = 0.0\n",
    "\n",
    "        y_true = []\n",
    "        y_pred = []\n",
    "\n",
    "        if phase == 'train':\n",
    "            model.train()\n",
    "        else:\n",
    "            model.eval()\n",
    "\n",
    "        for i, batch in enumerate(dataloaders[phase]):\n",
    "            token = batch[\"token\"]\n",
    "            images = batch[\"image\"]\n",
    "            labels = batch[\"label\"]\n",
    "            text = batch[\"text\"]\n",
    "            attribute = batch[\"attribute\"]\n",
    "        \n",
    "            token = Variable(token.cuda())\n",
    "            images = Variable(images.cuda())\n",
    "            labels = Variable(labels.cuda())\n",
    "            text = Variable(text.cuda())\n",
    "            attribute = Variable(attribute.cuda())\n",
    "            \n",
    "            scores = model(images, token, text, attribute)\n",
    "\n",
    "            loss = loss_fn(scores, labels)\n",
    "            \n",
    "            if (phase == 'train'):\n",
    "                model.zero_grad()\n",
    "                loss.backward()\n",
    "                torch.nn.utils.clip_grad_norm_(model.parameters(), 1)\n",
    "                optimizer.step()\n",
    "                \n",
    "        \n",
    "            running_loss += loss.item() * labels.size(0)\n",
    "            \n",
    "            if scores is not None:\n",
    "                _, preds = scores.data.max(1)\n",
    "                running_corrects += (preds == labels).sum()\n",
    "                y_pred.extend(preds.tolist())\n",
    "                y_true.extend(labels.tolist())\n",
    "\n",
    "            if (i%100 == 0) and phase == 'train':\n",
    "                print(i, running_loss/((i+1)*labels.size(0)))\n",
    "\n",
    "            del loss, scores, token, images, labels, text, attribute\n",
    "\n",
    "        epoch_loss = running_loss / (len(dataloaders[phase]) * BATCH_SIZE)\n",
    "        epoch_acc = float(running_corrects) / (len(dataloaders[phase]) * BATCH_SIZE)\n",
    "        \n",
    "        print(confusion_matrix(y_true, y_pred))\n",
    "        pre = precision_score(y_true, y_pred)\n",
    "        recall = recall_score(y_true, y_pred)\n",
    "        f1 = f1_score(y_true, y_pred)\n",
    "\n",
    "        print(phase + \": F1: {:.4f}, Precision: {:.4f}, Recall : {:.4f}, Accuracy: {:.4f}, Loss: {:.4f}.\".format(f1, pre, recall, epoch_acc, epoch_loss))\n",
    "        \n",
    "        # Uncomment below lines to save the weights of the modle\n",
    "        # if phase == \"train\":\n",
    "        #     save_dir = './saved_models/film_electra_' + str(epoch+1) + \"_\" + str(f1)\n",
    "        #     torch.save({'model_state_dict': model.state_dict(),                                                 \n",
    "        #         'optimizer_state_dict': optimizer.state_dict()}, save_dir)\n",
    "        \n",
    "print(\"Time taken to compute the results:\", time.time() - tick)"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "film.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}