Seq2Seq_GRU_decompression_attention.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Ramubala/text-translation/blob/main/Seq2Seq_GRU_decompression_attention.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "wsfWBukxnk9J"
      },
      "outputs": [],
      "source": [
        "#! git clone https://github.com/Ramubala/text-translation.git\n",
        "#%rm -rf '/content/text-translation'"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# !pip install -q condacolab\n",
        "# import condacolab\n",
        "# condacolab.install()\n",
        "# !conda install cudnn=8.3.2"
      ],
      "metadata": {
        "id": "oY0XgEwOV7FU"
      },
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "id": "ZHW0tXIrntnS",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b5579e5b-55d0-4d84-c403-0d4bb2861c9d"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "eng_-french.csv  \u001b[0m\u001b[01;34msample_data\u001b[0m/  translation_model_attention_epoch_4.pt\n"
          ]
        }
      ],
      "source": [
        "%ls\n",
        "#%cd text-translation\n",
        "#!unzip eng_-french.csv.zip -d eng_-french"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 224
        },
        "id": "i6z0RfGdqzGJ",
        "outputId": "1d9407bb-adc4-4bda-cf11-0eaa4745b1e4"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(100000, 2)\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "  English words/sentences French words/sentences\n",
              "0                     Hi.                 Salut!\n",
              "1                    Run!                Cours !\n",
              "2                    Run!               Courez !\n",
              "3                    Who?                  Qui ?\n",
              "4                    Wow!             Ça alors !"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-fbd272b7-4936-400f-b1da-e26ce92a116e\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>English words/sentences</th>\n",
              "      <th>French words/sentences</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Hi.</td>\n",
              "      <td>Salut!</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Run!</td>\n",
              "      <td>Cours !</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Run!</td>\n",
              "      <td>Courez !</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Who?</td>\n",
              "      <td>Qui ?</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Wow!</td>\n",
              "      <td>Ça alors !</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-fbd272b7-4936-400f-b1da-e26ce92a116e')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-fbd272b7-4936-400f-b1da-e26ce92a116e button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-fbd272b7-4936-400f-b1da-e26ce92a116e');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ],
      "source": [
        "import pandas as pd\n",
        "data = pd.read_csv('/content/eng_-french.csv')\n",
        "data = data[:100000]\n",
        "print(data.shape)\n",
        "data.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6hn6ngup5_sG",
        "outputId": "52e6fc98-e0ac-4225-d19c-5175518442b8"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((100000, 2),\n",
              " Index(['English words/sentences', 'French words/sentences'], dtype='object'))"
            ]
          },
          "metadata": {},
          "execution_count": 6
        }
      ],
      "source": [
        "data.shape, data.columns"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fe8FSJ_eraJn"
      },
      "source": [
        "# Train a simple seq2seq model based on GRU"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WOOvfZlT1UlL"
      },
      "source": [
        "### Data Preparation"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "id": "jjv1Z5kP1eYu"
      },
      "outputs": [],
      "source": [
        "# we use spacy tokenizers\n",
        "import spacy\n",
        "import torchtext as tt\n",
        "import collections\n",
        "import numpy as np\n",
        "from torchtext.vocab import vocab\n",
        "from torch.utils.data import DataLoader\n",
        "from torch.nn.utils.rnn import pad_sequence\n",
        "import torch\n",
        "import torch.nn as nn"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "I_xRo4wV15ZZ",
        "outputId": "e9f60a35-6660-4ac5-cd20-8f00045bab2b"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2022-09-22 16:41:12.849291: E tensorflow/stream_executor/cuda/cuda_driver.cc:271] failed call to cuInit: CUDA_ERROR_NO_DEVICE: no CUDA-capable device is detected\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting en-core-web-sm==3.4.0\n",
            "  Downloading https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.4.0/en_core_web_sm-3.4.0-py3-none-any.whl (12.8 MB)\n",
            "\u001b[K     |████████████████████████████████| 12.8 MB 4.0 MB/s \n",
            "\u001b[?25hRequirement already satisfied: spacy<3.5.0,>=3.4.0 in /usr/local/lib/python3.7/dist-packages (from en-core-web-sm==3.4.0) (3.4.1)\n",
            "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.0->en-core-web-sm==3.4.0) (3.0.7)\n",
            "Requirement already satisfied: typer<0.5.0,>=0.3.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.0->en-core-web-sm==3.4.0) (0.4.2)\n",
            "Requirement already satisfied: srsly<3.0.0,>=2.4.3 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.0->en-core-web-sm==3.4.0) (2.4.4)\n",
            "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.0->en-core-web-sm==3.4.0) (1.0.8)\n",
            "Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.0->en-core-web-sm==3.4.0) (4.64.1)\n",
            "Requirement already satisfied: spacy-legacy<3.1.0,>=3.0.9 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.0->en-core-web-sm==3.4.0) (3.0.10)\n",
            "Requirement already satisfied: spacy-loggers<2.0.0,>=1.0.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.0->en-core-web-sm==3.4.0) (1.0.3)\n",
            "Requirement already satisfied: requests<3.0.0,>=2.13.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.0->en-core-web-sm==3.4.0) (2.23.0)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.0->en-core-web-sm==3.4.0) (57.4.0)\n",
            "Requirement already satisfied: thinc<8.2.0,>=8.1.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.0->en-core-web-sm==3.4.0) (8.1.0)\n",
            "Requirement already satisfied: langcodes<4.0.0,>=3.2.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.0->en-core-web-sm==3.4.0) (3.3.0)\n",
            "Requirement already satisfied: numpy>=1.15.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.0->en-core-web-sm==3.4.0) (1.21.6)\n",
            "Requirement already satisfied: catalogue<2.1.0,>=2.0.6 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.0->en-core-web-sm==3.4.0) (2.0.8)\n",
            "Requirement already satisfied: pathy>=0.3.5 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.0->en-core-web-sm==3.4.0) (0.6.2)\n",
            "Requirement already satisfied: typing-extensions<4.2.0,>=3.7.4 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.0->en-core-web-sm==3.4.0) (4.1.1)\n",
            "Requirement already satisfied: pydantic!=1.8,!=1.8.1,<1.10.0,>=1.7.4 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.0->en-core-web-sm==3.4.0) (1.9.2)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.0->en-core-web-sm==3.4.0) (2.11.3)\n",
            "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.0->en-core-web-sm==3.4.0) (2.0.6)\n",
            "Requirement already satisfied: wasabi<1.1.0,>=0.9.1 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.0->en-core-web-sm==3.4.0) (0.10.1)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.0->en-core-web-sm==3.4.0) (21.3)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from catalogue<2.1.0,>=2.0.6->spacy<3.5.0,>=3.4.0->en-core-web-sm==3.4.0) (3.8.1)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=20.0->spacy<3.5.0,>=3.4.0->en-core-web-sm==3.4.0) (3.0.9)\n",
            "Requirement already satisfied: smart-open<6.0.0,>=5.2.1 in /usr/local/lib/python3.7/dist-packages (from pathy>=0.3.5->spacy<3.5.0,>=3.4.0->en-core-web-sm==3.4.0) (5.2.1)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.5.0,>=3.4.0->en-core-web-sm==3.4.0) (1.24.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.5.0,>=3.4.0->en-core-web-sm==3.4.0) (2022.6.15)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.5.0,>=3.4.0->en-core-web-sm==3.4.0) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.5.0,>=3.4.0->en-core-web-sm==3.4.0) (2.10)\n",
            "Requirement already satisfied: blis<0.8.0,>=0.7.8 in /usr/local/lib/python3.7/dist-packages (from thinc<8.2.0,>=8.1.0->spacy<3.5.0,>=3.4.0->en-core-web-sm==3.4.0) (0.7.8)\n",
            "Requirement already satisfied: click<9.0.0,>=7.1.1 in /usr/local/lib/python3.7/dist-packages (from typer<0.5.0,>=0.3.0->spacy<3.5.0,>=3.4.0->en-core-web-sm==3.4.0) (7.1.2)\n",
            "Requirement already satisfied: MarkupSafe>=0.23 in /usr/local/lib/python3.7/dist-packages (from jinja2->spacy<3.5.0,>=3.4.0->en-core-web-sm==3.4.0) (2.0.1)\n",
            "\u001b[38;5;2m✔ Download and installation successful\u001b[0m\n",
            "You can now load the package via spacy.load('en_core_web_sm')\n",
            "2022-09-22 16:41:24.136375: E tensorflow/stream_executor/cuda/cuda_driver.cc:271] failed call to cuInit: CUDA_ERROR_NO_DEVICE: no CUDA-capable device is detected\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting fr-core-news-sm==3.4.0\n",
            "  Downloading https://github.com/explosion/spacy-models/releases/download/fr_core_news_sm-3.4.0/fr_core_news_sm-3.4.0-py3-none-any.whl (16.3 MB)\n",
            "\u001b[K     |████████████████████████████████| 16.3 MB 4.1 MB/s \n",
            "\u001b[?25hRequirement already satisfied: spacy<3.5.0,>=3.4.0 in /usr/local/lib/python3.7/dist-packages (from fr-core-news-sm==3.4.0) (3.4.1)\n",
            "Requirement already satisfied: spacy-legacy<3.1.0,>=3.0.9 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.0->fr-core-news-sm==3.4.0) (3.0.10)\n",
            "Requirement already satisfied: thinc<8.2.0,>=8.1.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.0->fr-core-news-sm==3.4.0) (8.1.0)\n",
            "Requirement already satisfied: numpy>=1.15.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.0->fr-core-news-sm==3.4.0) (1.21.6)\n",
            "Requirement already satisfied: requests<3.0.0,>=2.13.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.0->fr-core-news-sm==3.4.0) (2.23.0)\n",
            "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.0->fr-core-news-sm==3.4.0) (2.0.6)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.0->fr-core-news-sm==3.4.0) (2.11.3)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.0->fr-core-news-sm==3.4.0) (21.3)\n",
            "Requirement already satisfied: srsly<3.0.0,>=2.4.3 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.0->fr-core-news-sm==3.4.0) (2.4.4)\n",
            "Requirement already satisfied: pathy>=0.3.5 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.0->fr-core-news-sm==3.4.0) (0.6.2)\n",
            "Requirement already satisfied: pydantic!=1.8,!=1.8.1,<1.10.0,>=1.7.4 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.0->fr-core-news-sm==3.4.0) (1.9.2)\n",
            "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.0->fr-core-news-sm==3.4.0) (1.0.8)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.0->fr-core-news-sm==3.4.0) (57.4.0)\n",
            "Requirement already satisfied: wasabi<1.1.0,>=0.9.1 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.0->fr-core-news-sm==3.4.0) (0.10.1)\n",
            "Requirement already satisfied: catalogue<2.1.0,>=2.0.6 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.0->fr-core-news-sm==3.4.0) (2.0.8)\n",
            "Requirement already satisfied: langcodes<4.0.0,>=3.2.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.0->fr-core-news-sm==3.4.0) (3.3.0)\n",
            "Requirement already satisfied: typer<0.5.0,>=0.3.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.0->fr-core-news-sm==3.4.0) (0.4.2)\n",
            "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.0->fr-core-news-sm==3.4.0) (3.0.7)\n",
            "Requirement already satisfied: spacy-loggers<2.0.0,>=1.0.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.0->fr-core-news-sm==3.4.0) (1.0.3)\n",
            "Requirement already satisfied: typing-extensions<4.2.0,>=3.7.4 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.0->fr-core-news-sm==3.4.0) (4.1.1)\n",
            "Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in /usr/local/lib/python3.7/dist-packages (from spacy<3.5.0,>=3.4.0->fr-core-news-sm==3.4.0) (4.64.1)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from catalogue<2.1.0,>=2.0.6->spacy<3.5.0,>=3.4.0->fr-core-news-sm==3.4.0) (3.8.1)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=20.0->spacy<3.5.0,>=3.4.0->fr-core-news-sm==3.4.0) (3.0.9)\n",
            "Requirement already satisfied: smart-open<6.0.0,>=5.2.1 in /usr/local/lib/python3.7/dist-packages (from pathy>=0.3.5->spacy<3.5.0,>=3.4.0->fr-core-news-sm==3.4.0) (5.2.1)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.5.0,>=3.4.0->fr-core-news-sm==3.4.0) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.5.0,>=3.4.0->fr-core-news-sm==3.4.0) (2.10)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.5.0,>=3.4.0->fr-core-news-sm==3.4.0) (1.24.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests<3.0.0,>=2.13.0->spacy<3.5.0,>=3.4.0->fr-core-news-sm==3.4.0) (2022.6.15)\n",
            "Requirement already satisfied: blis<0.8.0,>=0.7.8 in /usr/local/lib/python3.7/dist-packages (from thinc<8.2.0,>=8.1.0->spacy<3.5.0,>=3.4.0->fr-core-news-sm==3.4.0) (0.7.8)\n",
            "Requirement already satisfied: click<9.0.0,>=7.1.1 in /usr/local/lib/python3.7/dist-packages (from typer<0.5.0,>=0.3.0->spacy<3.5.0,>=3.4.0->fr-core-news-sm==3.4.0) (7.1.2)\n",
            "Requirement already satisfied: MarkupSafe>=0.23 in /usr/local/lib/python3.7/dist-packages (from jinja2->spacy<3.5.0,>=3.4.0->fr-core-news-sm==3.4.0) (2.0.1)\n",
            "Installing collected packages: fr-core-news-sm\n",
            "Successfully installed fr-core-news-sm-3.4.0\n",
            "\u001b[38;5;2m✔ Download and installation successful\u001b[0m\n",
            "You can now load the package via spacy.load('fr_core_news_sm')\n"
          ]
        }
      ],
      "source": [
        "!spacy download en_core_web_sm\n",
        "!spacy download fr_core_news_sm\n",
        "english_tokenizer = spacy.load('en_core_web_sm')\n",
        "french_tokenizer = spacy.load('fr_core_news_sm')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "id": "yu-YL6aS3fy-"
      },
      "outputs": [],
      "source": [
        "def make_vocab(df: pd.Series, tokenizer):\n",
        "  counter_obj = collections.Counter()\n",
        "  for item in df:\n",
        "    line = item.strip()\n",
        "    tokenized_words = [token.text for token in tokenizer(line)]\n",
        "    counter_obj.update(tokenized_words)\n",
        "  return vocab(counter_obj, min_freq = 1, specials=['<unk>', '<pad>', '<bos>', '<eos>'])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "id": "rkIiZZln4Ww8"
      },
      "outputs": [],
      "source": [
        "en_vocab = make_vocab(data.loc[:,'English words/sentences'], english_tokenizer)\n",
        "fr_vocab = make_vocab(data.loc[:,'French words/sentences'], french_tokenizer)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FDURL3T_7hP5",
        "outputId": "47830910-e10e-4d16-c7d7-376f1d9cf6b8"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((60000, 2), (20000, 2), (20000, 2))"
            ]
          },
          "metadata": {},
          "execution_count": 11
        }
      ],
      "source": [
        "#train, test, val split\n",
        "train, validate, test = np.split(data.sample(frac=1, random_state=42), [int(.6*len(data)), int(.8*len(data))])\n",
        "train.reset_index(drop=True,inplace=True)\n",
        "validate.reset_index(drop=True,inplace=True)\n",
        "test.reset_index(drop=True,inplace=True)\n",
        "\n",
        "train.shape, validate.shape, test.shape"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "id": "t74h-zMadYSk"
      },
      "outputs": [],
      "source": [
        "bos_id = en_vocab.__getitem__('<bos>')\n",
        "eos_id = en_vocab.__getitem__('<eos>')\n",
        "pad_id = en_vocab.__getitem__('<pad>')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qfgDXFIAU5AH",
        "outputId": "166dc42a-caf8-4c69-8821-d51264d59bf1"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(3, 1)"
            ]
          },
          "metadata": {},
          "execution_count": 13
        }
      ],
      "source": [
        "eos_id, pad_id"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {
        "id": "uGjihFUYeYd_"
      },
      "outputs": [],
      "source": [
        "def prepare_source_target(df):\n",
        "  tuple_list = []\n",
        "  for i in range(len(df)):\n",
        "    eng_sentence = df.loc[i,'English words/sentences'].strip()\n",
        "    french_sentence = df.loc[i,'French words/sentences'].strip()\n",
        "    src = torch.tensor([en_vocab.__getitem__(token.text) for token in english_tokenizer(eng_sentence)])\n",
        "    target = torch.tensor([fr_vocab.__getitem__(token.text) for token in french_tokenizer(french_sentence)])\n",
        "    tuple_list.append((src, target))\n",
        "  return tuple_list\n",
        "\n",
        "train_data = prepare_source_target(train)\n",
        "val_data = prepare_source_target(validate)\n",
        "test_data = prepare_source_target(test)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {
        "id": "1qnRRGwu6Ilp"
      },
      "outputs": [],
      "source": [
        "# prepare dataloaders\n",
        "def collate_fn(batch_sampler_data):\n",
        "  src_batch = []\n",
        "  target_batch =[] \n",
        "  for i,(src, target) in enumerate(batch_sampler_data):\n",
        "    src_tensor = torch.tensor(torch.cat([torch.tensor([bos_id]), src, torch.tensor([eos_id])], dim=0))\n",
        "    target_tensor = torch.tensor(torch.cat([torch.tensor([bos_id]), target, torch.tensor([eos_id])], dim=0))\n",
        "    src_batch.append(src_tensor)\n",
        "    target_batch.append(target_tensor)\n",
        "  src_batch = pad_sequence(src_batch, padding_value=pad_id)\n",
        "  target_batch = pad_sequence(target_batch, padding_value=pad_id)\n",
        "\n",
        "  return src_batch, target_batch\n",
        "\n",
        "train_dataloader = DataLoader(train_data,batch_size=128, shuffle=True,collate_fn=collate_fn)\n",
        "val_dataloader = DataLoader(val_data,batch_size=128, shuffle=True,collate_fn=collate_fn)\n",
        "test_dataloader = DataLoader(test_data,batch_size=128, shuffle=True,collate_fn=collate_fn)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 26,
      "metadata": {
        "id": "CLOGbBuBrpZl"
      },
      "outputs": [],
      "source": [
        "class Encoder(torch.nn.Module):\n",
        "  def __init__(self, embedding_size: int, vocab_size : int, gru_layers: int):\n",
        "    super().__init__()\n",
        "    self.embedding_size = embedding_size\n",
        "    self.hidden_size = 2*self.embedding_size\n",
        "    self.embedding = nn.Embedding(num_embeddings=vocab_size, embedding_dim=self.embedding_size)\n",
        "    self.GRU = nn.GRU(input_size=self.embedding_size, hidden_size=self.hidden_size,num_layers=gru_layers, bidirectional=True)\n",
        "    self.l1_layer = nn.Linear(in_features=2*self.hidden_size,out_features=self.hidden_size)\n",
        "\n",
        "  def forward(self, src_batch):\n",
        "    # src_len*batch_size\n",
        "    src_embedding = self.embedding(src_batch)\n",
        "    #print(\"Encoder:: src embedding shape: {}\".format(src_embedding.shape))\n",
        "    # src_len*batch_size* embedding_dim\n",
        "    output, hidden = self.GRU(src_embedding)\n",
        "    context = torch.tanh(self.l1_layer(torch.cat((hidden[-2,:,:],hidden[-1,:,:]), dim=1)))\n",
        "    #print(\"Encoder:: output shape: {}, hidden shape {}\".format(output.shape, hidden.shape))\n",
        "    return context, output"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "class Attention(nn.Module):\n",
        "    def __init__(self, enc_hid_dim, dec_hid_dim):\n",
        "        super().__init__()\n",
        "        self.attn = nn.Linear((enc_hid_dim * 2) + dec_hid_dim, dec_hid_dim)\n",
        "        self.v = nn.Linear(dec_hid_dim, 1, bias = False)\n",
        "        \n",
        "    def forward(self, hidden, encoder_outputs):\n",
        "        \n",
        "        #hidden = [batch size, dec hid dim]\n",
        "        #encoder_outputs = [src len, batch size, enc hid dim * 2]\n",
        "        \n",
        "        batch_size = encoder_outputs.shape[1]\n",
        "        src_len = encoder_outputs.shape[0]\n",
        "        #repeat decoder hidden state src_len times\n",
        "        \n",
        "        hidden = hidden.permute(1, 0, 2)\n",
        "        hidden = hidden.repeat(1, src_len, 1)\n",
        "\n",
        "        encoder_outputs = encoder_outputs.permute(1, 0, 2)\n",
        "        \n",
        "        #hidden = [batch size, src len, dec hid dim]\n",
        "        #encoder_outputs = [batch size, src len, enc hid dim * 2]\n",
        "        energy = torch.tanh(self.attn(torch.cat((hidden, encoder_outputs), dim = 2))) \n",
        "        \n",
        "        #energy = [batch size, src len, dec hid dim]\n",
        "\n",
        "        attention = self.v(energy).squeeze(2)\n",
        "        \n",
        "        #attention= [batch size, src len]\n",
        "        \n",
        "        return torch.nn.functional.softmax(attention, dim=1)"
      ],
      "metadata": {
        "id": "AlXiRK-tR-2W"
      },
      "execution_count": 25,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": 27,
      "metadata": {
        "id": "ENTUfN4bvXNx"
      },
      "outputs": [],
      "source": [
        "class Decoder(torch.nn.Module):\n",
        "  def __init__(self, embedding_size: int, target_vocab_size : int, gru_layers: int):\n",
        "    super().__init__()\n",
        "    self.embedding_size = embedding_size\n",
        "    self.hidden_size = 2*self.embedding_size\n",
        "    self.output_size = self.hidden_size\n",
        "    self.embedding = nn.Embedding(num_embeddings=target_vocab_size, embedding_dim=self.embedding_size)\n",
        "    self.GRU = nn.GRU(input_size=self.embedding_size+2*self.hidden_size, hidden_size=self.hidden_size,num_layers=gru_layers)\n",
        "    self.attention_layer = Attention(self.hidden_size,self.hidden_size)\n",
        "    self.output_layer = nn.Linear(in_features=self.embedding_size+3*self.hidden_size,out_features=target_vocab_size)\n",
        "\n",
        "  def forward(self, hidden, previous_prediction, context, encoder_outputs):\n",
        "    word_embedding = self.embedding(previous_prediction)\n",
        "    word_embedding = torch.unsqueeze(word_embedding, dim=0)\n",
        "    #print(\"Decoder:: src embedding shape: {}\".format(word_embedding.shape))\n",
        "    # 1*batch_size*embedding_length\n",
        "    #print(word_embedding.shape, context.shape)\n",
        "    if hidden.dim()<3:\n",
        "        hidden = hidden.unsqueeze(0)\n",
        "    attention = self.attention_layer(hidden, encoder_outputs)\n",
        "    # batch_size*src_len\n",
        "    attention = attention.unsqueeze(1)\n",
        "\n",
        "    # batch_size*1*src_len\n",
        "    encoder_outputs = encoder_outputs.permute(1, 0, 2)\n",
        "\n",
        "    # encoder_outputs batch_size*src_len*hidden_dim\n",
        "    new_context = torch.bmm(attention, encoder_outputs).permute(1,0,2)\n",
        "    concat_embedding_context = torch.cat((word_embedding, new_context), dim=2)\n",
        "    # 1*batch_size*(embedding_length+hidden_dim)\n",
        "\n",
        "    \n",
        "    output, hidden = self.GRU(concat_embedding_context, hidden)\n",
        "    #print(\"Decoder:: output shape: {}, hidden shape {}\".format(output.shape, hidden.shape))\n",
        "    # output shape == (1,batch_size,hidden_dim)\n",
        "    # hidden shape == (num_layers*num_directions,batch_size, hidden_dim)\n",
        "    output = torch.concat((output, concat_embedding_context),dim=2)\n",
        "    output = self.output_layer(output.squeeze(0))\n",
        "    #print(\"Decoder:: output shape: {} after linear layer\".format(output.shape))\n",
        "    # 1*batch_size*target_vocab_size\n",
        "    return output, hidden"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 28,
      "metadata": {
        "id": "AlTxrktp4pXU"
      },
      "outputs": [],
      "source": [
        "import random\n",
        "class Seq2Seq(torch.nn.Module):\n",
        "  def __init__(self):\n",
        "    super().__init__()\n",
        "    self.encoder = Encoder(embedding_size=100, vocab_size=len(en_vocab), gru_layers=1)\n",
        "    self.decoder = Decoder(embedding_size=100, target_vocab_size=len(fr_vocab), gru_layers=1)\n",
        "\n",
        "  def forward(self, src_batch, target_batch, teacher_forcing_ratio: int):\n",
        "    hidden, output = self.encoder(src_batch)\n",
        "    context = hidden\n",
        "\n",
        "    # output_shape: src_len*num_directions, batch_size, hidden_dim\n",
        "    # hidden: num_layers*num_directions,batch_size, hidden_dim\n",
        "\n",
        "    batch_size = src_batch.shape[1]\n",
        "    src_length = src_batch.shape[0]\n",
        "    max_len = target_batch.shape[0]\n",
        "\n",
        "    final_decoder_output = torch.zeros(size=(max_len, batch_size, len(fr_vocab)), device= device)\n",
        "    input = target_batch[0,:]\n",
        "    for i in range(1,max_len):\n",
        "      decoder_output, hidden = self.decoder(hidden, input, context, output)\n",
        "      # decoder_output: 1*batch_size*target_vocab_size\n",
        "      random_probab = random.random() > teacher_forcing_ratio\n",
        "      prediction_word = decoder_output.argmax(dim=1)\n",
        "      final_decoder_output[i,:,:] = decoder_output  \n",
        "      input = prediction_word if random_probab else target_batch[i,:]\n",
        "      input = input.squeeze()\n",
        "    return final_decoder_output"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 36,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "oz5NCLMWDV6i",
        "outputId": "e5cedf03-701e-4244-97d3-04ad7811716a"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "cpu\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Seq2Seq(\n",
              "  (encoder): Encoder(\n",
              "    (embedding): Embedding(9644, 100)\n",
              "    (GRU): GRU(100, 200, bidirectional=True)\n",
              "    (l1_layer): Linear(in_features=400, out_features=200, bias=True)\n",
              "  )\n",
              "  (decoder): Decoder(\n",
              "    (embedding): Embedding(17025, 100)\n",
              "    (GRU): GRU(500, 200)\n",
              "    (attention_layer): Attention(\n",
              "      (attn): Linear(in_features=600, out_features=200, bias=True)\n",
              "      (v): Linear(in_features=200, out_features=1, bias=False)\n",
              "    )\n",
              "    (output_layer): Linear(in_features=700, out_features=17025, bias=True)\n",
              "  )\n",
              ")"
            ]
          },
          "metadata": {},
          "execution_count": 36
        }
      ],
      "source": [
        "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
        "print(device)\n",
        "model = Seq2Seq().to(device)\n",
        "\n",
        "def init_weights(m):\n",
        "    for name, param in m.named_parameters():\n",
        "        nn.init.uniform_(param.data, -0.05, 0.05)\n",
        "        \n",
        "model.apply(init_weights)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "9so7L3aI_pNz"
      },
      "outputs": [],
      "source": [
        "# define criterion and optimizer\n",
        "import torch.optim as optim\n",
        "\n",
        "criterion = nn.CrossEntropyLoss(ignore_index = pad_id)\n",
        "optimizer = optim.Adam(model.parameters())"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "9bxOTP-3Exvn"
      },
      "outputs": [],
      "source": [
        "def train_model(dataloader):\n",
        "  clip = 0.1\n",
        "  model.train()\n",
        "  epoch_loss = 0\n",
        "\n",
        "  for i, (src_batch, target_batch) in enumerate(dataloader):\n",
        "    prediction = model(src_batch=src_batch.cuda(), target_batch=target_batch.cuda(), teacher_forcing_ratio=0.7)\n",
        "    prediction = prediction[1:].view(-1, len(fr_vocab))\n",
        "    target_batch = target_batch[1:].view(-1).cuda()\n",
        "    loss = criterion(prediction, target_batch)\n",
        "    loss.backward()\n",
        "    torch.nn.utils.clip_grad_norm_(model.parameters(), clip)\n",
        "    optimizer.step()\n",
        "    epoch_loss += loss.item()\n",
        "  \n",
        "  return epoch_loss / len(dataloader)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Z5y8OF-IDA3j"
      },
      "outputs": [],
      "source": [
        "def eval_model(dataloader):\n",
        "  model.eval()\n",
        "  epoch_loss = 0\n",
        "\n",
        "  for i, (src_batch, target_batch) in enumerate(dataloader):\n",
        "    prediction = model(src_batch=src_batch.cuda(), target_batch=target_batch.cuda(), teacher_forcing_ratio=0)\n",
        "    prediction = prediction[1:].view(-1, len(fr_vocab))\n",
        "    target_batch = target_batch[1:].view(-1).cuda()\n",
        "    loss = criterion(prediction, target_batch)\n",
        "    #loss.backward()\n",
        "    #optimizer.step()\n",
        "    epoch_loss += loss.item()\n",
        "  \n",
        "  return epoch_loss / len(dataloader)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "yR8JWPH_DWGV",
        "outputId": "d303e3f4-9f34-41f3-b742-c2367e6f2761"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:6: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
            "  \n",
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:7: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
            "  import sys\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch: 01 | Time: 0m 45s\n",
            "\tTrain Loss: 4.222 | Train PPL:  68.173\n",
            "\t Val. Loss: 3.954 |  Val. PPL:  52.132\n",
            "Epoch: 02 | Time: 0m 45s\n",
            "\tTrain Loss: 2.576 | Train PPL:  13.140\n",
            "\t Val. Loss: 3.201 |  Val. PPL:  24.559\n",
            "Epoch: 03 | Time: 0m 45s\n",
            "\tTrain Loss: 1.791 | Train PPL:   5.998\n",
            "\t Val. Loss: 2.863 |  Val. PPL:  17.522\n",
            "Epoch: 04 | Time: 0m 45s\n",
            "\tTrain Loss: 1.341 | Train PPL:   3.822\n",
            "\t Val. Loss: 2.765 |  Val. PPL:  15.886\n",
            "Epoch: 05 | Time: 0m 45s\n",
            "\tTrain Loss: 1.082 | Train PPL:   2.951\n",
            "\t Val. Loss: 2.715 |  Val. PPL:  15.107\n",
            "Epoch: 06 | Time: 0m 45s\n",
            "\tTrain Loss: 0.891 | Train PPL:   2.437\n",
            "\t Val. Loss: 2.756 |  Val. PPL:  15.738\n",
            "Epoch: 07 | Time: 0m 45s\n",
            "\tTrain Loss: 0.768 | Train PPL:   2.155\n",
            "\t Val. Loss: 2.795 |  Val. PPL:  16.369\n",
            "Epoch: 08 | Time: 0m 45s\n",
            "\tTrain Loss: 0.676 | Train PPL:   1.966\n",
            "\t Val. Loss: 2.850 |  Val. PPL:  17.289\n",
            "Epoch: 09 | Time: 0m 45s\n",
            "\tTrain Loss: 0.608 | Train PPL:   1.836\n",
            "\t Val. Loss: 2.870 |  Val. PPL:  17.630\n",
            "Epoch: 10 | Time: 0m 45s\n",
            "\tTrain Loss: 0.556 | Train PPL:   1.743\n",
            "\t Val. Loss: 2.956 |  Val. PPL:  19.218\n",
            "Epoch: 11 | Time: 0m 45s\n",
            "\tTrain Loss: 0.506 | Train PPL:   1.659\n",
            "\t Val. Loss: 3.003 |  Val. PPL:  20.153\n",
            "Epoch: 12 | Time: 0m 45s\n",
            "\tTrain Loss: 0.457 | Train PPL:   1.580\n",
            "\t Val. Loss: 3.112 |  Val. PPL:  22.466\n",
            "Epoch: 13 | Time: 0m 45s\n",
            "\tTrain Loss: 0.441 | Train PPL:   1.555\n",
            "\t Val. Loss: 3.118 |  Val. PPL:  22.600\n",
            "Epoch: 14 | Time: 0m 45s\n",
            "\tTrain Loss: 0.402 | Train PPL:   1.495\n",
            "\t Val. Loss: 3.239 |  Val. PPL:  25.514\n",
            "Epoch: 15 | Time: 0m 45s\n",
            "\tTrain Loss: 0.382 | Train PPL:   1.465\n",
            "\t Val. Loss: 3.273 |  Val. PPL:  26.391\n",
            "Epoch: 16 | Time: 0m 45s\n",
            "\tTrain Loss: 0.365 | Train PPL:   1.440\n",
            "\t Val. Loss: 3.335 |  Val. PPL:  28.069\n",
            "Epoch: 17 | Time: 0m 45s\n",
            "\tTrain Loss: 0.352 | Train PPL:   1.422\n",
            "\t Val. Loss: 3.351 |  Val. PPL:  28.541\n",
            "Epoch: 18 | Time: 0m 45s\n",
            "\tTrain Loss: 0.339 | Train PPL:   1.403\n",
            "\t Val. Loss: 3.403 |  Val. PPL:  30.060\n",
            "Epoch: 19 | Time: 0m 45s\n",
            "\tTrain Loss: 0.331 | Train PPL:   1.392\n",
            "\t Val. Loss: 3.445 |  Val. PPL:  31.329\n",
            "Epoch: 20 | Time: 0m 45s\n",
            "\tTrain Loss: 0.316 | Train PPL:   1.371\n",
            "\t Val. Loss: 3.498 |  Val. PPL:  33.050\n",
            "Epoch: 21 | Time: 0m 45s\n",
            "\tTrain Loss: 0.315 | Train PPL:   1.371\n",
            "\t Val. Loss: 3.483 |  Val. PPL:  32.566\n",
            "Epoch: 22 | Time: 0m 45s\n",
            "\tTrain Loss: 0.309 | Train PPL:   1.362\n",
            "\t Val. Loss: 3.534 |  Val. PPL:  34.276\n",
            "Epoch: 23 | Time: 0m 45s\n",
            "\tTrain Loss: 0.301 | Train PPL:   1.352\n",
            "\t Val. Loss: 3.536 |  Val. PPL:  34.341\n",
            "Epoch: 24 | Time: 0m 45s\n",
            "\tTrain Loss: 0.294 | Train PPL:   1.341\n",
            "\t Val. Loss: 3.583 |  Val. PPL:  35.968\n",
            "Epoch: 25 | Time: 0m 45s\n",
            "\tTrain Loss: 0.288 | Train PPL:   1.333\n",
            "\t Val. Loss: 3.645 |  Val. PPL:  38.290\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-20-62f0bd07ea3a>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     16\u001b[0m     \u001b[0mstart_time\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 18\u001b[0;31m     \u001b[0mtrain_loss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrain_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdataloader\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrain_dataloader\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     19\u001b[0m     \u001b[0mvalid_loss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0meval_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdataloader\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mval_dataloader\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     20\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-18-617dab170ab4>\u001b[0m in \u001b[0;36mtrain_model\u001b[0;34m(dataloader)\u001b[0m\n\u001b[1;32m     12\u001b[0m     \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mutils\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclip_grad_norm_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparameters\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mclip\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     13\u001b[0m     \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 14\u001b[0;31m     \u001b[0mepoch_loss\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     15\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     16\u001b[0m   \u001b[0;32mreturn\u001b[0m \u001b[0mepoch_loss\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdataloader\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ],
      "source": [
        "import time\n",
        "import math\n",
        "\n",
        "N_EPOCHS = 50\n",
        "\n",
        "best_valid_loss = float('inf')\n",
        "\n",
        "def epoch_time(start_time, end_time):\n",
        "    elapsed_time = end_time - start_time\n",
        "    elapsed_mins = int(elapsed_time / 60)\n",
        "    elapsed_secs = int(elapsed_time - (elapsed_mins * 60))\n",
        "    return elapsed_mins, elapsed_secs\n",
        "\n",
        "for epoch in range(N_EPOCHS):\n",
        "    \n",
        "    start_time = time.time()\n",
        "    \n",
        "    train_loss = train_model(dataloader = train_dataloader)\n",
        "    valid_loss = eval_model(dataloader = val_dataloader)\n",
        "    \n",
        "    end_time = time.time()\n",
        "    \n",
        "    epoch_mins, epoch_secs = epoch_time(start_time, end_time)\n",
        "    \n",
        "    if valid_loss < best_valid_loss:\n",
        "        best_valid_loss = valid_loss\n",
        "        torch.save(model.state_dict(), '/content/translation_model_attention_epoch_{}.pt'.format(epoch))\n",
        "    \n",
        "    print(f'Epoch: {epoch+1:02} | Time: {epoch_mins}m {epoch_secs}s')\n",
        "    print(f'\\tTrain Loss: {train_loss:.3f} | Train PPL: {math.exp(train_loss):7.3f}')\n",
        "    print(f'\\t Val. Loss: {valid_loss:.3f} |  Val. PPL: {math.exp(valid_loss):7.3f}')"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# from google.colab import drive\n",
        "# drive.mount('/content/drive')\n",
        "model.load_state_dict(torch.load('/content/translation_model_attention_epoch_4.pt',map_location=torch.device('cpu')))"
      ],
      "metadata": {
        "id": "3Qtj9pQEprKi",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "f2816784-29fb-4fea-e70e-b84ebe6017de"
      },
      "execution_count": 37,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<All keys matched successfully>"
            ]
          },
          "metadata": {},
          "execution_count": 37
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def translate(dataloader):\n",
        "   model.eval()\n",
        "   translations=[]\n",
        "   for i, (src_batch, target_batch) in enumerate(dataloader):\n",
        "        x = model(src_batch, target_batch, 0)\n",
        "        batch_size = x.size()[1]\n",
        "        #x = .view(-1, batch_size, len(fr_vocab))\n",
        "        src_batch = src_batch[1:-1].permute(1,0)\n",
        "        target_batch = target_batch[1:-1].permute(1,0)\n",
        "        prediction_word = x[1:-1].argmax(dim=2).permute(1,0)\n",
        "        preds = []\n",
        "        for i in range(prediction_word.shape[0]):\n",
        "          output = prediction_word[i,:]\n",
        "          input = src_batch[i,:]\n",
        "          target = target_batch[i,:]\n",
        "          input = [en_vocab.vocab.get_itos()[item] for item in input]\n",
        "          output = [fr_vocab.vocab.get_itos()[item] for item in output]\n",
        "          target = [fr_vocab.vocab.get_itos()[item] for item in target]\n",
        "          output = output[: output.index('<eos>')] if '<eos>' in output else output\n",
        "          input = input[: input.index('<eos>')] if '<eos>' in input else input\n",
        "          target = target[: target.index('<eos>')] if '<eos>' in target else target\n",
        "          translations.append((\" \".join(input), \" \".join(output) ,\" \".join(target)))\n",
        "          #print((\" \".join(input), \" \".join(output) ,\" \".join(target)))\n",
        "   return translations \n",
        "\n",
        "translations = translate(test_dataloader)"
      ],
      "metadata": {
        "id": "Ek4uMlF9qHZZ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "1b870d45-fe7b-4da4-f26c-f9f06be24fcc"
      },
      "execution_count": 71,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:6: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
            "  \n",
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:7: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
            "  import sys\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "[x[1].split(\" \") for x in translations][0], [x[2].split(\" \") for x in translations][0]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "X1bRykylqoYD",
        "outputId": "a30182b5-9e51-42ff-b58b-8f4243ab88d3"
      },
      "execution_count": 83,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(['Cette', 'corde', \"n'\", 'est', 'pas', 'assez', '.'],\n",
              " ['Cette', 'corde', \"n'\", 'est', 'pas', 'assez', 'solide', '.'])"
            ]
          },
          "metadata": {},
          "execution_count": 83
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from torchtext.data.metrics import bleu_score\n",
        "bleu_score([x[1].split(\" \") for x in translations], [[x[2].split(\" \")] for x in translations])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bAhlJl8ikQF6",
        "outputId": "1512cc79-34a1-49d6-bc5b-b6b1a3ca6848"
      },
      "execution_count": 85,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.35945024809113896"
            ]
          },
          "metadata": {},
          "execution_count": 85
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "Rbdje06ui83Q"
      },
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "colab": {
      "collapsed_sections": [],
      "provenance": [],
      "authorship_tag": "ABX9TyPzv3gbueWuPOH1SLhYsEGY",
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}