3_pytorch_transfomer_translator_100epoch_ipynb의_사본.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "3-pytorch transfomer translator_100epoch.ipynb의 사본",
      "provenance": [],
      "collapsed_sections": [],
      "machine_shape": "hm",
      "mount_file_id": "1qHKw5X4jzn9ynt1jqBkUVQysJa0jijrm",
      "authorship_tag": "ABX9TyNl354k5tSxzq61lLTKhqNb",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/da-head0/youreditor/blob/main/3_pytorch_transfomer_translator_100epoch_ipynb%EC%9D%98_%EC%82%AC%EB%B3%B8.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "z3-RdVmo3FLT",
        "outputId": "165f677f-bf87-4089-f377-e8417f5c02fa"
      },
      "source": [
        "!git clone https://github.com/Huffon/pytorch-transformer-kor-eng.git"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Cloning into 'pytorch-transformer-kor-eng'...\n",
            "remote: Enumerating objects: 270, done.\u001b[K\n",
            "remote: Counting objects: 100% (30/30), done.\u001b[K\n",
            "remote: Compressing objects: 100% (23/23), done.\u001b[K\n",
            "remote: Total 270 (delta 13), reused 16 (delta 6), pack-reused 240\u001b[K\n",
            "Receiving objects: 100% (270/270), 10.28 MiB | 25.93 MiB/s, done.\n",
            "Resolving deltas: 100% (164/164), done.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8C7QXfA83iIk",
        "outputId": "5233e7f8-a5d1-4963-c835-ea8eee530206"
      },
      "source": [
        "# 버전을 업그레이드 하면 torch Dataset이 지원되지 않음. torch 1.3까지는 지원되는 듯 하다.\n",
        "\n",
        "!pip install torchvision==0.4.2\n",
        "!pip install torchtext==0.4\n",
        "!pip install torch==1.3.1 -f https://download.pytorch.org/whl/torch_stable.html"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting torchvision==0.4.2\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/74/ee/d872c12ea508f9cca4cf0d7b91fc5a5e476cec77628a66795ec5a585a67f/torchvision-0.4.2-cp37-cp37m-manylinux1_x86_64.whl (10.2MB)\n",
            "\u001b[K     |████████████████████████████████| 10.2MB 6.6MB/s \n",
            "\u001b[?25hRequirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from torchvision==0.4.2) (1.15.0)\n",
            "Collecting torch==1.3.1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/f9/34/2107f342d4493b7107a600ee16005b2870b5a0a5a165bdf5c5e7168a16a6/torch-1.3.1-cp37-cp37m-manylinux1_x86_64.whl (734.6MB)\n",
            "\u001b[K     |██████████████████████████▏     | 600.6MB 1.3MB/s eta 0:01:45"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0TyCdHrM3w_Q",
        "outputId": "7256abde-59f7-4197-8fb8-eac8400f0aa7"
      },
      "source": [
        "!nvidia-smi # GPU 정보 확인"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Sat Apr 24 10:11:34 2021       \n",
            "+-----------------------------------------------------------------------------+\n",
            "| NVIDIA-SMI 465.19.01    Driver Version: 460.32.03    CUDA Version: 11.2     |\n",
            "|-------------------------------+----------------------+----------------------+\n",
            "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
            "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
            "|                               |                      |               MIG M. |\n",
            "|===============================+======================+======================|\n",
            "|   0  Tesla V100-SXM2...  Off  | 00000000:00:04.0 Off |                    0 |\n",
            "| N/A   32C    P0    24W / 300W |      0MiB / 16160MiB |      0%      Default |\n",
            "|                               |                      |                  N/A |\n",
            "+-------------------------------+----------------------+----------------------+\n",
            "                                                                               \n",
            "+-----------------------------------------------------------------------------+\n",
            "| Processes:                                                                  |\n",
            "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n",
            "|        ID   ID                                                   Usage      |\n",
            "|=============================================================================|\n",
            "|  No running processes found                                                 |\n",
            "+-----------------------------------------------------------------------------+\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "g2bOhFlr3zpG"
      },
      "source": [
        "!python -m spacy download en_core_web_sm\n",
        "!python -m spacy download en\n",
        "!pip install soynlp"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xKC08ZNj364o"
      },
      "source": [
        "!pip install ipython-autotime\n",
        "\n",
        "%load_ext autotime"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "irna1kmc4gpR",
        "outputId": "5280a660-3bb9-4f07-e996-e7f7285c5a62"
      },
      "source": [
        "%cd /content/pytorch-transformer-kor-eng"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content/pytorch-transformer-kor-eng\n",
            "time: 2.1 ms (started: 2021-04-24 10:11:46 +00:00)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TOtAguE84LQ5",
        "outputId": "39f74445-ce4c-4189-f47d-b9a6034af2b7"
      },
      "source": [
        "!python build_pickles.py"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Now building soy-nlp tokenizer . . .\n",
            "training was done. used memory 0.486 Gb\n",
            "all cohesion probabilities was computed. # words = 46586\n",
            "all branching entropies was computed # words = 112737\n",
            "all accessor variety was computed # words = 112737\n",
            "Build vocabulary using torchtext . . .\n",
            "Unique tokens in Korean vocabulary: 55002\n",
            "Unique tokens in English vocabulary: 19545\n",
            "Most commonly used Korean words are as follows:\n",
            "[('이', 10659), ('는', 8160), ('을', 6736), ('에', 6573), ('은', 6520), ('수', 6470), ('가', 6153), ('를', 5501), ('당신', 5190), ('해요', 5073), ('나는', 4894), ('요', 4798), ('있어', 4539), ('우리', 4500), ('의', 4392), ('할', 4200), ('서', 4020), ('것', 4013), ('한', 3634), ('있는', 3539)]\n",
            "Most commonly used English words are as follows:\n",
            "[('i', 46668), ('the', 44623), ('to', 32009), ('you', 27361), ('a', 22986), ('is', 18900), ('it', 18156), ('in', 12905), ('and', 11584), ('of', 11465), ('for', 10258), ('that', 9814), ('do', 9322), ('have', 9088), (\"'s\", 9083), ('we', 8864), ('my', 8595), ('this', 7980), (\"n't\", 7901), ('are', 7729)]\n",
            "time: 2min 20s (started: 2021-04-24 10:11:46 +00:00)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MBpABQqECS1Z"
      },
      "source": [
        "- 배치 사이즈 512로 늘려봄.\n",
        "- epoch는 15"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6C9_iNct4um-",
        "outputId": "284d05b5-b1e0-4247-96f8-1ee889b50ae5"
      },
      "source": [
        "!python main.py"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Loading AI Hub Kor-Eng translation dataset and converting it to pandas DataFrame . . .\n",
            "Number of training examples: 92000\n",
            "Number of validation examples: 11500\n",
            "Make Iterators for training . . .\n",
            "Transformer(\n",
            "  (encoder): Encoder(\n",
            "    (token_embedding): Embedding(55002, 512, padding_idx=1)\n",
            "    (pos_embedding): Embedding(65, 512)\n",
            "    (encoder_layers): ModuleList(\n",
            "      (0): EncoderLayer(\n",
            "        (layer_norm): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
            "        (self_attention): MultiHeadAttention(\n",
            "          (attentions): ModuleList(\n",
            "            (0): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (1): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (2): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (3): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (4): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (5): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (6): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (7): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "          )\n",
            "          (o_w): Linear(in_features=512, out_features=512, bias=False)\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (position_wise_ffn): PositionWiseFeedForward(\n",
            "          (conv1): Conv1d(512, 2048, kernel_size=(1,), stride=(1,))\n",
            "          (conv2): Conv1d(2048, 512, kernel_size=(1,), stride=(1,))\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "      (1): EncoderLayer(\n",
            "        (layer_norm): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
            "        (self_attention): MultiHeadAttention(\n",
            "          (attentions): ModuleList(\n",
            "            (0): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (1): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (2): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (3): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (4): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (5): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (6): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (7): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "          )\n",
            "          (o_w): Linear(in_features=512, out_features=512, bias=False)\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (position_wise_ffn): PositionWiseFeedForward(\n",
            "          (conv1): Conv1d(512, 2048, kernel_size=(1,), stride=(1,))\n",
            "          (conv2): Conv1d(2048, 512, kernel_size=(1,), stride=(1,))\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "      (2): EncoderLayer(\n",
            "        (layer_norm): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
            "        (self_attention): MultiHeadAttention(\n",
            "          (attentions): ModuleList(\n",
            "            (0): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (1): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (2): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (3): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (4): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (5): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (6): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (7): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "          )\n",
            "          (o_w): Linear(in_features=512, out_features=512, bias=False)\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (position_wise_ffn): PositionWiseFeedForward(\n",
            "          (conv1): Conv1d(512, 2048, kernel_size=(1,), stride=(1,))\n",
            "          (conv2): Conv1d(2048, 512, kernel_size=(1,), stride=(1,))\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "      (3): EncoderLayer(\n",
            "        (layer_norm): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
            "        (self_attention): MultiHeadAttention(\n",
            "          (attentions): ModuleList(\n",
            "            (0): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (1): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (2): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (3): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (4): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (5): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (6): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (7): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "          )\n",
            "          (o_w): Linear(in_features=512, out_features=512, bias=False)\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (position_wise_ffn): PositionWiseFeedForward(\n",
            "          (conv1): Conv1d(512, 2048, kernel_size=(1,), stride=(1,))\n",
            "          (conv2): Conv1d(2048, 512, kernel_size=(1,), stride=(1,))\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "      (4): EncoderLayer(\n",
            "        (layer_norm): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
            "        (self_attention): MultiHeadAttention(\n",
            "          (attentions): ModuleList(\n",
            "            (0): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (1): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (2): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (3): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (4): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (5): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (6): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (7): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "          )\n",
            "          (o_w): Linear(in_features=512, out_features=512, bias=False)\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (position_wise_ffn): PositionWiseFeedForward(\n",
            "          (conv1): Conv1d(512, 2048, kernel_size=(1,), stride=(1,))\n",
            "          (conv2): Conv1d(2048, 512, kernel_size=(1,), stride=(1,))\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "      (5): EncoderLayer(\n",
            "        (layer_norm): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
            "        (self_attention): MultiHeadAttention(\n",
            "          (attentions): ModuleList(\n",
            "            (0): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (1): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (2): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (3): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (4): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (5): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (6): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (7): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "          )\n",
            "          (o_w): Linear(in_features=512, out_features=512, bias=False)\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (position_wise_ffn): PositionWiseFeedForward(\n",
            "          (conv1): Conv1d(512, 2048, kernel_size=(1,), stride=(1,))\n",
            "          (conv2): Conv1d(2048, 512, kernel_size=(1,), stride=(1,))\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (dropout): Dropout(p=0.1, inplace=False)\n",
            "    (layer_norm): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
            "  )\n",
            "  (decoder): Decoder(\n",
            "    (token_embedding): Embedding(19545, 512, padding_idx=1)\n",
            "    (pos_embedding): Embedding(65, 512)\n",
            "    (decoder_layers): ModuleList(\n",
            "      (0): DecoderLayer(\n",
            "        (layer_norm): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
            "        (self_attention): MultiHeadAttention(\n",
            "          (attentions): ModuleList(\n",
            "            (0): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (1): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (2): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (3): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (4): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (5): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (6): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (7): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "          )\n",
            "          (o_w): Linear(in_features=512, out_features=512, bias=False)\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (encoder_attention): MultiHeadAttention(\n",
            "          (attentions): ModuleList(\n",
            "            (0): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (1): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (2): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (3): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (4): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (5): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (6): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (7): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "          )\n",
            "          (o_w): Linear(in_features=512, out_features=512, bias=False)\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (position_wise_ffn): PositionWiseFeedForward(\n",
            "          (conv1): Conv1d(512, 2048, kernel_size=(1,), stride=(1,))\n",
            "          (conv2): Conv1d(2048, 512, kernel_size=(1,), stride=(1,))\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "      (1): DecoderLayer(\n",
            "        (layer_norm): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
            "        (self_attention): MultiHeadAttention(\n",
            "          (attentions): ModuleList(\n",
            "            (0): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (1): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (2): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (3): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (4): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (5): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (6): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (7): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "          )\n",
            "          (o_w): Linear(in_features=512, out_features=512, bias=False)\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (encoder_attention): MultiHeadAttention(\n",
            "          (attentions): ModuleList(\n",
            "            (0): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (1): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (2): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (3): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (4): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (5): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (6): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (7): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "          )\n",
            "          (o_w): Linear(in_features=512, out_features=512, bias=False)\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (position_wise_ffn): PositionWiseFeedForward(\n",
            "          (conv1): Conv1d(512, 2048, kernel_size=(1,), stride=(1,))\n",
            "          (conv2): Conv1d(2048, 512, kernel_size=(1,), stride=(1,))\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "      (2): DecoderLayer(\n",
            "        (layer_norm): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
            "        (self_attention): MultiHeadAttention(\n",
            "          (attentions): ModuleList(\n",
            "            (0): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (1): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (2): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (3): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (4): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (5): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (6): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (7): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "          )\n",
            "          (o_w): Linear(in_features=512, out_features=512, bias=False)\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (encoder_attention): MultiHeadAttention(\n",
            "          (attentions): ModuleList(\n",
            "            (0): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (1): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (2): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (3): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (4): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (5): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (6): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (7): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "          )\n",
            "          (o_w): Linear(in_features=512, out_features=512, bias=False)\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (position_wise_ffn): PositionWiseFeedForward(\n",
            "          (conv1): Conv1d(512, 2048, kernel_size=(1,), stride=(1,))\n",
            "          (conv2): Conv1d(2048, 512, kernel_size=(1,), stride=(1,))\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "      (3): DecoderLayer(\n",
            "        (layer_norm): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
            "        (self_attention): MultiHeadAttention(\n",
            "          (attentions): ModuleList(\n",
            "            (0): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (1): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (2): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (3): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (4): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (5): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (6): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (7): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "          )\n",
            "          (o_w): Linear(in_features=512, out_features=512, bias=False)\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (encoder_attention): MultiHeadAttention(\n",
            "          (attentions): ModuleList(\n",
            "            (0): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (1): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (2): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (3): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (4): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (5): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (6): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (7): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "          )\n",
            "          (o_w): Linear(in_features=512, out_features=512, bias=False)\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (position_wise_ffn): PositionWiseFeedForward(\n",
            "          (conv1): Conv1d(512, 2048, kernel_size=(1,), stride=(1,))\n",
            "          (conv2): Conv1d(2048, 512, kernel_size=(1,), stride=(1,))\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "      (4): DecoderLayer(\n",
            "        (layer_norm): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
            "        (self_attention): MultiHeadAttention(\n",
            "          (attentions): ModuleList(\n",
            "            (0): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (1): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (2): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (3): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (4): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (5): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (6): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (7): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "          )\n",
            "          (o_w): Linear(in_features=512, out_features=512, bias=False)\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (encoder_attention): MultiHeadAttention(\n",
            "          (attentions): ModuleList(\n",
            "            (0): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (1): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (2): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (3): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (4): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (5): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (6): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (7): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "          )\n",
            "          (o_w): Linear(in_features=512, out_features=512, bias=False)\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (position_wise_ffn): PositionWiseFeedForward(\n",
            "          (conv1): Conv1d(512, 2048, kernel_size=(1,), stride=(1,))\n",
            "          (conv2): Conv1d(2048, 512, kernel_size=(1,), stride=(1,))\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "      (5): DecoderLayer(\n",
            "        (layer_norm): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
            "        (self_attention): MultiHeadAttention(\n",
            "          (attentions): ModuleList(\n",
            "            (0): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (1): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (2): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (3): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (4): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (5): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (6): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (7): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "          )\n",
            "          (o_w): Linear(in_features=512, out_features=512, bias=False)\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (encoder_attention): MultiHeadAttention(\n",
            "          (attentions): ModuleList(\n",
            "            (0): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (1): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (2): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (3): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (4): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (5): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (6): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "            (7): SelfAttention(\n",
            "              (q_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (k_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (v_w): Linear(in_features=512, out_features=64, bias=False)\n",
            "              (dropout): Dropout(p=0.1, inplace=False)\n",
            "            )\n",
            "          )\n",
            "          (o_w): Linear(in_features=512, out_features=512, bias=False)\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "        (position_wise_ffn): PositionWiseFeedForward(\n",
            "          (conv1): Conv1d(512, 2048, kernel_size=(1,), stride=(1,))\n",
            "          (conv2): Conv1d(2048, 512, kernel_size=(1,), stride=(1,))\n",
            "          (dropout): Dropout(p=0.1, inplace=False)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "    (dropout): Dropout(p=0.1, inplace=False)\n",
            "    (layer_norm): LayerNorm((512,), eps=1e-06, elementwise_affine=True)\n",
            "  )\n",
            ")\n",
            "The model has 82,253,312 trainable parameters\n",
            "Epoch: 01 | Epoch Time: 3m 38s\n",
            "\tTrain Loss: 5.563 | Val. Loss: 5.031\n",
            "Epoch: 02 | Epoch Time: 3m 39s\n",
            "\tTrain Loss: 4.899 | Val. Loss: 4.824\n",
            "Epoch: 03 | Epoch Time: 3m 41s\n",
            "\tTrain Loss: 4.728 | Val. Loss: 4.738\n",
            "Epoch: 04 | Epoch Time: 3m 40s\n",
            "\tTrain Loss: 4.629 | Val. Loss: 4.676\n",
            "Epoch: 05 | Epoch Time: 3m 39s\n",
            "\tTrain Loss: 4.552 | Val. Loss: 4.618\n",
            "Epoch: 06 | Epoch Time: 3m 39s\n",
            "\tTrain Loss: 4.481 | Val. Loss: 4.586\n",
            "Epoch: 07 | Epoch Time: 3m 42s\n",
            "\tTrain Loss: 4.424 | Val. Loss: 4.561\n",
            "Epoch: 08 | Epoch Time: 3m 41s\n",
            "\tTrain Loss: 4.378 | Val. Loss: 4.494\n",
            "Epoch: 09 | Epoch Time: 3m 40s\n",
            "\tTrain Loss: 4.339 | Val. Loss: 4.504\n",
            "Epoch: 10 | Epoch Time: 3m 40s\n",
            "\tTrain Loss: 4.294 | Val. Loss: 4.448\n",
            "Epoch: 11 | Epoch Time: 3m 42s\n",
            "\tTrain Loss: 4.259 | Val. Loss: 4.447\n",
            "Epoch: 12 | Epoch Time: 3m 42s\n",
            "\tTrain Loss: 4.224 | Val. Loss: 4.399\n",
            "Epoch: 13 | Epoch Time: 3m 41s\n",
            "\tTrain Loss: 4.188 | Val. Loss: 4.396\n",
            "Epoch: 14 | Epoch Time: 3m 42s\n",
            "\tTrain Loss: 4.166 | Val. Loss: 4.391\n",
            "Epoch: 15 | Epoch Time: 3m 42s\n",
            "\tTrain Loss: 4.138 | Val. Loss: 4.370\n",
            "Epoch: 16 | Epoch Time: 3m 41s\n",
            "\tTrain Loss: 4.110 | Val. Loss: 4.368\n",
            "Epoch: 17 | Epoch Time: 3m 41s\n",
            "\tTrain Loss: 4.085 | Val. Loss: 4.343\n",
            "Epoch: 18 | Epoch Time: 3m 42s\n",
            "\tTrain Loss: 4.060 | Val. Loss: 4.360\n",
            "Epoch: 19 | Epoch Time: 3m 42s\n",
            "\tTrain Loss: 4.033 | Val. Loss: 4.313\n",
            "Epoch: 20 | Epoch Time: 3m 43s\n",
            "\tTrain Loss: 4.010 | Val. Loss: 4.287\n",
            "Epoch: 21 | Epoch Time: 3m 42s\n",
            "\tTrain Loss: 3.988 | Val. Loss: 4.285\n",
            "Epoch: 22 | Epoch Time: 3m 42s\n",
            "\tTrain Loss: 3.968 | Val. Loss: 4.265\n",
            "Epoch: 23 | Epoch Time: 3m 44s\n",
            "\tTrain Loss: 3.946 | Val. Loss: 4.275\n",
            "Epoch: 24 | Epoch Time: 3m 44s\n",
            "\tTrain Loss: 3.930 | Val. Loss: 4.280\n",
            "Epoch: 25 | Epoch Time: 3m 44s\n",
            "\tTrain Loss: 3.911 | Val. Loss: 4.259\n",
            "Epoch: 26 | Epoch Time: 3m 44s\n",
            "\tTrain Loss: 3.886 | Val. Loss: 4.238\n",
            "Epoch: 27 | Epoch Time: 3m 45s\n",
            "\tTrain Loss: 3.868 | Val. Loss: 4.243\n",
            "Epoch: 28 | Epoch Time: 3m 45s\n",
            "\tTrain Loss: 3.853 | Val. Loss: 4.213\n",
            "Epoch: 29 | Epoch Time: 3m 45s\n",
            "\tTrain Loss: 3.833 | Val. Loss: 4.217\n",
            "Epoch: 30 | Epoch Time: 3m 44s\n",
            "\tTrain Loss: 3.815 | Val. Loss: 4.205\n",
            "Epoch: 31 | Epoch Time: 3m 46s\n",
            "\tTrain Loss: 3.796 | Val. Loss: 4.191\n",
            "Epoch: 32 | Epoch Time: 3m 44s\n",
            "\tTrain Loss: 3.780 | Val. Loss: 4.186\n",
            "Epoch: 33 | Epoch Time: 3m 42s\n",
            "\tTrain Loss: 3.763 | Val. Loss: 4.177\n",
            "Epoch: 34 | Epoch Time: 3m 43s\n",
            "\tTrain Loss: 3.744 | Val. Loss: 4.161\n",
            "Epoch: 35 | Epoch Time: 3m 43s\n",
            "\tTrain Loss: 3.733 | Val. Loss: 4.167\n",
            "Epoch: 36 | Epoch Time: 3m 43s\n",
            "\tTrain Loss: 3.713 | Val. Loss: 4.162\n",
            "Epoch: 37 | Epoch Time: 3m 43s\n",
            "\tTrain Loss: 3.697 | Val. Loss: 4.146\n",
            "Epoch: 38 | Epoch Time: 3m 42s\n",
            "\tTrain Loss: 3.684 | Val. Loss: 4.155\n",
            "Epoch: 39 | Epoch Time: 3m 42s\n",
            "\tTrain Loss: 3.670 | Val. Loss: 4.133\n",
            "Epoch: 40 | Epoch Time: 3m 44s\n",
            "\tTrain Loss: 3.658 | Val. Loss: 4.151\n",
            "Epoch: 41 | Epoch Time: 3m 43s\n",
            "\tTrain Loss: 3.643 | Val. Loss: 4.142\n",
            "Epoch: 42 | Epoch Time: 3m 43s\n",
            "\tTrain Loss: 3.628 | Val. Loss: 4.110\n",
            "Epoch: 43 | Epoch Time: 3m 43s\n",
            "\tTrain Loss: 3.611 | Val. Loss: 4.115\n",
            "Epoch: 44 | Epoch Time: 3m 42s\n",
            "\tTrain Loss: 3.593 | Val. Loss: 4.117\n",
            "Epoch: 45 | Epoch Time: 3m 43s\n",
            "\tTrain Loss: 3.582 | Val. Loss: 4.126\n",
            "Epoch: 46 | Epoch Time: 3m 42s\n",
            "\tTrain Loss: 3.565 | Val. Loss: 4.114\n",
            "Epoch: 47 | Epoch Time: 3m 44s\n",
            "\tTrain Loss: 3.553 | Val. Loss: 4.104\n",
            "Epoch: 48 | Epoch Time: 3m 42s\n",
            "\tTrain Loss: 3.534 | Val. Loss: 4.112\n",
            "Epoch: 49 | Epoch Time: 3m 42s\n",
            "\tTrain Loss: 3.518 | Val. Loss: 4.094\n",
            "Epoch: 50 | Epoch Time: 3m 42s\n",
            "\tTrain Loss: 3.513 | Val. Loss: 4.091\n",
            "Epoch: 51 | Epoch Time: 3m 43s\n",
            "\tTrain Loss: 3.495 | Val. Loss: 4.090\n",
            "Epoch: 52 | Epoch Time: 3m 42s\n",
            "\tTrain Loss: 3.481 | Val. Loss: 4.082\n",
            "Epoch: 53 | Epoch Time: 3m 42s\n",
            "\tTrain Loss: 3.471 | Val. Loss: 4.076\n",
            "Epoch: 54 | Epoch Time: 3m 41s\n",
            "\tTrain Loss: 3.457 | Val. Loss: 4.095\n",
            "Epoch: 55 | Epoch Time: 3m 41s\n",
            "\tTrain Loss: 3.444 | Val. Loss: 4.077\n",
            "Epoch: 56 | Epoch Time: 3m 41s\n",
            "\tTrain Loss: 3.435 | Val. Loss: 4.059\n",
            "Epoch: 57 | Epoch Time: 3m 41s\n",
            "\tTrain Loss: 3.422 | Val. Loss: 4.064\n",
            "Epoch: 58 | Epoch Time: 3m 40s\n",
            "\tTrain Loss: 3.409 | Val. Loss: 4.051\n",
            "Epoch: 59 | Epoch Time: 3m 41s\n",
            "\tTrain Loss: 3.392 | Val. Loss: 4.053\n",
            "Epoch: 60 | Epoch Time: 3m 40s\n",
            "\tTrain Loss: 3.380 | Val. Loss: 4.060\n",
            "Epoch: 61 | Epoch Time: 3m 38s\n",
            "\tTrain Loss: 3.374 | Val. Loss: 4.050\n",
            "Epoch: 62 | Epoch Time: 3m 40s\n",
            "\tTrain Loss: 3.361 | Val. Loss: 4.039\n",
            "Epoch: 63 | Epoch Time: 3m 40s\n",
            "\tTrain Loss: 3.351 | Val. Loss: 4.044\n",
            "Epoch: 64 | Epoch Time: 3m 41s\n",
            "\tTrain Loss: 3.340 | Val. Loss: 4.034\n",
            "Epoch: 65 | Epoch Time: 3m 41s\n",
            "\tTrain Loss: 3.330 | Val. Loss: 4.037\n",
            "Epoch: 66 | Epoch Time: 3m 40s\n",
            "\tTrain Loss: 3.315 | Val. Loss: 4.030\n",
            "Epoch: 67 | Epoch Time: 3m 44s\n",
            "\tTrain Loss: 3.300 | Val. Loss: 4.027\n",
            "Epoch: 68 | Epoch Time: 3m 45s\n",
            "\tTrain Loss: 3.291 | Val. Loss: 4.041\n",
            "Epoch: 69 | Epoch Time: 3m 42s\n",
            "\tTrain Loss: 3.282 | Val. Loss: 4.019\n",
            "Epoch: 70 | Epoch Time: 3m 42s\n",
            "\tTrain Loss: 3.269 | Val. Loss: 4.009\n",
            "Epoch: 71 | Epoch Time: 3m 44s\n",
            "\tTrain Loss: 3.258 | Val. Loss: 4.036\n",
            "Epoch: 72 | Epoch Time: 3m 43s\n",
            "\tTrain Loss: 3.247 | Val. Loss: 4.016\n",
            "Epoch: 73 | Epoch Time: 3m 42s\n",
            "\tTrain Loss: 3.237 | Val. Loss: 3.993\n",
            "Epoch: 74 | Epoch Time: 3m 39s\n",
            "\tTrain Loss: 3.225 | Val. Loss: 4.029\n",
            "Epoch: 75 | Epoch Time: 3m 41s\n",
            "\tTrain Loss: 3.211 | Val. Loss: 3.988\n",
            "Epoch: 76 | Epoch Time: 3m 41s\n",
            "\tTrain Loss: 3.203 | Val. Loss: 3.983\n",
            "Epoch: 77 | Epoch Time: 3m 41s\n",
            "\tTrain Loss: 3.192 | Val. Loss: 3.995\n",
            "Epoch: 78 | Epoch Time: 3m 41s\n",
            "\tTrain Loss: 3.180 | Val. Loss: 3.992\n",
            "Epoch: 79 | Epoch Time: 3m 41s\n",
            "\tTrain Loss: 3.171 | Val. Loss: 3.981\n",
            "Epoch: 80 | Epoch Time: 3m 43s\n",
            "\tTrain Loss: 3.162 | Val. Loss: 3.983\n",
            "Epoch: 81 | Epoch Time: 3m 42s\n",
            "\tTrain Loss: 3.148 | Val. Loss: 3.988\n",
            "Epoch: 82 | Epoch Time: 3m 41s\n",
            "\tTrain Loss: 3.140 | Val. Loss: 3.976\n",
            "Epoch: 83 | Epoch Time: 3m 42s\n",
            "\tTrain Loss: 3.131 | Val. Loss: 3.966\n",
            "Epoch: 84 | Epoch Time: 3m 42s\n",
            "\tTrain Loss: 3.121 | Val. Loss: 3.968\n",
            "Epoch: 85 | Epoch Time: 3m 41s\n",
            "\tTrain Loss: 3.114 | Val. Loss: 3.964\n",
            "Epoch: 86 | Epoch Time: 3m 42s\n",
            "\tTrain Loss: 3.103 | Val. Loss: 3.963\n",
            "Epoch: 87 | Epoch Time: 3m 41s\n",
            "\tTrain Loss: 3.090 | Val. Loss: 3.942\n",
            "Epoch: 88 | Epoch Time: 3m 41s\n",
            "\tTrain Loss: 3.083 | Val. Loss: 3.968\n",
            "Epoch: 89 | Epoch Time: 3m 42s\n",
            "\tTrain Loss: 3.074 | Val. Loss: 3.948\n",
            "Epoch: 90 | Epoch Time: 3m 40s\n",
            "\tTrain Loss: 3.062 | Val. Loss: 3.934\n",
            "Epoch: 91 | Epoch Time: 3m 40s\n",
            "\tTrain Loss: 3.053 | Val. Loss: 3.940\n",
            "Epoch: 92 | Epoch Time: 3m 41s\n",
            "\tTrain Loss: 3.040 | Val. Loss: 3.940\n",
            "Epoch: 93 | Epoch Time: 3m 40s\n",
            "\tTrain Loss: 3.033 | Val. Loss: 3.950\n",
            "Epoch: 94 | Epoch Time: 3m 39s\n",
            "\tTrain Loss: 3.029 | Val. Loss: 3.949\n",
            "Epoch: 95 | Epoch Time: 3m 41s\n",
            "\tTrain Loss: 3.020 | Val. Loss: 3.943\n",
            "Epoch: 96 | Epoch Time: 3m 40s\n",
            "\tTrain Loss: 3.008 | Val. Loss: 3.941\n",
            "Epoch: 97 | Epoch Time: 3m 40s\n",
            "\tTrain Loss: 2.998 | Val. Loss: 3.933\n",
            "Epoch: 98 | Epoch Time: 3m 41s\n",
            "\tTrain Loss: 2.987 | Val. Loss: 3.936\n",
            "Epoch: 99 | Epoch Time: 3m 39s\n",
            "\tTrain Loss: 2.978 | Val. Loss: 3.934\n",
            "Epoch: 100 | Epoch Time: 3m 40s\n",
            "\tTrain Loss: 2.968 | Val. Loss: 3.925\n",
            "time: 6h 13min 14s (started: 2021-04-24 10:14:06 +00:00)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vdHshEw2kMM9"
      },
      "source": [
        "# 이미 베스트 모델만 저장됨\n",
        "# 구글 드라이브에 학습된 모델 업로드\n",
        "path = \"/content/drive/MyDrive/model_100.pt\"\n",
        "torch.save(model.state_dict(), path)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ajZi05_BlpWc"
      },
      "source": [
        "# 학습된 모델 로드 - main.py 돌릴 필요 없음\n",
        "path = \"/content/drive/MyDrive/pytorchtr/model.pt\"\n",
        "model.load_state_dict(torch.load(path))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "juHa8uqfw1ke"
      },
      "source": [
        "- predict py의 model 경로를 바꾸면 된다."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lUpmq5g24yJp",
        "outputId": "9878a04c-375e-4149-eac8-f7e069c7efcf"
      },
      "source": [
        "!python predict.py --input \"너 참 예쁘다\""
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "kor> 너 참 예쁘다\n",
            "eng> I hope you are a great person\n",
            "<Figure size 1000x1000 with 1 Axes>\n",
            "time: 11.8 s (started: 2021-04-24 16:46:05 +00:00)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 230
        },
        "id": "TYJFvI6DUlTJ",
        "outputId": "7a870cf3-154f-454b-e694-9f54032596f6"
      },
      "source": [
        "weights = '/content/pytorch-transformer-kor-eng/model.pt'\n",
        "\n",
        "model.save_weights('content/drive/MyDrive/')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-10-43a1c0528117>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mweights\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m'/content/pytorch-transformer-kor-eng/model.pt'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave_weights\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'content/drive/MyDrive/'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m: name 'model' is not defined"
          ]
        },
        {
          "output_type": "stream",
          "text": [
            "time: 69.7 ms (started: 2021-04-24 16:27:32 +00:00)\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}