lab7.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "I1lc3xnYPSlw"
      },
      "source": [
        "First we import libraries and set the random seed:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "pJXe-HNwOvWq"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "<torch._C.Generator at 0x1ef003e88b0>"
            ]
          },
          "execution_count": 2,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "from torchvision.datasets.utils import download_url\n",
        "from torch.utils.data import Dataset, DataLoader, random_split\n",
        "from torch.nn.utils.rnn import pad_sequence\n",
        "from torch import nn\n",
        "from torch.nn import functional as F\n",
        "from torch import optim\n",
        "from tqdm.autonotebook import tqdm\n",
        "from textwrap import wrap\n",
        "import random\n",
        "import sys\n",
        "import io\n",
        "import random\n",
        "import numpy as np\n",
        "import torch\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "torch.manual_seed(0)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zvgBTudyPamX"
      },
      "source": [
        "The following block defines a dataset object which parses our data file and performs numericalisation of the sequence data:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "id": "obU0x4DzO2ce"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Using downloaded and verified file: .\\dataset.txt\n"
          ]
        }
      ],
      "source": [
        "class CodeDataset(Dataset):\n",
        "    def __init__(self):\n",
        "        download_url('https://github.com/ecs-vlc/COMP6248/raw/master/exercises/lab7/dataset.txt', '.', 'dataset.txt', None)\n",
        "        with io.open('dataset.txt', 'r') as f:\n",
        "            self.data = f.readlines()\n",
        "\n",
        "        self.PAD='_'\n",
        "        self.SOS='^'\n",
        "        self.EOS='$'\n",
        "        self.PAD_IDX=0\n",
        "\n",
        "        # construct the vocabularies to numericalise the data\n",
        "        self.alphabet = \"*\".join(self.PAD+self.SOS+self.EOS+\"abcdefghijklmnopqrstuvwxyz \").split('*')\n",
        "\n",
        "        self.alphabet_indices = dict((c, i) for i, c in enumerate(self.alphabet))\n",
        "        self.indices_alphabet = dict((i, c) for i, c in enumerate(self.alphabet))\n",
        "\n",
        "        self.morsebet = self.PAD+self.SOS+self.EOS+'.- /'\n",
        "        self.morse_indices = dict((c, i) for i, c in enumerate(self.morsebet))\n",
        "        self.indices_morse = dict((i, c) for i, c in enumerate(self.morsebet))\n",
        "\n",
        "    def encode_alpha(self, inp):\n",
        "        x = torch.zeros(len(inp), dtype=torch.long)\n",
        "        for t, char in enumerate(inp):\n",
        "            x[t] = self.alphabet_indices[char]\n",
        "\n",
        "        return x\n",
        "\n",
        "    def decode_alpha(self, ten, skip_tok=False):\n",
        "        s = ''\n",
        "        ten = ten.view(-1)\n",
        "        for v in ten.view(-1):\n",
        "            if not skip_tok:\n",
        "                s += self.indices_alphabet[v.item()]\n",
        "            elif v>2:\n",
        "                s += self.indices_alphabet[v.item()]\n",
        "        return s\n",
        "\n",
        "    def encode_morse(self, inp):\n",
        "        x = torch.zeros(len(inp), dtype=torch.long)\n",
        "        for t, char in enumerate(inp):\n",
        "            x[t] = self.morse_indices[char]\n",
        "\n",
        "        return x\n",
        "\n",
        "    def decode_morse(self, ten):\n",
        "        s = ''\n",
        "        for v in ten:\n",
        "          s += self.indices_morse[v]\n",
        "        return s\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.data)\n",
        "\n",
        "    def __getitem__(self, i):\n",
        "        inp, out = self.data[i].strip().split('|')\n",
        "        x = self.encode_morse(inp)\n",
        "        y = self.encode_alpha(out[::-1])\n",
        "        return x, y\n",
        "\n",
        "\n",
        "# This will be used to automatically pad all batch items to the same length\n",
        "def pad_collate(batch): \n",
        "    data = [item[0] for item in batch]\n",
        "    data = pad_sequence(data)\n",
        "    targets = [item[1] for item in batch]\n",
        "    targets = pad_sequence(targets)\n",
        "    return [data, targets]\n",
        "\n",
        "# Load the data and split randomly into training and val subsets\n",
        "ds = CodeDataset()\n",
        "tr, va = random_split(ds, [len(ds) - len(ds)//3, len(ds)//3])\n",
        "trainloader = DataLoader(tr, batch_size=1024, shuffle=True, collate_fn=pad_collate)\n",
        "valloader = DataLoader(va, batch_size=1024, shuffle=False, collate_fn=pad_collate)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "__Fs_h2PPqWa"
      },
      "source": [
        "We next define the model architecture, which is made up of an encoder, a decoder and the Seq2Seq class which ties everything together:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "id": "0JxbT3pFO9Em"
      },
      "outputs": [],
      "source": [
        "class Encoder(nn.Module):\n",
        "    def __init__(self, input_dim, emb_dim, hid_dim):\n",
        "        super().__init__()\n",
        "        \n",
        "        self.hid_dim = hid_dim\n",
        "        self.embedding = nn.Embedding(input_dim, emb_dim)\n",
        "        self.rnn = nn.LSTM(emb_dim, hid_dim)\n",
        "        \n",
        "    def forward(self, src):\n",
        "        # TODO\n",
        "        out = self.embedding(src)\n",
        "        _, hidden = self.rnn(out)\n",
        "        return hidden\n",
        "\n",
        "\n",
        "class Decoder(nn.Module):\n",
        "    def __init__(self, output_dim, emb_dim, hid_dim):\n",
        "        super().__init__()\n",
        "        \n",
        "        self.output_dim = output_dim\n",
        "        self.hid_dim = hid_dim\n",
        "        self.embedding = nn.Embedding(output_dim, emb_dim)\n",
        "        self.rnn = nn.LSTM(emb_dim, hid_dim)\n",
        "        self.fc_out = nn.Linear(hid_dim, output_dim)\n",
        "        \n",
        "    def forward(self, input, hidden, cell):\n",
        "        input = input.unsqueeze(0)\n",
        "        embedded = self.embedding(input)\n",
        "        output, (hidden, cell) = self.rnn(embedded, (hidden, cell))        \n",
        "        prediction = self.fc_out(output.squeeze(0))\n",
        "        return prediction, hidden, cell\n",
        "\n",
        "\n",
        "class Seq2Seq(nn.Module):\n",
        "    def __init__(self, encoder, decoder):\n",
        "        super().__init__()\n",
        "        \n",
        "        self.encoder = encoder\n",
        "        self.decoder = decoder\n",
        "        \n",
        "    def forward(self, src, trg=None, teacher_forcing_ratio = 0.5, maxlen=5, padding_idx=0):\n",
        "        batch_size = src.shape[1]\n",
        "        trg_len = trg.shape[0] if trg is not None else maxlen\n",
        "        trg_vocab_size = self.decoder.output_dim\n",
        "\n",
        "        outputs = torch.zeros(trg_len, batch_size, trg_vocab_size).to(src.device)\n",
        "        hidden, cell = self.encoder(src)\n",
        "\n",
        "        input = torch.ones(batch_size, dtype=torch.long, device=src.device) * padding_idx\n",
        "        for t in range(1, trg_len):\n",
        "            output, hidden, cell = self.decoder(input, hidden, cell)\n",
        "            outputs[t] = output\n",
        "            \n",
        "            teacher_force = random.random() < teacher_forcing_ratio\n",
        "            top1 = output.argmax(1) \n",
        "            input = trg[t] if teacher_force and trg is not None else top1\n",
        "        \n",
        "        return outputs\n",
        "\n",
        "\n",
        "INPUT_DIM = len(ds.morsebet)\n",
        "OUTPUT_DIM = len(ds.alphabet)\n",
        "ENC_EMB_DIM = 128\n",
        "DEC_EMB_DIM = 128\n",
        "HID_DIM = 256\n",
        "\n",
        "enc = Encoder(INPUT_DIM, ENC_EMB_DIM, HID_DIM)\n",
        "dec = Decoder(OUTPUT_DIM, DEC_EMB_DIM, HID_DIM)\n",
        "model = Seq2Seq(enc, dec).cuda()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iY4Y5DPVP7Ih"
      },
      "source": [
        "Finally, we can train and evaluate the model:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "id": "qAkkOs_tPC68"
      },
      "outputs": [
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "a94f2ad092924548884b622a0875c305",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "train:   0%|          | 0/163 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "a9f6a4bd406044c3ada515299cf734c5",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "val:   0%|          | 0/82 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "4dde1e620cd24341844ffd4532338d41",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "train:   0%|          | 0/163 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "a8e9ce5d7f454edc838516ef3f42057e",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "val:   0%|          | 0/82 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "93042c2dbbad43558ac94673d3b00d57",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "train:   0%|          | 0/163 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "1cdec67f27754952ae3a557d9a3a847b",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "val:   0%|          | 0/82 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "c7258e2b7385405bac5c2220100be00c",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "train:   0%|          | 0/163 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "c26a84405ec74e02a4c29790a8c75850",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "val:   0%|          | 0/82 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "2c8d22dfc87945ada6b20fc04b7802d6",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "train:   0%|          | 0/163 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "3f887aa6daa04b349e4ee834e4fc9133",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "val:   0%|          | 0/82 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "7792ddf34ca64c68a0f5e66a5fac29cb",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "train:   0%|          | 0/163 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "143254afeb2b4456a463e0f89c71359e",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "val:   0%|          | 0/82 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "ab8231c4830e4b09a3e2723affed8890",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "train:   0%|          | 0/163 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "80333c8620154d7a9ff7cc667b8c63fd",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "val:   0%|          | 0/82 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "ea76a1eb5aa34a08a556c2b09b74568e",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "train:   0%|          | 0/163 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "e70a978b330043df8b0f0fd8585726aa",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "val:   0%|          | 0/82 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "31c2a4b6aaff4146924076d6f3f8180e",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "train:   0%|          | 0/163 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "cfee829897a247d197ded6cd2172a5f6",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "val:   0%|          | 0/82 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "3d3d2d3ce2e844948e77442e2434aebb",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "train:   0%|          | 0/163 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "fbfd249a85014ea69940576f83242c39",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "val:   0%|          | 0/82 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "The PostScript backend does not support transparency; partially transparent artists will be rendered opaque.\n"
          ]
        },
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAu6ElEQVR4nO3deXhUVbb38e+qSmVOIBCGhEAAAYGEQEIIQURAUEFURFFRwKG1FbSvQ9u2tn1bHLrv9e3rtWnndvYqCoqCA4jdKiraMg8hCAoiSCBAGBISMlft949TRIYkVUAlNWR9nqeeFOfsc2pR6K9Odu2ztxhjUEopFfxs/i5AKaWUb2igK6VUiNBAV0qpEKGBrpRSIUIDXSmlQkSYv144MTHRdO3a1V8vr5RSQWnVqlX7jDHt6tvnt0Dv2rUrK1eu9NfLK6VUUBKR7Q3t0y4XpZQKERroSikVIjTQlVIqRPitD10p1fxqamooKCigsrLS36UoDyIjI0lJScHhcHh9jAa6Ui1IQUEBcXFxdO3aFRHxdzmqAcYY9u/fT0FBAd26dfP6OO1yUaoFqayspG3bthrmAU5EaNu27Un/JqWBrlQLo2EeHE7l3ynoAr3gYDkPfbiBGqfL36UopVRACbpA31hYyivfbOPlr3/ydylKqZNUXFzMM888c0rHXnjhhRQXFzfa5oEHHuDTTz89pfMfr2vXruzbt88n52ouQRfo5/XtwHl9OzDz080UHCz3dzlKqZPQWKA7nc5Gj124cCGtW7dutM3DDz/M6NGjT7W8oBd0gQ7w4CVpiMCM9zegKy4pFTzuu+8+fvzxRwYMGMA999zDF198wciRI7nmmmvo168fAJdeeikDBw4kLS2N559/vu7YI1fM27Zto0+fPvz6178mLS2N888/n4qKCgCuv/565s6dW9d+xowZZGVl0a9fPzZt2gRAUVER5513HllZWdxyyy2kpqZ6vBJ//PHHSU9PJz09nZkzZwJw+PBhxo0bR//+/UlPT2fOnDl1f8e+ffuSkZHB7373O5++f54E5bDFTq2juGt0L/6ycCOfbNjDmPSO/i5JqaDz0Icb+G7XIZ+es29yPDMuTmtw/6OPPkp+fj5r164F4IsvvmD58uXk5+fXDc97+eWXadOmDRUVFQwaNIjLL7+ctm3bHnOezZs389Zbb/HCCy9w5ZVX8u677zJlypQTXi8xMZHVq1fzzDPP8Nhjj/Hiiy/y0EMPce655/KHP/yBRYsWHfOhUZ9Vq1bxyiuvsGzZMowxDB48mOHDh7N161aSk5NZsGABACUlJRw4cIB58+axadMmRMRjF5GvBeUVOsANQ7vSJymehz7cQFlVrb/LUUqdopycnGPGWj/xxBP079+f3NxcduzYwebNm084plu3bgwYMACAgQMHsm3btnrPfdlll53Q5uuvv2bSpEkAjBkzhoSEhEbr+/rrr5kwYQIxMTHExsZy2WWXsWTJEvr168enn37Kvffey5IlS2jVqhXx8fFERkZy00038d577xEdHX2S78bpCcordIAwu42/TEjn8mf/zd/+9QN/uqivv0tSKqg0diXdnGJiYuqef/HFF3z66ad8++23REdHM2LEiHrHYkdERNQ9t9vtdV0uDbWz2+3U1loXfifbTdtQ+169erFq1SoWLlzIH/7wB84//3weeOABli9fzmeffcbs2bN56qmn+Pzzz0/q9U5H0F6hA2R1SeCanC688s1P5O8s8Xc5SikP4uLiKC0tbXB/SUkJCQkJREdHs2nTJpYuXerzGs4++2zefvttAP75z39y8ODBRtufc845zJ8/n/Lycg4fPsy8efMYNmwYu3btIjo6milTpvC73/2O1atXU1ZWRklJCRdeeCEzZ86s61pqLkF7hX7E7y/ozScbdvPH+fm8N/0s7Da9aUKpQNW2bVuGDh1Keno6Y8eOZdy4ccfsHzNmDM899xwZGRmceeaZ5Obm+ryGGTNmcPXVVzNnzhyGDx9OUlIScXFxDbbPysri+uuvJycnB4CbbrqJzMxMPvnkE+655x5sNhsOh4Nnn32W0tJSxo8fT2VlJcYY/va3v/m8/saIv0aJZGdnG18tcPH+2p3cMXstj1yaztTcVJ+cU6lQtHHjRvr06ePvMvyqqqoKu91OWFgY3377LdOnT2/2K2lv1ffvJSKrjDHZ9bUP+it0gEv6J/POygL+umgTF6R1oH1cpL9LUkoFqJ9//pkrr7wSl8tFeHg4L7zwgr9L8pmg7kM/QkR45NJ0qmpd/Pmjjf4uRykVwHr27MmaNWtYt24dK1asYNCgQf4uyWeCL9C3fgnPDYOKY7/I6JYYw60jzuCDdbv46ociPxWnlFL+E3yBHt0WdufB8hN/TZo+4gy6J8bwp/fzqaxp/DZipZQKNcEX6B3TodcYWPoMVJUdsysizM6fL01n+/5ynlm8xU8FKqWUfwRfoAMMu9vqcln16gm7zuqRyITMTjz75Y9s2Vt24rFKKRWigjPQO+dA12Hw7yehtuqE3fdf2Icoh53/nL9eJ+9SKsjFxsYCsGvXLiZOnFhvmxEjRuBpGPTMmTMpL/9lhlZvpuP1xoMPPshjjz122ufxheAMdLCu0st2w9o3T9jVLi6C+8b2YenWA7y3eqcfilNK+VpycnLdTIqn4vhA92Y63mATvIHefQQkZ8E3M8F54uRckwZ1JqtLa/6ycCPF5dXNXp5S6kT33nvvMfOhP/jgg/zv//4vZWVljBo1qm6q2/fff/+EY7dt20Z6ejoAFRUVTJo0iYyMDK666qpj5nKZPn062dnZpKWlMWPGDMCa8GvXrl2MHDmSkSNHAscuYFHf9LiNTdPbkLVr15Kbm0tGRgYTJkyom1bgiSeeqJtS98jEYF9++SUDBgxgwIABZGZmNjolgreC98YiEesqfc5k2PAeZFx5zG6bTfjLhH5c9OTXPPrxJh69PMNPhSoVoD6+D3av9+05O/aDsY82uHvSpEnceeed3HrrrQC8/fbbLFq0iMjISObNm0d8fDz79u0jNzeXSy65pMF1NZ999lmio6PJy8sjLy+PrKysun1/+ctfaNOmDU6nk1GjRpGXl8ftt9/O448/zuLFi0lMTDzmXA1Nj5uQkOD1NL1HXHvttTz55JMMHz6cBx54gIceeoiZM2fy6KOP8tNPPxEREVHXzfPYY4/x9NNPM3ToUMrKyoiMPP0bIoP3Ch3gzAuhXR9Y8ji4TlxjtE9SPDee3Y3ZK3awctsBPxSolDpaZmYme/fuZdeuXaxbt46EhAS6dOmCMYb777+fjIwMRo8ezc6dO9mzZ0+D5/nqq6/qgjUjI4OMjF8u2N5++22ysrLIzMxkw4YNfPfdd43W1ND0uOD9NL1gTSxWXFzM8OHDAbjuuuv46quv6mqcPHkyb7zxBmFh1nX00KFD+e1vf8sTTzxBcXFx3fbTEbxX6AA2Gwz7Lbz3a/h+IfS56IQmd47uyYK8Qv44L5+Pbj8bhz24P8OU8plGrqSb0sSJE5k7dy67d++u636YNWsWRUVFrFq1CofDQdeuXeudNvdo9V29//TTTzz22GOsWLGChIQErr/+eo/naWzghLfT9HqyYMECvvrqKz744AMeeeQRNmzYwH333ce4ceNYuHAhubm5fPrpp/Tu3fuUzn9E8Kdb2mWQ0BWW/C/U8w8THR7Gg5ek8f2eUl7ShaWV8rtJkyYxe/Zs5s6dWzdqpaSkhPbt2+NwOFi8eDHbt29v9BznnHMOs2bNAiA/P5+8vDwADh06RExMDK1atWLPnj18/PHHdcc0NHVvQ9PjnqxWrVqRkJBQd3X/+uuvM3z4cFwuFzt27GDkyJH89a9/pbi4mLKyMn788Uf69evHvffeS3Z2dt0SeacjuK/QAexhMPRO+OhO2PoFnDHyhCbn9e3A+X07MPPTHxjXL4nObZp3FRGl1C/S0tIoLS2lU6dOJCUlATB58mQuvvhisrOzGTBggMcr1enTp3PDDTeQkZHBgAED6qa27d+/P5mZmaSlpdG9e3eGDh1ad8zNN9/M2LFjSUpKYvHixXXbG5oet7HulYa89tprTJs2jfLycrp3784rr7yC0+lkypQplJSUYIzhrrvuonXr1vzpT39i8eLF2O12+vbty9ixY0/69Y4XEtPnUlsFMzMgsSdc/1G9TXYVVzD68S/J7d6Wl67LbvDLFqVCmU6fG1xOdvrc4O9yAQiLgLP+A7YtgR3L622S3DqK357Xi8837eWTDbubuUCllGp6oRHoAAOvh6g2Vl96A64/y1pY+sEPvtOFpZVSISd0Aj0iFnKnww+LGhxbG2a38V8T0tlTWsnj//yhmQtUKjDodBjB4VT+nTwGuoh0FpHFIrJRRDaIyB31tBEReUJEtohInohk1XeuJpfzawiPs8alNyDTvbD0q//WhaVVyxMZGcn+/fs11AOcMYb9+/ef9M1G3oxyqQXuNsasFpE4YJWI/MsYc/Ro/bFAT/djMPCs+2fzikqAQTfCN3+HkX+ExB71Nvv9GPfC0vPW896tQ3VhadVipKSkUFBQQFGRLgIT6CIjI0lJSTmpYzwGujGmECh0Py8VkY1AJ+DoQB8P/J+xPvaXikhrEUlyH9u8htwGy56z5ngZ/1S9TVpFOfjTRX25Y/ZaZi3bzrVDujZriUr5i8PhoFu3bv4uQzWRk+pDF5GuQCaw7LhdnYAdR/25wL3t+ONvFpGVIrKyya4QYttD5lRYNxtKChpsdkn/ZM7ukcj/LPqevYcav5NMKaWCgdeBLiKxwLvAncaYQ8fvrueQEzrpjDHPG2OyjTHZ7dq1O7lKT8bQ262X//eTDTapW1ja6eLhjxqf60EppYKBV4EuIg6sMJ9ljHmvniYFQOej/pwC7Dr98k5R6y6QcRWseg3KGv5NoFtiDLeN6MFHeYV8qQtLK6WCnDejXAR4CdhojGlo+MgHwLXu0S65QIlf+s+PdvZdUFtprT3aiGkjulsLS8/XhaWVUsHNmyv0ocBU4FwRWet+XCgi00RkmrvNQmArsAV4Abi1aco9CYk9oe94WPEiVBQ32CwizM6fJ6Tz84FyntaFpZVSQcybUS5fU38f+dFtDHCbr4rymWG/he/mW6F+zu8abHbWGYlcltmJ5778kfEDkunRPq75alRKKR8JnTtF65PUH3qcZ3W7VJc32vT+cX2IDg/jj/Py9aYLpVRQCu1AB+vKvHw/rH6t0WaJsRHcN7Y3y346wLu6sLRSKgiFfqB3yYXUofDNE1Db+GLRV2V3ZmBqAv+1cCMHD+vC0kqp4BL6gQ5WX3rpLsib3Wgza2HpdEoqanj049NfPUQppZpTywj0M0ZB0gD4+m/gbHza3N4d47np7G7MWbmDFbqwtFIqiLSMQBeBYXfDga3WqBcP7hjdk06to/jjvPVU17qavj6llPKBlhHoAL0vgsQzral1PYxiiQ4P46FL0vhhTxkvfr21mQpUSqnT03IC3Waz+tL3brAWwfBgdN8OXJDWgSc+28yOA40PeVRKqUDQcgIdIP1ya56Xrx7zeJUOMOPiNOwiPPC+jk1XSgW+lhXodgcMvQN2rrQWlPYguXUUd53Xi8XfF7EoXxeWVkoFtpYV6AADpkBsh0YXkz7a9Wd1pW9SPA9+uIHSypomLk4ppU5dywt0RyQM+Q1s/QIKVnlsHma38ZcJ6ewtreLxf+nC0kqpwNXyAh0g+waIbO31VXpmlwQmD+7Ca//epgtLK6UCVssM9Ig4GDwNvl8Ae7xbreieC3rTJiaC++etx+nSL0iVUoGnZQY6wOBbwBEDXze0ZsexrIWl+5BXUMIbS7c3cXFKKXXyWm6gR7eBQb+C/HetO0i9cEn/ZIb1TOR/PvmePbqwtFIqwLTcQAfry1GbA775u1fNRYRHxqdTrQtLK6UCUMsO9LiOkDkF1r4Jh7xb07prYgy/GdmDBXmFfPH93iYuUCmlvNeyAx1g6O3gcsK/n/L6kFuGd6d7uxhmfLCBWqdO3qWUCgwa6Aldod8VsOoVOLzfq0Miwuz8/oLebN9fzmeb9CpdKRUYNNDBmrSrpgKWPev1IaP7tCepVaSOeFFKBQwNdIB2Z0Kfi2DZ81B5yKtDwuw2rsnpwpLN+9haVNbEBSqllGca6EcMuxuqSmDlS14fclVOZ8JswqxlPzdhYUop5R0N9COSM62l6r592up+8UL7uEjGpHfknZU7qKh2NnGBSinVOA30ow27Gw4XwerXvT5kam4qhypr+XCdd8MelVKqqWigHy31LOica91oVFvt1SE53drQq0Ms/7d0my6CoZTyKw30o4nAOb+DQwWw/h0vDxGm5qaSv/MQ6wp0JkallP9ooB+vx2jo2M+atMvlXb/4pZmdiAm36xBGpZRfaaAfT8TqS9+/BTZ+4NUhcZEOJmR14sN1uzh42LuuGqWU8jUN9Pr0uQTa9rQWwPCyX3xKbipVtS7mripo4uKUUqp+Guj1sdnh7Ltg93rY/C+vDundMZ6crm14Y9l2XLoAhlLKDzTQG5JxJbTqDEse8/4qfUgq2/eXs2TLviYuTimlTqSB3hC7A4beATuWwfZ/e3XImLSOJMaG8/q3+uWoUqr5eQx0EXlZRPaKSH4D+0eISImIrHU/HvB9mX6SOQVi2llX6V4ID7Nx1aDOfL5pDwUHy5u4OKWUOpY3V+ivAmM8tFlijBngfjx8+mUFCEcUDLkNfvwcdq726pCrc7oA8NZynd9FKdW8PAa6MeYr4EAz1BKYsm+EyFZeLyadkhDNub07MGfFDqpqdX4XpVTz8VUf+hARWSciH4tIWkONRORmEVkpIiuLiop89NJNLDIecm6BjR/C3k1eHTJ1SCr7yqpZlL+7iYtTSqlf+CLQVwOpxpj+wJPA/IYaGmOeN8ZkG2Oy27Vr54OXbiaDp4EjGr7+m1fNh/VIJLVttN45qpRqVqcd6MaYQ8aYMvfzhYBDRBJPu7JAEtMWsn9lze9ycJvH5jabMGVwKiu2HWRjoXcLZiil1Ok67UAXkY4iIu7nOe5zerc4ZzAZcpt1w9E3T3jVfOLAFCLCbHqVrpRqNt4MW3wL+BY4U0QKRORGEZkmItPcTSYC+SKyDngCmGRCcR7Z+GQYcA2seQNKPfeNJ8SEc3H/ZOav2UlpZU0zFKiUaum8GeVytTEmyRjjMMakGGNeMsY8Z4x5zr3/KWNMmjGmvzEm1xjj3V04wWjoneCqgW+f8qr51NxUDlc7mb9mZ9PWpZRS6J2iJ6dNN0ifCCtehnLPIzn7d25NRkorXl+6XRe/UEo1OQ30k3X2XVBzGJb9w6vmU3JT+WFPGct/arlD+ZVSzUMD/WR16Au9L4Jlz0FVqcfmF2ck0yrKwev65ahSqolpoJ+Ks38LlcWw8hWPTaPC7VwxMIVF+bvZW1rZ9LUppVosDfRTkTIQuo+wvhytrfLYfHJuKrUuw5zlO5q+NqVUi6WBfqqG3gFle2DDPI9NuyXGMKxnIm8u/5lap6sZilNKtUQa6Keq+0hIPBOWPuvVAhhTclMpLKnks017m6E4pVRLpIF+qkRg8C1QuNZaBMODUb3bk9QqUu8cVUo1GQ3009F/kjW17tJnPTYNs9u4JqcLSzbvY2tRWTMUp5RqaTTQT0d4DGRdZ02tW1LgsflVOZ0JswmzluniF0op39NAP105vwYMrHjRY9P2cZGMSe/IOyt3UFGti18opXxLA/10te4CvcfBqleh2vM6olNzUzlUWcuH63Y1fW1KqRZFA90XBk+HioOw/m2PTXO6taFXh1jeWKZfjiqlfEsD3RdSz4KO/WDpcx6HMIoIU3NTySsoYd2O4uapTynVImig+4KIdZVetBF++tJj80szOxETbtf5XZRSPqWB7ivpl0N0olezMMZFOpiQ1YkP1+3i4OHqZihOKdUSaKD7iiPSWnf0+4/hwFaPzafkplJV62LuKs/DHZVSyhsa6L6U/Str3dHlL3hs2rtjPDld2/DGsu24XLr4hVLq9Gmg+1J8EqRNsNYd9WKu9ClDUtm+v5wlW/Y1Q3FKqVCnge5rg6dD1SFY+6bHpmPSOpIYG87r3+qXo0qp06eB7mspAyFlkPXlqKvxqXLDw2xcNagzn2/aQ8FBzzclKaVUYzTQm8LgaXDgR9jyqcemV+d0AeCt5Tq/i1Lq9GigN4W+4yEuCZZ5noUxJSGac3t3YM6KHVTV6vwuSqlTp4HeFOwOGHQj/Pg5FH3vsfnUIansK6tmUf7uZihOKRWqNNCbysAbwB4By57z2HRYj0RS20br4hdKqdOigd5UYhIh4wpYN9uauKsRNpswZXAqK7YdZNPuQ81UoFIq1GigN6XB06GmHFa/7rHpxIEpRITZ9CpdKXXKNNCbUsd06DrMunPUWdto04SYcC7un8y81TspraxppgKVUqFEA72pDb4FSn6G7xd6bDo1N5XD1U7mr9nZDIUppUKNBnpTO/NCa1UjL74c7d+5NRkprXh96XaMh3nVlVLqeBroTc1mh5ybYfs3UJjnsfmU3FR+2FPG8p8ONENxSqlQooHeHDKngiPGq6v0izOSaRXl0MUvlFInTQO9OUS1hgFXw/p3oKyo8abhdq4YmMKi/N3sLa1snvqUUiHBY6CLyMsisldE8hvYLyLyhIhsEZE8EcnyfZkhIOcWcFbDqlc9Np2cm0qtyzBn+Y6mr0spFTK8uUJ/FRjTyP6xQE/342bA8wQmLVG7XnDGKFjxItQ2vuxct8QYhvVM5M3lP1PrbHzGRqWUOsJjoBtjvgIa+4ZuPPB/xrIUaC0iSb4qMKTkToey3fDd+x6bTslNpbCkks827W2GwpRSocAXfeidgKP7Bgrc204gIjeLyEoRWVlU1Hhfckg6YxS07eHVLIyjercnqVWk3jmqlPKaLwJd6tlW7yBqY8zzxphsY0x2u3btfPDSQcZms+ZK37kKdqxotGmY3cY1OV1YsnkfW4vKmqlApVQw80WgFwCdj/pzCrDLB+cNTf2vhoh4r4YwXpXTmTCbMGuZLn6hlPLMF4H+AXCte7RLLlBijCn0wXlDU0SsNS79u/lwqPHPvfZxkYxJ78g7K3dQUa2LXyilGufNsMW3gG+BM0WkQERuFJFpIjLN3WQhsBXYArwA3Npk1YaKnF+DywkrXvLYdGpuKocqa/kwT3/pUUo1LsxTA2PM1R72G+A2n1XUErTpZs3xsuoVOOcecEQ22DSnWxt6dYjljaXbuTK7c4PtlFJK7xT1l9xpUL7funu0ESLC1NxU8gpKWLejuHlqU0oFJQ10f+k6DNqnwbJ/gIeZFS/N7ERMuF3nd1FKNUoD3V9ErLnS96y3ZmJsRFykgwlZnfhw3S4OHm78LlOlVMulge5PGVdCVBtY6vlGoym5qVTVupi7qqAZClNKBSMNdH9yRMHA663VjA423p3Su2M8OV3b8May7bhcuviFUupEGuj+NugmQGD58x6bThmSyvb95SzZsq/p61JKBR0NdH9r1Qn6jofVr0NV47f4j0nrSGJsOK9/q1+OKqVOpIEeCHKnQ1UJ5M1utFl4mI2rBnXm8017KDhY3kzFKaWChQZ6IEgZBMmZ1hBGV+Pzn1+d0wWAt5br/C5KqWNpoAcCERg8Hfb9AFs/b7RpSkI05/buwJwVO6iq1fldlFK/0EAPFGkTILYDLPU8C+PUIansK6tmUf7uZihMKRUsNNADRVg4ZN8IW/4F+zY32nRYj0RS20br4hdKqWNooAeS7BvAHm71pTfCZhOmDE5lxbaDbNp9qJmKU0oFOg30QBLbHtIvh7VvQmVJo00nDkwhIsymV+lKqToa6IFm8DSoOQxr3mi0WUJMOBf3T2be6p2UVtY0U3FKqUCmgR5okgdAlyHuIYyNj2KZmpvK4Won89fsbJ7alFIBTQM9EA2eBsXb4YdFjTbr37k1GSmteO7LrZSU61W6Ui2dBnog6n0RtOrs1SyMMy5OY8+hSu5+Zx3Gw7zqSqnQpoEeiOxh1qRd25bAng2NNh2YmsD9F/bh0417eP6rrc1UoFIqEGmgB6qsayEsCpZ5vtHohqFdGZvekb9+8j3Ltu5vhuKUUoFIAz1QRbeB/ldB3ttwuPGQFhH+OjGDLm2i+Y+31lBUWtVMRSqlAokGeiAbPA1qK2H1qx6bxkU6eGZyFiUVNdz+1hqcugiGUi2OBnoga98Huo+A5S+C0/Molj5J8TxyaTrfbt3P3/71Q9PXp5QKKBrogW7wdCjdBRs/8Kr5ldmduTI7hacWb2Hxpr1NXJxSKpBooAe6nudDQjeP87sc7eHx6fTuGMddb6/VhTCUakE00AOdzQaDb4Edy2Dnaq8OiXTYeXbKQGqdhtveXEN1beOLZiilQoMGejAYMBnC47wawnhEt8QY/mdiBut2FPNfCzc2YXFKqUChgR4MIuMhczLkvwel3i9qMbZfEjee3Y1X/72ND9ftasIClVKBQAM9WOTcDK5aWPnySR1239jeZHVpzX3v5vFjUVkTFaeUCgQa6MGi7RnQ6wIr0Gu9v3HIYbfx1DVZRDjsTH9jFeXVtU1YpFLKnzTQg8ngW+BwkdX1chKSW0cx86oBbN5bxn/Oy9dJvJQKURrowaT7SGjXG5Y9CycZyuf0asft5/bkvTU7mb1iRxMVqJTyJw30YCJiXaUXroOfl5704beP6smwnonM+GAD+TsbX+JOKRV8vAp0ERkjIt+LyBYRua+e/SNEpERE1rofD/i+VAVAxiSIbG1dpZ8ku02YedUA2kSHc+us1ZRU6KIYSoUSj4EuInbgaWAs0Be4WkT61tN0iTFmgPvxsI/rVEeER8PA62DjR1B88l0nbWMjeHpyJruKK7hHF8VQKqR4c4WeA2wxxmw1xlQDs4HxTVuWatSgXwMGVrx4SocPTG3DfWN788/v9vDikp98W5tSym+8CfROwNGXggXubccbIiLrRORjEUmr70QicrOIrBSRlUVFRadQrgKgdWdrmbpVr0L1qc3VcuPZ3RiT1pFHF21ixbYDvq1PKeUX3gS61LPt+N/TVwOpxpj+wJPA/PpOZIx53hiTbYzJbteu3UkVqo6TOx0qiyFvzikdLiL89YoMOidE8Zs3V7OvTBfFUCrYeRPoBUDno/6cAhxzH7kx5pAxpsz9fCHgEJFEn1WpTtRlCHTMsGZhPMV+8PhIB09PzqK4vIY7ZuuiGEoFO28CfQXQU0S6iUg4MAk4ZnJuEekoIuJ+nuM+ry5u2ZRErKv0oo2w9YtTPk1aciseGZ/ON1v28/dPdVEMpYKZx0A3xtQCvwE+ATYCbxtjNojINBGZ5m42EcgXkXXAE8Ako8Mnml765RDTDj7/M5Sfej/4lYM6M3FgCk8u3sIX3+uiGEoFK/FX7mZnZ5uVK1f65bVDSt47MH86xHWEia9A50GndJqKaicTnvmGPYcq+ej2YXRqHeXjQpVSviAiq4wx2fXt0ztFg13GFXDjJ1YXzCtj4NunT6lPPSrczjOTs6hxGm6btVoXxVAqCGmgh4JOA+GWr6DnBfDJ/TBnClQcPOnTdG8Xy18nZrB2RzH//bEuiqFUsNFADxVRCTBpFlzwX/DDIvjHObBz1Umf5sJ+SdwwtCuvfLONBXmFTVCoUqqpaKCHEhEYchvcsMjqdnnpglMa1viHsX3I7NKae9/NY6suiqFU0NBAD0WdB1ldMD1Gwce/h3eug0rvZ1cMD7MWxXDYhVtnraai2tmExSqlfEUDPVRFt4FJb8F5D1sTef1jOOxa6/XhnVpH8berBvD9nlL+9H5+09WplPIZDfRQZrPB0DvghoXWsnUvnWdN6OVlF8yIM9vzHyN7MHdVAW/rohhKBTwN9JagSy5M+xq6nQML7oa5v4KqUq8OvWN0L4b2aMuf3s9nwy5dFEOpQKaB3lLEtIVr3oFRD8B3860umN3rPR5mtwl/n5RJ62gHt85azaFKXRRDqUClgd6S2Gww7G647iOoPgwvjram4PXQBZMYG8FT12RRcLCC37+Tp4tiKBWgNNBboq5DrS6YLkPgwztg3i1Q1fjwxEFd23DfmN4s2rCbl77WRTGUCkQa6C1VbDuY8i6M/COsfwdeGAl7vmv0kJuGdeP8vh149ONNrNqui2IoFWg00Fsymx2G/x6ufR8qiuGFc2HNrAabiwj/c0V/kltHcdusNezXRTGUCiga6Moa/TLta0jJhvdvhfm3Nri0XasoB89MzuJAeTV3zlmri2IoFUA00JUlroN1pT78Xlj7pnW1XvR9vU3TO7Xi4UvSWLJ5H09+vrmZC1VKNUQDXf3CZoeR98PU9+BwETw/AtbNrrfpVYM6c1lWJ/7+2Wa++kEX/FYqEGigqxOdca7VBZOcaY2Aef83UFNxTBMR4c+XptOrfRx3zllLYUlFAydTSjUXDXRVv/gkuPYDa9z6mtfhhVGw79julejwMJ6ZkkVVjZPbZq2mxqmLYijlTxroqmH2MOvO0snvQmmh1QWzfu4xTc5oF8v/m5jB6p+Lueof3/LG0u06+kUpP9E1RZV3SnZac8DsWArZv4IL/hsckXW7X1+6nVe++YmtRYex24Qh3dsyLiOJMWkdSYgJ92PhSoWWxtYU1UBX3nPWwOePwDd/h4794IrXoO0ZdbuNMWzaXcqCvEI+ytvFtv3l2G3C0B6JXNQviQvSOtIq2uHHv4BSwU8DXfnW94usL0tdThj/JKRNOKGJMYYNuw6xYH0hC/IK+flAOQ67cHaPRMZlJHNe3w60itJwV+pkaaAr3yveAXNvgIIVkHMznP9nCIuot6kxhvU7S9xX7oXsLK4g3G7jnF6JjMtIYnSfDsRFargr5Q0NdNU0aqvhs4fg26cgaYAV6skDICKuwUOMMazdUcyCvEIWrC+ksKSS8DAbw3u146KMJEb16UBsRFiz/RWUCjYa6KppbVoA86e71y0VSOxpBXxyphXwHTMgIvaEw1wuwxp3uC9cX8juQ5VEhNkYeWZ7xmUkMapPe6LDNdyVOpoGump6FQdhx3Jr3dLCtbBrjTXUEbBCvpcV7keCvmO/Y0Le5TKs+vlg3ZV7UWkVkQ4bo3p3YFxGEiPPbE9UuL35/15KBRgNdOUfpXt+CfcjQX9CyGf+EvTukHe6DCu2HWBBXiEf5xeyr6ya6HA7o/p0YFy/JEac2Y5Ih4a7apk00FXgKN197FX8rrVQttvaJzYr5I/qrnG2T2dZQSUfrS9kUf5uDhyuJibcznl9OzAuI5lzeiUSEabhrloODXQV2A4VugN+7S9BX7bH2ic2SDzTCveO/VlvuvPuzgQ+3FhMcXkNcRFhnNe3Axf1T+LsHu0ID9Obn1Vo00BXwacu5N1X8bvWwOG91j6xYRJ7sSe2D0sruvDu7kRWVKYQHhnDeX070jc5nvZxEbSPi6BDfCTt4yP0y1UVMjTQVfAzxup/P767xh3yRmzsDk9leVUXfq5NoNjEctDEUkwsxSaW6vDWhMW1JSquDe1bxbgD3wr7Iz87xEfqkEkV8BoLdP2vVwUHEYhPth69L7S21YX8GmTXWpIK13JJYR4c3ouYemZ+LANXmVBWGMMBE0uxieGgiaWQODaZGA6aOA7b4zFRCdhi2uCIbUtUq/bEtk4kIaENHeKjaB8fQbu4SOIjwxCR5n0PlPJAA10Fr2NCfpy1CcDlgqoSayhl+UGoOADlB6DiILaKA8SXHyCu4iCdyvbjPLwfyrdhryrGUVtmnbfK/ThqHexqY6fEfbX/PbEckjiqwlrhjGiNK6oNtpg2RMQlEhmfSGxCe1q1aUdkVCzhkVFERMUQGR5GuN2mHwKqSXkV6CIyBvg7YAdeNMY8etx+ce+/ECgHrjfGrPZxrUp5x2aDqATr0ab+JgI43I86zhrrQ6DioPsDwPogqCrbT3lxEa5D+4gpP0BM+QHsVQeJqNlGdHkJ4eU1sL/xkqqNnTLCqcZBtTiokXBqJJxaCafWFo7TFoHTFo7LFoHLbj0Ii8CERYA9EnEceURhc0RiD4/EHh6FPTySsIgoHOHROCKiCI+MIjwimvCoKCIjYrA5IsDuAFuY9QGoQprHQBcRO/A0cB5QAKwQkQ+MMd8d1Wws0NP9GAw86/6pVPCwOyC2vfU4SoT70aDqcqg4QMWhfZTs30Ppgb1UlO7HVV2Bq6YSU1MJtUceVYizCqmtwuaqwuaswu6qJtJVjr22GIepJsxUE26qCaeGcFNNpNT45K9Xi41awnBipxY7TrHjJAynWNucEoZL7LgkzP3cehibtc1IGC6bA2OzY2xhGHGAPQxjc4DYMXYH2Kxt2BxI3U/rA0VsNsRmB7F+is2GiB2b3Q42GyI2bDa7e9+R/TZs9jD3T7t7vw2x2bC5z2mzh7m3C3ZbmNXObu0/0t5ut2O32xFbGCDW6Ck58rOxhwTVB6E3V+g5wBZjzFYAEZkNjAeODvTxwP8Z6xvWpSLSWkSSjDGFJ55OqRATHg3h0US1SiGqM3T08emdThdVVRVUVVZQVVFOdWU51VUV1FSVU1NVgbO6gtqqCpw1lTirK3HV/PJBYmqrEFcNOGsRUwuuGsT9XFzuh6nF5qrBZpyIqcVuarG53A9ThZ1awo0Tm6kljFpsxml9LBjr48H9cYADJ2HUYhf/DLRoKi4EF4JBcGFzP7cds83UtbH9sk1O3AbgEhuFZ1xJ7uQZPq/Vm0DvBOw46s8FnHj1XV+bTsAxgS4iNwM3A3Tp0uVka1WqRbLbbURHxxAdHePvUhpkjMFloMbposLpxFlTQ01NFa7aGmprqnA5a3E6nRiXC5fLaT2cLoyrFqfLhXFZz10uF8ZZi8sc2ebE5XRijLXdGIMxTsyRcxknuJwYl8GYWvcxLqjb7gLjso53OcE4EWPAuAD3T2MAF2Jc7n1OMMb9xbqrrr1gtbV+WtuPfn7kHEfaWOezXkfML/GPMTjiOzTJv4M3gV7f7xvHfwR70wZjzPPA82ANW/TitZVSQUBEsAvYbXZw2CEyHAjcD6BQ5c1tdQVA56P+nALsOoU2SimlmpA3gb4C6Cki3UQkHJgEfHBcmw+Aa8WSC5Ro/7lSSjUvj10uxphaEfkN8AnWsMWXjTEbRGSae/9zwEKsIYtbsIYt3tB0JSullKqPV+PQjTELsUL76G3PHfXcALf5tjSllFInQ6emU0qpEKGBrpRSIUIDXSmlQoQGulJKhQi/zYcuIkXAdr+8uO8kAvv8XUQA0ffjWPp+/ELfi2OdzvuRaoxpV98OvwV6KBCRlQ1NNN8S6ftxLH0/fqHvxbGa6v3QLhellAoRGuhKKRUiNNBPz/P+LiDA6PtxLH0/fqHvxbGa5P3QPnSllAoReoWulFIhQgNdKaVChAb6KRCRziKyWEQ2isgGEbnD3zX5m4jYRWSNiHzk71r8zb0E41wR2eT+b2SIv2vyJxG5y/3/Sb6IvCUikf6uqTmJyMsisldE8o/a1kZE/iUim90/E3zxWhrop6YWuNsY0wfIBW4Tkb5+rsnf7gA2+ruIAPF3YJExpjfQnxb8vohIJ+B2INsYk441Bfck/1bV7F4Fxhy37T7gM2NMT+Az959Pmwb6KTDGFBpjVrufl2L9D9vJv1X5j4ikAOOAF/1di7+JSDxwDvASgDGm2hhT7Nei/C8MiBKRMCCaFraamTHmK+DAcZvHA6+5n78GXOqL19JAP00i0hXIBJb5uRR/mgn8HnD5uY5A0B0oAl5xd0G9KCItdnFNY8xO4DHgZ6xF40uMMf/0b1UBocORVd3cP9v74qQa6KdBRGKBd4E7jTGH/F2PP4jIRcBeY8wqf9cSIMKALOBZY0wmcBgf/TodjNx9w+OBbkAyECMiU/xbVejSQD9FIuLACvNZxpj3/F2PHw0FLhGRbcBs4FwRecO/JflVAVBgjDnyG9tcrIBvqUYDPxljiowxNcB7wFl+rikQ7BGRJAD3z72+OKkG+ikQEcHqI91ojHnc3/X4kzHmD8aYFGNMV6wvuz43xrTYKzBjzG5gh4ic6d40CvjOjyX5289ArohEu/+/GUUL/pL4KB8A17mfXwe874uTerWmqDrBUGAqsF5E1rq33e9ee1Wp/wBmiUg4sJUWvGi6MWaZiMwFVmONDltDC5sGQETeAkYAiSJSAMwAHgXeFpEbsT70rvDJa+mt/0opFRq0y0UppUKEBrpSSoUIDXSllAoRGuhKKRUiNNCVUipEaKArpVSI0EBXSqkQ8f8BjcxcxraK038AAAAASUVORK5CYII=",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          },
          "output_type": "display_data"
        }
      ],
      "source": [
        "crit = nn.CrossEntropyLoss(ignore_index=ds.PAD_IDX)\n",
        "opt = optim.Adam(model.parameters())\n",
        "train_losses = []\n",
        "val_losses = []\n",
        "\n",
        "for e in range(10):\n",
        "    model.train()\n",
        "\n",
        "    with tqdm(total=len(trainloader), desc='train') as t:\n",
        "        epoch_loss = 0\n",
        "        for i, (x, y) in enumerate(trainloader):\n",
        "            x = x.cuda()\n",
        "            y = y.cuda()\n",
        "\n",
        "            opt.zero_grad()\n",
        "            pred = model(x, y, padding_idx=ds.PAD_IDX)\n",
        "\n",
        "            pred_dim = pred.shape[-1]\n",
        "            pred = pred[1:].view(-1, pred_dim)\n",
        "            y = y[1:].view(-1)\n",
        "\n",
        "            loss = crit(pred, y)\n",
        "            loss.backward()\n",
        "            opt.step()\n",
        "\n",
        "            epoch_loss = (epoch_loss*i + loss.item()) / (i+1)\n",
        "\n",
        "            t.set_postfix(loss='{:05.3f}'.format(epoch_loss))\n",
        "            t.update()\n",
        "        train_losses.append(epoch_loss)\n",
        "\n",
        "    model.eval()\n",
        "    with tqdm(total=len(valloader), desc='val') as t:\n",
        "        with torch.no_grad():\n",
        "            epoch_loss = 0\n",
        "            for i, (x, y) in enumerate(valloader):\n",
        "                x = x.cuda()\n",
        "                y = y.cuda()\n",
        "\n",
        "                pred = model(x, y, teacher_forcing_ratio=0, padding_idx=ds.PAD_IDX)\n",
        "\n",
        "                pred_dim = pred.shape[-1]\n",
        "                pred = pred[1:].view(-1, pred_dim)\n",
        "                y = y[1:].view(-1)\n",
        "\n",
        "                loss = crit(pred, y)\n",
        "                epoch_loss = (epoch_loss*i + loss.item()) / (i+1)\n",
        "\n",
        "                t.set_postfix(loss='{:05.3f}'.format(epoch_loss))\n",
        "                t.update()\n",
        "            val_losses.append(epoch_loss)\n",
        "plt.plot(range(1, 11), train_losses, label='training loss')\n",
        "plt.plot(range(1, 11), val_losses, label='validation loss')\n",
        "plt.legend()\n",
        "plt.savefig('loss.eps')\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 17,
      "metadata": {
        "id": "8EjXSAOUPFfE"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "answer the following\n",
            "why is the order of the output reversed\n",
            "what is the point of teacher forcing\n"
          ]
        }
      ],
      "source": [
        "def decode(code):\n",
        "  out = ''\n",
        "  for chunk in code.split(' '):\n",
        "    num = ds.encode_morse('^ ' + chunk + ' $').unsqueeze(1)\n",
        "    pred = model(num.cuda(), maxlen=2)\n",
        "    pred = pred[1:].view(-1, pred_dim).argmax(-1)\n",
        "    out += ds.decode_alpha(pred.cpu())[::-1]\n",
        "  return out\n",
        "print(decode('.- -. ... .-- . .-. / - .... . / ..-. --- .-.. .-.. --- .-- .. -. --.'))\n",
        "print(decode('.-- .... -.-- / .. ... / - .... . / --- .-. -.. . .-. / --- ..-. / - .... . / --- ..- - .--. ..- - / .-. . ...- . .-. ... . -..'))\n",
        "print(decode('.-- .... .- - / .. ... / - .... . / .--. --- .. -. - / --- ..-. / - . .- -.-. .... . .-. / ..-. --- .-. -.-. .. -. --.'))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": []
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "collapsed_sections": [],
      "name": "labex7-seq2seq.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.9.12"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}