notebooks/cis6930_week6a_recurrent_neural_networks.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "cis6930-week5a-recurrent-neural-networks.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "machine_shape": "hm"
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oRgpqL0KVO43"
      },
      "source": [
        "# CIS6930 Week 6a: Recurrent Neural Networks (Student version)\n",
        "\n",
        "---\n",
        "\n",
        "Preparation: Go to `Runtime > Change runtime type` and choose `GPU` for the hardware accelerator.\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uayE3ioc3k_h"
      },
      "source": [
        "gpu_info = !nvidia-smi -L\n",
        "gpu_info = \"\\n\".join(gpu_info)\n",
        "if gpu_info.find(\"failed\") >= 0:\n",
        "    print(\"Not connected to a GPU\")\n",
        "else:\n",
        "    print(gpu_info)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NoLdSlk0ensC"
      },
      "source": [
        "!pip install pytorch-lightning"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lTahfFKhDOjM"
      },
      "source": [
        "## Whitespace Tokenizer\n",
        "\n",
        "Let's take a look at this \"naive\" whitespace tokenizer class. In addition to `fit()` function that simply assigns token IDs to unseen tokens, it has `encode()` that returns a list of token IDs after tokenization. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "syP-NP-Pv0r2"
      },
      "source": [
        "class WhiteSpaceTokenizer:\n",
        "    \"\"\"Simple tokenizer based on whitespace splitting.\"\"\"\n",
        "    def __init__(self,\n",
        "                 max_size: int = 30000,\n",
        "                 unk_token: str = \"[UNK]\"):\n",
        "        self.token2id = {}\n",
        "        self.id2token = {}\n",
        "        self.max_size = max_size\n",
        "        self.unk_token_id = 1  # 0 is reserved for padding token\n",
        "        self.unk_token = unk_token\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.token2id)\n",
        "\n",
        "    def tokenize(self, text):\n",
        "        \"\"\"Simple whitespace splitting after lower casing.\n",
        "        To use more sophisticated option, you can overwrite this logic.\"\"\"\n",
        "        return text.lower().split(\" \")\n",
        "\n",
        "    def fit(self, text):\n",
        "        # Initialize\n",
        "        self.token2id = {self.unk_token: self.unk_token_id}\n",
        "        for token in self.tokenize(text):\n",
        "            if (len(self.token2id) < self.max_size) and (\n",
        "                            token not in self.token2id):\n",
        "                self.token2id[token] = len(self.token2id)\n",
        "\n",
        "        # id2token is reverse mapping\n",
        "        self.id2token = {int(v): k for k, v in self.token2id.items()}\n",
        "\n",
        "    def encode(self, text):\n",
        "        token_ids = []\n",
        "        for token in self.tokenize(text):\n",
        "            if token in self.token2id:\n",
        "                token_ids.append(self.token2id[token])\n",
        "            else:\n",
        "                token_ids.append(self.unk_token_id)\n",
        "        return token_ids\n",
        "\n",
        "    def decode(self, token_ids):\n",
        "        tokens = []\n",
        "        for token_id in token_ids:\n",
        "            if (token_id > 1) and (token_id not in self.id2token):\n",
        "                # token_id=1 is [UNK]\n",
        "                # Any token_ids should be in the dictionary\n",
        "                print(\"WARNING: token_id={} not found in the vocabulary.\".format(\n",
        "                    token_id))\n",
        "                token = \"N/A\"\n",
        "            else:\n",
        "                token = self.id2token[token_id]\n",
        "            tokens.append(token)\n",
        "        return tokens"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "O7MCQM7VyuEy"
      },
      "source": [
        "text = \"In machine learning, the perceptron is an algorithm for supervised learning of binary classifiers. A binary classifier is a function which can decide whether or not an input, represented by a vector of numbers, belongs to some specific class.[1] It is a type of linear classifier, i.e. a classification algorithm that makes its predictions based on a linear predictor function combining a set of weights with the feature vector.\"\n",
        "tokenizer = WhiteSpaceTokenizer()\n",
        "tokenizer.fit(text)\n",
        "\n",
        "token_ids = tokenizer.encode(\"Machine learning is fun\")\n",
        "print(token_ids)\n",
        "\n",
        "print(tokenizer.decode(token_ids))\n",
        "\n",
        "print(len(tokenizer))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3GedM368BfAp"
      },
      "source": [
        "tokenizer.token2id"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FVvktwJv9EWj"
      },
      "source": [
        "## Playing around with PyTorch components\n",
        "\n",
        "- `nn.Embedding`\n",
        "- `nn.RNN`\n",
        "- `nn.utils.rnn import pad_sequence`\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uRvia9Vg5IQY"
      },
      "source": [
        "import copy\n",
        "from time import time\n",
        "from typing import Any, Dict\n",
        "import random\n",
        "\n",
        "import numpy as np\n",
        "from sklearn.metrics import accuracy_score\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "from torch import optim\n",
        "from torch.utils.data import Dataset, TensorDataset, DataLoader"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0g2zAi-E9hDK"
      },
      "source": [
        "### `nn.Embedding`"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "H-txZl1C5SPj"
      },
      "source": [
        "# Embedding layer\n",
        "embedding_dim = 32\n",
        "\n",
        "emb_layer = nn.Embedding(num_embeddings=len(tokenizer),\n",
        "                         embedding_dim=embedding_dim)\n",
        "\n",
        "text_data1 = [\"Machine learning is fun\",\n",
        "              \"Deep learning is great\"]\n",
        "\n",
        "print([tokenizer.encode(text) for text in text_data1])\n",
        "\n",
        "X1 = torch.LongTensor([tokenizer.encode(text) for text in text_data1])\n",
        "out1 = emb_layer(X1)\n",
        "\n",
        "print(out1.shape) # (2, 4, 16)\n",
        "print(out1)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XLdaUyi7Jbt1"
      },
      "source": [
        "# This cell should return an error. Why?\n",
        "text_data2 = [\"Machine learning is really fun\",\n",
        "              \"Deep learning is great\"]\n",
        "\n",
        "torch.LongTensor([tokenizer.encode(text) for text in text_data2])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "D5lLWthQKBO4"
      },
      "source": [
        "[tokenizer.encode(text) for text in text_data2]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lf2LG94o9kHV"
      },
      "source": [
        "### `nn.utils.rnn.pad_sequence`"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "O_r5rJ73KHdB"
      },
      "source": [
        "# https://pytorch.org/docs/stable/generated/torch.nn.utils.rnn.pad_sequence.html\n",
        "\n",
        "from torch.nn.utils.rnn import pad_sequence\n",
        "seq = [torch.LongTensor(tokenizer.encode(text)) for text in text_data2]\n",
        "print(seq)\n",
        "\n",
        "padded_seq = pad_sequence(seq, batch_first=True)\n",
        "print(padded_seq)    "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kny2-2cMRFTh"
      },
      "source": [
        "emb_layer_ignore_pad = nn.Embedding(num_embeddings=len(tokenizer),\n",
        "                                    embedding_dim=embedding_dim,\n",
        "                                    padding_idx=0)\n",
        "\n",
        "no_padidx_out = emb_layer(padded_seq)\n",
        "padidx_out = emb_layer_ignore_pad(padded_seq)\n",
        "\n",
        "print(no_padidx_out[-1][-1])  # Some values are assigned to padding tokens!!\n",
        "print(padidx_out[-1][-1])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ytEkpMq79oTk"
      },
      "source": [
        "### nn.RNN\n",
        "\n",
        "This example only covers `nn.RNN`. `nn.LSTM` and `nn.GRU` generally have the same interface.\n",
        "\n",
        "`nn.LSTM` is significantly different from `nn.RNN` and `nn.GRU` in some sense. Please remember the lecture about what is the difference."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "o34eENIi-I4U"
      },
      "source": [
        "hidden_size = 32\n",
        "num_layers = 3\n",
        "\n",
        "rnn_layer = nn.RNN(input_size=embedding_dim,\n",
        "                    hidden_size=hidden_size,\n",
        "                    num_layers=num_layers,\n",
        "                    batch_first=True) # True: (B, L, V), False: (L, B, V)\n",
        "\n",
        "out2, h_n = rnn_layer(out1) # last hidden state of each layer"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rcojTi1g-Vch"
      },
      "source": [
        "print(out2.shape)  # (B, L, D)\n",
        "print(h_n.shape)     # (NumLayer, B, D)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "st4SuftbEYCR"
      },
      "source": [
        "out2[:, -1, :].shape"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "A-cral62dknb"
      },
      "source": [
        "fc_layer = nn.Linear(in_features=hidden_size,\n",
        "                     out_features=2)\n",
        "\n",
        "out3 = fc_layer(out2[:, -1, :])\n",
        "print(out3.shape)\n",
        "out3"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1QEplArcR-GT"
      },
      "source": [
        "## Putting It All Together!\n",
        "\n",
        "Let's put it all together to implement an RNN classifier model. \n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oyYfcXlPf05e"
      },
      "source": [
        "class SimpleRNN(nn.Module):\n",
        "    def __init__(self,\n",
        "                 vocab_size: int,\n",
        "                 embedding_dim: int = 128,\n",
        "                 hidden_size: int = 64,\n",
        "                 num_layers: int = 1,\n",
        "                 num_output: int = 3):\n",
        "        super().__init__()\n",
        "        self.emb = nn.Embedding(num_embeddings=vocab_size,\n",
        "                                embedding_dim=embedding_dim,\n",
        "                                padding_idx=0)\n",
        "        self.rnn = nn.RNN(input_size=embedding_dim,\n",
        "                          hidden_size=hidden_size,\n",
        "                          num_layers=num_layers,\n",
        "                          batch_first=True)\n",
        "        self.dropout = nn.Dropout(p=0.5)\n",
        "        self.linear = nn.Linear(hidden_size, num_output)\n",
        "    \n",
        "    def forward(self, x):\n",
        "        emb = self.emb(x)\n",
        "        hidden_states, _ = self.rnn(emb)\n",
        "        last_hidden_state = hidden_states[:, -1, :] # Last hidden state\n",
        "        out = self.linear(self.dropout(last_hidden_state))\n",
        "        return out, last_hidden_state                               "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YsuNygsS-GI6"
      },
      "source": [
        "### US Airline Sentiment Analysis dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HOT5rAx0ix1l"
      },
      "source": [
        "# https://www.kaggle.com/crowdflower/twitter-airline-sentiment\n",
        "# License CC BY-NC-SA 4.0\n",
        "!gdown --id 1BS_TIqm7crkBRr8p6REZrMv4Uk9_-e6W"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uVZchVhCi-1o"
      },
      "source": [
        "import pandas as pd\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.preprocessing import LabelEncoder\n",
        "\n",
        "# Loading dataset\n",
        "df = pd.read_csv(\"Tweets.csv\")\n",
        "\n",
        "# Tokenizer (Note: Technically, tokenizer should be fit only on training data)\n",
        "tokenizer = WhiteSpaceTokenizer()\n",
        "tokenizer.fit(\" \".join(df[\"text\"].tolist()))\n",
        "df[\"token_ids\"] = df[\"text\"].apply(lambda x: tokenizer.encode(x))\n",
        "\n",
        "# Label encoder\n",
        "le = LabelEncoder()\n",
        "y = le.fit_transform(df[\"airline_sentiment\"].values)\n",
        "df[\"label\"] = y\n",
        "\n",
        "df[[\"airline_sentiment\", \"label\", \"text\", \"token_ids\"]]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dy-rKLtokngB"
      },
      "source": [
        "# Splint into 60% train, 20% valid, 20% test\n",
        "train_df, test_df = train_test_split(\n",
        "    df, test_size=0.2, random_state=1)\n",
        "\n",
        "train_df, valid_df = train_test_split(\n",
        "    train_df, test_size=0.25, random_state=1)  # 0.25 x 0.8 = 0.2\n",
        "\n",
        "print(len(train_df), len(valid_df), len(test_df))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2wACD9Yb-Ml4"
      },
      "source": [
        "### Design Custom Dataset\n",
        "\n",
        "You can easily create a custom `Dataset` class. What you need to implement is\n",
        "\n",
        "- 1) `__len__(self)` that returns the total number of samples.\n",
        "- 2) `__getitem__(self, idx)` that returns the corresponding data for a given `idx`.\n",
        "\n",
        "Pandas `DataFrame` makes the custom Dataset design simple. Let's take a look."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "w4zhPQPskNDG"
      },
      "source": [
        "class TweetDataset(Dataset):\n",
        "    def __init__(self,\n",
        "                 df):\n",
        "        self.df = df\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.df)\n",
        "\n",
        "    def __getitem__(self, idx):\n",
        "        return {\"label\": torch.LongTensor([df.iloc[idx][\"label\"]]),\n",
        "                \"token_ids\": torch.LongTensor(df.iloc[idx][\"token_ids\"])}"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "H2sFrdaj-sYS"
      },
      "source": [
        "### Collate function\n",
        "\n",
        "The remainining challenge is that different tweets have different number of tokens. We have already learned how to use `pad_sequence`.\n",
        "\n",
        "By passing a collate function to `DataLoader`, we can process batches on the fly."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "J6vgCdd7-syP"
      },
      "source": [
        "def pad_collate(batch):\n",
        "    \"\"\"\n",
        "    batch = [{'label': tensor([0]), 'token_ids': tensor([1164,  122,   60,   28,   38, 1312,  163,  155,  235,  195, 1331,  340,\n",
        "        7283,  101, 6157,  128,  121,  588,  407, 3419, 2513,   13, 1470,   34, 9,  524,  121, 4436, 6153])},\n",
        "             {'label': tensor([2]), 'token_ids': tensor([ 1164,    48,  1391, 10195, 10196,   340,  1008,    39, 10197,  2384,\n",
        "          274, 10198,  2881,    83,   839,    32,  2849,  3389, 10199,   163, 167,   320, 10200])},\n",
        "          ...]\n",
        "    \"\"\"\n",
        "    token_ids = torch.nn.utils.rnn.pad_sequence([x[\"token_ids\"] for x in batch],\n",
        "                                                batch_first=True)\n",
        "    label = torch.cat([x[\"label\"] for x in batch])\n",
        "    #import pdb; pdb.set_trace()\n",
        "    batch = {\"token_ids\": token_ids, \"label\": label}\n",
        "    return batch"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2zKAdBtOmPEM"
      },
      "source": [
        "batch_size = 8 \n",
        "\n",
        "train_dataset = TweetDataset(train_df)\n",
        "valid_dataset = TweetDataset(valid_df)\n",
        "test_dataset = TweetDataset(test_df)\n",
        "\n",
        "train_dl = DataLoader(train_dataset,\n",
        "                      batch_size=batch_size,\n",
        "                      shuffle=True,\n",
        "                      drop_last=True,\n",
        "                      collate_fn=pad_collate) # <=\n",
        "valid_dl = DataLoader(valid_dataset,\n",
        "                      collate_fn=pad_collate) # <=\n",
        "test_dl = DataLoader(test_dataset,\n",
        "                     collate_fn=pad_collate) # <=\n",
        "\n",
        "# Check\n",
        "batch = next(iter(train_dl))\n",
        "batch"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5gz7og6QuR4T"
      },
      "source": [
        "def train(model: nn.Module,\n",
        "          train_dataset: Dataset,\n",
        "          valid_dataset: Dataset,\n",
        "          config: Dict[str, Any],\n",
        "          random_seed: int = 0):\n",
        "  \n",
        "    # Random Seeds ===============\n",
        "    torch.manual_seed(random_seed)\n",
        "    random.seed(random_seed)\n",
        "    np.random.seed(random_seed)\n",
        "    # Random Seeds ===============\n",
        "\n",
        "    # GPU configuration\n",
        "    device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "\n",
        "    dl_train = DataLoader(train_dataset,\n",
        "                          batch_size=config[\"batch_size\"],\n",
        "                          shuffle=True,\n",
        "                          drop_last=True,\n",
        "                          collate_fn=pad_collate)\n",
        "    dl_valid = DataLoader(valid_dataset,\n",
        "                          collate_fn=pad_collate)\n",
        "                  \n",
        "    # Model, Optimzier, Loss function\n",
        "    model = model.to(device)\n",
        "\n",
        "    # Optimizer\n",
        "    optimizer = config[\"optimizer_cls\"](model.parameters(), lr=config[\"lr\"])\n",
        "    loss_fn = nn.CrossEntropyLoss()\n",
        "\n",
        "    # For each epoch\n",
        "    eval_list = []\n",
        "    t0 = time()\n",
        "    best_val = None\n",
        "    best_model = None\n",
        "    for n in range(config[\"n_epochs\"]):\n",
        "        t1 = time()\n",
        "        print(\"Epoch {}\".format(n))\n",
        "        # Training\n",
        "        train_loss = 0.\n",
        "        train_pred_list = []\n",
        "        train_true_list = []\n",
        "        model.train()  # Switch to the training mode\n",
        "\n",
        "        # For each batch\n",
        "        for batch in dl_train:\n",
        "            optimizer.zero_grad()              # Initialize gradient information\n",
        "            X = batch[\"token_ids\"].to(device)\n",
        "            out, last_hidden_state = model(X)  # Call `forward()` function of the model\n",
        "            loss = loss_fn(out, batch[\"label\"].to(device))\n",
        "            loss.backward()                    # Backpropagate the loss value\n",
        "            optimizer.step()                   # Update the parameters\n",
        "            train_loss += loss.data.item()\n",
        "            train_pred_list += out.argmax(axis=1).detach().cpu().tolist()\n",
        "            train_true_list += batch[\"label\"].detach().cpu().tolist()\n",
        "\n",
        "        train_loss /= len(dl_train)\n",
        "        train_acc = accuracy_score(train_true_list, train_pred_list)\n",
        "        print(\"    Training loss: {:.4f}    Training acc: {:.4f}\".format(train_loss,\n",
        "                                                                         train_acc))\n",
        "\n",
        "        # Validation\n",
        "        valid_loss = 0.\n",
        "        valid_pred_list = []\n",
        "        valid_true_list = []\n",
        "\n",
        "        model.eval()  # Switch to the evaluation mode\n",
        "        valid_emb_list = []\n",
        "        valid_label_list = []\n",
        "        for i, batch in enumerate(dl_valid):\n",
        "            X = batch[\"token_ids\"].to(device)\n",
        "            out, last_hidden_state = model(X)  # Call `forward()` function of the model\n",
        "            loss = loss_fn(out, batch[\"label\"].to(device))\n",
        "            valid_loss += loss.data.item()\n",
        "            valid_pred_list += out.argmax(axis=1).detach().cpu().tolist()\n",
        "            valid_true_list += batch[\"label\"].detach().cpu().tolist()\n",
        "\n",
        "        valid_loss /= len(dl_valid)\n",
        "        valid_acc = accuracy_score(valid_true_list, valid_pred_list)\n",
        "        print(\"  Validation loss: {:.4f}  Validation acc: {:.4f}\".format(valid_loss,\n",
        "                                                                         valid_acc))\n",
        "\n",
        "        # Model selection\n",
        "        if best_val is None or valid_loss < best_val:\n",
        "            best_model = copy.deepcopy(model)\n",
        "            best_val = valid_loss\n",
        "\n",
        "        t2 = time()\n",
        "        print(\"     Elapsed time: {:.1f} [sec]\".format(t2 - t1))\n",
        "\n",
        "        # Store train/validation loss values\n",
        "        eval_list.append([n, train_loss, valid_loss, train_acc, valid_acc, t2 - t1])\n",
        "\n",
        "    eval_df = pd.DataFrame(eval_list, columns=[\"epoch\",\n",
        "                                               \"train_loss\", \"valid_loss\",\n",
        "                                               \"train_acc\", \"valid_acc\",\n",
        "                                               \"time\"])\n",
        "    eval_df.set_index(\"epoch\")\n",
        "\n",
        "    print(\"Total time: {:.1f} [sec]\".format(t2 - t0))\n",
        "\n",
        "    # Return the best model and trainining/validation information\n",
        "    return {\"model\": best_model,\n",
        "            \"best_val\": best_val,\n",
        "            \"eval_df\": eval_df}"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sRT-4GBHvHKw"
      },
      "source": [
        "config = {\"optimizer_cls\": optim.Adam,          \n",
        "          \"lr\": 0.001,\n",
        "          \"batch_size\": 16,\n",
        "          \"n_epochs\": 10}\n",
        "model = SimpleRNN(vocab_size=len(tokenizer))\n",
        "output = train(model, train_dataset, valid_dataset, config)"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}