Week_14_Class_12_LAB.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uacj685SvQM8"
      },
      "source": [
        "# Week 14-  Class 12\n",
        "This notebook have the most important exercises for theoretical class, which covers chapter 11 from the book. The labs from https://github.com/fchollet/deep-learning-with-python-notebooks are a guide for this notebook, mainly the labs from chapter 11 and 12"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "thupIQWrvQM-"
      },
      "source": [
        "####1. Downloading the data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "w1OzGCiZvQM-",
        "outputId": "b4d4ad35-b97b-409c-bf26-edc132750029"
      },
      "outputs": [],
      "source": [
        "!curl -O https://ai.stanford.edu/~amaas/data/sentiment/aclImdb_v1.tar.gz\n",
        "!tar -xf aclImdb_v1.tar.gz\n",
        "!rm -r aclImdb/train/unsup"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CqizRtEsvQM-"
      },
      "source": [
        "####2. Creating the dataset"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vlCI8oZzvQM_",
        "outputId": "5105a6c5-eeae-4b10-ccf9-a4103134b9b9"
      },
      "outputs": [],
      "source": [
        "import os, pathlib, shutil, random\n",
        "from tensorflow import keras\n",
        "\n",
        "batch_size = 32\n",
        "base_dir = pathlib.Path(\"aclImdb\")\n",
        "val_dir = base_dir / \"val\"\n",
        "train_dir = base_dir / \"train\"\n",
        "\n",
        "for category in (\"neg\", \"pos\"):\n",
        "    #os.makedirs(val_dir / category)\n",
        "    files = os.listdir(train_dir / category)\n",
        "    random.Random(1337).shuffle(files)\n",
        "    #20% for validation\n",
        "    num_val_samples = int(0.2 * len(files))\n",
        "    #select last num_val_samples (files) from array\n",
        "    val_files = files[-num_val_samples:]\n",
        "    #move last 20% of training files to validation\n",
        "    for fname in val_files:\n",
        "        shutil.move(train_dir / category / fname,\n",
        "                    val_dir / category / fname)\n",
        "\n",
        "train_ds = keras.utils.text_dataset_from_directory(\n",
        "    \"aclImdb/train\", batch_size=batch_size\n",
        ")\n",
        "val_ds = keras.utils.text_dataset_from_directory(\n",
        "    \"aclImdb/val\", batch_size=batch_size\n",
        ")\n",
        "test_ds = keras.utils.text_dataset_from_directory(\n",
        "    \"aclImdb/test\", batch_size=batch_size\n",
        ")\n",
        "text_only_train_ds = train_ds.map(lambda x, y: x)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WGI7z-I9wX9c",
        "outputId": "9007f9f3-5643-4ef0-ab6b-68dbb850bf64"
      },
      "outputs": [],
      "source": [
        "for x, y in train_ds:\n",
        "  print(x)\n",
        "  print ('\\n')\n",
        "  print(y)\n",
        "  break"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "N6fnX1lA2JoD"
      },
      "source": [
        "**Dummy Vectorization example**\n",
        "\n",
        "Understand The TextVectorization layer in keras"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "15Cbgwfg2IxV"
      },
      "outputs": [],
      "source": [
        "import string\n",
        "# The TextVectorization layer is just like this class example created here\n",
        "\n",
        "class Vectorizer:\n",
        "    def standardize(self, text):\n",
        "        text = text.lower()\n",
        "        return \"\".join(char for char in text if char not in string.punctuation)\n",
        "\n",
        "    def tokenize(self, text):\n",
        "        text = self.standardize(text)\n",
        "        return text.split()\n",
        "\n",
        "    def make_vocabulary(self, dataset):\n",
        "        self.vocabulary = {\"\": 0, \"[UNK]\": 1}\n",
        "        for text in dataset:\n",
        "            text = self.standardize(text)\n",
        "            tokens = self.tokenize(text)\n",
        "            for token in tokens:\n",
        "                if token not in self.vocabulary:\n",
        "                    self.vocabulary[token] = len(self.vocabulary)\n",
        "        self.inverse_vocabulary = dict(\n",
        "            (v, k) for k, v in self.vocabulary.items())\n",
        "\n",
        "    def encode(self, text):\n",
        "        text = self.standardize(text)\n",
        "        tokens = self.tokenize(text)\n",
        "        return [self.vocabulary.get(token, 1) for token in tokens]\n",
        "\n",
        "    def decode(self, int_sequence):\n",
        "        return \" \".join(\n",
        "            self.inverse_vocabulary.get(i, \"[UNK]\") for i in int_sequence)\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CIEPnowFeEXQ"
      },
      "source": [
        "Exercise"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "I4xouGR92Ini"
      },
      "outputs": [],
      "source": [
        "\n",
        "# initialize the class\n",
        "<your-answer>\n",
        "\n",
        "\n",
        "dataset = [\n",
        "    \"I write, erase, rewrite\",\n",
        "    \"Erase again, and then\",\n",
        "    \"A poppy blooms.\",\n",
        "]\n",
        "\n",
        "#create the vocabulary from the dataset\n",
        "<your-answer>\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "s6JrL_x8eHUo"
      },
      "source": [
        "Exercise"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bhiyhko33UAq",
        "outputId": "933231a9-daff-41c3-fcc4-a8fb47c9590c"
      },
      "outputs": [],
      "source": [
        "test_sentence = \"I write, rewrite, and still rewrite again\"\n",
        "#encode the test_sentence\n",
        "<your-answer>\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "54Cu5RmYeSnE"
      },
      "source": [
        "Exercise"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-o-HGoCq33Ma",
        "outputId": "10a777c8-a12e-47f5-ff12-b47afd70d133"
      },
      "outputs": [],
      "source": [
        "#decode the test_sentence\n",
        "<your-answer>\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CpcYEuwa4WqD"
      },
      "source": [
        "Optional example: if you need to pre process your text data in a specific way that the TextVectorizarion layer doesn't do in keras, you can define your own. "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "UnhTC2Xc4V67"
      },
      "outputs": [],
      "source": [
        "#this will be skipped in lab class\n",
        "from tensorflow.keras.layers import TextVectorization\n",
        "import re\n",
        "import string\n",
        "import tensorflow as tf\n",
        "\n",
        "#initialize\n",
        "text_vectorization = TextVectorization(\n",
        "    output_mode=\"int\",\n",
        ")\n",
        "\n",
        "#your custom functions\n",
        "\n",
        "def custom_standardization_fn(string_tensor):\n",
        "    lowercase_string = tf.strings.lower(string_tensor)\n",
        "    return tf.strings.regex_replace(\n",
        "        lowercase_string, f\"[{re.escape(string.punctuation)}]\", \"\")\n",
        "\n",
        "def custom_split_fn(string_tensor):\n",
        "    return tf.strings.split(string_tensor)\n",
        "\n",
        "#add the functions in the layer parameters\n",
        "text_vectorization = TextVectorization(\n",
        "    output_mode=\"int\",\n",
        "    #added here\n",
        "    standardize=custom_standardization_fn,\n",
        "    #added here\n",
        "    split=custom_split_fn,\n",
        ")\n",
        "\n",
        "#check results with example\n",
        "dataset = [\n",
        "    \"I write, erase, rewrite\",\n",
        "    \"Erase again, and then\",\n",
        "    \"A poppy blooms.\",\n",
        "]\n",
        "text_vectorization.adapt(dataset)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3wtlAuyrvQM_"
      },
      "source": [
        "####3. Vectorizing"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "LfQ0mhpAvQM_"
      },
      "outputs": [],
      "source": [
        "#Data Transformation\n",
        "\n",
        "from tensorflow.keras import layers\n",
        "#resulting in a tensor of shape [batch_size, output_sequence_length], can be padded or truncated\n",
        "max_length = 600\n",
        "#The maximum size of the vocabulary for this layer.\n",
        "max_tokens = 20000\n",
        "#if output mode is 'tf-idf'\n",
        "    #It weights a given term by taking “term frequency,” \n",
        "    #how many times the term appears in the current document, \n",
        "    #and dividing it by a measure of “document frequency,” \n",
        "    #which estimates how often the term comes up across the dataset\n",
        "text_vectorization = layers.TextVectorization(\n",
        "    max_tokens=max_tokens,\n",
        "    output_mode=\"int\",\n",
        "    output_sequence_length=max_length,\n",
        ")\n",
        "\n",
        "#just like fit\n",
        "text_vectorization.adapt(text_only_train_ds)\n",
        "\n",
        "int_train_ds = train_ds.map(\n",
        "    lambda x, y: (text_vectorization(x), y),\n",
        "    num_parallel_calls=4)\n",
        "int_val_ds = val_ds.map(\n",
        "    lambda x, y: (text_vectorization(x), y),\n",
        "    num_parallel_calls=4)\n",
        "int_test_ds = test_ds.map(\n",
        "    lambda x, y: (text_vectorization(x), y),\n",
        "    num_parallel_calls=4)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mYsRWnuh7Uqj"
      },
      "source": [
        "####Create Baseline model"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iWifVMhyemwP"
      },
      "source": [
        "Exercise"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "hmHuu1DI7wma"
      },
      "outputs": [],
      "source": [
        "max_tokens=20000\n",
        "hidden_dim=16\n",
        "\n",
        "inputs = <your-answer> \n",
        "x = <your-answer> #dense layer with relu\n",
        "x = <your-answer> #dropout 0.5\n",
        "outputs = <your-answer> #activation function is sigmoid\n",
        "model = keras.Model(inputs, outputs)\n",
        "<your-answer> #the optimizer is rmsprop\n",
        "\n",
        "<your-answer> #print the summary of the model\n",
        "\n",
        "<your-answer> #create callbacks array saving the best model\n",
        "\n",
        "<your-answer> #train the model\n",
        "\n",
        "<your-answer> #load the best model\n",
        "\n",
        "<your-answer> #evaluate and print the metric"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "idGE8OAbvQNA"
      },
      "source": [
        "####Main model (Embedding layer)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jh77jSnmf-vr",
        "outputId": "c09bda77-f2e1-4ab1-b8c6-c73820517650"
      },
      "outputs": [],
      "source": [
        "#this layer takes word index and transform that into a word vector of [1,output_dim] size \n",
        "#input=[batch_size, sequence_length]\n",
        "#output=[batch_size,sequence_length, embedding_ dimensionality]\n",
        "inputs = keras.Input(shape=(None,), dtype=\"int64\")\n",
        "embedded = layers.Embedding(input_dim=max_tokens, output_dim=256)(inputs)\n",
        "#Bidirectional: two RNN layers running in parallel, \n",
        "#with one processing the tokens in their natural order, \n",
        "#and the other processing the same tokens in reverse\n",
        "x = layers.Bidirectional(layers.LSTM(32))(embedded)\n",
        "x = layers.Dropout(0.5)(x)\n",
        "outputs = layers.Dense(1, activation=\"sigmoid\")(x)\n",
        "model_emb = keras.Model(inputs, outputs)\n",
        "model_emb.compile(optimizer=\"rmsprop\",\n",
        "              loss=\"binary_crossentropy\",\n",
        "              metrics=[\"accuracy\"])\n",
        "model_emb.summary()\n",
        "\n",
        "callbacks = [\n",
        "    keras.callbacks.ModelCheckpoint(\"embeddings_bidir_gru.keras\",\n",
        "                                    save_best_only=True)\n",
        "]\n",
        "model_emb.fit(int_train_ds, validation_data=int_val_ds, epochs=1, callbacks=callbacks)\n",
        "model_emb = keras.models.load_model(\"embeddings_bidir_gru.keras\")\n",
        "print(f\"Test acc: {model_emb.evaluate(int_test_ds)[1]:.3f}\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "b_wWgxrs-Tob"
      },
      "source": [
        "####Export and predict model"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QGnEy03skDfU"
      },
      "source": [
        "Exercise"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "UcOdmVackGdu"
      },
      "outputs": [],
      "source": [
        "#Based on the doc: https://keras.io/examples/nlp/pretrained_word_embeddings/\n",
        "# predict the sentence: 'This movie is complex and confusing, i didn't like it'\n",
        "\n",
        "<your-answer>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "qdYld0-_9zgz"
      },
      "outputs": [],
      "source": [
        "#Clustering the embedding space\n",
        "#https://stackoverflow.com/questions/51235118/how-to-get-word-vectors-from-keras-embedding-layer\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3UGpGEdQvQNB"
      },
      "source": [
        "#### Improvement step: Using pretrained word embeddings (optional)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "yWBzrOfdvQNB"
      },
      "outputs": [],
      "source": [
        "!wget http://nlp.stanford.edu/data/glove.6B.zip\n",
        "!unzip -q glove.6B.zip"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "sruGts72vQNB"
      },
      "outputs": [],
      "source": [
        "import numpy as np\n",
        "path_to_glove_file = \"glove.6B.100d.txt\"\n",
        "\n",
        "embeddings_index = {}\n",
        "with open(path_to_glove_file) as f:\n",
        "    for line in f:\n",
        "        word, coefs = line.split(maxsplit=1)\n",
        "        coefs = np.fromstring(coefs, \"f\", sep=\" \")\n",
        "        embeddings_index[word] = coefs\n",
        "\n",
        "print(f\"Found {len(embeddings_index)} word vectors.\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "OOOshtdTvQNB"
      },
      "outputs": [],
      "source": [
        "embedding_dim = 100\n",
        "\n",
        "vocabulary = text_vectorization.get_vocabulary()\n",
        "word_index = dict(zip(vocabulary, range(len(vocabulary))))\n",
        "\n",
        "embedding_matrix = np.zeros((max_tokens, embedding_dim))\n",
        "for word, i in word_index.items():\n",
        "    if i < max_tokens:\n",
        "        embedding_vector = embeddings_index.get(word)\n",
        "    if embedding_vector is not None:\n",
        "        embedding_matrix[i] = embedding_vector"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "_ezhgDWgvQNB"
      },
      "outputs": [],
      "source": [
        "embedding_layer = layers.Embedding(\n",
        "    max_tokens,\n",
        "    embedding_dim,\n",
        "    embeddings_initializer=keras.initializers.Constant(embedding_matrix),\n",
        "    trainable=False,\n",
        "    mask_zero=True,\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "xawZ8yCzvQNB"
      },
      "outputs": [],
      "source": [
        "inputs = keras.Input(shape=(None,), dtype=\"int64\")\n",
        "embedded = embedding_layer(inputs)\n",
        "x = layers.Bidirectional(layers.LSTM(32))(embedded)\n",
        "x = layers.Dropout(0.5)(x)\n",
        "outputs = layers.Dense(1, activation=\"sigmoid\")(x)\n",
        "model = keras.Model(inputs, outputs)\n",
        "model.compile(optimizer=\"rmsprop\",\n",
        "              loss=\"binary_crossentropy\",\n",
        "              metrics=[\"accuracy\"])\n",
        "model.summary()\n",
        "\n",
        "callbacks = [\n",
        "    keras.callbacks.ModelCheckpoint(\"glove_embeddings_sequence_model.keras\",\n",
        "                                    save_best_only=True)\n",
        "]\n",
        "model.fit(int_train_ds, validation_data=int_val_ds, epochs=10, callbacks=callbacks)\n",
        "model = keras.models.load_model(\"glove_embeddings_sequence_model.keras\")\n",
        "print(f\"Test acc: {model.evaluate(int_test_ds)[1]:.3f}\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "k0ds5eiabjao"
      },
      "source": [
        "####Main model 2: Transformer for text classification"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "-lUuaz-ckqpr"
      },
      "outputs": [],
      "source": [
        "class PositionalEmbedding(layers.Layer):\n",
        "    def __init__(self, sequence_length, input_dim, output_dim, **kwargs):\n",
        "        super().__init__(**kwargs)\n",
        "        self.token_embeddings = layers.Embedding(\n",
        "            input_dim=input_dim, output_dim=output_dim)\n",
        "        self.position_embeddings = layers.Embedding(\n",
        "            input_dim=sequence_length, output_dim=output_dim)\n",
        "        self.sequence_length = sequence_length\n",
        "        self.input_dim = input_dim\n",
        "        self.output_dim = output_dim\n",
        "\n",
        "    def call(self, inputs):\n",
        "        length = tf.shape(inputs)[-1]\n",
        "        positions = tf.range(start=0, limit=length, delta=1)\n",
        "        embedded_tokens = self.token_embeddings(inputs)\n",
        "        embedded_positions = self.position_embeddings(positions)\n",
        "        return embedded_tokens + embedded_positions\n",
        "\n",
        "    def compute_mask(self, inputs, mask=None):\n",
        "        return tf.math.not_equal(inputs, 0)\n",
        "\n",
        "    def get_config(self):\n",
        "        config = super().get_config()\n",
        "        config.update({\n",
        "            \"output_dim\": self.output_dim,\n",
        "            \"sequence_length\": self.sequence_length,\n",
        "            \"input_dim\": self.input_dim,\n",
        "        })\n",
        "        return config"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "8_90QtVlkrll"
      },
      "outputs": [],
      "source": [
        "import tensorflow as tf\n",
        "from tensorflow import keras\n",
        "from tensorflow.keras import layers\n",
        "\n",
        "class TransformerEncoder(layers.Layer):\n",
        "    def __init__(self, embed_dim, dense_dim, num_heads, **kwargs):\n",
        "        super().__init__(**kwargs)\n",
        "        self.embed_dim = embed_dim\n",
        "        self.dense_dim = dense_dim\n",
        "        self.num_heads = num_heads\n",
        "        self.attention = layers.MultiHeadAttention(\n",
        "            num_heads=num_heads, key_dim=embed_dim)\n",
        "        self.dense_proj = keras.Sequential(\n",
        "            [layers.Dense(dense_dim, activation=\"relu\"),\n",
        "             layers.Dense(embed_dim),]\n",
        "        )\n",
        "        self.layernorm_1 = layers.LayerNormalization()\n",
        "        self.layernorm_2 = layers.LayerNormalization()\n",
        "\n",
        "    def call(self, inputs, mask=None):\n",
        "        if mask is not None:\n",
        "            mask = mask[:, tf.newaxis, :]\n",
        "        attention_output = self.attention(\n",
        "            inputs, inputs, attention_mask=mask)\n",
        "        proj_input = self.layernorm_1(inputs + attention_output)\n",
        "        proj_output = self.dense_proj(proj_input)\n",
        "        return self.layernorm_2(proj_input + proj_output)\n",
        "\n",
        "    def get_config(self):\n",
        "        config = super().get_config()\n",
        "        config.update({\n",
        "            \"embed_dim\": self.embed_dim,\n",
        "            \"num_heads\": self.num_heads,\n",
        "            \"dense_dim\": self.dense_dim,\n",
        "        })\n",
        "        return config"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1-rVDl4xbidp",
        "outputId": "6ce1da13-1b0c-4b31-c6ed-7ba957531224"
      },
      "outputs": [],
      "source": [
        "\n",
        "\n",
        "vocab_size = 20000\n",
        "sequence_length = 600\n",
        "embed_dim = 256\n",
        "num_heads = 2\n",
        "dense_dim = 32\n",
        "\n",
        "inputs = keras.Input(shape=(None,), dtype=\"int64\")\n",
        "#order\n",
        "x = PositionalEmbedding(sequence_length, vocab_size, embed_dim)(inputs)\n",
        "#context\n",
        "x = TransformerEncoder(embed_dim, dense_dim, num_heads)(x)\n",
        "x = layers.GlobalMaxPooling1D()(x)\n",
        "x = layers.Dropout(0.5)(x)\n",
        "outputs = layers.Dense(1, activation=\"sigmoid\")(x)\n",
        "model = keras.Model(inputs, outputs)\n",
        "model.compile(optimizer=\"rmsprop\",\n",
        "              loss=\"binary_crossentropy\",\n",
        "              metrics=[\"accuracy\"])\n",
        "model.summary()\n",
        "\n",
        "callbacks = [\n",
        "    keras.callbacks.ModelCheckpoint(\"full_transformer_encoder.keras\",\n",
        "                                    save_best_only=True)\n",
        "]\n",
        "model.fit(int_train_ds, validation_data=int_val_ds, epochs=1, callbacks=callbacks)\n",
        "model = keras.models.load_model(\n",
        "    \"full_transformer_encoder.keras\",\n",
        "    custom_objects={\"TransformerEncoder\": TransformerEncoder,\n",
        "                    \"PositionalEmbedding\": PositionalEmbedding})\n",
        "print(f\"Test acc: {model.evaluate(int_test_ds)[1]:.3f}\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "QH9qU1VQSnqL"
      },
      "outputs": [],
      "source": [
        "#https://arxiv.org/abs/1907.11692\n",
        "#https://github.com/keras-team/keras-nlp/blob/v0.5.1/keras_nlp/models/roberta/roberta_backbone.py#L36"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": [],
      "toc_visible": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.0"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}