Lab12/Lab12_3_LEEJOOYOUNG.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Lab12-3_LEEJOOYOUNG.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "08s4f_GAoksI",
        "outputId": "492ab2f7-a480-47b9-b600-23bfabc788a4"
      },
      "source": [
        "# setup\n",
        "import numpy as np\n",
        "import tensorflow as tf\n",
        "import matplotlib.pyplot as plt\n",
        "from tensorflow import keras\n",
        "from tensorflow.keras import layers\n",
        "from tensorflow.keras import Sequential, Model\n",
        "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
        "from pprint import pprint\n",
        "%matplotlib inline\n",
        "\n",
        "print(tf.__version__)"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2.7.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "h8UGeNzAotE_"
      },
      "source": [
        "# example data\n",
        "sentences = [['I', 'feel', 'hungry'],\n",
        "     ['tensorflow', 'is', 'very', 'difficult'],\n",
        "     ['tensorflow', 'is', 'a', 'framework', 'for', 'deep', 'learning'],\n",
        "     ['tensorflow', 'is', 'very', 'fast', 'changing']]\n",
        "pos = [['pronoun', 'verb', 'adjective'],\n",
        "     ['noun', 'verb', 'adverb', 'adjective'],\n",
        "     ['noun', 'verb', 'determiner', 'noun', 'preposition', 'adjective', 'noun'],\n",
        "     ['noun', 'verb', 'adverb', 'adjective', 'verb']]"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "M8Wvei7kovl3",
        "outputId": "0d72d660-1d48-4f6e-c5e8-62ed4232fa55"
      },
      "source": [
        "# creating a token dictionary for word\n",
        "word_list = sum(sentences, [])\n",
        "word_list = sorted(set(word_list))\n",
        "word_list = ['<pad>'] + word_list\n",
        "word2idx = {word : idx for idx, word in enumerate(word_list)}\n",
        "idx2word = {idx : word for idx, word in enumerate(word_list)}\n",
        "\n",
        "print(word2idx)\n",
        "print(idx2word)\n",
        "print(len(idx2word))"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'<pad>': 0, 'I': 1, 'a': 2, 'changing': 3, 'deep': 4, 'difficult': 5, 'fast': 6, 'feel': 7, 'for': 8, 'framework': 9, 'hungry': 10, 'is': 11, 'learning': 12, 'tensorflow': 13, 'very': 14}\n",
            "{0: '<pad>', 1: 'I', 2: 'a', 3: 'changing', 4: 'deep', 5: 'difficult', 6: 'fast', 7: 'feel', 8: 'for', 9: 'framework', 10: 'hungry', 11: 'is', 12: 'learning', 13: 'tensorflow', 14: 'very'}\n",
            "15\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BCP7ih_Doyj4",
        "outputId": "e8c41759-db22-4a93-8d41-980135c7e598"
      },
      "source": [
        "# creating a token dictionary for part of speech\n",
        "pos_list = sum(pos, [])\n",
        "pos_list = sorted(set(pos_list))\n",
        "pos_list = ['<pad>'] + pos_list\n",
        "pos2idx = {pos : idx for idx, pos in enumerate(pos_list)}\n",
        "idx2pos = {idx : pos for idx, pos in enumerate(pos_list)}\n",
        "\n",
        "print(pos2idx)\n",
        "print(idx2pos)\n",
        "print(len(pos2idx))"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'<pad>': 0, 'adjective': 1, 'adverb': 2, 'determiner': 3, 'noun': 4, 'preposition': 5, 'pronoun': 6, 'verb': 7}\n",
            "{0: '<pad>', 1: 'adjective', 2: 'adverb', 3: 'determiner', 4: 'noun', 5: 'preposition', 6: 'pronoun', 7: 'verb'}\n",
            "8\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Rn4ybJPho0su",
        "outputId": "e71057c6-f2b8-46b4-87f9-8127240cb06b"
      },
      "source": [
        "# converting sequence of tokens to sequence of indices\n",
        "max_sequence = 10\n",
        "x_data = list(map(lambda sentence : [word2idx.get(token) for token in sentence], sentences))\n",
        "y_data = list(map(lambda sentence : [pos2idx.get(token) for token in sentence], pos))\n",
        "\n",
        "# padding the sequence of indices\n",
        "x_data = pad_sequences(sequences = x_data, maxlen = max_sequence, padding='post')\n",
        "x_data_mask = ((x_data != 0) * 1).astype(np.float32)\n",
        "x_data_len = list(map(lambda sentence : len(sentence), sentences))\n",
        "\n",
        "y_data = pad_sequences(sequences = y_data, maxlen = max_sequence, padding='post')\n",
        "\n",
        "# checking data\n",
        "print(x_data, x_data_len)\n",
        "print(x_data_mask)\n",
        "print(y_data)"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[[ 1  7 10  0  0  0  0  0  0  0]\n",
            " [13 11 14  5  0  0  0  0  0  0]\n",
            " [13 11  2  9  8  4 12  0  0  0]\n",
            " [13 11 14  6  3  0  0  0  0  0]] [3, 4, 7, 5]\n",
            "[[1. 1. 1. 0. 0. 0. 0. 0. 0. 0.]\n",
            " [1. 1. 1. 1. 0. 0. 0. 0. 0. 0.]\n",
            " [1. 1. 1. 1. 1. 1. 1. 0. 0. 0.]\n",
            " [1. 1. 1. 1. 1. 0. 0. 0. 0. 0.]]\n",
            "[[6 7 1 0 0 0 0 0 0 0]\n",
            " [4 7 2 1 0 0 0 0 0 0]\n",
            " [4 7 3 4 5 1 4 0 0 0]\n",
            " [4 7 2 1 7 0 0 0 0 0]]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jztr_4Ulo4YB"
      },
      "source": [
        "# creating rnn for \"many to many\" sequence tagging\n",
        "num_classes = len(pos2idx)\n",
        "hidden_dim = 10\n",
        "\n",
        "input_dim = len(word2idx)\n",
        "output_dim = len(word2idx)\n",
        "one_hot = np.eye(len(word2idx))\n",
        "\n",
        "model = Sequential()\n",
        "model.add(layers.Embedding(input_dim=input_dim, output_dim=output_dim, mask_zero=True,\n",
        "                           trainable=False, input_length=max_sequence,\n",
        "                           embeddings_initializer=keras.initializers.Constant(one_hot)))\n",
        "model.add(layers.SimpleRNN(units=hidden_dim, return_sequences=True))\n",
        "model.add(layers.TimeDistributed(layers.Dense(units=num_classes)))"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RHYpQHfro7Z3",
        "outputId": "edf81bd4-9a90-4ac7-b49c-e26308f16312"
      },
      "source": [
        "model.summary()"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"sequential\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " embedding (Embedding)       (None, 10, 15)            225       \n",
            "                                                                 \n",
            " simple_rnn (SimpleRNN)      (None, 10, 10)            260       \n",
            "                                                                 \n",
            " time_distributed (TimeDistr  (None, 10, 8)            88        \n",
            " ibuted)                                                         \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 573\n",
            "Trainable params: 348\n",
            "Non-trainable params: 225\n",
            "_________________________________________________________________\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vui5iO0Qo-Ac"
      },
      "source": [
        "# creating loss function\n",
        "def loss_fn(model, x, y, x_len, max_sequence):\n",
        "    masking = tf.sequence_mask(x_len, maxlen=max_sequence, dtype=tf.float32)\n",
        "    valid_time_step = tf.cast(x_len,dtype=tf.float32)    \n",
        "    sequence_loss = tf.keras.losses.sparse_categorical_crossentropy(\n",
        "        y_true=y, y_pred=model(x), from_logits=True) * masking    \n",
        "    sequence_loss = tf.reduce_sum(sequence_loss, axis=-1) / valid_time_step    \n",
        "    sequence_loss = tf.reduce_mean(sequence_loss)    \n",
        "    return sequence_loss\n",
        "\n",
        "# creating and optimizer\n",
        "lr = 0.1\n",
        "epochs = 30\n",
        "batch_size = 2 \n",
        "opt = tf.keras.optimizers.Adam(learning_rate = lr)"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IxhFtnKQpASH",
        "outputId": "df4a95bc-967c-4a2d-aa7f-273c44009908"
      },
      "source": [
        "# generating data pipeline\n",
        "tr_dataset = tf.data.Dataset.from_tensor_slices((x_data, y_data, x_data_len))\n",
        "tr_dataset = tr_dataset.shuffle(buffer_size=4)\n",
        "tr_dataset = tr_dataset.batch(batch_size = 2)\n",
        "\n",
        "print(tr_dataset)"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "<BatchDataset shapes: ((None, 10), (None, 10), (None,)), types: (tf.int32, tf.int32, tf.int32)>\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "oeOM5Bn0pDvC",
        "outputId": "f1ce6483-2314-4d07-86cf-a1d88b5fe367"
      },
      "source": [
        "# training\n",
        "tr_loss_hist = []\n",
        "\n",
        "for epoch in range(epochs):\n",
        "    avg_tr_loss = 0\n",
        "    tr_step = 0\n",
        "    \n",
        "    for x_mb, y_mb, x_mb_len in tr_dataset:\n",
        "        with tf.GradientTape() as tape:\n",
        "            tr_loss = loss_fn(model, x=x_mb, y=y_mb, x_len=x_mb_len, max_sequence=max_sequence)\n",
        "        grads = tape.gradient(target=tr_loss, sources=model.variables)\n",
        "        opt.apply_gradients(grads_and_vars=zip(grads, model.variables))\n",
        "        avg_tr_loss += tr_loss\n",
        "        tr_step += 1\n",
        "    else:\n",
        "        avg_tr_loss /= tr_step\n",
        "        tr_loss_hist.append(avg_tr_loss)\n",
        "    \n",
        "    if (epoch + 1) % 5 == 0:\n",
        "        print('epoch : {:3}, tr_loss : {:.3f}'.format(epoch + 1, avg_tr_loss))"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "epoch :   5, tr_loss : 0.167\n",
            "epoch :  10, tr_loss : 0.012\n",
            "epoch :  15, tr_loss : 0.003\n",
            "epoch :  20, tr_loss : 0.002\n",
            "epoch :  25, tr_loss : 0.001\n",
            "epoch :  30, tr_loss : 0.001\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "a12x8p4hpGrB",
        "outputId": "97af59fc-ec4d-4ecd-8963-cb44efc5eafa"
      },
      "source": [
        "yhat = model.predict(x_data)\n",
        "yhat = np.argmax(yhat, axis=-1) * x_data_mask\n",
        "\n",
        "pprint(list(map(lambda row : [idx2pos.get(elm) for elm in row],yhat.astype(np.int32).tolist())), width = 120)\n",
        "pprint(pos)"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[['pronoun', 'verb', 'adjective', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>'],\n",
            " ['noun', 'verb', 'adverb', 'adjective', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>'],\n",
            " ['noun', 'verb', 'determiner', 'noun', 'preposition', 'adjective', 'noun', '<pad>', '<pad>', '<pad>'],\n",
            " ['noun', 'verb', 'adverb', 'adjective', 'verb', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>']]\n",
            "[['pronoun', 'verb', 'adjective'],\n",
            " ['noun', 'verb', 'adverb', 'adjective'],\n",
            " ['noun', 'verb', 'determiner', 'noun', 'preposition', 'adjective', 'noun'],\n",
            " ['noun', 'verb', 'adverb', 'adjective', 'verb']]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 283
        },
        "id": "cSUysvS6pJiu",
        "outputId": "779fc5f7-ace6-4f99-abe4-b544b58e540a"
      },
      "source": [
        "plt.plot(tr_loss_hist)"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[<matplotlib.lines.Line2D at 0x7fa1d045b790>]"
            ]
          },
          "metadata": {},
          "execution_count": 12
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXwAAAD4CAYAAADvsV2wAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAeCUlEQVR4nO3de5ScdZ3n8fenL+nOpdOdpDv3GyRBLnKTJlzCDHgLweOKzrqSoA66atZZWXVmzsw67jni4tlz3HHGcV1RJqsZdFYCLMiYmYNiHGEiIJAOJlwFmoAknZB0EpKQhFy6+7t/1NOh0ulLdbrS1fU8n9c5dbrq9/yequ9jyaeePM/ze36KCMzMLP0qSl2AmZkNDwe+mVlGOPDNzDLCgW9mlhEOfDOzjKgqdQG9aWxsjLlz55a6DDOzsrF+/fqdEdHUX58RGfhz586lpaWl1GWYmZUNSb8fqM+Ah3QkzZL0gKRnJT0j6Qu99JGkb0tqlfSkpHfkLbtB0ovJ44bBb4aZmRVDIXv4HcCfR8QTkuqA9ZLWRMSzeX2uARYkj0uA7wGXSJoI3AQ0A5GsuzoiXi/qVpiZ2YAG3MOPiG0R8UTy/A3gOWBGj27XAj+KnEeBBknTgKuBNRGxOwn5NcCSom6BmZkVZFBX6UiaC1wIPNZj0Qxgc97rLUlbX+29vfdySS2SWtrb2wdTlpmZFaDgwJc0DrgH+GJE7Ct2IRGxIiKaI6K5qanfE81mZnYSCgp8SdXkwv7HEfGTXrq0AbPyXs9M2vpqNzOzYVbIVToCfgA8FxHf7KPbauCPk6t1LgX2RsQ24H5gsaQJkiYAi5M2MzMbZoVcpbMI+DjwlKQNSduXgdkAEXErcB/wPqAVOAh8Mlm2W9LXgHXJejdHxO7ilf+Wrq7guw+2cu7MBq48w4eEzMx6GjDwI+IhQAP0CeBzfSxbCaw8qeoGoaJC/P3aTXzowhkOfDOzXqTqXjrT60ezbe+hUpdhZjYipSrwp9bX8poD38ysV6kK/OkNtWzb+2apyzAzG5FSFfhTx49m5/4jHO7oLHUpZmYjTqoCf1p9LQDb9x4ucSVmZiNPugK/IRf4PqxjZnaidAV+sof/2j6fuDUz6ylVgT+1fjQAW/c48M3MekpV4I+rqaKutorXfEjHzOwEqQp8yB3W2epr8c3MTpDCwB/twVdmZr1IYeDX+vYKZma9SGHgj2bn/sMefGVm1kMKAz93aeaOfR58ZWaWL32Bnwy+2rrHV+qYmeVLX+B78JWZWa8GnABF0krg/cCOiHh7L8v/Avho3vudBTQls129ArwBdAIdEdFcrML70j34yiduzcyOV8ge/m3Akr4WRsQ3IuKCiLgA+Cvg33pMY/jOZPkpD3t4a/DVNh/SMTM7zoCBHxFrgULnoV0GrBpSRUXgSzPNzE5UtGP4ksaQ+5fAPXnNAfxC0npJy4v1WQOZ5qkOzcxOMOAx/EH4d8DDPQ7nXBERbZImA2sk/S75F8MJkh+E5QCzZ88eUiHT6mt5Zuu+Ib2HmVnaFPMqnaX0OJwTEW3J3x3AvcDCvlaOiBUR0RwRzU1NTUMqZGp9LTv3H+ZIR9eQ3sfMLE2KEviS6oErgZ/mtY2VVNf9HFgMPF2MzxvI9ORKne2+NNPM7JhCLstcBVwFNEraAtwEVANExK1Jtw8Bv4iIA3mrTgHuldT9ObdHxM+LV3rfptZ3z3x1iFkTxwzHR5qZjXgDBn5ELCugz23kLt/Mb9sEnH+yhQ3FdE91aGZ2gtSNtAUPvjIz600qA39cTRV1NVW+L76ZWZ5UBj7kbqLmG6iZmb0ltYE/tX60b6BmZpYntYE/vb6WrXsc+GZm3VIb+B58ZWZ2vNQGfvd98T34yswsJ8WB70szzczypTjwPfjKzCxfegO/wXv4Zmb5Uhv4HnxlZna81AY+5AZf+ZCOmVlOqgN/qme+MjM7JtWBP22857Y1M+uW7sBv8OArM7Nu6Q78+loiPPjKzAxSH/i5SzN9EzUzswICX9JKSTsk9TofraSrJO2VtCF5fCVv2RJJz0tqlfSlYhZeiO7BV75NsplZYXv4twFLBujz64i4IHncDCCpErgFuAY4G1gm6eyhFDtY3XPb+lp8M7MCAj8i1gK7T+K9FwKtEbEpIo4AdwDXnsT7nLS62mrqaqp8pY6ZGcU7hn+ZpI2SfibpnKRtBrA5r8+WpK1XkpZLapHU0t7eXqSycnv5HnxlZlacwH8CmBMR5wP/G/ink3mTiFgREc0R0dzU1FSEsnKmNXjwlZkZFCHwI2JfROxPnt8HVEtqBNqAWXldZyZtw8qDr8zMcoYc+JKmSlLyfGHynruAdcACSadJGgUsBVYP9fMGyzNfmZnlVA3UQdIq4CqgUdIW4CagGiAibgU+DPyJpA7gTWBpRATQIelG4H6gElgZEc+ckq3ox/SGtwZfzZo4Zrg/3sxsxBgw8CNi2QDLvwN8p49l9wH3nVxpxTE1b/CVA9/MsizVI20BpnvwlZkZkIHA9+ArM7Oc1Ad+XW014zz4ysws/YEPuXvqePCVmWVdJgJ/an2tD+mYWeZlIvCn149mqwPfzDIuE4HvwVdmZhkJ/O7BVzve8F6+mWVXJgK/e/CVr9QxsyzLROB3z3zlwDezLMtW4Hu0rZllWCYC34OvzMwyEviQ28v3tfhmlmWZCXxPdWhmWZeZwM/dXsF7+GaWXRkK/NG0e/CVmWXYgIEvaaWkHZKe7mP5RyU9KekpSY9IOj9v2StJ+wZJLcUsfLCm1XvwlZllWyF7+LcBS/pZ/jJwZUScC3wNWNFj+Tsj4oKIaD65EotjWkMy85UP65hZRhUyxeFaSXP7Wf5I3stHgZlDL6v4uq/F903UzCyrin0M/1PAz/JeB/ALSeslLe9vRUnLJbVIamlvby9yWfkzX/lKHTPLpgH38Asl6Z3kAv+KvOYrIqJN0mRgjaTfRcTa3taPiBUkh4Oam5ujWHV1G58Mvtq6x3v4ZpZNRdnDl3Qe8H3g2ojY1d0eEW3J3x3AvcDCYnzeyfJEKGaWZUMOfEmzgZ8AH4+IF/Lax0qq634OLAZ6vdJnuEyrr2XbPge+mWXTgId0JK0CrgIaJW0BbgKqASLiVuArwCTgu5IAOpIrcqYA9yZtVcDtEfHzU7ANBZtWX8vzrxX//ICZWTko5CqdZQMs/zTw6V7aNwHnn7hG6UxNBl8d7eyiujIzY87MzIAMjbQFmJ4MvtruwzpmlkGZCvy3Ls104JtZ9mQq8Kc3eKpDM8uuTAX+1GNTHXrwlZllT6YCf7xnvjKzDMtU4EMyEYpH25pZBmUu8D34ysyyKpOB7xuomVkWZS7wp9aPZscbucFXZmZZkrnAn35s5qvDpS7FzGxYZS7wj12auceHdcwsWzIX+NPqPfjKzLIpe4Hf4NsrmFk2ZS7w62qqGDuqkq2+UsfMMiZzgS+JaQ2jvYdvZpmTucCH3LX4Wx34ZpYxBQW+pJWSdkjqdYpC5XxbUqukJyW9I2/ZDZJeTB43FKvwoZg63oOvzCx7Ct3Dvw1Y0s/ya4AFyWM58D0ASRPJTYl4CbkJzG+SNOFkiy2WaQ25wVeHOzpLXYqZ2bApKPAjYi2wu58u1wI/ipxHgQZJ04CrgTURsTsiXgfW0P8Px7A4Z/p4ImDDq3tKXYqZ2bAp1jH8GcDmvNdbkra+2kvq0tMnUSF4+KVdpS7FzGzYjJiTtpKWS2qR1NLe3n5KP6t+dDXnzmzgkdadp/RzzMxGkmIFfhswK+/1zKStr/YTRMSKiGiOiOampqYildW3RfMmsWHzHvYf7jjln2VmNhIUK/BXA3+cXK1zKbA3IrYB9wOLJU1ITtYuTtpKbtH8Rjq6gsdf9mEdM8uGqkI6SVoFXAU0StpC7sqbaoCIuBW4D3gf0AocBD6ZLNst6WvAuuStbo6I/k7+DpuL5kxgVFUFD7fu4l1nTil1OWZmp1xBgR8RywZYHsDn+li2Elg5+NJOrdrqSi6eO4GHfRzfzDJixJy0LYXL5zXyu9feYOd+3xvfzNIv04G/aH4jAI/48kwzy4BMB/65M+qpq63y5ZlmlgmZDvzKCnHZ6ZN4yIFvZhmQ6cCH3GGdLa+/yau7Dpa6FDOzU8qBnxzHf/gl7+WbWbplPvDnNY1lyvgaX55pZqmX+cCXxKJ5jTzy0i66uqLU5ZiZnTKZD3yAy+c3svvAEX732hulLsXM7JRx4AOL5k8C4BEfxzezFHPgA9PqR3N601hfnmlmqebATyya18jjL+/mSEdXqUsxMzslHPiJRfMbOXikk41bPO2hmaWTAz9xWfe0hz6sY2Yp5cBP1I+p5u0z6h34ZpZaDvw8l89r5Lev7uGApz00sxRy4Oe5onvaw1dGxKRcZmZFVVDgS1oi6XlJrZK+1Mvyv5O0IXm8IGlP3rLOvGWri1l8sTXPTaY9fNGHdcwsfQac4lBSJXAL8F5gC7BO0uqIeLa7T0T8aV7//wJcmPcWb0bEBcUr+dSpra7kotkTeNgTophZChWyh78QaI2ITRFxBLgDuLaf/suAVcUorhSuWNDIc9v2scvTHppZyhQS+DOAzXmvtyRtJ5A0BzgN+FVec62kFkmPSvpgXx8iaXnSr6W9vb2Ask6Ny+flbrPwm03eyzezdCn2SdulwN0R0ZnXNicimoHrgW9JmtfbihGxIiKaI6K5qampyGUV7twZ9dTVVPnyTDNLnUICvw2Ylfd6ZtLWm6X0OJwTEW3J303Agxx/fH/Eqaqs4NJ5k3i41Xv4ZpYuhQT+OmCBpNMkjSIX6idcbSPpTGAC8Ju8tgmSapLnjcAi4Nme6440i+ZN4tXdB9m829Memll6DBj4EdEB3AjcDzwH3BURz0i6WdIH8rouBe6IiPxZRM4CWiRtBB4Avp5/dc9IdWzaQx/WMbMUGfCyTICIuA+4r0fbV3q8/mov6z0CnDuE+kpi/uRxTK6r4eGXdrF04exSl2NmVhQeadsLSSya38hvXtrJ8f9gMTMrXw78Plw+bxI79x/h+e2e9tDM0sGB34fu4/gP+TYLZpYSDvw+TG8YzemNY3nEt1kws5Rw4Pfj8vmTeGzTLo52etpDMyt/Dvx+LJrXyIEjnWzc7GkPzaz8OfD7cdm8SUiw9oXS3dvHzKxYHPj9aBgziivmN3L3+i10dvnyTDMrbw78AXz0ktls3XuIB5/fUepSzMyGxIE/gHefNYWmuhpuf+zVUpdiZjYkDvwBVFdWcF3zLB54fgdb97xZ6nLMzE6aA78ASxfOIoA71m0esK+Z2UjlwC/AzAljuPKMJu5c9yodvibfzMqUA79A1y+czfZ9h/nV73zy1szKkwO/QO86czJTx9fyY5+8NbMy5cAvUFVlBR+5eBZrX2z3TFhmVpYKCnxJSyQ9L6lV0pd6Wf4JSe2SNiSPT+ctu0HSi8njhmIWP9yWXjwLAXes816+mZWfAQNfUiVwC3ANcDawTNLZvXS9MyIuSB7fT9adCNwEXAIsBG6SNKFo1Q+z6Q2jedeZk7mrZYtvqGZmZaeQPfyFQGtEbIqII8AdwLUFvv/VwJqI2B0RrwNrgCUnV+rIcP0ls2l/4zC/fHZ7qUsxMxuUQgJ/BpB/AfqWpK2nfy/pSUl3S5o1yHWRtFxSi6SW9vaRe7OyK8+YzIyG0dz+uA/rmFl5KdZJ238G5kbEeeT24n842DeIiBUR0RwRzU1NTUUqq/gqK8R1F8/i1y/u5JWdB0pdjplZwQoJ/DZgVt7rmUnbMRGxKyIOJy+/D1xU6Lrl6LqLZ1FZIVb55K2ZlZFCAn8dsEDSaZJGAUuB1fkdJE3Le/kB4Lnk+f3AYkkTkpO1i5O2sjZlfC3vPnMyd7ds4UiHT96aWXkYMPAjogO4kVxQPwfcFRHPSLpZ0geSbp+X9IykjcDngU8k6+4GvkbuR2MdcHPSVvauv2Q2uw4c4f5nXit1KWZmBVHEyJvYo7m5OVpaWkpdRr+6uoI//MYDzJowhlXLLy11OWaWcZLWR0Rzf3080vYkVVSIZQtn85tNu3ipfX+pyzEzG5ADfwj+Q/NMqirEKt9fx8zKgAN/CCbX1bL4nCnc/cQWDh3tLHU5Zmb9cuAP0fUL57Dn4FF+/rRP3prZyObAH6LL501izqQxnvPWzEY8B/4QdZ+8ffyV3by4/Y1Sl2Nm1icHfhF8+KKZVFfK99cxsxHNgV8EjeNquPqcqdyzfgv7D3eUuhwzs1458IvkM39wOvsOdfAPD71c6lLMzHrlwC+S82c18N6zp7Di15vYe/BoqcsxMzuBA7+I/nzxGew/3MGKX79U6lLMzE7gwC+iM6eO5/3nTecfHn6FnfsPD7yCmdkwcuAX2Rffs4BDRzu59UHv5ZvZyOLAL7J5TeP4o3fM5B8f/T2v7T1U6nLMzI5x4J8CX3j3Aroi+M4DL5a6FDOzYxz4p8CsiWO47uJZ3LluM5t3Hyx1OWZmQIGBL2mJpOcltUr6Ui/L/0zSs5KelPSvkubkLeuUtCF5rO65blrd+M4FVEh8+1+9l29mI8OAgS+pErgFuAY4G1gm6ewe3X4LNEfEecDdwF/nLXszIi5IHh8gI6bW1/LxS+dwzxNbPEGKmY0IhezhLwRaI2JTRBwB7gCuze8QEQ9ERPexi0eBmcUtszx99qp51FZX8q1fei/fzEqvkMCfAWzOe70laevLp4Cf5b2uldQi6VFJHzyJGstW47gaPrloLv+8cSvPbdtX6nLMLOOKetJW0seAZuAbec1zkol1rwe+JWleH+suT34YWtrb24tZVkkt/4N51NVW8c01L5S6FDPLuEICvw2Ylfd6ZtJ2HEnvAf4b8IGIODbMNCLakr+bgAeBC3v7kIhYERHNEdHc1NRU8AaMdPVjqln+B6ez5tntbNy8p9TlmFmGFRL464AFkk6TNApYChx3tY2kC4G/Jxf2O/LaJ0iqSZ43AouAZ4tVfLn45BWnMXHsKP7We/lmVkIDBn5EdAA3AvcDzwF3RcQzkm6W1H3VzTeAccD/63H55VlAi6SNwAPA1yMic4E/rqaKP7lyHmtfaOfxl3eXuhwzyyhFRKlrOEFzc3O0tLSUuoyievNIJ1d+4wHmNo7lzuWXIqnUJZlZikhan5wv7ZNH2g6T0aMqufFd83n85d081Lqz1OWYWQY58IfRdRfPYkbDaP7mFy8wEv9lZWbp5sAfRjVVlXzh3QvYuHkPqzduLXU5ZpYxDvxh9kfvmMH5M+v50zs3sPKhl72nb2bDxoE/zKoqK7j9M5fynrOmcPO/PMuX732ao51dpS7LzDLAgV8CY2uquPVjF/Gfr5rHqsdf5eM/eIzXDxwpdVlmlnIO/BKpqBB/ueRM/u6683ni1T188LsP07rjjVKXZWYp5sAvsQ9dOJNVn7mUA4c7+NAtj/Dg8zsGXsnM7CQ48EeAi+ZM4Kc3XsHMiWP4j7et4wc+mWtmp4ADf4SY0TCauz97Ge85awpf+5dn+fK9T3Gkwydzzax4HPgjSPfJ3M+9cx6rHt/sk7lmVlRVpS7AjldRIf7i6jNZMLmOv7znSRZ/ay1XnzOFq86YzGXzJjG2xl+ZmZ0cp8cI9cELZzC3cSzf+VUrP3mijf/76KuMqqzg4tMmcNUZk7nybU0smDzON2Ezs4L5bpll4HBHJy2vvM6/vdDOg8/v4IXtuUnRp9fXcuXbmrjyjMksmj+JutrqEldqZqVSyN0yHfhlaOueN4+F/8Otu9h/uIOqCnHWtPG8fUY95yaPt02tY1SVT9OYZYEDPwOOdnax/vevs/aFdjZs3sPTbXvZd6gDgFGVFbxtap1/BMwywIGfQRHBq7sP8lTbXp5q28vTbXt5asvxPwKnN41lWn0tU+trmVxXy5TxtUytr2FyXa5t4phRVFT43IBZOSkk8As6aStpCfC/gErg+xHx9R7La4AfARcBu4DrIuKVZNlfAZ8COoHPR8T9g9wOGwRJzJk0ljmTxvL+86YDuR+Bzbvf5Mm2PTzVtpfW7ft5bd8hnmrbx64Dh+n5m19VISbX1TClvpZJY2uYOLaaicnfCWNGMXHsW48JY0dRV1Plk8dmZWDAwJdUCdwCvBfYAqyTtLrH3LSfAl6PiPmSlgL/E7hO0tnkJj0/B5gO/FLSGRHRWewNsb5JYvakMcyeNObYj0C3o51dtL9xmO37DiWPw7x27Pkhtrx+kKfajrD7wBGOdvb+r8HqSlE/upqaqkpqqyuora5MHhXUVuWe11RXMDppr66soKpCVFXq2PPKiuR5paiuyP2tTNorJSqSv5UVQuL49gpRIVEhqMjrk2tL2vP6iGR5hRC5fhLH1jmuDUGPZcp7D3q8VvK/d/fPX+59/WNoI0Mhe/gLgdaI2AQg6Q7gWiA/8K8Fvpo8vxv4jnL/L78WuCMiDgMvS2pN3u83xSnfhqq6soLpDaOZ3jC6334RwYEjnezef4TdB4/w+oHcj8DrB3N/97x5lENHOzl8tItDRzs51NHJoaNd7DmYaz/U3X60k6OdQUdXF10j72jiKdX9g5B7fvyPR66RY225l3k/KnnrHXvd17K835de18urp0eFfS7rbz0VvF7fP3wnrNdLV3Fi42B+S3vr2ltNg/p57rXOwj6nNxPHjOKuz142mAoGpZDAnwFsznu9Bbikrz4R0SFpLzApaX+0x7ozevsQScuB5QCzZ88upHYbRpIYV1PFuJoqZk8aU5T37OoKjnZ10dEZdHQFHZ1ddHQFRzu76OwKjnYGXRF0duUe3c+7greedwWd8VZbV97ySNq7143keZD7AYvufnBseUT++hzrCxzXP7csjh0Oi+P6J2281UayHj3W7dnGsXWP/9zutuP65i3j2LK3GvL7d9dDj7aefXtbRn/rRX6v6GdZ4eud0Ln3pkHdb6r39Qvr1+d79vIGva4/iDetqz21Q6NGzMCriFgBrIDcSdsSl2PDoKJC1FRU4sHDZsOjkOvz2oBZea9nJm299pFUBdSTO3lbyLpmZjYMCgn8dcACSadJGkXuJOzqHn1WAzckzz8M/Cpy/95ZDSyVVCPpNGAB8HhxSjczs8EY8B/TyTH5G4H7yV2WuTIinpF0M9ASEauBHwD/mJyU3U3uR4Gk313kTvB2AJ/zFTpmZqXhgVdmZilQyMArj7E3M8sIB76ZWUY48M3MMsKBb2aWESPypK2kduD3J7l6I7CziOWUWtq2B9K3TWnbHkjfNqVte+DEbZoTEU39rTAiA38oJLUMdKa6nKRteyB925S27YH0bVPatgdObpt8SMfMLCMc+GZmGZHGwF9R6gKKLG3bA+nbprRtD6Rvm9K2PXAS25S6Y/hmZta7NO7hm5lZLxz4ZmYZkZrAl7RE0vOSWiV9qdT1FIOkVyQ9JWmDpLK8m5yklZJ2SHo6r22ipDWSXkz+TihljYPRx/Z8VVJb8j1tkPS+UtY4GJJmSXpA0rOSnpH0haS9nL+jvrapLL8nSbWSHpe0Mdme/560nybpsSTz7kxuX9//e6XhGH4y0foL5E20DizrMdF62ZH0CtAcEWU7YETSHwL7gR9FxNuTtr8GdkfE15Mf5wkR8V9LWWeh+tierwL7I+JvSlnbyZA0DZgWEU9IqgPWAx8EPkH5fkd9bdNHKMPvKZkffGxE7JdUDTwEfAH4M+AnEXGHpFuBjRHxvf7eKy17+McmWo+II0D3ROtWYhGxltwcCfmuBX6YPP8huf8Yy0If21O2ImJbRDyRPH8DeI7cvNPl/B31tU1lKXL2Jy+rk0cA7wLuTtoL+o7SEvi9TbRetl9wngB+IWl9Msl7WkyJiG3J89eAKaUspkhulPRkcsinbA5/5JM0F7gQeIyUfEc9tgnK9HuSVClpA7ADWAO8BOyJiI6kS0GZl5bAT6srIuIdwDXA55LDCamSTIVZ7scVvwfMAy4AtgF/W9pyBk/SOOAe4IsRsS9/Wbl+R71sU9l+TxHRGREXkJsXfCFw5sm8T1oCP5WTpUdEW/J3B3AvuS86DbYnx1m7j7fuKHE9QxIR25P/ILuA/0OZfU/JceF7gB9HxE+S5rL+jnrbpnL/ngAiYg/wAHAZ0CCpe5ragjIvLYFfyETrZUXS2OSEE5LGAouBp/tfq2zkT3p/A/DTEtYyZN3BmPgQZfQ9JScEfwA8FxHfzFtUtt9RX9tUrt+TpCZJDcnz0eQuTnmOXPB/OOlW0HeUiqt0AJJLrL7FWxOt/48SlzQkkk4nt1cPucnmby/HbZK0CriK3K1ctwM3Af8E3AXMJncb7I9ERFmcCO1je64id5gggFeA/5R3/HtEk3QF8GvgKaAraf4yuWPe5fod9bVNyyjD70nSeeROylaS20m/KyJuTjLiDmAi8FvgYxFxuN/3Skvgm5lZ/9JySMfMzAbgwDczywgHvplZRjjwzcwywoFvZpYRDnwzs4xw4JuZZcT/B9cS5hlK5DZaAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    }
  ]
}