Lab12/Lab12_2_LEEJOOYOUNG.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Lab12-2_LEEJOOYOUNG.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "sZq-P-r-m-Ee",
        "outputId": "9c09d4e8-9b8b-4b26-9071-31115d35ae83"
      },
      "source": [
        "# setup\n",
        "import numpy as np\n",
        "import tensorflow as tf\n",
        "import matplotlib.pyplot as plt\n",
        "from tensorflow import keras\n",
        "from tensorflow.keras import layers\n",
        "from tensorflow.keras import Sequential, Model\n",
        "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
        "from pprint import pprint\n",
        "%matplotlib inline\n",
        "\n",
        "print(tf.__version__)"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2.7.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0iPmIALOnPPA"
      },
      "source": [
        "# example data\n",
        "sentences = ['What I cannot create, I do not understand.',\n",
        "             'Intellecuals solve problems, geniuses prevent them',\n",
        "             'A person who never made a mistake never tied anything new.',\n",
        "             'The same equations have the same solutions.']\n",
        "y_data = [1,0,0,1] # 1: richard feynman, 0: albert einstein"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "a4AsTIIBnRr4",
        "outputId": "1be2b300-f42d-4544-9cdb-0253059eed3a"
      },
      "source": [
        "# creating a token dictionary\n",
        "char_set = ['<pad>'] + sorted(list(set(''.join(sentences))))\n",
        "idx2char = {idx : char for idx, char in enumerate(char_set)}\n",
        "char2idx = {char : idx for idx, char in enumerate(char_set)}\n",
        "\n",
        "print(char_set)\n",
        "print(idx2char)\n",
        "print(char2idx)"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['<pad>', ' ', ',', '.', 'A', 'I', 'T', 'W', 'a', 'b', 'c', 'd', 'e', 'g', 'h', 'i', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'y']\n",
            "{0: '<pad>', 1: ' ', 2: ',', 3: '.', 4: 'A', 5: 'I', 6: 'T', 7: 'W', 8: 'a', 9: 'b', 10: 'c', 11: 'd', 12: 'e', 13: 'g', 14: 'h', 15: 'i', 16: 'k', 17: 'l', 18: 'm', 19: 'n', 20: 'o', 21: 'p', 22: 'q', 23: 'r', 24: 's', 25: 't', 26: 'u', 27: 'v', 28: 'w', 29: 'y'}\n",
            "{'<pad>': 0, ' ': 1, ',': 2, '.': 3, 'A': 4, 'I': 5, 'T': 6, 'W': 7, 'a': 8, 'b': 9, 'c': 10, 'd': 11, 'e': 12, 'g': 13, 'h': 14, 'i': 15, 'k': 16, 'l': 17, 'm': 18, 'n': 19, 'o': 20, 'p': 21, 'q': 22, 'r': 23, 's': 24, 't': 25, 'u': 26, 'v': 27, 'w': 28, 'y': 29}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JS2oOwPfnVYU",
        "outputId": "983b7fb7-56e9-4fca-8934-6f304a8e0cb4"
      },
      "source": [
        "# converting sequence of tokens to sequence of indices\n",
        "x_data = list(map(lambda sentence : [char2idx.get(char) for char in sentence], sentences))\n",
        "x_data_len = list(map(lambda sentence : len(sentence), sentences))\n",
        "\n",
        "print(x_data)\n",
        "print(x_data_len)\n",
        "print(y_data)"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[[7, 14, 8, 25, 1, 5, 1, 10, 8, 19, 19, 20, 25, 1, 10, 23, 12, 8, 25, 12, 2, 1, 5, 1, 11, 20, 1, 19, 20, 25, 1, 26, 19, 11, 12, 23, 24, 25, 8, 19, 11, 3], [5, 19, 25, 12, 17, 17, 12, 10, 26, 8, 17, 24, 1, 24, 20, 17, 27, 12, 1, 21, 23, 20, 9, 17, 12, 18, 24, 2, 1, 13, 12, 19, 15, 26, 24, 12, 24, 1, 21, 23, 12, 27, 12, 19, 25, 1, 25, 14, 12, 18], [4, 1, 21, 12, 23, 24, 20, 19, 1, 28, 14, 20, 1, 19, 12, 27, 12, 23, 1, 18, 8, 11, 12, 1, 8, 1, 18, 15, 24, 25, 8, 16, 12, 1, 19, 12, 27, 12, 23, 1, 25, 15, 12, 11, 1, 8, 19, 29, 25, 14, 15, 19, 13, 1, 19, 12, 28, 3], [6, 14, 12, 1, 24, 8, 18, 12, 1, 12, 22, 26, 8, 25, 15, 20, 19, 24, 1, 14, 8, 27, 12, 1, 25, 14, 12, 1, 24, 8, 18, 12, 1, 24, 20, 17, 26, 25, 15, 20, 19, 24, 3]]\n",
            "[42, 50, 58, 43]\n",
            "[1, 0, 0, 1]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xFjrR8cDnX7g",
        "outputId": "902a3d01-5790-41fb-c8ab-c4bb979881d8"
      },
      "source": [
        "# padding the sequence of indices\n",
        "max_sequence = 55\n",
        "x_data = pad_sequences(sequences = x_data, maxlen = max_sequence,\n",
        "                       padding = 'post', truncating = 'post')\n",
        "\n",
        "# checking data\n",
        "print(x_data)\n",
        "print(x_data_len)\n",
        "print(y_data)"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[[ 7 14  8 25  1  5  1 10  8 19 19 20 25  1 10 23 12  8 25 12  2  1  5  1\n",
            "  11 20  1 19 20 25  1 26 19 11 12 23 24 25  8 19 11  3  0  0  0  0  0  0\n",
            "   0  0  0  0  0  0  0]\n",
            " [ 5 19 25 12 17 17 12 10 26  8 17 24  1 24 20 17 27 12  1 21 23 20  9 17\n",
            "  12 18 24  2  1 13 12 19 15 26 24 12 24  1 21 23 12 27 12 19 25  1 25 14\n",
            "  12 18  0  0  0  0  0]\n",
            " [ 4  1 21 12 23 24 20 19  1 28 14 20  1 19 12 27 12 23  1 18  8 11 12  1\n",
            "   8  1 18 15 24 25  8 16 12  1 19 12 27 12 23  1 25 15 12 11  1  8 19 29\n",
            "  25 14 15 19 13  1 19]\n",
            " [ 6 14 12  1 24  8 18 12  1 12 22 26  8 25 15 20 19 24  1 14  8 27 12  1\n",
            "  25 14 12  1 24  8 18 12  1 24 20 17 26 25 15 20 19 24  3  0  0  0  0  0\n",
            "   0  0  0  0  0  0  0]]\n",
            "[42, 50, 58, 43]\n",
            "[1, 0, 0, 1]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "faW6N49cnaiz"
      },
      "source": [
        "# creating stacked rnn for \"many to one\" classification with dropout\n",
        "num_classes = 2\n",
        "hidden_dims = [10,10]\n",
        "\n",
        "input_dim = len(char2idx)\n",
        "output_dim = len(char2idx)\n",
        "one_hot = np.eye(len(char2idx))\n",
        "\n",
        "model = Sequential()\n",
        "model.add(layers.Embedding(input_dim=input_dim, output_dim=output_dim,\n",
        "                           trainable=False, mask_zero=True, input_length=max_sequence,\n",
        "                           embeddings_initializer=keras.initializers.Constant(one_hot)))\n",
        "model.add(layers.SimpleRNN(units=hidden_dims[0], return_sequences=True))\n",
        "model.add(layers.TimeDistributed(layers.Dropout(rate = .2)))\n",
        "model.add(layers.SimpleRNN(units=hidden_dims[1]))\n",
        "model.add(layers.Dropout(rate = .2))\n",
        "model.add(layers.Dense(units=num_classes))"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "i5yT99QYneO_",
        "outputId": "8ff30092-cdce-4495-8680-66bb09b37dea"
      },
      "source": [
        "model.summary()"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"sequential\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " embedding (Embedding)       (None, 55, 30)            900       \n",
            "                                                                 \n",
            " simple_rnn (SimpleRNN)      (None, 55, 10)            410       \n",
            "                                                                 \n",
            " time_distributed (TimeDistr  (None, 55, 10)           0         \n",
            " ibuted)                                                         \n",
            "                                                                 \n",
            " simple_rnn_1 (SimpleRNN)    (None, 10)                210       \n",
            "                                                                 \n",
            " dropout_1 (Dropout)         (None, 10)                0         \n",
            "                                                                 \n",
            " dense (Dense)               (None, 2)                 22        \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 1,542\n",
            "Trainable params: 642\n",
            "Non-trainable params: 900\n",
            "_________________________________________________________________\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XkUihB-Gngzx"
      },
      "source": [
        "# creating loss function\n",
        "def loss_fn(model, x, y, training):    \n",
        "    return tf.reduce_mean(tf.keras.losses.sparse_categorical_crossentropy(\n",
        "        y_true=y, y_pred=model(x, training), from_logits=True))\n",
        "\n",
        "# creating and optimizer\n",
        "lr = .01\n",
        "epochs = 30\n",
        "batch_size = 2\n",
        "opt = tf.keras.optimizers.Adam(learning_rate = lr)"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "X_GJSSIQnj1r",
        "outputId": "751a3f09-a2f0-478d-999c-071ee73a3189"
      },
      "source": [
        "# generating data pipeline\n",
        "tr_dataset = tf.data.Dataset.from_tensor_slices((x_data, y_data))\n",
        "tr_dataset = tr_dataset.shuffle(buffer_size=4)\n",
        "tr_dataset = tr_dataset.batch(batch_size=batch_size)\n",
        "\n",
        "print(tr_dataset)"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "<BatchDataset shapes: ((None, 55), (None,)), types: (tf.int32, tf.int32)>\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "h-95WeUOnmnv",
        "outputId": "8b92c485-f87a-4c71-cd02-ce31b1da8ac9"
      },
      "source": [
        "# training\n",
        "tr_loss_hist = []\n",
        "\n",
        "for epoch in range(epochs):\n",
        "    avg_tr_loss = 0\n",
        "    tr_step = 0\n",
        "    \n",
        "    for x_mb, y_mb in tr_dataset:\n",
        "        with tf.GradientTape() as tape:\n",
        "            tr_loss = loss_fn(model, x=x_mb, y=y_mb, training=True)\n",
        "        grads = tape.gradient(target=tr_loss, sources=model.variables)\n",
        "        opt.apply_gradients(grads_and_vars=zip(grads, model.variables))\n",
        "        avg_tr_loss += tr_loss\n",
        "        tr_step += 1\n",
        "    else:\n",
        "        avg_tr_loss /= tr_step\n",
        "        tr_loss_hist.append(avg_tr_loss)\n",
        "    \n",
        "    if (epoch + 1) % 5 ==0:\n",
        "        print('epoch : {:3}, tr_loss : {:.3f}'.format(epoch + 1, avg_tr_loss.numpy()))"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "epoch :   5, tr_loss : 0.125\n",
            "epoch :  10, tr_loss : 0.022\n",
            "epoch :  15, tr_loss : 0.014\n",
            "epoch :  20, tr_loss : 0.016\n",
            "epoch :  25, tr_loss : 0.087\n",
            "epoch :  30, tr_loss : 0.004\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Lfh4kF-dnpVK",
        "outputId": "100187f6-3f1b-4713-eb24-3c7ed1cf03ed"
      },
      "source": [
        "yhat = model.predict(x_data)\n",
        "yhat = np.argmax(yhat, axis=-1)\n",
        "print('accuracy : {:.2%}'.format(np.mean(yhat == y_data)))"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "accuracy : 100.00%\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 283
        },
        "id": "R5zKdw0fnsWM",
        "outputId": "652545e1-1908-4274-e1d7-85cd2bf9c39b"
      },
      "source": [
        "plt.plot(tr_loss_hist)"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[<matplotlib.lines.Line2D at 0x7fdea82e12d0>]"
            ]
          },
          "metadata": {},
          "execution_count": 12
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXUAAAD4CAYAAAATpHZ6AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXyU5bn/8c+VneyZEMKSTBL2HYWAotadilq1ttpCa7U9nqpV/Nl6elqtv3os7fnZ1W6H1rq16qmlaNVipUXcNxQCsoWwQ0hYA1nInkxy/f6YCYaYZRImmcwz1/v1youZZ+7MXE+f+p0n93M/9y2qijHGGGeICHYBxhhjAsdC3RhjHMRC3RhjHMRC3RhjHMRC3RhjHCQqWB88dOhQzc3NDdbHG2NMSFq3bt0xVc3o6vWghXpubi4FBQXB+nhjjAlJIlLc3evW/WKMMQ5ioW6MMQ5ioW6MMQ5ioW6MMQ5ioW6MMQ5ioW6MMQ5ioW6MMQ4ScqG+dl85P/nXNmzKYGOM+aSQC/XNpVX8/s3dlNc2BbsUY4wZdEIu1HPS4wEoLq8LciXGGDP4+BXqIjJfRLaLyC4RuaeT138pIht8PztEpDLwpXq1hfr+4xbqxhjTUY9zv4hIJLAEmAeUAmtFZLmqbm1ro6rfatf+TuDMfqgVgKw0X6jbmboxxnyCP2fqc4BdqrpHVZuApcA13bRfCPwlEMV1Ji46kuHJcRTbmboxxnyCP6E+Cihp97zUt+0TRCQHyANe7+L1W0SkQEQKysrKelvrSe70ePaX1/b5940xxqkCfaF0AfCcqrZ09qKqPqKq+aqan5HR5XTAPcpxxduZujHGdMKfUD8AZLd7nuXb1pkF9GPXS5uc9HiOVjdS39Tpd4cxxoQtf0J9LTBORPJEJAZvcC/v2EhEJgJpwOrAlvhJ2S7vxdKSCjtbN8aY9noMdVX1AIuAlUARsExVC0VksYhc3a7pAmCpDsCtnjnpCQDWBWOMMR34tZydqq4AVnTYdn+H5w8Erqzu5fjO1IuP28VSY4xpL+TuKAVIjY8mKS6KEhurbowxpwjJUBcR3K54myrAGGM6CMlQB+8IGJsqwBhjThWyoe52JVBSUUdLq03Ba4wxbUI41ONpblEOn2gIdinGGDNohGyon5yC10bAGGPMSSEb6m6XTcFrjDEdhWyoj0wdQlSE2AgYY4xpJ2RDPTJCyEobYvOqG2NMOyEb6gDu9ATrfjHGmHZCOtS9U/DahVJjjGkT2qGeHs+JBg+VdU3BLsUYYwaFkA71til4rV/dGGO8QjrUPx6rbqFujDEQ4qHutjN1Y4w5RUiHenxMFBlJsXax1BhjfEI61MF7tm5n6sYY4xXyoZ7jsil4jTGmTciHujs9nkMnGmj0tAS7FGOMCTq/Ql1E5ovIdhHZJSL3dNHmCyKyVUQKReSZwJbZtZz0eFShtKJ+oD7SGGMGrR4XnhaRSGAJMA8oBdaKyHJV3dquzTjgXuBcVa0QkWH9VXBH7WdrHJOROFAfa4wxg5I/Z+pzgF2qukdVm4ClwDUd2nwdWKKqFQCqejSwZXbN7UoAbF51Y4wB/0J9FFDS7nmpb1t744HxIvKeiHwgIvM7eyMRuUVECkSkoKysrG8VdzA0MYb4mEibgtcYYwjchdIoYBxwIbAQeFREUjs2UtVHVDVfVfMzMjIC8sEigtsVT4mFujHG+BXqB4Dsds+zfNvaKwWWq2qzqu4FduAN+QHhdsXbVAHGGIN/ob4WGCcieSISAywAlndo8yLes3REZCje7pg9AayzWznp3huQWlt1oD7SGGMGpR5DXVU9wCJgJVAELFPVQhFZLCJX+5qtBI6LyFbgDeA/VfV4fxXdkTs9gUZPK0erGwfqI40xZlDqcUgjgKquAFZ02HZ/u8cK3O37GXA57Sb2Gp4SF4wSjDFmUAj5O0rh47HqNqzRGBPuHBHqo9KGEBkhNrGXMSbsOSLUoyMjGJkaZyNgjDFhzxGhDpDjSrAzdWNM2HNMqGfbvOrGGOOcUM9Jj6e8tonqhuZgl2KMMUHjnFB32SLUxhjjmFDP9oW6zQFjjAlnjgn1nHTfmbqFujEmjDkm1JPionElxFj3izEmrDkm1MF7Z+n+crur1BgTvhwY6nambowJX44K9Zz0eA5WNtDc0hrsUowxJigcFepuVzwtrcqBivpgl2KMMUHhqFDPSfctQm1dMMaYMOWoUHe3m1fdGGPCkaNCfVhSLLFREey3edWNMWHKUaEeESG2CLUxJqw5KtTh40WojTEmHPkV6iIyX0S2i8guEbmnk9e/KiJlIrLB9/PvgS/VP21T8HqXTTXGmPDS48LTIhIJLAHmAaXAWhFZrqpbOzT9q6ou6ocaeyXHFU9dUwvHaprISIoNdjnGGDOg/DlTnwPsUtU9qtoELAWu6d+y+q5tWKNNF2CMCUf+hPoooKTd81Lfto4+LyKbROQ5Ecnu7I1E5BYRKRCRgrKysj6U2zN3us2rbowJX4G6UPoSkKuq04FVwJOdNVLVR1Q1X1XzMzIyAvTRp8pKG4KIjVU3xoQnf0L9AND+zDvLt+0kVT2uqo2+p48BswJTXu/FRkUyIjmO/XambowJQ/6E+lpgnIjkiUgMsABY3r6BiIxo9/RqoChwJfaeOz3epgowxoSlHke/qKpHRBYBK4FI4AlVLRSRxUCBqi4H/o+IXA14gHLgq/1Yc49yXAm8tu1oMEswxpig6DHUAVR1BbCiw7b72z2+F7g3sKX1nTs9nmM1jdQ1eYiP8WsXjTHGERx3RynYxF7GmPDlyFDPsWGNxpgw5cxQd/luQLJQN8aEGUeGekp8NMlxUdb9YowJO44MdfBOF2DDGo0x4caxoe5Oj7fFMowxYcexoZ7jiqe0oh5PS2uwSzHGmAHj2FB3u+LxtCqHqhqCXYoxxgwY54Z6uo1VN8aEH8eGetu86jZW3RgTThwb6sOT44iJimBPWU2wSzHGmAHj2FCPjBCmjUph3f6KYJdijDEDxrGhDjA718Xm0irqm1qCXYoxxgwIR4f6WXkuPK3KRyV2tm6MCQ+ODvWZOWmIwJq95cEuxRhjBoSjQz1lSDQThyezdp+FujEmPDg61AHm5KaxvriSZruz1BgTBhwf6rPzXNQ3t1B48ESwSzHGmH7nV6iLyHwR2S4iu0Tknm7afV5EVETyA1fi6ZmT6wJgzd7jQa7EGGP6X4+hLiKRwBLgcmAysFBEJnfSLgm4C/gw0EWejmHJceSmx7Nmr42AMcY4nz9n6nOAXaq6R1WbgKXANZ20+yHwE2DQzaA1O9dFQXE5ra0a7FKMMaZf+RPqo4CSds9LfdtOEpGZQLaqvtzdG4nILSJSICIFZWVlvS62r2bnuaisa2aXTRlgjHG4075QKiIRwEPAf/TUVlUfUdV8Vc3PyMg43Y/221l53n71D228ujHG4fwJ9QNAdrvnWb5tbZKAqcCbIrIPOBtYPpgulrpd8QxLimWthboxxuH8CfW1wDgRyRORGGABsLztRVWtUtWhqpqrqrnAB8DVqlrQLxX3gYgwO8/F2n3lqFq/ujHGuXoMdVX1AIuAlUARsExVC0VksYhc3d8FBsqcXBeHqhooragPdinGGNNvovxppKorgBUdtt3fRdsLT7+swJvj61dfu6+cbFd8kKsxxpj+4fg7SttMyEwiOS7KJvcyxjha2IR6RISQn+tijU3uZYxxsLAJdfDehLSnrJZjNY3BLsUYY/pFWIX6nLw0AArsbN0Y41BhFerTRqUSGxVhNyEZYxwrrEI9JiqCM92ptmiGMcaxwirUwTtefevBE1Q3NAe7FGOMCbiwC/XZeS5aFdbvrwx2KcYYE3BhF+oz3WlERogtmmGMcaSwC/WE2CimjkxmrS2aYYxxoLALdfCOV99QWkmjpyXYpRhjTECFZ6jnuWjytLKptCrYpRhjTECFZ6ifXIzahjYaY5wlLEPdlRDDuGGJFurGGMcJy1AHbxfM+uIKWmwxamOMg4RtqM/JdVHd6KHo0Ilgl2KMMQETvqHebtEMY4xxirAN9ZGpQxiVOsT61Y0xjhK2oQ7es3VbjNoY4yR+hbqIzBeR7SKyS0Tu6eT120Rks4hsEJF3RWRy4EsNvNm5Lo7VNLH3WG2wSzHGmIDoMdRFJBJYAlwOTAYWdhLaz6jqNFU9A/gp8FDAK+0HbYtmWL+6McYp/DlTnwPsUtU9qtoELAWuad9AVdsPIUkAQqI/Y0xGIq6EGFs0wxjjGFF+tBkFlLR7Xgqc1bGRiNwB3A3EABd39kYicgtwC4Db7e5trQEnIszOTbMzdWOMYwTsQqmqLlHVMcB3gf/bRZtHVDVfVfMzMjIC9dGnZXaui5Lyeg5XNQS7FGOMOW3+hPoBILvd8yzftq4sBT57OkUNpLbx6mvsbN0Y4wD+hPpaYJyI5IlIDLAAWN6+gYiMa/f0SmBn4ErsX5NHJJMQE2mLZhhjHKHHPnVV9YjIImAlEAk8oaqFIrIYKFDV5cAiEbkUaAYqgJv6s+hAioqMYGZOmi2aYYxxBH8ulKKqK4AVHbbd3+7xXQGua0Cdlefi56/soKy6kYyk2GCXY4wxfRbWd5S2uWjiMABe33YkyJUYY8zpsVDH268+KnUIq7ZaqBtjQpuFOt7x6vMmZ/LOzmPUNXmCXY4xxvSZhbrPvMmZNHpaeXvHsWCXYowxfWah7jMnz0VyXJR1wRhjQpqFuk90ZAQXTxzG69uO4GlpDXY5xhjTJxbq7cybPJyKumbWFduYdWNMaLJQb+eCCRnEREZYF4wxJmRZqLeTGBvF3DHprCo6YqshGWNCkoV6B/MmZ1J8vI6dR2uCXYoxxvSahXoH8yZnAlgXjDEmJFmod5CZHMeMrBResVA3xoQgC/VOzJucycaSSo6csIUzjDGhxUK9E/MmDwfg1SI7WzfGhBYL9U6Mz0zE7Yq3fnVjTMixUO9E2wRf7+86Tk2jTfBljAkdFupdmDc5k6aWVt7aXhbsUowxxm8W6l3Iz0kjNT6aVVsPB7sUY4zxm4V6F6IiI7hkYiavbztKs03wZYwJEX6FuojMF5HtIrJLRO7p5PW7RWSriGwSkddEJCfwpQ68eZMzOdHgYe3e8mCXYowxfukx1EUkElgCXA5MBhaKyOQOzT4C8lV1OvAc8NNAFxoM548fSmxUhN2IZIwJGf6cqc8BdqnqHlVtApYC17RvoKpvqGqd7+kHQFZgywyO+Jgozhs7lFVbbYIvY0xo8CfURwEl7Z6X+rZ15Wbgn529ICK3iEiBiBSUlYXGqJJ5kzM5UFlP0aHqYJdijDE9CuiFUhG5AcgHftbZ66r6iKrmq2p+RkZGID+631wyKRMRm+CrNxo9LRysrA92GcaEJX9C/QCQ3e55lm/bKUTkUuA+4GpVbQxMecGXkRTLmdmprCqyoY3+evjNPcx76C0amluCXYoxYcefUF8LjBORPBGJARYAy9s3EJEzgT/gDfSjgS8zuOZNHs6WAyfs7NNP7+8+Rm1TC5tKq4JdijFhp8dQV1UPsAhYCRQBy1S1UEQWi8jVvmY/AxKBZ0Vkg4gs7+LtQlLbHOs2wVfPmlta2VhaCWBrvRoTBFH+NFLVFcCKDtvub/f40gDXNaiMHZbI6KEJrNp6hBvn5ga7nEGt6NAJGpq9N2ut32+hbsxAsztK/TRvciardx+nqr452KUMam1n5+eNHcr64gobCmrMALNQ99O8yZl4WpU3tzvukkFArSuuYGRKHFdOH8Hx2iaKj9f1/EvGmICxUPfTme40hibG2NDGHqwvrmBmThoz3WmA9asbM9As1P0UGSFcMjGTt7aX0eSxCb46c7CynoNVDczKSWPcsESSYqNYZ/3qxgwoC/VemDc5k+pGDx/sOR7sUgaltgujs3LSiIgQzsxJY72dqRszoCzUe+G8cUNJiovi/r9vYe+x2mCXM+isK64gLjqCSSOSAZjlTmP7kWqqG+zisjEDxUK9F+KiI/nT12ZzosHDtb97jw/tjP0U64srmJGVSnSk9/9Ws3LSUIUNJZVBrsyY8GGh3kuzcly8cPs5uBJiuOHxD3nho9JglzQo1De1UHjwBLNy0k5um5GdgohdLDVmIFmo90FOegIvfONc8nNcfOuvG/nlqh1hPx57U2klnlY9JdST4qKZkJnE+v12pm7MQLFQ76OU+Gie/Lc5XD8ri1+/tpNv/XUDjZ7wncCqbZTLme60U7bPyknjo+IKWlvD+0vPmIFioX4aYqIi+Ol10/nPyybw4oaD3PDYh5TXNgW7rKBYX1zB6IwEXAkxp2yflZNGdaOHnUdrglSZMeHFQv00iQh3XDSW3y48k42lVVz7u/fYUxZeAaaqrN9fyawOZ+mA3YRkzACzUA+Qq2aM5C9fP4vqBg/X/u79sBrLvu94HeW1Taf0p7fJSY8nPSHGJvcyZoBYqAfQrBwXL95+LkMTY/jK4x/yt3XhMTKm7Sy8s1AXEWbaTUjGDBgL9QBzp8fz/DfOZXaui/94diM//uc2Whx+kXBdcQXJcVGMyUjs9PVZOWnsOVYbttcbjBlIFur9oG1kzJfPcvPwW7v5+lMFjr6rsm0Sr4gI6fT1tn51O1s3pv9ZqPeT6MgI/vvaafzws1N5a0cZ1/7uffY5cGqBqvpmdhyt7vQiaZvpWSlERYhN7mXMALBQ72dfOTuHp2+ew7GaRq5Z8h7v7ToW7JICakNJJaqd96e3iYuOZMqoFDtTN2YA+BXqIjJfRLaLyC4RuaeT188XkfUi4hGR6wJfZmg7Z8xQlt9xHpnJsdz4xBr+9N5ex9yBuq64ggiBGdmp3bab5U5jY2klzS02bbEx/anHUBeRSGAJcDkwGVgoIpM7NNsPfBV4JtAFOoU7PZ7nbz+XiyYM44GXtnLv85sdMS/7+uIKJg5PJiG2++VuZ+ak0tDcStGhEwNUmTHhyZ8z9TnALlXdo6pNwFLgmvYNVHWfqm4CQj+l+lFibBSPfGUWiy4ay9K1JXz5sQ84VtMY7LL6rKVV+Wh/RbddL23a2thNSMb0L39CfRRQ0u55qW9br4nILSJSICIFZWVlfXmLkBcRIXz7sgn8ZuGZbCqt4pr/eY/Cg1XBLqtPth+uprapxa9QH5EyhJEpcTa5lzH9bEAvlKrqI6qar6r5GRkZA/nRg87VM0by7G1zaWlVrvv9at7aEXpfcuv2d33TUWfsJiRj+p8/oX4AyG73PMu3zZym6VmpLF90LrlDE7jjz+vZdbQ62CX1yvriCjKSYslKG+JX+1k5aRyorOdQVX0/V2ZM+PIn1NcC40QkT0RigAXA8v4tK3wMS47j8ZvyiYuO4N+fLKCqrv9vUmpobuHtHWX86B9b+fQv3+L8n77Rp89dV1zBLHcaIp3fdNTRxzchWReM8c+KzYf43gubHTNabCD0GOqq6gEWASuBImCZqhaKyGIRuRpARGaLSClwPfAHESnsz6KdZmTqEB6+YRYHKutZ9Jf1eAI87E9V2XW0msff3ctNT6zhjMWvcOMTa3hqdTHpCbGUVtTxq9d29Oo9j1Y3sL+8zu+uF4DJI5OJi46wi6XGLw3NLfzgpUKe+XA/7+8OnwnyTlf349B8VHUFsKLDtvvbPV6Lt1vG9FF+rovF10zl3uc385N/beO+KzuOGu2dqvpm3tt1jLd3lPH2jjIOVjUAMDojgQWz3VwwPoOzRruIj4niey9s5qnVxXxpjptxmUl+vX/b2fbMXoR6dGQE07NSbcZG45dn15Vy5EQjQ6Ij+f2buzl37NBglxQS/Ap1MzAWznFTdOgEj76zl4nDk/n8rL59T64sPMy3l22kutFDUmwU544dyqKLMzh//FCy0uI/0f4/5o3npY0HWfyPrTz1b3P86k5Zv7+CmMgIpo5K7lVts3LSeOydPTQ0txAXHdmr3zXho8nTysNv7mZWThqXTsrkJ//axubSKqZlpQS7tEHPpgkYZL7/mcnMHZ3OvS9sZkNJ7/qeW1qVn63cxq1PryMvI4Flt85l/f3zePgrs/jSWe5OAx0gPTGWb146nnd2HuP1bUf9+qx1xRVMy0ohNqp3wTzTnUZzi7L5QGgO4zQD44WPSjlQWc+dF4/lhrPdJMVF8fBbu4NdVkiwUB9koiMjWPLlmQxLiuWWpwo4cqLBr9+rqG3iq39cw5I3drNgdjbLbp3LnDwX0ZH+HeIb5+YwJiOBH/5ja493ujZ6WthcWtWr/vQ2M93e6QSsX910xdPSypI3djM9K4ULxmeQFBfNDWfnsGLLIfY6cFK8QLNQH4RcCTE8dlM+NY0ebn16HQ3N3S9oveVAFVf9z7t8uKecBz83jR9/fnqvuzaiIyO4/6op7Dtexx/f29vD552gqaX15GiW3khPjCVvaIKNVzdd+vuGg+wvr+POi8ed7Ar82rm5REdG8MjbdrbeEwv1QWri8GQe+sIMNpRUct8LW7oc0vXculI+//v3aWlVlt02l4Vz3H3+zAvGZ3DJxGH89vVdHK3u+i+EtkCemdP9JF5dmelOY/3+ChumZj6hpVVZ8sYuJo1I5tJJw05uH5YUx/WzsvjbugMc9fOv13BloT6IzZ86grsuGcff1pfyxHv7TnmtydPK91/cwref3chMdxov3XkeZ/QwU6I/7rtyEo2eFn6+cnuXbdbvr8DtimdYUlyfPmNmTirHaprYX17X1zKNQ728+RB7jtVy58VjP3HB/pbzR+NpbeXxHv6SDHcW6oPcXZeM47Ipmfz3y1t5Z6d3KoHDVQ0seGQ1T39QzK3nj+bpm+cwNDE2IJ83OiORr52bx7PrStlU+skLtapKQbF/k3h1xSb3Mp1pbVX+5/WdjBuWyPwpwz/xek56AldOH8mfP9hPVb1zVxI7XRbqg1xEhPDQF85gfGYSi575iOfXl/KZ377LtsPVLPnSTO69YhJRfl4M9dedF48lPSGGH7y09RNdJKUV9ZRVN/ZqfHpH44YlkRQbZaFuTvHK1sPsOFLDoovHdrk04m0XjKam0cP/flA8wNWFDgv1EJAQG8WjN+YjAncv20hyXBQv3nEuV04f0S+flxQXzXcum8i64gqWbzx4ymttNw51t3xdTyIjhDPcqQGfsbGqvpkXPirlO89t5I3t/g3NNIODqvLb13cxemgCn5k+sst2U0amcP74DP743t4eBxCEKwv1EJHtiueJr87m65/K48VF5zLezzs/++q6WVlMG5XCgyu2UdfkObl9XXEFCTGRTBh+ep8/KyeN7YdPnPaC3GXVjTzz4X5ufGINs364im/9dSMvfnSQm/+0lqdW7zut9zYD5/VtRyk8eILbLxpLZBdn6W2+ccEYjtU08ey60gGqLrTYHaUhZKY7rU/DCPsiIkL4r6smc93Dq3n4zd3c/ekJgDfUz3Sn9fgfXk9mutNoVdhYUsV543p3+3dJeR0rCw+zsvAwBcUVqEJuejw3fyqP+VOGMz4ziW/+dQP3/72QkvI67r18Upd/zvtjx5FqHlxRxLSsVL7+qTyS4qL7/F59pap+T5wWalSV37y2k2zXEK45o+uz9DZnj3ZxRnYqj769h4WzswPe/RjqLNRNl/JzXVw9YyR/eHsP1+dn40qIoejQCRZdPO603/sMdyoi3i8Jf0J9d1kN/9x8iH8VHmbLAe+SeBOHJ3HXJeOYP3U4EzKTTgm9h2+YxQ//sZVH39lLaUU9v/ziGb0eu9/aqjzx3l5+unI7MZERvLG9jKdX7+OOi8Zyw9k5AzLNQaOnhXv+tpl3dpZx1yXjWDjH7bgQe3vnMTaWVvHg56b5dbOciPCNC8dw69PrWLHlMFfP6PmLIJxYqJtu3XvFRFZtPcKD/yzihrNyaFX/F8XoTnJcNBMyk7qd3Ku0oo5/bDrESxsPUnjQG+Qz3al874qJXDZlODnpCV3+bmSE8MDVU8h2xfOjl7dy+NEPeOzGfNL9HCVUUl7Ht5/dyId7y5k3OZMHPzeNg5X1/Gzldn70chFPvLuXb146ns/NHNVvIVtZ18QtT61jzb5yJo1I5vt/L+TJ1cXcd+UkLpowrOc3CAGqym9f28nIlDg+P9P/uY7mTcpkTEYCv39zN1dNH+HYv2L6wkLddGtEyhC+ceEYHlq1g+oGDyIEZDw8eGd4fGnjQVpb9WT3yNHqBlZsOsRLmw6dHB1zRnYq3//MZK6cNoLhKb0bG3/zeXmMSo3jrqUbuPZ37/Onr81mdEZil+1VlWcLSln8j60A/Oy66Vw3KwsRYWhiLE/ffBbv7zrGT1Zu5zt/28Qf3t7Ntz89gflThwc0WErK67jpj2soLa/ntwvP5DPTR/DK1iM8uKKIr/1xLeePz+C+Kyad9rWNQGr0tFDb2IIrIcbv31m95zgFxRUsvmYKMVH+fzlGRAi3XjCG7zy3ibd2lHGhQ77kAkGCdVdffn6+FhQUBOWzTe80NLdwyS/e4kBlPRMyk1j5rfMD8r7PrSvl289uZNmtc9lTVsNLmw6yevdxWtXbtXLVjJFcNX0k7vTOJyLrjY/2V/DvTxbQosqjN+YzO9f1iTZl1Y3c+/xmXi06wtmjXfz8+hldToKmqqwsPMLPX9nOrqM1TM9K4TuXTez19YHObCyp5OYn19Lc4q11Tt7HtTZ5Wnn6g2J+/eoOaho9LJjj5luXjicjKTD3KfSFqrJi82H+34oiDlXVc/nUEdx2wRi/ZlRc+MgH7C6r4e3vXNTr7qwmTyvn//QNcofGs/SWuX0tP+SIyDpVze/ydQt144+XNx3ijmfWs3COmwc/Ny0g77n3WC0X/fzNk89z0+O5esZIrpox0u953Xuj+HgtX/vjWkor6vnFF2ZwVbu+2H9tOcz3XthMTaOH786fyNfOyfXr4mpLq/L8+lJ+9epODlTWc86YdL592YQ+X9BetfUId/5lPRlJsfzxq3MYO6zzvyoqapv49Ws7+d8PiomLjuT2i8bwb+fmDfh0xkWHTvDA8kI+3OvtIjp3TDp/LSihusHDOWPSue2CMXxq3NBO/4op2FfOdQ+v5vufmczN5+X16fMfe2cPP3q5iOdvP2fABhEEm4W6CQhV5dF39nDhhGEBG06pqtz34hYSY6O4avpIpo5K7ve+0fb91N+dP5EvnRqN0SYAAAmjSURBVOXmBy8V8vz6A0wdlcwvv3BGn75QGj0t/PmD/Sx5YxfHa5uYkZ3KTXNzuGLaCL+D9qnV+3hgeSHTRqXw2E2z/Tr73l1Ww4MrtvFq0RFGpQ7hu5dP5OKJw0iIiezX/y0rapv4xartPPPhflKGRPMfn57AwjluIiOE6oZm/rJmP4+/u5cjJxqZMjKZWy8YwxVTh59y/eHGJ9aw9WAV73znYobE9O3LqLbRwzk/fp2z8lw8cmOXOecoFurGdNDQ3MJ/PreJlzYeJCk2irrmFu64aCx3XjzW76mKu1LT6OG5ghKe+qCYPWW1uBJi+OLsbL7czXz2ra3Kg/8s4tF39nLppEx+s/AM4mN6d7nr/V3H+OHLRRQd8l5QjomKYGhCDK7EGFwJsd7HCTGkJ8aSnhBDemIMI1OHMHZYYq/22dPSyjNr9vOLV7zdPzec5eZb88aTGv/JfvRGTwt//+ggf3h7N7vLasl2DeHrnxrN9bOy2X6kms8ueY97Lp/IbReM6dW+dvTQqh385rWdvHr3+Ywd1v0X8tHqBgoPnKCyvomstHiy0+IZlhR7WkNeB5qFujGdaG1VfvXqDt7aeYwfXD0lYBd/26gq7+8+zpPv7+PVoiMAXDwxkxvn5nDe2KEnQ6ShuYW7l21gxebD3DQ3h/uvmtLnewBaWpVXi45QfLyW4zVNHK9t4nhNI+W1TRyraeJ4bSMNzafOlR8bFcHkkcnMyEplelYK07NSGD00sdOQe3/3MRa/tJVth6s5Z0w6/3XVFL8u1Lb66nr4rd2s31+Jy/cFc6ymkXe/ezGJsac3XqO8tolzfvwan5k+kp9fPwPw/u9fWlFP4cEqCg+eYMuBKrYcPEFZdeMnfj8mKoKstCG4Xd6Qd7viyXYNIdsVT7YrnqTYqEE1uiYgoS4i84FfA5HAY6r64w6vxwJPAbOA48AXVXVfd+9poW7CxYHKep75sJila0o4XtvE6KEJ3HB2DpdMGsbdyzayfn8F910xiZvPy+v38Khr8pwM/OLjtWwurWJTaRVbDlZR1+S97T4xNoopI5OZkZ3KtFEpuF3x/OHt3azYfJhRqUP4v1dO6tNoH1Vl7b4KHn5rN69vO8p350/kGxee3ll6mweWF/K/HxRz49xcth0+QeHBEycn/YqMEMZmJDJlVDJTRqYwdWQy6YneBddLKuopKa+jpLyO/b6f6gbPKe8dHSmkDIkhZUgUqfExpAyJPuUnNd77b3JcNPGxkcTHRJEQE0l8rPffITGRxERGBOzYnnaoi0gksAOYB5QCa4GFqrq1XZvbgemqepuILACuVdUvdve+Fuom3DR6Wvjn5sM8tXrfyXlvYqIi+NUXz+CKaf0zj4+/WlqV3WU1bCqtYlNpJRtLqyg66F0MBSAuOoLbLxzLLeePDsjF2LLqRoYmxgQs6Eor6rj0obdobYWJI5KYMjKFKSOTmToqhYnDk3pVc1VdMyUV3oAvKa+jvK6JE/XNVPl+Kus+ftzxC6ArURFCfEwkCbFRDImJ5JuXju/zTVOBCPW5wAOqepnv+b0AqvpguzYrfW1Wi0gUcBjI0G7e3ELdhLMtB6p44aMDXDl9xKAdtdHkaWXHkWp2HKnm7NHpjEwdEuySulVV30x8TORpXxfpDU9LK9UNHqrqmznR0ExdUwt1TR7vv40t1LY9bvJQ2/jxa1+cnc2nxmX06TN7CnV/OrNGASXtnpcCZ3XVRlU9IlIFpAPHOhRzC3ALgNvd9xV6jAl1U0elMHVUz+O4gykmKiIk6myTMmTg5+SJiowgLSGGtF7ccNXfBnQSCVV9RFXzVTU/I6Nv31LGGGO65k+oHwCy2z3P8m3rtI2v+yUF7wVTY4wxA8ifUF8LjBORPBGJARYAyzu0WQ7c5Ht8HfB6d/3pxhhj+kePfeq+PvJFwEq8QxqfUNVCEVkMFKjqcuBx4GkR2QWU4w1+Y4wxA8yvUf+qugJY0WHb/e0eNwDXB7Y0Y4wxveWs2faNMSbMWagbY4yDWKgbY4yDBG1CLxEpA4r7+OtD6XBjkwM4bZ+ctj/gvH1y2v6A8/aps/3JUdUub/QJWqifDhEp6O422VDktH1y2v6A8/bJafsDztunvuyPdb8YY4yDWKgbY4yDhGqoPxLsAvqB0/bJafsDztsnp+0POG+fer0/IdmnbowxpnOheqZujDGmExbqxhjjICEX6iIyX0S2i8guEbkn2PWcLhHZJyKbRWSDiITkUlAi8oSIHBWRLe22uURklYjs9P07OJf36UQX+/OAiBzwHacNInJFMGvsLRHJFpE3RGSriBSKyF2+7SF5nLrZn5A9TiISJyJrRGSjb59+4NueJyIf+jLvr77Zcrt+n1DqU/dnvdRQIyL7gHxVDdkbJkTkfKAGeEpVp/q2/RQoV9Uf+75801T1u8Gs019d7M8DQI2q/jyYtfWViIwARqjqehFJAtYBnwW+Sggep2725wuE6HES74KtCapaIyLRwLvAXcDdwPOqulREHgY2qurvu3qfUDtTnwPsUtU9qtoELAWuCXJNYU9V38Y75XJ71wBP+h4/ifc/uJDQxf6ENFU9pKrrfY+rgSK8y1CG5HHqZn9ClnrV+J5G+34UuBh4zre9x2MUaqHe2XqpIX0g8R60V0RknW8NV6fIVNVDvseHgcxgFhMgi0Rkk697JiS6KTojIrnAmcCHOOA4ddgfCOHjJCKRIrIBOAqsAnYDlarq8TXpMfNCLdSd6DxVnQlcDtzh+9PfUXyrYIVOP1/nfg+MAc4ADgG/CG45fSMiicDfgG+q6on2r4Xicepkf0L6OKlqi6qegXfZ0DnAxN6+R6iFuj/rpYYUVT3g+/co8ALeA+kER3z9nm39n0eDXM9pUdUjvv/gWoFHCcHj5Oun/RvwZ1V93rc5ZI9TZ/vjhOMEoKqVwBvAXCDVt/Yz+JF5oRbq/qyXGjJEJMF3kQcRSQA+DWzp/rdCRvt1a28C/h7EWk5bW/D5XEuIHSffRbjHgSJVfajdSyF5nLran1A+TiKSISKpvsdD8A4IKcIb7tf5mvV4jEJq9AuAb4jSr/h4vdT/DnJJfSYio/GenYN3acFnQnF/ROQvwIV4pwk9AvwX8CKwDHDjnWL5C6oaEhcfu9ifC/H+Sa/APuDWdn3Rg56InAe8A2wGWn2bv4e3HzrkjlM3+7OQED1OIjId74XQSLwn3MtUdbEvJ5YCLuAj4AZVbezyfUIt1I0xxnQt1LpfjDHGdMNC3RhjHMRC3RhjHMRC3RhjHMRC3RhjHMRC3RhjHMRC3RhjHOT/A8MR/UxCh6+WAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    }
  ]
}