main_ipynb.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.3"
    },
    "colab": {
      "name": "COGS 225 Seq2Seq final project-Recipe.ipynb",
      "provenance": []
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "scrolled": true,
        "id": "YNmQWgsF6R_7",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 513
        },
        "outputId": "603b95fa-74ae-4fe6-e844-e52b94a0f150"
      },
      "source": [
        "!pip install --user -q ktext\n",
        "!pip install --user  -q annoy\n",
        "!pip install --user  tensorflow_hub"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\u001b[K     |████████████████████████████████| 143kB 10.4MB/s \n",
            "\u001b[K     |████████████████████████████████| 163kB 18.4MB/s \n",
            "\u001b[K     |████████████████████████████████| 317kB 9.6MB/s \n",
            "\u001b[K     |████████████████████████████████| 51kB 5.3MB/s \n",
            "\u001b[K     |████████████████████████████████| 481kB 25.0MB/s \n",
            "\u001b[K     |████████████████████████████████| 245kB 26.2MB/s \n",
            "\u001b[K     |████████████████████████████████| 102kB 12.5MB/s \n",
            "\u001b[K     |████████████████████████████████| 51kB 7.1MB/s \n",
            "\u001b[K     |████████████████████████████████| 3.0MB 33.3MB/s \n",
            "\u001b[K     |████████████████████████████████| 71kB 9.9MB/s \n",
            "\u001b[K     |████████████████████████████████| 112kB 54.8MB/s \n",
            "\u001b[?25h  Building wheel for pathos (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for python-levenshtein (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for cytoolz (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for ftfy (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for ppft (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for pox (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[33m  WARNING: The script unidecode is installed in '/root/.local/bin' which is not on PATH.\n",
            "  Consider adding this directory to PATH or, if you prefer to suppress this warning, use --no-warn-script-location.\u001b[0m\n",
            "\u001b[33m  WARNING: The script ftfy is installed in '/root/.local/bin' which is not on PATH.\n",
            "  Consider adding this directory to PATH or, if you prefer to suppress this warning, use --no-warn-script-location.\u001b[0m\n",
            "\u001b[K     |████████████████████████████████| 645kB 7.6MB/s \n",
            "\u001b[?25h  Building wheel for annoy (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: tensorflow_hub in /usr/local/lib/python3.6/dist-packages (0.8.0)\n",
            "Requirement already satisfied: protobuf>=3.8.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow_hub) (3.10.0)\n",
            "Requirement already satisfied: numpy>=1.12.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow_hub) (1.18.5)\n",
            "Requirement already satisfied: six>=1.12.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow_hub) (1.12.0)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from protobuf>=3.8.0->tensorflow_hub) (47.1.1)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xsOJnlAx6_ML",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "7b92fdd1-bb76-45cd-9818-e0a71ab9d7ac"
      },
      "source": [
        "!pip install annoy"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: annoy in /root/.local/lib/python3.6/site-packages (1.16.3)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "scrolled": true,
        "id": "dnFbhSne6SAD",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import json\n",
        "from urllib.request import urlopen\n",
        "\n",
        "from annoy import AnnoyIndex\n",
        "from tensorflow.python.keras.layers import Input, Dense, LSTM, GRU, Embedding, Lambda, BatchNormalization\n",
        "from tensorflow.python.keras.models import load_model, Model\n",
        "from tensorflow.python.keras import optimizer\n",
        "from tensorflow.python.keras.preprocessing.sequence import pad_sequences\n",
        "from keras.utils import get_file, to_categorical\n",
        "from ktext.preprocess import processor\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import random\n",
        "from tqdm import tqdm"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "scrolled": true,
        "id": "77O7Qlj26SAG",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# get training data\n",
        "\n",
        "# data = pd.read_csv('final_recipes.csv')\n",
        "data = pd.read_csv('final_two.csv')\n",
        "# data = data.replace(to_replace= [\"[\",\"'\",\"]\"], value= '', regex=True)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ElQbWhvj6SAJ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "data.dropna(inplace=True)\n",
        "data['ingredients'] = data['ingredients'].apply(lambda x: x.replace(\"[\",\"\").replace(\"'\",\"\").replace(\"]\",\"\").replace(\" , \",\"\").replace(\",,,,,,\",\"\").replace(\",,\",\"\").replace(\",,,\",\"\").replace(\",,,,\",\"\").replace(\",,,,,\",\"\"))\n",
        "data['name'] = data['name'].apply(lambda x: x.replace('  ',' ').replace('  ',' '))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GjOdqjEq6SAO",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 714
        },
        "outputId": "f58960ff-e66c-4bbd-926e-3d978bfceed5"
      },
      "source": [
        "data['name'].head(40)"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0              arriba baked winter squash mexican style\n",
              "1                       a bit different breakfast pizza\n",
              "2                              all in the kitchen chili\n",
              "3                                     alouette potatoes\n",
              "4                      amish tomato ketchup for canning\n",
              "5                                apple a day milk shake\n",
              "6                                  aww marinated olives\n",
              "7                         backyard style barbecued ribs\n",
              "8                               bananas 4 ice cream pie\n",
              "9                                beat this banana bread\n",
              "10                           berry good sandwich spread\n",
              "11                         better than sex strawberries\n",
              "12                       better then bush s baked beans\n",
              "13                            boat house collard greens\n",
              "14                               calm your nerves tonic\n",
              "15                       chicken lickin good pork chops\n",
              "16                                       chile rellenos\n",
              "17                                        chinese candy\n",
              "18                                    chinese chop suey\n",
              "19                      cream of cauliflower soup vegan\n",
              "20                                cream of spinach soup\n",
              "21                          cream of spinach soup vegan\n",
              "22                               crispy crunchy chicken\n",
              "23                           deep fried dessert thingys\n",
              "24                       easiest ever hollandaise sauce\n",
              "25                      emotional balance spice mixture\n",
              "26                           fool the meat eaters chili\n",
              "27                                 forgotten minestrone\n",
              "28                                       fried potatoes\n",
              "29                 george s at the cove black bean soup\n",
              "30                           get the sensation brownies\n",
              "31                        global gourmet taco casserole\n",
              "32                                  grilled ranch bread\n",
              "33                              grilled venison burgers\n",
              "34                    healthy for them yogurt popsicles\n",
              "35                   homemade vegetable soup from a can\n",
              "36    how i got my family to eat spinach spinach cas...\n",
              "37     i can t believe it s low fat creamy potato salad\n",
              "38                         i can t believe it s spinach\n",
              "39          i don t feel like cooking tonight casserole\n",
              "Name: name, dtype: object"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KeRvE-Q66SAS",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UafvYdJL6SAW",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "#target - recipe name\n",
        "#source = ingredients\n",
        "source_docs = data['ingredients'].tolist()\n",
        "target_docs = data['name'].tolist()\n",
        "\n",
        "# source_docs_t = data['ingredients'].sample(2000, random_state=7).tolist()\n",
        "# target_docs_t = data['name'].sample(2000, random_state=7).tolist()\n",
        "\n",
        "train_source_docs, test_source_docs, train_target_docs, test_target_docs = train_test_split(source_docs, target_docs, test_size=0.33, random_state=42)\n",
        "\n",
        "for i in range(len(train_source_docs)):\n",
        "    train_source_docs[i] = train_source_docs[i].replace(\"'\",\"\").replace(\"  \",\" \").replace(\"[\",\"\").replace(\"]\",\"\").replace(\" ,\",\"\")\n",
        "\n",
        "test_docs = pd.Series(test_target_docs).sample(10000).tolist()\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NJpp2uhw6SAZ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pD5lJrqR6SAd",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# len(test_docs)\n",
        "# test_docs\n",
        "# train_source_docs\n",
        "# train_target_docs"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "scrolled": true,
        "id": "ZF6aWdp76SAg",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 187
        },
        "outputId": "286d2536-d5f5-4aa3-9de5-8be2bd804065"
      },
      "source": [
        "#processor adds padding to all text entries in order to make all inputs the same length\n",
        "proc = processor(heuristic_pct_padding=.95, keep_n=10000)\n",
        "vecs = proc.fit_transform(target_docs)\n",
        "\n"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:root:....tokenizing data\n",
            "WARNING:root:Setting maximum document length to 10 based upon heuristic of 0.95 percentile.\n",
            " See full histogram by insepecting the `document_length_stats` attribute.\n",
            "WARNING:root:(1/2) done. 16 sec\n",
            "WARNING:root:....building corpus\n",
            "WARNING:root:(2/2) done. 0 sec\n",
            "WARNING:root:Finished parsing 231,636 documents.\n",
            "WARNING:root:...fit is finished, beginning transform\n",
            "WARNING:root:...padding data\n",
            "WARNING:root:done. 1 sec\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "scrolled": true,
        "id": "9IStIGAa6SAk",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# proc.token_count_pandas().head(20)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CJS_4BZA6SAo",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 173
        },
        "outputId": "b4f6ec70-579f-46ed-82e8-03432bb8bb7b"
      },
      "source": [
        "proc.document_length_stats"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>bin</th>\n",
              "      <th>doc_count</th>\n",
              "      <th>cumsum_pct</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0</td>\n",
              "      <td>25419</td>\n",
              "      <td>0.109737</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>5</td>\n",
              "      <td>191905</td>\n",
              "      <td>0.938213</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>10</td>\n",
              "      <td>14253</td>\n",
              "      <td>0.999745</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>15</td>\n",
              "      <td>59</td>\n",
              "      <td>1.000000</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   bin  doc_count  cumsum_pct\n",
              "1    0      25419    0.109737\n",
              "0    5     191905    0.938213\n",
              "2   10      14253    0.999745\n",
              "3   15         59    1.000000"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5ME7igm26SAt",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "assert vecs.shape[0] == len(target_docs)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8wajEh496SAx",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "f0c82cb5-425d-47b3-d35d-7b6b94054f44"
      },
      "source": [
        "print('original list: ', target_docs[0])\n",
        "print('tokenized list: ', vecs[0])"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "original list:  arriba baked winter squash mexican style\n",
            "tokenized list:  [   0    0    0    0 7897   22  587  121  106   45]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KO-XKkep6SA1",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "4ee44764-4c90-483f-a889-ecb681668e0f"
      },
      "source": [
        "vocab_size = max(proc.id2token.keys()) + 1\n",
        "max_length = proc.padding_maxlen\n",
        "\n",
        "print('vocab size: ', vocab_size)\n",
        "print('max length allowed for documents: ', max_length)"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "vocab size:  10002\n",
            "max length allowed for documents:  10\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QYMz6aYM6SA4",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "f51d076f-9977-4b18-dae0-1003dd779e82"
      },
      "source": [
        "sequences = []\n",
        "for arr in tqdm(vecs):\n",
        "    non_zero = (arr != 0).argmax()\n",
        "    for i in range(non_zero, len(arr)):\n",
        "        sequences.append(arr[:i+1])\n",
        "sequences = pad_sequences(sequences, maxlen=max_length, padding='pre')\n",
        "sequences = np.array(sequences)\n",
        "X, y = sequences[:,:-1], sequences[:,-1]\n",
        "# y = to_categorical(y, num_classes=vocab_size)"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 231636/231636 [00:01<00:00, 225069.32it/s]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "a2thBWwG6SA8",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 323
        },
        "outputId": "3409f5a6-8b11-4a04-d61b-3556ee16d58c"
      },
      "source": [
        "i = Input(shape=(max_length-1,))\n",
        "x = Embedding(vocab_size, 256, input_length=max_length-1)(i)\n",
        "x = LSTM(256, return_sequences=True)(x)\n",
        "last_timestep = Lambda(lambda x: x[:, -1, :])(x)\n",
        "last_timestep = Dense(vocab_size, activation='softmax')(last_timestep)\n",
        "model = Model(i, last_timestep)\n",
        "model.summary()"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"model\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "input_1 (InputLayer)         [(None, 9)]               0         \n",
            "_________________________________________________________________\n",
            "embedding (Embedding)        (None, 9, 256)            2560512   \n",
            "_________________________________________________________________\n",
            "lstm (LSTM)                  (None, 9, 256)            525312    \n",
            "_________________________________________________________________\n",
            "lambda (Lambda)              (None, 256)               0         \n",
            "_________________________________________________________________\n",
            "dense (Dense)                (None, 10002)             2570514   \n",
            "=================================================================\n",
            "Total params: 5,656,338\n",
            "Trainable params: 5,656,338\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "scrolled": true,
        "id": "SiVs2uM96SBE",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 547
        },
        "outputId": "66e4878d-87d5-4686-80a9-d18d3acfbed9"
      },
      "source": [
        "model.compile(loss='sparse_categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
        "history = model.fit(X, y, epochs=15, batch_size=256, validation_split=0.2)"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/15\n",
            "3162/3162 [==============================] - 110s 35ms/step - loss: 5.9032 - accuracy: 0.1255 - val_loss: 6.3126 - val_accuracy: 0.1560\n",
            "Epoch 2/15\n",
            "3162/3162 [==============================] - 106s 34ms/step - loss: 5.2054 - accuracy: 0.1845 - val_loss: 6.2665 - val_accuracy: 0.1786\n",
            "Epoch 3/15\n",
            "3162/3162 [==============================] - 106s 34ms/step - loss: 4.9643 - accuracy: 0.2000 - val_loss: 6.3648 - val_accuracy: 0.1838\n",
            "Epoch 4/15\n",
            "3162/3162 [==============================] - 107s 34ms/step - loss: 4.8068 - accuracy: 0.2096 - val_loss: 6.3939 - val_accuracy: 0.1866\n",
            "Epoch 5/15\n",
            "3162/3162 [==============================] - 107s 34ms/step - loss: 4.6867 - accuracy: 0.2170 - val_loss: 6.5207 - val_accuracy: 0.1886\n",
            "Epoch 6/15\n",
            "3162/3162 [==============================] - 107s 34ms/step - loss: 4.5870 - accuracy: 0.2236 - val_loss: 6.5653 - val_accuracy: 0.1874\n",
            "Epoch 7/15\n",
            "3162/3162 [==============================] - 106s 34ms/step - loss: 4.5002 - accuracy: 0.2300 - val_loss: 6.6254 - val_accuracy: 0.1881\n",
            "Epoch 8/15\n",
            "3162/3162 [==============================] - 106s 34ms/step - loss: 4.4241 - accuracy: 0.2367 - val_loss: 6.7240 - val_accuracy: 0.1869\n",
            "Epoch 9/15\n",
            "3162/3162 [==============================] - 108s 34ms/step - loss: 4.3554 - accuracy: 0.2435 - val_loss: 6.7713 - val_accuracy: 0.1861\n",
            "Epoch 10/15\n",
            "3162/3162 [==============================] - 106s 34ms/step - loss: 4.2930 - accuracy: 0.2496 - val_loss: 6.8075 - val_accuracy: 0.1837\n",
            "Epoch 11/15\n",
            "3162/3162 [==============================] - 106s 33ms/step - loss: 4.2354 - accuracy: 0.2564 - val_loss: 6.9219 - val_accuracy: 0.1835\n",
            "Epoch 12/15\n",
            "3162/3162 [==============================] - 106s 34ms/step - loss: 4.1833 - accuracy: 0.2627 - val_loss: 6.9669 - val_accuracy: 0.1832\n",
            "Epoch 13/15\n",
            "3162/3162 [==============================] - 106s 34ms/step - loss: 4.1341 - accuracy: 0.2688 - val_loss: 7.0206 - val_accuracy: 0.1816\n",
            "Epoch 14/15\n",
            "3162/3162 [==============================] - 106s 34ms/step - loss: 4.0888 - accuracy: 0.2744 - val_loss: 7.0848 - val_accuracy: 0.1808\n",
            "Epoch 15/15\n",
            "3162/3162 [==============================] - 106s 34ms/step - loss: 4.0467 - accuracy: 0.2802 - val_loss: 7.1421 - val_accuracy: 0.1771\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "h7__MU746SBI",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 295
        },
        "outputId": "967dd778-4469-4a20-81f6-9a8936664be4"
      },
      "source": [
        "from matplotlib import pyplot as plt\n",
        "plt.plot(history.history['accuracy'])\n",
        "plt.plot(history.history['val_accuracy'])\n",
        "plt.title('model accuracy')\n",
        "plt.ylabel('accuracy')\n",
        "plt.xlabel('epoch')\n",
        "plt.legend(['train', 'val'], loc='upper left')\n",
        "plt.show()"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEWCAYAAAB8LwAVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3deXxU9b3/8dcnGyELWxK2LBL2RRE0IK6IooKC6FULrqW20lZtbb3ee21tbWuX66/e29va2talWG0VRNxQQUQLriBJ2HcQIRshCUtIQvb5/P44JzAJIQyQYSYzn+fjMY/MnGXmM3nAeed8v+d8v6KqGGOMMS1FBLoAY4wxwckCwhhjTKssIIwxxrTKAsIYY0yrLCCMMca0ygLCGGNMqywgjAFE5O8i8isft90lIhP9XZMxgWYBYYwxplUWEMaEEBGJCnQNJnRYQJgOw23a+Q8RWSciVSLyNxHpJSKLRKRCRD4Qke5e218vIhtF5KCILBORYV7rRovIKne/V4DYFp81RUTWuPt+LiIjfazxOhFZLSKHRCRfRH7eYv0l7vsddNfPdJd3FpH/FZHdIlIuIp+6yy4XkYJWfg8T3ec/F5H5IvJPETkEzBSRsSKy3P2MPSLyJxGJ8dp/hIgsEZH9IrJXRH4sIr1F5LCIJHltd56IlIpItC/f3YQeCwjT0dwEXAUMBqYCi4AfAyk4/56/DyAig4E5wA/cdQuBt0Ukxj1Yvgn8A+gBvOq+L+6+o4HZwLeBJOBpYIGIdPKhvirgLqAbcB3wXRG5wX3fs9x6/+jWNApY4+73P8D5wEVuTf8JeHz8nUwD5ruf+RLQCPwQSAYuBK4E7nVrSAQ+AN4D+gIDgQ9VtRhYBnzN633vBOaqar2PdZgQYwFhOpo/qupeVS0EPgG+UNXVqloDvAGMdrebDryrqkvcA9z/AJ1xDsDjgGjg96par6rzgWyvz5gFPK2qX6hqo6q+ANS6+7VJVZep6npV9ajqOpyQGu+uvg34QFXnuJ+7T1XXiEgEcDfwgKoWup/5uarW+vg7Wa6qb7qfWa2quaq6QlUbVHUXTsA11TAFKFbV/1XVGlWtUNUv3HUvAHcAiEgkcCtOiJowZQFhOpq9Xs+rW3md4D7vC+xuWqGqHiAfSHXXFWrzkSp3ez0/C/h3t4nmoIgcBNLd/dokIheIyFK3aaYc+A7OX/K47/FlK7sl4zRxtbbOF/ktahgsIu+ISLHb7PQbH2oAeAsYLiKZOGdp5aq68hRrMiHAAsKEqiKcAz0AIiI4B8dCYA+Q6i5rkuH1PB/4tap283rEqeocHz73ZWABkK6qXYG/Ak2fkw8MaGWfMqDmOOuqgDiv7xGJ0zzlreWQzH8BtgCDVLULThOcdw39WyvcPQubh3MWcSd29hD2LCBMqJoHXCciV7qdrP+O00z0ObAcaAC+LyLRIvJvwFivfZ8FvuOeDYiIxLudz4k+fG4isF9Va0RkLE6zUpOXgIki8jURiRKRJBEZ5Z7dzAZ+JyJ9RSRSRC50+zy2AbHu50cDPwFO1BeSCBwCKkVkKPBdr3XvAH1E5Aci0klEEkXkAq/1LwIzgeuxgAh7FhAmJKnqVpy/hP+I8xf6VGCqqtapah3wbzgHwv04/RWve+2bA9wD/Ak4AOxwt/XFvcBjIlIBPIoTVE3vmwdcixNW+3E6qM91Vz8ErMfpC9kP/D8gQlXL3fd8DufspwpodlVTKx7CCaYKnLB7xauGCpzmo6lAMbAdmOC1/jOczvFVqurd7GbCkNiEQcYYbyLyL+BlVX0u0LWYwLKAMMYcISJjgCU4fSgVga7HBJY1MRljABCRF3DukfiBhYMBO4MwxhhzHHYGYYwxplUhM7BXcnKy9uvXL9BlGGNMh5Kbm1umqi3vrQFCKCD69etHTk5OoMswxpgORUSOezmzNTEZY4xplQWEMcaYVllAGGOMaVXI9EG0pr6+noKCAmpqagJdit/FxsaSlpZGdLTN7WKMaR8hHRAFBQUkJibSr18/mg/cGVpUlX379lFQUEBmZmagyzHGhAi/NjGJyCQR2SoiO0Tk4VbWPygim8SZQvJDd8atpnW/daeL3CwiT8opHOFrampISkoK6XAAEBGSkpLC4kzJGHPm+C0g3HHrnwImA8OBW0VkeIvNVgNZqjoSZ8rE37r7XgRcDIwEzgbGcHRGrJOt45Tq72jC5XsaY84cf55BjAV2qOpOd3jluThz5x6hqktV9bD7cgWQ1rQKZ4atGJyx76NpPnOYMcaEvUM19czPLeDlL/L88v7+DIhUmk+FWOAuO55v4kzojqouB5bizPy1B1isqpv9VKdfHTx4kD//+c8nvd+1117LwYMH/VCRMaYjq6pt4K01hXzrhRyyfvkBD726lldz80+84ykIik5qEbkDyMJtRhKRgcAwjp5RLBGRS1X1kxb7zcKZYJ6MDO8ZI4NHU0Dce++9zZY3NDQQFXX8X//ChQv9XZoxpoOormtk6dYS3llXxL+2lFBT76F3l1juvPAspozsw6j0bn75XH8GRCHOHMBN0txlzYjIROARYLyq1rqLbwRWqGqlu80i4EKgWUCo6jPAMwBZWVlBOSztww8/zJdffsmoUaOIjo4mNjaW7t27s2XLFrZt28YNN9xAfn4+NTU1PPDAA8yaNQs4OnRIZWUlkydP5pJLLuHzzz8nNTWVt956i86dOwf4mxlj/Km2oZGPt5Xx9toiPti8l8N1jSQnxPC1rHSmjOxL1lndiYjwb9+jPwMiGxgkIpk4wTCD5vPzIiKjgaeBSapa4rUqD7hHRP4bZ7L18cDvT6eYX7y9kU1Fh07nLY4xvG8XfjZ1RJvbPP7442zYsIE1a9awbNkyrrvuOjZs2HDkctTZs2fTo0cPqqurGTNmDDfddBNJSUnN3mP79u3MmTOHZ599lq997Wu89tpr3HHHHe36XYwxgVff6OHTHWW8s3YP728qpqKmgW5x0Uwb1ZepI/tyQf8kIv0cCt78FhCq2iAi9wOLgUhgtqpuFJHHgBxVXQA8ASQAr7pX4eSp6vU4VzRdgTNHrwLvqerb/qr1TBo7dmyzexWefPJJ3njjDQDy8/PZvn37MQGRmZnJqFGjADj//PPZtWvXGavXGONfDY0evvhqP++sK2LRhmIOHq4nMTaKa0b0ZsrIPlw8MJnoyMAMeuHXPghVXQgsbLHsUa/nE4+zXyPw7fas5UR/6Z8p8fHxR54vW7aMDz74gOXLlxMXF8fll1/e6r0MnTp1OvI8MjKS6urqM1KrMcY/PB4le9d+3lm3h0Ub9lBWWUd8TCQTh/diysi+XDY4mU5RkYEuMzg6qUNZYmIiFRWtz95YXl5O9+7diYuLY8uWLaxYseIMV2eMOVNUldX5B3ln7R7eXV/E3kO1xEZHcOXQXkwZ2YcJQ3sSGx34UPBmAeFnSUlJXHzxxZx99tl07tyZXr16HVk3adIk/vrXvzJs2DCGDBnCuHHjAlipMaa9NYXCwnV7WLShmMKD1cRERjB+SApTRvZh4rBexHcK3sNwyMxJnZWVpS0nDNq8eTPDhg0LUEVnXrh9X2OCkcfjhsL6PSxav4ei8hpiIiO4dFAyk8/pw9UjetElNngG1RSRXFXNam1d8EaXMcZ0EE4oHODddcUs2rCHPW4oXDY4mYeuGcLE4cEVCr6ygDDGmFPg8Sir8g7w7vo9LFpfTPGhplBI4T8nDeHKYR0zFLxZQBhjjI88HiU37wDvrtvDexvcUIiKYPzgFB4+ZyhXDutJYgcPBW8WEMYY0waPR8nZfcDpU9iwh72HaomJiuDywSn8aORQrhgaWqHgzQLCGGNaaPQoObv2u6FQTElFLZ2iIrh8SArXntOHK4f1IiGIrz5qL6H/DY0xxgeqyvrCct5cXcQ764qOhMKEIT25dmQfrhjaMyxCwVt4fdsOICEhgcrKykCXYUzY2FVWxVtrinhrTSE7y6qIiYxgwtAUpozsyxVDewb1fQr+Fr7f3BgTtsoqa3lnbRFvriliTf5BROCCzB7Muqw/k8/uQ9e40OxTOFkWEH728MMPk56ezn333QfAz3/+c6Kioli6dCkHDhygvr6eX/3qV0ybNu0E72SMOR1VtQ28v6mYN1cX8emOMho9yrA+XfjR5KFcP6ovfbraEPothU9ALHoYite373v2PgcmP97mJtOnT+cHP/jBkYCYN28eixcv5vvf/z5dunShrKyMcePGcf3119u80sa0s/pGD59sL+XN1UUs2bSX6vpGUrt15tuX9eeG0akM7pUY6BKDWvgERICMHj2akpISioqKKC0tpXv37vTu3Zsf/vCHfPzxx0RERFBYWMjevXvp3bt3oMs1psNTdW5ge3N1Ee+u38P+qjq6xUXzb+elcsPoVM7P8P9EO6EifALiBH/p+9Mtt9zC/PnzKS4uZvr06bz00kuUlpaSm5tLdHQ0/fr1a3WYb2OM73aUVPDm6iLeWltI/v5qOkVFcNXwXtwwKpXLBqcQExWYORU6svAJiACaPn0699xzD2VlZXz00UfMmzePnj17Eh0dzdKlS9m9e3egSzSmQ9p7qIYFa4p4c00hG4sOESFw8cBkfnDlYK45u3fYXZba3uy3dwaMGDGCiooKUlNT6dOnD7fffjtTp07lnHPOISsri6FDhwa6RGM6jJr6RhZvLOa1VYV8ur0Uj8K5aV15dMpwppzbh56JsYEuMWT4NSBEZBLwB5wpR59T1cdbrH8Q+BbQAJQCd6vqbnddBvAckI4z7ei1qrrLn/X60/r1RzvIk5OTWb58eavb2T0QxhyrqV9hfm4B76zdQ0VtA6ndOnPfhIHcODqV/ikJgS4xJPktIEQkEngKuAooALJFZIGqbvLabDWQpaqHReS7wG+B6e66F4Ffq+oSEUkAPP6q1RgTnAoPVvPGqgJeW1XIV2VVdI6OZPI5vbn5vDTG9U+yzmY/8+cZxFhgh6ruBBCRucA04EhAqOpSr+1XAHe42w4HolR1ibud/VltTJg4XNfAexuKeW1VAZ9/uQ9V5ya2ey8fwORz+li/whnkz990KpDv9boAuKCN7b8JLHKfDwYOisjrQCbwAfCwqjZ67yAis4BZABkZGa2+qaqGxf0FoTIzoAlPqsrKr/bz2qoC3l23h6q6RtJ7dOaBKwdx03lppPeIC3SJYSkoolhE7gCygPHuoijgUmA0kAe8AswE/ua9n6o+AzwDzpSjLd83NjaWffv2kZSUFNIhoars27eP2FjrnDMdS/7+w7y2qoDXVxWSt/8w8TGRXDeyDzedl8aYfj2sCSnA/BkQhTgdzE3S3GXNiMhE4BFgvKrWuosLgDVezVNvAuNoERAnkpaWRkFBAaWlpadQfscSGxtLWlpaoMsw5oSqahtYuH4P83ML+OKr/YjARQOS+OFVg7hmRG/iYoLi71aDfwMiGxgkIpk4wTADuM17AxEZDTwNTFLVkhb7dhORFFUtBa4Ack62gOjoaDIzM0+1fmNMO/F4lBU79zF/VQGL1hdTXd9IZnI8D109mBvPSyO1m42DFIz8FhCq2iAi9wOLcS5zna2qG0XkMSBHVRcATwAJwKtuE1Ceql6vqo0i8hDwoTgrcoFn/VWrMcY/Sg7V8GpuAXOz88jfX01ipyhuGJ3Kzeencl5G95Bu+g0FEiqdm1lZWZqTc9InGcaYdtboUT7ZXsqclXl8sLmERo9y0YAkpo9J55oRvYmNjgx0icaLiOSqalZr66yxzxjTLvYeqmFedj5zs/MpPFhNUnwM37o0kxljMshMjg90eeYUWEAYY05Zo0f5eFspL6/M419bnLOFSwYm8+Nrh3HV8F42QF4HZwFhjDlpe8qrmZddwCvZeRSV15CcEMOsy/ozY0w6ZyXZ2UKosIAwxvik0aMs21rCHPdswaNw6aBkfjplOFcOs7OFUGQBYYxpU9HBal7JzmdeTj57ymtITujEd8YPYMaYDDKS7A7nUGYBYYw5RkOjh6VbnSuRlm0tQYFLB6Xws6nO2UJ0pJ0thAMLCGPMEbv3VfFabgGv5OSz91AtPRM7ce/lA5k+Jt3GQwpDFhDGhLmyylreXbeHN9cUsjrvICIwfnAKj03L4IqhPe1sIYxZQBgThg7XNfD+xr28uaaQT7aX0ehRhvZO5OHJQ5l6bl8b+sIAFhDGhI36Rg+fbi/jzTWFvL9xL9X1jfTtGss9l/bnhtF9Gdq7S6BLNEHGAsKYEKaqrM4/yFurC3ln3R72VdXRtXM0N4xO5YZRfW1IbdMmCwhjQtCXpZW8tbqQt9YWsXvfYTpFRTBxWC+mjerL+CEpdIqy8ZDMiVlAGBMiSg7VsGBtEW+tKWJ9YTkRAhcNSOb+CQOZdHZvEmOjA12i6WAsIIzpwCpq6lm8cS9vrSnksx1leBTOTu3CT64bxtRz+9Kri80yaE6dBYQxHYzHo3z+5T7mZuexZNNeahs8ZPSI474JA5k2KpWBPRMCXaIJERYQxnQQZZW1vJrjTL6ze99husVFM31MOtNGpXJeRjebfMe0O78GhIhMAv6AM6Pcc6r6eIv1DwLfAhqAUuBuVd3ttb4LsAl4U1Xv92etxgQjj0dZvnMfL6/M4/2NxdQ3KmMze/DgVYNt8h3jd34LCBGJBJ4CrgIKgGwRWaCqm7w2Ww1kqephEfku8Ftgutf6XwIf+6tGY4LVvspa5ucWMGdlHrvcs4W7LuzHrWPTGdgzMdDlmTDhzzOIscAOVd0JICJzgWk4ZwQAqOpSr+1XAHc0vRCR84FewHtAq9PhGRNKVJ2zhTkr81m8oZi6Rg9j+nXngYmDmHx2HztbMGecPwMiFcj3el0AXNDG9t8EFgGISATwvziBMdFfBRoTDPZX1fGae7aws6yKLrFR3D4ug1vHZjC4l50tmMAJik5qEbkD5yxhvLvoXmChqha01fEmIrOAWQAZGRn+LtOYdqOqfPHVfl7+Io/33LOFrLO6c9+EgVw30s4WTHDwZ0AUAuler9PcZc2IyETgEWC8qta6iy8ELhWRe4EEIEZEKlX1Ye99VfUZ4BmArKwsbf+vYEz7OlBVx2urCnh5ZR47S6tIjI3itgucs4Uhve1swQQXfwZENjBIRDJxgmEGcJv3BiIyGngamKSqJU3LVfV2r21m4nRkNwsHYzoKVSV71wFe/mI3CzcUU9fg4byMbjxx80imjOxL5xg7WzDByW8BoaoNInI/sBjnMtfZqrpRRB4DclR1AfAEzhnCq25TUp6qXu+vmow5k/ZX1fH6qgLmZuezo6SSxE5RzBiTzq1jMxjWx0ZONcFPVEOjZSYrK0tzcnICXYYJc033LcxZmcf7G/dS1+hhdEY3bh2TwZRz+xAXExTdfsYcISK5qtrqlaL2r9WYdlByqIZXcwt4JTufvP2H6do5mtsuyGDG2HSbZ8F0WBYQxpyiRo/y0bYS5q7M58MtJTR6lAvcu5wnnW13OZuOzwLCmJNUeLCaV7LzeTUnnz3lNSTFx/CtSzKZPiad/ik2UJ4JHRYQxvigvtHDh5v3MmdlPh9vLwXgkoHJ/HTKcCYO60VMVESAKzSm/VlAGNOGXWVVzM3OZ35uAWWVtfTuEsv3Jgzklqx00nvEBbo8Y/zKAsKYFmrqG1m8sZi5K/NZvnMfkRHChCE9uXVsOuMHpxAVaWcLJjxYQBjj2lpcwdzsPN5YXcjBw/Wk9+jMQ1cP5pasdJuZzYQlCwgT1g5U1bFgbRHzcwtYX1hOdKRw9fDezBibzsUDkomIsEl4TPiygDBhp6HRw0fbSpmfW8AHm/dS36gM79OFR6cMZ9qoviQldAp0icYEBQsIEza2FlcwPzefN1YXUVZZS4/4GO4c14+bzk9lRN+ugS7PmKBjAWFCWssmpKgI4YqhPbn5/DQuH9LTLk81pg0WECbkWBOSMe3DAsKEDGtCMqZ9WUCYDs2akIzxHwsI0+G01oQ0om8XfjZ1ONefa01IxrQXCwjTYewsrWReTgGvrSqgtOJoE9LN56cxvK8NqW1Me7OAMEHtcF0DC9cXMy87n5W79rvDXqRwS1Y6E6wJyRi/8mtAiMgk4A84U44+p6qPt1j/IPAtoAEoBe5W1d0iMgr4C9AFaAR+raqv+LNWEzxUlbUF5bySnc/ba4uorG0gMzme/5w0hJvOS7NhL4w5Q/wWECISCTwFXAUUANkiskBVN3ltthrIUtXDIvJd4LfAdOAwcJeqbheRvkCuiCxW1YP+qtcEXtMczq/mFLB1bwWx0RFce04fpmelMzazB+685caYM8SfZxBjgR2quhNAROYC04AjAaGqS722XwHc4S7f5rVNkYiUACmABUSIafQon2wvZV5OPks2OR3O56Z34zc3nsPUc/uQGBsd6BKNCVv+DIhUIN/rdQFwQRvbfxNY1HKhiIwFYoAvW1k3C5gFkJGRcTq1mjMsf/9hXs1x5lkoKq+he1w0d47rx/Qx6QzpnRjo8owxBEkntYjcAWQB41ss7wP8A/i6qnpa7qeqzwDPAGRlZekZKNWchqZ5Fubl5PPZjn2IwKWDUnjkuuFMHN6TTlE2h7MxwcSngBCR14G/AYtaO1AfRyGQ7vU6zV3W8r0nAo8A41W11mt5F+Bd4BFVXeHjZ5ogtLGonHnZ+by5pojy6nrSunfmwasGc9P5aaR26xzo8owxx+HrGcSfgW8AT4rIq8Dzqrr1BPtkA4NEJBMnGGYAt3lvICKjgaeBSapa4rU8BngDeFFV5/tYowkiNfWNvL22iBeW72JD4SFioiKYNKI308ekc2H/JJtnwZgOwKeAUNUPgA9EpCtwq/s8H3gW+Keq1reyT4OI3A8sxrnMdbaqbhSRx4AcVV0APAEkAK+6V6jkqer1wNeAy4AkEZnpvuVMVV1zGt/VnAElh2r454rdvPRFHvuq6hjcK4FfXD+CaaP60i0uJtDlGWNOgqj61nQvIkk4VxndCRQBLwGXAOeo6uX+KtBXWVlZmpOTE+gywtb6gnJmf/YV76wrosGjXDGkJ3dfkslFA5Ls8lRjgpiI5KpqVmvrfO2DeAMYgtNhPFVV97irXhEROyqHqYZGD4s37uX5z74iZ/cB4mMiuf2Cs5h5UT/6JccHujxjzGnytQ/iyRb3LBxxvOQxoevg4TrmZufz4ue7KCqvIaNHHD+dMpxbstLoYvctGBMyfA2I4SKyuulOZhHpDtyqqn/2X2km2OwoqeD5z3bx2qoCauo9XNg/iZ9fP4Irh/Ui0jqdjQk5vgbEPar6VNMLVT0gIvfgXN1kQpjHo3y0vZTZn37FJ9vLiImK4MZRqcy8uB/D+tgIqsaEMl8DIlJERN0ebXecJbskJYRV1Tbw2qoC/v7ZLnaWVdEzsRMPXT2YW8dm2HwLxoQJXwPiPZwO6afd1992l5kQk7//MC8u38Xc7Hwqaho4N70bf5gxisln97GhtY0JM74GxH/hhMJ33ddLgOf8UpEJiNzdB3j24528v6kYEWHy2b25+5JMzsvoHujSjDEB4uuNch6c+Rn+4t9yzJmWv/8w/71oMwvXF9MtLprvjB/AnReeRZ+uNgSGMeHO1/sgBgH/DQwHjszWoqr9/VSX8bOKmnr+tHQHz3+6i8gI4YcTB3PPZZnExQTF+I3GmCDg69HgeeBnwP8BE3DGZbIG6Q6o0aO8kp3P75ZspayyjpvOS+M/rhlC7642S5sxpjlfA6Kzqn7oXsm0G/i5iOQCj/qxNtPOPt1exq/e3cSW4grG9OvO7JljGJnWLdBlGWOClK8BUSsiEcB2dwC+QpxB9kwH8GVpJb95dzMfbikhvUdn/nz7eUw+u7eNkWSMaZOvAfEAEAd8H/glTjPT1/1VlGkfBw/X8YcPt/OP5buJjY7k4clDmXlRP2KjbWIeY8yJnTAg3JvipqvqQ0AlTv+DCWL1jR5eWrGb33+4nUPV9Uwfk8GDVw0mJdFucDPG+O6EAaGqjSJyyZkoxpweVWXp1hJ+/e5mviyt4pKByfxkyjCG9rYhMYwxJ8/XJqbVIrIAeBWoalqoqq/7pSpz0rYWV/CrdzfxyfYy+ifH87evZ3HF0J7Wz2CMOWW+BkQssA+4wmuZAhYQAbavspbfLdnGnJV5JMZG8+iU4dwx7iwbFsMYc9p8vZP6lPodRGQS8AecKUefU9XHW6x/EPgW0ACUAne7l9EiIl8HfuJu+itVfeFUaghVtQ2N/P2zXfzpXzs4XN/IXRf244ErB9E93sZQNMa0D1/vpH4e54yhGVW9u419IoGngKuAAiBbRBao6iavzVYDWap6WES+C/wWmC4iPXBuzMtyPzfX3feAj98rZKkqizcW85uFW8jbf5grhvbkx9cOY2BPu+rYGNO+fG1iesfreSxwI8681G0ZC+xQ1Z0AIjIXmAYcCYgWs9StwJnzGuAaYImq7nf3XQJMAub4WG9Iqqlv5MF5a1i4vpghvRL5xzfHcumglECXZYwJUb42Mb3m/VpE5gCfnmC3VCDf63UBcEEb238TWNTGvqktdxCRWcAsgIyMjBOU07Edqqln1os5rNi5n/+aNJR7Ls0kKtL6GYwx/nOqI7MNAnq2VxEicgdOc9L4k9lPVZ8BngHIyso6pgksVJQcquHrz2ezfW8Ff5gximmjjslKY4xpd772QVTQvA+iGGeOiLYUAuler9PcZS3feyLwCDBeVWu99r28xb7LfKk11Owqq+LO2V+wr7KO2TPHcNlga1IyxpwZvjYxJZ7Ce2cDg0QkE+eAPwO4zXsDERkNPA1MUtUSr1WLgd+ISNNsNVcDPzqFGjq09QXlzHx+JR5VXr5nHKPSbWA9Y8yZ41MjtojcKCJdvV53E5Eb2tpHVRuA+3EO9puBeaq6UUQeE5Hr3c2ewBn071URWePejIfbOf1LnJDJBh5r6rAOF5/tKGPGM8uJjY5k/ncvsnAwxpxxonripnsRWaOqo1osW62qo/1W2UnKysrSnJycQJfRLt5ZV8SDr6wlMzmeF+4ea3M1GGP8RkRyVTWrtXW+dlK3dqZhU4/5wYvLd/GzBRvJOqs7z901hq5x0YEuyRgTpnw9yOeIyO9wbnwDuA/I9U9J4UlV+d2SbfzxXzuYOKwXf7pttA3LbYwJKF8vpP8eUAe8ApEgqSQAABgZSURBVMwFanBCwrSDRo/y4zc28Md/7eBrWWn89Y7zLByMMQHn61VMVcDDfq4lLNXUN/LA3NUs3riX+yYM4KGrh9gIrMaYoODrVUxLRKSb1+vuIrLYf2WFh/Lqeu6avZLFG/fys6nD+Y9rhlo4GGOChq99EMmqerDphaoeEJF2u5M6HJUcquGu2Sv5srSSJ28dzfXn9g10ScYY04yvAeERkQxVzQMQkX60Mrqr8c1XZVXc+bcv2F/l3B1tA+4ZY4KRrwHxCPCpiHwECHAp7iB55uQ03R2twJx7xnFuqN0A52mEimIoL4BDBVCxF6JjoXP3o4/Ybs7PTolgTWrGBC1fO6nfE5EsnFBYDbwJVPuzsFD06fYyvv2PHLrFxfCPb46lf0oHm8NBFaoPOAf/8gI4VAjl+VBeePT1oSLQRt/eTyKbB0fnbi1etwiUI6+7QmSUU0/9Yag7DHWV7vMq53G8522tqz/s1NXy8+N6uM97tHjtLouySZpMaPJ1sL5vAQ/gDJq3BhgHLKf5FKSmDW+vLeLBeWsYkJLAC3ePpVeXILw7ur7aPdjnuwf/ghZhUHD0INokIhq69IWu6XDWRdA1DbqkOq+7pkJiH2iodYKl+gDUHDz6vOWjsgRKt0L1Qagtb7vWqM7QcJJ/o0TGQEw8RMdDTNzR5wm9nNfR8YAeradsGxzeD9X7wdNw/PeNjndDo1srIeL+jO3ifF5Mgvtwn3dKcOqyMykThHxtYnoAGAOsUNUJIjIU+I3/ygotf//sK37xzibGnNWDZ7+eRdfOAb47umoflG6B0s3OAbl0i/Ozcu+x2yb0cg74KUNh4MRjAyC+J0T4cDFclz4nV2NjA9SUHz9Y6iohOs55xMS7B/s49+Ab1/x5UxBEnuLN/6rO51UfcAOjqQ73+eEWr0s2Hd3Ol7OpiCj3OyQe/S6dWgSJd6AceZ4I3TKge6bzPY1pZ77+j6lR1RoRQUQ6qeoWERni18pCgPfd0VcP78WTt57hu6MrS92D/5ajIVCyGQ6XHd0mJgFShjgH/x6ZzoG/S6obBH0hqtOZq9dbZBTEJzmPQBNxDsZNB2RfqULtIScs6iqhttJtzqp0H1UtlldBXcXR5wfzm2/X8uzNW2If6NG/lUemU7cxp8DXgChw74N4E1giIgeA3f4rq+Nr9Cg/eXM9c1bmM2NMOr+64Wz/zACnClWlzoH/yNmA+zi87+h2nbo4QTBkEqQMc84Ieg51wsCaN/xDxOkvie164m194Wl0QqIpUGrK4eBu2L/z6GP7+8eeCSb0ah4YPQYcfR3bpX1qMyHJ107qG92nPxeRpUBX4D2/VRUC3lpTyJyV+e17d7Qq7FkLeSuaB0H1gaPbdOrqHPiHXucGwRAnDLr0tSDo6CIij57JNEk7/9jtaitg/1dewfGl8/rLf8GaPc23jUuGpAHNzzq6pjl9SxERzoUEEZFePyOcR7NlTT+llWWRvjVBmqB00o2yqvqRPwoJNR9uLqFnYqfTD4f6Gtj1CWxdCFvfg4oiZ3lsN+g5DIZPcwKg6ZHY24Ig3HVKhD4jnUdLdVUtwsN9fPUxrJ3jv5okwulriYh2mg+PPI92n0e18rzltu5P720jo53/Cwm9IKGn8+8/oad74UG8/75PmLAhu/2gvtHDx9tLufbsPqcWDlVlsG2xEwpfLoX6KqeTdeAVMOSn0H+CBYE5NTHx0Pts59FSfTUc2OVcseZpdB7q/dMD6mmxrNFZ1vTzmO291nsajj4a68FT71yM4Glwn9e3WN/gXMJ8ZL3709Nw9HljndPUpp5WvmuCGxZeoXHkZy9IdH/GJZ/6BQwhzq+/FRGZBPwBiASeU9XHW6y/DPg9MBKYoarzvdb9FrgOZ7yoJcAD6svsRkFg1e4DVNQ0MGGoj3dIqzqXVG5dCFsXQf5KQCGxL5w7A4ZcC/0ucW44M8Zfojs7Z6U9hwW6kpPjaXQuBKgsdvpfKkua/6zYC3s3On9stXr5tEB88tHgSOgFCSnOmdiRK8jij//8dK6QC3J++1YiEokzf8RVQAGQLSILVHWT12Z5wEzgoRb7XgRcjBMcAJ8C44Fl/qq3PS3bVkpUhHDxwOTjb9TYAHnLnUDYtsg5zQfocy5c/jAMmQy9R9pZgjEnEhHpHNATUoBz2t62vtoNjpLjB0rZNudnY53vNUTFnjhIYuIhLgn6joa+53WICwT8GXtjgR2quhNAROYC04AjAaGqu9x1Lc8PFYgFYnCG9ogGWrlIPzgt3VJCVr/uJMa2uN+hphx2fOD0JWx/37m2PzIGMsfDhffD4EnOvQXGGP+I7gzdz3IeJ9JQ5zTvHrkEubL5HfnNXh/neVVZ89dHLlUWp88wLQvSxjg/U4Y6YRdE/BkQqUC+1+sC4AJfdlTV5e7VUntwAuJPqrq55XYiMgt3TKiMjJO4Pt2P9pRXs6W4gh9NHuosOLAbtr3nNB/t+tRpP41Lcq4yGjwJBlzh3PxkjAkuUTHOo3P39nvP6gNQuAoKcqAgG7a8A6v/4ayLSYDU8yDVKzQSAjtodlA2nInIQGAYztAe4Nx7camqfuK9nao+AzwDkJWVFRT9E8u2lgJwTUoZ/OXrsHe9syJ5CFx4n9OfkDYm6P5SMMacAZ27w8ArnQc4/Y/7dzph0RQanz95dGiXbhnO8aIpNPqMPKM3r/ozIAqBdK/Xae4yX9yIM6xHJYCILAIuBD5pc68gsHRLCYO6Kmct+bbT3nn1r53+hKQBgS7NGBNsRJxjQ9IA54IUcI4be9YeDY28L2DDa866yBjofY57hjEGUs+H7v381lfpz4DIBgaJSCZOMMwAbvNx3zzgHhH5b5wmpvE4VzsFtboGD5/tKGVO978hB/PgGwshY1ygyzLGdCTRnZ3jhvex49AeKMw5GhqrXoQv/uqsi0t2RkiY9lS7l+K3gFDVBhG5H1iMc5nrbFXdKCKPATmqukBExgBvAN2BqSLyC1UdAczHGSl2PU6H9Xuq+ra/am0vObv2c2PjYkaW/wsm/sLCwRjTPrr0gS5TYdhU53VjgzMoZEE2FOY6/Rd+IB3k1oITysrK0pycnIDW8Ny8N7hr47eIGDiBqNvn2RADxpigJyK5qprV2jo7grWXmnImb36YiqhuRP3b0xYOxpgOz45i7UGVw/Pvo5enhM9HPxEcQ1QbY8xpsoBoD9nPEbfjbZ5omM6IC64OdDXGGNMuLCBOV9EaWPxj1nUey3tdbiEz2UaQNMaEBguI01FTDq/OROOSmVV5D5cP7dU+8z4YY0wQsIA4Vaqw4HtwMI81435PcX08lw8N7G3xxhjTniwgTlX2c7DpLZj4M97al0anqAgu7G+d08aY0GEBcSqKVsPiH8Oga+DC7/HRtlIuGpBEbLSNr2SMCR0WECfL7XcgPgVu/Ctf7a/mq7IqLh9izUvGmNBiAXEymvodygvg5uchrgfLtpYAMMECwhgTYiwgTsbKZ51+hyt/BhnO1BZLt5bSPyWejKS4ABdnjDHtywLCV4Wr4P1HnEl+LrwfgOq6Rlbs3GdnD8aYkGQB4Yvqg26/Q0+44S9HxllavrOMugaPBYQxJiQF5YxyQaWp3+FQIXxjEcT1OLJq6ZZS4mIiGZPZjlMSGmNMkLAziBNZ+SxsXuD0O6SPPbJYVVm6tYSLBiTTKcoubzXGhB4LiLa00u/Q5MvSSgoOVDNhaEqAijPGGP+ygDie4/Q7NFm6pRTA7n8wxoQsvwaEiEwSka0iskNEHm5l/WUiskpEGkTk5hbrMkTkfRHZLCKbRKSfP2ttRhUW3O/0O9zyfLN+hybLtpUwpFciqd06n7GyjDHmTPJbQIhIJPAUMBkYDtwqIsNbbJYHzARebuUtXgSeUNVhwFigxF+1HmPlM7D5bZj482b9Dk0qaxtY+dV+Lh9izUvGmNDlz6uYxgI7VHUngIjMBaYBm5o2UNVd7jqP945ukESp6hJ3u0o/1tlc4SpY/AgMnnxMv0OTz3aUUd+o1rxkjAlp/mxiSgXyvV4XuMt8MRg4KCKvi8hqEXnCPSNpRkRmiUiOiOSUlpaefsVN/Q6JveGGP8Nx5nZYtrWEhE5RZPWzy1uNMaErWDupo4BLgYeAMUB/nKaoZlT1GVXNUtWslJTTbO5Rhbfuc/odbp7dar+D+5ks21rKpYOSiY4M1l+fMcacPn8e4QqBdK/Xae4yXxQAa1R1p6o2AG8C57Vzfc198TRseee4/Q5Ntu6tYE95jd09bYwJef4MiGxgkIhkikgMMANYcBL7dhORptOCK/Dqu2h3hbnw/k/a7Hdo0nR563jroDbGhDi/BYT7l//9wGJgMzBPVTeKyGMicj2AiIwRkQLgFuBpEdno7tuI07z0oYisBwR41i+F+tjv0GTp1hKG9+lCry6xfinHGGOChV/HYlLVhcDCFsse9XqejdP01Nq+S4CR/qwPgIZa6J4JV/z0uP0OTcqr68ndfYDvjO/v97KMMSbQbLC+xF5w11snPHMA5/LWRo9a/4MxJizYZTjgUzgALN1SQtfO0YxK7+bngowxJvAsIHzk8SjLtjmXt0bZ5a3GmDBgRzofbdpziNKKWmteMsaEDQsIHy3d4gwFZZe3GmPChQWEj5ZtK+XctK4kJ3QKdCnGGHNGWED44EBVHavzDtjgfMaYsGIB4YOPt5fiUWx4b2NMWLGA8MGyraX0iI9hZJpd3mqMCR8WECfg8SgfbStl/OAUIiN8u1/CGGNCgQXECawrLGd/VZ01Lxljwo4FxAks3VJChMBlgywgjDHhxQLiBJZtLWFUeje6x8cEuhRjjDmjLCDaUFpRy9qCcrt72hgTliwg2vDxNmdyoAlDLSCMMeHHAqINy7aVkpLYieF9ugS6FGOMOeP8GhAiMklEtorIDhF5uJX1l4nIKhFpEJGbW1nfRUQKRORP/qyzNQ2NHj52L2+NsMtbjTFhyG8BISKRwFPAZGA4cKuIDG+xWR4wE3j5OG/zS+Bjf9XYljX5Bymvrrf+B2NM2PLnGcRYYIeq7lTVOmAuMM17A1XdparrAE/LnUXkfKAX8L4fazyupVtLiIwQLhmUHIiPN8aYgPNnQKQC+V6vC9xlJyQiEcD/Ag+dYLtZIpIjIjmlpaWnXGhrlm4p5fyzutO1c3S7vq8xxnQUwdpJfS+wUFUL2tpIVZ9R1SxVzUpJab8b2fYeqmHTnkPWvGSMCWtRfnzvQiDd63Wau8wXFwKXisi9QAIQIyKVqnpMR7c/fLTVORux4TWMMeHMnwGRDQwSkUycYJgB3ObLjqp6e9NzEZkJZJ2pcACn/6F3l1iG9k48Ux9pjDFBx29NTKraANwPLAY2A/NUdaOIPCYi1wOIyBgRKQBuAZ4WkY3+qsdX9Y0ePtlexoShKYjY5a3GmPDlzzMIVHUhsLDFske9nmfjND219R5/B/7uh/JalbPrAJW1DTZ7nDEm7AVrJ3XALNtWQnSkcPFAu7zVGBPeLCBaWLallDH9epDQya8nV8YYE/QsILwUHqxm694Ku7zVGGOwgGhm2dYSACYMtctbjTHGAsLL0i2lpHXvzICUhECXYowxAWcB4aptaOTzL8uYMKSnXd5qjDFYQByR/dUBDtc12t3TxhjjsoBwLd1aQkxUBBcOSAp0KcYYExQsIFxLt5Ywrn8ScTF2easxxoAFBAB5+w6zs7SKCda8ZIwxR1hA4Nw9Ddj9D8YY48UCAli6pYR+SXH0S44PdCnGGBM0wj4gauob+fzLfTY4nzHGtBD2AXGoup5rRvTmmhG9A12KMcYElbC/ZKdnl1ievHV0oMswxpigE/ZnEMYYY1pnAWGMMaZVfg0IEZkkIltFZIeIHDOntIhcJiKrRKRBRG72Wj5KRJaLyEYRWSci0/1ZpzHGmGP5LSBEJBJ4CpgMDAduFZHhLTbLA2YCL7dYfhi4S1VHAJOA34tIN3/Vaowx5lj+7KQeC+xQ1Z0AIjIXmAZsatpAVXe56zzeO6rqNq/nRSJSAqQAB/1YrzHGGC/+bGJKBfK9Xhe4y06KiIwFYoAvW1k3S0RyRCSntLT0lAs1xhhzrKDupBaRPsA/gG+oqqflelV9RlWzVDUrJcXGUTLGmPbkz4AoBNK9Xqe5y3wiIl2Ad4FHVHVFO9dmjDHmBPzZB5ENDBKRTJxgmAHc5suOIhIDvAG8qKrzfdknNze3TER2n2qxQDJQdhr7n0kdqVboWPV2pFqhY9XbkWqFjlXv6dR61vFWiKqe4nuemIhcC/weiARmq+qvReQxIEdVF4jIGJwg6A7UAMWqOkJE7gCeBzZ6vd1MVV3jx1pzVDXLX+/fnjpSrdCx6u1ItULHqrcj1Qodq15/1erXoTZUdSGwsMWyR72eZ+M0PbXc75/AP/1ZmzHGmLYFdSe1McaYwLGAOOqZQBdwEjpSrdCx6u1ItULHqrcj1Qodq16/1OrXPghjjDEdl51BGGOMaZUFhDHGmFaFfUCcaMTZYCIi6SKyVEQ2uSPdPhDomk5ERCJFZLWIvBPoWk5ERLqJyHwR2SIim0XkwkDXdDwi8kP338AGEZkjIrGBrsmbiMwWkRIR2eC1rIeILBGR7e7P7oGssclxan3C/XewTkTeCKbBQlur12vdv4uIikhye3xWWAeEjyPOBpMG4N9VdTgwDrgvyOsFeADYHOgifPQH4D1VHQqcS5DWLSKpwPeBLFU9G+c+oxmBreoYf8cZidnbw8CHqjoI+NB9HQz+zrG1LgHOVtWRwDbgR2e6qDb8nWPrRUTSgatxRsluF2EdEHiNOKuqdUDTiLNBSVX3qOoq93kFzgHspAdAPFNEJA24Dngu0LWciIh0BS4D/gagqnWqGsyjB0cBnUUkCogDigJcTzOq+jGwv8XiacAL7vMXgBvOaFHH0Vqtqvq+qja4L1fQyv1agXKc3y3A/wH/CbTblUfhHhDtMuJsIIhIP2A08EVgK2nT73H+wR4z0GIQygRKgefdJrHnRCQ+0EW1RlULgf/B+UtxD1Cuqu8Htiqf9FLVPe7zYqBXIIs5CXcDiwJdRFtEZBpQqKpr2/N9wz0gOiQRSQBeA36gqocCXU9rRGQKUKKquYGuxUdRwHnAX1R1NFBF8DSBNOO23U/DCbW+QLw7PE2Hoc719UF/jb2IPILTtPtSoGs5HhGJA34MPHqibU9WuAfEaY04GwgiEo0TDi+p6uuBrqcNFwPXi8gunKa7K0QkmIdPKQAKVLXpjGw+TmAEo4nAV6paqqr1wOvARQGuyRd73SH8m4byLwlwPW0SkZnAFOB2De4bxgbg/LGw1v3/lgasEpHep/vG4R4QR0acdUeQnQEsCHBNxyUigtNGvllVfxfoetqiqj9S1TRV7Yfze/2XqgbtX7mqWgzki8gQd9GVeM1+GGTygHEiEuf+m7iSIO1Qb2EB8HX3+deBtwJYS5tEZBJO8+j1qno40PW0RVXXq2pPVe3n/n8rAM5z/02flrAOCLcT6n5gMc5/sHmqurHtvQLqYuBOnL/G17iPawNdVAj5HvCSiKwDRgG/CXA9rXLPcuYDq4D1OP+Pg2pYCBGZAywHhohIgYh8E3gcuEpEtuOcBT0eyBqbHKfWPwGJwBL3/9lfA1qkl+PU65/PCu4zJ2OMMYES1mcQxhhjjs8CwhhjTKssIIwxxrTKAsIYY0yrLCCMMca0ygLCmCAgIpd3hBFvTXixgDDGGNMqCwhjToKI3CEiK92bp55257uoFJH/c+dn+FBEUtxtR4nICq85Bbq7yweKyAcislZEVonIAPftE7zmo3jJvUvamICxgDDGRyIyDJgOXKyqo4BG4HYgHshR1RHAR8DP3F1eBP7LnVNgvdfyl4CnVPVcnDGUmkY4HQ38AGdukv44d84bEzBRgS7AmA7kSuB8INv9474zzoBzHuAVd5t/Aq+780t0U9WP3OUvAK+KSCKQqqpvAKhqDYD7fitVtcB9vQboB3zq/69lTOssIIzxnQAvqGqz2cVE5KcttjvV8WtqvZ43Yv8/TYBZE5MxvvsQuFlEesKROZbPwvl/dLO7zW3Ap6paDhwQkUvd5XcCH7kzARaIyA3ue3Ryx/M3JujYXyjG+EhVN4nIT4D3RSQCqAfuw5lcaKy7rgSnnwKcIa3/6gbATuAb7vI7gadF5DH3PW45g1/DGJ/ZaK7GnCYRqVTVhEDXYUx7syYmY4wxrbIzCGOMMa2yMwhjjDGtsoAwxhjTKgsIY4wxrbKAMMYY0yoLCGOMMa36/27fFU14lvLTAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EBR9nNUl6SBL",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def generate_seq(model, proc, n_words, seed_text):\n",
        "    in_text = seed_text\n",
        "    for _ in range(n_words):\n",
        "        vec = proc.transform([in_text])[:,1:]\n",
        "        index = np.argmax(model.predict(vec, verbose=0), axis=1)[0]\n",
        "        out_word = ''\n",
        "        if index == 1:\n",
        "            out_word = '_unk_'\n",
        "        else:\n",
        "            out_word = proc.id2token[index]\n",
        "        in_text += ' ' + out_word\n",
        "    return in_text"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "roauYvKE6SBR",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "e4fbc74b-0107-47e0-fb17-a6134a737af9"
      },
      "source": [
        "generate_seq(model, proc, max_length, 'cold')"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'cold cucumber and radish salad with lime dressing dressing salad salad'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1M9peJaz6SBU",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "k9b3TPOb6SBY",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "embedding_model = Model(inputs=model.inputs, outputs=model.layers[-3].output)\n",
        "import tensorflow as tf\n",
        "# import tensorflow_hub as hub\n",
        "\n",
        "#using goolge embeddings\n",
        "# embed = hub.load(\"https://tfhub.dev/google/universal-sentence-encoder/4\")\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1r0ZHudr6SBc",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# embedding_model = embed(Model(inputs=model.inputs, outputs=model.layers[-3].output))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kjjW3Tre6SBf",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 323
        },
        "outputId": "df187521-37e1-463a-a2b4-cdcaa28a3cbb"
      },
      "source": [
        "input_sequence = test_docs[random.randint(0, len(test_docs))]\n",
        "print('input sequence: ', input_sequence, '\\n\\nhidden states:\\n')\n",
        "vec = proc.transform([input_sequence])[:,1:]\n",
        "embedding_model.predict(vec)"
      ],
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "input sequence:  greek country chicken \n",
            "\n",
            "hidden states:\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[[-5.42407681e-04,  6.79876626e-01, -6.25794637e-05, ...,\n",
              "         -4.59693978e-03, -5.75528204e-01,  1.30559877e-02],\n",
              "        [-2.56459498e-05,  5.66342711e-01, -4.34591101e-07, ...,\n",
              "         -2.33814735e-05, -7.82317761e-03, -1.34587707e-03],\n",
              "        [-2.72134512e-05,  8.61239731e-01, -9.25380959e-07, ...,\n",
              "         -8.50662618e-05, -7.37224892e-03, -1.27710449e-03],\n",
              "        ...,\n",
              "        [-1.19688184e-06,  2.14657299e-02, -1.39537747e-06, ...,\n",
              "         -2.58620867e-06, -2.99489126e-03, -1.19166132e-02],\n",
              "        [-1.22945441e-03,  1.17683625e-02, -2.97509786e-03, ...,\n",
              "         -2.38019181e-03, -1.72720514e-02, -9.95290279e-01],\n",
              "        [-9.29809563e-11,  5.44131435e-05, -1.45661124e-05, ...,\n",
              "         -8.28980592e-06, -8.71996939e-01, -3.08233539e-05]]],\n",
              "      dtype=float32)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5BYWE3HG6SBj",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "test_vecs = proc.transform(test_docs)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "d4fCdgNP6SBn",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "hidden_states = embedding_model.predict(test_vecs[:, 1:])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UhRk4_wP6SBq",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "mean_vecs = np.mean(hidden_states, axis=1)\n",
        "max_vecs = np.max(hidden_states, axis=1)\n",
        "sum_vecs = np.sum(hidden_states, axis=1)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LYxbJCH56SBs",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 88
        },
        "outputId": "f590aa90-486b-48c1-e255-fcab84927028"
      },
      "source": [
        "dimension = hidden_states.shape[-1]\n",
        "index = AnnoyIndex(dimension)\n",
        "for i, v in enumerate(sum_vecs):\n",
        "    index.add_item(i, v)\n",
        "index.build(10)"
      ],
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:2: FutureWarning: The default argument for metric will be removed in future version of Annoy. Please pass metric='angular' explicitly.\n",
            "  \n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 29
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1RNXAP4Q6SBw",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 221
        },
        "outputId": "6a4f304c-be32-49a8-b0da-a4b619d7bc10"
      },
      "source": [
        "input_sequence = test_docs[random.randint(0, len(test_docs))]\n",
        "print('Query: ', input_sequence)\n",
        "\n",
        "vec = proc.transform([input_sequence])[:,1:]\n",
        "vec = np.sum(embedding_model.predict(vec), axis=1)\n",
        "ids, _ = index.get_nns_by_vector(vec.T, 10, include_distances=True)\n",
        "\n",
        "print('\\nSearch Results:')\n",
        "[test_docs[i] for i in ids][1:]"
      ],
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Query:  crack dip\n",
            "\n",
            "Search Results:\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['incredible omelet',\n",
              " 'favorite brownies',\n",
              " 'gooey rolls',\n",
              " 'creole cake',\n",
              " 'best burgers',\n",
              " 'classic blondies',\n",
              " 'bisquick shortcakes',\n",
              " 'good sandwich',\n",
              " 'baked cheesecake']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 30
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jc2GlKgv6SB0",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 187
        },
        "outputId": "041f838c-9fd4-42cf-9152-134fecd0854c"
      },
      "source": [
        "source_proc = processor(heuristic_pct_padding=.95, keep_n=50000)\n",
        "source_vecs = source_proc.fit_transform(source_docs)"
      ],
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:root:....tokenizing data\n",
            "WARNING:root:Setting maximum document length to 30 based upon heuristic of 0.95 percentile.\n",
            " See full histogram by insepecting the `document_length_stats` attribute.\n",
            "WARNING:root:(1/2) done. 44 sec\n",
            "WARNING:root:....building corpus\n",
            "WARNING:root:(2/2) done. 1 sec\n",
            "WARNING:root:Finished parsing 231,636 documents.\n",
            "WARNING:root:...fit is finished, beginning transform\n",
            "WARNING:root:...padding data\n",
            "WARNING:root:done. 2 sec\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "otBNULHu6SB3",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 187
        },
        "outputId": "688cb410-9b82-4dea-e35f-b17296996652"
      },
      "source": [
        "target_proc = processor(append_indicators=True, heuristic_pct_padding=.95, keep_n=12000, padding ='post')\n",
        "target_vecs = target_proc.fit_transform(target_docs)"
      ],
      "execution_count": 32,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:root:....tokenizing data\n",
            "WARNING:root:Setting maximum document length to 10 based upon heuristic of 0.95 percentile.\n",
            " See full histogram by insepecting the `document_length_stats` attribute.\n",
            "WARNING:root:(1/2) done. 16 sec\n",
            "WARNING:root:....building corpus\n",
            "WARNING:root:(2/2) done. 0 sec\n",
            "WARNING:root:Finished parsing 231,636 documents.\n",
            "WARNING:root:...fit is finished, beginning transform\n",
            "WARNING:root:...padding data\n",
            "WARNING:root:done. 1 sec\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SvI9y2686SB5",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8p05g_5g6SB8",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "encoder_input_data = source_vecs\n",
        "encoder_seq_len = encoder_input_data.shape[1]\n",
        "\n",
        "decoder_input_data = target_vecs[:, :-1]\n",
        "decoder_target_data = target_vecs[:, 1:]\n",
        "\n",
        "num_encoder_tokens = max(source_proc.id2token.keys()) + 1\n",
        "num_decoder_tokens = max(target_proc.id2token.keys()) + 1"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iXjEwNPu6SB_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Create encoder"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "scrolled": true,
        "id": "dR4fjSLi6SCC",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "word_emb_dim=512\n",
        "hidden_state_dim=1024\n",
        "encoder_seq_len=encoder_seq_len\n",
        "num_encoder_tokens=num_encoder_tokens\n",
        "num_decoder_tokens=num_decoder_tokens\n",
        "\n",
        "encoder_inputs = Input(shape=(encoder_seq_len,), name='Encoder-Input')\n",
        "x = Embedding(num_encoder_tokens, word_emb_dim, name='Body-Word-Embedding', mask_zero=False)(encoder_inputs)\n",
        "x = BatchNormalization(name='Encoder-Batchnorm-1')(x)\n",
        "_, state_h = GRU(hidden_state_dim, return_state=True, name='Encoder-Last-GRU', dropout=.5)(x)\n",
        "encoder_model = Model(inputs=encoder_inputs, outputs=state_h, name='Encoder-Model')\n",
        "seq2seq_encoder_out = encoder_model(encoder_inputs)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mlYVEJx16SCE",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 289
        },
        "outputId": "f6432fcd-7103-4c16-b77e-8b9b6b0ed2d3"
      },
      "source": [
        "encoder_model.summary()"
      ],
      "execution_count": 36,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"Encoder-Model\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "Encoder-Input (InputLayer)   [(None, 30)]              0         \n",
            "_________________________________________________________________\n",
            "Body-Word-Embedding (Embeddi (None, 30, 512)           2163712   \n",
            "_________________________________________________________________\n",
            "Encoder-Batchnorm-1 (BatchNo (None, 30, 512)           2048      \n",
            "_________________________________________________________________\n",
            "Encoder-Last-GRU (GRU)       [(None, 1024), (None, 102 4724736   \n",
            "=================================================================\n",
            "Total params: 6,890,496\n",
            "Trainable params: 6,889,472\n",
            "Non-trainable params: 1,024\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "StbOHa956SCI",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "decoder_inputs = Input(shape=(None,), name='Decoder-Input')\n",
        "dec_emb = Embedding(num_decoder_tokens, word_emb_dim, name='Decoder-Word-Embedding', mask_zero=False)(decoder_inputs)\n",
        "dec_bn = BatchNormalization(name='Decoder-Batchnorm-1')(dec_emb)\n",
        "decoder_gru = GRU(hidden_state_dim, return_state=True, return_sequences=True, name='Decoder-GRU', dropout=.5)\n",
        "decoder_gru_output, _ = decoder_gru(dec_bn, initial_state=seq2seq_encoder_out)\n",
        "x = BatchNormalization(name='Decoder-Batchnorm-2')(decoder_gru_output)\n",
        "decoder_dense = Dense(num_decoder_tokens, activation='softmax', name='Final-Output-Dense')\n",
        "decoder_outputs = decoder_dense(x)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AncfwqrG6SCK",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "seq2seq_model = Model([encoder_inputs, decoder_inputs], decoder_outputs)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-Hl4-N166SCO",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 442
        },
        "outputId": "04ca8f96-e374-4687-c6d2-c0ca7dea29e0"
      },
      "source": [
        "seq2seq_model.summary()"
      ],
      "execution_count": 39,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"model_2\"\n",
            "__________________________________________________________________________________________________\n",
            "Layer (type)                    Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            "Decoder-Input (InputLayer)      [(None, None)]       0                                            \n",
            "__________________________________________________________________________________________________\n",
            "Decoder-Word-Embedding (Embeddi (None, None, 512)    6145024     Decoder-Input[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "Encoder-Input (InputLayer)      [(None, 30)]         0                                            \n",
            "__________________________________________________________________________________________________\n",
            "Decoder-Batchnorm-1 (BatchNorma (None, None, 512)    2048        Decoder-Word-Embedding[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "Encoder-Model (Model)           (None, 1024)         6890496     Encoder-Input[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "Decoder-GRU (GRU)               [(None, None, 1024), 4724736     Decoder-Batchnorm-1[0][0]        \n",
            "                                                                 Encoder-Model[1][0]              \n",
            "__________________________________________________________________________________________________\n",
            "Decoder-Batchnorm-2 (BatchNorma (None, None, 1024)   4096        Decoder-GRU[0][0]                \n",
            "__________________________________________________________________________________________________\n",
            "Final-Output-Dense (Dense)      (None, None, 12002)  12302050    Decoder-Batchnorm-2[0][0]        \n",
            "==================================================================================================\n",
            "Total params: 30,068,450\n",
            "Trainable params: 30,064,354\n",
            "Non-trainable params: 4,096\n",
            "__________________________________________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "scrolled": true,
        "id": "oTdG9sy66SCR",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 547
        },
        "outputId": "120b2819-912e-4e31-f628-2e2fd51bc736"
      },
      "source": [
        "# from keras import optimizers\n",
        "# model.compile(loss='sparse_categorical_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
        "batch_size = 512\n",
        "epochs = 15\n",
        "#i upped the learning rate by x10,see how that works, its at 21% acc at epoch 5 elsewise\n",
        "# seq2seq_model.compile(optimizer=optimizers.Nadam(lr=0.0005), loss='sparse_categorical_crossentropy', metrics=['accuracy'])\n",
        "seq2seq_model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy'])\n",
        "history = seq2seq_model.fit([encoder_input_data, decoder_input_data],\n",
        "                            np.expand_dims(decoder_target_data, -1),\n",
        "                            batch_size=batch_size,\n",
        "                            epochs=epochs,\n",
        "                            validation_split=0.1)"
      ],
      "execution_count": 43,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/15\n",
            "408/408 [==============================] - 165s 405ms/step - loss: 2.3158 - accuracy: 0.6261 - val_loss: 9.0808 - val_accuracy: 0.1316\n",
            "Epoch 2/15\n",
            "408/408 [==============================] - 167s 408ms/step - loss: 1.6608 - accuracy: 0.6783 - val_loss: 2.3518 - val_accuracy: 0.6333\n",
            "Epoch 3/15\n",
            "408/408 [==============================] - 166s 407ms/step - loss: 1.4554 - accuracy: 0.6964 - val_loss: 2.3974 - val_accuracy: 0.6364\n",
            "Epoch 4/15\n",
            "408/408 [==============================] - 167s 409ms/step - loss: 1.3166 - accuracy: 0.7118 - val_loss: 2.4636 - val_accuracy: 0.6363\n",
            "Epoch 5/15\n",
            "408/408 [==============================] - 166s 407ms/step - loss: 1.2136 - accuracy: 0.7252 - val_loss: 2.5169 - val_accuracy: 0.6362\n",
            "Epoch 6/15\n",
            "408/408 [==============================] - 166s 408ms/step - loss: 1.1291 - accuracy: 0.7372 - val_loss: 2.5795 - val_accuracy: 0.6337\n",
            "Epoch 7/15\n",
            "408/408 [==============================] - 167s 409ms/step - loss: 1.0572 - accuracy: 0.7482 - val_loss: 2.6580 - val_accuracy: 0.6337\n",
            "Epoch 8/15\n",
            "408/408 [==============================] - 167s 410ms/step - loss: 0.9927 - accuracy: 0.7587 - val_loss: 2.7128 - val_accuracy: 0.6321\n",
            "Epoch 9/15\n",
            "408/408 [==============================] - 167s 409ms/step - loss: 0.9341 - accuracy: 0.7686 - val_loss: 2.7710 - val_accuracy: 0.6307\n",
            "Epoch 10/15\n",
            "408/408 [==============================] - 167s 409ms/step - loss: 0.8808 - accuracy: 0.7779 - val_loss: 2.8341 - val_accuracy: 0.6303\n",
            "Epoch 11/15\n",
            "408/408 [==============================] - 166s 408ms/step - loss: 0.8323 - accuracy: 0.7865 - val_loss: 2.8999 - val_accuracy: 0.6282\n",
            "Epoch 12/15\n",
            "408/408 [==============================] - 166s 407ms/step - loss: 0.7872 - accuracy: 0.7951 - val_loss: 2.9532 - val_accuracy: 0.6291\n",
            "Epoch 13/15\n",
            "408/408 [==============================] - 166s 408ms/step - loss: 0.7465 - accuracy: 0.8027 - val_loss: 3.0114 - val_accuracy: 0.6266\n",
            "Epoch 14/15\n",
            "408/408 [==============================] - 167s 408ms/step - loss: 0.7084 - accuracy: 0.8102 - val_loss: 3.0641 - val_accuracy: 0.6271\n",
            "Epoch 15/15\n",
            "408/408 [==============================] - 167s 408ms/step - loss: 0.6732 - accuracy: 0.8173 - val_loss: 3.1172 - val_accuracy: 0.6243\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RjQEZKPE6SCT",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 295
        },
        "outputId": "43004204-8ea1-4313-cacb-2bd716f9aad3"
      },
      "source": [
        "from matplotlib import pyplot as plt\n",
        "plt.plot(history.history['accuracy'])\n",
        "plt.plot(history.history['val_accuracy'])\n",
        "plt.title('model accuracy')\n",
        "plt.ylabel('accuracy')\n",
        "plt.xlabel('epoch')\n",
        "plt.legend(['train', 'val'], loc='upper left')\n",
        "plt.show()"
      ],
      "execution_count": 44,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3deXxd5X3n8c9Pi7VZu7xLtsS+Y4NwSMm0JMAMhICTpgTIMqFLmDbQQJpOS9oMSZl2Xsy0odOFNiEpDW1Y4hDSuBknBCiQpoHEJnEAG4zkDcurdGXJkq1dv/njnCsfyZJ9bevqXt3zfb9eet2z3XN/V7ae3znP85znMXdHRETiKy/TAYiISGYpEYiIxJwSgYhIzCkRiIjEnBKBiEjMKRGIiMScEoHEipl9zcz+NMVjt5vZ1emOSSTTlAhERGJOiUBkFjKzgkzHILlDiUCyTlgl89/N7FUzO2Rm/2BmC8zse2bWY2bPmll15PgbzWyjmXWZ2Qtmdm5k3woz+1n4vm8AxRM+631mtiF874/N7KIUY7zezH5uZgfNbKeZfWHC/neF5+sK998Wbi8xsy+a2Q4z6zazH4XbrjSztkl+D1eHy18wsyfN7OtmdhC4zcxWmtlL4WfsMbO/NbM5kfefb2bPmFmnme0zsz8ys4VmdtjMaiPHXWJm7WZWmMp3l9yjRCDZ6oPANcBZwA3A94A/AuYR/L/9FICZnQU8Dtwd7lsL/KuZzQkLxX8B/hmoAb4ZnpfwvSuAh4H/BtQCXwbWmFlRCvEdAv4rUAVcD/yOmb0/PO+yMN6/CWNaDmwI3/cXwKXAL4Ux/QEwmuLvZBXwZPiZjwIjwKeBOuCdwFXAJ8MYyoFnge8Di4EzgOfcfS/wAvChyHk/Bjzh7kMpxiE5RolAstXfuPs+d98F/DvwE3f/ubv3A98GVoTH3Qz8P3d/JizI/gIoIShoLwcKgf/r7kPu/iSwLvIZtwNfdvefuPuIuz8CDITvOyZ3f8HdX3P3UXd/lSAZ/Uq4+8PAs+7+ePi5CXffYGZ5wG8Ad7n7rvAzf+zuAyn+Tl5y938JP7PP3V9x95fdfdjdtxMksmQM7wP2uvsX3b3f3Xvc/SfhvkeAjwKYWT5wK0GylJhSIpBstS+y3DfJ+txweTGwI7nD3UeBncCScN8uHz+y4o7I8jLgM2HVSpeZdQEN4fuOyczeYWbPh1Uq3cBvE1yZE55jyyRvqyOomppsXyp2TojhLDP7rpntDauL/lcKMQB8BzjPzJoI7rq63f2nJxmT5AAlApntdhMU6ACYmREUgruAPcCScFvS0sjyTuDP3L0q8lPq7o+n8LmPAWuABnevBL4EJD9nJ3D6JO/pAPqn2HcIKI18j3yCaqWoiUMF/z3wJnCmu1cQVJ1FYzhtssDDu6rVBHcFH0N3A7GnRCCz3WrgejO7Kmzs/AxB9c6PgZeAYeBTZlZoZr8KrIy89yvAb4dX92ZmZWEjcHkKn1sOdLp7v5mtJKgOSnoUuNrMPmRmBWZWa2bLw7uVh4EHzGyxmeWb2TvDNom3gOLw8wuBzwHHa6soBw4CvWZ2DvA7kX3fBRaZ2d1mVmRm5Wb2jsj+fwJuA25EiSD2lAhkVnP3zQRXtn9DcMV9A3CDuw+6+yDwqwQFXidBe8JTkfeuBz4B/C1wAGgNj03FJ4H7zKwHuJcgISXP+zbwXoKk1EnQUHxxuPv3gdcI2io6gf8N5Ll7d3jOrxLczRwCxvUimsTvEySgHoKk9o1IDD0E1T43AHuBFuDdkf3/QdBI/TN3j1aXSQyZJqYRiScz+zfgMXf/aqZjkcxSIhCJITO7DHiGoI2jJ9PxSGapakgkZszsEYJnDO5WEhDQHYGISOzpjkBEJOZm3cBVdXV13tjYmOkwRERmlVdeeaXD3Sc+mwLMwkTQ2NjI+vXrMx2GiMisYmZTdhNW1ZCISMwpEYiIxJwSgYhIzM26NoLJDA0N0dbWRn9/f6ZDSavi4mLq6+spLNT8ISIyfXIiEbS1tVFeXk5jYyPjB5rMHe5OIpGgra2NpqamTIcjIjkkJ6qG+vv7qa2tzdkkAGBm1NbW5vxdj4jMvJxIBEBOJ4GkOHxHEZl5OVE1JCKSSwaGR+joHWT/wX7aewZo7x2gvWeA95wzn4vqq6b985QIpkFXVxePPfYYn/zkJ0/ofe9973t57LHHqKqa/n9YEcku7k7X4SH29wyEhXtQyO8/eKSgb+8ZYH/PAN19Q5Oeo25ukRJBturq6uLv/u7vjkoEw8PDFBRM/Steu3ZtukMTkTQbGhkdK8D3HewPCvqD/UcV7h29AwyNHD3IZ3FhHvPLi5lfXsQZ8+fyztNrmTe3iPkVRcwrL2Le3GLmVxRRUzaHwvz01OanNRGY2bXAXwH5wFfd/f4J+5cCjwBV4TH3uPusKx3vuecetmzZwvLlyyksLKS4uJjq6mrefPNN3nrrLd7//vezc+dO+vv7ueuuu7j99tuBI8Nl9Pb2ct111/Gud72LH//4xyxZsoTvfOc7lJSUZPibicTX4PAo7b1h4X5wgP09wWuysN8XVtskDg0e9V4zqC0LCvL55UWcuaB8bDko3MN9FcWUzcnPePtf2hJBOPn2gwTT5bUB68xsjbtvihz2OWC1u/+9mZ0HrAUaT+Vz/+RfN7Jp98FTOcVRzltcwedvOH/K/ffffz+vv/46GzZs4IUXXuD666/n9ddfH+vm+fDDD1NTU0NfXx+XXXYZH/zgB6mtrR13jpaWFh5//HG+8pWv8KEPfYhvfetbfPSjH53W7yEiQf17ULAPsD9SqEcL930H+zlw+Ojqmfw8o27uHBZUFFNfXcIly6qZX17Egorgin5+eXD1Xls2h4I0Xb2nQzrvCFYCre6+FcDMngBWAdFE4EBFuFwJ7E5jPDNm5cqV4/r6//Vf/zXf/va3Adi5cyctLS1HJYKmpiaWL18OwKWXXsr27dtnLF6RXDD+Cr6ffeFV/L6Dxy/gC/Js7Iq9oaaUS5dVjxXuCyqKw6v3ImrLisjPy73ee+lMBEuAnZH1NuAdE475AvADM/tdoAy4erITmdntwO0AS5cuPeaHHuvKfaaUlZWNLb/wwgs8++yzvPTSS5SWlnLllVdO+ixAUVHR2HJ+fj59fX0zEqtIthseGR27Wt93cID2SOG+L3JV3zlJFU1+noVX6kEB39xYzYLwqn1+pKCvKZ1DXg4W8KnKdGPxrcDX3P2LZvZO4J/N7AJ3H40e5O4PAQ8BNDc3Z92UauXl5fT0TD7jX3d3N9XV1ZSWlvLmm2/y8ssvz3B0ItnL3UkcGmR3Vx+7u/rZ090XLHf3syfctr+nn9EJf/X5eTbWoFpfHVzBzy8vZkFFWE1TEVTT1JbFu4BPVToTwS6gIbJeH26L+k3gWgB3f8nMioE6YH8a45p2tbW1XHHFFVxwwQWUlJSwYMGCsX3XXnstX/rSlzj33HM5++yzufzyyzMYqcjMOtg/xJ6ufnaHBfy45e5+9nT3Mzg87rqPOQV5LK4sZnFVCVecUcfiqmIWVZawsDJaB5+bVTSZkrY5i82sAHgLuIogAawDPuzuGyPHfA/4hrt/zczOBZ4DlvgxgmpubvaJE9O88cYbnHvuuWn4FtknTt9Vslv/0MhYgX6kYI9e2ffTOzA87j15BgsrillUVcKiymKWhK+LqkrGlmvK5mS8F00uMrNX3L15sn1puyNw92EzuxN4mqBr6MPuvtHM7gPWu/sa4DPAV8zs0wQNx7cdKwmIyMwYHB5l38EjBfzu7uBqPlrQT9boWls2h0VVxTTWlvFLp9dFCvngqn5+edGs6k0TF2ltIwifCVg7Ydu9keVNwBXpjEFExhsdddp7B9jVdXThnqyf7+gdYOIlWUVxAYvDq/blS6tYXBkU7ouqillcWcLCymKKC/Mz86XklGS6sVhEptnIqLPvYD+7uvpoO3CYts6+cDlY393Vz+DI+Hr50jn5LArr5c9eWM6iypKxuvnka1mRiotcpX9ZkVlmeGSUvQf7aTvQx64DRwr4tgNBgb+7q4/hCd1s6uYWUV9dwgVLKvkvFyykvrp0rLpmcWUJFSUFqpePMSUCkSwzOursPdjPzs7DvN15mJ1jBX5Q2O892M/IhIJ+QUURS6pKWN5QxfsuWhQU9NUl1FcHjbCqspFjUSIQyYDuviF2dh6OFPaHebuzj7bOoLCPVt1Y2NOmvrqElU01LKkKCvhkYb9IdfNyipQIMmDu3Ln09vZmOgxJo8HhUXZ39fF2pKAfK/Q7+44aZriqtJCG6lLOXVTBNecvYGlNKUtrSmmoLmVxVQlzCtTTRtJHiUDkJHX3DfF24jDbE4fYkTjEjkRQ0Lcd6GNPd9+4p2Hn5OdRX1NCQ3UpKxqqaagpYWlNKfXVpTTUlFJZUpi5LyKxp0QwDe655x4aGhq44447APjCF75AQUEBzz//PAcOHGBoaIg//dM/ZdWqVRmOVE5EcviDZCG/PXF4bHlH4tBR/ejnlxextKaUdzTV0FATFPBLa0ppqClhQXmxhjqQrJV7ieB798De16b3nAsvhOvun3L3zTffzN133z2WCFavXs3TTz/Npz71KSoqKujo6ODyyy/nxhtvVM+MLJNsmE0W7tsTh3m78xDbO4Kr++iTsXkGi6tKaKwt47oLF9FYW8qy2jKW1QYFfumc3PtzknjQ/9xpsGLFCvbv38/u3btpb2+nurqahQsX8ulPf5of/vCH5OXlsWvXLvbt28fChQszHW7sjI46ew72s639ENsSh9jecaQqZ0fn4XFj3RTmGw3VpSyrLWVlUw3LaktprC1jaW0p9dUlFBWoUVZyT+4lgmNcuafTTTfdxJNPPsnevXu5+eabefTRR2lvb+eVV16hsLCQxsbGSYeflunh7uzvGWBbR1DQb0scYlv7obD+/jADkcK+qCCPxtoymurKuPLseSyrLaMxvLJfXFWiwcwkdnIvEWTIzTffzCc+8Qk6Ojp48cUXWb16NfPnz6ewsJDnn3+eHTt2ZDrEWc/d6Tw0yPbEIbZ1HGZbRy/bOw4HhX/iEIcHR8aOLcw3ltaU0lRXxq+cNY/GujKaastorCtjYYXq60WilAimyfnnn09PTw9Llixh0aJFfOQjH+GGG27gwgsvpLm5mXPOOSfTIc4afYMjbGnvZUt7L1vDq/rtHYfY2nGInv4jdfb5eUZDdQmNdWWsbKqhqS4o6E+rK9OVvcgJUCKYRq+9dqSRuq6ujpdeemnS4/QMQeDAoUFa23tp3d/Llv29Y8u7uvrGBjwzg8WVJTTVlbFq+WKa6ubSVBfU2zfUlFKokSxFTpkSgaTV6Kizu7uPLe2HaN1/pNDf0t5LIjK1YFFBHqfNm8uKpdXcdGkDZ8yfy+nzg7p7PTUrkl5KBDItBodH2ZEICvst4ZV9a3svW/Yfom/oSN19VWkhZ8yby9XnLuCM+XPHflSVI5I5OZMI3D3n++hnw5w97k7bgT427+3hzb0HeXNvD5v39rCt49C4ES8XVxZz+vy53LKyJri6nxcU+LWafUok6+REIiguLiaRSFBbW5uzhYy7k0gkKC4unrHP7O4bYvPeHjbvPcgbYYH/1t4eeiIPWdVXl3DOwnKuOW8BZy6YyxnzyjltXpnGrheZRXLir7W+vp62tjba29szHUpaFRcXU19fP+3nHRweZWtHL5v39vDGnqDg37y3h93dR557qCgu4JyFFXzgkiWcvbCccxaWc9aCcsqLNUaOyGyXE4mgsLCQpqamTIeR9dyD4RTe3NPDm2HVzua9PWxp72VoJKjWKcw3Tp83l5VNNZy9sIJzFpZzzqJyFlYU5+zdlkjc5UQikMm19wzwalsXr7Z182pbF6/t6qaj90hPncWVxZyzqIJ3nzM/KPAXVtBUV6Yhj0ViJq2JwMyuBf4KyAe+6u73T9j/l8C7w9VSYL67V6UzplzVdXiQV9u6eW1X91jhvyes2skzOGP+XK48ez4XLqnkvMUVnLWgXEMfiwiQxkRgZvnAg8A1QBuwzszWuPum5DHu/unI8b8LrEhXPLmkd2CY1yMF/qtt3bzdeXhsf1P4pO2FSyq5qL6K8xdXqPFWRKaUztJhJdDq7lsBzOwJYBWwaYrjbwU+n8Z4ZqW+wRE27TkYVO20dfOLti62dhwae/J2SVUJF9VXcuvKpVxUX8kFSyp1pS8iJySdiWAJsDOy3ga8Y7IDzWwZ0AT82xT7bwduB1i6dOn0RplFkn30f7qtk/U7Ovn521207O8dm6h8XnkRF9dXsmr5Ei6sr+TCJZXUzS3KcNQiMttlS33BLcCT7j4y2U53fwh4CKC5uTnzT1VNk9FRp7W9l59s62Tdtk7Wbe8cq9evKC5g+dJqrj53ARfVB1U8Cytn7hkCEYmPdCaCXUBDZL0+3DaZW4A70hhLVhgaGWXj7oOs29bJT8Kr/q5wusP55UVc1lTDO5pquKyxhrMXlGuoZBGZEelMBOuAM82siSAB3AJ8eOJBZnYOUA1MPlTnLNY3OMLPdx5g3bYDrNveyc/ePjA2Zn5jbSnXnLtgrPBfWlOqfvoikhFpSwTuPmxmdwJPE3QffdjdN5rZfcB6d18THnoL8IRnw0A6p6i7b4hXdnSOVfW8tquboRHHDM5eUM5Nl9ZzWVMNKxtrmF+hah4RyQ4228rf5uZmX79+fabDGPPKjgOs2bCLn2zrZPO+HtyhIM+4qL5yrNBvXlZDZal68ohI5pjZK+7ePNm+bGksnnX6h0Z44Jm3+Mq/b6W4IJ9LllVx9wVncVlTNSsaqimZozH0RWR2UCI4CZt2H+TT39jA5n09fPgdS/nj956rB7ZEZNZS6XUCRkadh364lQee2UxlyRwevq2Z95yzINNhiYicEiWCFL2dOMxnvrmBddsPcN0FC/mzD1xITdmcTIclInLKlAiOw91ZvX4n9/3rJvLMeOBDF/OBFUvU1VNEcoYSwTG09wzw2ade5dk39nP5aTV88UPLWVJVkumwRESmlRLBFH6wcS+ffeo1egaG+dz15/IbVzTpSV8RyUlKBBP09A/xP7+7idXr2zhvUQWP37KcsxaUZzosEZG0USKI+Om2Tn5v9QZ2d/Vxx7tP566rztJsXSKS85QIgIHh4OGwh364lYbqUr752+/k0mU1mQ5LRGRGxD4RvLEneDjszb093LpyKZ+7Xg+HiUi8xLbEGxl1vvrvW/niD96ioqSQf/h4M1edO4MPh40Mw9DhyE8fDB4ev+1ExoGasjvrJNvHjrVT35Zct3woKIKC4uC1sCSyXnxku7rdimSdWCaCnZ2H+czqX/DT7Z1ce/5C/uwDF1B7sjN97d4ALT8ICu7BSKGeLMwHJ6wnt40OTe+Xmi2SCWFigigohsLoephI8gqmSB7HSnApHJtXAPmF4c+cyHpyeU4K+wohr/Do5bxCyMuPLBdAntqaJHvFKhG4O998pY0/WbMRM+OLN13Mr15yig+HPf1HsOM/gsKhsAQKy8LXUpgT/pTVBeuFJTAnsj+5LXnsuG0lQQFy9JeY7JtN9YWnPnbcvlS2RfZNtm10BIYHYLg/8jMQJMET2d5/cPz20eH0/A5GR2BkCEYGYfKJ8aaX5R1JCvkFkaRRcCTRHLVcGB5bAFhwDgtfsXDZptg32Trj1/MKgmSbP+fIa3R53GsRFMyZ8DrFccnfr48E/36jI+CjkW3R1xS348G5CyMXEGN3nSXB70x3myctNomgo3eAzz71Gs9s2sflp9XwFzddTH116TSc+C1Y8TFY9benfi7JjNHR4A4tmRhGhycsD4brQ5HjksuT7BsdiSwPHznf6NCRBDQa7hsZjixPPH44vHscBjxIaj4aWZ64nlwePc6xo2FBOwzDgzAyMHnCnU0sb/xd5lQJY6zaMtyXX3jk9wHjf3cTf18e+d1Ouj1yfF5B+DklQSyFpZFYwvXC4nB/SWT7hOWC4hlJcLFJBF9/eQcvbm6f3ofD+rrgUDvUnXnq55LMycuDvPAKN65GR4OEMDwQJrfBI8vjXgeOJI+x1wnHYUHVWF5+0HY07jXvBLbnHVnHgs8aitxBjt1V9k3YPskx/d0wvD9cjxw3OjThril5JxVZP2pfdPsk+7AgsQ71RWLrO/l/m2QyKSiBqz8PF98yHf/i4z9i2s+YpX7nytN530WLOGP+ND4cltgSvNaeMX3nFMmEvDzIC69GZfq5RxJT+Dq2fPhIshi3vW98Mhnqg4rFaQkvNomgqCB/epMAQKI1eK3VHYGIHIPZkWqfLKSuDKci0RLculY3ZjoSEZGTpkRwKhKtUL0s6EUhIjJLpTURmNm1ZrbZzFrN7J4pjvmQmW0ys41m9lg645l2Ha1qHxCRWS9tbQRmlg88CFwDtAHrzGyNu2+KHHMm8FngCnc/YGbz0xXPtBsdhc4t0PTLmY5EROSUpPOOYCXQ6u5b3X0QeAJYNeGYTwAPuvsBAHffn8Z4plfP7qC1v/b0TEciInJK0pkIlgA7I+tt4baos4CzzOw/zOxlM7t2shOZ2e1mtt7M1re3t6cp3BOU7DGkZwhEZJbLdGNxAXAmcCVwK/AVM6uaeJC7P+Tuze7ePG/evBkOcQodLcGr2ghEZJZLZyLYBTRE1uvDbVFtwBp3H3L3bcBbBIkh+yW2BOMKlS/KdCQiIqcknYlgHXCmmTWZ2RzgFmDNhGP+heBuADOrI6gq2prGmKZPoiVoH9BAVyIyy6UtEbj7MHAn8DTwBrDa3Tea2X1mdmN42NNAwsw2Ac8D/93dE+mKaVol1HVURHJDWoeYcPe1wNoJ2+6NLDvwe+HP7DE8AF1vw0U3ZzoSEZFTlunG4tmpc1sw1KzuCEQkBygRnIyEegyJSO5QIjgZY6OOKhGIyOynRHAyEq0wdwEUV2Q6EhGRU6ZEcDI02JyI5BAlgpOhrqMikkOUCE5U3wE43KFEICI5Q4ngRCXnKdZgcyKSI5QITpQGmxORHKNEcKISrZqnWERySkqJwMyeMrPrzUyJI9ESJIH8wkxHIiIyLVIt2P8O+DDQYmb3m9nZaYwpuyW2qH1ARHJKSonA3Z91948AlwDbgWfN7Mdm9utmFp9L49HRIBGofUBEckjKVT1mVgvcBvwW8HPgrwgSwzNpiSwbHdwFw31KBCKSU1IahtrMvg2cDfwzcIO77wl3fcPM1qcruKyjweZEJAelOh/BX7v785PtcPfmaYwnu+kZAhHJQalWDZ0XnVTezKrN7JNpiil7dbTAnLnBgHMiIjki1UTwCXfvSq64+wHgE+kJKYslxxjSPMUikkNSTQT5ZkdKPzPLB+akJ6QslmhR+4CI5JxUE8H3CRqGrzKzq4DHw23xMdQPXTvVPiAiOSfVRPCHwPPA74Q/zwF/cLw3mdm1ZrbZzFrN7J5J9t9mZu1mtiH8+a0TCX5GdW4FXHcEIpJzUuo15O6jwN+HPykJq48eBK4B2oB1ZrbG3TdNOPQb7n5nqufNGE1PKSI5KtWxhs40syfNbJOZbU3+HOdtK4FWd9/q7oPAE8CqUw04Y8aeITg9s3GIiEyzVKuG/pHgbmAYeDfwT8DXj/OeJcDOyHpbuG2iD5rZq2GiaZjsRGZ2u5mtN7P17e3tKYY8zRJboHwRFJVn5vNFRNIk1URQ4u7PAebuO9z9C8D10/D5/wo0uvtFBENVPDLZQe7+kLs3u3vzvHnzpuFjT0KHegyJSG5KNREMhENQt5jZnWb2AWDucd6zC4he4deH28a4e8LdB8LVrwKXphjPzNM8xSKSo1JNBHcBpcCnCArrjwIfP8571gFnmlmTmc0BbgHWRA8ws0WR1RuBN1KMZ2Yd7oS+TiUCEclJx+01FPb+udndfx/oBX49lRO7+7CZ3Qk8DeQDD7v7RjO7D1jv7muAT5nZjQRtD50Eo5tmn2SPIT1DICI56LiJwN1HzOxdJ3Nyd18LrJ2w7d7I8meBz57MuWeUuo6KSA5LdfTRn5vZGuCbwKHkRnd/Ki1RZZuOFsgrgKqlmY5ERGTapZoIioEE8J7INgfikQgSrVDdpHmKRSQnpfpkcUrtAjlLPYZEJIelOkPZPxLcAYzj7r8x7RFlm+Q8xWdclelIRETSItWqoe9GlouBDwC7pz+cLNS9E0YGdEcgIjkr1aqhb0XXzexx4EdpiSjbjPUYUtdREclNqT5QNtGZwPzpDCRrqeuoiOS4VNsIehjfRrCXYI6C3JdohaIKmBuPvCci8ZNq1VB8h9zsaAmGntY8xSKSo1Kdj+ADZlYZWa8ys/enL6wsktii9gERyWmpthF83t27kyvu3gV8Pj0hZZGhvqDXkNoHRCSHpZoIJjsu1a6ns1dynuI6JQIRyV2pJoL1ZvaAmZ0e/jwAvJLOwLJCR3J6SiUCEcldqSaC3wUGgW8QzD3cD9yRrqCyRrLraI3mKRaR3JVqr6FDwD1pjiX7JFqhfDEUHW8yNhGR2SvVXkPPmFlVZL3azJ5OX1hZItGq9gERyXmpVg3VhT2FAHD3A+T6k8XumrBeRGIh1UQwamZjs7KYWSOTjEaaUw53Qn+XniEQkZyXahfQPwZ+ZGYvAgb8J+D2tEWVDRLqMSQi8ZDSHYG7fx9oBjYDjwOfAfqO9z4zu9bMNptZq5lN2dhsZh80Mzez5hTjTr+xCeuVCEQkt6U66NxvAXcB9cAG4HLgJcZPXTnxPfnAg8A1QBuwzszWuPumCceVh+f+ycl8gbTpaIG8QqjUPMUikttSbSO4C7gM2OHu7wZWAF3HfgsrgVZ33+rugwTPH6ya5Lj/CfxvgmcTskeiFWpOg/zcf4BaROIt1UTQ7+79AGZW5O5vAmcf5z1LgJ2R9bZw2xgzuwRocPf/l2IcM0fzFItITKR6udsWPkfwL8AzZnYA2HEqH2xmecADwG0pHHs7YeP00qUzUFUzOhKMM3TWf0n/Z4mIZFiqTxZ/IFz8gpk9D1QC3z/O23YBDZH1+nBbUjlwAfCCBWP9LwTWmNmN7r5+wuc/BDwE0NzcnP5uq907YWRQdwQiEgsnXAHu7i+meOg64EwzayJIALcAH46cpxuoS66b2QvA709MAhnRoXmKRSQ+TnbO4uNy92HgTuBp4A1gtbtvNLP7zOzGdDPAWTcAAAwYSURBVH3utNA8xSISI2ntEuPua4G1E7bdO8WxV6YzlhOSaIHiSiirO/6xIiKzXNruCGa1ZI8hzVMsIjGgRDCZjla1D4hIbCgRTDR4GA62qX1ARGJDiWCizi3Ba61mJROReFAimGhssDlVDYlIPCgRTJR8hqDmtMzGISIyQ5QIJkq0QkU9zCnLdCQiIjNCiWCiRIvaB0QkVpQIotzDCevVPiAi8aFEEHWoA/q71XVURGJFiSAqocHmRCR+lAiixiasVxuBiMSHEkFUohXy50CV5ikWkfhQIojqCOcpzsvPdCQiIjNGiSBK8xSLSAwpESSNDAfzFCsRiEjMKBEkdb8No0N6hkBEYkeJIKlD01OKSDwpESTpGQIRiSklgqRECxRXQWlNpiMREZlRaU0EZnatmW02s1Yzu2eS/b9tZq+Z2QYz+5GZnZfOeI4pOcaQ5ikWkZhJWyIws3zgQeA64Dzg1kkK+sfc/UJ3Xw78H+CBdMVzXIktah8QkVhK5x3BSqDV3be6+yDwBLAqeoC7H4yslgGexnimNngIDu5SIhCRWCpI47mXADsj623AOyYeZGZ3AL8HzAHeM9mJzOx24HaApUvTMPxDIjlPsRKBiMRPxhuL3f1Bdz8d+EPgc1Mc85C7N7t787x586Y/iORgc3qGQERiKJ2JYBfQEFmvD7dN5Qng/WmMZ2qJLYBpnmIRiaV0JoJ1wJlm1mRmc4BbgDXRA8wsegl+PdCSxnim1tEClQ1QWJKRjxcRyaS0tRG4+7CZ3Qk8DeQDD7v7RjO7D1jv7muAO83samAIOAB8PF3xHFOiVXMQiEhspbOxGHdfC6ydsO3eyPJd6fz8lCTnKb74lkxHIiKSERlvLM64Q+0wcFA9hkQktpQIOpLTUyoRiEg8KREkNOqoiMSbEkGiBfKLgl5DIiIxpESQ2BL0GMrTr0JE4kmlX0eLuo6KSKzFOxGMDMOBbZqMRkRiLd6JoGsHjA6roVhEYi3eiSDZY0iDzYlIjMU7EegZAhGRmCeCRCuU1GieYhGJNSUC3Q2ISMwpEah9QERiLr6JYKAHevboGQIRib34JoKxeYp1RyAi8RbjRKDB5kREIPaJQPMUi4jEOxFUNUBhcaYjERHJqPgmgo4WtQ+IiBDXROAeDj+t9gERkbQmAjO71sw2m1mrmd0zyf7fM7NNZvaqmT1nZsvSGc+Y3n0w2KNnCERESGMiMLN84EHgOuA84FYzO2/CYT8Hmt39IuBJ4P+kK55xxnoM6RkCEZF03hGsBFrdfau7DwJPAKuiB7j78+5+OFx9GahPYzxHjA02pzsCEZF0JoIlwM7Ielu4bSq/CXxvsh1mdruZrTez9e3t7aceWaIVCkqg4ljhiIjEQ1Y0FpvZR4Fm4M8n2+/uD7l7s7s3z5s379Q/MNGqeYpFREIFaTz3LqAhsl4fbhvHzK4G/hj4FXcfSGM8RyRaYcH5M/JRIiLZLp2XxOuAM82syczmALcAa6IHmNkK4MvAje6+P42xHDEyBAe2q31ARCSUtkTg7sPAncDTwBvAanffaGb3mdmN4WF/DswFvmlmG8xszRSnmz4HNE+xiEhUOquGcPe1wNoJ2+6NLF+dzs+fVCLsMaRnCEREgCxpLJ5ReoZARGSc+CWCjhYorYOS6kxHIiKSFeKXCDTGkIjIODFMBC1Qp0QgIpIUr0TQfzAYcE53BCIiY+KVCMYaitVjSEQkKWaJIDlhve4IRESSYpYIWsDyoKYp05GIiGSNmCWCVqhaCgVFmY5ERCRrxCsRdLSoWkhEZIL4JIKxeYrVUCwiEhWfRNCzF4YOaWgJEZEJ4pMINNiciMikYpQIks8QqI1ARCQqPolg7gI4+3ooX5zpSEREskpa5yPIKudcH/yIiMg48bkjEBGRSSkRiIjEnBKBiEjMKRGIiMRcWhOBmV1rZpvNrNXM7plk/y+b2c/MbNjMfi2dsYiIyOTSlgjMLB94ELgOOA+41czOm3DY28BtwGPpikNERI4tnd1HVwKt7r4VwMyeAFYBm5IHuPv2cN9oGuMQEZFjSGfV0BJgZ2S9Ldx2wszsdjNbb2br29vbpyU4EREJzIoHytz9IeAhADNrN7MdJ3mqOqBj2gJLv9kU72yKFWZXvLMpVphd8c6mWOHU4l021Y50JoJdQENkvT7cdkrcfd7JvtfM1rt786nGMFNmU7yzKVaYXfHOplhhdsU7m2KF9MWbzqqhdcCZZtZkZnOAW4A1afw8ERE5CWlLBO4+DNwJPA28Aax2941mdp+Z3QhgZpeZWRtwE/BlM9uYrnhERGRyaW0jcPe1wNoJ2+6NLK8jqDKaKQ/N4GdNh9kU72yKFWZXvLMpVphd8c6mWCFN8Zq7p+O8IiIyS2iICRGRmFMiEBGJudgkguONe5QtzKzBzJ43s01mttHM7sp0TKkws3wz+7mZfTfTsRyLmVWZ2ZNm9qaZvWFm78x0TMdiZp8O/x+8bmaPm1lxpmOKMrOHzWy/mb0e2VZjZs+YWUv4Wp3JGJOmiPXPw/8Lr5rZt82sKpMxJk0Wa2TfZ8zMzaxuuj4vFokgxXGPssUw8Bl3Pw+4HLgji2ONuougd1i2+yvg++5+DnAxWRyzmS0BPgU0u/sFQD5BN+xs8jXg2gnb7gGec/czgefC9WzwNY6O9RngAne/CHgL+OxMBzWFr3F0rJhZA/CfCcZpmzaxSARExj1y90EgOe5R1nH3Pe7+s3C5h6CgOqmhOWaKmdUD1wNfzXQsx2JmlcAvA/8A4O6D7t6V2aiOqwAoMbMCoBTYneF4xnH3HwKdEzavAh4Jlx8B3j+jQU1hsljd/QdhV3eAl5nZXoxTmuL3CvCXwB8A09rLJy6JYNrGPZpJZtYIrAB+ktlIjuv/EvznzPbBA5uAduAfw2qsr5pZWaaDmoq77wL+guDqbw/Q7e4/yGxUKVng7nvC5b3AgkwGcwJ+A/hepoOYipmtAna5+y+m+9xxSQSzjpnNBb4F3O3uBzMdz1TM7H3Afnd/JdOxpKAAuAT4e3dfARwie6otjhLWra8iSGCLgTIz+2hmozoxHvRPz/o+6mb2xwTVso9mOpbJmFkp8EfAvcc79mTEJRGkZdyjdDGzQoIk8Ki7P5XpeI7jCuBGM9tOUOX2HjP7emZDmlIb0ObuyTusJwkSQ7a6Gtjm7u3uPgQ8BfxShmNKxT4zWwQQvu7PcDzHZGa3Ae8DPuLZ+2DV6QQXBL8I/9bqgZ+Z2cLpOHlcEsGsGffIzIygDvsNd38g0/Ecj7t/1t3r3b2R4Pf6b+6elVet7r4X2GlmZ4ebriIyP0YWehu43MxKw/8XV5HFjdsRa4CPh8sfB76TwViOycyuJajWvNHdD2c6nqm4+2vuPt/dG8O/tTbgkvD/9CmLRSKYatyjzEY1pSuAjxFcWW8If96b6aByyO8Cj5rZq8By4H9lOJ4phXcuTwI/A14j+HvNqiERzOxx4CXgbDNrM7PfBO4HrjGzFoK7mvszGWPSFLH+LVAOPBP+rX0po0GGpog1fZ+XvXdCIiIyE2JxRyAiIlNTIhARiTklAhGRmFMiEBGJOSUCEZGYUyIQmUFmdmW2j9Aq8aNEICISc0oEIpMws4+a2U/Dh4y+HM630GtmfxnOD/Ccmc0Lj11uZi9HxrSvDrefYWbPmtkvzOxnZnZ6ePq5kTkRHg2fGhbJGCUCkQnM7FzgZuAKd18OjAAfAcqA9e5+PvAi8PnwLf8E/GE4pv1rke2PAg+6+8UEYwQlR+RcAdxNMDfGaQRPk4tkTEGmAxDJQlcBlwLrwov1EoKB00aBb4THfB14KpzjoMrdXwy3PwJ808zKgSXu/m0Ad+8HCM/3U3dvC9c3AI3Aj9L/tUQmp0QgcjQDHnH3cbNVmdn/mHDcyY7PMhBZHkF/h5JhqhoSOdpzwK+Z2XwYm4N3GcHfy6+Fx3wY+JG7dwMHzOw/hds/BrwYzi7XZmbvD89RFI4pL5J1dCUiMoG7bzKzzwE/MLM8YAi4g2Aim5Xhvv0E7QgQDLX8pbCg3wr8erj9Y8CXzey+8Bw3zeDXEEmZRh8VSZGZ9br73EzHITLdVDUkIhJzuiMQEYk53RGIiMScEoGISMwpEYiIxJwSgYhIzCkRiIjE3P8HjasjIveTnLkAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GTrpHjZ66SCa",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# !wget https://storage.googleapis.com/kdd-seq2seq-2018/kdd_seq2seq_weights.h5"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AycV7qCf6SCd",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# seq2seq_model.load_weights('kdd_seq2seq_weights.h5')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ic7wjxjK6SCg",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def extract_decoder_model(model):\n",
        "    latent_dim = model.get_layer('Encoder-Model').output_shape[-1]\n",
        "    decoder_inputs = model.get_layer('Decoder-Input').input\n",
        "    dec_emb = model.get_layer('Decoder-Word-Embedding')(decoder_inputs)\n",
        "    dec_bn = model.get_layer('Decoder-Batchnorm-1')(dec_emb)\n",
        "    gru_inference_state_input = Input(shape=(latent_dim,), name='hidden_state_input')\n",
        "    gru_out, gru_state_out = model.get_layer('Decoder-GRU')([dec_bn, gru_inference_state_input])\n",
        "    dec_bn2 = model.get_layer('Decoder-Batchnorm-2')(gru_out)\n",
        "    dense_out = model.get_layer('Final-Output-Dense')(dec_bn2)\n",
        "    decoder_model = Model([decoder_inputs, gru_inference_state_input], [dense_out, gru_state_out])\n",
        "    return decoder_model"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "scrolled": true,
        "id": "9h5v7SKG6SCn",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 408
        },
        "outputId": "deb60a19-0848-4e33-a343-d8a0756e4b95"
      },
      "source": [
        "encoder_model = seq2seq_model.get_layer('Encoder-Model')\n",
        "for layer in encoder_model.layers:\n",
        "    layer.trainable = False\n",
        "\n",
        "decoder_model = extract_decoder_model(seq2seq_model)\n",
        "decoder_model.summary()"
      ],
      "execution_count": 47,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"model_3\"\n",
            "__________________________________________________________________________________________________\n",
            "Layer (type)                    Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            "Decoder-Input (InputLayer)      [(None, None)]       0                                            \n",
            "__________________________________________________________________________________________________\n",
            "Decoder-Word-Embedding (Embeddi (None, None, 512)    6145024     Decoder-Input[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "Decoder-Batchnorm-1 (BatchNorma (None, None, 512)    2048        Decoder-Word-Embedding[1][0]     \n",
            "__________________________________________________________________________________________________\n",
            "hidden_state_input (InputLayer) [(None, 1024)]       0                                            \n",
            "__________________________________________________________________________________________________\n",
            "Decoder-GRU (GRU)               [(None, None, 1024), 4724736     Decoder-Batchnorm-1[1][0]        \n",
            "                                                                 hidden_state_input[0][0]         \n",
            "__________________________________________________________________________________________________\n",
            "Decoder-Batchnorm-2 (BatchNorma (None, None, 1024)   4096        Decoder-GRU[1][0]                \n",
            "__________________________________________________________________________________________________\n",
            "Final-Output-Dense (Dense)      (None, None, 12002)  12302050    Decoder-Batchnorm-2[1][0]        \n",
            "==================================================================================================\n",
            "Total params: 23,177,954\n",
            "Trainable params: 23,174,882\n",
            "Non-trainable params: 3,072\n",
            "__________________________________________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7jujjain6SCq",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 224
        },
        "outputId": "752c8316-eb77-4a93-966c-82db884efec1"
      },
      "source": [
        "i = random.randint(0, len(test_source_docs))\n",
        "\n",
        "max_len = target_proc.padding_maxlen\n",
        "raw_input_text = test_source_docs[i]\n",
        "\n",
        "raw_tokenized = source_proc.transform([raw_input_text])\n",
        "encoding = encoder_model.predict(raw_tokenized)\n",
        "original_encoding = encoding\n",
        "state_value = np.array(target_proc.token2id['_start_']).reshape(1, 1)\n",
        "\n",
        "decoded_sentence = []\n",
        "stop_condition = False\n",
        "while not stop_condition:\n",
        "    preds, st = decoder_model.predict([state_value, encoding])\n",
        "    pred_idx = np.argmax(preds[:, :, 2:]) + 2\n",
        "    pred_word_str = target_proc.id2token[pred_idx]\n",
        "\n",
        "    if pred_word_str == '_end_' or len(decoded_sentence) >= max_len:\n",
        "        stop_condition = True\n",
        "        break\n",
        "    decoded_sentence.append(pred_word_str)\n",
        "\n",
        "    # update the decoder for the next word\n",
        "    encoding = st\n",
        "    state_value = np.array(pred_idx).reshape(1, 1)\n",
        "\n",
        "print('sample code from test set:\\n------------------------\\n', raw_input_text)\n",
        "print('\\nground truth:\\n------------------------\\n', test_target_docs[i])\n",
        "print('\\npredicted description:\\n------------------------')\n",
        "print(' '.join(decoded_sentence))"
      ],
      "execution_count": 48,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "sample code from test set:\n",
            "------------------------\n",
            " frozen chopped spinach, salted butter, fresh garlic, onions, flour, heavy cream, fresh lemon juice, pecorino romano cheese, sour cream, sharp white cheddar cheese, mozzarella cheese\n",
            "\n",
            "ground truth:\n",
            "------------------------\n",
            " your last spinach dip recipe\n",
            "\n",
            "predicted description:\n",
            "------------------------\n",
            "paula deen s spinach and cheese casserole\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Fcqpl7jJ6SCs",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#generate embeddings"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YZL-CpL_6SCu",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "train_source_emb = encoder_model.predict(source_proc.transform(source_docs))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fKPWtglo6SCw",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "train_target_vecs = proc.transform(target_docs)\n",
        "hidden_states = embedding_model.predict(train_target_vecs[:, 1:])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IsepAYYe6SCy",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "mean_vecs = np.mean(hidden_states, axis=1)\n",
        "max_vecs = np.max(hidden_states, axis=1)\n",
        "sum_vecs = np.sum(hidden_states, axis=1)\n",
        "# train_target_emb = mean_vecs\n",
        "train_target_emb = max_vecs"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "o41uJq356SC1",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "83a5e2c6-2dd2-443d-f5d8-27ce2c9fdb88"
      },
      "source": [
        "print('source embedding shape on training set: ', train_source_emb.shape)\n",
        "print('target embedding shape on training set: ', train_target_emb.shape)"
      ],
      "execution_count": 53,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "source embedding shape on training set:  (231636, 1024)\n",
            "target embedding shape on training set:  (231636, 256)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ium1lybM6SC5",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 88
        },
        "outputId": "250e92b4-5449-4e8a-e5a5-bfd5b704b91c"
      },
      "source": [
        "dimension = hidden_states.shape[-1]\n",
        "index = AnnoyIndex(dimension)\n",
        "for i, v in enumerate(sum_vecs):\n",
        "    index.add_item(i, v)\n",
        "index.build(10)"
      ],
      "execution_count": 54,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:2: FutureWarning: The default argument for metric will be removed in future version of Annoy. Please pass metric='angular' explicitly.\n",
            "  \n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 54
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "D0svDuBt6SC_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#create joint vector space for embeddings"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gcAnNNA46SDC",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 221
        },
        "outputId": "764d36b2-b46b-494a-c278-9518e74cbf83"
      },
      "source": [
        "inp = Input(shape=(train_source_emb.shape[1],))\n",
        "x = Dense(train_target_emb.shape[1], use_bias=False)(inp)\n",
        "# x = BatchNormalization()(x)\n",
        "# x = Dense(train_target_emb.shape[1])(x)\n",
        "modal_model = Model([inp], x)\n",
        "modal_model.summary()"
      ],
      "execution_count": 56,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"model_4\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "input_2 (InputLayer)         [(None, 1024)]            0         \n",
            "_________________________________________________________________\n",
            "dense_1 (Dense)              (None, 256)               262144    \n",
            "=================================================================\n",
            "Total params: 262,144\n",
            "Trainable params: 262,144\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "scrolled": true,
        "id": "dDkJBoyT6SDE",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "3a667d34-d5f4-4c90-d121-dadc9dae5440"
      },
      "source": [
        "# modal_model.compile(optimizer=optimizers.Nadam(lr=2), loss='cosine_proximity', metrics=['accuracy'])\n",
        "modal_model.compile(optimizer = 'adam', loss='cosine_similarity', metrics=['accuracy'])\n",
        "batch_size = 512\n",
        "epochs = 50\n",
        "history = modal_model.fit([train_source_emb], train_target_emb,\n",
        "                          batch_size=batch_size, epochs=epochs, validation_split=0.1)"
      ],
      "execution_count": 61,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/50\n",
            "408/408 [==============================] - 2s 5ms/step - loss: -0.8544 - accuracy: 0.0364 - val_loss: -0.8709 - val_accuracy: 0.0452\n",
            "Epoch 2/50\n",
            "408/408 [==============================] - 2s 4ms/step - loss: -0.8788 - accuracy: 0.0485 - val_loss: -0.8730 - val_accuracy: 0.0491\n",
            "Epoch 3/50\n",
            "408/408 [==============================] - 2s 4ms/step - loss: -0.8807 - accuracy: 0.0525 - val_loss: -0.8738 - val_accuracy: 0.0516\n",
            "Epoch 4/50\n",
            "408/408 [==============================] - 2s 4ms/step - loss: -0.8814 - accuracy: 0.0546 - val_loss: -0.8740 - val_accuracy: 0.0500\n",
            "Epoch 5/50\n",
            "408/408 [==============================] - 2s 4ms/step - loss: -0.8818 - accuracy: 0.0564 - val_loss: -0.8741 - val_accuracy: 0.0519\n",
            "Epoch 6/50\n",
            "408/408 [==============================] - 2s 4ms/step - loss: -0.8820 - accuracy: 0.0573 - val_loss: -0.8742 - val_accuracy: 0.0515\n",
            "Epoch 7/50\n",
            "408/408 [==============================] - 2s 4ms/step - loss: -0.8821 - accuracy: 0.0581 - val_loss: -0.8742 - val_accuracy: 0.0518\n",
            "Epoch 8/50\n",
            "408/408 [==============================] - 2s 4ms/step - loss: -0.8822 - accuracy: 0.0587 - val_loss: -0.8741 - val_accuracy: 0.0547\n",
            "Epoch 9/50\n",
            "408/408 [==============================] - 2s 4ms/step - loss: -0.8823 - accuracy: 0.0594 - val_loss: -0.8742 - val_accuracy: 0.0541\n",
            "Epoch 10/50\n",
            "408/408 [==============================] - 2s 4ms/step - loss: -0.8823 - accuracy: 0.0597 - val_loss: -0.8744 - val_accuracy: 0.0533\n",
            "Epoch 11/50\n",
            "408/408 [==============================] - 2s 4ms/step - loss: -0.8824 - accuracy: 0.0593 - val_loss: -0.8743 - val_accuracy: 0.0559\n",
            "Epoch 12/50\n",
            "408/408 [==============================] - 2s 4ms/step - loss: -0.8824 - accuracy: 0.0602 - val_loss: -0.8743 - val_accuracy: 0.0548\n",
            "Epoch 13/50\n",
            "408/408 [==============================] - 2s 4ms/step - loss: -0.8824 - accuracy: 0.0600 - val_loss: -0.8743 - val_accuracy: 0.0523\n",
            "Epoch 14/50\n",
            "408/408 [==============================] - 2s 4ms/step - loss: -0.8825 - accuracy: 0.0597 - val_loss: -0.8743 - val_accuracy: 0.0550\n",
            "Epoch 15/50\n",
            "408/408 [==============================] - 2s 4ms/step - loss: -0.8825 - accuracy: 0.0602 - val_loss: -0.8743 - val_accuracy: 0.0536\n",
            "Epoch 16/50\n",
            "408/408 [==============================] - 2s 4ms/step - loss: -0.8825 - accuracy: 0.0600 - val_loss: -0.8744 - val_accuracy: 0.0549\n",
            "Epoch 17/50\n",
            "408/408 [==============================] - 2s 4ms/step - loss: -0.8826 - accuracy: 0.0604 - val_loss: -0.8743 - val_accuracy: 0.0527\n",
            "Epoch 18/50\n",
            "408/408 [==============================] - 2s 4ms/step - loss: -0.8826 - accuracy: 0.0604 - val_loss: -0.8743 - val_accuracy: 0.0571\n",
            "Epoch 19/50\n",
            "408/408 [==============================] - 2s 4ms/step - loss: -0.8826 - accuracy: 0.0601 - val_loss: -0.8743 - val_accuracy: 0.0550\n",
            "Epoch 20/50\n",
            "408/408 [==============================] - 2s 4ms/step - loss: -0.8826 - accuracy: 0.0604 - val_loss: -0.8744 - val_accuracy: 0.0567\n",
            "Epoch 21/50\n",
            "408/408 [==============================] - 2s 4ms/step - loss: -0.8826 - accuracy: 0.0603 - val_loss: -0.8744 - val_accuracy: 0.0575\n",
            "Epoch 22/50\n",
            "408/408 [==============================] - 2s 4ms/step - loss: -0.8827 - accuracy: 0.0603 - val_loss: -0.8745 - val_accuracy: 0.0536\n",
            "Epoch 23/50\n",
            "408/408 [==============================] - 2s 4ms/step - loss: -0.8827 - accuracy: 0.0605 - val_loss: -0.8744 - val_accuracy: 0.0540\n",
            "Epoch 24/50\n",
            "408/408 [==============================] - 2s 4ms/step - loss: -0.8827 - accuracy: 0.0606 - val_loss: -0.8743 - val_accuracy: 0.0539\n",
            "Epoch 25/50\n",
            "408/408 [==============================] - 2s 4ms/step - loss: -0.8827 - accuracy: 0.0602 - val_loss: -0.8744 - val_accuracy: 0.0584\n",
            "Epoch 26/50\n",
            "408/408 [==============================] - 2s 4ms/step - loss: -0.8827 - accuracy: 0.0607 - val_loss: -0.8744 - val_accuracy: 0.0544\n",
            "Epoch 27/50\n",
            "408/408 [==============================] - 2s 4ms/step - loss: -0.8827 - accuracy: 0.0607 - val_loss: -0.8744 - val_accuracy: 0.0563\n",
            "Epoch 28/50\n",
            "408/408 [==============================] - 2s 4ms/step - loss: -0.8827 - accuracy: 0.0605 - val_loss: -0.8745 - val_accuracy: 0.0564\n",
            "Epoch 29/50\n",
            "408/408 [==============================] - 2s 4ms/step - loss: -0.8827 - accuracy: 0.0607 - val_loss: -0.8745 - val_accuracy: 0.0551\n",
            "Epoch 30/50\n",
            "408/408 [==============================] - 2s 4ms/step - loss: -0.8828 - accuracy: 0.0607 - val_loss: -0.8745 - val_accuracy: 0.0531\n",
            "Epoch 31/50\n",
            "408/408 [==============================] - 2s 4ms/step - loss: -0.8828 - accuracy: 0.0604 - val_loss: -0.8744 - val_accuracy: 0.0552\n",
            "Epoch 32/50\n",
            "408/408 [==============================] - 2s 4ms/step - loss: -0.8828 - accuracy: 0.0603 - val_loss: -0.8744 - val_accuracy: 0.0581\n",
            "Epoch 33/50\n",
            "408/408 [==============================] - 2s 4ms/step - loss: -0.8828 - accuracy: 0.0603 - val_loss: -0.8744 - val_accuracy: 0.0509\n",
            "Epoch 34/50\n",
            "408/408 [==============================] - 2s 4ms/step - loss: -0.8828 - accuracy: 0.0605 - val_loss: -0.8745 - val_accuracy: 0.0561\n",
            "Epoch 35/50\n",
            "408/408 [==============================] - 2s 4ms/step - loss: -0.8828 - accuracy: 0.0605 - val_loss: -0.8745 - val_accuracy: 0.0569\n",
            "Epoch 36/50\n",
            "408/408 [==============================] - 2s 4ms/step - loss: -0.8828 - accuracy: 0.0608 - val_loss: -0.8744 - val_accuracy: 0.0537\n",
            "Epoch 37/50\n",
            "408/408 [==============================] - 2s 4ms/step - loss: -0.8828 - accuracy: 0.0606 - val_loss: -0.8744 - val_accuracy: 0.0548\n",
            "Epoch 38/50\n",
            "408/408 [==============================] - 2s 4ms/step - loss: -0.8828 - accuracy: 0.0609 - val_loss: -0.8744 - val_accuracy: 0.0550\n",
            "Epoch 39/50\n",
            "408/408 [==============================] - 2s 4ms/step - loss: -0.8828 - accuracy: 0.0607 - val_loss: -0.8745 - val_accuracy: 0.0526\n",
            "Epoch 40/50\n",
            "408/408 [==============================] - 2s 4ms/step - loss: -0.8828 - accuracy: 0.0606 - val_loss: -0.8744 - val_accuracy: 0.0591\n",
            "Epoch 41/50\n",
            "408/408 [==============================] - 2s 4ms/step - loss: -0.8828 - accuracy: 0.0608 - val_loss: -0.8745 - val_accuracy: 0.0537\n",
            "Epoch 42/50\n",
            "408/408 [==============================] - 2s 4ms/step - loss: -0.8828 - accuracy: 0.0609 - val_loss: -0.8744 - val_accuracy: 0.0544\n",
            "Epoch 43/50\n",
            "408/408 [==============================] - 2s 4ms/step - loss: -0.8829 - accuracy: 0.0609 - val_loss: -0.8745 - val_accuracy: 0.0551\n",
            "Epoch 44/50\n",
            "408/408 [==============================] - 2s 5ms/step - loss: -0.8829 - accuracy: 0.0607 - val_loss: -0.8745 - val_accuracy: 0.0559\n",
            "Epoch 45/50\n",
            "408/408 [==============================] - 2s 4ms/step - loss: -0.8829 - accuracy: 0.0612 - val_loss: -0.8743 - val_accuracy: 0.0566\n",
            "Epoch 46/50\n",
            "408/408 [==============================] - 2s 4ms/step - loss: -0.8829 - accuracy: 0.0604 - val_loss: -0.8744 - val_accuracy: 0.0559\n",
            "Epoch 47/50\n",
            "408/408 [==============================] - 2s 4ms/step - loss: -0.8829 - accuracy: 0.0609 - val_loss: -0.8745 - val_accuracy: 0.0572\n",
            "Epoch 48/50\n",
            "408/408 [==============================] - 2s 4ms/step - loss: -0.8829 - accuracy: 0.0606 - val_loss: -0.8744 - val_accuracy: 0.0555\n",
            "Epoch 49/50\n",
            "408/408 [==============================] - 2s 4ms/step - loss: -0.8829 - accuracy: 0.0603 - val_loss: -0.8745 - val_accuracy: 0.0570\n",
            "Epoch 50/50\n",
            "408/408 [==============================] - 2s 4ms/step - loss: -0.8829 - accuracy: 0.0605 - val_loss: -0.8744 - val_accuracy: 0.0540\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jd1PRqAF6SDL",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 295
        },
        "outputId": "ce63dcc3-e6c5-4f62-b1de-f156c9f4ce5e"
      },
      "source": [
        "from matplotlib import pyplot as plt\n",
        "plt.plot(history.history['accuracy'])\n",
        "plt.plot(history.history['val_accuracy'])\n",
        "plt.title('model accuracy')\n",
        "plt.ylabel('accuracy')\n",
        "plt.xlabel('epoch')\n",
        "plt.legend(['train', 'val'], loc='upper left')\n",
        "plt.show()"
      ],
      "execution_count": 64,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEWCAYAAABxMXBSAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nOzdd3icxbX48e9R77KabbnIlnvBvZtiMB0bTO+E0AkhIbnJLyHlckkuKeQmIYGQQoDQWyBgAzbGBkyzDe69yN2ybFldVtdq5/fH7EorebVaybuq5/M8frTl3d1Z2X7PO3NmzogxBqWUUspfIR3dAKWUUl2LBg6llFKtooFDKaVUq2jgUEop1SoaOJRSSrWKBg6llFKtooFDKR9E5DkRecTPYw+IyHnBbpNSHU0Dh1JKqVbRwKFUDyAiYR3dBtV9aOBQXZ5riOj/ichmESkXkWdEpI+ILBGREyKyXESSPI6/TES2iUixiKwQkdEez00SkfWu170ORDX5rPkistH12pUiMt7PNs4TkQ0iUioih0Xk4SbPn+F6v2LX8990PR4tIn8QkYMiUiIiX7geO1tEsr38Hs5z3X5YRN4UkZdEpBT4pohMF5FVrs84KiJ/EZEIj9ePFZFlIlIoIrki8lMR6SsiFSKS4nHcZBHJE5Fwf7676n40cKju4irgfGAEcCmwBPgpkIb9d/5dABEZAbwKfM/13GLgXRGJcJ1E3wFeBJKBf7veF9drJwHPAvcAKcA/gEUiEulH+8qBbwC9gHnAt0Tkctf7DnK19wlXmyYCG12v+z0wBZjtatOPAKefv5MFwJuuz3wZqAO+D6QCs4BzgftcbYgHlgMfAP2AYcBHxphjwArgWo/3vQV4zRhT62c7VDejgUN1F08YY3KNMUeAz4GvjDEbjDFVwNvAJNdx1wHvG2OWuU58vweisSfmmUA48CdjTK0x5k1gjcdn3A38wxjzlTGmzhjzPFDtep1PxpgVxpgtxhinMWYzNnjNcT19I7DcGPOq63MLjDEbRSQEuB14wBhzxPWZK40x1X7+TlYZY95xfWalMWadMWa1McZhjDmADXzuNswHjhlj/mCMqTLGnDDGfOV67nngZgARCQVuwAZX1UNp4FDdRa7H7Uov9+Nct/sBB91PGGOcwGGgv+u5I6Zx5c+DHrcHAT9wDfUUi0gxMND1Op9EZIaIfOIa4ikB7sVe+eN6j71eXpaKHSrz9pw/DjdpwwgReU9EjrmGr37tRxsAFgJjRCQT26srMcZ83cY2qW5AA4fqaXKwAQAAERHsSfMIcBTo73rMLcPj9mHgV8aYXh5/Yowxr/rxua8Ai4CBxphE4O+A+3MOA0O9vCYfqGrmuXIgxuN7hGKHuTw1LX39N2AnMNwYk4AdyvNswxBvDXf12t7A9jpuQXsbPZ4GDtXTvAHME5FzXcndH2CHm1YCqwAH8F0RCReRK4HpHq/9J3Cvq/cgIhLrSnrH+/G58UChMaZKRKZjh6fcXgbOE5FrRSRMRFJEZKKrN/Qs8EcR6ScioSIyy5VT2Q1EuT4/HPg50FKuJR4oBcpEZBTwLY/n3gPSReR7IhIpIvEiMsPj+ReAbwKXoYGjx9PAoXoUY8wu7JXzE9gr+kuBS40xNcaYGuBK7AmyEJsP+Y/Ha9cCdwF/AYqAPa5j/XEf8EsROQE8hA1g7vc9BFyCDWKF2MT4BNfTPwS2YHMthcCjQIgxpsT1nk9je0vlQKNZVl78EBuwTmCD4OsebTiBHYa6FDgGZAHneDz/JTYpv94Y4zl8p3og0Y2clFL+EJGPgVeMMU93dFtUx9LAoZRqkYhMA5ZhczQnOro9qmPpUJVSyicReR67xuN7GjQUaI9DKaVUK2mPQymlVKv0iMJnqampZvDgwR3dDKWU6lLWrVuXb4xpuj6oZwSOwYMHs3bt2o5uhlJKdSki4nXqtQ5VKaWUahUNHEoppVpFA4dSSqlW6RE5Dm9qa2vJzs6mqqqqo5sSVFFRUQwYMIDwcN1zRykVGD02cGRnZxMfH8/gwYNpXAy1+zDGUFBQQHZ2NpmZmR3dHKVUN9Fjh6qqqqpISUnptkEDQERISUnp9r0qpVT76rGBA+jWQcOtJ3xHpVT76tGBQymlgmn9oSLeWpdNdyvtpIGjgxQXF/PXv/611a+75JJLKC4uDkKLlFKBcrCgnG+/vJ4r/7qSH/x7Ew8v2obT2X2CR49Njnc0d+C47777Gj3ucDgIC2v+r2Xx4sXBbppS7crpNISE+DekerCgnPc2HyU1LoLZQ1MZmBzT8ova0J7Cihoqa+qorXNSW2eorXNS7XDiqHPSNzGKjOQYr8PAxRU1PPHxHl5YdYCwkBAeOHc4ZdUOnvliPxU1dfz2qvGE+viuVbV1vL/5KKPTExjTLyHg3y1QNHB0kAcffJC9e/cyceJEwsPDiYqKIikpiZ07d7J7924uv/xyDh8+TFVVFQ888AB333030FA+paysjIsvvpgzzjiDlStX0r9/fxYuXEh0dHQHfzMVKDUOJ7mlVVTU1DG8d5zfJ9euYvW+Av68PIs1BwqZMSSZ80f34bwxfRiQ1DgYVNXW8cHWY7y+5jCr9hU0em5gcjSzh6Qye1gKs4ak0DshqsXPNcZwrLSK3bll7Msr41hJFTklVRwrqSSnuIrc0iocLfQO4iPDGN0vgbH9EhjbL5Ex6Qms3JvP4x9lUVbt4JopA/mvC0bQJyEKYwxxkWH8+aMsKmvreOy6iYSHnjzY83lWHj97eyuHCisAmDUkhdvPyGTuqN4+g01H6BFl1adOnWqa1qrasWMHo0ePBuAX725je05pQD9zTL8E/ufSsc0+f+DAAebPn8/WrVtZsWIF8+bNY+vWrfXTZgsLC0lOTqayspJp06bx6aefkpKS0ihwDBs2jLVr1zJx4kSuvfZaLrvsMm6++eaTPsvzu6rOac2BQt7ffJSjJZUcLaniaEkV+WXVuP979kmI5OLT0pk3Pp0pGUleg8jRkkq+3l/I2gNFHD9RRWWtk8oaBxU1dVTW1FFRU0f/pGi+MWsQl4xL93ryag+r9xXwp+W7Wb2vkLT4SM4f04ev9hWwN68cgNHpCZw/pg+TM3rx0Y7jLNx4hNIqBxnJMVw7dQBXTRlAWZWDlXsLWLk3n1V7CyitcgCQGhdJWnwkqXERpMXb22lxkRgDWcdPkHW8jD25ZZyodtS3JyIshPTEKPomRJGeGEV6r2j6xEcSExlGRGgIEWEhhIeGEB4qhIWEcLiogm05JWzLKWXH0VKqap317zVnRBo/uWQUo/qe3Fv4x6d7+c2SnZw3ujd/uXEyUeGhAOSXVfPIe9t5Z2MOQ1Jj+fn80ezOLeP5lQc4WlLFoJQYvjl7MNdMHUhcZPte64vIOmPM1KaPa4+jk5g+fXqjtRaPP/44b7/9NgCHDx8mKyuLlJSURq/JzMxk4sSJAEyZMoUDBw60W3t7uuKKGj7Pyuer/QUUVdRSXu1w/amjvMZBZU0dM4ak8J25wxjRJ77Z9zlWUsWvF+9g0aYcYiJC6d8rmr6JUYzum0DfxCj69YpCRFi+PZdXvj7EcysP1AeRuaN6c7Skkq/2F7LmQCGHCysBiI0IpX9SNNERYcSEh9I3IZzoiFBiIkJZe6CIB17byG8W7+SWWYO4cXoGSbERAfmdHC6sYPW+Aipr60iOjSA5JoLkOPszKTaCtQeK+PNHDQHjofljuHFGRv0JdF9eGct35LJsey5PfJyFMfakfslpfbl22kBmZqY0BMxEGN4nnltnD6bOadieU8rKvfkcKCgn70Q1eSeq2Zdnb9fU2RN7alwEw3rHccXk/gzvHcfwPvEMTYsjNS6izbMP65yG/fllbMsppU9CFDOHpDR77D1zhhITEcp/L9zGnc+v5R+3TOG9zTn8evFOKmocfPfc4dx39lCiwkOZO6oPd5yRydJtx/jXlwf4xbvb+eOHu7l4XF/OGdmbM4anEh/V8qLe2jpnUC4QNHCAz55Be4mNja2/vWLFCpYvX86qVauIiYnh7LPP9roWIzIysv52aGgolZWV7dLW9maM4a31R1i48QhVtXXUOJzUuMada+ucCHDu6D7cMH0gw3o3f5IGqHbUsXpfIRXVDjJSYhiUEuvXVZzTadhypIQVu/L4dPdxNh4uxmnskEVaQiRxkWHERoTRr1c4sZFhhIqwdNsx3tucwyWnpXP/3GGMTm+4Cq121PHsFwd44uMsHE7Dd+cO41tnDyM6ItTr5187dSBl1Q4+2pHL4i1HedUVRACSYyOYNjiJb87OZEZmMqP6xhPWzMnC6TR8ujuPZ7/cz/8t3cUTH2dxxaQBXDN1AOmJUSTFRNSfyFty/EQVq/YWsGpvASv3FtQPsfjiLWC4DUmL4+60OO4+aygFZdVszi5hckYSiTG+T5ChIcK4AYmMG5B40nPGGEorHTiNCViAbPrZw3rHt/jvzu2WWYOJjgjjR29uYtZvPqK0ysH0wcn8+srTTnqP8NAQ5o/vx/zx/dh4uJjnVx5gydZjvLE2m7AQYdrgZM4ZlcY5I3vTNzGKvXnl7DleVv9nb14ZR0sq2fLwhQEPHkENHCJyEfBnIBR42hjz2ybPRwIvAFOAAuA6Y8wB13PjgX8ACYATmGaMqRKRKcBzQDSwGHjAdMHxtvj4eE6c8L4LZ0lJCUlJScTExLBz505Wr17dzq3rPLJyT/Czd7by9f5ChqbF0jcxitjIMMJDQ+qHEU5U1fLCqgM888V+pg5K4vrpGcwbl15/Ei6tquWTncf5cHsuK3Yep7ymrtFnpMRGkJESQ0ZyDEkxEZRX2+GdMncvoqaOYyWVFFXUIgLjB/Ti/rnDOXtkGhMG9Gp2/LmovIZnv9zPc18e4P0tR7lwbB++M3c4eWXV/PLd7ezPL+e80X14aP4YMlJaTvLGRYaxYGJ/FkzsT1m1gzX7CxmYHM3QtDi/r5hDQoRzRvXmnFG92XXsBM+t3M9/1mfz6teH6o+JCg8hKSaCxOhwEqLDMcbgcBocrmDtcBoqa+o4UmwvVOKjwpg5JIXbTx/MrKGpJMWGU1ReS2F5jf1TUUNReQ0pcRFcNXmAX4EpJS6Sc0b19us7+SIiLQae9nb1lAFEh4fy2PLd/GxeJtdMGdhi/mriwF5MvG4itXVO1h8s4pNdeazYdZxfL97JrxfvbHRseKgwOCWWUX3jmT8+nRpH4HsdQQscIhIKPAmcD2QDa0RkkTFmu8dhdwBFxphhInI98ChwnYiEAS8BtxhjNolIClDres3fgLuAr7CB4yJgSbC+R7CkpKRw+umnc9pppxEdHU2fPn3qn7vooov4+9//zujRoxk5ciQzZ87swJY2WLjxCC9/dYipg5KYPTSVqYOT/L46dTPGsGx7Lr9dshME5o9LZ/6EficN51TW1PH4x1n887N9xEWF8ehV43z+B8svq+Y/67N57evD/PDfm/jFu9uYNy6dnJIqVu3Np7bOkBoXyWUT+3PB2D6kxUVysKCCQ4UVHCos51BhBesOFlFSWUtcZBgxEaGun2H07xXBuP4JzB6aypnDU0mJi/TahqaSYiP4wQUjufOMITz75X6e/XI/S7flApCZGstzt03j7JFtOznGRYad8ol1ZN94fnPleP7fhaP4en8BheW1FFXUUFxRQ1FFLcUVNZRWOQgLCSEqXAgLEcI8xvpvmTWI2UNTGNsv8aTg2Tu+5SR1TzZvvM1XtVZ4aAgzhqQwY0gKD148ipziSlbsyqO4soahaXEM6x1HRnJM0PNXQUuOi8gs4GFjzIWu+z8BMMb8xuOYpa5jVrmCxTEgDbgYuNEYc3OT90wHPjHGjHLdvwE42xhzj6+2tJQc7+4C8V2/3l/ITU+vpldMBEXlNTichojQECYP6sXsoamcPiyFiQOTfM7+OFhQzsOLtvHJrjxG9IkjJTaS1fsLMAZG9Ilj3rh+zJ+QzsGCch5auI3sokqunjKAn1w8yu+TtTGGr/cX8tqaw7y/5Sj9EqO4cGxfLhjbh0kDvSeV20tpVS2vfnWIyLAQbpiRQWRY64KuUu2tI5Lj/YHDHvezgRnNHWOMcYhICZACjACMK7CkAa8ZY37nOj67yXv29/bhInI3cDdARkbGKX+ZnuxwYQX3vrSOgUkxvH3f6YSGCmsOFLJqbwFf7snnseW7+eMyO+Rz3ug+XDC2D6cPS63vjVTV1vG3FXv526d7CQ8Rfj5vNLfOHkx4aAjHT1SxZMsx3t98lD99tJvHlu8GYHjvOF6/eyYzfCQbvRGR+iuy318zgRDpPGVXEqLCuWfO0I5uhlKnrLMmx8OAM4BpQAXwkYisA0r8fQNjzFPAU2B7HMFoZFdXUeMgMizUZy+hrNrBnc+vxVHn5Olbp9aPF58zsjfnuIZZispr+GJPPsu228Tt62sPExMRytkj05ickcQLqw5yqLCCSyf04+fzRtPHY6597/gobp09mFtnD+ZYSRWLtxwlPFS4bloGEWGn1t3ubHPfleoughk4jgADPe4PcD3m7Zhs11BVIjZJng18ZozJBxCRxcBkbN5jQAvvqVqw9UgJz3yxn3c35TAoJYb/XXAas4elnnRcndPwwKsb2JNXxvO3TWdIWpzX90uKjeDSCf24dEK/+llLS7cdcwWSYwxNi+WVO2d4/QxPfROjuP0MLf+uVGcXzMCxBhguIpnYk/v1wI1NjlkE3AqsAq4GPjbGuIeofiQiMUANMAd4zBhzVERKRWQmNjn+DeCJIH6HbqPOafhoRy5Pf7Gfr/cXEhsRyrXTBvJFVj43Pv0VCyb242eXjG608vbRD3by0c7j/O+CsZwx3PdJ3y0yLJQ5I9KYMyKNRxacxr78cjKSY06596CU6jyCFjhcOYv7gaXY6bjPGmO2icgvgbXGmEXAM8CLIrIHKMQGF4wxRSLyR2zwMcBiY8z7rre+j4bpuEvogjOq2pMxhrJqB3P/sIKDBRX07xXNzy4ZzXXTB5IQFd6Qf1ixl493HOcHF4zg5pmDeHvDEZ76bB/fmDWIW2YNbtNnh4QIw3p776UopbouLTnSjdU5nRwurCRr9y6e3FDJnWdmctHYvl4Xh+3PL+ehhVv5PCufUX3j2ZtXxozMFJ67bVqzi8mUUt2blhzp4uLi4igrK/P7+GpHHQcLKqiuddIrJpy375vkc3ZRZmosL9w+nSVbj/HLd7czMDmGJ2+crEFDKXUSDRzdUFmVg0OF5RggMzWGwyfC/JqSKiJcMi6d80b3wWlMqxf3KdUp7V4KB1fC+b/o6JZ0Gxo4OsiDDz7IwIED+fa3vw3Aww8/TFhYGJ988glFRUXU1tbyyCOPsGDBgla9b0FZNTnFVUSEhTA4JYbINpz8NZGtupVNr8H2hTD35xDaucqPdFUaOACWPAjHtgT2PfuOg4t/2+zT1113Hd/73vfqA8cbb7zB0qVL+e53v0tCQgL5+fnMnDmTyy67zK/egtMYjhZXUVBeTXxUOBnJ0YSGaABQipLDYOqg6CCkDuvo1nQLGjg6yKRJkzh+/Dg5OTnk5eWRlJRE3759+f73v89nn31GSEgIR44cITc3l759+/p8r1qHk0OFFZTXOEiLj6RvQlSnWS2tVIcrdhWwKNyrgSNANHCAz55BMF1zzTW8+eabHDt2jOuuu46XX36ZvLw81q1bR3h4OIMHD/ZaTt3TiapaDhdW4jSGjOQYesUEvnS0Ul2WoxrKjtnbBXuACzu0Od2FBo4OdN1113HXXXeRn5/Pp59+yhtvvEHv3r0JDw/nk08+4eDBg82+1hjD8RPV5JZWERUeSkZyrCazu4KyPBCBWP8WVKpTVOJR2q5gb8e1o5vRwNGBxo4dy4kTJ+jfvz/p6encdNNNXHrppYwbN46pU6cyatQor69z1NmhqbJqB0kxEfTrFa11mbqKN28DCYFbF3V0S3qGEtcwlYTaoSoVEBo4OtiWLQ1J+dTUVFatWuX1OPcajooaBwcLKnA4Df2TokmOafu2l6qdGQM5G8A4wVkHIV2gh1iaAwn9OroVbefObwyYCgX7OrYt3YhOu+lCqmrr2J9fjgDD0mJJiY3UoNGVlByGmjKorXCNt3dy+1bAH0dD7raObknblRy2PbzBZ9jbtb5zhso/Gji6iFqHkwP55QhCZlos0RHaWexycj02v8zZ2HHt8NcuVxm4/N0d245TUXwY4tMhbTRgoGh/R7eoW+jRgaOr1OlyOJ3sLyjH4TRkpsa0aue4rvIde4TjrsARGgFHu0Dg2PuJ/Vl6tGPbcSpKDkPiQEgZYu9rgjwgemzgiIqKoqCgoNOfWJ1OY2tOOZwMSolpVU/DGENBQQFRUbr/c6dwfDskDID0CZ2/x1FyBPJ32dulXXjLm+JD0GsgJLt2XtQEeUD02PGOAQMGkJ2dTV5eXkc3pVnGQGF5DZW1dSTHRpBd2vpkalRUFAMGDGj5QGVLUxTshbk/C877H98BfcZAr0Gw6VVwOqGzru7ft8L+DI20CfKuyFlng17iQIjuBTGpXSO31AX02MARHh5OZmbn3W3OGMPP39nKy19l89D8MZw/ofO2tVvY9Bq8fQ+EhMFZP4SwyMC+f10t5O2CYedC6ghY80/XSubhgf2cQNn3CcSmQepIONFFh6pOHAOnw/Y4AFKG6syqAOmklzvq8Y/28PJXh7h3zlDdTjXYti+Cd74FcX3tiSYYs4gK9oKzFnqPtUNV0HmHq5xO2+PInAOJ/bvuUJV7DUdihv2ZPFSHqgJEA0cn9NLqgzy2fDdXTR7Ajy8a2dHN6d6ylsObt8OAaXDLf+xjRzcF/nPcifHeoyFtlB0C6qwJ8uPboDwPhp5j13CUHrXBpKtxr+Go73EMsb2nav/3tWl3xYfgyZmQtayjW+KTBo5OZvGWo/z3wq3MHdWb3141TtdptKT4EDwx1eYPWuvAF/D6TfZkfuMb0HsMRCUGL3BIqB2mCg2HvqcF53MCwT2basg5EN/P9pQqCjq2TW1Rcsj+THTl+FJcBQ4LO+lwlTGw+P9B3g74+H/tfX/s+Qi2vhXctjWhgaMTWbknn++9tpHJGUk8eeNkwnX3vZbt/xwKsmDts617XfY6eOU6m6i+5W2bPBWxw0hBCRw77Bh7uGuGW/pE+zmd8Up+3yc2wCX2b1g1fqrDVQdXwc7Fp9621ig+DDEpEBFr73f2mVU73oXdH9je79FNcODzll9TUw7/uRve+76dDNBO9MzUSWw9UsJdL6wlMzWWZ2+dRnREFyhH0Rm48xFb3gRHjX+vObYVXrrSnlS+8U7jgoPpE+x71tUGvp29Pfa4T58A1aWdb0FabZXdLW/IOfZ+feA4hZlVez+GFy6D/9wFdY5Tb6O/3Gs43JI78VqO6hOw5MfQZxzc8o6dmLDyLy2/bs0zUJEPVSXt2oPVwNEJ7M8v59Znv6ZXTATP3z6dxBjdpcxvx7fZfEFlIezxY1zY6YR/3wrhMbbQYNM6TOkToa4a8nYGro015VB0wCbG3fpNtD9zNgTucwLh8FfgqLL5DWj4/ZxoY+A4uApeu8n+HdWUQW6AN0zzpfhwQ34DIDLOToDojIHj41/Z/Mulf7LtnHYXZC21M/GaU1MOX/4Z+k2y9/d/2j5tRQNHh8streKWZ77CAC/cMZ2+ibpYr1Vyt8FpV9ortE2vtnz87g/sXP4LfwVJg09+Pt11Qg/k1VveLsA07nGkjXatIO9keY59n9gpyYPPsPdj0+z9tvQ4cjbAK9fa4HPb+/axgysD11ZfjHH1ODIaP54yrH2Hqr7+J7x6Q0Oi3pucDfD1P2DaHbYYI8C0OyEsClb56HWsedr2Ni561E642KeBo0eocxpuf24NReU1PHfbNIamxXV0k7qWsuN29k/6BBh3Lez6ACoKfb9m9V/t6u3Rl3l/PnkIRMQF9oReP6NqTMNjYRHQZ6x/M6tOHINDq20CdOUTdqvj12+Bf10CR9YHrp1gE+MDpkFkvL0fEmprPbU2cBzfAS9eCVG94BsL7d9R0uD2CxwVhbaYpGePA+zMqvbocTid8MFPYfEP7cXKP86ySeym6hzw7gM2QJ/7UMPjsSkw8Ua7vqjs+Mmvc/c2hs6FjBl26vSh1XbjqnaggaMDLd+Ry7acUn595TjGD+jV0c3penK32p99xsKE6+3sH1+zS45utgnHGXdDaDNrX0NCoO/4wK6xOL7DXj0mN1mP407E+5o9s3uprVD77IV22vCHP4cNL9peTO5We9IJVFK0otC2x53fcGtt4CjYCy8ssD2qWxc2zGrKmG0DR3uU+amfUdUkcCQPtVfplcXB+2xHNbx1O6x+EqbfA/d9BfF94aWr4NP/azwhYs0/7e/8ot/YGX2eZn7b5tq+/ufJn7HmaTvT7eyf2PtD5oCjErLXBO97edDA0YH+9eV++veKZt649I5uStfkrjbbeyz0HWd/bnqt+eNX/w3CY2Hyrb7ft99EOLYlcCfk3G2QNvLk/TfSJ9qkpq8E+conbA/pprfgvtXw4CH4STbc/zXM+yMc22wDSSDsWwGYhvyGW0I//wNHSbYNGnW1tqfhTkgDDJptc1G+xu0DpekaDreUIM+sqiyyPa1tb8P5v4SLH4W0EXDnchh3DXzyCLx6vT2u5Ah8/AgMOw/GXnnye6UOg5GX2CBRU9HweHWZq7dxLgycbh8bdLotH99Ow1UaODrIjqOlrN5XyDdmDSJMp922Te42m+yMTbFTaSfeAEfWQn7WyceeyIWtb8Kkm+zUW1/SJ9irN2/v0xbHdzROjLvVJ8ib6d3k7bI9pKm3wfDzbI4kKtF+V4DTrrJX8R/9MjBX0Ps+gchE6De58eMJ/W3g8Ken8NadNhje8jb0brKD5aDZ9uehdhiuql813jRwuNZyBKP0SEk2PHuxnWBw5dNw+gMNf1cRsXDlU3DJ7+0ss3/MsSVunA77WHPrtWZ/xwbbTa80PNa0twH233S/Se2WINczVgf515f7iQ4P5fppGS0f3N5Kc6Co+f3OO43crXaYym3cNfaqy1uvY83T9ip4xr0tv6+7JEgg8hwVhVB2rHFi3K33GAgJb/5z1j5rn590i/fnReDi39rP+PTRU2unMbB3BWSeefIwXkI61Jbb6cO+OCR7JS0AACAASURBVOvgyDqY8s2GoOgpeQjE9WmfPEfxYZurik5q/HhSJiCB73HkboOnz7frXW5+C8Zfc/IxIjD9LrhtiQ0YBz6HOT86eQjTU8ZM6D8FVj1pf7/VZbDycdtLGTit8bGZc+zvv/pEYL+bFxo4OkBBWTXvbMzhysn9O+fU2zdvh3/OtUnZzqrOYa/IPQNHfF+bLNz8euNx5NoqWPsMjLioYajCl5ThEBYdmJIg3hLjbmGRtlqut8+pKYeNr8KYBRCX1vz7p0+AKbfC10+d2hBQ4T6bFxhy9snP+buWo/gQ1NU0X7hRxPY62iPP4V7D0fRKPjzK5lwCmSA/uslOVMDYoDBkju/jB06Dez6Dy/4Cs7/r+1gR2+so3Gc31lrzT9vbmPPgyccOmWMDUjsEZg0cHeCVrw5R43By2+mDO7opJyvLs7MzKvJt4b/OuLIZ7BVjXTX0Oa3x4xNusCeNg180PLblDfufbdZ9/r13aFjgSoK4S6H08RI4oGFvjqYn0q1vQXWJnZbZkrn/bXM3HzzY9hPy3o/tz6FzT34uob/92dLqcffJ2D0c5E3GbPs+xYda38bWcO/D4U3K0MCVV8/ZCM9fZmeh3f6B/Xfjj9hUmHyLLT/TklGXQq8M+PwP8GUzvQ2AgTPsepl2yHNo4GhnNQ4nL64+yFkj0hjWO76jm3OyrA8BA1PvsCeT1U92dIu8q59R1eSEPGoeRCY0DFcZA6v+agPM4DP9f//0iXYW1qkGztxtNi8R38wEiPSJUFUMxU2GBtc8Y3spGTNb/ozYVDjnJ/bvy73da2vtW2HXPHgms93cbW9pJ8ACV04oxUepeHeeI9hXxU1XjXtyV8k91V5PzkY7ESAyAb75nvd1QYEQGmZnWOWst/kOz9yGp/BoOzW3HfIcGjja2ZKtRzl+orpz9jYAdi+xV5jz/gCj5sPyX3S+1c1gT8ghYbamkqfwaDu8s32hHe7Zt8IWjZt5X/MJSG/SJ0DNiVMvCeJOjDf32f28LDg8ss4OX0293f82T7vTLgJb+tPWz+Wvc8D+z2Do2d4/rz5wtDBUVbDHJtc9S7g05S4kGcwEeXWZnbXkq8dRVdLymh9f2itouE262eZrhl/QsEjQm8w59qKqLLgb1AU1cIjIRSKyS0T2iMhJg3IiEikir7ue/0pEBrseHywilSKy0fXn7x6vWeF6T/dzvYP5HQLt2S8PMCQ1ljnDfYxbdxRHtV0ANuJCewK57Am7MOnNOzpfKercbTZoeNtwacINtrzFzvftgr/Y3jDu6ta9f/2eGacQNI1xBQ4viXG33mNtAPScWbXmWTv0NP46/z8rNNyuBSjabxOprZGz3ia+m67fcAuLsL/Dloaq8rPsFFJfwS4kBDJmBbfH0dyMKrdTLXaYs8HW3qoPGoPa9j6tERkHd38KVz3t+7ghZ9ufBz4LanOCFjhEJBR4ErgYGAPcICJNB3rvAIqMMcOAxwDPqSF7jTETXX+aToW5yeM5L8sqO6f1h4rYdLiYb54+mJCQTlgu/cDn9oQ74mJ7PybZTiEs3GcLsHUmudsbJ8Y9ZcxqGBPO+tBVvqGVO/qljTr1kiClR2yeorn8BthkbdrohgR5ZZHNb4y/FqISWvd5Q+fCyHnw2e9bHlbylLUMEHu12pyE9JZ3AizY6zu/4TZotu2dnMj1v42tUb+Go5kZi/VTctuQ58jZYHsaUYntFzTckgadvEiwqfSJNqAFOc8RzB7HdGCPMWafMaYGeA1Y0OSYBcDzrttvAudKN96A4tkv9hMfFcZVkzvpHuC7PrDF/zLPangs80w48wew8aW21fwPdJVZsMMMJYe8z1QCe1U74QZbqDA00g75tFZ9SZBTCBzuxHhz7XTr55Eg3/iqXUMy7Y62feaFj9gV9J/9n3/HG2MnD2SeZdfDNMe9lqM5NeVQmu07v+GW4V7Pscq/NnoqPgSb3/A9FNPcqnG3pEF2b5TWzqzKWm4T4VGJcGs7Bw1/hbrqjAU5zxHMwNEf8Kzsle16zOsxxhgHUAK4//VmisgGEflURJpmNf/lGqb67+YCjYjcLSJrRWRtXl5wx/v8cbSkkiVbj3Hd1IHERnbCrd6NsTV1hpzTsGeE29kP2vpF736/des7ygvgd0PsGgp/Fe5vudyHe8V40xlVntzDPOOv8T2d1Rd/SoL44i757muoCuxVYmWhHWJZ+ywMmG5XwrdF8hC7CnnLvxuvNm5O9hpbuXfC9b6PS+jne6jKvTmSP9Od0yfYCxR/hqtqymH3h7bH+8RU+NM4W579k181/5riw7a3GNfH+/Oh4bY34u9QlXuCxSvX2GBx25LOGTTcMufYv9MgrsXqrMnxo0CGMWYS8F/AKyLi7rffZIwZB5zp+uN1dZQx5iljzFRjzNS0tI7PJ7y46iDGGG6dPbijm+Jd7jZ74hp50cnPhYbbsVXjtKtd/d6ZbLkdO//of/1LRDqq7T4ZL17he28NzxpVzUkZakteXPCIf231Jn2Ca8ZTG6eOHt9hd9BrugitKXdZ7JV/sTOT2trbcJt0s/2973i35WM3vWbXrIya7/u4+HQ7jFZb6f159yr75tZweAqLsAleXwnysuPw0tXw6GB7wl73vD1ZX/gbW2pjx7vNl4QpOWx7SCE+Tm8pQ/3rcThq4N3vwtKf2PIfty9tqL3VWbnXkewPXp4jmIHjCODZVxzgeszrMSISBiQCBcaYamNMAYAxZh2wFxjhun/E9fME8Ap2SKxTq6qt49WvD3H+mD4MTI4JzoeU5sDiH7V9YdNu1zTO4Rd6fz5pMMz9uR1e8Heb1qyldry1uhQ+/V3Lx6983F65VhbaoNOc49tt1dWme2k0NeTslk/avtSvIG/jQsDj21rubYANgBJqF3dFJ8OYy9v2eW6Dz7ArpFuqYeWogW3/gVGXtJxPqV/L0cxwlfvfnbfpvN4MOt1uqOWtVIoxsPB+m3ObfrctX/LjA3ZF9qz77PqHivzmeyxN9+HwJmWYbbOvi6DyAnjxclj/Apz1/+DaFxt2E+zM0kbZ3lYQh6uCGTjWAMNFJFNEIoDrgUVNjlkEuCvOXQ18bIwxIpLmSq4jIkOA4cA+EQkTkVTX4+HAfGBrEL9DQCzddoyiilpunTU4eB+y7jlb0/9vp9tifq1df7DrA1vaIL6Z7j3A2CsAsbOVWlLnsGWkR19qiwqu+afv2k9FB+GzP9jkbkyqXf3dnNxtrpNtkNNh7hlPbclz1Dkgb7fvxLhbeLT9z26ctpZW06HC1hKx73Pgc9/7a+9ZbnsR41sYpoKWV48XZNlijP6eWAfNBoyt69TUuufsRcf5v7T7pgyd2/h3Mux820vavtD7e3vbh6Op5KG2jEpZMwn64zvgn+dA9lq46hl70eSrB9OZiNic1f7PgrZCP2i/CVfO4n5gKbADeMMYs01Efiki7s0QngFSRGQPdkjKPWX3LGCziGzEJs3vNcYUApHAUhHZDGzE9li81BzuXN7ZcIR+iVHMHOIj+Xiq9nxkT3SZZ9kVxM9d4n/vo+y4XTvgnk3VnPg+dohhlx+BI3uNHeYZfj6c81P7H33ZQ80fv/SnrtpLj9rifbuWNH816mtGVSDVz3hqQ+Ao2m9XtreUGHdzr+eYclvrP8ubCTfaul0bX2n+mM2v2SDdtBquN/U7ATYzs6pgj3/5Dbf+U20drqa9hoK9sPRntrc47S7vr42Ms0Ufdyw6+QLJUWNL5bTY4/Cxjez+z23dKUeVzWe0dip3Z5A5xwbFQO5k6SGoIdQYs9gYM8IYM9QY8yvXYw8ZYxa5blcZY64xxgwzxkw3xuxzPf6WMWasa7rtZGPMu67Hy40xU4wx413PP2CMab8d2tsgv6yaz7LyWTCpf/Cm4FYU2rn4oy+FG1+Hy/9uh3P+NruhOJovu5cCxnt+o6lR8+yUxJJs38dlfWiHX4bOhbjecNYPYNdi79MEd38IO9+zwwG9BtrEdl21PTE0VXzILsxrj8ABzZcEaYm/iXG3M/7LXtm25uTrS2J/mwvY+Ir3v//KYtvLHHe1f2Uv6hcBekmQGwP5e/zLb7hFxNjcjmfgqHPA2/famUEL/ur7Cn/M5fbE2LTHUpoNmOZnVLk1t5Zj91J4+Wqbx7jrExgwxe+v1Km48xxBmpbbRfpeXdd7m3KocxqumNR0QlkA7VthhzmGndtQXvy+r+xV29Kfwr8u9j2VcvcHdpjB1ywlt5Hz7M+WSltkfWjXU7jnnc/4lh0+WPqzxiey2ipY8iM7jXPW/fax/pPtGPTmN05+X/cJ2Z+2BkL6BDue3pYd8BA7BOWP1GGBv7KddLM90e/95OTnti+0wXn8tf69V2Sc/bv09nsoz7frVfxZw+Fp0Cx7EeKe/fXlY5Dt2mcksYX/L8MvsFOtmw5XNbcPR1O9MmyPx3Mtx9a34LUbbbC/bXHLbejMemXYPFeQ8hwaOILs7Y05jElPYESfINal2vuR/U/tuY9CQjrc8Bpc8Q97sn3xSjue3VRtla1x5F4t3pK0EfYkv/O95o8pOWJnPo24oOGx8Cg4/2HI3dJ4+GTl43ZY55Lf2dk2YNsx/jo7Rt90r2Z34PD3hHyqfJVYL8uzRec2vATZ6xqvrj++zSaKw6Pbp53ejLzYJtu9Jck3v2FP9E333vAlvpkNnfypUeXNoNPtmpMja22vbsVv7TClPwE0KsFeKDUdrmpp1bhbSKgtZ+4eqlr3nK2QMHAGfGORXfza1Q2ZAwe+sD25ANPAEUT78srYdLg4uL0NY2DPx7Z30XQfBRE7P//6V2yX/NUbTp5OeeBzuzfzyBbyG55GzbP/IJvbPCjrQ/tz+AWNHx97pV2j8PH/2j0Dig7Y1d1jLj+5Kus4134GW/7d+PHcrfZKKrKd9mfve5rNFXgGjpoKuzr78Umw7L9h4bfh6bnwm/7w2Gl2GumBL/1LjAdTWKT9+9/5vp0h5Fbsqh48/vrWTTBobidA91V7a4fZBs4AxObn/nO3LWtyye/9f/2YBbZHdWRdw2PFh+17Jvjxfy55qJ08sPIJuwXv8PPtzK3WrtjvrEZdan9HNYHfn0MDRxC9szHHlnya2MK00VORtxNO5Njx7OYMmWN7HodW2x3aPK9Adi2xdZFaUzl21Dxb9z9rmffns5bZYammvQIRW0+pLNduffnBT2we5MJfn/weyZkwcKadXeWZXzjeTolxt4hYWxPr6CZ7ZbvxFXhiig1+Q+bYIcHvrIfrXrYzbzJm2uSso8p7ifL2Nulme1W/xWPYz33b22ZDvjQXOPKz7IK75kp8NCe6lx1yXPkE5O+Cy59s3ZX+iIvscNP2dxoeKzls8zHu3qsvKUPtv6cPf25nDF73csf2EANt+Hmw4C+nNiW9GZ1wCXP3YIzhnQ1HOH1oKn0STnF6pS97PrI/h/kIHACnXQnleTafsPgHMP9P9vHdS+2smtZMAe0/1V4d7nzv5JOPo9rmXCY0czU7YKrtTXzxmA0+5/2i+bHk8dfC+/9l99VOn2B7SwV7vO/PHEzpE2wv6qmz7F7k/SbD1c80lAgHexIa7bGIzpjgTxf2R5+xNgm9/sWG3Q83vW7zT62t6JrQzwb9utrGCfWCvXZYrume6v4YNNsOX06/p/WBNrqX/be7fZFd6Cniex+OptyVlSd/w/5/aEv7eyjtcQTJ+kPFHCqs4PJgDlOBzW+kjvRvNeuMe+zsnXXP2fHkY1vsLJQRfsym8hQSYheN7Vl+cgnvg1/a+fEjmllICHDu/zSURJ/pY3OlsVfYK0p3kjxvp50E0N5DQP0m2fxQZYmd+XTnR42DhjedIWi4TbrF5lxyNtieU/4u/5PinhL6AebktQ8FWa1PjLtNvsW277yH2/b6MQtsbSp3FWNf+3A0Ne4auOlNuPRxDRqtpIEjSN7ZcISo8BAuHOtjQd2pqqmwY+kt9TY8nfsQTLwZPv2tLaWA+D7JN2fkPFtJt2lZg90fQliU76GvXgPt/Phb3vY9pBCTbNu25d92eK29Z1S5Tb7Vrhq+f41N3HaVhWBup11l/042vGSDcGhE21ane1s9Xuew9cXaGjj6jrPDKRFtrKgw8hJ7EbJ9oR1KLDnif48jIsbmNTpTkO8iutj/gK6hxuHkvc05nD+mL/FRQdxT/OBKO6XSV36jKRG49M+2tEjOBrtaPK4NW5pkngURcSfPrsr60AaNlk4E/Sf710saf629wt3/qV34Fx4T/E1zmoqIgTGXnfqK7o4S3QtGXwZb3rRBePgFbZs15G0tR/FBm0NpzRqOQIpJtv8Wty+EsmO2Lf72OFSbaeAIgs9251FUUcsVk4KYFAc7TBUa2fKwSVOhYXDNc3bK6xnfb9tnh0fZns6uJQ3TIQv22tlbbenBNGf4hXaq8ebX7Yyq3qN1WKEtJt9i11qUH2/dBlGe6suOeKwe92ef8WAbs8BO6XaXwmltkl61mgaOIHh74xGSYyM4M9i7/O1xjbW3pZsfEWM3aRrdQlVUX0bNt70B93TI+mm457f9PZsKj7LDKjvetePz/pbwUI0NOgN6uTYCamtgj06ypWM8exxtXcMRSKPm2ynTq/9q72uPI+g0cARYaVUty7fncun4dMJDg/jrLcm2Sc7W5DcCbfj5dnzZPVy1e6lN1Ad6KGnC9XatSVVx++c3uouQELji73Dl063fDdFNxC4s9cxxFOyxlYo7csFcbKqtCOwu6OhvjkO1mQaOAPtg6zGqHc7gz6aqn4Z7XnA/x5foJLv6d9diu2r64JeB7W24DZzZUO20PddwdDeDZjdezd8WCf0bFzrMz7L5jY5OMI9xbS4andw1Sp93cRo4AuydDUcYnBLDxIG9gvtBez+yJSDaq/RGc0bNh/zddpe/uprA5jfcQkJsryMkTANHR2u6E6C/+4wH26hLAdHeRjvRwBFAR0sqWbWvgMsn9SeoW6fXOewiu2FzO/5Kb9Ql9uenv4OIeNs7CIazfgj3fNY9agh1ZfHpNjnudNpe5omczhE44vvYaceZczq6JT2CrhwPoGXbczEGLpsQ5NlUOeuhqqR103CDJXFAw97coy/zr9RDW4RFam+jM0job6e8VhTYoAGdI3CAXc2v2oX2OAJoc3YJqXERZKYGeYx1z0d2FsmQs4P7Of5y71cdjGEq1bnUT8k90rp9xlW3oj2OANqWU8rYfonBHaYCm9/oN7nzDNtMutmeREadwtRe1TUkuBcB5rjWcIj/+4yrbkN7HAFSVVtHVu4JTusf5JLMFYV23URHTsNtKqEfXPVPu0JZdW/usiMncuwajsSB3auirPKLBo4A2Z17AofTcFq/RP9f5KiGv50Or1wPebv8e83ej22hv86Q31A9T2yand1WmtP6fcZVt6GBI0C2HikFYGxrAsf2hbaMxr5P4K+z4L3vQ9lx78fmbLSb3bx9L8T1tTWmlGpvIaH2319pTuv3GVfdhgaOANmaU0JiVAgDJbflg93WPGN3IfveVph2J6x/we4q99n/2cq3TifsXAzPzYen5thaPNPuhDuXnbzbn1LtJaGfvZCpOdF5ZlSpdqVnnwDZllPK9xM+Q5640a436DvO9wtyt8Hh1XDBryAuze65Pf1uWP4/8PEjsOZZO3ZcuNeOI1/wiN1wJqoVPRqlgiGhX8Ouexo4eiTtcQRAbZ2THUdLmetcafMPK37b8ovWPGP3SJh4Y8NjqcPg+pftXhXJmXY8+ep/wXc3wuzvaNBQnUOCxzolDRw9kvY4AmBvXhlxjmIGlm2GuD626N/RzZA+3vsLqk/YMuFjr/Q+pXbQbLhtcXAbrVRbuQNHaKRWou2h/OpxiMh/RGSeiGgPxYutR0o5N3Q9ghOuehoiE+HTR5t/weY37O550+5ov0YqFSjuwJEytOvthqgCwt+/9b8CNwJZIvJbERkZxDZ1OdtySrgobD0mcYDd/W7WfQ29jqaMgbXPQt/xOjNKdU3x7sChw1Q9lV+Bwxiz3BhzEzAZOAAsF5GVInKbiARxb9SuISv7OGfIFmTkPFt0cMa9zfc6Dn9tp+BOu6PjCxQq1RYJGjh6Or/7mSKSAnwTuBPYAPwZG0iWBaVlXYTTaUg6+gWRVMOoefbB6F4evY5NjV+w9hmITIBx17R/Y5UKhMQBMOWbcNqVHd0S1UH8zXG8DXwOxACXGmMuM8a8boz5DhAXzAZ2dgcKypljvqYmPKHx3t8z7rWzoFZ49DrKC2Db23ZvCd1sRnVVIaFw6Z9bnnKuui1/exyPG2PGGGN+Y4w56vmEMWZqENrVZWzLLmRuyHoqBp0LoR6jdtG9YOa3Ydf7Db2OjS/ZzY6m3t4xjVVKqQDwN3CMEZH6CnYikiQi9wWpTV1Kye7PSZYyYidcfvKTMz16HU4nrP2X3Wq19+j2b6hSSgWIv4HjLmNMsfuOMaYIuKulF4nIRSKyS0T2iMiDXp6PFJHXXc9/JSKDXY8PFpFKEdno+vN3j9dMEZEtrtc8LkGvYe5byuFl1BBO+Agve39HJcKs+22v48s/QdF+7W0opbo8fwNHqOcJWkRCAZ9bvbmOeRK4GBgD3CAiY5ocdgdQZIwZBjwGeE5D2muMmej6c6/H43/DBq3hrj8X+fkdAs44nYwr+5J98VMhsplUz4x7bAD56Bd2Jfjoy9q3kUopFWD+Bo4PgNdF5FwRORd41fWYL9OBPcaYfcaYGuA1YEGTYxYAz7tuvwmc66sHISLpQIIxZrUxxgAvAF7GiNrH8T3rGcBxijMuaP6gqESY9R17e9ItwdtaVSml2om/JUd+DNwDfMt1fxnwdAuv6Q8c9rifDcxo7hhjjENESoAU13OZIrIBKAV+boz53HV8dpP37O/tw0XkbuBugIyMjBaa2jalGxeSZoTY8S3sfDfzW1BdArO+HZR2KKVUe/IrcBhjnNghor8Ftzn1jgIZxpgCEZkCvCMiY1vzBsaYp4CnAKZOnWqC0EYSDn7IBjOcsUNa2MwmMs5Wt1VKqW7A33Ucw0XkTRHZLiL73H9aeNkRwLMC2gDXY16PEZEwIBEoMMZUG2MKAIwx64C9wAjX8QNaeM/2UXyYPuU7WR9zOlHhoR3SBKWU6gj+5jj+he1tOIBzsLmFl1p4zRpguIhkikgEcD2wqMkxi4BbXbevBj42xhgRSXMl1xGRIdgk+D7XGpJSEZnpyoV8A1jo53cIrF1LAMjvr1u4KqV6Fn8DR7Qx5iNAjDEHjTEPA/N8vcAY4wDuB5YCO4A3jDHbROSXIuKeWvQMkCIie4D/AtxTds8CNovIRmzS/F5jTKHrufuw+ZU92J7IEj+/Q0DVbFtElrM/fTJP64iPV0qpDuNvcrzaVVI9S0Tuxw4PtVhqxBizGFjc5LGHPG5XAScVbTLGvAW81cx7rgU69mxdWUTY4ZV86JzH1H4JHdoUpZRqb/72OB7A1qn6LjAFuJmGIaaeJ2sZIaaOZXVTGaOBQynVw7TY43DlGq4zxvwQKANuC3qrOrusZZSEJlGaPI74qB5fVV4p1cO02OMwxtQBZ7RDW7qOnPVsMiMY079Xy8cqpVQ342+OY4OILAL+DZS7HzTG/CcorerMqkqgYA9f1U7mtP6JHd0apZRqd/4GjiigAJjr8ZgBel7gcJVI32IyuUvzG0qpHsjfleOa13DL2QDAFmcmY/tpj0Mp1fP4FThE5F/YHkYjxpieVyM8ZyOF4X2JiEgjOVYLFiqleh5/h6re87gdBVwB5AS+OV1Azgb2hY8gOTKyo1uilFIdwt+hqkaL8UTkVeCLoLSoM6ssgqL97Io7h8Rof2OuUkp1L/4uAGxqONA7kA3pEnI2ArDFDCFB128opXoof3McJ2ic4ziG3aOjZ3ElxtfXZjAhWgOHUqpn8neoKj7YDekScjZAUiZHCiI5UwOHUqqH8nc/jitEJNHjfi8R6bAtWztMzkac6RMpr6kjUQOHUqqH8jfH8T/GmBL3HWNMMfA/wWlSJ1WeDyWHqEqbAEBClCbHlVI9k7+Bw9txPevM6UqMlyTZiu6JMdrjUEr1TP4GjrUi8kcRGer680dgXTAb1um4EuP58aMAdFaVUqrH8jdwfAeoAV4HXgOqgG8Hq1Gd0tGNkDKcorooAM1xKKV6LH9nVZXTsK1rz5SzAQadTmlVLQAJGjiUUj2Uv7OqlolIL4/7SSKyNHjN6mRO5ELpEeg3iZJKGzi0x6GU6qn8HapKdc2kAsAYU0RPWjl+1CbG6TeJ0koHoDkOpVTP5W/gcIpIhvuOiAzGS7XcbitnAyDQdxylVbVEhIYQFd7Wai1KKdW1+Tul9mfAFyLyKSDAmcDdQWtVZ5OzAdJGQmQcJZW1JESHISId3SqllOoQfl02G2M+AKYCu4BXgR8AlUFsV+dhjA0c/SYBUFpZq4lxpVSP5m+RwzuBB4ABwEZgJrCKxlvJdk8njkJZbn3gKKms1fyGUqpH83eg/gFgGnDQGHMOMAko9v2SbsK18K++x1Hl0BlVSqkezd/AUWWMqQIQkUhjzE5gZPCa1YnkbAAJhT621IgOVSmlejp/k+PZrnUc7wDLRKQIOBi8ZnUiORug92iIiAFcgUMLHCqlejB/V45f4br5sIh8AiQCHwStVZ2FMba44ciLXHcNJZW1OlSllOrRWn3pbIz5NBgN6ZRKsqEivz6/UVlbh8NpdKhKKdWj6So2X5okxrXciFJKaeDwLWcDhIR7JMa13IhSSgU1cIjIRSKyS0T2iMhJ1XVFJFJEXnc9/5WrlInn8xkiUiYiP/R47ICIbBGRjSKyNpjtr0+Mh0UC2uNQSikIYuAQkVDgSeBiYAxwg4iMaXLYHUCRMWYY8BjwaJPn/wgs8fL25xhjJhpjpga42Y2FhEHGrPq7pZXukuo6q0op1XMF8ww4HdhjjNkHICKvAQuA7R7HLAAedt1+E/iLiIgxxojI5cB+oDyIbfTt5jcb3dUeh1JKBXeoqj9wARBVQgAAC0pJREFU2ON+tusxr8cYYxxACZAiInHAj4FfeHlfA3woIutEpNlCiyJyt4isFZG1eXl5p/A1GtRv4qQ5DqVUD9ZZk+MPA48ZY8q8PHeGMWYydgjs2yJylrc3MMY8ZYyZaoyZmpaWFpBGuXsc8boAUCnVgwXzDHgEGOhxf4DrMW/HZItIGHZhYQEwA7haRH4H9MLuB1JljPmLMeYIgDHmuIi8jR0S+yyI36NeaaWDuMgwwkI7a7xVSqngC+YZcA0wXEQyRSQCuB5Y1OSYRcCtrttXAx8b60xjzGBjzGDgT8CvjTF/EZFYEYkHEJFY4AJgaxC/QyO6alwppYLY4zDGOETkfmApEAo8a4zZJiK/BNYaYxYBzwAvisgeoBAbXHzpA7zt2kQpDHjFtVdIuyitqtVhKqVUjxfUs6AxZjGwuMljD3ncrgKuaeE9Hva4vQ+YENhW+k97HEop1XmT452SllRXSikNHK1Sqj0OpZTSwNEapVUOXcOhlOrxNHD4yVHnpKxat41VSikNHH46UeWqjKt1qpRSPZwGDj9pnSqllLI0cPhJ61QppZSlgcNP9T2OGA0cSqmeTQOHn3T3P6WUsjRw+ElzHEopZWng8FN9jkNnVSmlejgNHH4qqawlPFSIDg/t6KYopVSH0sDhp9LKWhKiwnFV5lVKqR5LA4eftDKuUkpZGjj8VFrlIF4Dh1JKaeDwl1bGVUopSwOHn2yOQ2dUKaWUBg4/lVbpJk5KKQUaOPxijNHkuFJKuWjg8ENVrZPaOqPlRpRSCg0cftFyI0op1UADhx+03IhSSjXQwOEH7XEopVQDDRx+KK3UTZyUUspNA4cftMehlFINNHD4ob7HoYFDKaU0cPijpH73P02OK6WUBg4/lFbVEhsRSlio/rqUUkrPhH7QVeNKKdVAA4cfSiu1TpVSSrlp4PBDiQYOpZSqF9TAISIXicguEdkjIg96eT5SRF53Pf+ViAxu8nyGiJSJyA/9fc9gKK1y6BoOpZRyCVrgEJFQ4EngYmAMcIOIjGly2B1AkTFmGPAY8GiT5/8ILGnlewacbuKklFINgtnjmA7sMcbsM8bUAK8BC5ocswB43nX7TeBcEREAEbkc2A9sa+V7BpzNcehUXKWUguAGjv7AYY/72a7HvB5jjHEAJUCKiMQBPwZ+0Yb3BEBE7haRtSKyNi8vr81fos5pOFHt0B6HUkq5dNbk+MPAY8aYsra+gTHmKWPMVGPM1LS0tDY35ESV1qlSSilPwRx/OQIM9Lg/wPWYt2OyRSQMSAQKgBnA1SLyO6AX4BSRKmCdH+8ZUFqnSimlGgtm4FgDDBeRTOzJ/XrgxibHLAJuBVYBVwMfG2MMcKb7ABF5GCgzxvzFFVxaes+AKnWXG9HAoZRSQBADhzHGISL3A0uBUOBZY8w2EfklsNYYswh4BnhRRPYAhdhA0Or3DNZ3AO1xKKVUU0GdKmSMWQwsbvLYQx63q4BrWniPh1t6z2DS3f+UUqqxzpoc7zS0x6GUUo1p4GiB7v6nlFKNaeBoQUllLWEhQkxEaEc3RSmlOgUNHC0orbIFDl0L2pVSqsfTwNGCkkpdNa6UUp40cLSgtLJWt4xVSikPGjhaoHtxKKVUYxo4WuDOcSillLI0cLTADlVp4FBKKTcNHD4YYyjV5LhSSjWigcOHaoeTmjqnlhtRSikPGjh80HIjSil1Mg0cPmi5EaWUOpkGDh+0x6GUUifTwOFDQ0l1DRxKKeWmgcMH7XEopdTJNHD4UL9trJYcUUqpeho4fHD3OHSoSimlGmjg8KG0spaYiFDCQ/XXpJRSbnpG9KGkslbzG0op1YQGDh9Kq7ROlVJKNaWBwwftcSil1Mk0cPhQWunQOlVKKdWEnhV9mDU0hfTEqI5uhlJKdSoaOHz47/ljOroJSinV6ehQlVJKqVbRwKGUUqpVNHAopZRqFQ0cSimlWkUDh1JKqVbRwKGUUqpVNHAopZRqFQ0cSimlWkWMMR3dhqATkTzgYBtfngrkB7A5XYV+755Fv3fP4u/3HmSMSWv6YI8IHKdCRNYaY6Z2dDvam37vnkW/d89yqt9bh6qUUkq1igYOpZRSraKBo2VPdXQDOoh+755Fv3fPckrfW3McSimlWkV7HEoppVpFA4dSSqlW0cDRDBG5SER2icgeEXmwo9sTTCLyrIgcF5GtHo8li8gyEcly/UzqyDYGg4gMFJFPRGS7iGwTkQdcj3fr7y4iUSLytYhscn3vX7gezxSRr1z/5l8XkYiObmswiEioiGwQkfdc97v99xaRAyKyRUQ2isha12Nt/neugcMLEQkFngQuBsYAN4hId94O8Dn+f3v3E2JVGYdx/PukFqaRJCahlZhBf8BGArE0MKOQknRhf0hFImjjIqGojCIQXLTJWgQJFUxkf8y0XGaTWC4q06QiXZQEKeYs0sogS31anPfSbRjBO+Oda2eeD1zued97OLw/5r3zO+c997wvzO9T9yTQY/tqoKeU6+YE8Kjt64BZwIryd6577MeBebZvALqA+ZJmAc8Ba21PA44AD3Wwje30CLC3qTxc4r7VdlfT8xsD7udJHP2bCXxve7/tv4C3gYUdblPb2P4E+KVP9UKgu2x3A4uGtFFDwPYh27vL9u9U/0wmUfPYXTlWiqPKy8A8YGOpr13cAJImA3cBr5SyGAZxn8aA+3kSR/8mAT81lQ+UuuFkou1DZftnYGInG9NukqYAM4DPGQaxl+GaPUAvsBX4AThq+0TZpa59/gXgceBUKY9neMRt4ENJuyQ9XOoG3M9Hnu3WRf3YtqTa/m5b0ljgPWCl7d+qk9BKXWO3fRLokjQO2Axc0+EmtZ2kBUCv7V2S5na6PUNsju2Dki4Ftkra1/xhq/08Vxz9Owhc3lSeXOqGk8OSLgMo770dbk9bSBpFlTTW295UqodF7AC2jwLbgJuAcZIaJ5N17POzgbsl/Ug1/DwPeJH6x43tg+W9l+pEYSaD6OdJHP3bCVxdfm1xPnA/sKXDbRpqW4DlZXs58EEH29IWZXz7VWCv7eebPqp17JImlCsNJI0Gbqe6v7MNWFx2q13ctlfZnmx7CtV3+mPbS6h53JLGSLqosQ3cAXzLIPp5nhw/DUl3Uo2HjgBes72mw01qG0lvAXOpplo+DDwLvA9sAK6gmpL+Xtt9b6D/r0maA3wKfMO/Y95PUd3nqG3skqZT3QwdQXXyuMH2aklTqc7ELwG+ApbaPt65lrZPGap6zPaCusdd4ttciiOBN22vkTSeAfbzJI6IiGhJhqoiIqIlSRwREdGSJI6IiGhJEkdERLQkiSMiIlqSxBFxDpM0tzGLa8S5IokjIiJaksQRcRZIWlrWuNgjaV2ZRPCYpLVlzYseSRPKvl2SPpP0taTNjXUQJE2T9FFZJ2O3pKvK4cdK2ihpn6T1ap5MK6IDkjgiBknStcB9wGzbXcBJYAkwBvjS9vXAdqon8gFeB56wPZ3qqfVG/XrgpbJOxs1AY+bSGcBKqrVhplLNuRTRMZkdN2LwbgNuBHaWi4HRVBPGnQLeKfu8AWySdDEwzvb2Ut8NvFvmEppkezOA7T8ByvG+sH2glPcAU4Ad7Q8ron9JHBGDJ6Db9qr/VErP9NlvoPP7NM+bdJJ8b6PDMlQVMXg9wOKy1kFjLecrqb5fjVlXHwB22P4VOCLpllK/DNheViA8IGlROcYFki4c0igizlDOXCIGyfZ3kp6mWmHtPOBvYAXwBzCzfNZLdR8EqimsXy6JYT/wYKlfBqyTtLoc454hDCPijGV23Ig2kXTM9thOtyPibMtQVUREtCRXHBER0ZJccUREREuSOCIioiVJHBER0ZIkjoiIaEkSR0REtOQfZiJWxEiYtRMAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fUUvZqny6SDN",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "77890905-1a36-4a1b-8052-e6064d09687b"
      },
      "source": [
        "vec_code = modal_model.predict(encoder_model.predict(source_proc.transform(source_docs)))\n",
        "vec_code.shape"
      ],
      "execution_count": 65,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(231636, 256)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 65
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uTW0VhS26SDP",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 88
        },
        "outputId": "61949fcc-fe70-4058-dd36-cc189445c80c"
      },
      "source": [
        "#all code in a search box\n",
        "dimension = vec_code.shape[-1]\n",
        "index = AnnoyIndex(dimension)\n",
        "for i, v in enumerate(vec_code):\n",
        "    index.add_item(i, v)\n",
        "index.build(10)"
      ],
      "execution_count": 66,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:3: FutureWarning: The default argument for metric will be removed in future version of Annoy. Please pass metric='angular' explicitly.\n",
            "  This is separate from the ipykernel package so we can avoid doing imports until\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 66
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0_e2NEOr6SDR",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def vectorize_string(txt):\n",
        "  vec = proc.transform([txt])[:,1:]\n",
        "  emb = np.mean(embedding_model.predict(vec), axis=1)\n",
        "  return emb"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dGEZ8bpd6SDX",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def search(inp):\n",
        "  inp = vectorize_string(inp)\n",
        "  ids, dist = index.get_nns_by_vector(inp.squeeze(), 5, include_distances=True)\n",
        "\n",
        "  for i, dist in zip(ids, dist):\n",
        "    print(f'dist: {dist:.2f}\\n{source_docs[i]}\\n')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "scrolled": true,
        "id": "_8hdS8H86SDZ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# search('read csv file into pandas dataframe')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JLCfj7LB6SDb",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 292
        },
        "outputId": "03f31cd6-36c9-4dae-d32d-83c707984729"
      },
      "source": [
        " search('chicken pot pie')"
      ],
      "execution_count": 70,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "dist: 1.02\n",
            "boneless skinless chicken breasts, margarine, water, ritz cracker crumbs, parmesan cheese, cream of chicken soup, sour cream, mushrooms\n",
            "\n",
            "dist: 1.02\n",
            "chicken fat, onions, carrot, celery, green pepper, apple, chicken broth, all-purpose flour, canned tomato, salt, pepper, curry powder, cooked chicken\n",
            "\n",
            "dist: 1.03\n",
            "boneless skinless chicken breasts, \"aunt janes krazy mixed up salt\", olive oil, onion, cream of mushroom soup, cream of chicken soup, sour cream, ritz cracker, butter\n",
            "\n",
            "dist: 1.03\n",
            "milk, flour, cream of mushroom soup, dry onion soup mix, margarine, chicken\n",
            "\n",
            "dist: 1.03\n",
            "chicken breasts, mushroom soup, white wine, milk, pepperidge farm stuffing\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0L5siZ2_6SDd",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 272
        },
        "outputId": "cd3b0ec3-5cfc-4872-aea4-32e1bb689773"
      },
      "source": [
        " search('iced tea')"
      ],
      "execution_count": 71,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "dist: 1.02\n",
            "limoncello, vodka, cranberry juice\n",
            "\n",
            "dist: 1.02\n",
            "cranberries, vodka\n",
            "\n",
            "dist: 1.03\n",
            "triple sec, grand marnier, light rum, orange juice, creme de almond\n",
            "\n",
            "dist: 1.03\n",
            "vodka, pomegranate liqueur, unsweetened cranberry juice\n",
            "\n",
            "dist: 1.03\n",
            "rum, chambord raspberry liquor, apple juice\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ex_pXhIy6SDf",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 292
        },
        "outputId": "38b71632-2ee4-46fa-e8ec-1e57915a6766"
      },
      "source": [
        " search('chocolate chip cookie')"
      ],
      "execution_count": 72,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "dist: 1.00\n",
            "butter, graham cracker crumbs, sweetened condensed milk, butterscotch chips, semi-sweet chocolate chips, flaked coconut, nuts\n",
            "\n",
            "dist: 1.01\n",
            "graham cracker crumbs, butter, sweetened condensed milk, semi-sweet chocolate chips, milk chocolate chips, vanilla extract, pecans\n",
            "\n",
            "dist: 1.01\n",
            "butter, graham cracker crumbs, coconut, butterscotch chips, semi-sweet chocolate chips, sweetened condensed milk, nuts\n",
            "\n",
            "dist: 1.01\n",
            "butter, graham crackers, chocolate chips, butterscotch chips, coconut, walnuts, condensed milk\n",
            "\n",
            "dist: 1.01\n",
            "butter, graham wafer crumbs, sweetened flaked coconut, chocolate chips, butterscotch chips, sweetened condensed milk, walnuts\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-gIPbxKr6SDh",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}