part2-25k.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas\n",
    "pandas.set_option('max_colwidth', 200)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LentaRecord(\n",
       "    url='https://lenta.ru/news/2018/12/14/cancer/',\n",
       "    title='Названы регионы России с\\xa0самой высокой смертностью от\\xa0рака',\n",
       "    text='Вице-премьер по социальным вопросам Татьяна Голикова рассказала, в каких регионах России зафиксирована наиболее высокая смертность от рака, сообщает РИА Новости. По словам Голиковой, чаще всего онкологические заболевания становились причиной смерти в Псковской, Тверской, Тульской и Орловской областях, а также в Севастополе. Вице-премьер напомнила, что главные факторы смертности в России — рак и болезни системы кровообращения. В начале года стало известно, что смертность от онкологических заболеваний среди россиян снизилась впервые за три года. По данным Росстата, в 2017 году от рака умерли 289 тысяч человек. Это на 3,5 процента меньше, чем годом ранее.',\n",
       "    topic='Россия',\n",
       "    tags='Общество'\n",
       ")"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from corus import load_lenta\n",
    "path = 'data/lenta-ru-news.csv.gz'\n",
    "records = load_lenta(path)\n",
    "next(records)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#preprocessing: lower-case, commas, lemmatization\n",
    "#PRACTICE: implement preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['выплата', 'на', 'второй', 'ребенок']\n",
      "['выплата', 'на', 'второй', 'ребенок']\n"
     ]
    }
   ],
   "source": [
    "import razdel\n",
    "from pymystem3 import Mystem\n",
    "#from pymorphy2 import MorphAnalyzer\n",
    "\n",
    "_LEMMATIZER = None\n",
    "def tag(word):\n",
    "    global _LEMMATIZER\n",
    "    if _LEMMATIZER is None:\n",
    "        _LEMMATIZER = Mystem()\n",
    "        #_LEMMATIZER = MorphAnalyzer('ru')\n",
    "    try:\n",
    "        processed = _LEMMATIZER.analyze(word)[0]\n",
    "    except:\n",
    "        return word\n",
    "    if not processed.get('analysis'):\n",
    "        return word.strip().lower()\n",
    "    lemma = processed[\"analysis\"][0][\"lex\"].lower().strip()\n",
    "    return lemma\n",
    "\n",
    "import re\n",
    "def tokenize_re(doc):\n",
    "    return [tag(x) for x in re.split('(\\W+)', doc) if x.strip()]\n",
    "\n",
    "def tokenize(doc):\n",
    "    # упомянуть регулярные выражения\n",
    "    return [tag(t.text) for t in razdel.tokenize(doc)]\n",
    "\n",
    "import sentencepiece\n",
    "def tokenize_bpe(doc):\n",
    "    #FIXME\n",
    "    pass\n",
    "\n",
    "print(tokenize_re('Выплаты на второго ребёнка'))\n",
    "print(tokenize('Выплаты на второго ребёнка'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "012f892dc492483da006e3a17a42c867",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=25000), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "25000\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>class</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>[называть, регион, россия, с, самый, высокий, смертность, от, рак]</td>\n",
       "      <td>Россия</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>[австрия, не, представлять, доказательство, вина, российский, биатлонист]</td>\n",
       "      <td>Спорт</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>[обнаруживать, самый, счастливый, место, на, планета]</td>\n",
       "      <td>Путешествия</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>[в, сша, раскрывать, сумма, расход, на, расследование, «, российский, дело, »]</td>\n",
       "      <td>Мир</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>[хакер, рассказывать, о, план, великобритания, заминировать, севастополь]</td>\n",
       "      <td>Мир</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                            title  \\\n",
       "0              [называть, регион, россия, с, самый, высокий, смертность, от, рак]   \n",
       "1       [австрия, не, представлять, доказательство, вина, российский, биатлонист]   \n",
       "2                           [обнаруживать, самый, счастливый, место, на, планета]   \n",
       "3  [в, сша, раскрывать, сумма, расход, на, расследование, «, российский, дело, »]   \n",
       "4       [хакер, рассказывать, о, план, великобритания, заминировать, севастополь]   \n",
       "\n",
       "         class  \n",
       "0       Россия  \n",
       "1        Спорт  \n",
       "2  Путешествия  \n",
       "3          Мир  \n",
       "4          Мир  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas\n",
    "from tqdm import tqdm_notebook as tq\n",
    "\n",
    "records = load_lenta(path)\n",
    "# overfitting and underfitting\n",
    "# splitting the dataset. how many examples to take for a test?\n",
    "dataset = []\n",
    "for i, r in tq(enumerate(records), total=25000):\n",
    "    if not r.topic:\n",
    "        # есть примеры в датасете, где нет категории\n",
    "        continue\n",
    "    dataset.append((tokenize(r.title), r.topic))\n",
    "    if len(dataset) >= 25000:\n",
    "        break\n",
    "df = pandas.DataFrame(dataset, columns=['title', 'class'])\n",
    "print(len(df))\n",
    "df.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "17\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'69-я параллель': 0,\n",
       " 'Бизнес': 1,\n",
       " 'Бывший СССР': 2,\n",
       " 'Дом': 3,\n",
       " 'Из жизни': 4,\n",
       " 'Интернет и СМИ': 5,\n",
       " 'Крым': 6,\n",
       " 'Культпросвет ': 7,\n",
       " 'Культура': 8,\n",
       " 'Мир': 9,\n",
       " 'Наука и техника': 10,\n",
       " 'Путешествия': 11,\n",
       " 'Россия': 12,\n",
       " 'Силовые структуры': 13,\n",
       " 'Спорт': 14,\n",
       " 'Ценности': 15,\n",
       " 'Экономика': 16}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "classes = sorted(set([_[1] for _ in dataset]))\n",
    "y_size = len(classes)\n",
    "print(y_size)\n",
    "vec_y = {c: i for i, c in enumerate(classes)}\n",
    "vec_y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20000 20000 2500 2500 2500 2500\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>x_train</th>\n",
       "      <th>y_train</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>[в, россия, закрывать, чемпионат, мир, по, футбол]</td>\n",
       "      <td>Спорт</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>[назначать, орешкин, неизвестный, таджик, обретать, миссия]</td>\n",
       "      <td>Экономика</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>[в, киев, массово, лопнуть, труба, после, подача, горячий, вода]</td>\n",
       "      <td>Бывший СССР</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>[россиянин, избегать, тюрьма, за, убийство, оскорблять, он, девушка]</td>\n",
       "      <td>Россия</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>[умирать, режиссер, «, чегемский, детектив, », александр, светлов]</td>\n",
       "      <td>Культура</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                x_train  \\\n",
       "0                    [в, россия, закрывать, чемпионат, мир, по, футбол]   \n",
       "1           [назначать, орешкин, неизвестный, таджик, обретать, миссия]   \n",
       "2      [в, киев, массово, лопнуть, труба, после, подача, горячий, вода]   \n",
       "3  [россиянин, избегать, тюрьма, за, убийство, оскорблять, он, девушка]   \n",
       "4    [умирать, режиссер, «, чегемский, детектив, », александр, светлов]   \n",
       "\n",
       "       y_train  \n",
       "0        Спорт  \n",
       "1    Экономика  \n",
       "2  Бывший СССР  \n",
       "3       Россия  \n",
       "4     Культура  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>x_dev</th>\n",
       "      <th>y_dev</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>[телеведущая, отказывать, в, работа, из-за, «, недостаточно, большой, грудь, »]</td>\n",
       "      <td>Интернет и СМИ</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>[шарапов, отвечать, на, отказ, серена, уильямс, играть, против, она]</td>\n",
       "      <td>Спорт</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>[bvlgari, представлять, новый, вариация, на, змеиный, тема]</td>\n",
       "      <td>Ценности</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>[находить, новый, способ, побеждать, особо, опасный, рак]</td>\n",
       "      <td>Наука и техника</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>[анестезиолог, изнасиловать, россиянка, под, наркоз, и, пойти, под, суд]</td>\n",
       "      <td>Силовые структуры</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                             x_dev  \\\n",
       "0  [телеведущая, отказывать, в, работа, из-за, «, недостаточно, большой, грудь, »]   \n",
       "1             [шарапов, отвечать, на, отказ, серена, уильямс, играть, против, она]   \n",
       "2                      [bvlgari, представлять, новый, вариация, на, змеиный, тема]   \n",
       "3                        [находить, новый, способ, побеждать, особо, опасный, рак]   \n",
       "4         [анестезиолог, изнасиловать, россиянка, под, наркоз, и, пойти, под, суд]   \n",
       "\n",
       "               y_dev  \n",
       "0     Интернет и СМИ  \n",
       "1              Спорт  \n",
       "2           Ценности  \n",
       "3    Наука и техника  \n",
       "4  Силовые структуры  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#train, dev, test\n",
    "#20000, 2500, 2500 \n",
    "from sklearn.model_selection import train_test_split\n",
    "X_train, X_dev, y_train, y_dev = train_test_split([_[0] for _ in dataset], \n",
    "                                                  [_[1] for _ in dataset], test_size=2500, random_state=42)\n",
    "X_train, X_test, y_train, y_test = train_test_split(X_train, \n",
    "                                                    y_train, test_size=2500, random_state=42)\n",
    "print(len(X_train), len(y_train), len(X_dev), len(y_dev), len(X_test), len(y_test))\n",
    "display(pandas.DataFrame(zip(X_train[:5], y_train[:5]), columns=['x_train', 'y_train']))\n",
    "display(pandas.DataFrame(zip(X_dev[:5], y_dev[:5]), columns=['x_dev', 'y_dev']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['!',\n",
      " 'бомбить',\n",
      " 'гризли',\n",
      " 'игнашевич',\n",
      " 'линчевать',\n",
      " 'ничья',\n",
      " 'пломбир',\n",
      " 'пропадать',\n",
      " 'скульптура',\n",
      " 'ударный',\n",
      " 'ярошенко']\n",
      "Vocabulary size: 8499\n"
     ]
    }
   ],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "vec_x = CountVectorizer(ngram_range=(1, 1), tokenizer=lambda x: x, lowercase=False, min_df=2, binary=False)\n",
    "vec_x.fit_transform(X_train)\n",
    "x_size = len(vec_x.get_feature_names())\n",
    "import pprint; pprint.pprint(vec_x.get_feature_names()[::x_size//10])\n",
    "print(\"Vocabulary size:\", x_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm import tqdm_notebook as tq\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.utils.data import DataLoader\n",
    "from torch.utils.data.dataset import Dataset\n",
    "\n",
    "use_cuda = torch.cuda.is_available()\n",
    "device = torch.device(\"cuda:0\" if use_cuda else \"cpu\")\n",
    "#device = torch.device('cpu')\n",
    "params = {\n",
    "    'batch_size': 1,\n",
    "    'shuffle': True,\n",
    "    'num_workers': 0}\n",
    "eval_params = {\n",
    "    'batch_size': 64,\n",
    "    'shuffle': False,\n",
    "    'num_workers': 0}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([0., 0., 0.,  ..., 0., 0., 0.]), 14)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "[tensor(1.),\n",
       " tensor(1.),\n",
       " tensor(1.),\n",
       " tensor(1.),\n",
       " tensor(1.),\n",
       " tensor(1.),\n",
       " tensor(1.),\n",
       " tensor(0.),\n",
       " tensor(0.),\n",
       " tensor(0.)]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class VecDataset(Dataset):\n",
    "    def __init__(self, x, y, vec_x, vec_y):\n",
    "        self.x = x\n",
    "        self.y = y\n",
    "        self.vec_x = vec_x\n",
    "        self.vec_y = vec_y\n",
    "\n",
    "    def __getitem__(self, index):\n",
    "        x_list = self.vec_x.transform([self.x[index]])\n",
    "        y_id = self.vec_y[self.y[index]]\n",
    "        x = torch.from_numpy(x_list.todense()).squeeze(0).float()\n",
    "        #x /= max(1, x.sum())\n",
    "        return x, y_id\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.x)\n",
    "\n",
    "train_dataset = VecDataset(X_train, y_train, vec_x, vec_y)\n",
    "example = train_dataset[0]\n",
    "display(example)\n",
    "sorted(example[0], reverse=True)[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[0., 0., 0.,  ..., 0., 0., 0.]])\n",
      "tensor([8])\n"
     ]
    }
   ],
   "source": [
    "train_loader = DataLoader(train_dataset, **params)\n",
    "for x,y in train_loader:\n",
    "    print(x)\n",
    "    print(y)\n",
    "    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=40), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0772\n"
     ]
    }
   ],
   "source": [
    "class TextClassifier(nn.Module):\n",
    "    def __init__(self, vocab_size, hidden_size, num_classes):\n",
    "        super().__init__()\n",
    "        if hidden_size:\n",
    "            layers = [\n",
    "                nn.Linear(vocab_size, hidden_size),\n",
    "                nn.Linear(hidden_size, num_classes)\n",
    "            ]\n",
    "        else:\n",
    "            layers = [\n",
    "                nn.Linear(vocab_size, num_classes),\n",
    "            ]\n",
    "        self.fc = nn.Sequential(*layers)            \n",
    "\n",
    "    def forward(self, text):\n",
    "        x = self.fc(text)\n",
    "        return x\n",
    "\n",
    "from sklearn.metrics import accuracy_score\n",
    "def evaluate_model(model, dataset, params=eval_params):\n",
    "    criterion = torch.nn.CrossEntropyLoss()\n",
    "    model.eval()\n",
    "    val_loader = DataLoader(dataset, **params)\n",
    "    predicts = []\n",
    "    trues = []\n",
    "    with torch.no_grad():\n",
    "        total_loss = 0.\n",
    "        total_count = 0\n",
    "        with tq(val_loader, leave=False) as tq_batches:\n",
    "            for batch_X, batch_y in tq_batches: # aka inputs, expected\n",
    "                batch_X = batch_X.to(device)\n",
    "                batch_y = batch_y.to(device)\n",
    "                output = model(batch_X)\n",
    "                predicted = torch.argmax(output, dim=1)\n",
    "                predicts.extend([x.item() for x in predicted])\n",
    "                trues.extend([x.item() for x in batch_y])\n",
    "                loss = criterion(output, batch_y.long())\n",
    "                total_loss += loss.item() * len(batch_y)\n",
    "                total_count += len(batch_y)\n",
    "                tq_batches.set_postfix(loss=f\"{total_loss/total_count:.3f}\", \n",
    "                                       refresh=False)\n",
    "    return predicts, trues\n",
    "\n",
    "def train(model, dataset, epochs, lr_base, scale=1):\n",
    "    #print(\"Using params:\", params)\n",
    "    criterion = torch.nn.CrossEntropyLoss()\n",
    "    train_size = len(dataset)\n",
    "    optimizer = torch.optim.SGD(model.parameters(), lr=lr_base, )\n",
    "    train_loader = DataLoader(dataset, **params)\n",
    "    losses = []\n",
    "    with tq(range(epochs)) as tq_epochs:\n",
    "        for e in tq_epochs:\n",
    "            model.train()\n",
    "            total_loss = 0.\n",
    "            total_count = 0\n",
    "            with tq(train_loader, leave=False) as tq_batches:\n",
    "                for batch_X, batch_y in tq_batches:\n",
    "                    batch_X = batch_X.to(device)\n",
    "                    batch_y = batch_y.to(device)\n",
    "                    lr = lr_base * (train_size*(scale*epochs-e)-total_count) / (train_size * scale * epochs)\n",
    "                    for g in optimizer.param_groups:\n",
    "                        g['lr'] = lr\n",
    "                    optimizer.zero_grad()\n",
    "                    output = model(batch_X)\n",
    "                    loss = criterion(output, batch_y.long())\n",
    "                    loss = loss.to(device)\n",
    "                    loss.backward()\n",
    "                    loss_item = loss.item() \n",
    "                    total_loss += loss_item * len(batch_y)\n",
    "                    total_count += len(batch_y)\n",
    "                    losses.append(loss_item)\n",
    "                    optimizer.step()\n",
    "                    tq_batches.set_postfix(epoch=e, lr=f\"{lr:.3f}\",\n",
    "                                           loss=f\"{total_loss/total_count:.3f}\", \n",
    "                                           refresh=False)\n",
    "            predicted, expected = evaluate_model(model, val_dataset)\n",
    "            accuracy = accuracy_score(predicted, expected)\n",
    "            print(f'Epoch: {e:2d} Accuracy: {accuracy:.3f} Loss:{total_loss/total_count:.3f}')\n",
    "    return losses\n",
    "\n",
    "model = TextClassifier(x_size, 4, y_size).to(device)\n",
    "val_dataset = VecDataset(X_dev, y_dev, vec_x, vec_y)\n",
    "predicted, expected = evaluate_model(model, val_dataset)\n",
    "accuracy = accuracy_score(predicted, expected)\n",
    "print(accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a6c43c54dd8247b58041b7187033d341",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=20000), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=40), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch:  0 Accuracy: 0.659 Loss:1.510\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d9f865b1528b4bf58c9c2c46ec9fb72e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=10000), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=40), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch:  0 Accuracy: 0.696 Loss:0.998\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6133c6ee6cbf41c981535a3e29aba409",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=5000), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=40), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch:  0 Accuracy: 0.703 Loss:0.879\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b6e1e8a3f250424aa335c7b8d7d53db0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2500), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=40), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch:  0 Accuracy: 0.701 Loss:0.829\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "47b67cf92b0442bf96cb2acf5a907cd0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1250), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=40), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch:  0 Accuracy: 0.706 Loss:0.807\n",
      "\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWsAAAD4CAYAAAAqw8chAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO2deXwTdfrHP0+SphdQCi3IXUTFFV0RixfqeqyieOGqu+iu63r8WK/1XFdU1vtAcb1RRMUbb/AAFJBTbgoUCpSjLYW29L7vpsn390cmbZJmkplkJpm0z/v14kU6M/nOkzk+88zzfb7Pl4QQYBiGYYyNKdIGMAzDMIFhsWYYhokCWKwZhmGiABZrhmGYKIDFmmEYJgqw6NFoSkqKSEtL06NphmGYbsnWrVsrhBCpcut1Eeu0tDRkZGTo0TTDMEy3hIgO+VvPYRCGYZgogMWaYRgmCmCxZhiGiQJYrBmGYaIAFmuGYZgoQJFYE9H9RLSbiHYR0RdEFKe3YQzDMEwnAcWaiIYAuAdAuhDiRABmAFP0NoxhGIbpRGkYxAIgnogsABIAHNHPJIZh9GJfST0y8qsibQYTBAHFWghRBOBlAIcBFAOoFUIs9d6OiKYSUQYRZZSXl2tvKcMwITPxtTW4dvaGSJvBBIGSMEgygKsAjAQwGEAiEf3NezshxBwhRLoQIj01VXbEJMMwDBMESsIgfwRwUAhRLoSwAZgP4Cx9zWIYhmHcUSLWhwGcQUQJREQALgSQra9ZDMMwjDtKYtabAHwLYBuALOk7c3S2i2EYhnFDUdU9IcQTAJ7Q2RaGYRhGBh7ByDAMEwWwWDMMw0QBLNYMwzBRAIs1wzBMFMBizTAMEwWwWDMMw0QBLNYMwzBRAIs1wzBMFMBizTAMEwWwWDMMw0QBLNYMwzBRAIs1wzBMFMBizTAMEwWwWDMMgMLqJmQV1kbaDIaRRVGJVIbp7pz94koAQP6MyyJsCcP4hj1rhmGYKEDJhLmjiSjT7V8dEd0XDuMYJy02O+74bCsOVzZF2hSGYSKEkmm99gkhxgohxgI4FUATgAW6W8Z0sC6nAj/vKsGTP+2OtClMDyJt2iI8/dOeSJvBSKgNg1wIIFcIcUgPYxiGMRZz1x2MtAmMhFqxngLgC18riGgqEWUQUUZ5eXnoljEMwzAdKBZrIrICuBLAN77WCyHmCCHShRDpqampWtnHMAzDQJ1nfSmAbUKIUr2MYRiGYXyjRqyvh0wIhGEYhtEXRWJNRIkALgIwX19zGF8IEWkLGEZ/hBAorOb0VDkUibUQolEI0V8IweNxIwhF2gCG0ZE5a/Jw9osrsb+0PtKmGBIewcgwjCHYmFcJAOxdy8BizTAMEwWwWEcBHLJmGIbFOoogDlozDACgqKYZadMWYeW+skibEjZYrBkmSvg6o6BHiZM/Mg/XAAC+ySiIsCXhg+tZRwGCc/cYAP/5dicArrndU2HPOqrgOAjD9FRYrBmG8aCxtR1p0xZF2gzGCxbrHkx2cR3OfGE5qhrbIm0KoxOHKhuxr0TdIJOSuhadrGFCgcW6B/POqlwU17bgtwNc0ra78oeZqzDxtTWRNoPRABZrhmFQ1diGtGmL8MXmw5E2hZGBxToK4FwQ7ViwvRAr9/a89LeLXlmNrYeqZNcXVDmHeLNYGxdO3YsieFBM6Nz/1Q4APS/97UBZA55dlB1pM5gQYM+aYRgPoimtP5psDRUW6yigJ12QDKOEnviWqXTygb5E9C0R7SWibCI6U2/DmK5ofX3yM4DRina7Aw98lYmcsoZIm9JtUepZvw7gFyHE8QBOBsDBL4ZhOth1pA7ztxfhwa8zw7K/nvi2GbCDkYiSAJwL4B8AIIRoA8CjKBiGiTg9KRyixLMeCaAcwIdEtJ2I3pfmZGTCRg90IxiG8UCJWFsAjAPwjhDiFACNAKZ5b0REU4kog4gyyst5RJwe9CQvgum59MQQhxKUiHUhgEIhxCbp72/hFG8PhBBzhBDpQoj01NRULW1kGCasREYtyUDeyL6Sejz0zQ7YHcZ5cgQUayFECYACIhotLboQwB5drWI6yCmrx/Tvd+nSNtfJZhjf3P7ZVnyztRCHKhsjbUoHSkcw/gvA50RkBZAH4Gb9TNKfoppmpPSyItZijrQpAZn6yVZUNOjTn+uSaiN5NAzD+EZR6p4QIlMKcfxeCDFZCFGtt2F60dpux4QZK/Dg1zsibYph6I5S3dxmR0VDa6TN6LZc9dZaPPXT7kib0aPi2z1uBKPN7jy7PbGYj57YHQILdx4xTGjl6rfXIf3ZXyNtRrdlR2EtPlyXH7H998SXwR4n1tEMGdgH/nh9Pu6etx1zI3gDu7NXZcF9hjE6LNZRhJG9iayiWgDAMwu57znaCeXlyBjvVaFjlDdEd1isezIaXo8tNrt2jalkY14l0qYtwu4jtRGzIRqxOwTa7Q7V33PVvnZHCz/CJZBGmlbMSJ3vLNZhQgiBwuquF3nA72mw7z1H6tDcJi+mBroeg2Lp7lIAwIbcyghbEl386e11OOaxn1V/75yXVupgDXCo0nl/PLZAn1TVaIfF2geHKhuxYHuhpm1+s7UQZ7+4Elvy5Wfr0IO6FhsmvfEbHghTgR0methRGPk3kfU5FThS0wwAaDfQABQjwmLtg8vfXNsxo4hWbDvkzHYMdwnJFsmjzjgUtdmWTBgJd6j2hvc34eJX1U/oa8CQsu6wWPugvqU90iZEhEU7i7E+tyKo70Z7KKUn4O8URfL8NbQGf7/1pOuux4m1EXt5laL3hXnXvG244b1NgTfUiZLaFqRNW4TfDnAhsEgSvXeIdhjxGPQ4sXZhpF5eF7XNtqB654NFGOyS3H7YGar5fCPPsK0Hxjrb2hCM73WgtB5z1x5UtK2RVKLHinW4CXRR2R0CJz+1FNPmZ4V9/0YebMPow0frlImVWrR8cX1+cTbSpi1Co48wSSi+1mVvrsXTUTgegMU6zMhdYw7pKv9+e5HH8pDDNqzDmlFU0+wzxzga+WJzgabtafGi6v2mN2dNHgCgptkGwBnb1uL4t7WH7+1VS1isezBaekGBvPOCqibUt9i022GINLfZ8cDXmahqVF7RcMKMFbrlGMshhMCTP+5GTllkh88X1TSjuLY54HZ6htaum70h7MffSLBYRxF6hSvCEb4/56WV+NPb62XX55TVdwxZDwffbi3A/G1FeGXZPl3aL6xuwgcK46L+OFzVhI/W5+Pmj7ZoYFXwTJixAme+sEJ2vRbXZqA2sovrQt6HUoyYh6C0njXTg6ltsqGuxYZh/RJCaueAnxzzP76iPtfWhQH7ivH3uZuRV96IK08ejNTesSG3F85+BTVC1W53oNHP6FhV+zVgF6iRri32rKOJkC6crjeC0pvywldW9+jXz2Coa3Z2ikVCgOpbbHhkfhaa2vQfL/Cf73bi5KeWRlRoHQ6Bv7y7Aav2de+yx4rEmojyiSiLiDKJKENvo7oj7hfzxrzKsOV7K/HIAm3BRfyji7dX5eKLzYfxyYZDstto5THO31bkd31pXYvuRb4a29qx6WAV7p63Xbd9tNsdmLlkL2qbItfvosazPl8IMVYIka6bNT2A7zOLMGXOxoAXOcP4Y/bqXNnCYK7MIiPEXU9/fjlu/lBZvN3IKaQ/7yrBrJW5eG5x5FL+elwYJNLX72GpstghKQWpqrENDj8FbNTY+96aPKzU4FWwqrFNVZYEAE4R7II2V5ovwT1S04wZP+9VLIKRZkNe4GqIq/cHN2q1ttnmd6KJm+ZuxpLdJUG17U67w5nuF8m0P6ViLQAsJaKtRDTV1wZENJWIMogoo7zc+MOFw6EtBysakTZtEZbKXCyVDa0Y98wyvLxUm4yE5xZna3IDj3tmGcY9s0wDi7RjU14lDlYYZ6ZppWjlLbqHLVyec5NGHXtG4Ka5m3FYJofa3xFcn1uJ62Zv6Pjb7hBIm7YIr/26H4DzIfDPT7d2rFOKETs7lYr12UKIcQAuBXAXEZ3rvYEQYo40qW56amqqpkZGmrUHKoJKxt9RUAMAWJRV7HN9peS9LttTqvsrqxFeiQHAFuRw+r/M2YjzX17lc51Rflso2OwOrAnSu9SSrKLasKZQBsPMJXt9llMVQnRcX++syvVYt7+0HqMeXazayzZSaEbp7OZF0v9lABYAOE1Po9Ry/ZyN+OMrq3Vr/28fbJIVCqUoFZRQO37y424AvvqbovYi4T08tyi7y7LW9uj0ENvtDnyw9qAm9r+ybD/+PnczNh/Urt65mvP75I+dM5Wv9SqkFclZgHwxa2WuatHNlBynZXtKg9qnERyCgGJNRIlE1Nv1GcDFAAw1lcOGvErd60QboTA6AfhxxxGMf+5X/wWfsn/q+KjkItMyl7Q3moA2+bcQX2K0ep+nODQbTBzk+DqjEM8s3IPZq/K6rFN7c+eVO6/fqsbAmTdnv6h9GuW2wzWy6xwqfoxr0xabHbNW5oRqlix2e2Tux0gWgFPiWQ8EsJaIdgDYDGCREOIXfc3ShtZ2O578cXdE021cuM6x6xKTO+WB7ovpC7JQXt+KxlZ1gqb3NeZqPivuNuDNU2W321NcFzAVMJy3YSgeU0OrzeN/X0RyUIW/V/jaZm3vCe99vbH8AGYu0Wd0qFZUNLTis43y6Y3uRIVnLYTIE0KcLP0bI4R4LhyGBYsQokOcF2wrwkfr8zFz6d6g2tKz59f73Cu5qUN5qof1Yqs/4nf11W+v87t+zf5yVfNVBnVYNFTRtnYHbv5wM/aXal+/Q6/TVlwrPynt95n+z58SfFXK0xu1x+ruedsw/ftdPt/K5e6XSEawu13q3kfr83Hy00txqLIRdumI+4oYtCno6Jq5JDiR94fakx1y0T3j9I90UFAVuCDQ3+duDoMloeE6N9sLarByXzmmfx98dNAInltPo7rR6dS50vJqm2xY7JUMYKT7p9uJ9fJsZ57x4aomvzN6tyrwmg9WaFcO09/NGI77tNJH3rSRBSKaplbT9n4mP39p2XL3I9Tr+e4vtuHOz7f5fKsL1PSRmmZM+25n0NlOSuh2Yu3Osx2ZB52HOhIClV/ZefKPuL1+Brp5Fu484pF7WlwT2CP1R6VXrLjzUGh/G7+5/IDmbUYTvi6znLIGzFmT62NNVw5WNIZcuzmc2T567suXdyu3v1Du78Jq5/3lN/wpc6s8Mj8LX24pwNoDwc1hqoRuLdZ6I4TAe2vyAnZguvKt3VEij961DkKdobxF5iLU4lXPO57+v2X7Q2+0G+B+VK5+ex2eX7xXkfd1/surAhbPKpK8ue6O0lxndwGXu6Z1d9Z0fH2JSrGuaGhVNRpJLzbmVeG5xdl4dEHoU3GpvYjq3TIQVu4tQ9q0RSHb4M0LP2f7fNAYFX9hLzl+3lUSdB6xklNms4uOa9U14tB7NqBQ9vHlFvkZXyIxoEMPMVy9v2sJBbW/zd/WSmz2V3hNCBH0cHk1RJVYF9U0o6apDenP/ooXFncdXKEENV5kTVObhwi22OyYsya34+ZzdVLWaTgDSiD76qRY7r1fZnYs+2RDvnx7Idjy7uo8XDXLM3Njb0kdapqc8e/SuhZcNWsdyutbg68imLsCEF3FUo3drjcONd68q/2qxjYc/98QM1F9nDTX8TjjheW4drbnpAsPfdvVGw509LSQ3cMKOnaDRc8Hw8Pf6TMvqfcbjvs17H05d6bcdv2dW/JDe+NVStSI9eKsYkyYsQI/7XT21i7dUwohhKyH7X6wg9aRcs+UnucWZeP5xXuxQIVn1GFDcCbIkleu7yAgOS557TdMlgT8o/X52FFQg68zgpzP78Ay4NOrMXLve8q/01IHlO72WNTg9bBsiEDamDfu53u7NOBEydugntkHcvntd36+VfN9+UoztTsELnpltSaFldTi0gD32WbyZGrN+EuR/W5bIcY+vbTj79ommy7HzxdRI9au4aLuB/vphXsw6tHFHtvpebG7BhIEM8Lu262FsutyyhqwYLv8el8EI/4XvbLawwsPduite4dpSNQ7H7zxDYeVf+fTycA7Z/nd5O8fbArFKsW4BEBJqMjIQ+oXZykXz0APZrnrsqapDXXNNhwoa8DDUpw9Iz/4ofW+Ohj9OWUFUobHHgVTg+0srEGRn878Grc+qjm/5aKiQWWFyiCJGrH2RkDg4/X5Edn3kRCzMly4P1j0etVzp6nNjsd/2B14wzAzNP875RsXOb2YfSX1HaVlh9sLMAidZTj9DZ3WEjUZEGf5mb9QL/TI0Ji/zbdTEchJKqvv9OprmmxIm7YI17pVy1NLu4bDzb1F/lW3cBrnWYeA32O3/i18XDBRs31Vy2R5eFf0ksNIJ1pv1I6urGxoxY5CeVEN1NzE19bgndXO8/Bh093YEPcvVfsP97nxlefujpxXaISOdLWU1bfgxx2hj4L0x/K9/uu2BxP61OJI63lZRY1Yu4L/HTU2fB3ZpY/BJPN6tPVQVdCv/R3teP2tx4mx2QXe/y1P0VD3RTuL8cuuEuwolC9p6UtEvYtAReKZMvXTrfh8k4rwhw+y/Pxub/aV1Osey3Y/jsH2k1Q3tnmc+y82B3+M9Oj0C9SmEAI3f7ily+QVwWTqBEMoD2HXOfPpeBjgmRl1s5sHewFe847zlWvnkxcHvW+1GQ/B3rDPLsp2G9Djm1abA3fN2xZU+8v3lmHimKOC+q5WHKlpxqgw7UsIgYmvrcFpI/tp3K427Rz72GLYpNf6afOzsC63M6RTLWXeBFMXZsXe0JwTF9MXKB9Gv7ekHv0SrV2WXzVrHTIfv0gTe+QQ0Kamh/u2ruNuhMkIok6slR40ua2MPMRaDWrKVnb5rkFerUWYfXqtakUfrGhEfIxZk7YAdAi1C+/6FIB/R2HWyhycOao/XvplL8andT6QtEopm++W/WTo0J7bITK0nUESFWJdUNXU+Yoi3eBqtMpd4LWspPeNnwyPQITrYtJyFvWmtvClxCl5g/pldwlOenIJ/HXNltdrP4DKNRHFQxNHa9quC3d7lRwH91KkG/Oq8M8/HK2LXYBzGi2j0mZ34HePq8+Zd+mDEbxnfxherHcfqcVlb6yFSbpm3UWOiFS7yj9kBj+ruPueCqqa8FMInSjl9YGLzBuNEx5fIrtO7bNnjGMfzAj9wVnf0g7Eya8f/9yvIe8jkiyXQhkttshN1KoWIzi1LTYHrvMajBQKRngjN3wH4yEpp9flbKySZhU5UtusqcdUVteCtGmLFPdiu9dtcJ3IxtZ2zFyyV5H37uxc0/+yVhLrDLZOdrDWj6UcvG97BPdY5vvdLm3aIkz/Pqsjxz4cpE1bpKqWthxavdG4cnpL6uTrT3sT6WHmcpkvY59WPgnz5Fn+a54rIVwjC93RcyYZxWJNRGYi2k5EC3Wzxgfe17yrCp2ae0HJxbu/1Dki8O0QpiJ6Y/kBzFqZi2+2Kh3RF/wNraTEqxxzfsvrGDIOBC+6wVo/kJw30WAKHEP+bONh1Gs4nF8J63K6Vk4z2jyE/jD667wSwvmALq1rwcp9AVIBZb43a6WyNF4tUBMGuRdANoA+OtnShV92FWsy8sv94v01238P+d6S4Gf7cN3Q+0rqZUe1JaMOp5hysMIxDp9tDC11LVi2H67BI/P1H4TTnfAVttKyP0BTDGqWUbnlowwAwNDk+C7r/Dkyj4b5HlLkWRPRUACXAXhfX3M62ZRXids/24YXfg5utha5G2lfCGIsO2+i193xyYZDXQoguZhrfRlzrS+jN5qwRsdKXbXNNuRXNHapYe3CPef4tk8yfHqTftn8Hk4/8ikAILV+D5Jt2qSJuYhkb777pWN3CKyXOTZG1ep313SdwFdv9D5fanLqQ8bHb/HV1xVKRlYwKPWsXwPwHwC95TYgoqkApgLA8OHDQzbMVYdD644495hS2rRFuOD4AZj7j/GKvit3atblKO8hTyNnHQYz9H2tPvmppX7X/+ZVJP21X1XWn178b5wHADgLf952IwDgY8xT10YEURrXfWdVDl5euh/PX31Sl3VZRfICYlAdj1queGttaA0oeJq4Jh/IK+9a4GljnjZpn6EQ0LMmossBlAkh/JaWEkLMEUKkCyHSU1NTQzYsXFO+r5CGrSrZXWuAuGU442zRwsq9ZXjgq0yvpYGlTM/JitXgqsxW6qODb0Oe50OayFl2s7nNHvAnCiHQ3GbHrBD6SJjwsMvPQzmcKPGsJwC4kogmwZkk1YeIPhNC/E1f0/xjggPxaEUjusaZAHhMh+VZLjV4n+fXbP+dEP6GfXtDPcT3uvmjLUF9L1AtjVDx93D2dWZafPSd+Jon8vo5G5FxqBq9Y/3fWvd9lYkfNJhFnFFGdQjX04oAdUjciWhtECHEI0KIoUKINABTAKwIh1AH+tHPWD7E7rhbYYHvgRpGqC5n0BIDBiAyAemzTLvQC86HeKPKOiHvrlYWB1Y69RoLdXipbAwunOrvoR6ut38Xhs+zluMa8xoAUD2wQq6SXk/HqJ1lWtEPdZhnfR6zYt4AgIC1V0Klmx/OqENNzW6jomoEoxBiFYBVulgSYfR4Rv7ox3siGOuG3l8afJaM3mgxyCMWzof0sabAJQKEAO7/KhO/7inF7wYFl6kazAQVDOMPw3rWWr5h6CWKl5s24B9m+VoEvmruuhcvOigzrZAcer511fmIv+qH/zPSB4340voMhqBc2jr8j7UF24tQ39qOzUHOZhKNdagZ3xjlrdOwYh0INboVyGssqVU+lNedt6xv4smYTwBAl9nFw8lIKkYfqHt4BMJXBoUcyeicbuky80acYcrGXZbvVe+TRTIyyM3v2J0Jd8+LYcVaSy9yZ4AsjTNeWK7dzhQSTDZIME/4YVSKoRR48M3K2Afxg3W6+h344fTnO4/rUutDeEuKF/vixRgVk+b6QYuBCt1huHa4+ceHwWX9GB0hgFd9jEEoqmmG3ce1pufbr2HFuqE1zDG/cJUsDfPz+LfY+7E29l5F2440dY5CvMS0GcdRkLOW++A4UxEuN2+U/up6DC42azNDdKD8bCNUhGOiB7maMBNmrOgoKhcuDCvW93yxPdIm+OX3FL4CLpFgtvU1LI19OCL79hZUNTeFe0nUJDRgfezdGEMHVe3fKDFKhnHHkGKdr7LjLdwMp1L8GPvfSJthOO6zfKtwS6Vq6JRtNTXIm9zm+jvTtAeDqQp3u8W+WYcZPdHzQW9IsT5PmonDqPRFQ6RNiAh9UY9Jpo2y6+8LUJ86Wpj+vfI5BxkmXBh+pphwEagTsicxmnyXbp1tfQ1nmLJxestxQbd9vEzb7vSUofgMowYWa4kZikqxCmTH3oyF9jNC3l+kOrpSUY0EasUhIT+7+ZLYaT6XD4GzUl8MBZ+TPYgCVyh0ibUenbHcwcioQe2Q8h6ZDWI0hlI5BqEK8dSG6yxrgm7Hn884zTIPV5nUl4IcRqWYYl6haNstcXdhdewDqvcRCVzHSos4IPvqTLRjKLHeX1qvYnBJeG+/tbH3YkPcv3Tdx+2WhXjd+rbHsnvN32Fv7E0AgDGUj4ctX8D7t39nfQozYt6XLWplJIIJcQSTOz2MSjHb+prbfhkmujGUWG9TWLHMnXDnLTcRId8SevRIqWjdH/Md4shZ12K+9QncYfmpo86Fi2QYt65HKLjObTDFty4wddbQdr9C2MNmohVDiXUkp3JSyr0DU3DFsMEhtND5I0fTYVxhWt9li1Npn8x39Yvlas2JlKcilU97nor5OGL7Zhg9MJZYByFC4c4c2BjfdbKD80zeM6EoY0nsNLxpfavL8u9inwqqPSOxMHa6z1Q+rc+XSWWJXOM/5hgjYaTrxVBire7IGOcwDiTf4ZsxlI/c2L/iKFQiDq2YaNrcsW6yWX1HolFS2vS2Q037fzd7zjU5lMpwpqnrxBPhOHJGmKeP0Ra1b/talPOVw1CpeyZVR8Z5+42kEuwVoU/Qqwd/My+DmQTOM+/AyZSL6y0rO9Y9FhP85LKRCoMMM3Ud9n2hSZuaHu64xPpicwaebL8Jdphlt00mz3j92tj7fG43znRAOwOZHsM9Xwb31qwHSibMjSOizUS0g4h2E5Fu7+h2h/pJUn+RyQkOBhMcSKNi1d9T4gkqqXznj4tNW2Al7YpbWaGu004u1POB9X+q2lHzmBlINbjbrL5Mqi/etjor/hnj3YSJFnYYaBJsJWGQVgAXCCFOBjAWwCVEFPqoEB88/F2Wj6UCyahDP9ThGctcxOiYnvaQ5Susin0Qw6jUY/kI6jolkPtNTxAYhEpkxN7eIfbXm5d7eNL+yI+7Aanwf1HMsb4asJ0pZmX7A9Q/PEZ4HRM9cRf0+2O+Qxz81UpWOWghKIsYJvIomTBXCCFcxTBipH9hc1CuM6/G9rjb8an1Bdxo+RWX+ahNcYZpjyb7Ot3knJcvFZ5Dz3+wBi7adJV5HVKoDlPMqwAA91gWqNr3iSZ1leF88WzMh7LrLjVtUt0ZZxT+apavN86eMtNTUNTBSERmIsoEUAZgmRBik49tphJRBhFllJeHXuf1FDqAxyyf4RyT09seYzrk3I90e7qHHr60PosTKF9x22bYoeY270v+qwCG21tTYvmJlOcxSOYd6+u4xfyz6n310bholVadk33QgBi0Q4jAR98oHbMMEwqKxFoIYRdCjAUwFMBpRHSij23mCCHShRDpqampIRu2IPYJ/J9lcZflcdQGX3Ll3dEkhwXtyI27Ed9Yn8Ilps2Yb33cZ3tqSEALjiLPTAA9BcJbnnztaWHsdEyzfOGxzFfWyhsxb+JWs/yo0Z1xU2X3oZQUrzeVQPIqd+z+7Bbm2Rk3FR/EzAzBKobRHj3Hiqid3byGiFYCuARAROpIvhDzAQb7KAYUA2Wdb67Rf+NN+zHe2nW6HqUIdIrO9JjP/W57rXlNF8HyRk2H3ysxb+Mx2y2oQy+/252kILRypXkDrjRvULxvtYygEsW1SMbRfpSIfkgm3978S15Tf51rzsJWR/AVABkmmlCSDZJKRH2lz/EALgKgpESdJvye8rosu8bctZDSR9aXOj5PNf+Ec007AACfxjyPvNi/4gTKl+1US5ImilWTEucA8HXvXmhTsG26aT/STP476J6QJt5VwhXmjbjdsrDjb3A+ieUAABujSURBVKVZIsF6+/1Ri6eDHBE4zOuYk58ZDufHPon1cffg3i6DaeTt/peCvoGJ5oyA2zCM0VHiWQ8C8DERmeEU96+FEAsDfEczfIlcIFF9NMb5+n9h60ycY3a+ACyOfRQAMKblgy7bz7XOxJf283GqilzcnxMT8ExKP5RYzLinuqvXrFYYB5O6ARVKPPHTTZ7PVFf8Xy3eOcopbjORB0LveLGForPTlGHUElCshRA7AZwSBltUocSb/NHHbN3uUzy5GE0FuM68WtX+G0zOl5Jak++Xk1BDV6OoCLliiOz64VSmus3RpsKOz2eZduMu0w9dtunvI1xzmdkzAycpQIerP+LRihZYg/4+w/RUjDXcXCEWKT6dF2PBpKGDUC0jmInUNT/3DstPAdt39waDneGbQkyTWx77EJJVeLBqeTbmQ1xj/q3L8tdiZnVZNtnctdiUUrw96wRqVf0g+2/M58iKvTXgds9b3lfZMsNED1Ep1mZJCOcm9UFBTAxWJXQtrqQVE0y++1FdElRosSA3Rp9R+ynkKda5MRacNHI4fouPw8XmrZhhmYNHLP47N9XiChtphVad472pOeA2N1iUTcDAMHqhZxqvoWqDKEXrKGgvasFpbmVJ3WPi3gefhIBwy89ZnxCPyQnxyDoYeG5Bf7RKTca6/bhlsf/x2GZ7XCwAYHliAs5pbsEUy6qQ9tkTGaSyb4BhjELUeNY/JyZgueRBWyI4Ek8oTKRU+4Q9fcQwnDV8mHqDooxoqMXNMEbEUGI9nEqRH3cDxlPXzMD/DEjBfQOdg21cebh63fbzY59EHwTfiRYMdiK0mXqGkPVTOICJYaKNuevydWvbUGJ9llSH+E8+Or78oUdy2JlSvRG51LMCmTg1wYH8uBu6xJvd+aZ3Iu4fkKLKHgcAh0G8UjXesa/j93xM1/RJhukO/JqtX8EzQ8WsXTf2aJOyDAw9pet+y7dY29ZlVH0HnyX18bncrCBE83RKf9X2nDzSrWZ3VJW68DR2RgxnbDBMMBjKs3YxzpQTaRNwvKkA/7Z8HWkzDImaZ8VFpm262cEwPQlDedbBopejmUo1aHUbwHHnwMAFqk4zqRuJP7NfXwxq125SgXCgJgzyV4t8eVOGYZQT1WIdTBhEANgSF4vxLYEHZ1xu9qwE+5uCfO7fq6xL/YlXOKWJCAnC2HGOT60zIm0Cw/Q4DBUGCTYGraCkcQeLExNw66CBWNArscu6PJ0Gt6jh9eS+AIB2AD/1SojS6QIYhtEaQ4m1L/ZZY3DSSN8T4gYj7kUWpyAXegnzkoR4XDV0cEcud6RolNL3Pkrqg0dTU7DQx0OFYZieh+HFellCgsffNSaTbC0QAMj+cjDW5/RV5JEWm81okga57Ldapf9jVE4lqw+VZudv9PVbeeYThul5GFKsCyxmHJQ8X29ZOmfEUJw7Yqjf7ydnJPgMc3hz8fAhmHrUAI9lbyf3xTg3T/7GQQPxx2GDsSY+TpnxbuRb1I+1dP1eY2RUMwxjFAwl1i6PcdKwIbhy6GAAgePRp+Q4MOa9JGS1xXmESyrNZkX73CHV25AjMy4WpRYL7vISdW+8HypLEuJxxbDBuH9ACjYF2Ievthr9vD0wDNPzUDJTzDAiWklEe4hoNxHdGw7DAOCvgwbivb5JsusrzGZcsMMpk9salcd2n/+oHad95T82XWsiv+EWb77q3Qvzevfq8KT/LaX5rUhMwG2DBna0edAtVu5LxPdarXgypR/m93ZO2eUr4FFlNqNU4cOIYZjugZL0h3YADwohthFRbwBbiWiZEGKP1sac5DWF184AHunqhHiMl5l7cb81BsVmMwbZu64/phgI9Jw6e4S6okrPpfQDAPQWDhRYYnxuM2XwUSiM6Vzn6ux0Z3+sFftj/RfnX5GYgBWJCSFX+mMYJnoI6DoKIYqFENukz/UAsgHIT2ESAmpKfjZIHYMkuZ7NJgLc8pOX9ErExcOHYH9MDI5YOr1QuajKVpWhCjk2xsXhneSubwMnjRzuIdRKUZOWyDBM90VVYJSI0uCc4muTj3VTiSiDiDLKy31PTBuI9hYTKrJ7eYiuHG1epUo3J8T5FOJrhg7CxGFDcNhikZ3cNjfGgq1BdCD6wqZiLvqv+vifnRxwhkHkOkvf7pvUkc3CMEz3RvEoECLqBeA7APcJIbqUlBNCzAEwBwDS09ODyi3bldEPsYVW/LmXA9+faUJbjFOIzt/hQJsFWDem89ly4+CBnvt3++wrA+OyYYNxSUMjjmvrmpg3WerM1IKfVeRF74kN7M2/2i9Zdt07yUloIcID1TWK98kwTHSiyLMmohg4hfpzIcR8vYw5KNXhuHadwK1LOyX3jsUO3PujpwQflkIKrjCIIATMe9ugwnvuXytw4Xbjjx9sZc+aYXoESrJBCMAHALKFEK/ob5KTgdXKnHNy28zkEm6ZbWvNZmQqjE0//oUd//zFgYQWYw9AMfEAGYbpESjxrCcAuBHABUSUKf2bpIs1brrTrx4gh3IhEkBHGp93p9wpOQ58/UI7+tcJrFE4nLxPk/N/MrgWNnE+NsP0CALGrIUQaxGmAXXuOzmqBrhhtQOfn+87nzixWaAxzvM7/7fEd9jiQknEjy4WqOyjbg5Fo2djuGe6MAzTfTG0W5a+v6tbm1wv8PUL7fjwNTsmbxCwxZ0IAfIQVYtMqDmlDohtU+gqBwipGAWT0Q1kGEYTDCXWI6Xpy1acNwu5I6/AkCqnB+3OFZs6lXhM8Vg0pt6OgqHn+W3X1cLNvzrw6f88B8mQQ3RJFUxsFkhw5fkZ3LM2uHkMw2iEocTa2t75+dCISwAAL83tFNdRRwQu39IprHaLc/BJa2w/DKvw07CMos19tR1fvWjHvT94uuJPzOvcp9EdVxZrhukZGEqsfZHqltH9wse+h5YLAv62Uj7Nrn+tb8nt1eL8f0J25/r/+9mOtDL1dkYKFmuG6RkYXqz90SnkhIbEwWiJ7dux7tEv7R3hjREKB1S+9XY7Lso0ui/tSVSfQIZhFGPoe72uV2cxpeKBp2HdGc/KTta6efxjWH/mc8ge/VcIAGMPdm4Zo2A+2iEVAgNquy43ejYIGXy+RoZhtMHQYp2RPq3j897RN6A1LhmC/KeqFQ86C42JzuHjw8uAeIWDWl59z7eiv/mOHQ9/bUdSozFF0eDPEoZhNCLyM8QqoDmuP4TJR8U6yaus7+VZztRhcv6smXMVuNQB6NsEnJorcPV6Bz66yHg5zd4FrRiG6Z4Y2rN2seGMp93+6urh1vY9xuPvdnMcHBQVz6GQWRfhCX4ZhgkPhhJrmyURrdY+AbYKHI7IHHsvtp5yvzZGMQzDGABDuZ+/nf1Sl2WlA071+HvthBcxPmMG4lsqYTfLz6hS3ydNU9uOLjZmzJphmJ6BoTxrX+w+4RaPv9stCchLuxwA0GaVn58R0HZAS1KTho0xDMOoxPBi7YvSo06DzRI4Vntk0Fma7ZP9aoZhIklUijUAOExWFAaoCVKbNEqz/Q2u1qwphmEY1UStWLfE9Qu4TWuAMAnDMEy0ELVivXXcvwNu05QwMOA2DMMw0YCSab3mElEZEe0Kh0Fa0qrA+2YYhokGlHjWHwG4RGc7GIZhGD8EFGshxBoAVWGwhWEYhpFBs5g1EU0logwiyigvV1iTlGEYhlGEZmIthJgjhEgXQqSnpqZq1SzDMAyDKM4GYRiG6UmwWDMMw0QBSlL3vgCwAcBoIiokolv1N4thGIZxJ2DVPSHE9eEwhGEYhpGHwyAMwzBRAIs1wzBMFMBizTAMEwWwWDMMw0QBLNYMwzBRQLcW66Ta3EibwDAMowndWqxNDlukTWAYhtGEbi3WDMMw3QUWa4U8/jdzpE1gGKYHYzix7lVfgJOyZivaduTBn/yuN9vbtDAJAFAfeDJ1hmEY3Qg43DycXLDqro7PE9Y/ioa4VlSnXIrW2CSUDhzvse3JO95C/+ps9K/KhrWtFpvHP4Z2S0LH+pEHF2LIkd8U7bfZCsQH0PWq3sp/B8MwjNYYzrN2cfudDbjjrnYck7cAY7I/6lh+UtZszD7jPvSvzgYA9Kk/hLjWGiRX7/P4/shDP8Nqa8A7k0x47i8mvDPJ90/dPxj4592BQxzNsRT8j2EYhgkRQ4r1K5NNaIx3iuNNDziF9Hd7P0Vq+XakVO4CSGDqv8zYNqpTQAX5/il7hxJ2HG3CypNNqEnsXF6dCFz/HzOm32RBSwAh3nyccYV6WiXPuMYwPQFDhUFcbPxdp/A2xxJWnkSIbd+EdydugZjoXFfTi7DwNMK4XIH9g4E9KfMxuEmgPWFsx3drEoDi/m5CKzo/Fg0WsJs717XEAHE2YNF4wi+nmhDXBsycawcAvHyNvp2L91dV49V+yUF99691DRpbwzCMETGkZ+3NwfNa8NpkM5pjycMLLu3r/LzuBBPmn1mDty78EBVJmTiu6ksU9gem3uv5LNozwrn9wAsq8c2lwmPdu5c6D8WCM00oTSYUpjiXt4UhCSRYv90kROCNGIbpFhhSrL8tKvb4+8XySqw5VNjx97aDhwEA5X0JN91vhun45o51tmFv4rjff4cHpnYK9VCbc3DMrMtNOPrSMvQb0IrBaPfYx+gh9fjzIxbUJTqlc3ZZGb6ZQHjsJnm1/ktdfVC/7x81dR5t/KWuAfdXVWNqdS0AINHhUNTOMxW+QyD/bLsvKLsYhjEuisIgRHQJgNcBmAG8L4SYoYcxR08qg8nigN3W6TE+WlEFE4BkNwGLAfBGaTnuGZiK5jhC39bOdQSgr8Pu0e6f6xvwSr9kTK2vRWySU6SfK6/E1fUN2BNrxcqEBNxfXYv7qmuRPmIY2kyEM1pa8eAEoM5MuKWmFm1E+CypDwBgXlEJegkHRtracXtNLS4cNgQOIqS12XB9XT1eSOkHANiSX4CfExOwz2rF50m98VppOSrNZvy5vgFXNTQiK9aKqxsaAQC31NZDADjWZsP5TU04ZInB0TYb7h6YinUJ8Xiwsho/9E5EncmEMosFm/MLEC/jWS9xnIa0lnlYaH0UJ5ryQzsp3YxtjmMibQLDBEVAsSYiM4BZAC4CUAhgCxH9KITYo7UxsX2cQhojgKyDh4FTbgTOuAw46veAcIC+vxQuP/f8pmaMb25BksOBe6prkGu1YGN8PIYMPxdJFd9joyRmJgBITMWokjKc1dzSsa9EIfCHa7/AHz69GndIni4B2HCoAE0mEzBlHu6r24ensz/EntKb8J71DVxX34CY0+/CsIOvd7STYndgR34BtsbG4pTWVpgA9BYOVLYPwHUtz+DdswhX73kPtxwuwAB750PkGJsNx9g8h8MTgMHJVyO28XMcJ61zfee4NhsWFJWgiQhNJpIVancmtz2Nc0xZ+NA6E6+3X41GEYfbLT+hH3nGuVfYx+ICc2bgExQkL9uuw79jvlG0ba5jEEaZigNvCOD6tsdwq3kx/mjertiWdvDgJiY6URIGOQ1AjhAiTwjRBuBLAFfpYs0/FgNTvgBuWeL8d9VbwOhLgaQhQN9h2Hz0Tdgw8kZgWgEw8CTMFQPwalkFEqauwZx/HsBbF7yF2y7/AACQ2DcNpofzgf8cBB7KwbkPl8LyeDVsl78FESOlhYy6ALgnE7jmA+DEa4FJL8M6ZR76PlYOHH8ZrjvtAWTdlIX3/vsgAODosTdj2LnTOu3tdRTw4H58OnEHrm/8GN/3n4qFOBdXNDRhc92VqEoag8EX3Q386T2n6I77u/N7E+4Drn4X+P1fgMteAZ6sBeL6AgDqznmis/2Jz2MaUjA9Zhi+j3sGAJAgBFLsXcMkL9g8Z197/uqT0A4LVjpOwZ7rVuOep+eibtyduLLtObzffqnHtnvHPorptpuxLuFCAECTiAUA3Jj0Eea1XwAA2GA/we+py3QcDQB4t/2yLuu+sp/n8fcjtlvxr7a7O/5eaj+14/OlbTPwhO0mTGp9vmPZevsJ+Nh8DW5qexijWj7FvW134tLWF7DBMQa32R5CWss8/LH1pY7t9zmG4orWZ/GS7c9dbHnAdqff36GWX+47B7edPVLTNo1EUnwM1jx0vsey+Xeepft+TxvZL6Tvjx3WVyNLjAOJAB4aEV0L4BIhxG3S3zcCOF0IcbfXdlMBTAWA4cOHn3ro0CF9LFbCkUyg73AgQeaE21oACCBGxbDE+hIgIQUwB3gZcTiA3BU4nHwmUvvEId7q5sk57MD6N4DTb++674ocIHcFcPpUlO/fhNThxwNxSW7NCoiKAzAn9nf+LocdaCh1PsgAHK5sgsVMaGhtxzGpvWAyEZra2pFg7bTX7hBYsbcM6SOS0YeaAFsTzH0GAUTIKWvAkL7xiLeaUVDVBJvdgRH9EyGEQOnhA9heacblvXNQZbNguxiNCcOsqNi9AkNPm4yDtQ7szS/CpcMd+OZwIk507MPv4msgTrwGueWNyClrwPjBVvSPdQDWXmh0WJBf2Ygx7XsBawJ22IZBOOw4vp8J+Y1mLNhWhDvOG4Wawn1Yd6AEfzhrAgYlxeO5Rdn45x+OxsA+cThY0YjHf9iF9BH9cPcFx+BITTNsDVVIbS/G2sahOOe4VFQ2tGJQUjxWZezEjB+2oHfKULxw/QT8dqAcpx/dH73jLIiPMaNfohXbDlfjk/WH8MDFx2H3kVpsyqvCqSOSMfmUIWhqs6NPnAUZh6pxx2fbcPLQJFw5djDOPiYF/Xs5H2xFNc34JqMA5xybggG94zA0OR7NNjteX34A144bise+34Xbzh6J4toWxFvNeOmXffjj7wbg2IG9YTUTVu+vwDnHpuCJH3fj9SljUd/SjpqmNkw6aRCmf78L9154LPIqGnGkphmHKpsw4Zj+SO0di3U5lTh5WF/EWkxI6WXFnuJ6XHj8AJw1YwVevu5kNLa2w2Z3oLy+FWsOVMBmd+DYAb1w1qj+OHZgb1Q0tOKYAb2wu6gOmw5W4pnJJ6Kp1Y7pP+zCqJRE/PMPo5AYa0F2cR36JsQgpVcsYswmLNx5BKeOSMbfP9iM9LR+mDJ+GJZnl2LUgF6obmzDhb8biLU5FUiwmjFxzFHYmFeJCcekoKS2Ba8u24+rxw3B7iN1KK5pxlNXnYjG1nY88eNuJFrNuHnCSKSlJKKuxYZH5meh3e7Avy8ejSd/2o3bzjkab6/MwZb8ajw0cTROHJKET9bn46pThiC1VyysFkJJbSsu+/0gZBbU4KstBcgqqsGsG8ZhRP9EbD9cjavfXo9xw/vi5etOxqKdxTgqKQ4frsvH1HOPxvmjB+Da2etx1/nHYG1OBZ6dfCLiYsx48sfduHlCGv4wcxWmX/Y7/LjjCP5+ZhquPXUoFmwvxJsrcnDr2SNx1dgh6BUbXJIdEW0VQqTLrtdKrN1JT08XGRkZQRnMMAzTEwkk1krCIEUAhrn9PVRaxjAMw4QJJWK9BcCxRDSSiKwApgD4UV+zGIZhGHcCBleEEO1EdDeAJXCm7s0VQuzW3TKGYRimA0WRcCHEYgCLdbaFYRiGkcGQIxgZhmEYT1isGYZhogAWa4ZhmCiAxZphGCYKCDgoJqhGicoBBDuEMQVAhYbmaAXbpQ62Sx1slzq6o10jhBCpcit1EetQIKIMf6N4IgXbpQ62Sx1slzp6ol0cBmEYhokCWKwZhmGiACOK9ZxIGyAD26UOtksdbJc6epxdhotZMwzDMF0xomfNMAzDeMFizTAMEwUYRqyJ6BIi2kdEOUQ0LfA3Qt7fMCJaSUR7iGg3Ed0rLX+SiIqIKFP6N8ntO49I9u0jool62U5E+USUJe0/Q1rWj4iWEdEB6f9kaTkR0RvSvncS0Ti3dm6Stj9ARDeFaNNot2OSSUR1RHRfJI4XEc0lojIi2uW2TLPjQ0SnSsc/R/ouhWDXTCLaK+17ARH1lZanEVGz23GbHWj/cr8xSLs0O2/kLJ+8SVr+FTlLKQdr11duNuUTUWYEjpecNkT2GhNCRPwfnKVXcwEcDcAKYAeAE3Te5yAA46TPvQHsB3ACgCcB/NvH9idIdsUCGCnZa9bDdgD5AFK8lr0EYJr0eRqAF6XPkwD8DOd8u2cA2CQt7wcgT/o/WfqcrOH5KgEwIhLHC8C5AMYB2KXH8QGwWdqWpO9eGoJdFwOwSJ9fdLMrzX07r3Z87l/uNwZpl2bnDcDXAKZIn2cDuCNYu7zW/w/A4xE4XnLaENFrzCiedfgm5ZUQQhQLIbZJn+sBZAMY4ucrVwH4UgjRKoQ4CCBHsjtctl8F4GPp88cAJrst/0Q42QigLxENAjARwDIhRJUQohrAMgCXaGTLhQByhRD+RqnqdryEEGsAVPnYX8jHR1rXRwixUTjvqk/c2lJtlxBiqRCiXfpzI5wzLckSYP9yv1G1XX5Qdd4kj/ACAN9qaZfU7p8BfOGvDZ2Ol5w2RPQaM4pYDwFQ4PZ3IfwLp6YQURqAUwBskhbdLb3OzHV7dZKzUQ/bBYClRLSVnBMRA8BAIUSx9LkEwMAI2OViCjxvokgfL0C74zNE+qy1fQBwC5xelIuRRLSdiFYT0Tlu9srtX+43BosW560/gBq3B5JWx+scAKVCiANuy8J+vLy0IaLXmFHEOmIQUS8A3wG4TwhRB+AdAKMAjAVQDOerWLg5WwgxDsClAO4ionPdV0pP44jkXErxyCsBfCMtMsLx8iCSx0cOInoMQDuAz6VFxQCGCyFOAfAAgHlE1Edpexr8RsOdNy+uh6dDEPbj5UMbQmovVIwi1hGZlJeIYuA8GZ8LIeYDgBCiVAhhF0I4ALwH5+ufPxs1t10IUST9XwZggWRDqfT65Hr1Kwu3XRKXAtgmhCiVbIz48ZLQ6vgUwTNUEbJ9RPQPAJcD+Kt0k0MKM1RKn7fCGQ8+LsD+5X6jajQ8b5VwvvZbvJYHjdTWnwB85WZvWI+XL23w0154rjElAXe9/8E5vVgenB0ars6LMTrvk+CMFb3mtXyQ2+f74YzfAcAYeHa85MHZ6aKp7QASAfR2+7wezljzTHh2brwkfb4Mnp0bm0Vn58ZBODs2kqXP/TQ4bl8CuDnSxwteHU5aHh907fyZFIJdlwDYAyDVa7tUAGbp89Fw3qx+9y/3G4O0S7PzBudblnsH453B2uV2zFZH6nhBXhsieo3pJoZB3HyT4Ox1zQXwWBj2dzacrzE7AWRK/yYB+BRAlrT8R6+L+jHJvn1w673V0nbpQtwh/dvtag/O2OByAAcA/Op20gnALGnfWQDS3dq6Bc4Oohy4CWwItiXC6UkluS0L+/GC8/W4GIANznjfrVoeHwDpAHZJ33kL0kjfIO3KgTNu6brGZkvbXiOd30wA2wBcEWj/cr8xSLs0O2/SNbtZ+q3fAIgN1i5p+UcAbvfaNpzHS04bInqN8XBzhmGYKMAoMWuGYRjGDyzWDMMwUQCLNcMwTBTAYs0wDBMFsFgzDMNEASzWDMMwUQCLNcMwTBTw/xKBiIcnfEU6AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "model = TextClassifier(x_size, 0, y_size).to(device)\n",
    "params['batch_size'] = 1\n",
    "losses = train(model, train_dataset, 1, 0.1, 2)\n",
    "plt.plot(losses);\n",
    "params['batch_size'] = 2\n",
    "losses = train(model, train_dataset, 1, 0.1, 2)\n",
    "plt.plot(losses);\n",
    "params['batch_size'] = 4\n",
    "losses = train(model, train_dataset, 1, 0.1, 2)\n",
    "plt.plot(losses);\n",
    "params['batch_size'] = 8\n",
    "losses = train(model, train_dataset, 1, 0.1, 2)\n",
    "plt.plot(losses);\n",
    "params['batch_size'] = 16\n",
    "losses = train(model, train_dataset, 1, 0.2, 2)\n",
    "plt.plot(losses);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3149b944b4b14174a3ca4fc8936f16bd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=40), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=625), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=40), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch:  0 Accuracy: 0.709 Loss:0.787\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=625), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=40), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch:  1 Accuracy: 0.710 Loss:0.762\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=625), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=40), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch:  2 Accuracy: 0.713 Loss:0.740\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=625), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=40), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch:  3 Accuracy: 0.717 Loss:0.720\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=625), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=40), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch:  4 Accuracy: 0.716 Loss:0.702\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=625), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=40), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch:  5 Accuracy: 0.718 Loss:0.685\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=625), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=40), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch:  6 Accuracy: 0.721 Loss:0.670\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=625), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=40), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch:  7 Accuracy: 0.722 Loss:0.655\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=625), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=40), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch:  8 Accuracy: 0.724 Loss:0.642\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=625), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=40), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch:  9 Accuracy: 0.722 Loss:0.630\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=625), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=40), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 10 Accuracy: 0.722 Loss:0.619\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=625), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=40), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 11 Accuracy: 0.724 Loss:0.608\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=625), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=40), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 12 Accuracy: 0.725 Loss:0.599\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=625), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=40), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 13 Accuracy: 0.725 Loss:0.590\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=625), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=40), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 14 Accuracy: 0.724 Loss:0.581\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=625), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=40), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 15 Accuracy: 0.726 Loss:0.573\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=625), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=40), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 16 Accuracy: 0.725 Loss:0.566\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=625), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=40), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 17 Accuracy: 0.728 Loss:0.559\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=625), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=40), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 18 Accuracy: 0.728 Loss:0.553\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=625), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=40), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 19 Accuracy: 0.729 Loss:0.547\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=625), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=40), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 20 Accuracy: 0.729 Loss:0.542\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=625), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=40), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 21 Accuracy: 0.727 Loss:0.537\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=625), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=40), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 22 Accuracy: 0.728 Loss:0.532\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=625), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=40), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 23 Accuracy: 0.731 Loss:0.527\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=625), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=40), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 24 Accuracy: 0.730 Loss:0.523\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=625), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=40), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 25 Accuracy: 0.729 Loss:0.519\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=625), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=40), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 26 Accuracy: 0.730 Loss:0.516\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=625), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=40), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 27 Accuracy: 0.730 Loss:0.513\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=625), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=40), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 28 Accuracy: 0.732 Loss:0.510\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=625), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=40), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 29 Accuracy: 0.731 Loss:0.507\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=625), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=40), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 30 Accuracy: 0.732 Loss:0.505\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=625), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=40), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 31 Accuracy: 0.733 Loss:0.502\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=625), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=40), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 32 Accuracy: 0.732 Loss:0.500\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=625), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=40), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 33 Accuracy: 0.733 Loss:0.499\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=625), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=40), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 34 Accuracy: 0.732 Loss:0.497\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=625), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=40), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 35 Accuracy: 0.733 Loss:0.496\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=625), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=40), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 36 Accuracy: 0.734 Loss:0.495\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=625), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=40), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 37 Accuracy: 0.734 Loss:0.494\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=625), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=40), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 38 Accuracy: 0.734 Loss:0.493\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=625), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=40), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 39 Accuracy: 0.734 Loss:0.493\n",
      "\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD5CAYAAAA3Os7hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3dd3wUZf4H8M83IRBKqKEoxVBFBKSELopKFRUsp+jZC/fTs53lDht62BDPehZERU/vBBvYAEEQRDqhdwg9EUhoIZSElOf3x+6Gze7szuzuzM7u5PN+vXixOzs7+8zu5rvPPOX7iFIKREQU/xLsLgAREZmDAZ2IyCEY0ImIHIIBnYjIIRjQiYgcggGdiMghKuntICITAVwBIEcp1T7Ift0ALAYwQin1jd5xU1NTVVpaWghFJSKiFStWHFRK1dd6TDegA/gUwDsAPgu0g4gkAngFwCyjhUpLS0NGRobR3YmICICI7A70mG6Ti1JqPoDDOrs9AOBbADmhFY2IiMwScRu6iDQGcDWA9yMvDhERhcuMTtE3AfxDKVWqt6OIjBSRDBHJyM3NNeGliYjIw0gbup50AJNFBABSAVwuIsVKqe98d1RKTQAwAQDS09OZRIaIyEQRB3SlVHPPbRH5FMBPWsGciIisZWTY4iQA/QCkikgWgGcBJAGAUmq8paUjIiLDdAO6UupGowdTSt0eUWmIiChsjpgpqpTCNyuyUFBUYndRiIhs44iA/tvWXDz29RqMnbHZ7qIQEdnGEQE9v6AYAJB7vNDmkhAR2ccRAZ2IiBjQiYgcgwGdiMghHB/QlVLY8Eee3cUgIrKc4wP61FXZGPr2AszcsN/uohARWcrxAX3LgXwAwI7cEzaXhIjIWo4P6EREFQUDOhGRQ5iRPtc2w99diLNrJ2NI+7PsLgoRke3iuoa+eu9RTF+3H3mnilwbmGGdiCqwuA7oHlv254f93MLiElw/fjFW7z1qYomIiKLPEQE9EtsOHMeyXYfx1NR1dheFiCgiFT6gExE5RYUJ6IoN7ETkcBUmoBMROV1cDlv8bPEuNK5dNaTnCMSawhARxYi4DOijv99gdxGIiGIOm1zcFJvYiSjOOSqgT1u3D4qRmYgqKEcFdAAoLmVAJ6KKyREBXdjfGXeu+PfvGPTGfLuLQeQocdkpGg6OQ48t67OP2V0EIsfRraGLyEQRyRGR9QEe/7OIrBWRdSKySEQuML+Y4eNwRSKqKIw0uXwKYHCQx3cCuFgp1QHA8wAmmFCuqGP9nYjinW6Ti1JqvoikBXl8kdfdJQCaRF6s6GH7OxE5hdmdoncBmBHoQREZKSIZIpKRm5tr8kv7Ky4pxaHjhZa/TrgKi0s4zJKITGNaQBeRS+AK6P8ItI9SaoJSKl0plV6/fn2zXrp8ObxuP/P9Bny9Isu9Pbaq4vvyTuHcp3/Gf5fuCbpfZs5x7Dl0MkqlIqJ4ZkpAF5GOAD4CMEwpdciMY5phxvp9dhchoF0HXUH6pzV/BN2v/+u/4aJX50ajSEQU5yIO6CLSDMAUALcopbZGXqTQebdasAGDiCoq3U5REZkEoB+AVBHJAvAsgCQAUEqNBzAaQD0A74mrh7FYKZVuVYHN8Pu2XKzPPoZ7+7Us28a2bCKKd0ZGudyo8/jdAO42rUQW8Z5YdMvHywAA9/ZrGXNt62ZYn52HGlUqIS21ut1FIaIocvTUf+eFamOu+PcC9PvXPLuLYbmpq7KQNmoa9uWdsrsoRDHBEQHdW6AgvjP3RFTLEaniklKcOl1idzFi2rcrsgG4RgIRkQMDeiCe4YuBbN6fj7xTRWEfP/voKRQUGQ/Aerll7vvfSpw3+uewy0NEFY8jAvrxwmJTjrNlf37Yz+0z9lfc81lGyM8L1Fw0a+OBsMtCRBWTI7ItTlmZ7bftv0t248hJ/Rq3mVP/f9920NB+ny/ehed/2mTeCxMRwSEBXcvT32kmh/RjRy6XZ7gmKhFZwBFNLt6sGE3+ycKdSBs1DXsPn0RxSWlEx5rtwKaUrQfysWRHzEwQJqqwHBfQrfD6L64JsH3HzcVL0zdHdKy7w2hnj3UD35iPEROW2F0MogqPAd2A/IIzna7ztubgwLECKKWQm1+IF6dtREkE65g6cWITEdnDsW3o4TDSnr4j9wR6vDQHz17ZDou3H8KsjQfQu2Vq2eN7D59E07rVTH1NIiIjWEP3si4rD/kFxsaiL8w8hCJ3e7r3mPK+42IvM2JhcQlem7UlpHHyRBR/GNC9jPlpI+74ZLmhfWdvip/Ozc8W7ca/f83EhPk77C4KEVnIkQH9penGx3j7tmGv3HMk5NfbtC/8CUnRUFhcUu5/PRMXuEb12JV64JaPl+IHnTzxROTPkQE9kppoON2br87cEvbrFZfEXtreD393vX9HTp625fV/33YQD05aZctrE8WzuOsU/W1r+GuRztuSU27ESizYciC2a/dEFD/irob+8/r9QR/fdiBw5r3bP1mOB7xqfu/8ug1rs46W20cpRDx56Ikpaw3va3SUy/JdhzW3ny4uRU5+geHXIyLniruArmf8b9sN7/uvWVvx+Df+wbf9czMjKsOkZXsjer6WP41frLn98W/WoPuLc8pG3Jgp9hqDiCgYxwV0MxQUuYLj+uw8dIgwuOvxXfnu4PHCkJ4/w33FEsnkJl92Do2ftGyPja9OFN/iLqDrNVGYWav8eMHOqLa5r957FOkvzMaUlcFztzvZE1PWRe21lFIY9MZ8fL/aP1snUTyKu4CuZ1GmsRS2ZtD78cgvKNJdNMP7B2rL/mMAwERXUaKUq1P64S9X210UIlPE3SiXHbnBlxs7dMKcoXb/W7o74mN0eG4WAOCtEZ0wsF0jVK2c6LdPNJs3fJt39PdnKzpRPIm7GnrWkegsCPzUVP186sGC8QmvVZQemrwaz0/baEKpwiMa7VS5+YHb6rX213LoeGHIbf7hKigqwZNT1+GoTWPjieJB3AX0WDJ3S+Ax8e/PKz/a5kCeRUMLw6hEz9uSg24vzsbczTkRvXTXF2Yj/YXZER3DqCkrs/HF0j0YF8EkLiKnY0APIpImh6JS84cRBhNK1sbVe11j71ftPaqzZ+zwJEBjKxBRYLoBXUQmikiOiGi2QYjL2yKSKSJrRaSL+cWMP7/4rEw0Z3MOvlvlP5riyMkiU7MgbnZ3rHqsz87TTS9w6Hgh9lt1BUHl5BwrwNtztrF/gixhpIb+KYDBQR4fAqC1+99IAO9HXqzYYLQtWcuO3BN+2wKNplix21hCsB25x5E2alrQZewGv/l72e1tB/Jxxb8X4I3ZW4Met+sLs9Hz5Tl+2xlzzPfQ5NV4/ZetWJedZ3dRHKukVCHryEm7i2EL3YCulJoPQHveucswAJ8plyUAaovIWWYVsCL6KkN7HLqnqWTaun0oKinVrdnn+nRYvjdve9xmMbTix8WO36uT7s/MzIlgVN64nzfjwlfmYl9edAZQxBIzhi02BuA91z3LvW2fCce2lRXT6c1yxdsLsOVAPionun6TX56+2a/Wd9OHS/2e9+CkVXi4f2vNYxYUldiWYdEoz0XT1gP5WGDSnAM7ZsYynFvn922u78Wh46dxVq2qNpcmuqLaKSoiI0UkQ0QycnPDz5oYLT+tjd3fJN8sjZ8u2mW46SaQByatQq+Xf43oGL5+3XwAvV+egz+Ohldbuv2TZZim8Tn8d4mxeQKLtx/Cff9bETNt1lxxkKxkRkDPBtDU634T9zY/SqkJSql0pVR6/fr1w3qxGPm7NJXRc/LsN9Wrc1WZWNfz7cgN5JGvVuNlg4uI3PlpBv7IK8DwdxeGVaZ5W3Lx1y9WBt0n2ELbd3y6DNPX7S/LzwO4ZuL+tNbepqdY/B4rpWLmh4/CY0ZA/wHAre7RLj0B5CmlYrdqS7r0+oKnrMzGBzqLiJwuLsXYGZvL7ucEmcgUbSMmLMH9X9izgEYsLwo+5qeNaP7EdLuLQRHQbUMXkUkA+gFIFZEsAM8CSAIApdR4ANMBXA4gE8BJAHdYVViniuU/8lDN3ngAtaslITPneEipjPVYXW88cuI0tuceR3paXYtfySP2asKfLNxldxEoQroBXSl1o87jCsBfTStRBVZYHLwT9nihOZkfPdP1I13IQ8vdn2UAAJ4f3t70YwPWtUHf+OESbN6fj11jh1r0Ci4O+u2OWU6qIIWKM0VjgGfx5j+OBp/cs3m/OcvVTXYvwPHevO04opHMzNOM6t2cOmVlFtJGTdPsoIwVszceQM6x8u+h0SZhs95bslZmTj5mbgi+allFxoAeAzwzOfVqFodPmNMO7R3j+rziP6olW2NEyke/7wQA3Q7KaPF9q0pLFe7+LAM3TFiivb/Ge2tnB6DWS6/YfZidkjr6vz4ff/l8hd3FiFlxF9C1go1TLN8ZbP4WMHOD/yiUSP/+T54OPDnpZNGZJh47wky5mpjOiXoe3X3If4aunkhmBJv1Wr9uPoBr31+M/yzaFbWykPmyj54ql2k12uIuoDuRiCBj12FkhDGOvNjCGYef2txJplUTi5X20fyCIqSNmoZvV4S3upTvp+ZJC71dI2UEhceOi50+Y3/Fte8vCvj4oeOF2HvYurQEDOgx4roAi0DbyfvHwkhTwLifzwxTjJG4a9mVhedKcYLO8E1fVr0va7OOIm3UNK52Bf0f/Xlbcixd2SxYf0yPl+ag77i5lr02A3oFZDSPyDch1j7fm6c/TFEphZOn7bskjRV/Gr8YeSddyxN+vmQ3Rn+/IaLjLcx0BfK5WyLLcV8R3P7Jctz0kX9aDG9tnpqBIW/9HnSfcFh5RQ0woFOUfbxgJ9qNnhnxcXzboj1XEKUK5ZKWndYZChpt3sVeutMVhJ/5Tn91LIqu0yWl2LTvmP6OMYYBPQbc4x67XRFMX1d+2OMag4tshFKvafvMzyHsba0DxwpCGu8fK30EFJ8Y0MkQszqYfA8zLMQcL1p5W8LNZ2NlWyYAHD15Gj1emoMXphnLexPI7kMnDF9pfPDbDvQZa26CtXil972YtyUHaaOmhTUyKlYxoJMlAtU0Pe3GofKt2YdKqzz73Ks0WVUpzjvlOtdfA6zdqgDNVay8HT15Ghe/Og9PTV1n+HWNDO09cuK0paMt7BQsWZs3T5K7VXviZylGPQzoZIkXftKule44GF5taMmO4GP0rVRaqnSD34rdh7EswDyCPYdPYtF27VEVgVax8vCke1i03X/0yqLMg2GPW79o3FzLr1ACKSlVXODDIgzoZEhxiItenzJxnVQjrAwP7/+2HX3HzUVmjv9wtILiEhSVlOLa9xfj+g8CDz31LDbiXXs8XuA/2se7bqnX9n7TR0vx7A/hjY7Jt3HyS6d/zkLvsf5LHkabmePUV+05glkxkJKAAZ0MMWPCS06+dq6aN2dvxWGNnDJa7Og09Izt1hqWufvQSVz1Tni53h/9ek3Ax5btPIxWT83A8l2uWr+nGWXL/ny88+u2sF4vVuQXFuPAsdhJp2yGq99bhJExkJKAAZ2i5u052oHozdnb0OX5X7Aj97jhY3kH9ls+XoY9Ok0iwdpVjf5ITFmp3d7tPbyt24uzg6/bavC1PMvrLcos39Ry9XsL8a9ZW2NiOObqvUcx6I35lswrKCgqwdXvLQw4CmrR9oPY+EfwYYV6NXAnjihiQKeo+e+SPUEfN9J8UKoU3pq9za9zdbFGG7NRRSXGr70HvP4b+oz9NWCwyM0vxOjvrRlXnl9QVJZ7x+gVjRHbc4+HlSPpxWkbseVAPtZnmzde+7Gv12Dqqixs+OMYVu05iud+1P5O3PThUlz+tvbEH6OB2ol50BjQSVdBlNrDDx3XD1Lztx7EG7O3YorO6BBfQ96aj9kGl9i757MMtHxSe+WebTnGg5/vrMAF2w5itc64e98JU953Ozw3q+x2z5fNa4O+7LXfDA91LCwuwZgfN+JYgfZopYKiElz3/iLD8wt8fbMiC3/7MnBTlBWsqKkbXc7RbAzoFNSxgqKyBTGsttHAzLyiAB2FepWt7bkn8Pg3+oHi0PFC/LLxQLlRGOFkY9x2IB+XvfZbuW03f7zUsqaSaDUfTFmZjYkLd+K5AKkKNu07hozdRzA6zM5aX9EYUmhFTd2uyYIM6BRUx+dm4cc1sbOohWfsuK/nf9qo+9xSpb9f1xdm+20LNUf50ZNFAZsDrJJjYSdj3smisvfAc9UxZVV2zA89jO3SWYMBnXTN35prdxF0Gan55p0qwscLdpryeht0OuRCaZeP1M/r92HiQnPOy9fOgydwwZhZuH+S/6LaK92151hblCPUixUndY4yoJOu0hj7g402rSaXtVnWNAXsd1+BrNrjyo2/97B+e32gmajhUkrhpembsGnfMew86Bp5NG3tPmz4Iy+sWbVjftyItFHTTC2jt7VZRyP6UbHq6z1zw/6oj01nQCdd0eoUjTWLMg8ibdQ0bM8xPpwyUj+7A8Dv21zDFhcbyG/+VUZ4i2wEcuxUMSbM34EbfCZKHTWQtuGWj5e5bnhFSe+rhz8Mj6YxFmV/2XgAV72zEF8u36u7b1FJacgpobVk7DoccE6Ft798viLqY9MZ0EnXmqw8u4sQsmHvLMC6CMv96qwtALRzo1SEi5ZjBcWYvq58DVOreeKGCUswdsZm7Ms7VZaqIJDeJicO2+VOJZFp4Ef3nV8z8ZjGZK5Qm1yuG78YV/57QWhPihIGdHKkNVl5uPKdyP7o7Era9FWGfm3TSt5ZCo3WaCcu3Bl0pZ7QRd6w7dsMkxvBaK3C4hIcPXlmWO2BY4XY8EeeoavXzfuPYeWe0JeXDAcDOlGM+fs3a+0uAvm4feJydBrzS7ltQ99egKem6k8iG/zm77jmvcDrjJqJAZ0oDOHmYI93RlPTagl3spERMzfuxz98fgh9O7OnBkjdoJfxEgjcl/HtyixLlqoLl6GALiKDRWSLiGSKyCiNx5uJyFwRWSUia0XkcvOLShQ7jIw+iTUjJizGE1OM51UPhZEw77uYyTPfrY84J7vnh3Xv4VP4MmMv1mdr95s8MWVdwAygwfpD2jw9A+/OzQxahlhaqk43oItIIoB3AQwB0A7AjSLSzme3pwF8pZTqDGAEgPfMLihRLFm5OzptomZasuMwJi3bgw7PnlnT9dr3Q2sKMHPM9udLduPByf7j2yNR6DUfYca6fWUpiCct888jZOQa63RxKV6ducWUskVj7VgjNfTuADKVUjuUUqcBTAYwzGcfBaCm+3YtAEHSzRHFPzvziRux32tGbdaRkzhw7Mx977Kv0PhhKigKPT2B+ZNzIm/S+mD+Ds2Ux1q27M/H9hCyfYbj8yW7LT0+AFQysE9jAN7d7lkAevjs8xyAWSLyAIDqAPprHUhERgIYCQDNmjULtaxEZND23ONoVCsZAHDhK6GtTBQoPYJS/gnHYpLXr0uwdnvv36BBb84HAOwaOxRZR+J3aT4jAd2IGwF8qpR6TUR6AfhcRNorpcr91CulJgCYAADp6elx8M0gqnj2BghoT323DrsPaT+m1Q79wk8bdYfrnTqtP+yvuKQUew6f1Lya0JsPMCfILFqtp36/OhsPTdbvJI1VRgJ6NoCmXvebuLd5uwvAYABQSi0WkWQAqQDMnZNMRJZbG2BCVqBgDpRvuwZczTYfGcibE3js+pn6c6unZqBqUqLBZQ39w3T/13/T2E/bzBhYRi4SRtrQlwNoLSLNRaQyXJ2eP/jsswfAZQAgIucBSAYQ+xmdiMg03hN58k7ppwkIRSRr1AaaRfqjxspSvjNj441uQFdKFQO4H8BMAJvgGs2yQUTGiMhV7t0eBXCPiKwBMAnA7SrWUrARVTBFJaVRzcOT77Xo9f5j+rlOPAqLS5A2ahommpIJ00GpE8NgqA1dKTUdwHSfbaO9bm8E0MfcohFRuD78fQf+/NHSqL7mi9M2hfU8T23ee0RKqMMpPUpKVYUO6WZ1ihJRDJm3Jfotnjn54eVKyTpi3iSt630yRJrNrqXljOLUfyKy1YPuxTPCGcs+2UDaXDNpZWuMJQzoRGQrTw09nKaSne70ueTCgE5EMcFJS8HZhQGdiGJCJJkcyYUBnYhiQihDHe0S61cRDOhERAbF+uwaBnQiIoPMngFrNgZ0IqIoW7T9oCXHZUAnIoqyQCsrRYoBnYgoyqxqi2dAJyKKMqv6VhnQiYiijDV0IiKHUBbV0RnQiYiijDV0IiIKigGdiCjKSkvZ5EJE5Ai+i2qbhQGdiCjKvludbclxGdCJiKLsNGvoRETOYFETOgM6EZFTMKATETkEAzoRUZRZtfKRoYAuIoNFZIuIZIrIqAD7XC8iG0Vkg4h8YW4xiYicw6qZopX0dhCRRADvAhgAIAvAchH5QSm10Wuf1gCeANBHKXVERBpYU1wiIiewb2JRdwCZSqkdSqnTACYDGOazzz0A3lVKHQEApVSOucUkIiI9RgJ6YwB7ve5nubd5awOgjYgsFJElIjLYrAISETmPNY3ouk0uIRynNYB+AJoAmC8iHZRSR713EpGRAEYCQLNmzUx6aSKieGNfk0s2gKZe95u4t3nLAvCDUqpIKbUTwFa4Anw5SqkJSql0pVR6/fr1wypwh8a1wnoeEZHTGQnoywG0FpHmIlIZwAgAP/js8x1ctXOISCpcTTA7TCxnmccGnWvFYYmI4p5uQFdKFQO4H8BMAJsAfKWU2iAiY0TkKvduMwEcEpGNAOYCeFwpdciKAldNSrTisEREUWPV1H9DbehKqekApvtsG+11WwF4xP2PiIhswJmiREQOwYBOROQQDOhERFGmLJr7z4BORBRlR04WWXJcBnQiIodgQCcicoi4C+gJFuURJiKKd3EX0GtXS7K7CEREMSnuAnqrBil2F4GIKCbFXUAnIiJtDOhERA7BgE5E5BAVKqC3bcT2dyJyrrgM6Mueugx1whjt8vPDF1lQGiKi2BCXAb1BSjJWjR6I7//aJ+h+f+raJEolIiKyX1wGdI/UlCoAgLNrJePzu7r7PX7ZeQ2jXSQiItvEdUD31rd1+TVKm6dWR3KS/+nd0SctSiUiIoouxwR0AGjVoEbZ7e/u64OL2/gvRP3sledHs0hERFET1wHdN6fwqMFtAQAXtamPWtWSICKY8VBfO4pGRBR1cR3QPURcGbuqVnYtIJ1c6cxpJRtcVDqliqHlVYmIYpYjArpHrxb18HD/1nj5mg6Gn7N69AAAwB0XNg+637s3dSk3jr1hzSrhFZKIyCKOCugJCYKH+7dBvRpngq1W4P1zj2Zlt2tXq4wN/xyEhy9rHfTYQzueVe5+i9QaAfYkIrKHowK6lmqV/ZtSRg1pW+5+9SqVkJAgaN3AeJDu1Kx2xGUjIjJTXAf0RPdqFzVCbP9OSQ4+y/TJy9sGfRwAHh3QBo1rVw3pdS9slYpeLeqF9BwiIqPiOqCfVasqnh56Hibe0S3k56bWqIwh7RuV2xbKOtyVEv3fukcGtAn6nP/e3QOTRvYM4VWIiIwzFNBFZLCIbBGRTBEZFWS/a0VEiUi6eUUM7u6+LUKuKQNAxtMD8P7NXTUfa2Ww6eWDW7pieKezy+7f07cFrunSOOSyEBGZQTegi0gigHcBDAHQDsCNItJOY78UAA8BWGp2IaOtaZ1q2Pz8YN392jeuhTdHdC67X7VyIl6/vlPZ/cpewyfvv6SVuYUkIvJhpIbeHUCmUmqHUuo0gMkAhmns9zyAVwAUmFg+Uzx3ZbtywTUQ74lKWuPXPePdB50feo6YxwadG/JzgtFKa0BEFZuRqNAYwF6v+1nubWVEpAuApkqpacEOJCIjRSRDRDJyc3NDLmy4bu/THFtfGGJ4f3fc9uMJ+A/qDHH0eMjgfuFICFRIIqqwIq7miUgCgNcBPKq3r1JqglIqXSmVXr++f56VWONpm09JLj+KRmAsmF7atoGh/RaOutRv2yc6Hb16HbBEVPEYCejZAJp63W/i3uaRAqA9gHkisgtATwA/RLNj1CzNU6sDAKpUcjW3THvwQrz2pwsw+5GLgz7vjj5p6Ns61W/7eWfVNPS6Wp26l5zr/2PgfWVQq2roC3wQkbMZGcC9HEBrEWkOVyAfAeAmz4NKqTwAZdFMROYBeEwplWFuUa33xg2dsGznYTStWw2AaxbptQYWyYhWBsdHBrTB23O2ReW1iCj+6NbQlVLFAO4HMBPAJgBfKaU2iMgYEbnK6gJGU0pyUtwsiiFsQyciH4amWCqlpgOY7rNtdIB9+0VeLGfr2aIu1uzNw6miEruLgsqJCThdUmp3MYjIBBz7ZoPJI3thk4Fx7kZV1pi1CgDnNkzR3A4AN6S7ukX6tzPWcRuJqgZTGBNRZBjQLfbQZa1xncWLVbeoX11z+yCf1AbelDvRQTuDHbeRmHJfb8tfgyieGJ2NHioGdIv9bUAb/OtPF9jy2rf1OqfstmcEj0eNKq5RMtFoi9d6iZ8f5kpSRGbjMj0WqJQgKC7VT/X1wvD2ZYH2l79dhD2HT+Ls2lWRlKgfZL33qJ9SBZv35/vt450Xfu5j/ZA2yjXva9y1HbHj4And10hKFBSVuM7j6s6NMXVVts4zjGvbyPorAzN8dmd33Dpxmd3FIIexqhrFgG6Bnx68EPO36s+EvbnnmRp064YpaO3T5v300PPw2qyt+PSObpp53X9+uC+yDp/C1yv2+j2mx9Pk4l177t2yHhZtP1R2f9uLl6PVk9MN/TgF07ROtXL361WvHHT/lvWrY3uu/g9ONBhdwpAoFJH9RQXGJhcLtG1UEyMvahnxce7u2wKbnh+MHi3qoUOTWgCAOtXOTChq26gm+rdraHjmqkfvVvXKvlHRSCFQvUqlcqtEtQnSWbtr7FDc3if4coBEpI0BPc7005hBqse347NJnWoodeelsXM0+/ibu+LbezU6TJVV9Rd/F7byn+FLFK8Y0EPw5ohOGNrxLLRpaP96ot4V60HtA0+GmnJfb3xxTw8AwK+PXoyPb3NlZPDETL0Kuu/jr1zbAV/c3SPoc54fFnzmrKe5Z3D7Ruh6Tp3gBQjCSIpjPUaycDoBFzWvGCrGt9kkbRvVxLs3ddFcrchOV3dugi0vaAe3Ls3qoHY1V5t1i/o1ymbC9nHnnokkoGrJeLo/eok3se4AABBDSURBVLU8s8yeVsDUa5I3Wj9PTkos15QTzvBIpXM1YLRF6uVrOuDNGzrp72iDs2slo09LXonEEr3vXbhiKzJR2DwJxYy65NwG2DhmELqeU9fUcqTWKF8T/OH+PgCAvq31s2t6rh5C8eLVHcoSlTWvd2Zo5vKn+ht6fqA/q1ev64h7+7VE12aBf/D+edX5Zbnxb+zeDMM7N8ausUMN/whES4OaySHtv2vsUItKQlbjKBcHSUmuhPyCYsP7a42cCaR6FdcPhtYPR9WkRHx7b28cPF7o91iye//BQSY5eXiuHsKtvHgH0vopVTB5ZE+sz87DC9M2hXSc/9zZHRe30f8Buq13Gm7rneZfDpg7imHw+Y3w84b9YT+/QQqbW2JNpQRr6tKsoceZ1u72+0a1/Gtdsx+5OKJZmZd3OMtv21/dS+f9Y3Bb/H3wubjygrP99hEB2p1dExcZCIKAdZebvnq2qIf6XsFM673RKoqRYB4O36sXo+6/9MzyhR/dGndZqeOO0QVsIlGzqjV1aQb0OPN/F7XEN//XC7012kQb1kxGF3cTQa8W9fwe1/PnHs2w+fnBePDSVmWdaA/3b4NdY4ciJTkJ9/VrhcQE8Rsa4xsUvcdua4XucOJ5s7rVAj725ohO6NKsNlKS/XPEe5eli1fzyYhuTQOWL9Z459Xv364hFj/hvyBKML5DU1OqhBZMWgZILeFUoSweM+7ajmW3U2sEn1/hzao6DQN6nElIEKSnBW/3Xv5Uf90Vj7zVrV4ZD17aCiKC5KREPDLwXCx90lgbtJYmdQIHX0A/iHrX4Csl6DdIX3JuA0y5r4/rx8bHAJ90yP+9qwfmPHpxuZo7YLxzuHpl/b6KQOkUGtcOrS0bAF68ur3fedWpZixw1K6WhFt6noMxw31GHWkUr1ua9vn/rX8b9He/h6OGtEVH93yIQC7voN+05ivQzGijx0qrF/z75uvxMNb31XqNXWOH4vpuTTX29tfaotwtvhjQHah+SpWQZjiufGYAHhlo/EteV2emJxD6H1kgMx7qi8vaNsDdfV2TjTTHrQeR4BMML2ydipb1a+DmnuegQ+NaZTV1K3xwS9dy9z+5o3vQ/cff3MXQcasYHGpZKUHw/PD2aJCi/0MS6PvyUP/W5X6Av7inZ9DjvH59J1zcpj7GDDs/4HfAd0nHF4d3wGMDy9eK5z7WDw9dZqymHGo+Ik8zYijmPX6J7j5aV83N6lbDh7emY8ZD5XMXcaYoxYxQ8rCE017u/YzWDVPw8e3dcEvPc/D73y9B13PqGF6rNZiGNZPx4wMXlqupf3JHN9yu0ckZqpev7oB61StjYLvyVwfBfggTBLpXXh5hJVQzYeSNAKih01yTnJSI/9zZHbf2SsOg8/Vr2M1Tq6N/u4a4t18rvDWiU7nt3rXa6pUTcdeF/jOI2zYKPOvYiLdGdMKvjwZfYtKoV//UUXP7gHYNozbUmaNcyBLBgk6nprXDOp5nacCPbk0PWMN57sp2fk0+3knGfJVNsIKr6UZrLVffcui5vlvTsktxo3lpdrysPVQw0Hv11ohOOFFYgienrgt4zHrVz/xY6aWH8P3d7dWiHhbvOKS9s0FGfsrnPtav7LZvDdf76mp458Z45op26NmiHo4XFuFvX64BADw68Fy8OG1j2GU8/+yaaFG/Bra/dDlaPjld/wlBhDJ0uLZFawKzhk4ROydI84r3H7VnwQ29/PB6lfqEBNFsLweA2/s0R3+fmvGq0QOx5tmBQY9ptNLb/7zQrg5+eiD8NMFp9arh/LO126yHdWqMm7wmVXmrUy0J467riP/ceaaJp5pO27+nXdmzGMkX9/TAjpcuB3CmE7Wa+/+NYwZp5uBf/89BQV/DI5yLBc9cgwHtGuLqzuGvL+BZ2MW3ozfQ98moQIMQhgQYrmtVSm3W0CkiH92ajgs0apHB/jyiPfEmWDNBKE1Cm8YMxvJdh/Hd6j/Qt7WxmZdVfQLpt/f2QuXERFz5zgLd5xoJMq0a1EBmznEAwEVt6mP+1lw8MeQ8XJ9evm/A+1haRz3LPQy2uvu9EpGyz2nkxS1QtXIibnRfdVSrXAlJGuOo9ZpjwjFm2PmYv/VgwKGEwT6/KpUSUFjsWl7xhvSm+DJjLzo1c31Xx9/cFR8v2InmqZF3Vq59bmDZfIvOzWpj1Z6jAIDpD/bFuRpNQs1Tq6OOgX6ocLCGThHp366h34gRPXoxNNLaUig8RQnWJPHpHd1wQ3pTv+Acjq7n1C3LnGmGqV5j6xsG+RzOP/tMv4dvs1Hd6pXL3gett75KpUTc3bdFuXbgvxkY2jeiW1PUrpaEN2/ohHdu6qy7v3cmUY9be6Xho9vS/Tpt+3uNXgrUDLb4icvw+vUXYOmTl5XN3/D8cLVumIKx13YM67s2/uYu5WY110xOKktxMXlkz7KRULWrJWke38pvNwM6heWVazvgx/svDPj40I6uSUp1vYbYBaqZ+66JeoO7Jmh01IcpgvyV9Tu3AV65ztXh1atlPdyQ3hQvX9Mhopf74JauePbKdob3v7NPc9yt0SmoNfZei3czV7e0Ouh/XoOyoFSjSqWymrnW5DItg9s3QrO61YIuxtKifg2sHj0Qwzs3xhUdz0xIGzOsPQD4vYdmdxwmiuCaLk3QsGYy7uzTHF/9pZduttLGtav6bfvUZwjw4PZnlc1q9lWlUmLZZ+L7ffd0vlYysIBNuNjkQmG5oZt2+63H3/q3wT0XtUBNAwHnhwf6YM6mnLL7yUmJUcsnEuognKTEhLLgHgnPCJB//misQ290CMFfi3cttnKlBLz3567Y5V61SsQV1Fc+M6CsrdoIT4dmqJ2Jwzs3xvDOjUN6Tihu752GTxftKndFlZAg6N48+CiiL+7pobnWZzgpq7U0T62O+/q1xAidv51IMKCTJRISxFAwB1zDIO1akq5s5SZbXh3o3rwurvEKbt/8Xy8kJyXixWmb8Ngg4zMWvakwRzkbmV/gLZpNY75GDTkXh08Uoo9XPvuUKpWQX1iMBy5theeuCp7CWYvWOPJwnFOvGvYfK0CSzxWHiODvg9ua8hqBMKBT1EVx/Qpdnj+66hZ06HnMfuSigEMXv/pLr3L3PWPRJ40MPoEnHDd2b4ZJy/aEVAs3YvzNXZGTX2DqMfW0apCCKfe5Mnl6flY+ui0d+48VlFtL1w4f3NIVGbuOhJ27JxKGvsUiMhjAWwASAXyklBrr8/gjAO4GUAwgF8CdSqndJpeVyHTp59TBYwPb4Mbu1l0Gt2qQglYNIpsAE4pAHbztG9cs2wM4k0LAaLt5IEYyaQLAkicus3SEU2pKFfQII4eRlkY1k7H/WHg/UrWrVfYbOhstur0QIpII4F0AQwC0A3CjiPg26K0CkK6U6gjgGwDjzC4okRVEBPdf2tr2Wl0kUmtUKTcOOlCTi6dJYXgnVwdlrWpJWPPsQDweQtqHSDSqlYyGIeZmt8u8x/tFPAvVDkZq6N0BZCqldgCAiEwGMAxAWW+OUmqu1/5LANxsZiGJKLCMp12J1P7+zZqg+zVPre7X2Wx284tTJCclYvqD4U8Ks4uRcUKNAez1up/l3hbIXQBmRFIocibPpXmo49bJmFjqm4imcdd1RPfmdYOmWA5HQoL4JXeLdab2BInIzQDSAWhmuxGRkQBGAkCzZta1WVJsevDS1rijd3PU0phAQhSu9LS6fp3LFZWRgJ4NwHsecRP3tnJEpD+ApwBcrJTyX4sMgFJqAoAJAJCenl5B6xMVV0KCMJhbKNbWMg3HV3/pZdlqPpF6ZEAbW4dqGmHknVsOoLWINIcrkI8AcJP3DiLSGcAHAAYrpXL8D0FEVvNMP0+0aL3KaNCb/GOnaCxNFyndgK6UKhaR+wHMhGvY4kSl1AYRGQMgQyn1A4BXAdQA8LV7RtoepdRVFpabiHz8fXBbVK9SCcM6+a/7ShWDRGvBXl/p6ekqIyPDltcmIopXIrJCKaW5Wnj8XpsREVE5DOhERA7BgE5E5BAM6EREDsGATkTkEAzoREQOwYBOROQQDOhERA5h28QiEckFEO4iGKkADppYnHjAc64YeM4VQyTnfI5Sqr7WA7YF9EiISEagmVJOxXOuGHjOFYNV58wmFyIih2BAJyJyiHgN6BPsLoANeM4VA8+5YrDknOOyDZ2IiPzFaw2diIh8xF1AF5HBIrJFRDJFZJTd5YmUiOwSkXUislpEMtzb6orILyKyzf1/Hfd2EZG33ee+VkS6eB3nNvf+20TkNrvOR4uITBSRHBFZ77XNtHMUka7u9zDT/Vxb1wkLcL7PiUi2+3NeLSKXez32hLvsW0RkkNd2ze+6iDQXkaXu7V+KSOXonZ02EWkqInNFZKOIbBCRh9zbnfw5Bzpn+z5rpVTc/INrxaTtAFoAqAxgDYB2dpcrwnPaBSDVZ9s4AKPct0cBeMV9+3IAMwAIgJ4Alrq31wWww/1/HfftOnafm9f5XASgC4D1VpwjgGXufcX93CExeL7PAXhMY9927u9xFQDN3d/vxGDfdQBfARjhvj0ewL0x8BmfBaCL+3YKgK3uc3Py5xzonG37rOOtht4dQKZSaodS6jSAyQCG2VwmKwwD8B/37f8AGO61/TPlsgRAbRE5C8AgAL8opQ4rpY4A+AXA4GgXOhCl1HwAh302m3KO7sdqKqWWKNe3/jOvY9kiwPkGMgzAZKVUoVJqJ4BMuL7nmt91d630UgDfuJ/v/d7ZRim1Tym10n07H8AmAI3h7M850DkHYvlnHW8BvTGAvV73sxD8DYwHCsAsEVkhIiPd2xoqpfa5b+8H0NB9O9D5x+P7YtY5Nnbf9t0ei+53Ny9M9DQ9IPTzrQfgqFKq2Gd7zBCRNACdASxFBfmcfc4ZsOmzjreA7kQXKqW6ABgC4K8icpH3g+7aiKOHIlWEcwTwPoCWADoB2AfgNXuLYw0RqQHgWwAPK6WOeT/m1M9Z45xt+6zjLaBnA2jqdb+Je1vcUkplu//PATAVrsuvA+5LTLj/z3HvHuj84/F9Mescs923fbfHFKXUAaVUiVKqFMCHcH3OQOjnewiu5olKPtttJyJJcAW2/ymlprg3O/pz1jpnOz/reAvoywG0dvf8VgYwAsAPNpcpbCJSXURSPLcBDASwHq5z8vTu3wbge/ftHwDc6h4h0BNAnvtydiaAgSJSx315N9C9LZaZco7ux46JSE93m+OtXseKGZ6g5nY1XJ8z4DrfESJSRUSaA2gNV+ef5nfdXcudC+A69/O93zvbuN/7jwFsUkq97vWQYz/nQOds62dtZy9xOP/g6h3fClev8FN2lyfCc2kBV4/2GgAbPOcDV9vZHADbAMwGUNe9XQC86z73dQDSvY51J1ydLJkA7rD73HzOcxJcl55FcLUD3mXmOQJId//RbAfwDtwT5mLsfD93n89a9x/2WV77P+Uu+xZ4jdwI9F13f2+Wud+HrwFUiYHP+EK4mlPWAljt/ne5wz/nQOds22fNmaJERA4Rb00uREQUAAM6EZFDMKATETkEAzoRkUMwoBMROQQDOhGRQzCgExE5BAM6EZFD/D8SfuerR462vQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "params['batch_size'] = 32\n",
    "losses = train(model, train_dataset, 40, 0.4, 1)\n",
    "plt.plot(losses);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "predict_dev = clf.predict(dev_x_vec)\n",
    "pandas.DataFrame(zip(predict_dev, y_dev, X_dev), columns=['predicted', 'expected', 'title'])\n",
    "from sklearn.metrics import classification_report, confusion_matrix, accuracy_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "\"\"\"\n",
    ">>> # an Embedding module containing 10 tensors of size 3\n",
    ">>> embedding_sum = nn.EmbeddingBag(10, 3, mode='sum')\n",
    ">>> # a batch of 2 samples of 4 indices each\n",
    ">>> input = torch.LongTensor([1,2,4,5,4,3,2,9])\n",
    ">>> offsets = torch.LongTensor([0,4])\n",
    ">>> embedding_sum(input, offsets)\n",
    "tensor([[-0.8861, -5.4350, -0.0523],\n",
    "        [ 1.1306, -2.5798, -1.0044]])\n",
    "\"\"\"\n",
    "def generate_batch(batch):\n",
    "    #batch is (X, y)\n",
    "    print(len(batch), batch[0])\n",
    "    label = torch.tensor([entry[0] for entry in batch])\n",
    "    text = [entry[1] for entry in batch]\n",
    "    offsets = [0] + [len(entry) for entry in text]\n",
    "    offsets = torch.tensor(offsets[:-1]).cumsum(dim=0)\n",
    "    text = torch.cat(text)\n",
    "    return text, offsets, label\n",
    "\n",
    "train_loader = DataLoader(train_dataset, **params)\n",
    "for x,y in train_loader:\n",
    "    print(x)\n",
    "    print(y)\n",
    "    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch.nn as nn\n",
    "class TextSentiment(nn.Module):\n",
    "    def __init__(self, vocab_size, embed_dim, num_class):\n",
    "        super().__init__()\n",
    "        self.embedding = nn.EmbeddingBag(vocab_size, embed_dim, sparse=True)\n",
    "        self.fc = nn.Linear(embed_dim, num_class)\n",
    "        self.init_weights()\n",
    "\n",
    "    def init_weights(self):\n",
    "        initrange = 0.5\n",
    "        self.embedding.weight.data.uniform_(-initrange, initrange)\n",
    "        self.fc.weight.data.uniform_(-initrange, initrange)\n",
    "        self.fc.bias.data.zero_()\n",
    "\n",
    "    def forward(self, text, offsets):\n",
    "        r\"\"\"\n",
    "        Arguments:\n",
    "            text: 1-D tensor representing a bag of text tensors\n",
    "            offsets: a list of offsets to delimit the 1-D text tensor\n",
    "                into the individual sequences.\n",
    "        \"\"\"\n",
    "        return self.fc(self.embedding(text, offsets))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = TextSentiment(len(vocab), embed_dim, num_labels).to(device)\n",
    "criterion = torch.nn.CrossEntropyLoss().to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "predict_dev = clf.predict(dev_x_vec)\n",
    "pandas.DataFrame(zip(predict_dev, y_dev, X_dev), columns=['predicted', 'expected', 'title'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "clf.score(dev_x_vec, y_dev)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import classification_report, confusion_matrix, accuracy_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(classification_report(predict_dev, y_dev, digits=3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(confusion_matrix(predict_dev, y_dev))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lines = [' '.join(['__label__'+y] + x) for x, y in zip(X_train, y_train)]\n",
    "lines[:5]\n",
    "from pathlib import Path\n",
    "Path('data/train.txt').write_text('\\n'.join(lines))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lines = [' '.join(['__label__'+y] + x) for x, y in zip(X_dev, y_dev)]\n",
    "Path('data/dev.txt').write_text('\\n'.join(lines))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!dist/fastText-0.9.1/fasttext supervised -input data/train.txt -output data/ft.bin"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!dist/fastText-0.9.1/fasttext supervised"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!dist/fastText-0.9.1/fasttext test data/ft.bin.bin data/dev.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import fasttext"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = fasttext.load_model('data/ft.bin.bin')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "display(list(zip(*model.predict(' '.join(X_dev[0]), k=15))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train[0], y_train[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# NN\n",
    "# more complex models\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}