Lab12/Lab12-3_SurinKim.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Lab 12-3 many to many\n",
    "### simple pos-tagger training \n",
    "* many to many\n",
    "* variable input sequence length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.5.0\n"
     ]
    }
   ],
   "source": [
    "# setup\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import matplotlib.pyplot as plt\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras import layers\n",
    "from tensorflow.keras import Sequential, Model\n",
    "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
    "from pprint import pprint\n",
    "%matplotlib inline\n",
    "\n",
    "print(tf.__version__)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Prepairing dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# example data\n",
    "sentences = [['I', 'feel', 'hungry'],\n",
    "     ['tensorflow', 'is', 'very', 'difficult'],\n",
    "     ['tensorflow', 'is', 'a', 'framework', 'for', 'deep', 'learning'],\n",
    "     ['tensorflow', 'is', 'very', 'fast', 'changing']]\n",
    "pos = [['pronoun', 'verb', 'adjective'],\n",
    "     ['noun', 'verb', 'adverb', 'adjective'],\n",
    "     ['noun', 'verb', 'determiner', 'noun', 'preposition', 'adjective', 'noun'],\n",
    "     ['noun', 'verb', 'adverb', 'adjective', 'verb']]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Preprocessing dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'<pad>': 0, 'I': 1, 'a': 2, 'changing': 3, 'deep': 4, 'difficult': 5, 'fast': 6, 'feel': 7, 'for': 8, 'framework': 9, 'hungry': 10, 'is': 11, 'learning': 12, 'tensorflow': 13, 'very': 14}\n",
      "{0: '<pad>', 1: 'I', 2: 'a', 3: 'changing', 4: 'deep', 5: 'difficult', 6: 'fast', 7: 'feel', 8: 'for', 9: 'framework', 10: 'hungry', 11: 'is', 12: 'learning', 13: 'tensorflow', 14: 'very'}\n",
      "15\n"
     ]
    }
   ],
   "source": [
    "# creating a token dictionary for word\n",
    "word_list = sum(sentences, [])\n",
    "word_list = sorted(set(word_list))\n",
    "word_list = ['<pad>'] + word_list\n",
    "word2idx = {word : idx for idx, word in enumerate(word_list)}\n",
    "idx2word = {idx : word for idx, word in enumerate(word_list)}\n",
    "\n",
    "print(word2idx)\n",
    "print(idx2word)\n",
    "print(len(idx2word))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'<pad>': 0, 'adjective': 1, 'adverb': 2, 'determiner': 3, 'noun': 4, 'preposition': 5, 'pronoun': 6, 'verb': 7}\n",
      "{0: '<pad>', 1: 'adjective', 2: 'adverb', 3: 'determiner', 4: 'noun', 5: 'preposition', 6: 'pronoun', 7: 'verb'}\n",
      "8\n"
     ]
    }
   ],
   "source": [
    "# creating a token dictionary for part of speech\n",
    "pos_list = sum(pos, [])\n",
    "pos_list = sorted(set(pos_list))\n",
    "pos_list = ['<pad>'] + pos_list\n",
    "pos2idx = {pos : idx for idx, pos in enumerate(pos_list)}\n",
    "idx2pos = {idx : pos for idx, pos in enumerate(pos_list)}\n",
    "\n",
    "print(pos2idx)\n",
    "print(idx2pos)\n",
    "print(len(pos2idx))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 1  7 10  0  0  0  0  0  0  0]\n",
      " [13 11 14  5  0  0  0  0  0  0]\n",
      " [13 11  2  9  8  4 12  0  0  0]\n",
      " [13 11 14  6  3  0  0  0  0  0]] [3, 4, 7, 5]\n",
      "[[1. 1. 1. 0. 0. 0. 0. 0. 0. 0.]\n",
      " [1. 1. 1. 1. 0. 0. 0. 0. 0. 0.]\n",
      " [1. 1. 1. 1. 1. 1. 1. 0. 0. 0.]\n",
      " [1. 1. 1. 1. 1. 0. 0. 0. 0. 0.]]\n",
      "[[6 7 1 0 0 0 0 0 0 0]\n",
      " [4 7 2 1 0 0 0 0 0 0]\n",
      " [4 7 3 4 5 1 4 0 0 0]\n",
      " [4 7 2 1 7 0 0 0 0 0]]\n"
     ]
    }
   ],
   "source": [
    "# converting sequence of tokens to sequence of indices\n",
    "max_sequence = 10\n",
    "x_data = list(map(lambda sentence : [word2idx.get(token) for token in sentence], sentences))\n",
    "y_data = list(map(lambda sentence : [pos2idx.get(token) for token in sentence], pos))\n",
    "\n",
    "# padding the sequence of indices\n",
    "x_data = pad_sequences(sequences = x_data, maxlen = max_sequence, padding='post')\n",
    "x_data_mask = ((x_data != 0) * 1).astype(np.float32)\n",
    "x_data_len = list(map(lambda sentence : len(sentence), sentences))\n",
    "\n",
    "y_data = pad_sequences(sequences = y_data, maxlen = max_sequence, padding='post')\n",
    "\n",
    "# checking data\n",
    "print(x_data, x_data_len)\n",
    "print(x_data_mask)\n",
    "print(y_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Creating model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# creating rnn for \"many to many\" sequence tagging\n",
    "num_classes = len(pos2idx)\n",
    "hidden_dim = 10\n",
    "\n",
    "input_dim = len(word2idx)\n",
    "output_dim = len(word2idx)\n",
    "one_hot = np.eye(len(word2idx))\n",
    "\n",
    "model = Sequential()\n",
    "model.add(layers.Embedding(input_dim=input_dim, output_dim=output_dim, mask_zero=True,\n",
    "                           trainable=False, input_length=max_sequence,\n",
    "                           embeddings_initializer=keras.initializers.Constant(one_hot)))\n",
    "model.add(layers.SimpleRNN(units=hidden_dim, return_sequences=True))\n",
    "model.add(layers.TimeDistributed(layers.Dense(units=num_classes)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding (Embedding)        (None, 10, 15)            225       \n",
      "_________________________________________________________________\n",
      "simple_rnn (SimpleRNN)       (None, 10, 10)            260       \n",
      "_________________________________________________________________\n",
      "time_distributed (TimeDistri (None, 10, 8)             88        \n",
      "=================================================================\n",
      "Total params: 573\n",
      "Trainable params: 348\n",
      "Non-trainable params: 225\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# creating loss function\n",
    "def loss_fn(model, x, y, x_len, max_sequence):\n",
    "    masking = tf.sequence_mask(x_len, maxlen=max_sequence, dtype=tf.float32)\n",
    "    valid_time_step = tf.cast(x_len,dtype=tf.float32)    \n",
    "    sequence_loss = tf.keras.losses.sparse_categorical_crossentropy(\n",
    "        y_true=y, y_pred=model(x), from_logits=True) * masking    \n",
    "    sequence_loss = tf.reduce_sum(sequence_loss, axis=-1) / valid_time_step    \n",
    "    sequence_loss = tf.reduce_mean(sequence_loss)    \n",
    "    return sequence_loss\n",
    "\n",
    "# creating and optimizer\n",
    "lr = 0.1\n",
    "epochs = 30\n",
    "batch_size = 2 \n",
    "opt = tf.keras.optimizers.Adam(learning_rate = lr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<BatchDataset shapes: ((None, 10), (None, 10), (None,)), types: (tf.int32, tf.int32, tf.int32)>\n"
     ]
    }
   ],
   "source": [
    "# generating data pipeline\n",
    "tr_dataset = tf.data.Dataset.from_tensor_slices((x_data, y_data, x_data_len))\n",
    "tr_dataset = tr_dataset.shuffle(buffer_size=4)\n",
    "tr_dataset = tr_dataset.batch(batch_size = 2)\n",
    "\n",
    "print(tr_dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch :   5, tr_loss : 0.190\n",
      "epoch :  10, tr_loss : 0.008\n",
      "epoch :  15, tr_loss : 0.002\n",
      "epoch :  20, tr_loss : 0.001\n",
      "epoch :  25, tr_loss : 0.001\n",
      "epoch :  30, tr_loss : 0.001\n"
     ]
    }
   ],
   "source": [
    "# training\n",
    "tr_loss_hist = []\n",
    "\n",
    "for epoch in range(epochs):\n",
    "    avg_tr_loss = 0\n",
    "    tr_step = 0\n",
    "    \n",
    "    for x_mb, y_mb, x_mb_len in tr_dataset:\n",
    "        with tf.GradientTape() as tape:\n",
    "            tr_loss = loss_fn(model, x=x_mb, y=y_mb, x_len=x_mb_len, max_sequence=max_sequence)\n",
    "        grads = tape.gradient(target=tr_loss, sources=model.variables)\n",
    "        opt.apply_gradients(grads_and_vars=zip(grads, model.variables))\n",
    "        avg_tr_loss += tr_loss\n",
    "        tr_step += 1\n",
    "    else:\n",
    "        avg_tr_loss /= tr_step\n",
    "        tr_loss_hist.append(avg_tr_loss)\n",
    "    \n",
    "    if (epoch + 1) % 5 == 0:\n",
    "        print('epoch : {:3}, tr_loss : {:.3f}'.format(epoch + 1, avg_tr_loss))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Checking performance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[['pronoun', 'verb', 'adjective', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>'],\n",
      " ['noun', 'verb', 'adverb', 'adjective', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>'],\n",
      " ['noun', 'verb', 'determiner', 'noun', 'preposition', 'adjective', 'noun', '<pad>', '<pad>', '<pad>'],\n",
      " ['noun', 'verb', 'adverb', 'adjective', 'verb', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>']]\n",
      "[['pronoun', 'verb', 'adjective'],\n",
      " ['noun', 'verb', 'adverb', 'adjective'],\n",
      " ['noun', 'verb', 'determiner', 'noun', 'preposition', 'adjective', 'noun'],\n",
      " ['noun', 'verb', 'adverb', 'adjective', 'verb']]\n"
     ]
    }
   ],
   "source": [
    "yhat = model.predict(x_data)\n",
    "yhat = np.argmax(yhat, axis=-1) * x_data_mask\n",
    "\n",
    "pprint(list(map(lambda row : [idx2pos.get(elm) for elm in row],yhat.astype(np.int32).tolist())), width = 120)\n",
    "pprint(pos)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x1fe8a1b9790>]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXwAAAD4CAYAAADvsV2wAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAAeXUlEQVR4nO3de3Bc533e8e+zuBLgBSAB0hJJEJDMmKKtiLARWhfHkeREljKtlaROLLqOZVceuhlp4jSdTuxkarnydMZN2iRN4lplFEa+xJJVXxKlVSIrlhw5ligTkqgrdaFJSiRFEuD9ThDYX//YA2kJAsQCXPBg9zyfmR3svu85u78zO3z28D2XVxGBmZlVv1zaBZiZ2fnhwDczywgHvplZRjjwzcwywoFvZpYRtWkXMJq2trbo7OxMuwwzs4rx5JNP7omI9rMtMy0Dv7Ozk97e3rTLMDOrGJJeG28ZD+mYmWWEA9/MLCMc+GZmGeHANzPLCAe+mVlGOPDNzDLCgW9mlhHjBr6kxZIekfSipBckfWaUZSTpzyRtkvSspHcX9d0s6dXkcXO5N2DY4FCeLz+yiUdf6Z+qjzAzq2il7OEPAv8xIpYDlwO3Slo+YpkbgKXJYzXwFQBJc4HbgfcCK4HbJbWWqfbT1OTEmkc38/0Xd03F25uZVbxxAz8idkbEU8nzw8BGYOGIxW4EvhYF64AWSRcAHwQeioh9EbEfeAi4vqxbkJBEZ1szW/YcnYq3NzOreBMaw5fUCXQDT4zoWghsK3q9PWkbq320914tqVdSb3//5IZlLmprZku/A9/MbDQlB76kmcB3gN+JiEPlLiQi1kRET0T0tLef9f4/Y+pqa+aNgyc4cWqozNWZmVW+kgJfUh2FsP+biPjuKIvsABYXvV6UtI3VPiU625oB2LrXe/lmZiOVcpaOgL8CNkbEH4+x2P3Ax5OzdS4HDkbETuBB4DpJrcnB2uuStilxURL4HtYxMztTKbdHvgr4TeA5SRuStt8HOgAi4k7gAeCXgU3AMeCTSd8+SV8E1ifr3RER+8pW/QjDe/hbvIdvZnaGcQM/Iv4F0DjLBHDrGH1rgbWTqm6CZjbU0j6rwXv4ZmajqLorbbvamj2Gb2Y2iuoL/Hk+F9/MbDTVF/jtzew5MsChE6fSLsXMbFqpvsAfPjXTe/lmZqep2sD3sI6Z2emqLvA75jYhOfDNzEaqusBvrKvhwjkzHPhmZiNUXeADXNTuM3XMzEaqysDvSm6TXLgezMzMoEoDv3NeM4dPDLL36EDapZiZTRtVGfhd7T5Tx8xspKoM/It8aqaZ2RmqMvAXtsygNicHvplZkaoM/NqaHB3zmny1rZlZkaoMfPBN1MzMRiplxqu1kvokPT9G/3+StCF5PC9pSNLcpG+rpOeSvt5yF382w6dm5vM+NdPMDErbw78buH6szoj4o4hYERErgM8B/zxiVqtrkv6ec6p0grramzk5mGfXoRPn82PNzKatcQM/Ih4FSp2WcBVwzzlVVCZd83ymjplZsbKN4UtqovA/ge8UNQfwfUlPSlo9zvqrJfVK6u3v7z/nenwuvpnZ6cp50PZfAz8eMZzzvoh4N3ADcKuk94+1ckSsiYieiOhpb28/52IWzGpkRl2NA9/MLFHOwL+JEcM5EbEj+dsHfA9YWcbPO6tcTiyZ1+TANzNLlCXwJc0BfgH4u6K2Zkmzhp8D1wGjnukzVS5qb/a5+GZmidrxFpB0D3A10CZpO3A7UAcQEXcmi/0q8P2IKE7XBcD3JA1/zjcj4h/LV/r4Ouc18/0XdnNqKE9dTdVecmBmVpJxAz8iVpWwzN0UTt8sbtsMXDbZwsqhq62ZwXywff/xN6c+NDPLqqre7b2o3ROam5kNq+rA70zOxd/swDczq+7An9tcz+zGWu/hm5lR5YEv6c176piZZV1VBz7gwDczS2Qg8GfyxsHjnDg1lHYpZmapqvrA72xrIgJe23ss7VLMzFJV9YF/UdtMwDdRMzOr+sDvbGsCHPhmZlUf+LMa62ib2cCWPUfSLsXMLFVVH/gAF7U1s3WPx/DNLNsyEfidbU2+2tbMMi8Tgd/VNpM9R05y+MSptEsxM0tNRgJ/+CZqHtYxs+zKVOBv9oFbM8uwTAT+knlNSN7DN7NsGzfwJa2V1Cdp1OkJJV0t6aCkDcnj80V910t6WdImSZ8tZ+ET0VhXw4VzZvjUTDPLtFL28O8Grh9nmR9FxIrkcQeApBrgy8ANwHJglaTl51LsufBN1Mws68YN/Ih4FNg3ifdeCWyKiM0RMQDcC9w4ifcpi+HAj4i0SjAzS1W5xvCvkPSMpH+Q9M6kbSGwrWiZ7UnbqCStltQrqbe/v79MZb2ls62ZQycG2Xd0oOzvbWZWCcoR+E8BSyLiMuDPgb+dzJtExJqI6ImInvb29jKUdbqLkjN1PKxjZll1zoEfEYci4kjy/AGgTlIbsANYXLTooqQtFV0OfDPLuHMOfElvk6Tk+crkPfcC64Glkrok1QM3Afef6+dN1qLWGdTm5MA3s8yqHW8BSfcAVwNtkrYDtwN1ABFxJ/Bh4LckDQLHgZuicGR0UNJtwINADbA2Il6Ykq0oQW1Njo65TWzd68A3s2waN/AjYtU4/X8B/MUYfQ8AD0yutPLrbGtmc78D38yyKRNX2g7ramtm696j5PM+NdPMsidzgX/iVJ7dh0+kXYqZ2XmXucAH2OJhHTPLoGwGvg/cmlkGZSrw3za7kca6nPfwzSyTMhX4uZzonOebqJlZNmUq8CG5iZqHdMwsgzIX+J1tzby+9xiDQ/m0SzEzO68yF/hdbc0M5oPt+4+nXYqZ2XmVucC/yGfqmFlGZS7wO30uvpllVOYCf15zPbMaa30TNTPLnMwFviTPb2tmmZS5wIfCgVvfNdPMsiazgf/GweMcHxhKuxQzs/Nm3MCXtFZSn6Tnx+j/t5KelfScpMckXVbUtzVp3yCpt5yFn4t3XTiHCHj+jYNpl2Jmdt6Usod/N3D9Wfq3AL8QEZcCXwTWjOi/JiJWRETP5EosvxUdLQA8/fr+dAsxMzuPSpnx6lFJnWfpf6zo5ToKk5VPa20zG+iY28TTrx9IuxQzs/Om3GP4twD/UPQ6gO9LelLS6rOtKGm1pF5Jvf39/WUu60zdHS0OfDPLlLIFvqRrKAT+7xU1vy8i3g3cANwq6f1jrR8RayKiJyJ62tvby1XWmLoXt7Dr0Al2HvQtFswsG8oS+JJ+FrgLuDEi9g63R8SO5G8f8D1gZTk+rxy6O1oBvJdvZplxzoEvqQP4LvCbEfFKUXuzpFnDz4HrgFHP9EnDJRfMpr425wO3ZpYZ4x60lXQPcDXQJmk7cDtQBxARdwKfB+YB/0sSwGByRs4C4HtJWy3wzYj4xynYhkmpr81x6cI53sM3s8wo5SydVeP0fwr41Cjtm4HLzlxj+uhe3MLX173GwGCe+tpMXoNmZhmS6ZTr7mjl5GCel3YdSrsUM7Mpl/HAbwF84NbMsiHTgX/BnEYWzG7wgVszy4RMB74kuhe38vS2A2mXYmY25TId+FAY1nlt7zH2HjmZdilmZlPKgZ9cgLXBe/lmVuUyH/iXLpxDTU4+cGtmVS/zgT+jvoZLLpjF09t84NbMqlvmAx+ge3Erz2w7yFA+0i7FzGzKOPApHLg9cnKQTX1H0i7FzGzKOPApvnOmh3XMrHo58IHOeU20NNX5wK2ZVTUHPsMXYLX4wK2ZVTUHfqK7o5VX+45w6MSptEsxM5sSDvxEd0cLEfDstoNpl2JmNiUc+InLFrcg+cCtmVWvkgJf0lpJfZJGnaJQBX8maZOkZyW9u6jvZkmvJo+by1V4uc1urOPt7TN9IzUzq1ql7uHfDVx/lv4bgKXJYzXwFQBJcylMifheChOY3y6pdbLFTrXujhaefn0/Eb4Ay8yqT0mBHxGPAvvOssiNwNeiYB3QIukC4IPAQxGxLyL2Aw9x9h+OVHV3tLL/2Cle23ss7VLMzMquXGP4C4FtRa+3J21jtZ9B0mpJvZJ6+/v7y1TWxLw5A5ZPzzSzKjRtDtpGxJqI6ImInvb29lRqWDp/Fs31Nb4Ay8yqUrkCfwewuOj1oqRtrPZpqSYnLlvc4sA3s6pUrsC/H/h4crbO5cDBiNgJPAhcJ6k1OVh7XdI2bXV3tLBx5yGODwylXYqZWVnVlrKQpHuAq4E2SdspnHlTBxARdwIPAL8MbAKOAZ9M+vZJ+iKwPnmrOyLibAd/U9e9uJXBfPD8Gwf5uc65aZdjZlY2JQV+RKwapz+AW8foWwusnXhp6VgxfOD29f0OfDOrKtPmoO100TazgY65TR7HN7Oq48AfReECrANpl2FmVlYO/FF0L25h16ET7Dx4PO1SzMzKxoE/irdmwDqQbiFmZmXkwB/FJRfMpr425ztnmllVceCPor42x6UL53gP38yqigN/DN2LW3hux0EGBvNpl2JmVhYO/DF0d7RycjDPS7sOpV2KmVlZOPDH8OadMz2sY2ZVwoE/hgvmNLJgdoMP3JpZ1XDgj0ES3YtbPeWhmVUNB/5ZdHe08NreY+w9cjLtUszMzpkD/yyGL8Da4L18M6sCDvyzuHThHGpzYv1Wj+ObWeVz4J/FjPoaejpb+eHLfWmXYmZ2zkoKfEnXS3pZ0iZJnx2l/08kbUger0g6UNQ3VNR3fxlrPy+uXTafl3YdZscB30jNzCrbuIEvqQb4MnADsBxYJWl58TIR8R8iYkVErAD+HPhuUffx4b6I+FD5Sj8/rl22AICHX/JevplVtlL28FcCmyJic0QMAPcCN55l+VXAPeUobjq4uL2ZJfOaeHjj7rRLMTM7J6UE/kJgW9Hr7UnbGSQtAbqAh4uaGyX1Slon6VfG+hBJq5Plevv7+0so6/yQxLXL5vPYT/d6YnMzq2jlPmh7E/DtiChOxiUR0QN8FPhTSRePtmJErImInojoaW9vL3NZ5+YDyxZwcjDPYz/dk3YpZmaTVkrg7wAWF71elLSN5iZGDOdExI7k72bgh0D3hKtM2cquuTTX1/ADj+ObWQUrJfDXA0sldUmqpxDqZ5xtI2kZ0Ao8XtTWKqkhed4GXAW8WI7Cz6f62hw/v7Sdhzf2ERFpl2NmNinjBn5EDAK3AQ8CG4H7IuIFSXdIKj7r5ibg3jg9ES8BeiU9AzwCfCkiKi7wAa69ZD67Dp3gxZ2+XbKZVabaUhaKiAeAB0a0fX7E6y+Mst5jwKXnUN+0cc075gPw8MY+3nnhnJSrMTObOF9pW6L2WQ1ctrjF4/hmVrEc+BPwgWXzeWb7Afb47plmVoEc+BNw7bL5RMAPX54+1wmYmZXKgT8B77xwNgtmN/DwS77q1swqjwN/Aoavun30lT0MDObTLsfMbEIc+BN07bIFHDk5yPqt+9IuxcxsQhz4E3TV2+dRX5vjBxt9to6ZVRYH/gQ11ddy5cXz+MFLu33VrZlVFAf+JHxg2Xxe23uMzXuOpl2KmVnJHPiTcM2yt666NTOrFA78SVjU2sSyt83yLFhmVlEc+JN0zbL5rN+6j4PHT6VdiplZSRz4k/SBZfMZzAc/etVX3ZpZZXDgT1J3RystTXUexzeziuHAn6SanLjmHfN55OU+hvI+PdPMpj8H/jm4dtl89h87xYZt+9MuxcxsXCUFvqTrJb0saZOkz47S/wlJ/ZI2JI9PFfXdLOnV5HFzOYtP2/t/pp2anHzVrZlVhHEDX1IN8GXgBmA5sErS8lEW/VZErEgedyXrzgVuB94LrARul9RatupTNmdGHT/X2erTM82sIpSyh78S2BQRmyNiALgXuLHE9/8g8FBE7IuI/cBDwPWTK3V6unbZfF7adZgdB46nXYqZ2VmVEvgLgW1Fr7cnbSP9G0nPSvq2pMUTXBdJqyX1Surt76+cUx2vXbYAwHv5Zjbtleug7d8DnRHxsxT24r860TeIiDUR0RMRPe3t7WUqa+pd3N7MknlNPLzRk6KY2fRWSuDvABYXvV6UtL0pIvZGxPBEr3cB7yl13Uo3PCnKj3+6l2MDg2mXY2Y2plICfz2wVFKXpHrgJuD+4gUkXVD08kPAxuT5g8B1klqTg7XXJW1V5QPLFjAwmOexTXvTLsXMbEzjBn5EDAK3UQjqjcB9EfGCpDskfShZ7LclvSDpGeC3gU8k6+4DvkjhR2M9cEfSVlVWds2lub6GH3gc38ymsdpSFoqIB4AHRrR9vuj554DPjbHuWmDtOdQ47dXX5vj5pe088lIf+XyQyyntkszMzuArbcvkhkvfxq5DJ/hn30zNzKYpB36Z3PCuC5g/q4G//vHWtEsxMxuVA79M6mtzfOzyJTz6Sj+b+o6kXY6Z2Rkc+GX00fd2UF+T46uPbU27FDOzMzjwy6htZgMfWnEh33lqu2fCMrNpx4FfZp+4spNjA0Pct37b+AubmZ1HDvwye9fCOazsnMtXH9/qiVHMbFpx4E+BT17Vyfb9x/kn31/HzKYRB/4U+KXlC1jYMoO//vGWtEsxM3uTA38K1Nbk+PgVS1i3eR8bdx5KuxwzM8CBP2U+8nOLaazLcbcvxDKzacKBP0Vamur5tXcv4m837GDf0YG0yzEzc+BPpU9e2cnJwTz3/OT1tEsxM3PgT6WlC2bx80vb+Prjr3FqKJ92OWaWcQ78KfaJKzvZdegE//j8rrRLMbOMKynwJV0v6WVJmyR9dpT+35X0YjKJ+Q8kLSnqG5K0IXncP3LdanfNO+bTOa/Jp2iaWerGDXxJNcCXgRuA5cAqSctHLPY00JNMYv5t4A+L+o5HxIrk8SEyJpcTN1/ZyVOvH+CZbQfSLsfMMqyUPfyVwKaI2BwRA8C9wI3FC0TEIxFxLHm5jsJk5Zb48HsWMbOh1nv5ZpaqUgJ/IVB8J7DtSdtYbgH+oeh1o6ReSesk/cpYK0lanSzX299fXbNGzWqs49d7FvH/nttJ36ETaZdjZhlV1oO2kj4G9AB/VNS8JCJ6gI8Cfyrp4tHWjYg1EdETET3t7e3lLGtauPmKTgbzwTee8CmaZpaOUgJ/B7C46PWipO00kn4R+APgQxFxcrg9InYkfzcDPwS6z6HeitXZ1sy175jPN594jZODQ2mXY2YZVErgrweWSuqSVA/cBJx2to2kbuB/Uwj7vqL2VkkNyfM24CrgxXIVX2k+eVUXe44M8PfP7Ey7FDPLoHEDPyIGgduAB4GNwH0R8YKkOyQNn3XzR8BM4P+MOP3yEqBX0jPAI8CXIiKzgX/V2+exdP5M/vrHW4jwvfLN7PyqLWWhiHgAeGBE2+eLnv/iGOs9Blx6LgVWE0l84qpO/uB7z7N+635Wds1NuyQzyxBfaXue/Vr3ItpmNvCZe59mc/+RtMsxswxx4J9nM+pr+PotKxkYzPORNet4dffhtEsys4xw4Kfgkgtmc+/qywG4ac06T5JiZueFAz8lSxfM4r5PX0F9bY5Vf7mO57YfTLskM6tyDvwUdbU1c9+nr2BmQy0fvWsdT72+P+2SzKyKOfBTtnhuE9/69BXMba7nN+96gp9s2Zd2SWZWpRz408DClhnc9+kreNucRm5e+xN+vGlP2iWZWRVy4E8TC2Y3cu/qK+iY28S/u3s9P3y5b/yVzMwmwIE/jbTPauCe1Zfz9vkzWf21J3noxd1pl2RmVcSBP83Mba7nm5+6nEsunM1vfeNJ/u+zb6RdkplVCQf+NDSnqY5v3LKSFYtbuO2bT/Mbdz7OP724m3ze998xs8lz4E9Tsxrr+Man3st//lfL2XHgOJ/6Wi+/+Cf/zD0/eZ0Tp3x7ZTObOE3Huzb29PREb29v2mVMG4NDeR54fhdrHv0pz+84RNvMej5+RScfu3wJc5vr0y7PzKYBSU8mk02NvYwDv3JEBI9v3stfPrqZR17up7Eux6+/ZzG3vK+LzrbmtMszsxSVEvgl3R7ZpgdJXHlxG1de3MYruw9z148286312/jGE6/xweVv45pl7XS1zaSzrYn2mQ1ISrtkM5tGvIdf4foOneCrj2/lG+te5+DxU2+2N9fX0NnWTGdbM13zkr9tTXTOa2Zuc71/DMyqTNmGdCRdD/xPoAa4KyK+NKK/Afga8B5gL/CRiNia9H0OuAUYAn47Ih4c7/Mc+BM3OJTnjQMn2LL3KFv6j7B17zG27DnK1r1H2b7/OENFZ/jMbKhl/uwGFsxqZP7sBubPamDB7EbaZzUwf1YjC2Y3MH92IzMb/B9As0pRliEdSTXAl4FfArYD6yXdP2KqwluA/RHxdkk3Af8N+Iik5RTmwH0ncCHwT5J+JiJ8mkmZ1dbk6JjXRMe8Jn7hZ9pP6xsYzLNt/zG27jnKlj1H2bbvGH2HT9J3+CRPvb6f3YdOMjCYP+M9m+prmNVYS2NdDTPqamioq2FGXe7N1411NTQmrxvraqivyVFfm3vr78jntTkaanLU1uSoyUFOoiYnchK1NaJGIpcr/K3JvdWXU2E4K6fCOrlc0XMJicKDt5YVSZv/J2P2plJ24VYCmyJiM4Cke4EbOX0y8huBLyTPvw38hQr/0m4E7o2Ik8AWSZuS93u8POVbKeprc1zcPpOL22eO2h8RHDo+SN/hE+w+dJK+wycKPwiHTnL05CDHTw1x4tQQx08NcfJUnn1HBzg+MMSJwSGOD+Q5earw/NTQ9BsehOEfgxE/BBQah19DoW3ksknHm8+V/MAkzW/+oKjosyhqKV729P63PpMx+4q3YewfrtG6xlpcnNkx6vpjftpon1Xa0hP66Z2C3+mp+Okv9w7F3KZ67vv3V5T1PYuVEvgLgW1Fr7cD7x1rmYgYlHQQmJe0rxux7sLRPkTSamA1QEdHRym1W5lIYk5THXOa6li6YNak3yefDwaG8oXHYNFj6PS/Q/lgMB/k88FQPhiK5G8+yCfPh/sDyEeQj8IPU2GZwvPh9uHhqogggkI/hedR1H56W+E1yW/U6csV+gvtQfGo5/AQ6MhlOO01p71mRD9n9L/1HqP3lb7eqCucpXm0Id2J/GyXeghwYu9Z/h2HKdkVmYI3ndU4tcOo02aQNiLWAGugMIafcjk2CbmcaMwVhnfMbPop5UrbHcDioteLkrZRl5FUC8yhcPC2lHXNzOw8KCXw1wNLJXVJqqdwEPb+EcvcD9ycPP8w8HAU/l92P3CTpAZJXcBS4CflKd3MzCZi3CGdZEz+NuBBCqdlro2IFyTdAfRGxP3AXwFfTw7K7qPwo0Cy3H0UDvAOArf6DB0zs3T4wiszsypQynn4vlummVlGOPDNzDLCgW9mlhEOfDOzjJiWB20l9QOvTXL1NmBPGctJW7VtD1TfNlXb9kD1bVO1bQ+cuU1LIqJ9rIVhmgb+uZDUO96R6kpSbdsD1bdN1bY9UH3bVG3bA5PbJg/pmJllhAPfzCwjqjHw16RdQJlV2/ZA9W1TtW0PVN82Vdv2wCS2qerG8M3MbHTVuIdvZmajcOCbmWVE1QS+pOslvSxpk6TPpl1POUjaKuk5SRskVeTd5CStldQn6fmitrmSHpL0avK3Nc0aJ2KM7fmCpB3J97RB0i+nWeNESFos6RFJL0p6QdJnkvZK/o7G2qaK/J4kNUr6iaRnku35L0l7l6Qnksz7VnL7+rO/VzWM4ScTrb9C0UTrwKoRE61XHElbgZ6IqNgLRiS9HzgCfC0i3pW0/SGwLyK+lPw4t0bE76VZZ6nG2J4vAEci4r+nWdtkSLoAuCAinpI0C3gS+BXgE1TudzTWNv0GFfg9JfODN0fEEUl1wL8AnwF+F/huRNwr6U7gmYj4ytneq1r28N+caD0iBoDhidYtZRHxKIU5EordCHw1ef5VCv8YK8IY21OxImJnRDyVPD8MbKQw73Qlf0djbVNFioIjycu65BHAtcC3k/aSvqNqCfzRJlqv2C+4SADfl/RkMsl7tVgQETuT57uABWkWUya3SXo2GfKpmOGPYpI6gW7gCarkOxqxTVCh35OkGkkbgD7gIeCnwIGIGEwWKSnzqiXwq9X7IuLdwA3ArclwQlVJpsKs9HHFrwAXAyuAncD/SLWaSZA0E/gO8DsRcai4r1K/o1G2qWK/p4gYiogVFOYFXwksm8z7VEvgV+Vk6RGxI/nbB3yPwhddDXYn46zD4619KddzTiJid/IPMg/8JRX2PSXjwt8B/iYivps0V/R3NNo2Vfr3BBARB4BHgCuAFknD09SWlHnVEvilTLReUSQ1JweckNQMXAc8f/a1KkbxpPc3A3+XYi3nbDgYE79KBX1PyQHBvwI2RsQfF3VV7Hc01jZV6vckqV1SS/J8BoWTUzZSCP4PJ4uV9B1VxVk6AMkpVn/KWxOt/9d0Kzo3ki6isFcPhcnmv1mJ2yTpHuBqCrdy3Q3cDvwtcB/QQeE22L8RERVxIHSM7bmawjBBAFuBTxeNf09rkt4H/Ah4Dsgnzb9PYcy7Ur+jsbZpFRX4PUn6WQoHZWso7KTfFxF3JBlxLzAXeBr4WEScPOt7VUvgm5nZ2VXLkI6ZmY3DgW9mlhEOfDOzjHDgm5llhAPfzCwjHPhmZhnhwDczy4j/D4adppAk96RuAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(tr_loss_hist)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}