Lab12/Lab12-3_ChanjaeYu.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Lab 12-3 many to many\n",
    "\n",
    "### simple pos-tagger training\n",
    "\n",
    "- many to many\n",
    "- variable input sequence length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.4.1\n"
     ]
    }
   ],
   "source": [
    "# setup\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import matplotlib.pyplot as plt\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras import layers\n",
    "from tensorflow.keras import Sequential, Model\n",
    "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
    "from pprint import pprint\n",
    "%matplotlib inline\n",
    "\n",
    "print(tf.__version__)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Preparing Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# example data\n",
    "sentences = [['I', 'feel', 'hungry'],\n",
    "     ['tensorflow', 'is', 'very', 'difficult'],\n",
    "     ['tensorflow', 'is', 'a', 'framework', 'for', 'deep', 'learning'],\n",
    "     ['tensorflow', 'is', 'very', 'fast', 'changing']]\n",
    "pos = [['pronoun', 'verb', 'adjective'],\n",
    "     ['noun', 'verb', 'adverb', 'adjective'],\n",
    "     ['noun', 'verb', 'determiner', 'noun', 'preposition', 'adjective', 'noun'],\n",
    "     ['noun', 'verb', 'adverb', 'adjective', 'verb']]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Preprocessing data set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'<pad>': 0, 'I': 1, 'a': 2, 'changing': 3, 'deep': 4, 'difficult': 5, 'fast': 6, 'feel': 7, 'for': 8, 'framework': 9, 'hungry': 10, 'is': 11, 'learning': 12, 'tensorflow': 13, 'very': 14}\n",
      "{0: '<pad>', 1: 'I', 2: 'a', 3: 'changing', 4: 'deep', 5: 'difficult', 6: 'fast', 7: 'feel', 8: 'for', 9: 'framework', 10: 'hungry', 11: 'is', 12: 'learning', 13: 'tensorflow', 14: 'very'}\n",
      "15\n"
     ]
    }
   ],
   "source": [
    "# creating a token dictionary for word\n",
    "word_list = sum(sentences, [])\n",
    "word_list = sorted(set(word_list))\n",
    "word_list = ['<pad>'] + word_list\n",
    "word2idx = {word : idx for idx, word in enumerate(word_list)}\n",
    "idx2word = {idx : word for idx, word in enumerate(word_list)}\n",
    "\n",
    "print(word2idx)\n",
    "print(idx2word)\n",
    "print(len(idx2word))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'<pad>': 0, 'adjective': 1, 'adverb': 2, 'determiner': 3, 'noun': 4, 'preposition': 5, 'pronoun': 6, 'verb': 7}\n",
      "{0: '<pad>', 1: 'adjective', 2: 'adverb', 3: 'determiner', 4: 'noun', 5: 'preposition', 6: 'pronoun', 7: 'verb'}\n",
      "8\n"
     ]
    }
   ],
   "source": [
    "# creating a token dictionary for part of speech\n",
    "pos_list = sum(pos, [])\n",
    "pos_list = sorted(set(pos_list))\n",
    "pos_list = ['<pad>'] + pos_list\n",
    "pos2idx = {pos : idx for idx, pos in enumerate(pos_list)}\n",
    "idx2pos = {idx : pos for idx, pos in enumerate(pos_list)}\n",
    "\n",
    "print(pos2idx)\n",
    "print(idx2pos)\n",
    "print(len(pos2idx))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 1  7 10  0  0  0  0  0  0  0]\n",
      " [13 11 14  5  0  0  0  0  0  0]\n",
      " [13 11  2  9  8  4 12  0  0  0]\n",
      " [13 11 14  6  3  0  0  0  0  0]] [3, 4, 7, 5]\n",
      "[[1. 1. 1. 0. 0. 0. 0. 0. 0. 0.]\n",
      " [1. 1. 1. 1. 0. 0. 0. 0. 0. 0.]\n",
      " [1. 1. 1. 1. 1. 1. 1. 0. 0. 0.]\n",
      " [1. 1. 1. 1. 1. 0. 0. 0. 0. 0.]]\n",
      "[[6 7 1 0 0 0 0 0 0 0]\n",
      " [4 7 2 1 0 0 0 0 0 0]\n",
      " [4 7 3 4 5 1 4 0 0 0]\n",
      " [4 7 2 1 7 0 0 0 0 0]]\n"
     ]
    }
   ],
   "source": [
    "# converting sequence of tokens to sequence of indices\n",
    "max_sequence = 10\n",
    "x_data = list(map(lambda sentence : [word2idx.get(token) for token in sentence], sentences))\n",
    "y_data = list(map(lambda sentence : [pos2idx.get(token) for token in sentence], pos))\n",
    "\n",
    "# padding the sequence of indices\n",
    "x_data = pad_sequences(sequences = x_data, maxlen = max_sequence, padding='post')\n",
    "x_data_mask = ((x_data != 0) * 1).astype(np.float32)\n",
    "x_data_len = list(map(lambda sentence : len(sentence), sentences))\n",
    "\n",
    "y_data = pad_sequences(sequences = y_data, maxlen = max_sequence, padding='post')\n",
    "\n",
    "# checking data\n",
    "print(x_data, x_data_len)\n",
    "print(x_data_mask)\n",
    "print(y_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Creating Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# creating rnn for \"many to many\" sequence tagging\n",
    "num_classes = len(pos2idx)\n",
    "hidden_dim = 10\n",
    "\n",
    "input_dim = len(word2idx)\n",
    "output_dim = len(word2idx)\n",
    "one_hot = np.eye(len(word2idx))\n",
    "\n",
    "model = Sequential()\n",
    "model.add(layers.Embedding(input_dim=input_dim, output_dim=output_dim, mask_zero=True,\n",
    "                           trainable=False, input_length=max_sequence,\n",
    "                           embeddings_initializer=keras.initializers.Constant(one_hot)))\n",
    "model.add(layers.SimpleRNN(units=hidden_dim, return_sequences=True))\n",
    "model.add(layers.TimeDistributed(layers.Dense(units=num_classes)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding (Embedding)        (None, 10, 15)            225       \n",
      "_________________________________________________________________\n",
      "simple_rnn (SimpleRNN)       (None, 10, 10)            260       \n",
      "_________________________________________________________________\n",
      "time_distributed (TimeDistri (None, 10, 8)             88        \n",
      "=================================================================\n",
      "Total params: 573\n",
      "Trainable params: 348\n",
      "Non-trainable params: 225\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# creating loss function\n",
    "def loss_fn(model, x, y, x_len, max_sequence):\n",
    "    masking = tf.sequence_mask(x_len, maxlen=max_sequence, dtype=tf.float32)\n",
    "    valid_time_step = tf.cast(x_len,dtype=tf.float32)    \n",
    "    sequence_loss = tf.keras.losses.sparse_categorical_crossentropy(\n",
    "        y_true=y, y_pred=model(x), from_logits=True) * masking    \n",
    "    sequence_loss = tf.reduce_sum(sequence_loss, axis=-1) / valid_time_step    \n",
    "    sequence_loss = tf.reduce_mean(sequence_loss)    \n",
    "    return sequence_loss\n",
    "\n",
    "# creating and optimizer\n",
    "lr = 0.1\n",
    "epochs = 30\n",
    "batch_size = 2 \n",
    "opt = tf.keras.optimizers.Adam(learning_rate = lr)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<BatchDataset shapes: ((None, 10), (None, 10), (None,)), types: (tf.int32, tf.int32, tf.int32)>\n"
     ]
    }
   ],
   "source": [
    "# generating data pipeline\n",
    "tr_dataset = tf.data.Dataset.from_tensor_slices((x_data, y_data, x_data_len))\n",
    "tr_dataset = tr_dataset.shuffle(buffer_size=4)\n",
    "tr_dataset = tr_dataset.batch(batch_size = 2)\n",
    "\n",
    "print(tr_dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch :   5, tr_loss : 0.107\n",
      "epoch :  10, tr_loss : 0.008\n",
      "epoch :  15, tr_loss : 0.002\n",
      "epoch :  20, tr_loss : 0.001\n",
      "epoch :  25, tr_loss : 0.001\n",
      "epoch :  30, tr_loss : 0.001\n"
     ]
    }
   ],
   "source": [
    "# training\n",
    "tr_loss_hist = []\n",
    "\n",
    "for epoch in range(epochs):\n",
    "    avg_tr_loss = 0\n",
    "    tr_step = 0\n",
    "    \n",
    "    for x_mb, y_mb, x_mb_len in tr_dataset:\n",
    "        with tf.GradientTape() as tape:\n",
    "            tr_loss = loss_fn(model, x=x_mb, y=y_mb, x_len=x_mb_len, max_sequence=max_sequence)\n",
    "        grads = tape.gradient(target=tr_loss, sources=model.variables)\n",
    "        opt.apply_gradients(grads_and_vars=zip(grads, model.variables))\n",
    "        avg_tr_loss += tr_loss\n",
    "        tr_step += 1\n",
    "    else:\n",
    "        avg_tr_loss /= tr_step\n",
    "        tr_loss_hist.append(avg_tr_loss)\n",
    "    \n",
    "    if (epoch + 1) % 5 == 0:\n",
    "        print('epoch : {:3}, tr_loss : {:.3f}'.format(epoch + 1, avg_tr_loss))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Checking performance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[['pronoun', 'verb', 'adjective', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>'],\n",
      " ['noun', 'verb', 'adverb', 'adjective', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>'],\n",
      " ['noun', 'verb', 'determiner', 'noun', 'preposition', 'adjective', 'noun', '<pad>', '<pad>', '<pad>'],\n",
      " ['noun', 'verb', 'adverb', 'adjective', 'verb', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>']]\n",
      "[['pronoun', 'verb', 'adjective'],\n",
      " ['noun', 'verb', 'adverb', 'adjective'],\n",
      " ['noun', 'verb', 'determiner', 'noun', 'preposition', 'adjective', 'noun'],\n",
      " ['noun', 'verb', 'adverb', 'adjective', 'verb']]\n"
     ]
    }
   ],
   "source": [
    "yhat = model.predict(x_data)\n",
    "yhat = np.argmax(yhat, axis=-1) * x_data_mask\n",
    "\n",
    "pprint(list(map(lambda row : [idx2pos.get(elm) for elm in row],yhat.astype(np.int32).tolist())), width = 120)\n",
    "pprint(pos)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x268a34ed220>]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXwAAAD4CAYAAADvsV2wAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAaR0lEQVR4nO3dfXAc933f8ffnDk8kCJCQcJAlPkoyXYdyRFlEZNlybKnTypTHGSaZOBGTOInrlFVrzaTTSSdKZ2qn9nTiaepOa0u2hnZkOW0jRYktW51IljyJE9lWZAukKZm0nmjqgSBlERTFR5AEgfv2j1uQR/AAHMADF7f7ec3c4Pa3v7377tzwc8vf7W9XEYGZmWVfIe0CzMzswnDgm5nlhAPfzCwnHPhmZjnhwDczy4mWtAuopbe3N1atWpV2GWZmTWPLli37I6I0VZ95GfirVq1iYGAg7TLMzJqGpFem6+MhHTOznHDgm5nlhAPfzCwnHPhmZjnhwDczywkHvplZTjjwzcxyIjOBHxF8/u9e5B9fGEq7FDOzeSkzgS+Jzd/dxXee25d2KWZm81JmAh+gr6udfUdOpF2Gmdm8lKnAL3W1s+/wybTLMDOblzIV+H1dHQwddeCbmdWSscCvHOH7Pr1mZufKVOCXuto5fmqMoydH0y7FzGzeyVTg93W3AzB0xMM6ZmYTZSvwuzoA2OfANzM7R6YCv9TlI3wzs8lkKvD7ksD3Eb6Z2bkyFfiLF7TSVix48pWZWQ2ZCnxJlLraPaRjZlZDpgIfcOCbmU0ik4HvyyuYmZ0rc4Hf19XuyyuYmdWQwcDv4MCxEUZGy2mXYmY2r2Qu8MfPxd/vo3wzs7NMG/iS7pG0T9L2SdbfKOmQpG3J4xNV69ZLel7STkl3NLLwyfR58pWZWU31HOHfC6yfps93I+Ka5PEpAElF4C7gFmANsFHSmvMpth7j19Px5Cszs7NNG/gR8ThwYBavfR2wMyJ2RcQIcD+wYRavMyOl07NtPfnKzKxao8bw3y3paUmPSLoqaVsK7K7qM5i01SRpk6QBSQNDQ7O/EXnvonYkD+mYmU3UiMDfCqyMiLXA54FvJO2q0XfSO5NExOaI6I+I/lKpNOtiWosFLlrY5iEdM7MJzjvwI+JwRBxNnj8MtErqpXJEv7yq6zJg7/m+Xz0829bM7FznHfiS3iJJyfPrktd8A3gKWC3pckltwK3AQ+f7fvUodbX7CN/MbIKW6TpIug+4EeiVNAh8EmgFiIi7gV8D/q2kUeA4cGtUbio7Kul24FGgCNwTETvmZC8m6Ovq4Kf79l+ItzIzaxrTBn5EbJxm/Z3AnZOsexh4eHalzV4pubxCRJD858PMLPcyN9MWKpOvTo0FB4dPpV2Kmdm8kcnAL/nOV2Zm58hk4PvyCmZm58pm4Hd3AJ5ta2ZWLZOB7yEdM7NzZTLwF7W3sLCt6CEdM7MqmQx8qIzj+wjfzOyMzAZ+5d62HsM3MxuX2cDv6+rwvW3NzKpkNvBLXe0MHXbgm5mNy3TgHzk5yvGRsbRLMTObFzIb+J58ZWZ2tuwGvidfmZmdJbOBX1rkI3wzs2qZDfy+bs+2NTOrltnAv2hhG8WCPKRjZpbIbOAXCqJ3UZuHdMzMEpkNfKhMvvKQjplZRaYDv3J5BQe+mRlkPPD7knvbmplZHYEv6R5J+yRtn2T9b0l6Jnk8IWlt1bqXJf1Y0jZJA40svB59Xe28cfQkY+W40G9tZjbv1HOEfy+wfor1LwHvj4irgU8DmyesvykiromI/tmVOHulrnbKAW/4KN/MbPrAj4jHgQNTrH8iIt5MFp8EljWotvNW6hqfbevANzNr9Bj+x4BHqpYDeEzSFkmbGvxe0xqffOVTM83MoKVRLyTpJiqB/96q5hsiYq+kPuDbkp5L/sdQa/tNwCaAFStWNKSm8csrePKVmVmDjvAlXQ18GdgQEW+Mt0fE3uTvPuBB4LrJXiMiNkdEf0T0l0qlRpR1+mbmPsI3M2tA4EtaAXwd+EhEvFDV3impa/w5cDNQ80yfudLRWmTxglaP4ZuZUceQjqT7gBuBXkmDwCeBVoCIuBv4BHAx8AVJAKPJGTmXAA8mbS3AX0bEt+ZgH6ZU6mr3Eb6ZGXUEfkRsnGb97wO/X6N9F7D23C0urL6udh/hm5mR8Zm2kFxewT/ampllP/D7kiGdCM+2NbN8y0Hgd3DiVJkjJ0fTLsXMLFWZD/zxUzN91Uwzy7vMB36fz8U3MwPyEPjdnm1rZgY5CPzSosoF1HyEb2Z5l/nA717QQltLwYFvZrmX+cCX5MlXZmbkIPDBk6/MzCAngd/n6+mYmeUl8Ds8pGNmuZeLwC91tXNw+BQnR8fSLsXMLDW5CPzxyVf7j46kXImZWXpyEfhnLq/gH27NLL9yEfh9XZ58ZWaWj8A/fXkFB76Z5VcuAv/izjYkB76Z5VsuAr+lWODizjYP6ZhZruUi8AFKXR0MebatmeVYjgLf19Mxs3ybNvAl3SNpn6Ttk6yXpM9J2inpGUnXVq1bL+n5ZN0djSx8pnx5BTPLu3qO8O8F1k+x/hZgdfLYBHwRQFIRuCtZvwbYKGnN+RR7PsYDv1z2zczNLJ+mDfyIeBw4MEWXDcBfRMWTwBJJlwLXATsjYldEjAD3J31TUepqZ7QcvDns2bZmlk+NGMNfCuyuWh5M2iZrr0nSJkkDkgaGhoYaUNbZTk++OuphHTPLp0YEvmq0xRTtNUXE5ojoj4j+UqnUgLLOdnry1WEHvpnlU0sDXmMQWF61vAzYC7RN0p6K0iLPtjWzfGvEEf5DwO8kZ+tcDxyKiNeAp4DVki6X1AbcmvRNxfgF1Hymjpnl1bRH+JLuA24EeiUNAp8EWgEi4m7gYeCDwE5gGPhosm5U0u3Ao0ARuCcidszBPtSls72Fzraib3VoZrk1beBHxMZp1gfw8UnWPUzlC2Fe6Ovu8BG+meVWbmbagmfbmlm+5S7wfYRvZnmVq8D35RXMLM9yFvgdHD05yvDIaNqlmJldcLkK/DP3tvVRvpnlT64Cv2/8XHxfXsHMcihfge/LK5hZjuUq8M9cXsGTr8wsf3IV+D0L22gpyGfqmFku5SrwCwV58pWZ5VauAh8829bM8it3ge/JV2aWV7kL/FJXB0P+0dbMciiHgd/OG8dGGB0rp12KmdkFlbvA7+tqJwLeOOabmZtZvuQu8H15BTPLq9wF/pnLK3gc38zyJX+B390B+AjfzPInd4Hfu6gNwOfim1nu5C7w21uKLFnY6nPxzSx3chf4UBnH9wXUzCxv6gp8SeslPS9pp6Q7aqz/j5K2JY/tksYkXZSse1nSj5N1A43egdnw5RXMLI+mDXxJReAu4BZgDbBR0prqPhHxZxFxTURcA/wx8I8RcaCqy03J+v7GlT57fV0dHtIxs9yp5wj/OmBnROyKiBHgfmDDFP03Avc1ori50pcc4UdE2qWYmV0w9QT+UmB31fJg0nYOSQuB9cDXqpoDeEzSFkmbJnsTSZskDUgaGBoaqqOs2Vvas4CR0TKvHfI4vpnlRz2Brxptkx0a/xLw/QnDOTdExLVUhoQ+Lul9tTaMiM0R0R8R/aVSqY6yZu+dy3sA2Prqm3P6PmZm80k9gT8ILK9aXgbsnaTvrUwYzomIvcnffcCDVIaIUvX2S7tY0FpkyysOfDPLj3oC/ylgtaTLJbVRCfWHJnaStBh4P/DNqrZOSV3jz4Gbge2NKPx8tBYLrF2+2IFvZrkybeBHxChwO/Ao8CzwQETskHSbpNuquv4K8FhEHKtquwT4nqSngR8CfxsR32pc+bO3bmUPO/YeZnhkNO1SzMwuiJZ6OkXEw8DDE9runrB8L3DvhLZdwNrzqnCOrFvZw1g5eGbwENdfcXHa5ZiZzblczrSFMz/celjHzPIit4Hf09nGlaVOtjrwzSwnchv4UBnW2fLqm56AZWa5kPvAPzh8il37j03f2cysyeU88C8CPI5vZvmQ68C/oreTJQtbPY5vZrmQ68AvFMS1K3oYcOCbWQ7kOvChMo6/c99RDg6PpF2Kmdmcyn3gX7uicj7+j149mG4hZmZzLPeBv3b5YooF+YdbM8u83Af+wrYW1lza7cA3s8zLfeBDZRx/2+6DjI6V0y7FzGzOOPCBa1f2cPzUGM/97EjapZiZzRkHPpUjfPAELDPLNgc+sHTJAi5d3OHAN7NMc+Anrl3Z48A3s0xz4CfWrehhz8HjvHboeNqlmJnNCQd+Ynwcf+srB9MtxMxsjjjwE2su66ajteBhHTPLLAd+orVY4OplS9jyqgPfzLKprsCXtF7S85J2SrqjxvobJR2StC15fKLebeeTdSt72LHnECdOjaVdiplZw00b+JKKwF3ALcAaYKOkNTW6fjcirkken5rhtvPCuhU9jJaDZwYPpV2KmVnD1XOEfx2wMyJ2RcQIcD+woc7XP59tL7hrPQHLzDKsnsBfCuyuWh5M2iZ6t6SnJT0i6aoZbjsvXNTZxhW9nWx55UDapZiZNVw9ga8abTFheSuwMiLWAp8HvjGDbSsdpU2SBiQNDA0N1VHW3FiXTMCKqFmmmVnTqifwB4HlVcvLgL3VHSLicEQcTZ4/DLRK6q1n26rX2BwR/RHRXyqVZrALjbVuZQ9vDp/ipf3HUqvBzGwu1BP4TwGrJV0uqQ24FXiouoOkt0hS8vy65HXfqGfb+cYXUjOzrJo28CNiFLgdeBR4FnggInZIuk3SbUm3XwO2S3oa+Bxwa1TU3HYudqRRriwtorujha0+H9/MMqalnk7JMM3DE9rurnp+J3BnvdvOZ4WCfCE1M8skz7StYd2KHl54/SiHjp9KuxQzs4Zx4NcwPo7/Iw/rmFmGOPBrWLt8CQXBVg/rmFmGOPBr6Gxv4ecu7WbAgW9mGeLAn8S6lT1s232Q0bFy2qWYmTWEA38S61b2MDwyxnM/O5J2KWZmDeHAn8TpO2D5h1szywgH/iSWLlnAJd3tPh/fzDLDgT8JSacvpGZmlgUO/Clcu6KHwTeP8/rhE2mXYmZ23hz4Uzg9ju+jfDPLAAf+FK66bDELWos8sv1naZdiZnbeHPhTaGsp8NEbVvHQ03vZvsf3uTWz5ubAn8ZtN15Jz8JWPvPIc2mXYmZ2Xhz40+juaOX2f76a7+3cz+MvpHfrRTOz8+XAr8NvX7+C5Rct4E8feY5y2fe6NbPm5MCvQ3tLkT+8+Z/x7GuH+ca2PWmXY2Y2Kw78Ov3S1Zfx80sX89nHXuDEqbG0yzEzmzEHfp0KBfHHt7ydPQeP87//6ZW0yzEzmzEH/gy85629vP9tJe78zk4ODfv2h2bWXBz4M3THLW/n8IlTfOEfdqZdipnZjNQV+JLWS3pe0k5Jd9RY/1uSnkkeT0haW7XuZUk/lrRN0kAji0/Dz13aza++cxlfeeJl9hw8nnY5ZmZ1mzbwJRWBu4BbgDXARklrJnR7CXh/RFwNfBrYPGH9TRFxTUT0N6Dm1P2Hm98GwGcfez7lSszM6lfPEf51wM6I2BURI8D9wIbqDhHxRESMX2HsSWBZY8ucX5YuWcBH37OKB3+0h5/sPZx2OWZmdakn8JcCu6uWB5O2yXwMeKRqOYDHJG2RtGmyjSRtkjQgaWBoaP7PaP13N76V7o5WPvMtX3LBzJpDPYGvGm01p5tKuolK4P9RVfMNEXEtlSGhj0t6X61tI2JzRPRHRH+pVKqjrHQtXtjK7Te9lcdfGOL7O/enXY6Z2bTqCfxBYHnV8jJg78ROkq4GvgxsiIg3xtsjYm/ydx/wIJUhokz4yLtXsnTJAv70kWd9yQUzm/fqCfyngNWSLpfUBtwKPFTdQdIK4OvARyLihar2Tkld48+Bm4HtjSo+bR2tRf7wA29j+57D/L9nzvkONDObV6YN/IgYBW4HHgWeBR6IiB2SbpN0W9LtE8DFwBcmnH55CfA9SU8DPwT+NiK+1fC9SNGGtUtZc2k3f/bo85wc9SUXzGz+UsT8G4ro7++PgYHmOWX/uy8O8ZE//yH/+UNr+Nh7L0+7HDPLIUlbpjv13TNtG+AXV5f4xdW9fO7vXuTF14+kXY6ZWU0O/Ab59IZ30NZSYOOXfuDQN7N5yYHfIKt6O7nvX1+PhEPfzOYlB34DvbVvkUPfzOYtB36DOfTNbL5y4M8Bh76ZzUcO/Dni0Dez+caBP4fe2reI+zc59M1sfnDgz7ErSw59M5sfHPgXgEPfzOYDB/4FcnboP8kTP93PfLyshZlllwP/AhoP/dZigd/80g/40Oe/x99sGfRF18zsgvDF01JwfGSMb2zbw1e+/xIvvH6U3kVt/Oa7VvLb16+gr6sj7fLMrAnVc/E0B36KIoLv73yDr3z/Jf7++X20FMSHrr6Mf3XD5fz8ssVpl2dmTaSewG+5UMXYuSTx3tW9vHd1Ly/vP8a9T7zMXw/s5sEf7aF/ZQ8fveFyPnDVJbQUPfJmZufPR/jzzOETp/jrgUG++sTLvHpgmEsXd/Dhdcv4cP9yll+0MO3yzGye8pBOExsrB3//3D7+z5Ov8PiLQ0TAe668mN/4heV84Kq30NFaTLtEM5tHHPgZsffgcf5myyAPDOxm8M3jdHe08MvvXMqv9y/nHUs91m9mDvzMKZeDf9r1Bn/11G6+teNnjIyWueqybn7jF5azYe1SFi9sTbtEM0uJAz/DDg6P8M1te/mrp3bzk9cO095S4KrLurmitIgrSp1c0buIK0udrLh4Ie0tHv4xyzoHfk5s33OIr2/dw469h9i1/xhDR06eXlcQLOtZePpL4IpSJ6su7uSS7nb6ujroXtCCpBSrN7NGaNhpmZLWA/8LKAJfjojPTFivZP0HgWHg9yJiaz3b2vl7x9LFZ43lHzlxipf2H2PX0DF2DR3lp/uP8dLQMX6w6wDHT509q7etpUBpUTt93e30dVW+BPq6KsulrnYWL2ils72FzrYWOttbWNhWpL2l4C8JsyY0beBLKgJ3Af8SGASekvRQRPykqtstwOrk8S7gi8C76tzWGqyro5Wrly3h6mVLzmovl4OfHT7BqweG2XfkJPsOn2DoyMnK8yMneGn/MX7w0gEODp+a8vVbCkq+BIqVL4H2Fha2FmlrKZx+tBfPPG9LnrcWzywXC6K1KIqFAi1F0VIQLcUCrQUl6yp9igVRkGgpVv4WC6I4/rcgigVOtxeSL6FCQRRUaZdATFg+3Z48h9P9Kuurnk/s4y86a2L1HOFfB+yMiF0Aku4HNgDVob0B+IuojA89KWmJpEuBVXVsaxdIoSAuW7KAy5YsmLLfydGx018ER06MMnxylKMnRxkeGePoyVGOVT0fHhnl6Mkxjo+MMjw8ysnRMiNjZUZGk0fy/NRYmVNj82/4cLYm+8LgdHvSr8YXR7Li9J/xLxHprFVn2ie8b9VSjbap+ic1Trquerv6v9hqda3ZxrmNk71Nve8+ozob3rHxdV60sI0Hbnt3/QXMUD2BvxTYXbU8SOUofro+S+vcFgBJm4BNACtWrKijLJsr7S1FlvUsZFlPYyd6lcvByFiZ0XIwevpvcGqszFg5GC2faRvvM1YOxiIol2G0XKYcwVi5Mk/hzLqgHEE5oBwByd/x5YggkvcvV1ZX2gKC8b+cvRyTtFNpqNU+vkzyvXb2+1TWV9orT6p/Phv/LS1OL5/d95z+NdrOXnPuurO3jynWnWuyn/omvs5kL1D7NWu/aL2HBTP5+bH+16z/RevuOYM6uzrm9uIH9bx6ra+mibswWZ96tq00RmwGNkPlR9s66rImUyiIjoLPGDJLSz2BPwgsr1peBuyts09bHduamdkFUM9VuZ4CVku6XFIbcCvw0IQ+DwG/o4rrgUMR8Vqd25qZ2QUw7RF+RIxKuh14lMqplfdExA5JtyXr7wYepnJK5k4qp2V+dKpt52RPzMxsSp54ZWaWAfVMvPKF1s3McsKBb2aWEw58M7OccOCbmeXEvPzRVtIQ8MosN+8F9jewnLRlbX8ge/uUtf2B7O1T1vYHzt2nlRFRmmqDeRn450PSwHS/VDeTrO0PZG+fsrY/kL19ytr+wOz2yUM6ZmY54cA3M8uJLAb+5rQLaLCs7Q9kb5+ytj+QvX3K2v7ALPYpc2P4ZmZWWxaP8M3MrAYHvplZTmQm8CWtl/S8pJ2S7ki7nkaQ9LKkH0vaJqnpriYn6R5J+yRtr2q7SNK3Jb2Y/O1Js8aZmmSf/kTSnuRz2ibpg2nWOBOSlkv6jqRnJe2Q9AdJe9N+TlPsU1N+TpI6JP1Q0tPJ/vyXpH3Gn1EmxvCTm6W/QNXN0oGNzX6zdEkvA/0R0ZQTRiS9DzhK5X7H70ja/htwICI+k3wx90TEH6VZ50xMsk9/AhyNiP+eZm2zkdx7+tKI2CqpC9gC/DLwezTp5zTFPv06Tfg5qXJD3M6IOCqpFfge8AfArzLDzygrR/inb7QeESPA+M3SLUUR8ThwYELzBuCryfOvUvmH2DQm2aemFRGvRcTW5PkR4Fkq96Ju2s9pin1qSlFxNFlsTR7BLD6jrAT+ZDdRb3YBPCZpS3KT9yy4JLkbGsnfvpTraZTbJT2TDPk0zfBHNUmrgHcCPyAjn9OEfYIm/ZwkFSVtA/YB346IWX1GWQn8um+W3mRuiIhrgVuAjyfDCTb/fBG4ErgGeA34bKrVzIKkRcDXgH8fEYfTrqcRauxT035OETEWEddQuS/4dZLeMZvXyUrg13Oj9aYTEXuTv/uAB6kMXTW715Mx1vGx1n0p13PeIuL15B9kGfgSTfY5JePCXwP+b0R8PWlu6s+p1j41++cEEBEHgX8A1jOLzygrgZ+5m6VL6kx+cEJSJ3AzsH3qrZrCQ8DvJs9/F/hmirU0xPg/usSv0ESfU/KD4J8Dz0bE/6ha1bSf02T71Kyfk6SSpCXJ8wXAvwCeYxafUSbO0gFITrH6n5y5Wfp/Tbei8yPpCipH9VC52fxfNts+SboPuJHKZVxfBz4JfAN4AFgBvAp8OCKa5kfQSfbpRirDBAG8DPyb8bHV+U7Se4HvAj8Gyknzf6Iy5t2Un9MU+7SRJvycJF1N5UfZIpWD9Aci4lOSLmaGn1FmAt/MzKaWlSEdMzObhgPfzCwnHPhmZjnhwDczywkHvplZTjjwzcxywoFvZpYT/x+DJwHDmmw9LQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(tr_loss_hist)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}