Lab12/Lab 12-3 eunseoChoi.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# lab-12-3-many-to-many-keras-eager"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## simple pos-tagger training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.5.0\n"
     ]
    }
   ],
   "source": [
    "# setup\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import matplotlib.pyplot as plt\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras import layers\n",
    "from tensorflow.keras import Sequential, Model\n",
    "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
    "from pprint import pprint\n",
    "%matplotlib inline\n",
    "\n",
    "print(tf.__version__)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prepairing dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# example data\n",
    "sentences = [['I', 'feel', 'hungry'],\n",
    "     ['tensorflow', 'is', 'very', 'difficult'],\n",
    "     ['tensorflow', 'is', 'a', 'framework', 'for', 'deep', 'learning'],\n",
    "     ['tensorflow', 'is', 'very', 'fast', 'changing']]\n",
    "pos = [['pronoun', 'verb', 'adjective'],\n",
    "     ['noun', 'verb', 'adverb', 'adjective'],\n",
    "     ['noun', 'verb', 'determiner', 'noun', 'preposition', 'adjective', 'noun'],\n",
    "     ['noun', 'verb', 'adverb', 'adjective', 'verb']]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preprocessing dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'<pad>': 0, 'I': 1, 'a': 2, 'changing': 3, 'deep': 4, 'difficult': 5, 'fast': 6, 'feel': 7, 'for': 8, 'framework': 9, 'hungry': 10, 'is': 11, 'learning': 12, 'tensorflow': 13, 'very': 14}\n",
      "{0: '<pad>', 1: 'I', 2: 'a', 3: 'changing', 4: 'deep', 5: 'difficult', 6: 'fast', 7: 'feel', 8: 'for', 9: 'framework', 10: 'hungry', 11: 'is', 12: 'learning', 13: 'tensorflow', 14: 'very'}\n",
      "15\n"
     ]
    }
   ],
   "source": [
    "# creating a token dictionary for word\n",
    "word_list = sum(sentences, [])\n",
    "word_list = sorted(set(word_list))\n",
    "word_list = ['<pad>'] + word_list\n",
    "word2idx = {word : idx for idx, word in enumerate(word_list)}\n",
    "idx2word = {idx : word for idx, word in enumerate(word_list)}\n",
    "\n",
    "print(word2idx)\n",
    "print(idx2word)\n",
    "print(len(idx2word))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'<pad>': 0, 'adjective': 1, 'adverb': 2, 'determiner': 3, 'noun': 4, 'preposition': 5, 'pronoun': 6, 'verb': 7}\n",
      "{0: '<pad>', 1: 'adjective', 2: 'adverb', 3: 'determiner', 4: 'noun', 5: 'preposition', 6: 'pronoun', 7: 'verb'}\n",
      "8\n"
     ]
    }
   ],
   "source": [
    "# creating a token dictionary for part of speech\n",
    "pos_list = sum(pos, [])\n",
    "pos_list = sorted(set(pos_list))\n",
    "pos_list = ['<pad>'] + pos_list\n",
    "pos2idx = {pos : idx for idx, pos in enumerate(pos_list)}\n",
    "idx2pos = {idx : pos for idx, pos in enumerate(pos_list)}\n",
    "\n",
    "print(pos2idx)\n",
    "print(idx2pos)\n",
    "print(len(pos2idx))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 1  7 10  0  0  0  0  0  0  0]\n",
      " [13 11 14  5  0  0  0  0  0  0]\n",
      " [13 11  2  9  8  4 12  0  0  0]\n",
      " [13 11 14  6  3  0  0  0  0  0]] [3, 4, 7, 5]\n",
      "[[1. 1. 1. 0. 0. 0. 0. 0. 0. 0.]\n",
      " [1. 1. 1. 1. 0. 0. 0. 0. 0. 0.]\n",
      " [1. 1. 1. 1. 1. 1. 1. 0. 0. 0.]\n",
      " [1. 1. 1. 1. 1. 0. 0. 0. 0. 0.]]\n",
      "[[6 7 1 0 0 0 0 0 0 0]\n",
      " [4 7 2 1 0 0 0 0 0 0]\n",
      " [4 7 3 4 5 1 4 0 0 0]\n",
      " [4 7 2 1 7 0 0 0 0 0]]\n"
     ]
    }
   ],
   "source": [
    "# converting sequence of tokens to sequence of indices\n",
    "max_sequence = 10\n",
    "x_data = list(map(lambda sentence : [word2idx.get(token) for token in sentence], sentences))\n",
    "y_data = list(map(lambda sentence : [pos2idx.get(token) for token in sentence], pos))\n",
    "\n",
    "# padding the sequence of indices\n",
    "x_data = pad_sequences(sequences = x_data, maxlen = max_sequence, padding='post')\n",
    "x_data_mask = ((x_data != 0) * 1).astype(np.float32)\n",
    "x_data_len = list(map(lambda sentence : len(sentence), sentences))\n",
    "\n",
    "y_data = pad_sequences(sequences = y_data, maxlen = max_sequence, padding='post')\n",
    "\n",
    "# checking data\n",
    "print(x_data, x_data_len)\n",
    "print(x_data_mask)\n",
    "print(y_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Creating model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# creating rnn for \"many to many\" sequence tagging\n",
    "num_classes = len(pos2idx)\n",
    "hidden_dim = 10\n",
    "\n",
    "input_dim = len(word2idx)\n",
    "output_dim = len(word2idx)\n",
    "one_hot = np.eye(len(word2idx))\n",
    "\n",
    "model = Sequential()\n",
    "model.add(layers.Embedding(input_dim=input_dim, output_dim=output_dim, mask_zero=True,\n",
    "                           trainable=False, input_length=max_sequence,\n",
    "                           embeddings_initializer=keras.initializers.Constant(one_hot)))\n",
    "model.add(layers.SimpleRNN(units=hidden_dim, return_sequences=True))\n",
    "model.add(layers.TimeDistributed(layers.Dense(units=num_classes)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding (Embedding)        (None, 10, 15)            225       \n",
      "_________________________________________________________________\n",
      "simple_rnn (SimpleRNN)       (None, 10, 10)            260       \n",
      "_________________________________________________________________\n",
      "time_distributed (TimeDistri (None, 10, 8)             88        \n",
      "=================================================================\n",
      "Total params: 573\n",
      "Trainable params: 348\n",
      "Non-trainable params: 225\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# creating loss function\n",
    "def loss_fn(model, x, y, x_len, max_sequence):\n",
    "    masking = tf.sequence_mask(x_len, maxlen=max_sequence, dtype=tf.float32)\n",
    "    valid_time_step = tf.cast(x_len,dtype=tf.float32)    \n",
    "    sequence_loss = tf.keras.losses.sparse_categorical_crossentropy(\n",
    "        y_true=y, y_pred=model(x), from_logits=True) * masking    \n",
    "    sequence_loss = tf.reduce_sum(sequence_loss, axis=-1) / valid_time_step    \n",
    "    sequence_loss = tf.reduce_mean(sequence_loss)    \n",
    "    return sequence_loss\n",
    "\n",
    "# creating and optimizer\n",
    "lr = 0.1\n",
    "epochs = 30\n",
    "batch_size = 2 \n",
    "opt = tf.keras.optimizers.Adam(learning_rate = lr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<BatchDataset shapes: ((None, 10), (None, 10), (None,)), types: (tf.int32, tf.int32, tf.int32)>\n"
     ]
    }
   ],
   "source": [
    "# generating data pipeline\n",
    "tr_dataset = tf.data.Dataset.from_tensor_slices((x_data, y_data, x_data_len))\n",
    "tr_dataset = tr_dataset.shuffle(buffer_size=4)\n",
    "tr_dataset = tr_dataset.batch(batch_size = 2)\n",
    "\n",
    "print(tr_dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch :   5, tr_loss : 0.200\n",
      "epoch :  10, tr_loss : 0.010\n",
      "epoch :  15, tr_loss : 0.002\n",
      "epoch :  20, tr_loss : 0.001\n",
      "epoch :  25, tr_loss : 0.001\n",
      "epoch :  30, tr_loss : 0.001\n"
     ]
    }
   ],
   "source": [
    "# training\n",
    "tr_loss_hist = []\n",
    "\n",
    "for epoch in range(epochs):\n",
    "    avg_tr_loss = 0\n",
    "    tr_step = 0\n",
    "    \n",
    "    for x_mb, y_mb, x_mb_len in tr_dataset:\n",
    "        with tf.GradientTape() as tape:\n",
    "            tr_loss = loss_fn(model, x=x_mb, y=y_mb, x_len=x_mb_len, max_sequence=max_sequence)\n",
    "        grads = tape.gradient(target=tr_loss, sources=model.variables)\n",
    "        opt.apply_gradients(grads_and_vars=zip(grads, model.variables))\n",
    "        avg_tr_loss += tr_loss\n",
    "        tr_step += 1\n",
    "    else:\n",
    "        avg_tr_loss /= tr_step\n",
    "        tr_loss_hist.append(avg_tr_loss)\n",
    "    \n",
    "    if (epoch + 1) % 5 == 0:\n",
    "        print('epoch : {:3}, tr_loss : {:.3f}'.format(epoch + 1, avg_tr_loss))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Checking performance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[['pronoun', 'verb', 'adjective', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>'],\n",
      " ['noun', 'verb', 'adverb', 'adjective', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>'],\n",
      " ['noun', 'verb', 'determiner', 'noun', 'preposition', 'adjective', 'noun', '<pad>', '<pad>', '<pad>'],\n",
      " ['noun', 'verb', 'adverb', 'adjective', 'verb', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>']]\n",
      "[['pronoun', 'verb', 'adjective'],\n",
      " ['noun', 'verb', 'adverb', 'adjective'],\n",
      " ['noun', 'verb', 'determiner', 'noun', 'preposition', 'adjective', 'noun'],\n",
      " ['noun', 'verb', 'adverb', 'adjective', 'verb']]\n"
     ]
    }
   ],
   "source": [
    "yhat = model.predict(x_data)\n",
    "yhat = np.argmax(yhat, axis=-1) * x_data_mask\n",
    "\n",
    "pprint(list(map(lambda row : [idx2pos.get(elm) for elm in row],yhat.astype(np.int32).tolist())), width = 120)\n",
    "pprint(pos)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x212a85e5550>]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXwAAAD4CAYAAADvsV2wAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAdhklEQVR4nO3df5DcdZ3n8edrfmV+kJlkkiHJTDJJjg1KJgeBnY0gWxJK5YKuG7hTl5Tnr9OKWFClW5Yl59XprntX5a271q4nC2ZdDtlSkFsMRi8K7K4rIqBMIEACBsYYyC+SyQ/yO5nMzPv+6O8knUnPTCfTk57u7+tR1dXdn+/n2/3+Vlde/c2nP/P5KiIwM7PyV1HsAszM7MJw4JuZpYQD38wsJRz4ZmYp4cA3M0uJqmIXkMv06dNj3rx5xS7DzKxkrFu3bk9EtIzUZ0IG/rx58+jq6ip2GWZmJUPSa6P18ZCOmVlKOPDNzFLCgW9mlhIOfDOzlHDgm5mlhAPfzCwlHPhmZilRNoHfPxDc+bNuHn+lp9ilmJlNSKMGvqQ5kn4m6WVJGyV9JkcfSfqGpG5JL0i6KmvbMkmbkm13FPoABlVWiG/9/Lc89tKu8XoLM7OSls8Zfh/wuYi4DLgauE3SwiF9bgQWJLeVwF0AkiqBO5PtC4EVOfYtmPZp9by+7+h4vbyZWUkbNfAjYmdEPJs8PgS8DLQN6bYcuC8yngamSJoFLAG6I2JzRPQCDyR9x0V7cz1bHfhmZjmd0xi+pHnAlcCvhmxqA7ZmPd+WtA3Xnuu1V0rqktTV03N+4/BzmuvZtv8Y/QO+bKOZ2VB5B76ki4CHgM9GxMGhm3PsEiO0n90YsSoiOiOis6VlxAXfhtXeXE9v/wC7Dh4/r/3NzMpZXoEvqZpM2H83In6Qo8s2YE7W89nAjhHax0V7cz2Ax/HNzHLIZ5aOgH8AXo6Irw/TbQ3wkWS2ztXAgYjYCTwDLJA0X1INcEvSd1w48M3MhpfPevjXAh8GXpS0Pmn7ItAOEBF3A2uB9wDdwFHg48m2Pkm3A48AlcA9EbGxoEeQpXVKHRXCP9yameUwauBHxBPkHovP7hPAbcNsW0vmC2HcVVdW0Dqlzmf4ZmY5lM1f2g5qb/ZcfDOzXMoy8D2kY2Z2trIL/DnN9ew53MuRE33FLsXMbEIpu8AfnKmzdb/P8s3MspVt4L++14FvZpat7AJ/7jTPxTczy6XsAr+prprJtVX+4dbMbIiyC3xJnpppZpZD2QU+eC6+mVkuZRv4W/cfY8DLJJuZnVKWgT+nuZ7evgF2HzpR7FLMzCaMsgx8r5ppZnY2B76ZWUqUZeAPLpPswDczO60sA7+mqoJZTXWei29mlmXU9fAl3QP8EbA7Ihbl2P554ENZr3cZ0BIR+yRtAQ4B/UBfRHQWqvDReGqmmdmZ8jnDvxdYNtzGiPhaRCyOiMXAfwV+HhH7srpcn2y/YGEPDnwzs6FGDfyIeBzYN1q/xArg/jFVVCDt0+rpOXSCY739xS7FzGxCKNgYvqR6Mv8TeCirOYBHJa2TtHKU/VdK6pLU1dPTM+Z65niZZDOzMxTyR9v3Ab8cMpxzbURcBdwI3CbpHcPtHBGrIqIzIjpbWlrGXIyXSTYzO1MhA/8WhgznRMSO5H43sBpYUsD3G5Hn4puZnakggS+pCbgO+GFWW4OkyYOPgRuADYV4v3xMra/moklVDnwzs0Q+0zLvB5YC0yVtA74MVANExN1Jt5uBRyPiSNauM4DVkgbf53sR8dPClT5q3czxBc3NzE4ZNfAjYkUefe4lM30zu20zcMX5FlYI7c11bO45MnpHM7MUKMu/tB00OBc/wsskm5mVfeCf6Bugx8skm5mVd+APzsV/zeP4ZmblHfiei29mdlpZB37b1DrkZZLNzIAyD/xJVZXMaqz11EwzM8o88CEzju8zfDOzFAS+l0k2M8tIReDv9jLJZmYpCPxpmZk627xMspmlXNkH/hyvmmlmBqQg8L1MsplZRtkH/rSGGuprKh34ZpZ6ZR/4kmj3MslmZuUf+OC5+GZmkEfgS7pH0m5JOa9WJWmppAOS1ie3L2VtWyZpk6RuSXcUsvBz4WWSzczyO8O/F1g2Sp9fRMTi5PYVAEmVwJ1kLmC+EFghaeFYij1f7c31HD85QM9hL5NsZuk1auBHxOPAvvN47SVAd0Rsjohe4AFg+Xm8zpgNztTxOL6ZpVmhxvCvkfS8pJ9I6kja2oCtWX22JW0XnOfim5nlcU3bPDwLzI2Iw5LeAzwMLACUo++wg+iSVgIrAdrb2wtQ1mmzp9YB8PreYwV9XTOzUjLmM/yIOBgRh5PHa4FqSdPJnNHPyeo6G9gxwuusiojOiOhsaWkZa1lnqK2uZGZjrc/wzSzVxhz4kmZKUvJ4SfKae4FngAWS5kuqAW4B1oz1/c6X5+KbWdqNOqQj6X5gKTBd0jbgy0A1QETcDbwf+LSkPuAYcEtk5j/2SbodeASoBO6JiI3jchR5mNNczy+79xTr7c3Mim7UwI+IFaNs/ybwzWG2rQXWnl9phTV3Wj0PPXuc4yf7qa2uLHY5ZmYXXCr+0hZOT83ctt8/3JpZOqUm8Od4Lr6ZpVxqAt/LJJtZ2qUm8KdfVENdtZdJNrP0Sk3gDy6T7MA3s7RKTeBDZhzfY/hmllapCnwvk2xmaZaywK/jaG8/e4/0FrsUM7MLLl2BP80zdcwsvdIV+J6Lb2YplqrAnz01OcPf68A3s/RJVeDXVlcyo3GSh3TMLJVSFfiA5+KbWWqlLvA9F9/M0ip1gd/eXM/Og8c50ddf7FLMzC6oVAZ+BGz3MslmljKjBr6keyTtlrRhmO0fkvRCcntS0hVZ27ZIelHSekldhSz8fHnVTDNLq3zO8O8Flo2w/XfAdRFxOfAXwKoh26+PiMUR0Xl+JRaW5+KbWVrlc4nDxyXNG2H7k1lPnwZmj72s8dMyeRKTqip8hm9mqVPoMfxPAD/Jeh7Ao5LWSVo50o6SVkrqktTV09NT4LLOeB9PzTSzVBr1DD9fkq4nE/h/mNV8bUTskHQx8Jik30TE47n2j4hVJMNBnZ2d47qcZSbw/aOtmaVLQc7wJV0OfBtYHhF7B9sjYkdyvxtYDSwpxPuN1eBcfC+TbGZpMubAl9QO/AD4cES8ktXeIGny4GPgBiDnTJ8Lbf70Bg6f6GPXwRPFLsXM7IIZdUhH0v3AUmC6pG3Al4FqgIi4G/gSMA34O0kAfcmMnBnA6qStCvheRPx0HI7hnHW0NgKwcccBZjbVFrkaM7MLI59ZOitG2f5J4JM52jcDV5y9R/FdNqsRCTbuOMg7L5tR7HLMzC6I1P2lLUDDpCrmT2tgw/YDxS7FzOyCSWXgAyxsbWTjjoPFLsPM7IJJbeAvamti+5vHePOor29rZumQ2sA//cOtz/LNLB1SHPhNQGamjplZGqQ28JsbapjVVOszfDNLjdQGPmTO8h34ZpYWKQ/8Rn7bc5ijvX3FLsXMbNylPvAj4OWdh4pdipnZuEt14C9qy/xw+5J/uDWzFEh14M9qqmVqfbXH8c0sFVId+JLoaG1ig8/wzSwFUh34kBnHf+WNw5zsHyh2KWZm48qB39ZEb/8Ar+46XOxSzMzGlQM/WWLBwzpmVu5GDXxJ90jaLSnn1aqU8Q1J3ZJekHRV1rZlkjYl2+4oZOGFMn9aA/U1lbzkH27NrMzlc4Z/L7BshO03AguS20rgLgBJlcCdyfaFwApJC8dS7HioqBALZzV6TR0zK3ujBn5EPA7sG6HLcuC+yHgamCJpFpkLlndHxOaI6AUeSPpOOB2tjby04yADA76ouZmVr0KM4bcBW7Oeb0vahmvPSdJKSV2Sunp6egpQVv46Wps40tvPlr1HLuj7mpldSIUIfOVoixHac4qIVRHRGRGdLS0tBSgrfwu9Nr6ZpUAhAn8bMCfr+WxgxwjtE86lMyZTXSkHvpmVtUIE/hrgI8lsnauBAxGxE3gGWCBpvqQa4Jak74RTU1XBpTMm+4dbMytrVaN1kHQ/sBSYLmkb8GWgGiAi7gbWAu8BuoGjwMeTbX2SbgceASqBeyJi4zgcQ0F0tDbyzy/vJiKQco1GmZmVtlEDPyJWjLI9gNuG2baWzBfChNfR2sSDXdt44+BxZjXVFbscM7OCS/1f2g5a1Jb8cLvd4/hmVp4c+Im3zmxE8hILZla+HPiJhklVzJ/e4Jk6Zla2HPhZFrU2eU0dMytbDvwsHa2NbH/zGPuP9Ba7FDOzgnPgZ+lozVzj1sM6ZlaOHPhZOk4tseAfbs2s/Djws0xtqKFtSp3P8M2sLDnwh1jY2uipmWZWlhz4Q3S0NvK7PUc4cqKv2KWYmRWUA3+IRa1NRMBv3vCwjpmVFwf+EB1tXhvfzMqTA3+ImY21NDfUsGG7x/HNrLw48IeQREdro8/wzazsOPBz6Ght4pVdh+jtGyh2KWZmBePAz6GjtZGT/cGruw8VuxQzs4LJK/AlLZO0SVK3pDtybP+8pPXJbYOkfknNybYtkl5MtnUV+gDGw6m/uPXa+GZWRvK5xGElcCfwbjIXJn9G0pqIeGmwT0R8Dfha0v99wJ9GxL6sl7k+IvYUtPJxNG9aAw01lckSC3NG7W9mVgryOcNfAnRHxOaI6AUeAJaP0H8FcH8hiiuWigqx0D/cmlmZySfw24CtWc+3JW1nkVQPLAMeymoO4FFJ6yStHO5NJK2U1CWpq6enJ4+yxldHaxMv7TxI/0AUuxQzs4LIJ/CVo224FHwf8MshwznXRsRVwI3AbZLekWvHiFgVEZ0R0dnS0pJHWeNrYWsjR3v72bL3SLFLMTMriHwCfxtnDmTPBnYM0/cWhgznRMSO5H43sJrMENGEt8hr45tZmckn8J8BFkiaL6mGTKivGdpJUhNwHfDDrLYGSZMHHwM3ABsKUfh4WzDjImoqK7w2vpmVjVFn6UREn6TbgUeASuCeiNgo6dZk+91J15uBRyMiewxkBrBa0uB7fS8iflrIAxgv1ZUVXDrzIk/NNLOyMWrgA0TEWmDtkLa7hzy/F7h3SNtm4IoxVVhEHbOaePSlN4gIki8tM7OS5b+0HcGitkb2Hz3JzgPHi12KmdmYOfBHsNA/3JpZGXHgj+CyWZOpEKzfur/YpZiZjZkDfwT1NVW8bf40fvJiZhzfzKyUOfBHcfOVbWzec4QXtnl6ppmVNgf+KJb9+5nUVFWw+rntxS7FzGxMHPijaKyt5l2XXcyPX9hBX78viGJmpcuBn4fli9vYc7iXJ7pLZoVnM7OzOPDzsPQtLTTVVfOwh3XMrIQ58PMwqaqS914+i0c27uLIib5il2Nmdl4c+Hm6aXEbx07289hLu4pdipnZeXHg56lz7lTaptR5to6ZlSwHfp4qKsRNV7byRPceeg6dKHY5ZmbnzIF/Dm5a3Eb/QPDjF4a7/ouZ2cTlwD8HC2ZMpqO10bN1zKwk5RX4kpZJ2iSpW9IdObYvlXRA0vrk9qV89y01N1/ZxvPbDrC553CxSzEzOyejBr6kSuBOMhchXwiskLQwR9dfRMTi5PaVc9y3ZLzvilYkeHi9h3XMrLTkc4a/BOiOiM0R0Qs8ACzP8/XHsu+ENKOxlmsvmc7Dz233CppmVlLyCfw2YGvW821J21DXSHpe0k8kdZzjvkhaKalLUldPT08eZRXPTVe28fq+ozz7+pvFLsXMLG/5BH6ui7kOPbV9FpgbEVcA/xt4+Bz2zTRGrIqIzojobGlpyaOs4vkPHTOYVFXBD9f7x1szKx35BP42YE7W89nAGQPYEXEwIg4nj9cC1ZKm57NvKZpcW827F87gR8/v4KRX0DSzEpFP4D8DLJA0X1INcAuwJruDpJmSlDxekrzu3nz2LVU3X9nG/qMnefyViT38ZGY2qGq0DhHRJ+l24BGgErgnIjZKujXZfjfwfuDTkvqAY8AtkflFM+e+43QsF9Q7Lm1han01D6/fwTsvm1HscszMRjVq4MOpYZq1Q9ruznr8TeCb+e5bDqorK/ijy1t5sGsrh46fZHJtdbFLMjMbkf/SdgxuurKNE30DPLLRK2ia2cTnwB+Dq9qn0N5c79k6ZlYSHPhjIImbFrfyy+497Dp4vNjlmJmNyIE/RsuvbGMg4EfPl/xsUzMrcw78Mbqk5SIun93Ewx7WMbMJzoFfADctbmPD9oO8uutQsUsxMxuWA78A3ndFK5UV8lm+mU1oDvwCaJk8iWt/bzoPP+elFsxs4nLgF8hHrp7L9jePcde//bbYpZiZ5eTAL5B3LZzB8sWtfONfXmXD9gPFLsfM7CwO/AL68z/uoLmhhs89+Dwn+vqLXY6Z2Rkc+AU0pb6G//WfLmfTrkP8zT+/WuxyzMzO4MAvsOvfejG3/MEcvvXz37LutX3FLsfM7BQH/jj4b++9jFlNdXzuwec52ttX7HLMzAAH/riYXFvN1z5wOVv2HuUvf7qp2OWYmQF5Br6kZZI2SeqWdEeO7R+S9EJye1LSFVnbtkh6UdJ6SV2FLH4ie/sl0/nY2+dx75NbeLJ7T7HLMTMbPfAlVQJ3AjcCC4EVkhYO6fY74LqIuBz4C2DVkO3XR8TiiOgsQM0l4wvL3sr86Q18/p9e4NDxk8Uux8xSLp8z/CVAd0Rsjohe4AFgeXaHiHgyIvYnT58mc7Hy1KurqeSvPnAFOw8c43/8+OVil2NmKZdP4LcBW7Oeb0vahvMJ4CdZzwN4VNI6SSvPvcTS9vtzp/Kp6y7h+11b+dff+MpYZlY8+QS+crRFzo7S9WQC/wtZzddGxFVkhoRuk/SOYfZdKalLUldPT08eZZWOz75rAW+dOZkvPPQi+4/0FrscM0upfAJ/GzAn6/ls4KyrfUi6HPg2sDwi9g62R8SO5H43sJrMENFZImJVRHRGRGdLS0v+R1ACJlVV8tcfvIL9R3r50pqNxS7HzFIqn8B/Blggab6kGuAWYE12B0ntwA+AD0fEK1ntDZImDz4GbgA2FKr4UtLR2sRn3rmAHz2/gx+/4KtjmdmFN2rgR0QfcDvwCPAy8GBEbJR0q6Rbk25fAqYBfzdk+uUM4AlJzwO/Bv5fRPy04EdRIj699BKumN3Ef394A7sP+Rq4ZnZhKSLncHxRdXZ2RldXeU7Z7959iPd+4wkuabmI+z6xhOkXTSp2SWZWBiStG23qu//S9gL7vYsns+ojnWzec5gPfuspdh44VuySzCwlHPhFcN2lLdz3X95Gz8ETvP+up9iy50ixSzKzFHDgF8mS+c3cv/Jqjvb28YFvPcWmN3wBdDMbXw78IlrU1sSDn7qGCsGfrHqK9VvfLHZJZlbGHPhFtmDGZP7p1rfTWFvNh/7+aZ767d7RdzIzOw8O/AlgTnM9//fWa2idUsdH/8+v+ZeXvQSDmRWeA3+CmNFYy/c/dQ1vnTmZT/3jOtY87z/OMrPCcuBPIM0NNXz3k2/jqrlT+cwDz/G9X71e7JLMrIw48CeYybXVfOfjS7ju0ha+uPpF7vxZN719A8Uuy8zKgAN/AqqrqWTVhzt57+Wz+Nojm3j7V/+Vrz/2CrsOejkGMzt/XlphAosIHn91D995cgs/27SbSolli2bysbfP4/fnTkXKtXK1maVRPksrVF2oYuzcSeK6S1u47tIWXtt7hH986jW+37WVH7+wk47WRj56zTz+eHErtdWVxS7VzEqAz/BLzNHePlY/t537nnyNTbsOMbW+mj/5g3b+89XtzJ5aX+zyzKxI8jnDd+CXqIjg6c37+M6TW3j0pTcAWNjayKLWJjpaG+loa+KymY3U1fjs3ywNPKRTxiRxzSXTuOaSaWx/8xgPPrOVZ1/fzyMb3+CBZzKXIK4QXNJyEYvaki+B1iYWtjbSVFdd5OrNrBgc+GWgbUodf/ruS4HMmf+OA8fZuP0AG3YcZOP2Azz1272sfm77qf6zp9Yxf3oD86Y1MHdaPfOnNzB3WgPtzfXUVHnillm5yivwJS0D/haoBL4dEV8dsl3J9vcAR4GPRcSz+exrhSWJtil1tE2p44aOmafaew6dYOOOA2zccZBNbxxiy94jPLx+O4eO953qUyFonXLml0HL5Ek01lXTVFdNY23mvqmu2l8MZiVo1MCXVAncCbybzAXNn5G0JiJeyup2I7Agub0NuAt4W5772gXQMnkSS99yMUvfcvGptohg/9GTbNl7hC17jrBl71G27DnCa3uP8MP12zmY9WUwVG11xRlfApNrq6itrkxuFUyqqmRSdQW1VafbBu+rKiqoqhCVFaKqUlRmPz91X0FFBVQo87xCmS+zColKCQkqKjKPB7dJmf4iua/g1OOKZDuD25TZpqS/ktcwK2f5nOEvAbojYjOApAeA5UB2aC8H7ovML8BPS5oiaRYwL499rUgk0dxQQ3NDDVe1Tz1jW0Tw5tGT7D3Sy8HjJzlw7CQHk9uBY4PP+0493nO4l+Mn+zne18/xkwMcP9nPiZMD9PaX1l8JD/0iON2WPEu2D23X6c3Jc53xnOG2Z33HDLfvmd9Dp58Mfc+h7ZltGnbb2a+Y3S//L7+cr5nzfc5uHO5t8n33c6qz4B0LX2dzfQ0P3npN/gWco3wCvw3YmvV8G5mz+NH6tOW5LwCSVgIrAdrb2/Moy8aTJKY21DC1oWZMr9M/EPT2DZzxZdDXP0B/BH39Qf9A0DcweD9w+nl/5j4iGAgYiDh9G8h+nnmPIPMlFUnf7Pvg9GsMTkobOLVPZnvmPtMwkLRxajtZj5NnZ7Sf7ptsGvL8zO0M2S/ffbJ3P91+5oue0X/o+w3pm6vP2a84fL/hXjPPJoabIZjvvMFzmWCY/2vm/6J59zyHOifXju/Pqvm8eq6vpqGHMFyffPbNNEasAlZBZlpmHnVZCaisEHU1lZ4eajYB5BP424A5Wc9nA0PX7h2uT00e+5qZ2QWQz1SLZ4AFkuZLqgFuAdYM6bMG+IgyrgYORMTOPPc1M7MLYNQz/Ijok3Q78AiZqZX3RMRGSbcm2+8G1pKZktlNZlrmx0fad1yOxMzMRuSlFczMykA+Syv4r2fMzFLCgW9mlhIOfDOzlHDgm5mlxIT80VZSD/Daee4+HdhTwHKKrdyOB8rvmMrteKD8jqncjgfOPqa5EdEy0g4TMvDHQlLXaL9Ul5JyOx4ov2Mqt+OB8jumcjseOL9j8pCOmVlKOPDNzFKiHAN/VbELKLByOx4ov2Mqt+OB8jumcjseOI9jKrsxfDMzy60cz/DNzCwHB76ZWUqUTeBLWiZpk6RuSXcUu55CkLRF0ouS1ksqudXkJN0jabekDVltzZIek/Rqcj91pNeYaIY5pj+TtD35nNZLek8xazwXkuZI+pmklyVtlPSZpL1kP6cRjqkkPydJtZJ+Len55Hj+PGk/58+oLMbwk4ulv0LWxdKBFaV+sXRJW4DOiCjJPxiR9A7gMJnrHS9K2v4S2BcRX02+mKdGxBeKWee5GOaY/gw4HBF/Vczazkdy7elZEfGspMnAOuAm4GOU6Oc0wjF9kBL8nJS5IG5DRByWVA08AXwG+I+c42dULmf4py60HhG9wODF0q2IIuJxYN+Q5uXAd5LH3yHzD7FkDHNMJSsidkbEs8njQ8DLZK5FXbKf0wjHVJIi43DytDq5BefxGZVL4A93EfVSF8CjktYlF3kvBzOSq6GR3F9c5HoK5XZJLyRDPiUz/JFN0jzgSuBXlMnnNOSYoEQ/J0mVktYDu4HHIuK8PqNyCfy8L5ZeYq6NiKuAG4HbkuEEm3juAi4BFgM7gb8ubjnnTtJFwEPAZyPiYLHrKYQcx1Syn1NE9EfEYjLXBV8iadH5vE65BH4+F1ovORGxI7nfDawmM3RV6nYlY6yDY627i1zPmEXEruQf5ADw95TY55SMCz8EfDcifpA0l/TnlOuYSv1zAoiIN4F/A5ZxHp9RuQR+2V0sXVJD8oMTkhqAG4ANI+9VEtYAH00efxT4YRFrKYjBf3SJmymhzyn5QfAfgJcj4utZm0r2cxrumEr1c5LUImlK8rgOeBfwG87jMyqLWToAyRSrv+H0xdL/Z5FLGhNJ/47MWT1kLjb/vVI7Jkn3A0vJLOO6C/gy8DDwINAOvA58ICJK5kfQYY5pKZlhggC2AJ8aHFud6CT9IfAL4EVgIGn+Ipkx75L8nEY4phWU4Ock6XIyP8pWkjlJfzAiviJpGuf4GZVN4JuZ2cjKZUjHzMxG4cA3M0sJB76ZWUo48M3MUsKBb2aWEg58M7OUcOCbmaXE/wfZDL0lG7CBRQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(tr_loss_hist)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}