Lab12/Lab12-3 minkiKang.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 모두를 위한 딥러닝 시즌2 - TensorFlow\n",
    "\n",
    "## LAB 12 - 3 RNN many to many"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.5.0\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import matplotlib.pyplot as plt\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras import layers\n",
    "from tensorflow.keras import Sequential, Model\n",
    "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
    "from pprint import pprint\n",
    "%matplotlib inline\n",
    "\n",
    "print(tf.__version__)\n",
    "tf.compat.v1.enable_eager_execution()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Preparing dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#토큰에 대한 품사가 주어져있음\n",
    "sentences = [['I', 'feel', 'hungry'],\n",
    "     ['tensorflow', 'is', 'very', 'difficult'],\n",
    "     ['tensorflow', 'is', 'a', 'framework', 'for', 'deep', 'learning'],\n",
    "     ['tensorflow', 'is', 'very', 'fast', 'changing']]\n",
    "pos = [['pronoun', 'verb', 'adjective'],\n",
    "     ['noun', 'verb', 'adverb', 'adjective'],\n",
    "     ['noun', 'verb', 'determiner', 'noun', 'preposition', 'adjective', 'noun'],\n",
    "     ['noun', 'verb', 'adverb', 'adjective', 'verb']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'<pad>': 0, 'I': 1, 'a': 2, 'changing': 3, 'deep': 4, 'difficult': 5, 'fast': 6, 'feel': 7, 'for': 8, 'framework': 9, 'hungry': 10, 'is': 11, 'learning': 12, 'tensorflow': 13, 'very': 14}\n",
      "{0: '<pad>', 1: 'I', 2: 'a', 3: 'changing', 4: 'deep', 5: 'difficult', 6: 'fast', 7: 'feel', 8: 'for', 9: 'framework', 10: 'hungry', 11: 'is', 12: 'learning', 13: 'tensorflow', 14: 'very'}\n",
      "15\n"
     ]
    }
   ],
   "source": [
    "#sentence를 워드의 시퀀스로 간주\n",
    "#토큰인 워드를 인티저 인덱스로 매핑하고 있는 토큰의 딕셔너리 생성\n",
    "word_list = sum(sentences, [])\n",
    "word_list = sorted(set(word_list))\n",
    "word_list = ['<pad>'] + word_list\n",
    "word2idx = {word : idx for idx, word in enumerate(word_list)}\n",
    "idx2word = {idx : word for idx, word in enumerate(word_list)}\n",
    "\n",
    "print(word2idx)\n",
    "print(idx2word)\n",
    "print(len(idx2word))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'<pad>': 0, 'adjective': 1, 'adverb': 2, 'determiner': 3, 'noun': 4, 'preposition': 5, 'pronoun': 6, 'verb': 7}\n",
      "{0: '<pad>', 1: 'adjective', 2: 'adverb', 3: 'determiner', 4: 'noun', 5: 'preposition', 6: 'pronoun', 7: 'verb'}\n",
      "8\n"
     ]
    }
   ],
   "source": [
    "#품사또한 마찬가지\n",
    "pos_list = sum(pos, [])\n",
    "pos_list = sorted(set(pos_list))\n",
    "#입력과 출력의 길이가 강ㅌ음을 포함하기 위해 <pad>토큰 포함\n",
    "pos_list = ['<pad>'] + pos_list\n",
    "pos2idx = {pos : idx for idx, pos in enumerate(pos_list)}\n",
    "idx2pos = {idx : pos for idx, pos in enumerate(pos_list)}\n",
    "\n",
    "print(pos2idx)\n",
    "print(idx2pos)\n",
    "print(len(pos2idx))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 1  7 10  0  0  0  0  0  0  0]\n",
      " [13 11 14  5  0  0  0  0  0  0]\n",
      " [13 11  2  9  8  4 12  0  0  0]\n",
      " [13 11 14  6  3  0  0  0  0  0]] [3, 4, 7, 5]\n",
      "[[1. 1. 1. 0. 0. 0. 0. 0. 0. 0.]\n",
      " [1. 1. 1. 1. 0. 0. 0. 0. 0. 0.]\n",
      " [1. 1. 1. 1. 1. 1. 1. 0. 0. 0.]\n",
      " [1. 1. 1. 1. 1. 0. 0. 0. 0. 0.]]\n",
      "[[6 7 1 0 0 0 0 0 0 0]\n",
      " [4 7 2 1 0 0 0 0 0 0]\n",
      " [4 7 3 4 5 1 4 0 0 0]\n",
      " [4 7 2 1 7 0 0 0 0 0]]\n"
     ]
    }
   ],
   "source": [
    "max_sequence = 10\n",
    "x_data = list(map(lambda sentence : [word2idx.get(token) for token in sentence], sentences))\n",
    "y_data = list(map(lambda sentence : [pos2idx.get(token) for token in sentence], pos))\n",
    "\n",
    "# padding the sequence of indices\n",
    "x_data = pad_sequences(sequences = x_data, maxlen = max_sequence, padding='post')\n",
    "x_data_mask = ((x_data != 0) * 1).astype(np.float32)\n",
    "x_data_len = list(map(lambda sentence : len(sentence), sentences))\n",
    "\n",
    "y_data = pad_sequences(sequences = y_data, maxlen = max_sequence, padding='post')\n",
    "\n",
    "# checking data\n",
    "print(x_data, x_data_len)\n",
    "print(x_data_mask)\n",
    "print(y_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Creating model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_classes = len(pos2idx)\n",
    "hidden_dim = 10\n",
    "\n",
    "input_dim = len(word2idx)\n",
    "output_dim = len(word2idx)\n",
    "one_hot = np.eye(len(word2idx))\n",
    "\n",
    "model = Sequential()\n",
    "model.add(layers.Embedding(input_dim=input_dim, output_dim=output_dim, mask_zero=True,\n",
    "                           trainable=False, input_length=max_sequence,\n",
    "                           embeddings_initializer=keras.initializers.Constant(one_hot)))\n",
    "model.add(layers.SimpleRNN(units=hidden_dim, return_sequences=True))\n",
    "model.add(layers.TimeDistributed(layers.Dense(units=num_classes)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding (Embedding)        (None, 10, 15)            225       \n",
      "_________________________________________________________________\n",
      "simple_rnn (SimpleRNN)       (None, 10, 10)            260       \n",
      "_________________________________________________________________\n",
      "time_distributed (TimeDistri (None, 10, 8)             88        \n",
      "=================================================================\n",
      "Total params: 573\n",
      "Trainable params: 348\n",
      "Non-trainable params: 225\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training model\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#매 토큰마다 로스를 구해야하고, <pad>토큰은 반영하면 안됨\n",
    "def loss_fn(model, x, y, x_len, max_sequence):\n",
    "    masking = tf.sequence_mask(x_len, maxlen=max_sequence, dtype=tf.float32)\n",
    "    valid_time_step = tf.cast(x_len,dtype=tf.float32)\n",
    "    sequence_loss = tf.compat.v1.losses.sparse_softmax_cross_entropy(labels=y, logits=model(x),\n",
    "                                                           reduction='none') * masking\n",
    "    sequence_loss = tf.reduce_sum(sequence_loss, axis=-1) / valid_time_step\n",
    "    sequence_loss = tf.reduce_mean(sequence_loss)\n",
    "    return sequence_loss\n",
    "\n",
    "# creating and optimizer\n",
    "lr = 0.1\n",
    "epochs = 30\n",
    "batch_size = 2 \n",
    "opt = tf.compat.v1.train.AdamOptimizer(learning_rate = lr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<BatchDataset shapes: ((None, 10), (None, 10), (None,)), types: (tf.int32, tf.int32, tf.int32)>\n"
     ]
    }
   ],
   "source": [
    "tr_dataset = tf.data.Dataset.from_tensor_slices((x_data, y_data, x_data_len))\n",
    "tr_dataset = tr_dataset.shuffle(buffer_size=4)\n",
    "tr_dataset = tr_dataset.batch(batch_size = 2)\n",
    "\n",
    "print(tr_dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch :   5, tr_loss : 0.090\n",
      "epoch :  10, tr_loss : 0.004\n",
      "epoch :  15, tr_loss : 0.001\n",
      "epoch :  20, tr_loss : 0.001\n",
      "epoch :  25, tr_loss : 0.000\n",
      "epoch :  30, tr_loss : 0.000\n"
     ]
    }
   ],
   "source": [
    "tr_loss_hist = []\n",
    "\n",
    "for epoch in range(epochs):\n",
    "    avg_tr_loss = 0\n",
    "    tr_step = 0\n",
    "    \n",
    "    for x_mb, y_mb, x_mb_len in tr_dataset:\n",
    "        with tf.GradientTape() as tape:\n",
    "            tr_loss = loss_fn(model, x=x_mb, y=y_mb, x_len=x_mb_len, max_sequence=max_sequence)\n",
    "        grads = tape.gradient(target=tr_loss, sources=model.variables)\n",
    "        opt.apply_gradients(grads_and_vars=zip(grads, model.variables))\n",
    "        avg_tr_loss += tr_loss\n",
    "        tr_step += 1\n",
    "    else:\n",
    "        avg_tr_loss /= tr_step\n",
    "        tr_loss_hist.append(avg_tr_loss)\n",
    "    \n",
    "    if (epoch + 1) % 5 == 0:\n",
    "        print('epoch : {:3}, tr_loss : {:.3f}'.format(epoch + 1, avg_tr_loss))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Checking performance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[['pronoun', 'verb', 'adjective', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>'],\n",
      " ['noun', 'verb', 'adverb', 'adjective', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>'],\n",
      " ['noun', 'verb', 'determiner', 'noun', 'preposition', 'adjective', 'noun', '<pad>', '<pad>', '<pad>'],\n",
      " ['noun', 'verb', 'adverb', 'adjective', 'verb', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>']]\n",
      "[['pronoun', 'verb', 'adjective'],\n",
      " ['noun', 'verb', 'adverb', 'adjective'],\n",
      " ['noun', 'verb', 'determiner', 'noun', 'preposition', 'adjective', 'noun'],\n",
      " ['noun', 'verb', 'adverb', 'adjective', 'verb']]\n"
     ]
    }
   ],
   "source": [
    "yhat = model.predict(x_data)\n",
    "yhat = np.argmax(yhat, axis=-1) * x_data_mask\n",
    "\n",
    "pprint(list(map(lambda row : [idx2pos.get(elm) for elm in row],yhat.astype(np.int32).tolist())), width = 120)\n",
    "pprint(pos)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x17f82097640>]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXUAAAD4CAYAAAATpHZ6AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAW9ElEQVR4nO3dfZAcdZ3H8fd3ZnZ3NpudzcMuu2OegUhAUeBiAHmQqtMrQK8Qy/PAEpU7DSp6WuVVaVl16l2VVZ51R53KCUallCvB0kORq0M9zpMCTnlIcuExAUKAsLJJNgnsZvOwT/O9P6Y3TJbZ3dnsbHr7159X1db0dPf2fJsmn+n9dffvZ+6OiIiEIRN3ASIiUj8KdRGRgCjURUQColAXEQmIQl1EJCC5uD64vb3dV65cGdfHi4gk0qZNm/a6e8dEy2ML9ZUrV7Jx48a4Pl5EJJHM7MXJlqv5RUQkIAp1EZGAKNRFRAKiUBcRCYhCXUQkIAp1EZGAKNRFRAKSuFB/etcBvvHrbbx6aCjuUkRE5pzEhfoL+w7ynXuf46X9h+MuRURkzklcqHcV8gD09CnURUTGS1yoF9vKob67/0jMlYiIzD2JC/XF85vIZoxdCnURkdeZMtTNbJmZ/c7MtprZk2b22SrrmJl9y8y2m9ljZnbO7JQL2YxxUmsTu/oGZ+sjREQSq5ZeGkeAz7v7ZjNrBTaZ2T3u/lTFOpcBq6Ofc4GbotdZ0VnIq/lFRKSKKc/U3b3H3TdH0weArcCScatdAdzqZQ8CC8ysWPdqI12FvC6UiohUMa02dTNbCZwNPDRu0RLgpYr33bw++DGz9Wa20cw29vb2TrPU13S15dndr+YXEZHxag51M5sP3AF8zt37xy+u8iv+uhnuG9x9rbuv7eiYcOCOKXW15RkYHGFgcOS4tyEiEqKaQt3MGigH+o/d/edVVukGllW8Xwq8PPPyqhu7V31Xn9rVRUQq1XL3iwE/ALa6+w0TrHYX8OHoLpjzgD5376ljncfoLOhedRGRamq5++UC4BrgcTPbEs37ErAcwN1vBu4GLge2A4eAa+teaYWxB5B6dKYuInKMKUPd3R+gept55ToOXF+voqbSpadKRUSqStwTpQD5hixtzQ1qUxcRGSeRoQ7li6XqKkBE5FiJDfXONj1VKiIyXmJDvVjI60KpiMg4iQ31zrY8ewcGGR4txV2KiMickdhQ7yrkcYfeA+ouQERkTHJDva0JQBdLRUQqJDbUjz5VqnZ1EZGjEhvqxbZmQE+ViohUSmyoL5zXQGMuo9saRUQqJDbUzYzOQpPa1EVEKiQ21CF6qlTNLyIiRyU71NuadaYuIlIh2aFeaGJX3xHKnUSKiEiiQ72zkGdwpETf4eG4SxERmRMSHepj/aqrCUZEpCzZoa6xSkVEjpHsUG9TqIuIVEp0qJ/UquYXEZFKiQ71xlyG9vmNeqpURCSS6FCH8h0wan4RESlLfKiXxypVn+oiIhBCqLfl2dV3OO4yRETmhOSHeiHPK4eGOTI8GncpIiKxS3yod0a3Ne5RE4yISPJD/egDSLoDRkQkgFBXVwEiIkeFE+q6WCoikvxQb23KMa8xy64+tamLiCQ+1M2MrkJeT5WKiBBAqEP0VKlCXUQkjFAvtqmrABERCCTUO9vKzS+lkoa1E5F0CyLUuwp5RkrOvoNDcZciIhKrIEK9M3oASRdLRSTtggh1jYAkIlIWRKgXo1Dv0Zm6iKRcEKHePr+JbMbYrTN1EUm5IEI9mzE65jfpXnURSb0pQ93MbjGzPWb2xATLLzGzPjPbEv18uf5lTm3stkYRkTTL1bDOD4EbgVsnWed+d39PXSo6Tl2FJnb0HoyzBBGR2E15pu7u9wH7T0AtM1Jsa9bdLyKSevVqUz/fzB41s1+Z2ZsmWsnM1pvZRjPb2NvbW6ePLuss5DkwOMLBwZG6bldEJEnqEeqbgRXu/lbg28CdE63o7hvcfa27r+3o6KjDR7+mq60J0GAZIpJuMw51d+9394Fo+m6gwczaZ1zZNB19qlRNMCKSYjMOdTPrMjOLptdF29w30+1OV7GtGdCZuoik25R3v5jZ7cAlQLuZdQNfARoA3P1m4P3AJ81sBDgMXOXuJ7y7xLEBqHt0pi4iKTZlqLv71VMsv5HyLY+xam7MUsjndK+6iKRaEE+UjunSYBkiknJBhXqnxioVkZQLKtSLbRqrVETSLahQ7yrk6T0wyMhoKe5SRERiEVSod7blKTn0DgzGXYqISCyCCvWx2xp1sVRE0iqoUNdYpSKSdkGFelFjlYpIygUV6otaGmnMZjRWqYikVlChbmacVGhSp14iklpBhTqUL5bqXnURSavwQr0tz+5+3dIoIukUXqgX8vT0HSaGjiJFRGIXXqi35TkyXKL/sIa1E5H0CS7Ux+5VV7u6iKRRcKHe1aZQF5H0Ci/UNVapiKRYcKHeqWHtRCTFggv1xlyGxS2Nan4RkVQKLtRBIyCJSHoFGeoaq1RE0irYUNeZuoikUZihXsiz7+AQgyOjcZciInJCBRvqAHvUB4yIpEyQod6pB5BEJKWCDHWNgCQiaRVkqGusUhFJqyBDvZDP0dyQ1VOlIpI6QYa6mZXvVdeZuoikTJChDtCpsUpFJIWCDfViW7PO1EUkdYIN9c5Cnj39g5RKGtZORNIj2FDvKjQxNFpi/6GhuEsRETlhwg113asuIikUbKjrXnURSaNgQ73Y1gyoqwARSZdgQ719fiMZ01ilIpIuwYZ6Lpuho7VJT5WKSKpMGepmdouZ7TGzJyZYbmb2LTPbbmaPmdk59S/z+KxY1MLzew/GXYaIyAlTy5n6D4FLJ1l+GbA6+lkP3DTzsupjTbGVbbsO4K571UUkHaYMdXe/D9g/ySpXALd62YPAAjMr1qvAmVjTVWBgcITuVw7HXYqIyAlRjzb1JcBLFe+7o3mxW1NsBWDbrgMxVyIicmLUI9Styryq7R1mtt7MNprZxt7e3jp89ORO64xCvad/1j9LRGQuqEeodwPLKt4vBV6utqK7b3D3te6+tqOjow4fPbmWphwrFs/TmbqIpEY9Qv0u4MPRXTDnAX3u3lOH7dbFaZ2tbNulM3URSYfcVCuY2e3AJUC7mXUDXwEaANz9ZuBu4HJgO3AIuHa2ij0ea4oF/nvrbo4Mj5JvyMZdjojIrJoy1N396imWO3B93Sqqs9O7Wik5PLt7gDOXtsVdjojIrAr2idIxa4oFALaqCUZEUiD4UF++aB7NDVm29ehiqYiEL/hQz2aMN3bpYqmIpEPwoQ7ldnV1FyAiaZCKUF/T1cr+g0P0DgzGXYqIyKxKRaif1lW+WKp2dREJXSpCfU3XWB8walcXkbClItQXtjTSVcjrTF1EgpeKUIdyj41b1QeMiAQuPaHeVWD7ngMMj5biLkVEZNakJtRPL7YyPOrs6NXwdiISrtSE+pqxO2B0sVREApaaUD+5o4WGrKlvdREJWmpCvSGb4ZSO+RoFSUSClppQBzi9WNCZuogELVWhvqarlZ6+I7x6aCjuUkREZkW6Qr04drFUZ+siEqZUhfrpY90FqF1dRAKVqlDvaG1iUUujztRFJFipCnUzY03Ut7qISIhSFepQfgjp6V0HKJU0YIaIhCeFod7K4eFRdu4/FHcpIiJ1l75QL6pvdREJV+pCffVJrWQMtqpvdREJUOpCvbkxy8r2Fp2pi0iQUhfqAKd3qbsAEQlTKkN9TVcrO/cf4uDgSNyliIjUVTpDvVjAHZ7ZrbN1EQlLOkN9rLsANcGISGBSGepLFzYzvymnPmBEJDipDHUz47SuVrbqTF1EApPKUIdyE8y2nn7c1V2AiIQjvaFeLNB/ZISeviNxlyIiUjepDfWxvtWfVhOMiAQktaH+xijUt+rJUhEJSGpDvZBvYOnCZrapDxgRCUhqQx2ii6U6UxeRgKQ81As813uQwZHRuEsREamLdId6sZXRkrN9z0DcpYiI1EVNoW5ml5rZ02a23cy+WGX5JWbWZ2Zbop8v17/U+lvTVQBQu7qIBCM31QpmlgX+FXgX0A08YmZ3uftT41a9393fMws1zpqVi+fRlMuoXV1EglHLmfo6YLu773D3IeAnwBWzW9aJkctmeGNnqzr2EpFg1BLqS4CXKt53R/PGO9/MHjWzX5nZm6ptyMzWm9lGM9vY29t7HOXWX/kOGIW6iIShllC3KvPGd5iyGVjh7m8Fvg3cWW1D7r7B3de6+9qOjo5pFTpb1hQL9B4YZO/AYNyliIjMWC2h3g0sq3i/FHi5cgV373f3gWj6bqDBzNrrVuUsWqPuAkQkILWE+iPAajNbZWaNwFXAXZUrmFmXmVk0vS7a7r56FzsbxkJ9q/pWF5EATHn3i7uPmNmngd8AWeAWd3/SzD4RLb8ZeD/wSTMbAQ4DV3lC+rRdPL+JjtYmtauLSBCmDHU42qRy97h5N1dM3wjcWN/SThx1FyAioUj1E6VjTi8WeHb3ACOjpbhLERGZEYU6cEaxwOBIiUe7++IuRURkRhTqwDvP6KQ1n+OWB56PuxQRkRlRqAPzm3J86LwV/OqJHl7cdzDuckREjptCPXLt21eSy2T4/v06WxeR5FKoR04q5Hnv2W/gZ5teYp+eLhWRhFKoV1h/8ckcGS5x6x9ejLsUEZHjolCvcOpJrbzz9JO49Q8vcHhIoyGJSPIo1Me57h2n8MqhYX626aWpVxYRmWMU6uOsXbGQs5cv4Pv3P89oKRE9HYiIHKVQH8fMuO7iU9i5/xC/fmJX3OWIiEyLQr2Kd53Ryar2Fr5733MkpF8yERFAoV5VNmN8/KKTeay7jwd37I+7HBGRminUJ/C+c5bQPr+R7973XNyliIjUTKE+gXxDlo+cv5J7n+5Vt7wikhgK9Ulcc/4KmhuybLhvR9yliIjURKE+iQXzGvnLty3jri0v09N3OO5yRESmpFCfwl9fuAoHdcsrIomgUJ/CskXzePeZRW5/+CX6jwzHXY6IyKQU6jVYf/HJDAyOcNtDO+MuRURkUgr1Grx5SRsXntrOLQ88z+CIOvoSkblLoV6j695xMnsODPLLLS/HXYqIyIQU6jW68NR2zigW2HDfDkrq6EtE5iiFeo3MjPUXn8z2PQP8z7Y9cZcjIlKVQn0a3v2WIksXNvOFOx5j04uvxF2OiMjrKNSnoSGb4Ud/tY75+RxXf+9B/uNRta+LyNyiUJ+mUzrm84tPXcBZSxfwmdv/j2//9ll1zysic4ZC/Tgsamnk3z62jivPXsI/3/MMn//po7rVUUTmhFzcBSRVUy7LDR94K6vaW7jhnmfofvUw3/3Qn7CwpTHu0kQkxXSmPgNmxt/86Wq+edVZbNn5Kld+53/Z0TsQd1kikmIK9Tq44qwl3Pbxc+k/MsL7bvo9D+7YF3dJIpJSCvU6WbtyEXd+6gIWtzRyzQ8e4t83dcddkoikkEK9jpYvnsfPP3UB61Yt4m9/9ij/+OttHBoaibssEUkRhXqdtTU38MNr13HV25Zx073Pce7Xfsvf3fkET72sIfFEZPZZXPdYr1271jdu3BjLZ58I7s7GF1/htod28p+P9zA0UuKsZQv44LnL+fO3vIHmxmzcJYpIApnZJndfO+Fyhfrse/XQEHds/iO3PfQiz/UepDWf48qzl/DBc5ezpqsQd3kikiAK9TnE3Xn4+f3c/vBO7n5iF0MjJc5ZvoCr1y3n8jOLtDTpsQERmZxCfY565eAQd2zu5raHd7Kj9yANWePs5Qu56NR2LlzdzplL2shldclDRI6lUJ/j3J1HXniF327bzQPP7uXJ6IJqaz7H209ZzIWrO7jw1HZWLp6HmcVcrYjEbapQr+nvfTO7FPgmkAW+7+5fH7fcouWXA4eAj7r75uOuOkXMjHWrFrFu1SK4DPYNDPL75/bxwLN7eWD7Xn7z5G4Alixo5qLV7axbtYj2+U0Umhso5HMUmhtozedoyunCq4jUcKZuZlngGeBdQDfwCHC1uz9Vsc7lwGcoh/q5wDfd/dzJtqsz9am5Oy/sO8QDz/Zy/7N7+cOOfRw4Uv2+96Zc5pigL+QbaG7I0pDL0JjN0JgzGrMZGrKZinmZaJ7RmMuW30fzmnLldSvnNebK62bMyGSMrBkZo2LayGQgmylPm4Ex9lr+AssY+otDZAbqcaa+Dtju7juiDf4EuAJ4qmKdK4BbvfwN8aCZLTCzorv3zKD21DMzVrW3sKq9hWvOX8nIaInn9x6k7/Aw/UeGOXBkhP7Dw/QffX1t+tVDQ+waLjE8WmJwpPw6NFpieCR6HY2/u2AzyuEfTR/9AqiYHltOxRcDR9cf2040r2K7r70be0+VdcbmVV/32K1M78uo2qpV51F9m9XXrfWzp1Fn3Ves3Wx8tSflhOGqty3jYxedPCvbriXUlwAvVbzvpnw2PtU6S4BjQt3M1gPrAZYvXz7dWlMvl82wurO1Ltty96PhPjRSeu1ndJTBY94fOz0y6pS8/DNaomLaKTmUSs5oNM+9/Dnu4BC9ltfDHYfX1qtYPv73xtaprN2PTkev0Zyx32Hc8rG1xs87Zppjv+iOXTbRf8cq86qtXdusaJuvX1LrV/B0LpHVvs36nwDMyilF/OcpNWuf3zRr264l1Kt99Y3/z1fLOrj7BmADlJtfavhsmSVmRlMuS1MOmL3/v0TkBKvlnrluYFnF+6XA+HHcallHRERmWS2h/giw2sxWmVkjcBVw17h17gI+bGXnAX1qTxcROfGmbH5x9xEz+zTwG8q3NN7i7k+a2Sei5TcDd1O+82U75Vsar529kkVEZCI13afu7ndTDu7KeTdXTDtwfX1LExGR6dJz6CIiAVGoi4gERKEuIhIQhbqISEBi66XRzHqBF4/z19uBvXUsZy4IbZ9C2x8Ib59C2x8Ib5+q7c8Kd++Y6BdiC/WZMLONk3Vok0Sh7VNo+wPh7VNo+wPh7dPx7I+aX0REAqJQFxEJSFJDfUPcBcyC0PYptP2B8PYptP2B8PZp2vuTyDZ1ERGpLqln6iIiUoVCXUQkIIkLdTO71MyeNrPtZvbFuOupBzN7wcweN7MtZpa4gVvN7BYz22NmT1TMW2Rm95jZs9HrwjhrnK4J9umrZvbH6DhticbmTQQzW2ZmvzOzrWb2pJl9NpqfyOM0yf4k+RjlzexhM3s02qe/j+ZP6xglqk29lkGwk8jMXgDWunsiH5ows4uBAcrj1L45mvcNYL+7fz368l3o7l+Is87pmGCfvgoMuPs/xVnb8TCzIlB0981m1gpsAt4LfJQEHqdJ9ucDJPcYGdDi7gNm1gA8AHwWeB/TOEZJO1M/Ogi2uw8BY4NgS4zc/T5g/7jZVwA/iqZ/RPkfXGJMsE+J5e497r45mj4AbKU8jnAij9Mk+5NYXjYQvW2IfpxpHqOkhfpEA1wnnQP/ZWabosG5Q9A5NvpV9HpSzPXUy6fN7LGoeSYRTRXjmdlK4GzgIQI4TuP2BxJ8jMwsa2ZbgD3APe4+7WOUtFCvaYDrBLrA3c8BLgOuj/70l7nnJuAU4CygB/jnWKs5DmY2H7gD+Jy798ddz0xV2Z9EHyN3H3X3syiP87zOzN483W0kLdSDHODa3V+OXvcAv6DczJR0u6N2z7H2zz0x1zNj7r47+kdXAr5Hwo5T1E57B/Bjd/95NDuxx6na/iT9GI1x91eBe4FLmeYxSlqo1zIIdqKYWUt0oQczawH+DHhi8t9KhLuAj0TTHwF+GWMtdTH2DytyJQk6TtFFuB8AW939hopFiTxOE+1Pwo9Rh5ktiKabgXcC25jmMUrU3S8A0S1K/8Jrg2B/Ld6KZsbMTqZ8dg7lMWNvS9o+mdntwCWUuwndDXwFuBP4KbAc2An8hbsn5sLjBPt0CeU/6x14AbhurK1zrjOzC4H7gceBUjT7S5TboRN3nCbZn6tJ7jF6C+ULoVnKJ9w/dfd/MLPFTOMYJS7URURkYklrfhERkUko1EVEAqJQFxEJiEJdRCQgCnURkYAo1EVEAqJQFxEJyP8DXpsLAvIQ2lEAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(tr_loss_hist)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}