Lab12/Lab12-1_SurinKim.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Lab 12-1 many to one\n",
    "### word sentiment classification \n",
    "* many to one \n",
    "* variable input sequence length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.5.0\n"
     ]
    }
   ],
   "source": [
    "# setup\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import matplotlib.pyplot as plt\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras import layers\n",
    "from tensorflow.keras import Sequential, Model\n",
    "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
    "%matplotlib inline\n",
    "\n",
    "print(tf.__version__)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Prepairing dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['<pad>', ' ', 'a', 'b', 'd', 'e', 'g', 'o', 'r', 's', 'w']\n",
      "{0: '<pad>', 1: ' ', 2: 'a', 3: 'b', 4: 'd', 5: 'e', 6: 'g', 7: 'o', 8: 'r', 9: 's', 10: 'w'}\n",
      "{'<pad>': 0, ' ': 1, 'a': 2, 'b': 3, 'd': 4, 'e': 5, 'g': 6, 'o': 7, 'r': 8, 's': 9, 'w': 10}\n"
     ]
    }
   ],
   "source": [
    "# example data\n",
    "words = ['good', 'bad', 'worse', 'so good']\n",
    "y_data = [1,0,0,1]\n",
    "\n",
    "# creating a token dictionary\n",
    "char_set = ['<pad>'] + sorted(list(set(''.join(words))))\n",
    "idx2char = {idx : char for idx, char in enumerate(char_set)}\n",
    "char2idx = {char : idx for idx, char in enumerate(char_set)}\n",
    "\n",
    "print(char_set)\n",
    "print(idx2char)\n",
    "print(char2idx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[6, 7, 7, 4], [3, 2, 4], [10, 7, 8, 9, 5], [9, 7, 1, 6, 7, 7, 4]]\n",
      "[4, 3, 5, 7]\n"
     ]
    }
   ],
   "source": [
    "# converting sequence of tokens to sequence of indices\n",
    "x_data = list(map(lambda word : [char2idx.get(char) for char in word], words))\n",
    "x_data_len = list(map(lambda word : len(word), x_data))\n",
    "\n",
    "print(x_data)\n",
    "print(x_data_len)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 6  7  7  4  0  0  0  0  0  0]\n",
      " [ 3  2  4  0  0  0  0  0  0  0]\n",
      " [10  7  8  9  5  0  0  0  0  0]\n",
      " [ 9  7  1  6  7  7  4  0  0  0]]\n",
      "[4, 3, 5, 7]\n",
      "[1, 0, 0, 1]\n"
     ]
    }
   ],
   "source": [
    "# padding the sequence of indices\n",
    "max_sequence = 10\n",
    "x_data = pad_sequences(sequences = x_data, maxlen = max_sequence,\n",
    "                       padding = 'post', truncating = 'post')\n",
    "\n",
    "# checking data\n",
    "print(x_data)\n",
    "print(x_data_len)\n",
    "print(y_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Creating model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# creating simple rnn for \"many to one\" classification\n",
    "input_dim = len(char2idx)\n",
    "output_dim = len(char2idx)\n",
    "one_hot = np.eye(len(char2idx))\n",
    "hidden_size = 10\n",
    "num_classes = 2\n",
    "\n",
    "model = Sequential()\n",
    "model.add(layers.Embedding(input_dim=input_dim, output_dim=output_dim,\n",
    "                           trainable=False, mask_zero=True, input_length=max_sequence,\n",
    "                           embeddings_initializer=keras.initializers.Constant(one_hot)))\n",
    "model.add(layers.SimpleRNN(units=hidden_size))\n",
    "model.add(layers.Dense(units=num_classes))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding (Embedding)        (None, 10, 11)            121       \n",
      "_________________________________________________________________\n",
      "simple_rnn (SimpleRNN)       (None, 10)                220       \n",
      "_________________________________________________________________\n",
      "dense (Dense)                (None, 2)                 22        \n",
      "=================================================================\n",
      "Total params: 363\n",
      "Trainable params: 242\n",
      "Non-trainable params: 121\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# creating loss function\n",
    "def loss_fn(model, x, y):\n",
    "    return tf.reduce_mean(tf.keras.losses.sparse_categorical_crossentropy(\n",
    "        y_true=y, y_pred=model(x), from_logits=True))\n",
    "\n",
    "# creating an optimizer\n",
    "lr = .01\n",
    "epochs = 30\n",
    "batch_size = 2\n",
    "opt = tf.keras.optimizers.Adam(learning_rate = lr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<BatchDataset shapes: ((None, 10), (None,)), types: (tf.int32, tf.int32)>\n"
     ]
    }
   ],
   "source": [
    "# generating data pipeline\n",
    "tr_dataset = tf.data.Dataset.from_tensor_slices((x_data, y_data))\n",
    "tr_dataset = tr_dataset.shuffle(buffer_size = 4)\n",
    "tr_dataset = tr_dataset.batch(batch_size = batch_size)\n",
    "\n",
    "print(tr_dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch :   5, tr_loss : 0.235\n",
      "epoch :  10, tr_loss : 0.042\n",
      "epoch :  15, tr_loss : 0.013\n",
      "epoch :  20, tr_loss : 0.007\n",
      "epoch :  25, tr_loss : 0.004\n",
      "epoch :  30, tr_loss : 0.003\n"
     ]
    }
   ],
   "source": [
    "# training\n",
    "tr_loss_hist = []\n",
    "\n",
    "for epoch in range(epochs):\n",
    "    avg_tr_loss = 0\n",
    "    tr_step = 0\n",
    "    \n",
    "    for x_mb, y_mb in tr_dataset:\n",
    "        with tf.GradientTape() as tape:\n",
    "            tr_loss = loss_fn(model, x=x_mb, y=y_mb)\n",
    "        grads = tape.gradient(target=tr_loss, sources=model.variables)\n",
    "        opt.apply_gradients(grads_and_vars=zip(grads, model.variables))\n",
    "        avg_tr_loss += tr_loss\n",
    "        tr_step += 1\n",
    "    else:\n",
    "        avg_tr_loss /= tr_step\n",
    "        tr_loss_hist.append(avg_tr_loss)\n",
    "    \n",
    "    if (epoch + 1) % 5 ==0:        \n",
    "        print('epoch : {:3}, tr_loss : {:.3f}'.format(epoch + 1, avg_tr_loss.numpy()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Checking performance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "acc : 100.00%\n"
     ]
    }
   ],
   "source": [
    "yhat = model.predict(x_data)\n",
    "yhat = np.argmax(yhat, axis=-1)\n",
    "print('acc : {:.2%}'.format(np.mean(yhat == y_data)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x20af8fe42e0>]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXUAAAD4CAYAAAATpHZ6AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAAcXklEQVR4nO3de5hcdZ3n8fe3qvqSviSdpCsJJN2kO3SELCYEGwwSLqPigzICjspNV1RW2BlxcJjdFUfXUUZdLzs8OzoooiLCiojiJTPiMo4jQuSWDgnGJJKkQ+cGJJ3OpdPp9LW++0edTipNd7qSdPfpc+rzep5+6tSpX536Hg586nDOr34/c3dERCQeEmEXICIio0ehLiISIwp1EZEYUaiLiMSIQl1EJEZSYX1wdXW1z507N6yPFxGJpJUrV+529/Rwr4cW6nPnzqWpqSmsjxcRiSQz23Ks13X5RUQkRhTqIiIxolAXEYkRhbqISIwo1EVEYkShLiISIwp1EZEYiVyor2jZw5f/35/QkMEiIq8VuVBfs30/33y8md0dPWGXIiIy4UQu1OvT5QBsbu0IuRIRkYkncqE+L10BwEu7D4ZciYjIxBO5UD+1ahLFqQSbFeoiIq8RuVBPJoy508t0+UVEZAiRC3WA+uoKnamLiAwhmqGeLmdrWye9/ZmwSxERmVAiGuoV9GWcbXs6wy5FRGRCiWSo11UPdGvUJRgRkVyRDPV5QV91dWsUETlaJEO9qqyYaeXFbN6tHjAiIrkiGeoA9dXlNOvyi4jIUSIb6nXV5bqmLiIySGRDvT5dwe6Obtq7esMuRURkwohwqAc3S3W2LiJyWGRDfaAHjG6WiogcEdlQr5lWRsLUV11EJFdkQ70klaRmWpnGgBERyRHZUIdst0adqYuIHJFXqJvZZWb2opltMrPbh3i91sx+a2arzOwPZvaO0S/1teqqK3hpdweZjOYrFRGBPELdzJLAXcDbgQXAdWa2YFCzTwMPu/ti4FrgG6Nd6FDq0+V09WZ4tb1rPD5ORGTCy+dM/Txgk7tvdvce4CHgykFtHJgcLE8BXh69Eod3ZL5SXYIREYH8Qn02sC3n+fZgXa7PAu83s+3Ao8DHRqW6EQzMV6pujSIiWaN1o/Q64D53nwO8A3jAzF6zbTO7ycyazKyptbX1pD90RmUJ5cVJnamLiATyCfUdQE3O8znBulw3Ag8DuPvTQClQPXhD7n6Puze6e2M6nT6xinOYGXXpcnVrFBEJ5BPqK4AGM6szs2KyN0KXDWqzFXgLgJmdSTbUT/5UPA/11RWahFpEJDBiqLt7H3AL8Biwnmwvl7VmdoeZXRE0+1vgI2b2AvBD4IPuPi79DOvT5ezYd4iu3v7x+DgRkQktlU8jd3+U7A3Q3HWfyVleB1wwuqXlp666HHdoaTvIGbMmj/wGEZEYi/QvSuFIDxiN1igiEoNQPzwJtW6WiohEP9TLS1LMmlxKs26WiohEP9RBU9uJiAyIRajXp8vZ3NrBOHW4ERGZsGIS6hW0d/Wx52BP2KWIiIQqHqGum6UiIkBcQl2TUIuIADEJ9TlTyyhOJmjWaI0iUuBiEerJhHHa9DL1gBGRgheLUIeBbo06UxeRwhabUK9PV7B1Tyd9/ZmwSxERCU2MQr2c3n5n+95DYZciIhKa2IT6vIH5SnWzVEQKWGxCva46mK9UN0tFpIDFJtSnlRdTVVakHyCJSEGLTahD9pel6gEjIoUsXqGertDlFxEpaLEK9brqcnYd6OZAV2/YpYiIhCJWoT7QA6Zld2fIlYiIhCNWoV4fzFeqbo0iUqhiFeqnTS/DDJp1XV1EClSsQr0klWTO1Em8pG6NIlKgYhXqAPXVFerWKCIFK36hni7npd0HNV+piBSk+IV6dTmdPf282t4VdikiIuMufqEe9IDR1HYiUohiGOrZvurNulkqIgUodqE+a3Ipk4qSulkqIgUpdqFuZsHUdjpTF5HCE7tQhyM9YERECk1MQ72C7Xs76e7rD7sUEZFxFctQn5cuJ+OwpU0De4lIYYllqNdVB/OV6mapiBSYeIe6rquLSIHJK9TN7DIze9HMNpnZ7cO0udrM1pnZWjN7cHTLPD6VpUXMqCxRDxgRKTipkRqYWRK4C7gU2A6sMLNl7r4up00D8EngAnffa2YzxqrgfNWnNV+piBSefM7UzwM2uftmd+8BHgKuHNTmI8Bd7r4XwN13jW6Zx6+uukLdGkWk4OQT6rOBbTnPtwfrcs0H5pvZ783sGTO7bLQKPFHz0uXs7exl78GesEsRERk3o3WjNAU0AJcA1wHfNrOqwY3M7CYzazKzptbW1lH66KENjAGjqe1EpJDkE+o7gJqc53OCdbm2A8vcvdfdXwI2kA35o7j7Pe7e6O6N6XT6RGvOS111drRGTW0nIoUkn1BfATSYWZ2ZFQPXAssGtfk52bN0zKya7OWYzaNX5vGrmTqJ4mSCjTsPhFmGiMi4GjHU3b0PuAV4DFgPPOzua83sDjO7Imj2GNBmZuuA3wL/3d3bxqrofKSSCRbXVvFUc6hliIiMqxG7NAK4+6PAo4PWfSZn2YHbgr8J46L5ab762Ivs7uimuqIk7HJERMZcLH9ROmDp6dUA/H7T7pArEREZH7EO9bNmT6GqrIgnNyrURaQwxDrUkwnjgnnVLN+4m+wVIhGReIt1qANc2FDNq+1dbNql/uoiEn+xD/WlDdnr6k/oEoyIFIDYh/qcqWXUV5ezfOPY/oJVRGQiiH2oQ/Zs/ZnNezS9nYjEXkGE+oUNaQ719vP8ln1hlyIiMqYKItSX1E8jlTCe1CUYEYm5ggj1ytIiFtdWsVw/QhKRmCuIUIfsJZg1O/azR+Ori0iMFUyoL22oxl1DBohIvBVMqC+cPYXJpSmWq7+6iMRYwYR6KpngTfOqeXJjq4YMEJHYKphQB7hwfjUv7+9isyakFpGYKqxQPz07hd6TG9S1UUTiqaBCvXZ6GadNL1PXRhGJrYIKdchOnPF0cxs9fZmwSxERGXUFF+oXNqQ52NPPqq17wy5FRGTUFVyonz9vOsmE6RKMiMRSwYX6lElFLJozReOri0gsFVyoQzBkwPZ97OvUkAEiEi8FGurVZByeam4LuxQRkVFVkKG+qKaKypKUhuIVkdgpyFAvSiZYMm86T2zYrSEDRCRWCjLUAS5qqGbHvkO0tHWGXYqIyKgp2FBf2pAdMkATUotInBRsqM+dXsacqZPUtVFEYqVgQ93MuLAhzTPNbfT2a8gAEYmHgg11yHZtPNDdxwvb9oVdiojIqCjoUH/TvOkkDF2CEZHYKOhQryor5vVzqnSzVERio6BDHbJdG1dv28f+Q71hlyIictIKPtSXnp4dMuBpDRkgIjFQ8KG+uHYq5cVJDRkgIrGQV6ib2WVm9qKZbTKz24/R7t1m5mbWOHoljq3iVIIl9dN5UjdLRSQGRgx1M0sCdwFvBxYA15nZgiHaVQK3As+OdpFj7c1nzmDrnk5WtOwJuxQRkZOSz5n6ecAmd9/s7j3AQ8CVQ7T7B+DLQNco1jcu3rV4NlPLirj78eawSxEROSn5hPpsYFvO8+3BusPM7Bygxt1/OYq1jZuy4hQ3vGkuv/nTLl589UDY5YiInLCTvlFqZgngTuBv82h7k5k1mVlTa+vEujF5w/lzmVSU5J4nNoddiojICcsn1HcANTnP5wTrBlQCZwGPm1kLsARYNtTNUne/x90b3b0xnU6feNVjYGp5MdecW8MvVu/g5X2Hwi5HROSE5BPqK4AGM6szs2LgWmDZwIvuvt/dq919rrvPBZ4BrnD3pjGpeAzduLQOB767/KWwSxEROSEjhrq79wG3AI8B64GH3X2tmd1hZleMdYHjqWZaGe9ceAo/fG6rJqUWkUjK65q6uz/q7vPdfZ67fyFY9xl3XzZE20uieJY+4OaL59HZ088DT28JuxQRkeNW8L8oHezMUyZzyevS3PdUC129/WGXIyJyXBTqQ7j5onm0Hezhxyu3h12KiMhxUagPYUn9NBbVVPHtJzbTp1mRRCRCFOpDMDP+8uJ6tu7p5Fd/fDXsckRE8qZQH8alC2ZRX13O3b9rxt3DLkdEJC8K9WEkE8ZNF9Wz9uV2lm/SCI4iEg0K9WN41zmzSVeW8K3faegAEYkGhfoxlKSSfPiCOpZv2s2a7fvDLkdEZEQK9RG8b0ktlSUp7n5Cw/KKyMSnUB/B5NIirl9Sy6/WvMKWtoNhlyMickwK9TzceEEdqURCw/KKyISnUM/DjMml/MU5s/nxyu20HugOuxwRkWEp1PP0kYvq6e3P8P2nWsIuRURkWAr1PM1LV/C2BTO5/+kWOrr7wi5HRGRICvXj8F8vnkd7V5/O1kVkwlKoH4fFtVN565kz+dpvNtKyWz1hRGTiUagfp89fdRbFqQSfeOQPZDIaE0ZEJhaF+nGaNaWUT19+Js++tIcfPLc17HJERI6iUD8BVzfWsPT0ar706Hp27DsUdjkiIocp1E+AmfG//uL1OPDJn67R0LwiMmEo1E9QzbQyPnHZGTyxoZVHnt8RdjkiIoBC/aT85yWnce7cqdzxL2vZ1d4VdjkiIgr1k5FIGF9+90K6+zJ8+ud/1GUYEQmdQv0k1acruO3S+fzbup38cs0rYZcjIgVOoT4Kblxax8I5U/j7X6xlz8GesMsRkQKmUB8FqWSCr7xnIe1dvXzuX9aGXY6IFDCF+ig5Y9ZkPvpnp/OL1S/z7+t2hl2OiBQohfoo+qtLTueMWZV86udr2H+oN+xyRKQAKdRHUXEqwVffs4jdHT188Zfrwy5HRAqQQn2UvX7OFD5yYT0/atrGkxtbwy5HRAqMQn0MfPytDdSny7n9kTXs61RvGBEZPwr1MVBalOQf37uI1gPdfOyHq+jrz4RdkogUCIX6GFlcO5XPX3UWT27czZd+9aewyxGRApEKu4A4u/rcGta90s53lr/EmadM5t1vmBN2SSISczpTH2OfuvxM3jRvOp/82RpWbd0bdjkiEnN5hbqZXWZmL5rZJjO7fYjXbzOzdWb2BzP7jZmdNvqlRlNRMsFd15/DzMkl3PzASnZqNEcRGUMjhrqZJYG7gLcDC4DrzGzBoGargEZ3Xwj8BPjKaBcaZVPLi/n2Bxrp6O7j5gdW0tXbH3ZJIhJT+ZypnwdscvfN7t4DPARcmdvA3X/r7p3B02cAXTwe5IxZk7nz6kWs3rZPw/SKyJjJJ9RnA9tynm8P1g3nRuBXJ1NUXF121inc+pYGfrJyO9/7fUvY5YhIDI1q7xczez/QCFw8zOs3ATcB1NbWjuZHR8atb2lg/SvtfOHR9cyfWcnShuqwSxKRGMnnTH0HUJPzfE6w7ihm9lbgU8AV7t491Ibc/R53b3T3xnQ6fSL1Rl4iYdx5zdnMS5fz0QefZ0vbwbBLEpEYySfUVwANZlZnZsXAtcCy3AZmthj4FtlA3zX6ZcZLRUmK73zgXMzgI/c30dHdF3ZJIhITI4a6u/cBtwCPAeuBh919rZndYWZXBM2+ClQAPzaz1Wa2bJjNSaB2ehl3XX8Oza0Hue1Hq8lkdONURE6ehdULo7Gx0ZuamkL57Ink3uUvcce/ruOvLpnH/7jsjLDLEZEJzsxWunvjcK9rmICQfeiCuWzc1cE3Hm9mRmUJH7ygLuySRCTCFOohMzP+4cr/xO6Obj73r+tIV5Zy+cJTwi5LRCJKY79MAKlkgq9ft5jG06byNz9azVPNu8MuSUQiSqE+QZQWJfnOB85lbnUZN9+/knUvt4ddkohEkEJ9AplSVsR9HzqPitIUN3zvObbt6Rz5TSIiORTqE8ypVZO4/8Pn0dOX4QP3Pkdbx5C/4xIRGZJCfQJqmFnJd29o5OV9h/jwfSs4qB8niUieFOoTVOPcafzz9eewZsd+Pvrg8/RqnlMRyYNCfQK7dMFMvviu1/P4i6184pE/aLheERmR+qlPcNeeV8uuA93c+esNpCtL+OTbzwy7JBGZwBTqEfCxN5/OzvYuvvW7zaQrSvgvF9aHXZKITFAK9QgwM+648izaOnr4/C/Xs33vIf7uHWdSnNLVMxE5mlIhIpIJ4+vXL+bDF9Rx31MtXHPP07yy/1DYZYnIBKNQj5CiZILPvHMBd11/DhtePcDlX1vOkxtbwy5LRCYQhXoEXb7wFJZ9bCnVFcV84N7n+NpvNmo8dhEBFOqRNS9dwc8/egFXnT2bO3+9gQ/dt4K9B3vCLktEQqZQj7Cy4hR3Xr2Iz191Fk83t/HnX1/O6m37wi5LREKkUI84M+P9S07jJ395PgDvvfspHni6RT9UEilQCvWYWDinil/+9VKWnl7N//zFWv7mR6tp7+oNuywRGWcK9RipKivmuzecy39723yWvfAyS774Gz750zX8ccf+sEsTkXGiHx/FTCJh3PLmBi553Qy+/1QLP1u1nR8+t5Wza6p43xtr+fOFpzKpOBl2mSIyRiysa6+NjY3e1NQUymcXkv2dvTzy/HZ+8OwWmlsPMrk0xXveUMP1b6zl9BkVYZcnIsfJzFa6e+OwryvUC4O78+xLe/i/z2zhsbWv0tvvnF8/nfctqeVtC2ZpyAGRiBgp1HX5pUCYGUvqp7OkfjqtB7r58cptPPjsVm55cBUzJ5fwsTc3cM25NRQlFe4iUaYz9QLWn3Ge2NDKNx7fxIqWvZw2vYzbLp3POxeeSiJhYZcnIkMY6Uxdp2UFLJkw/uyMGTx88/l874PnMqkoya0Prebyry/nP/60U33dRSJIoS6YZcP90b++kH+69mw6e/r48H1NXP2tp1nRsifs8kTkOCjU5bBEwrjy7Nn8+20X8/mrzmJLWyfvvftpPvS951j3cnvY5YlIHnRNXYZ1qKef+55q4ZuPb6K9q48rFp3KFYtO5XWzKpldNUnX3UVCoC6NctL2H+rlnieauXd5C4d6+wEoK07SMKOC+TMred2sSubPzP7NnFyCmcJeZKwo1GXUdHT38eKrB9iw88jfi692sLuj+3CbyaUp5s+s5IxTKllcM5XFtVXUVZcr6EVGiUJdxtyegz05IX+AjTs7WPdKOx3dfQBUlRWxuKaKxbXZkF9UU8Xk0qKQqxaJJv34SMbctPLiwz9sGtCfcZpbO3h+y15Wbd3Hqm17eXxDK+5gBg0zKg6fyddOK2NaRTHTy0uYWlZESj+AEjlhOlOXcdPe1csL2/axaus+nt+aDfv9h44eHtgMpkwqYnp5NuSnlRczvaI4+7yihBmVJcyYXMKMylLSlSWUFmlwMiksOlOXCWNyaREXNqS5sCENZMej2dLWySv7u2g72M2egz20dfRkHw9209bRQ3NrBytaetjT2cNQ5x+Vpals0FeWBmGfXZ4yqYjykhTlJUkqSlJUlKYoL05RUZKivCSlsW4kthTqEhozY251OXOry0ds259x9nb2sKu9m10Huth1oJvWA93sas8u7zrQzfNb97KrvZvuvsyI2ytOJagoSVFWnKS0KElJKjHiY0lRkklFSUqLEkwqSjKpOElJKvtYmkpkH4O2RckEqaRRlAgek9l1SXUDlTGWV6ib2WXAPwFJ4Dvu/qVBr5cA9wNvANqAa9y9ZXRLlUKWTBjVFSVUV5SwgMnDtnN3DnT30X6ol4Pd/XR099HR3cfB4LGjK1juyS539vTT3ddPd2+GruCxvav3qOddvf109WXoyePLYiRmUJRIUJQ0UkHQFx0O/SPhP7BcnEqQSmTbJs1IJi37mAj+zEgkjFTwPGFGKpl9TCYgmQjeN7CcINtmoH3wnoHtDLyesCPbyz5ydNvc54nsF3QyeJ9Zdj8TOc8Tlm2TMMNgyHYJA3LfN9AuAcbA+8E48t6B5YH2FnxOIRsx1M0sCdwFXApsB1aY2TJ3X5fT7EZgr7ufbmbXAl8GrhmLgkWOxcyYXFo0Jr1rMhmnuy8b8oeCv67DfxkO9fTT1Zdd7uvP0Jtx+voz9PU7PcFjXyZDb7/T23+kTW9fht7By0Gbg919h5cz7vRlnEzG6Xenvz94zEB/JkN/xrN/7mQyBK8V5vg92cDP/vsw8OXA4XXZL4OBLxqDnNde+yVB8MURNDvqy2TgPUd97hCvH24RrPv4W+fzzkWnjsm+53Omfh6wyd03Z4u2h4ArgdxQvxL4bLD8E+CfzcxcI0JJjCQSxqTi7OWWqWEXkyd3J+McFfiHlzOO+5F1A18EGc/54gjWZwbW52wv40de6/dgW5kjnwnZx4w7Pugx49l27uA57TIO+NDvcwaeH1k+so3cdke2cbhd8DrBsvtrtzXwzysoIbu9YDnYavCZR2/38HsZtP2B9hy9bTzbzXes5BPqs4FtOc+3A28cro2795nZfmA6sDu3kZndBNwEUFtbe4Ili0i+spdF0LX8AjKuXQDc/R53b3T3xnQ6PZ4fLSJSEPIJ9R1ATc7zOcG6IduYWQqYQvaGqYiIjKN8Qn0F0GBmdWZWDFwLLBvUZhlwQ7D8HuA/dD1dRGT8jXhNPbhGfgvwGNkujfe6+1ozuwNocvdlwHeBB8xsE7CHbPCLiMg4y6ufurs/Cjw6aN1ncpa7gPeObmkiInK89FtpEZEYUaiLiMSIQl1EJEZCG3rXzFqBLSf49moG/bApBuK2T3HbH4jfPsVtfyB++zTU/pzm7sP+0Ce0UD8ZZtZ0rPGEoyhu+xS3/YH47VPc9gfit08nsj+6/CIiEiMKdRGRGIlqqN8TdgFjIG77FLf9gfjtU9z2B+K3T8e9P5G8pi4iIkOL6pm6iIgMQaEuIhIjkQt1M7vMzF40s01mdnvY9ZwsM2sxszVmttrMmsKu50SY2b1mtsvM/pizbpqZ/drMNgaPUZksaLj9+ayZ7QiO02oze0eYNR4vM6sxs9+a2TozW2tmtwbrI3mcjrE/kT1OZlZqZs+Z2QvBPn0uWF9nZs8GmfejYLTc4bcTpWvqwXypG8iZLxW4btB8qZFiZi1Ao7tH9gcTZnYR0AHc7+5nBeu+Auxx9y8FX75T3f0TYdaZr2H257NAh7v/7zBrO1Fmdgpwirs/b2aVwErgKuCDRPA4HWN/riaix8myk52Wu3uHmRUBy4FbgduAn7r7Q2Z2N/CCu39zuO1E7Uz98Hyp7t4DDMyXKiFy9yfIDrmc60rg+8Hy98n+BxcJw+xPpLn7K+7+fLB8AFhPdhrKSB6nY+xPZHlWR/C0KPhz4M1k536GPI5R1EJ9qPlSI30gyR60fzOzlcEcrnEx091fCZZfBWaGWcwoucXM/hBcnonEZYqhmNlcYDHwLDE4ToP2ByJ8nMwsaWargV3Ar4FmYJ+79wVNRsy8qIV6HC1193OAtwMfDf7XP1aCWbCic51vaN8E5gFnA68A/xhqNSfIzCqAR4CPu3t77mtRPE5D7E+kj5O797v72WSnDT0POON4txG1UM9nvtRIcfcdweMu4GdkD2Qc7Ayuew5c/9wVcj0nxd13Bv/BZYBvE8HjFFynfQT4gbv/NFgd2eM01P7E4TgBuPs+4LfA+UBVMPcz5JF5UQv1fOZLjQwzKw9u8mBm5cDbgD8e+12RkTtv7Q3AL0Ks5aQNBF/gXUTsOAU34b4LrHf3O3NeiuRxGm5/onyczCxtZlXB8iSyHULWkw339wTNRjxGker9AhB0Ufo/HJkv9QvhVnTizKye7Nk5ZKcWfDCK+2NmPwQuITtM6E7g74GfAw8DtWSHWL7a3SNx83GY/bmE7P/SO9AC3JxzLXrCM7OlwJPAGiATrP47stehI3ecjrE/1xHR42RmC8neCE2SPeF+2N3vCHLiIWAasAp4v7t3D7udqIW6iIgML2qXX0RE5BgU6iIiMaJQFxGJEYW6iEiMKNRFRGJEoS4iEiMKdRGRGPn/sthUFAdKl5gAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(tr_loss_hist)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}