Lab12/Lab 12-2 eunseoChoi.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# lab-12-2-many-to-one-stacking-keras-eager.ipynb"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## sentence sentiment classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.5.0\n"
     ]
    }
   ],
   "source": [
    "# setup\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import matplotlib.pyplot as plt\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras import layers\n",
    "from tensorflow.keras import Sequential, Model\n",
    "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
    "from pprint import pprint\n",
    "%matplotlib inline\n",
    "\n",
    "print(tf.__version__)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prepairing dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# example data\n",
    "sentences = ['What I cannot create, I do not understand.',\n",
    "             'Intellecuals solve problems, geniuses prevent them',\n",
    "             'A person who never made a mistake never tied anything new.',\n",
    "             'The same equations have the same solutions.']\n",
    "y_data = [1,0,0,1] # 1: richard feynman, 0: albert einstein"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['<pad>', ' ', ',', '.', 'A', 'I', 'T', 'W', 'a', 'b', 'c', 'd', 'e', 'g', 'h', 'i', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'y']\n",
      "{0: '<pad>', 1: ' ', 2: ',', 3: '.', 4: 'A', 5: 'I', 6: 'T', 7: 'W', 8: 'a', 9: 'b', 10: 'c', 11: 'd', 12: 'e', 13: 'g', 14: 'h', 15: 'i', 16: 'k', 17: 'l', 18: 'm', 19: 'n', 20: 'o', 21: 'p', 22: 'q', 23: 'r', 24: 's', 25: 't', 26: 'u', 27: 'v', 28: 'w', 29: 'y'}\n",
      "{'<pad>': 0, ' ': 1, ',': 2, '.': 3, 'A': 4, 'I': 5, 'T': 6, 'W': 7, 'a': 8, 'b': 9, 'c': 10, 'd': 11, 'e': 12, 'g': 13, 'h': 14, 'i': 15, 'k': 16, 'l': 17, 'm': 18, 'n': 19, 'o': 20, 'p': 21, 'q': 22, 'r': 23, 's': 24, 't': 25, 'u': 26, 'v': 27, 'w': 28, 'y': 29}\n"
     ]
    }
   ],
   "source": [
    "# creating a token dictionary\n",
    "char_set = ['<pad>'] + sorted(list(set(''.join(sentences))))\n",
    "idx2char = {idx : char for idx, char in enumerate(char_set)}\n",
    "char2idx = {char : idx for idx, char in enumerate(char_set)}\n",
    "\n",
    "print(char_set)\n",
    "print(idx2char)\n",
    "print(char2idx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[7, 14, 8, 25, 1, 5, 1, 10, 8, 19, 19, 20, 25, 1, 10, 23, 12, 8, 25, 12, 2, 1, 5, 1, 11, 20, 1, 19, 20, 25, 1, 26, 19, 11, 12, 23, 24, 25, 8, 19, 11, 3], [5, 19, 25, 12, 17, 17, 12, 10, 26, 8, 17, 24, 1, 24, 20, 17, 27, 12, 1, 21, 23, 20, 9, 17, 12, 18, 24, 2, 1, 13, 12, 19, 15, 26, 24, 12, 24, 1, 21, 23, 12, 27, 12, 19, 25, 1, 25, 14, 12, 18], [4, 1, 21, 12, 23, 24, 20, 19, 1, 28, 14, 20, 1, 19, 12, 27, 12, 23, 1, 18, 8, 11, 12, 1, 8, 1, 18, 15, 24, 25, 8, 16, 12, 1, 19, 12, 27, 12, 23, 1, 25, 15, 12, 11, 1, 8, 19, 29, 25, 14, 15, 19, 13, 1, 19, 12, 28, 3], [6, 14, 12, 1, 24, 8, 18, 12, 1, 12, 22, 26, 8, 25, 15, 20, 19, 24, 1, 14, 8, 27, 12, 1, 25, 14, 12, 1, 24, 8, 18, 12, 1, 24, 20, 17, 26, 25, 15, 20, 19, 24, 3]]\n",
      "[42, 50, 58, 43]\n",
      "[1, 0, 0, 1]\n"
     ]
    }
   ],
   "source": [
    "# converting sequence of tokens to sequence of indices\n",
    "x_data = list(map(lambda sentence : [char2idx.get(char) for char in sentence], sentences))\n",
    "x_data_len = list(map(lambda sentence : len(sentence), sentences))\n",
    "\n",
    "print(x_data)\n",
    "print(x_data_len)\n",
    "print(y_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 7 14  8 25  1  5  1 10  8 19 19 20 25  1 10 23 12  8 25 12  2  1  5  1\n",
      "  11 20  1 19 20 25  1 26 19 11 12 23 24 25  8 19 11  3  0  0  0  0  0  0\n",
      "   0  0  0  0  0  0  0]\n",
      " [ 5 19 25 12 17 17 12 10 26  8 17 24  1 24 20 17 27 12  1 21 23 20  9 17\n",
      "  12 18 24  2  1 13 12 19 15 26 24 12 24  1 21 23 12 27 12 19 25  1 25 14\n",
      "  12 18  0  0  0  0  0]\n",
      " [ 4  1 21 12 23 24 20 19  1 28 14 20  1 19 12 27 12 23  1 18  8 11 12  1\n",
      "   8  1 18 15 24 25  8 16 12  1 19 12 27 12 23  1 25 15 12 11  1  8 19 29\n",
      "  25 14 15 19 13  1 19]\n",
      " [ 6 14 12  1 24  8 18 12  1 12 22 26  8 25 15 20 19 24  1 14  8 27 12  1\n",
      "  25 14 12  1 24  8 18 12  1 24 20 17 26 25 15 20 19 24  3  0  0  0  0  0\n",
      "   0  0  0  0  0  0  0]]\n",
      "[42, 50, 58, 43]\n",
      "[1, 0, 0, 1]\n"
     ]
    }
   ],
   "source": [
    "# padding the sequence of indices\n",
    "max_sequence = 55\n",
    "x_data = pad_sequences(sequences = x_data, maxlen = max_sequence,\n",
    "                       padding = 'post', truncating = 'post')\n",
    "\n",
    "# checking data\n",
    "print(x_data)\n",
    "print(x_data_len)\n",
    "print(y_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Creating model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# creating stacked rnn for \"many to one\" classification with dropout\n",
    "num_classes = 2\n",
    "hidden_dims = [10,10]\n",
    "\n",
    "input_dim = len(char2idx)\n",
    "output_dim = len(char2idx)\n",
    "one_hot = np.eye(len(char2idx))\n",
    "\n",
    "model = Sequential()\n",
    "model.add(layers.Embedding(input_dim=input_dim, output_dim=output_dim,\n",
    "                           trainable=False, mask_zero=True, input_length=max_sequence,\n",
    "                           embeddings_initializer=keras.initializers.Constant(one_hot)))\n",
    "model.add(layers.SimpleRNN(units=hidden_dims[0], return_sequences=True))\n",
    "model.add(layers.TimeDistributed(layers.Dropout(rate = .2)))\n",
    "model.add(layers.SimpleRNN(units=hidden_dims[1]))\n",
    "model.add(layers.Dropout(rate = .2))\n",
    "model.add(layers.Dense(units=num_classes))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding (Embedding)        (None, 55, 30)            900       \n",
      "_________________________________________________________________\n",
      "simple_rnn (SimpleRNN)       (None, 55, 10)            410       \n",
      "_________________________________________________________________\n",
      "time_distributed (TimeDistri (None, 55, 10)            0         \n",
      "_________________________________________________________________\n",
      "simple_rnn_1 (SimpleRNN)     (None, 10)                210       \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 10)                0         \n",
      "_________________________________________________________________\n",
      "dense (Dense)                (None, 2)                 22        \n",
      "=================================================================\n",
      "Total params: 1,542\n",
      "Trainable params: 642\n",
      "Non-trainable params: 900\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# creating loss function\n",
    "def loss_fn(model, x, y, training):    \n",
    "    return tf.reduce_mean(tf.keras.losses.sparse_categorical_crossentropy(\n",
    "        y_true=y, y_pred=model(x, training), from_logits=True))\n",
    "\n",
    "# creating and optimizer\n",
    "lr = .01\n",
    "epochs = 30\n",
    "batch_size = 2\n",
    "opt = tf.keras.optimizers.Adam(learning_rate = lr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<BatchDataset shapes: ((None, 55), (None,)), types: (tf.int32, tf.int32)>\n"
     ]
    }
   ],
   "source": [
    "# generating data pipeline\n",
    "tr_dataset = tf.data.Dataset.from_tensor_slices((x_data, y_data))\n",
    "tr_dataset = tr_dataset.shuffle(buffer_size=4)\n",
    "tr_dataset = tr_dataset.batch(batch_size=batch_size)\n",
    "\n",
    "print(tr_dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch :   5, tr_loss : 0.289\n",
      "epoch :  10, tr_loss : 0.077\n",
      "epoch :  15, tr_loss : 0.051\n",
      "epoch :  20, tr_loss : 0.009\n",
      "epoch :  25, tr_loss : 0.038\n",
      "epoch :  30, tr_loss : 0.006\n"
     ]
    }
   ],
   "source": [
    "# training\n",
    "tr_loss_hist = []\n",
    "\n",
    "for epoch in range(epochs):\n",
    "    avg_tr_loss = 0\n",
    "    tr_step = 0\n",
    "    \n",
    "    for x_mb, y_mb in tr_dataset:\n",
    "        with tf.GradientTape() as tape:\n",
    "            tr_loss = loss_fn(model, x=x_mb, y=y_mb, training=True)\n",
    "        grads = tape.gradient(target=tr_loss, sources=model.variables)\n",
    "        opt.apply_gradients(grads_and_vars=zip(grads, model.variables))\n",
    "        avg_tr_loss += tr_loss\n",
    "        tr_step += 1\n",
    "    else:\n",
    "        avg_tr_loss /= tr_step\n",
    "        tr_loss_hist.append(avg_tr_loss)\n",
    "    \n",
    "    if (epoch + 1) % 5 ==0:\n",
    "        print('epoch : {:3}, tr_loss : {:.3f}'.format(epoch + 1, avg_tr_loss.numpy()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Checking performance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy : 100.00%\n"
     ]
    }
   ],
   "source": [
    "yhat = model.predict(x_data)\n",
    "yhat = np.argmax(yhat, axis=-1)\n",
    "print('accuracy : {:.2%}'.format(np.mean(yhat == y_data)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x1b0f3840fd0>]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXUAAAD4CAYAAAATpHZ6AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXBc5Z3u8e9PrV2WtUu2JWvxihdsB4TNEsAODjYkgxMKBkhIQpIJIQEqmZrMZLk3MySZDDMhYcxUIIQhuRPuDVDMQBKSGAw47GbxDpZl2fImyZt2WbZ29Xv/6JaRhZaWLbt1up9Plat1Tp/u/p065adPv+973mPOOUREJDLEhLsAEREZOwp1EZEIolAXEYkgCnURkQiiUBcRiSCx4frg7OxsV1xcHK6PFxHxpE2bNtU753KGej5soV5cXMzGjRvD9fEiIp5kZgeGe17NLyIiEUShLiISQRTqIiIRRKEuIhJBFOoiIhFEoS4iEkEU6iIiEcRzob7zyDH+9bmdHOvoDncpIiLjjudCvbqxnYdf3cPeuhPhLkVEZNzxXKgXZyUDcKBBoS4iMpDnQn1qZjJmsL++LdyliIiMO54L9cQ4H1PSktivM3URkQ/xXKgDFGUlK9RFRAbh0VBP4UCDml9ERAbyZKiXZCfTeKKLlnYNaxQR6c+ToV6UlQJoBIyIyECeDPXiYKjvVxOMiMgpPBnqRcGx6vvrdaYuItKfJ0M9Mc7H5LREjYARERnAk6EOgbN1jYARETmVZ0O9JDtFHaUiIgN4NtSLslKoP95Fq2ZrFBE5ybOh/sHEXmqCERHp49lQLzo5rFFNMCIifTwc6hrWKCIykGdDPTk+lryJCboASUSkH8+GOvRN7KUzdRGRPp4O9ZKsFPbpZhkiIid5OtSLspOpP97J8c6ecJciIjIueDrUizVbo4jIKSIk1NUEIyICHg/1vmGN+zSsUUQE8HiopyTEkpOaoOYXEZEgT4c6BEbAaKy6iEiA50O9KCtZV5WKiAR5PtSLs1Oobe2krUvDGkVEQgp1M1tpZhVmVmlm3xnk+TQz+6OZbTOzMjP74tiXOjiNgBER+cCIoW5mPuBB4BpgLnCLmc0dsNmdwA7n3EJgKfAzM4sf41oHpYm9REQ+EMqZ+mKg0jm31znXBTwJrBqwjQNSzcyACUAjcE7aQ06Gus7URURCCvV8oLrfck1wXX8/B+YAh4D3gW845/wD38jMbjezjWa2sa6u7jRLPlVqYhzZE+I1rFFEhNBC3QZZ5wYsrwC2AlOARcDPzWzih17k3CPOuVLnXGlOTs6oix1KcVaKLkASESG0UK8BpvZbLiBwRt7fF4FnXEAlsA84b2xKHFlgCl41v4iIhBLqG4CZZlYS7Py8GXh2wDZVwFUAZpYHzAb2jmWhwynOSubIsQ7au3rP1UeKiIxLI4a6c64HuAtYC5QDTznnyszsDjO7I7jZj4BLzex9YB3wbedc/dkqeqDi7MCwxqpGna2LSHSLDWUj59waYM2AdQ/3+/sQcPXYlha6vrHq++pPMHtSarjKEBEJO89fUQqBm2WA5lUXEYmIUJ+YGEdWSrzGqotI1IuIUAdN7CUiAhEU6sVZKWp+EZGoFzmhnp3CoZYOOro1rFFEolfEhHrfHDAa1igi0SxiQr1vWKPa1UUkmkVcqGu6ABGJZhET6mnJcWQkx7FPnaUiEsUiJtShb2IvhbqIRK+ICvXirGT216v5RUSiV2SFenYKh1ra6ezRsEYRiU6RFepZKTgH1RrWKCJRKqJC/YObUCvURSQ6RVSolwTnVd+vzlIRiVIRFerpyfGkJcUp1EUkakVUqENgBIwuQBKRaBV5oZ6dojN1EYlaERfqRVkpHGxqp6vHH+5SRETOuYgL9eKsZPwOqpvUBCMi0SfyQj27b2IvNcGISPSJvFA/OQWvztRFJPpEXKhnJMeRmhirzlIRiUoRF+pmRkl2Cvs1rFFEolDEhTpoCl4RiV4RGerFWcnUNLXT3athjSISXSIy1IuyUuj1O2qa2sNdiojIORWRoV6SHZytUU0wIhJlIjLUi04Oa1Soi0h0ichQz0qJZ0JCrCb2EpGoE5GhbmYUZyer+UVEok5EhjoEmmDU/CIi0SZiQ13DGkUkGoUU6ma20swqzKzSzL4zxDZLzWyrmZWZ2atjW+boFWel0ON3HGrWsEYRiR4jhrqZ+YAHgWuAucAtZjZ3wDbpwEPAdc65ecCNZ6HWUSk+eb9SdZaKSPQI5Ux9MVDpnNvrnOsCngRWDdjmM8AzzrkqAOdc7diWOXpFWcGx6mpXF5EoEkqo5wPV/ZZrguv6mwVkmNkrZrbJzD4/2BuZ2e1mttHMNtbV1Z1exSHKmZBASryPfQp1EYkioYS6DbLODViOBS4EPgGsAL5vZrM+9CLnHnHOlTrnSnNyckZd7GiYGR8pzODpzTXsqTt+Vj9LRGS8CCXUa4Cp/ZYLgEODbPO8c+6Ec64eeA1YODYlnr5/u2EB8b4YvvLYRo51dIe7HBGRsy6UUN8AzDSzEjOLB24Gnh2wzR+Ay80s1sySgSVA+diWOnr56Uk89NkLqGpo4xtPbKHXP/AHhohIZBkx1J1zPcBdwFoCQf2Uc67MzO4wszuC25QDzwPvAe8Cjzrntp+9skO3ZFoW91w3j5cr6vjpCxXhLkdE5KyKDWUj59waYM2AdQ8PWL4PuG/sShs7t15cxI7Dx/jFK3s4b1IqqxYN7OcVEYkMEXtF6UD3/NU8Fhdn8u2n32P7wZZwlyMiclZETajHx8bw0K0XkJkcz+2PbaSutTPcJYmIjLmoCXWA7AkJPPL5Uhrbuvj6bzfR1aN5YUQkskRVqAPMz0/jJzcsZMP+Ju75Y1m4yxERGVMhdZRGmusWTqE82HE6d/JEbr24KNwliYiMiag7U+/zratns2x2Dvc8W8bbexvCXY6IyJiI2lD3xRgP3PIRCrOS+fpvN1PTpNkcRcT7ojbUASYmxvGfny+lu9fP7Y9torOnN9wliYickagOdYDpORO49/rz2XH4GG/srg93OSIiZyTqQx3g6rmTmJAQy0vlR8NdiojIGVGoE7gw6cpZOawrr8WvSb9ExMMU6kHL5+ZS29rJ+5pCQEQ8TKEetGx2Lr4YUxOMiHiaQj0oPTmeC4syeHGHQl1EvEuh3s/H5+Sx80irxqyLiGcp1PtZPjcPgHXltWGuRETk9CjU+ynJTmF6Tora1UXEsxTqAyyfk8fbexto1Y2qRcSDFOoDLJ+bR3ev47VdurpURLxHoT7ABYUZZCTHqQlGRDxJoT6AL8ZYdl4uf9lZS0+v7owkIt6iUB/Ex+fk0dLezcYDTeEuRURkVBTqg7h8Vg7xvhjWqQlGRDxGoT6ICQmxXDI9ixd3HMU5TfAlIt6hUB/C8jm57G9oY0/diXCXIiISMoX6EK6a03d1qZpgRMQ7FOpDmJKexLwpEzW0UUQ8RaE+jOVz8th0oImG453hLkVEJCQK9WEsn5OH38HLFXXhLkVEJCQK9WHMz59I3sQEtauLiGco1IdhZiyfk8eru+ro6O4NdzkiIiNSqI9g+Zw82rp6eXtvQ7hLEREZkUJ9BJdMzyIpzqdRMCLiCSGFupmtNLMKM6s0s+8Ms91FZtZrZjeMXYnhlRjn44pZ2awrr9XVpSIy7o0Y6mbmAx4ErgHmAreY2dwhtvs3YO1YFxluy+fkcbilg7JDx8JdiojIsEI5U18MVDrn9jrnuoAngVWDbHc38DQQcTf4XHZeLmaoCUZExr1QQj0fqO63XBNcd5KZ5QOfBh4e7o3M7HYz22hmG+vqvDP2O3tCAhcUZijURWTcCyXUbZB1AxuXVwPfds4NO+7POfeIc67UOVeak5MTao3jwvI5eWw/eIzDLe3hLkVEZEihhHoNMLXfcgFwaMA2pcCTZrYfuAF4yMw+NSYVjhPL5+QCsK484lqXRCSChBLqG4CZZlZiZvHAzcCz/TdwzpU454qdc8XA/wBfd879fsyrDaMZuRMoykpWE4yIjGsjhrpzrge4i8ColnLgKedcmZndYWZ3nO0Cx4u+q0vX72ngRGdPuMsRERlUbCgbOefWAGsGrBu0U9Q5d9uZlzU+LZ+Tx6/e2Mfru+tZOX9SuMsREfkQXVE6CqXFGaQlxfG7LTXhLkVEZFAK9VGI88XwpctKWFt2lFcq1GEqIuOPQn2U7lg6jek5Kfzv32+nrUtt6yIyvijURykh1se91y+gpqmdB17aHe5yREROoVA/DYtLMrn5oqk8+sY+yg61hLscEZGTFOqn6bvXzCEjOY7vPvM+vX7N3igi44NC/TSlJcfx/U/O5b2aFh57a3+4yxERARTqZ+S6hVO4clYOP11bwaFmzQkjIuGnUD8DZsY/f2o+vc7xT8+WhbscERGF+pmampnM3y6fxYs7jvL89iPhLkdEopxCfQx86aMlzJk8kXueLaO1ozvc5YhIFFOoj4E4Xwz3Xn8+R1s7+OnainCXIyJRTKE+RhZNTecLlxTz2NsH2FzVFO5yRCRKKdTH0N9dPYu81ES+98z7dPf6w12OiEQhhfoYSk2M4wer5rHzSCuPvr4v3OWISBRSqI+xFfMmsWJeHg+s20VVQ1u4yxGRKBPSTTJkdH5w3XyW3/8q//D0Nm67tITkeB9J8T6S4nwkxvkCy3GBdQmxMZgNdm9vEZHRU6ifBZPSEvnutefxv363nbf3Ng67rRkkxfn42pXTufuqmeeoQhGJVAr1s+SzS4pYOjuX5rYuOrp7ae/y09bVQ3t3L+1dvYHH4N8b9zexet1urjl/MjNyJ4S7dBHxMIX6WZSfnkR+etKI29Uf72Tpfa9w75pyfnXbReegMhGJVOooHQeyJyRw57IZrNtZy/rK+nCXIyIeplAfJ754WTH56Un885/LNT+7iJw2hfo4kRjn4x9WzmbH4WM8vbkm3OWIiEcp1MeR6xZOYeHUdH66tkI3tRaR06JQH0fMjO9/Yg61rZ088trecJcjIh6kUB9nSoszufb8Sfzy1b0cPdYR7nJExGMU6uPQt1eeR4/fr2l8RWTUFOrjUFFWCrddWsz/bK6h7FBLuMsREQ9RqI9Tdy2bSVpSHD/+cznOaYijiIRGoT5OpSXH8Y2rZrJ+TwN/2Vkb7nJExCMU6uPYrRcXMS07hX9ZU66bbohISBTq41icL4bvXHMee+pO8OS7VeEuR0Q8QKE+zn18bh5LSjL595d2c6yjO9zliMg4F1Kom9lKM6sws0oz+84gz3/WzN4L/ltvZgvHvtToZGZ8/5NzaWrr4sGXK8NdjoiMcyOGupn5gAeBa4C5wC1mNnfAZvuAK51zC4AfAY+MdaHRbH5+Gp/+SD7/5439VDfqFnkiMrRQztQXA5XOub3OuS7gSWBV/w2cc+udc03BxbeBgrEtU/5+xWxiYuAnY3xB0mNv7Wfl6teoa+0c0/cVkfAIJdTzgep+yzXBdUP5MvDcYE+Y2e1mttHMNtbV1YVepTA5LYnbL5/GH7cd4sGXK8947LpzjtUv7eIf/1DGziOtPLWxeuQXici4F0qoD3ZX5EETxcyWEQj1bw/2vHPuEedcqXOuNCcnJ/QqBYA7PzaDVYumcN/aCr713+/R1XN6wxz9fsc9z5ax+qXd3HBhAYtLMnlyQxV+zeMu4nmhhHoNMLXfcgFwaOBGZrYAeBRY5ZxrGJvypL+EWB+rb1rEN5fP5OnNNXzuV+/QdKJrVO/R3evnb5/aym/eOsBXLi/hvhsWcOvFRVQ3tvOG7rok4nmhhPoGYKaZlZhZPHAz8Gz/DcysEHgG+JxzbtfYlyl9zIxvLp/F6psWsaWqmet/sZ599SdCem17Vy9f/b+b+MPWQ/z9itl879o5mBkr5uWRmRLP4+9oLLyI140Y6s65HuAuYC1QDjzlnCszszvM7I7gZv8IZAEPmdlWM9t41ioWAD71kXwe/8oSWtq7+fRDb/L23uF/HLW0d/P5X7/DyxW1/PjT87lz2QzMAi1rCbE+briwgJfKj1Kr6X5FPM3CNVlUaWmp27hR2X+mDjSc4Ev/tYGqxjbuvX4BN1z44YFHta0dfOHXG6isbeXfb1rEJxdM+dA2e+uO87Gfvcrfr5jNnctmnIvSReQ0mNkm51zpUM/rilKPK8pK4ZmvXcZFxZl867+38dO1Fad0eFY3tnHjw2+xv/4Ej37hokEDHWBazgQumZbFE++qw1TEyxTqESAtOY7ffGkxN180lZ+/XMndT26ho7uXiiOt3PDweprbuvl/f7OEK2cNP+LoliWF1DS187o6TEU8KzbcBcjYiPPFcO/15zMtJ4V7n9tJVUMbVY1tJMTG8NRXL2H2pNQR36Ovw/SJd6pG/AIQkfFJZ+oRxMy4/Yrp/OKzF7K7tpX05Die/tqlIQU6fNBh+qI6TEU8S6EegVbOn8TL31rKn+7+KFMzk0f12psvmkqv3/Hfm2rOUnUicjYp1CPU5LQkUhPjRv06dZiKeJtCXT7kM+owFfEshbp8yNUnrzA9EO5SRGSUFOryIQmxPm68sICXymvVYSriMQp1GdTNiwvVYSriQQp1GVRJdgqXTleHqYjXKNRlSLcsVoepiNco1GVIK+ZNIksdpiKeolCXIcXHxgSn5FWHqYhXKNRlWH0dpqO5h+nxzh62VTefxapEZCgKdRnWBx2m1SN2mFY1tPHDP+7gkn9Zx6oH3+TBlyvPUZUi0kezNMqIPrOkkLse38Jru+tYOjv3lOecc7y1p4Ffv7mfdTuP4jPjEwsm093r5761FQC66YbIOaRQlxFdPTfQYfrEu1UnQ72ju5ffbznIf63fz84jrWSmxHPXshncenEReRMT6fU74nxbuW9tBTFmfG3p9DDvhUh0UKjLiOJjY7ihtIBHX9/Htupm1pYd4Yl3q2hq62bO5In85IYFXLdwColxvpOv8cUYP7txIc7Bvz2/kxiDr16pYBc52xTqEpJbLirkl6/uZdWDbxJj8PG5eXzxshKWlGSevIH1QLG+GO7/64X4nePe53YSY8ZXrph2jisXiS4KdQlJcXYK37hqJu3dvXzu4qKQ52mP9cWw+qZFOODHa8oxg7+5XMEucrYo1CVkf/vxWaf1ulhfDA/ctAgc/POfyzEzvvzRkjGuLmD7wRb+Z1MNyfE+VsybxIKCtCF/SYhEIoW6nBOxvhhW37wIv3P86E87iDH44mVjE+ztXb388b1D/PadKrZVN5MQG0OP3/HQK3uYNDGRq+flsWLeJBaXZBLn0yheiWwKdTln4nwx/MctH+Hux7fwgz/uwIDbziDYK2tb+e07VTy9qYZjHT3MzJ3APX81l09fUIBzjnXltawtO8JTG6t57K0DpCXFcdV5uVw9bxJXzsohKd438oeIeIw5F54Z+EpLS93GjRvD8tkSXt29fu56fDNry47yw1Xz+PwlxSG/tqvHz9qyI/z2nQO8vbeROJ9xzfzJfHZJIYuH6LRt7+rltd11rC07wrryWlrau0mMi+HymTlce/4kPnH+FOJjdQYv3mBmm5xzpUM+r1CXcOjq8XPn45t5ccdRLpuRRUp8LAlxPhJiY4L/fCTE9fs7NoajrR08vamG+uNdTM1M4jOLi7ixtIDsCQkhf253r5939zXyQtkRXthxlMMtHeSnJ3HnshnccGGBwl3GPYW6jFtdPX5++Kcy3q9pobPHT2ePn64eP509vXR2B5d7/Se3jzG4ak4en11SyBUzc4iJObMOUOccr+yq44GXdrO1upn89CS+vmw6N144VeEu45ZCXTzN73d09frp7PYTEwOpiXFj/hnOOV7dVcfqfuH+taXTubG0gIRYtbuHS9+XbtnBFm66qJCc1NB/kUUyhbpIiJxzvLa7ntUv7WJLVTNT0hL5+rIZYxruJzp72H6whfdqWth+qIXEWB8z8yYwI3cCM/NSmZKWGPVDMJ1zrN/TwM9eqGBzVWC2zwkJsdy5bAZf+mhx1H/RKtRFRsk5x+vBcN9c1czkYLhfdV4uqYmxpMTHhtT009Xjp+JIK9tqmtlW3cx7NS3srm2lb7LLyWmJdPX4aTjRdfI1KfE+ZuROYEZuKjPzJjAzdwIzc1MpyEg64+YmL3h3XyM/e6GCd/Y1Mjktkbs+NoOLijP5yfM7eam8lsLMZL537RxWzMuL2i8/hbrIaXLO8UZlPatf2s2mA00n15tBakIsqYlxpCbGMjExjolJHyz7nWP7wWPsOHyMrp5An0BGchwLCtJZWJDGwqnpLChIP9mc0HC8k8ra4+yuPR58bGX30ePUtnae/MzUhFiumpPLyvmTWTo755R5dkar9lgHr+yqY2/dCS4oTGfJtCzSkk6/Wcs5x776E7y5p4GqhhPMz0/jgsIMCjKSQg7eLVVN3P/iLl7fXU9OagJ3Lp3OzYsLT9nP13fX8aM/7WDX0eNcMi2L739yLnOnTDztur1KoS5yhpxzbNjfxN664xzr6Ka1o4fWjh6OdXRzrL2H1r51nYFlv98xZ8pEFhaksaAgnUVT00cVcH1a2rqprAsE/OaqJl7YcZTmtm6S430sOy+Xa+ZPYtnsXFIShr/cpNfv2FrdzCsVtbxcUcv2g8eAQMez3wUezy9I59LpWVw2PZvS4owRvzSOHuvgzcp63qxsYP2eeg63BO6MFRtj9AR/iuSmJnBhUQYXFGZwQVEG8/MnfqjpZPvBFu5/cRd/2VlLZko8X7tyOrdeXDTkNQQ9vX6eeLeK+1/cRUt7NzddVMjfXT1rVCOgzga/39Ht95+TpiGFukiE6O71887eRtZsP8wLZUeoP95FQmwMS2fncM38yXxsTi4Tgx3JjSe6eG1XHS9X1PLqrjqa27rxxRgXFmaw9Lwcls3OZVpOCluqmlm/p4H1lfVsrW6mx++Ij43hwsIMLp2exaUzsllYkMaJzl7e2lvP+j0NvFlZz566E0DgF8il07O5dEbgC6EgI4mKo61sPtDEpgNNbKpqorqxHYB4XwznF6RxQWE68/PTeO79IzxfdoS0pDhuv2Iat11aPOIXVJ+Wtm4eWLebx97aT1Kcj7uvmsFtl5acHLXknKP+eBdVjW3UNLVR1dBGVWPgX3VjG83t3UyamEh+RhJT0pICj+lJ5Af/TUpLPGUElHOOhhNd1DS1U93YFnhsCjzWNLZR09xOT6+fmbmpLJyaxqKpGSycmsbsvFRix/gq5jEJdTNbCTwA+IBHnXP/OuB5Cz5/LdAG3Oac2zzceyrURU5fr9+xYX8jz28/wnPbD3P0WCfxvhgum5FFc3s3W6ubcQ6yJ8Rz5axclp2Xw+UzckhLHrqZ5XhnDxv2NbJ+T+AMfMfhwBl9cryP9u5enAv8vbgkk8uCQT5n0sQR2/prWzvYfKCZzVWBoH+/poWuXj+pCbF86aMlfPnykpNfRqNVWXucf1lTzl921lKclcyM3FSqg+Hd3t17yrZ5ExMozExmakYyaclxHD3WwcGmdg42d1B/vPOUbc0CvzQmpyVxorOHmqb2D71fRnIcUzOTKchIYmpGMgmxMbx3sIVt1c00tXUDkBgXw/n5aSwsSGdRYToLC07vV9uptZ1hqJuZD9gFfByoATYAtzjndvTb5lrgbgKhvgR4wDm3ZLj3VaiLjA2/37Glupnn3j/MS+VHSUuO52OzA0E+f0raaXewNp7o4q09Dby7r4GMlHgum5HNwoL0Mx7D39nTy64jxynMTB72S2Y0Xt1Vx/0vVNDZ42dqMLgLM5MozEqmMDOZgozkYZuUOrp7OdwSCPlDze3UNAceD7e0kxIfe0p4F2QmUZCRzIQhflU456hubGdLdRPbqlvYVtPM9oOBazEg8EV7x5XTT3u20rEI9UuAe5xzK4LL3w0Wfm+/bX4JvOKceyK4XAEsdc4dHup9FeoiEi26ewMjobZUB0ZCXTErh+sWTjmt9xop1ENpwMoH+t9KvobA2fhI2+QDp4S6md0O3A5QWFgYwkeLiHhfnC+G+flpzM9P43MXF53Vzwrld9Rgv90Gnt6Hsg3OuUecc6XOudKcnJxQ6hMRkVEIJdRrgKn9lguAQ6exjYiInGWhhPoGYKaZlZhZPHAz8OyAbZ4FPm8BFwMtw7Wni4jI2TFim7pzrsfM7gLWEhjS+GvnXJmZ3RF8/mFgDYGRL5UEhjR+8eyVLCIiQwlppL9zbg2B4O6/7uF+fzvgzrEtTURERkuTRouIRBCFuohIBFGoi4hEkLBN6GVmdcCB03x5NlA/huWMB5G2T5G2PxB5+xRp+wORt0+D7U+Rc27IC33CFupnwsw2DneZrBdF2j5F2v5A5O1TpO0PRN4+nc7+qPlFRCSCKNRFRCKIV0P9kXAXcBZE2j5F2v5A5O1TpO0PRN4+jXp/PNmmLiIig/PqmbqIiAxCoS4iEkE8F+pmttLMKsys0sy+E+56xoKZ7Tez981sq5l57nZQZvZrM6s1s+391mWa2Ytmtjv4mBHOGkdriH26x8wOBo/T1uBtHD3BzKaa2ctmVm5mZWb2jeB6Tx6nYfbHy8co0czeNbNtwX36QXD9qI6Rp9rUQ7lfqheZ2X6g1DnnyYsmzOwK4DjwmHNufnDdT4BG59y/Br98M5xz3w5nnaMxxD7dAxx3zv00nLWdDjObDEx2zm02s1RgE/Ap4DY8eJyG2Z+/xrvHyIAU59xxM4sD3gC+AVzPKI6R187UFwOVzrm9zrku4ElgVZhrinrOudeAxgGrVwG/Cf79GwL/4TxjiH3yLOfcYefc5uDfrUA5gVtOevI4DbM/nuUCjgcX44L/HKM8Rl4L9aHuhep1DnjBzDYF7+MaCfL6bpQSfMwNcz1j5S4zey/YPOOJpoqBzKwY+AjwDhFwnAbsD3j4GJmZz8y2ArXAi865UR8jr4V6SPdC9aDLnHMXANcAdwZ/+sv48wtgOrCIwE3VfxbeckbPzCYATwPfdM4dC3c9Z2qQ/fH0MXLO9TrnFhG4JehiM5s/2vfwWqhH5L1QnXOHgo+1wO8INDN53dFgu2df+2dtmOs5Y865o8H/dH7gP/HYcQq20z4N/NY590xwtWeP02D74/Vj1Mc51wy8AqxklMfIa6Eeyv1SPcXMUoIdPZhZCnA1sH34V3nCs8AXgn9/AQ5yULYAAADbSURBVPhDGGsZE33/sYI+jYeOU7AT7ldAuXPu/n5PefI4DbU/Hj9GOWaWHvw7CVgO7GSUx8hTo18AgkOUVvPB/VJ/HOaSzoiZTSNwdg6B2ws+7rV9MrMngKUEpgk9CvwT8HvgKaAQqAJudM55puNxiH1aSuBnvQP2A1/1yg3WzeyjwOvA+4A/uPp7BNqhPXechtmfW/DuMVpAoCPUR+CE+ynn3A/NLItRHCPPhbqIiAzNa80vIiIyDIW6iEgEUaiLiEQQhbqISARRqIuIRBCFuohIBFGoi4hEkP8PU3X6OnmOxHgAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(tr_loss_hist)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}