Lab12/lab-12-1_JijoongHong.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Lab 12-1 many to one\n",
    "### word sentiment classification \n",
    "* many to one \n",
    "* variable input sequence length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.1.0\n"
     ]
    }
   ],
   "source": [
    "# setup\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import matplotlib.pyplot as plt\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras import layers\n",
    "from tensorflow.keras import Sequential, Model\n",
    "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
    "%matplotlib inline\n",
    "\n",
    "print(tf.__version__)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Prepairing dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['<pad>', ' ', 'a', 'b', 'd', 'e', 'g', 'o', 'r', 's', 'w']\n",
      "{0: '<pad>', 1: ' ', 2: 'a', 3: 'b', 4: 'd', 5: 'e', 6: 'g', 7: 'o', 8: 'r', 9: 's', 10: 'w'}\n",
      "{'<pad>': 0, ' ': 1, 'a': 2, 'b': 3, 'd': 4, 'e': 5, 'g': 6, 'o': 7, 'r': 8, 's': 9, 'w': 10}\n"
     ]
    }
   ],
   "source": [
    "# example data\n",
    "words = ['good', 'bad', 'worse', 'so good']\n",
    "y_data = [1,0,0,1]\n",
    "\n",
    "# creating a token dictionary\n",
    "char_set = ['<pad>'] + sorted(list(set(''.join(words))))\n",
    "idx2char = {idx : char for idx, char in enumerate(char_set)}\n",
    "char2idx = {char : idx for idx, char in enumerate(char_set)}\n",
    "\n",
    "print(char_set)\n",
    "print(idx2char)\n",
    "print(char2idx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[6, 7, 7, 4], [3, 2, 4], [10, 7, 8, 9, 5], [9, 7, 1, 6, 7, 7, 4]]\n",
      "[4, 3, 5, 7]\n"
     ]
    }
   ],
   "source": [
    "# converting sequence of tokens to sequence of indices\n",
    "x_data = list(map(lambda word : [char2idx.get(char) for char in word], words))\n",
    "x_data_len = list(map(lambda word : len(word), x_data))\n",
    "\n",
    "print(x_data)\n",
    "print(x_data_len)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 6  7  7  4  0  0  0  0  0  0]\n",
      " [ 3  2  4  0  0  0  0  0  0  0]\n",
      " [10  7  8  9  5  0  0  0  0  0]\n",
      " [ 9  7  1  6  7  7  4  0  0  0]]\n",
      "[4, 3, 5, 7]\n",
      "[1, 0, 0, 1]\n"
     ]
    }
   ],
   "source": [
    "# padding the sequence of indices\n",
    "max_sequence = 10\n",
    "x_data = pad_sequences(sequences = x_data, maxlen = max_sequence,\n",
    "                       padding = 'post', truncating = 'post')\n",
    "\n",
    "# checking data\n",
    "print(x_data)\n",
    "print(x_data_len)\n",
    "print(y_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Creating model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# creating simple rnn for \"many to one\" classification\n",
    "input_dim = len(char2idx)\n",
    "output_dim = len(char2idx)\n",
    "one_hot = np.eye(len(char2idx))\n",
    "hidden_size = 10\n",
    "num_classes = 2\n",
    "\n",
    "model = Sequential()\n",
    "model.add(layers.Embedding(input_dim=input_dim, output_dim=output_dim,\n",
    "                           trainable=False, mask_zero=True, input_length=max_sequence,\n",
    "                           embeddings_initializer=keras.initializers.Constant(one_hot)))\n",
    "model.add(layers.SimpleRNN(units=hidden_size))\n",
    "model.add(layers.Dense(units=num_classes))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding (Embedding)        (None, 10, 11)            121       \n",
      "_________________________________________________________________\n",
      "simple_rnn (SimpleRNN)       (None, 10)                220       \n",
      "_________________________________________________________________\n",
      "dense (Dense)                (None, 2)                 22        \n",
      "=================================================================\n",
      "Total params: 363\n",
      "Trainable params: 242\n",
      "Non-trainable params: 121\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# creating loss function\n",
    "def loss_fn(model, x, y):\n",
    "    return tf.reduce_mean(tf.keras.losses.sparse_categorical_crossentropy(\n",
    "        y_true=y, y_pred=model(x), from_logits=True))\n",
    "\n",
    "# creating an optimizer\n",
    "lr = .01\n",
    "epochs = 30\n",
    "batch_size = 2\n",
    "opt = tf.keras.optimizers.Adam(learning_rate = lr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<BatchDataset shapes: ((None, 10), (None,)), types: (tf.int32, tf.int32)>\n"
     ]
    }
   ],
   "source": [
    "# generating data pipeline\n",
    "tr_dataset = tf.data.Dataset.from_tensor_slices((x_data, y_data))\n",
    "tr_dataset = tr_dataset.shuffle(buffer_size = 4)\n",
    "tr_dataset = tr_dataset.batch(batch_size = batch_size)\n",
    "\n",
    "print(tr_dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch :   5, tr_loss : 0.238\n",
      "epoch :  10, tr_loss : 0.055\n",
      "epoch :  15, tr_loss : 0.021\n",
      "epoch :  20, tr_loss : 0.011\n",
      "epoch :  25, tr_loss : 0.007\n",
      "epoch :  30, tr_loss : 0.006\n"
     ]
    }
   ],
   "source": [
    "# training\n",
    "tr_loss_hist = []\n",
    "\n",
    "for epoch in range(epochs):\n",
    "    avg_tr_loss = 0\n",
    "    tr_step = 0\n",
    "    \n",
    "    for x_mb, y_mb in tr_dataset:\n",
    "        with tf.GradientTape() as tape:\n",
    "            tr_loss = loss_fn(model, x=x_mb, y=y_mb)\n",
    "        grads = tape.gradient(target=tr_loss, sources=model.variables)\n",
    "        opt.apply_gradients(grads_and_vars=zip(grads, model.variables))\n",
    "        avg_tr_loss += tr_loss\n",
    "        tr_step += 1\n",
    "    else:\n",
    "        avg_tr_loss /= tr_step\n",
    "        tr_loss_hist.append(avg_tr_loss)\n",
    "    \n",
    "    if (epoch + 1) % 5 ==0:        \n",
    "        print('epoch : {:3}, tr_loss : {:.3f}'.format(epoch + 1, avg_tr_loss.numpy()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Checking performance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "acc : 100.00%\n"
     ]
    }
   ],
   "source": [
    "yhat = model.predict(x_data)\n",
    "yhat = np.argmax(yhat, axis=-1)\n",
    "print('acc : {:.2%}'.format(np.mean(yhat == y_data)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x19a98a83cf8>]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXUAAAD4CAYAAAATpHZ6AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAc0ElEQVR4nO3de3gcd33v8fd3d3WXZVnWyvHdsmMbm8S56XHCJYkhkJMEE9OeAjGHlvbwEMohh1ByekppHwoBWm6FlpKGkwIPlx5IU+AQE5yEW+6QECexHSeOE9lxbPkq+SbL1m213/PHjpy1LFlre+XRzH5eT/bZ3Znfjr7jiT8a//Y3vzF3R0RE4iERdgEiIlI8CnURkRhRqIuIxIhCXUQkRhTqIiIxkgrrBzc2NvqcOXPC+vEiIpH01FNPdbh7eqT1oYX6nDlzWLNmTVg/XkQkkszslZOtV/eLiEiMKNRFRGJEoS4iEiMKdRGRGFGoi4jEiEJdRCRGFOoiIjESuVBfs3U/X7jvBTRlsIjIiSIX6s/uOMTtD26mo6sv7FJERMadyIX63HQtAFvau0KuRERk/IleqDfWALC5/UjIlYiIjD+RC/Xp9VVUpBI6UxcRGUbkQj2RMJoba9jSoTN1EZGhIhfqAHPTNTpTFxEZRjRDvbGW7Qe66ctkwy5FRGRciWaop2sYyDrb9qsLRkQkX0RDPTesUSNgRESOF9FQzw1r3KJQFxE5TiRDva6yjPSECn1ZKiIyRCRDHXIXIWlYo4jI8aIb6ulanamLiAwR2VCfl67hwNF+9h/RxF4iIoNGDXUz+7aZ7TWzDSOsNzP7mpm1mtl6M7u4+GWe6NUvS3W2LiIyqJAz9e8A15xk/bXA/OBxI3D7mZc1urmNg7M1ql9dRGTQqKHu7g8D+0/SZAXwPc95HKg3s6nFKnAkMyZVUZY0NnfoTF1EZFAx+tSnA9vz3rcFy05gZjea2RozW9Pe3n5GPzSVTDB7co3O1EVE8hQj1G2YZcPea87d73D3FndvSafTZ/yD5zZqYi8RkXzFCPU2YGbe+xnAziJsd1Rz07Vs23+UzIAm9hIRgeKE+irgT4JRMJcBh9x9VxG2O6q56Rr6B5ztB7rPxo8TERn3UqM1MLMfAsuARjNrA/4OKANw928Aq4HrgFbgKPBnY1XsUPPy7lfaHNzmTkSklI0a6u6+cpT1Dny4aBWdgnl5E3tdtSiMCkRExpfIXlEKUF9dTkNNOVs0rFFEBIh4qENuBMzmvRrWKCICcQj1dI3O1EVEAjEI9Vo6uvo41N0fdikiIqGLfqg3amIvEZFB0Q/1tCb2EhEZFPlQn9VQTTJh6lcXESEGoV6eSjCroVpn6iIixCDUIXcRkkJdRCQmoT43XcvL+44wkB12ckgRkZIRj1BvrKEvk2XnQU3sJSKlLR6hHoyAadWwRhEpcTEJ9Vcn9hIRKWWxCPXJNeXUVaZ0AZKIlLxYhLqZMTddqzN1ESl5sQh10MReIiIQo1Cfl65lT2cvXb2ZsEsREQlNbEJ9cGKvl9UFIyIlLDahPq8pmNhLXTAiUsJiE+qzJ1eTMNisM3URKWGxCfWKVJIZk6o1rFFESlpsQh2CETA6UxeREhavUG+sZUtHF1lN7CUiJSpeoZ6uoac/y67OnrBLEREJRexCHXS/UhEpXbEK9Xm6X6mIlLhYhXrThApqypM6UxeRkhWrUD82sVeHztRFpDQVFOpmdo2ZbTKzVjP7+DDrZ5nZA2b2jJmtN7Pril9qYTSsUURK2aihbmZJ4DbgWmAxsNLMFg9p9rfAXe5+EXAD8K/FLrRQ89K17DjYTXffQFgliIiEppAz9aVAq7tvcfc+4E5gxZA2DtQFrycCO4tX4qkZHAHzsrpgRKQEFRLq04Htee/bgmX5PgW818zagNXA/xxuQ2Z2o5mtMbM17e3tp1Hu6OY2amIvESldhYS6DbNs6CWbK4HvuPsM4Drg+2Z2wrbd/Q53b3H3lnQ6ferVFqA5mIJ3816dqYtI6Skk1NuAmXnvZ3Bi98r7gbsA3P13QCXQWIwCT1VVeZLp9VU6UxeRklRIqD8JzDezZjMrJ/dF6KohbbYBVwGY2SJyoT42/SsF0AgYESlVo4a6u2eAm4D7gY3kRrk8Z2a3mtn1QbNbgA+Y2Trgh8Cfuntos2rNbaxhS3sXIZYgIhKKVCGN3H01uS9A85d9Mu/188Abilva6ZubruVI3wB7D/cypa4y7HJERM6aWF1ROmhwWONmTRcgIiUmpqGuib1EpDTFMtSn1lVSVZZUqItIyYllqCcSRnNjjYY1ikjJiWWog4Y1ikhpinGo19J24Cg9/ZrYS0RKR2xDfV66hqzDK/uOhl2KiMhZE9tQPzaxl4Y1ikgJiW2oN2usuoiUoNiGem1FirnpGn6/9UDYpYiInDWxDXWAKxekeWLLPn1ZKiIlI9ahvmxhE72ZLL/bsi/sUkREzopYh/qlzQ1UpBI8tCm0WYBFRM6qWId6ZVmS182bzMMvKtRFpDTEOtQh16++peMI2zReXURKQOxDfdnCJgAeenFvyJWIiIy92If6nMnVzGqo5kH1q4tICYh9qJsZyxam+e3mffRmNLRRROIt9qEOuX717v4BnnxZFyKJSLyVRKi/bt5kypMJ9auLSOyVRKhXl6dY2tygfnURib2SCHWAZQvTvLS3ix0Hu8MuRURkzJRMqF+5IA2gq0tFJNZKJtTPbapl2sRK9auLSKyVTKibGVcubOKx1n30ZbJhlyMiMiZKJtQh1wXT1Zvh6W0a2igi8VRSof6GcyeTSphGwYhIbBUU6mZ2jZltMrNWM/v4CG3eZWbPm9lzZvaD4pZZHBMqy7hk9iQe0qyNIhJTo4a6mSWB24BrgcXASjNbPKTNfOCvgTe4+2uBj45BrUWxbGETG3d1sqezJ+xSRESKrpAz9aVAq7tvcfc+4E5gxZA2HwBuc/cDAO4+boeYHBvaqLN1EYmhQkJ9OrA9731bsCzfAmCBmT1mZo+b2TXDbcjMbjSzNWa2pr09nFBdNHUCTRMqNF5dRGKpkFC3YZb5kPcpYD6wDFgJfNPM6k/4kPsd7t7i7i3pdPpUay0KM+PKBWkeeamdzICGNopIvBQS6m3AzLz3M4Cdw7S529373f1lYBO5kB+Xli1sorMnw9rtB8MuRUSkqAoJ9SeB+WbWbGblwA3AqiFtfgq8CcDMGsl1x2wpZqHF9MZzG0mY+tVFJH5GDXV3zwA3AfcDG4G73P05M7vVzK4Pmt0P7DOz54EHgL90931jVfSZmlhdxsWzJmm8uojETqqQRu6+Glg9ZNkn81478LHgEQlXLkjzj798kY6uXhprK8IuR0SkKErqitJ8gzekfuQlna2LSHyUbKi/dlodjbXl6oIRkVgp2VBPJIwr5qd5+MV2BrJDR2iKiERTyYY6wJUL0xw42s+zOw6FXYqISFGUdKhfPj+Nme6GJCLxUdKh3lBTzpIZ9TyouyGJSEyUdKgDLFuQZt32gxw40hd2KSIiZ6zkQ/3KhWmyDo+0doRdiojIGSv5UL9gRj311WXqVxeRWCj5UE8mjMvnp3noxXayGtooIhFX8qEO8JZFTXR09eqG1CISeQp14KpFU6hIJfjZuqEzCouIRItCHaitSPGmhU2s3rBbV5eKSKQp1APLL5hK++Fennh53M4YLCIyKoV64M2vaaKqLMk963eFXYqIyGlTqAeqy1NctaiJ+zbs1r1LRSSyFOp5li+Zxv4jffx2s7pgRCSaFOp5li1MU1uR4p71GgUjItGkUM9TWZbkrYuncN+G3fRl1AUjItGjUB9i+ZKpdPZkeLRV0waISPQo1Ie4fH6ausoU96zTKBgRiR6F+hDlqQTXnHcOv3h+Dz39A2GXIyJyShTqw1i+ZBpdvRkeelFdMCISLQr1Ybx+3mQaasp1IZKIRI5CfRipZK4L5tcb99Ddpy4YEYkOhfoIli+ZytG+AX7zgu5fKiLRoVAfwaXNk0lPqNB0vCISKQWFupldY2abzKzVzD5+knZ/ZGZuZi3FKzEcyYRx3Xnn8MCmvXT1ZsIuR0SkIKOGupklgduAa4HFwEozWzxMuwnAR4Anil1kWJZfMI3eTJZfPb8n7FJERApSyJn6UqDV3be4ex9wJ7BimHafAb4I9BSxvlBdMmsS59RVai4YEYmMQkJ9OrA9731bsOwYM7sImOnu95xsQ2Z2o5mtMbM17e3jfwx4ImG8bclUHnqxnUPd/WGXIyIyqkJC3YZZduyeb2aWAL4K3DLahtz9DndvcfeWdDpdeJUhWr5kKv0Dzi+e2x12KSIioyok1NuAmXnvZwD5/RETgPOAB81sK3AZsCoOX5YCXDiznhmTqnQhkohEQiGh/iQw38yazawcuAFYNbjS3Q+5e6O7z3H3OcDjwPXuvmZMKj7LzHJdMI+1dnDgSF/Y5YiInNSooe7uGeAm4H5gI3CXuz9nZrea2fVjXeB48PYl08hknfvUBSMi41yqkEbuvhpYPWTZJ0dou+zMyxpfXjutjubGGu5Zv5OVS2eFXY6IyIh0RWkBzIzlS6byu837aD/cG3Y5IiIjUqgXaPmSaWQd7tugL0xFZPxSqBdo4TkTmN9Uy880CkZExjGF+ilYvmQaT27dz57O2Fw0KyIxo1A/BcsvmIo73L12R9iliIgMS6F+Cuala3nd3Ml846EtdPZo2gARGX8U6qfoE9ctYv+RPv71gc1hlyIicgKF+ik6f8ZE/vCi6Xz7sZfZvv9o2OWIiBxHoX4a/td/WYgBX/7FprBLERE5jkL9NEyrr+IDl8/l7rU7Wbv9YNjliIgco1A/TX++bB6NteV87ufP4+6jf0BE5CxQqJ+m2ooUf/HWBTy59QD3a6IvERknFOpn4N0tM5nfVMvn732Bvkw27HJERBTqZyKVTPCJty1i676jfP/xV8IuR0REoX6mli1Ic/n8Rr7265c4eFQ30RCRcCnUz5CZ8YnrFtHZ08+//KY17HJEpMQp1Itg0dQ63nXJTL73u628su9I2OWISAlTqBfJLVcvoCyZ4Av3vRB2KSJSwhTqRdJUV8kHr5jH6md3s2br/rDLEZESpVAvog9c0cyUugo++/ONuiBJREKhUC+i6vIUt1y9kLXbD+oOSSISCoV6kf3Xi2ewaGodX7j3BXr6B8IuR0RKjEK9yJIJ42/ftogdB7v5zm+3hl2OiJQYhfoYeMO5jbz5NU3c9ptWtu3TnOsicvYo1MfIp69/LWbwP37wlLphROSsUaiPkZkN1XzlXReyYUcnn/7Z82GXIyIlQqE+ht6yeAofWjaPH/5+Gz9+qi3sckSkBBQU6mZ2jZltMrNWM/v4MOs/ZmbPm9l6M/u1mc0ufqnRdMtbF3DZ3Ab+5qfP8sLuzrDLEZGYGzXUzSwJ3AZcCywGVprZ4iHNngFa3H0J8CPgi8UuNKpSyQRfW3kRdZVlfOjfn+ZwT3/YJYlIjBVypr4UaHX3Le7eB9wJrMhv4O4PuPvgMI/HgRnFLTPamiZU8vX3XMy2/Uf53z9ar6tNRWTMFBLq04Htee/bgmUjeT9w75kUFUdLmxv4q2sWcu+G3Xzr0ZfDLkdEYqqQULdhlg17qmlm7wVagC+NsP5GM1tjZmva29sLrzImPnD5XK5ePIXP3/uCJv0SkTFRSKi3ATPz3s8Adg5tZGZvAf4GuN7de4fbkLvf4e4t7t6STqdPp95IMzO+9M4LmD6pig//4Gk6uob9YxIROW2FhPqTwHwzazazcuAGYFV+AzO7CPg/5AJ9b/HLjI+JVWXc/t8u4eDRfm6+8xkGsupfF5HiGTXU3T0D3ATcD2wE7nL358zsVjO7Pmj2JaAW+E8zW2tmq0bYnACLp9XxmXecx2Ot+/jqL18MuxwRiZFUIY3cfTWwesiyT+a9fkuR64q9d7XM5KmtB/j6A61cPLueN79mStgliUgM6IrSEH16xWtZPLWOv/iPdbq3qYgUhUI9RJVlSW5/78W4O2//l0dZte6E759FRE6JQj1ksyfXsOqmNzKvqZaP/PAZbr7zGQ5166pTETk9CvVxYE5jDf/5wdfxsbcu4J71u7j2nx7md5v3hV2WiESQQn2cSCUTfOSq+fz4Q6+noizJe775OH+/eiO9Gc3FLiKFU6iPMxfOrOfnH3kj71k6izse3sKKrz/Gpt2Hwy5LRCJCoT4OVZen+NwfnM+33tdCR1cvb//6o3zzkS1kdaGSiIxCoT6OXbVoCvd99AqumJ/msz/fyB9/+wl2HeoOuywRGccU6uNcY20F//Ynl/APf3g+z2w7yNVfeZg7Ht6svnYRGZZCPQLMjJVLZ7H6I5dzyZxJ/P3qF7j6qw9z34ZdmptdRI6jUI+QOY01fOfPlvLd/76U8mSCP//3p3n3HY/zbNuhsEsTkXFCoR5BVy5Ic+/Nl/PZd5xH694urr/tUW65ax17OnvCLk1EQqZQj6hUMsF7L5vNg3+5jBsvn8vP1u1k2Zce5J9/9RLdfepvFylVCvWIq6ss46+vW8SvPnYlyxam+eqvXuRNX36QnzzdpiGQIiXIwvqiraWlxdesWRPKz46z37+8n8/c8zzP7jjE9Poq3n7BNN5x0TRec05d2KWJSBGY2VPu3jLieoV6/GSzzuoNu/jRU2088lIHA1nnNedMYMWF07n+wmlMr68Ku0QROU0K9RLX0dXLz9fv4qdrd/DMtoMALG1u4B0XTue688+hvro85ApF5FQo1OWYV/Yd4e61O/np2h1saT9CWdJYtrCJ5Uum8rp5k2maUBl2iSIyCoW6nMDd2bCjk7vX7mDVup3sPdwLwNzGGi6d28DS5gYubZ7MNHXTiIw7CnU5qYGss77tIL9/eX/usXU/h3syAMyYVMWlzZO5dG4DlzY3MKuhGjMLuWKR0qZQl1MykHVe2N3JE1teDfn9R/oAOKeukgtn1nP+jImcPz33mFSjPnmRs0mhLmfE3Wnd28XjwZn8+raDvLLv6LH10+urcgGvoBc5K0YL9dTZLEaix8yYP2UC86dM4I8vmw3Aoe5+nttxiGeDx4Ydh7jvud3HPjO9vorF0+qY1VDNrIZqZjZUMXNSNTMbqqksS4a1KyIlQaEup2xiVRmvP7eR15/beGzZ0KDftPswj7zUTk9/9rjPpidU5IJ+UhWzGqqZ0VDN9Poqpk6sZOrEKqrKFfoiZ0KhLkUxXNC7Ox1dfWzbf5S2A0fZtu8o2w8cZfv+bp7ceoBV63YydCaDiVVlQcBXcs7EKqZNrOScIPDPmVhBY20FE6vK9IWtyAgU6jJmzIz0hArSEyq4ZPakE9b3D2TZebCbnQd72N3Zza5DPew62JN7PtTN+rZD7Au+pM1XljQm11QwubacxtqK4BG8npB7rq8qp64qxYTKMiZUpihLapojKQ0KdQlNWTLB7Mk1zJ5cM2Kbnv4B9nTmgn5PZw8dXX10dPXScbiXfUdyr1/ac5iOrj76BrIjbqe6PEldZRl1VSnqgqCvq8o911SkqC0PnitSVFckj72uKQ+eK5JUl6eoLEvoXwkyrinUZVyrLEuOGvyQ6+o53Juh43AvHV19HOrup7O7n8M9/XT2ZOjs7qezp5/O7gydPf20d/Wyuf0Ih3v6OdI7cNJfCENVlSWpKk8O+1xdnqSyLElFKpF7BK/LkwkqyhJUpIJ1ZQnKk0nKU4ncI5mgPGXHlpUlbci6BKlEbrl+qcjJFBTqZnYN8M9AEvimu39+yPoK4HvAJcA+4N3uvrW4pYqMzMxyZ+KVZcxNn/rn+zJZjvRmONKX4UjvAF29mdz73syx1939Wbr7B+juy9DdP8DRvgF6+gfo7su9Pni0j50HB+juH6Avk6U3k6Uvk6UnM0AxRw6nEkYqaZQlE8HDSCUGg99IJRPH2qQSRjKRa5tM5N6nEgmSeetyzwmSCXLrguWD6xIWPA8ut1fXJ4L3r66HhOU+k0wYCRvyPliWtNwvp2Ntgp+TW06wPLc9Cz4/uC3L22bCBte/ugzjhOXACe3j+stx1FA3syRwG/BWoA140sxWufvzec3eDxxw93PN7AbgC8C7x6JgkbGQOysuH5Mx9u5OJuv0ZrL09uf+VdDbnwv7/ozTNzBAX8bpG8jSn8nSN5D7ZTD43B88Z7JO/0A2eLz6OjOQ+2wmWJbJOpngeSDrZAacI5lMsDy3rD+bZSBYP5DN1ZcNnvOXD3juOa6C3wHHflmY2XHvE8F7G/LLIGEAg23AOP4XheX9Asp9/vjt3HzVfN5+wbQx2adCztSXAq3uviVXrN0JrADyQ30F8Kng9Y+Ar5uZue6KLIKZURacWddWRK/H0/3VgM9mORb0g4/sMO9zyzi2LreM49pn3fG8Zdm87Xve5wfbDW4jG6zP5i0brHGwHeSvy1uWzdtGbudybXi1rQfrBts6fvzyY/UAHF+bB+2D/4619eDPcXD9xKqyMTtehfwfNh3Ynve+Dbh0pDbunjGzQ8BkoCO/kZndCNwIMGvWrNMsWUTOJrOgKyfsQqQghYzzGq7jaegZeCFtcPc73L3F3VvS6dPo+BQRkZMqJNTbgJl572cAO0dqY2YpYCKwvxgFiohI4QoJ9SeB+WbWbGblwA3AqiFtVgHvC17/EfAb9aeLiJx9o3aTBX3kNwH3kxvS+G13f87MbgXWuPsq4FvA982sldwZ+g1jWbSIiAyvoO8+3H01sHrIsk/mve4B3lnc0kRE5FRpQgwRkRhRqIuIxIhCXUQkRkK7nZ2ZtQOvnObHGxlyYVMMxG2f4rY/EL99itv+QPz2abj9me3uI17oE1qonwkzW3Oye/RFUdz2KW77A/Hbp7jtD8Rvn05nf9T9IiISIwp1EZEYiWqo3xF2AWMgbvsUt/2B+O1T3PYH4rdPp7w/kexTFxGR4UX1TF1ERIahUBcRiZHIhbqZXWNmm8ys1cw+HnY9Z8rMtprZs2a21szWhF3P6TCzb5vZXjPbkLeswcx+aWYvBc+TwqzxVIywP58ysx3BcVprZteFWeOpMrOZZvaAmW00s+fM7OZgeSSP00n2J7LHycwqzez3ZrYu2KdPB8ubzeyJ4Bj9RzBb7sjbiVKfenC/1BfJu18qsHLI/VIjxcy2Ai3uHtkLJszsCqAL+J67nxcs+yKw390/H/zyneTufxVmnYUaYX8+BXS5+5fDrO10mdlUYKq7P21mE4CngHcAf0oEj9NJ9uddRPQ4We4GpzXu3mVmZcCjwM3Ax4CfuPudZvYNYJ273z7SdqJ2pn7sfqnu3gcM3i9VQuTuD3PiTVFWAN8NXn+X3F+4SBhhfyLN3Xe5+9PB68PARnK3oYzkcTrJ/kSW53QFb8uChwNvJnfvZyjgGEUt1Ie7X2qkDyS5g/YLM3squIdrXExx912Q+wsINIVcTzHcZGbrg+6ZSHRTDMfM5gAXAU8Qg+M0ZH8gwsfJzJJmthbYC/wS2AwcdPdM0GTUzItaqBd0L9SIeYO7XwxcC3w4+Ke/jD+3A/OAC4FdwD+GW87pMbNa4MfAR929M+x6ztQw+xPp4+TuA+5+Ibnbhi4FFg3X7GTbiFqoF3K/1Ehx953B817g/5E7kHGwJ+j3HOz/3BtyPWfE3fcEf+GywL8RweMU9NP+GPi/7v6TYHFkj9Nw+xOH4wTg7geBB4HLgPrg3s9QQOZFLdQLuV9qZJhZTfAlD2ZWA1wNbDj5pyIj/7617wPuDrGWMzYYfIE/IGLHKfgS7lvARnf/St6qSB6nkfYnysfJzNJmVh+8rgLeQu67ggfI3fsZCjhGkRr9AhAMUfonXr1f6udCLum0mdlccmfnkLu14A+iuD9m9kNgGblpQvcAfwf8FLgLmAVsA97p7pH48nGE/VlG7p/0DmwFPjjYFx0FZvZG4BHgWSAbLP4EuX7oyB2nk+zPSiJ6nMxsCbkvQpPkTrjvcvdbg5y4E2gAngHe6+69I24naqEuIiIji1r3i4iInIRCXUQkRhTqIiIxolAXEYkRhbqISIwo1EVEYkShLiISI/8fN62ZA4yA888AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(tr_loss_hist)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}