Lab12/lab-12-3_JijoongHong.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Lab 12-3 many to many\n",
    "### simple pos-tagger training \n",
    "* many to many\n",
    "* variable input sequence length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.1.0\n"
     ]
    }
   ],
   "source": [
    "# setup\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import matplotlib.pyplot as plt\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras import layers\n",
    "from tensorflow.keras import Sequential, Model\n",
    "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
    "from pprint import pprint\n",
    "%matplotlib inline\n",
    "\n",
    "print(tf.__version__)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Prepairing dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# example data\n",
    "sentences = [['I', 'feel', 'hungry'],\n",
    "     ['tensorflow', 'is', 'very', 'difficult'],\n",
    "     ['tensorflow', 'is', 'a', 'framework', 'for', 'deep', 'learning'],\n",
    "     ['tensorflow', 'is', 'very', 'fast', 'changing']]\n",
    "pos = [['pronoun', 'verb', 'adjective'],\n",
    "     ['noun', 'verb', 'adverb', 'adjective'],\n",
    "     ['noun', 'verb', 'determiner', 'noun', 'preposition', 'adjective', 'noun'],\n",
    "     ['noun', 'verb', 'adverb', 'adjective', 'verb']]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Preprocessing dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'<pad>': 0, 'I': 1, 'a': 2, 'changing': 3, 'deep': 4, 'difficult': 5, 'fast': 6, 'feel': 7, 'for': 8, 'framework': 9, 'hungry': 10, 'is': 11, 'learning': 12, 'tensorflow': 13, 'very': 14}\n",
      "{0: '<pad>', 1: 'I', 2: 'a', 3: 'changing', 4: 'deep', 5: 'difficult', 6: 'fast', 7: 'feel', 8: 'for', 9: 'framework', 10: 'hungry', 11: 'is', 12: 'learning', 13: 'tensorflow', 14: 'very'}\n",
      "15\n"
     ]
    }
   ],
   "source": [
    "# creating a token dictionary for word\n",
    "word_list = sum(sentences, [])\n",
    "word_list = sorted(set(word_list))\n",
    "word_list = ['<pad>'] + word_list\n",
    "word2idx = {word : idx for idx, word in enumerate(word_list)}\n",
    "idx2word = {idx : word for idx, word in enumerate(word_list)}\n",
    "\n",
    "print(word2idx)\n",
    "print(idx2word)\n",
    "print(len(idx2word))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'<pad>': 0, 'adjective': 1, 'adverb': 2, 'determiner': 3, 'noun': 4, 'preposition': 5, 'pronoun': 6, 'verb': 7}\n",
      "{0: '<pad>', 1: 'adjective', 2: 'adverb', 3: 'determiner', 4: 'noun', 5: 'preposition', 6: 'pronoun', 7: 'verb'}\n",
      "8\n"
     ]
    }
   ],
   "source": [
    "# creating a token dictionary for part of speech\n",
    "pos_list = sum(pos, [])\n",
    "pos_list = sorted(set(pos_list))\n",
    "pos_list = ['<pad>'] + pos_list\n",
    "pos2idx = {pos : idx for idx, pos in enumerate(pos_list)}\n",
    "idx2pos = {idx : pos for idx, pos in enumerate(pos_list)}\n",
    "\n",
    "print(pos2idx)\n",
    "print(idx2pos)\n",
    "print(len(pos2idx))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 1  7 10  0  0  0  0  0  0  0]\n",
      " [13 11 14  5  0  0  0  0  0  0]\n",
      " [13 11  2  9  8  4 12  0  0  0]\n",
      " [13 11 14  6  3  0  0  0  0  0]] [3, 4, 7, 5]\n",
      "[[1. 1. 1. 0. 0. 0. 0. 0. 0. 0.]\n",
      " [1. 1. 1. 1. 0. 0. 0. 0. 0. 0.]\n",
      " [1. 1. 1. 1. 1. 1. 1. 0. 0. 0.]\n",
      " [1. 1. 1. 1. 1. 0. 0. 0. 0. 0.]]\n",
      "[[6 7 1 0 0 0 0 0 0 0]\n",
      " [4 7 2 1 0 0 0 0 0 0]\n",
      " [4 7 3 4 5 1 4 0 0 0]\n",
      " [4 7 2 1 7 0 0 0 0 0]]\n"
     ]
    }
   ],
   "source": [
    "# converting sequence of tokens to sequence of indices\n",
    "max_sequence = 10\n",
    "x_data = list(map(lambda sentence : [word2idx.get(token) for token in sentence], sentences))\n",
    "y_data = list(map(lambda sentence : [pos2idx.get(token) for token in sentence], pos))\n",
    "\n",
    "# padding the sequence of indices\n",
    "x_data = pad_sequences(sequences = x_data, maxlen = max_sequence, padding='post')\n",
    "x_data_mask = ((x_data != 0) * 1).astype(np.float32)\n",
    "x_data_len = list(map(lambda sentence : len(sentence), sentences))\n",
    "\n",
    "y_data = pad_sequences(sequences = y_data, maxlen = max_sequence, padding='post')\n",
    "\n",
    "# checking data\n",
    "print(x_data, x_data_len)\n",
    "print(x_data_mask)\n",
    "print(y_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Creating model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# creating rnn for \"many to many\" sequence tagging\n",
    "num_classes = len(pos2idx)\n",
    "hidden_dim = 10\n",
    "\n",
    "input_dim = len(word2idx)\n",
    "output_dim = len(word2idx)\n",
    "one_hot = np.eye(len(word2idx))\n",
    "\n",
    "model = Sequential()\n",
    "model.add(layers.Embedding(input_dim=input_dim, output_dim=output_dim, mask_zero=True,\n",
    "                           trainable=False, input_length=max_sequence,\n",
    "                           embeddings_initializer=keras.initializers.Constant(one_hot)))\n",
    "model.add(layers.SimpleRNN(units=hidden_dim, return_sequences=True))\n",
    "model.add(layers.TimeDistributed(layers.Dense(units=num_classes)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding (Embedding)        (None, 10, 15)            225       \n",
      "_________________________________________________________________\n",
      "simple_rnn (SimpleRNN)       (None, 10, 10)            260       \n",
      "_________________________________________________________________\n",
      "time_distributed (TimeDistri (None, 10, 8)             88        \n",
      "=================================================================\n",
      "Total params: 573\n",
      "Trainable params: 348\n",
      "Non-trainable params: 225\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# creating loss function\n",
    "def loss_fn(model, x, y, x_len, max_sequence):\n",
    "    masking = tf.sequence_mask(x_len, maxlen=max_sequence, dtype=tf.float32)\n",
    "    valid_time_step = tf.cast(x_len,dtype=tf.float32)    \n",
    "    sequence_loss = tf.keras.losses.sparse_categorical_crossentropy(\n",
    "        y_true=y, y_pred=model(x), from_logits=True) * masking    \n",
    "    sequence_loss = tf.reduce_sum(sequence_loss, axis=-1) / valid_time_step    \n",
    "    sequence_loss = tf.reduce_mean(sequence_loss)    \n",
    "    return sequence_loss\n",
    "\n",
    "# creating and optimizer\n",
    "lr = 0.1\n",
    "epochs = 30\n",
    "batch_size = 2 \n",
    "opt = tf.keras.optimizers.Adam(learning_rate = lr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<BatchDataset shapes: ((None, 10), (None, 10), (None,)), types: (tf.int32, tf.int32, tf.int32)>\n"
     ]
    }
   ],
   "source": [
    "# generating data pipeline\n",
    "tr_dataset = tf.data.Dataset.from_tensor_slices((x_data, y_data, x_data_len))\n",
    "tr_dataset = tr_dataset.shuffle(buffer_size=4)\n",
    "tr_dataset = tr_dataset.batch(batch_size = 2)\n",
    "\n",
    "print(tr_dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch :   5, tr_loss : 0.136\n",
      "epoch :  10, tr_loss : 0.008\n",
      "epoch :  15, tr_loss : 0.003\n",
      "epoch :  20, tr_loss : 0.002\n",
      "epoch :  25, tr_loss : 0.001\n",
      "epoch :  30, tr_loss : 0.001\n"
     ]
    }
   ],
   "source": [
    "# training\n",
    "tr_loss_hist = []\n",
    "\n",
    "for epoch in range(epochs):\n",
    "    avg_tr_loss = 0\n",
    "    tr_step = 0\n",
    "    \n",
    "    for x_mb, y_mb, x_mb_len in tr_dataset:\n",
    "        with tf.GradientTape() as tape:\n",
    "            tr_loss = loss_fn(model, x=x_mb, y=y_mb, x_len=x_mb_len, max_sequence=max_sequence)\n",
    "        grads = tape.gradient(target=tr_loss, sources=model.variables)\n",
    "        opt.apply_gradients(grads_and_vars=zip(grads, model.variables))\n",
    "        avg_tr_loss += tr_loss\n",
    "        tr_step += 1\n",
    "    else:\n",
    "        avg_tr_loss /= tr_step\n",
    "        tr_loss_hist.append(avg_tr_loss)\n",
    "    \n",
    "    if (epoch + 1) % 5 == 0:\n",
    "        print('epoch : {:3}, tr_loss : {:.3f}'.format(epoch + 1, avg_tr_loss))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Checking performance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[['pronoun', 'verb', 'adjective', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>'],\n",
      " ['noun', 'verb', 'adverb', 'adjective', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>'],\n",
      " ['noun', 'verb', 'determiner', 'noun', 'preposition', 'adjective', 'noun', '<pad>', '<pad>', '<pad>'],\n",
      " ['noun', 'verb', 'adverb', 'adjective', 'verb', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>']]\n",
      "[['pronoun', 'verb', 'adjective'],\n",
      " ['noun', 'verb', 'adverb', 'adjective'],\n",
      " ['noun', 'verb', 'determiner', 'noun', 'preposition', 'adjective', 'noun'],\n",
      " ['noun', 'verb', 'adverb', 'adjective', 'verb']]\n"
     ]
    }
   ],
   "source": [
    "yhat = model.predict(x_data)\n",
    "yhat = np.argmax(yhat, axis=-1) * x_data_mask\n",
    "\n",
    "pprint(list(map(lambda row : [idx2pos.get(elm) for elm in row],yhat.astype(np.int32).tolist())), width = 120)\n",
    "pprint(pos)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x1f78e82add8>]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXwAAAD4CAYAAADvsV2wAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAceklEQVR4nO3de5hcdZ3n8fen+kqS7nQn6VxMuhMEVohcOtBGHBwHWWWCrqK77khcHdzVycwoOrvzPKvM5RHF0XHH1XFWUIyaB/RZYBgUjSsMZLyhIkpHIgTCJcZgYkLSIbdOyK27v/tHne6udPpS3V2d6j7n83qeerrqd36n6nss+dTJ75zzO4oIzMws/XLlLsDMzE4PB76ZWUY48M3MMsKBb2aWEQ58M7OMqCx3AYOZM2dOLFmypNxlmJlNGevXr98TEU3D9ZmUgb9kyRLa29vLXYaZ2ZQh6bmR+nhIx8wsIxz4ZmYZ4cA3M8sIB76ZWUY48M3MMsKBb2aWEQ58M7OMSE3g9/QEn//eszz4TEe5SzEzm5RSE/i5nFj94Ba+/9TucpdiZjYppSbwAebW17C782i5yzAzm5TSFfh1tew+eKzcZZiZTUrpCvz6GnZ3OvDNzAaTrsCvq2HXwaP4Pr1mZqdKWeDXcqyrh4NHu8pdipnZpJOuwK+vAaDDB27NzE6RrsCvqwXwgVszs0GkK/CTPXwfuDUzO1W6Ar+uN/A9pGNmNlCqAn9GTSVnVFWwy0M6ZmanSFXgS/K5+GZmQ0hV4EN+WGf3QQ/pmJkNNGLgS1ojabekjUMs/5+SNiSPjZK6Jc1Klm2V9HiyrL3UxQ9mbn0tHd7DNzM7RTF7+LcCK4ZaGBGfjojWiGgF/gr4UUTsLejy2mR52/hKLc7cOg/pmJkNZsTAj4gHgb0j9UusBO4YV0XjNLeulkPHujh8zFfbmpkVKtkYvqRp5P8l8I2C5gAekLRe0qoR1l8lqV1Se0fH2G9i0n9qpvfyzcwKlfKg7ZuAnw4YzrksIi4GrgLeL+k1Q60cEasjoi0i2pqamsZcRN/FVz5wa2Z2klIG/jUMGM6JiB3J393APcDyEn7eoPqmV/AevpnZSUoS+JJmAn8AfLugbbqkut7nwJXAoGf6lNI8T69gZjaoypE6SLoDuByYI2k7cANQBRARtyTd3go8EBGHC1adB9wjqfdzbo+Ify1d6YObeUYV1ZU5T69gZjbAiIEfESuL6HMr+dM3C9u2ABeNtbCxkkTTjBo6PL2CmdlJUnelLeQP3O7yHr6Z2UnSGfh1NZ4T38xsgJQGfq0P2pqZDZDKwJ9XX8OBIyc4eqK73KWYmU0aqQz83nPxPYmamVm/VAZ+k8/FNzM7RSoDv28+HU+vYGbWJ6WB7+kVzMwGSmXgz55eTUVOvtrWzKxAKgM/l8tfbetz8c3M+qUy8AHfzNzMbID0Br5vdWhmdpLUBn5TXa3P0jEzK5DawJ9bV8MLh49zorun3KWYmU0K6Q385OKrPYc8rGNmBikO/Hm95+L7TB0zMyDFgT/X0yuYmZ1kxMCXtEbSbkmD3o9W0uWSDkjakDw+UrBshaSnJW2WdH0pCx9J/9W2PnBrZgbF7eHfCqwYoc+PI6I1edwIIKkCuBm4ClgKrJS0dDzFjsacGdVIsMtDOmZmQBGBHxEPAnvH8N7Lgc0RsSUijgN3AleP4X3GpLIix+zp1XR4D9/MDCjdGP6rJP1K0n2SXp60LQS2FfTZnrQNStIqSe2S2js6OkpSVP5cfO/hm5lBaQL/l8DiiLgI+DzwraRdg/SNod4kIlZHRFtEtDU1NZWgrPydr3zQ1swsb9yBHxEHI+JQ8vxeoErSHPJ79M0FXRcBO8b7eaORn17BQzpmZlCCwJc0X5KS58uT93wBeAQ4R9KZkqqBa4C14/280ZhbV8ueQ8fp7hnyHxZmZplROVIHSXcAlwNzJG0HbgCqACLiFuBtwJ9L6gKOANdERABdkq4D7gcqgDUR8cSEbMUQ5tbX0N0TvHD4WN9pmmZmWTVi4EfEyhGW3wTcNMSye4F7x1ba+PXf6tCBb2aW2ittIX+WDkCHD9yamaU78Of1Ta/gA7dmZqkO/KaCIR0zs6xLdeDXVFbQMK3K5+KbmZHywIf8gdtdvvOVmVkWAr/We/hmZmQi8Gt8lo6ZGRkI/Kb6fODnrwUzM8uu1Af+vLpajnf3sP/FE+UuxcysrFIf+L7VoZlZXvoD37c6NDMDMhH4+T183+rQzLIu/YHv6RXMzIAMBP606kpm1FR6egUzy7zUBz7k9/J9Lr6ZZV02At+3OjQzy0rge3oFM7MRA1/SGkm7JW0cYvl/kfRY8nhI0kUFy7ZKelzSBkntpSx8NHonUPPVtmaWZcXs4d8KrBhm+W+AP4iIC4GPA6sHLH9tRLRGRNvYShy/ufU1HD3RQ+exrnKVYGZWdiMGfkQ8COwdZvlDEbEvefkwsKhEtZVM38VXPlPHzDKs1GP47wHuK3gdwAOS1ktaNdyKklZJapfU3tHRUdKifC6+mRlUluqNJL2WfOC/uqD5sojYIWkusE7SU8m/GE4REatJhoPa2tpKOtg+1zczNzMrzR6+pAuBrwBXR8QLve0RsSP5uxu4B1heis8brb49fA/pmFmGjTvwJbUA3wTeFRHPFLRPl1TX+xy4Ehj0TJ+JVldTSW1Vzrc6NLNMG3FIR9IdwOXAHEnbgRuAKoCIuAX4CDAb+IIkgK7kjJx5wD1JWyVwe0T86wRsw4gk+Vx8M8u8EQM/IlaOsPy9wHsHad8CXHTqGuXhq23NLOsycaUtwLx67+GbWbZlJvCb6mro8EFbM8uwzAT+3PoaOo918eJxX21rZtmUncD31bZmlnEZCnzfzNzMsi07ge/pFcws4zIT+PM8pGNmGZeZwG+YVkV1Rc5DOmaWWZkJfEk0+eIrM8uwzAQ+5M/F95COmWVVpgLf0yuYWZZlK/DrazyGb2aZlanAn1dXy/4XT3Csq7vcpZiZnXaZCvzec/F95yszy6JsBX7vufgOfDPLoEwFflPv9Aq+85WZZVCmAr9/egXv4ZtZ9hQV+JLWSNotadB70irv/0jaLOkxSRcXLLtW0rPJ49pSFT4Ws6fXkJOnVzCzbCp2D/9WYMUwy68Czkkeq4AvAkiaRf4euK8ElgM3SGoca7HjVZHz1bZmll1FBX5EPAjsHabL1cDXIu9hoEHSAuAPgXURsTci9gHrGP6HY8L5ZuZmllWlGsNfCGwreL09aRuq/RSSVklql9Te0dFRorJONdfTK5hZRpUq8DVIWwzTfmpjxOqIaIuItqamphKVdar81bYe0jGz7ClV4G8HmgteLwJ2DNNeNk11tbxw+Dhd3T3lLMPM7LQrVeCvBf44OVvnUuBAROwE7geulNSYHKy9Mmkrm7l1NUTAnkPHy1mGmdlpV1lMJ0l3AJcDcyRtJ3/mTRVARNwC3Au8AdgMvAj812TZXkkfBx5J3urGiBju4O+Em1ffe7XtUebPrC1nKWZmp1VRgR8RK0dYHsD7h1i2Blgz+tImRt/NzH3g1swyJlNX2oKvtjWz7Mpc4M+ZUYMEuzyfjpllTOYCv6oix6xp1d7DN7PMyVzgQ37WzA6fi29mGZPJwJ9X7+kVzCx7Mhn4nl7BzLIom4FfX0PHoWN09ww6y4OZWSplMvAXz55Od0/wmz2Hy12Kmdlpk8nAb21uAGDDtv1lrsTM7PTJZOCf1TSDGTWVbNi2r9ylmJmdNpkM/IqcuKh5pvfwzSxTMhn4kB/W2bSzkyPHu8tdipnZaZHhwG+kuyfYuONAuUsxMzstMhz4yYHb33pYx8yyIbOB31RXw8KGMzyOb2aZkdnAB2htaXDgm1lmZDrwlzU38Lv9R3xTczPLhKICX9IKSU9L2izp+kGW/6OkDcnjGUn7C5Z1FyxbW8rix8vj+GaWJSPe4lBSBXAz8HpgO/CIpLUR8WRvn4j4HwX9PwAsK3iLIxHRWrqSS+f8hTOpzIkN2/Zz5cvnl7scM7MJVcwe/nJgc0RsiYjjwJ3A1cP0XwncUYriJlptVQXnLajnUe/hm1kGFBP4C4FtBa+3J22nkLQYOBP4fkFzraR2SQ9LestQHyJpVdKvvaOjo4iySqO1uYHHtu/3zJlmlnrFBL4GaRsqHa8B7o6IwstXWyKiDXgH8DlJZw22YkSsjoi2iGhramoqoqzSaG1u4PDxbjbvPnTaPtPMrByKCfztQHPB60XAjiH6XsOA4ZyI2JH83QL8kJPH98uutaV35kxPpGZm6VZM4D8CnCPpTEnV5EP9lLNtJL0MaAR+VtDWKKkmeT4HuAx4cuC65XTm7OnU11b6fHwzS70Rz9KJiC5J1wH3AxXAmoh4QtKNQHtE9Ib/SuDOiCgc7jkP+JKkHvI/Lp8qPLtnMsjlxEXNDT5wa2apN2LgA0TEvcC9A9o+MuD1RwdZ7yHggnHUd1osa2nkpu8/y+FjXUyvKep/EjOzKSfTV9r2WtbcQE/A47/zzJlmll4OfOCi5IpbD+uYWZo58IFZ06tZPHuaz9Qxs1Rz4Cdamz1zppmlmwM/0drcwK6Dx9h54Ei5SzEzmxAO/IRnzjSztHPgJ5a+pJ7qipyHdcwstRz4iZrKCpa+pJ5HHfhmllIO/AKtzQ08vv0AXd095S7FzKzkHPgFlrU0cOREN0/v6ix3KWZmJefAL9B34NbDOmaWQg78Ai2zpjFrerXP1DGzVHLgF5DERYtmeg/fzFLJgT/AspZGNnccovPoiXKXYmZWUg78AVqbG4iAx7Z75kwzSxcH/gAX+cCtmaWUA3+AmWdU8dKm6Z4q2cxSp6jAl7RC0tOSNku6fpDl75bUIWlD8nhvwbJrJT2bPK4tZfETJT9z5j5OvlujmdnUNmLgS6oAbgauApYCKyUtHaTrP0dEa/L4SrLuLOAG4JXAcuAGSY0lq36CLGtuYM+h42zf55kzzSw9itnDXw5sjogtEXEcuBO4usj3/0NgXUTsjYh9wDpgxdhKPX1am/O/SR7HN7M0KSbwFwLbCl5vT9oG+k+SHpN0t6TmUa6LpFWS2iW1d3R0FFHWxDl3QR01lZ4508zSpZjA1yBtAwe3vwMsiYgLgX8DbhvFuvnGiNUR0RYRbU1NTUWUNXGqKnJcsNAXYJlZuhQT+NuB5oLXi4AdhR0i4oWIOJa8/DJwSbHrTlatzQ1s/N0BTnjmTDNLiWIC/xHgHElnSqoGrgHWFnaQtKDg5ZuBTcnz+4ErJTUmB2uvTNomvdaWBo519fDUTs+caWbpUDlSh4joknQd+aCuANZExBOSbgTaI2It8EFJbwa6gL3Au5N190r6OPkfDYAbI2LvBGxHyfXOnPnotn1csGhmmasxMxs/TcZzzdva2qK9vb2sNUQEr/jE93jNOXP47Ntby1qLmdlIJK2PiLbh+vhK2yFISi7A8oFbM0sHB/4wlrU0sGXPYQ686JkzzWzqc+APY1nvRGrbvZdvZlOfA38YFzY3IMFDv95T7lLMzMbNgT+MGTWVvOH8BXz9Z8+x++DRcpdjZjYuDvwRfGjFyzjR3cNn1z1T7lLMzMbFgT+CxbOn88evWsJd7dt46vmD5S7HzGzMHPhF+MAVZ1NXW8Un732q3KWYmY2ZA78IDdOq+cAVZ/PgMx386JnyzuRpZjZWDvwivetVi2mZNY2/v3cT3T2T7+pkM7OROPCLVFNZwYdXnMtTz3dy9/ptI69gZjbJOPBH4Q0XzOfilgY+88AzHD7WVe5yzMxGxYE/CpL4mzcuZXfnMb784y3lLsfMbFQc+KN0yeJG3njBAr70oy3s8sVYZjaFOPDH4MMrzqWrp4fPPuCLscxs6nDgj0HL7Glc+6ol3LV+G5t2+mIsM5saHPhjdN0VZ1NfW8Un7900cmczs0mgqMCXtELS05I2S7p+kOV/KelJSY9J+p6kxQXLuiVtSB5rB647VTVMq+aD//4cfvzsHl+MZWZTwoiBL6kCuBm4ClgKrJS0dEC3R4G2iLgQuBv4h4JlRyKiNXm8uUR1TwrvunQxi2dP45Pf9cVYZjb5FbOHvxzYHBFbIuI4cCdwdWGHiPhBRLyYvHwYWFTaMien6soc1684l6d3dfIv7b4Yy8wmt2ICfyFQmGbbk7ahvAe4r+B1raR2SQ9LestQK0lalfRr7+iYOkMkK86fzyWLG/nMOl+MZWaTWzGBr0HaBh2/kPROoA34dEFzS3In9XcAn5N01mDrRsTqiGiLiLampqYiypoc8hdjnUdH5zG+9KAvxjKzyauYwN8ONBe8XgTsGNhJ0uuAvwHeHBHHetsjYkfydwvwQ2DZOOqdlC5uaeQ/XLiAW370a9Y9uavc5ZiZDaqYwH8EOEfSmZKqgWuAk862kbQM+BL5sN9d0N4oqSZ5Pge4DHiyVMVPJh9788s5b34df/r1dm57aGu5yzEzO8WIgR8RXcB1wP3AJuCuiHhC0o2Ses+6+TQwA/iXAadfnge0S/oV8APgUxGRysCfPaOGO1ZdyhXnzuOGtU/wie8+SY/P3DGzSUQRky+U2traor29vdxljEl3T/Dx//cktz60lTdcMJ/P/lErtVUV5S7LzFJO0vrkeOmQfKVtiVXkxA1vWsrfvvE87tv4PO/48sPsPXy83GWZmTnwJ4Ik3vv7L+UL77iYJ3Yc5D9+4ads3XO43GWZWcY58CfQVRcs4PY/uZSDR7t46xd+yvrn9pW7JDPLMAf+BLtkcSPf/PPfY+YZVbzjyw9z3+M7y12SmWWUA/80WDJnOt9832W8/CX1vO/2X/KVH29hMh4sN7N0c+CfJrOmV3P7n1zKVefP5+++u4m/vmcjx7q6y12WmWWIA/80qq2q4KaVF/O+y8/ijl/8lrd/6WF2HjhS7rLMLCMc+KdZLic+tOJcbnnnxTy7q5M3ff4n/OzXL5S7LDPLAAd+maw4fwHfvu4yZp5RxTu/+nOP65vZhHPgl9HZc+v41vsv43XnzeXvvruJD965gRePe4plM5sYDvwyq6ut4pZ3XsKHVryM7z62g7fe/JAv0jKzCeHAnwQk8b7Lz+a2/7acXZ1HedNNP+F7mzzNspmVlgN/Evn9c5r4znWvZvHsabzntnb+cd0znnHTzEqmstwF2MmaZ03j7j/7Pf72Wxv5p+89y8NbXmDF+fN5xZJZnDu/jsoK/0ab2dg48Ceh2qoKPv22C1nW0sDN39/Mx76Tv4XAtOoKlrU0cMniWbxiSSPLWhqZUeOv0MyK4/nwp4Ad+4/Q/tw+1m/dyyNb9/HU8wfpCcgJzp1fT9uSRi5Z3MhFixpomTWNXG6w2xCbWZoVMx++A38K6jx6gg3b9tO+dR/tz+3l0d/u58Xj+WkapldX8LL5dZy3oL7vce78Oqb7XwJmqVaywJe0AvgnoAL4SkR8asDyGuBrwCXAC8DbI2JrsuyvgPcA3cAHI+L+kT7PgT86Xd09PPV8J0/sOMCmnZ08ufMgm3YepPNo/px+CRbPmtb3A3BW0wwaplVRV1tJXW3v30pqKn1nLrOpqpjAH3G3T1IFcDPwemA78IiktQPuTfseYF9EnC3pGuB/AW+XtJT8Tc9fDrwE+DdJ/y4iPGtYCVVW5Dh/4UzOXzizry0i+N3+I2za2cmm5AfgyZ0HuW/j80O+T01ljrraKuprK6k7I/+3uiJHZYWozOWoyCl5LipyOSoHvK6qUL5PwfLedQrbK3KQk5BEhURO+VNTc8q3V+SEkuc5iVyu/3lFjoL18v0K+4v8cgkEfX1yyg9z5XK9ffr7M2DdnEDk36D3faT+9fKrqP+5+l8P7Ct5eM0mj2L+nb8c2BwRWwAk3QlcDRQG/tXAR5PndwM3Kf//9KuBOyPiGPAbSZuT9/tZacq3oUhiUeM0FjVO4/VL5/W1HzrWxW9feJHOoyc4eLSLzqMn6Cz4e/Ck5yc40d1DV3fQ1RN09wQnunvo7sm/7uru6WvP9+nBZ5EOrvDHADjlx4O+50P8qPS9T//6DLWs4Ddm0PUG9Olv5ZTPG9jeV++QywauN/QP3knrDeg28DMG6zPY5w2n2B/fQXsNsWqxn1/sZ8+aVs1df/aqIt919IoJ/IXAtoLX24FXDtUnIrokHQBmJ+0PD1h34WAfImkVsAqgpaWlmNptDGbUVLL0JfUT9v49PUF3RN+PQnfyQ9D3uifoid6/+X+JdEfQ0wM9EUSQfx1BT9KnJ3kdQd/6hc97Xwf979Hfln/v/mX9ffv6kX/RW0++L31zG/W+T+/oZwzWNuB9e5f1vndve379U9+rt52T2vs/v7etsJ7CZfQt628o7N/3/gPWKVy9v/3kNz2p/8DPK+h76rLi1mOY9frXP7V1NPsWxR6qLPazR/X5oyi0rnZij7UV8+6D/TQN3ISh+hSzbr4xYjWwGvJj+EXUZZNQLidyiCofDjCbdIq5imc70FzwehGwY6g+kiqBmcDeItc1M7PToJjAfwQ4R9KZkqrJH4RdO6DPWuDa5PnbgO9H/t9Ba4FrJNVIOhM4B/hFaUo3M7PRGHFIJxmTvw64n/xpmWsi4glJNwLtEbEW+Crw9eSg7F7yPwok/e4if4C3C3i/z9AxMysPX3hlZpYCxZyH75m4zMwywoFvZpYRDnwzs4xw4JuZZcSkPGgrqQN4boyrzwH2lLCcckvb9kD6tilt2wPp26a0bQ+cuk2LI6JpuBUmZeCPh6T2kY5UTyVp2x5I3zalbXsgfduUtu2BsW2Th3TMzDLCgW9mlhFpDPzV5S6gxNK2PZC+bUrb9kD6tilt2wNj2KbUjeGbmdng0riHb2Zmg3Dgm5llRGoCX9IKSU9L2izp+nLXUwqStkp6XNIGSVNyNjlJayTtlrSxoG2WpHWSnk3+NpazxtEYYns+Kul3yfe0QdIbylnjaEhqlvQDSZskPSHpL5L2qfwdDbVNU/J7klQr6ReSfpVsz8eS9jMl/Tz5jv45mb5++PdKwxh+cqP1Zyi40TqwcsCN1qccSVuBtoiYsheMSHoNcAj4WkScn7T9A7A3Ij6V/Dg3RsSHy1lnsYbYno8ChyLif5eztrGQtABYEBG/lFQHrAfeArybqfsdDbVNf8QU/J6S+4NPj4hDkqqAnwB/Afwl8M2IuFPSLcCvIuKLw71XWvbw+260HhHHgd4brVuZRcSD5O+RUOhq4Lbk+W3k/2OcEobYnikrInZGxC+T553AJvL3nZ7K39FQ2zQlRd6h5GVV8gjgCuDupL2o7ygtgT/Yjdan7BdcIIAHJK1PbvKeFvMiYifk/+ME5pa5nlK4TtJjyZDPlBn+KCRpCbAM+Dkp+Y4GbBNM0e9JUoWkDcBuYB3wa2B/RHQlXYrKvLQEftE3S59iLouIi4GrgPcnwwk2+XwROAtoBXYCnylvOaMnaQbwDeC/R8TBctdTCoNs05T9niKiOyJayd8XfDlw3mDdRnqftAR+Km+WHhE7kr+7gXvIf9FpsCsZZ+0db91d5nrGJSJ2Jf9B9gBfZop9T8m48DeA/xsR30yap/R3NNg2TfXvCSAi9gM/BC4FGiT13qa2qMxLS+AXc6P1KUXS9OSAE5KmA1cCG4dfa8oovOn9tcC3y1jLuPUGY+KtTKHvKTkg+FVgU0R8tmDRlP2Ohtqmqfo9SWqS1JA8PwN4HfnjEj8A3pZ0K+o7SsVZOgDJKVafo/9G658oc0njIuml5PfqIX+z+dun4jZJugO4nPxUrruAG4BvAXcBLcBvgf8cEVPiQOgQ23M5+WGCALYCf9o7/j3ZSXo18GPgcaAnaf5r8mPeU/U7GmqbVjIFvydJF5I/KFtBfif9roi4McmIO4FZwKPAOyPi2LDvlZbANzOz4aVlSMfMzEbgwDczywgHvplZRjjwzcwywoFvZpYRDnwzs4xw4JuZZcT/B/mERvKl+DdoAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(tr_loss_hist)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}