Lab12/lab-12-3 JihoHong.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Lab 12-3 many to many\n",
    "### simple pos-tagger training \n",
    "* many to many\n",
    "* variable input sequence length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.12.0\n"
     ]
    }
   ],
   "source": [
    "# setup\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import matplotlib.pyplot as plt\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras import layers\n",
    "from tensorflow.keras import Sequential, Model\n",
    "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
    "from pprint import pprint\n",
    "%matplotlib inline\n",
    "\n",
    "print(tf.__version__)\n",
    "tf.enable_eager_execution()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Prepairing dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# example data\n",
    "sentences = [['I', 'feel', 'hungry'],\n",
    "     ['tensorflow', 'is', 'very', 'difficult'],\n",
    "     ['tensorflow', 'is', 'a', 'framework', 'for', 'deep', 'learning'],\n",
    "     ['tensorflow', 'is', 'very', 'fast', 'changing']]\n",
    "pos = [['pronoun', 'verb', 'adjective'],\n",
    "     ['noun', 'verb', 'adverb', 'adjective'],\n",
    "     ['noun', 'verb', 'determiner', 'noun', 'preposition', 'adjective', 'noun'],\n",
    "     ['noun', 'verb', 'adverb', 'adjective', 'verb']]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Preprocessing dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'<pad>': 0, 'I': 1, 'a': 2, 'changing': 3, 'deep': 4, 'difficult': 5, 'fast': 6, 'feel': 7, 'for': 8, 'framework': 9, 'hungry': 10, 'is': 11, 'learning': 12, 'tensorflow': 13, 'very': 14}\n",
      "{0: '<pad>', 1: 'I', 2: 'a', 3: 'changing', 4: 'deep', 5: 'difficult', 6: 'fast', 7: 'feel', 8: 'for', 9: 'framework', 10: 'hungry', 11: 'is', 12: 'learning', 13: 'tensorflow', 14: 'very'}\n",
      "15\n"
     ]
    }
   ],
   "source": [
    "# creating a token dictionary for word\n",
    "word_list = sum(sentences, [])\n",
    "word_list = sorted(set(word_list))\n",
    "word_list = ['<pad>'] + word_list\n",
    "word2idx = {word : idx for idx, word in enumerate(word_list)}\n",
    "idx2word = {idx : word for idx, word in enumerate(word_list)}\n",
    "\n",
    "print(word2idx)\n",
    "print(idx2word)\n",
    "print(len(idx2word))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'<pad>': 0, 'adjective': 1, 'adverb': 2, 'determiner': 3, 'noun': 4, 'preposition': 5, 'pronoun': 6, 'verb': 7}\n",
      "{0: '<pad>', 1: 'adjective', 2: 'adverb', 3: 'determiner', 4: 'noun', 5: 'preposition', 6: 'pronoun', 7: 'verb'}\n",
      "8\n"
     ]
    }
   ],
   "source": [
    "# creating a token dictionary for part of speech\n",
    "pos_list = sum(pos, [])\n",
    "pos_list = sorted(set(pos_list))\n",
    "pos_list = ['<pad>'] + pos_list\n",
    "pos2idx = {pos : idx for idx, pos in enumerate(pos_list)}\n",
    "idx2pos = {idx : pos for idx, pos in enumerate(pos_list)}\n",
    "\n",
    "print(pos2idx)\n",
    "print(idx2pos)\n",
    "print(len(pos2idx))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 1  7 10  0  0  0  0  0  0  0]\n",
      " [13 11 14  5  0  0  0  0  0  0]\n",
      " [13 11  2  9  8  4 12  0  0  0]\n",
      " [13 11 14  6  3  0  0  0  0  0]] [3, 4, 7, 5]\n",
      "[[1. 1. 1. 0. 0. 0. 0. 0. 0. 0.]\n",
      " [1. 1. 1. 1. 0. 0. 0. 0. 0. 0.]\n",
      " [1. 1. 1. 1. 1. 1. 1. 0. 0. 0.]\n",
      " [1. 1. 1. 1. 1. 0. 0. 0. 0. 0.]]\n",
      "[[6 7 1 0 0 0 0 0 0 0]\n",
      " [4 7 2 1 0 0 0 0 0 0]\n",
      " [4 7 3 4 5 1 4 0 0 0]\n",
      " [4 7 2 1 7 0 0 0 0 0]]\n"
     ]
    }
   ],
   "source": [
    "# converting sequence of tokens to sequence of indices\n",
    "max_sequence = 10\n",
    "x_data = list(map(lambda sentence : [word2idx.get(token) for token in sentence], sentences))\n",
    "y_data = list(map(lambda sentence : [pos2idx.get(token) for token in sentence], pos))\n",
    "\n",
    "# padding the sequence of indices\n",
    "x_data = pad_sequences(sequences = x_data, maxlen = max_sequence, padding='post')\n",
    "x_data_mask = ((x_data != 0) * 1).astype(np.float32)\n",
    "x_data_len = list(map(lambda sentence : len(sentence), sentences))\n",
    "\n",
    "y_data = pad_sequences(sequences = y_data, maxlen = max_sequence, padding='post')\n",
    "\n",
    "# checking data\n",
    "print(x_data, x_data_len)\n",
    "print(x_data_mask)\n",
    "print(y_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Creating model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# creating rnn for \"many to many\" sequence tagging\n",
    "num_classes = len(pos2idx)\n",
    "hidden_dim = 10\n",
    "\n",
    "input_dim = len(word2idx)\n",
    "output_dim = len(word2idx)\n",
    "one_hot = np.eye(len(word2idx))\n",
    "\n",
    "model = Sequential()\n",
    "model.add(layers.Embedding(input_dim=input_dim, output_dim=output_dim, mask_zero=True,\n",
    "                           trainable=False, input_length=max_sequence,\n",
    "                           embeddings_initializer=keras.initializers.Constant(one_hot)))\n",
    "model.add(layers.SimpleRNN(units=hidden_dim, return_sequences=True))\n",
    "model.add(layers.TimeDistributed(layers.Dense(units=num_classes)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding (Embedding)        (None, 10, 15)            225       \n",
      "_________________________________________________________________\n",
      "simple_rnn (SimpleRNN)       (None, 10, 10)            260       \n",
      "_________________________________________________________________\n",
      "time_distributed (TimeDistri (None, 10, 8)             88        \n",
      "=================================================================\n",
      "Total params: 573\n",
      "Trainable params: 348\n",
      "Non-trainable params: 225\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# creating loss function\n",
    "def loss_fn(model, x, y, x_len, max_sequence):\n",
    "    masking = tf.sequence_mask(x_len, maxlen=max_sequence, dtype=tf.float32)\n",
    "    valid_time_step = tf.cast(x_len,dtype=tf.float32)\n",
    "    sequence_loss = tf.losses.sparse_softmax_cross_entropy(labels=y, logits=model(x),\n",
    "                                                           reduction='none') * masking\n",
    "    sequence_loss = tf.reduce_sum(sequence_loss, axis=-1) / valid_time_step\n",
    "    sequence_loss = tf.reduce_mean(sequence_loss)\n",
    "    return sequence_loss\n",
    "\n",
    "# creating and optimizer\n",
    "lr = 0.1\n",
    "epochs = 30\n",
    "batch_size = 2 \n",
    "opt = tf.train.AdamOptimizer(learning_rate = lr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<BatchDataset shapes: ((?, 10), (?, 10), (?,)), types: (tf.int32, tf.int32, tf.int32)>\n"
     ]
    }
   ],
   "source": [
    "# generating data pipeline\n",
    "tr_dataset = tf.data.Dataset.from_tensor_slices((x_data, y_data, x_data_len))\n",
    "tr_dataset = tr_dataset.shuffle(buffer_size=4)\n",
    "tr_dataset = tr_dataset.batch(batch_size = 2)\n",
    "\n",
    "print(tr_dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch :   5, tr_loss : 0.249\n",
      "epoch :  10, tr_loss : 0.040\n",
      "epoch :  15, tr_loss : 0.006\n",
      "epoch :  20, tr_loss : 0.002\n",
      "epoch :  25, tr_loss : 0.001\n",
      "epoch :  30, tr_loss : 0.001\n"
     ]
    }
   ],
   "source": [
    "# training\n",
    "tr_loss_hist = []\n",
    "\n",
    "for epoch in range(epochs):\n",
    "    avg_tr_loss = 0\n",
    "    tr_step = 0\n",
    "    \n",
    "    for x_mb, y_mb, x_mb_len in tr_dataset:\n",
    "        with tf.GradientTape() as tape:\n",
    "            tr_loss = loss_fn(model, x=x_mb, y=y_mb, x_len=x_mb_len, max_sequence=max_sequence)\n",
    "        grads = tape.gradient(target=tr_loss, sources=model.variables)\n",
    "        opt.apply_gradients(grads_and_vars=zip(grads, model.variables))\n",
    "        avg_tr_loss += tr_loss\n",
    "        tr_step += 1\n",
    "    else:\n",
    "        avg_tr_loss /= tr_step\n",
    "        tr_loss_hist.append(avg_tr_loss)\n",
    "    \n",
    "    if (epoch + 1) % 5 == 0:\n",
    "        print('epoch : {:3}, tr_loss : {:.3f}'.format(epoch + 1, avg_tr_loss))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Checking performance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[['pronoun', 'verb', 'adjective', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>'],\n",
      " ['noun', 'verb', 'adverb', 'adjective', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>'],\n",
      " ['noun', 'verb', 'determiner', 'noun', 'preposition', 'adjective', 'noun', '<pad>', '<pad>', '<pad>'],\n",
      " ['noun', 'verb', 'adverb', 'adjective', 'verb', '<pad>', '<pad>', '<pad>', '<pad>', '<pad>']]\n",
      "[['pronoun', 'verb', 'adjective'],\n",
      " ['noun', 'verb', 'adverb', 'adjective'],\n",
      " ['noun', 'verb', 'determiner', 'noun', 'preposition', 'adjective', 'noun'],\n",
      " ['noun', 'verb', 'adverb', 'adjective', 'verb']]\n"
     ]
    }
   ],
   "source": [
    "yhat = model.predict(x_data)\n",
    "yhat = np.argmax(yhat, axis=-1) * x_data_mask\n",
    "\n",
    "pprint(list(map(lambda row : [idx2pos.get(elm) for elm in row],yhat.astype(np.int32).tolist())), width = 120)\n",
    "pprint(pos)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7fd564027c50>]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAD8CAYAAABw1c+bAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAHfhJREFUeJzt3X2UXHWd5/H3t6q7uvLUnYfudEMeCQQC0Ty2mbAwik8xuAjIeDTRWcGjk505oOO6szu4nhUHZ4+Mujs66shEjeiuEhFBo/KoMIJiIJ1IgDxBSAIkJOlOAp2E9FN1ffePup1UOlXdle7qrq57P69z6nTV795b9b3U4VM3v/v73WvujoiIREes1AWIiMjwUvCLiESMgl9EJGIU/CIiEaPgFxGJGAW/iEjEKPhFRCJGwS8iEjEKfhGRiKkodQG51NbW+syZM0tdhohI2di4ceMhd68rZN0RGfwzZ86kqamp1GWIiJQNM3up0HXV1SMiEjEKfhGRiFHwi4hEjIJfRCRiFPwiIhGj4BcRiRgFv4hIxIQm+NNp55uPvMDvnm8pdSkiIiNaaII/FjP+7bFdPLLtYKlLEREZ0UIT/ADn1CTZ39pe6jJEREa0UAV/fXWSg0cV/CIifQlV8DdUJzmg4BcR6VOogv+cmiQtxzpIdadLXYqIyIgVquCvr0mSdmg53lHqUkRERqxQBX9DdRKAAzrBKyKSV6iCvz4Ifp3gFRHJL1TBf05NJvg1pFNEJL9+78BlZmuAq4Bmd39TjuX/DfhI1vtdDNS5+xEz2wMcA7qBlLs3FqvwXCaOSZCIxzSyR0SkD4Uc8d8BLM+30N2/4u4L3H0B8Fngd+5+JGuVtwfLhzT0AcyMydVVHNQRv4hIXv0Gv7s/Bhzpb73ASuDOQVU0SJq9KyLSt6L18ZvZaDL/MvhZVrMDD5nZRjNbVazP6otm74qI9K2YJ3ffB/yhVzfP5e6+CLgSuNHM3ppvYzNbZWZNZtbU0jLwK2z2zN519wG/h4hImBUz+FfQq5vH3fcFf5uBe4El+TZ299Xu3ujujXV1dQMuoqEmSXtXmta2rgG/h4hImBUl+M2sBngb8IustjFmNq7nObAMeK4Yn9eXhmBIp0b2iIjkVshwzjuBK4BaM9sL3AJUArj77cFq7wcecvc3sjatB+41s57P+bG7P1C80nPLnr07p6F6qD9ORKTs9Bv87r6ygHXuIDPsM7ttFzB/oIUNlGbvioj0LVQzd+FU8GtIp4hIbqEL/kRFjNqxCR3xi4jkEbrgh8xRv67QKSKSWyiDX7N3RUTyC2Xwa/auiEh+oQz+huokr53oor2ru9SliIiMOOEM/mASV/NR3YJRRKS3UAf//ta2ElciIjLyhDP4q3XZBhGRfEIZ/PU1mr0rIpJPKIN/XFUFYxJxDekUEckhlMFvZtTXaEiniEguoQx+CG7IoiN+EZEzhDf4axT8IiK5hDf4q5M0H+sgndYtGEVEsoU3+GuSpNLOoTc0iUtEJFt4g7/nhiytCn4RkWzhDX7N3hURyanf4DezNWbWbGY5b5RuZleYWauZPR08Pp+1bLmZ7TCznWZ2czEL70+DbsEoIpJTIUf8dwDL+1nncXdfEDxuBTCzOPAt4ErgEmClmV0ymGLPxqSxVcRjpss2iIj00m/wu/tjwJEBvPcSYKe773L3TmAtcM0A3mdA4jGjflyVZu+KiPRSrD7+S81ss5ndb2Zzg7YpwCtZ6+wN2nIys1Vm1mRmTS0tLUUpSrN3RUTOVIzg3wTMcPf5wDeAnw/kTdx9tbs3untjXV1dEcrS7F0RkVwGHfzuftTdjwfP7wMqzawW2AdMy1p1atA2bBpqkhzUzVhERE4z6OA3swYzs+D5kuA9DwMbgNlmdp6ZJYAVwLrBft7ZaKhOcrwjxbH2ruH8WBGREa2ivxXM7E7gCqDWzPYCtwCVAO5+O/AB4G/MLAW0ASvc3YGUmd0EPAjEgTXuvmVI9iKPhqzr8o9LVg7nR4uIjFj9Br+7r+xn+TeBb+ZZdh9w38BKG7yTd+Jq7eCCyeNKVYaIyIgS2pm7oNm7IiK5hDr46zV7V0TkDKEO/mRlnPGjKzV7V0QkS6iDHzSWX0Skt/AHf01SR/wiIlnCH/zVSQ7omvwiIieFP/hrkhx+o4POVLrUpYiIjAjhD/7qJO7QfEzdPSIiEIHgr6/RkE4RkWyhD/5zak7N3hURkQgEf89lGzR7V0QkI/TBXzOqkqqKmLp6REQCoQ9+MwvG8qurR0QEIhD8kOnuOajZuyIiQFSCvybJ/qPq4xcRgagEf3XmFoyZ+8OIiERbNIK/JklnKs1rJ3QLRhGRfoPfzNaYWbOZPZdn+UfM7Bkze9bMnjCz+VnL9gTtT5tZUzELPxsa0ikickohR/x3AMv7WL4beJu7vxn4IrC61/K3u/sCd28cWImDp9m7IiKnFHLP3cfMbGYfy5/IerkemDr4sopLs3dFRE4pdh//x4H7s1478JCZbTSzVUX+rILVja0iZnBAXT0iIv0f8RfKzN5OJvgvz2q+3N33mdlk4GEz2+7uj+XZfhWwCmD69OnFKguAiniM2rFVuiGLiAhFOuI3s3nAd4Fr3P1wT7u77wv+NgP3AkvyvYe7r3b3RndvrKurK0ZZp9HsXRGRjEEHv5lNB+4B/pO7P5/VPsbMxvU8B5YBOUcGDQfN3hURyei3q8fM7gSuAGrNbC9wC1AJ4O63A58HJgH/amYAqWAETz1wb9BWAfzY3R8Ygn0oSENNkvW7Dve/oohIyBUyqmdlP8s/AXwiR/suYP6ZW5RGfXWSo+0p2jq7GZWIl7ocEZGSicTMXcga0qkTvCIScZEJfs3eFRHJiEzwa/auiEhGZIK/54hfs3dFJOoiE/xjqioYl6zQEb+IRF5kgh8yR/3q4xeRqItW8Gv2rohIxIJfs3dFRCIW/DVJmo+1k+pOl7oUEZGSiVTw11cnSTscOt5Z6lJEREomUsGv2bsiIhEL/vqTY/k1skdEoitSwd9w8haMOuIXkeiKVPBPHJ0gEY9pSKeIRFqkgj8WMyZXV2n2rohEWqSCHzR7V0QkcsFfX5PkoLp6RCTCIhf851QnOdDajruXuhQRkZIoKPjNbI2ZNZtZzpulW8a/mNlOM3vGzBZlLbvezF4IHtcXq/CBaqhJ0tbVzdG2VKlLEREpiUKP+O8Alvex/EpgdvBYBXwbwMwmkrk5+58BS4BbzGzCQIsthpNj+XWCV0QiqqDgd/fHgCN9rHIN8EPPWA+MN7NzgPcAD7v7EXd/DXiYvn9Ahpxm74pI1BWrj38K8ErW671BW772M5jZKjNrMrOmlpaWIpV1pp4jfl2lU0SiasSc3HX31e7e6O6NdXV1Q/Y59Sdvuq7gF5FoKlbw7wOmZb2eGrTlay+ZREWM2rEJdfWISGQVK/jXAR8NRvcsBVrdfT/wILDMzCYEJ3WXBW0lVV+d1OxdEYmsikJWMrM7gSuAWjPbS2akTiWAu98O3Ae8F9gJnAA+Fiw7YmZfBDYEb3Wru/d1knhYNFQneVVdPSISUQUFv7uv7Ge5AzfmWbYGWHP2pQ2d+pokf3rl9VKXISJSEiPm5O5wOrcmyZE3OnmjQ5O4RCR6Ihn886aOB2DTy6+VuBIRkeEXyeBfPGMCFTFj/a7DpS5FRGTYRTL4x1RVMG9qDX98UcEvItETyeAHWDprEs/sbVU/v4hETmSD/9LzJ5FKOxtfUj+/iERLZINf/fwiElWRDf7RiQrmTxuv4BeRyIls8AMsnTVR/fwiEjkRD/5MP3+T+vlFJEIiHfyLZ0ygMq5+fhGJlkgH/+hEBfOnqp9fRKIl0sEPGs8vItGj4J81iW7184tIhEQ++BfNGE9l3HT5BhGJjMgHv/r5RSRqIh/8kOnueXZfK8fVzy8iEVBQ8JvZcjPbYWY7zezmHMv/2cyeDh7Pm9nrWcu6s5atK2bxxXLp+UE//56S3xVSRGTI9XvrRTOLA98C3g3sBTaY2Tp339qzjrv/l6z1PwkszHqLNndfULySi2/R9J7x/Ee44qLJpS5HRGRIFXLEvwTY6e673L0TWAtc08f6K4E7i1HccBmViLNA1+0RkYgoJPinAK9kvd4btJ3BzGYA5wGPZDUnzazJzNab2bUDrnSIqZ9fRKKi2Cd3VwB3u3t3VtsMd28EPgx8zczOz7Whma0KfiCaWlpailxW/3rG829QP7+IhFwhwb8PmJb1emrQlssKenXzuPu+4O8u4N85vf8/e73V7t7o7o11dXUFlFVcp/r51d0jIuFWSPBvAGab2XlmliAT7meMzjGzOcAE4I9ZbRPMrCp4XgtcBmztve1IcKqfX0f8IhJu/Qa/u6eAm4AHgW3AXe6+xcxuNbOrs1ZdAax1d89quxhoMrPNwKPAbdmjgUaaS2dN4rl9rRxr7yp1KSIiQ6bf4ZwA7n4fcF+vts/3ev2FHNs9Abx5EPUNq6WzJvEvj+yk6aXXeLuGdYpISGnmbpaF0yeQiMdYr+v2iEiIKfizaDy/iESBgr+XpbMm8qz6+UUkxBT8vSydNYm0Q9MeXZ9fRMJJwd/LohlBP7+6e0QkpBT8vSQr4yyYrn5+EQkvBX8OPdftOap+fhEJIQV/DktnTQz6+TWLV0TCR8Gfw6Ke8fy6fIOIhJCCPwf184tImCn481gaXLdH/fwiEjYK/jwuDcbzb9it7h4RCRcFfx4Lp48nUaHx/CISPgr+PJKVcRbq+vwiEkIK/j4snTWJLa+20tqmfn4RCQ8Ffx9OXbdHR/0iEh4K/j4snD6eUZVxVj+2i45Ud/8biIiUgYKC38yWm9kOM9tpZjfnWH6DmbWY2dPB4xNZy643sxeCx/XFLH6oJSvjfOm6N/Pk7iP83U+fIZ32/jcSERnh+r31opnFgW8B7wb2AhvMbF2Oe+f+xN1v6rXtROAWoBFwYGOwbdlc8/jahVPY39rOPz2wnYbqKj73Hy8pdUkiIoNSyBH/EmCnu+9y905gLXBNge//HuBhdz8ShP3DwPKBlVo6f/22WVx/6Qy+8/huvvf73aUuR0RkUAoJ/inAK1mv9wZtvf2FmT1jZneb2bSz3HZEMzM+/765LJ/bwD/+eiu/eubVUpckIjJgxTq5+0tgprvPI3NU/4OzfQMzW2VmTWbW1NLSUqSyiiceM762YgGNMybwmZ9s1sQuESlbhQT/PmBa1uupQdtJ7n7Y3TuCl98FFhe6bdZ7rHb3RndvrKurK6T2YZesjPOdjzYyfdJo/uqHTew4cKzUJYmInLVCgn8DMNvMzjOzBLACWJe9gpmdk/XyamBb8PxBYJmZTTCzCcCyoK1sjR+d4I6PvYVRlXGuX/MU+1vbSl2SiMhZ6Tf43T0F3EQmsLcBd7n7FjO71cyuDlb7lJltMbPNwKeAG4JtjwBfJPPjsQG4NWgra1MnjOaOjy3heEeKG9Zs0MxeESkr5j7yxqY3NjZ6U1NTqcvo1x92HuKG7z/FoukT+OHHl1BVES91SSISUWa20d0bC1lXM3cH4bILavnKB+bz5O4jfOauzZrgJSJlod8JXNK3axdO4eDRdr50/3YaqpP8z6s0wUtERjYd8RfBqrfO4qOXzuB7v9+tC7qJyIin4C8CM+PmK+cweVwVX7p/OyPxvImISA8Ff5GMTlTw6XddyMaXXuOhrQdLXY6ISF4K/iL6YONUzq8bw5cf2E6qO13qckREclLwF1FFPMZ/Xz6HF1ve4K6mvaUuR0QkJwV/kS27pJ7FMybwz795nhOdqVKXIyJyBgV/kZkZn71yDi3HOlijSziLyAik4B8CjTMn8u5L6rn9d7s4fLyj/w1ERIaRgn+I/P3yizjRmeIbj+wsdSkiIqdR8A+RCyaP40NvmcaPnnyJlw+fKHU5IiInKfiH0KffdSHxmPHVh3aUuhQRkZMU/EOovjrJJy6fxbrNr/Ls3tZSlyMiAij4h9yqt81iwuhKbntgmy7lICIjgoJ/iFUnK/nkO2bzh52HeeyFQ6UuR0REwT8cPrJ0OtMmjuK2+7frmv0iUnIK/mFQVRHn75ZdxLb9R/nF5pz3mhcRGTYFBb+ZLTezHWa208xuzrH8M2a21cyeMbPfmtmMrGXdZvZ08FjXe9uoeN+8c3nTlGq++uDztHd1l7ocEYmwfoPfzOLAt4ArgUuAlWbW+zZTfwIa3X0ecDfw5axlbe6+IHhcTUTFYsbNyy9m3+tt/L/1L5W6HBGJsEKO+JcAO919l7t3AmuBa7JXcPdH3b1nltJ6YGpxywyHy2fX8ueza/nmoztpbesqdTkiElGFBP8U4JWs13uDtnw+Dtyf9TppZk1mtt7Mrh1AjaFy85VzaG3r4sYfbaL5WHupyxGRCCrqyV0z+0ugEfhKVvMMd28EPgx8zczOz7PtquAHoqmlpaWYZY0oc8+t4bbr3syGPUd479cf59EdzaUuSUQippDg3wdMy3o9NWg7jZm9C/gccLW7n7wkpbvvC/7uAv4dWJjrQ9x9tbs3untjXV1dwTtQjj70lun86pOXUzu2io99fwP/8MstOuErIsOmkODfAMw2s/PMLAGsAE4bnWNmC4F/IxP6zVntE8ysKnheC1wGbC1W8eVsdv04fn7jZdzwH2by/T/s4f3/+gQ7m4+VuiwRiYB+g9/dU8BNwIPANuAud99iZreaWc8ona8AY4Gf9hq2eTHQZGabgUeB29xdwR9IVsb5wtVzWXNDI81H27nqG7/nx0++rEs7iMiQspEYMo2Njd7U1FTqMoZV89F2/utPN/P4C4d4z9x6brtuHhPGJEpdloiUCTPbGJxP7Zdm7o4Qk6uT/OBjS/jcey/mke3NXPn1x/nji4dLXZaIhJCCfwSJxYy/euss7vmbyxiViPPh767nH3+1le0Hjqr7R0SKRl09I9QbHSlu/eVWftKUmUIxZfwo3nnxZN55cT1LZ02kqiJe4gpFZCQ5m64eBf8Id/BoO49sb+a325r5/c4W2rvSjE7E+fPZtbzz4nreMWcytWOrSl2miJSYgj+k2ru6eeLFQ/x2W+aH4MDRdsxg/tTxvOviybz7kgYurB+LmZW6VBEZZgr+CHB3trx6NPjXwEE2B7d2nDFpNMsuqWfZ3AYWTZ9APKYfAZEoUPBHUPPRdn6zrZmHth7giZ2H6exOM2lMgnddXM+yufVcdkEtyUqdFxAJKwV/xB1r7+J3z7fw0JaDPLq9mWMdKUZVxnnbhXUsm5s5LzB+tOYIiITJ2QR/xVAXI8NvXLKSq+ady1XzzqUzlebJ3Yd5aMtBHtp6gAe2HKAybrxjzmSuWzSVt180mUSFRvWKRImO+CMknXae3dfKLze/ys+ffpVDxzuYOCbB1fPP5bpFU3jzlBqdGBYpU+rqkX6lutM8/sIh7t60l4e3HqQzlWb25LH8xeKpvH/hFOqrk6UuUUTOgoJfzkrriS5+/ex+frZpLxtfeo2YwWUX1PKBxVN5z9wGnRQWKQMKfhmw3Yfe4J5Ne7ln0z72vd5GdbKC6xZNZcWSacxpqC51eSKSh4JfBi2ddtbvPszap17hgecO0NmdZuH08axcMp2r5p3D6ITGBYiMJAp+Kaojb3Ryz6a93PnUy7zY8gbjqiq4duEUViyZxtxza0pdnoig4Jch4u5s2PMadz71Mr9+dj+dqTTzp9awcsl03jf/XMZU6V8BIqWi4Jch9/qJTu790z7ufOplnj94nIqYMatuDBc1VHNR/VguaqhmTsM4powfRUyXjRAZckUPfjNbDnwdiAPfdffbei2vAn4ILAYOAx9y9z3Bss8CHwe6gU+5+4P9fZ6Cv3y4O5tefo3fbmtmx4FjbD9wjH2vt51cPiYR58KGcVxUP46LGjKPhuok40cnqE5WUBHX5DGRYijqzF0ziwPfAt4N7AU2mNm6XvfO/TjwmrtfYGYrgH8CPmRml5C5Oftc4FzgN2Z2obt3n90uyUhlZiyeMZHFMyaebDvW3sXzB4+z48Axdhw4yo6Dx3hwywHWbnjljO3HVVVQM7qS8aMrqRlVyfhRiczrUZWMS1YyOhFnVCLOqMp45nll8DoRZ3RlBclEjNGJChLxGJVx0wQ0kQIU0im7BNjp7rsAzGwtcA2QHfzXAF8Int8NfNMy/wdeA6x19w5gt5ntDN7vj8UpX0aicclKFs+YwOIZE062uTstxzrYcfAYh4530Hqii9fbunj9RBdH23qed3Kg9SitbV20tnXR1X323ZCJihiJeOy0v5VxI1ERJ1ERoyoeI5mIk6yIkayMk6yMMaoyTrIyTlWv15XxGBUxIx6zU3/jRjx2Zns8ZsQs88g8z9xRLR60xWIQjxlGZhkGMTOMzI9nzMAwLAZGsKynzTj5PGaZ9TPboR86GZBCgn8KkH2othf4s3zruHvKzFqBSUH7+l7bThlwtVK2zIzJ1UkmFzgj2N1p70pzojNFW1c3bZ3dtHV1cyL429aZeZzo6qatM0VnKk1nKk1Hd+ZvV/C3M5WmsztNZ8rp7E7T0dVNa1sXzV3dtHdl3qu9K017VzcdqfQQ/1cYGpkfhZ7nWT8KZBbYaetlfkiAkz86Pc/J1Z7VBqf/0GS/b3bLmdv0vD79Ryr7N6v3z1dfP2inbddrtezPOHNZYXJ9dt5tC3zTfKv1/qyJoxPc9deXFvamgzBihmGY2SpgFcD06dNLXI2Umpmd7NIZLum005HK/Ai0p7pJdTuptNOdTpNKO6lupzvd0+akutMnn6f91N+0k/Xc6U5n3rvbHXdwMusQrOvuOJx67pAO2nrW99OW9WoL6j9t3azXmYU9bZ617unb9TznZLv3bHrmsl7rn7lOr+W9/vF2qupcy+hjmede8YztPO+yvuQ65Zlv20IHxuRdK8eCccnhieRCPmUfMC3r9dSgLdc6e82sAqghc5K3kG0BcPfVwGrInNwtpHiRYorFhv/HRqQUChlSsQGYbWbnmVmCzMnadb3WWQdcHzz/APCIZ34O1wErzKzKzM4DZgNPFad0EREZiH6P+IM++5uAB8kM51zj7lvM7Fagyd3XAd8D/m9w8vYImR8HgvXuInMiOAXcqBE9IiKlpQlcIiIhcDbj+DV7RkQkYhT8IiIRo+AXEYkYBb+ISMQo+EVEImZEjuoxsxbgpQFuXgscKmI5pRa2/YHw7VPY9gfCt09h2x84c59muHtdIRuOyOAfDDNrKnRIUzkI2/5A+PYpbPsD4dunsO0PDG6f1NUjIhIxCn4RkYgJY/CvLnUBRRa2/YHw7VPY9gfCt09h2x8YxD6Fro9fRET6FsYjfhER6UNogt/MlpvZDjPbaWY3l7qeYjCzPWb2rJk9bWZledU6M1tjZs1m9lxW20Qze9jMXgj+TujrPUaSPPvzBTPbF3xPT5vZe0tZ49kws2lm9qiZbTWzLWb2t0F7OX9H+fapLL8nM0ua2VNmtjnYn38I2s8zsyeDzPtJcNn8wt4zDF09wQ3hnyfrhvDAyl43hC87ZrYHaHT3sh1/bGZvBY4DP3T3NwVtXwaOuPttwY/0BHf/+1LWWag8+/MF4Li7f7WUtQ2EmZ0DnOPum8xsHLARuBa4gfL9jvLt0wcpw+8puH/5GHc/bmaVwO+BvwU+A9zj7mvN7HZgs7t/u5D3DMsR/8kbwrt7J9BzQ3gpMXd/jMw9GrJdA/wgeP4DMv9TloU8+1O23H2/u28Knh8DtpG5L3Y5f0f59qksecbx4GVl8HDgHcDdQftZfUdhCf5cN4Qv2y86iwMPmdnG4J7EYVHv7vuD5weA+lIWUyQ3mdkzQVdQ2XSLZDOzmcBC4ElC8h312ico0+/JzOJm9jTQDDwMvAi87u6pYJWzyrywBH9YXe7ui4ArgRuDboZQCW7RWe79jd8GzgcWAPuB/13acs6emY0FfgZ82t2PZi8r1+8oxz6V7ffk7t3uvoDMfcuXAHMG835hCf6Cb+peTtx9X/C3GbiXzBceBgeDftie/tjmEtczKO5+MPgfMw18hzL7noJ+458BP3L3e4Lmsv6Ocu1TuX9PAO7+OvAocCkw3sx6bp97VpkXluAv5IbwZcXMxgQnpjCzMcAy4Lm+tyob64Drg+fXA78oYS2D1hOQgfdTRt9TcOLwe8A2d/8/WYvK9jvKt0/l+j2ZWZ2ZjQ+ejyIziGUbmR+ADwSrndV3FIpRPQDB0KyvceqG8P+rxCUNipnNInOUD1AB/Lgc98nM7gSuIHMlwYPALcDPgbuA6WSuwvpBdy+LE6Z59ucKMt0HDuwB/nNW//iIZmaXA48DzwLpoPl/kOkTL9fvKN8+raQMvyczm0fm5G2czMH6Xe5+a5ARa4GJwJ+Av3T3joLeMyzBLyIihQlLV4+IiBRIwS8iEjEKfhGRiFHwi4hEjIJfRCRiFPwiIhGj4BcRiRgFv4hIxPx/+ypeJpYek5wAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(tr_loss_hist)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}