Lab12/Lab 12-2 minkiKang.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 모두를 위한 딥러닝 시즌2 - TensorFlow\n",
    "\n",
    "## LAB 12 - 2 RNN many to one stacking"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.5.0\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "import matplotlib.pyplot as plt\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras import layers\n",
    "from tensorflow.keras import Sequential, Model\n",
    "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
    "from pprint import pprint\n",
    "%matplotlib inline\n",
    "\n",
    "print(tf.__version__)\n",
    "tf.compat.v1.enable_eager_execution()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Preparing dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentences = ['What I cannot create, I do not understand.',\n",
    "             'Intellecuals solve problems, geniuses prevent them',\n",
    "             'A person who never made a mistake never tied anything new.',\n",
    "             'The same equations have the same solutions.']\n",
    "y_data = [1,0,0,1] #richard feynman : 1, albert einstein : 0 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['<pad>', ' ', ',', '.', 'A', 'I', 'T', 'W', 'a', 'b', 'c', 'd', 'e', 'g', 'h', 'i', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'y']\n",
      "{0: '<pad>', 1: ' ', 2: ',', 3: '.', 4: 'A', 5: 'I', 6: 'T', 7: 'W', 8: 'a', 9: 'b', 10: 'c', 11: 'd', 12: 'e', 13: 'g', 14: 'h', 15: 'i', 16: 'k', 17: 'l', 18: 'm', 19: 'n', 20: 'o', 21: 'p', 22: 'q', 23: 'r', 24: 's', 25: 't', 26: 'u', 27: 'v', 28: 'w', 29: 'y'}\n",
      "{'<pad>': 0, ' ': 1, ',': 2, '.': 3, 'A': 4, 'I': 5, 'T': 6, 'W': 7, 'a': 8, 'b': 9, 'c': 10, 'd': 11, 'e': 12, 'g': 13, 'h': 14, 'i': 15, 'k': 16, 'l': 17, 'm': 18, 'n': 19, 'o': 20, 'p': 21, 'q': 22, 'r': 23, 's': 24, 't': 25, 'u': 26, 'v': 27, 'w': 28, 'y': 29}\n"
     ]
    }
   ],
   "source": [
    "char_set = ['<pad>'] + sorted(list(set(''.join(sentences))))\n",
    "idx2char = {idx : char for idx, char in enumerate(char_set)}\n",
    "char2idx = {char : idx for idx, char in enumerate(char_set)}\n",
    "\n",
    "print(char_set)\n",
    "print(idx2char)\n",
    "print(char2idx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[7, 14, 8, 25, 1, 5, 1, 10, 8, 19, 19, 20, 25, 1, 10, 23, 12, 8, 25, 12, 2, 1, 5, 1, 11, 20, 1, 19, 20, 25, 1, 26, 19, 11, 12, 23, 24, 25, 8, 19, 11, 3], [5, 19, 25, 12, 17, 17, 12, 10, 26, 8, 17, 24, 1, 24, 20, 17, 27, 12, 1, 21, 23, 20, 9, 17, 12, 18, 24, 2, 1, 13, 12, 19, 15, 26, 24, 12, 24, 1, 21, 23, 12, 27, 12, 19, 25, 1, 25, 14, 12, 18], [4, 1, 21, 12, 23, 24, 20, 19, 1, 28, 14, 20, 1, 19, 12, 27, 12, 23, 1, 18, 8, 11, 12, 1, 8, 1, 18, 15, 24, 25, 8, 16, 12, 1, 19, 12, 27, 12, 23, 1, 25, 15, 12, 11, 1, 8, 19, 29, 25, 14, 15, 19, 13, 1, 19, 12, 28, 3], [6, 14, 12, 1, 24, 8, 18, 12, 1, 12, 22, 26, 8, 25, 15, 20, 19, 24, 1, 14, 8, 27, 12, 1, 25, 14, 12, 1, 24, 8, 18, 12, 1, 24, 20, 17, 26, 25, 15, 20, 19, 24, 3]]\n",
      "[42, 50, 58, 43]\n",
      "[1, 0, 0, 1]\n"
     ]
    }
   ],
   "source": [
    "x_data = list(map(lambda sentence : [char2idx.get(char) for char in sentence], sentences))\n",
    "x_data_len = list(map(lambda sentence : len(sentence), sentences))\n",
    "\n",
    "#integer 인덱스의 시퀀스로 변환\n",
    "print(x_data)\n",
    "print(x_data_len)\n",
    "print(y_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 7 14  8 25  1  5  1 10  8 19 19 20 25  1 10 23 12  8 25 12  2  1  5  1\n",
      "  11 20  1 19 20 25  1 26 19 11 12 23 24 25  8 19 11  3  0  0  0  0  0  0\n",
      "   0  0  0  0  0  0  0]\n",
      " [ 5 19 25 12 17 17 12 10 26  8 17 24  1 24 20 17 27 12  1 21 23 20  9 17\n",
      "  12 18 24  2  1 13 12 19 15 26 24 12 24  1 21 23 12 27 12 19 25  1 25 14\n",
      "  12 18  0  0  0  0  0]\n",
      " [ 4  1 21 12 23 24 20 19  1 28 14 20  1 19 12 27 12 23  1 18  8 11 12  1\n",
      "   8  1 18 15 24 25  8 16 12  1 19 12 27 12 23  1 25 15 12 11  1  8 19 29\n",
      "  25 14 15 19 13  1 19]\n",
      " [ 6 14 12  1 24  8 18 12  1 12 22 26  8 25 15 20 19 24  1 14  8 27 12  1\n",
      "  25 14 12  1 24  8 18 12  1 24 20 17 26 25 15 20 19 24  3  0  0  0  0  0\n",
      "   0  0  0  0  0  0  0]]\n",
      "[42, 50, 58, 43]\n",
      "[1, 0, 0, 1]\n"
     ]
    }
   ],
   "source": [
    "max_sequence = 55\n",
    "#배치단위로 처리하려면 sequence의 길이를 맞춰줌\n",
    "x_data = pad_sequences(sequences = x_data, maxlen = max_sequence, padding = 'post', truncating = 'post')\n",
    "#기본적으로 0값으로 패딩\n",
    "\n",
    "print(x_data)\n",
    "print(x_data_len)\n",
    "print(y_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Creating model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#stacked rnn\n",
    "num_classes = 2\n",
    "hidden_dims = [10,10]\n",
    "\n",
    "input_dim = len(char2idx)\n",
    "output_dim = len(char2idx)\n",
    "one_hot = np.eye(len(char2idx)) \n",
    "\n",
    "model = Sequential()\n",
    "#각각의 토큰을 원핫 벡터로\n",
    "model.add(layers.Embedding(input_dim=input_dim, output_dim=output_dim,\n",
    "                           trainable=False, mask_zero=True, input_length=max_sequence,\n",
    "                           embeddings_initializer=keras.initializers.Constant(one_hot)))\n",
    "model.add(layers.SimpleRNN(units=hidden_dims[0], return_sequences=True))\n",
    "model.add(layers.TimeDistributed(layers.Dropout(rate = .2)))\n",
    "model.add(layers.SimpleRNN(units=hidden_dims[1]))\n",
    "model.add(layers.Dropout(rate = .2))\n",
    "model.add(layers.Dense(units=num_classes))#dense 레이어가 classification\n",
    "#[data dimension, max sequence, input dimension]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding (Embedding)        (None, 55, 30)            900       \n",
      "_________________________________________________________________\n",
      "simple_rnn (SimpleRNN)       (None, 55, 10)            410       \n",
      "_________________________________________________________________\n",
      "time_distributed (TimeDistri (None, 55, 10)            0         \n",
      "_________________________________________________________________\n",
      "simple_rnn_1 (SimpleRNN)     (None, 10)                210       \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 10)                0         \n",
      "_________________________________________________________________\n",
      "dense (Dense)                (None, 2)                 22        \n",
      "=================================================================\n",
      "Total params: 1,542\n",
      "Trainable params: 642\n",
      "Non-trainable params: 900\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "#cross entropy 계산\n",
    "def loss_fn(model, x, y, training):\n",
    "    return tf.compat.v1.losses.sparse_softmax_cross_entropy(labels=y, logits=model(x, training))\n",
    "\n",
    "#하이퍼 파라미터\n",
    "lr = .01\n",
    "epochs = 30\n",
    "batch_size = 2\n",
    "opt = tf.compat.v1.train.AdamOptimizer(learning_rate = lr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<BatchDataset shapes: ((None, 55), (None,)), types: (tf.int32, tf.int32)>\n"
     ]
    }
   ],
   "source": [
    "tr_dataset = tf.data.Dataset.from_tensor_slices((x_data, y_data))\n",
    "tr_dataset = tr_dataset.shuffle(buffer_size=4)\n",
    "tr_dataset = tr_dataset.batch(batch_size=batch_size)\n",
    "\n",
    "print(tr_dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch :   5, tr_loss : 0.026\n",
      "epoch :  10, tr_loss : 0.006\n",
      "epoch :  15, tr_loss : 0.006\n",
      "epoch :  20, tr_loss : 0.007\n",
      "epoch :  25, tr_loss : 0.002\n",
      "epoch :  30, tr_loss : 0.004\n"
     ]
    }
   ],
   "source": [
    "tr_loss_hist = []\n",
    "\n",
    "for epoch in range(epochs):\n",
    "    avg_tr_loss = 0\n",
    "    tr_step = 0\n",
    "    \n",
    "    for x_mb, y_mb in tr_dataset:\n",
    "        with tf.GradientTape() as tape:\n",
    "            tr_loss = loss_fn(model, x=x_mb, y=y_mb, training=True)\n",
    "        grads = tape.gradient(target=tr_loss, sources=model.variables)\n",
    "        opt.apply_gradients(grads_and_vars=zip(grads, model.variables))\n",
    "        avg_tr_loss += tr_loss\n",
    "        tr_step += 1\n",
    "    else:\n",
    "        avg_tr_loss /= tr_step\n",
    "        tr_loss_hist.append(avg_tr_loss)\n",
    "    \n",
    "    if (epoch + 1) % 5 ==0:\n",
    "        print('epoch : {:3}, tr_loss : {:.3f}'.format(epoch + 1, avg_tr_loss))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Checking performance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy : 100.00%\n"
     ]
    }
   ],
   "source": [
    "yhat = model.predict(x_data)\n",
    "yhat = np.argmax(yhat, axis=-1)\n",
    "print('accuracy : {:.2%}'.format(np.mean(yhat == y_data)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x1f6b9e79bb0>]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXUAAAD4CAYAAAATpHZ6AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAcr0lEQVR4nO3dfXRcd33n8fd3niSN/CApVhI/yXaCIQl5wIlsp1sCboHiBNiUZWHthPKwpWlYwmH3jz1h+aOwy+EsS8uetgsh62ZTYAsJ6ZIWQwMJhYakDSGWg3HiGAfHjm3Zji1bkm1Jlubpu3/MHXkk62Fkjzy+dz6vc+bMnXuvZ75XV/7oN7977++auyMiItEQq3UBIiJSPQp1EZEIUaiLiESIQl1EJEIU6iIiEZKo1QcvWLDAly9fXquPFxEJpa1btx5z9/bJltcs1JcvX05XV1etPl5EJJTMbN9Uy9X9IiISIQp1EZEIUaiLiESIQl1EJEIU6iIiEaJQFxGJkGlD3cweNLOjZvbiJMvvNLPtweMZM7uh+mWKiEglKmmpfx1YP8XyvcBb3f164PPApirUNaldr53izx7fRd9gZjY/RkQklKYNdXd/CuidYvkz7t4XvHwWWFKl2ia099ggX/mn3Rw6cXo2P0ZEJJSq3af+h8APJ1toZneZWZeZdfX09JzTB7SmkwD0DWbP6d+LiERZ1ULdzH6HYqjfO9k67r7J3TvdvbO9fdKhC6bU1pwCoG9I3S8iIuNVZewXM7seeAC41d2PV+M9J9OSVqiLiEzmvFvqZtYBPAr8gbu/fP4lTa1F3S8iIpOatqVuZg8B64AFZtYNfBZIArj7/cCfAJcA95kZQM7dO2er4GQ8xtzGhFrqIiITmDbU3X3jNMs/BnysahVVoK05Ra9OaRQROUsoryhtSafUUhcRmUAoQ70tnVSoi4hMIJSh3ppO6UCpiMgEwhnqzep+ERGZSDhDPZ1kKJNnOJuvdSkiIheVcIZ6cFVp/5C6YEREyoUz1HVVqYjIhMId6jpXXURkjHCGenMwVIC6X0RExghlqLcFLfVedb+IiIwRylAvjdTYr+4XEZExQhnqqUSMOQ0JtdRFRMYJZahDcQhendIoIjJWaENdIzWKiJwttKHekk7Rr+4XEZExQhvqbemk+tRFRMYJbai3pFP0a6RGEZExQhvqbc0pTo3kyOQKtS5FROSiEdpQbw1uQN1/Wl0wIiIl4Q315tL4L+qCEREpCW+oa6RGEZGzhD/Uda66iMio8Ia6RmoUETlLeENd3S8iImeZNtTN7EEzO2pmL06y3MzsL81st5ltN7Mbq1/m2RqTcZqScXW/iIiUqaSl/nVg/RTLbwVWBo+7gK+df1mVaWtO6apSEZEy04a6uz8F9E6xyu3AN73oWaDFzBZWq8CpaKRGEZGxqtGnvhg4UPa6O5h3FjO7y8y6zKyrp6fnvD9YIzWKiIxVjVC3Ceb5RCu6+yZ373T3zvb29vP+YI3UKCIyVjVCvRtYWvZ6CXCoCu87rbZ0Ui11EZEy1Qj1zcCHgrNgbgZOuPvhKrzvtFrSKU4O58jlNaiXiAhAYroVzOwhYB2wwMy6gc8CSQB3vx94DLgN2A0MAR+drWLHawvGf+k/nWXBnIYL9bEiIhetaUPd3TdOs9yBT1StohloKY3UOJRRqIuIEOIrSuFMS71XIzWKiAAhD3UNFSAiMla4Q71ZIzWKiJQLd6inNVKjiEi5UId6UzJOQyKm7hcRkUCoQ93MaE2n1P0iIhIIdahDsV9dLXURkaLwh3o6qT51EZFA+EO9Wd0vIiIl4Q/1dFLdLyIigdCHels6Rf/pLPnChKP9iojUldCHeks6hTucPK1+dRGR0If66Pgv6oIREQl/qJeP1CgiUu9CH+oaqVFE5IzQh7pGahQROSP8oa6RGkVERoU+1JtTcVLxmK4qFREhAqFuZrSkk2qpi4gQgVCH4sFS9amLiEQk1Fs0VICICBCRUC+21NWnLiISiVBv0Y0yRESAiIR6aVCvggb1EpE6V1Gom9l6M9tlZrvN7NMTLJ9vZt83s1+Z2Q4z+2j1S51cSzpJvuCcGs5dyI8VEbnoTBvqZhYHvgrcClwDbDSza8at9gngJXe/AVgHfNnMUlWudVKloQJ0sFRE6l0lLfU1wG533+PuGeBh4PZx6zgw18wMmAP0Ahes2VwaKkAjNYpIvask1BcDB8pedwfzyn0FuBo4BLwAfMrdC+PfyMzuMrMuM+vq6ek5x5LPVhoqQCM1iki9qyTUbYJ5449IvhPYBiwC3gR8xczmnfWP3De5e6e7d7a3t8+w1Mm1BsPvaqRGEal3lYR6N7C07PUSii3ych8FHvWi3cBe4KrqlDg9tdRFRIoqCfUtwEozWxEc/NwAbB63zn7gbQBmdhnwBmBPNQudytyGBImY0atz1UWkziWmW8Hdc2Z2D/A4EAcedPcdZnZ3sPx+4PPA183sBYrdNfe6+7FZrHuM4qBeuqpURGTaUAdw98eAx8bNu79s+hDwe9UtbWZaNVKjiEg0riiFYr+6zlMXkXoXnVDXSI0iItEJdY3UKCISoVAvjdTorkG9RKR+RSbU29IpcgVnYESDeolI/YpMqLcEV5X26apSEaljkQl1jdQoIhKhUG/RSI0iItEJ9dGWui5AEpE6FplQL43UqNMaRaSeRSbU5zUmiZla6iJS3yIT6rFYaVAvhbqI1K/IhDpoqAARkYiFekrnqYtIXYtWqGukRhGpc9EKdXW/iEidi1aoNxe7XzSol4jUq2iFejpFJl9gKJOvdSkiIjURqVBvKw0VoHPVRaRORSrUSyM19uuqUhGpU5EK9dL4LxrUS0TqVaRCvTRSY79CXUTqVKRCfbSlrj51EalTFYW6ma03s11mttvMPj3JOuvMbJuZ7TCzn1W3zMrMb0pippEaRaR+JaZbwcziwFeBdwDdwBYz2+zuL5Wt0wLcB6x39/1mduks1TuleMyY35TUSI0iUrcqaamvAXa7+x53zwAPA7ePW+cO4FF33w/g7kerW2blWjVSo4jUsUpCfTFwoOx1dzCv3OuBVjN70sy2mtmHqlXgTGmoABGpZ9N2vwA2wbzx1+EngJuAtwFNwM/N7Fl3f3nMG5ndBdwF0NHRMfNqK9CaTnH4xPCsvLeIyMWukpZ6N7C07PUS4NAE6/zI3Qfd/RjwFHDD+Ddy903u3unune3t7eda85Q0UqOI1LNKQn0LsNLMVphZCtgAbB63zveAW8wsYWZpYC2ws7qlVkbdLyJSz6btfnH3nJndAzwOxIEH3X2Hmd0dLL/f3Xea2Y+A7UABeMDdX5zNwifT2pxiOFvgdCZPUypeixJERGqmkj513P0x4LFx8+4f9/pPgT+tXmnnpjW4qrRvKENTqqnG1YiIXFiRuqIUzoS6rioVkXoUwVDXSI0iUr8iF+oaqVFE6lnkQl0jNYpIPYtgqBe7X9SnLiL1KHKhnozHmNuYUJ+6iNSlyIU6FPvV1VIXkXoUyVBv0UiNIlKnIhnqbRoqQETqVCRDvTWdom9QfeoiUn+iGeoaqVFE6lQ0Qz2dZCiTZzibr3UpIiIXVDRDvbl0AZK6YESkvkQz1MtGahQRqSfRDnWdqy4idSaaod5cHCqgT90vIlJnIhnqbWmN1Cgi9SmSoT46UqO6X0SkzkQy1FOJGHMaEmqpi0jdiWSoQ3EIXp3SKCL1JrKhrpEaRaQeRTbUW9Ip3f1IROpOZEO9LZ1Un7qI1J3IhnpLOkW/RmoUkTpTUaib2Xoz22Vmu83s01Ost9rM8mb2b6tX4rlpa05xaiRHJleodSkiIhfMtKFuZnHgq8CtwDXARjO7ZpL1/gfweLWLPBetwQ2o+0+rC0ZE6kclLfU1wG533+PuGeBh4PYJ1vsk8F3gaBXrO2elkRp1swwRqSeVhPpi4EDZ6+5g3igzWwy8F7h/qjcys7vMrMvMunp6emZa64xopEYRqUeVhLpNMM/Hvf5z4F53n/KuFO6+yd073b2zvb29whLPjUZqFJF6lKhgnW5gadnrJcChcet0Ag+bGcAC4DYzy7n731ejyHOhkRpFpB5VEupbgJVmtgI4CGwA7ihfwd1XlKbN7OvAD2oZ6KDuFxGpT9OGurvnzOweime1xIEH3X2Hmd0dLJ+yH71WGpNxmpJxdb+ISF2ppKWOuz8GPDZu3oRh7u4fOf+yqqOtOcVxhbqI1JHIXlEK8MZF83j6Nz1k87oASUTqQ6RDfcOapRwbyPCTnUdqXYqIyAUR6VB/y8p2Lp/XyEPPHZh+ZRGRCIh0qCfiMT7QuYSnftNDd99QrcsREZl1kQ51gA+sLp5i/0hXd40rERGZfZEP9SWtaW5Z2c7fdh0gXxh/IayISLREPtQBNq5eyuETw/zs5YtirDERkVlTF6H+tqsvY8GclA6Yikjk1UWopxIx3nfTEn7666McPTlc63JERGZNXYQ6wIbVHeQLzt9u1QFTEYmuugn1FQuaWbuije9sOUBBB0xFJKLqJtQBNq7pYH/vED/fc7zWpYiIzIq6CvX1117O/KYkDz23v9aliIjMiroK9cZknPeuWswTO47Qq9EbRSSC6irUodgFk8kXePR5HTAVkeipu1B/w+VzWdXRwkPP7cddB0xFJFrqLtQBNq7u4JWeQbr29dW6FBGRqqrLUH/X9QuZ05DQAVMRiZy6DPXmhgTvuWERj71wmBOns7UuR0Skauoy1AE2rlnKcLbA5m0Ha12KiEjV1G2oX7d4PtcsnMdDzx3QAVMRiYy6DXUzY+Oapbx0+CQvHDxR63JERKqibkMd4PZVi2lMxjQkr4hERl2H+rzGJO+6bhGbtx1kcCRX63JERM5bRaFuZuvNbJeZ7TazT0+w/E4z2x48njGzG6pf6uzYuGYpg5k8P9h+qNaliIict2lD3cziwFeBW4FrgI1mds241fYCb3X364HPA5uqXehsuWlZK6+7dI66YEQkEippqa8Bdrv7HnfPAA8Dt5ev4O7PuHvp8sxngSXVLXP2mBkbVi9l24F+dh4+WetyRETOSyWhvhgob8Z2B/Mm84fADydaYGZ3mVmXmXX19PRUXuUse9+NS0glYnzrF/tqXYqIyHmpJNRtgnkTnthtZr9DMdTvnWi5u29y905372xvb6+8ylnW2pzi3dct5O+eP8iADpiKSIhVEurdwNKy10uAs44qmtn1wAPA7e4eulsL3XnzMgYzeb6nK0xFJMQqCfUtwEozW2FmKWADsLl8BTPrAB4F/sDdX65+mbPvxo4Wrrp8Ln/zrIbkFZHwmjbU3T0H3AM8DuwEHnH3HWZ2t5ndHaz2J8AlwH1mts3Mumat4lliZnzw5mXsPHySXx7or3U5IiLnxGrVKu3s7PSurosr+wdGcqz9wj+y/tqFfPkDoTnVXkTqiJltdffOyZbX9RWl481pSPD7qxbzg+2H6B/SPUxFJHwU6uPcuXYZI7kC/2+r7mEqIuGjUB/nmkXzuLGjhW//QgdMRSR8FOoT+ODNy9hzbJCfvxK6MzNFpM4p1Cdw23ULaUkn+dYvdA9TEQkXhfoEGpNx3n/TEh7f8RpHTw7XuhwRkYop1Cdxx9pl5ArOI10avVFEwkOhPokVC5p58+sW8NBzB8gXdMBURMJBoT6FO9d2cLD/NE/uOlrrUkREKqJQn8Lbr7mMS+c28DfPakheEQkHhfoUkvEYG1Yv5cmXezjQO1TrckREpqVQn8aGNR0Y8NBzOr1RRC5+CvVpLGpp4nevuoxHug6QyRVqXY6IyJQU6hX44M0dHBvI8PiO12pdiojIlBTqFXjLynaWtjXpHqYictFTqFcgFjPuWLOMZ/f0svvoqVqXIyIyKYV6hd7fuYRk3DQejIhc1BTqFVowp4Fbr13Id7d2czqTr3U5IiITUqjPwJ1rOzg5nOOvn9lLNn/hz4QZzub5598c45f7+9h/fIhTw1mN+S4iYyRqXUCYrFnRxqqOFr70o1088PRebr32ct5zwyJWL28jHrNZ+9xCwdn8q0N86Ue/5tCJsaNGpuIx2ppTZz0uaU5x2bxGFrU0sbi1iYXzG2lMxmetRhG5OOjG0zM0nM3z1Ms9fH/7Yf7xpSOczua5dG4D77p+Ie+5YRGrlrZgVr2Af25vL1/4h5f4VfcJ3rhoHp/83ZWkEsbxgQy9gxl6hzL0lk8PFl+fGsmd9V4L5jSwuCUI+pam0cC/YkEzKy+bW7WaZ0MmV+DIyWGWtqVrXYpITU1342mF+nkYyuT4yc6jfP9Xh3hyVw+ZfIElrU28+/pFvPv6hbxx0bxzDvi9xwb54g938viOI1w+r5H//M438N5Vi4lV+I1gJJfn6MkRDvaf5mDfaQ71ny5OB49D/acZzp7pQrppWSt/dMsK3nHN5bP6rWOmdh89xXe2HODR5w9yfDDD6uWt/Id1r2PdG9qr+sdTJCwU6hfIyeEsT+w4wg+2H+Lp3xwjX3CWX5Lm5isu4cZlrXQua2XFguZpg6h/KMNf/mQ3//fZV0nGY3z8rVfysVuuoClV3a4Td6d3MMOh/mG2vNrLXz+zlwO9p+loS/Pvf3s57+9cSnNDbXrnhjI5/mH7Yb6z5QBd+/pIxIy3X30Z1y6ex7d/sZ9DJ4a5euE8Pr7uSt513cKL6o+QyGxTqNdA72CGH734Gj9+6TW27uvj5HCxK6StOcWNHa10Li+G/LWL54/2c2dyBb7581f5Xz/dzanhLP9u9VL+0ztez6VzGy9IzfmC88SO1/irp/fw/P5+5jcluWNtBx/5V8u5bN7s1+DuvHDwBA9vOcDmbYcYGMlxRXszG1Yv5b2rltA+twEo/py+t+0g9//sFV7pGWTZJWn++C1X8r6bFtOQ0DEDib6qhLqZrQf+AogDD7j7F8ctt2D5bcAQ8BF3f36q94xyqJcrFJxXegbo2tdH16t9PL+/j73HBoHiQc5rF8/jhqUt/PTXR9l3fIi3vL6dz9x2FVddPq9mNW/d18cDT+/h8R2vEY8Z77lhEX90yxVcvbC6NWXzBY4NjPDEjiM8vOUAOw+fpDEZ47brFrJxTQedy1on/WZTKDhPvPQa9z35Ctu7T3Dp3AY+dssK7li7jDk1+oYhciGcd6ibWRx4GXgH0A1sATa6+0tl69wGfJJiqK8F/sLd1071vvUS6hM5NjDC1n19PL+vj659fbzQfYLlC9J85rarWfeGS2td3qj9x4d48F/28kjXAYYyed78ugXcfEUbiXiMRMyKj3iMZNyIx4rPiVhstDukf6h48LZvMEPvYJa+4EBu6fnU8JmDudctns8HVi/lX9+wiPlNyYprdHf+Zfdx7ntyN8+8cpz5TUk+/FvLuGl5G5lcofjI50enR3IFMvnCmWXBvJFcvvicLZsuzc8Wp+MxoyERozEZH31uTAbPieJ0QzJOYyJGMh4b/dkk46XXRip4TgbLUvE4qUSMhkSMVPAoTTfE4zQkY6TixTOPBzM5Tg7nODWc5eTp4HnMdPE5ZsachgTNwWNOQ7xsOkFzqvjckIxxOpNnMJMLnvOczuQYKpseHMlzOpvHDBoTcZpSxe0rbnv5NgfTiTjxmBGPQcyMeMxGn+MxwwziwevRn08sVvGxomy+wOBIjlPDxcfASI6BkezotGFjf4ajz8V9VnodjxnD2QLD2TzD2eI2ll6fzuYZKZuXSsSY05BgbmPxMachGTwnRp8T8bFnh7s7+YKTyRfI5orPxekC2XyB+enkOX8Lr0ao/xbwOXd/Z/D6vwRF//eydf438KS7PxS83gWsc/fDk71vPYf6eLl8IfiFvzj7hk8MZfn2c/v5xjOv8to53Ii7IRHjkuYUrWWnXLamS89JVnUUu6LO1y/39/G1J1/hiZeOVLR+MgjZhuSZ//ANiWKQNpQHQRCseS+e/TSSK/7nHykFQS4fhENhdHktNCZjzGlIUnBnYCRXtVFFGxIx3CEzi9dmxINGwtg/esXQBxgYyTMwkh1zcP9i0hT8gcvlnZF8MbinitaPr7uSe9dfdU6fNV2oV/I9dTFQfvflboqt8enWWQyMCXUzuwu4C6Cjo6OCj64P4//KX2zmp5N8fN2V3P3WK8jmiy2QbKFALu/kSs+l6YKP/kK3NqdoS6eqfpB3Mqs6Wtn0oU72Hx+iZ2CYVFlLN1XWEk7Fi49KW4czVWqlZfPFn1M2V/y5ZHJnfj7Z4NtCNu+j3yZGssXW3Ei2wEi+9C2i+C2jUHDmNhZbiPOakswrmy61IMcfUyi1agdGii3u4nNudF4mX6ApGSedSpBOxYNH2XRDgqZkfPSbV77gjOTyY1q45X/USt9y8gUouFMIfg75Qmka8u4Ugnm5QnH7s/ni71A2aM2Ony64l7WMk8xpSDCnMcHc0nNpXtDtlsmN/bZV/nMs/XzzBaex7FtH8RtInKZU8Y95Uyo++o0smy8wMFz8ljQwUvw2NDCc41TwjWEg+IY0nMuTjJ/5HStNJ+MxUnE7M52I8bpL58zK7x5UFuoT/eaP/xtUyTq4+yZgExRb6hV8tlxEzIxUorirm7h4D0p2XJKm45Lanc9uZiTiRiJe259TMh6jJZ2iJZ2qyvvFYxaEflXeLjSS8RjpVIJLa3eYa0YqaSJ2A0vLXi8BDp3DOiIiMssqCfUtwEozW2FmKWADsHncOpuBD1nRzcCJqfrTRURkdkzb/eLuOTO7B3ic4imND7r7DjO7O1h+P/AYxTNfdlM8pfGjs1eyiIhMpqITet39MYrBXT7v/rJpBz5R3dJERGSmLu7TLkREZEYU6iIiEaJQFxGJEIW6iEiE1GyURjPrAfad4z9fAByrYjkXg6htU9S2B6K3TVHbHojeNk20PcvcvX2yf1CzUD8fZtY11dgHYRS1bYra9kD0tilq2wPR26Zz2R51v4iIRIhCXUQkQsIa6ptqXcAsiNo2RW17IHrbFLXtgeht04y3J5R96iIiMrGwttRFRGQCCnURkQgJXaib2Xoz22Vmu83s07WupxrM7FUze8HMtplZ6O7xZ2YPmtlRM3uxbF6bmf3YzH4TPLfWssaZmmSbPmdmB4P9tC24N28omNlSM/snM9tpZjvM7FPB/FDupym2J8z7qNHMnjOzXwXb9F+D+TPaR6HqU6/kJthhZGavAp3uHsqLJszsLcAA8E13vzaY9yWg192/GPzxbXX3e2tZ50xMsk2fAwbc/c9qWdu5MLOFwEJ3f97M5gJbgd8HPkII99MU2/MBwruPDGh29wEzSwL/DHwK+DfMYB+FraW+Btjt7nvcPQM8DNxe45rqnrs/BfSOm3078I1g+hsU/8OFxiTbFFruftjdnw+mTwE7Kd5HOJT7aYrtCS0vGgheJoOHM8N9FLZQn+wG12HnwBNmtjW4OXcUXFa6+1XwfGmN66mWe8xse9A9E4quivHMbDmwCvgFEdhP47YHQryPzCxuZtuAo8CP3X3G+yhsoV7RDa5D6Lfd/UbgVuATwVd/ufh8DbgSeBNwGPhyTas5B2Y2B/gu8B/d/WSt6zlfE2xPqPeRu+fd/U0U7/O8xsyunel7hC3UI3mDa3c/FDwfBf6OYjdT2B0J+j1L/Z9Ha1zPeXP3I8F/ugLwV4RsPwX9tN8FvuXujwazQ7ufJtqesO+jEnfvB54E1jPDfRS2UK/kJtihYmbNwYEezKwZ+D3gxan/VShsBj4cTH8Y+F4Na6mK0n+swHsJ0X4KDsL9H2Cnu//PskWh3E+TbU/I91G7mbUE003A24FfM8N9FKqzXwCCU5T+nDM3wf5CbSs6P2Z2BcXWORTvGfvtsG2TmT0ErKM4TOgR4LPA3wOPAB3AfuD97h6aA4+TbNM6il/rHXgV+ONSX+fFzszeDDwNvAAUgtmfodgPHbr9NMX2bCS8++h6igdC4xQb3I+4+38zs0uYwT4KXaiLiMjkwtb9IiIiU1Coi4hEiEJdRCRCFOoiIhGiUBcRiRCFuohIhCjURUQi5P8DZ5BDT8DZAIQAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(tr_loss_hist)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}