grp14_final_submission.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5ZsFDG9BGzQ6"
      },
      "source": [
        "# **Relationship between Self-Attention and Convolution**\n",
        "This notebook intends to provide an empirical evidence for the theoretical proof concerning the relationship between Self-Attention layers and Convolutional layers. The idea for this work has been adopted from the original paper titled: [**On the Relationship between Self-Attention and Convolutional Layers**](https://arxiv.org/pdf/1911.03584.pdf) (by Cordonnier et al, ICLR 2020).\n",
        "<br>\n",
        "We have performed a moderately complex task of Time-Series Classification in this notebook, as of yet. Our final objective is, to give an empirical evidence of why Self-Attention is capable of replacing Convolution without any significant reduction in performance. More details about the dataset can be found later in this notebook."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8s1C93BYGzQ9"
      },
      "source": [
        "### Importing Libraries"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "wAl75RNgGzQ9"
      },
      "outputs": [],
      "source": [
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "import os, sys, random\n",
        "from sklearn.model_selection import train_test_split"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "-Q3_2ZenGzQ_"
      },
      "outputs": [],
      "source": [
        "%matplotlib inline"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "id": "fzXxf6pkGzRA"
      },
      "outputs": [],
      "source": [
        "from torch.utils.data import Dataset, TensorDataset, DataLoader"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "id": "mUqvA8daGzRA"
      },
      "outputs": [],
      "source": [
        "import models, utils"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "id": "tBQ-k0sdGzRB"
      },
      "outputs": [],
      "source": [
        "from torchsummary import summary"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iENb-vswGzRB"
      },
      "source": [
        "### Dataset Implementation"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "id": "zGDiTSC0GzRB"
      },
      "outputs": [],
      "source": [
        "raw_data_dir = './whole_data'"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LOQGzZn3GzRC",
        "outputId": "244bc0c3-58ad-4056-9baa-16d2d30ea7e9"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "69\n"
          ]
        }
      ],
      "source": [
        "subject_list = os.listdir(raw_data_dir)\n",
        "print(len(subject_list))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "id": "2mdLAF2kGzRC"
      },
      "outputs": [],
      "source": [
        "random.seed(1)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 270
        },
        "id": "eVY_BRZbGzRD",
        "outputId": "5df1304a-a218-4137-8d54-d9d2b5b9c39c"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "     AB_I_O  AB_PHI_O   AB_I_DO  AB_PHI_DO    CD_I_O  CD_PHI_O   CD_I_DO  \\\n",
              "0  0.275960  0.051303  0.091763  -0.044598  0.014088  0.283515  0.328312   \n",
              "1  0.247073  0.041880  0.087941  -0.035670 -0.040156  0.444357  0.333307   \n",
              "2  0.219966  0.037286  0.084897  -0.030853 -0.089000  0.616255  0.336260   \n",
              "3  0.195418  0.038564  0.082599  -0.029996 -0.131033  0.792589  0.336908   \n",
              "4  0.174229  0.046410  0.080965  -0.032637 -0.165057  0.967083  0.334975   \n",
              "\n",
              "   CD_PHI_DO  label  \n",
              "0   0.088877      0  \n",
              "1   0.057275      0  \n",
              "2   0.022443      0  \n",
              "3  -0.014349      0  \n",
              "4  -0.051811      0  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-62bab382-1288-4986-ab15-790149b8eb9f\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>AB_I_O</th>\n",
              "      <th>AB_PHI_O</th>\n",
              "      <th>AB_I_DO</th>\n",
              "      <th>AB_PHI_DO</th>\n",
              "      <th>CD_I_O</th>\n",
              "      <th>CD_PHI_O</th>\n",
              "      <th>CD_I_DO</th>\n",
              "      <th>CD_PHI_DO</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0.275960</td>\n",
              "      <td>0.051303</td>\n",
              "      <td>0.091763</td>\n",
              "      <td>-0.044598</td>\n",
              "      <td>0.014088</td>\n",
              "      <td>0.283515</td>\n",
              "      <td>0.328312</td>\n",
              "      <td>0.088877</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0.247073</td>\n",
              "      <td>0.041880</td>\n",
              "      <td>0.087941</td>\n",
              "      <td>-0.035670</td>\n",
              "      <td>-0.040156</td>\n",
              "      <td>0.444357</td>\n",
              "      <td>0.333307</td>\n",
              "      <td>0.057275</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0.219966</td>\n",
              "      <td>0.037286</td>\n",
              "      <td>0.084897</td>\n",
              "      <td>-0.030853</td>\n",
              "      <td>-0.089000</td>\n",
              "      <td>0.616255</td>\n",
              "      <td>0.336260</td>\n",
              "      <td>0.022443</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0.195418</td>\n",
              "      <td>0.038564</td>\n",
              "      <td>0.082599</td>\n",
              "      <td>-0.029996</td>\n",
              "      <td>-0.131033</td>\n",
              "      <td>0.792589</td>\n",
              "      <td>0.336908</td>\n",
              "      <td>-0.014349</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0.174229</td>\n",
              "      <td>0.046410</td>\n",
              "      <td>0.080965</td>\n",
              "      <td>-0.032637</td>\n",
              "      <td>-0.165057</td>\n",
              "      <td>0.967083</td>\n",
              "      <td>0.334975</td>\n",
              "      <td>-0.051811</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-62bab382-1288-4986-ab15-790149b8eb9f')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-62bab382-1288-4986-ab15-790149b8eb9f button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-62bab382-1288-4986-ab15-790149b8eb9f');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ],
      "source": [
        "subject_df = pd.read_csv(os.path.join(raw_data_dir, random.choice(subject_list)))\n",
        "subject_df.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UKLHyP12GzRD",
        "outputId": "4a4115f1-5aa7-4fe6-847a-9bf8256af071"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0    1704\n",
              "1    1704\n",
              "2    1704\n",
              "3    1704\n",
              "Name: label, dtype: int64"
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ],
      "source": [
        "# checking if the dataset is balanced\n",
        "subject_df.iloc[:, -1].value_counts()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "id": "AoSZVFNmGzRD"
      },
      "outputs": [],
      "source": [
        "# for a window_size = 150 (~30s) and stride = 3 (~0.6s)\n",
        "# sampling frequency for the data = 5.2Hz (Ts~0.2s)\n",
        "slide_window_data = utils.extract_sliding_window(subject_df)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 270
        },
        "id": "Fbs441J8GzRE",
        "outputId": "2a7682fe-0cd4-44db-e358-94b3a5589910"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "     AB_I_O  AB_PHI_O   AB_I_DO  AB_PHI_DO    CD_I_O  CD_PHI_O   CD_I_DO  \\\n",
              "0  0.251090 -0.127196  0.014890   0.027926 -0.227493  0.191236  0.015056   \n",
              "1  0.239895 -0.095349  0.015898  -0.000577 -0.231132  0.161360  0.016941   \n",
              "2  0.232039 -0.067809  0.016425  -0.031034 -0.234026  0.159570  0.019698   \n",
              "3  0.227504 -0.045332  0.016339  -0.062451 -0.236548  0.187071  0.023307   \n",
              "4  0.226260 -0.028519  0.015534  -0.093858 -0.238990  0.243260  0.027707   \n",
              "\n",
              "   CD_PHI_DO  label  window  \n",
              "0  -0.232111      0       0  \n",
              "1  -0.215199      0       0  \n",
              "2  -0.205297      0       0  \n",
              "3  -0.203764      0       0  \n",
              "4  -0.211319      0       0  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-2457e528-3a31-4325-892f-d60ddd48ae28\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>AB_I_O</th>\n",
              "      <th>AB_PHI_O</th>\n",
              "      <th>AB_I_DO</th>\n",
              "      <th>AB_PHI_DO</th>\n",
              "      <th>CD_I_O</th>\n",
              "      <th>CD_PHI_O</th>\n",
              "      <th>CD_I_DO</th>\n",
              "      <th>CD_PHI_DO</th>\n",
              "      <th>label</th>\n",
              "      <th>window</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0.251090</td>\n",
              "      <td>-0.127196</td>\n",
              "      <td>0.014890</td>\n",
              "      <td>0.027926</td>\n",
              "      <td>-0.227493</td>\n",
              "      <td>0.191236</td>\n",
              "      <td>0.015056</td>\n",
              "      <td>-0.232111</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0.239895</td>\n",
              "      <td>-0.095349</td>\n",
              "      <td>0.015898</td>\n",
              "      <td>-0.000577</td>\n",
              "      <td>-0.231132</td>\n",
              "      <td>0.161360</td>\n",
              "      <td>0.016941</td>\n",
              "      <td>-0.215199</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0.232039</td>\n",
              "      <td>-0.067809</td>\n",
              "      <td>0.016425</td>\n",
              "      <td>-0.031034</td>\n",
              "      <td>-0.234026</td>\n",
              "      <td>0.159570</td>\n",
              "      <td>0.019698</td>\n",
              "      <td>-0.205297</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0.227504</td>\n",
              "      <td>-0.045332</td>\n",
              "      <td>0.016339</td>\n",
              "      <td>-0.062451</td>\n",
              "      <td>-0.236548</td>\n",
              "      <td>0.187071</td>\n",
              "      <td>0.023307</td>\n",
              "      <td>-0.203764</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0.226260</td>\n",
              "      <td>-0.028519</td>\n",
              "      <td>0.015534</td>\n",
              "      <td>-0.093858</td>\n",
              "      <td>-0.238990</td>\n",
              "      <td>0.243260</td>\n",
              "      <td>0.027707</td>\n",
              "      <td>-0.211319</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-2457e528-3a31-4325-892f-d60ddd48ae28')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-2457e528-3a31-4325-892f-d60ddd48ae28 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-2457e528-3a31-4325-892f-d60ddd48ae28');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 12
        }
      ],
      "source": [
        "slide_window_data.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lJzu_49OGzRE",
        "outputId": "13cc77c5-3817-4397-83ec-4c246dbfa523"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Number of windows (size=150, stride=3) in the dataset: 1216\n"
          ]
        }
      ],
      "source": [
        "num_windows = len(slide_window_data.iloc[:, -1].unique())\n",
        "print(\"Number of windows (size=150, stride=3) in the dataset: {}\".format(num_windows))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2QBCr_VOGzRE",
        "outputId": "6b10c859-7788-4912-d382-c4389e1e4a32"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "shape features: (1216, 150, 8), shape labels: (1216,)\n"
          ]
        }
      ],
      "source": [
        "features, labels = utils.data_loading_function(slide_window_data)\n",
        "print(\"shape features: {}, shape labels: {}\".format(features.shape, labels.shape))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {
        "id": "ew1f853RGzRF"
      },
      "outputs": [],
      "source": [
        "# train-val-test split (70/15/15)\n",
        "trainval_features, test_features = train_test_split(features, test_size=0.15)\n",
        "trainval_labels, test_labels = train_test_split(labels, test_size=0.15)\n",
        "train_features, val_features = train_test_split(trainval_features, test_size=0.15)\n",
        "train_labels, val_labels = train_test_split(trainval_labels, test_size=0.15)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6XyYGDsXGzRF",
        "outputId": "a1fd3fc7-9801-4923-a75f-a201e5844928"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "shape train: (878, 150, 8), shape validation: (155, 150, 8), shape test: (183, 150, 8)\n"
          ]
        }
      ],
      "source": [
        "print(\"shape train: {}, shape validation: {}, shape test: {}\".format(\n",
        "    train_features.shape, val_features.shape, test_features.shape\n",
        "))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 17,
      "metadata": {
        "id": "hhwRROz_GzRF"
      },
      "outputs": [],
      "source": [
        "train_feature_tensor = torch.Tensor(train_features)\n",
        "train_label_tensor = torch.Tensor(train_labels)\n",
        "val_feature_tensor = torch.Tensor(val_features)\n",
        "val_label_tensor = torch.Tensor(val_labels)\n",
        "test_feature_tensor = torch.Tensor(test_features)\n",
        "test_label_tensor = torch.Tensor(test_labels)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 18,
      "metadata": {
        "id": "mNYJlVKwGzRG"
      },
      "outputs": [],
      "source": [
        "trainset = TensorDataset(train_feature_tensor, train_label_tensor)\n",
        "valset = TensorDataset(val_feature_tensor, val_label_tensor)\n",
        "testset = TensorDataset(test_feature_tensor, test_label_tensor)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5HX7omzLGzRG"
      },
      "source": [
        "### Model Implementation\n",
        "Transformer classifier and Deep Convnet"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 19,
      "metadata": {
        "id": "u7MT8TVWGzRH"
      },
      "outputs": [],
      "source": [
        "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 20,
      "metadata": {
        "id": "Jw-wGLvUGzRH"
      },
      "outputs": [],
      "source": [
        "num_features = 8\n",
        "num_classes = 4\n",
        "hidden_size = 100"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 21,
      "metadata": {
        "id": "AnmO6jKXGzRI"
      },
      "outputs": [],
      "source": [
        "%%capture\n",
        "convnet = models.ConvNet(window_size=150, num_features=num_features, num_classes=num_classes)\n",
        "tfclassifier = models.TransformerClassifier(num_labels=num_classes, embed_size=num_features, \n",
        "                                            hidden_size=hidden_size, device=device)\n",
        "convnet.to(device)\n",
        "tfclassifier.to(device)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 22,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1rBbysafGzRI",
        "outputId": "6766c776-914e-4736-c1c9-8da51c3c4471"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "----------------------------------------------------------------\n",
            "        Layer (type)               Output Shape         Param #\n",
            "================================================================\n",
            "            Conv2d-1           [-1, 25, 8, 143]             225\n",
            "            Conv2d-2           [-1, 25, 1, 143]           5,000\n",
            "       BatchNorm2d-3           [-1, 25, 1, 143]              50\n",
            "              ReLU-4           [-1, 25, 1, 143]               0\n",
            "         MaxPool2d-5            [-1, 25, 1, 71]               0\n",
            "           Dropout-6            [-1, 25, 1, 71]               0\n",
            "            Conv2d-7            [-1, 50, 1, 64]          10,000\n",
            "       BatchNorm2d-8            [-1, 50, 1, 64]             100\n",
            "              ReLU-9            [-1, 50, 1, 64]               0\n",
            "        MaxPool2d-10            [-1, 50, 1, 32]               0\n",
            "          Dropout-11            [-1, 50, 1, 32]               0\n",
            "           Conv2d-12            [-1, 50, 1, 25]          20,000\n",
            "      BatchNorm2d-13            [-1, 50, 1, 25]             100\n",
            "             ReLU-14            [-1, 50, 1, 25]               0\n",
            "        MaxPool2d-15            [-1, 50, 1, 12]               0\n",
            "           Linear-16                    [-1, 4]           2,404\n",
            "================================================================\n",
            "Total params: 37,879\n",
            "Trainable params: 37,879\n",
            "Non-trainable params: 0\n",
            "----------------------------------------------------------------\n",
            "Input size (MB): 0.00\n",
            "Forward/backward pass size (MB): 0.46\n",
            "Params size (MB): 0.14\n",
            "Estimated Total Size (MB): 0.61\n",
            "----------------------------------------------------------------\n"
          ]
        }
      ],
      "source": [
        "summary(convnet, (150, 8))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 23,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6zvlitiiGzRI",
        "outputId": "743ddf5c-defc-41ef-8424-014e71b97d0f"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "----------------------------------------------------------------\n",
            "        Layer (type)               Output Shape         Param #\n",
            "================================================================\n",
            "           Dropout-1               [-1, 150, 8]               0\n",
            "            Linear-2            [-1, 150, 2, 4]              16\n",
            "            Linear-3            [-1, 150, 2, 4]              16\n",
            "            Linear-4            [-1, 150, 2, 4]              16\n",
            "            Linear-5               [-1, 150, 8]              72\n",
            "     SelfAttention-6               [-1, 150, 8]               0\n",
            "         LayerNorm-7               [-1, 150, 8]              16\n",
            "           Dropout-8               [-1, 150, 8]               0\n",
            "            Linear-9              [-1, 150, 16]             144\n",
            "             ReLU-10              [-1, 150, 16]               0\n",
            "           Linear-11               [-1, 150, 8]             136\n",
            "        LayerNorm-12               [-1, 150, 8]              16\n",
            "          Dropout-13               [-1, 150, 8]               0\n",
            " TransformerBlock-14               [-1, 150, 8]               0\n",
            "           Linear-15            [-1, 150, 2, 4]              16\n",
            "           Linear-16            [-1, 150, 2, 4]              16\n",
            "           Linear-17            [-1, 150, 2, 4]              16\n",
            "           Linear-18               [-1, 150, 8]              72\n",
            "    SelfAttention-19               [-1, 150, 8]               0\n",
            "        LayerNorm-20               [-1, 150, 8]              16\n",
            "          Dropout-21               [-1, 150, 8]               0\n",
            "           Linear-22              [-1, 150, 16]             144\n",
            "             ReLU-23              [-1, 150, 16]               0\n",
            "           Linear-24               [-1, 150, 8]             136\n",
            "        LayerNorm-25               [-1, 150, 8]              16\n",
            "          Dropout-26               [-1, 150, 8]               0\n",
            " TransformerBlock-27               [-1, 150, 8]               0\n",
            "TransformerEncoder-28               [-1, 150, 8]               0\n",
            "          Flatten-29                 [-1, 1200]               0\n",
            "           Linear-30                  [-1, 100]         120,100\n",
            "             ReLU-31                  [-1, 100]               0\n",
            "           Linear-32                    [-1, 4]             404\n",
            "================================================================\n",
            "Total params: 121,368\n",
            "Trainable params: 121,368\n",
            "Non-trainable params: 0\n",
            "----------------------------------------------------------------\n",
            "Input size (MB): 0.00\n",
            "Forward/backward pass size (MB): 0.30\n",
            "Params size (MB): 0.46\n",
            "Estimated Total Size (MB): 0.77\n",
            "----------------------------------------------------------------\n"
          ]
        }
      ],
      "source": [
        "summary(tfclassifier, (150, 8))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MEOQRiHCGzRJ"
      },
      "source": [
        "### Training and Validation"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 24,
      "metadata": {
        "id": "aV20O0_VGzRJ"
      },
      "outputs": [],
      "source": [
        "import trainer"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 25,
      "metadata": {
        "id": "CM0xUFCHGzRJ"
      },
      "outputs": [],
      "source": [
        "batch_size = 32\n",
        "num_epochs = 200\n",
        "learn_rate = 0.01"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 26,
      "metadata": {
        "id": "YLwJOq9vGzRJ"
      },
      "outputs": [],
      "source": [
        "trainloader = DataLoader(trainset, batch_size=batch_size, shuffle=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 27,
      "metadata": {
        "id": "c5w5EAtdGzRJ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "0d678286-1383-490b-de3a-cdcb20468865"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch [0/200], Step[1/28], Loss: 1.4900\n",
            "Epoch [0/200], Step[2/28], Loss: 2.2834\n",
            "Epoch [0/200], Step[3/28], Loss: 3.0367\n",
            "Epoch [0/200], Step[4/28], Loss: 1.9285\n",
            "Epoch [0/200], Step[5/28], Loss: 1.8117\n",
            "Epoch [0/200], Step[6/28], Loss: 2.1847\n",
            "Epoch [0/200], Step[7/28], Loss: 1.6045\n",
            "Epoch [0/200], Step[8/28], Loss: 1.5799\n",
            "Epoch [0/200], Step[9/28], Loss: 2.2196\n",
            "Epoch [0/200], Step[10/28], Loss: 1.5826\n",
            "Epoch [0/200], Step[11/28], Loss: 1.8525\n",
            "Epoch [0/200], Step[12/28], Loss: 1.7473\n",
            "Epoch [0/200], Step[13/28], Loss: 1.7258\n",
            "Epoch [0/200], Step[14/28], Loss: 1.5589\n",
            "Epoch [0/200], Step[15/28], Loss: 1.7782\n",
            "Epoch [0/200], Step[16/28], Loss: 1.6048\n",
            "Epoch [0/200], Step[17/28], Loss: 1.5904\n",
            "Epoch [0/200], Step[18/28], Loss: 1.4936\n",
            "Epoch [0/200], Step[19/28], Loss: 1.6224\n",
            "Epoch [0/200], Step[20/28], Loss: 1.6571\n",
            "Epoch [0/200], Step[21/28], Loss: 1.6696\n",
            "Epoch [0/200], Step[22/28], Loss: 1.5881\n",
            "Epoch [0/200], Step[23/28], Loss: 1.4875\n",
            "Epoch [0/200], Step[24/28], Loss: 1.4532\n",
            "Epoch [0/200], Step[25/28], Loss: 1.7910\n",
            "Epoch [0/200], Step[26/28], Loss: 1.3710\n",
            "Epoch [0/200], Step[27/28], Loss: 1.5689\n",
            "Epoch [0/200], Step[28/28], Loss: 1.2133\n",
            "Epoch [10/200], Step[1/28], Loss: 1.2363\n",
            "Epoch [10/200], Step[2/28], Loss: 1.2952\n",
            "Epoch [10/200], Step[3/28], Loss: 1.2383\n",
            "Epoch [10/200], Step[4/28], Loss: 1.2451\n",
            "Epoch [10/200], Step[5/28], Loss: 1.2493\n",
            "Epoch [10/200], Step[6/28], Loss: 1.1686\n",
            "Epoch [10/200], Step[7/28], Loss: 1.4197\n",
            "Epoch [10/200], Step[8/28], Loss: 1.4635\n",
            "Epoch [10/200], Step[9/28], Loss: 1.3118\n",
            "Epoch [10/200], Step[10/28], Loss: 1.2942\n",
            "Epoch [10/200], Step[11/28], Loss: 1.2565\n",
            "Epoch [10/200], Step[12/28], Loss: 1.3529\n",
            "Epoch [10/200], Step[13/28], Loss: 1.3719\n",
            "Epoch [10/200], Step[14/28], Loss: 1.1851\n",
            "Epoch [10/200], Step[15/28], Loss: 1.3398\n",
            "Epoch [10/200], Step[16/28], Loss: 1.2877\n",
            "Epoch [10/200], Step[17/28], Loss: 1.2721\n",
            "Epoch [10/200], Step[18/28], Loss: 1.3361\n",
            "Epoch [10/200], Step[19/28], Loss: 1.3442\n",
            "Epoch [10/200], Step[20/28], Loss: 1.2887\n",
            "Epoch [10/200], Step[21/28], Loss: 1.2654\n",
            "Epoch [10/200], Step[22/28], Loss: 1.3593\n",
            "Epoch [10/200], Step[23/28], Loss: 1.2331\n",
            "Epoch [10/200], Step[24/28], Loss: 1.2615\n",
            "Epoch [10/200], Step[25/28], Loss: 1.2213\n",
            "Epoch [10/200], Step[26/28], Loss: 1.3467\n",
            "Epoch [10/200], Step[27/28], Loss: 1.3101\n",
            "Epoch [10/200], Step[28/28], Loss: 1.2800\n",
            "Epoch [20/200], Step[1/28], Loss: 1.0616\n",
            "Epoch [20/200], Step[2/28], Loss: 1.0299\n",
            "Epoch [20/200], Step[3/28], Loss: 1.1365\n",
            "Epoch [20/200], Step[4/28], Loss: 1.1862\n",
            "Epoch [20/200], Step[5/28], Loss: 1.1654\n",
            "Epoch [20/200], Step[6/28], Loss: 1.2157\n",
            "Epoch [20/200], Step[7/28], Loss: 1.2899\n",
            "Epoch [20/200], Step[8/28], Loss: 1.0499\n",
            "Epoch [20/200], Step[9/28], Loss: 0.9173\n",
            "Epoch [20/200], Step[10/28], Loss: 1.0795\n",
            "Epoch [20/200], Step[11/28], Loss: 1.2940\n",
            "Epoch [20/200], Step[12/28], Loss: 1.1795\n",
            "Epoch [20/200], Step[13/28], Loss: 1.1063\n",
            "Epoch [20/200], Step[14/28], Loss: 1.0307\n",
            "Epoch [20/200], Step[15/28], Loss: 1.2205\n",
            "Epoch [20/200], Step[16/28], Loss: 1.2912\n",
            "Epoch [20/200], Step[17/28], Loss: 1.1253\n",
            "Epoch [20/200], Step[18/28], Loss: 1.4409\n",
            "Epoch [20/200], Step[19/28], Loss: 1.3545\n",
            "Epoch [20/200], Step[20/28], Loss: 1.1999\n",
            "Epoch [20/200], Step[21/28], Loss: 1.2326\n",
            "Epoch [20/200], Step[22/28], Loss: 1.1012\n",
            "Epoch [20/200], Step[23/28], Loss: 1.2161\n",
            "Epoch [20/200], Step[24/28], Loss: 1.1698\n",
            "Epoch [20/200], Step[25/28], Loss: 1.1602\n",
            "Epoch [20/200], Step[26/28], Loss: 1.1540\n",
            "Epoch [20/200], Step[27/28], Loss: 1.1553\n",
            "Epoch [20/200], Step[28/28], Loss: 1.2221\n",
            "Epoch [30/200], Step[1/28], Loss: 0.9368\n",
            "Epoch [30/200], Step[2/28], Loss: 1.1578\n",
            "Epoch [30/200], Step[3/28], Loss: 0.8240\n",
            "Epoch [30/200], Step[4/28], Loss: 0.9492\n",
            "Epoch [30/200], Step[5/28], Loss: 1.0743\n",
            "Epoch [30/200], Step[6/28], Loss: 1.1322\n",
            "Epoch [30/200], Step[7/28], Loss: 0.9743\n",
            "Epoch [30/200], Step[8/28], Loss: 0.9512\n",
            "Epoch [30/200], Step[9/28], Loss: 1.1158\n",
            "Epoch [30/200], Step[10/28], Loss: 1.0572\n",
            "Epoch [30/200], Step[11/28], Loss: 1.1591\n",
            "Epoch [30/200], Step[12/28], Loss: 1.1668\n",
            "Epoch [30/200], Step[13/28], Loss: 1.0348\n",
            "Epoch [30/200], Step[14/28], Loss: 1.1648\n",
            "Epoch [30/200], Step[15/28], Loss: 1.2085\n",
            "Epoch [30/200], Step[16/28], Loss: 1.0577\n",
            "Epoch [30/200], Step[17/28], Loss: 1.0664\n",
            "Epoch [30/200], Step[18/28], Loss: 1.2435\n",
            "Epoch [30/200], Step[19/28], Loss: 1.0733\n",
            "Epoch [30/200], Step[20/28], Loss: 1.1272\n",
            "Epoch [30/200], Step[21/28], Loss: 1.0188\n",
            "Epoch [30/200], Step[22/28], Loss: 0.9881\n",
            "Epoch [30/200], Step[23/28], Loss: 1.3987\n",
            "Epoch [30/200], Step[24/28], Loss: 1.1819\n",
            "Epoch [30/200], Step[25/28], Loss: 1.1401\n",
            "Epoch [30/200], Step[26/28], Loss: 1.1431\n",
            "Epoch [30/200], Step[27/28], Loss: 0.9771\n",
            "Epoch [30/200], Step[28/28], Loss: 1.1562\n",
            "Epoch [40/200], Step[1/28], Loss: 0.9802\n",
            "Epoch [40/200], Step[2/28], Loss: 0.8456\n",
            "Epoch [40/200], Step[3/28], Loss: 0.9766\n",
            "Epoch [40/200], Step[4/28], Loss: 0.9474\n",
            "Epoch [40/200], Step[5/28], Loss: 0.8538\n",
            "Epoch [40/200], Step[6/28], Loss: 1.0576\n",
            "Epoch [40/200], Step[7/28], Loss: 0.8959\n",
            "Epoch [40/200], Step[8/28], Loss: 0.9649\n",
            "Epoch [40/200], Step[9/28], Loss: 0.9710\n",
            "Epoch [40/200], Step[10/28], Loss: 1.0084\n",
            "Epoch [40/200], Step[11/28], Loss: 0.9623\n",
            "Epoch [40/200], Step[12/28], Loss: 0.9046\n",
            "Epoch [40/200], Step[13/28], Loss: 0.9779\n",
            "Epoch [40/200], Step[14/28], Loss: 1.0122\n",
            "Epoch [40/200], Step[15/28], Loss: 1.2948\n",
            "Epoch [40/200], Step[16/28], Loss: 1.0632\n",
            "Epoch [40/200], Step[17/28], Loss: 0.9165\n",
            "Epoch [40/200], Step[18/28], Loss: 1.1415\n",
            "Epoch [40/200], Step[19/28], Loss: 1.1488\n",
            "Epoch [40/200], Step[20/28], Loss: 1.1219\n",
            "Epoch [40/200], Step[21/28], Loss: 0.9590\n",
            "Epoch [40/200], Step[22/28], Loss: 1.0035\n",
            "Epoch [40/200], Step[23/28], Loss: 0.9945\n",
            "Epoch [40/200], Step[24/28], Loss: 1.0818\n",
            "Epoch [40/200], Step[25/28], Loss: 1.2429\n",
            "Epoch [40/200], Step[26/28], Loss: 1.0905\n",
            "Epoch [40/200], Step[27/28], Loss: 1.1186\n",
            "Epoch [40/200], Step[28/28], Loss: 1.3024\n",
            "Epoch [50/200], Step[1/28], Loss: 1.0787\n",
            "Epoch [50/200], Step[2/28], Loss: 0.7351\n",
            "Epoch [50/200], Step[3/28], Loss: 1.0372\n",
            "Epoch [50/200], Step[4/28], Loss: 0.9144\n",
            "Epoch [50/200], Step[5/28], Loss: 1.0032\n",
            "Epoch [50/200], Step[6/28], Loss: 0.7340\n",
            "Epoch [50/200], Step[7/28], Loss: 0.9151\n",
            "Epoch [50/200], Step[8/28], Loss: 0.9236\n",
            "Epoch [50/200], Step[9/28], Loss: 0.9741\n",
            "Epoch [50/200], Step[10/28], Loss: 0.9966\n",
            "Epoch [50/200], Step[11/28], Loss: 0.9780\n",
            "Epoch [50/200], Step[12/28], Loss: 1.0509\n",
            "Epoch [50/200], Step[13/28], Loss: 0.9398\n",
            "Epoch [50/200], Step[14/28], Loss: 1.0418\n",
            "Epoch [50/200], Step[15/28], Loss: 1.0023\n",
            "Epoch [50/200], Step[16/28], Loss: 0.9537\n",
            "Epoch [50/200], Step[17/28], Loss: 1.1516\n",
            "Epoch [50/200], Step[18/28], Loss: 1.0864\n",
            "Epoch [50/200], Step[19/28], Loss: 0.9024\n",
            "Epoch [50/200], Step[20/28], Loss: 0.9422\n",
            "Epoch [50/200], Step[21/28], Loss: 0.8628\n",
            "Epoch [50/200], Step[22/28], Loss: 1.2810\n",
            "Epoch [50/200], Step[23/28], Loss: 1.0503\n",
            "Epoch [50/200], Step[24/28], Loss: 1.1317\n",
            "Epoch [50/200], Step[25/28], Loss: 0.9140\n",
            "Epoch [50/200], Step[26/28], Loss: 0.9617\n",
            "Epoch [50/200], Step[27/28], Loss: 0.9110\n",
            "Epoch [50/200], Step[28/28], Loss: 1.1423\n",
            "Epoch [60/200], Step[1/28], Loss: 0.8845\n",
            "Epoch [60/200], Step[2/28], Loss: 0.7432\n",
            "Epoch [60/200], Step[3/28], Loss: 0.8690\n",
            "Epoch [60/200], Step[4/28], Loss: 0.5910\n",
            "Epoch [60/200], Step[5/28], Loss: 0.9459\n",
            "Epoch [60/200], Step[6/28], Loss: 0.8408\n",
            "Epoch [60/200], Step[7/28], Loss: 0.8228\n",
            "Epoch [60/200], Step[8/28], Loss: 0.8956\n",
            "Epoch [60/200], Step[9/28], Loss: 0.9253\n",
            "Epoch [60/200], Step[10/28], Loss: 1.2948\n",
            "Epoch [60/200], Step[11/28], Loss: 0.9732\n",
            "Epoch [60/200], Step[12/28], Loss: 0.9539\n",
            "Epoch [60/200], Step[13/28], Loss: 1.0416\n",
            "Epoch [60/200], Step[14/28], Loss: 0.8385\n",
            "Epoch [60/200], Step[15/28], Loss: 0.8845\n",
            "Epoch [60/200], Step[16/28], Loss: 0.9417\n",
            "Epoch [60/200], Step[17/28], Loss: 0.9403\n",
            "Epoch [60/200], Step[18/28], Loss: 0.9540\n",
            "Epoch [60/200], Step[19/28], Loss: 0.9936\n",
            "Epoch [60/200], Step[20/28], Loss: 1.0223\n",
            "Epoch [60/200], Step[21/28], Loss: 0.9944\n",
            "Epoch [60/200], Step[22/28], Loss: 0.9726\n",
            "Epoch [60/200], Step[23/28], Loss: 0.9025\n",
            "Epoch [60/200], Step[24/28], Loss: 1.0836\n",
            "Epoch [60/200], Step[25/28], Loss: 0.8143\n",
            "Epoch [60/200], Step[26/28], Loss: 0.9777\n",
            "Epoch [60/200], Step[27/28], Loss: 1.1204\n",
            "Epoch [60/200], Step[28/28], Loss: 1.0650\n",
            "Epoch [70/200], Step[1/28], Loss: 0.8856\n",
            "Epoch [70/200], Step[2/28], Loss: 0.8720\n",
            "Epoch [70/200], Step[3/28], Loss: 0.7991\n",
            "Epoch [70/200], Step[4/28], Loss: 0.6083\n",
            "Epoch [70/200], Step[5/28], Loss: 0.9158\n",
            "Epoch [70/200], Step[6/28], Loss: 0.9179\n",
            "Epoch [70/200], Step[7/28], Loss: 0.8649\n",
            "Epoch [70/200], Step[8/28], Loss: 0.8787\n",
            "Epoch [70/200], Step[9/28], Loss: 0.9289\n",
            "Epoch [70/200], Step[10/28], Loss: 0.6468\n",
            "Epoch [70/200], Step[11/28], Loss: 0.9053\n",
            "Epoch [70/200], Step[12/28], Loss: 0.8201\n",
            "Epoch [70/200], Step[13/28], Loss: 0.8184\n",
            "Epoch [70/200], Step[14/28], Loss: 0.7133\n",
            "Epoch [70/200], Step[15/28], Loss: 0.7671\n",
            "Epoch [70/200], Step[16/28], Loss: 0.7162\n",
            "Epoch [70/200], Step[17/28], Loss: 0.8598\n",
            "Epoch [70/200], Step[18/28], Loss: 1.1036\n",
            "Epoch [70/200], Step[19/28], Loss: 0.7885\n",
            "Epoch [70/200], Step[20/28], Loss: 0.8814\n",
            "Epoch [70/200], Step[21/28], Loss: 0.8439\n",
            "Epoch [70/200], Step[22/28], Loss: 0.7780\n",
            "Epoch [70/200], Step[23/28], Loss: 0.7216\n",
            "Epoch [70/200], Step[24/28], Loss: 0.7809\n",
            "Epoch [70/200], Step[25/28], Loss: 1.0229\n",
            "Epoch [70/200], Step[26/28], Loss: 0.8379\n",
            "Epoch [70/200], Step[27/28], Loss: 0.9469\n",
            "Epoch [70/200], Step[28/28], Loss: 0.7616\n",
            "Epoch [80/200], Step[1/28], Loss: 0.5730\n",
            "Epoch [80/200], Step[2/28], Loss: 0.5726\n",
            "Epoch [80/200], Step[3/28], Loss: 0.7344\n",
            "Epoch [80/200], Step[4/28], Loss: 0.7942\n",
            "Epoch [80/200], Step[5/28], Loss: 0.8853\n",
            "Epoch [80/200], Step[6/28], Loss: 0.9018\n",
            "Epoch [80/200], Step[7/28], Loss: 0.6647\n",
            "Epoch [80/200], Step[8/28], Loss: 0.5407\n",
            "Epoch [80/200], Step[9/28], Loss: 0.8109\n",
            "Epoch [80/200], Step[10/28], Loss: 0.7317\n",
            "Epoch [80/200], Step[11/28], Loss: 0.8505\n",
            "Epoch [80/200], Step[12/28], Loss: 0.5542\n",
            "Epoch [80/200], Step[13/28], Loss: 0.7766\n",
            "Epoch [80/200], Step[14/28], Loss: 0.8036\n",
            "Epoch [80/200], Step[15/28], Loss: 1.0016\n",
            "Epoch [80/200], Step[16/28], Loss: 0.6041\n",
            "Epoch [80/200], Step[17/28], Loss: 0.9248\n",
            "Epoch [80/200], Step[18/28], Loss: 1.0165\n",
            "Epoch [80/200], Step[19/28], Loss: 1.0163\n",
            "Epoch [80/200], Step[20/28], Loss: 0.8427\n",
            "Epoch [80/200], Step[21/28], Loss: 0.8308\n",
            "Epoch [80/200], Step[22/28], Loss: 0.7906\n",
            "Epoch [80/200], Step[23/28], Loss: 0.8227\n",
            "Epoch [80/200], Step[24/28], Loss: 0.8494\n",
            "Epoch [80/200], Step[25/28], Loss: 0.7380\n",
            "Epoch [80/200], Step[26/28], Loss: 1.0956\n",
            "Epoch [80/200], Step[27/28], Loss: 0.6470\n",
            "Epoch [80/200], Step[28/28], Loss: 0.8233\n",
            "Epoch [90/200], Step[1/28], Loss: 0.7680\n",
            "Epoch [90/200], Step[2/28], Loss: 0.9246\n",
            "Epoch [90/200], Step[3/28], Loss: 0.8880\n",
            "Epoch [90/200], Step[4/28], Loss: 0.7262\n",
            "Epoch [90/200], Step[5/28], Loss: 0.8544\n",
            "Epoch [90/200], Step[6/28], Loss: 0.7957\n",
            "Epoch [90/200], Step[7/28], Loss: 0.7193\n",
            "Epoch [90/200], Step[8/28], Loss: 0.6764\n",
            "Epoch [90/200], Step[9/28], Loss: 0.5673\n",
            "Epoch [90/200], Step[10/28], Loss: 0.7761\n",
            "Epoch [90/200], Step[11/28], Loss: 0.9167\n",
            "Epoch [90/200], Step[12/28], Loss: 0.7579\n",
            "Epoch [90/200], Step[13/28], Loss: 0.7179\n",
            "Epoch [90/200], Step[14/28], Loss: 1.0798\n",
            "Epoch [90/200], Step[15/28], Loss: 0.6678\n",
            "Epoch [90/200], Step[16/28], Loss: 0.5438\n",
            "Epoch [90/200], Step[17/28], Loss: 0.7090\n",
            "Epoch [90/200], Step[18/28], Loss: 1.0824\n",
            "Epoch [90/200], Step[19/28], Loss: 0.7898\n",
            "Epoch [90/200], Step[20/28], Loss: 0.8487\n",
            "Epoch [90/200], Step[21/28], Loss: 0.8547\n",
            "Epoch [90/200], Step[22/28], Loss: 0.9363\n",
            "Epoch [90/200], Step[23/28], Loss: 0.8795\n",
            "Epoch [90/200], Step[24/28], Loss: 0.8259\n",
            "Epoch [90/200], Step[25/28], Loss: 1.0398\n",
            "Epoch [90/200], Step[26/28], Loss: 0.8668\n",
            "Epoch [90/200], Step[27/28], Loss: 0.7472\n",
            "Epoch [90/200], Step[28/28], Loss: 0.5718\n",
            "Epoch [100/200], Step[1/28], Loss: 0.7194\n",
            "Epoch [100/200], Step[2/28], Loss: 0.5890\n",
            "Epoch [100/200], Step[3/28], Loss: 0.8991\n",
            "Epoch [100/200], Step[4/28], Loss: 0.7672\n",
            "Epoch [100/200], Step[5/28], Loss: 0.6822\n",
            "Epoch [100/200], Step[6/28], Loss: 0.6133\n",
            "Epoch [100/200], Step[7/28], Loss: 0.7928\n",
            "Epoch [100/200], Step[8/28], Loss: 0.7966\n",
            "Epoch [100/200], Step[9/28], Loss: 0.9083\n",
            "Epoch [100/200], Step[10/28], Loss: 0.6956\n",
            "Epoch [100/200], Step[11/28], Loss: 1.0078\n",
            "Epoch [100/200], Step[12/28], Loss: 0.6991\n",
            "Epoch [100/200], Step[13/28], Loss: 0.8103\n",
            "Epoch [100/200], Step[14/28], Loss: 0.7197\n",
            "Epoch [100/200], Step[15/28], Loss: 0.8561\n",
            "Epoch [100/200], Step[16/28], Loss: 0.6451\n",
            "Epoch [100/200], Step[17/28], Loss: 0.8012\n",
            "Epoch [100/200], Step[18/28], Loss: 0.6777\n",
            "Epoch [100/200], Step[19/28], Loss: 0.7237\n",
            "Epoch [100/200], Step[20/28], Loss: 0.7327\n",
            "Epoch [100/200], Step[21/28], Loss: 0.9554\n",
            "Epoch [100/200], Step[22/28], Loss: 0.7882\n",
            "Epoch [100/200], Step[23/28], Loss: 0.9743\n",
            "Epoch [100/200], Step[24/28], Loss: 0.6516\n",
            "Epoch [100/200], Step[25/28], Loss: 0.5922\n",
            "Epoch [100/200], Step[26/28], Loss: 0.8538\n",
            "Epoch [100/200], Step[27/28], Loss: 0.7875\n",
            "Epoch [100/200], Step[28/28], Loss: 0.7052\n",
            "Epoch [110/200], Step[1/28], Loss: 0.8851\n",
            "Epoch [110/200], Step[2/28], Loss: 0.6240\n",
            "Epoch [110/200], Step[3/28], Loss: 0.5878\n",
            "Epoch [110/200], Step[4/28], Loss: 0.7313\n",
            "Epoch [110/200], Step[5/28], Loss: 0.6767\n",
            "Epoch [110/200], Step[6/28], Loss: 0.6502\n",
            "Epoch [110/200], Step[7/28], Loss: 0.5852\n",
            "Epoch [110/200], Step[8/28], Loss: 0.7013\n",
            "Epoch [110/200], Step[9/28], Loss: 0.7530\n",
            "Epoch [110/200], Step[10/28], Loss: 0.6369\n",
            "Epoch [110/200], Step[11/28], Loss: 0.7738\n",
            "Epoch [110/200], Step[12/28], Loss: 0.6049\n",
            "Epoch [110/200], Step[13/28], Loss: 0.7033\n",
            "Epoch [110/200], Step[14/28], Loss: 0.7463\n",
            "Epoch [110/200], Step[15/28], Loss: 0.7392\n",
            "Epoch [110/200], Step[16/28], Loss: 0.7902\n",
            "Epoch [110/200], Step[17/28], Loss: 0.6691\n",
            "Epoch [110/200], Step[18/28], Loss: 0.7792\n",
            "Epoch [110/200], Step[19/28], Loss: 1.0033\n",
            "Epoch [110/200], Step[20/28], Loss: 0.8691\n",
            "Epoch [110/200], Step[21/28], Loss: 0.8687\n",
            "Epoch [110/200], Step[22/28], Loss: 0.7167\n",
            "Epoch [110/200], Step[23/28], Loss: 0.6025\n",
            "Epoch [110/200], Step[24/28], Loss: 0.7911\n",
            "Epoch [110/200], Step[25/28], Loss: 0.7066\n",
            "Epoch [110/200], Step[26/28], Loss: 0.6798\n",
            "Epoch [110/200], Step[27/28], Loss: 0.7680\n",
            "Epoch [110/200], Step[28/28], Loss: 0.9258\n",
            "Epoch [120/200], Step[1/28], Loss: 0.6719\n",
            "Epoch [120/200], Step[2/28], Loss: 0.8344\n",
            "Epoch [120/200], Step[3/28], Loss: 0.3781\n",
            "Epoch [120/200], Step[4/28], Loss: 0.6946\n",
            "Epoch [120/200], Step[5/28], Loss: 0.6543\n",
            "Epoch [120/200], Step[6/28], Loss: 0.3849\n",
            "Epoch [120/200], Step[7/28], Loss: 0.7537\n",
            "Epoch [120/200], Step[8/28], Loss: 0.6633\n",
            "Epoch [120/200], Step[9/28], Loss: 0.6966\n",
            "Epoch [120/200], Step[10/28], Loss: 1.0318\n",
            "Epoch [120/200], Step[11/28], Loss: 1.0336\n",
            "Epoch [120/200], Step[12/28], Loss: 1.1259\n",
            "Epoch [120/200], Step[13/28], Loss: 0.7440\n",
            "Epoch [120/200], Step[14/28], Loss: 0.6221\n",
            "Epoch [120/200], Step[15/28], Loss: 0.8472\n",
            "Epoch [120/200], Step[16/28], Loss: 0.4073\n",
            "Epoch [120/200], Step[17/28], Loss: 0.5242\n",
            "Epoch [120/200], Step[18/28], Loss: 0.7995\n",
            "Epoch [120/200], Step[19/28], Loss: 0.6970\n",
            "Epoch [120/200], Step[20/28], Loss: 0.7749\n",
            "Epoch [120/200], Step[21/28], Loss: 0.6097\n",
            "Epoch [120/200], Step[22/28], Loss: 0.7198\n",
            "Epoch [120/200], Step[23/28], Loss: 1.0873\n",
            "Epoch [120/200], Step[24/28], Loss: 0.9724\n",
            "Epoch [120/200], Step[25/28], Loss: 0.8808\n",
            "Epoch [120/200], Step[26/28], Loss: 0.7606\n",
            "Epoch [120/200], Step[27/28], Loss: 1.0100\n",
            "Epoch [120/200], Step[28/28], Loss: 1.0197\n",
            "Epoch [130/200], Step[1/28], Loss: 0.7171\n",
            "Epoch [130/200], Step[2/28], Loss: 0.5661\n",
            "Epoch [130/200], Step[3/28], Loss: 0.6737\n",
            "Epoch [130/200], Step[4/28], Loss: 0.5542\n",
            "Epoch [130/200], Step[5/28], Loss: 0.7214\n",
            "Epoch [130/200], Step[6/28], Loss: 0.7873\n",
            "Epoch [130/200], Step[7/28], Loss: 0.7135\n",
            "Epoch [130/200], Step[8/28], Loss: 0.8129\n",
            "Epoch [130/200], Step[9/28], Loss: 0.4285\n",
            "Epoch [130/200], Step[10/28], Loss: 0.5825\n",
            "Epoch [130/200], Step[11/28], Loss: 0.7889\n",
            "Epoch [130/200], Step[12/28], Loss: 0.6789\n",
            "Epoch [130/200], Step[13/28], Loss: 0.4008\n",
            "Epoch [130/200], Step[14/28], Loss: 0.8153\n",
            "Epoch [130/200], Step[15/28], Loss: 0.6105\n",
            "Epoch [130/200], Step[16/28], Loss: 0.9206\n",
            "Epoch [130/200], Step[17/28], Loss: 0.5041\n",
            "Epoch [130/200], Step[18/28], Loss: 0.9518\n",
            "Epoch [130/200], Step[19/28], Loss: 0.5219\n",
            "Epoch [130/200], Step[20/28], Loss: 0.8058\n",
            "Epoch [130/200], Step[21/28], Loss: 0.7831\n",
            "Epoch [130/200], Step[22/28], Loss: 0.7494\n",
            "Epoch [130/200], Step[23/28], Loss: 0.7535\n",
            "Epoch [130/200], Step[24/28], Loss: 0.9171\n",
            "Epoch [130/200], Step[25/28], Loss: 0.8177\n",
            "Epoch [130/200], Step[26/28], Loss: 0.6463\n",
            "Epoch [130/200], Step[27/28], Loss: 0.7652\n",
            "Epoch [130/200], Step[28/28], Loss: 0.7547\n",
            "Epoch [140/200], Step[1/28], Loss: 0.7176\n",
            "Epoch [140/200], Step[2/28], Loss: 0.5436\n",
            "Epoch [140/200], Step[3/28], Loss: 0.7742\n",
            "Epoch [140/200], Step[4/28], Loss: 0.6529\n",
            "Epoch [140/200], Step[5/28], Loss: 0.4717\n",
            "Epoch [140/200], Step[6/28], Loss: 0.8296\n",
            "Epoch [140/200], Step[7/28], Loss: 0.7953\n",
            "Epoch [140/200], Step[8/28], Loss: 0.6804\n",
            "Epoch [140/200], Step[9/28], Loss: 0.6521\n",
            "Epoch [140/200], Step[10/28], Loss: 0.7136\n",
            "Epoch [140/200], Step[11/28], Loss: 0.9638\n",
            "Epoch [140/200], Step[12/28], Loss: 0.3634\n",
            "Epoch [140/200], Step[13/28], Loss: 0.5701\n",
            "Epoch [140/200], Step[14/28], Loss: 0.7228\n",
            "Epoch [140/200], Step[15/28], Loss: 0.7832\n",
            "Epoch [140/200], Step[16/28], Loss: 0.7163\n",
            "Epoch [140/200], Step[17/28], Loss: 0.6657\n",
            "Epoch [140/200], Step[18/28], Loss: 0.6817\n",
            "Epoch [140/200], Step[19/28], Loss: 0.8354\n",
            "Epoch [140/200], Step[20/28], Loss: 0.7787\n",
            "Epoch [140/200], Step[21/28], Loss: 0.9358\n",
            "Epoch [140/200], Step[22/28], Loss: 0.5849\n",
            "Epoch [140/200], Step[23/28], Loss: 0.7526\n",
            "Epoch [140/200], Step[24/28], Loss: 0.5831\n",
            "Epoch [140/200], Step[25/28], Loss: 0.6762\n",
            "Epoch [140/200], Step[26/28], Loss: 0.5691\n",
            "Epoch [140/200], Step[27/28], Loss: 0.6812\n",
            "Epoch [140/200], Step[28/28], Loss: 0.8262\n",
            "Epoch [150/200], Step[1/28], Loss: 0.7250\n",
            "Epoch [150/200], Step[2/28], Loss: 0.4998\n",
            "Epoch [150/200], Step[3/28], Loss: 0.3861\n",
            "Epoch [150/200], Step[4/28], Loss: 0.5898\n",
            "Epoch [150/200], Step[5/28], Loss: 0.7399\n",
            "Epoch [150/200], Step[6/28], Loss: 0.6388\n",
            "Epoch [150/200], Step[7/28], Loss: 0.6837\n",
            "Epoch [150/200], Step[8/28], Loss: 0.7274\n",
            "Epoch [150/200], Step[9/28], Loss: 0.8598\n",
            "Epoch [150/200], Step[10/28], Loss: 0.7304\n",
            "Epoch [150/200], Step[11/28], Loss: 0.6844\n",
            "Epoch [150/200], Step[12/28], Loss: 0.5277\n",
            "Epoch [150/200], Step[13/28], Loss: 0.4195\n",
            "Epoch [150/200], Step[14/28], Loss: 1.0540\n",
            "Epoch [150/200], Step[15/28], Loss: 0.6195\n",
            "Epoch [150/200], Step[16/28], Loss: 0.7017\n",
            "Epoch [150/200], Step[17/28], Loss: 0.6906\n",
            "Epoch [150/200], Step[18/28], Loss: 0.5313\n",
            "Epoch [150/200], Step[19/28], Loss: 0.6039\n",
            "Epoch [150/200], Step[20/28], Loss: 0.6306\n",
            "Epoch [150/200], Step[21/28], Loss: 0.5617\n",
            "Epoch [150/200], Step[22/28], Loss: 0.7522\n",
            "Epoch [150/200], Step[23/28], Loss: 0.6614\n",
            "Epoch [150/200], Step[24/28], Loss: 0.7584\n",
            "Epoch [150/200], Step[25/28], Loss: 0.6892\n",
            "Epoch [150/200], Step[26/28], Loss: 0.8736\n",
            "Epoch [150/200], Step[27/28], Loss: 0.4758\n",
            "Epoch [150/200], Step[28/28], Loss: 0.9057\n",
            "Epoch [160/200], Step[1/28], Loss: 0.4717\n",
            "Epoch [160/200], Step[2/28], Loss: 0.4778\n",
            "Epoch [160/200], Step[3/28], Loss: 0.4462\n",
            "Epoch [160/200], Step[4/28], Loss: 0.3847\n",
            "Epoch [160/200], Step[5/28], Loss: 0.6797\n",
            "Epoch [160/200], Step[6/28], Loss: 0.8205\n",
            "Epoch [160/200], Step[7/28], Loss: 0.6360\n",
            "Epoch [160/200], Step[8/28], Loss: 0.6565\n",
            "Epoch [160/200], Step[9/28], Loss: 0.7034\n",
            "Epoch [160/200], Step[10/28], Loss: 0.6002\n",
            "Epoch [160/200], Step[11/28], Loss: 0.5661\n",
            "Epoch [160/200], Step[12/28], Loss: 0.5174\n",
            "Epoch [160/200], Step[13/28], Loss: 0.6248\n",
            "Epoch [160/200], Step[14/28], Loss: 0.6824\n",
            "Epoch [160/200], Step[15/28], Loss: 0.5634\n",
            "Epoch [160/200], Step[16/28], Loss: 0.5118\n",
            "Epoch [160/200], Step[17/28], Loss: 0.6547\n",
            "Epoch [160/200], Step[18/28], Loss: 0.9611\n",
            "Epoch [160/200], Step[19/28], Loss: 0.7369\n",
            "Epoch [160/200], Step[20/28], Loss: 0.9031\n",
            "Epoch [160/200], Step[21/28], Loss: 0.5710\n",
            "Epoch [160/200], Step[22/28], Loss: 0.5367\n",
            "Epoch [160/200], Step[23/28], Loss: 0.6440\n",
            "Epoch [160/200], Step[24/28], Loss: 0.6661\n",
            "Epoch [160/200], Step[25/28], Loss: 0.4941\n",
            "Epoch [160/200], Step[26/28], Loss: 0.7898\n",
            "Epoch [160/200], Step[27/28], Loss: 0.7635\n",
            "Epoch [160/200], Step[28/28], Loss: 0.4416\n",
            "Epoch [170/200], Step[1/28], Loss: 0.4387\n",
            "Epoch [170/200], Step[2/28], Loss: 0.7222\n",
            "Epoch [170/200], Step[3/28], Loss: 0.4522\n",
            "Epoch [170/200], Step[4/28], Loss: 0.4388\n",
            "Epoch [170/200], Step[5/28], Loss: 0.8027\n",
            "Epoch [170/200], Step[6/28], Loss: 0.5887\n",
            "Epoch [170/200], Step[7/28], Loss: 0.6707\n",
            "Epoch [170/200], Step[8/28], Loss: 0.6603\n",
            "Epoch [170/200], Step[9/28], Loss: 0.5441\n",
            "Epoch [170/200], Step[10/28], Loss: 0.6747\n",
            "Epoch [170/200], Step[11/28], Loss: 0.7312\n",
            "Epoch [170/200], Step[12/28], Loss: 0.9492\n",
            "Epoch [170/200], Step[13/28], Loss: 0.8629\n",
            "Epoch [170/200], Step[14/28], Loss: 0.7723\n",
            "Epoch [170/200], Step[15/28], Loss: 0.5111\n",
            "Epoch [170/200], Step[16/28], Loss: 0.5944\n",
            "Epoch [170/200], Step[17/28], Loss: 0.7759\n",
            "Epoch [170/200], Step[18/28], Loss: 0.7007\n",
            "Epoch [170/200], Step[19/28], Loss: 0.7063\n",
            "Epoch [170/200], Step[20/28], Loss: 0.7452\n",
            "Epoch [170/200], Step[21/28], Loss: 0.9572\n",
            "Epoch [170/200], Step[22/28], Loss: 0.7745\n",
            "Epoch [170/200], Step[23/28], Loss: 0.6970\n",
            "Epoch [170/200], Step[24/28], Loss: 0.7854\n",
            "Epoch [170/200], Step[25/28], Loss: 0.6244\n",
            "Epoch [170/200], Step[26/28], Loss: 0.4417\n",
            "Epoch [170/200], Step[27/28], Loss: 0.4482\n",
            "Epoch [170/200], Step[28/28], Loss: 0.6949\n",
            "Epoch [180/200], Step[1/28], Loss: 0.4979\n",
            "Epoch [180/200], Step[2/28], Loss: 0.6211\n",
            "Epoch [180/200], Step[3/28], Loss: 0.7277\n",
            "Epoch [180/200], Step[4/28], Loss: 0.4755\n",
            "Epoch [180/200], Step[5/28], Loss: 0.4907\n",
            "Epoch [180/200], Step[6/28], Loss: 0.4886\n",
            "Epoch [180/200], Step[7/28], Loss: 0.5552\n",
            "Epoch [180/200], Step[8/28], Loss: 0.3725\n",
            "Epoch [180/200], Step[9/28], Loss: 0.4605\n",
            "Epoch [180/200], Step[10/28], Loss: 0.8224\n",
            "Epoch [180/200], Step[11/28], Loss: 0.5755\n",
            "Epoch [180/200], Step[12/28], Loss: 0.5997\n",
            "Epoch [180/200], Step[13/28], Loss: 0.6915\n",
            "Epoch [180/200], Step[14/28], Loss: 0.3871\n",
            "Epoch [180/200], Step[15/28], Loss: 0.5343\n",
            "Epoch [180/200], Step[16/28], Loss: 0.4963\n",
            "Epoch [180/200], Step[17/28], Loss: 0.5054\n",
            "Epoch [180/200], Step[18/28], Loss: 0.7682\n",
            "Epoch [180/200], Step[19/28], Loss: 0.4437\n",
            "Epoch [180/200], Step[20/28], Loss: 0.7729\n",
            "Epoch [180/200], Step[21/28], Loss: 0.4501\n",
            "Epoch [180/200], Step[22/28], Loss: 0.5997\n",
            "Epoch [180/200], Step[23/28], Loss: 0.6121\n",
            "Epoch [180/200], Step[24/28], Loss: 0.7747\n",
            "Epoch [180/200], Step[25/28], Loss: 0.6727\n",
            "Epoch [180/200], Step[26/28], Loss: 0.6858\n",
            "Epoch [180/200], Step[27/28], Loss: 1.0377\n",
            "Epoch [180/200], Step[28/28], Loss: 0.8287\n",
            "Epoch [190/200], Step[1/28], Loss: 0.5390\n",
            "Epoch [190/200], Step[2/28], Loss: 0.6070\n",
            "Epoch [190/200], Step[3/28], Loss: 0.3959\n",
            "Epoch [190/200], Step[4/28], Loss: 0.3262\n",
            "Epoch [190/200], Step[5/28], Loss: 0.9102\n",
            "Epoch [190/200], Step[6/28], Loss: 0.6219\n",
            "Epoch [190/200], Step[7/28], Loss: 0.6540\n",
            "Epoch [190/200], Step[8/28], Loss: 0.5629\n",
            "Epoch [190/200], Step[9/28], Loss: 0.4481\n",
            "Epoch [190/200], Step[10/28], Loss: 0.8896\n",
            "Epoch [190/200], Step[11/28], Loss: 0.5867\n",
            "Epoch [190/200], Step[12/28], Loss: 0.7159\n",
            "Epoch [190/200], Step[13/28], Loss: 0.4583\n",
            "Epoch [190/200], Step[14/28], Loss: 0.4047\n",
            "Epoch [190/200], Step[15/28], Loss: 0.7792\n",
            "Epoch [190/200], Step[16/28], Loss: 0.7681\n",
            "Epoch [190/200], Step[17/28], Loss: 0.6841\n",
            "Epoch [190/200], Step[18/28], Loss: 0.6064\n",
            "Epoch [190/200], Step[19/28], Loss: 0.7387\n",
            "Epoch [190/200], Step[20/28], Loss: 0.4400\n",
            "Epoch [190/200], Step[21/28], Loss: 0.4118\n",
            "Epoch [190/200], Step[22/28], Loss: 0.6853\n",
            "Epoch [190/200], Step[23/28], Loss: 0.7058\n",
            "Epoch [190/200], Step[24/28], Loss: 0.5260\n",
            "Epoch [190/200], Step[25/28], Loss: 0.7213\n",
            "Epoch [190/200], Step[26/28], Loss: 0.5087\n",
            "Epoch [190/200], Step[27/28], Loss: 0.7531\n",
            "Epoch [190/200], Step[28/28], Loss: 0.8072\n"
          ]
        }
      ],
      "source": [
        "train_loss_history = trainer.train(convnet, trainloader, learning_rate=learn_rate,\n",
        "                                    num_epochs=num_epochs, batch_size=batch_size, device=device)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 28,
      "metadata": {
        "id": "W_6b0CT2GzRK",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 296
        },
        "outputId": "d8885353-4249-4108-c79a-3de99dca76e0"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<matplotlib.legend.Legend at 0x7f61954a3050>"
            ]
          },
          "metadata": {},
          "execution_count": 28
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEGCAYAAABy53LJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdd3xV9f348dc7N3uQkMFKgIS9ZxgyXCjiAlTctmpR26/a5Vdb2q911f5aW6uto1UUraOKE0VFRRwIyN5bQgIkYWWTve7798c9iSEJyWVcEuD9fDzyyD3nfM657+PgzWeLqmKMMcZ4y6+lAzDGGHNqscRhjDHmqFjiMMYYc1QscRhjjDkqljiMMcYcFf+WDuBkiI2N1cTExJYOwxhjTimrV6/OVtW4+ufPiMSRmJjIqlWrWjoMY4w5pYjI7sbO+7SpSkQmich2EUkRkRmNXA8Skbec68tFJNE5nygipSKyzvl5rs493zjPrLnWzpfvYIwx5nA+q3GIiAt4FrgQyABWishcVd1Sp9h0IE9Ve4jIdcBjwLXOtZ2qOuQIj79RVa0KYYwxLcCXNY6RQIqqpqpqBTAbmFKvzBTgFefzu8AEEREfxmSMMeY4+bKPIx5Ir3OcAYw6UhlVrRKRAiDGuZYkImuBQ8D9qrqozn0vi0g18B7wqNq6KcYcl8rKSjIyMigrK2vpUEwLCA4OJiEhgYCAAK/Kt9bO8X1AF1XNEZHhwAci0l9VD+FppsoUkQg8ieNHwKv1HyAidwB3AHTp0uUkhm7MqScjI4OIiAgSExOxSv+ZRVXJyckhIyODpKQkr+7xZVNVJtC5znGCc67RMiLiD0QCOaparqo5AKq6GtgJ9HKOM53fhcAbeJrEGlDVmaqarKrJcXENRpMZY+ooKysjJibGksYZSESIiYk5qtqmLxPHSqCniCSJSCBwHTC3Xpm5wM3O52nAV6qqIhLndK4jIt2AnkCqiPiLSKxzPgC4DNjkw3cw5oxhSePMdbT/7n3WVOX0WdwNfA64gJdUdbOIPAKsUtW5wCzgNRFJAXLxJBeAs4FHRKQScAM/U9VcEQkDPneShgtYALzgq3d45btdRIcFcvngTr76CmOMOeX4dB6Hqs5T1V6q2l1V/+Sce8BJGqhqmaperao9VHWkqqY6599T1f6qOkRVh6nqR875YlUdrqqDnOu/VNVqX8X/xvI9fLJhn68eb4xx5Ofn869//euY7r3kkkvIz89vsswDDzzAggULjun59SUmJpKdnX1CnnWqsrWqmhAc4EdZlc/ykjHG0VTiqKqqavLeefPmERUV1WSZRx55hAsuuOCY42vtmvtndKJZ4mhCkL+LskpLHMb42owZM9i5cydDhgzhvvvu45tvvmH8+PFMnjyZfv36ATB16lSGDx9O//79mTlzZu29NTWAXbt20bdvX26//Xb69+/PxIkTKS0tBeCWW27h3XffrS3/4IMPMmzYMAYOHMi2bdsAyMrK4sILL6R///7cdtttdO3atdmaxRNPPMGAAQMYMGAA//jHPwAoLi7m0ksvZfDgwQwYMIC33nqr9h379evHoEGDuPfeexs8q6ioiFtvvZWBAwcyaNAg3nvvPQDCw8Nry7z77rvccsstte/0s5/9jFGjRvGb3/yGxMTEw2pePXv25MCBA2RlZXHVVVcxYsQIRowYwZIlS7z/F3MErXU4bqsQFOBHYdnJzeTGtLSHP9rMlr2HTugz+3Vqw4OX9z/i9b/85S9s2rSJdevWAfDNN9+wZs0aNm3aVDtE9KWXXiI6OprS0lJGjBjBVVddRUxMzGHP2bFjB2+++SYvvPAC11xzDe+99x433XRTg++LjY1lzZo1/Otf/+Lxxx/nxRdf5OGHH+b888/nd7/7HZ999hmzZs1q8p1Wr17Nyy+/zPLly1FVRo0axTnnnENqaiqdOnXik08+AaCgoICcnBzmzJnDtm3bEJFGm9b++Mc/EhkZycaNGwHIy8tr8vvBM4z6u+++w+VyUV1dzZw5c7j11ltZvnw5Xbt2pX379txwww38+te/Zty4cezZs4eLLrqIrVu3NvvspliNownBAVbjMKaljBw58rB5BU899RSDBw9m9OjRpKens2PHjgb3JCUlMWSIZ6Wi4cOHs2vXrkaffeWVVzYos3jxYq67zjM+Z9KkSbRt27bJ+BYvXswVV1xBWFgY4eHhXHnllSxatIiBAwfyxRdf8Nvf/pZFixYRGRlJZGQkwcHBTJ8+nffff5/Q0NAGz1uwYAF33XVX7XFz3w9w9dVX43K5ALj22mtrazezZ8/m2muvrX3u3XffzZAhQ5g8eTKHDh2iqKio2Wc3xWocTQgOcFFR5W7pMIw5qZqqGZxMYWFhtZ+/+eYbFixYwNKlSwkNDeXcc89tdN5BUFBQ7WeXy1XbVHWkci6X64T3D/Tq1Ys1a9Ywb9487r//fiZMmMADDzzAihUr+PLLL3n33Xd55pln+Oqrr7x6Xt2hsvXfue4/o7POOouUlBSysrL44IMPuP/++wFwu90sW7aM4ODgE/B2HlbjaEKwv5/VOIw5CSIiIigsLDzi9YKCAtq2bUtoaCjbtm1j2bJlJzyGsWPH8vbbbwMwf/78ZpuKxo8fzwcffEBJSQnFxcXMmTOH8ePHs3fvXkJDQ7npppu47777WLNmDUVFRRQUFHDJJZfw5JNPsn79+gbPu/DCC3n22Wdrj2u+v3379mzduhW3282cOXOOGI+IcMUVV3DPPffQt2/f2ma8iRMn8vTTT9eWq2kOPB6WOJoQHOCizGocxvhcTEwMY8eOZcCAAdx3330Nrk+aNImqqir69u3LjBkzGD169AmP4cEHH2T+/PkMGDCAd955hw4dOhAREXHE8sOGDeOWW25h5MiRjBo1ittuu42hQ4eyceNGRo4cyZAhQ3j44Ye5//77KSws5LLLLmPQoEGMGzeOJ554osHz7r//fvLy8hgwYACDBw/m66+/Bjz9P5dddhljxoyhY8eOTb7Dtddey+uvv17bTAWeJr5Vq1YxaNAg+vXrx3PPPdfEE7wjZ8L6gMnJyXosGzn96ZMt/Hf5HrY8MskHURnTemzdupW+ffu2dBgtqry8HJfLhb+/P0uXLuV//ud/Tsjfzk8Vjf03ICKrVTW5flnr42hCzXBcVbXlGIw5ze3Zs4drrrkGt9tNYGAgL7zgs0UpTnmWOJoQHOCHW6GyWgn0t8RhzOmsZ8+erF27tqXDOCVYH0cTggM8w9xs9rg5E5wJzdamcUf7794SRxOCahKHjawyp7ng4GBycnIseZyBavbjOJrhutZU1YRgf09eLa+0kVXm9JaQkEBGRgZZWVktHYppATU7AHrLEkcTapqqyq2pypzmAgICvN79zRhrqmpCbR+H1TiMMaaWJY4mBDlNVdbHYYwxP7DE0QSrcRhjTEOWOJoQHGA1DmOMqc+niUNEJonIdhFJEZEZjVwPEpG3nOvLRSTROZ8oIqUiss75ea7OPcNFZKNzz1PiwyndNo/DGGMa8lniEBEX8CxwMdAPuF5E+tUrNh3IU9UewJPAY3Wu7XT2HB+iqj+rc/7fwO1AT+fHZwtJBfs7o6qsqcoYY2r5ssYxEkhR1VRVrQBmA1PqlZkCvOJ8fheY0FQNQkQ6Am1UdZl6Ziq9Ckw98aF71DZVWY3DGGNq+TJxxAPpdY4znHONllHVKqAAqNkLMklE1orIQhEZX6d8RjPPBEBE7hCRVSKy6lgnNQVZ57gxxjTQWjvH9wFdVHUocA/whoi0OZoHqOpMVU1W1eS4uLhjCsKG4xpjTEO+TByZQOc6xwnOuUbLiIg/EAnkqGq5quYAqOpqYCfQyylfd158Y888YYL8/RCBckscxhhTy5eJYyXQU0SSRCQQuA6YW6/MXOBm5/M04CtVVRGJczrXEZFueDrBU1V1H3BIREY7fSE/Bj701QuICEH+frYLoDHG1OGztapUtUpE7gY+B1zAS6q6WUQeAVap6lxgFvCaiKQAuXiSC8DZwCMiUgm4gZ+paq5z7U7gP0AI8Knz4zPBAS5rqjLGmDp8usihqs4D5tU790Cdz2XA1Y3c9x7w3hGeuQoYcGIjPbJgf0scxhhTV2vtHG81ggP8KLemKmOMqWWJoxnWVGWMMYezxNGMIH8/m8dhjDF1WOJoRpDVOIwx5jCWOJoRHOCy4bjGGFOHJY5mBPv72QRAY4ypwxJHM6xz3BhjDmeJoxnBAdY5bowxdVniaEZwgItyW1bdGGNqWeJohg3HNcaYw1niaIZnVFU1nn2jjDHGWOJoRnCAC1WoqLZahzHGgBeJQ0SuFpEI5/P9IvK+iAzzfWitww+bOVniMMYY8K7G8QdVLRSRccAFeJZC/7dvw2o9gp3tY20uhzHGeHiTOGr+xLwUmKmqnwCBvgupdQm2fceNMeYw3iSOTBF5HrgWmCciQV7ed1oIDvC8qg3JNcYYD28SwDV4dvG7SFXzgWjgPp9G1YqEBXr2uiosr2rhSIwxpnXwZgfAjsAnqlouIucCg4BXfRpVK9IhMhiA/QVlLRyJMca0Dt7UON4DqkWkBzAT6Ay84c3DRWSSiGwXkRQRmdHI9SARecu5vlxEEutd7yIiRSJyb51zu0Rko4isE5FV3sRxPDpFhgCwN7/U119ljDGnBG8Sh1tVq4ArgadV9T48tZAmiYgLeBa4GOgHXC8i/eoVmw7kqWoP4EngsXrXnwA+beTx56nqEFVN9iL+49ImxJ+wQBeZljiMMQbwLnFUisj1wI+Bj51zAV7cNxJIUdVUVa0AZgNT6pWZArzifH4XmCAiAiAiU4E0YLMX3+UzIkKnqBCrcRhjjMObxHErcBbwJ1VNE5Ek4DUv7osH0uscZzjnGi3j1GoKgBgRCQd+CzzcyHMVmC8iq0XkjiN9uYjcISKrRGRVVlaWF+EemSdxWB+HMcaAF4lDVbcA9wIbRWQAkKGq9ZuUTrSHgCdVtaiRa+NUdRieJrC7ROTsxh6gqjNVNVlVk+Pi4o4rGKtxGGPMD5odVeWMpHoF2AUI0FlEblbVb5u5NRNPR3qNBOdcY2UyRMQfiARygFHANBH5KxAFuEWkTFWfUdVMAFU9KCJz8DSJNRfLcekUGUxOcQVlldW1EwKNMeZM5c1w3L8DE1V1O4CI9ALeBIY3c99KoKfTtJUJXAfcUK/MXOBmYCkwDfhKPcvQjq8pICIPAUWq+oyIhAF+zhIoYcBE4BEv3uG4dIryjKzaV1BGUmyYr7/OGGNaNW8SR0BN0gBQ1e9FpNnOcVWtEpG78UwedAEvqepmEXkEWKWqc/Gse/WaiKQAuXiSS1PaA3Oc/nN/4A1V/cyLdzguNYljb36pJQ5jzBnPm8SxSkReBF53jm8EvJo/oarzgHn1zj1Q53MZcHUzz3iozudUYLA3330ixTuJw4bkGmOMd4njf4C7gF84x4uAf/ksolaofWQQIjYJ0BhjwIvEoarleCbiPeH7cFqnIH8XseFBljiMMYYmEoeIbMQzZ6JRqjrIJxG1Ul2iQ9my71BLh2GMMS2uqRrHZSctilPA5YM68tBHW9iQkc+ghKiWDscYY1rMEScAqurupn5OZpCtwZXDEwgNdPHq0jPu1Y0x5jBnzIZMx6tNcABXDotn7vq95BSVt3Q4xhjTYixxHIVbxiRR7VYen7+9+cLGGHOa8ipxiEiIiPT2dTCtXY924fxkbCJvrkhn9e7clg7HGGNaRLOJQ0QuB9YBnznHQ0Rkrq8Da61+dUEvOkUG839zNlFZ7W7pcIwx5qTzpsbxEJ6FBPMBVHUdkOTDmFq1sCB/Hprcn237C3lpcVpLh2OMMSedNzPHK1W1wFkfqsYR53ecCSb278CF/drzjwU7cCtcMrADXWNsDStjzJnBmxrHZhG5AXCJSE8ReRr4zsdxtXoPT+5PYmwYj322jWnPLaW8qpq1e/J4b3VGS4dmjDE+5U3i+DnQHyjHs5z6IeBXvgzqVNApKoRPfzmel28dQVZhOR+szeSXs9cx4/0NFJdXtXR4xhjjM96sVVUC/J/zY+o5t1ccPdqF88CHmymv8nSWr9iVy3m927VwZMYY4xtNrVX1EU2vVTXZJxGdYkSEm8/qyh8+3Mzwrm3ZmFnAdynZljiMMaetpmocj5+0KE5xVw1PYOWuPO48rzuPfLSFxSk5LR2SMcb4zBETh6ourPksIoFAHzw1kO2qWnESYjtlhAb689T1QwEY2yOWv32+neyicmLDg6h2K37iqZkYY8zpwJsJgJcCO4GngGeAFBG52NeBnarG9YgF4G+fbef5hTsZ+NDnvLxkV8sGZYwxJ5A3o6r+Dpynqueq6jnAecCT3jxcRCaJyHYRSRGRGY1cDxKRt5zry0Uksd71LiJSJCL3evvMljYwPpJrkhN4Z3U6f/50GwCzFqdR7T6jp74YY04j3kwALFTVlDrHqUBhczeJiAt4FrgQyABWishcVd1Sp9h0IE9Ve4jIdcBjwLV1rj8BfHqUz2xRfn7CX6cN5q7zepBdVM7e/DJ+/uZaFu3I4lzrMDfGnAa8qXGsEpF5InKLiNwMfITnD+wrReTKJu4bCaSoaqrTJzIbmFKvzBTgFefzu8AEcToDRGQqkAZsPspntgpdY8IY3jWai/p3ICYskDeW72npkIwx5oTwJnEEAweAc4BzgSwgBLicpncJjAfS6xxnOOcaLaOqVUABECMi4cBvgYeP4ZkAiMgdIrJKRFZlZWU1EaZvBfr7MW14Al9uO0husY0pMMac+ryZAHjryQiknoeAJ1W16FhHI6nqTGAmQHJycot2MFw6qCPPf5vKV9sOMm14QkuGYowxx63ZxCEiSXiWHUmsW96LCYCZQOc6xwnOucbKZIiIPxAJ5ACjgGki8lcgCnCLSBmw2otntjoD4yPp0CaY+Zv3W+IwxpzyvOkc/wCYhadv42g2oFgJ9HQSTyZwHXBDvTJzgZuBpcA04CtVVWB8TQEReQgoUtVnnOTS3DNbHRFhYv/2vL0qndKKakICXS0dkjHGHDNv+jjKVPUpVf1aVRfW/DR3k9NncTfwObAVeFtVN4vIIyJSU1uZhadPIwW4B2hyeO2RnunFO7S4if06UFbp5rVlu9hfUNbS4RhjzDETz1/wmyjgWVK9JzAfzwq5AKjqGt+GduIkJyfrqlWrWjSGymo3Z/35S7KLKgjy9+OTX4ynR7vwFo3JGGOaIiKrVTW5/nlvmqoGAj8CzueHpip1jo2XAlx+fPHrc9i6/xC3vbKKJxd8z7M3DGvpsIwx5qh5kziuBrrZ+lTHr21YIGO6x/KTsUk883UKd55bQP9OkS0dljHGHBVv+jg24RnZZE6Q28/uRmRIAP/z+hp2HGh2Er4xxrQq3iSOKGCbiHwuInNrfnwd2OksMiSAl28dQUlFNZc9vZjrZy5jSUp2S4dljDFe8aZz/JzGznszsqq1aA2d443Zm1/KC4tS+XTjfgL8hYX3noefny2/boxpHY65c/xUShCnmk5RITx4eX+GdI7il7PXsTQ1h7HOsuwAT8zfzqGyKh6a3L8FozTGmMN5sx/HaBFZ6SxvXiEi1SJy6GQEd6a4qH8HIkMCmL3yh2W4VJXZK9N5e1U6VdVHM+/SGGN8y5s+jmeA64EdeBY3vA3P0ubmBAkOcHHF0Hg+37SflINFAOzJLeFgYTklFdVs2Wd52hjTeniTOHD243CparWqvgxM8m1YZ55bxiQSHuzP1GeX8M32g6xIy629tnJXXgtGZowxh/MmcZQ4e46vE5G/isivvbzPHIXE2DA++vk4EtqGcO876/l2RzaRIQHER4WwMi2XLXsP8Z2NvDLGtALeJIAfOeXuBorxrE57lS+DOlPFR4Xw6NQBZBdV8NH6vSR3bcuopGiWpeXwo1nLufONNbYFrTGmxTWbOFR1t6qWqeoh4CngP/W2kjUnUHJiNON7ekZWjUiKJjkxmvySSnKKK8gvqWTz3oIWjtAYc6bzZlTVNyLSRkSigTXACyLyhO9DO3Pdd1FvYsMDOb9PO8b3jCU00MX9l/YFYHFKNtv3F/LZpv0tHKUx5kzlzQTAtao6VERuAzqr6oMiskFVB52cEI9fa50A6K1qt+LyEyb941vahASQU1TO7pwSlv9+AjHhQS0dnjHmNHWkCYDe9HH4i0hH4Brg4xMemWmWy5lNPq5HLCvSctmZVUyVW/lw3d4WjswYcybyJnE8gmfjpBRVXSki3fDM6TAn2Tin72NUUjQD4yN5d3VGC0dkjDkTNdtUdTo41ZuqapRXVfPAB5u5/ewkvtuZwwMfbqZfxzYE+PvxwZ1jELF1rowxJ87xNFXVfcgps+vf6SjI38Vj0wbRo10Elw/qRExYIAcLy1mfns+e3JKWDs8Yc4Y42ol8R/VXWhGZJCLbRSRFRBrsJy4iQSLylnN9uYgkOudHisg652e9iFxR555dIrLRuXbqVyOOUduwQFbdfwGv/GQEAOvS89mbX8r7a6z5yhjjW00mDhFxOTPFa3zi7YNFxIVnTauLgX7A9SLSr16x6UCeqvYAngQec85vApJVdQie5U2eF5G6K/mep6pDGqtCnUlEhN7tIwgO8GPtnnz+uWAH97y9nv0FZXy2aR/D//gFBSWVLR2mMeY002TiUNVqPAsc1hzffxTPHomnQz3V2XZ2NjClXpkpwCvO53eBCSIiqlqiqlXO+WA8e5ybRvi7/BgYH8maPXnM3+KZ27FmTx6fbz5ATnEF3+7IauEIjTGnG2+aqpaIyDMiMl5EhtX8eHFfPJBe5zjDOddoGSdRFAAxACIySkQ2AxuBn9VJJArMF5HVInLHkb5cRO4QkVUisior6/T+w3NI5yg2ZBSQ59Qu1uzOq10k8evtB1syNGPMaajZjZyAIc7vR+qcU+D8Ex9OnS9QXQ70F5G+wCsi8qmqlgHjVDVTRNoBX4jINlX9tpH7ZwIzwTOqypextrQhndsCaYQEuOgWF8anm/aTmV9KoL8f336fhduttrOgMeaE8WatqvMa+fEmaWTiWRCxRoJzrtEyTh9GJJBT7/u3AkXAAOc40/l9EJiDp0nsjDa4cyQA5/aOY0z3GDLzSwG4+ayuZBdVsMnWtzLGnEDerFXVXkRmicinznE/EZnuxbNXAj1FJMlZlv06YG69MnOBm53P04CvVFWde/yd7+sK9AF2iUiYiEQ458OAiXg60s9o8VEh/HJCT+46rwfDu7YFICLIn9vP7oYIfLnVmquMMSeON30c/8Ezc7yTc/w98KvmbnL6JO527t0KvK2qm0XkERGZ7BSbBcSISApwD1AzZHccsF5E1uGpVdypqtlAe2CxiKwHVgCfqOpnXrzDaU1E+PWFvRgQH8mwLp7EMTyxLe0ighnbPZb/Lt9DaUX1YffMXrGHeRv3tUS4xphTnDd9HLGq+raI/A48CUFEqpu7ySk7D5hX79wDdT6XAVc3ct9rwGuNnE8FBnvz3Weqdm2CuX5kZ87v0x6AX0zoyTXPL+X1Zbu5/exuAHy4LpMZ72+kR7twLhnYsSXDNcacgrxJHMUiEoMzJFZERuMZ/WRaqT9f+cPCxSOTohnXI5bnFu5k2vAE0nKKue/dDQT5+5FysIiCkkpeXJxKZl4pf79msC1bYoxpljdNVffg6YvoLiJLgFeBn/s0KnNC/XZSHwrLqrjlPyu549XVdGgTzN+v8VTcVu3O5dWlu3l/bSazV6Y38yRjjPFuVNUa4BxgDPBToL+qbvB1YObEGZgQyT+vG8KGjHzKK6uZdXMy5/Zuh5/ArMVpFJRWEhseyKMfb2GvMyLLGGOOxJtRVcHAL4A/Ag8DdznnzCnk4oEdeeXWkbx5x2h6to8gPMifXu0j+G5nDv5+wn9uHUlxRTWfbLAOc2NM07xpqnoV6A88DTzjfG7QcW1av7N7xTEgPrL2eJgzdHdUt2gGxEfSPS6MxSnZLRWeMeYU4U3iGKCq01X1a+fndjzJw5ziaobu1ozAGuvsMFhR5W7JsIwxrZw3iWONM5IK8KwhBZyxy5mfTib0acfUIZ2YMsQzRWdsj1hKK6tZuycP8Kxz9cxXttmjMeZw3iSO4cB3zj4Yu4ClwAhnTwzrJD+FtQ0L5B/XDSU2PAiA0d1i8BNY4jRXPf75dv6xYAflVV5N2zHGnCG8mccxyedRmFYhMiSAgQlRfL09iylD49m89xAAKQeL6N8pskH5l5ekMSghkuFdo092qMaYFuTNcNzdTf2cjCDNyTNteAIbMwv437fX157btq+wQbmKKjd//HgLt768kt05xSczRGNMCzvarWPNae6GkV3o17EN69LzGZHYliB/P7buO9SgXEZeCW6FQ2VV/PS11VRWW4e6MWcKSxzmMC4/4ZEp/fETuHp4Z3q1j2Db/kLeWZXODS8sq+3v2J1bAsCNo7qwbX8hq3fntWTYxpiTyKvEISJdReQC53NIzdLm5vSUnBjNdzMmcHVyAn07RrBl3yH+sWAH3+3M4eUluwDYk+NJHNPHJeHvJ3yz/fTeZdEY8wNvZo7fjmc/8OedUwnAB74MyrS8DpHBiAh9OrQht7iCzPxSOkUG88xXKRwsLGNXTjGhgS6SYsNITmzLN7ZFrTFnDG9qHHcBY4FDAKq6A2jny6BM69G3YxsAOkUG8+r0UZRWVvPqd7vZk1NCl+hQRIRze7dj2/5C9hXYOlfGnAm8SRzlqlpRc+DszHda7+FtftCvUxtCAlzcOjaJHu3CGdo5ikU7stidW0LXmFDAs2UtwGOfbuOJL77nkw37KC6vasmwjTE+5M08joUi8nsgREQuBO4EPvJtWKa1iAwJYNFvzyMmLBCAcT1j+eeXO/D3E87v46l49m4fQbfYMD5Yt7f2vpFJ0bx1x2jKKt2IQHCAi+cX7qS4vIp7JvZukXcxxpwY3tQ4ZgBZwEY8y6rPA+735uEiMklEtotIiojMaOR6kIi85VxfLiKJzvmRIrLO+VkvIld4+0xz4sWGB9Vu8DSuRyyqUFmttTUOEeHjX4xj/QMT2f7oJH53cR9WpOXy5op0Jv5jIVc/t5T03BIen7+dZ75OIbPe0u1z1+/lgQ/P+K3jjTlleDMB0K2qL6jq1ao6zfncbFOViC19YQIAACAASURBVLiAZ4GLgX7A9SLSr16x6UCeqvYAngQec85vApJVdQiemevPi4i/l880PjS4cxThQZ6KatfosNrzoYH+RIYGEOTvYvq4JLrFhfH7ORvZX1DGxswCrn1+aW3Z15cdPm/0+YU7eXXpbnKLKzDGtH7ejKraKCIb6v0sEpEnnS1lj2QkkKKqqU4fyWxgSr0yU4BXnM/vAhNERFS1RFVrGsmD+aFPxZtnGh8KcPkxuptniZGaGkd9/i4//nBZP9oE+/PvG4dzYb/27C0o49oRnbmgb3tmr9hDWaVnPsje/NLapU1WpOWenJcwxhwXb/o4PgWqgTec4+uAUGA/8B/g8iPcFw/U3Ys0Axh1pDKqWiUiBUAMkO2swvsS0BX4kXPdm2caH5s2PIGc4go6Rh55P6/zerdj7QMTcfkJAxMiad8miF9M6EnKgSLmbznAZ5v2M3VoPF9u8wzj9RNP4pg0oMPJeg1jzDHyJnFcoKrD6hxvFJE1qjpMRG7yVWCquhzoLyJ9gVdE5NOjuV9E7gDuAOjSpYsPIjxzTRrQkUkDOjZbzuXn6Rdp3yaYR6cOBCA2LIiOkcF8vGEvU4fGs2DLAZJiw2jfJojlaTk+jdsYc2J40znuEpGRNQciMgJwOYdNjbnMBDrXOU5wzjVaxhnmGwkc9qeHqm4FioABXj6z5r6ZqpqsqslxcXFNhGlOJj8/4dKBHVn4fRZp2cUs3ZnDhD7tGJUUw5Z9hygorcTtVj5Ym8nBwrKWDtcY0whvEsdtwCwRSXP245gF3C4iYcCfm7hvJdBTRJJEJBBPE9fcemXmAjc7n6cBX6mqOvf4g2e5E6APsMvLZ5pW7rLBnaisVq5+ztNhfnVyZ0Z1i0YVXlu6i9+8t4FfvbWO5xemtmygxphGNdtUpaorgYEiEukcF9S5/HYT91WJyN3A53hqKC+p6mYReQRYpapz8SSh10QkBcjFkwgAxgEzRKQScAN3qmo2QGPPPKo3Ni1ucEIknaNDSM8t5Y9TB9C7QwRllaH07diGx+d/D0BIgKvRVXmNMS1PvBhZi4hcimef8dreUFV9xIdxnVDJycm6apXtdtuafLguk+8PFHLvxN61c0TcbmX1njwOlVbyxZYDfL55P2v+cCHPf5vKiMS2DO8azXUzl5IYE8afrxxYe58xxjdEZLWqJtc/32yNQ0SewzOK6jzgRTxNSitOeITmjDJlSHyDc35+wohEz1DfjLxSZq9MZ2NmAX/5dBsX9G3HH6eGsCw1l2WpucRFBPG/NgPdmBbhTR/HGFX9MZ6Jeg8DZwG9fBuWOdP16eBZuX/mt55+jmWpuSz63rMX+pjuMTz9VQobMvJbLD5jzmTeJI6aoS0lItIJqASaH4tpzHHo46zKO2/jPgCKyqt4/tudRIUG8O8bhxMS4GowA90Yc3J4kzg+EpEo4G/AGjyjm95o8g5jjlNkSADxUSG4FSb0aYefwM6sYkYnxRAZGsDUoZ2Yu34vBSWVLR2qMWecJhOHiPgBX6pqvqq+h2cWdx9VfeCkRGfOaH07epqrLhvckcGdowAY08Ozys1No7tSVunmndXpR7zfGOMbTSYOVXXjWVSw5ri83nBcY3xmYHwULj9hXI84xvf0TOI8q5sncfTvFMnobtE8+cX3pBwsPOy+5xbu5NGPtwAwa3EaD831jNj+cF0mv3t/w0l8A2NOT940VX0pIleJjX00J9n08UnMuXMMcRFBTB+bxLM3DKNn+x+2u3/imiGEBLq449XVlFZU155/9btdvLg4je9Ssnn88+28tTIdt1t5f00mb65IJzWrqCVex5jThjeJ46fAO0CFiBwSkUIRsZlZxufCg/wZlOBpoooMDeDSQYePyegUFcKfrhhIanYxK3Z5VtbNyCthb4FnPMdtr66itLKa0spq9uSW8P0BT82kpsPdGHNsvNmPI0JV/VQ1QFXbOMdtTkZwxjRnVJJn3kfNLPNVu/IAOK93HCUV1fRoFw7Ail257HMSyscbLHEYczy82Y9DROQmEfmDc9y57qKHxrSkqNBAOkYGs81JHCt25RIR5M/frh7MxQM68NR1QxGBj9Z7trU9u1cc2/YXsmZPHt6smmCMacibpqp/4Zn0d4NzXESdDnNjWlrfjm3Yus/TDLUyLZfhiW2JDQ/i3zcNp1+nNnSJDmVJimfy4K8u6ElwgB9X/us7Jj+zhIoqd0uGbswpyZvEMUpV78KZCKiqeUCgT6My5ij07RjBzqwiDhwqY8fBotplS2r0bh+BWyEi2J+hnaP49Jdn88sJPdmYWVBbEwFYlppjCysa4wVvEkels9e3AohIHJ4Va41pFfp0aEOVW/nHAs/KumO6x9S7HlH7W0RIig3jVxf0pFf7cF5YlIqqUu1Wbn5pBRf/cxEz3tuA292wGauxc8acibxJHE8Bc4B2IvInYDHw/3walTFHoa+zPMmbK9IZ2iWKIc5kwRq9O7Rxfv8wlFdEuH18N7btL+TbHdlk5pVSXuWmR7twZq9MZ/uBw+eGrE/PZ9DD8w+roRhzpvJmVNV/gd/g2bRpHzBVVd/xdWDGeCspNowgf89/yned26PBcuv9O7Vxfkcedn7KkHjCAl18tfUAqdmeuR13nN0N8CSKutZn5FNUXsUvZ6/l3dUZPnkPY04V3iyr/hQwW1WtQ9y0Si4/YWB8JEXlVZzfp12D64mxYbz907Ma1EQC/f3o3SGCbfsLSYwNA+C83u2IDAlgXXo+1438Ya/6PTklBPn7MSIxmnvfWc+u7GKGdI6iZ/twusaE+fYFjWllmk0cwGrgfhHpjafJaraq2q5IplX5103D8BPBz6/xBQ5GJkU3er5PxzZ8smEfvdpHEBHsT2x4IIM7R7GuXo1jd24JXaJDeemWEcx4fwPPfJ0CQNeYUL6591zbVMqcUbxpqnpFVS8BRgDbgcdEZIfPIzPmKLSLCCY2POio7+vbIYKC0kqWpuaQFBuGiDAkIZLvDxRSUlFVWy49t4SuMaEE+vvx96sHM/fusdx3UW9255SwenfeiXwVY1o9bzrHa/QA+uBZIXebNzeIyCQR2S4iKSIyo5HrQSLylnN9uYgkOucvFJHVIrLR+X1+nXu+cZ65zvlp2DZhjJdq9v1IOVhEktNcNaRLFG6FjRme9TxVlT25JXSJ9lwXEQYlRHHzmESCA/yYszaT3OIK9uSUAJ69Qw4cKmvk24w5PXjTx/FX4ApgJ/AW8EdVbXbrNWcI77PAhUAGsFJE5qrqljrFpuPZWbCHiFwHPAZcC2QDl6vqXhEZAHwO1N1r9EZrLjMnQt2RVjWJY7CzPtba9HxGdYshu6iCkopqukSHHHZveJA/F/XvwNx1e/ls034OlVXyk7FJfLhuL25Vlv1uwhGbzo6kuLyK15btZvq4JAJcR/P3OmNOHm/+y9wJnKWqk1T1ZW+ShmMkkKKqqapaAcwGptQrMwV4xfn8LjBBRERV16pqzbjHzUCIiBx9O4QxzWgTHEBCW09CqEkcMeFB9O3Yhqe/3MFX2w6wJ7cYoNFO8CuHJVBYXkV0WCDje8bx/LepFJVXcbCwnC1NTCZcnprDTS8up6yy+rDz8zbu4y+fbqud6W5Ma+RNH8fzQLWIjBSRs2t+vHh2PFB3l50MDq81HFZGVauAAiCmXpmrgDWqWl7n3MtOM9UfjrTcu4jcISKrRGRVVlaWF+GaM1UfZ55Ht9jw2nMv3zKCpLgwbntlFQu2HgSgc3Rog3vP7hnLf28bxYd3j2XWzcnMvmM0H/18HOCZiX4k/1iwg8Up2ayp1z+yea8n2WzZdwhVZX16vq2pZVodbxY5vA34Fk9z0cPO74d8G1btd/fH03z10zqnb1TVgcB45+dHjd2rqjNVNVlVk+Pi4nwfrDll9e/UBpefkBj7Q2LoEBnMqz8ZRaC/H7MWpSFCbc2kLhFhbI9YQgP9ERFGd4shKTaMbrFhfLfz8MSxdGcOv5q9lhVpuSx1ksrytNzDytTUUrbsPcTilGymPLuEFfXKGNPSvGmq+iWeEVW7VfU8YCjgTXNVJtC5znGCc67RMiLiD0QCOc5xAp7hvz9W1Z01N6hqpvO7EM/e57ZSrzku08cn8fZPRxMRHHDY+eiwQK4alkBFtZsObYIJDnB5/czR3WNYkZbLnpwSFn6fRVllNfe+s54P1u3lhheWEejvR1JsGMvTfkguqsrWOjWOhds9NeXduSUn4C2NOXG8SRxlqloGnlFQqroN6O3FfSuBniKSJCKBwHXA3Hpl5gI3O5+nAV+pqopIFPAJMENVl9QUFhF/EYl1PgcAlwGbvIjFmCNqExzA8K6Nz/OYPi4JgC6NNFM1ZUz3GIrKq7jgyYXc/NIKpj67hMz8Un5+fg/8RLhqWDzn92nH2j35lFd5+jnSc0spLK+iU2QwadnFfLnN00S2v8BGaJnWxZvEkeH8Qf4B8IWIfAjsbu4mp8/ibjxNW1uBt1V1s4g8IiKTnWKzgBgRSQHuAWqG7N6NZ/jvA/WG3QYBn4vIBmAdnhrLC96+rDFHq1tcOL+Y0JNrkjs3X7iOs7rFEBzgR7+ObbhlTCLb9hdyycAO/O/E3iz67Xk8PHkAI5OiKa9ys8EZ9rtln+f3VcMTUIW0bE+n/H4b2mtamWaH46rqFc7Hh0TkazzNSZ9583BVnQfMq3fugTqfy4CrG7nvUeDRIzx2uDffbcyJcs+FvY76npjwIBb95nzahgbg7/LjiqHx9Gzv6Xxv3yYYgJHO8u+fb9pPcte2bN57CJefMHVoPE9/5ZmZHujv55Mah6qyNj2fYV3anvBnm9OfN0uO1FLVhb4KxJjTTVzEDyPIB9dbJwugbVggF/Rtx4uL09iy7xD7C8roHufpWI8KDaC6WhnWtW1t4qh2K/e9s54FWw/Qr1Mb/nrVYLrEHF0TWo3PNx/gZ6+vZu7dY2v3dTfGWzbDyJgW9PyPkrn/0r6k55WQkVfKub3bISJM7NeeyUM6Ed82hP2HylBV7v9gE++vzWRsj1jW7sln5qKdhz1LVXlo7maWNzEMuEbNSK1t+wubKWlMQ0dV4zDGnFguP+G28d24bXw3VLV2scS/ThsMwNNf7iC3uILVu/N4c8Uefnp2N353SV9+/uZaPtmwjwcv7187w3xpag7/+W4X8zfvZ/495xAe9MP/3s8t3Emgy48bR3chyN/F6j2e+SOpWcUn+Y3N6cBqHMa0Eo3NZW0f6ekP+XTTfgBuGZsIwJTBncgrqWTxjh9mmP93+R5CA13sO1TG459vrz2/eEc2f/l0G498vIWLnvyWfQWlbNnr6YhPzSry1euY05glDmNasY5O4vhs037aRQTRwelYP7tXHJEhAXy4zjM1KquwnM837eeGkV340eiu/Oe7Xcx4bwNb9x3ioY820zUmlOduGsaunBJ+8+4GKquVsEAXqdnN1zg+3biP38/Z2GAG+/trMhj2xy+Y+e1OKqpsN+kziTVVGdOK1SSKzPxSLuzXvrZWEujvx8UDOvDR+r1UVrt5b00GVW7l+lFd6Nw2lLAgf55buJPZKz2r/sy6OZkJfdszoU+72vkhlwzsyAfrMqmqduPv8qOq2o1Cg8UV56zNZP6WA5zVLYbLB3eqPf/hur0UllXy/+Zto6C0kvsu6nMS/omY1sBqHMa0Yh2cGgfQYAfDc3vHUVxRzdo9+czfvJ9BCZF0jwsn0N+P307qw/xfnc2zNwxj5o+GM6Fve+CHrXETY0IZkRhNZbWSkVcKwO2vruKGF5Y1qFnU1Er+8um22kUZyyqrWZ6Ww42jujK2RwxfbDngm38AplWyxGFMKxYRHEBYoGepk8H1hs2e1T0WP4G56zNZm57Peb0P35qmZ/sILh3UkYn9O9SeG5kUzQV923HZoE50b+dZ7Tc1u4hFO7L4ensWK3flsWZPPlmF5aRmFVHtVnbnFDO8a1sy80tr91tfvTuPsko343vGMr5nHN8fKLI9SM4g1lRlTCvXITKYnVnFDEyIPOx8ZEgAgztH8eaKdFRpdL/1+kSEF28eAUBecQUAOw8W88G6TOKjQigsq+SpL3ewM6sIVXjz9tFUVivXJCeQkefZ7fCm0V1ZtCObAJdnUceaCY1LUrK5cljCCX570xpZ4jCmlUtoG4qIEBkS0ODa+J5xrN2TT2x4EAPjIxu5+8jahgXSNjSAx+dvp7zKzZPXDmbL3kO8sCittsxiZ1+QbnHhDEqIYr2zF/uiHVkM7dKWsCB/+nVsQ0xYIIt3ZDO4cxQxYYFEhQYexxt7r7CsssHilMb3rKnKmFbuwcv78cwNQxu9dnbPWMDT33G0uw2CZ0Z7SKCLx64ayNQh8dw8JpG+Hdvwi/N7APDOak/nelJsGEM6R5GaXcz3BwrZvPcQ5/TybFfg5yeM6RHLxxv2MeHvC5ny7BIy8hpf0be0oprc4gqq3Z5+lIfmbuYn/1l51HEDfLn1AMP/uOCI32V8x2ocxrRy3eLCj3htSOcopg1P4OazEo/p2c/eMAwRCA30/FGQ0DaUT385nspqNy8sSmPtnnzaBPsTExZY28fy2KfbAJg04Ie+k8mDO/Ht91ncMLQL76/J4Nrnl/HZr8YTERxAaUU1//4mhXmb9pNy0DNvZHBCJH+/ZjCvLdtNgEsOm/zorUU7sqmodrM8NZeE4ce29Io5NlbjMOYU5u/y4/GrBzfo//BWWJB/bdKoK8DlVzuKq1tcOCJS+x1fbjtInw4RdK+T0C7s1571D07kocn9eer6oWTml7IkJYe9+aVc+ORCnvoqhY6Rwfz6gl784vwerM8o4Jrnl1HtVsoq3RwsLG8QQ40vthzghheWUVV9+FyRtc7s9zV78hq7zfiQ1TiMMY0a3rUtS1Nz6ObsxR4ZEkC32DBSs4u5dGDHI943pnssoYEulqRks23/ITLzS3nj9lGM6R5bWya3pILXl+2hZ7twdhwsYndOSW0ne12qyl8/28aOg0WkZBXVbvNbVlldu83u6jrb787fvJ9Afz/O7d38QAFz7KzGYYxp1PBEz5Lr3eLCas/VrPJ7yaAjJ45Afz9GJkWzZGc2nzlLxtdNGgD/d0k/fn9JHx6/2rMm166cxmewf7sjmx1O89ZGZ98SgE2ZBVS5lf6d2vD9gUIKyypxu5Xfz9nEP7/ccQxva46GJQ5jTKNGJEYzulv0YX97v2VMIvdd1PuwZqrGjO0eS2pWMdv2F3JRnXkkNUICXdxxdnf6Ofu978lp2MFdVe3m+YU7aRcRRFigy5Msqt1syiyobZ6aPi4Jt8L69AI27S0gu6icAz7aMfHLrQesWcxhTVXGmEaFB/kz+46zDjs3uHNUo3uL1De2xw81jLqd6PUFuPyIjwppsK96WnYxP39zDZsyD3H/pX2Zv/kAGzMLeP7bVP72+XYigvzpHB3ChL7tEfH0c9SM1DpQWE61W3EdwyizI1FV7nt3A307RvDf20afsOeeqixxGGNOuD4dIogJC6RTVAgJbZse8dQ1JpTdOcUcLCxjf0EZgxKiePTjLezOKeHZG4ZxycAO7Cso4/Vlu8kqKqd9myCyCsuZ2L8DkSEB9O/UhrdWphPizLCvdis5ReW0a6TP5FjtzComt7iCnQdtGXrwcVOViEwSke0ikiIiMxq5HiQibznXl4tIonP+QhFZLSIbnd/n17lnuHM+RUSekqMdw2eM8Tk/P+HpG4byl6sGNlvWkzg8q/ZO+/dSFu3I4qvtB7llTCKXDuroGdEVH0l5lZv03FJ+O6kPX997Lg9c3g+A/3fFQLKLykk5WFQ7EmxfQRmvfLeLJ7/4/ojfuyItt3btreas2uXZ+Gr/oTKKyqu8uud05rPEISIu4FngYqAfcL2I9KtXbDqQp6o9gCeBx5zz2cDlqjoQuBl4rc49/wZuB3o6P5N89Q7GmGM3pnss/Ts1P0y4a3QYBaWVfLM9i4pqN9NfWYWfCDeM6lJbZoAzKz48yJ+LB3Ska0xY7Uz6QQlR/HXaIEIDXdzq7Fey/1AZs1emM/Pb1EaXfH/lu11c8/xS/jxvq1fvsnLXD30baUex+VVFlZvrZy7ju5Ts5gufQnxZ4xgJpKhqqqpWALOBKfXKTAFecT6/C0wQEVHVtaq61zm/GQhxaicdgTaqukw9S3i+Ckz14TsYY3ysZt/0IH8/7j6vBxVVbi7s256OkSG1ZbrFhhEbHsjUoZ1qm6TqmjIknk0PXVQ7emtvfimpWUWUVlbXzveo8fW2gzz80WZCAly8vSqjds0ugJyicj7ZsK92wcayymrcbmXV7tza0WU7j2Lzq905xSxNzeGjDfu8vudY1Z/n4ku+TBzxQHqd4wznXKNlVLUKKABi6pW5ClijquVO+YxmngmAiNwhIqtEZFVWVtYxv4QxxrcSYzx/IF85LJ5fXdCTn57Tjf+d2OuwMn5+wrxfjuf+S+s3WhxeJiYskACXsHp3HuVOTWNxnb/tF5dX8bv3N9KrfQRv3jGa0spq/rt8NwDPL9xJ8p8WcNcba3jk4y1Uu5VLnlrEBU8uZHdOCVcNS8DlJ0e1a2KasyR9zRpf3vguJZslR1FD2ZCRz00vLqf3Hz5j7vq9zd9wArTq4bgi0h9P89VPj/ZeVZ2pqsmqmhwXF3figzPGnBA924Xzm0m9+fUFvfB3+fG7i/vSs31Eg3LtIoIJDmhY26jLz09oFxHMdztzAAgLdB2WOJ79OoX9h8p4dOoAhnSO4tzecTy3MJXfvb+RP3+6jYn92nPJwA4s2HKAL7bsJzWrmL35nv1KxvWIpXPbEHYeRVNVTeLYfqCQ0gpPf8qu7GJeXbqrwb4nNX7z3gZufHE5j3685YhlalRWu/nl7HVs21+In8CGo0hQx8OXiSMT6FznOME512gZEfEHIoEc5zgBmAP8WFV31ilfd93mxp5pjDmF+PkJd57b44SNguoQGUyu0/x01fAE1qfnc6iskoOHynhxURpXDI0nOTEagEcmD2BolyjeXLGH0d2ieer6oUwfl0R5lZv/m7OJNsH+fHPvecy6Obl2o6yjaaqqmdhY7VY27S2g2q38YvZaHvhwM+m5pQ3Kl1VWk5lfSsfIYF5cnMbS1Jwmnz97ZTpp2cX85cqBdI4OJTO/4TN9wZeJYyXQU0SSRCQQuA6YW6/MXDyd3wDTgK9UVUUkCvgEmKGqS2oKq+o+4JCIjHZGU/0Y+NCH72CMOcXUbLcbGx7EpQM74lZPv8bc9XupqHZz13k9ast2iQnltemj+OxX43n5lpEE+bsY1qUtCW1DyCmuYPKQTnSIDHbmiwjd24WTml1cO2ekOWnZxXR1+nDWp+fz+rLdbHBmwC9La5gUMvJKUIU7z+2OCKxMy6Oiys3sFXs4VFZ5WNmUg4X8c8H3jEyMZkLfdsRHhZz6icPps7gb+BzYCrytqptF5BERmewUmwXEiEgKcA9QM2T3bqAH8ICIrHN+aqav3gm8CKQAO4FPffUOxphTT82aVz3ahZGcGE1iTCizFqcxZ20mgxIi6dGu4az3Ph3a1Ha6iwiTnb3Vpw3vfFi57nFhVFS5a5uv6sopKsddL6Hsyi5heNe2xEeF8N6aTB77bBvje8YSHRbIsjq1ibziCsoqq0nL9kyEHBAfSZ8ObVi1O5eP1u9lxvsbufuNtVRVu3G7lZeXpHHpU4txKzw4uR8iQkLbEDLzTk7i8OkEQFWdB8yrd+6BOp/LgKsbue9R4NEjPHMVMODERmqMOV10jKxJHOG4/ITbxnfj/g82AfDAZUfuXK/rf87tztAubRvs816zxH1KVhGdo3+Y2LghI5+pzy6hbWggPxmXxF3n9aCkoor9h8pIigmjvNLNJxv30at9OI9fPZiH5m5meapnbkh2UTkXPfktE/t3qF1QMik2jBGJbXlvdQYuPyE4wI9vv8/iupnLcKuyZk8+5/dpx2NXDSIuIgiA+ChPLamssrrZvqDj1ao7x40x5mi1r0kczh/y04YnEBMWiMtPuNypSTQnIjiAC/u1b3C+Zo2unQeLKCip5J6315GeW8LzC1MJC/KnT8cI/vb5dr4/UMgup/aQGBvGDaO6cMXQeN7+6Vm0bxPMqKRoMvNLSc8t4cG5m8kpruDb77NIyykmKjSAqNBAkhOjKa6o5pvtWdw4qiszLu5DUXkV+SWVPDp1ALNuTq5NGgCdojzDl09Gc5UtOWKMOa30bBeOyA8r+QYHuHhocn/25JYc9gftsYh2tttNzS7m2x1ZvL8mk3Xp+ezKLuaOs7vz07O7Mfaxr/jX1ylMdBZ3TIoNY0B85GHrd43u7pl18LPXV7N57yG6x4WxM6uYpTtzaocnj3BWJwa4eEAHkhOj+dk53Y8YW3xN4sgrbXYRyuNlicMYc1rp27ENK35/wWFJwtuahje6xYWz82AREcH+uPyEtOxi/P2EW8cm0jYskBtHdWHW4jRynJFdibFhDZ7Rq10EXaJDOXCojFvGJHLlsHgmP7OEtOxirhjqmZrWMTKE+KgQKqvdDOvStsEz6otvazUOY4w5Zsdbs2hK97iw/9/evcfKUZZxHP/+dntoKe1pqS2UnkJPT0FCUS4VEbmJwQvlVhAUFLmIwWgwkaBiSVGJ/6FBEwyxaCQWrYIgxMbEiBRTJQZK6R1oaakIraWFSkAoBdo+/jHvttvD7uFMz9mZRX+fZHPmvDu7++wzs/PsOzM7Lw+ueoGOaoWpB3Vy5cndbHtr566D8led0sOfHt/E39a8SM+4/Rgx9O2b2UpFzP/6R6hIVCtix86gc9gQXtm2fVePA+CGs45AUr/Gkx/fOYxqRYUcIHfhMDPLYcq4Efx20XqWPLudc46ewPnHTtzj/gM6h7Hgm6fx6hvb6ag2P4xcf1+1Ij7YPYb5qzbTPXb3QffpfYy02NuQaoXxncN29Tie3bKVJc+9xNlHTRjUS8yDD46bmeVSO7PqtTd3cGRX44s4SmLksI5cZzd9qCf7mFWJdwAAB0tJREFUUeLkBru2+qtr9O5Tcn/84Bquu2c5W15rPp773nKPw8wshyl1Q+keOaFz0J73ouMOQYj39eOKws107b8vj6zbwrNbtnLvkg1cesIkDhg5eOOS1LjHYWaWw8FjhtNRFRXBEeMHr3CMGt7BVaf29Ot4RjPv7xrFv17exgWz/061Ir5yWvOzsAbCPQ4zsxw6qhUOGTOcakUNL/FepitO7Ob1t3Zw8/2ruezD3bsO2A82Fw4zs5yu/fjh9HHcuzSVirj6o4dywbSJjB2xT8tex4XDzCyns47q/9lOZRg/qjU9jZo2rJlmZtbOXDjMzCwXFw4zM8vFhcPMzHJx4TAzs1xcOMzMLBcXDjMzy8WFw8zMclFEvPNc73KSXgD+uZcPHwu8OIjhDBbHlV+7xua48mnXuKB9Y9vbuCZFxLjejf8XhWMgJC2KiOPKjqM3x5Vfu8bmuPJp17igfWMb7Li8q8rMzHJx4TAzs1xcON7ZT8sOoAnHlV+7xua48mnXuKB9YxvUuHyMw8zMcnGPw8zMcnHhMDOzXFw4mpB0hqTVktZKmllyLAdL+oukJyQ9Lulrqf1GSRskLU23M0uI7RlJK9LrL0ptYyT9WdKa9Hf/gmM6vC4nSyW9IumasvIl6XZJmyWtrGtrmCNlbknr3XJJ0wqO6weSVqXXvk/S6NTeLen1utzNLjiupstO0vUpX6slfbLguO6qi+kZSUtTe5H5arZ9aN06FhG+9boBVeBpoAfYB1gGTC0xnoOAaWl6JPAUMBW4EfhGybl6Bhjbq+37wMw0PRO4qeRl+Twwqax8AacC04CV75Qj4Ezgj4CAE4BHCo7rE8CQNH1TXVzd9fOVkK+Gyy59DpYBQ4HJ6XNbLSquXvffDHynhHw12z60bB1zj6Ox44G1EbEuIt4E7gRmlBVMRGyMiMVp+j/Ak0BXWfH0wwxgTpqeA5xXYiynA09HxN5eOWDAIuKvwL97NTfL0Qzgjsg8DIyW1JJxShvFFRH3R8T29O/DwMRWvHbeuPowA7gzIt6IiH8Aa8k+v4XGJUnAZ4DftOK1+9LH9qFl65gLR2NdwHN1/6+nTTbUkrqBY4FHUtNXU3fz9qJ3CSUB3C/pMUlfSm0HRsTGNP08cGAJcdVczJ4f5rLzVdMsR+207l1J9s20ZrKkJZIWSDqlhHgaLbt2ydcpwKaIWFPXVni+em0fWraOuXC8i0gaAfwOuCYiXgF+AkwBjgE2knWVi3ZyREwDpgNXSzq1/s7I+salnPMtaR/gXODu1NQO+XqbMnPUjKRZwHZgbmraCBwSEccC1wK/ltRZYEhtuezqfJY9v6AUnq8G24ddBnsdc+FobANwcN3/E1NbaSR1kK0UcyPiXoCI2BQROyJiJ/AzWtRF70tEbEh/NwP3pRg21bq+6e/mouNKpgOLI2JTirH0fNVplqPS1z1JVwBnA5ekDQ5pV9CWNP0Y2bGE9xYVUx/Lrh3yNQT4FHBXra3ofDXaPtDCdcyFo7FHgcMkTU7fWi8G5pUVTNp/+nPgyYj4YV17/X7J84GVvR/b4rj2kzSyNk12YHUlWa4uT7NdDvy+yLjq7PEtsOx89dIsR/OAy9KZLycAL9ftbmg5SWcA1wHnRsTWuvZxkqppugc4DFhXYFzNlt084GJJQyVNTnEtLCqu5GPAqohYX2soMl/Ntg+0ch0r4qj/u/FGdubBU2TfFGaVHMvJZN3M5cDSdDsT+CWwIrXPAw4qOK4esjNalgGP1/IEvAeYD6wBHgDGlJCz/YAtwKi6tlLyRVa8NgJvke1P/mKzHJGd6XJrWu9WAMcVHNdasv3ftfVsdpr3grSMlwKLgXMKjqvpsgNmpXytBqYXGVdq/wXw5V7zFpmvZtuHlq1jvuSImZnl4l1VZmaWiwuHmZnl4sJhZma5uHCYmVkuLhxmZpaLC4dZG5N0mqQ/lB2HWT0XDjMzy8WFw2wQSPq8pIVp7IXbJFUlvSrpR2mMhPmSxqV5j5H0sHaPeVEbJ+FQSQ9IWiZpsaQp6elHSLpH2TgZc9Mvhc1K48JhNkCSjgAuAk6KiGOAHcAlZL9eXxQRRwILgO+mh9wBfCsijiL75W6tfS5wa0QcDZxI9itlyK52eg3ZGAs9wEktf1NmfRhSdgBm/wNOBz4APJo6A/uSXVBuJ7svfPcr4F5Jo4DREbEgtc8B7k7X/OqKiPsAImIbQHq+hZGug6RshLlu4KHWvy2zxlw4zAZOwJyIuH6PRunbvebb2+v7vFE3vQN/bq1k3lVlNnDzgQslHQC7xnqeRPb5ujDN8zngoYh4GXipbmCfS4EFkY3ctl7Seek5hkoaXui7MOsnf3MxG6CIeELSDWQjIVbIrp56NfAacHy6bzPZcRDILnE9OxWGdcAXUvulwG2Svpee49MFvg2zfvPVcc1aRNKrETGi7DjMBpt3VZmZWS7ucZiZWS7ucZiZWS4uHGZmlosLh5mZ5eLCYWZmubhwmJlZLv8FyX1EE+aqirAAAAAASUVORK5CYII=\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ],
      "source": [
        "# average per-sample loss per epoch\n",
        "plt.plot(np.arange(num_epochs), train_loss_history)\n",
        "plt.xlabel('epoch')\n",
        "plt.ylabel('average per-sample loss')\n",
        "plt.legend(['training loss curve'])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 29,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2KVRF-C5GzRK",
        "outputId": "4da40b29-9e50-4eaa-f2b9-ea86395b09f6"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch [0/200], Step[1/28], Loss: 1.3810\n",
            "Epoch [0/200], Step[2/28], Loss: 1.9341\n",
            "Epoch [0/200], Step[3/28], Loss: 1.6035\n",
            "Epoch [0/200], Step[4/28], Loss: 2.1889\n",
            "Epoch [0/200], Step[5/28], Loss: 2.4075\n",
            "Epoch [0/200], Step[6/28], Loss: 2.1386\n",
            "Epoch [0/200], Step[7/28], Loss: 1.6518\n",
            "Epoch [0/200], Step[8/28], Loss: 1.5466\n",
            "Epoch [0/200], Step[9/28], Loss: 1.6941\n",
            "Epoch [0/200], Step[10/28], Loss: 1.9644\n",
            "Epoch [0/200], Step[11/28], Loss: 1.4389\n",
            "Epoch [0/200], Step[12/28], Loss: 1.4649\n",
            "Epoch [0/200], Step[13/28], Loss: 1.4987\n",
            "Epoch [0/200], Step[14/28], Loss: 1.4579\n",
            "Epoch [0/200], Step[15/28], Loss: 1.4323\n",
            "Epoch [0/200], Step[16/28], Loss: 1.3720\n",
            "Epoch [0/200], Step[17/28], Loss: 1.3427\n",
            "Epoch [0/200], Step[18/28], Loss: 1.4011\n",
            "Epoch [0/200], Step[19/28], Loss: 1.4078\n",
            "Epoch [0/200], Step[20/28], Loss: 1.3738\n",
            "Epoch [0/200], Step[21/28], Loss: 1.3927\n",
            "Epoch [0/200], Step[22/28], Loss: 1.5748\n",
            "Epoch [0/200], Step[23/28], Loss: 1.4042\n",
            "Epoch [0/200], Step[24/28], Loss: 1.3895\n",
            "Epoch [0/200], Step[25/28], Loss: 1.3797\n",
            "Epoch [0/200], Step[26/28], Loss: 1.3854\n",
            "Epoch [0/200], Step[27/28], Loss: 1.3806\n",
            "Epoch [0/200], Step[28/28], Loss: 1.3792\n",
            "Epoch [10/200], Step[1/28], Loss: 1.3852\n",
            "Epoch [10/200], Step[2/28], Loss: 1.3891\n",
            "Epoch [10/200], Step[3/28], Loss: 1.3823\n",
            "Epoch [10/200], Step[4/28], Loss: 1.3826\n",
            "Epoch [10/200], Step[5/28], Loss: 1.3784\n",
            "Epoch [10/200], Step[6/28], Loss: 1.3793\n",
            "Epoch [10/200], Step[7/28], Loss: 1.3883\n",
            "Epoch [10/200], Step[8/28], Loss: 1.3977\n",
            "Epoch [10/200], Step[9/28], Loss: 1.3888\n",
            "Epoch [10/200], Step[10/28], Loss: 1.3874\n",
            "Epoch [10/200], Step[11/28], Loss: 1.3858\n",
            "Epoch [10/200], Step[12/28], Loss: 1.3874\n",
            "Epoch [10/200], Step[13/28], Loss: 1.3983\n",
            "Epoch [10/200], Step[14/28], Loss: 1.3853\n",
            "Epoch [10/200], Step[15/28], Loss: 1.3784\n",
            "Epoch [10/200], Step[16/28], Loss: 1.3740\n",
            "Epoch [10/200], Step[17/28], Loss: 1.3837\n",
            "Epoch [10/200], Step[18/28], Loss: 1.4102\n",
            "Epoch [10/200], Step[19/28], Loss: 1.3777\n",
            "Epoch [10/200], Step[20/28], Loss: 1.3855\n",
            "Epoch [10/200], Step[21/28], Loss: 1.3873\n",
            "Epoch [10/200], Step[22/28], Loss: 1.3858\n",
            "Epoch [10/200], Step[23/28], Loss: 1.3914\n",
            "Epoch [10/200], Step[24/28], Loss: 1.3745\n",
            "Epoch [10/200], Step[25/28], Loss: 1.3965\n",
            "Epoch [10/200], Step[26/28], Loss: 1.3998\n",
            "Epoch [10/200], Step[27/28], Loss: 1.3872\n",
            "Epoch [10/200], Step[28/28], Loss: 1.3744\n",
            "Epoch [20/200], Step[1/28], Loss: 1.3864\n",
            "Epoch [20/200], Step[2/28], Loss: 1.3903\n",
            "Epoch [20/200], Step[3/28], Loss: 1.3848\n",
            "Epoch [20/200], Step[4/28], Loss: 1.3848\n",
            "Epoch [20/200], Step[5/28], Loss: 1.3780\n",
            "Epoch [20/200], Step[6/28], Loss: 1.3884\n",
            "Epoch [20/200], Step[7/28], Loss: 1.3894\n",
            "Epoch [20/200], Step[8/28], Loss: 1.3945\n",
            "Epoch [20/200], Step[9/28], Loss: 1.3871\n",
            "Epoch [20/200], Step[10/28], Loss: 1.3815\n",
            "Epoch [20/200], Step[11/28], Loss: 1.3889\n",
            "Epoch [20/200], Step[12/28], Loss: 1.3825\n",
            "Epoch [20/200], Step[13/28], Loss: 1.3886\n",
            "Epoch [20/200], Step[14/28], Loss: 1.3873\n",
            "Epoch [20/200], Step[15/28], Loss: 1.3874\n",
            "Epoch [20/200], Step[16/28], Loss: 1.3922\n",
            "Epoch [20/200], Step[17/28], Loss: 1.3832\n",
            "Epoch [20/200], Step[18/28], Loss: 1.3855\n",
            "Epoch [20/200], Step[19/28], Loss: 1.3803\n",
            "Epoch [20/200], Step[20/28], Loss: 1.3920\n",
            "Epoch [20/200], Step[21/28], Loss: 1.3887\n",
            "Epoch [20/200], Step[22/28], Loss: 1.3831\n",
            "Epoch [20/200], Step[23/28], Loss: 1.3830\n",
            "Epoch [20/200], Step[24/28], Loss: 1.3872\n",
            "Epoch [20/200], Step[25/28], Loss: 1.3826\n",
            "Epoch [20/200], Step[26/28], Loss: 1.3862\n",
            "Epoch [20/200], Step[27/28], Loss: 1.3893\n",
            "Epoch [20/200], Step[28/28], Loss: 1.3965\n",
            "Epoch [30/200], Step[1/28], Loss: 1.3866\n",
            "Epoch [30/200], Step[2/28], Loss: 1.3928\n",
            "Epoch [30/200], Step[3/28], Loss: 1.3898\n",
            "Epoch [30/200], Step[4/28], Loss: 1.3855\n",
            "Epoch [30/200], Step[5/28], Loss: 1.3880\n",
            "Epoch [30/200], Step[6/28], Loss: 1.3828\n",
            "Epoch [30/200], Step[7/28], Loss: 1.3793\n",
            "Epoch [30/200], Step[8/28], Loss: 1.3855\n",
            "Epoch [30/200], Step[9/28], Loss: 1.3841\n",
            "Epoch [30/200], Step[10/28], Loss: 1.3868\n",
            "Epoch [30/200], Step[11/28], Loss: 1.3942\n",
            "Epoch [30/200], Step[12/28], Loss: 1.3937\n",
            "Epoch [30/200], Step[13/28], Loss: 1.3868\n",
            "Epoch [30/200], Step[14/28], Loss: 1.3785\n",
            "Epoch [30/200], Step[15/28], Loss: 1.3905\n",
            "Epoch [30/200], Step[16/28], Loss: 1.3865\n",
            "Epoch [30/200], Step[17/28], Loss: 1.3867\n",
            "Epoch [30/200], Step[18/28], Loss: 1.3901\n",
            "Epoch [30/200], Step[19/28], Loss: 1.3859\n",
            "Epoch [30/200], Step[20/28], Loss: 1.3881\n",
            "Epoch [30/200], Step[21/28], Loss: 1.3823\n",
            "Epoch [30/200], Step[22/28], Loss: 1.3873\n",
            "Epoch [30/200], Step[23/28], Loss: 1.3840\n",
            "Epoch [30/200], Step[24/28], Loss: 1.3772\n",
            "Epoch [30/200], Step[25/28], Loss: 1.3899\n",
            "Epoch [30/200], Step[26/28], Loss: 1.3890\n",
            "Epoch [30/200], Step[27/28], Loss: 1.3888\n",
            "Epoch [30/200], Step[28/28], Loss: 1.3839\n",
            "Epoch [40/200], Step[1/28], Loss: 1.3747\n",
            "Epoch [40/200], Step[2/28], Loss: 1.3861\n",
            "Epoch [40/200], Step[3/28], Loss: 1.3907\n",
            "Epoch [40/200], Step[4/28], Loss: 1.3939\n",
            "Epoch [40/200], Step[5/28], Loss: 1.3838\n",
            "Epoch [40/200], Step[6/28], Loss: 1.3865\n",
            "Epoch [40/200], Step[7/28], Loss: 1.3829\n",
            "Epoch [40/200], Step[8/28], Loss: 1.3872\n",
            "Epoch [40/200], Step[9/28], Loss: 1.3861\n",
            "Epoch [40/200], Step[10/28], Loss: 1.3891\n",
            "Epoch [40/200], Step[11/28], Loss: 1.3889\n",
            "Epoch [40/200], Step[12/28], Loss: 1.3887\n",
            "Epoch [40/200], Step[13/28], Loss: 1.3856\n",
            "Epoch [40/200], Step[14/28], Loss: 1.3806\n",
            "Epoch [40/200], Step[15/28], Loss: 1.3891\n",
            "Epoch [40/200], Step[16/28], Loss: 1.3836\n",
            "Epoch [40/200], Step[17/28], Loss: 1.3819\n",
            "Epoch [40/200], Step[18/28], Loss: 1.3867\n",
            "Epoch [40/200], Step[19/28], Loss: 1.3929\n",
            "Epoch [40/200], Step[20/28], Loss: 1.3866\n",
            "Epoch [40/200], Step[21/28], Loss: 1.3830\n",
            "Epoch [40/200], Step[22/28], Loss: 1.3850\n",
            "Epoch [40/200], Step[23/28], Loss: 1.3901\n",
            "Epoch [40/200], Step[24/28], Loss: 1.3890\n",
            "Epoch [40/200], Step[25/28], Loss: 1.3837\n",
            "Epoch [40/200], Step[26/28], Loss: 1.3882\n",
            "Epoch [40/200], Step[27/28], Loss: 1.3890\n",
            "Epoch [40/200], Step[28/28], Loss: 1.3844\n",
            "Epoch [50/200], Step[1/28], Loss: 1.3865\n",
            "Epoch [50/200], Step[2/28], Loss: 1.3831\n",
            "Epoch [50/200], Step[3/28], Loss: 1.3862\n",
            "Epoch [50/200], Step[4/28], Loss: 1.3819\n",
            "Epoch [50/200], Step[5/28], Loss: 1.3849\n",
            "Epoch [50/200], Step[6/28], Loss: 1.3962\n",
            "Epoch [50/200], Step[7/28], Loss: 1.3894\n",
            "Epoch [50/200], Step[8/28], Loss: 1.3874\n",
            "Epoch [50/200], Step[9/28], Loss: 1.3886\n",
            "Epoch [50/200], Step[10/28], Loss: 1.3922\n",
            "Epoch [50/200], Step[11/28], Loss: 1.3837\n",
            "Epoch [50/200], Step[12/28], Loss: 1.3848\n",
            "Epoch [50/200], Step[13/28], Loss: 1.3752\n",
            "Epoch [50/200], Step[14/28], Loss: 1.3777\n",
            "Epoch [50/200], Step[15/28], Loss: 1.3881\n",
            "Epoch [50/200], Step[16/28], Loss: 1.3880\n",
            "Epoch [50/200], Step[17/28], Loss: 1.3884\n",
            "Epoch [50/200], Step[18/28], Loss: 1.3889\n",
            "Epoch [50/200], Step[19/28], Loss: 1.3814\n",
            "Epoch [50/200], Step[20/28], Loss: 1.3858\n",
            "Epoch [50/200], Step[21/28], Loss: 1.3894\n",
            "Epoch [50/200], Step[22/28], Loss: 1.3833\n",
            "Epoch [50/200], Step[23/28], Loss: 1.3952\n",
            "Epoch [50/200], Step[24/28], Loss: 1.3847\n",
            "Epoch [50/200], Step[25/28], Loss: 1.3864\n",
            "Epoch [50/200], Step[26/28], Loss: 1.3924\n",
            "Epoch [50/200], Step[27/28], Loss: 1.3875\n",
            "Epoch [50/200], Step[28/28], Loss: 1.4003\n",
            "Epoch [60/200], Step[1/28], Loss: 1.3819\n",
            "Epoch [60/200], Step[2/28], Loss: 1.3786\n",
            "Epoch [60/200], Step[3/28], Loss: 1.3847\n",
            "Epoch [60/200], Step[4/28], Loss: 1.3859\n",
            "Epoch [60/200], Step[5/28], Loss: 1.3768\n",
            "Epoch [60/200], Step[6/28], Loss: 1.3726\n",
            "Epoch [60/200], Step[7/28], Loss: 1.4013\n",
            "Epoch [60/200], Step[8/28], Loss: 1.3866\n",
            "Epoch [60/200], Step[9/28], Loss: 1.3989\n",
            "Epoch [60/200], Step[10/28], Loss: 1.3736\n",
            "Epoch [60/200], Step[11/28], Loss: 1.3818\n",
            "Epoch [60/200], Step[12/28], Loss: 1.4001\n",
            "Epoch [60/200], Step[13/28], Loss: 1.4023\n",
            "Epoch [60/200], Step[14/28], Loss: 1.3775\n",
            "Epoch [60/200], Step[15/28], Loss: 1.4015\n",
            "Epoch [60/200], Step[16/28], Loss: 1.3841\n",
            "Epoch [60/200], Step[17/28], Loss: 1.4014\n",
            "Epoch [60/200], Step[18/28], Loss: 1.3881\n",
            "Epoch [60/200], Step[19/28], Loss: 1.3863\n",
            "Epoch [60/200], Step[20/28], Loss: 1.3912\n",
            "Epoch [60/200], Step[21/28], Loss: 1.3917\n",
            "Epoch [60/200], Step[22/28], Loss: 1.3863\n",
            "Epoch [60/200], Step[23/28], Loss: 1.3864\n",
            "Epoch [60/200], Step[24/28], Loss: 1.3838\n",
            "Epoch [60/200], Step[25/28], Loss: 1.3899\n",
            "Epoch [60/200], Step[26/28], Loss: 1.3750\n",
            "Epoch [60/200], Step[27/28], Loss: 1.3944\n",
            "Epoch [60/200], Step[28/28], Loss: 1.3836\n",
            "Epoch [70/200], Step[1/28], Loss: 1.3929\n",
            "Epoch [70/200], Step[2/28], Loss: 1.3844\n",
            "Epoch [70/200], Step[3/28], Loss: 1.3787\n",
            "Epoch [70/200], Step[4/28], Loss: 1.3832\n",
            "Epoch [70/200], Step[5/28], Loss: 1.3880\n",
            "Epoch [70/200], Step[6/28], Loss: 1.3894\n",
            "Epoch [70/200], Step[7/28], Loss: 1.3922\n",
            "Epoch [70/200], Step[8/28], Loss: 1.3883\n",
            "Epoch [70/200], Step[9/28], Loss: 1.3777\n",
            "Epoch [70/200], Step[10/28], Loss: 1.3908\n",
            "Epoch [70/200], Step[11/28], Loss: 1.3870\n",
            "Epoch [70/200], Step[12/28], Loss: 1.3846\n",
            "Epoch [70/200], Step[13/28], Loss: 1.3847\n",
            "Epoch [70/200], Step[14/28], Loss: 1.3870\n",
            "Epoch [70/200], Step[15/28], Loss: 1.3821\n",
            "Epoch [70/200], Step[16/28], Loss: 1.3815\n",
            "Epoch [70/200], Step[17/28], Loss: 1.3891\n",
            "Epoch [70/200], Step[18/28], Loss: 1.3838\n",
            "Epoch [70/200], Step[19/28], Loss: 1.3807\n",
            "Epoch [70/200], Step[20/28], Loss: 1.3879\n",
            "Epoch [70/200], Step[21/28], Loss: 1.3846\n",
            "Epoch [70/200], Step[22/28], Loss: 1.3891\n",
            "Epoch [70/200], Step[23/28], Loss: 1.3899\n",
            "Epoch [70/200], Step[24/28], Loss: 1.3870\n",
            "Epoch [70/200], Step[25/28], Loss: 1.3966\n",
            "Epoch [70/200], Step[26/28], Loss: 1.3899\n",
            "Epoch [70/200], Step[27/28], Loss: 1.3884\n",
            "Epoch [70/200], Step[28/28], Loss: 1.3829\n",
            "Epoch [80/200], Step[1/28], Loss: 1.3858\n",
            "Epoch [80/200], Step[2/28], Loss: 1.3863\n",
            "Epoch [80/200], Step[3/28], Loss: 1.3874\n",
            "Epoch [80/200], Step[4/28], Loss: 1.3857\n",
            "Epoch [80/200], Step[5/28], Loss: 1.3870\n",
            "Epoch [80/200], Step[6/28], Loss: 1.3845\n",
            "Epoch [80/200], Step[7/28], Loss: 1.3865\n",
            "Epoch [80/200], Step[8/28], Loss: 1.3872\n",
            "Epoch [80/200], Step[9/28], Loss: 1.3901\n",
            "Epoch [80/200], Step[10/28], Loss: 1.3808\n",
            "Epoch [80/200], Step[11/28], Loss: 1.3817\n",
            "Epoch [80/200], Step[12/28], Loss: 1.3816\n",
            "Epoch [80/200], Step[13/28], Loss: 1.3888\n",
            "Epoch [80/200], Step[14/28], Loss: 1.3930\n",
            "Epoch [80/200], Step[15/28], Loss: 1.3891\n",
            "Epoch [80/200], Step[16/28], Loss: 1.3839\n",
            "Epoch [80/200], Step[17/28], Loss: 1.3786\n",
            "Epoch [80/200], Step[18/28], Loss: 1.3795\n",
            "Epoch [80/200], Step[19/28], Loss: 1.3913\n",
            "Epoch [80/200], Step[20/28], Loss: 1.3872\n",
            "Epoch [80/200], Step[21/28], Loss: 1.3878\n",
            "Epoch [80/200], Step[22/28], Loss: 1.3854\n",
            "Epoch [80/200], Step[23/28], Loss: 1.3917\n",
            "Epoch [80/200], Step[24/28], Loss: 1.3988\n",
            "Epoch [80/200], Step[25/28], Loss: 1.3870\n",
            "Epoch [80/200], Step[26/28], Loss: 1.3894\n",
            "Epoch [80/200], Step[27/28], Loss: 1.3841\n",
            "Epoch [80/200], Step[28/28], Loss: 1.3846\n",
            "Epoch [90/200], Step[1/28], Loss: 1.3900\n",
            "Epoch [90/200], Step[2/28], Loss: 1.3791\n",
            "Epoch [90/200], Step[3/28], Loss: 1.3835\n",
            "Epoch [90/200], Step[4/28], Loss: 1.3865\n",
            "Epoch [90/200], Step[5/28], Loss: 1.3849\n",
            "Epoch [90/200], Step[6/28], Loss: 1.3799\n",
            "Epoch [90/200], Step[7/28], Loss: 1.3828\n",
            "Epoch [90/200], Step[8/28], Loss: 1.3895\n",
            "Epoch [90/200], Step[9/28], Loss: 1.4007\n",
            "Epoch [90/200], Step[10/28], Loss: 1.3813\n",
            "Epoch [90/200], Step[11/28], Loss: 1.3895\n",
            "Epoch [90/200], Step[12/28], Loss: 1.3976\n",
            "Epoch [90/200], Step[13/28], Loss: 1.3951\n",
            "Epoch [90/200], Step[14/28], Loss: 1.3930\n",
            "Epoch [90/200], Step[15/28], Loss: 1.3882\n",
            "Epoch [90/200], Step[16/28], Loss: 1.3797\n",
            "Epoch [90/200], Step[17/28], Loss: 1.3823\n",
            "Epoch [90/200], Step[18/28], Loss: 1.3907\n",
            "Epoch [90/200], Step[19/28], Loss: 1.3827\n",
            "Epoch [90/200], Step[20/28], Loss: 1.3962\n",
            "Epoch [90/200], Step[21/28], Loss: 1.3862\n",
            "Epoch [90/200], Step[22/28], Loss: 1.3778\n",
            "Epoch [90/200], Step[23/28], Loss: 1.3827\n",
            "Epoch [90/200], Step[24/28], Loss: 1.3912\n",
            "Epoch [90/200], Step[25/28], Loss: 1.3852\n",
            "Epoch [90/200], Step[26/28], Loss: 1.3823\n",
            "Epoch [90/200], Step[27/28], Loss: 1.3905\n",
            "Epoch [90/200], Step[28/28], Loss: 1.3840\n",
            "Epoch [100/200], Step[1/28], Loss: 1.3856\n",
            "Epoch [100/200], Step[2/28], Loss: 1.3860\n",
            "Epoch [100/200], Step[3/28], Loss: 1.3872\n",
            "Epoch [100/200], Step[4/28], Loss: 1.3834\n",
            "Epoch [100/200], Step[5/28], Loss: 1.3880\n",
            "Epoch [100/200], Step[6/28], Loss: 1.3880\n",
            "Epoch [100/200], Step[7/28], Loss: 1.3832\n",
            "Epoch [100/200], Step[8/28], Loss: 1.3793\n",
            "Epoch [100/200], Step[9/28], Loss: 1.3838\n",
            "Epoch [100/200], Step[10/28], Loss: 1.3859\n",
            "Epoch [100/200], Step[11/28], Loss: 1.3916\n",
            "Epoch [100/200], Step[12/28], Loss: 1.3933\n",
            "Epoch [100/200], Step[13/28], Loss: 1.3850\n",
            "Epoch [100/200], Step[14/28], Loss: 1.3926\n",
            "Epoch [100/200], Step[15/28], Loss: 1.3776\n",
            "Epoch [100/200], Step[16/28], Loss: 1.3947\n",
            "Epoch [100/200], Step[17/28], Loss: 1.3917\n",
            "Epoch [100/200], Step[18/28], Loss: 1.3732\n",
            "Epoch [100/200], Step[19/28], Loss: 1.3923\n",
            "Epoch [100/200], Step[20/28], Loss: 1.3859\n",
            "Epoch [100/200], Step[21/28], Loss: 1.3887\n",
            "Epoch [100/200], Step[22/28], Loss: 1.3744\n",
            "Epoch [100/200], Step[23/28], Loss: 1.3810\n",
            "Epoch [100/200], Step[24/28], Loss: 1.3930\n",
            "Epoch [100/200], Step[25/28], Loss: 1.3876\n",
            "Epoch [100/200], Step[26/28], Loss: 1.3859\n",
            "Epoch [100/200], Step[27/28], Loss: 1.3921\n",
            "Epoch [100/200], Step[28/28], Loss: 1.3966\n",
            "Epoch [110/200], Step[1/28], Loss: 1.3876\n",
            "Epoch [110/200], Step[2/28], Loss: 1.3853\n",
            "Epoch [110/200], Step[3/28], Loss: 1.3841\n",
            "Epoch [110/200], Step[4/28], Loss: 1.3863\n",
            "Epoch [110/200], Step[5/28], Loss: 1.3826\n",
            "Epoch [110/200], Step[6/28], Loss: 1.3785\n",
            "Epoch [110/200], Step[7/28], Loss: 1.3912\n",
            "Epoch [110/200], Step[8/28], Loss: 1.3982\n",
            "Epoch [110/200], Step[9/28], Loss: 1.3859\n",
            "Epoch [110/200], Step[10/28], Loss: 1.3883\n",
            "Epoch [110/200], Step[11/28], Loss: 1.3840\n",
            "Epoch [110/200], Step[12/28], Loss: 1.3855\n",
            "Epoch [110/200], Step[13/28], Loss: 1.3913\n",
            "Epoch [110/200], Step[14/28], Loss: 1.3866\n",
            "Epoch [110/200], Step[15/28], Loss: 1.3868\n",
            "Epoch [110/200], Step[16/28], Loss: 1.3775\n",
            "Epoch [110/200], Step[17/28], Loss: 1.3833\n",
            "Epoch [110/200], Step[18/28], Loss: 1.3919\n",
            "Epoch [110/200], Step[19/28], Loss: 1.3931\n",
            "Epoch [110/200], Step[20/28], Loss: 1.3881\n",
            "Epoch [110/200], Step[21/28], Loss: 1.3865\n",
            "Epoch [110/200], Step[22/28], Loss: 1.3834\n",
            "Epoch [110/200], Step[23/28], Loss: 1.3822\n",
            "Epoch [110/200], Step[24/28], Loss: 1.3871\n",
            "Epoch [110/200], Step[25/28], Loss: 1.3879\n",
            "Epoch [110/200], Step[26/28], Loss: 1.3867\n",
            "Epoch [110/200], Step[27/28], Loss: 1.3786\n",
            "Epoch [110/200], Step[28/28], Loss: 1.3939\n",
            "Epoch [120/200], Step[1/28], Loss: 1.3699\n",
            "Epoch [120/200], Step[2/28], Loss: 1.3854\n",
            "Epoch [120/200], Step[3/28], Loss: 1.3914\n",
            "Epoch [120/200], Step[4/28], Loss: 1.3859\n",
            "Epoch [120/200], Step[5/28], Loss: 1.3758\n",
            "Epoch [120/200], Step[6/28], Loss: 1.3779\n",
            "Epoch [120/200], Step[7/28], Loss: 1.4013\n",
            "Epoch [120/200], Step[8/28], Loss: 1.3932\n",
            "Epoch [120/200], Step[9/28], Loss: 1.3934\n",
            "Epoch [120/200], Step[10/28], Loss: 1.3890\n",
            "Epoch [120/200], Step[11/28], Loss: 1.3892\n",
            "Epoch [120/200], Step[12/28], Loss: 1.3922\n",
            "Epoch [120/200], Step[13/28], Loss: 1.4029\n",
            "Epoch [120/200], Step[14/28], Loss: 1.3809\n",
            "Epoch [120/200], Step[15/28], Loss: 1.3847\n",
            "Epoch [120/200], Step[16/28], Loss: 1.3836\n",
            "Epoch [120/200], Step[17/28], Loss: 1.3766\n",
            "Epoch [120/200], Step[18/28], Loss: 1.3715\n",
            "Epoch [120/200], Step[19/28], Loss: 1.3879\n",
            "Epoch [120/200], Step[20/28], Loss: 1.3893\n",
            "Epoch [120/200], Step[21/28], Loss: 1.3752\n",
            "Epoch [120/200], Step[22/28], Loss: 1.3985\n",
            "Epoch [120/200], Step[23/28], Loss: 1.3891\n",
            "Epoch [120/200], Step[24/28], Loss: 1.3910\n",
            "Epoch [120/200], Step[25/28], Loss: 1.3892\n",
            "Epoch [120/200], Step[26/28], Loss: 1.3772\n",
            "Epoch [120/200], Step[27/28], Loss: 1.3872\n",
            "Epoch [120/200], Step[28/28], Loss: 1.4094\n",
            "Epoch [130/200], Step[1/28], Loss: 1.3817\n",
            "Epoch [130/200], Step[2/28], Loss: 1.3905\n",
            "Epoch [130/200], Step[3/28], Loss: 1.3864\n",
            "Epoch [130/200], Step[4/28], Loss: 1.3776\n",
            "Epoch [130/200], Step[5/28], Loss: 1.3880\n",
            "Epoch [130/200], Step[6/28], Loss: 1.3873\n",
            "Epoch [130/200], Step[7/28], Loss: 1.3854\n",
            "Epoch [130/200], Step[8/28], Loss: 1.3914\n",
            "Epoch [130/200], Step[9/28], Loss: 1.3850\n",
            "Epoch [130/200], Step[10/28], Loss: 1.3837\n",
            "Epoch [130/200], Step[11/28], Loss: 1.3845\n",
            "Epoch [130/200], Step[12/28], Loss: 1.3909\n",
            "Epoch [130/200], Step[13/28], Loss: 1.3793\n",
            "Epoch [130/200], Step[14/28], Loss: 1.3888\n",
            "Epoch [130/200], Step[15/28], Loss: 1.3877\n",
            "Epoch [130/200], Step[16/28], Loss: 1.3917\n",
            "Epoch [130/200], Step[17/28], Loss: 1.3875\n",
            "Epoch [130/200], Step[18/28], Loss: 1.3902\n",
            "Epoch [130/200], Step[19/28], Loss: 1.3845\n",
            "Epoch [130/200], Step[20/28], Loss: 1.3886\n",
            "Epoch [130/200], Step[21/28], Loss: 1.3867\n",
            "Epoch [130/200], Step[22/28], Loss: 1.3896\n",
            "Epoch [130/200], Step[23/28], Loss: 1.3859\n",
            "Epoch [130/200], Step[24/28], Loss: 1.3869\n",
            "Epoch [130/200], Step[25/28], Loss: 1.3911\n",
            "Epoch [130/200], Step[26/28], Loss: 1.3839\n",
            "Epoch [130/200], Step[27/28], Loss: 1.3834\n",
            "Epoch [130/200], Step[28/28], Loss: 1.3823\n",
            "Epoch [140/200], Step[1/28], Loss: 1.3842\n",
            "Epoch [140/200], Step[2/28], Loss: 1.3830\n",
            "Epoch [140/200], Step[3/28], Loss: 1.3816\n",
            "Epoch [140/200], Step[4/28], Loss: 1.3801\n",
            "Epoch [140/200], Step[5/28], Loss: 1.3834\n",
            "Epoch [140/200], Step[6/28], Loss: 1.3789\n",
            "Epoch [140/200], Step[7/28], Loss: 1.3856\n",
            "Epoch [140/200], Step[8/28], Loss: 1.3856\n",
            "Epoch [140/200], Step[9/28], Loss: 1.3800\n",
            "Epoch [140/200], Step[10/28], Loss: 1.3821\n",
            "Epoch [140/200], Step[11/28], Loss: 1.3964\n",
            "Epoch [140/200], Step[12/28], Loss: 1.3873\n",
            "Epoch [140/200], Step[13/28], Loss: 1.3775\n",
            "Epoch [140/200], Step[14/28], Loss: 1.3898\n",
            "Epoch [140/200], Step[15/28], Loss: 1.3888\n",
            "Epoch [140/200], Step[16/28], Loss: 1.3967\n",
            "Epoch [140/200], Step[17/28], Loss: 1.4003\n",
            "Epoch [140/200], Step[18/28], Loss: 1.3950\n",
            "Epoch [140/200], Step[19/28], Loss: 1.3898\n",
            "Epoch [140/200], Step[20/28], Loss: 1.4037\n",
            "Epoch [140/200], Step[21/28], Loss: 1.3906\n",
            "Epoch [140/200], Step[22/28], Loss: 1.3863\n",
            "Epoch [140/200], Step[23/28], Loss: 1.3758\n",
            "Epoch [140/200], Step[24/28], Loss: 1.3894\n",
            "Epoch [140/200], Step[25/28], Loss: 1.3827\n",
            "Epoch [140/200], Step[26/28], Loss: 1.3966\n",
            "Epoch [140/200], Step[27/28], Loss: 1.3857\n",
            "Epoch [140/200], Step[28/28], Loss: 1.3770\n",
            "Epoch [150/200], Step[1/28], Loss: 1.3891\n",
            "Epoch [150/200], Step[2/28], Loss: 1.3849\n",
            "Epoch [150/200], Step[3/28], Loss: 1.3893\n",
            "Epoch [150/200], Step[4/28], Loss: 1.3869\n",
            "Epoch [150/200], Step[5/28], Loss: 1.3885\n",
            "Epoch [150/200], Step[6/28], Loss: 1.3858\n",
            "Epoch [150/200], Step[7/28], Loss: 1.3926\n",
            "Epoch [150/200], Step[8/28], Loss: 1.3948\n",
            "Epoch [150/200], Step[9/28], Loss: 1.3829\n",
            "Epoch [150/200], Step[10/28], Loss: 1.3858\n",
            "Epoch [150/200], Step[11/28], Loss: 1.3933\n",
            "Epoch [150/200], Step[12/28], Loss: 1.3886\n",
            "Epoch [150/200], Step[13/28], Loss: 1.3810\n",
            "Epoch [150/200], Step[14/28], Loss: 1.3858\n",
            "Epoch [150/200], Step[15/28], Loss: 1.3853\n",
            "Epoch [150/200], Step[16/28], Loss: 1.3836\n",
            "Epoch [150/200], Step[17/28], Loss: 1.3842\n",
            "Epoch [150/200], Step[18/28], Loss: 1.3913\n",
            "Epoch [150/200], Step[19/28], Loss: 1.3809\n",
            "Epoch [150/200], Step[20/28], Loss: 1.3847\n",
            "Epoch [150/200], Step[21/28], Loss: 1.3813\n",
            "Epoch [150/200], Step[22/28], Loss: 1.3885\n",
            "Epoch [150/200], Step[23/28], Loss: 1.3855\n",
            "Epoch [150/200], Step[24/28], Loss: 1.3817\n",
            "Epoch [150/200], Step[25/28], Loss: 1.3912\n",
            "Epoch [150/200], Step[26/28], Loss: 1.3827\n",
            "Epoch [150/200], Step[27/28], Loss: 1.3886\n",
            "Epoch [150/200], Step[28/28], Loss: 1.3956\n",
            "Epoch [160/200], Step[1/28], Loss: 1.3871\n",
            "Epoch [160/200], Step[2/28], Loss: 1.3774\n",
            "Epoch [160/200], Step[3/28], Loss: 1.3879\n",
            "Epoch [160/200], Step[4/28], Loss: 1.3906\n",
            "Epoch [160/200], Step[5/28], Loss: 1.3912\n",
            "Epoch [160/200], Step[6/28], Loss: 1.3817\n",
            "Epoch [160/200], Step[7/28], Loss: 1.3873\n",
            "Epoch [160/200], Step[8/28], Loss: 1.3944\n",
            "Epoch [160/200], Step[9/28], Loss: 1.3923\n",
            "Epoch [160/200], Step[10/28], Loss: 1.3844\n",
            "Epoch [160/200], Step[11/28], Loss: 1.3878\n",
            "Epoch [160/200], Step[12/28], Loss: 1.3824\n",
            "Epoch [160/200], Step[13/28], Loss: 1.3877\n",
            "Epoch [160/200], Step[14/28], Loss: 1.3892\n",
            "Epoch [160/200], Step[15/28], Loss: 1.3893\n",
            "Epoch [160/200], Step[16/28], Loss: 1.3871\n",
            "Epoch [160/200], Step[17/28], Loss: 1.3808\n",
            "Epoch [160/200], Step[18/28], Loss: 1.3849\n",
            "Epoch [160/200], Step[19/28], Loss: 1.3892\n",
            "Epoch [160/200], Step[20/28], Loss: 1.3844\n",
            "Epoch [160/200], Step[21/28], Loss: 1.3832\n",
            "Epoch [160/200], Step[22/28], Loss: 1.3824\n",
            "Epoch [160/200], Step[23/28], Loss: 1.3891\n",
            "Epoch [160/200], Step[24/28], Loss: 1.3893\n",
            "Epoch [160/200], Step[25/28], Loss: 1.3847\n",
            "Epoch [160/200], Step[26/28], Loss: 1.3834\n",
            "Epoch [160/200], Step[27/28], Loss: 1.3864\n",
            "Epoch [160/200], Step[28/28], Loss: 1.3879\n",
            "Epoch [170/200], Step[1/28], Loss: 1.3853\n",
            "Epoch [170/200], Step[2/28], Loss: 1.3930\n",
            "Epoch [170/200], Step[3/28], Loss: 1.3797\n",
            "Epoch [170/200], Step[4/28], Loss: 1.3861\n",
            "Epoch [170/200], Step[5/28], Loss: 1.3769\n",
            "Epoch [170/200], Step[6/28], Loss: 1.3840\n",
            "Epoch [170/200], Step[7/28], Loss: 1.3814\n",
            "Epoch [170/200], Step[8/28], Loss: 1.3906\n",
            "Epoch [170/200], Step[9/28], Loss: 1.3805\n",
            "Epoch [170/200], Step[10/28], Loss: 1.3857\n",
            "Epoch [170/200], Step[11/28], Loss: 1.3928\n",
            "Epoch [170/200], Step[12/28], Loss: 1.3879\n",
            "Epoch [170/200], Step[13/28], Loss: 1.3913\n",
            "Epoch [170/200], Step[14/28], Loss: 1.3926\n",
            "Epoch [170/200], Step[15/28], Loss: 1.3980\n",
            "Epoch [170/200], Step[16/28], Loss: 1.3846\n",
            "Epoch [170/200], Step[17/28], Loss: 1.4038\n",
            "Epoch [170/200], Step[18/28], Loss: 1.3894\n",
            "Epoch [170/200], Step[19/28], Loss: 1.3832\n",
            "Epoch [170/200], Step[20/28], Loss: 1.3795\n",
            "Epoch [170/200], Step[21/28], Loss: 1.3870\n",
            "Epoch [170/200], Step[22/28], Loss: 1.3901\n",
            "Epoch [170/200], Step[23/28], Loss: 1.3828\n",
            "Epoch [170/200], Step[24/28], Loss: 1.3800\n",
            "Epoch [170/200], Step[25/28], Loss: 1.3894\n",
            "Epoch [170/200], Step[26/28], Loss: 1.3860\n",
            "Epoch [170/200], Step[27/28], Loss: 1.3973\n",
            "Epoch [170/200], Step[28/28], Loss: 1.3867\n",
            "Epoch [180/200], Step[1/28], Loss: 1.3890\n",
            "Epoch [180/200], Step[2/28], Loss: 1.3810\n",
            "Epoch [180/200], Step[3/28], Loss: 1.3764\n",
            "Epoch [180/200], Step[4/28], Loss: 1.3833\n",
            "Epoch [180/200], Step[5/28], Loss: 1.3884\n",
            "Epoch [180/200], Step[6/28], Loss: 1.3844\n",
            "Epoch [180/200], Step[7/28], Loss: 1.3861\n",
            "Epoch [180/200], Step[8/28], Loss: 1.3766\n",
            "Epoch [180/200], Step[9/28], Loss: 1.3930\n",
            "Epoch [180/200], Step[10/28], Loss: 1.3859\n",
            "Epoch [180/200], Step[11/28], Loss: 1.3875\n",
            "Epoch [180/200], Step[12/28], Loss: 1.3894\n",
            "Epoch [180/200], Step[13/28], Loss: 1.3826\n",
            "Epoch [180/200], Step[14/28], Loss: 1.3965\n",
            "Epoch [180/200], Step[15/28], Loss: 1.3921\n",
            "Epoch [180/200], Step[16/28], Loss: 1.3883\n",
            "Epoch [180/200], Step[17/28], Loss: 1.3887\n",
            "Epoch [180/200], Step[18/28], Loss: 1.3953\n",
            "Epoch [180/200], Step[19/28], Loss: 1.3890\n",
            "Epoch [180/200], Step[20/28], Loss: 1.3818\n",
            "Epoch [180/200], Step[21/28], Loss: 1.3868\n",
            "Epoch [180/200], Step[22/28], Loss: 1.3910\n",
            "Epoch [180/200], Step[23/28], Loss: 1.3808\n",
            "Epoch [180/200], Step[24/28], Loss: 1.3925\n",
            "Epoch [180/200], Step[25/28], Loss: 1.3868\n",
            "Epoch [180/200], Step[26/28], Loss: 1.3849\n",
            "Epoch [180/200], Step[27/28], Loss: 1.3863\n",
            "Epoch [180/200], Step[28/28], Loss: 1.3741\n",
            "Epoch [190/200], Step[1/28], Loss: 1.3870\n",
            "Epoch [190/200], Step[2/28], Loss: 1.3869\n",
            "Epoch [190/200], Step[3/28], Loss: 1.3798\n",
            "Epoch [190/200], Step[4/28], Loss: 1.3882\n",
            "Epoch [190/200], Step[5/28], Loss: 1.3907\n",
            "Epoch [190/200], Step[6/28], Loss: 1.3889\n",
            "Epoch [190/200], Step[7/28], Loss: 1.3927\n",
            "Epoch [190/200], Step[8/28], Loss: 1.3889\n",
            "Epoch [190/200], Step[9/28], Loss: 1.3814\n",
            "Epoch [190/200], Step[10/28], Loss: 1.3909\n",
            "Epoch [190/200], Step[11/28], Loss: 1.3777\n",
            "Epoch [190/200], Step[12/28], Loss: 1.3861\n",
            "Epoch [190/200], Step[13/28], Loss: 1.3907\n",
            "Epoch [190/200], Step[14/28], Loss: 1.3906\n",
            "Epoch [190/200], Step[15/28], Loss: 1.3791\n",
            "Epoch [190/200], Step[16/28], Loss: 1.3917\n",
            "Epoch [190/200], Step[17/28], Loss: 1.3873\n",
            "Epoch [190/200], Step[18/28], Loss: 1.3873\n",
            "Epoch [190/200], Step[19/28], Loss: 1.3809\n",
            "Epoch [190/200], Step[20/28], Loss: 1.3799\n",
            "Epoch [190/200], Step[21/28], Loss: 1.3885\n",
            "Epoch [190/200], Step[22/28], Loss: 1.3919\n",
            "Epoch [190/200], Step[23/28], Loss: 1.3846\n",
            "Epoch [190/200], Step[24/28], Loss: 1.3850\n",
            "Epoch [190/200], Step[25/28], Loss: 1.3809\n",
            "Epoch [190/200], Step[26/28], Loss: 1.3872\n",
            "Epoch [190/200], Step[27/28], Loss: 1.3850\n",
            "Epoch [190/200], Step[28/28], Loss: 1.3853\n"
          ]
        }
      ],
      "source": [
        "# training the transformer model\n",
        "train_loss_history_tf = trainer.train(tfclassifier, trainloader, learning_rate=learn_rate,\n",
        "                                        num_epochs=200, batch_size=batch_size, device=device)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 30,
      "metadata": {
        "id": "yD9wXSLSGzRK",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 296
        },
        "outputId": "7936dfec-072f-44fc-925f-17764d227b1a"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<matplotlib.legend.Legend at 0x7f61954b2610>"
            ]
          },
          "metadata": {},
          "execution_count": 30
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEGCAYAAABy53LJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXxV9Z3/8dfnZiVkAcK+GRBUdtQIuFBtLRatFfetTrWjtdOp82t1tNpfHUVm6oydVluXjsVqx9a2Lij+sGq1arWuSEAUECwBEYIsYUsIIfvn98c5iTe5WW6Am1zh/Xw88sg953zPuZ977s395Luc7zF3R0REJF6R7g5AREQ+X5Q4RESkU5Q4RESkU5Q4RESkU5Q4RESkU1K7O4Cu0LdvXy8oKOjuMEREPjcWL168zd37tbbtkEgcBQUFFBUVdXcYIiKfG2b2SVvb1FQlIiKdosQhIiKdosQhIiKdckj0cYi0pba2lpKSEqqqqro7FJFukZmZydChQ0lLS4t7HyUOOaSVlJSQk5NDQUEBZtbd4Yh0KXdn+/btlJSUMGLEiLj3U1OVHNKqqqrIz89X0pBDkpmRn5/f6Rq3Eocc8pQ05FC2L59/JY523PPyal77e2l3hyEiklQSmjjMbKaZfWRmxWZ2UyvbM8zssXD7QjMraLF9uJlVmNn1Ueu+Z2bLzWyFmX0/kfH/z2treGO1Eockzq5du/jlL3+5T/ueccYZ7Nq1q90yt9xyCy+99NI+Hb+lgoICtm3bdkCOlYz2573orEsuuYSJEydy1113dcnzHWgJSxxmlgLcB5wOjAUuMbOxLYpdCex091HAXcAdLbbfCTwfdczxwLeAKcAk4EwzG5WYVwARMxp0nytJoPa+rOrq6trd97nnnqNXr17tlpkzZw5f/vKX9zm+ZNfROeqM/XkvOmPz5s0sWrSIDz74gGuvvTaufQ7k8wPU19fv1/6JrHFMAYrdfa271wCPArNalJkFPBw+ngecamGDm5mdDXwMrIgqPwZY6O6V7l4HvAacm6gXYAYNukOiJNBNN93EmjVrmDx5MjfccAOvvvoq06dP56yzzmLs2OD/rLPPPptjjz2WcePGMXfu3KZ9G2sA69atY8yYMXzrW99i3LhxnHbaaezduxeAK664gnnz5jWVv/XWWznmmGOYMGECq1atAqC0tJQZM2Ywbtw4rrrqKg477LAOaxZ33nkn48ePZ/z48fz85z8HYM+ePXz1q19l0qRJjB8/nscee6zpNY4dO5aJEydy/fXXxxyroqKCb37zm0yYMIGJEyfy5JNPApCdnd1UZt68eVxxxRVNr+mf/umfmDp1Kj/4wQ8oKChoVvMaPXo0W7ZsobS0lPPOO4/jjjuO4447jjfffDNh70V2djY/+tGPmDRpEtOmTWPLli0APPHEE4wfP55JkybxhS98AYDTTjuNjRs3MnnyZF5//XWWLl3KtGnTmDhxIueccw47d+4E4JRTTuH73/8+hYWF/OIXv+CUU07h2muvpbCwkDFjxrBo0SLOPfdcRo8ezc0339wUyyOPPMKUKVOYPHky3/72t5uSRHZ2Nv/6r//KpEmTePvtt9s9Fx1J5HDcIcCGqOUSYGpbZdy9zszKgHwzqwJuBGYA0Z+05cCPzSwf2AucAbQ6CZWZXQ1cDTB8+PB9egERM5Q3Dh23PbOCDz8tP6DHHDs4l1u/Nq7N7f/1X//F8uXLWbp0KQCvvvoqS5YsYfny5U3DIx966CH69OnD3r17Oe644zjvvPPIz89vdpzVq1fzxz/+kQceeIALL7yQJ598kssuuyzm+fr27cuSJUv45S9/yU9/+lN+/etfc9ttt/GlL32JH/7wh/z5z3/mwQcfbPc1LV68mN/85jcsXLgQd2fq1KmcfPLJrF27lsGDB/Pss88CUFZWxvbt25k/fz6rVq3CzFptWvv3f/938vLyWLZsGUDTF2d7SkpKeOutt0hJSaG+vp758+fzzW9+k4ULF3LYYYcxYMAALr30Uq699lpOOukk1q9fz1e+8hVWrlxJUVER999/P7/+9a8P2HuxZ88epk2bxo9//GN+8IMf8MADD3DzzTczZ84cXnjhBYYMGdL02hcsWMCZZ57Z9DwTJ07knnvu4eSTT+aWW27htttua0rGNTU1TfPsPfPMM6Snp1NUVMQvfvELZs2axeLFi+nTpw+HH3441157LVu3buWxxx7jzTffJC0tjX/+53/m97//Pd/4xjfYs2cPU6dO5Wc/+1mH57cjydo5Phu4y90role6+0qC5qwXgT8DS4FW61zuPtfdC929sF+/Vid47FBENQ7pBlOmTGk2pv7uu+9u+k92w4YNrF69OmafESNGMHnyZACOPfZY1q1b1+qxzz333Jgyb7zxBhdffDEAM2fOpHfv3u3G98Ybb3DOOefQs2dPsrOzOffcc3n99deZMGECf/nLX7jxxht5/fXXycvLIy8vj8zMTK688kqeeuopsrKyYo730ksv8d3vfrdpuaPnB7jgggtISUkB4KKLLmqq3Tz66KNcdNFFTce95pprmDx5MmeddRbl5eVUVFRQWFgYkzTaEu97kZ6ezplnngk0P7cnnngiV1xxBQ888ECrzUNlZWXs2rWLk08+GYDLL7+cv/3tb03bG19Lo7POOguACRMmMG7cOAYNGkRGRgYjR45kw4YNvPzyyyxevJjjjjuOyZMn8/LLL7N27VoAUlJSOO+88+J63R1JZI1jIzAsanlouK61MiVmlgrkAdsJaibnm9lPgF5Ag5lVufu97v4g8CCAmd1OUJNJiKCPQ4njUNFezaAr9ezZs+nxq6++yksvvcTbb79NVlYWp5xySqtj7jMyMpoep6SkNDVVtVUuJSXlgLebH3HEESxZsoTnnnuOm2++mVNPPZVbbrmFd999l5dffpl58+Zx77338sorr8R1vOhhoi1fc/Q5Ov744ykuLqa0tJSnn366qdmmoaGBd955h8zMzH1+TfG+F2lpaU3xRp/b+++/n4ULF/Lss89y7LHHsnjx4n1+fvjs/YtEIs3e80gkQl1dHe7O5Zdfzn/+53/GHCszM7Mp2e6vRNY4FgGjzWyEmaUDFwMLWpRZAFwePj4feMUD0929wN0LgJ8Dt7v7vQBm1j/8PZygf+MPiXoBps5xSbCcnBx2797d5vaysjJ69+5NVlYWq1at4p133jngMZx44ok8/vjjALz44osdNhVNnz6dp59+msrKSvbs2cP8+fOZPn06n376KVlZWVx22WXccMMNLFmyhIqKCsrKyjjjjDO46667eP/992OON2PGDO67776m5cbnHzBgACtXrqShoYH58+e3GY+Zcc4553DdddcxZsyYpma80047jXvuuaepXGPTUFsS8V6sWbOGqVOnMmfOHPr168eGDRuabc/Ly6N37968/vrrAPzud79rqn3si1NPPZV58+axdetWAHbs2MEnn7Q5O/o+S1jiCDuvrwFeAFYCj7v7CjObY2ZnhcUeJOjTKAauA2KG7LbiSTP7EHgG+K67tz8ecT9ELLgkXyRR8vPzOfHEExk/fjw33HBDzPaZM2dSV1fHmDFjuOmmm5g2bdoBj+HWW2/lxRdfZPz48TzxxBMMHDiQnJycNssfc8wxXHHFFUyZMoWpU6dy1VVXcfTRR7Ns2bKmTtnbbruNm2++md27d3PmmWcyceJETjrpJO68886Y4918883s3LmzqRP5r3/9KxD0OZx55pmccMIJDBo0qN3XcNFFF/HII480a9q5++67KSoqYuLEiYwdO5b7778fgKKiIq666qqYYyTivbjhhhuYMGEC48eP54QTTmDSpEkxZR5++GFuuOEGJk6cyNKlS7nllls6PG5bxo4dy3/8x39w2mmnMXHiRGbMmMGmTZv2+XhtsUPhi7GwsND35UZO025/mZOP6Mcd509MQFSSDFauXMmYMWO6O4xuVV1dTUpKCqmpqbz99tt85zvf6fC/czm4tPZ3YGaL3b2wtfKa5LAd6hyXQ8H69eu58MILaWhoID09nQceeKC7Q5Ikp8TRDvVxyKFg9OjRvPfee90dhnyOJOtw3KQQiaiP41Cg91gOZfvy+VfiaIeG4x78MjMz2b59u5KHHJIa78fR2SHLaqpqh+aqOvgNHTqUkpISSks1maUcmhrvANgZShzt0FxVB7+0tLRO3flMRNRU1S7NVSUiEkuJox0ajisiEkuJox3qHBcRiaXE0Q5dxyEiEkuJox2aq0pEJJYSRzs0HFdEJJYSRzvUOS4iEkuJox3q4xARiaXE0Q71cYiIxFLiaIeG44qIxFLiaEfEjIaG7o5CRCS5KHG0Q3NViYjEUuJoh+aqEhGJpcTRjkhENQ4RkZaUONqhznERkVhKHO3QdRwiIrGUONqh6zhERGIpcbRDc1WJiMRS4miH5qoSEYmV0MRhZjPN7CMzKzazm1rZnmFmj4XbF5pZQYvtw82swsyuj1p3rZmtMLPlZvZHM8tMYPyqcYiItJCwxGFmKcB9wOnAWOASMxvbotiVwE53HwXcBdzRYvudwPNRxxwC/B+g0N3HAynAxYl5BerjEBFpTSJrHFOAYndf6+41wKPArBZlZgEPh4/nAaeamQGY2dnAx8CKFvukAj3MLBXIAj5NUPwajisi0opEJo4hwIao5ZJwXatl3L0OKAPyzSwbuBG4Lbqwu28EfgqsBzYBZe7+YmtPbmZXm1mRmRWVlpbu0wtQ57iISKxk7RyfDdzl7hXRK82sN0EtZQQwGOhpZpe1dgB3n+vuhe5e2K9fv30KwgwalDlERJpJTeCxNwLDopaHhutaK1MSNj3lAduBqcD5ZvYToBfQYGZVwBbgY3cvBTCzp4ATgEcS8QLUVCUiEqvDGoeZXWBmOeHjm83sKTM7Jo5jLwJGm9kIM0sn6MRe0KLMAuDy8PH5wCsemO7uBe5eAPwcuN3d7yVooppmZllhX8ipwMo4YtknwXDcRB1dROTzKZ6mqn9z991mdhLwZeBB4H862inss7gGeIHgy/1xd19hZnPM7Kyw2IMEfRrFwHVAzJDdFsdcSNCJvgRYFsY/N47XsE8iEdU4RERaiqepqj78/VVgrrs/a2b/Ec/B3f054LkW626JelwFXNDBMWa3WL4VuDWe599fmlZdRCRWPDWOjWb2K+Ai4Dkzy4hzv889XTkuIhIrngRwIUFz01fcfRfQB7ghoVElCXWOi4jEiqepahDwrLtXm9kpwETgtwmNKkloyhERkVjx1DieBOrNbBRBR/Qw4A8JjSpJaMoREZFY8SSOhnCE1LnAPe5+A0Et5KCnK8dFRGLFkzhqzewS4BvAn8J1aYkLKXmoc1xEJFY8ieObwPHAj939YzMbAfwusWElBzPTlCMiIi10mDjc/UPgemCZmY0HSty95fTnByVdxyEiEqvDUVXhSKqHgXWAAcPM7HJ3/1tiQ+t+aqoSEYkVz3DcnwGnuftHAGZ2BPBH4NhEBpYMgilHujsKEZHkEk8fR1pj0gBw979ziHSOm2ocIiIx4qlxFJnZr/ls6vKvA0WJCyl5qI9DRCRWPInjO8B3Ce71DfA68MuERZRE1MchIhKrw8Th7tXAneHPIUVzVYmIxGozcZjZMqDNb013n5iQiJKI5qoSEYnVXo3jzC6LIklFLPjt7gQ3HBQRkTYTh7t/0pWBJKNImCwaHFKUN0REgEPkhkz7qrHGoX4OEZHPKHG0w5pqHEocIiKN4kocZtbDzI5MdDDJprGpSnlDROQzHSYOM/sasBT4c7g82cwWJDqwZKCmKhGRWPHUOGYDU4BdAO6+FBiRwJiSRnTnuIiIBOK6kZO7l7VYd0h8lZpqHCIiMeKZcmSFmV0KpJjZaIKpR95KbFjJoamPo6GbAxERSSLx1Dj+BRgHVBNMp14OfD+eg5vZTDP7yMyKzeymVrZnmNlj4faFZlbQYvtwM6sws+vD5SPNbGnUT7mZxRXLvlAfh4hIrHjmqqoEfhT+xM3MUoD7gBlACbDIzBaEdxRsdCWw091HmdnFwB3ARVHb7wSej4rlI2By1PE3AvM7E1dnRCIajisi0lJ7c1U9Q/tzVZ3VwbGnAMXuvjY83qPALCA6ccwi6HwHmAfca2bm7m5mZwMfA3vaOP6pwJpEXuFu6hwXEYnRXo3jp/t57CHAhqjlEmBqW2Xcvc7MyoB8M6sCbiSorVzfxvEvJmg6a5WZXQ1cDTB8+PB9ib/ZXFUiIhJob66q1xofm1k6cBRBDeQjd69JcFyzgbvcvaK1yQXDeM4CftjWAdx9LjAXoLCwcJ+++TUcV0QkVod9HGb2VeB+YA1gwAgz+7a7P9/+nmwEhkUtDw3XtVamxMxSgTxgO0HN5Hwz+wnQC2gwsyp3vzfc73Rgibtv6Sj+/aHOcRGRWPEMx/0Z8EV3LwYws8OBZ4nqtG7DImC0mY0gSBAXA5e2KLMAuBx4GzgfeMWDdqHpjQXMbDZQEZU0AC6hnWaqA0VzVYmIxIoncexuTBqhtcDujnYK+yyuAV4AUoCH3H2Fmc0Bitx9AfAg8DszKwZ2ECSXdplZT4K+j2/HEft+0VxVIiKx4kkcRWb2HPA4QR/HBQRDa88FcPen2trR3Z8Dnmux7paox1Xh8drk7rNbLO8B8uOIe7+pqUpEJFY8iSMT2AKcHC6XAj2ArxEkkjYTx+edOsdFRGLFcwHgN7sikGSkuapERGLFM6pqBMG0IwXR5eO4APBz77M+DiUOEZFG8TRVPU3Qif0McEhN99eYOOoPqVctItK+eBJHlbvfnfBIkpA6x0VEYsWTOH5hZrcCLxLMkAuAuy9JWFRJQtdxiIjEiidxTAD+AfgSnzVVebh8UEuJ6DoOEZGW4kkcFwAju2B+qqSjpioRkVjx3MhpOcF8UYccXcchIhIrnhpHL2CVmS2ieR/HQT8cV9dxiIjEiidx3JrwKJKUruMQEYkVz5Xjr3VU5mClpioRkVgd9nGY2TQzW2RmFWZWY2b1ZlbeFcF1t6bOcWUOEZEm8XSO30tw/4vVBJMbXgXcl8igkoXuOS4iEiuexEF4P44Ud693998AMxMbVnLQPcdFRGLF0zleGd7je2l4K9dNxJlwPu8iEdU4RERaiicB/ENY7hpgD8E9ws9LZFDJQhcAiojEimdU1SfhwyozuxsY1uJWsgctzVUlIhIrnlFVr5pZrpn1AZYAD5jZnYkPrfvpnuMiIrHiaarKc/dy4Fzgt+4+FfhyYsNKDmqqEhGJFU/iSDWzQcCFwJ8SHE9S0QWAIiKx4kkcc4AXgGJ3X2RmIwmu6Tjoaa4qEZFY8XSOPwE8EbW8lkNmVJXmqhIRaalT12OY2UF/179oaqoSEYnV2Qv5LCFRJCl1jouIxGo3cZhZipldG7Xq2c4c3MxmmtlHZlZsZje1sj3DzB4Lty80s4IW24eHkyteH7Wul5nNM7NVZrbSzI7vTEydjB9QjUNEJFq7icPd6wkmOGxcvjneA5tZCsFkiKcDY4FLzGxsi2JXAjvdfRRwF3BHi+13As+3WPcL4M/ufhQwCVgZb0ydpbmqRERixTNX1Ztmdi/wGMGUIwC4e0f9HVMIRmKtBTCzR4FZwIdRZWYBs8PH84B7zczc3c3sbODj6Oc0szzgC8AVYQw1QMLuhR7RleMiIjHiSRyTw99zotY58KUO9hsCbIhaLgGmtlXG3evMrAzIN7Mq4EZgBnB9VPkRQCnwGzObBCwGvufue2jBzK4GrgYYPnx4B6G2rilxNOzT7iIiB6V4huN+sSsCaWE2cJe7VzT2M4RSgWOAf3H3hWb2C+Am4N9aHsDd5wJzAQoLC/epyqDrOEREYnWYOMxsAHA7MNjdTw/7KY539wc72HUjwUy6jYaG61orU2JmqUAesJ2gZnJ+OI17L6AhrIXMA0rcfWG4/zyCxJEQjdOqK2+IiHwmnuG4/0tw5fjgcPnvwPfj2G8RMNrMRoT387gYWNCizALg8vDx+cArHpju7gXuXgD8HLjd3e91983ABjM7MtznVJr3mRxQGo4rIhIrnsTR190fBxog6IsA6jvaKSx3DUHSWQk87u4rzGyOmZ0VFnuQoE+jGLiO+GoP/wL83sw+IOh/uT2OffaJLgAUEYkVT+f4HjPLJ+gQx8ymAWXxHNzdnwOea7HulqjHVcAFHRxjdovlpUBhPM+/v9THISISK57EcR1Bk9LhZvYm0I+gWemgp7mqRERixTOqaomZnQwcSTDlyEfuXpvwyJKAmqpERGLFM6oqE/hn4CSC5qrXzez+sJnpoKbOcRGRWPE0Vf0W2A3cEy5fCvyODvomDgaaq0pEJFY8iWO8u0fPMfVXM0vYENhkormqRERixTMcd0k4kgoAM5sKFCUupOShuapERGLFU+M4FnjLzNaHy8OBj8xsGeDuPjFh0XUzdY6LiMSKJ3HMTHgUSUrXcYiIxIpnOO4nXRFIMvrsOo5uDkREJIl09taxh5TGzvF6tVWJiDRR4miHOsdFRGLFlTjM7DAz+3L4uIeZ5SQ2rOTwWR9H98YhIpJMOkwcZvYtgvte/CpcNRR4OpFBJQszw0zXcYiIRIunxvFd4ESgHMDdVwP9ExlUMkkxU1OViEiUeBJHtbvXNC6Ed+o7ZL5JI2ZqqhIRiRJP4njNzP4v0MPMZgBPAM8kNqzkYabOcRGRaPEkjpuAUmAZ8G2CGzPdnMigkknETNdxiIhEiecCwAbggfDnkBMxaFBblYhIk3jux7GM2D6NMoKJDv/D3bcnIrBkoT4OEZHm4pmr6nmgHvhDuHwxkAVsBv4X+FpCIksS6uMQEWkunsTxZXc/Jmp5mZktcfdjzOyyRAWWLCIR03UcIiJR4ukcTzGzKY0LZnYckBIu1iUkqiSipioRkebiqXFcBTxkZtmAEVwIeJWZ9QT+M5HBJYOImqpERJqJZ1TVImCCmeWFy2VRmx9PVGDJwlTjEBFpJp4aB2b2VWAckGlN96jwOXHsNxP4BUHT1q/d/b9abM8Afktwl8HtwEXuvi5q+3DgQ2C2u/80XLcO2E3QYV/n7oXxvIZ9FdFcVSIizcQzyeH9wEXAvxA0VV0AHBbHfinAfcDpwFjgEjMb26LYlcBOdx8F3AXc0WL7nQSjulr6ortPTnTSgMY+DiUOEZFG8XSOn+Du3yD4gr8NOB44Io79pgDF7r42nOvqUWBWizKzgIfDx/OAUy2s0pjZ2cDHwIo4nith1DkuItJcPImjKvxdaWaDgVpgUBz7DQE2RC2XhOtaLePudQQXFuaHHfE3Are1clwHXjSzxWZ2dRxx7BddxyEi0lw8fRzPmFkv4L+BJQRf3ImefmQ2cJe7VzT2qUQ5yd03mll/4C9mtsrd/9ayUJhUrgYYPnz4PgeiuapERJprN3GYWQR42d13AU+a2Z+AzBYjq9qyERgWtTw0XNdamZJwuvY8gk7yqcD5ZvYToBfQYGZV7n6vu28EcPetZjafoEksJnG4+1xgLkBhYeE+f/VrOK6ISHPtNlWFExzeF7VcHWfSAFgEjDazEWaWTjBVyYIWZRYAl4ePzwde8cB0dy9w9wLg58Dt7n6vmfVsvG1teB3JacDyOOPZJ+rjEBFpLp6mqpfN7DzgKe/EuFR3rzOza4AXCIbjPuTuK8xsDlDk7guAB4HfmVkxsIMgubRnADA/bL5KBf7g7n+ON6Z9oT4OEZHm4kkc3wauA+rNbC/BkFx399yOdnT35wju3xG97paox1UEw3vbO8bsqMdrgUlxxHzABH0cShwiIo3iuXI8pysCSVYRMxoaujsKEZHkEc8FgGZml5nZv4XLw6InPTzYqalKRKS5eK7j+CXBRX+XhssVRHWYH+zUOS4i0lw8fRxTw3tvvAfg7jvDUVKHhEhEc1WJiESLp8ZRG8475QBm1g84ZFr9NVeViEhz8SSOu4H5QH8z+zHwBnB7QqNKIppWXUSkuXhGVf3ezBYDpxIMxT3b3VcmPLIkoSvHRUSa6zBxmNndwKPufsh0iEfTXFUiIs3F01S1GLjZzNaY2U/NLOH3wEgmqnGIiDTXYeJw94fd/QzgOOAj4A4zW53wyJKEqXNcRKSZeGocjUYBRxHc/W9VYsJJPkGNo7ujEBFJHvFcOf6TsIYxh2Am2kJ3/1rCI0sSmqtKRKS5eC4AXAMc7+7bEh1MMtKV4yIizcUzHPdXZtY7nJ8qM2p9zM2TDkaaq0pEpLl4huNeBXyP4A5+S4FpwNvAlxIbWnJQjUNEpLl4Ose/RzCi6hN3/yJwNLAroVElkYhprioRkWjxJI6q8IZLmFmGu68CjkxsWMkjYka9qhwiIk3i6RwvMbNewNPAX8xsJ/BJYsNKHpqrSkSkuXg6x88JH842s78CeUBC7/OdTNRUJSLSXDw1jibu/lqiAklWmlZdRKS5zlw5fkhKiaipSkQkmhJHB3Qdh4hIc0ocHdC06iIizSlxdEDTqouINKfE0QF1jouINJfQxGFmM83sIzMrNrObWtmeYWaPhdsXmllBi+3DzazCzK5vsT7FzN4zsz8lMv7wuWhoSPSziIh8fiQscZhZCnAfcDowFrjEzMa2KHYlsNPdRwF3AXe02H4n8Hwrh/8e0CX3Pdd1HCIizSWyxjEFKHb3te5eAzwKzGpRZhbwcPh4HnCqmRmAmZ0NfAysiN7BzIYCXwV+ncDYm2iSQxGR5hKZOIYAG6KWS8J1rZZx9zqgDMg3s2zgRuC2Vo77c+AHQLsNSGZ2tZkVmVlRaWnpvr0CIBJR57iISLRk7RyfDdzl7hXRK83sTGCruy/u6ADuPtfdC929sF+/fvsciOaqEhFprlNTjnTSRmBY1PLQcF1rZUrMLJVgHqztwFTgfDP7CdALaDCzKoIayllmdgbBTaVyzewRd78sUS9CfRwiIs0lMnEsAkab2QiCBHExcGmLMguAywluDHU+8IoH39LTGwuY2Wygwt3vDVf9MFx/CnB9IpMGaDiuiEhLCUsc7l5nZtcALwApwEPuvsLM5gBF7r4AeBD4nZkVAzsIkktSUee4iEhziaxx4O7PAc+1WHdL1OMq4IIOjjG7jfWvAq/ub4wd0VxVIiLNJWvneNLQXFUiIs0pcXRAc1WJiDSnxNEBdY6LiM0fwMMAABP+SURBVDSnxNEBXcchItKcEkcHdB2HiEhzShwd0HBcEZHmlDg6oM5xEZHmlDg6YOFwXDVXiYgElDg6EAlmede1HCIiISWODkSCvKHmKhGRkBJHByJh5lAHuYhIQImjA6Yah4hIM0ocHVAfh4hIc0ocHVAfh4hIc0ocHWiscXRH4thWUc2nu/Z+7oYCxxtvQ4OzrKSMhi7oQKqrb/cW9R2qrqtnTWlFxwWBiuo6yvbWtrm9rLKWPyxcz69fX0ttO3Ft3LWXt4q37fP5ORCfG3fnLx9u4Zn3P93vc3ig1dQ1sGNPTZc9X7L+HS5Zv5O3ird1aXwJvR/HwcDCxPHfL3zEByVlVFTX8fOLJjNucC479tSQn53R6n479tSwalM563dU8qWj+rOjsobrHnufLx7Vj0unHsZzH2zirTXbKK+q4+tThzO4Vw/e37CL55dvJiczlf45mSx4fyO19U5OZioDcjM5amAOM8cPZNLQXuysrGHh2h0M69ODIwbk0CsrnZq6BnZX1bJ9Tw1vFm9j2cYyKqrqGJ6fRW5mGq+vLuXYw3rzozPGkpeVxgclu3izeDsF+Vks3bCLhR/v4NKpw8nrkcafl2/myIE5TB7Wi/45GaRGIiz/tIyXVm4hMy2FcYNzOe+YoWSkRliyfidPLtlI6e5qtldUs2rzbqaO6MMNXzmKvbX1VNbUNTs3qZEIQ3r14PbnVvLnFZv58pgBjBmUw2/eXMcxh/VmSkFvtlXUMKRXD8YOzmV0/2zWba9kS3kVvbPSeeq9Epau38XEoXkcNSiXYb2zGJiXwbMfbObdddv51vSRHH94PsVbKjAznlpSwpNLSuiXk8GYQbkc3i+b0t3V1DU0cHi/bEb1z6Zsby3PfrCJiuo6+uVkcPFxwxiQm0nZ3lp2VdZy98urWbttD9+aPoIvHtmfj7fvYcygXLbtruaDkjIG5mUybnAuDnznkcXUNzgP/+MU8ntmsGpzOWV7a+mRlkLRJzt5+K11VNcFX8Ivr9zKP540gqz0FLLSUzhyYA4V1XXc8MQHvPb3UgBmjB3A8SPzeWHFZv6+ZTcRM44alMPp4wdxxoRB1DU08IN5H/D3zbuZOX4QsyYPZu22CmYv+JDpo/ty7YwjcHdWfFrO5rIq+udmsGJjOcWlFRxX0IcRfXuSlhIhLcUwMyqq6vigZBebyqoo3lrBh5vKARjeJ4sLC4fyhSP6ETFj7t/WUrKzkvOPHYZZ8JnvlZXGW8Xb+bRsL989ZRQnjurL6q27efq9TxnRN4tzjhnK/Pc2kp2RwtcmDmbjrr08/d6nvL66lJH9ejIwN5PyqjpSIsbA3EyOGJjDA39by/JPy5gwJI/+OZn0zkpjUK8ePPLOJ6zbvodzjx7K5GF5bCmvZltFNUcOzOELR/TjhRWbyeuRxhnjB/HssuC9PXJADu98vJ0NOypJS4kwa/Jgvnhkf2rrnWfe/5Ti0goO75fNnjD5Z6WnsH5HJR+UlLFyUzl9szM49rDeFBb0pqaugdLd1RxX0Aez4At86YZdZGekcubEwUwe1ovKmnqWbSzjqIE5jB2U2zTYxt2pqW/g6fc2snRDGUN796C8qpbyvXUclp9F2d5adlTUMHZwLscfns/o/tkAPLZoA3NfX8ulU4YzZlAuf3x3PX/6YBMAUwr6cPqEgaSnRli5qZzjCvowc/xAMlJTDuyXImDJmkUPpMLCQi8qKtqnff/47np++NQy0lKMo4f1Zv2OSuoaGhjVP5t31u7gnkuO5muTBjeV37Cjkm889C4fb9vTtC4nIxUs6CepqP7sS3RU+GEo3vrZf7ITh+axp7qO9TsqubBwGEcOzKF4awVby6tZtG4H2+P8DyticNTAXHIyUyneWsHuqjqOHt6Lok92kp4SIbdHKlvKq5vKp0SMYb17sG57JQC5mamUV9XFHLd3VhoAOytr6ZudQX1DAzsra+mZnsKwPlnk9UijIL8nf/rgU/bU1HcY41mTBvPMB5uob3C+eGQ/Vm7azebyKnqmp7S5f4+0FKaO7MOHn5azdXfbr6FRekqE844dSnVtPR9uKmfttj1hMjTW76hsGjF31MAchvTqwcpN5XxaVtXsGMP7ZHHsYb2Z/97Gdl8TwJBePXB3Siuqqa1v/vdlBudMHsI/njSC1Vt3c+OTy6ip++w/+az0FDLTUthbU88/n3I4qSkRfvriR9Q3OGMH5TJpWC8aGpwl63eyOvzcpKdEiERg2sh83ireTk1YMzhqYA5rS/c0Lbc8J0P79GBt6Z6YbY3bB/fKJK9HGl+fdhi9s9KZ+7c1LFq3s6lMj7QUhvTu0ezzC5DfM53szFQ+iXofUiNGXYOTlmJN5yQnM5Xd4WdswpA8SnZWsmtvLdkZqTQ0eNP73ysrjVOPGsBHW8rZuaeWbRXVVNc1cHi/nhx/eD6PLyqhpr6BiEGvrPQOayGpEaOgb092VdawraKG3llp1NY7FdV14UwRzctnZ6QydnAuYwflUlpRTdG6HU1/O42vC4LP35hBOWwtr272uWzUMz2Fw/J7sqW8ih2VNaRFItTUN5CTkcru6jrSUyL0zEhhZ2UtqREjJzOVnZVBzXVAbgY5mWkUb61gcF5m0+czMy3Ct79wOH2z07n/tbVs3LUXgIzUCNV1DQzKy+TVG07Zp+RhZovdvbDVbUoc7ausqWPRup0cPbwXueEbd+Gv3sbd6Z+TySc79vDUd05k7OBcIPhv89WPSrluxhGMGZRLXo80fvriR2wpr+KBbxSyprSC9zeUceakQRzeL5uGBuedtdupa3BG9c9mcK8eANQ3OCmNHSyh+gbn/ZJdrNq0m8y0CCeN6svGXXtZt30PuypryUhNISczlZzMVCYN7UXvnulA8N9Nbb2Tnhph+cYynijawO7qOiYMyePM8L++AbkZDMjJ5MUPN9PgcNrYAWzfU8PqLRWUVlRR3wCD8jKZNjKflIjxVvE2Hn57Hb16pFNY0JszJgyiZ8ZnFdhPd+3ltb+XMjA3k5zM5hXbqtoG1m6rYMygXI4r6MP7G3axt7aeaSPzqW9wquvqyUpPZXtFNSs37Wb11t0M7Z3FsD492FJezfjBuU01vd1VtWzYsZeNu/Zy5IAchvTuwZNLSijfW8uYQbk0uDOyXzZDwvPaUnVdPeu2VRIxGD0gBwiatd5cs536hgbyeqSTnhJh9IBsMtNSeHvNdvbW1jGqXw4fbiojJzONwrCGtHT9LtZt38PFxw2jqq6Be18pZnifLAoLetM7K52q2nryeqQxrE9W0/Nvq6hmc1kVlTX1lO2t5ZVVWynZWcmPvjqGowYGn6m1pRXU1jtHDsxp2s/dWb6xnLfWbKNk516+Pm04Rw3MpWxvLS8s30yDOxcUDmP9jkreWrONnumpjOqfzbA+WWwtr2JgXiY5mWlsq6hme0UNtfUNTbWgzLQIo/pnt/pls3HXXpZvLKN8by2nHNmfvtnprPi0nNzMNPrnZlC6u5qBeZkAzH9vI9srauibnc5p4wby9pptvLxyK+cfO5Rde2t5ftkmJg7txYyxAxjWJwt3x/2zIfCby6p4v2QXU0f0oVdWerO/g01lexmYm0lqSoSde2qobWggv2cGKRHj/Q27WLRuBzPGDmDjrr28+lEpXxk3gOF9evL3LbsZPySPvB5p1NY3MP+9jby3fhcpEZgxdiDHj8xnw85KcjJT6dUjncqaOnIz05piajz3n5ZVNf0DtviTnaSYMXFoL3qkpzT9nX60eTepkWD9h5vKeH9DGR9v28PA3Ez652ZQWVPPyUf0Y/rovuytrScjNYWUiFFeFdROUyNGyc69vL56G0XrdrCzsoapI/P51vSRFK3bwe6qOk4YlU9W+md/X5vK9lJT18DQ3lm8UbyN1Vt2c9X0ka1+9juixLEfiaM1O/bUkJ4aobKmjrPueZOsjBSe/9503lu/i4vnvsN1M47g/5w6+oA9n4hIV2svcahzfB/06ZlOdkbQD/Ff501gbekebnl6Bdc9tpTBeZl8ax8zvIjI54ESx3465cj+fG3SYB4r2kBtgzP3G4X0SD/wnVEiIslCo6oOgNlfG0v/nAyuOKGgWfu1iMjBSInjAMjPzuDfzhzb3WGIiHQJNVWJiEinJDRxmNlMM/vIzIrN7KZWtmeY2WPh9oVmVtBi+3AzqzCz68PlTDN718zeN7MVZnZbIuMXEZFYCUscZpYC3AecDowFLjGzlu05VwI73X0UcBdwR4vtdwLPRy1XA19y90nAZGCmmU1LRPwiItK6RNY4pgDF7r7W3WuAR4FZLcrMAh4OH88DTrVwjg8zOxv4GFjRWNgDjZeppoU/B/+FKCIiSSSRiWMIsCFquSRc12oZd68DyoB8M8sGbgRimqLMLMXMlgJbgb+4+8IExC4iIm1I1s7x2cBdUbWLJu5e7+6TgaHAFDMb39oBzOxqMysys6LS0tLERisicghJZOLYCAyLWh4armu1jJmlAnnAdmAq8BMzWwd8H/i/ZnZN9I7uvgv4KzCztSd397nuXujuhf369dv/VyMiIkBiE8ciYLSZjTCzdOBiYEGLMguAy8PH5wOvhP0Y0929wN0LgJ8Dt7v7vWbWz8x6AZhZD2AGsCqBr0FERFpI2AWA7l4X1hJeAFKAh9x9hZnNAYrcfQHwIPA7MysGdhAkl/YMAh4OR2xFgMfd/U8dxbJ48eJtZvbJPr6UvsC2fdw3kRRX5yVrbIqrcxRX5+1LbIe1teGQmB13f5hZUVszRHYnxdV5yRqb4uocxdV5Bzq2ZO0cFxGRJKXEISIinaLE0bG53R1AGxRX5yVrbIqrcxRX5x3Q2NTHISIinaIah4iIdIoSh4iIdIoSRxs6mhK+C+MYZmZ/NbMPw6nkvxeun21mG81safhzRjfFt87MloUxFIXr+pjZX8xsdfi7dxfHdGTUeVlqZuVm9v3uOGdm9pCZbTWz5VHrWj0/Frg7/Mx9YGbHdENs/21mq8Lnnx91wW2Bme2NOnf3d3Fcbb53ZvbD8Jx9ZGZf6eK4HouKaV04j15Xn6+2viMS9zlzd/20+CG4YHENMBJIB94HxnZTLIOAY8LHOcDfCaapnw1cnwTnah3Qt8W6nwA3hY9vAu7o5vdyM8HFTF1+zoAvAMcAyzs6P8AZBLcRMGAasLAbYjsNSA0f3xEVW0F0uW6Iq9X3LvxbeB/IAEaEf7cpXRVXi+0/A27phvPV1ndEwj5nqnG0Lp4p4buEu29y9yXh493ASmJnGU420dPlPwyc3Y2xnAqscfd9nTlgv7j73whmRYjW1vmZBfzWA+8AvcxsUFfG5u4vejBTNcA7BHPMdak2zllbZgGPunu1u38MFBP8/XZpXGZmwIXAHxPx3O1p5zsiYZ8zJY7WxTMlfJez4A6JRwONU8lfE1Y1H+rq5qAoDrxoZovN7Opw3QB33xQ+3gwM6J7QgGAam+g/5mQ4Z22dn2T73P0jzW+kNsLM3jOz18xsejfE09p7lyznbDqwxd1XR63r8vPV4jsiYZ8zJY7PCQvuUfIk8H13Lwf+Bzic4E6Imwiqyd3hJHc/huBOj981sy9Eb/SgbtwtY74tmFzzLOCJcFWynLMm3Xl+2mNmPwLqgN+HqzYBw939aOA64A9mltuFISXde9fCJTT/B6XLz1cr3xFNDvTnTImjdfFMCd9lzCyN4APxe3d/CsDdt3hwb5IG4AESVD3viLtvDH9vBeaHcWxprPqGv7d2R2wEyWyJu28JY0yKc0bb5ycpPndmdgVwJvD18AuHsCloe/h4MUFfwhFdFVM77123nzMLbglxLvBY47quPl+tfUeQwM+ZEkfr4pkSvkuEbacPAivd/c6o9dFtkucAy1vu2wWx9TSznMbHBB2ry2k+Xf7lwP/r6thCzf4LTIZzFmrr/CwAvhGOepkGlEU1NXQJM5sJ/AA4y90ro9b3s2BWasxsJDAaWNuFcbX13i0ALjazDDMbEcb1blfFFfoysMrdSxpXdOX5aus7gkR+zrqi1//z+EMw8uDvBP8p/Kgb4ziJoIr5AbA0/DkD+B2wLFy/ABjUDbGNJBjR8j7BveF/FK7PB14GVgMvAX26IbaeBDcFy4ta1+XnjCBxbQJqCdqSr2zr/BCMcrkv/MwtAwq7IbZigvbvxs/a/WHZ88L3eCmwBPhaF8fV5nsH/Cg8Zx8Bp3dlXOH6/wX+qUXZrjxfbX1HJOxzpilHRESkU9RUJSIinaLEISIinaLEISIinaLEISIinaLEISIinaLEIZLEzOwUM/tTd8chEk2JQ0REOkWJQ+QAMLPLzOzd8N4LvzKzFDOrMLO7wnskvGxm/cKyk83sHfvsnheN90kYZWYvmdn7ZrbEzA4PD59tZvMsuE/G78MrhUW6jRKHyH4yszHARcCJ7j4ZqAe+TnD1epG7jwNeA24Nd/ktcKO7TyS4crdx/e+B+9x9EnACwVXKEMx2+n2CeyyMBE5M+IsSaUdqdwcgchA4FTgWWBRWBnoQTCjXwGcT3z0CPGVmeUAvd38tXP8w8EQ459cQd58P4O5VAOHx3vVwHiQL7jBXALyR+Jcl0jolDpH9Z8DD7v7DZivN/q1FuX2d36c66nE9+ruVbqamKpH99zJwvpn1h6Z7PR9G8Pd1fljmUuANdy8Ddkbd2OcfgNc8uHNbiZmdHR4jw8yyuvRViMRJ/7mI7Cd3/9DMbia4E2KEYPbU7wJ7gCnhtq0E/SAQTHF9f5gY1gLfDNf/A/ArM5sTHuOCLnwZInHT7LgiCWJmFe6e3d1xiBxoaqoSEZFOUY1DREQ6RTUOERHpFCUOERHpFCUOERHpFCUOERHpFCUOERHplP8PhlvXs5/ArFoAAAAASUVORK5CYII=\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ],
      "source": [
        "plt.plot(np.arange(200), train_loss_history_tf)\n",
        "plt.xlabel('epoch')\n",
        "plt.ylabel('average per-sample loss')\n",
        "plt.legend(['training loss curve: transformer'])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7GPandFfGzRK"
      },
      "source": [
        "### Testing"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 31,
      "metadata": {
        "id": "CnibzOn9GzRL"
      },
      "outputs": [],
      "source": [
        "import validate"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 32,
      "metadata": {
        "id": "K5PwsJ3TGzRL"
      },
      "outputs": [],
      "source": [
        "testloader = DataLoader(testset, batch_size=batch_size, shuffle=False)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 33,
      "metadata": {
        "id": "LeXwJId3GzRL"
      },
      "outputs": [],
      "source": [
        "criterion = nn.CrossEntropyLoss()"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "test_loss1, test_acc1 = validate.eval_model(convnet, testloader, criterion, batch_size, device)\n",
        "print(f\"Accuracy on test dataset: {(100 * test_acc1):.2f}, Average per-sample loss: {(test_loss1):.4f}\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZV3iQe0qGIAy",
        "outputId": "3735dd0b-179f-47d1-9252-5c39ad019981"
      },
      "execution_count": 34,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Accuracy on test dataset: 24.59, Average per-sample loss: 0.1031\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 35,
      "metadata": {
        "id": "kAOMwS4fGzRL",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "4a1c6269-232e-4ff2-b100-e5c7cb767fcb"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Accuracy on test dataset: 22.95, Average per-sample loss: 0.0434\n"
          ]
        }
      ],
      "source": [
        "test_loss2, test_acc2 = validate.eval_model(tfclassifier, testloader, criterion, batch_size, device)\n",
        "print(f\"Accuracy on test dataset: {(100 * test_acc2):.2f}, Average per-sample loss: {(test_loss2):.4f}\")"
      ]
    }
  ],
  "metadata": {
    "interpreter": {
      "hash": "80d90075c5a86a80d8d1073f976de278ef7ed64e8960fcf294f942c6a259a6a1"
    },
    "kernelspec": {
      "display_name": "Python 3.9.12 ('torchenv')",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.9.12"
    },
    "orig_nbformat": 4,
    "colab": {
      "name": "grp14-final-submission.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "8s1C93BYGzQ9",
        "iENb-vswGzRB"
      ]
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}