Lab10/Lab10-2-2_SurinKim.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Xavier Deep network version"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# import"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.5.0\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "from tensorflow.keras.utils import to_categorical\n",
    "from tensorflow.keras.datasets import mnist\n",
    "from time import time\n",
    "import os\n",
    "print(tf.__version__)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Checkpoint function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load(model, checkpoint_dir):\n",
    "    print(\" [*] Reading checkpoints...\")\n",
    "\n",
    "    ckpt = tf.train.get_checkpoint_state(checkpoint_dir)\n",
    "    if ckpt :\n",
    "        ckpt_name = os.path.basename(ckpt.model_checkpoint_path)\n",
    "        checkpoint = tf.train.Checkpoint(dnn=model)\n",
    "        checkpoint.restore(save_path=os.path.join(checkpoint_dir, ckpt_name))\n",
    "        counter = int(ckpt_name.split('-')[1])\n",
    "        print(\" [*] Success to read {}\".format(ckpt_name))\n",
    "        return True, counter\n",
    "    else:\n",
    "        print(\" [*] Failed to find a checkpoint\")\n",
    "        return False, 0\n",
    "\n",
    "def check_folder(dir):\n",
    "    if not os.path.exists(dir):\n",
    "        os.makedirs(dir)\n",
    "    return dir"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data load & pre-processing function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_mnist() :\n",
    "    (train_data, train_labels), (test_data, test_labels) = mnist.load_data()\n",
    "    train_data = np.expand_dims(train_data, axis=-1) # [N, 28, 28] -> [N, 28, 28, 1]\n",
    "    test_data = np.expand_dims(test_data, axis=-1) # [N, 28, 28] -> [N, 28, 28, 1]\n",
    "\n",
    "    train_data, test_data = normalize(train_data, test_data)\n",
    "\n",
    "    train_labels = to_categorical(train_labels, 10) # [N,] -> [N, 10]\n",
    "    test_labels = to_categorical(test_labels, 10) # [N,] -> [N, 10]\n",
    "\n",
    "    return train_data, train_labels, test_data, test_labels\n",
    "\n",
    "def normalize(train_data, test_data):\n",
    "    train_data = train_data.astype(np.float32) / 255.0\n",
    "    test_data = test_data.astype(np.float32) / 255.0\n",
    "\n",
    "    return train_data, test_data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Performance function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def loss_fn(model, images, labels):\n",
    "    logits = model(images, training=True)\n",
    "    loss = tf.reduce_mean(tf.keras.losses.categorical_crossentropy(y_pred=logits, y_true=labels, \n",
    "                                                                   from_logits=True))\n",
    "    return loss\n",
    "\n",
    "def accuracy_fn(model, images, labels):\n",
    "    logits = model(images, training=False)\n",
    "    prediction = tf.equal(tf.argmax(logits, -1), tf.argmax(labels, -1))\n",
    "    accuracy = tf.reduce_mean(tf.cast(prediction, tf.float32))\n",
    "    return accuracy\n",
    "\n",
    "def grad(model, images, labels):\n",
    "    with tf.GradientTape() as tape:\n",
    "        loss = loss_fn(model, images, labels)\n",
    "    return tape.gradient(loss, model.variables)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def flatten() :\n",
    "    return tf.keras.layers.Flatten()\n",
    "\n",
    "def dense(label_dim, weight_init) :\n",
    "    return tf.keras.layers.Dense(units=label_dim, use_bias=True, kernel_initializer=weight_init)\n",
    "\n",
    "def relu() :\n",
    "    return tf.keras.layers.Activation(tf.keras.activations.relu)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create model (class version)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "class create_model_class(tf.keras.Model):\n",
    "    def __init__(self, label_dim):\n",
    "        super(create_model_class, self).__init__()\n",
    "        weight_init = tf.keras.initializers.glorot_uniform()\n",
    "\n",
    "        self.model = tf.keras.Sequential()\n",
    "        self.model.add(flatten())\n",
    "\n",
    "        for i in range(4):\n",
    "            self.model.add(dense(512, weight_init))\n",
    "            self.model.add(relu())\n",
    "\n",
    "        self.model.add(dense(label_dim, weight_init))\n",
    "\n",
    "    def call(self, x, training=None, mask=None):\n",
    "\n",
    "        x = self.model(x)\n",
    "\n",
    "        return x"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create model (function version)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_model_function(label_dim) :\n",
    "    weight_init = tf.keras.initializers.glorot_uniform()\n",
    "\n",
    "    model = tf.keras.Sequential()\n",
    "    model.add(flatten())\n",
    "\n",
    "    for i in range(4) :\n",
    "        model.add(dense(512, weight_init))\n",
    "        model.add(relu())\n",
    "\n",
    "    model.add(dense(label_dim, weight_init))\n",
    "\n",
    "    return model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Define data & hyper-parameter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\" dataset \"\"\"\n",
    "train_x, train_y, test_x, test_y = load_mnist()\n",
    "\n",
    "\"\"\" parameters \"\"\"\n",
    "learning_rate = 0.001\n",
    "batch_size = 128\n",
    "\n",
    "training_epochs = 1\n",
    "training_iterations = len(train_x) // batch_size\n",
    "\n",
    "label_dim = 10\n",
    "\n",
    "train_flag = True\n",
    "\n",
    "\"\"\" Graph Input using Dataset API \"\"\"\n",
    "train_dataset = tf.data.Dataset.from_tensor_slices((train_x, train_y)).\\\n",
    "    shuffle(buffer_size=100000).\\\n",
    "    prefetch(buffer_size=batch_size).\\\n",
    "    batch(batch_size, drop_remainder=True)\n",
    "\n",
    "test_dataset = tf.data.Dataset.from_tensor_slices((test_x, test_y)).\\\n",
    "    shuffle(buffer_size=100000).\\\n",
    "    prefetch(buffer_size=len(test_x)).\\\n",
    "    batch(len(test_x))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Define model & optimizer & writer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\" Model \"\"\"\n",
    "network = create_model_function(label_dim)\n",
    "\n",
    "\"\"\" Training \"\"\"\n",
    "optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate)\n",
    "\n",
    "\"\"\" Writer \"\"\"\n",
    "checkpoint_dir = 'checkpoints'\n",
    "logs_dir = 'logs'\n",
    "\n",
    "model_dir = 'nn_deep'\n",
    "\n",
    "checkpoint_dir = os.path.join(checkpoint_dir, model_dir)\n",
    "check_folder(checkpoint_dir)\n",
    "checkpoint_prefix = os.path.join(checkpoint_dir, model_dir)\n",
    "logs_dir = os.path.join(logs_dir, model_dir)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Restore checkpoint & start train or test phase"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " [*] Reading checkpoints...\n",
      " [*] Failed to find a checkpoint\n",
      " [!] Load failed...\n",
      "Epoch: [ 0] [    0/  468] time: 1.6200, train_loss: 2.07296610, train_accuracy: 0.3672, test_Accuracy: 0.2060\n",
      "Epoch: [ 0] [    1/  468] time: 2.1840, train_loss: 1.98973250, train_accuracy: 0.4922, test_Accuracy: 0.4592\n",
      "Epoch: [ 0] [    2/  468] time: 2.7820, train_loss: 1.79307199, train_accuracy: 0.7266, test_Accuracy: 0.6365\n",
      "Epoch: [ 0] [    3/  468] time: 3.3850, train_loss: 1.55289590, train_accuracy: 0.7031, test_Accuracy: 0.6821\n",
      "Epoch: [ 0] [    4/  468] time: 4.0160, train_loss: 1.24313354, train_accuracy: 0.7109, test_Accuracy: 0.6572\n",
      "Epoch: [ 0] [    5/  468] time: 4.5480, train_loss: 1.05351233, train_accuracy: 0.7656, test_Accuracy: 0.7052\n",
      "Epoch: [ 0] [    6/  468] time: 5.0830, train_loss: 0.79410684, train_accuracy: 0.7656, test_Accuracy: 0.6976\n",
      "Epoch: [ 0] [    7/  468] time: 5.6131, train_loss: 0.71339667, train_accuracy: 0.8047, test_Accuracy: 0.7586\n",
      "Epoch: [ 0] [    8/  468] time: 6.1400, train_loss: 0.61694992, train_accuracy: 0.8203, test_Accuracy: 0.7493\n",
      "Epoch: [ 0] [    9/  468] time: 6.6971, train_loss: 0.60380387, train_accuracy: 0.8047, test_Accuracy: 0.7802\n",
      "Epoch: [ 0] [   10/  468] time: 7.4100, train_loss: 0.51666415, train_accuracy: 0.8672, test_Accuracy: 0.7997\n",
      "Epoch: [ 0] [   11/  468] time: 8.0040, train_loss: 0.46789104, train_accuracy: 0.7969, test_Accuracy: 0.7960\n",
      "Epoch: [ 0] [   12/  468] time: 8.5500, train_loss: 0.27675343, train_accuracy: 0.9375, test_Accuracy: 0.8164\n",
      "Epoch: [ 0] [   13/  468] time: 9.1350, train_loss: 0.40834841, train_accuracy: 0.8594, test_Accuracy: 0.8184\n",
      "Epoch: [ 0] [   14/  468] time: 9.7030, train_loss: 0.48936716, train_accuracy: 0.8594, test_Accuracy: 0.8410\n",
      "Epoch: [ 0] [   15/  468] time: 10.2840, train_loss: 0.46179444, train_accuracy: 0.9141, test_Accuracy: 0.8750\n",
      "Epoch: [ 0] [   16/  468] time: 10.9650, train_loss: 0.47885528, train_accuracy: 0.8125, test_Accuracy: 0.8573\n",
      "Epoch: [ 0] [   17/  468] time: 11.5241, train_loss: 0.29649672, train_accuracy: 0.9062, test_Accuracy: 0.8449\n",
      "Epoch: [ 0] [   18/  468] time: 12.1180, train_loss: 0.49491674, train_accuracy: 0.8672, test_Accuracy: 0.8640\n",
      "Epoch: [ 0] [   19/  468] time: 12.7270, train_loss: 0.43332180, train_accuracy: 0.8750, test_Accuracy: 0.8449\n",
      "Epoch: [ 0] [   20/  468] time: 13.2610, train_loss: 0.27583966, train_accuracy: 0.9219, test_Accuracy: 0.8494\n",
      "Epoch: [ 0] [   21/  468] time: 13.8650, train_loss: 0.38809550, train_accuracy: 0.9141, test_Accuracy: 0.8701\n",
      "Epoch: [ 0] [   22/  468] time: 14.3920, train_loss: 0.25226825, train_accuracy: 0.9375, test_Accuracy: 0.8854\n",
      "Epoch: [ 0] [   23/  468] time: 15.0760, train_loss: 0.46387011, train_accuracy: 0.8594, test_Accuracy: 0.8884\n",
      "Epoch: [ 0] [   24/  468] time: 15.7610, train_loss: 0.43351778, train_accuracy: 0.8906, test_Accuracy: 0.8711\n",
      "Epoch: [ 0] [   25/  468] time: 16.4200, train_loss: 0.37885702, train_accuracy: 0.8828, test_Accuracy: 0.8693\n",
      "Epoch: [ 0] [   26/  468] time: 17.0060, train_loss: 0.26066247, train_accuracy: 0.9141, test_Accuracy: 0.8915\n",
      "Epoch: [ 0] [   27/  468] time: 17.5441, train_loss: 0.34344113, train_accuracy: 0.8984, test_Accuracy: 0.8965\n",
      "Epoch: [ 0] [   28/  468] time: 18.0730, train_loss: 0.30107200, train_accuracy: 0.8984, test_Accuracy: 0.8857\n",
      "Epoch: [ 0] [   29/  468] time: 18.6210, train_loss: 0.32276401, train_accuracy: 0.9062, test_Accuracy: 0.8764\n",
      "Epoch: [ 0] [   30/  468] time: 19.2500, train_loss: 0.32246023, train_accuracy: 0.8750, test_Accuracy: 0.8860\n",
      "Epoch: [ 0] [   31/  468] time: 19.7830, train_loss: 0.31645542, train_accuracy: 0.9141, test_Accuracy: 0.9028\n",
      "Epoch: [ 0] [   32/  468] time: 20.3153, train_loss: 0.40209168, train_accuracy: 0.8750, test_Accuracy: 0.9116\n",
      "Epoch: [ 0] [   33/  468] time: 20.8780, train_loss: 0.31920868, train_accuracy: 0.9062, test_Accuracy: 0.9126\n",
      "Epoch: [ 0] [   34/  468] time: 21.4290, train_loss: 0.21960577, train_accuracy: 0.9453, test_Accuracy: 0.9068\n",
      "Epoch: [ 0] [   35/  468] time: 21.9750, train_loss: 0.39277899, train_accuracy: 0.8906, test_Accuracy: 0.9104\n",
      "Epoch: [ 0] [   36/  468] time: 22.4980, train_loss: 0.37757879, train_accuracy: 0.8828, test_Accuracy: 0.9127\n",
      "Epoch: [ 0] [   37/  468] time: 23.0610, train_loss: 0.35164696, train_accuracy: 0.8984, test_Accuracy: 0.9148\n",
      "Epoch: [ 0] [   38/  468] time: 23.5910, train_loss: 0.27464515, train_accuracy: 0.9375, test_Accuracy: 0.9145\n",
      "Epoch: [ 0] [   39/  468] time: 24.1710, train_loss: 0.17540959, train_accuracy: 0.9531, test_Accuracy: 0.9108\n",
      "Epoch: [ 0] [   40/  468] time: 24.7150, train_loss: 0.35366103, train_accuracy: 0.8984, test_Accuracy: 0.9132\n",
      "Epoch: [ 0] [   41/  468] time: 25.2550, train_loss: 0.31902066, train_accuracy: 0.8984, test_Accuracy: 0.9141\n",
      "Epoch: [ 0] [   42/  468] time: 25.8180, train_loss: 0.36251149, train_accuracy: 0.9297, test_Accuracy: 0.9116\n",
      "Epoch: [ 0] [   43/  468] time: 26.3657, train_loss: 0.27452102, train_accuracy: 0.9375, test_Accuracy: 0.9095\n",
      "Epoch: [ 0] [   44/  468] time: 26.9660, train_loss: 0.38903010, train_accuracy: 0.8750, test_Accuracy: 0.9178\n",
      "Epoch: [ 0] [   45/  468] time: 27.5220, train_loss: 0.20542121, train_accuracy: 0.9453, test_Accuracy: 0.9211\n",
      "Epoch: [ 0] [   46/  468] time: 28.0670, train_loss: 0.19860451, train_accuracy: 0.9297, test_Accuracy: 0.9078\n",
      "Epoch: [ 0] [   47/  468] time: 28.6780, train_loss: 0.33717352, train_accuracy: 0.9062, test_Accuracy: 0.8970\n",
      "Epoch: [ 0] [   48/  468] time: 29.1780, train_loss: 0.34175086, train_accuracy: 0.8984, test_Accuracy: 0.9001\n",
      "Epoch: [ 0] [   49/  468] time: 29.7250, train_loss: 0.20922524, train_accuracy: 0.9375, test_Accuracy: 0.9131\n",
      "Epoch: [ 0] [   50/  468] time: 30.2650, train_loss: 0.34488538, train_accuracy: 0.8750, test_Accuracy: 0.9172\n",
      "Epoch: [ 0] [   51/  468] time: 30.8930, train_loss: 0.35631350, train_accuracy: 0.9062, test_Accuracy: 0.9082\n",
      "Epoch: [ 0] [   52/  468] time: 31.4350, train_loss: 0.25169861, train_accuracy: 0.9141, test_Accuracy: 0.9104\n",
      "Epoch: [ 0] [   53/  468] time: 32.0330, train_loss: 0.22645612, train_accuracy: 0.9297, test_Accuracy: 0.9122\n",
      "Epoch: [ 0] [   54/  468] time: 32.5951, train_loss: 0.16671260, train_accuracy: 0.9375, test_Accuracy: 0.9096\n",
      "Epoch: [ 0] [   55/  468] time: 33.1620, train_loss: 0.20279855, train_accuracy: 0.9453, test_Accuracy: 0.9123\n",
      "Epoch: [ 0] [   56/  468] time: 33.7470, train_loss: 0.25407344, train_accuracy: 0.9219, test_Accuracy: 0.9159\n",
      "Epoch: [ 0] [   57/  468] time: 34.2870, train_loss: 0.11515539, train_accuracy: 0.9844, test_Accuracy: 0.9167\n",
      "Epoch: [ 0] [   58/  468] time: 34.8290, train_loss: 0.23758422, train_accuracy: 0.9297, test_Accuracy: 0.9217\n",
      "Epoch: [ 0] [   59/  468] time: 35.3760, train_loss: 0.22540334, train_accuracy: 0.9375, test_Accuracy: 0.9247\n",
      "Epoch: [ 0] [   60/  468] time: 35.9600, train_loss: 0.18388437, train_accuracy: 0.9609, test_Accuracy: 0.9263\n",
      "Epoch: [ 0] [   61/  468] time: 36.5670, train_loss: 0.28061193, train_accuracy: 0.9141, test_Accuracy: 0.9170\n",
      "Epoch: [ 0] [   62/  468] time: 37.1701, train_loss: 0.25171977, train_accuracy: 0.9297, test_Accuracy: 0.9148\n",
      "Epoch: [ 0] [   63/  468] time: 37.7250, train_loss: 0.25846419, train_accuracy: 0.8984, test_Accuracy: 0.9121\n",
      "Epoch: [ 0] [   64/  468] time: 38.3040, train_loss: 0.26872316, train_accuracy: 0.9375, test_Accuracy: 0.9039\n",
      "Epoch: [ 0] [   65/  468] time: 38.8251, train_loss: 0.25158483, train_accuracy: 0.9219, test_Accuracy: 0.9136\n",
      "Epoch: [ 0] [   66/  468] time: 39.3800, train_loss: 0.22125967, train_accuracy: 0.9375, test_Accuracy: 0.9162\n",
      "Epoch: [ 0] [   67/  468] time: 39.9180, train_loss: 0.22251774, train_accuracy: 0.9297, test_Accuracy: 0.9053\n",
      "Epoch: [ 0] [   68/  468] time: 40.5800, train_loss: 0.23242614, train_accuracy: 0.9297, test_Accuracy: 0.9060\n",
      "Epoch: [ 0] [   69/  468] time: 41.1910, train_loss: 0.18593200, train_accuracy: 0.9453, test_Accuracy: 0.9209\n",
      "Epoch: [ 0] [   70/  468] time: 41.7321, train_loss: 0.18956444, train_accuracy: 0.9375, test_Accuracy: 0.9220\n",
      "Epoch: [ 0] [   71/  468] time: 42.3500, train_loss: 0.28348362, train_accuracy: 0.9219, test_Accuracy: 0.9118\n",
      "Epoch: [ 0] [   72/  468] time: 42.9970, train_loss: 0.28160089, train_accuracy: 0.8984, test_Accuracy: 0.9150\n",
      "Epoch: [ 0] [   73/  468] time: 43.5410, train_loss: 0.26474172, train_accuracy: 0.9062, test_Accuracy: 0.9134\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [ 0] [   74/  468] time: 44.0860, train_loss: 0.25231022, train_accuracy: 0.9297, test_Accuracy: 0.9222\n",
      "Epoch: [ 0] [   75/  468] time: 44.6290, train_loss: 0.26006931, train_accuracy: 0.9219, test_Accuracy: 0.9260\n",
      "Epoch: [ 0] [   76/  468] time: 45.2141, train_loss: 0.26549917, train_accuracy: 0.9297, test_Accuracy: 0.9239\n",
      "Epoch: [ 0] [   77/  468] time: 45.7860, train_loss: 0.18340820, train_accuracy: 0.9453, test_Accuracy: 0.9150\n",
      "Epoch: [ 0] [   78/  468] time: 46.3090, train_loss: 0.29419795, train_accuracy: 0.8906, test_Accuracy: 0.9198\n",
      "Epoch: [ 0] [   79/  468] time: 46.8690, train_loss: 0.31250411, train_accuracy: 0.9219, test_Accuracy: 0.9282\n",
      "Epoch: [ 0] [   80/  468] time: 47.4220, train_loss: 0.14052300, train_accuracy: 0.9688, test_Accuracy: 0.9326\n",
      "Epoch: [ 0] [   81/  468] time: 47.9790, train_loss: 0.14809769, train_accuracy: 0.9688, test_Accuracy: 0.9337\n",
      "Epoch: [ 0] [   82/  468] time: 48.5150, train_loss: 0.22265276, train_accuracy: 0.9453, test_Accuracy: 0.9279\n",
      "Epoch: [ 0] [   83/  468] time: 49.1590, train_loss: 0.22111467, train_accuracy: 0.9531, test_Accuracy: 0.9199\n",
      "Epoch: [ 0] [   84/  468] time: 49.6540, train_loss: 0.17060511, train_accuracy: 0.9531, test_Accuracy: 0.9178\n",
      "Epoch: [ 0] [   85/  468] time: 50.2440, train_loss: 0.23577307, train_accuracy: 0.9297, test_Accuracy: 0.9232\n",
      "Epoch: [ 0] [   86/  468] time: 50.8720, train_loss: 0.21976037, train_accuracy: 0.9141, test_Accuracy: 0.9333\n",
      "Epoch: [ 0] [   87/  468] time: 51.3950, train_loss: 0.14991733, train_accuracy: 0.9453, test_Accuracy: 0.9373\n",
      "Epoch: [ 0] [   88/  468] time: 52.0490, train_loss: 0.19860277, train_accuracy: 0.9297, test_Accuracy: 0.9363\n",
      "Epoch: [ 0] [   89/  468] time: 52.6730, train_loss: 0.22059375, train_accuracy: 0.9141, test_Accuracy: 0.9340\n",
      "Epoch: [ 0] [   90/  468] time: 53.2050, train_loss: 0.15755659, train_accuracy: 0.9453, test_Accuracy: 0.9346\n",
      "Epoch: [ 0] [   91/  468] time: 53.8490, train_loss: 0.25274441, train_accuracy: 0.9297, test_Accuracy: 0.9308\n",
      "Epoch: [ 0] [   92/  468] time: 54.5310, train_loss: 0.26918009, train_accuracy: 0.9062, test_Accuracy: 0.9308\n",
      "Epoch: [ 0] [   93/  468] time: 55.1600, train_loss: 0.11871406, train_accuracy: 0.9531, test_Accuracy: 0.9334\n",
      "Epoch: [ 0] [   94/  468] time: 55.6940, train_loss: 0.17498043, train_accuracy: 0.9453, test_Accuracy: 0.9367\n",
      "Epoch: [ 0] [   95/  468] time: 56.2670, train_loss: 0.13742703, train_accuracy: 0.9609, test_Accuracy: 0.9381\n",
      "Epoch: [ 0] [   96/  468] time: 56.7930, train_loss: 0.16404195, train_accuracy: 0.9688, test_Accuracy: 0.9372\n",
      "Epoch: [ 0] [   97/  468] time: 57.3150, train_loss: 0.19278055, train_accuracy: 0.9219, test_Accuracy: 0.9390\n",
      "Epoch: [ 0] [   98/  468] time: 57.8631, train_loss: 0.18924546, train_accuracy: 0.9219, test_Accuracy: 0.9362\n",
      "Epoch: [ 0] [   99/  468] time: 58.4000, train_loss: 0.22716832, train_accuracy: 0.9219, test_Accuracy: 0.9337\n",
      "Epoch: [ 0] [  100/  468] time: 58.9650, train_loss: 0.23513024, train_accuracy: 0.9375, test_Accuracy: 0.9321\n",
      "Epoch: [ 0] [  101/  468] time: 59.5510, train_loss: 0.19816707, train_accuracy: 0.9453, test_Accuracy: 0.9305\n",
      "Epoch: [ 0] [  102/  468] time: 60.1220, train_loss: 0.19212350, train_accuracy: 0.9453, test_Accuracy: 0.9252\n",
      "Epoch: [ 0] [  103/  468] time: 60.7360, train_loss: 0.14418261, train_accuracy: 0.9453, test_Accuracy: 0.9217\n",
      "Epoch: [ 0] [  104/  468] time: 61.2210, train_loss: 0.26640022, train_accuracy: 0.9297, test_Accuracy: 0.9260\n",
      "Epoch: [ 0] [  105/  468] time: 61.6430, train_loss: 0.09590835, train_accuracy: 0.9766, test_Accuracy: 0.9323\n",
      "Epoch: [ 0] [  106/  468] time: 62.1500, train_loss: 0.12920535, train_accuracy: 0.9609, test_Accuracy: 0.9386\n",
      "Epoch: [ 0] [  107/  468] time: 62.5960, train_loss: 0.08990520, train_accuracy: 0.9844, test_Accuracy: 0.9355\n",
      "Epoch: [ 0] [  108/  468] time: 63.1770, train_loss: 0.17951441, train_accuracy: 0.9375, test_Accuracy: 0.9303\n",
      "Epoch: [ 0] [  109/  468] time: 63.7110, train_loss: 0.21421194, train_accuracy: 0.9297, test_Accuracy: 0.9352\n",
      "Epoch: [ 0] [  110/  468] time: 64.3030, train_loss: 0.08640997, train_accuracy: 0.9766, test_Accuracy: 0.9392\n",
      "Epoch: [ 0] [  111/  468] time: 64.8471, train_loss: 0.11435477, train_accuracy: 0.9609, test_Accuracy: 0.9381\n",
      "Epoch: [ 0] [  112/  468] time: 65.3911, train_loss: 0.16592017, train_accuracy: 0.9453, test_Accuracy: 0.9373\n",
      "Epoch: [ 0] [  113/  468] time: 65.9330, train_loss: 0.20715207, train_accuracy: 0.9219, test_Accuracy: 0.9371\n",
      "Epoch: [ 0] [  114/  468] time: 66.5280, train_loss: 0.17877106, train_accuracy: 0.9688, test_Accuracy: 0.9397\n",
      "Epoch: [ 0] [  115/  468] time: 67.0640, train_loss: 0.15029988, train_accuracy: 0.9297, test_Accuracy: 0.9420\n",
      "Epoch: [ 0] [  116/  468] time: 67.5760, train_loss: 0.15006664, train_accuracy: 0.9609, test_Accuracy: 0.9399\n",
      "Epoch: [ 0] [  117/  468] time: 68.0670, train_loss: 0.10655106, train_accuracy: 0.9766, test_Accuracy: 0.9408\n",
      "Epoch: [ 0] [  118/  468] time: 68.6670, train_loss: 0.17586148, train_accuracy: 0.9531, test_Accuracy: 0.9425\n",
      "Epoch: [ 0] [  119/  468] time: 69.1840, train_loss: 0.09863048, train_accuracy: 0.9688, test_Accuracy: 0.9431\n",
      "Epoch: [ 0] [  120/  468] time: 69.5574, train_loss: 0.10502079, train_accuracy: 0.9531, test_Accuracy: 0.9448\n",
      "Epoch: [ 0] [  121/  468] time: 70.1710, train_loss: 0.10646906, train_accuracy: 0.9688, test_Accuracy: 0.9444\n",
      "Epoch: [ 0] [  122/  468] time: 70.7110, train_loss: 0.14808220, train_accuracy: 0.9531, test_Accuracy: 0.9390\n",
      "Epoch: [ 0] [  123/  468] time: 71.2910, train_loss: 0.25736052, train_accuracy: 0.9453, test_Accuracy: 0.9337\n",
      "Epoch: [ 0] [  124/  468] time: 71.8800, train_loss: 0.11400273, train_accuracy: 0.9609, test_Accuracy: 0.9300\n",
      "Epoch: [ 0] [  125/  468] time: 72.4860, train_loss: 0.15343049, train_accuracy: 0.9375, test_Accuracy: 0.9320\n",
      "Epoch: [ 0] [  126/  468] time: 73.0390, train_loss: 0.11724199, train_accuracy: 0.9531, test_Accuracy: 0.9340\n",
      "Epoch: [ 0] [  127/  468] time: 73.5250, train_loss: 0.13368291, train_accuracy: 0.9688, test_Accuracy: 0.9336\n",
      "Epoch: [ 0] [  128/  468] time: 74.0350, train_loss: 0.16349617, train_accuracy: 0.9531, test_Accuracy: 0.9351\n",
      "Epoch: [ 0] [  129/  468] time: 74.5890, train_loss: 0.07394172, train_accuracy: 0.9766, test_Accuracy: 0.9372\n",
      "Epoch: [ 0] [  130/  468] time: 75.1041, train_loss: 0.09151301, train_accuracy: 0.9688, test_Accuracy: 0.9418\n",
      "Epoch: [ 0] [  131/  468] time: 75.5970, train_loss: 0.12854883, train_accuracy: 0.9688, test_Accuracy: 0.9438\n",
      "Epoch: [ 0] [  132/  468] time: 76.0890, train_loss: 0.17723808, train_accuracy: 0.9297, test_Accuracy: 0.9413\n",
      "Epoch: [ 0] [  133/  468] time: 76.7440, train_loss: 0.16007259, train_accuracy: 0.9688, test_Accuracy: 0.9390\n",
      "Epoch: [ 0] [  134/  468] time: 77.3090, train_loss: 0.22783563, train_accuracy: 0.9297, test_Accuracy: 0.9352\n",
      "Epoch: [ 0] [  135/  468] time: 77.9030, train_loss: 0.22783759, train_accuracy: 0.9375, test_Accuracy: 0.9368\n",
      "Epoch: [ 0] [  136/  468] time: 78.4700, train_loss: 0.10438246, train_accuracy: 0.9688, test_Accuracy: 0.9381\n",
      "Epoch: [ 0] [  137/  468] time: 78.9510, train_loss: 0.08183163, train_accuracy: 0.9766, test_Accuracy: 0.9390\n",
      "Epoch: [ 0] [  138/  468] time: 79.4080, train_loss: 0.23601469, train_accuracy: 0.9453, test_Accuracy: 0.9367\n",
      "Epoch: [ 0] [  139/  468] time: 79.9870, train_loss: 0.13883348, train_accuracy: 0.9531, test_Accuracy: 0.9263\n",
      "Epoch: [ 0] [  140/  468] time: 80.5581, train_loss: 0.12096363, train_accuracy: 0.9531, test_Accuracy: 0.9199\n",
      "Epoch: [ 0] [  141/  468] time: 81.1140, train_loss: 0.19103537, train_accuracy: 0.9375, test_Accuracy: 0.9336\n",
      "Epoch: [ 0] [  142/  468] time: 81.6801, train_loss: 0.16594157, train_accuracy: 0.9375, test_Accuracy: 0.9451\n",
      "Epoch: [ 0] [  143/  468] time: 82.2180, train_loss: 0.08295668, train_accuracy: 0.9844, test_Accuracy: 0.9503\n",
      "Epoch: [ 0] [  144/  468] time: 82.7400, train_loss: 0.10995138, train_accuracy: 0.9531, test_Accuracy: 0.9493\n",
      "Epoch: [ 0] [  145/  468] time: 83.3220, train_loss: 0.14591654, train_accuracy: 0.9609, test_Accuracy: 0.9453\n",
      "Epoch: [ 0] [  146/  468] time: 83.8620, train_loss: 0.18153635, train_accuracy: 0.9375, test_Accuracy: 0.9424\n",
      "Epoch: [ 0] [  147/  468] time: 84.4400, train_loss: 0.15177771, train_accuracy: 0.9375, test_Accuracy: 0.9461\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [ 0] [  148/  468] time: 84.9850, train_loss: 0.16724753, train_accuracy: 0.9531, test_Accuracy: 0.9483\n",
      "Epoch: [ 0] [  149/  468] time: 85.5510, train_loss: 0.15006250, train_accuracy: 0.9531, test_Accuracy: 0.9502\n",
      "Epoch: [ 0] [  150/  468] time: 86.0600, train_loss: 0.21851031, train_accuracy: 0.9531, test_Accuracy: 0.9511\n",
      "Epoch: [ 0] [  151/  468] time: 86.5231, train_loss: 0.21135142, train_accuracy: 0.9297, test_Accuracy: 0.9532\n",
      "Epoch: [ 0] [  152/  468] time: 87.1170, train_loss: 0.16381854, train_accuracy: 0.9609, test_Accuracy: 0.9532\n",
      "Epoch: [ 0] [  153/  468] time: 87.6471, train_loss: 0.19881748, train_accuracy: 0.9453, test_Accuracy: 0.9524\n",
      "Epoch: [ 0] [  154/  468] time: 88.1860, train_loss: 0.14133179, train_accuracy: 0.9453, test_Accuracy: 0.9496\n",
      "Epoch: [ 0] [  155/  468] time: 88.6460, train_loss: 0.15566699, train_accuracy: 0.9531, test_Accuracy: 0.9468\n",
      "Epoch: [ 0] [  156/  468] time: 89.1810, train_loss: 0.09260961, train_accuracy: 0.9688, test_Accuracy: 0.9453\n",
      "Epoch: [ 0] [  157/  468] time: 89.6740, train_loss: 0.13476688, train_accuracy: 0.9609, test_Accuracy: 0.9417\n",
      "Epoch: [ 0] [  158/  468] time: 90.1960, train_loss: 0.19862774, train_accuracy: 0.9453, test_Accuracy: 0.9397\n",
      "Epoch: [ 0] [  159/  468] time: 90.8320, train_loss: 0.16228610, train_accuracy: 0.9688, test_Accuracy: 0.9383\n",
      "Epoch: [ 0] [  160/  468] time: 91.4050, train_loss: 0.18646052, train_accuracy: 0.9531, test_Accuracy: 0.9404\n",
      "Epoch: [ 0] [  161/  468] time: 91.9070, train_loss: 0.17229369, train_accuracy: 0.9297, test_Accuracy: 0.9427\n",
      "Epoch: [ 0] [  162/  468] time: 92.4680, train_loss: 0.16709262, train_accuracy: 0.9375, test_Accuracy: 0.9499\n",
      "Epoch: [ 0] [  163/  468] time: 93.0140, train_loss: 0.18944794, train_accuracy: 0.9219, test_Accuracy: 0.9523\n",
      "Epoch: [ 0] [  164/  468] time: 93.5354, train_loss: 0.16132355, train_accuracy: 0.9453, test_Accuracy: 0.9517\n",
      "Epoch: [ 0] [  165/  468] time: 94.1010, train_loss: 0.23562904, train_accuracy: 0.9219, test_Accuracy: 0.9500\n",
      "Epoch: [ 0] [  166/  468] time: 94.7070, train_loss: 0.13536812, train_accuracy: 0.9609, test_Accuracy: 0.9496\n",
      "Epoch: [ 0] [  167/  468] time: 95.2580, train_loss: 0.23910654, train_accuracy: 0.9375, test_Accuracy: 0.9515\n",
      "Epoch: [ 0] [  168/  468] time: 95.9320, train_loss: 0.09903075, train_accuracy: 0.9766, test_Accuracy: 0.9498\n",
      "Epoch: [ 0] [  169/  468] time: 96.6280, train_loss: 0.09221114, train_accuracy: 0.9766, test_Accuracy: 0.9509\n",
      "Epoch: [ 0] [  170/  468] time: 97.1990, train_loss: 0.15548152, train_accuracy: 0.9688, test_Accuracy: 0.9529\n",
      "Epoch: [ 0] [  171/  468] time: 97.7725, train_loss: 0.09735110, train_accuracy: 0.9844, test_Accuracy: 0.9530\n",
      "Epoch: [ 0] [  172/  468] time: 98.3890, train_loss: 0.11742820, train_accuracy: 0.9766, test_Accuracy: 0.9525\n",
      "Epoch: [ 0] [  173/  468] time: 98.9370, train_loss: 0.17028302, train_accuracy: 0.9453, test_Accuracy: 0.9524\n",
      "Epoch: [ 0] [  174/  468] time: 99.4960, train_loss: 0.10485269, train_accuracy: 0.9688, test_Accuracy: 0.9511\n",
      "Epoch: [ 0] [  175/  468] time: 100.1151, train_loss: 0.17002639, train_accuracy: 0.9531, test_Accuracy: 0.9505\n",
      "Epoch: [ 0] [  176/  468] time: 100.6810, train_loss: 0.13149534, train_accuracy: 0.9688, test_Accuracy: 0.9490\n",
      "Epoch: [ 0] [  177/  468] time: 101.2380, train_loss: 0.10865042, train_accuracy: 0.9688, test_Accuracy: 0.9499\n",
      "Epoch: [ 0] [  178/  468] time: 101.8060, train_loss: 0.10101214, train_accuracy: 0.9688, test_Accuracy: 0.9510\n",
      "Epoch: [ 0] [  179/  468] time: 102.3381, train_loss: 0.12549177, train_accuracy: 0.9609, test_Accuracy: 0.9491\n",
      "Epoch: [ 0] [  180/  468] time: 102.8800, train_loss: 0.04849917, train_accuracy: 0.9922, test_Accuracy: 0.9460\n",
      "Epoch: [ 0] [  181/  468] time: 103.4100, train_loss: 0.10508437, train_accuracy: 0.9766, test_Accuracy: 0.9458\n",
      "Epoch: [ 0] [  182/  468] time: 103.9270, train_loss: 0.13063063, train_accuracy: 0.9688, test_Accuracy: 0.9444\n",
      "Epoch: [ 0] [  183/  468] time: 104.5720, train_loss: 0.11436771, train_accuracy: 0.9609, test_Accuracy: 0.9413\n",
      "Epoch: [ 0] [  184/  468] time: 105.1291, train_loss: 0.13060670, train_accuracy: 0.9609, test_Accuracy: 0.9395\n",
      "Epoch: [ 0] [  185/  468] time: 105.6630, train_loss: 0.12860055, train_accuracy: 0.9531, test_Accuracy: 0.9462\n",
      "Epoch: [ 0] [  186/  468] time: 106.2240, train_loss: 0.15339005, train_accuracy: 0.9531, test_Accuracy: 0.9515\n",
      "Epoch: [ 0] [  187/  468] time: 106.8570, train_loss: 0.14604746, train_accuracy: 0.9609, test_Accuracy: 0.9537\n",
      "Epoch: [ 0] [  188/  468] time: 107.4680, train_loss: 0.12746412, train_accuracy: 0.9688, test_Accuracy: 0.9561\n",
      "Epoch: [ 0] [  189/  468] time: 108.1760, train_loss: 0.16300173, train_accuracy: 0.9453, test_Accuracy: 0.9566\n",
      "Epoch: [ 0] [  190/  468] time: 108.7260, train_loss: 0.09220982, train_accuracy: 0.9766, test_Accuracy: 0.9514\n",
      "Epoch: [ 0] [  191/  468] time: 109.2620, train_loss: 0.11011918, train_accuracy: 0.9609, test_Accuracy: 0.9511\n",
      "Epoch: [ 0] [  192/  468] time: 109.7560, train_loss: 0.09945024, train_accuracy: 0.9688, test_Accuracy: 0.9566\n",
      "Epoch: [ 0] [  193/  468] time: 110.3620, train_loss: 0.11619335, train_accuracy: 0.9531, test_Accuracy: 0.9588\n",
      "Epoch: [ 0] [  194/  468] time: 110.9590, train_loss: 0.13624866, train_accuracy: 0.9688, test_Accuracy: 0.9545\n",
      "Epoch: [ 0] [  195/  468] time: 111.3840, train_loss: 0.10997127, train_accuracy: 0.9609, test_Accuracy: 0.9507\n",
      "Epoch: [ 0] [  196/  468] time: 111.7920, train_loss: 0.08914462, train_accuracy: 0.9844, test_Accuracy: 0.9503\n",
      "Epoch: [ 0] [  197/  468] time: 112.1820, train_loss: 0.12706186, train_accuracy: 0.9766, test_Accuracy: 0.9487\n",
      "Epoch: [ 0] [  198/  468] time: 112.7260, train_loss: 0.15720893, train_accuracy: 0.9609, test_Accuracy: 0.9502\n",
      "Epoch: [ 0] [  199/  468] time: 113.2470, train_loss: 0.11740199, train_accuracy: 0.9531, test_Accuracy: 0.9500\n",
      "Epoch: [ 0] [  200/  468] time: 113.8620, train_loss: 0.15210384, train_accuracy: 0.9297, test_Accuracy: 0.9503\n",
      "Epoch: [ 0] [  201/  468] time: 114.4010, train_loss: 0.17230707, train_accuracy: 0.9375, test_Accuracy: 0.9513\n",
      "Epoch: [ 0] [  202/  468] time: 114.9340, train_loss: 0.14336300, train_accuracy: 0.9531, test_Accuracy: 0.9499\n",
      "Epoch: [ 0] [  203/  468] time: 115.4551, train_loss: 0.09662810, train_accuracy: 0.9688, test_Accuracy: 0.9462\n",
      "Epoch: [ 0] [  204/  468] time: 116.0040, train_loss: 0.11799964, train_accuracy: 0.9688, test_Accuracy: 0.9463\n",
      "Epoch: [ 0] [  205/  468] time: 116.6360, train_loss: 0.10861127, train_accuracy: 0.9844, test_Accuracy: 0.9468\n",
      "Epoch: [ 0] [  206/  468] time: 117.1660, train_loss: 0.12042696, train_accuracy: 0.9766, test_Accuracy: 0.9507\n",
      "Epoch: [ 0] [  207/  468] time: 117.8270, train_loss: 0.14010791, train_accuracy: 0.9688, test_Accuracy: 0.9564\n",
      "Epoch: [ 0] [  208/  468] time: 118.5274, train_loss: 0.05712730, train_accuracy: 0.9844, test_Accuracy: 0.9580\n",
      "Epoch: [ 0] [  209/  468] time: 119.1973, train_loss: 0.22437137, train_accuracy: 0.9375, test_Accuracy: 0.9561\n",
      "Epoch: [ 0] [  210/  468] time: 119.9370, train_loss: 0.16471060, train_accuracy: 0.9297, test_Accuracy: 0.9551\n",
      "Epoch: [ 0] [  211/  468] time: 120.4940, train_loss: 0.16628711, train_accuracy: 0.9375, test_Accuracy: 0.9530\n",
      "Epoch: [ 0] [  212/  468] time: 121.1440, train_loss: 0.12052640, train_accuracy: 0.9609, test_Accuracy: 0.9512\n",
      "Epoch: [ 0] [  213/  468] time: 121.6710, train_loss: 0.18634848, train_accuracy: 0.9453, test_Accuracy: 0.9495\n",
      "Epoch: [ 0] [  214/  468] time: 122.2000, train_loss: 0.10067044, train_accuracy: 0.9766, test_Accuracy: 0.9474\n",
      "Epoch: [ 0] [  215/  468] time: 122.7841, train_loss: 0.17534699, train_accuracy: 0.9375, test_Accuracy: 0.9518\n",
      "Epoch: [ 0] [  216/  468] time: 123.4670, train_loss: 0.07597819, train_accuracy: 0.9766, test_Accuracy: 0.9564\n",
      "Epoch: [ 0] [  217/  468] time: 124.0030, train_loss: 0.07302228, train_accuracy: 0.9922, test_Accuracy: 0.9565\n",
      "Epoch: [ 0] [  218/  468] time: 124.4930, train_loss: 0.07810147, train_accuracy: 0.9766, test_Accuracy: 0.9550\n",
      "Epoch: [ 0] [  219/  468] time: 125.1210, train_loss: 0.11237311, train_accuracy: 0.9609, test_Accuracy: 0.9549\n",
      "Epoch: [ 0] [  220/  468] time: 125.8100, train_loss: 0.08191387, train_accuracy: 0.9844, test_Accuracy: 0.9545\n",
      "Epoch: [ 0] [  221/  468] time: 126.3810, train_loss: 0.12175871, train_accuracy: 0.9609, test_Accuracy: 0.9569\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [ 0] [  222/  468] time: 126.9470, train_loss: 0.18550387, train_accuracy: 0.9297, test_Accuracy: 0.9583\n",
      "Epoch: [ 0] [  223/  468] time: 127.4790, train_loss: 0.13875702, train_accuracy: 0.9609, test_Accuracy: 0.9588\n",
      "Epoch: [ 0] [  224/  468] time: 128.1800, train_loss: 0.09703319, train_accuracy: 0.9766, test_Accuracy: 0.9592\n",
      "Epoch: [ 0] [  225/  468] time: 128.8780, train_loss: 0.11523120, train_accuracy: 0.9609, test_Accuracy: 0.9603\n",
      "Epoch: [ 0] [  226/  468] time: 129.4720, train_loss: 0.15489781, train_accuracy: 0.9688, test_Accuracy: 0.9599\n",
      "Epoch: [ 0] [  227/  468] time: 130.1101, train_loss: 0.06454811, train_accuracy: 0.9766, test_Accuracy: 0.9559\n",
      "Epoch: [ 0] [  228/  468] time: 130.7050, train_loss: 0.07364275, train_accuracy: 0.9688, test_Accuracy: 0.9512\n",
      "Epoch: [ 0] [  229/  468] time: 131.3451, train_loss: 0.08109593, train_accuracy: 0.9766, test_Accuracy: 0.9490\n",
      "Epoch: [ 0] [  230/  468] time: 131.8820, train_loss: 0.14399084, train_accuracy: 0.9609, test_Accuracy: 0.9477\n",
      "Epoch: [ 0] [  231/  468] time: 132.4890, train_loss: 0.10062284, train_accuracy: 0.9688, test_Accuracy: 0.9478\n",
      "Epoch: [ 0] [  232/  468] time: 133.1750, train_loss: 0.10330565, train_accuracy: 0.9688, test_Accuracy: 0.9522\n",
      "Epoch: [ 0] [  233/  468] time: 133.7500, train_loss: 0.14116479, train_accuracy: 0.9688, test_Accuracy: 0.9553\n",
      "Epoch: [ 0] [  234/  468] time: 134.4020, train_loss: 0.09485675, train_accuracy: 0.9609, test_Accuracy: 0.9531\n",
      "Epoch: [ 0] [  235/  468] time: 135.1120, train_loss: 0.12568706, train_accuracy: 0.9609, test_Accuracy: 0.9493\n",
      "Epoch: [ 0] [  236/  468] time: 135.6540, train_loss: 0.14450029, train_accuracy: 0.9609, test_Accuracy: 0.9481\n",
      "Epoch: [ 0] [  237/  468] time: 136.2130, train_loss: 0.09488904, train_accuracy: 0.9688, test_Accuracy: 0.9492\n",
      "Epoch: [ 0] [  238/  468] time: 136.8000, train_loss: 0.07076455, train_accuracy: 0.9922, test_Accuracy: 0.9510\n",
      "Epoch: [ 0] [  239/  468] time: 137.4630, train_loss: 0.06647948, train_accuracy: 0.9844, test_Accuracy: 0.9515\n",
      "Epoch: [ 0] [  240/  468] time: 138.0000, train_loss: 0.12051255, train_accuracy: 0.9688, test_Accuracy: 0.9496\n",
      "Epoch: [ 0] [  241/  468] time: 138.5670, train_loss: 0.10159263, train_accuracy: 0.9766, test_Accuracy: 0.9491\n",
      "Epoch: [ 0] [  242/  468] time: 139.1341, train_loss: 0.09885356, train_accuracy: 0.9922, test_Accuracy: 0.9496\n",
      "Epoch: [ 0] [  243/  468] time: 139.8180, train_loss: 0.14129616, train_accuracy: 0.9766, test_Accuracy: 0.9522\n",
      "Epoch: [ 0] [  244/  468] time: 140.3740, train_loss: 0.09410521, train_accuracy: 0.9766, test_Accuracy: 0.9511\n",
      "Epoch: [ 0] [  245/  468] time: 140.9130, train_loss: 0.09396940, train_accuracy: 0.9766, test_Accuracy: 0.9508\n",
      "Epoch: [ 0] [  246/  468] time: 141.4320, train_loss: 0.12596256, train_accuracy: 0.9766, test_Accuracy: 0.9498\n",
      "Epoch: [ 0] [  247/  468] time: 142.0920, train_loss: 0.07341173, train_accuracy: 0.9766, test_Accuracy: 0.9515\n",
      "Epoch: [ 0] [  248/  468] time: 142.6500, train_loss: 0.07107027, train_accuracy: 0.9844, test_Accuracy: 0.9545\n",
      "Epoch: [ 0] [  249/  468] time: 143.2690, train_loss: 0.04688641, train_accuracy: 0.9844, test_Accuracy: 0.9588\n",
      "Epoch: [ 0] [  250/  468] time: 143.8140, train_loss: 0.07796918, train_accuracy: 0.9688, test_Accuracy: 0.9603\n",
      "Epoch: [ 0] [  251/  468] time: 144.3520, train_loss: 0.03744017, train_accuracy: 1.0000, test_Accuracy: 0.9618\n",
      "Epoch: [ 0] [  252/  468] time: 144.9360, train_loss: 0.09180084, train_accuracy: 0.9766, test_Accuracy: 0.9620\n",
      "Epoch: [ 0] [  253/  468] time: 145.6530, train_loss: 0.10944663, train_accuracy: 0.9688, test_Accuracy: 0.9617\n",
      "Epoch: [ 0] [  254/  468] time: 146.2510, train_loss: 0.15756759, train_accuracy: 0.9688, test_Accuracy: 0.9595\n",
      "Epoch: [ 0] [  255/  468] time: 146.8560, train_loss: 0.09875406, train_accuracy: 0.9688, test_Accuracy: 0.9573\n",
      "Epoch: [ 0] [  256/  468] time: 147.5240, train_loss: 0.09316830, train_accuracy: 0.9609, test_Accuracy: 0.9564\n",
      "Epoch: [ 0] [  257/  468] time: 148.1050, train_loss: 0.06330768, train_accuracy: 0.9766, test_Accuracy: 0.9561\n",
      "Epoch: [ 0] [  258/  468] time: 148.6610, train_loss: 0.20381925, train_accuracy: 0.9453, test_Accuracy: 0.9590\n",
      "Epoch: [ 0] [  259/  468] time: 149.3460, train_loss: 0.05260941, train_accuracy: 0.9766, test_Accuracy: 0.9608\n",
      "Epoch: [ 0] [  260/  468] time: 149.8330, train_loss: 0.04975913, train_accuracy: 0.9922, test_Accuracy: 0.9646\n",
      "Epoch: [ 0] [  261/  468] time: 150.5880, train_loss: 0.14282206, train_accuracy: 0.9531, test_Accuracy: 0.9620\n",
      "Epoch: [ 0] [  262/  468] time: 151.2870, train_loss: 0.15203407, train_accuracy: 0.9531, test_Accuracy: 0.9583\n",
      "Epoch: [ 0] [  263/  468] time: 151.8650, train_loss: 0.12207782, train_accuracy: 0.9609, test_Accuracy: 0.9560\n",
      "Epoch: [ 0] [  264/  468] time: 152.4001, train_loss: 0.15864664, train_accuracy: 0.9531, test_Accuracy: 0.9532\n",
      "Epoch: [ 0] [  265/  468] time: 153.0090, train_loss: 0.18481834, train_accuracy: 0.9297, test_Accuracy: 0.9550\n",
      "Epoch: [ 0] [  266/  468] time: 153.5155, train_loss: 0.04047289, train_accuracy: 1.0000, test_Accuracy: 0.9541\n",
      "Epoch: [ 0] [  267/  468] time: 154.0930, train_loss: 0.10389079, train_accuracy: 0.9531, test_Accuracy: 0.9531\n",
      "Epoch: [ 0] [  268/  468] time: 154.5570, train_loss: 0.09989340, train_accuracy: 0.9766, test_Accuracy: 0.9511\n",
      "Epoch: [ 0] [  269/  468] time: 155.0180, train_loss: 0.20657647, train_accuracy: 0.9453, test_Accuracy: 0.9547\n",
      "Epoch: [ 0] [  270/  468] time: 155.6500, train_loss: 0.07543568, train_accuracy: 0.9766, test_Accuracy: 0.9567\n",
      "Epoch: [ 0] [  271/  468] time: 156.2451, train_loss: 0.10067772, train_accuracy: 0.9844, test_Accuracy: 0.9572\n",
      "Epoch: [ 0] [  272/  468] time: 156.8571, train_loss: 0.15770909, train_accuracy: 0.9609, test_Accuracy: 0.9576\n",
      "Epoch: [ 0] [  273/  468] time: 157.4350, train_loss: 0.14094067, train_accuracy: 0.9531, test_Accuracy: 0.9588\n",
      "Epoch: [ 0] [  274/  468] time: 158.0981, train_loss: 0.12653351, train_accuracy: 0.9531, test_Accuracy: 0.9600\n",
      "Epoch: [ 0] [  275/  468] time: 158.6580, train_loss: 0.07999712, train_accuracy: 0.9766, test_Accuracy: 0.9607\n",
      "Epoch: [ 0] [  276/  468] time: 159.3250, train_loss: 0.04773492, train_accuracy: 0.9922, test_Accuracy: 0.9602\n",
      "Epoch: [ 0] [  277/  468] time: 159.9680, train_loss: 0.15913337, train_accuracy: 0.9531, test_Accuracy: 0.9601\n",
      "Epoch: [ 0] [  278/  468] time: 160.5860, train_loss: 0.11837241, train_accuracy: 0.9688, test_Accuracy: 0.9607\n",
      "Epoch: [ 0] [  279/  468] time: 161.0960, train_loss: 0.16017795, train_accuracy: 0.9531, test_Accuracy: 0.9593\n",
      "Epoch: [ 0] [  280/  468] time: 161.6620, train_loss: 0.15481925, train_accuracy: 0.9688, test_Accuracy: 0.9575\n",
      "Epoch: [ 0] [  281/  468] time: 162.1990, train_loss: 0.08602744, train_accuracy: 0.9688, test_Accuracy: 0.9593\n",
      "Epoch: [ 0] [  282/  468] time: 162.7621, train_loss: 0.08295472, train_accuracy: 0.9844, test_Accuracy: 0.9605\n",
      "Epoch: [ 0] [  283/  468] time: 163.3500, train_loss: 0.11799352, train_accuracy: 0.9766, test_Accuracy: 0.9623\n",
      "Epoch: [ 0] [  284/  468] time: 163.9151, train_loss: 0.15898038, train_accuracy: 0.9609, test_Accuracy: 0.9629\n",
      "Epoch: [ 0] [  285/  468] time: 164.5574, train_loss: 0.14982410, train_accuracy: 0.9609, test_Accuracy: 0.9630\n",
      "Epoch: [ 0] [  286/  468] time: 165.1670, train_loss: 0.12499452, train_accuracy: 0.9609, test_Accuracy: 0.9638\n",
      "Epoch: [ 0] [  287/  468] time: 165.7800, train_loss: 0.11308035, train_accuracy: 0.9766, test_Accuracy: 0.9635\n",
      "Epoch: [ 0] [  288/  468] time: 166.4170, train_loss: 0.18815672, train_accuracy: 0.9531, test_Accuracy: 0.9616\n",
      "Epoch: [ 0] [  289/  468] time: 167.0180, train_loss: 0.21467242, train_accuracy: 0.9375, test_Accuracy: 0.9613\n",
      "Epoch: [ 0] [  290/  468] time: 167.6140, train_loss: 0.07052885, train_accuracy: 0.9688, test_Accuracy: 0.9609\n",
      "Epoch: [ 0] [  291/  468] time: 168.1940, train_loss: 0.07511517, train_accuracy: 0.9766, test_Accuracy: 0.9615\n",
      "Epoch: [ 0] [  292/  468] time: 168.8340, train_loss: 0.12516868, train_accuracy: 0.9688, test_Accuracy: 0.9623\n",
      "Epoch: [ 0] [  293/  468] time: 169.4390, train_loss: 0.06675108, train_accuracy: 0.9844, test_Accuracy: 0.9621\n",
      "Epoch: [ 0] [  294/  468] time: 170.0030, train_loss: 0.06935921, train_accuracy: 0.9844, test_Accuracy: 0.9634\n",
      "Epoch: [ 0] [  295/  468] time: 170.4690, train_loss: 0.10515544, train_accuracy: 0.9766, test_Accuracy: 0.9634\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [ 0] [  296/  468] time: 171.0190, train_loss: 0.12335230, train_accuracy: 0.9844, test_Accuracy: 0.9642\n",
      "Epoch: [ 0] [  297/  468] time: 171.6250, train_loss: 0.16499320, train_accuracy: 0.9531, test_Accuracy: 0.9643\n",
      "Epoch: [ 0] [  298/  468] time: 172.2380, train_loss: 0.05222566, train_accuracy: 0.9922, test_Accuracy: 0.9633\n",
      "Epoch: [ 0] [  299/  468] time: 172.8250, train_loss: 0.09088504, train_accuracy: 0.9766, test_Accuracy: 0.9609\n",
      "Epoch: [ 0] [  300/  468] time: 173.3740, train_loss: 0.24009612, train_accuracy: 0.9141, test_Accuracy: 0.9614\n",
      "Epoch: [ 0] [  301/  468] time: 173.9550, train_loss: 0.07219054, train_accuracy: 0.9766, test_Accuracy: 0.9601\n",
      "Epoch: [ 0] [  302/  468] time: 174.5390, train_loss: 0.06463647, train_accuracy: 0.9844, test_Accuracy: 0.9587\n",
      "Epoch: [ 0] [  303/  468] time: 175.1000, train_loss: 0.14290556, train_accuracy: 0.9531, test_Accuracy: 0.9596\n",
      "Epoch: [ 0] [  304/  468] time: 175.7410, train_loss: 0.03129309, train_accuracy: 1.0000, test_Accuracy: 0.9612\n",
      "Epoch: [ 0] [  305/  468] time: 176.3390, train_loss: 0.13452664, train_accuracy: 0.9453, test_Accuracy: 0.9616\n",
      "Epoch: [ 0] [  306/  468] time: 176.9690, train_loss: 0.10086493, train_accuracy: 0.9688, test_Accuracy: 0.9640\n",
      "Epoch: [ 0] [  307/  468] time: 177.5476, train_loss: 0.07384108, train_accuracy: 0.9844, test_Accuracy: 0.9642\n",
      "Epoch: [ 0] [  308/  468] time: 177.9366, train_loss: 0.07199305, train_accuracy: 0.9766, test_Accuracy: 0.9616\n",
      "Epoch: [ 0] [  309/  468] time: 178.4066, train_loss: 0.12084097, train_accuracy: 0.9609, test_Accuracy: 0.9582\n",
      "Epoch: [ 0] [  310/  468] time: 179.0396, train_loss: 0.08147168, train_accuracy: 0.9688, test_Accuracy: 0.9536\n",
      "Epoch: [ 0] [  311/  468] time: 179.6326, train_loss: 0.07958452, train_accuracy: 0.9766, test_Accuracy: 0.9508\n",
      "Epoch: [ 0] [  312/  468] time: 180.3126, train_loss: 0.08632886, train_accuracy: 0.9844, test_Accuracy: 0.9497\n",
      "Epoch: [ 0] [  313/  468] time: 180.9936, train_loss: 0.10686567, train_accuracy: 0.9609, test_Accuracy: 0.9499\n",
      "Epoch: [ 0] [  314/  468] time: 181.7206, train_loss: 0.04632832, train_accuracy: 0.9844, test_Accuracy: 0.9549\n",
      "Epoch: [ 0] [  315/  468] time: 182.4656, train_loss: 0.10704137, train_accuracy: 0.9609, test_Accuracy: 0.9595\n",
      "Epoch: [ 0] [  316/  468] time: 183.1896, train_loss: 0.09955443, train_accuracy: 0.9688, test_Accuracy: 0.9600\n",
      "Epoch: [ 0] [  317/  468] time: 183.8568, train_loss: 0.21205318, train_accuracy: 0.9375, test_Accuracy: 0.9565\n",
      "Epoch: [ 0] [  318/  468] time: 184.5979, train_loss: 0.06161603, train_accuracy: 0.9922, test_Accuracy: 0.9505\n",
      "Epoch: [ 0] [  319/  468] time: 185.3548, train_loss: 0.06340843, train_accuracy: 0.9766, test_Accuracy: 0.9420\n",
      "Epoch: [ 0] [  320/  468] time: 186.0208, train_loss: 0.15034534, train_accuracy: 0.9609, test_Accuracy: 0.9407\n",
      "Epoch: [ 0] [  321/  468] time: 186.6788, train_loss: 0.18666077, train_accuracy: 0.9688, test_Accuracy: 0.9510\n",
      "Epoch: [ 0] [  322/  468] time: 187.3788, train_loss: 0.10662464, train_accuracy: 0.9688, test_Accuracy: 0.9617\n",
      "Epoch: [ 0] [  323/  468] time: 188.1468, train_loss: 0.13043380, train_accuracy: 0.9531, test_Accuracy: 0.9628\n",
      "Epoch: [ 0] [  324/  468] time: 188.8438, train_loss: 0.08392312, train_accuracy: 0.9531, test_Accuracy: 0.9535\n",
      "Epoch: [ 0] [  325/  468] time: 189.5098, train_loss: 0.08595818, train_accuracy: 0.9766, test_Accuracy: 0.9387\n",
      "Epoch: [ 0] [  326/  468] time: 190.2289, train_loss: 0.11925355, train_accuracy: 0.9688, test_Accuracy: 0.9269\n",
      "Epoch: [ 0] [  327/  468] time: 190.9709, train_loss: 0.13924912, train_accuracy: 0.9531, test_Accuracy: 0.9309\n",
      "Epoch: [ 0] [  328/  468] time: 191.7818, train_loss: 0.14554642, train_accuracy: 0.9453, test_Accuracy: 0.9464\n",
      "Epoch: [ 0] [  329/  468] time: 192.4571, train_loss: 0.04099531, train_accuracy: 1.0000, test_Accuracy: 0.9538\n",
      "Epoch: [ 0] [  330/  468] time: 193.1608, train_loss: 0.03775168, train_accuracy: 1.0000, test_Accuracy: 0.9560\n",
      "Epoch: [ 0] [  331/  468] time: 193.8828, train_loss: 0.12460242, train_accuracy: 0.9688, test_Accuracy: 0.9529\n",
      "Epoch: [ 0] [  332/  468] time: 194.5848, train_loss: 0.28276333, train_accuracy: 0.9062, test_Accuracy: 0.9520\n",
      "Epoch: [ 0] [  333/  468] time: 195.2328, train_loss: 0.06032225, train_accuracy: 0.9922, test_Accuracy: 0.9534\n",
      "Epoch: [ 0] [  334/  468] time: 195.9689, train_loss: 0.15336765, train_accuracy: 0.9688, test_Accuracy: 0.9592\n",
      "Epoch: [ 0] [  335/  468] time: 196.7008, train_loss: 0.09759131, train_accuracy: 0.9609, test_Accuracy: 0.9630\n",
      "Epoch: [ 0] [  336/  468] time: 197.4368, train_loss: 0.10350218, train_accuracy: 0.9766, test_Accuracy: 0.9626\n",
      "Epoch: [ 0] [  337/  468] time: 198.2058, train_loss: 0.08437761, train_accuracy: 0.9844, test_Accuracy: 0.9563\n",
      "Epoch: [ 0] [  338/  468] time: 198.8968, train_loss: 0.11393605, train_accuracy: 0.9453, test_Accuracy: 0.9503\n",
      "Epoch: [ 0] [  339/  468] time: 199.6388, train_loss: 0.25118476, train_accuracy: 0.9375, test_Accuracy: 0.9446\n",
      "Epoch: [ 0] [  340/  468] time: 200.3979, train_loss: 0.21358563, train_accuracy: 0.9375, test_Accuracy: 0.9489\n",
      "Epoch: [ 0] [  341/  468] time: 201.1188, train_loss: 0.06774819, train_accuracy: 0.9922, test_Accuracy: 0.9562\n",
      "Epoch: [ 0] [  342/  468] time: 201.8019, train_loss: 0.08115909, train_accuracy: 0.9688, test_Accuracy: 0.9611\n",
      "Epoch: [ 0] [  343/  468] time: 202.5208, train_loss: 0.16469935, train_accuracy: 0.9766, test_Accuracy: 0.9628\n",
      "Epoch: [ 0] [  344/  468] time: 203.1968, train_loss: 0.09429554, train_accuracy: 0.9766, test_Accuracy: 0.9640\n",
      "Epoch: [ 0] [  345/  468] time: 203.8798, train_loss: 0.06794266, train_accuracy: 0.9922, test_Accuracy: 0.9646\n",
      "Epoch: [ 0] [  346/  468] time: 204.5129, train_loss: 0.11119788, train_accuracy: 0.9609, test_Accuracy: 0.9655\n",
      "Epoch: [ 0] [  347/  468] time: 205.2348, train_loss: 0.10591757, train_accuracy: 0.9688, test_Accuracy: 0.9633\n",
      "Epoch: [ 0] [  348/  468] time: 205.8128, train_loss: 0.11301786, train_accuracy: 0.9531, test_Accuracy: 0.9630\n",
      "Epoch: [ 0] [  349/  468] time: 206.2698, train_loss: 0.14356032, train_accuracy: 0.9609, test_Accuracy: 0.9647\n",
      "Epoch: [ 0] [  350/  468] time: 206.7614, train_loss: 0.07287177, train_accuracy: 0.9844, test_Accuracy: 0.9640\n",
      "Epoch: [ 0] [  351/  468] time: 207.4311, train_loss: 0.11768156, train_accuracy: 0.9609, test_Accuracy: 0.9651\n",
      "Epoch: [ 0] [  352/  468] time: 208.1478, train_loss: 0.04479520, train_accuracy: 0.9922, test_Accuracy: 0.9660\n",
      "Epoch: [ 0] [  353/  468] time: 208.8533, train_loss: 0.08636111, train_accuracy: 0.9766, test_Accuracy: 0.9658\n",
      "Epoch: [ 0] [  354/  468] time: 209.3873, train_loss: 0.08233154, train_accuracy: 0.9609, test_Accuracy: 0.9664\n",
      "Epoch: [ 0] [  355/  468] time: 210.0113, train_loss: 0.07136191, train_accuracy: 0.9766, test_Accuracy: 0.9661\n",
      "Epoch: [ 0] [  356/  468] time: 210.6803, train_loss: 0.05393302, train_accuracy: 0.9766, test_Accuracy: 0.9646\n",
      "Epoch: [ 0] [  357/  468] time: 211.5553, train_loss: 0.05697193, train_accuracy: 0.9766, test_Accuracy: 0.9643\n",
      "Epoch: [ 0] [  358/  468] time: 212.2533, train_loss: 0.11453854, train_accuracy: 0.9766, test_Accuracy: 0.9637\n",
      "Epoch: [ 0] [  359/  468] time: 212.9053, train_loss: 0.08265205, train_accuracy: 0.9844, test_Accuracy: 0.9631\n",
      "Epoch: [ 0] [  360/  468] time: 213.6183, train_loss: 0.08360368, train_accuracy: 0.9844, test_Accuracy: 0.9634\n",
      "Epoch: [ 0] [  361/  468] time: 214.4273, train_loss: 0.08829644, train_accuracy: 0.9609, test_Accuracy: 0.9633\n",
      "Epoch: [ 0] [  362/  468] time: 215.1063, train_loss: 0.10920300, train_accuracy: 0.9688, test_Accuracy: 0.9625\n",
      "Epoch: [ 0] [  363/  468] time: 215.7973, train_loss: 0.04876321, train_accuracy: 0.9844, test_Accuracy: 0.9622\n",
      "Epoch: [ 0] [  364/  468] time: 216.5003, train_loss: 0.09753934, train_accuracy: 0.9922, test_Accuracy: 0.9620\n",
      "Epoch: [ 0] [  365/  468] time: 217.2403, train_loss: 0.07465774, train_accuracy: 0.9766, test_Accuracy: 0.9606\n",
      "Epoch: [ 0] [  366/  468] time: 218.0413, train_loss: 0.08449982, train_accuracy: 0.9766, test_Accuracy: 0.9601\n",
      "Epoch: [ 0] [  367/  468] time: 218.7103, train_loss: 0.06106004, train_accuracy: 0.9766, test_Accuracy: 0.9594\n",
      "Epoch: [ 0] [  368/  468] time: 219.3943, train_loss: 0.15932575, train_accuracy: 0.9609, test_Accuracy: 0.9608\n",
      "Epoch: [ 0] [  369/  468] time: 220.2024, train_loss: 0.10242432, train_accuracy: 0.9688, test_Accuracy: 0.9625\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [ 0] [  370/  468] time: 220.9284, train_loss: 0.07010594, train_accuracy: 0.9766, test_Accuracy: 0.9611\n",
      "Epoch: [ 0] [  371/  468] time: 221.6873, train_loss: 0.11954963, train_accuracy: 0.9531, test_Accuracy: 0.9627\n",
      "Epoch: [ 0] [  372/  468] time: 222.4103, train_loss: 0.04568664, train_accuracy: 0.9922, test_Accuracy: 0.9608\n",
      "Epoch: [ 0] [  373/  468] time: 223.1173, train_loss: 0.07847105, train_accuracy: 0.9844, test_Accuracy: 0.9592\n",
      "Epoch: [ 0] [  374/  468] time: 223.7603, train_loss: 0.12567250, train_accuracy: 0.9766, test_Accuracy: 0.9575\n",
      "Epoch: [ 0] [  375/  468] time: 224.3713, train_loss: 0.04500348, train_accuracy: 0.9844, test_Accuracy: 0.9598\n",
      "Epoch: [ 0] [  376/  468] time: 225.0703, train_loss: 0.12583478, train_accuracy: 0.9531, test_Accuracy: 0.9619\n",
      "Epoch: [ 0] [  377/  468] time: 225.7933, train_loss: 0.11606584, train_accuracy: 0.9609, test_Accuracy: 0.9605\n",
      "Epoch: [ 0] [  378/  468] time: 226.5433, train_loss: 0.06774351, train_accuracy: 0.9766, test_Accuracy: 0.9592\n",
      "Epoch: [ 0] [  379/  468] time: 227.2813, train_loss: 0.11407441, train_accuracy: 0.9609, test_Accuracy: 0.9578\n",
      "Epoch: [ 0] [  380/  468] time: 227.9733, train_loss: 0.18297160, train_accuracy: 0.9688, test_Accuracy: 0.9570\n",
      "Epoch: [ 0] [  381/  468] time: 228.7073, train_loss: 0.14466608, train_accuracy: 0.9688, test_Accuracy: 0.9569\n",
      "Epoch: [ 0] [  382/  468] time: 229.5103, train_loss: 0.10994981, train_accuracy: 0.9609, test_Accuracy: 0.9582\n",
      "Epoch: [ 0] [  383/  468] time: 230.1803, train_loss: 0.07578496, train_accuracy: 0.9766, test_Accuracy: 0.9582\n",
      "Epoch: [ 0] [  384/  468] time: 230.8953, train_loss: 0.09822007, train_accuracy: 0.9766, test_Accuracy: 0.9604\n",
      "Epoch: [ 0] [  385/  468] time: 231.5873, train_loss: 0.11537924, train_accuracy: 0.9766, test_Accuracy: 0.9618\n",
      "Epoch: [ 0] [  386/  468] time: 232.3243, train_loss: 0.08824974, train_accuracy: 0.9922, test_Accuracy: 0.9629\n",
      "Epoch: [ 0] [  387/  468] time: 233.0553, train_loss: 0.11937916, train_accuracy: 0.9766, test_Accuracy: 0.9625\n",
      "Epoch: [ 0] [  388/  468] time: 233.8043, train_loss: 0.09179685, train_accuracy: 0.9688, test_Accuracy: 0.9638\n",
      "Epoch: [ 0] [  389/  468] time: 234.4323, train_loss: 0.14483558, train_accuracy: 0.9531, test_Accuracy: 0.9655\n",
      "Epoch: [ 0] [  390/  468] time: 234.9673, train_loss: 0.05654416, train_accuracy: 0.9844, test_Accuracy: 0.9666\n",
      "Epoch: [ 0] [  391/  468] time: 235.5007, train_loss: 0.07038720, train_accuracy: 0.9844, test_Accuracy: 0.9661\n",
      "Epoch: [ 0] [  392/  468] time: 236.2039, train_loss: 0.05664115, train_accuracy: 0.9922, test_Accuracy: 0.9618\n",
      "Epoch: [ 0] [  393/  468] time: 236.8565, train_loss: 0.16646229, train_accuracy: 0.9453, test_Accuracy: 0.9583\n",
      "Epoch: [ 0] [  394/  468] time: 237.5909, train_loss: 0.10928351, train_accuracy: 0.9688, test_Accuracy: 0.9588\n",
      "Epoch: [ 0] [  395/  468] time: 238.3439, train_loss: 0.15044110, train_accuracy: 0.9609, test_Accuracy: 0.9601\n",
      "Epoch: [ 0] [  396/  468] time: 239.0629, train_loss: 0.11278442, train_accuracy: 0.9453, test_Accuracy: 0.9613\n",
      "Epoch: [ 0] [  397/  468] time: 239.7000, train_loss: 0.03931206, train_accuracy: 0.9922, test_Accuracy: 0.9632\n",
      "Epoch: [ 0] [  398/  468] time: 240.3980, train_loss: 0.07752967, train_accuracy: 0.9609, test_Accuracy: 0.9636\n",
      "Epoch: [ 0] [  399/  468] time: 241.1379, train_loss: 0.13396166, train_accuracy: 0.9766, test_Accuracy: 0.9651\n",
      "Epoch: [ 0] [  400/  468] time: 241.8490, train_loss: 0.14158040, train_accuracy: 0.9531, test_Accuracy: 0.9655\n",
      "Epoch: [ 0] [  401/  468] time: 242.4789, train_loss: 0.09618479, train_accuracy: 0.9688, test_Accuracy: 0.9653\n",
      "Epoch: [ 0] [  402/  468] time: 243.1999, train_loss: 0.12372931, train_accuracy: 0.9766, test_Accuracy: 0.9661\n",
      "Epoch: [ 0] [  403/  468] time: 243.9309, train_loss: 0.06709152, train_accuracy: 0.9766, test_Accuracy: 0.9670\n",
      "Epoch: [ 0] [  404/  468] time: 244.6089, train_loss: 0.13239932, train_accuracy: 0.9453, test_Accuracy: 0.9668\n",
      "Epoch: [ 0] [  405/  468] time: 245.2689, train_loss: 0.09303625, train_accuracy: 0.9688, test_Accuracy: 0.9660\n",
      "Epoch: [ 0] [  406/  468] time: 245.9759, train_loss: 0.07064044, train_accuracy: 0.9766, test_Accuracy: 0.9645\n",
      "Epoch: [ 0] [  407/  468] time: 246.7639, train_loss: 0.09900379, train_accuracy: 0.9531, test_Accuracy: 0.9644\n",
      "Epoch: [ 0] [  408/  468] time: 247.4299, train_loss: 0.13429573, train_accuracy: 0.9688, test_Accuracy: 0.9643\n",
      "Epoch: [ 0] [  409/  468] time: 248.1590, train_loss: 0.05059946, train_accuracy: 0.9844, test_Accuracy: 0.9638\n",
      "Epoch: [ 0] [  410/  468] time: 248.9059, train_loss: 0.10259293, train_accuracy: 0.9688, test_Accuracy: 0.9635\n",
      "Epoch: [ 0] [  411/  468] time: 249.6019, train_loss: 0.07096517, train_accuracy: 0.9844, test_Accuracy: 0.9640\n",
      "Epoch: [ 0] [  412/  468] time: 250.3059, train_loss: 0.11301866, train_accuracy: 0.9609, test_Accuracy: 0.9632\n",
      "Epoch: [ 0] [  413/  468] time: 251.0249, train_loss: 0.05248319, train_accuracy: 0.9844, test_Accuracy: 0.9640\n",
      "Epoch: [ 0] [  414/  468] time: 251.7009, train_loss: 0.11124451, train_accuracy: 0.9453, test_Accuracy: 0.9643\n",
      "Epoch: [ 0] [  415/  468] time: 252.4399, train_loss: 0.11690925, train_accuracy: 0.9531, test_Accuracy: 0.9647\n",
      "Epoch: [ 0] [  416/  468] time: 253.1999, train_loss: 0.07192250, train_accuracy: 0.9766, test_Accuracy: 0.9643\n",
      "Epoch: [ 0] [  417/  468] time: 253.8769, train_loss: 0.09121324, train_accuracy: 0.9844, test_Accuracy: 0.9635\n",
      "Epoch: [ 0] [  418/  468] time: 254.5239, train_loss: 0.14162572, train_accuracy: 0.9375, test_Accuracy: 0.9642\n",
      "Epoch: [ 0] [  419/  468] time: 255.2399, train_loss: 0.16959941, train_accuracy: 0.9531, test_Accuracy: 0.9663\n",
      "Epoch: [ 0] [  420/  468] time: 255.9189, train_loss: 0.07850392, train_accuracy: 0.9766, test_Accuracy: 0.9648\n",
      "Epoch: [ 0] [  421/  468] time: 256.6869, train_loss: 0.06780591, train_accuracy: 0.9922, test_Accuracy: 0.9639\n",
      "Epoch: [ 0] [  422/  468] time: 257.3659, train_loss: 0.02597680, train_accuracy: 0.9922, test_Accuracy: 0.9602\n",
      "Epoch: [ 0] [  423/  468] time: 258.0549, train_loss: 0.14472452, train_accuracy: 0.9688, test_Accuracy: 0.9598\n",
      "Epoch: [ 0] [  424/  468] time: 258.7929, train_loss: 0.05135126, train_accuracy: 0.9922, test_Accuracy: 0.9637\n",
      "Epoch: [ 0] [  425/  468] time: 259.5679, train_loss: 0.07612247, train_accuracy: 0.9688, test_Accuracy: 0.9643\n",
      "Epoch: [ 0] [  426/  468] time: 260.2859, train_loss: 0.09943993, train_accuracy: 0.9688, test_Accuracy: 0.9648\n",
      "Epoch: [ 0] [  427/  468] time: 260.9399, train_loss: 0.06218453, train_accuracy: 0.9922, test_Accuracy: 0.9647\n",
      "Epoch: [ 0] [  428/  468] time: 261.6739, train_loss: 0.08946741, train_accuracy: 0.9688, test_Accuracy: 0.9629\n",
      "Epoch: [ 0] [  429/  468] time: 262.4310, train_loss: 0.10339064, train_accuracy: 0.9688, test_Accuracy: 0.9604\n",
      "Epoch: [ 0] [  430/  468] time: 263.1899, train_loss: 0.03059090, train_accuracy: 0.9922, test_Accuracy: 0.9601\n",
      "Epoch: [ 0] [  431/  468] time: 263.8809, train_loss: 0.11303329, train_accuracy: 0.9688, test_Accuracy: 0.9610\n",
      "Epoch: [ 0] [  432/  468] time: 264.6209, train_loss: 0.06383786, train_accuracy: 0.9766, test_Accuracy: 0.9623\n",
      "Epoch: [ 0] [  433/  468] time: 265.3029, train_loss: 0.05309214, train_accuracy: 0.9766, test_Accuracy: 0.9612\n",
      "Epoch: [ 0] [  434/  468] time: 266.0129, train_loss: 0.16666353, train_accuracy: 0.9609, test_Accuracy: 0.9588\n",
      "Epoch: [ 0] [  435/  468] time: 266.6909, train_loss: 0.11850189, train_accuracy: 0.9844, test_Accuracy: 0.9578\n",
      "Epoch: [ 0] [  436/  468] time: 267.3519, train_loss: 0.12947004, train_accuracy: 0.9688, test_Accuracy: 0.9576\n",
      "Epoch: [ 0] [  437/  468] time: 268.0929, train_loss: 0.05211730, train_accuracy: 0.9766, test_Accuracy: 0.9587\n",
      "Epoch: [ 0] [  438/  468] time: 268.7939, train_loss: 0.07299586, train_accuracy: 0.9844, test_Accuracy: 0.9627\n",
      "Epoch: [ 0] [  439/  468] time: 269.4220, train_loss: 0.11013507, train_accuracy: 0.9688, test_Accuracy: 0.9664\n",
      "Epoch: [ 0] [  440/  468] time: 270.0919, train_loss: 0.04263991, train_accuracy: 0.9844, test_Accuracy: 0.9656\n",
      "Epoch: [ 0] [  441/  468] time: 270.8009, train_loss: 0.03104535, train_accuracy: 0.9922, test_Accuracy: 0.9638\n",
      "Epoch: [ 0] [  442/  468] time: 271.5029, train_loss: 0.05414761, train_accuracy: 0.9844, test_Accuracy: 0.9620\n",
      "Epoch: [ 0] [  443/  468] time: 272.2359, train_loss: 0.06779005, train_accuracy: 0.9844, test_Accuracy: 0.9598\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [ 0] [  444/  468] time: 272.9319, train_loss: 0.05604490, train_accuracy: 0.9766, test_Accuracy: 0.9603\n",
      "Epoch: [ 0] [  445/  468] time: 273.6129, train_loss: 0.04780240, train_accuracy: 0.9922, test_Accuracy: 0.9637\n",
      "Epoch: [ 0] [  446/  468] time: 274.3469, train_loss: 0.11859503, train_accuracy: 0.9609, test_Accuracy: 0.9660\n",
      "Epoch: [ 0] [  447/  468] time: 275.0980, train_loss: 0.06332577, train_accuracy: 0.9766, test_Accuracy: 0.9681\n",
      "Epoch: [ 0] [  448/  468] time: 275.7459, train_loss: 0.06270567, train_accuracy: 0.9844, test_Accuracy: 0.9699\n",
      "Epoch: [ 0] [  449/  468] time: 276.4629, train_loss: 0.07870732, train_accuracy: 0.9766, test_Accuracy: 0.9671\n",
      "Epoch: [ 0] [  450/  468] time: 277.2299, train_loss: 0.05496416, train_accuracy: 0.9844, test_Accuracy: 0.9635\n",
      "Epoch: [ 0] [  451/  468] time: 278.0229, train_loss: 0.21635082, train_accuracy: 0.9531, test_Accuracy: 0.9629\n",
      "Epoch: [ 0] [  452/  468] time: 278.7259, train_loss: 0.07686342, train_accuracy: 0.9766, test_Accuracy: 0.9612\n",
      "Epoch: [ 0] [  453/  468] time: 279.3739, train_loss: 0.05174074, train_accuracy: 0.9844, test_Accuracy: 0.9605\n",
      "Epoch: [ 0] [  454/  468] time: 280.0929, train_loss: 0.06739145, train_accuracy: 0.9844, test_Accuracy: 0.9609\n",
      "Epoch: [ 0] [  455/  468] time: 280.8709, train_loss: 0.09866680, train_accuracy: 0.9688, test_Accuracy: 0.9607\n",
      "Epoch: [ 0] [  456/  468] time: 281.5500, train_loss: 0.05800449, train_accuracy: 0.9844, test_Accuracy: 0.9612\n",
      "Epoch: [ 0] [  457/  468] time: 282.1829, train_loss: 0.18936977, train_accuracy: 0.9297, test_Accuracy: 0.9650\n",
      "Epoch: [ 0] [  458/  468] time: 282.9259, train_loss: 0.08236560, train_accuracy: 0.9844, test_Accuracy: 0.9666\n",
      "Epoch: [ 0] [  459/  468] time: 283.6700, train_loss: 0.05981808, train_accuracy: 0.9844, test_Accuracy: 0.9626\n",
      "Epoch: [ 0] [  460/  468] time: 284.4239, train_loss: 0.11934618, train_accuracy: 0.9766, test_Accuracy: 0.9583\n",
      "Epoch: [ 0] [  461/  468] time: 285.1239, train_loss: 0.04971399, train_accuracy: 0.9922, test_Accuracy: 0.9575\n",
      "Epoch: [ 0] [  462/  468] time: 285.8329, train_loss: 0.15048918, train_accuracy: 0.9453, test_Accuracy: 0.9562\n",
      "Epoch: [ 0] [  463/  468] time: 286.5259, train_loss: 0.09582371, train_accuracy: 0.9688, test_Accuracy: 0.9571\n",
      "Epoch: [ 0] [  464/  468] time: 287.2639, train_loss: 0.02580918, train_accuracy: 0.9922, test_Accuracy: 0.9581\n",
      "Epoch: [ 0] [  465/  468] time: 287.9883, train_loss: 0.07720602, train_accuracy: 0.9766, test_Accuracy: 0.9608\n",
      "Epoch: [ 0] [  466/  468] time: 288.7049, train_loss: 0.03650981, train_accuracy: 0.9922, test_Accuracy: 0.9615\n",
      "Epoch: [ 0] [  467/  468] time: 289.4369, train_loss: 0.13912648, train_accuracy: 0.9531, test_Accuracy: 0.9626\n"
     ]
    }
   ],
   "source": [
    "if train_flag :\n",
    "\n",
    "    checkpoint = tf.train.Checkpoint(dnn=network)\n",
    "\n",
    "    # create writer for tensorboard\n",
    "    summary_writer = tf.summary.create_file_writer(logdir=logs_dir)\n",
    "    start_time = time()\n",
    "\n",
    "    # restore check-point if it exits\n",
    "    could_load, checkpoint_counter = load(network, checkpoint_dir)    \n",
    "\n",
    "    if could_load:\n",
    "        start_epoch = (int)(checkpoint_counter / training_iterations)        \n",
    "        counter = checkpoint_counter        \n",
    "        print(\" [*] Load SUCCESS\")\n",
    "    else:\n",
    "        start_epoch = 0\n",
    "        start_iteration = 0\n",
    "        counter = 0\n",
    "        print(\" [!] Load failed...\")\n",
    "    \n",
    "    # train phase\n",
    "    with summary_writer.as_default():  # for tensorboard\n",
    "        for epoch in range(start_epoch, training_epochs):\n",
    "            for idx, (train_input, train_label) in enumerate(train_dataset):            \n",
    "                grads = grad(network, train_input, train_label)\n",
    "                optimizer.apply_gradients(grads_and_vars=zip(grads, network.variables))\n",
    "\n",
    "                train_loss = loss_fn(network, train_input, train_label)\n",
    "                train_accuracy = accuracy_fn(network, train_input, train_label)\n",
    "                \n",
    "                for test_input, test_label in test_dataset:                \n",
    "                    test_accuracy = accuracy_fn(network, test_input, test_label)\n",
    "\n",
    "                tf.summary.scalar(name='train_loss', data=train_loss, step=counter)\n",
    "                tf.summary.scalar(name='train_accuracy', data=train_accuracy, step=counter)\n",
    "                tf.summary.scalar(name='test_accuracy', data=test_accuracy, step=counter)\n",
    "\n",
    "                print(\n",
    "                    \"Epoch: [%2d] [%5d/%5d] time: %4.4f, train_loss: %.8f, train_accuracy: %.4f, test_Accuracy: %.4f\" \\\n",
    "                    % (epoch, idx, training_iterations, time() - start_time, train_loss, train_accuracy,\n",
    "                       test_accuracy))\n",
    "                counter += 1                \n",
    "        checkpoint.save(file_prefix=checkpoint_prefix + '-{}'.format(counter))\n",
    "        \n",
    "# test phase      \n",
    "else :\n",
    "    _, _ = load(network, checkpoint_dir)\n",
    "    for test_input, test_label in test_dataset:    \n",
    "        test_accuracy = accuracy_fn(network, test_input, test_label)\n",
    "\n",
    "    print(\"test_Accuracy: %.4f\" % (test_accuracy))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Test accuracy : 96.26 %"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}