Lab10/Lab10-4_SurinKim.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# import"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.5.0\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "from tensorflow.keras.utils import to_categorical\n",
    "from tensorflow.keras.datasets import mnist\n",
    "from time import time\n",
    "import os\n",
    "print(tf.__version__)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Checkpoint function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load(model, checkpoint_dir):\n",
    "    print(\" [*] Reading checkpoints...\")\n",
    "\n",
    "    ckpt = tf.train.get_checkpoint_state(checkpoint_dir)\n",
    "    if ckpt :\n",
    "        ckpt_name = os.path.basename(ckpt.model_checkpoint_path)\n",
    "        checkpoint = tf.train.Checkpoint(dnn=model)\n",
    "        checkpoint.restore(save_path=os.path.join(checkpoint_dir, ckpt_name))\n",
    "        counter = int(ckpt_name.split('-')[1])\n",
    "        print(\" [*] Success to read {}\".format(ckpt_name))\n",
    "        return True, counter\n",
    "    else:\n",
    "        print(\" [*] Failed to find a checkpoint\")\n",
    "        return False, 0\n",
    "\n",
    "def check_folder(dir):\n",
    "    if not os.path.exists(dir):\n",
    "        os.makedirs(dir)\n",
    "    return dir"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data load & pre-processing function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_mnist() :\n",
    "    (train_data, train_labels), (test_data, test_labels) = mnist.load_data()\n",
    "    train_data = np.expand_dims(train_data, axis=-1) # [N, 28, 28] -> [N, 28, 28, 1]\n",
    "    test_data = np.expand_dims(test_data, axis=-1) # [N, 28, 28] -> [N, 28, 28, 1]\n",
    "\n",
    "    train_data, test_data = normalize(train_data, test_data)\n",
    "\n",
    "    train_labels = to_categorical(train_labels, 10) # [N,] -> [N, 10]\n",
    "    test_labels = to_categorical(test_labels, 10) # [N,] -> [N, 10]\n",
    "\n",
    "    return train_data, train_labels, test_data, test_labels\n",
    "\n",
    "def normalize(train_data, test_data):\n",
    "    train_data = train_data.astype(np.float32) / 255.0\n",
    "    test_data = test_data.astype(np.float32) / 255.0\n",
    "\n",
    "    return train_data, test_data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Performance function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def loss_fn(model, images, labels):\n",
    "    logits = model(images, training=True)\n",
    "    loss = tf.reduce_mean(tf.keras.losses.categorical_crossentropy(y_pred=logits, y_true=labels, \n",
    "                                                                   from_logits=True))\n",
    "    return loss\n",
    "\n",
    "def accuracy_fn(model, images, labels):\n",
    "    logits = model(images, training=False)\n",
    "    prediction = tf.equal(tf.argmax(logits, -1), tf.argmax(labels, -1))\n",
    "    accuracy = tf.reduce_mean(tf.cast(prediction, tf.float32))\n",
    "    return accuracy\n",
    "\n",
    "def grad(model, images, labels):\n",
    "    with tf.GradientTape() as tape:\n",
    "        loss = loss_fn(model, images, labels)\n",
    "    return tape.gradient(loss, model.trainable_variables)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def flatten() :\n",
    "    return tf.keras.layers.Flatten()\n",
    "\n",
    "def dense(label_dim, weight_init) :\n",
    "    return tf.keras.layers.Dense(units=label_dim, use_bias=True, kernel_initializer=weight_init)\n",
    "\n",
    "def relu() :\n",
    "    return tf.keras.layers.Activation(tf.keras.activations.relu)\n",
    "\n",
    "def batch_norm() :\n",
    "    return tf.keras.layers.BatchNormalization()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create model (class version)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "class create_model_class(tf.keras.Model):\n",
    "    def __init__(self, label_dim):\n",
    "        super(create_model_class, self).__init__()\n",
    "        weight_init = tf.keras.initializers.glorot_uniform()\n",
    "\n",
    "        self.model = tf.keras.Sequential()\n",
    "        self.model.add(flatten())\n",
    "\n",
    "        for i in range(4):\n",
    "            self.model.add(dense(512, weight_init))\n",
    "            self.model.add(batch_norm())\n",
    "            self.model.add(relu())\n",
    "\n",
    "        self.model.add(dense(label_dim, weight_init))\n",
    "\n",
    "    def call(self, x, training=None, mask=None):\n",
    "\n",
    "        x = self.model(x)\n",
    "\n",
    "        return x"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create model (function version)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_model_function(label_dim) :\n",
    "    weight_init = tf.keras.initializers.glorot_uniform()\n",
    "\n",
    "    model = tf.keras.Sequential()\n",
    "    model.add(flatten())\n",
    "\n",
    "    for i in range(4) :\n",
    "        model.add(dense(512, weight_init))\n",
    "        model.add(batch_norm())\n",
    "        model.add(relu())\n",
    "\n",
    "    model.add(dense(label_dim, weight_init))\n",
    "\n",
    "    return model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Define data & hyper-parameter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\" dataset \"\"\"\n",
    "train_x, train_y, test_x, test_y = load_mnist()\n",
    "\n",
    "\"\"\" parameters \"\"\"\n",
    "learning_rate = 0.001\n",
    "batch_size = 128\n",
    "\n",
    "training_epochs = 1\n",
    "training_iterations = len(train_x) // batch_size\n",
    "\n",
    "label_dim = 10\n",
    "\n",
    "train_flag = True\n",
    "\n",
    "\"\"\" Graph Input using Dataset API \"\"\"\n",
    "train_dataset = tf.data.Dataset.from_tensor_slices((train_x, train_y)).\\\n",
    "    shuffle(buffer_size=100000).\\\n",
    "    prefetch(buffer_size=batch_size).\\\n",
    "    batch(batch_size, drop_remainder=True)\n",
    "\n",
    "test_dataset = tf.data.Dataset.from_tensor_slices((test_x, test_y)).\\\n",
    "    shuffle(buffer_size=100000).\\\n",
    "    prefetch(buffer_size=len(test_x)).\\\n",
    "    batch(len(test_x))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Define model & optimizer & writer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\" Model \"\"\"\n",
    "network = create_model_function(label_dim)\n",
    "\n",
    "\"\"\" Training \"\"\"\n",
    "optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate)\n",
    "\n",
    "\"\"\" Writer \"\"\"\n",
    "checkpoint_dir = 'checkpoints'\n",
    "logs_dir = 'logs'\n",
    "\n",
    "model_dir = 'nn_batchnorm'\n",
    "\n",
    "checkpoint_dir = os.path.join(checkpoint_dir, model_dir)\n",
    "check_folder(checkpoint_dir)\n",
    "checkpoint_prefix = os.path.join(checkpoint_dir, model_dir)\n",
    "logs_dir = os.path.join(logs_dir, model_dir)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Restore checkpoint & start train or test phase"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " [*] Reading checkpoints...\n",
      " [*] Failed to find a checkpoint\n",
      " [!] Load failed...\n",
      "Epoch: [ 0] [    0/  468] time: 2.8620, train_loss: 0.27537161, train_accuracy: 0.5312, test_Accuracy: 0.3502\n",
      "Epoch: [ 0] [    1/  468] time: 3.8740, train_loss: 0.34401697, train_accuracy: 0.5234, test_Accuracy: 0.4880\n",
      "Epoch: [ 0] [    2/  468] time: 4.8634, train_loss: 0.27018720, train_accuracy: 0.5938, test_Accuracy: 0.5739\n",
      "Epoch: [ 0] [    3/  468] time: 5.8232, train_loss: 0.30082202, train_accuracy: 0.6797, test_Accuracy: 0.6101\n",
      "Epoch: [ 0] [    4/  468] time: 6.8242, train_loss: 0.34786552, train_accuracy: 0.6953, test_Accuracy: 0.6344\n",
      "Epoch: [ 0] [    5/  468] time: 7.7182, train_loss: 0.29289582, train_accuracy: 0.7031, test_Accuracy: 0.6690\n",
      "Epoch: [ 0] [    6/  468] time: 8.6012, train_loss: 0.27103245, train_accuracy: 0.6875, test_Accuracy: 0.6954\n",
      "Epoch: [ 0] [    7/  468] time: 9.5582, train_loss: 0.27577981, train_accuracy: 0.7344, test_Accuracy: 0.7141\n",
      "Epoch: [ 0] [    8/  468] time: 10.4733, train_loss: 0.23698965, train_accuracy: 0.7656, test_Accuracy: 0.7248\n",
      "Epoch: [ 0] [    9/  468] time: 11.3202, train_loss: 0.25550038, train_accuracy: 0.7031, test_Accuracy: 0.7359\n",
      "Epoch: [ 0] [   10/  468] time: 12.3342, train_loss: 0.24627964, train_accuracy: 0.7578, test_Accuracy: 0.7574\n",
      "Epoch: [ 0] [   11/  468] time: 13.3592, train_loss: 0.24591868, train_accuracy: 0.8203, test_Accuracy: 0.7726\n",
      "Epoch: [ 0] [   12/  468] time: 14.2662, train_loss: 0.30439779, train_accuracy: 0.7812, test_Accuracy: 0.7855\n",
      "Epoch: [ 0] [   13/  468] time: 15.2552, train_loss: 0.21470609, train_accuracy: 0.8516, test_Accuracy: 0.8072\n",
      "Epoch: [ 0] [   14/  468] time: 16.2173, train_loss: 0.30335063, train_accuracy: 0.8047, test_Accuracy: 0.8258\n",
      "Epoch: [ 0] [   15/  468] time: 17.0912, train_loss: 0.26848161, train_accuracy: 0.8516, test_Accuracy: 0.8364\n",
      "Epoch: [ 0] [   16/  468] time: 18.0962, train_loss: 0.25166994, train_accuracy: 0.8516, test_Accuracy: 0.8473\n",
      "Epoch: [ 0] [   17/  468] time: 18.9993, train_loss: 0.21424377, train_accuracy: 0.8516, test_Accuracy: 0.8583\n",
      "Epoch: [ 0] [   18/  468] time: 19.9642, train_loss: 0.23342279, train_accuracy: 0.8750, test_Accuracy: 0.8635\n",
      "Epoch: [ 0] [   19/  468] time: 20.9293, train_loss: 0.15941752, train_accuracy: 0.8281, test_Accuracy: 0.8687\n",
      "Epoch: [ 0] [   20/  468] time: 21.8642, train_loss: 0.19214478, train_accuracy: 0.8672, test_Accuracy: 0.8669\n",
      "Epoch: [ 0] [   21/  468] time: 22.8163, train_loss: 0.23223698, train_accuracy: 0.8828, test_Accuracy: 0.8661\n",
      "Epoch: [ 0] [   22/  468] time: 23.7822, train_loss: 0.24826619, train_accuracy: 0.8906, test_Accuracy: 0.8698\n",
      "Epoch: [ 0] [   23/  468] time: 24.7173, train_loss: 0.23924139, train_accuracy: 0.8672, test_Accuracy: 0.8715\n",
      "Epoch: [ 0] [   24/  468] time: 25.5312, train_loss: 0.24209005, train_accuracy: 0.8672, test_Accuracy: 0.8750\n",
      "Epoch: [ 0] [   25/  468] time: 26.3752, train_loss: 0.26189458, train_accuracy: 0.8438, test_Accuracy: 0.8801\n",
      "Epoch: [ 0] [   26/  468] time: 27.1962, train_loss: 0.19513167, train_accuracy: 0.8672, test_Accuracy: 0.8857\n",
      "Epoch: [ 0] [   27/  468] time: 27.8272, train_loss: 0.14152125, train_accuracy: 0.9453, test_Accuracy: 0.8939\n",
      "Epoch: [ 0] [   28/  468] time: 28.5095, train_loss: 0.15522140, train_accuracy: 0.9062, test_Accuracy: 0.8995\n",
      "Epoch: [ 0] [   29/  468] time: 29.5036, train_loss: 0.19580574, train_accuracy: 0.9062, test_Accuracy: 0.9016\n",
      "Epoch: [ 0] [   30/  468] time: 30.4526, train_loss: 0.17872396, train_accuracy: 0.9062, test_Accuracy: 0.8988\n",
      "Epoch: [ 0] [   31/  468] time: 31.2047, train_loss: 0.20068786, train_accuracy: 0.8906, test_Accuracy: 0.8967\n",
      "Epoch: [ 0] [   32/  468] time: 32.0408, train_loss: 0.14087908, train_accuracy: 0.9375, test_Accuracy: 0.8951\n",
      "Epoch: [ 0] [   33/  468] time: 33.0737, train_loss: 0.21250369, train_accuracy: 0.9219, test_Accuracy: 0.8936\n",
      "Epoch: [ 0] [   34/  468] time: 33.9366, train_loss: 0.21180722, train_accuracy: 0.9062, test_Accuracy: 0.8965\n",
      "Epoch: [ 0] [   35/  468] time: 34.9126, train_loss: 0.16797295, train_accuracy: 0.8984, test_Accuracy: 0.9001\n",
      "Epoch: [ 0] [   36/  468] time: 36.0041, train_loss: 0.17035146, train_accuracy: 0.9297, test_Accuracy: 0.9022\n",
      "Epoch: [ 0] [   37/  468] time: 36.8907, train_loss: 0.17173383, train_accuracy: 0.9141, test_Accuracy: 0.9026\n",
      "Epoch: [ 0] [   38/  468] time: 37.8398, train_loss: 0.28933579, train_accuracy: 0.8750, test_Accuracy: 0.9010\n",
      "Epoch: [ 0] [   39/  468] time: 38.7917, train_loss: 0.24485992, train_accuracy: 0.8906, test_Accuracy: 0.9033\n",
      "Epoch: [ 0] [   40/  468] time: 39.7576, train_loss: 0.16189623, train_accuracy: 0.8906, test_Accuracy: 0.9077\n",
      "Epoch: [ 0] [   41/  468] time: 40.7266, train_loss: 0.18265268, train_accuracy: 0.9062, test_Accuracy: 0.9121\n",
      "Epoch: [ 0] [   42/  468] time: 41.7657, train_loss: 0.15081738, train_accuracy: 0.9375, test_Accuracy: 0.9124\n",
      "Epoch: [ 0] [   43/  468] time: 42.6830, train_loss: 0.11902346, train_accuracy: 0.9297, test_Accuracy: 0.9129\n",
      "Epoch: [ 0] [   44/  468] time: 43.6176, train_loss: 0.14607213, train_accuracy: 0.9531, test_Accuracy: 0.9125\n",
      "Epoch: [ 0] [   45/  468] time: 44.4927, train_loss: 0.12607712, train_accuracy: 0.9297, test_Accuracy: 0.9102\n",
      "Epoch: [ 0] [   46/  468] time: 45.3266, train_loss: 0.15003528, train_accuracy: 0.9375, test_Accuracy: 0.9085\n",
      "Epoch: [ 0] [   47/  468] time: 46.1436, train_loss: 0.08804449, train_accuracy: 0.9375, test_Accuracy: 0.9064\n",
      "Epoch: [ 0] [   48/  468] time: 47.1478, train_loss: 0.13058044, train_accuracy: 0.9531, test_Accuracy: 0.9049\n",
      "Epoch: [ 0] [   49/  468] time: 48.1067, train_loss: 0.08491370, train_accuracy: 0.9609, test_Accuracy: 0.9042\n",
      "Epoch: [ 0] [   50/  468] time: 49.0386, train_loss: 0.19439186, train_accuracy: 0.8906, test_Accuracy: 0.9038\n",
      "Epoch: [ 0] [   51/  468] time: 49.9997, train_loss: 0.26905358, train_accuracy: 0.8906, test_Accuracy: 0.9050\n",
      "Epoch: [ 0] [   52/  468] time: 50.9567, train_loss: 0.16140009, train_accuracy: 0.9141, test_Accuracy: 0.9093\n",
      "Epoch: [ 0] [   53/  468] time: 51.8736, train_loss: 0.13528177, train_accuracy: 0.9531, test_Accuracy: 0.9139\n",
      "Epoch: [ 0] [   54/  468] time: 52.7166, train_loss: 0.09648548, train_accuracy: 0.9219, test_Accuracy: 0.9179\n",
      "Epoch: [ 0] [   55/  468] time: 53.6926, train_loss: 0.08855483, train_accuracy: 0.9531, test_Accuracy: 0.9227\n",
      "Epoch: [ 0] [   56/  468] time: 54.6277, train_loss: 0.14348119, train_accuracy: 0.9297, test_Accuracy: 0.9270\n",
      "Epoch: [ 0] [   57/  468] time: 55.5727, train_loss: 0.12022658, train_accuracy: 0.9609, test_Accuracy: 0.9316\n",
      "Epoch: [ 0] [   58/  468] time: 56.3456, train_loss: 0.14717881, train_accuracy: 0.9297, test_Accuracy: 0.9357\n",
      "Epoch: [ 0] [   59/  468] time: 57.0524, train_loss: 0.14634405, train_accuracy: 0.9453, test_Accuracy: 0.9373\n",
      "Epoch: [ 0] [   60/  468] time: 57.9336, train_loss: 0.21061462, train_accuracy: 0.9219, test_Accuracy: 0.9356\n",
      "Epoch: [ 0] [   61/  468] time: 58.8793, train_loss: 0.13906209, train_accuracy: 0.9531, test_Accuracy: 0.9357\n",
      "Epoch: [ 0] [   62/  468] time: 59.8023, train_loss: 0.15122026, train_accuracy: 0.9375, test_Accuracy: 0.9353\n",
      "Epoch: [ 0] [   63/  468] time: 60.6963, train_loss: 0.29094410, train_accuracy: 0.9141, test_Accuracy: 0.9340\n",
      "Epoch: [ 0] [   64/  468] time: 61.5213, train_loss: 0.13714415, train_accuracy: 0.9531, test_Accuracy: 0.9324\n",
      "Epoch: [ 0] [   65/  468] time: 62.5783, train_loss: 0.17643018, train_accuracy: 0.9453, test_Accuracy: 0.9284\n",
      "Epoch: [ 0] [   66/  468] time: 63.4853, train_loss: 0.19431746, train_accuracy: 0.9219, test_Accuracy: 0.9245\n",
      "Epoch: [ 0] [   67/  468] time: 64.3863, train_loss: 0.08297083, train_accuracy: 0.9609, test_Accuracy: 0.9234\n",
      "Epoch: [ 0] [   68/  468] time: 65.3803, train_loss: 0.10782666, train_accuracy: 0.9688, test_Accuracy: 0.9195\n",
      "Epoch: [ 0] [   69/  468] time: 66.2523, train_loss: 0.10296622, train_accuracy: 0.9297, test_Accuracy: 0.9163\n",
      "Epoch: [ 0] [   70/  468] time: 67.1974, train_loss: 0.12909220, train_accuracy: 0.9375, test_Accuracy: 0.9174\n",
      "Epoch: [ 0] [   71/  468] time: 68.1893, train_loss: 0.07130645, train_accuracy: 0.9453, test_Accuracy: 0.9195\n",
      "Epoch: [ 0] [   72/  468] time: 69.1033, train_loss: 0.20556661, train_accuracy: 0.9297, test_Accuracy: 0.9201\n",
      "Epoch: [ 0] [   73/  468] time: 69.9723, train_loss: 0.13544376, train_accuracy: 0.9297, test_Accuracy: 0.9209\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [ 0] [   74/  468] time: 70.9623, train_loss: 0.07822019, train_accuracy: 0.9609, test_Accuracy: 0.9227\n",
      "Epoch: [ 0] [   75/  468] time: 71.8633, train_loss: 0.15872654, train_accuracy: 0.9297, test_Accuracy: 0.9263\n",
      "Epoch: [ 0] [   76/  468] time: 72.7673, train_loss: 0.09403252, train_accuracy: 0.9766, test_Accuracy: 0.9291\n",
      "Epoch: [ 0] [   77/  468] time: 73.6963, train_loss: 0.12394293, train_accuracy: 0.9453, test_Accuracy: 0.9321\n",
      "Epoch: [ 0] [   78/  468] time: 74.6493, train_loss: 0.25185615, train_accuracy: 0.9062, test_Accuracy: 0.9344\n",
      "Epoch: [ 0] [   79/  468] time: 75.5424, train_loss: 0.16334257, train_accuracy: 0.9297, test_Accuracy: 0.9369\n",
      "Epoch: [ 0] [   80/  468] time: 76.4423, train_loss: 0.07922050, train_accuracy: 0.9766, test_Accuracy: 0.9377\n",
      "Epoch: [ 0] [   81/  468] time: 77.3075, train_loss: 0.15742485, train_accuracy: 0.9609, test_Accuracy: 0.9388\n",
      "Epoch: [ 0] [   82/  468] time: 78.2463, train_loss: 0.10561615, train_accuracy: 0.9453, test_Accuracy: 0.9396\n",
      "Epoch: [ 0] [   83/  468] time: 79.1223, train_loss: 0.14852531, train_accuracy: 0.9531, test_Accuracy: 0.9399\n",
      "Epoch: [ 0] [   84/  468] time: 80.0803, train_loss: 0.06496614, train_accuracy: 0.9922, test_Accuracy: 0.9405\n",
      "Epoch: [ 0] [   85/  468] time: 81.0413, train_loss: 0.07934989, train_accuracy: 0.9766, test_Accuracy: 0.9413\n",
      "Epoch: [ 0] [   86/  468] time: 81.9233, train_loss: 0.08318973, train_accuracy: 0.9688, test_Accuracy: 0.9405\n",
      "Epoch: [ 0] [   87/  468] time: 82.7573, train_loss: 0.14091170, train_accuracy: 0.9453, test_Accuracy: 0.9405\n",
      "Epoch: [ 0] [   88/  468] time: 83.7213, train_loss: 0.10369065, train_accuracy: 0.9453, test_Accuracy: 0.9404\n",
      "Epoch: [ 0] [   89/  468] time: 84.6873, train_loss: 0.10485943, train_accuracy: 0.9453, test_Accuracy: 0.9410\n",
      "Epoch: [ 0] [   90/  468] time: 85.5453, train_loss: 0.17018737, train_accuracy: 0.9453, test_Accuracy: 0.9410\n",
      "Epoch: [ 0] [   91/  468] time: 86.4483, train_loss: 0.07859549, train_accuracy: 0.9609, test_Accuracy: 0.9421\n",
      "Epoch: [ 0] [   92/  468] time: 87.4623, train_loss: 0.16666940, train_accuracy: 0.9453, test_Accuracy: 0.9410\n",
      "Epoch: [ 0] [   93/  468] time: 88.3433, train_loss: 0.13826665, train_accuracy: 0.9375, test_Accuracy: 0.9415\n",
      "Epoch: [ 0] [   94/  468] time: 89.2163, train_loss: 0.11263660, train_accuracy: 0.9609, test_Accuracy: 0.9432\n",
      "Epoch: [ 0] [   95/  468] time: 90.2103, train_loss: 0.16506022, train_accuracy: 0.9453, test_Accuracy: 0.9449\n",
      "Epoch: [ 0] [   96/  468] time: 91.0823, train_loss: 0.13241605, train_accuracy: 0.9453, test_Accuracy: 0.9456\n",
      "Epoch: [ 0] [   97/  468] time: 91.8623, train_loss: 0.13089949, train_accuracy: 0.9453, test_Accuracy: 0.9466\n",
      "Epoch: [ 0] [   98/  468] time: 92.8463, train_loss: 0.10762928, train_accuracy: 0.9609, test_Accuracy: 0.9473\n",
      "Epoch: [ 0] [   99/  468] time: 93.8343, train_loss: 0.09860646, train_accuracy: 0.9453, test_Accuracy: 0.9479\n",
      "Epoch: [ 0] [  100/  468] time: 94.7013, train_loss: 0.30474201, train_accuracy: 0.9062, test_Accuracy: 0.9469\n",
      "Epoch: [ 0] [  101/  468] time: 95.6603, train_loss: 0.13157821, train_accuracy: 0.9453, test_Accuracy: 0.9471\n",
      "Epoch: [ 0] [  102/  468] time: 96.6443, train_loss: 0.13422912, train_accuracy: 0.9453, test_Accuracy: 0.9477\n",
      "Epoch: [ 0] [  103/  468] time: 97.5343, train_loss: 0.06655706, train_accuracy: 0.9844, test_Accuracy: 0.9474\n",
      "Epoch: [ 0] [  104/  468] time: 98.4663, train_loss: 0.06437741, train_accuracy: 0.9766, test_Accuracy: 0.9468\n",
      "Epoch: [ 0] [  105/  468] time: 99.4953, train_loss: 0.16689488, train_accuracy: 0.9531, test_Accuracy: 0.9467\n",
      "Epoch: [ 0] [  106/  468] time: 100.4473, train_loss: 0.24070930, train_accuracy: 0.9141, test_Accuracy: 0.9478\n",
      "Epoch: [ 0] [  107/  468] time: 101.3103, train_loss: 0.12291798, train_accuracy: 0.9688, test_Accuracy: 0.9487\n",
      "Epoch: [ 0] [  108/  468] time: 102.3133, train_loss: 0.13276953, train_accuracy: 0.9531, test_Accuracy: 0.9495\n",
      "Epoch: [ 0] [  109/  468] time: 103.2136, train_loss: 0.06395997, train_accuracy: 0.9922, test_Accuracy: 0.9497\n",
      "Epoch: [ 0] [  110/  468] time: 104.0953, train_loss: 0.08103774, train_accuracy: 0.9688, test_Accuracy: 0.9508\n",
      "Epoch: [ 0] [  111/  468] time: 105.0123, train_loss: 0.18112347, train_accuracy: 0.9453, test_Accuracy: 0.9517\n",
      "Epoch: [ 0] [  112/  468] time: 106.0033, train_loss: 0.04046202, train_accuracy: 0.9922, test_Accuracy: 0.9523\n",
      "Epoch: [ 0] [  113/  468] time: 106.8973, train_loss: 0.09748377, train_accuracy: 0.9609, test_Accuracy: 0.9519\n",
      "Epoch: [ 0] [  114/  468] time: 107.9073, train_loss: 0.08163347, train_accuracy: 0.9688, test_Accuracy: 0.9524\n",
      "Epoch: [ 0] [  115/  468] time: 108.8423, train_loss: 0.17556396, train_accuracy: 0.9766, test_Accuracy: 0.9537\n",
      "Epoch: [ 0] [  116/  468] time: 109.7453, train_loss: 0.11581197, train_accuracy: 0.9453, test_Accuracy: 0.9546\n",
      "Epoch: [ 0] [  117/  468] time: 110.6553, train_loss: 0.08169508, train_accuracy: 0.9609, test_Accuracy: 0.9548\n",
      "Epoch: [ 0] [  118/  468] time: 111.3723, train_loss: 0.11038373, train_accuracy: 0.9453, test_Accuracy: 0.9546\n",
      "Epoch: [ 0] [  119/  468] time: 112.1784, train_loss: 0.13424855, train_accuracy: 0.9531, test_Accuracy: 0.9545\n",
      "Epoch: [ 0] [  120/  468] time: 113.0233, train_loss: 0.16606402, train_accuracy: 0.9531, test_Accuracy: 0.9541\n",
      "Epoch: [ 0] [  121/  468] time: 113.8473, train_loss: 0.07690522, train_accuracy: 0.9844, test_Accuracy: 0.9544\n",
      "Epoch: [ 0] [  122/  468] time: 114.6963, train_loss: 0.13567019, train_accuracy: 0.9375, test_Accuracy: 0.9549\n",
      "Epoch: [ 0] [  123/  468] time: 115.4473, train_loss: 0.10975484, train_accuracy: 0.9453, test_Accuracy: 0.9542\n",
      "Epoch: [ 0] [  124/  468] time: 116.3423, train_loss: 0.14850734, train_accuracy: 0.9531, test_Accuracy: 0.9534\n",
      "Epoch: [ 0] [  125/  468] time: 117.2523, train_loss: 0.08153007, train_accuracy: 0.9688, test_Accuracy: 0.9523\n",
      "Epoch: [ 0] [  126/  468] time: 118.1433, train_loss: 0.09607461, train_accuracy: 0.9453, test_Accuracy: 0.9509\n",
      "Epoch: [ 0] [  127/  468] time: 118.9223, train_loss: 0.10482172, train_accuracy: 0.9609, test_Accuracy: 0.9499\n",
      "Epoch: [ 0] [  128/  468] time: 119.7283, train_loss: 0.11212655, train_accuracy: 0.9609, test_Accuracy: 0.9487\n",
      "Epoch: [ 0] [  129/  468] time: 120.4313, train_loss: 0.05904022, train_accuracy: 0.9922, test_Accuracy: 0.9471\n",
      "Epoch: [ 0] [  130/  468] time: 121.2063, train_loss: 0.09338720, train_accuracy: 0.9688, test_Accuracy: 0.9466\n",
      "Epoch: [ 0] [  131/  468] time: 121.9493, train_loss: 0.07401651, train_accuracy: 0.9688, test_Accuracy: 0.9449\n",
      "Epoch: [ 0] [  132/  468] time: 122.6213, train_loss: 0.10992053, train_accuracy: 0.9609, test_Accuracy: 0.9459\n",
      "Epoch: [ 0] [  133/  468] time: 123.3693, train_loss: 0.05047295, train_accuracy: 0.9766, test_Accuracy: 0.9460\n",
      "Epoch: [ 0] [  134/  468] time: 124.0843, train_loss: 0.14582781, train_accuracy: 0.9531, test_Accuracy: 0.9467\n",
      "Epoch: [ 0] [  135/  468] time: 124.8313, train_loss: 0.11469387, train_accuracy: 0.9609, test_Accuracy: 0.9483\n",
      "Epoch: [ 0] [  136/  468] time: 125.5163, train_loss: 0.10786319, train_accuracy: 0.9688, test_Accuracy: 0.9503\n",
      "Epoch: [ 0] [  137/  468] time: 126.2773, train_loss: 0.03220951, train_accuracy: 0.9922, test_Accuracy: 0.9518\n",
      "Epoch: [ 0] [  138/  468] time: 127.0593, train_loss: 0.05643971, train_accuracy: 0.9844, test_Accuracy: 0.9520\n",
      "Epoch: [ 0] [  139/  468] time: 127.7893, train_loss: 0.14675188, train_accuracy: 0.9609, test_Accuracy: 0.9538\n",
      "Epoch: [ 0] [  140/  468] time: 128.5593, train_loss: 0.10121212, train_accuracy: 0.9609, test_Accuracy: 0.9542\n",
      "Epoch: [ 0] [  141/  468] time: 129.3243, train_loss: 0.16515273, train_accuracy: 0.9609, test_Accuracy: 0.9546\n",
      "Epoch: [ 0] [  142/  468] time: 130.0527, train_loss: 0.15015122, train_accuracy: 0.9609, test_Accuracy: 0.9558\n",
      "Epoch: [ 0] [  143/  468] time: 130.8863, train_loss: 0.06165957, train_accuracy: 0.9844, test_Accuracy: 0.9557\n",
      "Epoch: [ 0] [  144/  468] time: 131.6933, train_loss: 0.06306495, train_accuracy: 0.9844, test_Accuracy: 0.9551\n",
      "Epoch: [ 0] [  145/  468] time: 132.6063, train_loss: 0.16565900, train_accuracy: 0.9688, test_Accuracy: 0.9544\n",
      "Epoch: [ 0] [  146/  468] time: 133.5053, train_loss: 0.05256497, train_accuracy: 1.0000, test_Accuracy: 0.9544\n",
      "Epoch: [ 0] [  147/  468] time: 134.2344, train_loss: 0.04763848, train_accuracy: 0.9922, test_Accuracy: 0.9560\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [ 0] [  148/  468] time: 135.0223, train_loss: 0.09208603, train_accuracy: 0.9766, test_Accuracy: 0.9563\n",
      "Epoch: [ 0] [  149/  468] time: 135.8293, train_loss: 0.14874858, train_accuracy: 0.9531, test_Accuracy: 0.9562\n",
      "Epoch: [ 0] [  150/  468] time: 136.5744, train_loss: 0.09304371, train_accuracy: 0.9609, test_Accuracy: 0.9560\n",
      "Epoch: [ 0] [  151/  468] time: 137.3163, train_loss: 0.13258657, train_accuracy: 0.9453, test_Accuracy: 0.9560\n",
      "Epoch: [ 0] [  152/  468] time: 138.0483, train_loss: 0.16539337, train_accuracy: 0.9453, test_Accuracy: 0.9558\n",
      "Epoch: [ 0] [  153/  468] time: 138.7403, train_loss: 0.10524774, train_accuracy: 0.9688, test_Accuracy: 0.9560\n",
      "Epoch: [ 0] [  154/  468] time: 139.5113, train_loss: 0.11404105, train_accuracy: 0.9766, test_Accuracy: 0.9554\n",
      "Epoch: [ 0] [  155/  468] time: 140.2373, train_loss: 0.09369515, train_accuracy: 0.9688, test_Accuracy: 0.9557\n",
      "Epoch: [ 0] [  156/  468] time: 140.9523, train_loss: 0.06039258, train_accuracy: 0.9922, test_Accuracy: 0.9567\n",
      "Epoch: [ 0] [  157/  468] time: 141.6603, train_loss: 0.09749892, train_accuracy: 0.9609, test_Accuracy: 0.9552\n",
      "Epoch: [ 0] [  158/  468] time: 142.4083, train_loss: 0.12102440, train_accuracy: 0.9688, test_Accuracy: 0.9556\n",
      "Epoch: [ 0] [  159/  468] time: 143.1463, train_loss: 0.07244458, train_accuracy: 0.9844, test_Accuracy: 0.9558\n",
      "Epoch: [ 0] [  160/  468] time: 143.9183, train_loss: 0.09334658, train_accuracy: 0.9688, test_Accuracy: 0.9571\n",
      "Epoch: [ 0] [  161/  468] time: 144.6283, train_loss: 0.08084169, train_accuracy: 0.9688, test_Accuracy: 0.9580\n",
      "Epoch: [ 0] [  162/  468] time: 145.3393, train_loss: 0.04678861, train_accuracy: 0.9688, test_Accuracy: 0.9586\n",
      "Epoch: [ 0] [  163/  468] time: 146.0663, train_loss: 0.09548113, train_accuracy: 0.9688, test_Accuracy: 0.9585\n",
      "Epoch: [ 0] [  164/  468] time: 146.8179, train_loss: 0.05018300, train_accuracy: 0.9844, test_Accuracy: 0.9587\n",
      "Epoch: [ 0] [  165/  468] time: 147.5483, train_loss: 0.06349960, train_accuracy: 0.9844, test_Accuracy: 0.9593\n",
      "Epoch: [ 0] [  166/  468] time: 148.3543, train_loss: 0.13359307, train_accuracy: 0.9688, test_Accuracy: 0.9591\n",
      "Epoch: [ 0] [  167/  468] time: 149.1073, train_loss: 0.12884128, train_accuracy: 0.9375, test_Accuracy: 0.9587\n",
      "Epoch: [ 0] [  168/  468] time: 149.9223, train_loss: 0.07132173, train_accuracy: 0.9766, test_Accuracy: 0.9587\n",
      "Epoch: [ 0] [  169/  468] time: 150.7073, train_loss: 0.07443382, train_accuracy: 0.9609, test_Accuracy: 0.9589\n",
      "Epoch: [ 0] [  170/  468] time: 151.2923, train_loss: 0.07066710, train_accuracy: 0.9922, test_Accuracy: 0.9593\n",
      "Epoch: [ 0] [  171/  468] time: 152.0433, train_loss: 0.10081662, train_accuracy: 0.9688, test_Accuracy: 0.9587\n",
      "Epoch: [ 0] [  172/  468] time: 152.8433, train_loss: 0.03853673, train_accuracy: 0.9922, test_Accuracy: 0.9584\n",
      "Epoch: [ 0] [  173/  468] time: 153.5753, train_loss: 0.21248636, train_accuracy: 0.9375, test_Accuracy: 0.9585\n",
      "Epoch: [ 0] [  174/  468] time: 154.3133, train_loss: 0.05093906, train_accuracy: 0.9844, test_Accuracy: 0.9586\n",
      "Epoch: [ 0] [  175/  468] time: 155.0536, train_loss: 0.08797800, train_accuracy: 0.9609, test_Accuracy: 0.9592\n",
      "Epoch: [ 0] [  176/  468] time: 155.7353, train_loss: 0.15156710, train_accuracy: 0.9609, test_Accuracy: 0.9588\n",
      "Epoch: [ 0] [  177/  468] time: 156.4943, train_loss: 0.11666347, train_accuracy: 0.9688, test_Accuracy: 0.9590\n",
      "Epoch: [ 0] [  178/  468] time: 157.2003, train_loss: 0.04714293, train_accuracy: 0.9844, test_Accuracy: 0.9599\n",
      "Epoch: [ 0] [  179/  468] time: 158.0243, train_loss: 0.05792861, train_accuracy: 0.9766, test_Accuracy: 0.9592\n",
      "Epoch: [ 0] [  180/  468] time: 158.7943, train_loss: 0.04929895, train_accuracy: 0.9922, test_Accuracy: 0.9580\n",
      "Epoch: [ 0] [  181/  468] time: 159.5613, train_loss: 0.08016926, train_accuracy: 0.9844, test_Accuracy: 0.9580\n",
      "Epoch: [ 0] [  182/  468] time: 160.3483, train_loss: 0.10088962, train_accuracy: 0.9844, test_Accuracy: 0.9576\n",
      "Epoch: [ 0] [  183/  468] time: 161.0943, train_loss: 0.08313914, train_accuracy: 0.9688, test_Accuracy: 0.9575\n",
      "Epoch: [ 0] [  184/  468] time: 161.7853, train_loss: 0.03846072, train_accuracy: 1.0000, test_Accuracy: 0.9568\n",
      "Epoch: [ 0] [  185/  468] time: 162.5283, train_loss: 0.04812165, train_accuracy: 1.0000, test_Accuracy: 0.9569\n",
      "Epoch: [ 0] [  186/  468] time: 163.2690, train_loss: 0.07566922, train_accuracy: 0.9688, test_Accuracy: 0.9574\n",
      "Epoch: [ 0] [  187/  468] time: 164.1163, train_loss: 0.12355383, train_accuracy: 0.9688, test_Accuracy: 0.9592\n",
      "Epoch: [ 0] [  188/  468] time: 164.8453, train_loss: 0.14783257, train_accuracy: 0.9531, test_Accuracy: 0.9593\n",
      "Epoch: [ 0] [  189/  468] time: 165.5843, train_loss: 0.16222797, train_accuracy: 0.9531, test_Accuracy: 0.9607\n",
      "Epoch: [ 0] [  190/  468] time: 166.3063, train_loss: 0.16053697, train_accuracy: 0.9688, test_Accuracy: 0.9620\n",
      "Epoch: [ 0] [  191/  468] time: 167.0533, train_loss: 0.07088930, train_accuracy: 0.9766, test_Accuracy: 0.9611\n",
      "Epoch: [ 0] [  192/  468] time: 167.8843, train_loss: 0.07239842, train_accuracy: 0.9766, test_Accuracy: 0.9590\n",
      "Epoch: [ 0] [  193/  468] time: 168.6523, train_loss: 0.11921092, train_accuracy: 0.9531, test_Accuracy: 0.9585\n",
      "Epoch: [ 0] [  194/  468] time: 169.3953, train_loss: 0.06535142, train_accuracy: 0.9844, test_Accuracy: 0.9554\n",
      "Epoch: [ 0] [  195/  468] time: 170.1663, train_loss: 0.05756409, train_accuracy: 0.9688, test_Accuracy: 0.9530\n",
      "Epoch: [ 0] [  196/  468] time: 170.9183, train_loss: 0.07089400, train_accuracy: 0.9688, test_Accuracy: 0.9498\n",
      "Epoch: [ 0] [  197/  468] time: 171.6930, train_loss: 0.09982342, train_accuracy: 0.9766, test_Accuracy: 0.9483\n",
      "Epoch: [ 0] [  198/  468] time: 172.4293, train_loss: 0.06751824, train_accuracy: 0.9531, test_Accuracy: 0.9467\n",
      "Epoch: [ 0] [  199/  468] time: 173.1813, train_loss: 0.07462548, train_accuracy: 0.9531, test_Accuracy: 0.9453\n",
      "Epoch: [ 0] [  200/  468] time: 173.9453, train_loss: 0.06986484, train_accuracy: 0.9688, test_Accuracy: 0.9441\n",
      "Epoch: [ 0] [  201/  468] time: 174.6464, train_loss: 0.11107142, train_accuracy: 0.9531, test_Accuracy: 0.9433\n",
      "Epoch: [ 0] [  202/  468] time: 175.3743, train_loss: 0.05339327, train_accuracy: 0.9766, test_Accuracy: 0.9433\n",
      "Epoch: [ 0] [  203/  468] time: 176.0993, train_loss: 0.08801641, train_accuracy: 0.9844, test_Accuracy: 0.9439\n",
      "Epoch: [ 0] [  204/  468] time: 176.8363, train_loss: 0.12763298, train_accuracy: 0.9688, test_Accuracy: 0.9442\n",
      "Epoch: [ 0] [  205/  468] time: 177.5033, train_loss: 0.09181051, train_accuracy: 0.9609, test_Accuracy: 0.9455\n",
      "Epoch: [ 0] [  206/  468] time: 178.2083, train_loss: 0.21677098, train_accuracy: 0.9453, test_Accuracy: 0.9461\n",
      "Epoch: [ 0] [  207/  468] time: 179.0063, train_loss: 0.10556160, train_accuracy: 0.9453, test_Accuracy: 0.9465\n",
      "Epoch: [ 0] [  208/  468] time: 179.7563, train_loss: 0.05139014, train_accuracy: 0.9766, test_Accuracy: 0.9476\n",
      "Epoch: [ 0] [  209/  468] time: 180.4923, train_loss: 0.06470494, train_accuracy: 0.9453, test_Accuracy: 0.9468\n",
      "Epoch: [ 0] [  210/  468] time: 181.2633, train_loss: 0.05525700, train_accuracy: 0.9766, test_Accuracy: 0.9479\n",
      "Epoch: [ 0] [  211/  468] time: 182.0543, train_loss: 0.12497290, train_accuracy: 0.9531, test_Accuracy: 0.9496\n",
      "Epoch: [ 0] [  212/  468] time: 182.7744, train_loss: 0.15509960, train_accuracy: 0.9375, test_Accuracy: 0.9519\n",
      "Epoch: [ 0] [  213/  468] time: 183.3383, train_loss: 0.09039202, train_accuracy: 0.9531, test_Accuracy: 0.9522\n",
      "Epoch: [ 0] [  214/  468] time: 184.1273, train_loss: 0.07944962, train_accuracy: 0.9688, test_Accuracy: 0.9520\n",
      "Epoch: [ 0] [  215/  468] time: 184.8533, train_loss: 0.04522765, train_accuracy: 0.9844, test_Accuracy: 0.9531\n",
      "Epoch: [ 0] [  216/  468] time: 185.5873, train_loss: 0.05234852, train_accuracy: 0.9766, test_Accuracy: 0.9539\n",
      "Epoch: [ 0] [  217/  468] time: 186.3853, train_loss: 0.11822089, train_accuracy: 0.9766, test_Accuracy: 0.9541\n",
      "Epoch: [ 0] [  218/  468] time: 187.0883, train_loss: 0.04831637, train_accuracy: 0.9844, test_Accuracy: 0.9551\n",
      "Epoch: [ 0] [  219/  468] time: 187.9223, train_loss: 0.05573606, train_accuracy: 0.9922, test_Accuracy: 0.9556\n",
      "Epoch: [ 0] [  220/  468] time: 188.6653, train_loss: 0.07396816, train_accuracy: 0.9766, test_Accuracy: 0.9556\n",
      "Epoch: [ 0] [  221/  468] time: 189.2141, train_loss: 0.10056847, train_accuracy: 0.9609, test_Accuracy: 0.9573\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [ 0] [  222/  468] time: 189.6821, train_loss: 0.14285710, train_accuracy: 0.9688, test_Accuracy: 0.9575\n",
      "Epoch: [ 0] [  223/  468] time: 190.1750, train_loss: 0.10288278, train_accuracy: 0.9766, test_Accuracy: 0.9590\n",
      "Epoch: [ 0] [  224/  468] time: 190.6980, train_loss: 0.09679151, train_accuracy: 0.9609, test_Accuracy: 0.9603\n",
      "Epoch: [ 0] [  225/  468] time: 191.2430, train_loss: 0.09821682, train_accuracy: 0.9688, test_Accuracy: 0.9617\n",
      "Epoch: [ 0] [  226/  468] time: 191.7140, train_loss: 0.12260213, train_accuracy: 0.9688, test_Accuracy: 0.9613\n",
      "Epoch: [ 0] [  227/  468] time: 192.2260, train_loss: 0.03272253, train_accuracy: 0.9922, test_Accuracy: 0.9601\n",
      "Epoch: [ 0] [  228/  468] time: 192.9660, train_loss: 0.08307144, train_accuracy: 0.9688, test_Accuracy: 0.9592\n",
      "Epoch: [ 0] [  229/  468] time: 193.7671, train_loss: 0.14641729, train_accuracy: 0.9531, test_Accuracy: 0.9594\n",
      "Epoch: [ 0] [  230/  468] time: 194.5211, train_loss: 0.04234621, train_accuracy: 1.0000, test_Accuracy: 0.9599\n",
      "Epoch: [ 0] [  231/  468] time: 195.3191, train_loss: 0.12712181, train_accuracy: 0.9688, test_Accuracy: 0.9605\n",
      "Epoch: [ 0] [  232/  468] time: 196.0920, train_loss: 0.04038307, train_accuracy: 1.0000, test_Accuracy: 0.9607\n",
      "Epoch: [ 0] [  233/  468] time: 196.8381, train_loss: 0.06349017, train_accuracy: 0.9766, test_Accuracy: 0.9602\n",
      "Epoch: [ 0] [  234/  468] time: 197.5631, train_loss: 0.11036237, train_accuracy: 0.9688, test_Accuracy: 0.9600\n",
      "Epoch: [ 0] [  235/  468] time: 198.3170, train_loss: 0.06960584, train_accuracy: 0.9766, test_Accuracy: 0.9581\n",
      "Epoch: [ 0] [  236/  468] time: 199.0971, train_loss: 0.06867573, train_accuracy: 0.9766, test_Accuracy: 0.9576\n",
      "Epoch: [ 0] [  237/  468] time: 199.8310, train_loss: 0.07354413, train_accuracy: 0.9688, test_Accuracy: 0.9563\n",
      "Epoch: [ 0] [  238/  468] time: 200.6810, train_loss: 0.05854303, train_accuracy: 0.9766, test_Accuracy: 0.9570\n",
      "Epoch: [ 0] [  239/  468] time: 201.4050, train_loss: 0.06357927, train_accuracy: 0.9688, test_Accuracy: 0.9568\n",
      "Epoch: [ 0] [  240/  468] time: 202.1480, train_loss: 0.06612024, train_accuracy: 0.9766, test_Accuracy: 0.9560\n",
      "Epoch: [ 0] [  241/  468] time: 202.9441, train_loss: 0.13104168, train_accuracy: 0.9531, test_Accuracy: 0.9540\n",
      "Epoch: [ 0] [  242/  468] time: 203.6690, train_loss: 0.08917443, train_accuracy: 0.9453, test_Accuracy: 0.9539\n",
      "Epoch: [ 0] [  243/  468] time: 204.4590, train_loss: 0.11034942, train_accuracy: 0.9531, test_Accuracy: 0.9529\n",
      "Epoch: [ 0] [  244/  468] time: 205.2501, train_loss: 0.06202883, train_accuracy: 0.9531, test_Accuracy: 0.9527\n",
      "Epoch: [ 0] [  245/  468] time: 205.9890, train_loss: 0.15612537, train_accuracy: 0.9609, test_Accuracy: 0.9533\n",
      "Epoch: [ 0] [  246/  468] time: 206.7190, train_loss: 0.04014712, train_accuracy: 0.9844, test_Accuracy: 0.9539\n",
      "Epoch: [ 0] [  247/  468] time: 207.4970, train_loss: 0.07186662, train_accuracy: 0.9609, test_Accuracy: 0.9543\n",
      "Epoch: [ 0] [  248/  468] time: 208.2982, train_loss: 0.04845825, train_accuracy: 0.9688, test_Accuracy: 0.9545\n",
      "Epoch: [ 0] [  249/  468] time: 209.0620, train_loss: 0.07950900, train_accuracy: 0.9688, test_Accuracy: 0.9536\n",
      "Epoch: [ 0] [  250/  468] time: 209.8110, train_loss: 0.07212938, train_accuracy: 0.9844, test_Accuracy: 0.9536\n",
      "Epoch: [ 0] [  251/  468] time: 210.5900, train_loss: 0.06064429, train_accuracy: 0.9766, test_Accuracy: 0.9536\n",
      "Epoch: [ 0] [  252/  468] time: 211.2132, train_loss: 0.10221150, train_accuracy: 0.9609, test_Accuracy: 0.9548\n",
      "Epoch: [ 0] [  253/  468] time: 211.9570, train_loss: 0.12585644, train_accuracy: 0.9453, test_Accuracy: 0.9559\n",
      "Epoch: [ 0] [  254/  468] time: 212.7600, train_loss: 0.12158588, train_accuracy: 0.9531, test_Accuracy: 0.9561\n",
      "Epoch: [ 0] [  255/  468] time: 213.5161, train_loss: 0.06000080, train_accuracy: 0.9766, test_Accuracy: 0.9565\n",
      "Epoch: [ 0] [  256/  468] time: 214.2320, train_loss: 0.07245884, train_accuracy: 0.9688, test_Accuracy: 0.9577\n",
      "Epoch: [ 0] [  257/  468] time: 214.9681, train_loss: 0.04747723, train_accuracy: 0.9766, test_Accuracy: 0.9584\n",
      "Epoch: [ 0] [  258/  468] time: 215.7680, train_loss: 0.13051300, train_accuracy: 0.9453, test_Accuracy: 0.9592\n",
      "Epoch: [ 0] [  259/  468] time: 216.5320, train_loss: 0.09544870, train_accuracy: 0.9531, test_Accuracy: 0.9608\n",
      "Epoch: [ 0] [  260/  468] time: 217.2981, train_loss: 0.09021926, train_accuracy: 0.9688, test_Accuracy: 0.9603\n",
      "Epoch: [ 0] [  261/  468] time: 218.0271, train_loss: 0.02723446, train_accuracy: 0.9688, test_Accuracy: 0.9608\n",
      "Epoch: [ 0] [  262/  468] time: 218.8431, train_loss: 0.10102180, train_accuracy: 0.9609, test_Accuracy: 0.9605\n",
      "Epoch: [ 0] [  263/  468] time: 219.6120, train_loss: 0.07345863, train_accuracy: 0.9844, test_Accuracy: 0.9600\n",
      "Epoch: [ 0] [  264/  468] time: 220.3591, train_loss: 0.08725977, train_accuracy: 0.9844, test_Accuracy: 0.9595\n",
      "Epoch: [ 0] [  265/  468] time: 221.0570, train_loss: 0.11164836, train_accuracy: 0.9766, test_Accuracy: 0.9602\n",
      "Epoch: [ 0] [  266/  468] time: 221.8151, train_loss: 0.08848435, train_accuracy: 0.9531, test_Accuracy: 0.9597\n",
      "Epoch: [ 0] [  267/  468] time: 222.6580, train_loss: 0.08624767, train_accuracy: 0.9844, test_Accuracy: 0.9585\n",
      "Epoch: [ 0] [  268/  468] time: 223.4831, train_loss: 0.08180870, train_accuracy: 0.9609, test_Accuracy: 0.9571\n",
      "Epoch: [ 0] [  269/  468] time: 224.1580, train_loss: 0.06582399, train_accuracy: 0.9609, test_Accuracy: 0.9552\n",
      "Epoch: [ 0] [  270/  468] time: 224.8950, train_loss: 0.07073337, train_accuracy: 0.9688, test_Accuracy: 0.9533\n",
      "Epoch: [ 0] [  271/  468] time: 225.6201, train_loss: 0.08801375, train_accuracy: 0.9219, test_Accuracy: 0.9508\n",
      "Epoch: [ 0] [  272/  468] time: 226.3511, train_loss: 0.13655567, train_accuracy: 0.9453, test_Accuracy: 0.9506\n",
      "Epoch: [ 0] [  273/  468] time: 227.0680, train_loss: 0.07324696, train_accuracy: 0.9609, test_Accuracy: 0.9495\n",
      "Epoch: [ 0] [  274/  468] time: 227.8362, train_loss: 0.02063957, train_accuracy: 0.9922, test_Accuracy: 0.9481\n",
      "Epoch: [ 0] [  275/  468] time: 228.5610, train_loss: 0.08510844, train_accuracy: 0.9609, test_Accuracy: 0.9455\n",
      "Epoch: [ 0] [  276/  468] time: 229.2561, train_loss: 0.11049830, train_accuracy: 0.9219, test_Accuracy: 0.9450\n",
      "Epoch: [ 0] [  277/  468] time: 230.0090, train_loss: 0.09629494, train_accuracy: 0.9297, test_Accuracy: 0.9449\n",
      "Epoch: [ 0] [  278/  468] time: 230.7460, train_loss: 0.07488570, train_accuracy: 0.9609, test_Accuracy: 0.9447\n",
      "Epoch: [ 0] [  279/  468] time: 231.4570, train_loss: 0.04095918, train_accuracy: 0.9766, test_Accuracy: 0.9453\n",
      "Epoch: [ 0] [  280/  468] time: 232.2270, train_loss: 0.06651571, train_accuracy: 0.9375, test_Accuracy: 0.9457\n",
      "Epoch: [ 0] [  281/  468] time: 232.9271, train_loss: 0.07924362, train_accuracy: 0.9688, test_Accuracy: 0.9471\n",
      "Epoch: [ 0] [  282/  468] time: 233.7070, train_loss: 0.06030045, train_accuracy: 0.9766, test_Accuracy: 0.9490\n",
      "Epoch: [ 0] [  283/  468] time: 234.5101, train_loss: 0.05088545, train_accuracy: 0.9766, test_Accuracy: 0.9505\n",
      "Epoch: [ 0] [  284/  468] time: 235.3340, train_loss: 0.06368218, train_accuracy: 0.9688, test_Accuracy: 0.9521\n",
      "Epoch: [ 0] [  285/  468] time: 236.1299, train_loss: 0.09646538, train_accuracy: 0.9453, test_Accuracy: 0.9540\n",
      "Epoch: [ 0] [  286/  468] time: 236.8801, train_loss: 0.18289402, train_accuracy: 0.9609, test_Accuracy: 0.9555\n",
      "Epoch: [ 0] [  287/  468] time: 237.5981, train_loss: 0.08302071, train_accuracy: 0.9375, test_Accuracy: 0.9579\n",
      "Epoch: [ 0] [  288/  468] time: 238.2951, train_loss: 0.09670565, train_accuracy: 0.9922, test_Accuracy: 0.9600\n",
      "Epoch: [ 0] [  289/  468] time: 239.0520, train_loss: 0.04804638, train_accuracy: 0.9688, test_Accuracy: 0.9606\n",
      "Epoch: [ 0] [  290/  468] time: 239.7530, train_loss: 0.08607031, train_accuracy: 0.9609, test_Accuracy: 0.9599\n",
      "Epoch: [ 0] [  291/  468] time: 240.5390, train_loss: 0.07207949, train_accuracy: 0.9453, test_Accuracy: 0.9598\n",
      "Epoch: [ 0] [  292/  468] time: 241.3320, train_loss: 0.03807389, train_accuracy: 1.0000, test_Accuracy: 0.9579\n",
      "Epoch: [ 0] [  293/  468] time: 242.1241, train_loss: 0.04387049, train_accuracy: 0.9531, test_Accuracy: 0.9581\n",
      "Epoch: [ 0] [  294/  468] time: 242.9191, train_loss: 0.04976565, train_accuracy: 0.9844, test_Accuracy: 0.9595\n",
      "Epoch: [ 0] [  295/  468] time: 243.6580, train_loss: 0.07104042, train_accuracy: 0.9766, test_Accuracy: 0.9597\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [ 0] [  296/  468] time: 244.3805, train_loss: 0.06681886, train_accuracy: 0.9688, test_Accuracy: 0.9598\n",
      "Epoch: [ 0] [  297/  468] time: 245.1610, train_loss: 0.13260171, train_accuracy: 0.9688, test_Accuracy: 0.9599\n",
      "Epoch: [ 0] [  298/  468] time: 245.9450, train_loss: 0.07641497, train_accuracy: 0.9531, test_Accuracy: 0.9605\n",
      "Epoch: [ 0] [  299/  468] time: 246.6971, train_loss: 0.06312172, train_accuracy: 0.9844, test_Accuracy: 0.9603\n",
      "Epoch: [ 0] [  300/  468] time: 247.4770, train_loss: 0.10858423, train_accuracy: 0.9766, test_Accuracy: 0.9603\n",
      "Epoch: [ 0] [  301/  468] time: 248.1900, train_loss: 0.10263545, train_accuracy: 0.9219, test_Accuracy: 0.9600\n",
      "Epoch: [ 0] [  302/  468] time: 248.9960, train_loss: 0.04709834, train_accuracy: 0.9609, test_Accuracy: 0.9601\n",
      "Epoch: [ 0] [  303/  468] time: 249.7480, train_loss: 0.04987021, train_accuracy: 0.9922, test_Accuracy: 0.9606\n",
      "Epoch: [ 0] [  304/  468] time: 250.4860, train_loss: 0.09253675, train_accuracy: 0.9609, test_Accuracy: 0.9594\n",
      "Epoch: [ 0] [  305/  468] time: 251.2430, train_loss: 0.08923784, train_accuracy: 0.9766, test_Accuracy: 0.9582\n",
      "Epoch: [ 0] [  306/  468] time: 252.0070, train_loss: 0.07184166, train_accuracy: 0.9844, test_Accuracy: 0.9578\n",
      "Epoch: [ 0] [  307/  468] time: 252.7331, train_loss: 0.10408597, train_accuracy: 0.9766, test_Accuracy: 0.9586\n",
      "Epoch: [ 0] [  308/  468] time: 253.4890, train_loss: 0.10787521, train_accuracy: 0.9688, test_Accuracy: 0.9591\n",
      "Epoch: [ 0] [  309/  468] time: 254.2230, train_loss: 0.04450674, train_accuracy: 0.9844, test_Accuracy: 0.9591\n",
      "Epoch: [ 0] [  310/  468] time: 254.9480, train_loss: 0.09032636, train_accuracy: 0.9688, test_Accuracy: 0.9600\n",
      "Epoch: [ 0] [  311/  468] time: 255.7601, train_loss: 0.09022585, train_accuracy: 0.9609, test_Accuracy: 0.9610\n",
      "Epoch: [ 0] [  312/  468] time: 256.5141, train_loss: 0.07810824, train_accuracy: 0.9688, test_Accuracy: 0.9623\n",
      "Epoch: [ 0] [  313/  468] time: 257.3200, train_loss: 0.12797645, train_accuracy: 0.9688, test_Accuracy: 0.9631\n",
      "Epoch: [ 0] [  314/  468] time: 258.0941, train_loss: 0.08346761, train_accuracy: 0.9766, test_Accuracy: 0.9644\n",
      "Epoch: [ 0] [  315/  468] time: 258.9391, train_loss: 0.06723658, train_accuracy: 0.9844, test_Accuracy: 0.9673\n",
      "Epoch: [ 0] [  316/  468] time: 259.6861, train_loss: 0.21216640, train_accuracy: 0.9609, test_Accuracy: 0.9676\n",
      "Epoch: [ 0] [  317/  468] time: 260.4881, train_loss: 0.16710338, train_accuracy: 0.9531, test_Accuracy: 0.9668\n",
      "Epoch: [ 0] [  318/  468] time: 261.2051, train_loss: 0.05772084, train_accuracy: 0.9844, test_Accuracy: 0.9660\n",
      "Epoch: [ 0] [  319/  468] time: 261.9141, train_loss: 0.08418943, train_accuracy: 0.9844, test_Accuracy: 0.9653\n",
      "Epoch: [ 0] [  320/  468] time: 262.6781, train_loss: 0.04798830, train_accuracy: 0.9922, test_Accuracy: 0.9646\n",
      "Epoch: [ 0] [  321/  468] time: 263.4141, train_loss: 0.10580473, train_accuracy: 0.9609, test_Accuracy: 0.9640\n",
      "Epoch: [ 0] [  322/  468] time: 264.1690, train_loss: 0.14722688, train_accuracy: 0.9375, test_Accuracy: 0.9644\n",
      "Epoch: [ 0] [  323/  468] time: 264.8460, train_loss: 0.04357320, train_accuracy: 0.9844, test_Accuracy: 0.9639\n",
      "Epoch: [ 0] [  324/  468] time: 265.5561, train_loss: 0.08093192, train_accuracy: 0.9766, test_Accuracy: 0.9633\n",
      "Epoch: [ 0] [  325/  468] time: 266.3070, train_loss: 0.17001489, train_accuracy: 0.9531, test_Accuracy: 0.9623\n",
      "Epoch: [ 0] [  326/  468] time: 267.0500, train_loss: 0.08885923, train_accuracy: 0.9844, test_Accuracy: 0.9628\n",
      "Epoch: [ 0] [  327/  468] time: 267.7800, train_loss: 0.10411654, train_accuracy: 0.9766, test_Accuracy: 0.9639\n",
      "Epoch: [ 0] [  328/  468] time: 268.4800, train_loss: 0.10707551, train_accuracy: 0.9766, test_Accuracy: 0.9640\n",
      "Epoch: [ 0] [  329/  468] time: 269.2530, train_loss: 0.08787949, train_accuracy: 0.9844, test_Accuracy: 0.9645\n",
      "Epoch: [ 0] [  330/  468] time: 269.9840, train_loss: 0.07971095, train_accuracy: 0.9766, test_Accuracy: 0.9646\n",
      "Epoch: [ 0] [  331/  468] time: 270.7810, train_loss: 0.08687460, train_accuracy: 0.9766, test_Accuracy: 0.9631\n",
      "Epoch: [ 0] [  332/  468] time: 271.3880, train_loss: 0.07839122, train_accuracy: 0.9766, test_Accuracy: 0.9622\n",
      "Epoch: [ 0] [  333/  468] time: 272.1310, train_loss: 0.04581277, train_accuracy: 0.9844, test_Accuracy: 0.9608\n",
      "Epoch: [ 0] [  334/  468] time: 272.8640, train_loss: 0.08064426, train_accuracy: 0.9688, test_Accuracy: 0.9597\n",
      "Epoch: [ 0] [  335/  468] time: 273.6793, train_loss: 0.09198060, train_accuracy: 0.9688, test_Accuracy: 0.9595\n",
      "Epoch: [ 0] [  336/  468] time: 274.4170, train_loss: 0.13279784, train_accuracy: 0.9688, test_Accuracy: 0.9607\n",
      "Epoch: [ 0] [  337/  468] time: 275.2060, train_loss: 0.17496145, train_accuracy: 0.9609, test_Accuracy: 0.9601\n",
      "Epoch: [ 0] [  338/  468] time: 276.0170, train_loss: 0.04196960, train_accuracy: 0.9922, test_Accuracy: 0.9605\n",
      "Epoch: [ 0] [  339/  468] time: 276.8141, train_loss: 0.09309861, train_accuracy: 0.9531, test_Accuracy: 0.9619\n",
      "Epoch: [ 0] [  340/  468] time: 277.6420, train_loss: 0.04955897, train_accuracy: 0.9766, test_Accuracy: 0.9626\n",
      "Epoch: [ 0] [  341/  468] time: 278.3890, train_loss: 0.11240053, train_accuracy: 0.9688, test_Accuracy: 0.9634\n",
      "Epoch: [ 0] [  342/  468] time: 279.1360, train_loss: 0.04402017, train_accuracy: 0.9922, test_Accuracy: 0.9633\n",
      "Epoch: [ 0] [  343/  468] time: 279.8990, train_loss: 0.10795650, train_accuracy: 0.9609, test_Accuracy: 0.9638\n",
      "Epoch: [ 0] [  344/  468] time: 280.5970, train_loss: 0.12647447, train_accuracy: 0.9531, test_Accuracy: 0.9625\n",
      "Epoch: [ 0] [  345/  468] time: 281.3180, train_loss: 0.04451795, train_accuracy: 0.9844, test_Accuracy: 0.9622\n",
      "Epoch: [ 0] [  346/  468] time: 282.0925, train_loss: 0.07331892, train_accuracy: 0.9766, test_Accuracy: 0.9604\n",
      "Epoch: [ 0] [  347/  468] time: 282.8931, train_loss: 0.04797748, train_accuracy: 0.9609, test_Accuracy: 0.9579\n",
      "Epoch: [ 0] [  348/  468] time: 283.6240, train_loss: 0.07250322, train_accuracy: 0.9688, test_Accuracy: 0.9569\n",
      "Epoch: [ 0] [  349/  468] time: 284.3920, train_loss: 0.07686163, train_accuracy: 0.9844, test_Accuracy: 0.9551\n",
      "Epoch: [ 0] [  350/  468] time: 285.1360, train_loss: 0.19531041, train_accuracy: 0.9219, test_Accuracy: 0.9535\n",
      "Epoch: [ 0] [  351/  468] time: 285.9321, train_loss: 0.06751107, train_accuracy: 0.9688, test_Accuracy: 0.9519\n",
      "Epoch: [ 0] [  352/  468] time: 286.7850, train_loss: 0.07886258, train_accuracy: 0.9531, test_Accuracy: 0.9548\n",
      "Epoch: [ 0] [  353/  468] time: 287.5931, train_loss: 0.10225227, train_accuracy: 0.9688, test_Accuracy: 0.9571\n",
      "Epoch: [ 0] [  354/  468] time: 288.3040, train_loss: 0.07607591, train_accuracy: 0.9766, test_Accuracy: 0.9580\n",
      "Epoch: [ 0] [  355/  468] time: 289.0100, train_loss: 0.12483019, train_accuracy: 0.9609, test_Accuracy: 0.9594\n",
      "Epoch: [ 0] [  356/  468] time: 289.7511, train_loss: 0.07179616, train_accuracy: 0.9688, test_Accuracy: 0.9612\n",
      "Epoch: [ 0] [  357/  468] time: 290.4610, train_loss: 0.06741067, train_accuracy: 0.9688, test_Accuracy: 0.9629\n",
      "Epoch: [ 0] [  358/  468] time: 291.3690, train_loss: 0.07087290, train_accuracy: 0.9922, test_Accuracy: 0.9643\n",
      "Epoch: [ 0] [  359/  468] time: 292.2020, train_loss: 0.11343046, train_accuracy: 0.9609, test_Accuracy: 0.9653\n",
      "Epoch: [ 0] [  360/  468] time: 293.0500, train_loss: 0.04864784, train_accuracy: 0.9688, test_Accuracy: 0.9666\n",
      "Epoch: [ 0] [  361/  468] time: 293.8101, train_loss: 0.07611997, train_accuracy: 0.9844, test_Accuracy: 0.9683\n",
      "Epoch: [ 0] [  362/  468] time: 294.6011, train_loss: 0.07507885, train_accuracy: 0.9531, test_Accuracy: 0.9692\n",
      "Epoch: [ 0] [  363/  468] time: 295.2911, train_loss: 0.10555187, train_accuracy: 0.9609, test_Accuracy: 0.9691\n",
      "Epoch: [ 0] [  364/  468] time: 296.0290, train_loss: 0.05705535, train_accuracy: 0.9766, test_Accuracy: 0.9701\n",
      "Epoch: [ 0] [  365/  468] time: 296.7831, train_loss: 0.08199726, train_accuracy: 0.9688, test_Accuracy: 0.9697\n",
      "Epoch: [ 0] [  366/  468] time: 297.5310, train_loss: 0.05760506, train_accuracy: 0.9922, test_Accuracy: 0.9696\n",
      "Epoch: [ 0] [  367/  468] time: 298.2431, train_loss: 0.14752318, train_accuracy: 0.9453, test_Accuracy: 0.9695\n",
      "Epoch: [ 0] [  368/  468] time: 299.0121, train_loss: 0.08777751, train_accuracy: 0.9844, test_Accuracy: 0.9689\n",
      "Epoch: [ 0] [  369/  468] time: 299.7900, train_loss: 0.04517556, train_accuracy: 0.9922, test_Accuracy: 0.9685\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [ 0] [  370/  468] time: 300.5381, train_loss: 0.05318908, train_accuracy: 0.9766, test_Accuracy: 0.9692\n",
      "Epoch: [ 0] [  371/  468] time: 301.3151, train_loss: 0.07588097, train_accuracy: 0.9688, test_Accuracy: 0.9700\n",
      "Epoch: [ 0] [  372/  468] time: 302.0831, train_loss: 0.05537463, train_accuracy: 0.9922, test_Accuracy: 0.9692\n",
      "Epoch: [ 0] [  373/  468] time: 302.9104, train_loss: 0.07047355, train_accuracy: 0.9766, test_Accuracy: 0.9695\n",
      "Epoch: [ 0] [  374/  468] time: 303.7120, train_loss: 0.08562930, train_accuracy: 0.9688, test_Accuracy: 0.9701\n",
      "Epoch: [ 0] [  375/  468] time: 304.4630, train_loss: 0.07618178, train_accuracy: 0.9766, test_Accuracy: 0.9704\n",
      "Epoch: [ 0] [  376/  468] time: 305.2440, train_loss: 0.07545616, train_accuracy: 0.9766, test_Accuracy: 0.9706\n",
      "Epoch: [ 0] [  377/  468] time: 306.0170, train_loss: 0.10796739, train_accuracy: 0.9609, test_Accuracy: 0.9703\n",
      "Epoch: [ 0] [  378/  468] time: 306.7301, train_loss: 0.03310665, train_accuracy: 0.9844, test_Accuracy: 0.9696\n",
      "Epoch: [ 0] [  379/  468] time: 307.4841, train_loss: 0.07917625, train_accuracy: 0.9688, test_Accuracy: 0.9691\n",
      "Epoch: [ 0] [  380/  468] time: 308.2140, train_loss: 0.06653400, train_accuracy: 0.9844, test_Accuracy: 0.9705\n",
      "Epoch: [ 0] [  381/  468] time: 308.9650, train_loss: 0.16219340, train_accuracy: 0.9219, test_Accuracy: 0.9710\n",
      "Epoch: [ 0] [  382/  468] time: 309.6611, train_loss: 0.04707218, train_accuracy: 0.9844, test_Accuracy: 0.9700\n",
      "Epoch: [ 0] [  383/  468] time: 310.4000, train_loss: 0.07958957, train_accuracy: 0.9844, test_Accuracy: 0.9704\n",
      "Epoch: [ 0] [  384/  468] time: 311.2004, train_loss: 0.06336832, train_accuracy: 0.9766, test_Accuracy: 0.9705\n",
      "Epoch: [ 0] [  385/  468] time: 311.9660, train_loss: 0.07320571, train_accuracy: 0.9766, test_Accuracy: 0.9708\n",
      "Epoch: [ 0] [  386/  468] time: 312.6960, train_loss: 0.06187940, train_accuracy: 0.9844, test_Accuracy: 0.9714\n",
      "Epoch: [ 0] [  387/  468] time: 313.4330, train_loss: 0.06929716, train_accuracy: 0.9844, test_Accuracy: 0.9709\n",
      "Epoch: [ 0] [  388/  468] time: 314.1440, train_loss: 0.12733877, train_accuracy: 0.9766, test_Accuracy: 0.9714\n",
      "Epoch: [ 0] [  389/  468] time: 314.8651, train_loss: 0.09144355, train_accuracy: 0.9609, test_Accuracy: 0.9711\n",
      "Epoch: [ 0] [  390/  468] time: 315.5960, train_loss: 0.06096213, train_accuracy: 0.9766, test_Accuracy: 0.9712\n",
      "Epoch: [ 0] [  391/  468] time: 316.3960, train_loss: 0.08051403, train_accuracy: 0.9922, test_Accuracy: 0.9708\n",
      "Epoch: [ 0] [  392/  468] time: 317.1080, train_loss: 0.04106642, train_accuracy: 1.0000, test_Accuracy: 0.9694\n",
      "Epoch: [ 0] [  393/  468] time: 317.8551, train_loss: 0.04631679, train_accuracy: 0.9844, test_Accuracy: 0.9694\n",
      "Epoch: [ 0] [  394/  468] time: 318.6020, train_loss: 0.04067344, train_accuracy: 0.9844, test_Accuracy: 0.9684\n",
      "Epoch: [ 0] [  395/  468] time: 319.3443, train_loss: 0.07364698, train_accuracy: 0.9766, test_Accuracy: 0.9676\n",
      "Epoch: [ 0] [  396/  468] time: 320.0670, train_loss: 0.09528542, train_accuracy: 0.9688, test_Accuracy: 0.9676\n",
      "Epoch: [ 0] [  397/  468] time: 320.7380, train_loss: 0.02653540, train_accuracy: 0.9844, test_Accuracy: 0.9665\n",
      "Epoch: [ 0] [  398/  468] time: 321.4390, train_loss: 0.09195470, train_accuracy: 0.9531, test_Accuracy: 0.9655\n",
      "Epoch: [ 0] [  399/  468] time: 322.1540, train_loss: 0.03710612, train_accuracy: 0.9844, test_Accuracy: 0.9657\n",
      "Epoch: [ 0] [  400/  468] time: 322.8850, train_loss: 0.09802213, train_accuracy: 0.9609, test_Accuracy: 0.9658\n",
      "Epoch: [ 0] [  401/  468] time: 323.5761, train_loss: 0.11326439, train_accuracy: 0.9531, test_Accuracy: 0.9668\n",
      "Epoch: [ 0] [  402/  468] time: 324.2881, train_loss: 0.05396402, train_accuracy: 0.9766, test_Accuracy: 0.9666\n",
      "Epoch: [ 0] [  403/  468] time: 325.0171, train_loss: 0.08938689, train_accuracy: 0.9688, test_Accuracy: 0.9660\n",
      "Epoch: [ 0] [  404/  468] time: 325.7152, train_loss: 0.07239303, train_accuracy: 0.9844, test_Accuracy: 0.9661\n",
      "Epoch: [ 0] [  405/  468] time: 326.4530, train_loss: 0.08957434, train_accuracy: 0.9531, test_Accuracy: 0.9677\n",
      "Epoch: [ 0] [  406/  468] time: 327.2040, train_loss: 0.11916006, train_accuracy: 0.9609, test_Accuracy: 0.9689\n",
      "Epoch: [ 0] [  407/  468] time: 327.9170, train_loss: 0.10378372, train_accuracy: 0.9844, test_Accuracy: 0.9691\n",
      "Epoch: [ 0] [  408/  468] time: 328.6171, train_loss: 0.06454840, train_accuracy: 0.9688, test_Accuracy: 0.9703\n",
      "Epoch: [ 0] [  409/  468] time: 329.3580, train_loss: 0.12059893, train_accuracy: 0.9609, test_Accuracy: 0.9712\n",
      "Epoch: [ 0] [  410/  468] time: 330.1190, train_loss: 0.08445223, train_accuracy: 0.9766, test_Accuracy: 0.9711\n",
      "Epoch: [ 0] [  411/  468] time: 330.8351, train_loss: 0.07267912, train_accuracy: 0.9766, test_Accuracy: 0.9703\n",
      "Epoch: [ 0] [  412/  468] time: 331.4420, train_loss: 0.06005546, train_accuracy: 0.9609, test_Accuracy: 0.9703\n",
      "Epoch: [ 0] [  413/  468] time: 332.1502, train_loss: 0.08348076, train_accuracy: 0.9922, test_Accuracy: 0.9711\n",
      "Epoch: [ 0] [  414/  468] time: 332.9510, train_loss: 0.15178844, train_accuracy: 0.9609, test_Accuracy: 0.9703\n",
      "Epoch: [ 0] [  415/  468] time: 333.6850, train_loss: 0.12283961, train_accuracy: 0.9688, test_Accuracy: 0.9709\n",
      "Epoch: [ 0] [  416/  468] time: 334.3811, train_loss: 0.11872555, train_accuracy: 0.9609, test_Accuracy: 0.9708\n",
      "Epoch: [ 0] [  417/  468] time: 335.1292, train_loss: 0.10556809, train_accuracy: 0.9688, test_Accuracy: 0.9704\n",
      "Epoch: [ 0] [  418/  468] time: 335.9460, train_loss: 0.05672432, train_accuracy: 0.9922, test_Accuracy: 0.9707\n",
      "Epoch: [ 0] [  419/  468] time: 336.6600, train_loss: 0.02759088, train_accuracy: 0.9922, test_Accuracy: 0.9713\n",
      "Epoch: [ 0] [  420/  468] time: 337.3491, train_loss: 0.15457928, train_accuracy: 0.9453, test_Accuracy: 0.9726\n",
      "Epoch: [ 0] [  421/  468] time: 338.0660, train_loss: 0.03081608, train_accuracy: 0.9922, test_Accuracy: 0.9733\n",
      "Epoch: [ 0] [  422/  468] time: 338.8041, train_loss: 0.14603615, train_accuracy: 0.9688, test_Accuracy: 0.9742\n",
      "Epoch: [ 0] [  423/  468] time: 339.5001, train_loss: 0.08920896, train_accuracy: 0.9609, test_Accuracy: 0.9746\n",
      "Epoch: [ 0] [  424/  468] time: 340.2431, train_loss: 0.07491840, train_accuracy: 0.9844, test_Accuracy: 0.9753\n",
      "Epoch: [ 0] [  425/  468] time: 341.0350, train_loss: 0.10380062, train_accuracy: 0.9609, test_Accuracy: 0.9756\n",
      "Epoch: [ 0] [  426/  468] time: 341.7931, train_loss: 0.03920209, train_accuracy: 1.0000, test_Accuracy: 0.9752\n",
      "Epoch: [ 0] [  427/  468] time: 342.5420, train_loss: 0.06725639, train_accuracy: 0.9766, test_Accuracy: 0.9755\n",
      "Epoch: [ 0] [  428/  468] time: 343.3112, train_loss: 0.03363968, train_accuracy: 0.9922, test_Accuracy: 0.9755\n",
      "Epoch: [ 0] [  429/  468] time: 344.1340, train_loss: 0.05101132, train_accuracy: 0.9844, test_Accuracy: 0.9751\n",
      "Epoch: [ 0] [  430/  468] time: 344.8430, train_loss: 0.04572973, train_accuracy: 0.9922, test_Accuracy: 0.9740\n",
      "Epoch: [ 0] [  431/  468] time: 345.5581, train_loss: 0.12961255, train_accuracy: 0.9688, test_Accuracy: 0.9731\n",
      "Epoch: [ 0] [  432/  468] time: 346.2681, train_loss: 0.06686341, train_accuracy: 0.9844, test_Accuracy: 0.9726\n",
      "Epoch: [ 0] [  433/  468] time: 347.0220, train_loss: 0.11540860, train_accuracy: 0.9609, test_Accuracy: 0.9729\n",
      "Epoch: [ 0] [  434/  468] time: 347.7630, train_loss: 0.07449942, train_accuracy: 0.9766, test_Accuracy: 0.9717\n",
      "Epoch: [ 0] [  435/  468] time: 348.5600, train_loss: 0.13715060, train_accuracy: 0.9766, test_Accuracy: 0.9709\n",
      "Epoch: [ 0] [  436/  468] time: 349.3000, train_loss: 0.08491931, train_accuracy: 0.9609, test_Accuracy: 0.9710\n",
      "Epoch: [ 0] [  437/  468] time: 350.0960, train_loss: 0.02771394, train_accuracy: 1.0000, test_Accuracy: 0.9712\n",
      "Epoch: [ 0] [  438/  468] time: 350.8860, train_loss: 0.04155041, train_accuracy: 0.9922, test_Accuracy: 0.9719\n",
      "Epoch: [ 0] [  439/  468] time: 351.6022, train_loss: 0.09909324, train_accuracy: 0.9766, test_Accuracy: 0.9707\n",
      "Epoch: [ 0] [  440/  468] time: 352.3401, train_loss: 0.10103295, train_accuracy: 0.9609, test_Accuracy: 0.9710\n",
      "Epoch: [ 0] [  441/  468] time: 353.0900, train_loss: 0.14494522, train_accuracy: 0.9766, test_Accuracy: 0.9707\n",
      "Epoch: [ 0] [  442/  468] time: 353.8481, train_loss: 0.04770607, train_accuracy: 0.9922, test_Accuracy: 0.9701\n",
      "Epoch: [ 0] [  443/  468] time: 354.5960, train_loss: 0.05771105, train_accuracy: 0.9922, test_Accuracy: 0.9697\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [ 0] [  444/  468] time: 355.4311, train_loss: 0.10404128, train_accuracy: 0.9766, test_Accuracy: 0.9692\n",
      "Epoch: [ 0] [  445/  468] time: 356.1300, train_loss: 0.05535939, train_accuracy: 0.9766, test_Accuracy: 0.9689\n",
      "Epoch: [ 0] [  446/  468] time: 356.8461, train_loss: 0.06696216, train_accuracy: 0.9844, test_Accuracy: 0.9700\n",
      "Epoch: [ 0] [  447/  468] time: 357.6070, train_loss: 0.05049480, train_accuracy: 0.9844, test_Accuracy: 0.9699\n",
      "Epoch: [ 0] [  448/  468] time: 358.3520, train_loss: 0.04898415, train_accuracy: 0.9688, test_Accuracy: 0.9702\n",
      "Epoch: [ 0] [  449/  468] time: 359.1020, train_loss: 0.04497945, train_accuracy: 0.9844, test_Accuracy: 0.9714\n",
      "Epoch: [ 0] [  450/  468] time: 359.8281, train_loss: 0.06713687, train_accuracy: 0.9688, test_Accuracy: 0.9710\n",
      "Epoch: [ 0] [  451/  468] time: 360.6420, train_loss: 0.09948654, train_accuracy: 0.9766, test_Accuracy: 0.9713\n",
      "Epoch: [ 0] [  452/  468] time: 361.4150, train_loss: 0.05150383, train_accuracy: 0.9922, test_Accuracy: 0.9715\n",
      "Epoch: [ 0] [  453/  468] time: 362.1790, train_loss: 0.07350683, train_accuracy: 0.9922, test_Accuracy: 0.9710\n",
      "Epoch: [ 0] [  454/  468] time: 362.8440, train_loss: 0.07543582, train_accuracy: 0.9766, test_Accuracy: 0.9718\n",
      "Epoch: [ 0] [  455/  468] time: 363.5700, train_loss: 0.04007170, train_accuracy: 0.9844, test_Accuracy: 0.9715\n",
      "Epoch: [ 0] [  456/  468] time: 364.3251, train_loss: 0.08139919, train_accuracy: 0.9844, test_Accuracy: 0.9713\n",
      "Epoch: [ 0] [  457/  468] time: 365.0650, train_loss: 0.03923596, train_accuracy: 0.9922, test_Accuracy: 0.9707\n",
      "Epoch: [ 0] [  458/  468] time: 365.8231, train_loss: 0.05173580, train_accuracy: 0.9844, test_Accuracy: 0.9706\n",
      "Epoch: [ 0] [  459/  468] time: 366.5461, train_loss: 0.04302656, train_accuracy: 0.9844, test_Accuracy: 0.9700\n",
      "Epoch: [ 0] [  460/  468] time: 367.2570, train_loss: 0.09822487, train_accuracy: 0.9453, test_Accuracy: 0.9693\n",
      "Epoch: [ 0] [  461/  468] time: 368.0341, train_loss: 0.04070392, train_accuracy: 0.9844, test_Accuracy: 0.9692\n",
      "Epoch: [ 0] [  462/  468] time: 368.8230, train_loss: 0.14335342, train_accuracy: 0.9688, test_Accuracy: 0.9685\n",
      "Epoch: [ 0] [  463/  468] time: 369.5620, train_loss: 0.03692505, train_accuracy: 0.9688, test_Accuracy: 0.9681\n",
      "Epoch: [ 0] [  464/  468] time: 370.2720, train_loss: 0.02146669, train_accuracy: 0.9922, test_Accuracy: 0.9673\n",
      "Epoch: [ 0] [  465/  468] time: 371.0041, train_loss: 0.09476610, train_accuracy: 0.9609, test_Accuracy: 0.9660\n",
      "Epoch: [ 0] [  466/  468] time: 371.7900, train_loss: 0.02315762, train_accuracy: 1.0000, test_Accuracy: 0.9658\n",
      "Epoch: [ 0] [  467/  468] time: 372.5491, train_loss: 0.06699377, train_accuracy: 0.9766, test_Accuracy: 0.9659\n"
     ]
    }
   ],
   "source": [
    "if train_flag :\n",
    "\n",
    "    checkpoint = tf.train.Checkpoint(dnn=network)\n",
    "\n",
    "    # create writer for tensorboard\n",
    "    summary_writer = tf.summary.create_file_writer(logdir=logs_dir)\n",
    "    start_time = time()\n",
    "\n",
    "    # restore check-point if it exits\n",
    "    could_load, checkpoint_counter = load(network, checkpoint_dir)    \n",
    "\n",
    "    if could_load:\n",
    "        start_epoch = (int)(checkpoint_counter / training_iterations)        \n",
    "        counter = checkpoint_counter        \n",
    "        print(\" [*] Load SUCCESS\")\n",
    "    else:\n",
    "        start_epoch = 0\n",
    "        start_iteration = 0\n",
    "        counter = 0\n",
    "        print(\" [!] Load failed...\")\n",
    "    \n",
    "    # train phase\n",
    "    with summary_writer.as_default():  # for tensorboard\n",
    "        for epoch in range(start_epoch, training_epochs):\n",
    "            for idx, (train_input, train_label) in enumerate(train_dataset):            \n",
    "                grads = grad(network, train_input, train_label)\n",
    "                optimizer.apply_gradients(grads_and_vars=zip(grads, network.trainable_variables))\n",
    "\n",
    "                train_loss = loss_fn(network, train_input, train_label)\n",
    "                train_accuracy = accuracy_fn(network, train_input, train_label)\n",
    "                \n",
    "                for test_input, test_label in test_dataset:                \n",
    "                    test_accuracy = accuracy_fn(network, test_input, test_label)\n",
    "\n",
    "                tf.summary.scalar(name='train_loss', data=train_loss, step=counter)\n",
    "                tf.summary.scalar(name='train_accuracy', data=train_accuracy, step=counter)\n",
    "                tf.summary.scalar(name='test_accuracy', data=test_accuracy, step=counter)\n",
    "\n",
    "                print(\n",
    "                    \"Epoch: [%2d] [%5d/%5d] time: %4.4f, train_loss: %.8f, train_accuracy: %.4f, test_Accuracy: %.4f\" \\\n",
    "                    % (epoch, idx, training_iterations, time() - start_time, train_loss, train_accuracy,\n",
    "                       test_accuracy))\n",
    "                counter += 1                \n",
    "        checkpoint.save(file_prefix=checkpoint_prefix + '-{}'.format(counter))\n",
    "        \n",
    "# test phase      \n",
    "else :\n",
    "    _, _ = load(network, checkpoint_dir)\n",
    "    for test_input, test_label in test_dataset:    \n",
    "        test_accuracy = accuracy_fn(network, test_input, test_label)\n",
    "\n",
    "    print(\"test_Accuracy: %.4f\" % (test_accuracy))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Test accuracy : 96.59 %"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}