Lab10/Lab10_1_2_JoongHoonKim.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "58391e4e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.5.0\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "from tensorflow.keras.utils import to_categorical\n",
    "from tensorflow.keras.datasets import mnist\n",
    "from time import time\n",
    "import os\n",
    "print(tf.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "fbf80d84",
   "metadata": {},
   "outputs": [],
   "source": [
    "def load(model, checkpoint_dir):\n",
    "    print(\" [*] Reading checkpoints...\")\n",
    "\n",
    "    ckpt = tf.train.get_checkpoint_state(checkpoint_dir)\n",
    "    if ckpt :\n",
    "        ckpt_name = os.path.basename(ckpt.model_checkpoint_path)\n",
    "        checkpoint = tf.train.Checkpoint(dnn=model)\n",
    "        checkpoint.restore(save_path=os.path.join(checkpoint_dir, ckpt_name))\n",
    "        counter = int(ckpt_name.split('-')[1])\n",
    "        print(\" [*] Success to read {}\".format(ckpt_name))\n",
    "        return True, counter\n",
    "    else:\n",
    "        print(\" [*] Failed to find a checkpoint\")\n",
    "        return False, 0\n",
    "\n",
    "def check_folder(dir):\n",
    "    if not os.path.exists(dir):\n",
    "        os.makedirs(dir)\n",
    "    return dir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "a894af21",
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_mnist() :\n",
    "    (train_data, train_labels), (test_data, test_labels) = mnist.load_data()\n",
    "    train_data = np.expand_dims(train_data, axis=-1) # [N, 28, 28] -> [N, 28, 28, 1]\n",
    "    test_data = np.expand_dims(test_data, axis=-1) # [N, 28, 28] -> [N, 28, 28, 1]\n",
    "\n",
    "    train_data, test_data = normalize(train_data, test_data)\n",
    "\n",
    "    train_labels = to_categorical(train_labels, 10) # [N,] -> [N, 10]\n",
    "    test_labels = to_categorical(test_labels, 10) # [N,] -> [N, 10]\n",
    "\n",
    "    return train_data, train_labels, test_data, test_labels\n",
    "\n",
    "def normalize(train_data, test_data):\n",
    "    train_data = train_data.astype(np.float32) / 255.0\n",
    "    test_data = test_data.astype(np.float32) / 255.0\n",
    "\n",
    "    return train_data, test_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "a91eed5e",
   "metadata": {},
   "outputs": [],
   "source": [
    "def loss_fn(model, images, labels):\n",
    "    logits = model(images, training=True)\n",
    "    loss = tf.reduce_mean(tf.keras.losses.categorical_crossentropy(y_pred=logits, y_true=labels, \n",
    "                                                                   from_logits=True))\n",
    "    return loss\n",
    "\n",
    "def accuracy_fn(model, images, labels):\n",
    "    logits = model(images, training=False)\n",
    "    prediction = tf.equal(tf.argmax(logits, -1), tf.argmax(labels, -1))\n",
    "    accuracy = tf.reduce_mean(tf.cast(prediction, tf.float32))\n",
    "    return accuracy\n",
    "\n",
    "def grad(model, images, labels):\n",
    "    with tf.GradientTape() as tape:\n",
    "        loss = loss_fn(model, images, labels)\n",
    "    return tape.gradient(loss, model.variables)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "e859db09",
   "metadata": {},
   "outputs": [],
   "source": [
    "def flatten() :\n",
    "    return tf.keras.layers.Flatten()\n",
    "\n",
    "def dense(label_dim, weight_init) :\n",
    "    return tf.keras.layers.Dense(units=label_dim, use_bias=True, kernel_initializer=weight_init)\n",
    "\n",
    "def relu() :\n",
    "    return tf.keras.layers.Activation(tf.keras.activations.relu)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "b6a87d48",
   "metadata": {},
   "outputs": [],
   "source": [
    "class create_model_class(tf.keras.Model):\n",
    "    def __init__(self, label_dim):\n",
    "        super(create_model_class, self).__init__()\n",
    "        weight_init = tf.keras.initializers.RandomNormal()\n",
    "\n",
    "        self.model = tf.keras.Sequential()\n",
    "        self.model.add(flatten())\n",
    "\n",
    "        for i in range(2):\n",
    "            self.model.add(dense(256, weight_init))\n",
    "            self.model.add(relu())\n",
    "\n",
    "        self.model.add(dense(label_dim, weight_init))\n",
    "\n",
    "    def call(self, x, training=None, mask=None):\n",
    "        x = self.model(x)\n",
    "\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "4c236fba",
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_model_function(label_dim) :\n",
    "    weight_init = tf.keras.initializers.RandomNormal()\n",
    "\n",
    "    model = tf.keras.Sequential()\n",
    "    model.add(flatten())\n",
    "\n",
    "    for i in range(2) :\n",
    "        model.add(dense(256, weight_init))\n",
    "        model.add(relu())\n",
    "\n",
    "    model.add(dense(label_dim, weight_init))\n",
    "\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "3566b8ef",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\" dataset \"\"\"\n",
    "train_x, train_y, test_x, test_y = load_mnist()\n",
    "\n",
    "\"\"\" parameters \"\"\"\n",
    "learning_rate = 0.001\n",
    "batch_size = 128\n",
    "\n",
    "training_epochs = 1\n",
    "training_iterations = len(train_x) // batch_size\n",
    "\n",
    "label_dim = 10\n",
    "\n",
    "train_flag = True\n",
    "\n",
    "\"\"\" Graph Input using Dataset API \"\"\"\n",
    "train_dataset = tf.data.Dataset.from_tensor_slices((train_x, train_y)).\\\n",
    "    shuffle(buffer_size=100000).\\\n",
    "    prefetch(buffer_size=batch_size).\\\n",
    "    batch(batch_size, drop_remainder=True)\n",
    "\n",
    "test_dataset = tf.data.Dataset.from_tensor_slices((test_x, test_y)).\\\n",
    "    shuffle(buffer_size=100000).\\\n",
    "    prefetch(buffer_size=len(test_x)).\\\n",
    "    batch(len(test_x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "56783f7a",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\" Model \"\"\"\n",
    "network = create_model_function(label_dim)\n",
    "\n",
    "\"\"\" Training \"\"\"\n",
    "optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate)\n",
    "\n",
    "\"\"\" Writer \"\"\"\n",
    "checkpoint_dir = 'checkpoints'\n",
    "logs_dir = 'logs'\n",
    "\n",
    "model_dir = 'nn_relu'\n",
    "\n",
    "checkpoint_dir = os.path.join(checkpoint_dir, model_dir)\n",
    "check_folder(checkpoint_dir)\n",
    "checkpoint_prefix = os.path.join(checkpoint_dir, model_dir)\n",
    "logs_dir = os.path.join(logs_dir, model_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "045991c9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " [*] Reading checkpoints...\n",
      " [*] Failed to find a checkpoint\n",
      " [!] Load failed...\n",
      "Epoch: [ 0] [    0/  468] time: 1.0582, train_loss: 2.16529417, train_accuracy: 0.3906, test_Accuracy: 0.2324\n",
      "Epoch: [ 0] [    1/  468] time: 1.3464, train_loss: 2.13100982, train_accuracy: 0.3672, test_Accuracy: 0.3172\n",
      "Epoch: [ 0] [    2/  468] time: 1.5808, train_loss: 2.10220575, train_accuracy: 0.4062, test_Accuracy: 0.4314\n",
      "Epoch: [ 0] [    3/  468] time: 1.7982, train_loss: 1.95755959, train_accuracy: 0.5938, test_Accuracy: 0.4809\n",
      "Epoch: [ 0] [    4/  468] time: 2.0056, train_loss: 1.94063807, train_accuracy: 0.4688, test_Accuracy: 0.5119\n",
      "Epoch: [ 0] [    5/  468] time: 2.2380, train_loss: 1.86165595, train_accuracy: 0.5078, test_Accuracy: 0.5544\n",
      "Epoch: [ 0] [    6/  468] time: 2.4425, train_loss: 1.71105075, train_accuracy: 0.6797, test_Accuracy: 0.5969\n",
      "Epoch: [ 0] [    7/  468] time: 2.6549, train_loss: 1.62779164, train_accuracy: 0.5859, test_Accuracy: 0.6343\n",
      "Epoch: [ 0] [    8/  468] time: 2.8434, train_loss: 1.50785625, train_accuracy: 0.6875, test_Accuracy: 0.6745\n",
      "Epoch: [ 0] [    9/  468] time: 3.0748, train_loss: 1.43100464, train_accuracy: 0.7734, test_Accuracy: 0.7176\n",
      "Epoch: [ 0] [   10/  468] time: 3.2752, train_loss: 1.32561946, train_accuracy: 0.6953, test_Accuracy: 0.7474\n",
      "Epoch: [ 0] [   11/  468] time: 3.4967, train_loss: 1.19922018, train_accuracy: 0.7500, test_Accuracy: 0.7748\n",
      "Epoch: [ 0] [   12/  468] time: 3.7929, train_loss: 1.11382723, train_accuracy: 0.8125, test_Accuracy: 0.7823\n",
      "Epoch: [ 0] [   13/  468] time: 4.0641, train_loss: 1.08622336, train_accuracy: 0.7266, test_Accuracy: 0.7937\n",
      "Epoch: [ 0] [   14/  468] time: 4.3504, train_loss: 0.94334298, train_accuracy: 0.8047, test_Accuracy: 0.8065\n",
      "Epoch: [ 0] [   15/  468] time: 4.6027, train_loss: 0.84667706, train_accuracy: 0.8359, test_Accuracy: 0.8135\n",
      "Epoch: [ 0] [   16/  468] time: 4.8480, train_loss: 0.80766201, train_accuracy: 0.8281, test_Accuracy: 0.8089\n",
      "Epoch: [ 0] [   17/  468] time: 5.0445, train_loss: 0.78835881, train_accuracy: 0.8047, test_Accuracy: 0.8053\n",
      "Epoch: [ 0] [   18/  468] time: 5.3198, train_loss: 0.89735055, train_accuracy: 0.7422, test_Accuracy: 0.8090\n",
      "Epoch: [ 0] [   19/  468] time: 5.5232, train_loss: 0.68021125, train_accuracy: 0.8359, test_Accuracy: 0.8298\n",
      "Epoch: [ 0] [   20/  468] time: 5.7576, train_loss: 0.63699913, train_accuracy: 0.8281, test_Accuracy: 0.8321\n",
      "Epoch: [ 0] [   21/  468] time: 6.0229, train_loss: 0.71797156, train_accuracy: 0.8047, test_Accuracy: 0.8004\n",
      "Epoch: [ 0] [   22/  468] time: 6.2672, train_loss: 0.69006050, train_accuracy: 0.7812, test_Accuracy: 0.7935\n",
      "Epoch: [ 0] [   23/  468] time: 6.5236, train_loss: 0.59961247, train_accuracy: 0.7969, test_Accuracy: 0.8175\n",
      "Epoch: [ 0] [   24/  468] time: 6.8048, train_loss: 0.49587709, train_accuracy: 0.8828, test_Accuracy: 0.8349\n",
      "Epoch: [ 0] [   25/  468] time: 6.9953, train_loss: 0.54735017, train_accuracy: 0.8750, test_Accuracy: 0.8447\n",
      "Epoch: [ 0] [   26/  468] time: 7.2197, train_loss: 0.50254565, train_accuracy: 0.8438, test_Accuracy: 0.8573\n",
      "Epoch: [ 0] [   27/  468] time: 7.4232, train_loss: 0.45793438, train_accuracy: 0.8594, test_Accuracy: 0.8677\n",
      "Epoch: [ 0] [   28/  468] time: 7.6097, train_loss: 0.45479161, train_accuracy: 0.8672, test_Accuracy: 0.8631\n",
      "Epoch: [ 0] [   29/  468] time: 7.8181, train_loss: 0.49466634, train_accuracy: 0.8594, test_Accuracy: 0.8485\n",
      "Epoch: [ 0] [   30/  468] time: 8.0225, train_loss: 0.42029691, train_accuracy: 0.8906, test_Accuracy: 0.8431\n",
      "Epoch: [ 0] [   31/  468] time: 8.2250, train_loss: 0.52159911, train_accuracy: 0.8359, test_Accuracy: 0.8449\n",
      "Epoch: [ 0] [   32/  468] time: 8.4175, train_loss: 0.57087958, train_accuracy: 0.8359, test_Accuracy: 0.8581\n",
      "Epoch: [ 0] [   33/  468] time: 8.6140, train_loss: 0.44425428, train_accuracy: 0.8984, test_Accuracy: 0.8695\n",
      "Epoch: [ 0] [   34/  468] time: 8.8623, train_loss: 0.41998467, train_accuracy: 0.8672, test_Accuracy: 0.8768\n",
      "Epoch: [ 0] [   35/  468] time: 9.0807, train_loss: 0.34357557, train_accuracy: 0.9219, test_Accuracy: 0.8792\n",
      "Epoch: [ 0] [   36/  468] time: 9.2942, train_loss: 0.58309180, train_accuracy: 0.8516, test_Accuracy: 0.8816\n",
      "Epoch: [ 0] [   37/  468] time: 9.6083, train_loss: 0.32885963, train_accuracy: 0.9141, test_Accuracy: 0.8783\n",
      "Epoch: [ 0] [   38/  468] time: 9.9893, train_loss: 0.36373675, train_accuracy: 0.9141, test_Accuracy: 0.8739\n",
      "Epoch: [ 0] [   39/  468] time: 10.2905, train_loss: 0.44569987, train_accuracy: 0.8672, test_Accuracy: 0.8818\n",
      "Epoch: [ 0] [   40/  468] time: 10.6276, train_loss: 0.44699365, train_accuracy: 0.8828, test_Accuracy: 0.8865\n",
      "Epoch: [ 0] [   41/  468] time: 10.8460, train_loss: 0.26096141, train_accuracy: 0.9219, test_Accuracy: 0.8858\n",
      "Epoch: [ 0] [   42/  468] time: 11.0774, train_loss: 0.31081498, train_accuracy: 0.8984, test_Accuracy: 0.8796\n",
      "Epoch: [ 0] [   43/  468] time: 11.3397, train_loss: 0.36790389, train_accuracy: 0.8750, test_Accuracy: 0.8739\n",
      "Epoch: [ 0] [   44/  468] time: 11.5391, train_loss: 0.44360715, train_accuracy: 0.8672, test_Accuracy: 0.8739\n",
      "Epoch: [ 0] [   45/  468] time: 11.7596, train_loss: 0.38833302, train_accuracy: 0.8828, test_Accuracy: 0.8790\n",
      "Epoch: [ 0] [   46/  468] time: 12.0149, train_loss: 0.52261925, train_accuracy: 0.8438, test_Accuracy: 0.8855\n",
      "Epoch: [ 0] [   47/  468] time: 12.2712, train_loss: 0.41083965, train_accuracy: 0.8906, test_Accuracy: 0.8911\n",
      "Epoch: [ 0] [   48/  468] time: 12.5345, train_loss: 0.30440649, train_accuracy: 0.8984, test_Accuracy: 0.8900\n",
      "Epoch: [ 0] [   49/  468] time: 12.7848, train_loss: 0.40194532, train_accuracy: 0.8906, test_Accuracy: 0.8885\n",
      "Epoch: [ 0] [   50/  468] time: 13.0102, train_loss: 0.29292271, train_accuracy: 0.9141, test_Accuracy: 0.8849\n",
      "Epoch: [ 0] [   51/  468] time: 13.1997, train_loss: 0.30977213, train_accuracy: 0.9297, test_Accuracy: 0.8828\n",
      "Epoch: [ 0] [   52/  468] time: 13.4291, train_loss: 0.34691155, train_accuracy: 0.9062, test_Accuracy: 0.8829\n",
      "Epoch: [ 0] [   53/  468] time: 13.6325, train_loss: 0.48497391, train_accuracy: 0.8906, test_Accuracy: 0.8844\n",
      "Epoch: [ 0] [   54/  468] time: 13.8260, train_loss: 0.37276092, train_accuracy: 0.8906, test_Accuracy: 0.8922\n",
      "Epoch: [ 0] [   55/  468] time: 14.0135, train_loss: 0.40453851, train_accuracy: 0.8906, test_Accuracy: 0.8991\n",
      "Epoch: [ 0] [   56/  468] time: 14.2070, train_loss: 0.35771781, train_accuracy: 0.8906, test_Accuracy: 0.8987\n",
      "Epoch: [ 0] [   57/  468] time: 14.3995, train_loss: 0.29711568, train_accuracy: 0.9219, test_Accuracy: 0.8932\n",
      "Epoch: [ 0] [   58/  468] time: 14.5940, train_loss: 0.35338491, train_accuracy: 0.8906, test_Accuracy: 0.8881\n",
      "Epoch: [ 0] [   59/  468] time: 14.7775, train_loss: 0.38394430, train_accuracy: 0.9062, test_Accuracy: 0.8858\n",
      "Epoch: [ 0] [   60/  468] time: 15.0288, train_loss: 0.26937130, train_accuracy: 0.9141, test_Accuracy: 0.8834\n",
      "Epoch: [ 0] [   61/  468] time: 15.2771, train_loss: 0.24251367, train_accuracy: 0.9219, test_Accuracy: 0.8843\n",
      "Epoch: [ 0] [   62/  468] time: 15.5404, train_loss: 0.32755667, train_accuracy: 0.8906, test_Accuracy: 0.8895\n",
      "Epoch: [ 0] [   63/  468] time: 15.7978, train_loss: 0.30304191, train_accuracy: 0.8828, test_Accuracy: 0.8955\n",
      "Epoch: [ 0] [   64/  468] time: 16.0032, train_loss: 0.26951200, train_accuracy: 0.8984, test_Accuracy: 0.8972\n",
      "Epoch: [ 0] [   65/  468] time: 16.2047, train_loss: 0.24148193, train_accuracy: 0.9375, test_Accuracy: 0.8952\n",
      "Epoch: [ 0] [   66/  468] time: 16.4161, train_loss: 0.44343626, train_accuracy: 0.8906, test_Accuracy: 0.8985\n",
      "Epoch: [ 0] [   67/  468] time: 16.6026, train_loss: 0.26998031, train_accuracy: 0.9453, test_Accuracy: 0.9017\n",
      "Epoch: [ 0] [   68/  468] time: 16.7702, train_loss: 0.33858341, train_accuracy: 0.8750, test_Accuracy: 0.9050\n",
      "Epoch: [ 0] [   69/  468] time: 16.9457, train_loss: 0.22178115, train_accuracy: 0.9609, test_Accuracy: 0.9023\n",
      "Epoch: [ 0] [   70/  468] time: 17.1741, train_loss: 0.32869893, train_accuracy: 0.8906, test_Accuracy: 0.9038\n",
      "Epoch: [ 0] [   71/  468] time: 17.3915, train_loss: 0.43940061, train_accuracy: 0.8750, test_Accuracy: 0.9041\n",
      "Epoch: [ 0] [   72/  468] time: 17.6149, train_loss: 0.20627174, train_accuracy: 0.9531, test_Accuracy: 0.9037\n",
      "Epoch: [ 0] [   73/  468] time: 17.8772, train_loss: 0.42829740, train_accuracy: 0.8906, test_Accuracy: 0.9002\n",
      "Epoch: [ 0] [   74/  468] time: 18.0896, train_loss: 0.46806699, train_accuracy: 0.8516, test_Accuracy: 0.8974\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [ 0] [   75/  468] time: 18.2781, train_loss: 0.41006184, train_accuracy: 0.8516, test_Accuracy: 0.9023\n",
      "Epoch: [ 0] [   76/  468] time: 18.4706, train_loss: 0.25366783, train_accuracy: 0.9219, test_Accuracy: 0.9060\n",
      "Epoch: [ 0] [   77/  468] time: 18.6631, train_loss: 0.39034373, train_accuracy: 0.8516, test_Accuracy: 0.9082\n",
      "Epoch: [ 0] [   78/  468] time: 18.8376, train_loss: 0.23831762, train_accuracy: 0.9219, test_Accuracy: 0.9040\n",
      "Epoch: [ 0] [   79/  468] time: 19.0152, train_loss: 0.41166735, train_accuracy: 0.8516, test_Accuracy: 0.8999\n",
      "Epoch: [ 0] [   80/  468] time: 19.2007, train_loss: 0.28209713, train_accuracy: 0.9297, test_Accuracy: 0.8971\n",
      "Epoch: [ 0] [   81/  468] time: 19.4640, train_loss: 0.33747217, train_accuracy: 0.9219, test_Accuracy: 0.8986\n",
      "Epoch: [ 0] [   82/  468] time: 19.7243, train_loss: 0.28696907, train_accuracy: 0.9375, test_Accuracy: 0.9054\n",
      "Epoch: [ 0] [   83/  468] time: 19.9826, train_loss: 0.29722124, train_accuracy: 0.9062, test_Accuracy: 0.9124\n",
      "Epoch: [ 0] [   84/  468] time: 20.2419, train_loss: 0.32530189, train_accuracy: 0.9062, test_Accuracy: 0.9157\n",
      "Epoch: [ 0] [   85/  468] time: 20.4413, train_loss: 0.28229380, train_accuracy: 0.9062, test_Accuracy: 0.9161\n",
      "Epoch: [ 0] [   86/  468] time: 20.6428, train_loss: 0.33767173, train_accuracy: 0.9062, test_Accuracy: 0.9133\n",
      "Epoch: [ 0] [   87/  468] time: 20.8433, train_loss: 0.30160233, train_accuracy: 0.8984, test_Accuracy: 0.9119\n",
      "Epoch: [ 0] [   88/  468] time: 21.0557, train_loss: 0.27693173, train_accuracy: 0.9141, test_Accuracy: 0.9114\n",
      "Epoch: [ 0] [   89/  468] time: 21.2432, train_loss: 0.36768532, train_accuracy: 0.9062, test_Accuracy: 0.9117\n",
      "Epoch: [ 0] [   90/  468] time: 21.4347, train_loss: 0.21877171, train_accuracy: 0.9297, test_Accuracy: 0.9119\n",
      "Epoch: [ 0] [   91/  468] time: 21.6451, train_loss: 0.29572797, train_accuracy: 0.9141, test_Accuracy: 0.9104\n",
      "Epoch: [ 0] [   92/  468] time: 21.8286, train_loss: 0.28507084, train_accuracy: 0.8984, test_Accuracy: 0.9136\n",
      "Epoch: [ 0] [   93/  468] time: 22.0071, train_loss: 0.26805550, train_accuracy: 0.9219, test_Accuracy: 0.9174\n",
      "Epoch: [ 0] [   94/  468] time: 22.1807, train_loss: 0.37065327, train_accuracy: 0.8984, test_Accuracy: 0.9181\n",
      "Epoch: [ 0] [   95/  468] time: 22.4041, train_loss: 0.36099732, train_accuracy: 0.8828, test_Accuracy: 0.9180\n",
      "Epoch: [ 0] [   96/  468] time: 22.6833, train_loss: 0.24193835, train_accuracy: 0.9297, test_Accuracy: 0.9169\n",
      "Epoch: [ 0] [   97/  468] time: 22.9496, train_loss: 0.27849573, train_accuracy: 0.8906, test_Accuracy: 0.9155\n",
      "Epoch: [ 0] [   98/  468] time: 23.2249, train_loss: 0.27461752, train_accuracy: 0.9219, test_Accuracy: 0.9145\n",
      "Epoch: [ 0] [   99/  468] time: 23.4433, train_loss: 0.41413265, train_accuracy: 0.8750, test_Accuracy: 0.9167\n",
      "Epoch: [ 0] [  100/  468] time: 23.6527, train_loss: 0.34427184, train_accuracy: 0.9062, test_Accuracy: 0.9185\n",
      "Epoch: [ 0] [  101/  468] time: 23.8722, train_loss: 0.17777191, train_accuracy: 0.9688, test_Accuracy: 0.9199\n",
      "Epoch: [ 0] [  102/  468] time: 24.0766, train_loss: 0.29147026, train_accuracy: 0.9219, test_Accuracy: 0.9199\n",
      "Epoch: [ 0] [  103/  468] time: 24.2731, train_loss: 0.30589038, train_accuracy: 0.8828, test_Accuracy: 0.9198\n",
      "Epoch: [ 0] [  104/  468] time: 24.4606, train_loss: 0.26743194, train_accuracy: 0.9297, test_Accuracy: 0.9189\n",
      "Epoch: [ 0] [  105/  468] time: 24.6481, train_loss: 0.29799688, train_accuracy: 0.8906, test_Accuracy: 0.9179\n",
      "Epoch: [ 0] [  106/  468] time: 24.8495, train_loss: 0.30963916, train_accuracy: 0.9141, test_Accuracy: 0.9169\n",
      "Epoch: [ 0] [  107/  468] time: 25.0420, train_loss: 0.25795555, train_accuracy: 0.8906, test_Accuracy: 0.9160\n",
      "Epoch: [ 0] [  108/  468] time: 25.2385, train_loss: 0.28078187, train_accuracy: 0.9297, test_Accuracy: 0.9171\n",
      "Epoch: [ 0] [  109/  468] time: 25.4479, train_loss: 0.36906606, train_accuracy: 0.9062, test_Accuracy: 0.9168\n",
      "Epoch: [ 0] [  110/  468] time: 25.6534, train_loss: 0.31524101, train_accuracy: 0.8984, test_Accuracy: 0.9197\n",
      "Epoch: [ 0] [  111/  468] time: 25.8329, train_loss: 0.26635182, train_accuracy: 0.9531, test_Accuracy: 0.9212\n",
      "Epoch: [ 0] [  112/  468] time: 26.0194, train_loss: 0.28678301, train_accuracy: 0.9062, test_Accuracy: 0.9205\n",
      "Epoch: [ 0] [  113/  468] time: 26.2189, train_loss: 0.19819805, train_accuracy: 0.9297, test_Accuracy: 0.9216\n",
      "Epoch: [ 0] [  114/  468] time: 26.3934, train_loss: 0.22360396, train_accuracy: 0.9453, test_Accuracy: 0.9213\n",
      "Epoch: [ 0] [  115/  468] time: 26.5570, train_loss: 0.24726634, train_accuracy: 0.9375, test_Accuracy: 0.9211\n",
      "Epoch: [ 0] [  116/  468] time: 26.7405, train_loss: 0.41756856, train_accuracy: 0.9141, test_Accuracy: 0.9228\n",
      "Epoch: [ 0] [  117/  468] time: 26.9589, train_loss: 0.24035823, train_accuracy: 0.9141, test_Accuracy: 0.9242\n",
      "Epoch: [ 0] [  118/  468] time: 27.2112, train_loss: 0.29957357, train_accuracy: 0.9062, test_Accuracy: 0.9247\n",
      "Epoch: [ 0] [  119/  468] time: 27.4386, train_loss: 0.21279271, train_accuracy: 0.9453, test_Accuracy: 0.9246\n",
      "Epoch: [ 0] [  120/  468] time: 27.6999, train_loss: 0.32301778, train_accuracy: 0.9141, test_Accuracy: 0.9246\n",
      "Epoch: [ 0] [  121/  468] time: 27.9104, train_loss: 0.27621523, train_accuracy: 0.9141, test_Accuracy: 0.9240\n",
      "Epoch: [ 0] [  122/  468] time: 28.1068, train_loss: 0.28635961, train_accuracy: 0.8984, test_Accuracy: 0.9247\n",
      "Epoch: [ 0] [  123/  468] time: 28.3153, train_loss: 0.19635284, train_accuracy: 0.9531, test_Accuracy: 0.9231\n",
      "Epoch: [ 0] [  124/  468] time: 28.4968, train_loss: 0.28238040, train_accuracy: 0.9141, test_Accuracy: 0.9220\n",
      "Epoch: [ 0] [  125/  468] time: 28.6693, train_loss: 0.33868542, train_accuracy: 0.9062, test_Accuracy: 0.9218\n",
      "Epoch: [ 0] [  126/  468] time: 28.8489, train_loss: 0.26888537, train_accuracy: 0.9219, test_Accuracy: 0.9228\n",
      "Epoch: [ 0] [  127/  468] time: 29.0393, train_loss: 0.27804431, train_accuracy: 0.9375, test_Accuracy: 0.9258\n",
      "Epoch: [ 0] [  128/  468] time: 29.2328, train_loss: 0.39556435, train_accuracy: 0.8984, test_Accuracy: 0.9269\n",
      "Epoch: [ 0] [  129/  468] time: 29.4233, train_loss: 0.19764732, train_accuracy: 0.9609, test_Accuracy: 0.9263\n",
      "Epoch: [ 0] [  130/  468] time: 29.6138, train_loss: 0.13928185, train_accuracy: 0.9531, test_Accuracy: 0.9228\n",
      "Epoch: [ 0] [  131/  468] time: 29.8083, train_loss: 0.38640535, train_accuracy: 0.9062, test_Accuracy: 0.9191\n",
      "Epoch: [ 0] [  132/  468] time: 30.0107, train_loss: 0.19653764, train_accuracy: 0.9531, test_Accuracy: 0.9171\n",
      "Epoch: [ 0] [  133/  468] time: 30.2042, train_loss: 0.26434427, train_accuracy: 0.9219, test_Accuracy: 0.9170\n",
      "Epoch: [ 0] [  134/  468] time: 30.3937, train_loss: 0.28190190, train_accuracy: 0.9141, test_Accuracy: 0.9202\n",
      "Epoch: [ 0] [  135/  468] time: 30.5782, train_loss: 0.29776582, train_accuracy: 0.8984, test_Accuracy: 0.9242\n",
      "Epoch: [ 0] [  136/  468] time: 30.7548, train_loss: 0.29853565, train_accuracy: 0.9141, test_Accuracy: 0.9254\n",
      "Epoch: [ 0] [  137/  468] time: 30.9522, train_loss: 0.14008594, train_accuracy: 0.9766, test_Accuracy: 0.9252\n",
      "Epoch: [ 0] [  138/  468] time: 31.1417, train_loss: 0.35953456, train_accuracy: 0.8672, test_Accuracy: 0.9251\n",
      "Epoch: [ 0] [  139/  468] time: 31.3362, train_loss: 0.23046395, train_accuracy: 0.9062, test_Accuracy: 0.9239\n",
      "Epoch: [ 0] [  140/  468] time: 31.5347, train_loss: 0.26375574, train_accuracy: 0.9375, test_Accuracy: 0.9227\n",
      "Epoch: [ 0] [  141/  468] time: 31.7331, train_loss: 0.21333036, train_accuracy: 0.9531, test_Accuracy: 0.9208\n",
      "Epoch: [ 0] [  142/  468] time: 31.9426, train_loss: 0.20855533, train_accuracy: 0.9375, test_Accuracy: 0.9224\n",
      "Epoch: [ 0] [  143/  468] time: 32.1809, train_loss: 0.25362819, train_accuracy: 0.9219, test_Accuracy: 0.9282\n",
      "Epoch: [ 0] [  144/  468] time: 32.4333, train_loss: 0.22355056, train_accuracy: 0.9375, test_Accuracy: 0.9310\n",
      "Epoch: [ 0] [  145/  468] time: 32.6816, train_loss: 0.18635532, train_accuracy: 0.9531, test_Accuracy: 0.9279\n",
      "Epoch: [ 0] [  146/  468] time: 32.9559, train_loss: 0.17599615, train_accuracy: 0.9531, test_Accuracy: 0.9235\n",
      "Epoch: [ 0] [  147/  468] time: 33.1922, train_loss: 0.28816822, train_accuracy: 0.9375, test_Accuracy: 0.9171\n",
      "Epoch: [ 0] [  148/  468] time: 33.4097, train_loss: 0.26270577, train_accuracy: 0.9219, test_Accuracy: 0.9147\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [ 0] [  149/  468] time: 33.6311, train_loss: 0.27538359, train_accuracy: 0.9141, test_Accuracy: 0.9244\n",
      "Epoch: [ 0] [  150/  468] time: 33.8245, train_loss: 0.24693757, train_accuracy: 0.9219, test_Accuracy: 0.9308\n",
      "Epoch: [ 0] [  151/  468] time: 34.0140, train_loss: 0.19848272, train_accuracy: 0.9453, test_Accuracy: 0.9333\n",
      "Epoch: [ 0] [  152/  468] time: 34.2115, train_loss: 0.11561541, train_accuracy: 0.9688, test_Accuracy: 0.9315\n",
      "Epoch: [ 0] [  153/  468] time: 34.4389, train_loss: 0.31930268, train_accuracy: 0.9453, test_Accuracy: 0.9277\n",
      "Epoch: [ 0] [  154/  468] time: 34.6234, train_loss: 0.32257593, train_accuracy: 0.9141, test_Accuracy: 0.9240\n",
      "Epoch: [ 0] [  155/  468] time: 34.8059, train_loss: 0.40391958, train_accuracy: 0.8984, test_Accuracy: 0.9222\n",
      "Epoch: [ 0] [  156/  468] time: 35.0164, train_loss: 0.23018883, train_accuracy: 0.9453, test_Accuracy: 0.9222\n",
      "Epoch: [ 0] [  157/  468] time: 35.2218, train_loss: 0.15983474, train_accuracy: 0.9609, test_Accuracy: 0.9234\n",
      "Epoch: [ 0] [  158/  468] time: 35.4133, train_loss: 0.27119082, train_accuracy: 0.9375, test_Accuracy: 0.9289\n",
      "Epoch: [ 0] [  159/  468] time: 35.6088, train_loss: 0.14184120, train_accuracy: 0.9688, test_Accuracy: 0.9327\n",
      "Epoch: [ 0] [  160/  468] time: 35.8072, train_loss: 0.22769043, train_accuracy: 0.9062, test_Accuracy: 0.9349\n",
      "Epoch: [ 0] [  161/  468] time: 36.0107, train_loss: 0.12379131, train_accuracy: 0.9766, test_Accuracy: 0.9354\n",
      "Epoch: [ 0] [  162/  468] time: 36.1912, train_loss: 0.29859668, train_accuracy: 0.8906, test_Accuracy: 0.9340\n",
      "Epoch: [ 0] [  163/  468] time: 36.3658, train_loss: 0.16166750, train_accuracy: 0.9453, test_Accuracy: 0.9317\n",
      "Epoch: [ 0] [  164/  468] time: 36.5762, train_loss: 0.19029242, train_accuracy: 0.9453, test_Accuracy: 0.9301\n",
      "Epoch: [ 0] [  165/  468] time: 36.8076, train_loss: 0.17970793, train_accuracy: 0.9297, test_Accuracy: 0.9311\n",
      "Epoch: [ 0] [  166/  468] time: 37.0260, train_loss: 0.32558873, train_accuracy: 0.9453, test_Accuracy: 0.9324\n",
      "Epoch: [ 0] [  167/  468] time: 37.2773, train_loss: 0.15388605, train_accuracy: 0.9531, test_Accuracy: 0.9356\n",
      "Epoch: [ 0] [  168/  468] time: 37.5097, train_loss: 0.18024147, train_accuracy: 0.9453, test_Accuracy: 0.9377\n",
      "Epoch: [ 0] [  169/  468] time: 37.6872, train_loss: 0.31778336, train_accuracy: 0.9219, test_Accuracy: 0.9362\n",
      "Epoch: [ 0] [  170/  468] time: 37.8817, train_loss: 0.30833125, train_accuracy: 0.8984, test_Accuracy: 0.9360\n",
      "Epoch: [ 0] [  171/  468] time: 38.0692, train_loss: 0.26156041, train_accuracy: 0.9297, test_Accuracy: 0.9358\n",
      "Epoch: [ 0] [  172/  468] time: 38.2557, train_loss: 0.13886851, train_accuracy: 0.9688, test_Accuracy: 0.9355\n",
      "Epoch: [ 0] [  173/  468] time: 38.4342, train_loss: 0.21237063, train_accuracy: 0.9688, test_Accuracy: 0.9354\n",
      "Epoch: [ 0] [  174/  468] time: 38.6008, train_loss: 0.14553784, train_accuracy: 0.9531, test_Accuracy: 0.9349\n",
      "Epoch: [ 0] [  175/  468] time: 38.7823, train_loss: 0.15396088, train_accuracy: 0.9531, test_Accuracy: 0.9342\n",
      "Epoch: [ 0] [  176/  468] time: 38.9708, train_loss: 0.19165987, train_accuracy: 0.9531, test_Accuracy: 0.9342\n",
      "Epoch: [ 0] [  177/  468] time: 39.1603, train_loss: 0.13392264, train_accuracy: 0.9688, test_Accuracy: 0.9336\n",
      "Epoch: [ 0] [  178/  468] time: 39.3398, train_loss: 0.25484598, train_accuracy: 0.8906, test_Accuracy: 0.9323\n",
      "Epoch: [ 0] [  179/  468] time: 39.5532, train_loss: 0.16100121, train_accuracy: 0.9375, test_Accuracy: 0.9326\n",
      "Epoch: [ 0] [  180/  468] time: 39.7467, train_loss: 0.18009266, train_accuracy: 0.9531, test_Accuracy: 0.9317\n",
      "Epoch: [ 0] [  181/  468] time: 39.9472, train_loss: 0.16974472, train_accuracy: 0.9531, test_Accuracy: 0.9334\n",
      "Epoch: [ 0] [  182/  468] time: 40.1456, train_loss: 0.16882470, train_accuracy: 0.9531, test_Accuracy: 0.9342\n",
      "Epoch: [ 0] [  183/  468] time: 40.4448, train_loss: 0.18698318, train_accuracy: 0.9609, test_Accuracy: 0.9349\n",
      "Epoch: [ 0] [  184/  468] time: 40.6992, train_loss: 0.22283790, train_accuracy: 0.9219, test_Accuracy: 0.9360\n",
      "Epoch: [ 0] [  185/  468] time: 40.9804, train_loss: 0.10428213, train_accuracy: 0.9688, test_Accuracy: 0.9362\n",
      "Epoch: [ 0] [  186/  468] time: 41.2706, train_loss: 0.28714427, train_accuracy: 0.9062, test_Accuracy: 0.9376\n",
      "Epoch: [ 0] [  187/  468] time: 41.4910, train_loss: 0.26755050, train_accuracy: 0.9219, test_Accuracy: 0.9373\n",
      "Epoch: [ 0] [  188/  468] time: 41.7105, train_loss: 0.30942816, train_accuracy: 0.8906, test_Accuracy: 0.9364\n",
      "Epoch: [ 0] [  189/  468] time: 41.9309, train_loss: 0.21609765, train_accuracy: 0.9375, test_Accuracy: 0.9342\n",
      "Epoch: [ 0] [  190/  468] time: 42.1383, train_loss: 0.15678331, train_accuracy: 0.9531, test_Accuracy: 0.9339\n",
      "Epoch: [ 0] [  191/  468] time: 42.3298, train_loss: 0.22719060, train_accuracy: 0.9219, test_Accuracy: 0.9327\n",
      "Epoch: [ 0] [  192/  468] time: 42.5153, train_loss: 0.26609570, train_accuracy: 0.9297, test_Accuracy: 0.9312\n",
      "Epoch: [ 0] [  193/  468] time: 42.7048, train_loss: 0.17715070, train_accuracy: 0.9609, test_Accuracy: 0.9312\n",
      "Epoch: [ 0] [  194/  468] time: 42.8923, train_loss: 0.40051228, train_accuracy: 0.8984, test_Accuracy: 0.9347\n",
      "Epoch: [ 0] [  195/  468] time: 43.0758, train_loss: 0.26537549, train_accuracy: 0.9453, test_Accuracy: 0.9388\n",
      "Epoch: [ 0] [  196/  468] time: 43.2663, train_loss: 0.12418461, train_accuracy: 0.9766, test_Accuracy: 0.9410\n",
      "Epoch: [ 0] [  197/  468] time: 43.4947, train_loss: 0.18898523, train_accuracy: 0.9297, test_Accuracy: 0.9404\n",
      "Epoch: [ 0] [  198/  468] time: 43.7440, train_loss: 0.20913559, train_accuracy: 0.9375, test_Accuracy: 0.9385\n",
      "Epoch: [ 0] [  199/  468] time: 44.0013, train_loss: 0.18666083, train_accuracy: 0.9375, test_Accuracy: 0.9337\n",
      "Epoch: [ 0] [  200/  468] time: 44.3295, train_loss: 0.27096996, train_accuracy: 0.8828, test_Accuracy: 0.9330\n",
      "Epoch: [ 0] [  201/  468] time: 44.6167, train_loss: 0.17121999, train_accuracy: 0.9453, test_Accuracy: 0.9328\n",
      "Epoch: [ 0] [  202/  468] time: 44.8570, train_loss: 0.17285785, train_accuracy: 0.9453, test_Accuracy: 0.9348\n",
      "Epoch: [ 0] [  203/  468] time: 45.0934, train_loss: 0.21872938, train_accuracy: 0.9453, test_Accuracy: 0.9341\n",
      "Epoch: [ 0] [  204/  468] time: 45.3019, train_loss: 0.19626966, train_accuracy: 0.9297, test_Accuracy: 0.9326\n",
      "Epoch: [ 0] [  205/  468] time: 45.4944, train_loss: 0.27166885, train_accuracy: 0.9219, test_Accuracy: 0.9291\n",
      "Epoch: [ 0] [  206/  468] time: 45.6858, train_loss: 0.15678877, train_accuracy: 0.9609, test_Accuracy: 0.9287\n",
      "Epoch: [ 0] [  207/  468] time: 45.9152, train_loss: 0.27418447, train_accuracy: 0.9219, test_Accuracy: 0.9302\n",
      "Epoch: [ 0] [  208/  468] time: 46.1526, train_loss: 0.25774843, train_accuracy: 0.9297, test_Accuracy: 0.9357\n",
      "Epoch: [ 0] [  209/  468] time: 46.3760, train_loss: 0.27426291, train_accuracy: 0.8906, test_Accuracy: 0.9383\n",
      "Epoch: [ 0] [  210/  468] time: 46.6632, train_loss: 0.16437739, train_accuracy: 0.9453, test_Accuracy: 0.9389\n",
      "Epoch: [ 0] [  211/  468] time: 46.8627, train_loss: 0.23142907, train_accuracy: 0.9141, test_Accuracy: 0.9350\n",
      "Epoch: [ 0] [  212/  468] time: 47.0522, train_loss: 0.30260748, train_accuracy: 0.9141, test_Accuracy: 0.9320\n",
      "Epoch: [ 0] [  213/  468] time: 47.2457, train_loss: 0.20683037, train_accuracy: 0.9453, test_Accuracy: 0.9280\n",
      "Epoch: [ 0] [  214/  468] time: 47.4292, train_loss: 0.19948125, train_accuracy: 0.9688, test_Accuracy: 0.9277\n",
      "Epoch: [ 0] [  215/  468] time: 47.6227, train_loss: 0.34986204, train_accuracy: 0.9297, test_Accuracy: 0.9363\n",
      "Epoch: [ 0] [  216/  468] time: 47.7872, train_loss: 0.10769662, train_accuracy: 0.9844, test_Accuracy: 0.9397\n",
      "Epoch: [ 0] [  217/  468] time: 47.9757, train_loss: 0.23278487, train_accuracy: 0.9453, test_Accuracy: 0.9415\n",
      "Epoch: [ 0] [  218/  468] time: 48.1502, train_loss: 0.27623296, train_accuracy: 0.9297, test_Accuracy: 0.9411\n",
      "Epoch: [ 0] [  219/  468] time: 48.3417, train_loss: 0.16618565, train_accuracy: 0.9453, test_Accuracy: 0.9392\n",
      "Epoch: [ 0] [  220/  468] time: 48.5202, train_loss: 0.18278503, train_accuracy: 0.9219, test_Accuracy: 0.9362\n",
      "Epoch: [ 0] [  221/  468] time: 48.6908, train_loss: 0.17992118, train_accuracy: 0.9531, test_Accuracy: 0.9359\n",
      "Epoch: [ 0] [  222/  468] time: 48.8733, train_loss: 0.17351848, train_accuracy: 0.9531, test_Accuracy: 0.9344\n",
      "Epoch: [ 0] [  223/  468] time: 49.0568, train_loss: 0.16237047, train_accuracy: 0.9453, test_Accuracy: 0.9339\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [ 0] [  224/  468] time: 49.2304, train_loss: 0.15858305, train_accuracy: 0.9766, test_Accuracy: 0.9346\n",
      "Epoch: [ 0] [  225/  468] time: 49.3930, train_loss: 0.16603613, train_accuracy: 0.9453, test_Accuracy: 0.9346\n",
      "Epoch: [ 0] [  226/  468] time: 49.5625, train_loss: 0.16736704, train_accuracy: 0.9297, test_Accuracy: 0.9346\n",
      "Epoch: [ 0] [  227/  468] time: 49.7918, train_loss: 0.29663378, train_accuracy: 0.9375, test_Accuracy: 0.9372\n",
      "Epoch: [ 0] [  228/  468] time: 50.0153, train_loss: 0.25693256, train_accuracy: 0.9219, test_Accuracy: 0.9397\n",
      "Epoch: [ 0] [  229/  468] time: 50.2367, train_loss: 0.23850469, train_accuracy: 0.9531, test_Accuracy: 0.9418\n",
      "Epoch: [ 0] [  230/  468] time: 50.4850, train_loss: 0.13297257, train_accuracy: 0.9531, test_Accuracy: 0.9427\n",
      "Epoch: [ 0] [  231/  468] time: 50.6835, train_loss: 0.16499759, train_accuracy: 0.9375, test_Accuracy: 0.9439\n",
      "Epoch: [ 0] [  232/  468] time: 50.8680, train_loss: 0.14171183, train_accuracy: 0.9766, test_Accuracy: 0.9446\n",
      "Epoch: [ 0] [  233/  468] time: 51.0615, train_loss: 0.21119973, train_accuracy: 0.9297, test_Accuracy: 0.9448\n",
      "Epoch: [ 0] [  234/  468] time: 51.2599, train_loss: 0.09326757, train_accuracy: 0.9844, test_Accuracy: 0.9449\n",
      "Epoch: [ 0] [  235/  468] time: 51.4315, train_loss: 0.22314660, train_accuracy: 0.9062, test_Accuracy: 0.9446\n",
      "Epoch: [ 0] [  236/  468] time: 51.6080, train_loss: 0.18818915, train_accuracy: 0.9453, test_Accuracy: 0.9465\n",
      "Epoch: [ 0] [  237/  468] time: 51.7885, train_loss: 0.30613756, train_accuracy: 0.9453, test_Accuracy: 0.9467\n",
      "Epoch: [ 0] [  238/  468] time: 51.9830, train_loss: 0.24109611, train_accuracy: 0.9453, test_Accuracy: 0.9455\n",
      "Epoch: [ 0] [  239/  468] time: 52.1835, train_loss: 0.10170273, train_accuracy: 0.9844, test_Accuracy: 0.9448\n",
      "Epoch: [ 0] [  240/  468] time: 52.3630, train_loss: 0.12257228, train_accuracy: 0.9531, test_Accuracy: 0.9430\n",
      "Epoch: [ 0] [  241/  468] time: 52.5515, train_loss: 0.14245857, train_accuracy: 0.9688, test_Accuracy: 0.9397\n",
      "Epoch: [ 0] [  242/  468] time: 52.7689, train_loss: 0.12919275, train_accuracy: 0.9688, test_Accuracy: 0.9385\n",
      "Epoch: [ 0] [  243/  468] time: 52.9684, train_loss: 0.13932258, train_accuracy: 0.9609, test_Accuracy: 0.9393\n",
      "Epoch: [ 0] [  244/  468] time: 53.1628, train_loss: 0.21781586, train_accuracy: 0.9141, test_Accuracy: 0.9416\n",
      "Epoch: [ 0] [  245/  468] time: 53.3603, train_loss: 0.25053519, train_accuracy: 0.9297, test_Accuracy: 0.9422\n",
      "Epoch: [ 0] [  246/  468] time: 53.5837, train_loss: 0.20949662, train_accuracy: 0.9219, test_Accuracy: 0.9406\n",
      "Epoch: [ 0] [  247/  468] time: 53.7642, train_loss: 0.21334890, train_accuracy: 0.9297, test_Accuracy: 0.9381\n",
      "Epoch: [ 0] [  248/  468] time: 53.9497, train_loss: 0.17998458, train_accuracy: 0.9688, test_Accuracy: 0.9364\n",
      "Epoch: [ 0] [  249/  468] time: 54.1961, train_loss: 0.14830236, train_accuracy: 0.9609, test_Accuracy: 0.9382\n",
      "Epoch: [ 0] [  250/  468] time: 54.4275, train_loss: 0.17782713, train_accuracy: 0.9375, test_Accuracy: 0.9403\n",
      "Epoch: [ 0] [  251/  468] time: 54.6618, train_loss: 0.15859468, train_accuracy: 0.9609, test_Accuracy: 0.9441\n",
      "Epoch: [ 0] [  252/  468] time: 54.8952, train_loss: 0.10897644, train_accuracy: 0.9844, test_Accuracy: 0.9447\n",
      "Epoch: [ 0] [  253/  468] time: 55.1146, train_loss: 0.13882375, train_accuracy: 0.9609, test_Accuracy: 0.9447\n",
      "Epoch: [ 0] [  254/  468] time: 55.4467, train_loss: 0.19478697, train_accuracy: 0.9531, test_Accuracy: 0.9436\n",
      "Epoch: [ 0] [  255/  468] time: 55.7210, train_loss: 0.18751051, train_accuracy: 0.9219, test_Accuracy: 0.9440\n",
      "Epoch: [ 0] [  256/  468] time: 55.9434, train_loss: 0.24667500, train_accuracy: 0.9375, test_Accuracy: 0.9461\n",
      "Epoch: [ 0] [  257/  468] time: 56.2117, train_loss: 0.20029943, train_accuracy: 0.9531, test_Accuracy: 0.9467\n",
      "Epoch: [ 0] [  258/  468] time: 56.4072, train_loss: 0.20862807, train_accuracy: 0.9375, test_Accuracy: 0.9479\n",
      "Epoch: [ 0] [  259/  468] time: 56.5996, train_loss: 0.18456256, train_accuracy: 0.9531, test_Accuracy: 0.9492\n",
      "Epoch: [ 0] [  260/  468] time: 56.7941, train_loss: 0.11634973, train_accuracy: 0.9609, test_Accuracy: 0.9490\n",
      "Epoch: [ 0] [  261/  468] time: 57.0215, train_loss: 0.25479597, train_accuracy: 0.9375, test_Accuracy: 0.9488\n",
      "Epoch: [ 0] [  262/  468] time: 57.2310, train_loss: 0.12392262, train_accuracy: 0.9844, test_Accuracy: 0.9483\n",
      "Epoch: [ 0] [  263/  468] time: 57.4394, train_loss: 0.19684878, train_accuracy: 0.9453, test_Accuracy: 0.9476\n",
      "Epoch: [ 0] [  264/  468] time: 57.6588, train_loss: 0.16292861, train_accuracy: 0.9531, test_Accuracy: 0.9465\n",
      "Epoch: [ 0] [  265/  468] time: 57.9710, train_loss: 0.16218618, train_accuracy: 0.9609, test_Accuracy: 0.9465\n",
      "Epoch: [ 0] [  266/  468] time: 58.2373, train_loss: 0.40214279, train_accuracy: 0.8594, test_Accuracy: 0.9475\n",
      "Epoch: [ 0] [  267/  468] time: 58.4936, train_loss: 0.38993225, train_accuracy: 0.8828, test_Accuracy: 0.9472\n",
      "Epoch: [ 0] [  268/  468] time: 58.7469, train_loss: 0.26499438, train_accuracy: 0.9219, test_Accuracy: 0.9474\n",
      "Epoch: [ 0] [  269/  468] time: 58.9663, train_loss: 0.21672043, train_accuracy: 0.9531, test_Accuracy: 0.9483\n",
      "Epoch: [ 0] [  270/  468] time: 59.1708, train_loss: 0.19207492, train_accuracy: 0.9375, test_Accuracy: 0.9479\n",
      "Epoch: [ 0] [  271/  468] time: 59.3643, train_loss: 0.13300875, train_accuracy: 0.9609, test_Accuracy: 0.9478\n",
      "Epoch: [ 0] [  272/  468] time: 59.5398, train_loss: 0.30070937, train_accuracy: 0.9375, test_Accuracy: 0.9480\n",
      "Epoch: [ 0] [  273/  468] time: 59.7313, train_loss: 0.18266203, train_accuracy: 0.9375, test_Accuracy: 0.9478\n",
      "Epoch: [ 0] [  274/  468] time: 59.9238, train_loss: 0.15984027, train_accuracy: 0.9609, test_Accuracy: 0.9468\n",
      "Epoch: [ 0] [  275/  468] time: 60.1063, train_loss: 0.21227598, train_accuracy: 0.9453, test_Accuracy: 0.9474\n",
      "Epoch: [ 0] [  276/  468] time: 60.2778, train_loss: 0.20443706, train_accuracy: 0.9531, test_Accuracy: 0.9491\n",
      "Epoch: [ 0] [  277/  468] time: 60.4454, train_loss: 0.15440464, train_accuracy: 0.9453, test_Accuracy: 0.9493\n",
      "Epoch: [ 0] [  278/  468] time: 60.6698, train_loss: 0.27569488, train_accuracy: 0.9453, test_Accuracy: 0.9499\n",
      "Epoch: [ 0] [  279/  468] time: 60.8932, train_loss: 0.18379158, train_accuracy: 0.9453, test_Accuracy: 0.9493\n",
      "Epoch: [ 0] [  280/  468] time: 61.1255, train_loss: 0.15486850, train_accuracy: 0.9688, test_Accuracy: 0.9495\n",
      "Epoch: [ 0] [  281/  468] time: 61.3878, train_loss: 0.21223259, train_accuracy: 0.9375, test_Accuracy: 0.9492\n",
      "Epoch: [ 0] [  282/  468] time: 61.6332, train_loss: 0.26240501, train_accuracy: 0.9219, test_Accuracy: 0.9491\n",
      "Epoch: [ 0] [  283/  468] time: 61.8336, train_loss: 0.13704655, train_accuracy: 0.9766, test_Accuracy: 0.9477\n",
      "Epoch: [ 0] [  284/  468] time: 62.0521, train_loss: 0.14571285, train_accuracy: 0.9688, test_Accuracy: 0.9472\n",
      "Epoch: [ 0] [  285/  468] time: 62.2844, train_loss: 0.17017174, train_accuracy: 0.9688, test_Accuracy: 0.9465\n",
      "Epoch: [ 0] [  286/  468] time: 62.5019, train_loss: 0.17712602, train_accuracy: 0.9609, test_Accuracy: 0.9473\n",
      "Epoch: [ 0] [  287/  468] time: 62.6953, train_loss: 0.08764905, train_accuracy: 0.9766, test_Accuracy: 0.9470\n",
      "Epoch: [ 0] [  288/  468] time: 62.9277, train_loss: 0.18108398, train_accuracy: 0.9531, test_Accuracy: 0.9478\n",
      "Epoch: [ 0] [  289/  468] time: 63.1541, train_loss: 0.22688746, train_accuracy: 0.9531, test_Accuracy: 0.9490\n",
      "Epoch: [ 0] [  290/  468] time: 63.3466, train_loss: 0.16212112, train_accuracy: 0.9531, test_Accuracy: 0.9503\n",
      "Epoch: [ 0] [  291/  468] time: 63.5431, train_loss: 0.15124145, train_accuracy: 0.9609, test_Accuracy: 0.9515\n",
      "Epoch: [ 0] [  292/  468] time: 63.7615, train_loss: 0.13903776, train_accuracy: 0.9766, test_Accuracy: 0.9504\n",
      "Epoch: [ 0] [  293/  468] time: 63.9640, train_loss: 0.18487599, train_accuracy: 0.9297, test_Accuracy: 0.9517\n",
      "Epoch: [ 0] [  294/  468] time: 64.1515, train_loss: 0.22035624, train_accuracy: 0.9219, test_Accuracy: 0.9524\n",
      "Epoch: [ 0] [  295/  468] time: 64.3409, train_loss: 0.10389019, train_accuracy: 0.9531, test_Accuracy: 0.9513\n",
      "Epoch: [ 0] [  296/  468] time: 64.5614, train_loss: 0.13865620, train_accuracy: 0.9453, test_Accuracy: 0.9514\n",
      "Epoch: [ 0] [  297/  468] time: 64.7648, train_loss: 0.26546121, train_accuracy: 0.9297, test_Accuracy: 0.9505\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [ 0] [  298/  468] time: 64.9603, train_loss: 0.14833972, train_accuracy: 0.9531, test_Accuracy: 0.9483\n",
      "Epoch: [ 0] [  299/  468] time: 65.1767, train_loss: 0.16294579, train_accuracy: 0.9531, test_Accuracy: 0.9462\n",
      "Epoch: [ 0] [  300/  468] time: 65.4201, train_loss: 0.16175169, train_accuracy: 0.9375, test_Accuracy: 0.9449\n",
      "Epoch: [ 0] [  301/  468] time: 65.6175, train_loss: 0.23955519, train_accuracy: 0.9297, test_Accuracy: 0.9449\n",
      "Epoch: [ 0] [  302/  468] time: 65.8150, train_loss: 0.11767542, train_accuracy: 0.9766, test_Accuracy: 0.9454\n",
      "Epoch: [ 0] [  303/  468] time: 66.0195, train_loss: 0.17728692, train_accuracy: 0.9219, test_Accuracy: 0.9457\n",
      "Epoch: [ 0] [  304/  468] time: 66.2239, train_loss: 0.08784426, train_accuracy: 0.9844, test_Accuracy: 0.9482\n",
      "Epoch: [ 0] [  305/  468] time: 66.4184, train_loss: 0.15665667, train_accuracy: 0.9531, test_Accuracy: 0.9498\n",
      "Epoch: [ 0] [  306/  468] time: 66.6109, train_loss: 0.17986161, train_accuracy: 0.9453, test_Accuracy: 0.9506\n",
      "Epoch: [ 0] [  307/  468] time: 66.7964, train_loss: 0.10429516, train_accuracy: 0.9688, test_Accuracy: 0.9512\n",
      "Epoch: [ 0] [  308/  468] time: 67.0118, train_loss: 0.18789040, train_accuracy: 0.9531, test_Accuracy: 0.9511\n",
      "Epoch: [ 0] [  309/  468] time: 67.2242, train_loss: 0.15670964, train_accuracy: 0.9453, test_Accuracy: 0.9515\n",
      "Epoch: [ 0] [  310/  468] time: 67.4197, train_loss: 0.27164274, train_accuracy: 0.9375, test_Accuracy: 0.9507\n",
      "Epoch: [ 0] [  311/  468] time: 67.6232, train_loss: 0.13805710, train_accuracy: 0.9688, test_Accuracy: 0.9486\n",
      "Epoch: [ 0] [  312/  468] time: 67.8904, train_loss: 0.15558115, train_accuracy: 0.9688, test_Accuracy: 0.9467\n",
      "Epoch: [ 0] [  313/  468] time: 68.1697, train_loss: 0.15620831, train_accuracy: 0.9609, test_Accuracy: 0.9440\n",
      "Epoch: [ 0] [  314/  468] time: 68.4629, train_loss: 0.17048782, train_accuracy: 0.9297, test_Accuracy: 0.9431\n",
      "Epoch: [ 0] [  315/  468] time: 68.7741, train_loss: 0.17916825, train_accuracy: 0.9531, test_Accuracy: 0.9418\n",
      "Epoch: [ 0] [  316/  468] time: 68.9855, train_loss: 0.15056920, train_accuracy: 0.9609, test_Accuracy: 0.9440\n",
      "Epoch: [ 0] [  317/  468] time: 69.1950, train_loss: 0.16415346, train_accuracy: 0.9453, test_Accuracy: 0.9451\n",
      "Epoch: [ 0] [  318/  468] time: 69.3954, train_loss: 0.12412199, train_accuracy: 0.9609, test_Accuracy: 0.9457\n",
      "Epoch: [ 0] [  319/  468] time: 69.5879, train_loss: 0.14571916, train_accuracy: 0.9609, test_Accuracy: 0.9478\n",
      "Epoch: [ 0] [  320/  468] time: 69.7764, train_loss: 0.17353272, train_accuracy: 0.9453, test_Accuracy: 0.9489\n",
      "Epoch: [ 0] [  321/  468] time: 69.9649, train_loss: 0.21222720, train_accuracy: 0.9531, test_Accuracy: 0.9495\n",
      "Epoch: [ 0] [  322/  468] time: 70.1554, train_loss: 0.07513399, train_accuracy: 0.9688, test_Accuracy: 0.9485\n",
      "Epoch: [ 0] [  323/  468] time: 70.3299, train_loss: 0.08484484, train_accuracy: 0.9766, test_Accuracy: 0.9488\n",
      "Epoch: [ 0] [  324/  468] time: 70.5045, train_loss: 0.15572321, train_accuracy: 0.9688, test_Accuracy: 0.9481\n",
      "Epoch: [ 0] [  325/  468] time: 70.6999, train_loss: 0.15851679, train_accuracy: 0.9531, test_Accuracy: 0.9471\n",
      "Epoch: [ 0] [  326/  468] time: 70.9253, train_loss: 0.20702997, train_accuracy: 0.9453, test_Accuracy: 0.9473\n",
      "Epoch: [ 0] [  327/  468] time: 71.1647, train_loss: 0.25276947, train_accuracy: 0.9453, test_Accuracy: 0.9481\n",
      "Epoch: [ 0] [  328/  468] time: 71.4170, train_loss: 0.18030038, train_accuracy: 0.9531, test_Accuracy: 0.9481\n",
      "Epoch: [ 0] [  329/  468] time: 71.6285, train_loss: 0.12254199, train_accuracy: 0.9766, test_Accuracy: 0.9484\n",
      "Epoch: [ 0] [  330/  468] time: 71.8229, train_loss: 0.25289935, train_accuracy: 0.9375, test_Accuracy: 0.9504\n",
      "Epoch: [ 0] [  331/  468] time: 72.0344, train_loss: 0.20052825, train_accuracy: 0.9219, test_Accuracy: 0.9512\n",
      "Epoch: [ 0] [  332/  468] time: 72.2189, train_loss: 0.09673398, train_accuracy: 0.9766, test_Accuracy: 0.9514\n",
      "Epoch: [ 0] [  333/  468] time: 72.4144, train_loss: 0.18671389, train_accuracy: 0.9219, test_Accuracy: 0.9513\n",
      "Epoch: [ 0] [  334/  468] time: 72.5869, train_loss: 0.12458614, train_accuracy: 0.9688, test_Accuracy: 0.9497\n",
      "Epoch: [ 0] [  335/  468] time: 72.7594, train_loss: 0.15409368, train_accuracy: 0.9297, test_Accuracy: 0.9477\n",
      "Epoch: [ 0] [  336/  468] time: 72.9589, train_loss: 0.09750119, train_accuracy: 0.9609, test_Accuracy: 0.9476\n",
      "Epoch: [ 0] [  337/  468] time: 73.1574, train_loss: 0.16905016, train_accuracy: 0.9531, test_Accuracy: 0.9471\n",
      "Epoch: [ 0] [  338/  468] time: 73.3419, train_loss: 0.24093828, train_accuracy: 0.9141, test_Accuracy: 0.9484\n",
      "Epoch: [ 0] [  339/  468] time: 73.5373, train_loss: 0.24403653, train_accuracy: 0.9062, test_Accuracy: 0.9516\n",
      "Epoch: [ 0] [  340/  468] time: 73.7258, train_loss: 0.29190207, train_accuracy: 0.9219, test_Accuracy: 0.9535\n",
      "Epoch: [ 0] [  341/  468] time: 73.9343, train_loss: 0.21227846, train_accuracy: 0.9297, test_Accuracy: 0.9535\n",
      "Epoch: [ 0] [  342/  468] time: 74.1258, train_loss: 0.19330871, train_accuracy: 0.9453, test_Accuracy: 0.9537\n",
      "Epoch: [ 0] [  343/  468] time: 74.3123, train_loss: 0.11892238, train_accuracy: 0.9766, test_Accuracy: 0.9522\n",
      "Epoch: [ 0] [  344/  468] time: 74.5806, train_loss: 0.14845948, train_accuracy: 0.9688, test_Accuracy: 0.9517\n",
      "Epoch: [ 0] [  345/  468] time: 74.8638, train_loss: 0.16297659, train_accuracy: 0.9531, test_Accuracy: 0.9528\n",
      "Epoch: [ 0] [  346/  468] time: 75.1580, train_loss: 0.12609690, train_accuracy: 0.9453, test_Accuracy: 0.9546\n",
      "Epoch: [ 0] [  347/  468] time: 75.4582, train_loss: 0.19159910, train_accuracy: 0.9531, test_Accuracy: 0.9549\n",
      "Epoch: [ 0] [  348/  468] time: 75.6976, train_loss: 0.16686814, train_accuracy: 0.9531, test_Accuracy: 0.9538\n",
      "Epoch: [ 0] [  349/  468] time: 75.9260, train_loss: 0.14607447, train_accuracy: 0.9609, test_Accuracy: 0.9535\n",
      "Epoch: [ 0] [  350/  468] time: 76.2551, train_loss: 0.10542786, train_accuracy: 0.9688, test_Accuracy: 0.9530\n",
      "Epoch: [ 0] [  351/  468] time: 76.5613, train_loss: 0.17415258, train_accuracy: 0.9375, test_Accuracy: 0.9538\n",
      "Epoch: [ 0] [  352/  468] time: 76.8495, train_loss: 0.17360279, train_accuracy: 0.9453, test_Accuracy: 0.9538\n",
      "Epoch: [ 0] [  353/  468] time: 77.1666, train_loss: 0.14569294, train_accuracy: 0.9688, test_Accuracy: 0.9531\n",
      "Epoch: [ 0] [  354/  468] time: 77.4170, train_loss: 0.16882363, train_accuracy: 0.9375, test_Accuracy: 0.9534\n",
      "Epoch: [ 0] [  355/  468] time: 77.6803, train_loss: 0.15008907, train_accuracy: 0.9375, test_Accuracy: 0.9531\n",
      "Epoch: [ 0] [  356/  468] time: 77.9127, train_loss: 0.14743698, train_accuracy: 0.9609, test_Accuracy: 0.9538\n",
      "Epoch: [ 0] [  357/  468] time: 78.1241, train_loss: 0.12375845, train_accuracy: 0.9609, test_Accuracy: 0.9532\n",
      "Epoch: [ 0] [  358/  468] time: 78.3654, train_loss: 0.13355158, train_accuracy: 0.9453, test_Accuracy: 0.9540\n",
      "Epoch: [ 0] [  359/  468] time: 78.6198, train_loss: 0.09655734, train_accuracy: 0.9766, test_Accuracy: 0.9557\n",
      "Epoch: [ 0] [  360/  468] time: 78.8083, train_loss: 0.24781582, train_accuracy: 0.9453, test_Accuracy: 0.9561\n",
      "Epoch: [ 0] [  361/  468] time: 79.0137, train_loss: 0.15261455, train_accuracy: 0.9609, test_Accuracy: 0.9542\n",
      "Epoch: [ 0] [  362/  468] time: 79.2192, train_loss: 0.10995415, train_accuracy: 0.9609, test_Accuracy: 0.9544\n",
      "Epoch: [ 0] [  363/  468] time: 79.4176, train_loss: 0.11107479, train_accuracy: 0.9688, test_Accuracy: 0.9541\n",
      "Epoch: [ 0] [  364/  468] time: 79.6001, train_loss: 0.08055730, train_accuracy: 0.9922, test_Accuracy: 0.9524\n",
      "Epoch: [ 0] [  365/  468] time: 79.7836, train_loss: 0.16761598, train_accuracy: 0.9688, test_Accuracy: 0.9523\n",
      "Epoch: [ 0] [  366/  468] time: 80.0380, train_loss: 0.27589908, train_accuracy: 0.9141, test_Accuracy: 0.9539\n",
      "Epoch: [ 0] [  367/  468] time: 80.2813, train_loss: 0.19045711, train_accuracy: 0.9219, test_Accuracy: 0.9540\n",
      "Epoch: [ 0] [  368/  468] time: 80.5536, train_loss: 0.14547712, train_accuracy: 0.9531, test_Accuracy: 0.9515\n",
      "Epoch: [ 0] [  369/  468] time: 80.8169, train_loss: 0.31174278, train_accuracy: 0.9141, test_Accuracy: 0.9495\n",
      "Epoch: [ 0] [  370/  468] time: 81.0044, train_loss: 0.08008161, train_accuracy: 0.9844, test_Accuracy: 0.9475\n",
      "Epoch: [ 0] [  371/  468] time: 81.1849, train_loss: 0.22344136, train_accuracy: 0.9297, test_Accuracy: 0.9471\n",
      "Epoch: [ 0] [  372/  468] time: 81.3754, train_loss: 0.23770492, train_accuracy: 0.9453, test_Accuracy: 0.9486\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [ 0] [  373/  468] time: 81.5559, train_loss: 0.12366275, train_accuracy: 0.9531, test_Accuracy: 0.9502\n",
      "Epoch: [ 0] [  374/  468] time: 81.7394, train_loss: 0.20562696, train_accuracy: 0.9453, test_Accuracy: 0.9530\n",
      "Epoch: [ 0] [  375/  468] time: 81.9060, train_loss: 0.10571376, train_accuracy: 0.9766, test_Accuracy: 0.9536\n",
      "Epoch: [ 0] [  376/  468] time: 82.0985, train_loss: 0.13228717, train_accuracy: 0.9375, test_Accuracy: 0.9527\n",
      "Epoch: [ 0] [  377/  468] time: 82.2820, train_loss: 0.25934079, train_accuracy: 0.9297, test_Accuracy: 0.9504\n",
      "Epoch: [ 0] [  378/  468] time: 82.4585, train_loss: 0.11433642, train_accuracy: 0.9531, test_Accuracy: 0.9493\n",
      "Epoch: [ 0] [  379/  468] time: 82.6360, train_loss: 0.25603846, train_accuracy: 0.9375, test_Accuracy: 0.9470\n",
      "Epoch: [ 0] [  380/  468] time: 82.8554, train_loss: 0.16462746, train_accuracy: 0.9609, test_Accuracy: 0.9462\n",
      "Epoch: [ 0] [  381/  468] time: 83.1506, train_loss: 0.11756428, train_accuracy: 0.9688, test_Accuracy: 0.9450\n",
      "Epoch: [ 0] [  382/  468] time: 83.4518, train_loss: 0.15862171, train_accuracy: 0.9453, test_Accuracy: 0.9444\n",
      "Epoch: [ 0] [  383/  468] time: 83.7171, train_loss: 0.20180663, train_accuracy: 0.9219, test_Accuracy: 0.9473\n",
      "Epoch: [ 0] [  384/  468] time: 83.9814, train_loss: 0.12960155, train_accuracy: 0.9688, test_Accuracy: 0.9499\n",
      "Epoch: [ 0] [  385/  468] time: 84.1909, train_loss: 0.23487133, train_accuracy: 0.9375, test_Accuracy: 0.9522\n",
      "Epoch: [ 0] [  386/  468] time: 84.5399, train_loss: 0.11772513, train_accuracy: 0.9609, test_Accuracy: 0.9518\n",
      "Epoch: [ 0] [  387/  468] time: 84.7992, train_loss: 0.17653070, train_accuracy: 0.9531, test_Accuracy: 0.9520\n",
      "Epoch: [ 0] [  388/  468] time: 85.1323, train_loss: 0.19725576, train_accuracy: 0.9453, test_Accuracy: 0.9501\n",
      "Epoch: [ 0] [  389/  468] time: 85.3677, train_loss: 0.30563176, train_accuracy: 0.8906, test_Accuracy: 0.9504\n",
      "Epoch: [ 0] [  390/  468] time: 85.5622, train_loss: 0.10926918, train_accuracy: 0.9766, test_Accuracy: 0.9520\n",
      "Epoch: [ 0] [  391/  468] time: 85.7637, train_loss: 0.20061027, train_accuracy: 0.9453, test_Accuracy: 0.9547\n",
      "Epoch: [ 0] [  392/  468] time: 85.9691, train_loss: 0.12484649, train_accuracy: 0.9766, test_Accuracy: 0.9556\n",
      "Epoch: [ 0] [  393/  468] time: 86.1815, train_loss: 0.14548409, train_accuracy: 0.9531, test_Accuracy: 0.9537\n",
      "Epoch: [ 0] [  394/  468] time: 86.3690, train_loss: 0.16337742, train_accuracy: 0.9297, test_Accuracy: 0.9535\n",
      "Epoch: [ 0] [  395/  468] time: 86.5635, train_loss: 0.09490243, train_accuracy: 0.9766, test_Accuracy: 0.9526\n",
      "Epoch: [ 0] [  396/  468] time: 86.8109, train_loss: 0.08507378, train_accuracy: 0.9844, test_Accuracy: 0.9529\n",
      "Epoch: [ 0] [  397/  468] time: 87.0263, train_loss: 0.08116265, train_accuracy: 0.9844, test_Accuracy: 0.9534\n",
      "Epoch: [ 0] [  398/  468] time: 87.2078, train_loss: 0.13087578, train_accuracy: 0.9609, test_Accuracy: 0.9536\n",
      "Epoch: [ 0] [  399/  468] time: 87.4152, train_loss: 0.11726468, train_accuracy: 0.9766, test_Accuracy: 0.9550\n",
      "Epoch: [ 0] [  400/  468] time: 87.6167, train_loss: 0.13526377, train_accuracy: 0.9688, test_Accuracy: 0.9550\n",
      "Epoch: [ 0] [  401/  468] time: 87.8122, train_loss: 0.13098171, train_accuracy: 0.9609, test_Accuracy: 0.9555\n",
      "Epoch: [ 0] [  402/  468] time: 87.9987, train_loss: 0.14634612, train_accuracy: 0.9688, test_Accuracy: 0.9535\n",
      "Epoch: [ 0] [  403/  468] time: 88.2510, train_loss: 0.18218949, train_accuracy: 0.9297, test_Accuracy: 0.9530\n",
      "Epoch: [ 0] [  404/  468] time: 88.5352, train_loss: 0.06404848, train_accuracy: 1.0000, test_Accuracy: 0.9526\n",
      "Epoch: [ 0] [  405/  468] time: 88.7507, train_loss: 0.11368668, train_accuracy: 0.9453, test_Accuracy: 0.9530\n",
      "Epoch: [ 0] [  406/  468] time: 88.9701, train_loss: 0.16540688, train_accuracy: 0.9453, test_Accuracy: 0.9559\n",
      "Epoch: [ 0] [  407/  468] time: 89.1985, train_loss: 0.09218720, train_accuracy: 0.9688, test_Accuracy: 0.9579\n",
      "Epoch: [ 0] [  408/  468] time: 89.4678, train_loss: 0.08089187, train_accuracy: 0.9766, test_Accuracy: 0.9588\n",
      "Epoch: [ 0] [  409/  468] time: 89.7819, train_loss: 0.09400706, train_accuracy: 0.9688, test_Accuracy: 0.9589\n",
      "Epoch: [ 0] [  410/  468] time: 90.0791, train_loss: 0.20608933, train_accuracy: 0.9453, test_Accuracy: 0.9585\n",
      "Epoch: [ 0] [  411/  468] time: 90.3913, train_loss: 0.08059357, train_accuracy: 0.9766, test_Accuracy: 0.9585\n",
      "Epoch: [ 0] [  412/  468] time: 90.7693, train_loss: 0.15349999, train_accuracy: 0.9531, test_Accuracy: 0.9567\n",
      "Epoch: [ 0] [  413/  468] time: 91.2101, train_loss: 0.08816801, train_accuracy: 0.9688, test_Accuracy: 0.9555\n",
      "Epoch: [ 0] [  414/  468] time: 91.5871, train_loss: 0.23725523, train_accuracy: 0.9531, test_Accuracy: 0.9550\n",
      "Epoch: [ 0] [  415/  468] time: 91.9591, train_loss: 0.19518417, train_accuracy: 0.9453, test_Accuracy: 0.9557\n",
      "Epoch: [ 0] [  416/  468] time: 92.2503, train_loss: 0.18610464, train_accuracy: 0.9375, test_Accuracy: 0.9569\n",
      "Epoch: [ 0] [  417/  468] time: 92.5535, train_loss: 0.15868935, train_accuracy: 0.9609, test_Accuracy: 0.9570\n",
      "Epoch: [ 0] [  418/  468] time: 92.8477, train_loss: 0.17908023, train_accuracy: 0.9375, test_Accuracy: 0.9583\n",
      "Epoch: [ 0] [  419/  468] time: 93.1100, train_loss: 0.19004807, train_accuracy: 0.9375, test_Accuracy: 0.9604\n",
      "Epoch: [ 0] [  420/  468] time: 93.3873, train_loss: 0.15000482, train_accuracy: 0.9531, test_Accuracy: 0.9607\n",
      "Epoch: [ 0] [  421/  468] time: 93.5967, train_loss: 0.16138330, train_accuracy: 0.9297, test_Accuracy: 0.9608\n",
      "Epoch: [ 0] [  422/  468] time: 93.8161, train_loss: 0.19156221, train_accuracy: 0.9531, test_Accuracy: 0.9618\n",
      "Epoch: [ 0] [  423/  468] time: 94.0884, train_loss: 0.11711767, train_accuracy: 0.9531, test_Accuracy: 0.9602\n",
      "Epoch: [ 0] [  424/  468] time: 94.4335, train_loss: 0.09217168, train_accuracy: 0.9688, test_Accuracy: 0.9585\n",
      "Epoch: [ 0] [  425/  468] time: 94.8055, train_loss: 0.09158284, train_accuracy: 0.9766, test_Accuracy: 0.9577\n",
      "Epoch: [ 0] [  426/  468] time: 95.1246, train_loss: 0.14356339, train_accuracy: 0.9453, test_Accuracy: 0.9564\n",
      "Epoch: [ 0] [  427/  468] time: 95.3600, train_loss: 0.06341392, train_accuracy: 0.9844, test_Accuracy: 0.9559\n",
      "Epoch: [ 0] [  428/  468] time: 95.5794, train_loss: 0.14057600, train_accuracy: 0.9453, test_Accuracy: 0.9567\n",
      "Epoch: [ 0] [  429/  468] time: 95.7968, train_loss: 0.06760108, train_accuracy: 0.9766, test_Accuracy: 0.9576\n",
      "Epoch: [ 0] [  430/  468] time: 96.0083, train_loss: 0.07039250, train_accuracy: 0.9844, test_Accuracy: 0.9590\n",
      "Epoch: [ 0] [  431/  468] time: 96.2247, train_loss: 0.22636662, train_accuracy: 0.9688, test_Accuracy: 0.9597\n",
      "Epoch: [ 0] [  432/  468] time: 96.4431, train_loss: 0.20718822, train_accuracy: 0.9531, test_Accuracy: 0.9597\n",
      "Epoch: [ 0] [  433/  468] time: 96.7194, train_loss: 0.12881795, train_accuracy: 0.9766, test_Accuracy: 0.9599\n",
      "Epoch: [ 0] [  434/  468] time: 96.9517, train_loss: 0.17091076, train_accuracy: 0.9531, test_Accuracy: 0.9575\n",
      "Epoch: [ 0] [  435/  468] time: 97.1672, train_loss: 0.12879655, train_accuracy: 0.9688, test_Accuracy: 0.9557\n",
      "Epoch: [ 0] [  436/  468] time: 97.3916, train_loss: 0.14638124, train_accuracy: 0.9531, test_Accuracy: 0.9516\n",
      "Epoch: [ 0] [  437/  468] time: 97.5970, train_loss: 0.17038360, train_accuracy: 0.9453, test_Accuracy: 0.9494\n",
      "Epoch: [ 0] [  438/  468] time: 97.8942, train_loss: 0.15950570, train_accuracy: 0.9531, test_Accuracy: 0.9495\n",
      "Epoch: [ 0] [  439/  468] time: 98.1366, train_loss: 0.22602020, train_accuracy: 0.9141, test_Accuracy: 0.9514\n",
      "Epoch: [ 0] [  440/  468] time: 98.3231, train_loss: 0.20618077, train_accuracy: 0.9297, test_Accuracy: 0.9548\n",
      "Epoch: [ 0] [  441/  468] time: 98.5146, train_loss: 0.18369497, train_accuracy: 0.9531, test_Accuracy: 0.9581\n",
      "Epoch: [ 0] [  442/  468] time: 98.7090, train_loss: 0.09464790, train_accuracy: 0.9688, test_Accuracy: 0.9589\n",
      "Epoch: [ 0] [  443/  468] time: 98.9055, train_loss: 0.11069962, train_accuracy: 0.9609, test_Accuracy: 0.9579\n",
      "Epoch: [ 0] [  444/  468] time: 99.1538, train_loss: 0.08618344, train_accuracy: 0.9766, test_Accuracy: 0.9546\n",
      "Epoch: [ 0] [  445/  468] time: 99.3852, train_loss: 0.10831747, train_accuracy: 0.9688, test_Accuracy: 0.9511\n",
      "Epoch: [ 0] [  446/  468] time: 99.6455, train_loss: 0.20375749, train_accuracy: 0.9297, test_Accuracy: 0.9494\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [ 0] [  447/  468] time: 99.8989, train_loss: 0.12970647, train_accuracy: 0.9531, test_Accuracy: 0.9502\n",
      "Epoch: [ 0] [  448/  468] time: 100.1901, train_loss: 0.10099588, train_accuracy: 0.9688, test_Accuracy: 0.9517\n",
      "Epoch: [ 0] [  449/  468] time: 100.3856, train_loss: 0.11357155, train_accuracy: 0.9688, test_Accuracy: 0.9544\n",
      "Epoch: [ 0] [  450/  468] time: 100.5950, train_loss: 0.11801068, train_accuracy: 0.9844, test_Accuracy: 0.9559\n",
      "Epoch: [ 0] [  451/  468] time: 100.7925, train_loss: 0.10552384, train_accuracy: 0.9766, test_Accuracy: 0.9580\n",
      "Epoch: [ 0] [  452/  468] time: 100.9800, train_loss: 0.19256677, train_accuracy: 0.9453, test_Accuracy: 0.9595\n",
      "Epoch: [ 0] [  453/  468] time: 101.2153, train_loss: 0.14474468, train_accuracy: 0.9531, test_Accuracy: 0.9585\n",
      "Epoch: [ 0] [  454/  468] time: 101.4118, train_loss: 0.15418515, train_accuracy: 0.9609, test_Accuracy: 0.9560\n",
      "Epoch: [ 0] [  455/  468] time: 101.6133, train_loss: 0.24077219, train_accuracy: 0.9219, test_Accuracy: 0.9549\n",
      "Epoch: [ 0] [  456/  468] time: 101.8087, train_loss: 0.06127919, train_accuracy: 0.9922, test_Accuracy: 0.9517\n",
      "Epoch: [ 0] [  457/  468] time: 102.0082, train_loss: 0.12892991, train_accuracy: 0.9766, test_Accuracy: 0.9504\n",
      "Epoch: [ 0] [  458/  468] time: 102.1997, train_loss: 0.26092318, train_accuracy: 0.9141, test_Accuracy: 0.9504\n",
      "Epoch: [ 0] [  459/  468] time: 102.3862, train_loss: 0.07898837, train_accuracy: 0.9688, test_Accuracy: 0.9522\n",
      "Epoch: [ 0] [  460/  468] time: 102.5707, train_loss: 0.10289016, train_accuracy: 0.9766, test_Accuracy: 0.9540\n",
      "Epoch: [ 0] [  461/  468] time: 102.8141, train_loss: 0.13644192, train_accuracy: 0.9531, test_Accuracy: 0.9560\n",
      "Epoch: [ 0] [  462/  468] time: 103.0624, train_loss: 0.06870986, train_accuracy: 0.9844, test_Accuracy: 0.9581\n",
      "Epoch: [ 0] [  463/  468] time: 103.3466, train_loss: 0.10453942, train_accuracy: 0.9531, test_Accuracy: 0.9589\n",
      "Epoch: [ 0] [  464/  468] time: 103.6648, train_loss: 0.10517713, train_accuracy: 0.9844, test_Accuracy: 0.9590\n",
      "Epoch: [ 0] [  465/  468] time: 103.9221, train_loss: 0.05283299, train_accuracy: 0.9844, test_Accuracy: 0.9587\n",
      "Epoch: [ 0] [  466/  468] time: 104.1425, train_loss: 0.10655379, train_accuracy: 0.9531, test_Accuracy: 0.9585\n",
      "Epoch: [ 0] [  467/  468] time: 104.3639, train_loss: 0.10589507, train_accuracy: 0.9766, test_Accuracy: 0.9584\n"
     ]
    }
   ],
   "source": [
    "\n",
    "if train_flag :\n",
    "\n",
    "    checkpoint = tf.train.Checkpoint(dnn=network)\n",
    "\n",
    "    # create writer for tensorboard\n",
    "    summary_writer = tf.summary.create_file_writer(logdir=logs_dir)\n",
    "    start_time = time()\n",
    "\n",
    "    # restore check-point if it exits\n",
    "    could_load, checkpoint_counter = load(network, checkpoint_dir)    \n",
    "\n",
    "    if could_load:\n",
    "        start_epoch = (int)(checkpoint_counter / training_iterations)        \n",
    "        counter = checkpoint_counter        \n",
    "        print(\" [*] Load SUCCESS\")\n",
    "    else:\n",
    "        start_epoch = 0\n",
    "        start_iteration = 0\n",
    "        counter = 0\n",
    "        print(\" [!] Load failed...\")\n",
    "    \n",
    "    # train phase\n",
    "    with summary_writer.as_default():  # for tensorboard\n",
    "        for epoch in range(start_epoch, training_epochs):\n",
    "            for idx, (train_input, train_label) in enumerate(train_dataset):                \n",
    "                grads = grad(network, train_input, train_label)\n",
    "                optimizer.apply_gradients(grads_and_vars=zip(grads, network.variables))\n",
    "\n",
    "                train_loss = loss_fn(network, train_input, train_label)\n",
    "                train_accuracy = accuracy_fn(network, train_input, train_label)\n",
    "\n",
    "                for test_input, test_label in test_dataset:                \n",
    "                    test_accuracy = accuracy_fn(network, test_input, test_label)\n",
    "\n",
    "                tf.summary.scalar(name='train_loss', data=train_loss, step=counter)\n",
    "                tf.summary.scalar(name='train_accuracy', data=train_accuracy, step=counter)\n",
    "                tf.summary.scalar(name='test_accuracy', data=test_accuracy, step=counter)\n",
    "\n",
    "                print(\n",
    "                    \"Epoch: [%2d] [%5d/%5d] time: %4.4f, train_loss: %.8f, train_accuracy: %.4f, test_Accuracy: %.4f\" \\\n",
    "                    % (epoch, idx, training_iterations, time() - start_time, train_loss, train_accuracy,\n",
    "                       test_accuracy))\n",
    "                counter += 1\n",
    "        checkpoint.save(file_prefix=checkpoint_prefix + '-{}'.format(counter))\n",
    "        \n",
    "# test phase      \n",
    "else :\n",
    "    _, _ = load(network, checkpoint_dir)\n",
    "    for test_input, test_label in test_dataset:    \n",
    "        test_accuracy = accuracy_fn(network, test_input, test_label)\n",
    "\n",
    "    print(\"test_Accuracy: %.4f\" % (test_accuracy))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e1e5ad37",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "py38tensorflow",
   "language": "python",
   "name": "py38tensorflow"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}