Lab10/Lab10_4_JoongHoonKim.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "f4bb60b7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.5.0\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "from tensorflow.keras.utils import to_categorical\n",
    "from tensorflow.keras.datasets import mnist\n",
    "from time import time\n",
    "import os\n",
    "print(tf.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "590cdd50",
   "metadata": {},
   "outputs": [],
   "source": [
    "def load(model, checkpoint_dir):\n",
    "    print(\" [*] Reading checkpoints...\")\n",
    "\n",
    "    ckpt = tf.train.get_checkpoint_state(checkpoint_dir)\n",
    "    if ckpt :\n",
    "        ckpt_name = os.path.basename(ckpt.model_checkpoint_path)\n",
    "        checkpoint = tf.train.Checkpoint(dnn=model)\n",
    "        checkpoint.restore(save_path=os.path.join(checkpoint_dir, ckpt_name))\n",
    "        counter = int(ckpt_name.split('-')[1])\n",
    "        print(\" [*] Success to read {}\".format(ckpt_name))\n",
    "        return True, counter\n",
    "    else:\n",
    "        print(\" [*] Failed to find a checkpoint\")\n",
    "        return False, 0\n",
    "\n",
    "def check_folder(dir):\n",
    "    if not os.path.exists(dir):\n",
    "        os.makedirs(dir)\n",
    "    return dir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "86feea7e",
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_mnist() :\n",
    "    (train_data, train_labels), (test_data, test_labels) = mnist.load_data()\n",
    "    train_data = np.expand_dims(train_data, axis=-1) # [N, 28, 28] -> [N, 28, 28, 1]\n",
    "    test_data = np.expand_dims(test_data, axis=-1) # [N, 28, 28] -> [N, 28, 28, 1]\n",
    "\n",
    "    train_data, test_data = normalize(train_data, test_data)\n",
    "\n",
    "    train_labels = to_categorical(train_labels, 10) # [N,] -> [N, 10]\n",
    "    test_labels = to_categorical(test_labels, 10) # [N,] -> [N, 10]\n",
    "\n",
    "    return train_data, train_labels, test_data, test_labels\n",
    "\n",
    "def normalize(train_data, test_data):\n",
    "    train_data = train_data.astype(np.float32) / 255.0\n",
    "    test_data = test_data.astype(np.float32) / 255.0\n",
    "\n",
    "    return train_data, test_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "ec8366b2",
   "metadata": {},
   "outputs": [],
   "source": [
    "def loss_fn(model, images, labels):\n",
    "    logits = model(images, training=True)\n",
    "    loss = tf.reduce_mean(tf.keras.losses.categorical_crossentropy(y_pred=logits, y_true=labels, \n",
    "                                                                   from_logits=True))\n",
    "    return loss\n",
    "\n",
    "def accuracy_fn(model, images, labels):\n",
    "    logits = model(images, training=False)\n",
    "    prediction = tf.equal(tf.argmax(logits, -1), tf.argmax(labels, -1))\n",
    "    accuracy = tf.reduce_mean(tf.cast(prediction, tf.float32))\n",
    "    return accuracy\n",
    "\n",
    "def grad(model, images, labels):\n",
    "    with tf.GradientTape() as tape:\n",
    "        loss = loss_fn(model, images, labels)\n",
    "    return tape.gradient(loss, model.trainable_variables)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "8537ab68",
   "metadata": {},
   "outputs": [],
   "source": [
    "def flatten() :\n",
    "    return tf.keras.layers.Flatten()\n",
    "\n",
    "def dense(label_dim, weight_init) :\n",
    "    return tf.keras.layers.Dense(units=label_dim, use_bias=True, kernel_initializer=weight_init)\n",
    "\n",
    "def relu() :\n",
    "    return tf.keras.layers.Activation(tf.keras.activations.relu)\n",
    "\n",
    "def batch_norm() :\n",
    "    return tf.keras.layers.BatchNormalization()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "ff121975",
   "metadata": {},
   "outputs": [],
   "source": [
    "class create_model_class(tf.keras.Model):\n",
    "    def __init__(self, label_dim):\n",
    "        super(create_model_class, self).__init__()\n",
    "        weight_init = tf.keras.initializers.glorot_uniform()\n",
    "\n",
    "        self.model = tf.keras.Sequential()\n",
    "        self.model.add(flatten())\n",
    "\n",
    "        for i in range(4):\n",
    "            self.model.add(dense(512, weight_init))\n",
    "            self.model.add(batch_norm())\n",
    "            self.model.add(relu())\n",
    "\n",
    "        self.model.add(dense(label_dim, weight_init))\n",
    "\n",
    "    def call(self, x, training=None, mask=None):\n",
    "\n",
    "        x = self.model(x)\n",
    "\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "3cd48733",
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_model_function(label_dim) :\n",
    "    weight_init = tf.keras.initializers.glorot_uniform()\n",
    "\n",
    "    model = tf.keras.Sequential()\n",
    "    model.add(flatten())\n",
    "\n",
    "    for i in range(4) :\n",
    "        model.add(dense(512, weight_init))\n",
    "        model.add(batch_norm())\n",
    "        model.add(relu())\n",
    "\n",
    "    model.add(dense(label_dim, weight_init))\n",
    "\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "95a102d8",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\" dataset \"\"\"\n",
    "train_x, train_y, test_x, test_y = load_mnist()\n",
    "\n",
    "\"\"\" parameters \"\"\"\n",
    "learning_rate = 0.001\n",
    "batch_size = 128\n",
    "\n",
    "training_epochs = 1\n",
    "training_iterations = len(train_x) // batch_size\n",
    "\n",
    "label_dim = 10\n",
    "\n",
    "train_flag = True\n",
    "\n",
    "\"\"\" Graph Input using Dataset API \"\"\"\n",
    "train_dataset = tf.data.Dataset.from_tensor_slices((train_x, train_y)).\\\n",
    "    shuffle(buffer_size=100000).\\\n",
    "    prefetch(buffer_size=batch_size).\\\n",
    "    batch(batch_size, drop_remainder=True)\n",
    "\n",
    "test_dataset = tf.data.Dataset.from_tensor_slices((test_x, test_y)).\\\n",
    "    shuffle(buffer_size=100000).\\\n",
    "    prefetch(buffer_size=len(test_x)).\\\n",
    "    batch(len(test_x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "02bad066",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\" Model \"\"\"\n",
    "network = create_model_function(label_dim)\n",
    "\n",
    "\"\"\" Training \"\"\"\n",
    "optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate)\n",
    "\n",
    "\"\"\" Writer \"\"\"\n",
    "checkpoint_dir = 'checkpoints'\n",
    "logs_dir = 'logs'\n",
    "\n",
    "model_dir = 'nn_batchnorm'\n",
    "\n",
    "checkpoint_dir = os.path.join(checkpoint_dir, model_dir)\n",
    "check_folder(checkpoint_dir)\n",
    "checkpoint_prefix = os.path.join(checkpoint_dir, model_dir)\n",
    "logs_dir = os.path.join(logs_dir, model_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "8e65542a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " [*] Reading checkpoints...\n",
      " [*] Failed to find a checkpoint\n",
      " [!] Load failed...\n",
      "Epoch: [ 0] [    0/  468] time: 0.8118, train_loss: 0.32644224, train_accuracy: 0.3750, test_Accuracy: 0.2346\n",
      "Epoch: [ 0] [    1/  468] time: 1.1365, train_loss: 0.40490192, train_accuracy: 0.4531, test_Accuracy: 0.3428\n",
      "Epoch: [ 0] [    2/  468] time: 1.4805, train_loss: 0.36095071, train_accuracy: 0.4922, test_Accuracy: 0.4294\n",
      "Epoch: [ 0] [    3/  468] time: 1.8307, train_loss: 0.40762043, train_accuracy: 0.5312, test_Accuracy: 0.5219\n",
      "Epoch: [ 0] [    4/  468] time: 2.2376, train_loss: 0.22052749, train_accuracy: 0.5938, test_Accuracy: 0.5722\n",
      "Epoch: [ 0] [    5/  468] time: 2.5418, train_loss: 0.29408538, train_accuracy: 0.6562, test_Accuracy: 0.6100\n",
      "Epoch: [ 0] [    6/  468] time: 2.8469, train_loss: 0.36145312, train_accuracy: 0.6328, test_Accuracy: 0.6408\n",
      "Epoch: [ 0] [    7/  468] time: 3.1491, train_loss: 0.25942695, train_accuracy: 0.6562, test_Accuracy: 0.6772\n",
      "Epoch: [ 0] [    8/  468] time: 3.4503, train_loss: 0.26186782, train_accuracy: 0.7031, test_Accuracy: 0.7172\n",
      "Epoch: [ 0] [    9/  468] time: 3.7585, train_loss: 0.30693120, train_accuracy: 0.7578, test_Accuracy: 0.7528\n",
      "Epoch: [ 0] [   10/  468] time: 4.0657, train_loss: 0.19924289, train_accuracy: 0.7969, test_Accuracy: 0.7865\n",
      "Epoch: [ 0] [   11/  468] time: 4.3769, train_loss: 0.38305706, train_accuracy: 0.7578, test_Accuracy: 0.8145\n",
      "Epoch: [ 0] [   12/  468] time: 4.6910, train_loss: 0.21709323, train_accuracy: 0.8672, test_Accuracy: 0.8339\n",
      "Epoch: [ 0] [   13/  468] time: 4.9872, train_loss: 0.20449118, train_accuracy: 0.8984, test_Accuracy: 0.8481\n",
      "Epoch: [ 0] [   14/  468] time: 5.3033, train_loss: 0.13365912, train_accuracy: 0.8750, test_Accuracy: 0.8545\n",
      "Epoch: [ 0] [   15/  468] time: 5.6051, train_loss: 0.18954282, train_accuracy: 0.8594, test_Accuracy: 0.8675\n",
      "Epoch: [ 0] [   16/  468] time: 5.9003, train_loss: 0.26600727, train_accuracy: 0.8672, test_Accuracy: 0.8725\n",
      "Epoch: [ 0] [   17/  468] time: 6.2000, train_loss: 0.22197466, train_accuracy: 0.8594, test_Accuracy: 0.8780\n",
      "Epoch: [ 0] [   18/  468] time: 6.5233, train_loss: 0.11953750, train_accuracy: 0.9219, test_Accuracy: 0.8798\n",
      "Epoch: [ 0] [   19/  468] time: 6.8405, train_loss: 0.27688631, train_accuracy: 0.8750, test_Accuracy: 0.8869\n",
      "Epoch: [ 0] [   20/  468] time: 7.1766, train_loss: 0.18502368, train_accuracy: 0.9062, test_Accuracy: 0.8885\n",
      "Epoch: [ 0] [   21/  468] time: 7.4928, train_loss: 0.19909385, train_accuracy: 0.9141, test_Accuracy: 0.8895\n",
      "Epoch: [ 0] [   22/  468] time: 7.8279, train_loss: 0.24870747, train_accuracy: 0.8516, test_Accuracy: 0.8897\n",
      "Epoch: [ 0] [   23/  468] time: 8.1224, train_loss: 0.16532166, train_accuracy: 0.8828, test_Accuracy: 0.8912\n",
      "Epoch: [ 0] [   24/  468] time: 8.4286, train_loss: 0.20341000, train_accuracy: 0.9141, test_Accuracy: 0.8929\n",
      "Epoch: [ 0] [   25/  468] time: 8.7537, train_loss: 0.16008326, train_accuracy: 0.9219, test_Accuracy: 0.8958\n",
      "Epoch: [ 0] [   26/  468] time: 9.0479, train_loss: 0.24377184, train_accuracy: 0.8594, test_Accuracy: 0.8989\n",
      "Epoch: [ 0] [   27/  468] time: 9.3611, train_loss: 0.20464407, train_accuracy: 0.8906, test_Accuracy: 0.8986\n",
      "Epoch: [ 0] [   28/  468] time: 9.6553, train_loss: 0.20603895, train_accuracy: 0.8984, test_Accuracy: 0.8979\n",
      "Epoch: [ 0] [   29/  468] time: 9.9585, train_loss: 0.09827521, train_accuracy: 0.9453, test_Accuracy: 0.8967\n",
      "Epoch: [ 0] [   30/  468] time: 10.3674, train_loss: 0.14144906, train_accuracy: 0.9219, test_Accuracy: 0.8983\n",
      "Epoch: [ 0] [   31/  468] time: 10.6726, train_loss: 0.08385802, train_accuracy: 0.9531, test_Accuracy: 0.8995\n",
      "Epoch: [ 0] [   32/  468] time: 10.9808, train_loss: 0.14633314, train_accuracy: 0.9062, test_Accuracy: 0.9033\n",
      "Epoch: [ 0] [   33/  468] time: 11.2840, train_loss: 0.18482044, train_accuracy: 0.9219, test_Accuracy: 0.9071\n",
      "Epoch: [ 0] [   34/  468] time: 11.5832, train_loss: 0.18997823, train_accuracy: 0.9219, test_Accuracy: 0.9105\n",
      "Epoch: [ 0] [   35/  468] time: 11.8943, train_loss: 0.15135202, train_accuracy: 0.8984, test_Accuracy: 0.9089\n",
      "Epoch: [ 0] [   36/  468] time: 12.1965, train_loss: 0.15722045, train_accuracy: 0.8828, test_Accuracy: 0.9094\n",
      "Epoch: [ 0] [   37/  468] time: 12.5017, train_loss: 0.17762375, train_accuracy: 0.9219, test_Accuracy: 0.9081\n",
      "Epoch: [ 0] [   38/  468] time: 12.8218, train_loss: 0.11956406, train_accuracy: 0.9375, test_Accuracy: 0.9086\n",
      "Epoch: [ 0] [   39/  468] time: 13.1220, train_loss: 0.07324212, train_accuracy: 0.9219, test_Accuracy: 0.9077\n",
      "Epoch: [ 0] [   40/  468] time: 13.4402, train_loss: 0.18202046, train_accuracy: 0.9062, test_Accuracy: 0.9075\n",
      "Epoch: [ 0] [   41/  468] time: 13.7414, train_loss: 0.15982696, train_accuracy: 0.9062, test_Accuracy: 0.9076\n",
      "Epoch: [ 0] [   42/  468] time: 14.0485, train_loss: 0.14749825, train_accuracy: 0.9375, test_Accuracy: 0.9060\n",
      "Epoch: [ 0] [   43/  468] time: 14.3607, train_loss: 0.13482910, train_accuracy: 0.9141, test_Accuracy: 0.9051\n",
      "Epoch: [ 0] [   44/  468] time: 14.7317, train_loss: 0.18960783, train_accuracy: 0.8984, test_Accuracy: 0.9047\n",
      "Epoch: [ 0] [   45/  468] time: 15.0628, train_loss: 0.15570931, train_accuracy: 0.8984, test_Accuracy: 0.9077\n",
      "Epoch: [ 0] [   46/  468] time: 15.4199, train_loss: 0.15846455, train_accuracy: 0.9375, test_Accuracy: 0.9078\n",
      "Epoch: [ 0] [   47/  468] time: 15.8139, train_loss: 0.13810328, train_accuracy: 0.9375, test_Accuracy: 0.9092\n",
      "Epoch: [ 0] [   48/  468] time: 16.1909, train_loss: 0.19844203, train_accuracy: 0.9219, test_Accuracy: 0.9081\n",
      "Epoch: [ 0] [   49/  468] time: 16.5549, train_loss: 0.16143627, train_accuracy: 0.8984, test_Accuracy: 0.9098\n",
      "Epoch: [ 0] [   50/  468] time: 16.9429, train_loss: 0.12701777, train_accuracy: 0.9297, test_Accuracy: 0.9089\n",
      "Epoch: [ 0] [   51/  468] time: 17.3468, train_loss: 0.05616979, train_accuracy: 0.9609, test_Accuracy: 0.9068\n",
      "Epoch: [ 0] [   52/  468] time: 17.7357, train_loss: 0.15207946, train_accuracy: 0.9141, test_Accuracy: 0.9069\n",
      "Epoch: [ 0] [   53/  468] time: 18.1032, train_loss: 0.12171713, train_accuracy: 0.9219, test_Accuracy: 0.9086\n",
      "Epoch: [ 0] [   54/  468] time: 18.4812, train_loss: 0.07833484, train_accuracy: 0.9375, test_Accuracy: 0.9120\n",
      "Epoch: [ 0] [   55/  468] time: 18.9090, train_loss: 0.07429391, train_accuracy: 0.9531, test_Accuracy: 0.9160\n",
      "Epoch: [ 0] [   56/  468] time: 19.3469, train_loss: 0.14098498, train_accuracy: 0.9375, test_Accuracy: 0.9201\n",
      "Epoch: [ 0] [   57/  468] time: 19.7627, train_loss: 0.19292921, train_accuracy: 0.9219, test_Accuracy: 0.9233\n",
      "Epoch: [ 0] [   58/  468] time: 20.1348, train_loss: 0.08800805, train_accuracy: 0.9844, test_Accuracy: 0.9269\n",
      "Epoch: [ 0] [   59/  468] time: 20.5028, train_loss: 0.04801080, train_accuracy: 0.9688, test_Accuracy: 0.9305\n",
      "Epoch: [ 0] [   60/  468] time: 20.8828, train_loss: 0.14401637, train_accuracy: 0.9375, test_Accuracy: 0.9324\n",
      "Epoch: [ 0] [   61/  468] time: 21.2515, train_loss: 0.13615331, train_accuracy: 0.9688, test_Accuracy: 0.9347\n",
      "Epoch: [ 0] [   62/  468] time: 21.6215, train_loss: 0.05379053, train_accuracy: 0.9766, test_Accuracy: 0.9365\n",
      "Epoch: [ 0] [   63/  468] time: 22.0444, train_loss: 0.16674565, train_accuracy: 0.9375, test_Accuracy: 0.9360\n",
      "Epoch: [ 0] [   64/  468] time: 22.4164, train_loss: 0.14176413, train_accuracy: 0.9297, test_Accuracy: 0.9364\n",
      "Epoch: [ 0] [   65/  468] time: 22.7964, train_loss: 0.15396187, train_accuracy: 0.9453, test_Accuracy: 0.9357\n",
      "Epoch: [ 0] [   66/  468] time: 23.1724, train_loss: 0.20022881, train_accuracy: 0.9531, test_Accuracy: 0.9359\n",
      "Epoch: [ 0] [   67/  468] time: 23.5434, train_loss: 0.14563712, train_accuracy: 0.9453, test_Accuracy: 0.9346\n",
      "Epoch: [ 0] [   68/  468] time: 23.9244, train_loss: 0.14974633, train_accuracy: 0.9375, test_Accuracy: 0.9328\n",
      "Epoch: [ 0] [   69/  468] time: 24.3004, train_loss: 0.10173944, train_accuracy: 0.9453, test_Accuracy: 0.9330\n",
      "Epoch: [ 0] [   70/  468] time: 24.6764, train_loss: 0.21415097, train_accuracy: 0.9375, test_Accuracy: 0.9328\n",
      "Epoch: [ 0] [   71/  468] time: 25.0584, train_loss: 0.07511362, train_accuracy: 0.9531, test_Accuracy: 0.9346\n",
      "Epoch: [ 0] [   72/  468] time: 25.4364, train_loss: 0.09814709, train_accuracy: 0.9219, test_Accuracy: 0.9367\n",
      "Epoch: [ 0] [   73/  468] time: 25.8144, train_loss: 0.14704019, train_accuracy: 0.9453, test_Accuracy: 0.9386\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [ 0] [   74/  468] time: 26.1889, train_loss: 0.23662567, train_accuracy: 0.8906, test_Accuracy: 0.9383\n",
      "Epoch: [ 0] [   75/  468] time: 26.5639, train_loss: 0.19752085, train_accuracy: 0.9141, test_Accuracy: 0.9377\n",
      "Epoch: [ 0] [   76/  468] time: 26.9448, train_loss: 0.11011227, train_accuracy: 0.9766, test_Accuracy: 0.9359\n",
      "Epoch: [ 0] [   77/  468] time: 27.3194, train_loss: 0.07351858, train_accuracy: 0.9688, test_Accuracy: 0.9347\n",
      "Epoch: [ 0] [   78/  468] time: 27.6983, train_loss: 0.12919027, train_accuracy: 0.9297, test_Accuracy: 0.9332\n",
      "Epoch: [ 0] [   79/  468] time: 28.0683, train_loss: 0.19565910, train_accuracy: 0.9375, test_Accuracy: 0.9306\n",
      "Epoch: [ 0] [   80/  468] time: 28.4613, train_loss: 0.11049391, train_accuracy: 0.9297, test_Accuracy: 0.9297\n",
      "Epoch: [ 0] [   81/  468] time: 28.8353, train_loss: 0.06912987, train_accuracy: 0.9688, test_Accuracy: 0.9301\n",
      "Epoch: [ 0] [   82/  468] time: 29.1914, train_loss: 0.12107599, train_accuracy: 0.9375, test_Accuracy: 0.9319\n",
      "Epoch: [ 0] [   83/  468] time: 29.5654, train_loss: 0.12450171, train_accuracy: 0.9375, test_Accuracy: 0.9316\n",
      "Epoch: [ 0] [   84/  468] time: 29.9573, train_loss: 0.09011208, train_accuracy: 0.9766, test_Accuracy: 0.9339\n",
      "Epoch: [ 0] [   85/  468] time: 30.3364, train_loss: 0.14636724, train_accuracy: 0.9453, test_Accuracy: 0.9347\n",
      "Epoch: [ 0] [   86/  468] time: 30.7065, train_loss: 0.12620839, train_accuracy: 0.9297, test_Accuracy: 0.9383\n",
      "Epoch: [ 0] [   87/  468] time: 31.0865, train_loss: 0.22316854, train_accuracy: 0.9297, test_Accuracy: 0.9397\n",
      "Epoch: [ 0] [   88/  468] time: 31.5303, train_loss: 0.14475861, train_accuracy: 0.9453, test_Accuracy: 0.9411\n",
      "Epoch: [ 0] [   89/  468] time: 31.8883, train_loss: 0.10517918, train_accuracy: 0.9531, test_Accuracy: 0.9425\n",
      "Epoch: [ 0] [   90/  468] time: 32.2763, train_loss: 0.16783252, train_accuracy: 0.9375, test_Accuracy: 0.9436\n",
      "Epoch: [ 0] [   91/  468] time: 32.6693, train_loss: 0.14670850, train_accuracy: 0.9922, test_Accuracy: 0.9431\n",
      "Epoch: [ 0] [   92/  468] time: 33.0602, train_loss: 0.16716349, train_accuracy: 0.9453, test_Accuracy: 0.9433\n",
      "Epoch: [ 0] [   93/  468] time: 33.4352, train_loss: 0.15838695, train_accuracy: 0.9375, test_Accuracy: 0.9444\n",
      "Epoch: [ 0] [   94/  468] time: 33.7993, train_loss: 0.08389749, train_accuracy: 0.9531, test_Accuracy: 0.9454\n",
      "Epoch: [ 0] [   95/  468] time: 34.1782, train_loss: 0.11524744, train_accuracy: 0.9688, test_Accuracy: 0.9461\n",
      "Epoch: [ 0] [   96/  468] time: 34.5412, train_loss: 0.15819493, train_accuracy: 0.9688, test_Accuracy: 0.9464\n",
      "Epoch: [ 0] [   97/  468] time: 34.9182, train_loss: 0.04491746, train_accuracy: 0.9922, test_Accuracy: 0.9462\n",
      "Epoch: [ 0] [   98/  468] time: 35.2992, train_loss: 0.12156998, train_accuracy: 0.9531, test_Accuracy: 0.9455\n",
      "Epoch: [ 0] [   99/  468] time: 35.6732, train_loss: 0.12503093, train_accuracy: 0.9453, test_Accuracy: 0.9449\n",
      "Epoch: [ 0] [  100/  468] time: 36.0492, train_loss: 0.11583728, train_accuracy: 0.9531, test_Accuracy: 0.9459\n",
      "Epoch: [ 0] [  101/  468] time: 36.4262, train_loss: 0.10815297, train_accuracy: 0.9688, test_Accuracy: 0.9445\n",
      "Epoch: [ 0] [  102/  468] time: 36.8102, train_loss: 0.06129438, train_accuracy: 0.9688, test_Accuracy: 0.9425\n",
      "Epoch: [ 0] [  103/  468] time: 37.1752, train_loss: 0.07776652, train_accuracy: 0.9609, test_Accuracy: 0.9419\n",
      "Epoch: [ 0] [  104/  468] time: 37.5492, train_loss: 0.09600806, train_accuracy: 0.9766, test_Accuracy: 0.9407\n",
      "Epoch: [ 0] [  105/  468] time: 37.9322, train_loss: 0.07708091, train_accuracy: 0.9688, test_Accuracy: 0.9398\n",
      "Epoch: [ 0] [  106/  468] time: 38.3221, train_loss: 0.24092813, train_accuracy: 0.9219, test_Accuracy: 0.9387\n",
      "Epoch: [ 0] [  107/  468] time: 38.7091, train_loss: 0.10864584, train_accuracy: 0.9531, test_Accuracy: 0.9411\n",
      "Epoch: [ 0] [  108/  468] time: 39.0861, train_loss: 0.06246479, train_accuracy: 0.9844, test_Accuracy: 0.9431\n",
      "Epoch: [ 0] [  109/  468] time: 39.4631, train_loss: 0.10125178, train_accuracy: 0.9531, test_Accuracy: 0.9450\n",
      "Epoch: [ 0] [  110/  468] time: 39.8331, train_loss: 0.06493419, train_accuracy: 0.9766, test_Accuracy: 0.9478\n",
      "Epoch: [ 0] [  111/  468] time: 40.1942, train_loss: 0.12586072, train_accuracy: 0.9375, test_Accuracy: 0.9488\n",
      "Epoch: [ 0] [  112/  468] time: 40.5672, train_loss: 0.13332079, train_accuracy: 0.9375, test_Accuracy: 0.9506\n",
      "Epoch: [ 0] [  113/  468] time: 40.9560, train_loss: 0.05263263, train_accuracy: 0.9766, test_Accuracy: 0.9514\n",
      "Epoch: [ 0] [  114/  468] time: 41.3409, train_loss: 0.06334065, train_accuracy: 0.9844, test_Accuracy: 0.9494\n",
      "Epoch: [ 0] [  115/  468] time: 41.7090, train_loss: 0.07558420, train_accuracy: 0.9688, test_Accuracy: 0.9473\n",
      "Epoch: [ 0] [  116/  468] time: 42.0814, train_loss: 0.11556935, train_accuracy: 0.9688, test_Accuracy: 0.9446\n",
      "Epoch: [ 0] [  117/  468] time: 42.4624, train_loss: 0.10643870, train_accuracy: 0.9609, test_Accuracy: 0.9439\n",
      "Epoch: [ 0] [  118/  468] time: 42.8364, train_loss: 0.08303730, train_accuracy: 0.9766, test_Accuracy: 0.9443\n",
      "Epoch: [ 0] [  119/  468] time: 43.2104, train_loss: 0.13758227, train_accuracy: 0.9297, test_Accuracy: 0.9474\n",
      "Epoch: [ 0] [  120/  468] time: 43.5969, train_loss: 0.13533886, train_accuracy: 0.9297, test_Accuracy: 0.9508\n",
      "Epoch: [ 0] [  121/  468] time: 43.9868, train_loss: 0.07146559, train_accuracy: 0.9688, test_Accuracy: 0.9532\n",
      "Epoch: [ 0] [  122/  468] time: 44.3728, train_loss: 0.09012058, train_accuracy: 0.9688, test_Accuracy: 0.9546\n",
      "Epoch: [ 0] [  123/  468] time: 44.7438, train_loss: 0.11966731, train_accuracy: 0.9766, test_Accuracy: 0.9545\n",
      "Epoch: [ 0] [  124/  468] time: 45.1282, train_loss: 0.11073019, train_accuracy: 0.9531, test_Accuracy: 0.9540\n",
      "Epoch: [ 0] [  125/  468] time: 45.4962, train_loss: 0.11758170, train_accuracy: 0.9688, test_Accuracy: 0.9533\n",
      "Epoch: [ 0] [  126/  468] time: 45.8952, train_loss: 0.08829757, train_accuracy: 0.9688, test_Accuracy: 0.9536\n",
      "Epoch: [ 0] [  127/  468] time: 46.2741, train_loss: 0.12794933, train_accuracy: 0.9688, test_Accuracy: 0.9524\n",
      "Epoch: [ 0] [  128/  468] time: 46.6552, train_loss: 0.10595708, train_accuracy: 0.9219, test_Accuracy: 0.9515\n",
      "Epoch: [ 0] [  129/  468] time: 47.0431, train_loss: 0.05982051, train_accuracy: 0.9531, test_Accuracy: 0.9521\n",
      "Epoch: [ 0] [  130/  468] time: 47.4221, train_loss: 0.09475179, train_accuracy: 0.9219, test_Accuracy: 0.9528\n",
      "Epoch: [ 0] [  131/  468] time: 47.8021, train_loss: 0.11626948, train_accuracy: 0.9609, test_Accuracy: 0.9532\n",
      "Epoch: [ 0] [  132/  468] time: 48.1871, train_loss: 0.08203171, train_accuracy: 0.9531, test_Accuracy: 0.9540\n",
      "Epoch: [ 0] [  133/  468] time: 48.5930, train_loss: 0.12111526, train_accuracy: 0.9453, test_Accuracy: 0.9538\n",
      "Epoch: [ 0] [  134/  468] time: 48.9999, train_loss: 0.07827674, train_accuracy: 0.9844, test_Accuracy: 0.9551\n",
      "Epoch: [ 0] [  135/  468] time: 49.4278, train_loss: 0.11195561, train_accuracy: 0.9609, test_Accuracy: 0.9535\n",
      "Epoch: [ 0] [  136/  468] time: 49.8447, train_loss: 0.12665345, train_accuracy: 0.9609, test_Accuracy: 0.9543\n",
      "Epoch: [ 0] [  137/  468] time: 50.3354, train_loss: 0.21372400, train_accuracy: 0.9375, test_Accuracy: 0.9549\n",
      "Epoch: [ 0] [  138/  468] time: 50.7947, train_loss: 0.20567992, train_accuracy: 0.9375, test_Accuracy: 0.9557\n",
      "Epoch: [ 0] [  139/  468] time: 51.1936, train_loss: 0.06522581, train_accuracy: 0.9766, test_Accuracy: 0.9556\n",
      "Epoch: [ 0] [  140/  468] time: 51.5981, train_loss: 0.12995706, train_accuracy: 0.9531, test_Accuracy: 0.9557\n",
      "Epoch: [ 0] [  141/  468] time: 51.9691, train_loss: 0.15120059, train_accuracy: 0.9688, test_Accuracy: 0.9566\n",
      "Epoch: [ 0] [  142/  468] time: 52.3811, train_loss: 0.08783147, train_accuracy: 0.9844, test_Accuracy: 0.9571\n",
      "Epoch: [ 0] [  143/  468] time: 52.8140, train_loss: 0.07652523, train_accuracy: 0.9922, test_Accuracy: 0.9569\n",
      "Epoch: [ 0] [  144/  468] time: 53.2067, train_loss: 0.09431807, train_accuracy: 0.9609, test_Accuracy: 0.9576\n",
      "Epoch: [ 0] [  145/  468] time: 53.6046, train_loss: 0.13835688, train_accuracy: 0.9688, test_Accuracy: 0.9577\n",
      "Epoch: [ 0] [  146/  468] time: 53.9846, train_loss: 0.06851411, train_accuracy: 0.9922, test_Accuracy: 0.9575\n",
      "Epoch: [ 0] [  147/  468] time: 54.3866, train_loss: 0.05972286, train_accuracy: 0.9844, test_Accuracy: 0.9580\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [ 0] [  148/  468] time: 54.7695, train_loss: 0.07885216, train_accuracy: 0.9844, test_Accuracy: 0.9580\n",
      "Epoch: [ 0] [  149/  468] time: 55.1685, train_loss: 0.07391743, train_accuracy: 0.9688, test_Accuracy: 0.9590\n",
      "Epoch: [ 0] [  150/  468] time: 55.5544, train_loss: 0.12036243, train_accuracy: 0.9609, test_Accuracy: 0.9582\n",
      "Epoch: [ 0] [  151/  468] time: 56.0132, train_loss: 0.11872225, train_accuracy: 0.9609, test_Accuracy: 0.9570\n",
      "Epoch: [ 0] [  152/  468] time: 56.4032, train_loss: 0.16013511, train_accuracy: 0.9688, test_Accuracy: 0.9573\n",
      "Epoch: [ 0] [  153/  468] time: 56.8221, train_loss: 0.10447432, train_accuracy: 0.9922, test_Accuracy: 0.9572\n",
      "Epoch: [ 0] [  154/  468] time: 57.2090, train_loss: 0.11679763, train_accuracy: 0.9766, test_Accuracy: 0.9571\n",
      "Epoch: [ 0] [  155/  468] time: 57.6110, train_loss: 0.15120855, train_accuracy: 0.9531, test_Accuracy: 0.9583\n",
      "Epoch: [ 0] [  156/  468] time: 57.9919, train_loss: 0.12321943, train_accuracy: 0.9609, test_Accuracy: 0.9582\n",
      "Epoch: [ 0] [  157/  468] time: 58.3899, train_loss: 0.06091445, train_accuracy: 0.9922, test_Accuracy: 0.9575\n",
      "Epoch: [ 0] [  158/  468] time: 58.8156, train_loss: 0.19431476, train_accuracy: 0.9609, test_Accuracy: 0.9570\n",
      "Epoch: [ 0] [  159/  468] time: 59.2095, train_loss: 0.23986164, train_accuracy: 0.9375, test_Accuracy: 0.9559\n",
      "Epoch: [ 0] [  160/  468] time: 59.6214, train_loss: 0.06998764, train_accuracy: 0.9844, test_Accuracy: 0.9548\n",
      "Epoch: [ 0] [  161/  468] time: 60.0333, train_loss: 0.11753121, train_accuracy: 0.9609, test_Accuracy: 0.9539\n",
      "Epoch: [ 0] [  162/  468] time: 60.4243, train_loss: 0.15717551, train_accuracy: 0.9297, test_Accuracy: 0.9528\n",
      "Epoch: [ 0] [  163/  468] time: 60.8132, train_loss: 0.10682364, train_accuracy: 0.9453, test_Accuracy: 0.9505\n",
      "Epoch: [ 0] [  164/  468] time: 61.2123, train_loss: 0.10661702, train_accuracy: 0.9297, test_Accuracy: 0.9496\n",
      "Epoch: [ 0] [  165/  468] time: 61.6510, train_loss: 0.04424205, train_accuracy: 0.9688, test_Accuracy: 0.9474\n",
      "Epoch: [ 0] [  166/  468] time: 62.0649, train_loss: 0.10793294, train_accuracy: 0.9531, test_Accuracy: 0.9458\n",
      "Epoch: [ 0] [  167/  468] time: 62.5087, train_loss: 0.10836346, train_accuracy: 0.9766, test_Accuracy: 0.9449\n",
      "Epoch: [ 0] [  168/  468] time: 62.9535, train_loss: 0.05158073, train_accuracy: 0.9609, test_Accuracy: 0.9437\n",
      "Epoch: [ 0] [  169/  468] time: 63.3535, train_loss: 0.12615767, train_accuracy: 0.9375, test_Accuracy: 0.9418\n",
      "Epoch: [ 0] [  170/  468] time: 63.7484, train_loss: 0.08070808, train_accuracy: 0.9688, test_Accuracy: 0.9425\n",
      "Epoch: [ 0] [  171/  468] time: 64.1393, train_loss: 0.03915595, train_accuracy: 0.9844, test_Accuracy: 0.9428\n",
      "Epoch: [ 0] [  172/  468] time: 64.5114, train_loss: 0.09942666, train_accuracy: 0.9531, test_Accuracy: 0.9437\n",
      "Epoch: [ 0] [  173/  468] time: 64.9103, train_loss: 0.06149308, train_accuracy: 0.9609, test_Accuracy: 0.9446\n",
      "Epoch: [ 0] [  174/  468] time: 65.3002, train_loss: 0.11062476, train_accuracy: 0.9531, test_Accuracy: 0.9457\n",
      "Epoch: [ 0] [  175/  468] time: 65.7131, train_loss: 0.03915454, train_accuracy: 0.9766, test_Accuracy: 0.9455\n",
      "Epoch: [ 0] [  176/  468] time: 66.1240, train_loss: 0.12599652, train_accuracy: 0.9375, test_Accuracy: 0.9473\n",
      "Epoch: [ 0] [  177/  468] time: 66.5549, train_loss: 0.07146777, train_accuracy: 0.9688, test_Accuracy: 0.9490\n",
      "Epoch: [ 0] [  178/  468] time: 66.9658, train_loss: 0.13225397, train_accuracy: 0.9766, test_Accuracy: 0.9485\n",
      "Epoch: [ 0] [  179/  468] time: 67.3787, train_loss: 0.03569590, train_accuracy: 0.9766, test_Accuracy: 0.9474\n",
      "Epoch: [ 0] [  180/  468] time: 67.8006, train_loss: 0.07016254, train_accuracy: 0.9609, test_Accuracy: 0.9450\n",
      "Epoch: [ 0] [  181/  468] time: 68.2002, train_loss: 0.10381630, train_accuracy: 0.9219, test_Accuracy: 0.9458\n",
      "Epoch: [ 0] [  182/  468] time: 68.6141, train_loss: 0.06265315, train_accuracy: 0.9531, test_Accuracy: 0.9486\n",
      "Epoch: [ 0] [  183/  468] time: 69.0409, train_loss: 0.21178918, train_accuracy: 0.9219, test_Accuracy: 0.9511\n",
      "Epoch: [ 0] [  184/  468] time: 69.4558, train_loss: 0.05699749, train_accuracy: 0.9766, test_Accuracy: 0.9533\n",
      "Epoch: [ 0] [  185/  468] time: 69.9076, train_loss: 0.10332922, train_accuracy: 0.9688, test_Accuracy: 0.9567\n",
      "Epoch: [ 0] [  186/  468] time: 70.3664, train_loss: 0.15109009, train_accuracy: 0.9453, test_Accuracy: 0.9577\n",
      "Epoch: [ 0] [  187/  468] time: 70.7833, train_loss: 0.09749690, train_accuracy: 0.9766, test_Accuracy: 0.9585\n",
      "Epoch: [ 0] [  188/  468] time: 71.2141, train_loss: 0.06470243, train_accuracy: 0.9844, test_Accuracy: 0.9589\n",
      "Epoch: [ 0] [  189/  468] time: 71.6250, train_loss: 0.07588044, train_accuracy: 0.9766, test_Accuracy: 0.9589\n",
      "Epoch: [ 0] [  190/  468] time: 72.0249, train_loss: 0.04300122, train_accuracy: 0.9922, test_Accuracy: 0.9588\n",
      "Epoch: [ 0] [  191/  468] time: 72.4648, train_loss: 0.12783396, train_accuracy: 0.9609, test_Accuracy: 0.9586\n",
      "Epoch: [ 0] [  192/  468] time: 72.8807, train_loss: 0.10037620, train_accuracy: 0.9688, test_Accuracy: 0.9582\n",
      "Epoch: [ 0] [  193/  468] time: 73.3075, train_loss: 0.04383227, train_accuracy: 0.9844, test_Accuracy: 0.9574\n",
      "Epoch: [ 0] [  194/  468] time: 73.7149, train_loss: 0.08476327, train_accuracy: 0.9766, test_Accuracy: 0.9575\n",
      "Epoch: [ 0] [  195/  468] time: 74.1438, train_loss: 0.13653316, train_accuracy: 0.9609, test_Accuracy: 0.9575\n",
      "Epoch: [ 0] [  196/  468] time: 74.5767, train_loss: 0.09161724, train_accuracy: 0.9688, test_Accuracy: 0.9569\n",
      "Epoch: [ 0] [  197/  468] time: 75.0198, train_loss: 0.05895252, train_accuracy: 0.9844, test_Accuracy: 0.9553\n",
      "Epoch: [ 0] [  198/  468] time: 75.4566, train_loss: 0.13051167, train_accuracy: 0.9375, test_Accuracy: 0.9541\n",
      "Epoch: [ 0] [  199/  468] time: 75.8736, train_loss: 0.07713450, train_accuracy: 0.9766, test_Accuracy: 0.9514\n",
      "Epoch: [ 0] [  200/  468] time: 76.2585, train_loss: 0.12405982, train_accuracy: 0.9609, test_Accuracy: 0.9513\n",
      "Epoch: [ 0] [  201/  468] time: 76.6664, train_loss: 0.06845763, train_accuracy: 0.9766, test_Accuracy: 0.9532\n",
      "Epoch: [ 0] [  202/  468] time: 77.0528, train_loss: 0.10383594, train_accuracy: 0.9688, test_Accuracy: 0.9573\n",
      "Epoch: [ 0] [  203/  468] time: 77.4508, train_loss: 0.07239037, train_accuracy: 0.9844, test_Accuracy: 0.9591\n",
      "Epoch: [ 0] [  204/  468] time: 77.8587, train_loss: 0.07536727, train_accuracy: 0.9766, test_Accuracy: 0.9600\n",
      "Epoch: [ 0] [  205/  468] time: 78.2327, train_loss: 0.13681331, train_accuracy: 0.9609, test_Accuracy: 0.9607\n",
      "Epoch: [ 0] [  206/  468] time: 78.6426, train_loss: 0.22077638, train_accuracy: 0.9531, test_Accuracy: 0.9612\n",
      "Epoch: [ 0] [  207/  468] time: 79.0445, train_loss: 0.10008207, train_accuracy: 0.9531, test_Accuracy: 0.9617\n",
      "Epoch: [ 0] [  208/  468] time: 79.4903, train_loss: 0.14179239, train_accuracy: 0.9688, test_Accuracy: 0.9605\n",
      "Epoch: [ 0] [  209/  468] time: 79.9491, train_loss: 0.06295654, train_accuracy: 0.9766, test_Accuracy: 0.9589\n",
      "Epoch: [ 0] [  210/  468] time: 80.3809, train_loss: 0.04048524, train_accuracy: 0.9688, test_Accuracy: 0.9556\n",
      "Epoch: [ 0] [  211/  468] time: 80.8726, train_loss: 0.21010697, train_accuracy: 0.9297, test_Accuracy: 0.9517\n",
      "Epoch: [ 0] [  212/  468] time: 81.3383, train_loss: 0.08395021, train_accuracy: 0.9688, test_Accuracy: 0.9525\n",
      "Epoch: [ 0] [  213/  468] time: 81.8140, train_loss: 0.09249760, train_accuracy: 0.9609, test_Accuracy: 0.9553\n",
      "Epoch: [ 0] [  214/  468] time: 82.3915, train_loss: 0.03754142, train_accuracy: 0.9766, test_Accuracy: 0.9562\n",
      "Epoch: [ 0] [  215/  468] time: 83.0647, train_loss: 0.10375114, train_accuracy: 0.9609, test_Accuracy: 0.9572\n",
      "Epoch: [ 0] [  216/  468] time: 83.4676, train_loss: 0.07116985, train_accuracy: 0.9609, test_Accuracy: 0.9576\n",
      "Epoch: [ 0] [  217/  468] time: 83.8815, train_loss: 0.06955357, train_accuracy: 0.9766, test_Accuracy: 0.9569\n",
      "Epoch: [ 0] [  218/  468] time: 84.2595, train_loss: 0.09872704, train_accuracy: 0.9531, test_Accuracy: 0.9567\n",
      "Epoch: [ 0] [  219/  468] time: 84.6445, train_loss: 0.05022692, train_accuracy: 0.9844, test_Accuracy: 0.9551\n",
      "Epoch: [ 0] [  220/  468] time: 85.0554, train_loss: 0.10100794, train_accuracy: 0.9766, test_Accuracy: 0.9549\n",
      "Epoch: [ 0] [  221/  468] time: 85.5211, train_loss: 0.07067011, train_accuracy: 0.9609, test_Accuracy: 0.9561\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [ 0] [  222/  468] time: 85.9171, train_loss: 0.11284730, train_accuracy: 0.9375, test_Accuracy: 0.9575\n",
      "Epoch: [ 0] [  223/  468] time: 86.3100, train_loss: 0.13075560, train_accuracy: 0.9297, test_Accuracy: 0.9576\n",
      "Epoch: [ 0] [  224/  468] time: 86.7508, train_loss: 0.03561377, train_accuracy: 0.9922, test_Accuracy: 0.9581\n",
      "Epoch: [ 0] [  225/  468] time: 87.1607, train_loss: 0.07297901, train_accuracy: 0.9766, test_Accuracy: 0.9556\n",
      "Epoch: [ 0] [  226/  468] time: 87.5766, train_loss: 0.10004246, train_accuracy: 0.9609, test_Accuracy: 0.9555\n",
      "Epoch: [ 0] [  227/  468] time: 87.9746, train_loss: 0.08218376, train_accuracy: 0.9688, test_Accuracy: 0.9549\n",
      "Epoch: [ 0] [  228/  468] time: 88.3615, train_loss: 0.10720508, train_accuracy: 0.9766, test_Accuracy: 0.9551\n",
      "Epoch: [ 0] [  229/  468] time: 88.7535, train_loss: 0.06159233, train_accuracy: 0.9688, test_Accuracy: 0.9554\n",
      "Epoch: [ 0] [  230/  468] time: 89.1434, train_loss: 0.05443198, train_accuracy: 0.9766, test_Accuracy: 0.9544\n",
      "Epoch: [ 0] [  231/  468] time: 89.5214, train_loss: 0.14438717, train_accuracy: 0.9453, test_Accuracy: 0.9545\n",
      "Epoch: [ 0] [  232/  468] time: 89.9273, train_loss: 0.10313471, train_accuracy: 0.9688, test_Accuracy: 0.9569\n",
      "Epoch: [ 0] [  233/  468] time: 90.3113, train_loss: 0.06662578, train_accuracy: 0.9609, test_Accuracy: 0.9581\n",
      "Epoch: [ 0] [  234/  468] time: 90.7092, train_loss: 0.09736969, train_accuracy: 0.9609, test_Accuracy: 0.9591\n",
      "Epoch: [ 0] [  235/  468] time: 91.0872, train_loss: 0.05556475, train_accuracy: 0.9609, test_Accuracy: 0.9599\n",
      "Epoch: [ 0] [  236/  468] time: 91.4832, train_loss: 0.07985953, train_accuracy: 0.9609, test_Accuracy: 0.9606\n",
      "Epoch: [ 0] [  237/  468] time: 91.8861, train_loss: 0.11165921, train_accuracy: 0.9844, test_Accuracy: 0.9618\n",
      "Epoch: [ 0] [  238/  468] time: 92.2551, train_loss: 0.13640419, train_accuracy: 0.9609, test_Accuracy: 0.9628\n",
      "Epoch: [ 0] [  239/  468] time: 92.6590, train_loss: 0.15591100, train_accuracy: 0.9453, test_Accuracy: 0.9629\n",
      "Epoch: [ 0] [  240/  468] time: 93.0560, train_loss: 0.09862821, train_accuracy: 0.9531, test_Accuracy: 0.9633\n",
      "Epoch: [ 0] [  241/  468] time: 93.4788, train_loss: 0.06356850, train_accuracy: 0.9844, test_Accuracy: 0.9638\n",
      "Epoch: [ 0] [  242/  468] time: 93.8748, train_loss: 0.04463405, train_accuracy: 0.9922, test_Accuracy: 0.9644\n",
      "Epoch: [ 0] [  243/  468] time: 94.2817, train_loss: 0.06221155, train_accuracy: 0.9766, test_Accuracy: 0.9654\n",
      "Epoch: [ 0] [  244/  468] time: 94.6537, train_loss: 0.16705593, train_accuracy: 0.9609, test_Accuracy: 0.9648\n",
      "Epoch: [ 0] [  245/  468] time: 95.0427, train_loss: 0.03423082, train_accuracy: 0.9844, test_Accuracy: 0.9639\n",
      "Epoch: [ 0] [  246/  468] time: 95.4206, train_loss: 0.08083087, train_accuracy: 0.9922, test_Accuracy: 0.9625\n",
      "Epoch: [ 0] [  247/  468] time: 95.7956, train_loss: 0.12671158, train_accuracy: 0.9531, test_Accuracy: 0.9616\n",
      "Epoch: [ 0] [  248/  468] time: 96.1906, train_loss: 0.07197145, train_accuracy: 0.9766, test_Accuracy: 0.9604\n",
      "Epoch: [ 0] [  249/  468] time: 96.5865, train_loss: 0.10742761, train_accuracy: 0.9688, test_Accuracy: 0.9593\n",
      "Epoch: [ 0] [  250/  468] time: 97.0523, train_loss: 0.08349770, train_accuracy: 0.9609, test_Accuracy: 0.9592\n",
      "Epoch: [ 0] [  251/  468] time: 97.5210, train_loss: 0.07579464, train_accuracy: 0.9531, test_Accuracy: 0.9588\n",
      "Epoch: [ 0] [  252/  468] time: 97.9698, train_loss: 0.07182691, train_accuracy: 0.9844, test_Accuracy: 0.9589\n",
      "Epoch: [ 0] [  253/  468] time: 98.3708, train_loss: 0.17058605, train_accuracy: 0.9688, test_Accuracy: 0.9586\n",
      "Epoch: [ 0] [  254/  468] time: 98.7966, train_loss: 0.05844752, train_accuracy: 0.9766, test_Accuracy: 0.9585\n",
      "Epoch: [ 0] [  255/  468] time: 99.2185, train_loss: 0.08986340, train_accuracy: 0.9688, test_Accuracy: 0.9600\n",
      "Epoch: [ 0] [  256/  468] time: 99.6164, train_loss: 0.04910367, train_accuracy: 0.9688, test_Accuracy: 0.9616\n",
      "Epoch: [ 0] [  257/  468] time: 100.0343, train_loss: 0.12585369, train_accuracy: 0.9688, test_Accuracy: 0.9634\n",
      "Epoch: [ 0] [  258/  468] time: 100.4671, train_loss: 0.11905136, train_accuracy: 0.9766, test_Accuracy: 0.9638\n",
      "Epoch: [ 0] [  259/  468] time: 100.9269, train_loss: 0.06631970, train_accuracy: 0.9688, test_Accuracy: 0.9646\n",
      "Epoch: [ 0] [  260/  468] time: 101.3478, train_loss: 0.08375350, train_accuracy: 0.9609, test_Accuracy: 0.9660\n",
      "Epoch: [ 0] [  261/  468] time: 101.7417, train_loss: 0.10610574, train_accuracy: 0.9766, test_Accuracy: 0.9674\n",
      "Epoch: [ 0] [  262/  468] time: 102.1377, train_loss: 0.03139750, train_accuracy: 0.9844, test_Accuracy: 0.9668\n",
      "Epoch: [ 0] [  263/  468] time: 102.5576, train_loss: 0.14397082, train_accuracy: 0.9531, test_Accuracy: 0.9665\n",
      "Epoch: [ 0] [  264/  468] time: 102.9934, train_loss: 0.05700008, train_accuracy: 0.9766, test_Accuracy: 0.9660\n",
      "Epoch: [ 0] [  265/  468] time: 103.4163, train_loss: 0.09284258, train_accuracy: 0.9688, test_Accuracy: 0.9652\n",
      "Epoch: [ 0] [  266/  468] time: 103.8092, train_loss: 0.14822131, train_accuracy: 0.9453, test_Accuracy: 0.9634\n",
      "Epoch: [ 0] [  267/  468] time: 104.2002, train_loss: 0.08874004, train_accuracy: 0.9688, test_Accuracy: 0.9619\n",
      "Epoch: [ 0] [  268/  468] time: 104.5951, train_loss: 0.03649638, train_accuracy: 0.9922, test_Accuracy: 0.9604\n",
      "Epoch: [ 0] [  269/  468] time: 105.0140, train_loss: 0.08955236, train_accuracy: 0.9531, test_Accuracy: 0.9609\n",
      "Epoch: [ 0] [  270/  468] time: 105.4219, train_loss: 0.08263347, train_accuracy: 0.9609, test_Accuracy: 0.9595\n",
      "Epoch: [ 0] [  271/  468] time: 105.8597, train_loss: 0.05846292, train_accuracy: 0.9922, test_Accuracy: 0.9580\n",
      "Epoch: [ 0] [  272/  468] time: 106.2896, train_loss: 0.03803501, train_accuracy: 0.9844, test_Accuracy: 0.9559\n",
      "Epoch: [ 0] [  273/  468] time: 106.6935, train_loss: 0.13545765, train_accuracy: 0.9688, test_Accuracy: 0.9565\n",
      "Epoch: [ 0] [  274/  468] time: 107.1174, train_loss: 0.16418031, train_accuracy: 0.9844, test_Accuracy: 0.9569\n",
      "Epoch: [ 0] [  275/  468] time: 107.5183, train_loss: 0.03582475, train_accuracy: 0.9766, test_Accuracy: 0.9564\n",
      "Epoch: [ 0] [  276/  468] time: 107.9302, train_loss: 0.10744724, train_accuracy: 0.9609, test_Accuracy: 0.9564\n",
      "Epoch: [ 0] [  277/  468] time: 108.3222, train_loss: 0.07610671, train_accuracy: 0.9531, test_Accuracy: 0.9557\n",
      "Epoch: [ 0] [  278/  468] time: 108.7273, train_loss: 0.14327937, train_accuracy: 0.9453, test_Accuracy: 0.9531\n",
      "Epoch: [ 0] [  279/  468] time: 109.1312, train_loss: 0.10791713, train_accuracy: 0.9375, test_Accuracy: 0.9525\n",
      "Epoch: [ 0] [  280/  468] time: 109.5221, train_loss: 0.14413662, train_accuracy: 0.9609, test_Accuracy: 0.9515\n",
      "Epoch: [ 0] [  281/  468] time: 109.9281, train_loss: 0.08325672, train_accuracy: 0.9688, test_Accuracy: 0.9526\n",
      "Epoch: [ 0] [  282/  468] time: 110.3290, train_loss: 0.04780755, train_accuracy: 0.9844, test_Accuracy: 0.9533\n",
      "Epoch: [ 0] [  283/  468] time: 110.7319, train_loss: 0.07556359, train_accuracy: 0.9766, test_Accuracy: 0.9551\n",
      "Epoch: [ 0] [  284/  468] time: 111.1279, train_loss: 0.09619659, train_accuracy: 0.9609, test_Accuracy: 0.9556\n",
      "Epoch: [ 0] [  285/  468] time: 111.5478, train_loss: 0.03165645, train_accuracy: 0.9766, test_Accuracy: 0.9566\n",
      "Epoch: [ 0] [  286/  468] time: 112.0324, train_loss: 0.09120616, train_accuracy: 0.9531, test_Accuracy: 0.9562\n",
      "Epoch: [ 0] [  287/  468] time: 112.4374, train_loss: 0.09166441, train_accuracy: 0.9688, test_Accuracy: 0.9570\n",
      "Epoch: [ 0] [  288/  468] time: 112.9001, train_loss: 0.12083428, train_accuracy: 0.9453, test_Accuracy: 0.9578\n",
      "Epoch: [ 0] [  289/  468] time: 113.3152, train_loss: 0.16024497, train_accuracy: 0.9688, test_Accuracy: 0.9599\n",
      "Epoch: [ 0] [  290/  468] time: 113.7201, train_loss: 0.04994825, train_accuracy: 0.9688, test_Accuracy: 0.9618\n",
      "Epoch: [ 0] [  291/  468] time: 114.1271, train_loss: 0.05749130, train_accuracy: 0.9766, test_Accuracy: 0.9610\n",
      "Epoch: [ 0] [  292/  468] time: 114.5420, train_loss: 0.04225855, train_accuracy: 0.9609, test_Accuracy: 0.9606\n",
      "Epoch: [ 0] [  293/  468] time: 114.9598, train_loss: 0.05459058, train_accuracy: 0.9766, test_Accuracy: 0.9594\n",
      "Epoch: [ 0] [  294/  468] time: 115.3667, train_loss: 0.08055097, train_accuracy: 0.9766, test_Accuracy: 0.9595\n",
      "Epoch: [ 0] [  295/  468] time: 115.7557, train_loss: 0.19183859, train_accuracy: 0.9375, test_Accuracy: 0.9612\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [ 0] [  296/  468] time: 116.1546, train_loss: 0.11596017, train_accuracy: 0.9453, test_Accuracy: 0.9620\n",
      "Epoch: [ 0] [  297/  468] time: 116.5526, train_loss: 0.07460530, train_accuracy: 0.9844, test_Accuracy: 0.9627\n",
      "Epoch: [ 0] [  298/  468] time: 116.9695, train_loss: 0.07105274, train_accuracy: 1.0000, test_Accuracy: 0.9630\n",
      "Epoch: [ 0] [  299/  468] time: 117.3685, train_loss: 0.15524851, train_accuracy: 0.9531, test_Accuracy: 0.9641\n",
      "Epoch: [ 0] [  300/  468] time: 117.7658, train_loss: 0.08990064, train_accuracy: 0.9766, test_Accuracy: 0.9634\n",
      "Epoch: [ 0] [  301/  468] time: 118.2127, train_loss: 0.06856830, train_accuracy: 0.9688, test_Accuracy: 0.9624\n",
      "Epoch: [ 0] [  302/  468] time: 118.6116, train_loss: 0.03602609, train_accuracy: 1.0000, test_Accuracy: 0.9629\n",
      "Epoch: [ 0] [  303/  468] time: 119.0424, train_loss: 0.05711953, train_accuracy: 0.9844, test_Accuracy: 0.9619\n",
      "Epoch: [ 0] [  304/  468] time: 119.4543, train_loss: 0.09863304, train_accuracy: 0.9688, test_Accuracy: 0.9622\n",
      "Epoch: [ 0] [  305/  468] time: 119.8772, train_loss: 0.09725485, train_accuracy: 0.9531, test_Accuracy: 0.9621\n",
      "Epoch: [ 0] [  306/  468] time: 120.2572, train_loss: 0.11071101, train_accuracy: 0.9688, test_Accuracy: 0.9629\n",
      "Epoch: [ 0] [  307/  468] time: 120.6714, train_loss: 0.09887822, train_accuracy: 0.9453, test_Accuracy: 0.9637\n",
      "Epoch: [ 0] [  308/  468] time: 121.1322, train_loss: 0.06875231, train_accuracy: 0.9844, test_Accuracy: 0.9633\n",
      "Epoch: [ 0] [  309/  468] time: 121.5444, train_loss: 0.10965094, train_accuracy: 0.9688, test_Accuracy: 0.9637\n",
      "Epoch: [ 0] [  310/  468] time: 121.9314, train_loss: 0.04640312, train_accuracy: 0.9844, test_Accuracy: 0.9633\n",
      "Epoch: [ 0] [  311/  468] time: 122.3224, train_loss: 0.06752665, train_accuracy: 0.9688, test_Accuracy: 0.9625\n",
      "Epoch: [ 0] [  312/  468] time: 122.7123, train_loss: 0.08603098, train_accuracy: 0.9766, test_Accuracy: 0.9637\n",
      "Epoch: [ 0] [  313/  468] time: 123.1163, train_loss: 0.07602779, train_accuracy: 0.9766, test_Accuracy: 0.9637\n",
      "Epoch: [ 0] [  314/  468] time: 123.5222, train_loss: 0.12083233, train_accuracy: 0.9766, test_Accuracy: 0.9642\n",
      "Epoch: [ 0] [  315/  468] time: 123.9617, train_loss: 0.07589366, train_accuracy: 0.9766, test_Accuracy: 0.9635\n",
      "Epoch: [ 0] [  316/  468] time: 124.3626, train_loss: 0.08384910, train_accuracy: 0.9375, test_Accuracy: 0.9628\n",
      "Epoch: [ 0] [  317/  468] time: 124.7446, train_loss: 0.09457807, train_accuracy: 0.9688, test_Accuracy: 0.9624\n",
      "Epoch: [ 0] [  318/  468] time: 125.1817, train_loss: 0.09370937, train_accuracy: 0.9688, test_Accuracy: 0.9615\n",
      "Epoch: [ 0] [  319/  468] time: 125.6116, train_loss: 0.14550926, train_accuracy: 0.9453, test_Accuracy: 0.9612\n",
      "Epoch: [ 0] [  320/  468] time: 126.0165, train_loss: 0.05486211, train_accuracy: 0.9844, test_Accuracy: 0.9605\n",
      "Epoch: [ 0] [  321/  468] time: 126.4164, train_loss: 0.07640672, train_accuracy: 0.9766, test_Accuracy: 0.9595\n",
      "Epoch: [ 0] [  322/  468] time: 126.8094, train_loss: 0.05925165, train_accuracy: 0.9609, test_Accuracy: 0.9593\n",
      "Epoch: [ 0] [  323/  468] time: 127.2273, train_loss: 0.04463160, train_accuracy: 0.9453, test_Accuracy: 0.9601\n",
      "Epoch: [ 0] [  324/  468] time: 127.6412, train_loss: 0.09565219, train_accuracy: 0.9688, test_Accuracy: 0.9604\n",
      "Epoch: [ 0] [  325/  468] time: 128.0351, train_loss: 0.04076861, train_accuracy: 0.9844, test_Accuracy: 0.9606\n",
      "Epoch: [ 0] [  326/  468] time: 128.4286, train_loss: 0.08921665, train_accuracy: 0.9688, test_Accuracy: 0.9616\n",
      "Epoch: [ 0] [  327/  468] time: 128.8106, train_loss: 0.15545815, train_accuracy: 0.9531, test_Accuracy: 0.9633\n",
      "Epoch: [ 0] [  328/  468] time: 129.1995, train_loss: 0.08750252, train_accuracy: 0.9453, test_Accuracy: 0.9638\n",
      "Epoch: [ 0] [  329/  468] time: 129.6213, train_loss: 0.10278354, train_accuracy: 0.9766, test_Accuracy: 0.9652\n",
      "Epoch: [ 0] [  330/  468] time: 130.0232, train_loss: 0.06956050, train_accuracy: 0.9609, test_Accuracy: 0.9653\n",
      "Epoch: [ 0] [  331/  468] time: 130.4321, train_loss: 0.06156391, train_accuracy: 0.9766, test_Accuracy: 0.9657\n",
      "Epoch: [ 0] [  332/  468] time: 130.8161, train_loss: 0.07230347, train_accuracy: 0.9766, test_Accuracy: 0.9642\n",
      "Epoch: [ 0] [  333/  468] time: 131.2272, train_loss: 0.03840768, train_accuracy: 0.9844, test_Accuracy: 0.9633\n",
      "Epoch: [ 0] [  334/  468] time: 131.6340, train_loss: 0.06667798, train_accuracy: 0.9688, test_Accuracy: 0.9627\n",
      "Epoch: [ 0] [  335/  468] time: 132.0240, train_loss: 0.09372687, train_accuracy: 0.9766, test_Accuracy: 0.9627\n",
      "Epoch: [ 0] [  336/  468] time: 132.4299, train_loss: 0.10575452, train_accuracy: 0.9609, test_Accuracy: 0.9619\n",
      "Epoch: [ 0] [  337/  468] time: 132.8340, train_loss: 0.03113023, train_accuracy: 0.9844, test_Accuracy: 0.9617\n",
      "Epoch: [ 0] [  338/  468] time: 133.2479, train_loss: 0.08278278, train_accuracy: 0.9609, test_Accuracy: 0.9624\n",
      "Epoch: [ 0] [  339/  468] time: 133.6517, train_loss: 0.07838713, train_accuracy: 0.9844, test_Accuracy: 0.9628\n",
      "Epoch: [ 0] [  340/  468] time: 134.0511, train_loss: 0.09338273, train_accuracy: 0.9531, test_Accuracy: 0.9628\n",
      "Epoch: [ 0] [  341/  468] time: 134.4611, train_loss: 0.07644390, train_accuracy: 0.9844, test_Accuracy: 0.9634\n",
      "Epoch: [ 0] [  342/  468] time: 134.8949, train_loss: 0.08412303, train_accuracy: 0.9766, test_Accuracy: 0.9640\n",
      "Epoch: [ 0] [  343/  468] time: 135.3068, train_loss: 0.08967604, train_accuracy: 0.9688, test_Accuracy: 0.9648\n",
      "Epoch: [ 0] [  344/  468] time: 135.7016, train_loss: 0.12392453, train_accuracy: 0.9531, test_Accuracy: 0.9650\n",
      "Epoch: [ 0] [  345/  468] time: 136.0976, train_loss: 0.12995562, train_accuracy: 0.9688, test_Accuracy: 0.9654\n",
      "Epoch: [ 0] [  346/  468] time: 136.4995, train_loss: 0.03096368, train_accuracy: 0.9766, test_Accuracy: 0.9652\n",
      "Epoch: [ 0] [  347/  468] time: 136.9094, train_loss: 0.06708542, train_accuracy: 0.9844, test_Accuracy: 0.9654\n",
      "Epoch: [ 0] [  348/  468] time: 137.3233, train_loss: 0.13652052, train_accuracy: 0.9531, test_Accuracy: 0.9660\n",
      "Epoch: [ 0] [  349/  468] time: 137.7552, train_loss: 0.08642238, train_accuracy: 0.9844, test_Accuracy: 0.9668\n",
      "Epoch: [ 0] [  350/  468] time: 138.1880, train_loss: 0.06395244, train_accuracy: 0.9766, test_Accuracy: 0.9670\n",
      "Epoch: [ 0] [  351/  468] time: 138.6797, train_loss: 0.05452550, train_accuracy: 0.9922, test_Accuracy: 0.9665\n",
      "Epoch: [ 0] [  352/  468] time: 139.1016, train_loss: 0.08038884, train_accuracy: 0.9609, test_Accuracy: 0.9656\n",
      "Epoch: [ 0] [  353/  468] time: 139.5125, train_loss: 0.09571484, train_accuracy: 0.9766, test_Accuracy: 0.9662\n",
      "Epoch: [ 0] [  354/  468] time: 139.9074, train_loss: 0.06516235, train_accuracy: 0.9688, test_Accuracy: 0.9664\n",
      "Epoch: [ 0] [  355/  468] time: 140.2945, train_loss: 0.05807012, train_accuracy: 0.9922, test_Accuracy: 0.9674\n",
      "Epoch: [ 0] [  356/  468] time: 140.6814, train_loss: 0.11262900, train_accuracy: 0.9688, test_Accuracy: 0.9674\n",
      "Epoch: [ 0] [  357/  468] time: 141.1093, train_loss: 0.05931497, train_accuracy: 0.9766, test_Accuracy: 0.9674\n",
      "Epoch: [ 0] [  358/  468] time: 141.5042, train_loss: 0.06383218, train_accuracy: 0.9922, test_Accuracy: 0.9670\n",
      "Epoch: [ 0] [  359/  468] time: 141.8982, train_loss: 0.05917484, train_accuracy: 0.9766, test_Accuracy: 0.9661\n",
      "Epoch: [ 0] [  360/  468] time: 142.2832, train_loss: 0.08826454, train_accuracy: 0.9766, test_Accuracy: 0.9655\n",
      "Epoch: [ 0] [  361/  468] time: 142.6740, train_loss: 0.11840162, train_accuracy: 0.9609, test_Accuracy: 0.9652\n",
      "Epoch: [ 0] [  362/  468] time: 143.1000, train_loss: 0.07490139, train_accuracy: 0.9609, test_Accuracy: 0.9650\n",
      "Epoch: [ 0] [  363/  468] time: 143.5098, train_loss: 0.02899475, train_accuracy: 1.0000, test_Accuracy: 0.9648\n",
      "Epoch: [ 0] [  364/  468] time: 143.9376, train_loss: 0.07696404, train_accuracy: 0.9922, test_Accuracy: 0.9646\n",
      "Epoch: [ 0] [  365/  468] time: 144.3326, train_loss: 0.09682354, train_accuracy: 0.9844, test_Accuracy: 0.9648\n",
      "Epoch: [ 0] [  366/  468] time: 144.7135, train_loss: 0.04610829, train_accuracy: 1.0000, test_Accuracy: 0.9645\n",
      "Epoch: [ 0] [  367/  468] time: 145.1225, train_loss: 0.06777280, train_accuracy: 0.9766, test_Accuracy: 0.9649\n",
      "Epoch: [ 0] [  368/  468] time: 145.5224, train_loss: 0.03668330, train_accuracy: 0.9922, test_Accuracy: 0.9649\n",
      "Epoch: [ 0] [  369/  468] time: 145.9373, train_loss: 0.19569549, train_accuracy: 0.9609, test_Accuracy: 0.9650\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [ 0] [  370/  468] time: 146.3602, train_loss: 0.07020974, train_accuracy: 0.9688, test_Accuracy: 0.9645\n",
      "Epoch: [ 0] [  371/  468] time: 146.7491, train_loss: 0.04084216, train_accuracy: 0.9688, test_Accuracy: 0.9647\n",
      "Epoch: [ 0] [  372/  468] time: 147.1361, train_loss: 0.09060808, train_accuracy: 0.9766, test_Accuracy: 0.9627\n",
      "Epoch: [ 0] [  373/  468] time: 147.5354, train_loss: 0.02485177, train_accuracy: 0.9766, test_Accuracy: 0.9622\n",
      "Epoch: [ 0] [  374/  468] time: 147.9293, train_loss: 0.08182291, train_accuracy: 0.9766, test_Accuracy: 0.9626\n",
      "Epoch: [ 0] [  375/  468] time: 148.3296, train_loss: 0.08229029, train_accuracy: 0.9766, test_Accuracy: 0.9629\n",
      "Epoch: [ 0] [  376/  468] time: 148.7193, train_loss: 0.10044748, train_accuracy: 0.9609, test_Accuracy: 0.9630\n",
      "Epoch: [ 0] [  377/  468] time: 149.1292, train_loss: 0.06574000, train_accuracy: 0.9688, test_Accuracy: 0.9632\n",
      "Epoch: [ 0] [  378/  468] time: 149.5371, train_loss: 0.06609713, train_accuracy: 0.9766, test_Accuracy: 0.9634\n",
      "Epoch: [ 0] [  379/  468] time: 149.9540, train_loss: 0.05791967, train_accuracy: 0.9609, test_Accuracy: 0.9635\n",
      "Epoch: [ 0] [  380/  468] time: 150.3808, train_loss: 0.08833417, train_accuracy: 0.9766, test_Accuracy: 0.9651\n",
      "Epoch: [ 0] [  381/  468] time: 150.7948, train_loss: 0.09698571, train_accuracy: 0.9609, test_Accuracy: 0.9647\n",
      "Epoch: [ 0] [  382/  468] time: 151.2386, train_loss: 0.04918626, train_accuracy: 0.9688, test_Accuracy: 0.9626\n",
      "Epoch: [ 0] [  383/  468] time: 151.6744, train_loss: 0.05412028, train_accuracy: 0.9766, test_Accuracy: 0.9612\n",
      "Epoch: [ 0] [  384/  468] time: 152.1344, train_loss: 0.03155838, train_accuracy: 0.9844, test_Accuracy: 0.9605\n",
      "Epoch: [ 0] [  385/  468] time: 152.5912, train_loss: 0.09986790, train_accuracy: 0.9688, test_Accuracy: 0.9599\n",
      "Epoch: [ 0] [  386/  468] time: 153.0350, train_loss: 0.08154238, train_accuracy: 0.9766, test_Accuracy: 0.9602\n",
      "Epoch: [ 0] [  387/  468] time: 153.4531, train_loss: 0.08645384, train_accuracy: 0.9609, test_Accuracy: 0.9612\n",
      "Epoch: [ 0] [  388/  468] time: 153.8680, train_loss: 0.08872828, train_accuracy: 0.9453, test_Accuracy: 0.9625\n",
      "Epoch: [ 0] [  389/  468] time: 154.2645, train_loss: 0.04141077, train_accuracy: 0.9766, test_Accuracy: 0.9639\n",
      "Epoch: [ 0] [  390/  468] time: 154.6614, train_loss: 0.05405395, train_accuracy: 0.9844, test_Accuracy: 0.9663\n",
      "Epoch: [ 0] [  391/  468] time: 155.0708, train_loss: 0.05702025, train_accuracy: 0.9844, test_Accuracy: 0.9673\n",
      "Epoch: [ 0] [  392/  468] time: 155.5516, train_loss: 0.11754791, train_accuracy: 0.9688, test_Accuracy: 0.9690\n",
      "Epoch: [ 0] [  393/  468] time: 156.1405, train_loss: 0.06786249, train_accuracy: 0.9766, test_Accuracy: 0.9696\n",
      "Epoch: [ 0] [  394/  468] time: 156.7619, train_loss: 0.05085824, train_accuracy: 0.9766, test_Accuracy: 0.9697\n",
      "Epoch: [ 0] [  395/  468] time: 157.2748, train_loss: 0.10567406, train_accuracy: 0.9766, test_Accuracy: 0.9698\n",
      "Epoch: [ 0] [  396/  468] time: 157.7146, train_loss: 0.03293435, train_accuracy: 1.0000, test_Accuracy: 0.9698\n",
      "Epoch: [ 0] [  397/  468] time: 158.1494, train_loss: 0.02860761, train_accuracy: 1.0000, test_Accuracy: 0.9692\n",
      "Epoch: [ 0] [  398/  468] time: 158.6271, train_loss: 0.07384257, train_accuracy: 0.9766, test_Accuracy: 0.9681\n",
      "Epoch: [ 0] [  399/  468] time: 159.1029, train_loss: 0.10032366, train_accuracy: 0.9531, test_Accuracy: 0.9676\n",
      "Epoch: [ 0] [  400/  468] time: 159.5547, train_loss: 0.08134373, train_accuracy: 0.9688, test_Accuracy: 0.9683\n",
      "Epoch: [ 0] [  401/  468] time: 160.0005, train_loss: 0.07152035, train_accuracy: 0.9844, test_Accuracy: 0.9679\n",
      "Epoch: [ 0] [  402/  468] time: 160.4433, train_loss: 0.13019931, train_accuracy: 0.9688, test_Accuracy: 0.9681\n",
      "Epoch: [ 0] [  403/  468] time: 160.9013, train_loss: 0.05347113, train_accuracy: 0.9766, test_Accuracy: 0.9678\n",
      "Epoch: [ 0] [  404/  468] time: 161.3551, train_loss: 0.01166768, train_accuracy: 1.0000, test_Accuracy: 0.9676\n",
      "Epoch: [ 0] [  405/  468] time: 161.8009, train_loss: 0.06769561, train_accuracy: 0.9844, test_Accuracy: 0.9672\n",
      "Epoch: [ 0] [  406/  468] time: 162.2509, train_loss: 0.07958454, train_accuracy: 0.9688, test_Accuracy: 0.9689\n",
      "Epoch: [ 0] [  407/  468] time: 162.7136, train_loss: 0.07816668, train_accuracy: 0.9688, test_Accuracy: 0.9681\n",
      "Epoch: [ 0] [  408/  468] time: 163.1616, train_loss: 0.04010294, train_accuracy: 0.9766, test_Accuracy: 0.9674\n",
      "Epoch: [ 0] [  409/  468] time: 163.6064, train_loss: 0.05857830, train_accuracy: 0.9766, test_Accuracy: 0.9678\n",
      "Epoch: [ 0] [  410/  468] time: 164.0441, train_loss: 0.07646947, train_accuracy: 0.9766, test_Accuracy: 0.9686\n",
      "Epoch: [ 0] [  411/  468] time: 164.4639, train_loss: 0.06612741, train_accuracy: 0.9688, test_Accuracy: 0.9687\n",
      "Epoch: [ 0] [  412/  468] time: 164.8848, train_loss: 0.07770774, train_accuracy: 0.9609, test_Accuracy: 0.9696\n",
      "Epoch: [ 0] [  413/  468] time: 165.3456, train_loss: 0.16523510, train_accuracy: 0.9453, test_Accuracy: 0.9699\n",
      "Epoch: [ 0] [  414/  468] time: 165.8153, train_loss: 0.08278086, train_accuracy: 0.9688, test_Accuracy: 0.9700\n",
      "Epoch: [ 0] [  415/  468] time: 166.2517, train_loss: 0.18882620, train_accuracy: 0.9531, test_Accuracy: 0.9702\n",
      "Epoch: [ 0] [  416/  468] time: 166.6835, train_loss: 0.06155796, train_accuracy: 0.9844, test_Accuracy: 0.9705\n",
      "Epoch: [ 0] [  417/  468] time: 167.1523, train_loss: 0.07773796, train_accuracy: 0.9844, test_Accuracy: 0.9707\n",
      "Epoch: [ 0] [  418/  468] time: 167.6001, train_loss: 0.04206124, train_accuracy: 0.9688, test_Accuracy: 0.9693\n",
      "Epoch: [ 0] [  419/  468] time: 168.0420, train_loss: 0.13071609, train_accuracy: 0.9688, test_Accuracy: 0.9691\n",
      "Epoch: [ 0] [  420/  468] time: 168.4823, train_loss: 0.09687222, train_accuracy: 0.9297, test_Accuracy: 0.9694\n",
      "Epoch: [ 0] [  421/  468] time: 168.9512, train_loss: 0.09134042, train_accuracy: 0.9688, test_Accuracy: 0.9699\n",
      "Epoch: [ 0] [  422/  468] time: 169.4409, train_loss: 0.05310880, train_accuracy: 0.9766, test_Accuracy: 0.9691\n",
      "Epoch: [ 0] [  423/  468] time: 169.9585, train_loss: 0.04420463, train_accuracy: 0.9922, test_Accuracy: 0.9687\n",
      "Epoch: [ 0] [  424/  468] time: 170.4274, train_loss: 0.07398434, train_accuracy: 0.9688, test_Accuracy: 0.9690\n",
      "Epoch: [ 0] [  425/  468] time: 170.8632, train_loss: 0.05866405, train_accuracy: 0.9688, test_Accuracy: 0.9672\n",
      "Epoch: [ 0] [  426/  468] time: 171.2941, train_loss: 0.05307076, train_accuracy: 0.9844, test_Accuracy: 0.9668\n",
      "Epoch: [ 0] [  427/  468] time: 171.7608, train_loss: 0.04460274, train_accuracy: 0.9844, test_Accuracy: 0.9666\n",
      "Epoch: [ 0] [  428/  468] time: 172.1957, train_loss: 0.19789895, train_accuracy: 0.9609, test_Accuracy: 0.9654\n",
      "Epoch: [ 0] [  429/  468] time: 172.6694, train_loss: 0.04730317, train_accuracy: 0.9766, test_Accuracy: 0.9639\n",
      "Epoch: [ 0] [  430/  468] time: 173.0694, train_loss: 0.03706501, train_accuracy: 0.9922, test_Accuracy: 0.9634\n",
      "Epoch: [ 0] [  431/  468] time: 173.4873, train_loss: 0.08158273, train_accuracy: 0.9766, test_Accuracy: 0.9625\n",
      "Epoch: [ 0] [  432/  468] time: 173.9111, train_loss: 0.06122440, train_accuracy: 0.9844, test_Accuracy: 0.9630\n",
      "Epoch: [ 0] [  433/  468] time: 174.3270, train_loss: 0.06733479, train_accuracy: 0.9844, test_Accuracy: 0.9644\n",
      "Epoch: [ 0] [  434/  468] time: 174.7439, train_loss: 0.08791436, train_accuracy: 0.9531, test_Accuracy: 0.9653\n",
      "Epoch: [ 0] [  435/  468] time: 175.1837, train_loss: 0.08324844, train_accuracy: 0.9844, test_Accuracy: 0.9654\n",
      "Epoch: [ 0] [  436/  468] time: 175.6176, train_loss: 0.06347284, train_accuracy: 0.9688, test_Accuracy: 0.9671\n",
      "Epoch: [ 0] [  437/  468] time: 176.0215, train_loss: 0.06152664, train_accuracy: 0.9531, test_Accuracy: 0.9670\n",
      "Epoch: [ 0] [  438/  468] time: 176.4474, train_loss: 0.04072528, train_accuracy: 0.9766, test_Accuracy: 0.9666\n",
      "Epoch: [ 0] [  439/  468] time: 176.8971, train_loss: 0.07449681, train_accuracy: 0.9844, test_Accuracy: 0.9665\n",
      "Epoch: [ 0] [  440/  468] time: 177.3280, train_loss: 0.09760790, train_accuracy: 0.9766, test_Accuracy: 0.9677\n",
      "Epoch: [ 0] [  441/  468] time: 177.7748, train_loss: 0.03927501, train_accuracy: 0.9844, test_Accuracy: 0.9677\n",
      "Epoch: [ 0] [  442/  468] time: 178.2366, train_loss: 0.06337216, train_accuracy: 0.9766, test_Accuracy: 0.9682\n",
      "Epoch: [ 0] [  443/  468] time: 178.6774, train_loss: 0.05398809, train_accuracy: 0.9922, test_Accuracy: 0.9674\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [ 0] [  444/  468] time: 179.1042, train_loss: 0.10397348, train_accuracy: 0.9688, test_Accuracy: 0.9674\n",
      "Epoch: [ 0] [  445/  468] time: 179.5321, train_loss: 0.08754227, train_accuracy: 0.9766, test_Accuracy: 0.9673\n",
      "Epoch: [ 0] [  446/  468] time: 179.9629, train_loss: 0.05286007, train_accuracy: 0.9922, test_Accuracy: 0.9666\n",
      "Epoch: [ 0] [  447/  468] time: 180.3748, train_loss: 0.05373925, train_accuracy: 0.9766, test_Accuracy: 0.9656\n",
      "Epoch: [ 0] [  448/  468] time: 180.7838, train_loss: 0.06947993, train_accuracy: 0.9609, test_Accuracy: 0.9658\n",
      "Epoch: [ 0] [  449/  468] time: 181.3103, train_loss: 0.05721885, train_accuracy: 0.9844, test_Accuracy: 0.9654\n",
      "Epoch: [ 0] [  450/  468] time: 181.8160, train_loss: 0.10717013, train_accuracy: 0.9688, test_Accuracy: 0.9653\n",
      "Epoch: [ 0] [  451/  468] time: 182.2778, train_loss: 0.05318617, train_accuracy: 0.9922, test_Accuracy: 0.9651\n",
      "Epoch: [ 0] [  452/  468] time: 182.7276, train_loss: 0.06814774, train_accuracy: 0.9844, test_Accuracy: 0.9636\n",
      "Epoch: [ 0] [  453/  468] time: 183.2483, train_loss: 0.10144079, train_accuracy: 0.9688, test_Accuracy: 0.9627\n",
      "Epoch: [ 0] [  454/  468] time: 183.7908, train_loss: 0.10518965, train_accuracy: 0.9531, test_Accuracy: 0.9629\n",
      "Epoch: [ 0] [  455/  468] time: 184.2695, train_loss: 0.08419921, train_accuracy: 0.9688, test_Accuracy: 0.9644\n",
      "Epoch: [ 0] [  456/  468] time: 184.6984, train_loss: 0.05442271, train_accuracy: 0.9844, test_Accuracy: 0.9650\n",
      "Epoch: [ 0] [  457/  468] time: 185.1302, train_loss: 0.04963174, train_accuracy: 0.9844, test_Accuracy: 0.9662\n",
      "Epoch: [ 0] [  458/  468] time: 185.5815, train_loss: 0.10189489, train_accuracy: 0.9297, test_Accuracy: 0.9657\n",
      "Epoch: [ 0] [  459/  468] time: 186.0542, train_loss: 0.07877752, train_accuracy: 0.9766, test_Accuracy: 0.9657\n",
      "Epoch: [ 0] [  460/  468] time: 186.5053, train_loss: 0.08519888, train_accuracy: 0.9766, test_Accuracy: 0.9657\n",
      "Epoch: [ 0] [  461/  468] time: 186.9521, train_loss: 0.11910671, train_accuracy: 0.9453, test_Accuracy: 0.9655\n",
      "Epoch: [ 0] [  462/  468] time: 187.4259, train_loss: 0.02652783, train_accuracy: 1.0000, test_Accuracy: 0.9656\n",
      "Epoch: [ 0] [  463/  468] time: 187.8467, train_loss: 0.02743277, train_accuracy: 0.9609, test_Accuracy: 0.9657\n",
      "Epoch: [ 0] [  464/  468] time: 188.2507, train_loss: 0.05252564, train_accuracy: 0.9844, test_Accuracy: 0.9656\n",
      "Epoch: [ 0] [  465/  468] time: 188.6835, train_loss: 0.03721134, train_accuracy: 0.9844, test_Accuracy: 0.9656\n",
      "Epoch: [ 0] [  466/  468] time: 189.1433, train_loss: 0.06141204, train_accuracy: 0.9688, test_Accuracy: 0.9658\n",
      "Epoch: [ 0] [  467/  468] time: 189.6768, train_loss: 0.12341328, train_accuracy: 0.9609, test_Accuracy: 0.9655\n"
     ]
    }
   ],
   "source": [
    "if train_flag :\n",
    "\n",
    "    checkpoint = tf.train.Checkpoint(dnn=network)\n",
    "\n",
    "    # create writer for tensorboard\n",
    "    summary_writer = tf.summary.create_file_writer(logdir=logs_dir)\n",
    "    start_time = time()\n",
    "\n",
    "    # restore check-point if it exits\n",
    "    could_load, checkpoint_counter = load(network, checkpoint_dir)    \n",
    "\n",
    "    if could_load:\n",
    "        start_epoch = (int)(checkpoint_counter / training_iterations)        \n",
    "        counter = checkpoint_counter        \n",
    "        print(\" [*] Load SUCCESS\")\n",
    "    else:\n",
    "        start_epoch = 0\n",
    "        start_iteration = 0\n",
    "        counter = 0\n",
    "        print(\" [!] Load failed...\")\n",
    "    \n",
    "    # train phase\n",
    "    with summary_writer.as_default():  # for tensorboard\n",
    "        for epoch in range(start_epoch, training_epochs):\n",
    "            for idx, (train_input, train_label) in enumerate(train_dataset):            \n",
    "                grads = grad(network, train_input, train_label)\n",
    "                optimizer.apply_gradients(grads_and_vars=zip(grads, network.trainable_variables))\n",
    "\n",
    "                train_loss = loss_fn(network, train_input, train_label)\n",
    "                train_accuracy = accuracy_fn(network, train_input, train_label)\n",
    "                \n",
    "                for test_input, test_label in test_dataset:                \n",
    "                    test_accuracy = accuracy_fn(network, test_input, test_label)\n",
    "\n",
    "                tf.summary.scalar(name='train_loss', data=train_loss, step=counter)\n",
    "                tf.summary.scalar(name='train_accuracy', data=train_accuracy, step=counter)\n",
    "                tf.summary.scalar(name='test_accuracy', data=test_accuracy, step=counter)\n",
    "\n",
    "                print(\n",
    "                    \"Epoch: [%2d] [%5d/%5d] time: %4.4f, train_loss: %.8f, train_accuracy: %.4f, test_Accuracy: %.4f\" \\\n",
    "                    % (epoch, idx, training_iterations, time() - start_time, train_loss, train_accuracy,\n",
    "                       test_accuracy))\n",
    "                counter += 1                \n",
    "        checkpoint.save(file_prefix=checkpoint_prefix + '-{}'.format(counter))\n",
    "        \n",
    "# test phase      \n",
    "else :\n",
    "    _, _ = load(network, checkpoint_dir)\n",
    "    for test_input, test_label in test_dataset:    \n",
    "        test_accuracy = accuracy_fn(network, test_input, test_label)\n",
    "\n",
    "    print(\"test_Accuracy: %.4f\" % (test_accuracy))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "439d2eea",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "py38tensorflow",
   "language": "python",
   "name": "py38tensorflow"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}