Lab10/Lab10-4 minkiKang.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 모두를 위한 딥러닝 시즌2 - TensorFlow\n",
    "\n",
    "## LAB10-4 MNIST NN BATCHNORMALIZATION"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.5.0\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "from tensorflow.keras.utils import to_categorical\n",
    "from tensorflow.keras.datasets import mnist\n",
    "from time import time\n",
    "import os\n",
    "print(tf.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load(model, checkpoint_dir):\n",
    "    print(\" [*] Reading checkpoints...\")\n",
    "\n",
    "    ckpt = tf.train.get_checkpoint_state(checkpoint_dir)\n",
    "    if ckpt :\n",
    "        ckpt_name = os.path.basename(ckpt.model_checkpoint_path)\n",
    "        checkpoint = tf.train.Checkpoint(dnn=model)\n",
    "        checkpoint.restore(save_path=os.path.join(checkpoint_dir, ckpt_name))\n",
    "        counter = int(ckpt_name.split('-')[1])\n",
    "        print(\" [*] Success to read {}\".format(ckpt_name))\n",
    "        return True, counter\n",
    "    else:\n",
    "        print(\" [*] Failed to find a checkpoint\")\n",
    "        return False, 0\n",
    "\n",
    "def check_folder(dir):\n",
    "    if not os.path.exists(dir):\n",
    "        os.makedirs(dir)\n",
    "    return dir"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 데이터 로드 & 전처리"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_mnist() :\n",
    "    (train_data, train_labels), (test_data, test_labels) = mnist.load_data()\n",
    "    train_data = np.expand_dims(train_data, axis=-1) # [N, 28, 28] -> [N, 28, 28, 1]\n",
    "    test_data = np.expand_dims(test_data, axis=-1) # [N, 28, 28] -> [N, 28, 28, 1]\n",
    "\n",
    "    train_data, test_data = normalize(train_data, test_data)\n",
    "\n",
    "    train_labels = to_categorical(train_labels, 10) # [N,] -> [N, 10]\n",
    "    test_labels = to_categorical(test_labels, 10) # [N,] -> [N, 10]\n",
    "\n",
    "    return train_data, train_labels, test_data, test_labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def normalize(train_data, test_data):\n",
    "    train_data = train_data.astype(np.float32) / 255.0\n",
    "    test_data = test_data.astype(np.float32) / 255.0\n",
    "\n",
    "    return train_data, test_data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### LOSS , accuracy, grad function 정의"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def loss_fn(model, images, labels):\n",
    "    logits = model(images, training=True)\n",
    "    loss = tf.reduce_mean(tf.keras.losses.categorical_crossentropy(y_pred=logits, y_true=labels,from_logits=True))\n",
    "    return loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def accuracy_fn(model, images, labels):\n",
    "    logits = model(images, training=False)\n",
    "    prediction = tf.equal(tf.argmax(logits, -1), tf.argmax(labels, -1))\n",
    "    accuracy = tf.reduce_mean(tf.cast(prediction, tf.float32))\n",
    "    return accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def grad(model, images, labels):\n",
    "    with tf.GradientTape() as tape:\n",
    "        loss = loss_fn(model, images, labels)\n",
    "    return tape.gradient(loss, model.trainable_variables)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model 함수"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def flatten() :\n",
    "    return tf.keras.layers.Flatten()\n",
    "\n",
    "def dense(label_dim, weight_init) :\n",
    "    return tf.keras.layers.Dense(units=label_dim, use_bias=True, kernel_initializer=weight_init)\n",
    "\n",
    "def relu() :\n",
    "    return tf.keras.layers.Activation(tf.keras.activations.relu)\n",
    "\n",
    "def batch_norm() :\n",
    "    return tf.keras.layers.BatchNormalization()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### MODEL 만들기\n",
    "1. class 버젼"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "class create_model_class(tf.keras.Model):\n",
    "    def __init__(self, label_dim):\n",
    "        super(create_model_class, self).__init__()\n",
    "        weight_init = tf.keras.initializers.glorot_uniform()\n",
    "\n",
    "        self.model = tf.keras.Sequential()\n",
    "        self.model.add(flatten())\n",
    "\n",
    "        for i in range(4):\n",
    "            self.model.add(dense(512, weight_init))\n",
    "            self.model.add(batch_norm())\n",
    "            self.model.add(relu())\n",
    "\n",
    "        self.model.add(dense(label_dim, weight_init))\n",
    "\n",
    "    def call(self, x, training=None, mask=None):\n",
    "\n",
    "        x = self.model(x)\n",
    "\n",
    "        return x"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### MODEL 만들기\n",
    "2. function 버젼"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_model_function(label_dim) :\n",
    "    weight_init = tf.keras.initializers.glorot_uniform()\n",
    "\n",
    "    model = tf.keras.Sequential()\n",
    "    model.add(flatten())\n",
    "\n",
    "    for i in range(4) :\n",
    "        model.add(dense(512, weight_init))\n",
    "        model.add(batch_norm())\n",
    "        model.add(relu())\n",
    "\n",
    "    model.add(dense(label_dim, weight_init))\n",
    "\n",
    "    return model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 데이터&하이퍼 파라미터 정의하기"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "#dataset\n",
    "train_x, train_y, test_x, test_y = load_mnist()\n",
    "\n",
    "#파라미터\n",
    "learning_rate = 0.001\n",
    "batch_size = 128\n",
    "\n",
    "training_epochs = 1\n",
    "training_iterations = len(train_x) // batch_size\n",
    "\n",
    "label_dim = 10\n",
    "\n",
    "train_flag = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dataset = tf.data.Dataset.from_tensor_slices((train_x, train_y)).\\\n",
    "    shuffle(buffer_size=100000).\\\n",
    "    prefetch(buffer_size=batch_size).\\\n",
    "    batch(batch_size, drop_remainder=True)\n",
    "\n",
    "test_dataset = tf.data.Dataset.from_tensor_slices((test_x, test_y)).\\\n",
    "    shuffle(buffer_size=100000).\\\n",
    "    prefetch(buffer_size=len(test_x)).\\\n",
    "    batch(len(test_x))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 모델&옵티마이저&writer 정의하기"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "network = create_model_function(label_dim)\n",
    "\n",
    "optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate)\n",
    "\n",
    "checkpoint_dir = 'checkpoints'\n",
    "logs_dir = 'logs'\n",
    "\n",
    "model_dir = 'nn_batchnorm'\n",
    "\n",
    "checkpoint_dir = os.path.join(checkpoint_dir, model_dir)\n",
    "check_folder(checkpoint_dir)\n",
    "checkpoint_prefix = os.path.join(checkpoint_dir, model_dir)\n",
    "logs_dir = os.path.join(logs_dir, model_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " [*] Reading checkpoints...\n",
      " [*] Failed to find a checkpoint\n",
      " [!] Load failed...\n",
      "Epoch: [ 0] [    0/  468] time: 2.2528, train_loss: 0.34585792, train_accuracy: 0.4688, test_Accuracy: 0.3171\n",
      "Epoch: [ 0] [    1/  468] time: 3.0273, train_loss: 0.34091747, train_accuracy: 0.4375, test_Accuracy: 0.4135\n",
      "Epoch: [ 0] [    2/  468] time: 3.7688, train_loss: 0.38450673, train_accuracy: 0.4219, test_Accuracy: 0.4738\n",
      "Epoch: [ 0] [    3/  468] time: 4.4965, train_loss: 0.32456967, train_accuracy: 0.5625, test_Accuracy: 0.4904\n",
      "Epoch: [ 0] [    4/  468] time: 4.9810, train_loss: 0.22497967, train_accuracy: 0.5078, test_Accuracy: 0.4816\n",
      "Epoch: [ 0] [    5/  468] time: 5.4954, train_loss: 0.25330222, train_accuracy: 0.4688, test_Accuracy: 0.4767\n",
      "Epoch: [ 0] [    6/  468] time: 6.1331, train_loss: 0.27595389, train_accuracy: 0.4453, test_Accuracy: 0.4987\n",
      "Epoch: [ 0] [    7/  468] time: 6.8272, train_loss: 0.24007842, train_accuracy: 0.4844, test_Accuracy: 0.5416\n",
      "Epoch: [ 0] [    8/  468] time: 7.5949, train_loss: 0.24562256, train_accuracy: 0.6172, test_Accuracy: 0.5650\n",
      "Epoch: [ 0] [    9/  468] time: 8.2877, train_loss: 0.24460925, train_accuracy: 0.6094, test_Accuracy: 0.5842\n",
      "Epoch: [ 0] [   10/  468] time: 8.9639, train_loss: 0.26961279, train_accuracy: 0.5391, test_Accuracy: 0.6067\n",
      "Epoch: [ 0] [   11/  468] time: 9.6154, train_loss: 0.24068870, train_accuracy: 0.5938, test_Accuracy: 0.6333\n",
      "Epoch: [ 0] [   12/  468] time: 10.3287, train_loss: 0.25856140, train_accuracy: 0.5859, test_Accuracy: 0.6540\n",
      "Epoch: [ 0] [   13/  468] time: 11.0008, train_loss: 0.22142561, train_accuracy: 0.6641, test_Accuracy: 0.6748\n",
      "Epoch: [ 0] [   14/  468] time: 11.6728, train_loss: 0.26824003, train_accuracy: 0.7266, test_Accuracy: 0.6989\n",
      "Epoch: [ 0] [   15/  468] time: 12.3599, train_loss: 0.18327667, train_accuracy: 0.7500, test_Accuracy: 0.7351\n",
      "Epoch: [ 0] [   16/  468] time: 13.1177, train_loss: 0.33341819, train_accuracy: 0.7812, test_Accuracy: 0.7568\n",
      "Epoch: [ 0] [   17/  468] time: 13.8768, train_loss: 0.16746172, train_accuracy: 0.8438, test_Accuracy: 0.7772\n",
      "Epoch: [ 0] [   18/  468] time: 14.5665, train_loss: 0.20234734, train_accuracy: 0.8047, test_Accuracy: 0.8005\n",
      "Epoch: [ 0] [   19/  468] time: 15.2928, train_loss: 0.25561062, train_accuracy: 0.7969, test_Accuracy: 0.8319\n",
      "Epoch: [ 0] [   20/  468] time: 15.9428, train_loss: 0.19369903, train_accuracy: 0.8672, test_Accuracy: 0.8576\n",
      "Epoch: [ 0] [   21/  468] time: 16.5945, train_loss: 0.22300760, train_accuracy: 0.8359, test_Accuracy: 0.8758\n",
      "Epoch: [ 0] [   22/  468] time: 17.3936, train_loss: 0.28933871, train_accuracy: 0.8594, test_Accuracy: 0.8872\n",
      "Epoch: [ 0] [   23/  468] time: 17.9488, train_loss: 0.19965185, train_accuracy: 0.9062, test_Accuracy: 0.8952\n",
      "Epoch: [ 0] [   24/  468] time: 18.6305, train_loss: 0.13754086, train_accuracy: 0.8828, test_Accuracy: 0.8980\n",
      "Epoch: [ 0] [   25/  468] time: 19.1807, train_loss: 0.14131913, train_accuracy: 0.9062, test_Accuracy: 0.8995\n",
      "Epoch: [ 0] [   26/  468] time: 19.7730, train_loss: 0.24071994, train_accuracy: 0.9375, test_Accuracy: 0.9005\n",
      "Epoch: [ 0] [   27/  468] time: 20.3917, train_loss: 0.17211589, train_accuracy: 0.8984, test_Accuracy: 0.9022\n",
      "Epoch: [ 0] [   28/  468] time: 21.0803, train_loss: 0.18986832, train_accuracy: 0.9219, test_Accuracy: 0.9016\n",
      "Epoch: [ 0] [   29/  468] time: 21.6530, train_loss: 0.25574142, train_accuracy: 0.8516, test_Accuracy: 0.9034\n",
      "Epoch: [ 0] [   30/  468] time: 22.3690, train_loss: 0.23266633, train_accuracy: 0.9141, test_Accuracy: 0.9066\n",
      "Epoch: [ 0] [   31/  468] time: 23.0007, train_loss: 0.14905444, train_accuracy: 0.9375, test_Accuracy: 0.9066\n",
      "Epoch: [ 0] [   32/  468] time: 23.7580, train_loss: 0.17037037, train_accuracy: 0.8828, test_Accuracy: 0.9091\n",
      "Epoch: [ 0] [   33/  468] time: 24.5122, train_loss: 0.16992897, train_accuracy: 0.9219, test_Accuracy: 0.9120\n",
      "Epoch: [ 0] [   34/  468] time: 25.2434, train_loss: 0.12978035, train_accuracy: 0.9297, test_Accuracy: 0.9140\n",
      "Epoch: [ 0] [   35/  468] time: 26.0224, train_loss: 0.19712970, train_accuracy: 0.9375, test_Accuracy: 0.9140\n",
      "Epoch: [ 0] [   36/  468] time: 26.8288, train_loss: 0.15464637, train_accuracy: 0.9297, test_Accuracy: 0.9140\n",
      "Epoch: [ 0] [   37/  468] time: 27.5556, train_loss: 0.23792383, train_accuracy: 0.8984, test_Accuracy: 0.9136\n",
      "Epoch: [ 0] [   38/  468] time: 28.1352, train_loss: 0.16585976, train_accuracy: 0.9297, test_Accuracy: 0.9142\n",
      "Epoch: [ 0] [   39/  468] time: 28.8009, train_loss: 0.17117395, train_accuracy: 0.9297, test_Accuracy: 0.9177\n",
      "Epoch: [ 0] [   40/  468] time: 29.5431, train_loss: 0.09304868, train_accuracy: 0.9531, test_Accuracy: 0.9216\n",
      "Epoch: [ 0] [   41/  468] time: 30.2093, train_loss: 0.13464811, train_accuracy: 0.9453, test_Accuracy: 0.9234\n",
      "Epoch: [ 0] [   42/  468] time: 30.9014, train_loss: 0.17427568, train_accuracy: 0.9453, test_Accuracy: 0.9248\n",
      "Epoch: [ 0] [   43/  468] time: 31.5790, train_loss: 0.11030427, train_accuracy: 0.9453, test_Accuracy: 0.9278\n",
      "Epoch: [ 0] [   44/  468] time: 32.3573, train_loss: 0.17637490, train_accuracy: 0.9453, test_Accuracy: 0.9314\n",
      "Epoch: [ 0] [   45/  468] time: 32.9837, train_loss: 0.14818847, train_accuracy: 0.9375, test_Accuracy: 0.9322\n",
      "Epoch: [ 0] [   46/  468] time: 33.7030, train_loss: 0.18896620, train_accuracy: 0.9375, test_Accuracy: 0.9325\n",
      "Epoch: [ 0] [   47/  468] time: 34.4360, train_loss: 0.13039464, train_accuracy: 0.9297, test_Accuracy: 0.9310\n",
      "Epoch: [ 0] [   48/  468] time: 35.0523, train_loss: 0.25724706, train_accuracy: 0.9062, test_Accuracy: 0.9316\n",
      "Epoch: [ 0] [   49/  468] time: 35.7240, train_loss: 0.11450672, train_accuracy: 0.9609, test_Accuracy: 0.9306\n",
      "Epoch: [ 0] [   50/  468] time: 36.6052, train_loss: 0.17243716, train_accuracy: 0.9297, test_Accuracy: 0.9284\n",
      "Epoch: [ 0] [   51/  468] time: 37.3047, train_loss: 0.14519250, train_accuracy: 0.9531, test_Accuracy: 0.9279\n",
      "Epoch: [ 0] [   52/  468] time: 38.0480, train_loss: 0.10716334, train_accuracy: 0.9609, test_Accuracy: 0.9264\n",
      "Epoch: [ 0] [   53/  468] time: 38.7977, train_loss: 0.09772661, train_accuracy: 0.9609, test_Accuracy: 0.9237\n",
      "Epoch: [ 0] [   54/  468] time: 39.5666, train_loss: 0.14765745, train_accuracy: 0.9219, test_Accuracy: 0.9262\n",
      "Epoch: [ 0] [   55/  468] time: 40.2358, train_loss: 0.16255233, train_accuracy: 0.9219, test_Accuracy: 0.9289\n",
      "Epoch: [ 0] [   56/  468] time: 40.9385, train_loss: 0.12944686, train_accuracy: 0.9531, test_Accuracy: 0.9313\n",
      "Epoch: [ 0] [   57/  468] time: 41.5956, train_loss: 0.10012159, train_accuracy: 0.9453, test_Accuracy: 0.9331\n",
      "Epoch: [ 0] [   58/  468] time: 42.3706, train_loss: 0.09866597, train_accuracy: 0.9609, test_Accuracy: 0.9330\n",
      "Epoch: [ 0] [   59/  468] time: 43.0427, train_loss: 0.13464442, train_accuracy: 0.9453, test_Accuracy: 0.9327\n",
      "Epoch: [ 0] [   60/  468] time: 43.7053, train_loss: 0.12912713, train_accuracy: 0.9141, test_Accuracy: 0.9300\n",
      "Epoch: [ 0] [   61/  468] time: 44.2659, train_loss: 0.19617733, train_accuracy: 0.9219, test_Accuracy: 0.9275\n",
      "Epoch: [ 0] [   62/  468] time: 44.7597, train_loss: 0.13253191, train_accuracy: 0.9688, test_Accuracy: 0.9272\n",
      "Epoch: [ 0] [   63/  468] time: 45.3551, train_loss: 0.14854452, train_accuracy: 0.9453, test_Accuracy: 0.9284\n",
      "Epoch: [ 0] [   64/  468] time: 45.9664, train_loss: 0.14350349, train_accuracy: 0.9297, test_Accuracy: 0.9319\n",
      "Epoch: [ 0] [   65/  468] time: 46.7275, train_loss: 0.10885827, train_accuracy: 0.9688, test_Accuracy: 0.9327\n",
      "Epoch: [ 0] [   66/  468] time: 47.4375, train_loss: 0.05591802, train_accuracy: 0.9766, test_Accuracy: 0.9350\n",
      "Epoch: [ 0] [   67/  468] time: 48.1912, train_loss: 0.07957158, train_accuracy: 0.9766, test_Accuracy: 0.9359\n",
      "Epoch: [ 0] [   68/  468] time: 48.8204, train_loss: 0.21554759, train_accuracy: 0.9375, test_Accuracy: 0.9376\n",
      "Epoch: [ 0] [   69/  468] time: 49.5341, train_loss: 0.12717173, train_accuracy: 0.9375, test_Accuracy: 0.9378\n",
      "Epoch: [ 0] [   70/  468] time: 50.2466, train_loss: 0.28212491, train_accuracy: 0.9219, test_Accuracy: 0.9394\n",
      "Epoch: [ 0] [   71/  468] time: 50.9128, train_loss: 0.11519174, train_accuracy: 0.9688, test_Accuracy: 0.9403\n",
      "Epoch: [ 0] [   72/  468] time: 51.5292, train_loss: 0.13215277, train_accuracy: 0.9609, test_Accuracy: 0.9418\n",
      "Epoch: [ 0] [   73/  468] time: 52.3082, train_loss: 0.25196671, train_accuracy: 0.9453, test_Accuracy: 0.9431\n",
      "Epoch: [ 0] [   74/  468] time: 53.0570, train_loss: 0.11474314, train_accuracy: 0.9453, test_Accuracy: 0.9430\n",
      "Epoch: [ 0] [   75/  468] time: 53.8263, train_loss: 0.15117860, train_accuracy: 0.9375, test_Accuracy: 0.9432\n",
      "Epoch: [ 0] [   76/  468] time: 54.4814, train_loss: 0.09879479, train_accuracy: 0.9609, test_Accuracy: 0.9431\n",
      "Epoch: [ 0] [   77/  468] time: 55.1223, train_loss: 0.07027308, train_accuracy: 0.9766, test_Accuracy: 0.9433\n",
      "Epoch: [ 0] [   78/  468] time: 55.8825, train_loss: 0.15985969, train_accuracy: 0.9297, test_Accuracy: 0.9450\n",
      "Epoch: [ 0] [   79/  468] time: 56.5548, train_loss: 0.17454404, train_accuracy: 0.9297, test_Accuracy: 0.9458\n",
      "Epoch: [ 0] [   80/  468] time: 57.2125, train_loss: 0.17309269, train_accuracy: 0.9531, test_Accuracy: 0.9467\n",
      "Epoch: [ 0] [   81/  468] time: 57.8728, train_loss: 0.14123265, train_accuracy: 0.9609, test_Accuracy: 0.9468\n",
      "Epoch: [ 0] [   82/  468] time: 58.4789, train_loss: 0.14783561, train_accuracy: 0.9531, test_Accuracy: 0.9464\n",
      "Epoch: [ 0] [   83/  468] time: 59.1869, train_loss: 0.08669256, train_accuracy: 0.9688, test_Accuracy: 0.9473\n",
      "Epoch: [ 0] [   84/  468] time: 59.8031, train_loss: 0.08311385, train_accuracy: 0.9688, test_Accuracy: 0.9467\n",
      "Epoch: [ 0] [   85/  468] time: 60.4483, train_loss: 0.16618049, train_accuracy: 0.9141, test_Accuracy: 0.9464\n",
      "Epoch: [ 0] [   86/  468] time: 61.1160, train_loss: 0.16933909, train_accuracy: 0.9219, test_Accuracy: 0.9473\n",
      "Epoch: [ 0] [   87/  468] time: 61.9015, train_loss: 0.18480355, train_accuracy: 0.9141, test_Accuracy: 0.9473\n",
      "Epoch: [ 0] [   88/  468] time: 62.5832, train_loss: 0.09813455, train_accuracy: 0.9531, test_Accuracy: 0.9471\n",
      "Epoch: [ 0] [   89/  468] time: 63.2648, train_loss: 0.12283978, train_accuracy: 0.9609, test_Accuracy: 0.9451\n",
      "Epoch: [ 0] [   90/  468] time: 63.8146, train_loss: 0.15379781, train_accuracy: 0.9609, test_Accuracy: 0.9441\n",
      "Epoch: [ 0] [   91/  468] time: 64.5637, train_loss: 0.08251641, train_accuracy: 0.9922, test_Accuracy: 0.9451\n",
      "Epoch: [ 0] [   92/  468] time: 65.1810, train_loss: 0.05463416, train_accuracy: 0.9844, test_Accuracy: 0.9440\n",
      "Epoch: [ 0] [   93/  468] time: 65.7442, train_loss: 0.11220920, train_accuracy: 0.9219, test_Accuracy: 0.9440\n",
      "Epoch: [ 0] [   94/  468] time: 66.4598, train_loss: 0.11256714, train_accuracy: 0.9453, test_Accuracy: 0.9443\n",
      "Epoch: [ 0] [   95/  468] time: 67.2182, train_loss: 0.09925130, train_accuracy: 0.9531, test_Accuracy: 0.9429\n",
      "Epoch: [ 0] [   96/  468] time: 67.9567, train_loss: 0.20342383, train_accuracy: 0.9219, test_Accuracy: 0.9439\n",
      "Epoch: [ 0] [   97/  468] time: 68.7304, train_loss: 0.12956595, train_accuracy: 0.9609, test_Accuracy: 0.9445\n",
      "Epoch: [ 0] [   98/  468] time: 69.4443, train_loss: 0.09568164, train_accuracy: 0.9609, test_Accuracy: 0.9455\n",
      "Epoch: [ 0] [   99/  468] time: 70.0651, train_loss: 0.05980282, train_accuracy: 0.9766, test_Accuracy: 0.9457\n",
      "Epoch: [ 0] [  100/  468] time: 70.8255, train_loss: 0.19753899, train_accuracy: 0.9297, test_Accuracy: 0.9471\n",
      "Epoch: [ 0] [  101/  468] time: 71.4860, train_loss: 0.08437028, train_accuracy: 0.9766, test_Accuracy: 0.9488\n",
      "Epoch: [ 0] [  102/  468] time: 72.1405, train_loss: 0.15144591, train_accuracy: 0.9453, test_Accuracy: 0.9504\n",
      "Epoch: [ 0] [  103/  468] time: 72.8950, train_loss: 0.04964989, train_accuracy: 0.9844, test_Accuracy: 0.9513\n",
      "Epoch: [ 0] [  104/  468] time: 73.4488, train_loss: 0.11022412, train_accuracy: 0.9688, test_Accuracy: 0.9516\n",
      "Epoch: [ 0] [  105/  468] time: 74.1794, train_loss: 0.16057310, train_accuracy: 0.9609, test_Accuracy: 0.9523\n",
      "Epoch: [ 0] [  106/  468] time: 74.8337, train_loss: 0.13242397, train_accuracy: 0.9609, test_Accuracy: 0.9528\n",
      "Epoch: [ 0] [  107/  468] time: 75.6186, train_loss: 0.13436209, train_accuracy: 0.9766, test_Accuracy: 0.9529\n",
      "Epoch: [ 0] [  108/  468] time: 76.2235, train_loss: 0.06026157, train_accuracy: 0.9766, test_Accuracy: 0.9541\n",
      "Epoch: [ 0] [  109/  468] time: 76.8201, train_loss: 0.08086041, train_accuracy: 0.9766, test_Accuracy: 0.9538\n",
      "Epoch: [ 0] [  110/  468] time: 77.4934, train_loss: 0.14166270, train_accuracy: 0.9688, test_Accuracy: 0.9529\n",
      "Epoch: [ 0] [  111/  468] time: 78.2925, train_loss: 0.07550307, train_accuracy: 0.9531, test_Accuracy: 0.9526\n",
      "Epoch: [ 0] [  112/  468] time: 79.0014, train_loss: 0.09139693, train_accuracy: 0.9609, test_Accuracy: 0.9512\n",
      "Epoch: [ 0] [  113/  468] time: 79.6723, train_loss: 0.16817153, train_accuracy: 0.9609, test_Accuracy: 0.9501\n",
      "Epoch: [ 0] [  114/  468] time: 80.3870, train_loss: 0.19765306, train_accuracy: 0.9453, test_Accuracy: 0.9493\n",
      "Epoch: [ 0] [  115/  468] time: 81.1048, train_loss: 0.07451216, train_accuracy: 0.9609, test_Accuracy: 0.9478\n",
      "Epoch: [ 0] [  116/  468] time: 81.7554, train_loss: 0.11485855, train_accuracy: 0.9609, test_Accuracy: 0.9466\n",
      "Epoch: [ 0] [  117/  468] time: 82.3917, train_loss: 0.13275670, train_accuracy: 0.9766, test_Accuracy: 0.9457\n",
      "Epoch: [ 0] [  118/  468] time: 83.0559, train_loss: 0.15395918, train_accuracy: 0.9453, test_Accuracy: 0.9455\n",
      "Epoch: [ 0] [  119/  468] time: 83.7554, train_loss: 0.13901263, train_accuracy: 0.9609, test_Accuracy: 0.9467\n",
      "Epoch: [ 0] [  120/  468] time: 84.4618, train_loss: 0.08392823, train_accuracy: 0.9688, test_Accuracy: 0.9483\n",
      "Epoch: [ 0] [  121/  468] time: 85.2163, train_loss: 0.17972594, train_accuracy: 0.9531, test_Accuracy: 0.9515\n",
      "Epoch: [ 0] [  122/  468] time: 85.9503, train_loss: 0.06222523, train_accuracy: 0.9922, test_Accuracy: 0.9509\n",
      "Epoch: [ 0] [  123/  468] time: 86.6714, train_loss: 0.06233963, train_accuracy: 0.9844, test_Accuracy: 0.9527\n",
      "Epoch: [ 0] [  124/  468] time: 87.3851, train_loss: 0.05008748, train_accuracy: 0.9688, test_Accuracy: 0.9533\n",
      "Epoch: [ 0] [  125/  468] time: 87.9973, train_loss: 0.11268435, train_accuracy: 0.9531, test_Accuracy: 0.9553\n",
      "Epoch: [ 0] [  126/  468] time: 88.5826, train_loss: 0.11749744, train_accuracy: 0.9531, test_Accuracy: 0.9557\n",
      "Epoch: [ 0] [  127/  468] time: 89.3577, train_loss: 0.08179180, train_accuracy: 0.9844, test_Accuracy: 0.9560\n",
      "Epoch: [ 0] [  128/  468] time: 90.0457, train_loss: 0.03292646, train_accuracy: 0.9844, test_Accuracy: 0.9557\n",
      "Epoch: [ 0] [  129/  468] time: 90.6264, train_loss: 0.06968250, train_accuracy: 0.9766, test_Accuracy: 0.9549\n",
      "Epoch: [ 0] [  130/  468] time: 91.2816, train_loss: 0.15327664, train_accuracy: 0.9219, test_Accuracy: 0.9543\n",
      "Epoch: [ 0] [  131/  468] time: 91.8525, train_loss: 0.08123964, train_accuracy: 0.9688, test_Accuracy: 0.9540\n",
      "Epoch: [ 0] [  132/  468] time: 92.5142, train_loss: 0.06281212, train_accuracy: 0.9766, test_Accuracy: 0.9541\n",
      "Epoch: [ 0] [  133/  468] time: 93.2609, train_loss: 0.09134784, train_accuracy: 0.9766, test_Accuracy: 0.9522\n",
      "Epoch: [ 0] [  134/  468] time: 93.9331, train_loss: 0.13143089, train_accuracy: 0.9375, test_Accuracy: 0.9513\n",
      "Epoch: [ 0] [  135/  468] time: 94.6350, train_loss: 0.09391910, train_accuracy: 0.9688, test_Accuracy: 0.9503\n",
      "Epoch: [ 0] [  136/  468] time: 95.3362, train_loss: 0.14435735, train_accuracy: 0.9531, test_Accuracy: 0.9507\n",
      "Epoch: [ 0] [  137/  468] time: 95.8928, train_loss: 0.07452278, train_accuracy: 0.9766, test_Accuracy: 0.9506\n",
      "Epoch: [ 0] [  138/  468] time: 96.5387, train_loss: 0.15471630, train_accuracy: 0.9453, test_Accuracy: 0.9520\n",
      "Epoch: [ 0] [  139/  468] time: 97.1872, train_loss: 0.14608271, train_accuracy: 0.9297, test_Accuracy: 0.9541\n",
      "Epoch: [ 0] [  140/  468] time: 97.8094, train_loss: 0.20271745, train_accuracy: 0.9375, test_Accuracy: 0.9558\n",
      "Epoch: [ 0] [  141/  468] time: 98.5507, train_loss: 0.04834566, train_accuracy: 0.9922, test_Accuracy: 0.9575\n",
      "Epoch: [ 0] [  142/  468] time: 99.3567, train_loss: 0.06337813, train_accuracy: 0.9922, test_Accuracy: 0.9583\n",
      "Epoch: [ 0] [  143/  468] time: 100.0050, train_loss: 0.05546976, train_accuracy: 0.9844, test_Accuracy: 0.9580\n",
      "Epoch: [ 0] [  144/  468] time: 100.5889, train_loss: 0.08307916, train_accuracy: 0.9922, test_Accuracy: 0.9567\n",
      "Epoch: [ 0] [  145/  468] time: 101.3308, train_loss: 0.03316782, train_accuracy: 0.9922, test_Accuracy: 0.9563\n",
      "Epoch: [ 0] [  146/  468] time: 102.1253, train_loss: 0.11436512, train_accuracy: 0.9609, test_Accuracy: 0.9552\n",
      "Epoch: [ 0] [  147/  468] time: 102.6766, train_loss: 0.09576402, train_accuracy: 0.9766, test_Accuracy: 0.9542\n",
      "Epoch: [ 0] [  148/  468] time: 103.2910, train_loss: 0.08698666, train_accuracy: 0.9844, test_Accuracy: 0.9537\n",
      "Epoch: [ 0] [  149/  468] time: 103.9420, train_loss: 0.11059934, train_accuracy: 0.9688, test_Accuracy: 0.9520\n",
      "Epoch: [ 0] [  150/  468] time: 104.6130, train_loss: 0.14113429, train_accuracy: 0.9453, test_Accuracy: 0.9519\n",
      "Epoch: [ 0] [  151/  468] time: 105.2501, train_loss: 0.13889682, train_accuracy: 0.9453, test_Accuracy: 0.9533\n",
      "Epoch: [ 0] [  152/  468] time: 106.0035, train_loss: 0.14137352, train_accuracy: 0.9453, test_Accuracy: 0.9540\n",
      "Epoch: [ 0] [  153/  468] time: 106.6139, train_loss: 0.03647290, train_accuracy: 0.9766, test_Accuracy: 0.9546\n",
      "Epoch: [ 0] [  154/  468] time: 107.4178, train_loss: 0.07196786, train_accuracy: 0.9688, test_Accuracy: 0.9540\n",
      "Epoch: [ 0] [  155/  468] time: 108.1175, train_loss: 0.03681887, train_accuracy: 0.9766, test_Accuracy: 0.9539\n",
      "Epoch: [ 0] [  156/  468] time: 108.7028, train_loss: 0.05335142, train_accuracy: 0.9766, test_Accuracy: 0.9541\n",
      "Epoch: [ 0] [  157/  468] time: 109.3374, train_loss: 0.06986955, train_accuracy: 0.9609, test_Accuracy: 0.9546\n",
      "Epoch: [ 0] [  158/  468] time: 110.0529, train_loss: 0.07012924, train_accuracy: 0.9766, test_Accuracy: 0.9537\n",
      "Epoch: [ 0] [  159/  468] time: 110.7599, train_loss: 0.08928666, train_accuracy: 0.9766, test_Accuracy: 0.9538\n",
      "Epoch: [ 0] [  160/  468] time: 111.4680, train_loss: 0.13115710, train_accuracy: 0.9609, test_Accuracy: 0.9530\n",
      "Epoch: [ 0] [  161/  468] time: 112.0716, train_loss: 0.04548813, train_accuracy: 0.9766, test_Accuracy: 0.9538\n",
      "Epoch: [ 0] [  162/  468] time: 112.7799, train_loss: 0.12929964, train_accuracy: 0.9688, test_Accuracy: 0.9548\n",
      "Epoch: [ 0] [  163/  468] time: 113.4295, train_loss: 0.11009499, train_accuracy: 0.9297, test_Accuracy: 0.9562\n",
      "Epoch: [ 0] [  164/  468] time: 114.1920, train_loss: 0.04802837, train_accuracy: 0.9922, test_Accuracy: 0.9571\n",
      "Epoch: [ 0] [  165/  468] time: 114.7399, train_loss: 0.06842713, train_accuracy: 0.9844, test_Accuracy: 0.9576\n",
      "Epoch: [ 0] [  166/  468] time: 115.3348, train_loss: 0.16356498, train_accuracy: 0.9453, test_Accuracy: 0.9572\n",
      "Epoch: [ 0] [  167/  468] time: 115.9471, train_loss: 0.05805834, train_accuracy: 1.0000, test_Accuracy: 0.9578\n",
      "Epoch: [ 0] [  168/  468] time: 116.6201, train_loss: 0.16335872, train_accuracy: 0.9375, test_Accuracy: 0.9559\n",
      "Epoch: [ 0] [  169/  468] time: 117.3421, train_loss: 0.13157174, train_accuracy: 0.9609, test_Accuracy: 0.9547\n",
      "Epoch: [ 0] [  170/  468] time: 117.9465, train_loss: 0.10908435, train_accuracy: 0.9766, test_Accuracy: 0.9537\n",
      "Epoch: [ 0] [  171/  468] time: 118.5639, train_loss: 0.08327985, train_accuracy: 0.9531, test_Accuracy: 0.9535\n",
      "Epoch: [ 0] [  172/  468] time: 119.1671, train_loss: 0.10285287, train_accuracy: 0.9609, test_Accuracy: 0.9536\n",
      "Epoch: [ 0] [  173/  468] time: 119.7715, train_loss: 0.11913291, train_accuracy: 0.9609, test_Accuracy: 0.9552\n",
      "Epoch: [ 0] [  174/  468] time: 120.4402, train_loss: 0.09930310, train_accuracy: 0.9453, test_Accuracy: 0.9574\n",
      "Epoch: [ 0] [  175/  468] time: 121.2378, train_loss: 0.04969289, train_accuracy: 0.9766, test_Accuracy: 0.9598\n",
      "Epoch: [ 0] [  176/  468] time: 121.8655, train_loss: 0.17172520, train_accuracy: 0.9531, test_Accuracy: 0.9613\n",
      "Epoch: [ 0] [  177/  468] time: 122.5676, train_loss: 0.11242764, train_accuracy: 0.9688, test_Accuracy: 0.9620\n",
      "Epoch: [ 0] [  178/  468] time: 123.2577, train_loss: 0.05732317, train_accuracy: 0.9844, test_Accuracy: 0.9615\n",
      "Epoch: [ 0] [  179/  468] time: 123.9530, train_loss: 0.11390969, train_accuracy: 0.9609, test_Accuracy: 0.9621\n",
      "Epoch: [ 0] [  180/  468] time: 124.6377, train_loss: 0.15259349, train_accuracy: 0.9609, test_Accuracy: 0.9607\n",
      "Epoch: [ 0] [  181/  468] time: 125.3201, train_loss: 0.13348675, train_accuracy: 0.9688, test_Accuracy: 0.9604\n",
      "Epoch: [ 0] [  182/  468] time: 125.9887, train_loss: 0.06936458, train_accuracy: 0.9766, test_Accuracy: 0.9600\n",
      "Epoch: [ 0] [  183/  468] time: 126.5783, train_loss: 0.09276107, train_accuracy: 0.9688, test_Accuracy: 0.9601\n",
      "Epoch: [ 0] [  184/  468] time: 127.2476, train_loss: 0.04936432, train_accuracy: 0.9844, test_Accuracy: 0.9606\n",
      "Epoch: [ 0] [  185/  468] time: 127.7174, train_loss: 0.05892342, train_accuracy: 0.9766, test_Accuracy: 0.9598\n",
      "Epoch: [ 0] [  186/  468] time: 128.3049, train_loss: 0.12827206, train_accuracy: 0.9609, test_Accuracy: 0.9597\n",
      "Epoch: [ 0] [  187/  468] time: 129.0479, train_loss: 0.07288121, train_accuracy: 0.9844, test_Accuracy: 0.9595\n",
      "Epoch: [ 0] [  188/  468] time: 129.7145, train_loss: 0.06963988, train_accuracy: 0.9766, test_Accuracy: 0.9586\n",
      "Epoch: [ 0] [  189/  468] time: 130.5008, train_loss: 0.06315818, train_accuracy: 0.9844, test_Accuracy: 0.9569\n",
      "Epoch: [ 0] [  190/  468] time: 131.1386, train_loss: 0.10049308, train_accuracy: 0.9219, test_Accuracy: 0.9570\n",
      "Epoch: [ 0] [  191/  468] time: 131.9010, train_loss: 0.11390968, train_accuracy: 0.9531, test_Accuracy: 0.9556\n",
      "Epoch: [ 0] [  192/  468] time: 132.5104, train_loss: 0.15797599, train_accuracy: 0.9375, test_Accuracy: 0.9542\n",
      "Epoch: [ 0] [  193/  468] time: 133.0829, train_loss: 0.19137534, train_accuracy: 0.9375, test_Accuracy: 0.9543\n",
      "Epoch: [ 0] [  194/  468] time: 133.7305, train_loss: 0.05143250, train_accuracy: 0.9922, test_Accuracy: 0.9524\n",
      "Epoch: [ 0] [  195/  468] time: 134.3807, train_loss: 0.11383257, train_accuracy: 0.9766, test_Accuracy: 0.9519\n",
      "Epoch: [ 0] [  196/  468] time: 135.0631, train_loss: 0.08224106, train_accuracy: 0.9766, test_Accuracy: 0.9526\n",
      "Epoch: [ 0] [  197/  468] time: 135.6510, train_loss: 0.08856806, train_accuracy: 0.9688, test_Accuracy: 0.9532\n",
      "Epoch: [ 0] [  198/  468] time: 136.3714, train_loss: 0.10459330, train_accuracy: 0.9766, test_Accuracy: 0.9530\n",
      "Epoch: [ 0] [  199/  468] time: 137.1235, train_loss: 0.08864300, train_accuracy: 0.9609, test_Accuracy: 0.9545\n",
      "Epoch: [ 0] [  200/  468] time: 137.6877, train_loss: 0.08205739, train_accuracy: 0.9766, test_Accuracy: 0.9561\n",
      "Epoch: [ 0] [  201/  468] time: 138.2697, train_loss: 0.13511239, train_accuracy: 0.9531, test_Accuracy: 0.9574\n",
      "Epoch: [ 0] [  202/  468] time: 138.9673, train_loss: 0.07544538, train_accuracy: 0.9766, test_Accuracy: 0.9570\n",
      "Epoch: [ 0] [  203/  468] time: 139.6019, train_loss: 0.13376284, train_accuracy: 0.9609, test_Accuracy: 0.9589\n",
      "Epoch: [ 0] [  204/  468] time: 140.3321, train_loss: 0.06295872, train_accuracy: 0.9922, test_Accuracy: 0.9606\n",
      "Epoch: [ 0] [  205/  468] time: 140.9458, train_loss: 0.21527946, train_accuracy: 0.9609, test_Accuracy: 0.9612\n",
      "Epoch: [ 0] [  206/  468] time: 141.5282, train_loss: 0.12031546, train_accuracy: 0.9609, test_Accuracy: 0.9613\n",
      "Epoch: [ 0] [  207/  468] time: 142.2228, train_loss: 0.09057175, train_accuracy: 0.9766, test_Accuracy: 0.9628\n",
      "Epoch: [ 0] [  208/  468] time: 142.9631, train_loss: 0.10973674, train_accuracy: 0.9766, test_Accuracy: 0.9630\n",
      "Epoch: [ 0] [  209/  468] time: 143.6550, train_loss: 0.08760460, train_accuracy: 0.9609, test_Accuracy: 0.9644\n",
      "Epoch: [ 0] [  210/  468] time: 144.3193, train_loss: 0.10092185, train_accuracy: 0.9766, test_Accuracy: 0.9636\n",
      "Epoch: [ 0] [  211/  468] time: 145.0235, train_loss: 0.10177996, train_accuracy: 0.9766, test_Accuracy: 0.9637\n",
      "Epoch: [ 0] [  212/  468] time: 145.6277, train_loss: 0.10309370, train_accuracy: 0.9688, test_Accuracy: 0.9640\n",
      "Epoch: [ 0] [  213/  468] time: 146.3323, train_loss: 0.04880822, train_accuracy: 0.9844, test_Accuracy: 0.9639\n",
      "Epoch: [ 0] [  214/  468] time: 147.0081, train_loss: 0.11874299, train_accuracy: 0.9531, test_Accuracy: 0.9631\n",
      "Epoch: [ 0] [  215/  468] time: 147.6205, train_loss: 0.12880261, train_accuracy: 0.9766, test_Accuracy: 0.9629\n",
      "Epoch: [ 0] [  216/  468] time: 148.3097, train_loss: 0.09089403, train_accuracy: 0.9688, test_Accuracy: 0.9627\n",
      "Epoch: [ 0] [  217/  468] time: 148.8715, train_loss: 0.13642222, train_accuracy: 0.9531, test_Accuracy: 0.9629\n",
      "Epoch: [ 0] [  218/  468] time: 149.6025, train_loss: 0.12241057, train_accuracy: 0.9688, test_Accuracy: 0.9624\n",
      "Epoch: [ 0] [  219/  468] time: 150.2451, train_loss: 0.04443483, train_accuracy: 0.9844, test_Accuracy: 0.9629\n",
      "Epoch: [ 0] [  220/  468] time: 150.9428, train_loss: 0.12271342, train_accuracy: 0.9531, test_Accuracy: 0.9631\n",
      "Epoch: [ 0] [  221/  468] time: 151.6115, train_loss: 0.10427508, train_accuracy: 0.9844, test_Accuracy: 0.9647\n",
      "Epoch: [ 0] [  222/  468] time: 152.3486, train_loss: 0.04300285, train_accuracy: 0.9766, test_Accuracy: 0.9645\n",
      "Epoch: [ 0] [  223/  468] time: 153.0159, train_loss: 0.06356932, train_accuracy: 0.9844, test_Accuracy: 0.9643\n",
      "Epoch: [ 0] [  224/  468] time: 153.5986, train_loss: 0.04986073, train_accuracy: 1.0000, test_Accuracy: 0.9643\n",
      "Epoch: [ 0] [  225/  468] time: 154.1997, train_loss: 0.12104169, train_accuracy: 0.9766, test_Accuracy: 0.9643\n",
      "Epoch: [ 0] [  226/  468] time: 154.8087, train_loss: 0.11449488, train_accuracy: 0.9609, test_Accuracy: 0.9644\n",
      "Epoch: [ 0] [  227/  468] time: 155.5092, train_loss: 0.10938435, train_accuracy: 0.9609, test_Accuracy: 0.9639\n",
      "Epoch: [ 0] [  228/  468] time: 156.1894, train_loss: 0.13114896, train_accuracy: 0.9688, test_Accuracy: 0.9641\n",
      "Epoch: [ 0] [  229/  468] time: 156.8676, train_loss: 0.02956719, train_accuracy: 0.9844, test_Accuracy: 0.9634\n",
      "Epoch: [ 0] [  230/  468] time: 157.6020, train_loss: 0.07966848, train_accuracy: 0.9688, test_Accuracy: 0.9632\n",
      "Epoch: [ 0] [  231/  468] time: 158.3367, train_loss: 0.06047951, train_accuracy: 0.9844, test_Accuracy: 0.9643\n",
      "Epoch: [ 0] [  232/  468] time: 158.9082, train_loss: 0.13991082, train_accuracy: 0.9688, test_Accuracy: 0.9639\n",
      "Epoch: [ 0] [  233/  468] time: 159.4669, train_loss: 0.06905517, train_accuracy: 0.9766, test_Accuracy: 0.9647\n",
      "Epoch: [ 0] [  234/  468] time: 160.2156, train_loss: 0.08893594, train_accuracy: 0.9688, test_Accuracy: 0.9647\n",
      "Epoch: [ 0] [  235/  468] time: 160.8742, train_loss: 0.04463880, train_accuracy: 0.9844, test_Accuracy: 0.9640\n",
      "Epoch: [ 0] [  236/  468] time: 161.6252, train_loss: 0.08657025, train_accuracy: 0.9453, test_Accuracy: 0.9631\n",
      "Epoch: [ 0] [  237/  468] time: 162.3266, train_loss: 0.06706942, train_accuracy: 0.9688, test_Accuracy: 0.9621\n",
      "Epoch: [ 0] [  238/  468] time: 163.0366, train_loss: 0.09011038, train_accuracy: 0.9766, test_Accuracy: 0.9619\n",
      "Epoch: [ 0] [  239/  468] time: 163.6588, train_loss: 0.10702685, train_accuracy: 0.9688, test_Accuracy: 0.9614\n",
      "Epoch: [ 0] [  240/  468] time: 164.3950, train_loss: 0.06383733, train_accuracy: 1.0000, test_Accuracy: 0.9604\n",
      "Epoch: [ 0] [  241/  468] time: 165.1325, train_loss: 0.09334537, train_accuracy: 0.9688, test_Accuracy: 0.9601\n",
      "Epoch: [ 0] [  242/  468] time: 165.9005, train_loss: 0.04649609, train_accuracy: 0.9766, test_Accuracy: 0.9598\n",
      "Epoch: [ 0] [  243/  468] time: 166.5785, train_loss: 0.05284949, train_accuracy: 0.9688, test_Accuracy: 0.9595\n",
      "Epoch: [ 0] [  244/  468] time: 167.2808, train_loss: 0.09123685, train_accuracy: 0.9688, test_Accuracy: 0.9590\n",
      "Epoch: [ 0] [  245/  468] time: 167.8237, train_loss: 0.06892475, train_accuracy: 0.9766, test_Accuracy: 0.9579\n",
      "Epoch: [ 0] [  246/  468] time: 168.4351, train_loss: 0.06700833, train_accuracy: 0.9609, test_Accuracy: 0.9588\n",
      "Epoch: [ 0] [  247/  468] time: 169.0685, train_loss: 0.08643517, train_accuracy: 0.9609, test_Accuracy: 0.9583\n",
      "Epoch: [ 0] [  248/  468] time: 169.6204, train_loss: 0.06283265, train_accuracy: 0.9688, test_Accuracy: 0.9574\n",
      "Epoch: [ 0] [  249/  468] time: 170.3075, train_loss: 0.10771643, train_accuracy: 0.9688, test_Accuracy: 0.9570\n",
      "Epoch: [ 0] [  250/  468] time: 171.0238, train_loss: 0.14149925, train_accuracy: 0.9609, test_Accuracy: 0.9572\n",
      "Epoch: [ 0] [  251/  468] time: 171.7113, train_loss: 0.06337883, train_accuracy: 0.9844, test_Accuracy: 0.9571\n",
      "Epoch: [ 0] [  252/  468] time: 172.4033, train_loss: 0.12554730, train_accuracy: 0.9453, test_Accuracy: 0.9583\n",
      "Epoch: [ 0] [  253/  468] time: 173.1499, train_loss: 0.04839912, train_accuracy: 0.9688, test_Accuracy: 0.9588\n",
      "Epoch: [ 0] [  254/  468] time: 173.7758, train_loss: 0.07694821, train_accuracy: 0.9688, test_Accuracy: 0.9598\n",
      "Epoch: [ 0] [  255/  468] time: 174.3746, train_loss: 0.06088535, train_accuracy: 0.9844, test_Accuracy: 0.9603\n",
      "Epoch: [ 0] [  256/  468] time: 175.0687, train_loss: 0.05477296, train_accuracy: 0.9766, test_Accuracy: 0.9614\n",
      "Epoch: [ 0] [  257/  468] time: 175.7967, train_loss: 0.03701895, train_accuracy: 0.9766, test_Accuracy: 0.9617\n",
      "Epoch: [ 0] [  258/  468] time: 176.5427, train_loss: 0.07563749, train_accuracy: 0.9609, test_Accuracy: 0.9629\n",
      "Epoch: [ 0] [  259/  468] time: 177.1873, train_loss: 0.09600101, train_accuracy: 0.9453, test_Accuracy: 0.9630\n",
      "Epoch: [ 0] [  260/  468] time: 177.8779, train_loss: 0.08709352, train_accuracy: 0.9844, test_Accuracy: 0.9637\n",
      "Epoch: [ 0] [  261/  468] time: 178.5299, train_loss: 0.09465097, train_accuracy: 0.9766, test_Accuracy: 0.9624\n",
      "Epoch: [ 0] [  262/  468] time: 179.1147, train_loss: 0.10095559, train_accuracy: 0.9531, test_Accuracy: 0.9621\n",
      "Epoch: [ 0] [  263/  468] time: 179.8531, train_loss: 0.06296206, train_accuracy: 0.9766, test_Accuracy: 0.9601\n",
      "Epoch: [ 0] [  264/  468] time: 180.5919, train_loss: 0.07136127, train_accuracy: 0.9375, test_Accuracy: 0.9574\n",
      "Epoch: [ 0] [  265/  468] time: 181.2467, train_loss: 0.12940358, train_accuracy: 0.9297, test_Accuracy: 0.9555\n",
      "Epoch: [ 0] [  266/  468] time: 182.0309, train_loss: 0.05928047, train_accuracy: 0.9766, test_Accuracy: 0.9520\n",
      "Epoch: [ 0] [  267/  468] time: 182.7252, train_loss: 0.17912716, train_accuracy: 0.9219, test_Accuracy: 0.9545\n",
      "Epoch: [ 0] [  268/  468] time: 183.3967, train_loss: 0.07998788, train_accuracy: 0.9688, test_Accuracy: 0.9566\n",
      "Epoch: [ 0] [  269/  468] time: 184.0179, train_loss: 0.05331543, train_accuracy: 0.9766, test_Accuracy: 0.9572\n",
      "Epoch: [ 0] [  270/  468] time: 184.7423, train_loss: 0.13080582, train_accuracy: 0.9609, test_Accuracy: 0.9579\n",
      "Epoch: [ 0] [  271/  468] time: 185.4781, train_loss: 0.08415094, train_accuracy: 0.9609, test_Accuracy: 0.9588\n",
      "Epoch: [ 0] [  272/  468] time: 186.0645, train_loss: 0.07786477, train_accuracy: 0.9844, test_Accuracy: 0.9573\n",
      "Epoch: [ 0] [  273/  468] time: 186.8167, train_loss: 0.06384912, train_accuracy: 0.9844, test_Accuracy: 0.9573\n",
      "Epoch: [ 0] [  274/  468] time: 187.5458, train_loss: 0.06932776, train_accuracy: 0.9844, test_Accuracy: 0.9574\n",
      "Epoch: [ 0] [  275/  468] time: 188.2022, train_loss: 0.11288540, train_accuracy: 0.9766, test_Accuracy: 0.9581\n",
      "Epoch: [ 0] [  276/  468] time: 188.7705, train_loss: 0.15382893, train_accuracy: 0.9766, test_Accuracy: 0.9590\n",
      "Epoch: [ 0] [  277/  468] time: 189.3919, train_loss: 0.09517558, train_accuracy: 0.9688, test_Accuracy: 0.9600\n",
      "Epoch: [ 0] [  278/  468] time: 190.1565, train_loss: 0.11797781, train_accuracy: 0.9844, test_Accuracy: 0.9603\n",
      "Epoch: [ 0] [  279/  468] time: 190.8729, train_loss: 0.10608258, train_accuracy: 0.9453, test_Accuracy: 0.9621\n",
      "Epoch: [ 0] [  280/  468] time: 191.5834, train_loss: 0.04257114, train_accuracy: 0.9766, test_Accuracy: 0.9618\n",
      "Epoch: [ 0] [  281/  468] time: 192.1917, train_loss: 0.06098468, train_accuracy: 0.9688, test_Accuracy: 0.9623\n",
      "Epoch: [ 0] [  282/  468] time: 192.8363, train_loss: 0.05307063, train_accuracy: 0.9688, test_Accuracy: 0.9618\n",
      "Epoch: [ 0] [  283/  468] time: 193.5622, train_loss: 0.08521695, train_accuracy: 0.9766, test_Accuracy: 0.9615\n",
      "Epoch: [ 0] [  284/  468] time: 194.2373, train_loss: 0.09721893, train_accuracy: 0.9609, test_Accuracy: 0.9610\n",
      "Epoch: [ 0] [  285/  468] time: 194.8537, train_loss: 0.03616147, train_accuracy: 0.9766, test_Accuracy: 0.9601\n",
      "Epoch: [ 0] [  286/  468] time: 195.4724, train_loss: 0.05952517, train_accuracy: 0.9844, test_Accuracy: 0.9611\n",
      "Epoch: [ 0] [  287/  468] time: 196.2198, train_loss: 0.11677387, train_accuracy: 0.9531, test_Accuracy: 0.9618\n",
      "Epoch: [ 0] [  288/  468] time: 196.9425, train_loss: 0.08237277, train_accuracy: 0.9531, test_Accuracy: 0.9632\n",
      "Epoch: [ 0] [  289/  468] time: 197.6820, train_loss: 0.07298543, train_accuracy: 0.9766, test_Accuracy: 0.9640\n",
      "Epoch: [ 0] [  290/  468] time: 198.4314, train_loss: 0.12085094, train_accuracy: 0.9766, test_Accuracy: 0.9636\n",
      "Epoch: [ 0] [  291/  468] time: 199.0603, train_loss: 0.04631646, train_accuracy: 0.9844, test_Accuracy: 0.9630\n",
      "Epoch: [ 0] [  292/  468] time: 199.7048, train_loss: 0.10268747, train_accuracy: 0.9531, test_Accuracy: 0.9639\n",
      "Epoch: [ 0] [  293/  468] time: 200.3285, train_loss: 0.06238950, train_accuracy: 0.9766, test_Accuracy: 0.9645\n",
      "Epoch: [ 0] [  294/  468] time: 200.9894, train_loss: 0.05673531, train_accuracy: 0.9922, test_Accuracy: 0.9640\n",
      "Epoch: [ 0] [  295/  468] time: 201.5803, train_loss: 0.08108363, train_accuracy: 0.9531, test_Accuracy: 0.9634\n",
      "Epoch: [ 0] [  296/  468] time: 202.3490, train_loss: 0.03855415, train_accuracy: 0.9844, test_Accuracy: 0.9637\n",
      "Epoch: [ 0] [  297/  468] time: 203.0163, train_loss: 0.12125564, train_accuracy: 0.9531, test_Accuracy: 0.9627\n",
      "Epoch: [ 0] [  298/  468] time: 203.7553, train_loss: 0.08598581, train_accuracy: 0.9531, test_Accuracy: 0.9623\n",
      "Epoch: [ 0] [  299/  468] time: 204.4429, train_loss: 0.07480349, train_accuracy: 0.9688, test_Accuracy: 0.9626\n",
      "Epoch: [ 0] [  300/  468] time: 205.1207, train_loss: 0.03702186, train_accuracy: 0.9688, test_Accuracy: 0.9632\n",
      "Epoch: [ 0] [  301/  468] time: 205.8111, train_loss: 0.03625770, train_accuracy: 1.0000, test_Accuracy: 0.9629\n",
      "Epoch: [ 0] [  302/  468] time: 206.4042, train_loss: 0.10964753, train_accuracy: 0.9531, test_Accuracy: 0.9633\n",
      "Epoch: [ 0] [  303/  468] time: 206.9976, train_loss: 0.07250279, train_accuracy: 0.9766, test_Accuracy: 0.9632\n",
      "Epoch: [ 0] [  304/  468] time: 207.6571, train_loss: 0.08608062, train_accuracy: 0.9844, test_Accuracy: 0.9662\n",
      "Epoch: [ 0] [  305/  468] time: 208.3644, train_loss: 0.06771036, train_accuracy: 0.9688, test_Accuracy: 0.9664\n",
      "Epoch: [ 0] [  306/  468] time: 209.0172, train_loss: 0.06614597, train_accuracy: 0.9844, test_Accuracy: 0.9673\n",
      "Epoch: [ 0] [  307/  468] time: 209.7412, train_loss: 0.04621911, train_accuracy: 0.9844, test_Accuracy: 0.9669\n",
      "Epoch: [ 0] [  308/  468] time: 210.4146, train_loss: 0.07607715, train_accuracy: 0.9766, test_Accuracy: 0.9663\n",
      "Epoch: [ 0] [  309/  468] time: 210.9846, train_loss: 0.09978026, train_accuracy: 0.9688, test_Accuracy: 0.9655\n",
      "Epoch: [ 0] [  310/  468] time: 211.7019, train_loss: 0.05943024, train_accuracy: 0.9766, test_Accuracy: 0.9644\n",
      "Epoch: [ 0] [  311/  468] time: 212.2864, train_loss: 0.05265046, train_accuracy: 0.9844, test_Accuracy: 0.9648\n",
      "Epoch: [ 0] [  312/  468] time: 212.9822, train_loss: 0.07675970, train_accuracy: 0.9688, test_Accuracy: 0.9656\n",
      "Epoch: [ 0] [  313/  468] time: 213.6612, train_loss: 0.09085212, train_accuracy: 0.9766, test_Accuracy: 0.9665\n",
      "Epoch: [ 0] [  314/  468] time: 214.2461, train_loss: 0.05590416, train_accuracy: 0.9688, test_Accuracy: 0.9682\n",
      "Epoch: [ 0] [  315/  468] time: 214.8981, train_loss: 0.03379161, train_accuracy: 1.0000, test_Accuracy: 0.9679\n",
      "Epoch: [ 0] [  316/  468] time: 215.6006, train_loss: 0.03287147, train_accuracy: 0.9766, test_Accuracy: 0.9681\n",
      "Epoch: [ 0] [  317/  468] time: 216.3226, train_loss: 0.03421836, train_accuracy: 0.9922, test_Accuracy: 0.9670\n",
      "Epoch: [ 0] [  318/  468] time: 217.1258, train_loss: 0.07728634, train_accuracy: 0.9766, test_Accuracy: 0.9651\n",
      "Epoch: [ 0] [  319/  468] time: 217.9211, train_loss: 0.13408408, train_accuracy: 0.9688, test_Accuracy: 0.9641\n",
      "Epoch: [ 0] [  320/  468] time: 218.5688, train_loss: 0.05582063, train_accuracy: 0.9766, test_Accuracy: 0.9638\n",
      "Epoch: [ 0] [  321/  468] time: 219.2466, train_loss: 0.13575837, train_accuracy: 0.9766, test_Accuracy: 0.9631\n",
      "Epoch: [ 0] [  322/  468] time: 219.9349, train_loss: 0.11788802, train_accuracy: 0.9766, test_Accuracy: 0.9629\n",
      "Epoch: [ 0] [  323/  468] time: 220.7144, train_loss: 0.05283915, train_accuracy: 0.9844, test_Accuracy: 0.9618\n",
      "Epoch: [ 0] [  324/  468] time: 221.3547, train_loss: 0.08901358, train_accuracy: 0.9688, test_Accuracy: 0.9612\n",
      "Epoch: [ 0] [  325/  468] time: 222.0787, train_loss: 0.09773392, train_accuracy: 0.9844, test_Accuracy: 0.9613\n",
      "Epoch: [ 0] [  326/  468] time: 222.6810, train_loss: 0.07770600, train_accuracy: 0.9688, test_Accuracy: 0.9622\n",
      "Epoch: [ 0] [  327/  468] time: 223.4422, train_loss: 0.08886354, train_accuracy: 0.9609, test_Accuracy: 0.9626\n",
      "Epoch: [ 0] [  328/  468] time: 224.0418, train_loss: 0.06539547, train_accuracy: 0.9844, test_Accuracy: 0.9630\n",
      "Epoch: [ 0] [  329/  468] time: 224.6610, train_loss: 0.06568410, train_accuracy: 0.9688, test_Accuracy: 0.9633\n",
      "Epoch: [ 0] [  330/  468] time: 225.3330, train_loss: 0.04791314, train_accuracy: 0.9844, test_Accuracy: 0.9631\n",
      "Epoch: [ 0] [  331/  468] time: 226.0408, train_loss: 0.08513408, train_accuracy: 0.9766, test_Accuracy: 0.9637\n",
      "Epoch: [ 0] [  332/  468] time: 226.7564, train_loss: 0.07949679, train_accuracy: 0.9766, test_Accuracy: 0.9633\n",
      "Epoch: [ 0] [  333/  468] time: 227.6379, train_loss: 0.05179018, train_accuracy: 0.9688, test_Accuracy: 0.9633\n",
      "Epoch: [ 0] [  334/  468] time: 228.4397, train_loss: 0.13139665, train_accuracy: 0.9609, test_Accuracy: 0.9632\n",
      "Epoch: [ 0] [  335/  468] time: 229.0707, train_loss: 0.19649693, train_accuracy: 0.9297, test_Accuracy: 0.9641\n",
      "Epoch: [ 0] [  336/  468] time: 229.6705, train_loss: 0.07841024, train_accuracy: 0.9922, test_Accuracy: 0.9639\n",
      "Epoch: [ 0] [  337/  468] time: 230.3036, train_loss: 0.04957681, train_accuracy: 0.9766, test_Accuracy: 0.9627\n",
      "Epoch: [ 0] [  338/  468] time: 230.9816, train_loss: 0.04165537, train_accuracy: 0.9688, test_Accuracy: 0.9621\n",
      "Epoch: [ 0] [  339/  468] time: 231.7542, train_loss: 0.06019108, train_accuracy: 0.9844, test_Accuracy: 0.9608\n",
      "Epoch: [ 0] [  340/  468] time: 232.4604, train_loss: 0.02714059, train_accuracy: 0.9844, test_Accuracy: 0.9610\n",
      "Epoch: [ 0] [  341/  468] time: 233.2130, train_loss: 0.07728606, train_accuracy: 0.9844, test_Accuracy: 0.9594\n",
      "Epoch: [ 0] [  342/  468] time: 233.9718, train_loss: 0.05967522, train_accuracy: 0.9688, test_Accuracy: 0.9588\n",
      "Epoch: [ 0] [  343/  468] time: 234.7317, train_loss: 0.13335375, train_accuracy: 0.9531, test_Accuracy: 0.9599\n",
      "Epoch: [ 0] [  344/  468] time: 235.4430, train_loss: 0.20380795, train_accuracy: 0.9453, test_Accuracy: 0.9630\n",
      "Epoch: [ 0] [  345/  468] time: 236.2299, train_loss: 0.06691815, train_accuracy: 0.9531, test_Accuracy: 0.9650\n",
      "Epoch: [ 0] [  346/  468] time: 236.9046, train_loss: 0.06462216, train_accuracy: 0.9844, test_Accuracy: 0.9650\n",
      "Epoch: [ 0] [  347/  468] time: 237.6107, train_loss: 0.06812809, train_accuracy: 0.9688, test_Accuracy: 0.9657\n",
      "Epoch: [ 0] [  348/  468] time: 238.1691, train_loss: 0.03189319, train_accuracy: 0.9766, test_Accuracy: 0.9655\n",
      "Epoch: [ 0] [  349/  468] time: 238.9389, train_loss: 0.03409208, train_accuracy: 0.9766, test_Accuracy: 0.9657\n",
      "Epoch: [ 0] [  350/  468] time: 239.4937, train_loss: 0.06042095, train_accuracy: 0.9688, test_Accuracy: 0.9648\n",
      "Epoch: [ 0] [  351/  468] time: 240.0992, train_loss: 0.05441787, train_accuracy: 0.9766, test_Accuracy: 0.9643\n",
      "Epoch: [ 0] [  352/  468] time: 240.7374, train_loss: 0.07975249, train_accuracy: 0.9844, test_Accuracy: 0.9640\n",
      "Epoch: [ 0] [  353/  468] time: 241.3856, train_loss: 0.13357633, train_accuracy: 0.9453, test_Accuracy: 0.9640\n",
      "Epoch: [ 0] [  354/  468] time: 242.1130, train_loss: 0.04916102, train_accuracy: 0.9688, test_Accuracy: 0.9632\n",
      "Epoch: [ 0] [  355/  468] time: 242.7252, train_loss: 0.04081810, train_accuracy: 0.9844, test_Accuracy: 0.9627\n",
      "Epoch: [ 0] [  356/  468] time: 243.3192, train_loss: 0.04933041, train_accuracy: 0.9922, test_Accuracy: 0.9629\n",
      "Epoch: [ 0] [  357/  468] time: 243.9942, train_loss: 0.18476191, train_accuracy: 0.9609, test_Accuracy: 0.9638\n",
      "Epoch: [ 0] [  358/  468] time: 244.6163, train_loss: 0.03438644, train_accuracy: 0.9922, test_Accuracy: 0.9654\n",
      "Epoch: [ 0] [  359/  468] time: 245.2045, train_loss: 0.07285838, train_accuracy: 0.9766, test_Accuracy: 0.9667\n",
      "Epoch: [ 0] [  360/  468] time: 245.8750, train_loss: 0.11103754, train_accuracy: 0.9688, test_Accuracy: 0.9679\n",
      "Epoch: [ 0] [  361/  468] time: 246.4594, train_loss: 0.05481048, train_accuracy: 0.9844, test_Accuracy: 0.9686\n",
      "Epoch: [ 0] [  362/  468] time: 247.1903, train_loss: 0.09756608, train_accuracy: 0.9688, test_Accuracy: 0.9688\n",
      "Epoch: [ 0] [  363/  468] time: 248.0679, train_loss: 0.04289299, train_accuracy: 0.9766, test_Accuracy: 0.9689\n",
      "Epoch: [ 0] [  364/  468] time: 248.7697, train_loss: 0.03403250, train_accuracy: 0.9922, test_Accuracy: 0.9683\n",
      "Epoch: [ 0] [  365/  468] time: 249.3787, train_loss: 0.04160696, train_accuracy: 0.9922, test_Accuracy: 0.9682\n",
      "Epoch: [ 0] [  366/  468] time: 250.1296, train_loss: 0.06000477, train_accuracy: 0.9844, test_Accuracy: 0.9679\n",
      "Epoch: [ 0] [  367/  468] time: 250.8544, train_loss: 0.05860392, train_accuracy: 0.9688, test_Accuracy: 0.9682\n",
      "Epoch: [ 0] [  368/  468] time: 251.5595, train_loss: 0.04354589, train_accuracy: 0.9766, test_Accuracy: 0.9689\n",
      "Epoch: [ 0] [  369/  468] time: 252.2776, train_loss: 0.06686028, train_accuracy: 0.9844, test_Accuracy: 0.9694\n",
      "Epoch: [ 0] [  370/  468] time: 252.9502, train_loss: 0.08050317, train_accuracy: 0.9844, test_Accuracy: 0.9690\n",
      "Epoch: [ 0] [  371/  468] time: 253.7410, train_loss: 0.05140321, train_accuracy: 0.9844, test_Accuracy: 0.9686\n",
      "Epoch: [ 0] [  372/  468] time: 254.4474, train_loss: 0.06362550, train_accuracy: 0.9844, test_Accuracy: 0.9688\n",
      "Epoch: [ 0] [  373/  468] time: 255.0603, train_loss: 0.06010653, train_accuracy: 0.9922, test_Accuracy: 0.9684\n",
      "Epoch: [ 0] [  374/  468] time: 255.5397, train_loss: 0.07526005, train_accuracy: 0.9688, test_Accuracy: 0.9681\n",
      "Epoch: [ 0] [  375/  468] time: 256.0371, train_loss: 0.08344273, train_accuracy: 0.9609, test_Accuracy: 0.9678\n",
      "Epoch: [ 0] [  376/  468] time: 256.7321, train_loss: 0.10241675, train_accuracy: 0.9688, test_Accuracy: 0.9672\n",
      "Epoch: [ 0] [  377/  468] time: 257.3485, train_loss: 0.08815560, train_accuracy: 0.9688, test_Accuracy: 0.9662\n",
      "Epoch: [ 0] [  378/  468] time: 257.9583, train_loss: 0.07421006, train_accuracy: 0.9688, test_Accuracy: 0.9650\n",
      "Epoch: [ 0] [  379/  468] time: 258.6175, train_loss: 0.09157377, train_accuracy: 0.9688, test_Accuracy: 0.9636\n",
      "Epoch: [ 0] [  380/  468] time: 259.2484, train_loss: 0.04630586, train_accuracy: 0.9844, test_Accuracy: 0.9614\n",
      "Epoch: [ 0] [  381/  468] time: 259.9179, train_loss: 0.04828179, train_accuracy: 0.9844, test_Accuracy: 0.9601\n",
      "Epoch: [ 0] [  382/  468] time: 260.5300, train_loss: 0.03725149, train_accuracy: 0.9766, test_Accuracy: 0.9594\n",
      "Epoch: [ 0] [  383/  468] time: 261.7369, train_loss: 0.08373509, train_accuracy: 0.9297, test_Accuracy: 0.9583\n",
      "Epoch: [ 0] [  384/  468] time: 262.4834, train_loss: 0.07964275, train_accuracy: 0.9609, test_Accuracy: 0.9576\n",
      "Epoch: [ 0] [  385/  468] time: 263.1978, train_loss: 0.05494954, train_accuracy: 0.9688, test_Accuracy: 0.9554\n",
      "Epoch: [ 0] [  386/  468] time: 263.9180, train_loss: 0.06646729, train_accuracy: 0.9766, test_Accuracy: 0.9540\n",
      "Epoch: [ 0] [  387/  468] time: 264.4514, train_loss: 0.03070001, train_accuracy: 0.9844, test_Accuracy: 0.9532\n",
      "Epoch: [ 0] [  388/  468] time: 265.1524, train_loss: 0.07574252, train_accuracy: 0.9609, test_Accuracy: 0.9528\n",
      "Epoch: [ 0] [  389/  468] time: 265.7896, train_loss: 0.12247865, train_accuracy: 0.9375, test_Accuracy: 0.9535\n",
      "Epoch: [ 0] [  390/  468] time: 266.4144, train_loss: 0.04663403, train_accuracy: 0.9844, test_Accuracy: 0.9537\n",
      "Epoch: [ 0] [  391/  468] time: 267.0266, train_loss: 0.09377765, train_accuracy: 0.9453, test_Accuracy: 0.9544\n",
      "Epoch: [ 0] [  392/  468] time: 267.7136, train_loss: 0.04698352, train_accuracy: 0.9688, test_Accuracy: 0.9553\n",
      "Epoch: [ 0] [  393/  468] time: 268.3208, train_loss: 0.06871545, train_accuracy: 0.9688, test_Accuracy: 0.9568\n",
      "Epoch: [ 0] [  394/  468] time: 268.9968, train_loss: 0.08138256, train_accuracy: 0.9766, test_Accuracy: 0.9580\n",
      "Epoch: [ 0] [  395/  468] time: 269.6140, train_loss: 0.12564583, train_accuracy: 0.9688, test_Accuracy: 0.9589\n",
      "Epoch: [ 0] [  396/  468] time: 270.2475, train_loss: 0.07851817, train_accuracy: 0.9531, test_Accuracy: 0.9614\n",
      "Epoch: [ 0] [  397/  468] time: 270.9180, train_loss: 0.16040665, train_accuracy: 0.9531, test_Accuracy: 0.9627\n",
      "Epoch: [ 0] [  398/  468] time: 271.5733, train_loss: 0.06223428, train_accuracy: 0.9844, test_Accuracy: 0.9636\n",
      "Epoch: [ 0] [  399/  468] time: 272.1671, train_loss: 0.06089347, train_accuracy: 0.9766, test_Accuracy: 0.9644\n",
      "Epoch: [ 0] [  400/  468] time: 272.8842, train_loss: 0.04824080, train_accuracy: 0.9766, test_Accuracy: 0.9652\n",
      "Epoch: [ 0] [  401/  468] time: 273.5348, train_loss: 0.03879559, train_accuracy: 0.9844, test_Accuracy: 0.9662\n",
      "Epoch: [ 0] [  402/  468] time: 274.1378, train_loss: 0.14840861, train_accuracy: 0.9531, test_Accuracy: 0.9680\n",
      "Epoch: [ 0] [  403/  468] time: 274.7352, train_loss: 0.03254579, train_accuracy: 1.0000, test_Accuracy: 0.9681\n",
      "Epoch: [ 0] [  404/  468] time: 275.4917, train_loss: 0.03653532, train_accuracy: 1.0000, test_Accuracy: 0.9693\n",
      "Epoch: [ 0] [  405/  468] time: 276.0861, train_loss: 0.07862192, train_accuracy: 0.9766, test_Accuracy: 0.9698\n",
      "Epoch: [ 0] [  406/  468] time: 276.7381, train_loss: 0.08559810, train_accuracy: 0.9922, test_Accuracy: 0.9696\n",
      "Epoch: [ 0] [  407/  468] time: 277.4502, train_loss: 0.09058286, train_accuracy: 0.9766, test_Accuracy: 0.9705\n",
      "Epoch: [ 0] [  408/  468] time: 278.1547, train_loss: 0.03484397, train_accuracy: 0.9844, test_Accuracy: 0.9710\n",
      "Epoch: [ 0] [  409/  468] time: 278.8406, train_loss: 0.02418350, train_accuracy: 0.9844, test_Accuracy: 0.9717\n",
      "Epoch: [ 0] [  410/  468] time: 279.5232, train_loss: 0.10014992, train_accuracy: 0.9688, test_Accuracy: 0.9720\n",
      "Epoch: [ 0] [  411/  468] time: 280.1490, train_loss: 0.06987220, train_accuracy: 0.9766, test_Accuracy: 0.9726\n",
      "Epoch: [ 0] [  412/  468] time: 280.7699, train_loss: 0.03696178, train_accuracy: 1.0000, test_Accuracy: 0.9728\n",
      "Epoch: [ 0] [  413/  468] time: 281.4786, train_loss: 0.05157215, train_accuracy: 1.0000, test_Accuracy: 0.9729\n",
      "Epoch: [ 0] [  414/  468] time: 282.0798, train_loss: 0.02651899, train_accuracy: 1.0000, test_Accuracy: 0.9717\n",
      "Epoch: [ 0] [  415/  468] time: 282.7441, train_loss: 0.04434667, train_accuracy: 0.9844, test_Accuracy: 0.9713\n",
      "Epoch: [ 0] [  416/  468] time: 283.3567, train_loss: 0.09676050, train_accuracy: 0.9766, test_Accuracy: 0.9702\n",
      "Epoch: [ 0] [  417/  468] time: 284.0608, train_loss: 0.04480298, train_accuracy: 0.9766, test_Accuracy: 0.9709\n",
      "Epoch: [ 0] [  418/  468] time: 284.5651, train_loss: 0.07859319, train_accuracy: 0.9688, test_Accuracy: 0.9719\n",
      "Epoch: [ 0] [  419/  468] time: 285.1013, train_loss: 0.15062362, train_accuracy: 0.9688, test_Accuracy: 0.9716\n",
      "Epoch: [ 0] [  420/  468] time: 285.5765, train_loss: 0.05170573, train_accuracy: 0.9766, test_Accuracy: 0.9710\n",
      "Epoch: [ 0] [  421/  468] time: 499.3186, train_loss: 0.08921999, train_accuracy: 0.9844, test_Accuracy: 0.9708\n",
      "Epoch: [ 0] [  422/  468] time: 501.2978, train_loss: 0.07845768, train_accuracy: 0.9609, test_Accuracy: 0.9711\n",
      "Epoch: [ 0] [  423/  468] time: 502.2635, train_loss: 0.05172268, train_accuracy: 0.9766, test_Accuracy: 0.9714\n",
      "Epoch: [ 0] [  424/  468] time: 503.1933, train_loss: 0.07561726, train_accuracy: 0.9844, test_Accuracy: 0.9713\n",
      "Epoch: [ 0] [  425/  468] time: 503.9594, train_loss: 0.09249184, train_accuracy: 0.9766, test_Accuracy: 0.9701\n",
      "Epoch: [ 0] [  426/  468] time: 504.6457, train_loss: 0.04065428, train_accuracy: 0.9844, test_Accuracy: 0.9691\n",
      "Epoch: [ 0] [  427/  468] time: 505.2318, train_loss: 0.02176042, train_accuracy: 0.9844, test_Accuracy: 0.9673\n",
      "Epoch: [ 0] [  428/  468] time: 505.8463, train_loss: 0.09526608, train_accuracy: 0.9844, test_Accuracy: 0.9648\n",
      "Epoch: [ 0] [  429/  468] time: 506.4899, train_loss: 0.04409396, train_accuracy: 0.9922, test_Accuracy: 0.9641\n",
      "Epoch: [ 0] [  430/  468] time: 507.2353, train_loss: 0.13508555, train_accuracy: 0.9375, test_Accuracy: 0.9626\n",
      "Epoch: [ 0] [  431/  468] time: 507.9550, train_loss: 0.03688682, train_accuracy: 0.9766, test_Accuracy: 0.9626\n",
      "Epoch: [ 0] [  432/  468] time: 508.6338, train_loss: 0.09486733, train_accuracy: 0.9609, test_Accuracy: 0.9614\n",
      "Epoch: [ 0] [  433/  468] time: 509.3864, train_loss: 0.02956546, train_accuracy: 0.9844, test_Accuracy: 0.9616\n",
      "Epoch: [ 0] [  434/  468] time: 510.0147, train_loss: 0.10037135, train_accuracy: 0.9766, test_Accuracy: 0.9609\n",
      "Epoch: [ 0] [  435/  468] time: 510.6633, train_loss: 0.09868997, train_accuracy: 0.9688, test_Accuracy: 0.9600\n",
      "Epoch: [ 0] [  436/  468] time: 511.3107, train_loss: 0.05922910, train_accuracy: 0.9609, test_Accuracy: 0.9597\n",
      "Epoch: [ 0] [  437/  468] time: 511.9281, train_loss: 0.07810225, train_accuracy: 0.9844, test_Accuracy: 0.9603\n",
      "Epoch: [ 0] [  438/  468] time: 512.4881, train_loss: 0.06775058, train_accuracy: 0.9609, test_Accuracy: 0.9617\n",
      "Epoch: [ 0] [  439/  468] time: 512.9860, train_loss: 0.04448014, train_accuracy: 0.9922, test_Accuracy: 0.9611\n",
      "Epoch: [ 0] [  440/  468] time: 513.5197, train_loss: 0.15971342, train_accuracy: 0.9688, test_Accuracy: 0.9613\n",
      "Epoch: [ 0] [  441/  468] time: 514.0623, train_loss: 0.03691919, train_accuracy: 0.9844, test_Accuracy: 0.9598\n",
      "Epoch: [ 0] [  442/  468] time: 514.6445, train_loss: 0.10132648, train_accuracy: 0.9609, test_Accuracy: 0.9590\n",
      "Epoch: [ 0] [  443/  468] time: 515.2739, train_loss: 0.11301972, train_accuracy: 0.9531, test_Accuracy: 0.9598\n",
      "Epoch: [ 0] [  444/  468] time: 515.8932, train_loss: 0.05612099, train_accuracy: 0.9688, test_Accuracy: 0.9613\n",
      "Epoch: [ 0] [  445/  468] time: 516.4747, train_loss: 0.09009154, train_accuracy: 0.9688, test_Accuracy: 0.9623\n",
      "Epoch: [ 0] [  446/  468] time: 517.0254, train_loss: 0.06797371, train_accuracy: 0.9766, test_Accuracy: 0.9631\n",
      "Epoch: [ 0] [  447/  468] time: 517.5416, train_loss: 0.08921000, train_accuracy: 0.9688, test_Accuracy: 0.9626\n",
      "Epoch: [ 0] [  448/  468] time: 518.1306, train_loss: 0.08014982, train_accuracy: 0.9688, test_Accuracy: 0.9628\n",
      "Epoch: [ 0] [  449/  468] time: 518.7722, train_loss: 0.09357028, train_accuracy: 0.9766, test_Accuracy: 0.9632\n",
      "Epoch: [ 0] [  450/  468] time: 519.3621, train_loss: 0.10975447, train_accuracy: 0.9531, test_Accuracy: 0.9630\n",
      "Epoch: [ 0] [  451/  468] time: 519.9352, train_loss: 0.16393076, train_accuracy: 0.9609, test_Accuracy: 0.9656\n",
      "Epoch: [ 0] [  452/  468] time: 520.5638, train_loss: 0.07498038, train_accuracy: 0.9609, test_Accuracy: 0.9668\n",
      "Epoch: [ 0] [  453/  468] time: 521.3028, train_loss: 0.12707488, train_accuracy: 0.9609, test_Accuracy: 0.9679\n",
      "Epoch: [ 0] [  454/  468] time: 522.0581, train_loss: 0.03395246, train_accuracy: 0.9922, test_Accuracy: 0.9688\n",
      "Epoch: [ 0] [  455/  468] time: 522.6406, train_loss: 0.16703451, train_accuracy: 0.9531, test_Accuracy: 0.9683\n",
      "Epoch: [ 0] [  456/  468] time: 523.3199, train_loss: 0.05177133, train_accuracy: 0.9766, test_Accuracy: 0.9673\n",
      "Epoch: [ 0] [  457/  468] time: 523.9626, train_loss: 0.07700650, train_accuracy: 0.9531, test_Accuracy: 0.9670\n",
      "Epoch: [ 0] [  458/  468] time: 524.6537, train_loss: 0.10292777, train_accuracy: 0.9375, test_Accuracy: 0.9660\n",
      "Epoch: [ 0] [  459/  468] time: 525.2433, train_loss: 0.04087880, train_accuracy: 0.9844, test_Accuracy: 0.9650\n",
      "Epoch: [ 0] [  460/  468] time: 525.8767, train_loss: 0.10622184, train_accuracy: 0.9375, test_Accuracy: 0.9640\n",
      "Epoch: [ 0] [  461/  468] time: 526.4278, train_loss: 0.02856546, train_accuracy: 0.9922, test_Accuracy: 0.9630\n",
      "Epoch: [ 0] [  462/  468] time: 527.1032, train_loss: 0.07705617, train_accuracy: 0.9609, test_Accuracy: 0.9631\n",
      "Epoch: [ 0] [  463/  468] time: 527.8332, train_loss: 0.09200602, train_accuracy: 0.9609, test_Accuracy: 0.9633\n",
      "Epoch: [ 0] [  464/  468] time: 528.3963, train_loss: 0.07107918, train_accuracy: 0.9688, test_Accuracy: 0.9634\n",
      "Epoch: [ 0] [  465/  468] time: 528.9541, train_loss: 0.03723445, train_accuracy: 0.9844, test_Accuracy: 0.9640\n",
      "Epoch: [ 0] [  466/  468] time: 529.5432, train_loss: 0.14253843, train_accuracy: 0.9609, test_Accuracy: 0.9647\n",
      "Epoch: [ 0] [  467/  468] time: 530.2690, train_loss: 0.09442705, train_accuracy: 0.9766, test_Accuracy: 0.9659\n"
     ]
    }
   ],
   "source": [
    "if train_flag :\n",
    "\n",
    "    checkpoint = tf.train.Checkpoint(dnn=network)\n",
    "\n",
    "    summary_writer = tf.summary.create_file_writer(logdir=logs_dir)\n",
    "    start_time = time()\n",
    "\n",
    "    could_load, checkpoint_counter = load(network, checkpoint_dir)    \n",
    "\n",
    "    if could_load:\n",
    "        start_epoch = (int)(checkpoint_counter / training_iterations)        \n",
    "        counter = checkpoint_counter        \n",
    "        print(\" [*] Load SUCCESS\")\n",
    "    else:\n",
    "        start_epoch = 0\n",
    "        start_iteration = 0\n",
    "        counter = 0\n",
    "        print(\" [!] Load failed...\")\n",
    "    \n",
    "    with summary_writer.as_default():\n",
    "        for epoch in range(start_epoch, training_epochs):\n",
    "            for idx, (train_input, train_label) in enumerate(train_dataset):            \n",
    "                grads = grad(network, train_input, train_label)\n",
    "                optimizer.apply_gradients(grads_and_vars=zip(grads, network.trainable_variables))\n",
    "\n",
    "                train_loss = loss_fn(network, train_input, train_label)\n",
    "                train_accuracy = accuracy_fn(network, train_input, train_label)\n",
    "                \n",
    "                for test_input, test_label in test_dataset:                \n",
    "                    test_accuracy = accuracy_fn(network, test_input, test_label)\n",
    "\n",
    "                tf.summary.scalar(name='train_loss', data=train_loss, step=counter)\n",
    "                tf.summary.scalar(name='train_accuracy', data=train_accuracy, step=counter)\n",
    "                tf.summary.scalar(name='test_accuracy', data=test_accuracy, step=counter)\n",
    "\n",
    "                print(\n",
    "                    \"Epoch: [%2d] [%5d/%5d] time: %4.4f, train_loss: %.8f, train_accuracy: %.4f, test_Accuracy: %.4f\" \\\n",
    "                    % (epoch, idx, training_iterations, time() - start_time, train_loss, train_accuracy,\n",
    "                       test_accuracy))\n",
    "                counter += 1                \n",
    "        checkpoint.save(file_prefix=checkpoint_prefix + '-{}'.format(counter))\n",
    "        \n",
    "else :\n",
    "    _, _ = load(network, checkpoint_dir)\n",
    "    for test_input, test_label in test_dataset:    \n",
    "        test_accuracy = accuracy_fn(network, test_input, test_label)\n",
    "\n",
    "\n",
    "    print(\"test_Accuracy: %.4f\" % (test_accuracy))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}