Lab10/Lab10-2-2_eunseoChoi.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Xavier initialization\n",
    "### global이 아닌 local minima를 찾는 문제가 있음 => initialization을 통해 해결\n",
    "### relu 함수에 최적화된 HE initialization => 분산=4/Channel_in + Channel_out"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Xavier Deep network version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.5.0\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "from tensorflow.keras.utils import to_categorical\n",
    "from tensorflow.keras.datasets import mnist\n",
    "from time import time\n",
    "import os\n",
    "print(tf.__version__)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Checkpoint function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load(model, checkpoint_dir):\n",
    "    print(\" [*] Reading checkpoints...\")\n",
    "\n",
    "    ckpt = tf.train.get_checkpoint_state(checkpoint_dir)\n",
    "    if ckpt :\n",
    "        ckpt_name = os.path.basename(ckpt.model_checkpoint_path)\n",
    "        checkpoint = tf.train.Checkpoint(dnn=model)\n",
    "        checkpoint.restore(save_path=os.path.join(checkpoint_dir, ckpt_name))\n",
    "        counter = int(ckpt_name.split('-')[1])\n",
    "        print(\" [*] Success to read {}\".format(ckpt_name))\n",
    "        return True, counter\n",
    "    else:\n",
    "        print(\" [*] Failed to find a checkpoint\")\n",
    "        return False, 0\n",
    "\n",
    "def check_folder(dir):\n",
    "    if not os.path.exists(dir):\n",
    "        os.makedirs(dir)\n",
    "    return dir"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data load & pre-processing function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_mnist() :\n",
    "    (train_data, train_labels), (test_data, test_labels) = mnist.load_data()\n",
    "    train_data = np.expand_dims(train_data, axis=-1) # [N, 28, 28] -> [N, 28, 28, 1]\n",
    "    test_data = np.expand_dims(test_data, axis=-1) # [N, 28, 28] -> [N, 28, 28, 1]\n",
    "\n",
    "    train_data, test_data = normalize(train_data, test_data)\n",
    "\n",
    "    train_labels = to_categorical(train_labels, 10) # [N,] -> [N, 10]\n",
    "    test_labels = to_categorical(test_labels, 10) # [N,] -> [N, 10]\n",
    "\n",
    "    return train_data, train_labels, test_data, test_labels\n",
    "\n",
    "def normalize(train_data, test_data):\n",
    "    train_data = train_data.astype(np.float32) / 255.0\n",
    "    test_data = test_data.astype(np.float32) / 255.0\n",
    "\n",
    "    return train_data, test_data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Performance function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def loss_fn(model, images, labels):\n",
    "    logits = model(images, training=True)\n",
    "    loss = tf.reduce_mean(tf.keras.losses.categorical_crossentropy(y_pred=logits, y_true=labels, \n",
    "                                                                   from_logits=True))\n",
    "    return loss\n",
    "\n",
    "def accuracy_fn(model, images, labels):\n",
    "    logits = model(images, training=False)\n",
    "    prediction = tf.equal(tf.argmax(logits, -1), tf.argmax(labels, -1))\n",
    "    accuracy = tf.reduce_mean(tf.cast(prediction, tf.float32))\n",
    "    return accuracy\n",
    "\n",
    "def grad(model, images, labels):\n",
    "    with tf.GradientTape() as tape:\n",
    "        loss = loss_fn(model, images, labels)\n",
    "    return tape.gradient(loss, model.variables)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def flatten() :\n",
    "    return tf.keras.layers.Flatten()\n",
    "\n",
    "def dense(label_dim, weight_init) :\n",
    "    return tf.keras.layers.Dense(units=label_dim, use_bias=True, kernel_initializer=weight_init)\n",
    "\n",
    "def relu() :\n",
    "    return tf.keras.layers.Activation(tf.keras.activations.relu)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create model (class version)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "class create_model_class(tf.keras.Model):\n",
    "    def __init__(self, label_dim):\n",
    "        super(create_model_class, self).__init__()\n",
    "        \n",
    "        weight_init = tf.keras.initializers.glorot_uniform() ## Initializator\n",
    "\n",
    "        self.model = tf.keras.Sequential()\n",
    "        self.model.add(flatten())\n",
    "\n",
    "        for i in range(4):\n",
    "            self.model.add(dense(512, weight_init))\n",
    "            self.model.add(relu())\n",
    "\n",
    "        self.model.add(dense(label_dim, weight_init))\n",
    "\n",
    "    def call(self, x, training=None, mask=None):\n",
    "\n",
    "        x = self.model(x)\n",
    "\n",
    "        return x"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create model (function version)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_model_function(label_dim) :\n",
    "    weight_init = tf.keras.initializers.glorot_uniform()\n",
    "\n",
    "    model = tf.keras.Sequential()\n",
    "    model.add(flatten())\n",
    "\n",
    "    for i in range(4) :\n",
    "        model.add(dense(512, weight_init))\n",
    "        model.add(relu())\n",
    "\n",
    "    model.add(dense(label_dim, weight_init))\n",
    "\n",
    "    return model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Define data & hyper-parameter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\" dataset \"\"\"\n",
    "train_x, train_y, test_x, test_y = load_mnist()\n",
    "\n",
    "\"\"\" parameters \"\"\"\n",
    "learning_rate = 0.001\n",
    "batch_size = 128\n",
    "\n",
    "training_epochs = 1\n",
    "training_iterations = len(train_x) // batch_size\n",
    "\n",
    "label_dim = 10\n",
    "\n",
    "train_flag = True\n",
    "\n",
    "\"\"\" Graph Input using Dataset API \"\"\"\n",
    "train_dataset = tf.data.Dataset.from_tensor_slices((train_x, train_y)).\\\n",
    "    shuffle(buffer_size=100000).\\\n",
    "    prefetch(buffer_size=batch_size).\\\n",
    "    batch(batch_size, drop_remainder=True)\n",
    "\n",
    "test_dataset = tf.data.Dataset.from_tensor_slices((test_x, test_y)).\\\n",
    "    shuffle(buffer_size=100000).\\\n",
    "    prefetch(buffer_size=len(test_x)).\\\n",
    "    batch(len(test_x))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Define model & optimizer & writer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\" Model \"\"\"\n",
    "network = create_model_function(label_dim)\n",
    "\n",
    "\"\"\" Training \"\"\"\n",
    "optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate)\n",
    "\n",
    "\"\"\" Writer \"\"\"\n",
    "checkpoint_dir = 'checkpoints'\n",
    "logs_dir = 'logs'\n",
    "\n",
    "model_dir = 'nn_deep'\n",
    "\n",
    "checkpoint_dir = os.path.join(checkpoint_dir, model_dir)\n",
    "check_folder(checkpoint_dir)\n",
    "checkpoint_prefix = os.path.join(checkpoint_dir, model_dir)\n",
    "logs_dir = os.path.join(logs_dir, model_dir)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Restore checkpoint & start train or test phase"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " [*] Reading checkpoints...\n",
      " [*] Failed to find a checkpoint\n",
      " [!] Load failed...\n",
      "Epoch: [ 0] [    0/  468] time: 0.9258, train_loss: 2.02812147, train_accuracy: 0.6094, test_Accuracy: 0.3990\n",
      "Epoch: [ 0] [    1/  468] time: 1.3499, train_loss: 1.89642477, train_accuracy: 0.5938, test_Accuracy: 0.4280\n",
      "Epoch: [ 0] [    2/  468] time: 1.8846, train_loss: 1.66584527, train_accuracy: 0.7031, test_Accuracy: 0.6344\n",
      "Epoch: [ 0] [    3/  468] time: 2.3100, train_loss: 1.51968563, train_accuracy: 0.7500, test_Accuracy: 0.7219\n",
      "Epoch: [ 0] [    4/  468] time: 2.6941, train_loss: 1.23590767, train_accuracy: 0.7188, test_Accuracy: 0.6958\n",
      "Epoch: [ 0] [    5/  468] time: 3.0764, train_loss: 0.91388392, train_accuracy: 0.7266, test_Accuracy: 0.6855\n",
      "Epoch: [ 0] [    6/  468] time: 3.6876, train_loss: 0.83714324, train_accuracy: 0.8125, test_Accuracy: 0.7773\n",
      "Epoch: [ 0] [    7/  468] time: 4.0460, train_loss: 0.72497058, train_accuracy: 0.8359, test_Accuracy: 0.7597\n",
      "Epoch: [ 0] [    8/  468] time: 4.6698, train_loss: 0.55585551, train_accuracy: 0.8516, test_Accuracy: 0.7695\n",
      "Epoch: [ 0] [    9/  468] time: 5.0229, train_loss: 0.62051022, train_accuracy: 0.7812, test_Accuracy: 0.7801\n",
      "Epoch: [ 0] [   10/  468] time: 5.4723, train_loss: 0.62323785, train_accuracy: 0.8203, test_Accuracy: 0.7792\n",
      "Epoch: [ 0] [   11/  468] time: 5.8136, train_loss: 0.65939832, train_accuracy: 0.8047, test_Accuracy: 0.7905\n",
      "Epoch: [ 0] [   12/  468] time: 6.1832, train_loss: 0.39051425, train_accuracy: 0.8672, test_Accuracy: 0.8021\n",
      "Epoch: [ 0] [   13/  468] time: 6.5821, train_loss: 0.47379956, train_accuracy: 0.8516, test_Accuracy: 0.8252\n",
      "Epoch: [ 0] [   14/  468] time: 6.9809, train_loss: 0.51604372, train_accuracy: 0.8359, test_Accuracy: 0.8597\n",
      "Epoch: [ 0] [   15/  468] time: 7.4693, train_loss: 0.26308298, train_accuracy: 0.9297, test_Accuracy: 0.8473\n",
      "Epoch: [ 0] [   16/  468] time: 7.9845, train_loss: 0.54132056, train_accuracy: 0.8203, test_Accuracy: 0.8237\n",
      "Epoch: [ 0] [   17/  468] time: 8.4109, train_loss: 0.42528412, train_accuracy: 0.8594, test_Accuracy: 0.8138\n",
      "Epoch: [ 0] [   18/  468] time: 8.8536, train_loss: 0.54568541, train_accuracy: 0.8203, test_Accuracy: 0.8243\n",
      "Epoch: [ 0] [   19/  468] time: 9.3073, train_loss: 0.52332914, train_accuracy: 0.8672, test_Accuracy: 0.8714\n",
      "Epoch: [ 0] [   20/  468] time: 9.6537, train_loss: 0.26069105, train_accuracy: 0.9062, test_Accuracy: 0.8505\n",
      "Epoch: [ 0] [   21/  468] time: 10.0207, train_loss: 0.50875753, train_accuracy: 0.8203, test_Accuracy: 0.8351\n",
      "Epoch: [ 0] [   22/  468] time: 10.4031, train_loss: 0.40789530, train_accuracy: 0.8750, test_Accuracy: 0.8376\n",
      "Epoch: [ 0] [   23/  468] time: 10.8957, train_loss: 0.41733664, train_accuracy: 0.9141, test_Accuracy: 0.8644\n",
      "Epoch: [ 0] [   24/  468] time: 11.3946, train_loss: 0.30155712, train_accuracy: 0.8984, test_Accuracy: 0.8885\n",
      "Epoch: [ 0] [   25/  468] time: 11.7867, train_loss: 0.42236710, train_accuracy: 0.8828, test_Accuracy: 0.8697\n",
      "Epoch: [ 0] [   26/  468] time: 12.2694, train_loss: 0.43558660, train_accuracy: 0.8594, test_Accuracy: 0.8444\n",
      "Epoch: [ 0] [   27/  468] time: 12.6273, train_loss: 0.45833337, train_accuracy: 0.8438, test_Accuracy: 0.8691\n",
      "Epoch: [ 0] [   28/  468] time: 12.9879, train_loss: 0.29823622, train_accuracy: 0.8984, test_Accuracy: 0.8939\n",
      "Epoch: [ 0] [   29/  468] time: 13.3683, train_loss: 0.25031954, train_accuracy: 0.9219, test_Accuracy: 0.8967\n",
      "Epoch: [ 0] [   30/  468] time: 13.8108, train_loss: 0.31303316, train_accuracy: 0.9453, test_Accuracy: 0.8808\n",
      "Epoch: [ 0] [   31/  468] time: 14.3741, train_loss: 0.45119646, train_accuracy: 0.8359, test_Accuracy: 0.8654\n",
      "Epoch: [ 0] [   32/  468] time: 14.7769, train_loss: 0.35627759, train_accuracy: 0.8672, test_Accuracy: 0.8671\n",
      "Epoch: [ 0] [   33/  468] time: 15.3071, train_loss: 0.33933607, train_accuracy: 0.8750, test_Accuracy: 0.8906\n",
      "Epoch: [ 0] [   34/  468] time: 15.7169, train_loss: 0.26302510, train_accuracy: 0.9297, test_Accuracy: 0.9021\n",
      "Epoch: [ 0] [   35/  468] time: 16.1009, train_loss: 0.40288115, train_accuracy: 0.8828, test_Accuracy: 0.8891\n",
      "Epoch: [ 0] [   36/  468] time: 16.4758, train_loss: 0.36802375, train_accuracy: 0.8906, test_Accuracy: 0.8761\n",
      "Epoch: [ 0] [   37/  468] time: 16.8502, train_loss: 0.26003379, train_accuracy: 0.9453, test_Accuracy: 0.8857\n",
      "Epoch: [ 0] [   38/  468] time: 17.1965, train_loss: 0.31793979, train_accuracy: 0.8906, test_Accuracy: 0.9065\n",
      "Epoch: [ 0] [   39/  468] time: 17.6345, train_loss: 0.23535013, train_accuracy: 0.9141, test_Accuracy: 0.9064\n",
      "Epoch: [ 0] [   40/  468] time: 18.0516, train_loss: 0.37133768, train_accuracy: 0.8750, test_Accuracy: 0.8955\n",
      "Epoch: [ 0] [   41/  468] time: 18.5356, train_loss: 0.34268233, train_accuracy: 0.8906, test_Accuracy: 0.8910\n",
      "Epoch: [ 0] [   42/  468] time: 18.9811, train_loss: 0.33669150, train_accuracy: 0.9141, test_Accuracy: 0.8965\n",
      "Epoch: [ 0] [   43/  468] time: 19.4032, train_loss: 0.29406512, train_accuracy: 0.9219, test_Accuracy: 0.9058\n",
      "Epoch: [ 0] [   44/  468] time: 19.8369, train_loss: 0.20649493, train_accuracy: 0.9453, test_Accuracy: 0.9096\n",
      "Epoch: [ 0] [   45/  468] time: 20.3055, train_loss: 0.32058623, train_accuracy: 0.9062, test_Accuracy: 0.9078\n",
      "Epoch: [ 0] [   46/  468] time: 20.6788, train_loss: 0.25523990, train_accuracy: 0.9141, test_Accuracy: 0.8992\n",
      "Epoch: [ 0] [   47/  468] time: 21.2364, train_loss: 0.45522365, train_accuracy: 0.8984, test_Accuracy: 0.8956\n",
      "Epoch: [ 0] [   48/  468] time: 21.7426, train_loss: 0.35172045, train_accuracy: 0.8906, test_Accuracy: 0.9057\n",
      "Epoch: [ 0] [   49/  468] time: 22.2508, train_loss: 0.35139424, train_accuracy: 0.8984, test_Accuracy: 0.9185\n",
      "Epoch: [ 0] [   50/  468] time: 22.5857, train_loss: 0.24803409, train_accuracy: 0.9531, test_Accuracy: 0.9202\n",
      "Epoch: [ 0] [   51/  468] time: 22.9412, train_loss: 0.21547967, train_accuracy: 0.9453, test_Accuracy: 0.9114\n",
      "Epoch: [ 0] [   52/  468] time: 23.3483, train_loss: 0.32731280, train_accuracy: 0.8984, test_Accuracy: 0.9125\n",
      "Epoch: [ 0] [   53/  468] time: 23.6751, train_loss: 0.19899292, train_accuracy: 0.9375, test_Accuracy: 0.9157\n",
      "Epoch: [ 0] [   54/  468] time: 24.1225, train_loss: 0.32990319, train_accuracy: 0.9141, test_Accuracy: 0.9180\n",
      "Epoch: [ 0] [   55/  468] time: 24.5931, train_loss: 0.21791190, train_accuracy: 0.9219, test_Accuracy: 0.9200\n",
      "Epoch: [ 0] [   56/  468] time: 24.9951, train_loss: 0.30609477, train_accuracy: 0.8906, test_Accuracy: 0.9200\n",
      "Epoch: [ 0] [   57/  468] time: 25.3622, train_loss: 0.18886995, train_accuracy: 0.9375, test_Accuracy: 0.9162\n",
      "Epoch: [ 0] [   58/  468] time: 25.6800, train_loss: 0.29916388, train_accuracy: 0.9141, test_Accuracy: 0.9152\n",
      "Epoch: [ 0] [   59/  468] time: 26.0902, train_loss: 0.25316837, train_accuracy: 0.8906, test_Accuracy: 0.9198\n",
      "Epoch: [ 0] [   60/  468] time: 26.4751, train_loss: 0.28936994, train_accuracy: 0.9141, test_Accuracy: 0.9258\n",
      "Epoch: [ 0] [   61/  468] time: 26.9032, train_loss: 0.29789144, train_accuracy: 0.8984, test_Accuracy: 0.9263\n",
      "Epoch: [ 0] [   62/  468] time: 27.3873, train_loss: 0.17869630, train_accuracy: 0.9609, test_Accuracy: 0.9192\n",
      "Epoch: [ 0] [   63/  468] time: 27.7220, train_loss: 0.19049358, train_accuracy: 0.9531, test_Accuracy: 0.9195\n",
      "Epoch: [ 0] [   64/  468] time: 28.1245, train_loss: 0.25738037, train_accuracy: 0.9141, test_Accuracy: 0.9253\n",
      "Epoch: [ 0] [   65/  468] time: 28.5962, train_loss: 0.28627872, train_accuracy: 0.8984, test_Accuracy: 0.9274\n",
      "Epoch: [ 0] [   66/  468] time: 28.9174, train_loss: 0.26520336, train_accuracy: 0.9297, test_Accuracy: 0.9211\n",
      "Epoch: [ 0] [   67/  468] time: 29.2495, train_loss: 0.25223699, train_accuracy: 0.9375, test_Accuracy: 0.9191\n",
      "Epoch: [ 0] [   68/  468] time: 29.5879, train_loss: 0.31318569, train_accuracy: 0.9062, test_Accuracy: 0.9245\n",
      "Epoch: [ 0] [   69/  468] time: 29.9496, train_loss: 0.28874701, train_accuracy: 0.9141, test_Accuracy: 0.9248\n",
      "Epoch: [ 0] [   70/  468] time: 30.4016, train_loss: 0.19176742, train_accuracy: 0.9609, test_Accuracy: 0.9276\n",
      "Epoch: [ 0] [   71/  468] time: 30.8477, train_loss: 0.14415595, train_accuracy: 0.9688, test_Accuracy: 0.9215\n",
      "Epoch: [ 0] [   72/  468] time: 31.3035, train_loss: 0.30835098, train_accuracy: 0.9062, test_Accuracy: 0.9198\n",
      "Epoch: [ 0] [   73/  468] time: 31.6414, train_loss: 0.21749753, train_accuracy: 0.9297, test_Accuracy: 0.9259\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [ 0] [   74/  468] time: 31.9933, train_loss: 0.25734606, train_accuracy: 0.9453, test_Accuracy: 0.9331\n",
      "Epoch: [ 0] [   75/  468] time: 32.3259, train_loss: 0.23960394, train_accuracy: 0.9141, test_Accuracy: 0.9285\n",
      "Epoch: [ 0] [   76/  468] time: 32.7200, train_loss: 0.19021530, train_accuracy: 0.9219, test_Accuracy: 0.9117\n",
      "Epoch: [ 0] [   77/  468] time: 33.0492, train_loss: 0.22819486, train_accuracy: 0.9375, test_Accuracy: 0.9113\n",
      "Epoch: [ 0] [   78/  468] time: 33.4821, train_loss: 0.15290941, train_accuracy: 0.9609, test_Accuracy: 0.9223\n",
      "Epoch: [ 0] [   79/  468] time: 33.8945, train_loss: 0.28150165, train_accuracy: 0.9219, test_Accuracy: 0.9350\n",
      "Epoch: [ 0] [   80/  468] time: 34.3008, train_loss: 0.21952650, train_accuracy: 0.9375, test_Accuracy: 0.9275\n",
      "Epoch: [ 0] [   81/  468] time: 34.7439, train_loss: 0.31461194, train_accuracy: 0.9141, test_Accuracy: 0.9185\n",
      "Epoch: [ 0] [   82/  468] time: 35.2436, train_loss: 0.15799782, train_accuracy: 0.9375, test_Accuracy: 0.9160\n",
      "Epoch: [ 0] [   83/  468] time: 35.6219, train_loss: 0.20442721, train_accuracy: 0.9453, test_Accuracy: 0.9223\n",
      "Epoch: [ 0] [   84/  468] time: 36.0022, train_loss: 0.14386936, train_accuracy: 0.9609, test_Accuracy: 0.9364\n",
      "Epoch: [ 0] [   85/  468] time: 36.3368, train_loss: 0.17619842, train_accuracy: 0.9531, test_Accuracy: 0.9405\n",
      "Epoch: [ 0] [   86/  468] time: 36.6664, train_loss: 0.11001732, train_accuracy: 0.9609, test_Accuracy: 0.9385\n",
      "Epoch: [ 0] [   87/  468] time: 37.1011, train_loss: 0.26522192, train_accuracy: 0.9062, test_Accuracy: 0.9353\n",
      "Epoch: [ 0] [   88/  468] time: 37.5475, train_loss: 0.17365605, train_accuracy: 0.9297, test_Accuracy: 0.9367\n",
      "Epoch: [ 0] [   89/  468] time: 37.8816, train_loss: 0.13189995, train_accuracy: 0.9609, test_Accuracy: 0.9356\n",
      "Epoch: [ 0] [   90/  468] time: 38.2852, train_loss: 0.16283517, train_accuracy: 0.9375, test_Accuracy: 0.9353\n",
      "Epoch: [ 0] [   91/  468] time: 38.6856, train_loss: 0.13795371, train_accuracy: 0.9688, test_Accuracy: 0.9304\n",
      "Epoch: [ 0] [   92/  468] time: 39.0989, train_loss: 0.29514128, train_accuracy: 0.9141, test_Accuracy: 0.9329\n",
      "Epoch: [ 0] [   93/  468] time: 39.4953, train_loss: 0.21491671, train_accuracy: 0.9531, test_Accuracy: 0.9399\n",
      "Epoch: [ 0] [   94/  468] time: 39.8368, train_loss: 0.15185522, train_accuracy: 0.9609, test_Accuracy: 0.9411\n",
      "Epoch: [ 0] [   95/  468] time: 40.3045, train_loss: 0.16611078, train_accuracy: 0.9609, test_Accuracy: 0.9404\n",
      "Epoch: [ 0] [   96/  468] time: 40.7823, train_loss: 0.12448362, train_accuracy: 0.9609, test_Accuracy: 0.9359\n",
      "Epoch: [ 0] [   97/  468] time: 41.1979, train_loss: 0.15546489, train_accuracy: 0.9453, test_Accuracy: 0.9364\n",
      "Epoch: [ 0] [   98/  468] time: 41.6432, train_loss: 0.21570593, train_accuracy: 0.9375, test_Accuracy: 0.9347\n",
      "Epoch: [ 0] [   99/  468] time: 41.9711, train_loss: 0.26247469, train_accuracy: 0.9062, test_Accuracy: 0.9352\n",
      "Epoch: [ 0] [  100/  468] time: 42.3464, train_loss: 0.19331631, train_accuracy: 0.9297, test_Accuracy: 0.9355\n",
      "Epoch: [ 0] [  101/  468] time: 42.6859, train_loss: 0.09069660, train_accuracy: 0.9844, test_Accuracy: 0.9326\n",
      "Epoch: [ 0] [  102/  468] time: 43.0375, train_loss: 0.19616607, train_accuracy: 0.9219, test_Accuracy: 0.9328\n",
      "Epoch: [ 0] [  103/  468] time: 43.5289, train_loss: 0.14942601, train_accuracy: 0.9609, test_Accuracy: 0.9308\n",
      "Epoch: [ 0] [  104/  468] time: 43.9140, train_loss: 0.18967956, train_accuracy: 0.9531, test_Accuracy: 0.9366\n",
      "Epoch: [ 0] [  105/  468] time: 44.3723, train_loss: 0.10633375, train_accuracy: 0.9609, test_Accuracy: 0.9380\n",
      "Epoch: [ 0] [  106/  468] time: 44.7349, train_loss: 0.10633491, train_accuracy: 0.9766, test_Accuracy: 0.9394\n",
      "Epoch: [ 0] [  107/  468] time: 45.1598, train_loss: 0.23199856, train_accuracy: 0.9141, test_Accuracy: 0.9383\n",
      "Epoch: [ 0] [  108/  468] time: 45.6290, train_loss: 0.18424277, train_accuracy: 0.9375, test_Accuracy: 0.9346\n",
      "Epoch: [ 0] [  109/  468] time: 45.9623, train_loss: 0.32345706, train_accuracy: 0.9141, test_Accuracy: 0.9382\n",
      "Epoch: [ 0] [  110/  468] time: 46.3506, train_loss: 0.16678257, train_accuracy: 0.9531, test_Accuracy: 0.9447\n",
      "Epoch: [ 0] [  111/  468] time: 46.8325, train_loss: 0.09042094, train_accuracy: 0.9688, test_Accuracy: 0.9425\n",
      "Epoch: [ 0] [  112/  468] time: 47.3365, train_loss: 0.19860235, train_accuracy: 0.9297, test_Accuracy: 0.9408\n",
      "Epoch: [ 0] [  113/  468] time: 47.7847, train_loss: 0.29740453, train_accuracy: 0.9219, test_Accuracy: 0.9371\n",
      "Epoch: [ 0] [  114/  468] time: 48.2829, train_loss: 0.22076201, train_accuracy: 0.9297, test_Accuracy: 0.9372\n",
      "Epoch: [ 0] [  115/  468] time: 48.6227, train_loss: 0.14008556, train_accuracy: 0.9453, test_Accuracy: 0.9401\n",
      "Epoch: [ 0] [  116/  468] time: 49.0000, train_loss: 0.09426758, train_accuracy: 0.9609, test_Accuracy: 0.9399\n",
      "Epoch: [ 0] [  117/  468] time: 49.3381, train_loss: 0.11651802, train_accuracy: 0.9609, test_Accuracy: 0.9378\n",
      "Epoch: [ 0] [  118/  468] time: 49.7885, train_loss: 0.14641938, train_accuracy: 0.9688, test_Accuracy: 0.9340\n",
      "Epoch: [ 0] [  119/  468] time: 50.2771, train_loss: 0.26607984, train_accuracy: 0.9141, test_Accuracy: 0.9345\n",
      "Epoch: [ 0] [  120/  468] time: 50.6067, train_loss: 0.14525571, train_accuracy: 0.9609, test_Accuracy: 0.9419\n",
      "Epoch: [ 0] [  121/  468] time: 51.0266, train_loss: 0.19561192, train_accuracy: 0.9453, test_Accuracy: 0.9449\n",
      "Epoch: [ 0] [  122/  468] time: 51.4633, train_loss: 0.18161654, train_accuracy: 0.9531, test_Accuracy: 0.9442\n",
      "Epoch: [ 0] [  123/  468] time: 51.8721, train_loss: 0.23177768, train_accuracy: 0.9219, test_Accuracy: 0.9418\n",
      "Epoch: [ 0] [  124/  468] time: 52.3411, train_loss: 0.07086547, train_accuracy: 0.9844, test_Accuracy: 0.9406\n",
      "Epoch: [ 0] [  125/  468] time: 52.8151, train_loss: 0.15031788, train_accuracy: 0.9531, test_Accuracy: 0.9394\n",
      "Epoch: [ 0] [  126/  468] time: 53.3819, train_loss: 0.16141790, train_accuracy: 0.9531, test_Accuracy: 0.9439\n",
      "Epoch: [ 0] [  127/  468] time: 53.7339, train_loss: 0.27825654, train_accuracy: 0.9219, test_Accuracy: 0.9420\n",
      "Epoch: [ 0] [  128/  468] time: 54.2073, train_loss: 0.16600770, train_accuracy: 0.9531, test_Accuracy: 0.9434\n",
      "Epoch: [ 0] [  129/  468] time: 54.6818, train_loss: 0.12119852, train_accuracy: 0.9609, test_Accuracy: 0.9417\n",
      "Epoch: [ 0] [  130/  468] time: 55.0523, train_loss: 0.06133670, train_accuracy: 0.9922, test_Accuracy: 0.9384\n",
      "Epoch: [ 0] [  131/  468] time: 55.4129, train_loss: 0.19677517, train_accuracy: 0.9141, test_Accuracy: 0.9351\n",
      "Epoch: [ 0] [  132/  468] time: 55.7347, train_loss: 0.32624930, train_accuracy: 0.9062, test_Accuracy: 0.9417\n",
      "Epoch: [ 0] [  133/  468] time: 56.1230, train_loss: 0.07578508, train_accuracy: 0.9844, test_Accuracy: 0.9452\n",
      "Epoch: [ 0] [  134/  468] time: 56.5247, train_loss: 0.15409312, train_accuracy: 0.9688, test_Accuracy: 0.9491\n",
      "Epoch: [ 0] [  135/  468] time: 56.8383, train_loss: 0.19731545, train_accuracy: 0.9453, test_Accuracy: 0.9497\n",
      "Epoch: [ 0] [  136/  468] time: 57.2689, train_loss: 0.08262183, train_accuracy: 0.9922, test_Accuracy: 0.9435\n",
      "Epoch: [ 0] [  137/  468] time: 57.6372, train_loss: 0.13690773, train_accuracy: 0.9531, test_Accuracy: 0.9344\n",
      "Epoch: [ 0] [  138/  468] time: 58.0316, train_loss: 0.18322200, train_accuracy: 0.9375, test_Accuracy: 0.9359\n",
      "Epoch: [ 0] [  139/  468] time: 58.4779, train_loss: 0.13150883, train_accuracy: 0.9531, test_Accuracy: 0.9377\n",
      "Epoch: [ 0] [  140/  468] time: 58.8890, train_loss: 0.12713379, train_accuracy: 0.9609, test_Accuracy: 0.9422\n",
      "Epoch: [ 0] [  141/  468] time: 59.3350, train_loss: 0.11753762, train_accuracy: 0.9531, test_Accuracy: 0.9457\n",
      "Epoch: [ 0] [  142/  468] time: 59.7796, train_loss: 0.20766415, train_accuracy: 0.9297, test_Accuracy: 0.9445\n",
      "Epoch: [ 0] [  143/  468] time: 60.2356, train_loss: 0.12873220, train_accuracy: 0.9609, test_Accuracy: 0.9400\n",
      "Epoch: [ 0] [  144/  468] time: 60.7150, train_loss: 0.14222141, train_accuracy: 0.9531, test_Accuracy: 0.9346\n",
      "Epoch: [ 0] [  145/  468] time: 61.0953, train_loss: 0.21013810, train_accuracy: 0.9453, test_Accuracy: 0.9354\n",
      "Epoch: [ 0] [  146/  468] time: 61.4934, train_loss: 0.16377215, train_accuracy: 0.9375, test_Accuracy: 0.9449\n",
      "Epoch: [ 0] [  147/  468] time: 61.8247, train_loss: 0.08106306, train_accuracy: 0.9766, test_Accuracy: 0.9479\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [ 0] [  148/  468] time: 62.1719, train_loss: 0.15495758, train_accuracy: 0.9453, test_Accuracy: 0.9433\n",
      "Epoch: [ 0] [  149/  468] time: 62.5161, train_loss: 0.09682118, train_accuracy: 0.9688, test_Accuracy: 0.9347\n",
      "Epoch: [ 0] [  150/  468] time: 62.8571, train_loss: 0.17017466, train_accuracy: 0.9531, test_Accuracy: 0.9366\n",
      "Epoch: [ 0] [  151/  468] time: 63.2884, train_loss: 0.18391971, train_accuracy: 0.9531, test_Accuracy: 0.9391\n",
      "Epoch: [ 0] [  152/  468] time: 63.6605, train_loss: 0.20483783, train_accuracy: 0.9531, test_Accuracy: 0.9418\n",
      "Epoch: [ 0] [  153/  468] time: 64.1583, train_loss: 0.20867747, train_accuracy: 0.9453, test_Accuracy: 0.9470\n",
      "Epoch: [ 0] [  154/  468] time: 64.5618, train_loss: 0.11759378, train_accuracy: 0.9609, test_Accuracy: 0.9452\n",
      "Epoch: [ 0] [  155/  468] time: 64.9894, train_loss: 0.26854244, train_accuracy: 0.9141, test_Accuracy: 0.9411\n",
      "Epoch: [ 0] [  156/  468] time: 65.5332, train_loss: 0.10505683, train_accuracy: 0.9609, test_Accuracy: 0.9376\n",
      "Epoch: [ 0] [  157/  468] time: 65.9424, train_loss: 0.15889817, train_accuracy: 0.9688, test_Accuracy: 0.9346\n",
      "Epoch: [ 0] [  158/  468] time: 66.4480, train_loss: 0.30732667, train_accuracy: 0.9219, test_Accuracy: 0.9307\n",
      "Epoch: [ 0] [  159/  468] time: 66.8369, train_loss: 0.21348423, train_accuracy: 0.9375, test_Accuracy: 0.9323\n",
      "Epoch: [ 0] [  160/  468] time: 67.2279, train_loss: 0.25996658, train_accuracy: 0.9297, test_Accuracy: 0.9423\n",
      "Epoch: [ 0] [  161/  468] time: 67.6826, train_loss: 0.18727523, train_accuracy: 0.9531, test_Accuracy: 0.9439\n",
      "Epoch: [ 0] [  162/  468] time: 68.0261, train_loss: 0.17691329, train_accuracy: 0.9297, test_Accuracy: 0.9407\n",
      "Epoch: [ 0] [  163/  468] time: 68.3927, train_loss: 0.16487667, train_accuracy: 0.9688, test_Accuracy: 0.9361\n",
      "Epoch: [ 0] [  164/  468] time: 68.7282, train_loss: 0.22712868, train_accuracy: 0.9297, test_Accuracy: 0.9336\n",
      "Epoch: [ 0] [  165/  468] time: 69.0881, train_loss: 0.17084309, train_accuracy: 0.9609, test_Accuracy: 0.9326\n",
      "Epoch: [ 0] [  166/  468] time: 69.5105, train_loss: 0.14429918, train_accuracy: 0.9453, test_Accuracy: 0.9351\n",
      "Epoch: [ 0] [  167/  468] time: 69.8360, train_loss: 0.12114835, train_accuracy: 0.9609, test_Accuracy: 0.9421\n",
      "Epoch: [ 0] [  168/  468] time: 70.1918, train_loss: 0.14472188, train_accuracy: 0.9531, test_Accuracy: 0.9456\n",
      "Epoch: [ 0] [  169/  468] time: 70.5665, train_loss: 0.13886112, train_accuracy: 0.9609, test_Accuracy: 0.9493\n",
      "Epoch: [ 0] [  170/  468] time: 70.9631, train_loss: 0.13665164, train_accuracy: 0.9531, test_Accuracy: 0.9522\n",
      "Epoch: [ 0] [  171/  468] time: 71.4122, train_loss: 0.09760745, train_accuracy: 0.9922, test_Accuracy: 0.9501\n",
      "Epoch: [ 0] [  172/  468] time: 71.8334, train_loss: 0.14703839, train_accuracy: 0.9609, test_Accuracy: 0.9477\n",
      "Epoch: [ 0] [  173/  468] time: 72.2963, train_loss: 0.15744290, train_accuracy: 0.9453, test_Accuracy: 0.9467\n",
      "Epoch: [ 0] [  174/  468] time: 72.7845, train_loss: 0.11654750, train_accuracy: 0.9688, test_Accuracy: 0.9460\n",
      "Epoch: [ 0] [  175/  468] time: 73.2117, train_loss: 0.14501974, train_accuracy: 0.9531, test_Accuracy: 0.9446\n",
      "Epoch: [ 0] [  176/  468] time: 73.6533, train_loss: 0.17357638, train_accuracy: 0.9609, test_Accuracy: 0.9441\n",
      "Epoch: [ 0] [  177/  468] time: 74.0955, train_loss: 0.06280127, train_accuracy: 0.9922, test_Accuracy: 0.9482\n",
      "Epoch: [ 0] [  178/  468] time: 74.5013, train_loss: 0.10559725, train_accuracy: 0.9531, test_Accuracy: 0.9528\n",
      "Epoch: [ 0] [  179/  468] time: 74.8170, train_loss: 0.12226695, train_accuracy: 0.9688, test_Accuracy: 0.9534\n",
      "Epoch: [ 0] [  180/  468] time: 75.1775, train_loss: 0.08129657, train_accuracy: 0.9766, test_Accuracy: 0.9518\n",
      "Epoch: [ 0] [  181/  468] time: 75.5326, train_loss: 0.10797435, train_accuracy: 0.9688, test_Accuracy: 0.9512\n",
      "Epoch: [ 0] [  182/  468] time: 75.9191, train_loss: 0.20749286, train_accuracy: 0.9375, test_Accuracy: 0.9518\n",
      "Epoch: [ 0] [  183/  468] time: 76.3656, train_loss: 0.09786822, train_accuracy: 0.9609, test_Accuracy: 0.9526\n",
      "Epoch: [ 0] [  184/  468] time: 76.7259, train_loss: 0.13208966, train_accuracy: 0.9688, test_Accuracy: 0.9540\n",
      "Epoch: [ 0] [  185/  468] time: 77.0754, train_loss: 0.06417471, train_accuracy: 0.9766, test_Accuracy: 0.9548\n",
      "Epoch: [ 0] [  186/  468] time: 77.4456, train_loss: 0.17730960, train_accuracy: 0.9688, test_Accuracy: 0.9580\n",
      "Epoch: [ 0] [  187/  468] time: 77.8219, train_loss: 0.25743642, train_accuracy: 0.9453, test_Accuracy: 0.9592\n",
      "Epoch: [ 0] [  188/  468] time: 78.2291, train_loss: 0.11870486, train_accuracy: 0.9766, test_Accuracy: 0.9590\n",
      "Epoch: [ 0] [  189/  468] time: 78.6314, train_loss: 0.08983512, train_accuracy: 0.9688, test_Accuracy: 0.9581\n",
      "Epoch: [ 0] [  190/  468] time: 79.0627, train_loss: 0.05964101, train_accuracy: 0.9922, test_Accuracy: 0.9559\n",
      "Epoch: [ 0] [  191/  468] time: 79.5014, train_loss: 0.09488717, train_accuracy: 0.9531, test_Accuracy: 0.9524\n",
      "Epoch: [ 0] [  192/  468] time: 79.9019, train_loss: 0.05184063, train_accuracy: 0.9922, test_Accuracy: 0.9509\n",
      "Epoch: [ 0] [  193/  468] time: 80.2655, train_loss: 0.11138333, train_accuracy: 0.9609, test_Accuracy: 0.9532\n",
      "Epoch: [ 0] [  194/  468] time: 80.6232, train_loss: 0.13586652, train_accuracy: 0.9609, test_Accuracy: 0.9549\n",
      "Epoch: [ 0] [  195/  468] time: 80.9404, train_loss: 0.24563777, train_accuracy: 0.9375, test_Accuracy: 0.9561\n",
      "Epoch: [ 0] [  196/  468] time: 81.2875, train_loss: 0.15952778, train_accuracy: 0.9531, test_Accuracy: 0.9556\n",
      "Epoch: [ 0] [  197/  468] time: 81.6513, train_loss: 0.04780570, train_accuracy: 0.9922, test_Accuracy: 0.9525\n",
      "Epoch: [ 0] [  198/  468] time: 82.0814, train_loss: 0.16040099, train_accuracy: 0.9375, test_Accuracy: 0.9490\n",
      "Epoch: [ 0] [  199/  468] time: 82.5009, train_loss: 0.11877906, train_accuracy: 0.9688, test_Accuracy: 0.9437\n",
      "Epoch: [ 0] [  200/  468] time: 82.8259, train_loss: 0.15418163, train_accuracy: 0.9375, test_Accuracy: 0.9411\n",
      "Epoch: [ 0] [  201/  468] time: 83.2562, train_loss: 0.21261832, train_accuracy: 0.9219, test_Accuracy: 0.9444\n",
      "Epoch: [ 0] [  202/  468] time: 83.7171, train_loss: 0.21886176, train_accuracy: 0.9453, test_Accuracy: 0.9471\n",
      "Epoch: [ 0] [  203/  468] time: 84.2098, train_loss: 0.11413106, train_accuracy: 0.9766, test_Accuracy: 0.9519\n",
      "Epoch: [ 0] [  204/  468] time: 84.6469, train_loss: 0.08668889, train_accuracy: 0.9844, test_Accuracy: 0.9564\n",
      "Epoch: [ 0] [  205/  468] time: 85.0645, train_loss: 0.16329277, train_accuracy: 0.9531, test_Accuracy: 0.9535\n",
      "Epoch: [ 0] [  206/  468] time: 85.4826, train_loss: 0.08244143, train_accuracy: 0.9766, test_Accuracy: 0.9449\n",
      "Epoch: [ 0] [  207/  468] time: 85.9309, train_loss: 0.23438005, train_accuracy: 0.9375, test_Accuracy: 0.9445\n",
      "Epoch: [ 0] [  208/  468] time: 86.3720, train_loss: 0.06047585, train_accuracy: 0.9922, test_Accuracy: 0.9457\n",
      "Epoch: [ 0] [  209/  468] time: 86.7979, train_loss: 0.11316349, train_accuracy: 0.9766, test_Accuracy: 0.9485\n",
      "Epoch: [ 0] [  210/  468] time: 87.1648, train_loss: 0.22105125, train_accuracy: 0.9219, test_Accuracy: 0.9537\n",
      "Epoch: [ 0] [  211/  468] time: 87.5301, train_loss: 0.14886665, train_accuracy: 0.9531, test_Accuracy: 0.9540\n",
      "Epoch: [ 0] [  212/  468] time: 87.8718, train_loss: 0.22806421, train_accuracy: 0.9531, test_Accuracy: 0.9516\n",
      "Epoch: [ 0] [  213/  468] time: 88.2619, train_loss: 0.14213523, train_accuracy: 0.9453, test_Accuracy: 0.9495\n",
      "Epoch: [ 0] [  214/  468] time: 88.6910, train_loss: 0.16023266, train_accuracy: 0.9453, test_Accuracy: 0.9510\n",
      "Epoch: [ 0] [  215/  468] time: 89.0451, train_loss: 0.15013865, train_accuracy: 0.9609, test_Accuracy: 0.9517\n",
      "Epoch: [ 0] [  216/  468] time: 89.4959, train_loss: 0.16576961, train_accuracy: 0.9453, test_Accuracy: 0.9509\n",
      "Epoch: [ 0] [  217/  468] time: 89.9050, train_loss: 0.11477132, train_accuracy: 0.9688, test_Accuracy: 0.9529\n",
      "Epoch: [ 0] [  218/  468] time: 90.3526, train_loss: 0.14516863, train_accuracy: 0.9609, test_Accuracy: 0.9552\n",
      "Epoch: [ 0] [  219/  468] time: 90.8124, train_loss: 0.11998009, train_accuracy: 0.9688, test_Accuracy: 0.9524\n",
      "Epoch: [ 0] [  220/  468] time: 91.3311, train_loss: 0.20687032, train_accuracy: 0.9375, test_Accuracy: 0.9535\n",
      "Epoch: [ 0] [  221/  468] time: 91.7433, train_loss: 0.08097938, train_accuracy: 0.9766, test_Accuracy: 0.9567\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [ 0] [  222/  468] time: 92.1666, train_loss: 0.06751882, train_accuracy: 0.9844, test_Accuracy: 0.9560\n",
      "Epoch: [ 0] [  223/  468] time: 92.6395, train_loss: 0.09426358, train_accuracy: 0.9688, test_Accuracy: 0.9545\n",
      "Epoch: [ 0] [  224/  468] time: 93.0761, train_loss: 0.08491783, train_accuracy: 0.9688, test_Accuracy: 0.9529\n",
      "Epoch: [ 0] [  225/  468] time: 93.4583, train_loss: 0.13329831, train_accuracy: 0.9766, test_Accuracy: 0.9507\n",
      "Epoch: [ 0] [  226/  468] time: 93.7867, train_loss: 0.14510013, train_accuracy: 0.9375, test_Accuracy: 0.9534\n",
      "Epoch: [ 0] [  227/  468] time: 94.1261, train_loss: 0.10130331, train_accuracy: 0.9766, test_Accuracy: 0.9563\n",
      "Epoch: [ 0] [  228/  468] time: 94.4739, train_loss: 0.05185942, train_accuracy: 0.9922, test_Accuracy: 0.9565\n",
      "Epoch: [ 0] [  229/  468] time: 94.8012, train_loss: 0.12026276, train_accuracy: 0.9609, test_Accuracy: 0.9582\n",
      "Epoch: [ 0] [  230/  468] time: 95.2094, train_loss: 0.08360844, train_accuracy: 0.9766, test_Accuracy: 0.9589\n",
      "Epoch: [ 0] [  231/  468] time: 95.5424, train_loss: 0.09527201, train_accuracy: 0.9766, test_Accuracy: 0.9571\n",
      "Epoch: [ 0] [  232/  468] time: 96.0026, train_loss: 0.17234316, train_accuracy: 0.9531, test_Accuracy: 0.9529\n",
      "Epoch: [ 0] [  233/  468] time: 96.4457, train_loss: 0.09569596, train_accuracy: 0.9844, test_Accuracy: 0.9517\n",
      "Epoch: [ 0] [  234/  468] time: 96.8631, train_loss: 0.11904091, train_accuracy: 0.9766, test_Accuracy: 0.9521\n",
      "Epoch: [ 0] [  235/  468] time: 97.2980, train_loss: 0.12687105, train_accuracy: 0.9766, test_Accuracy: 0.9530\n",
      "Epoch: [ 0] [  236/  468] time: 97.7205, train_loss: 0.22009906, train_accuracy: 0.9453, test_Accuracy: 0.9509\n",
      "Epoch: [ 0] [  237/  468] time: 98.1171, train_loss: 0.19093707, train_accuracy: 0.9141, test_Accuracy: 0.9513\n",
      "Epoch: [ 0] [  238/  468] time: 98.5419, train_loss: 0.11170199, train_accuracy: 0.9531, test_Accuracy: 0.9526\n",
      "Epoch: [ 0] [  239/  468] time: 98.9597, train_loss: 0.12569194, train_accuracy: 0.9609, test_Accuracy: 0.9552\n",
      "Epoch: [ 0] [  240/  468] time: 99.3578, train_loss: 0.14370316, train_accuracy: 0.9609, test_Accuracy: 0.9585\n",
      "Epoch: [ 0] [  241/  468] time: 99.7289, train_loss: 0.20056587, train_accuracy: 0.9531, test_Accuracy: 0.9605\n",
      "Epoch: [ 0] [  242/  468] time: 100.0845, train_loss: 0.05211062, train_accuracy: 0.9844, test_Accuracy: 0.9602\n",
      "Epoch: [ 0] [  243/  468] time: 100.4400, train_loss: 0.08607468, train_accuracy: 0.9688, test_Accuracy: 0.9603\n",
      "Epoch: [ 0] [  244/  468] time: 100.7615, train_loss: 0.07281457, train_accuracy: 0.9844, test_Accuracy: 0.9590\n",
      "Epoch: [ 0] [  245/  468] time: 101.1380, train_loss: 0.16051972, train_accuracy: 0.9609, test_Accuracy: 0.9586\n",
      "Epoch: [ 0] [  246/  468] time: 101.5675, train_loss: 0.11900007, train_accuracy: 0.9453, test_Accuracy: 0.9583\n",
      "Epoch: [ 0] [  247/  468] time: 101.9550, train_loss: 0.08104602, train_accuracy: 0.9766, test_Accuracy: 0.9578\n",
      "Epoch: [ 0] [  248/  468] time: 102.4245, train_loss: 0.07237560, train_accuracy: 0.9844, test_Accuracy: 0.9582\n",
      "Epoch: [ 0] [  249/  468] time: 102.8470, train_loss: 0.06957927, train_accuracy: 0.9844, test_Accuracy: 0.9583\n",
      "Epoch: [ 0] [  250/  468] time: 103.2987, train_loss: 0.10035516, train_accuracy: 0.9844, test_Accuracy: 0.9580\n",
      "Epoch: [ 0] [  251/  468] time: 103.7981, train_loss: 0.07026815, train_accuracy: 0.9688, test_Accuracy: 0.9592\n",
      "Epoch: [ 0] [  252/  468] time: 104.4105, train_loss: 0.16446900, train_accuracy: 0.9453, test_Accuracy: 0.9599\n",
      "Epoch: [ 0] [  253/  468] time: 104.8420, train_loss: 0.07632367, train_accuracy: 0.9922, test_Accuracy: 0.9582\n",
      "Epoch: [ 0] [  254/  468] time: 105.4051, train_loss: 0.23017491, train_accuracy: 0.9297, test_Accuracy: 0.9580\n",
      "Epoch: [ 0] [  255/  468] time: 105.8298, train_loss: 0.21123886, train_accuracy: 0.9297, test_Accuracy: 0.9567\n",
      "Epoch: [ 0] [  256/  468] time: 106.2303, train_loss: 0.09667993, train_accuracy: 0.9766, test_Accuracy: 0.9573\n",
      "Epoch: [ 0] [  257/  468] time: 106.6909, train_loss: 0.14823622, train_accuracy: 0.9453, test_Accuracy: 0.9578\n",
      "Epoch: [ 0] [  258/  468] time: 107.0368, train_loss: 0.08773297, train_accuracy: 0.9688, test_Accuracy: 0.9613\n",
      "Epoch: [ 0] [  259/  468] time: 107.3923, train_loss: 0.12285890, train_accuracy: 0.9766, test_Accuracy: 0.9644\n",
      "Epoch: [ 0] [  260/  468] time: 107.7368, train_loss: 0.08586143, train_accuracy: 0.9766, test_Accuracy: 0.9649\n",
      "Epoch: [ 0] [  261/  468] time: 108.0967, train_loss: 0.03546580, train_accuracy: 0.9922, test_Accuracy: 0.9635\n",
      "Epoch: [ 0] [  262/  468] time: 108.4353, train_loss: 0.07834642, train_accuracy: 0.9766, test_Accuracy: 0.9615\n",
      "Epoch: [ 0] [  263/  468] time: 108.8637, train_loss: 0.07085589, train_accuracy: 0.9844, test_Accuracy: 0.9582\n",
      "Epoch: [ 0] [  264/  468] time: 109.3072, train_loss: 0.16858946, train_accuracy: 0.9688, test_Accuracy: 0.9556\n",
      "Epoch: [ 0] [  265/  468] time: 109.8712, train_loss: 0.07094307, train_accuracy: 0.9844, test_Accuracy: 0.9572\n",
      "Epoch: [ 0] [  266/  468] time: 110.3668, train_loss: 0.15920654, train_accuracy: 0.9453, test_Accuracy: 0.9600\n",
      "Epoch: [ 0] [  267/  468] time: 110.9526, train_loss: 0.04426285, train_accuracy: 0.9922, test_Accuracy: 0.9599\n",
      "Epoch: [ 0] [  268/  468] time: 111.4233, train_loss: 0.05083879, train_accuracy: 0.9844, test_Accuracy: 0.9582\n",
      "Epoch: [ 0] [  269/  468] time: 111.8946, train_loss: 0.09645668, train_accuracy: 0.9688, test_Accuracy: 0.9581\n",
      "Epoch: [ 0] [  270/  468] time: 112.4078, train_loss: 0.10926576, train_accuracy: 0.9609, test_Accuracy: 0.9572\n",
      "Epoch: [ 0] [  271/  468] time: 112.8118, train_loss: 0.13918260, train_accuracy: 0.9688, test_Accuracy: 0.9601\n",
      "Epoch: [ 0] [  272/  468] time: 113.1974, train_loss: 0.07751463, train_accuracy: 0.9766, test_Accuracy: 0.9602\n",
      "Epoch: [ 0] [  273/  468] time: 113.6199, train_loss: 0.10360312, train_accuracy: 0.9766, test_Accuracy: 0.9604\n",
      "Epoch: [ 0] [  274/  468] time: 114.0402, train_loss: 0.12133615, train_accuracy: 0.9531, test_Accuracy: 0.9578\n",
      "Epoch: [ 0] [  275/  468] time: 114.4316, train_loss: 0.16075532, train_accuracy: 0.9609, test_Accuracy: 0.9552\n",
      "Epoch: [ 0] [  276/  468] time: 114.7924, train_loss: 0.05881250, train_accuracy: 0.9922, test_Accuracy: 0.9573\n",
      "Epoch: [ 0] [  277/  468] time: 115.2654, train_loss: 0.10037104, train_accuracy: 0.9766, test_Accuracy: 0.9568\n",
      "Epoch: [ 0] [  278/  468] time: 115.6934, train_loss: 0.19834584, train_accuracy: 0.9375, test_Accuracy: 0.9580\n",
      "Epoch: [ 0] [  279/  468] time: 116.3351, train_loss: 0.17641053, train_accuracy: 0.9531, test_Accuracy: 0.9615\n",
      "Epoch: [ 0] [  280/  468] time: 116.8719, train_loss: 0.11148795, train_accuracy: 0.9844, test_Accuracy: 0.9627\n",
      "Epoch: [ 0] [  281/  468] time: 117.4505, train_loss: 0.06938472, train_accuracy: 0.9766, test_Accuracy: 0.9622\n",
      "Epoch: [ 0] [  282/  468] time: 117.9328, train_loss: 0.07981482, train_accuracy: 0.9766, test_Accuracy: 0.9618\n",
      "Epoch: [ 0] [  283/  468] time: 118.3945, train_loss: 0.08174501, train_accuracy: 0.9688, test_Accuracy: 0.9624\n",
      "Epoch: [ 0] [  284/  468] time: 118.9650, train_loss: 0.06631982, train_accuracy: 0.9844, test_Accuracy: 0.9634\n",
      "Epoch: [ 0] [  285/  468] time: 119.3287, train_loss: 0.11699840, train_accuracy: 0.9688, test_Accuracy: 0.9634\n",
      "Epoch: [ 0] [  286/  468] time: 119.7102, train_loss: 0.09303334, train_accuracy: 0.9844, test_Accuracy: 0.9624\n",
      "Epoch: [ 0] [  287/  468] time: 120.1250, train_loss: 0.09692042, train_accuracy: 0.9609, test_Accuracy: 0.9625\n",
      "Epoch: [ 0] [  288/  468] time: 120.5132, train_loss: 0.08877415, train_accuracy: 0.9688, test_Accuracy: 0.9616\n",
      "Epoch: [ 0] [  289/  468] time: 120.9659, train_loss: 0.26291066, train_accuracy: 0.9141, test_Accuracy: 0.9603\n",
      "Epoch: [ 0] [  290/  468] time: 121.4105, train_loss: 0.10068501, train_accuracy: 0.9688, test_Accuracy: 0.9605\n",
      "Epoch: [ 0] [  291/  468] time: 121.9710, train_loss: 0.13173409, train_accuracy: 0.9609, test_Accuracy: 0.9623\n",
      "Epoch: [ 0] [  292/  468] time: 122.5013, train_loss: 0.10939828, train_accuracy: 0.9609, test_Accuracy: 0.9631\n",
      "Epoch: [ 0] [  293/  468] time: 122.9830, train_loss: 0.04874768, train_accuracy: 0.9766, test_Accuracy: 0.9650\n",
      "Epoch: [ 0] [  294/  468] time: 123.4416, train_loss: 0.03843886, train_accuracy: 0.9922, test_Accuracy: 0.9646\n",
      "Epoch: [ 0] [  295/  468] time: 123.8100, train_loss: 0.14577690, train_accuracy: 0.9609, test_Accuracy: 0.9642\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [ 0] [  296/  468] time: 124.3089, train_loss: 0.14005783, train_accuracy: 0.9531, test_Accuracy: 0.9638\n",
      "Epoch: [ 0] [  297/  468] time: 124.7685, train_loss: 0.17078030, train_accuracy: 0.9531, test_Accuracy: 0.9626\n",
      "Epoch: [ 0] [  298/  468] time: 125.1685, train_loss: 0.07599905, train_accuracy: 0.9766, test_Accuracy: 0.9634\n",
      "Epoch: [ 0] [  299/  468] time: 125.5960, train_loss: 0.06944077, train_accuracy: 0.9844, test_Accuracy: 0.9625\n",
      "Epoch: [ 0] [  300/  468] time: 126.0213, train_loss: 0.04944002, train_accuracy: 0.9922, test_Accuracy: 0.9614\n",
      "Epoch: [ 0] [  301/  468] time: 126.4350, train_loss: 0.17815892, train_accuracy: 0.9609, test_Accuracy: 0.9617\n",
      "Epoch: [ 0] [  302/  468] time: 126.7754, train_loss: 0.08685820, train_accuracy: 0.9844, test_Accuracy: 0.9621\n",
      "Epoch: [ 0] [  303/  468] time: 127.1730, train_loss: 0.13351323, train_accuracy: 0.9531, test_Accuracy: 0.9622\n",
      "Epoch: [ 0] [  304/  468] time: 127.5442, train_loss: 0.15919581, train_accuracy: 0.9531, test_Accuracy: 0.9622\n",
      "Epoch: [ 0] [  305/  468] time: 128.0156, train_loss: 0.13551135, train_accuracy: 0.9688, test_Accuracy: 0.9619\n",
      "Epoch: [ 0] [  306/  468] time: 128.5160, train_loss: 0.09435542, train_accuracy: 0.9766, test_Accuracy: 0.9613\n",
      "Epoch: [ 0] [  307/  468] time: 128.9658, train_loss: 0.10126722, train_accuracy: 0.9609, test_Accuracy: 0.9618\n",
      "Epoch: [ 0] [  308/  468] time: 129.4918, train_loss: 0.02944106, train_accuracy: 1.0000, test_Accuracy: 0.9625\n",
      "Epoch: [ 0] [  309/  468] time: 129.8419, train_loss: 0.14733608, train_accuracy: 0.9453, test_Accuracy: 0.9648\n",
      "Epoch: [ 0] [  310/  468] time: 130.2181, train_loss: 0.11212087, train_accuracy: 0.9766, test_Accuracy: 0.9651\n",
      "Epoch: [ 0] [  311/  468] time: 130.6029, train_loss: 0.18527547, train_accuracy: 0.9375, test_Accuracy: 0.9657\n",
      "Epoch: [ 0] [  312/  468] time: 131.0211, train_loss: 0.08815025, train_accuracy: 0.9688, test_Accuracy: 0.9645\n",
      "Epoch: [ 0] [  313/  468] time: 131.4002, train_loss: 0.09935060, train_accuracy: 0.9609, test_Accuracy: 0.9638\n",
      "Epoch: [ 0] [  314/  468] time: 131.8728, train_loss: 0.04872293, train_accuracy: 0.9922, test_Accuracy: 0.9631\n",
      "Epoch: [ 0] [  315/  468] time: 132.3338, train_loss: 0.16280507, train_accuracy: 0.9609, test_Accuracy: 0.9625\n",
      "Epoch: [ 0] [  316/  468] time: 132.6986, train_loss: 0.08444118, train_accuracy: 0.9844, test_Accuracy: 0.9643\n",
      "Epoch: [ 0] [  317/  468] time: 133.0613, train_loss: 0.09867431, train_accuracy: 0.9766, test_Accuracy: 0.9652\n",
      "Epoch: [ 0] [  318/  468] time: 133.4159, train_loss: 0.16604917, train_accuracy: 0.9531, test_Accuracy: 0.9648\n",
      "Epoch: [ 0] [  319/  468] time: 133.7301, train_loss: 0.07750934, train_accuracy: 0.9844, test_Accuracy: 0.9637\n",
      "Epoch: [ 0] [  320/  468] time: 134.0845, train_loss: 0.08556731, train_accuracy: 0.9766, test_Accuracy: 0.9625\n",
      "Epoch: [ 0] [  321/  468] time: 134.4787, train_loss: 0.12490956, train_accuracy: 0.9688, test_Accuracy: 0.9625\n",
      "Epoch: [ 0] [  322/  468] time: 134.9062, train_loss: 0.05591813, train_accuracy: 0.9844, test_Accuracy: 0.9601\n",
      "Epoch: [ 0] [  323/  468] time: 135.3632, train_loss: 0.07087298, train_accuracy: 0.9688, test_Accuracy: 0.9593\n",
      "Epoch: [ 0] [  324/  468] time: 135.7623, train_loss: 0.12290926, train_accuracy: 0.9531, test_Accuracy: 0.9609\n",
      "Epoch: [ 0] [  325/  468] time: 136.2028, train_loss: 0.10262008, train_accuracy: 0.9688, test_Accuracy: 0.9601\n",
      "Epoch: [ 0] [  326/  468] time: 136.6649, train_loss: 0.08423799, train_accuracy: 0.9844, test_Accuracy: 0.9587\n",
      "Epoch: [ 0] [  327/  468] time: 137.0495, train_loss: 0.12067786, train_accuracy: 0.9766, test_Accuracy: 0.9573\n",
      "Epoch: [ 0] [  328/  468] time: 137.5312, train_loss: 0.09925723, train_accuracy: 0.9844, test_Accuracy: 0.9587\n",
      "Epoch: [ 0] [  329/  468] time: 137.9576, train_loss: 0.06112389, train_accuracy: 0.9844, test_Accuracy: 0.9593\n",
      "Epoch: [ 0] [  330/  468] time: 138.3775, train_loss: 0.07513839, train_accuracy: 0.9688, test_Accuracy: 0.9570\n",
      "Epoch: [ 0] [  331/  468] time: 138.7622, train_loss: 0.13581565, train_accuracy: 0.9766, test_Accuracy: 0.9560\n",
      "Epoch: [ 0] [  332/  468] time: 139.1339, train_loss: 0.09835966, train_accuracy: 0.9609, test_Accuracy: 0.9583\n",
      "Epoch: [ 0] [  333/  468] time: 139.4716, train_loss: 0.11968108, train_accuracy: 0.9688, test_Accuracy: 0.9622\n",
      "Epoch: [ 0] [  334/  468] time: 139.8091, train_loss: 0.08057310, train_accuracy: 0.9688, test_Accuracy: 0.9648\n",
      "Epoch: [ 0] [  335/  468] time: 140.1700, train_loss: 0.10544690, train_accuracy: 0.9766, test_Accuracy: 0.9649\n",
      "Epoch: [ 0] [  336/  468] time: 140.5128, train_loss: 0.09621238, train_accuracy: 0.9609, test_Accuracy: 0.9612\n",
      "Epoch: [ 0] [  337/  468] time: 140.9607, train_loss: 0.09314644, train_accuracy: 0.9766, test_Accuracy: 0.9564\n",
      "Epoch: [ 0] [  338/  468] time: 141.4058, train_loss: 0.12815969, train_accuracy: 0.9609, test_Accuracy: 0.9552\n",
      "Epoch: [ 0] [  339/  468] time: 141.8965, train_loss: 0.11229791, train_accuracy: 0.9766, test_Accuracy: 0.9575\n",
      "Epoch: [ 0] [  340/  468] time: 142.3554, train_loss: 0.12818497, train_accuracy: 0.9609, test_Accuracy: 0.9610\n",
      "Epoch: [ 0] [  341/  468] time: 142.8120, train_loss: 0.06082499, train_accuracy: 0.9844, test_Accuracy: 0.9629\n",
      "Epoch: [ 0] [  342/  468] time: 143.2244, train_loss: 0.12930652, train_accuracy: 0.9531, test_Accuracy: 0.9634\n",
      "Epoch: [ 0] [  343/  468] time: 143.5615, train_loss: 0.14165969, train_accuracy: 0.9531, test_Accuracy: 0.9631\n",
      "Epoch: [ 0] [  344/  468] time: 143.9510, train_loss: 0.08099192, train_accuracy: 0.9766, test_Accuracy: 0.9616\n",
      "Epoch: [ 0] [  345/  468] time: 144.4000, train_loss: 0.13285807, train_accuracy: 0.9609, test_Accuracy: 0.9589\n",
      "Epoch: [ 0] [  346/  468] time: 144.8531, train_loss: 0.09268090, train_accuracy: 0.9688, test_Accuracy: 0.9609\n",
      "Epoch: [ 0] [  347/  468] time: 145.2748, train_loss: 0.05804043, train_accuracy: 0.9688, test_Accuracy: 0.9604\n",
      "Epoch: [ 0] [  348/  468] time: 145.7253, train_loss: 0.11358947, train_accuracy: 0.9609, test_Accuracy: 0.9618\n",
      "Epoch: [ 0] [  349/  468] time: 146.1666, train_loss: 0.07723351, train_accuracy: 0.9766, test_Accuracy: 0.9627\n",
      "Epoch: [ 0] [  350/  468] time: 146.5209, train_loss: 0.11248274, train_accuracy: 0.9688, test_Accuracy: 0.9608\n",
      "Epoch: [ 0] [  351/  468] time: 146.8471, train_loss: 0.17122728, train_accuracy: 0.9453, test_Accuracy: 0.9599\n",
      "Epoch: [ 0] [  352/  468] time: 147.3259, train_loss: 0.09375302, train_accuracy: 0.9609, test_Accuracy: 0.9575\n",
      "Epoch: [ 0] [  353/  468] time: 147.7458, train_loss: 0.10666378, train_accuracy: 0.9688, test_Accuracy: 0.9568\n",
      "Epoch: [ 0] [  354/  468] time: 148.1582, train_loss: 0.05011895, train_accuracy: 0.9922, test_Accuracy: 0.9563\n",
      "Epoch: [ 0] [  355/  468] time: 148.5779, train_loss: 0.13629261, train_accuracy: 0.9609, test_Accuracy: 0.9560\n",
      "Epoch: [ 0] [  356/  468] time: 149.0721, train_loss: 0.05424274, train_accuracy: 1.0000, test_Accuracy: 0.9589\n",
      "Epoch: [ 0] [  357/  468] time: 149.5486, train_loss: 0.08229746, train_accuracy: 0.9766, test_Accuracy: 0.9620\n",
      "Epoch: [ 0] [  358/  468] time: 150.0247, train_loss: 0.14868689, train_accuracy: 0.9844, test_Accuracy: 0.9652\n",
      "Epoch: [ 0] [  359/  468] time: 150.5117, train_loss: 0.05558958, train_accuracy: 0.9922, test_Accuracy: 0.9666\n",
      "Epoch: [ 0] [  360/  468] time: 150.9107, train_loss: 0.07208358, train_accuracy: 0.9844, test_Accuracy: 0.9672\n",
      "Epoch: [ 0] [  361/  468] time: 151.3112, train_loss: 0.03230317, train_accuracy: 0.9922, test_Accuracy: 0.9658\n",
      "Epoch: [ 0] [  362/  468] time: 151.7129, train_loss: 0.07999013, train_accuracy: 0.9688, test_Accuracy: 0.9644\n",
      "Epoch: [ 0] [  363/  468] time: 152.1103, train_loss: 0.17778239, train_accuracy: 0.9531, test_Accuracy: 0.9629\n",
      "Epoch: [ 0] [  364/  468] time: 152.4394, train_loss: 0.07817610, train_accuracy: 0.9766, test_Accuracy: 0.9626\n",
      "Epoch: [ 0] [  365/  468] time: 152.7701, train_loss: 0.08820732, train_accuracy: 0.9609, test_Accuracy: 0.9607\n",
      "Epoch: [ 0] [  366/  468] time: 153.1584, train_loss: 0.13658042, train_accuracy: 0.9531, test_Accuracy: 0.9621\n",
      "Epoch: [ 0] [  367/  468] time: 153.4913, train_loss: 0.14189261, train_accuracy: 0.9375, test_Accuracy: 0.9623\n",
      "Epoch: [ 0] [  368/  468] time: 153.9151, train_loss: 0.05258345, train_accuracy: 0.9844, test_Accuracy: 0.9634\n",
      "Epoch: [ 0] [  369/  468] time: 154.3580, train_loss: 0.10050838, train_accuracy: 0.9766, test_Accuracy: 0.9634\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [ 0] [  370/  468] time: 154.7909, train_loss: 0.04964381, train_accuracy: 0.9844, test_Accuracy: 0.9610\n",
      "Epoch: [ 0] [  371/  468] time: 155.2402, train_loss: 0.12169806, train_accuracy: 0.9531, test_Accuracy: 0.9643\n",
      "Epoch: [ 0] [  372/  468] time: 155.6352, train_loss: 0.11334420, train_accuracy: 0.9688, test_Accuracy: 0.9648\n",
      "Epoch: [ 0] [  373/  468] time: 156.0639, train_loss: 0.13712569, train_accuracy: 0.9609, test_Accuracy: 0.9645\n",
      "Epoch: [ 0] [  374/  468] time: 156.4895, train_loss: 0.16996419, train_accuracy: 0.9453, test_Accuracy: 0.9637\n",
      "Epoch: [ 0] [  375/  468] time: 156.9403, train_loss: 0.12139468, train_accuracy: 0.9688, test_Accuracy: 0.9632\n",
      "Epoch: [ 0] [  376/  468] time: 157.4251, train_loss: 0.08404960, train_accuracy: 0.9844, test_Accuracy: 0.9636\n",
      "Epoch: [ 0] [  377/  468] time: 157.8800, train_loss: 0.11493848, train_accuracy: 0.9688, test_Accuracy: 0.9628\n",
      "Epoch: [ 0] [  378/  468] time: 158.2765, train_loss: 0.16986760, train_accuracy: 0.9609, test_Accuracy: 0.9610\n",
      "Epoch: [ 0] [  379/  468] time: 158.6300, train_loss: 0.05882622, train_accuracy: 0.9922, test_Accuracy: 0.9578\n",
      "Epoch: [ 0] [  380/  468] time: 158.9642, train_loss: 0.07135676, train_accuracy: 0.9766, test_Accuracy: 0.9542\n",
      "Epoch: [ 0] [  381/  468] time: 159.3107, train_loss: 0.05717187, train_accuracy: 0.9922, test_Accuracy: 0.9543\n",
      "Epoch: [ 0] [  382/  468] time: 159.6906, train_loss: 0.09772324, train_accuracy: 0.9531, test_Accuracy: 0.9574\n",
      "Epoch: [ 0] [  383/  468] time: 160.0319, train_loss: 0.10074305, train_accuracy: 0.9844, test_Accuracy: 0.9617\n",
      "Epoch: [ 0] [  384/  468] time: 160.4515, train_loss: 0.06932638, train_accuracy: 0.9688, test_Accuracy: 0.9621\n",
      "Epoch: [ 0] [  385/  468] time: 160.8356, train_loss: 0.05467273, train_accuracy: 0.9844, test_Accuracy: 0.9603\n",
      "Epoch: [ 0] [  386/  468] time: 161.2457, train_loss: 0.07675275, train_accuracy: 0.9766, test_Accuracy: 0.9607\n",
      "Epoch: [ 0] [  387/  468] time: 161.6342, train_loss: 0.09733629, train_accuracy: 0.9844, test_Accuracy: 0.9616\n",
      "Epoch: [ 0] [  388/  468] time: 161.9887, train_loss: 0.06141218, train_accuracy: 0.9766, test_Accuracy: 0.9618\n",
      "Epoch: [ 0] [  389/  468] time: 162.4618, train_loss: 0.06843278, train_accuracy: 0.9766, test_Accuracy: 0.9620\n",
      "Epoch: [ 0] [  390/  468] time: 162.8418, train_loss: 0.09691471, train_accuracy: 0.9766, test_Accuracy: 0.9623\n",
      "Epoch: [ 0] [  391/  468] time: 163.2657, train_loss: 0.05604080, train_accuracy: 0.9922, test_Accuracy: 0.9619\n",
      "Epoch: [ 0] [  392/  468] time: 163.6291, train_loss: 0.11100458, train_accuracy: 0.9766, test_Accuracy: 0.9631\n",
      "Epoch: [ 0] [  393/  468] time: 164.0475, train_loss: 0.05863545, train_accuracy: 0.9844, test_Accuracy: 0.9630\n",
      "Epoch: [ 0] [  394/  468] time: 164.5719, train_loss: 0.16202980, train_accuracy: 0.9609, test_Accuracy: 0.9630\n",
      "Epoch: [ 0] [  395/  468] time: 164.9445, train_loss: 0.06998311, train_accuracy: 0.9766, test_Accuracy: 0.9626\n",
      "Epoch: [ 0] [  396/  468] time: 165.3308, train_loss: 0.13860083, train_accuracy: 0.9609, test_Accuracy: 0.9635\n",
      "Epoch: [ 0] [  397/  468] time: 165.6807, train_loss: 0.07461612, train_accuracy: 0.9766, test_Accuracy: 0.9655\n",
      "Epoch: [ 0] [  398/  468] time: 166.0175, train_loss: 0.04617611, train_accuracy: 0.9922, test_Accuracy: 0.9628\n",
      "Epoch: [ 0] [  399/  468] time: 166.3834, train_loss: 0.06598676, train_accuracy: 0.9688, test_Accuracy: 0.9595\n",
      "Epoch: [ 0] [  400/  468] time: 166.7818, train_loss: 0.08721571, train_accuracy: 0.9844, test_Accuracy: 0.9650\n",
      "Epoch: [ 0] [  401/  468] time: 167.1565, train_loss: 0.07528503, train_accuracy: 0.9766, test_Accuracy: 0.9668\n",
      "Epoch: [ 0] [  402/  468] time: 167.5939, train_loss: 0.08626909, train_accuracy: 0.9688, test_Accuracy: 0.9672\n",
      "Epoch: [ 0] [  403/  468] time: 167.9836, train_loss: 0.15353169, train_accuracy: 0.9609, test_Accuracy: 0.9661\n",
      "Epoch: [ 0] [  404/  468] time: 168.3973, train_loss: 0.08224523, train_accuracy: 0.9766, test_Accuracy: 0.9646\n",
      "Epoch: [ 0] [  405/  468] time: 168.8553, train_loss: 0.04199855, train_accuracy: 0.9844, test_Accuracy: 0.9645\n",
      "Epoch: [ 0] [  406/  468] time: 169.2635, train_loss: 0.12325668, train_accuracy: 0.9609, test_Accuracy: 0.9617\n",
      "Epoch: [ 0] [  407/  468] time: 169.6713, train_loss: 0.15873687, train_accuracy: 0.9531, test_Accuracy: 0.9636\n",
      "Epoch: [ 0] [  408/  468] time: 170.0801, train_loss: 0.10689491, train_accuracy: 0.9688, test_Accuracy: 0.9663\n",
      "Epoch: [ 0] [  409/  468] time: 170.4876, train_loss: 0.06418378, train_accuracy: 1.0000, test_Accuracy: 0.9682\n",
      "Epoch: [ 0] [  410/  468] time: 170.9037, train_loss: 0.09001511, train_accuracy: 0.9688, test_Accuracy: 0.9686\n",
      "Epoch: [ 0] [  411/  468] time: 171.3411, train_loss: 0.01397411, train_accuracy: 1.0000, test_Accuracy: 0.9683\n",
      "Epoch: [ 0] [  412/  468] time: 171.6790, train_loss: 0.03961156, train_accuracy: 1.0000, test_Accuracy: 0.9661\n",
      "Epoch: [ 0] [  413/  468] time: 172.0442, train_loss: 0.11951928, train_accuracy: 0.9531, test_Accuracy: 0.9639\n",
      "Epoch: [ 0] [  414/  468] time: 172.4640, train_loss: 0.17327981, train_accuracy: 0.9453, test_Accuracy: 0.9641\n",
      "Epoch: [ 0] [  415/  468] time: 172.8071, train_loss: 0.05106756, train_accuracy: 0.9844, test_Accuracy: 0.9634\n",
      "Epoch: [ 0] [  416/  468] time: 173.3284, train_loss: 0.11369896, train_accuracy: 0.9609, test_Accuracy: 0.9651\n",
      "Epoch: [ 0] [  417/  468] time: 173.7702, train_loss: 0.10753953, train_accuracy: 0.9609, test_Accuracy: 0.9640\n",
      "Epoch: [ 0] [  418/  468] time: 174.2449, train_loss: 0.06026758, train_accuracy: 0.9766, test_Accuracy: 0.9639\n",
      "Epoch: [ 0] [  419/  468] time: 174.7114, train_loss: 0.10568401, train_accuracy: 0.9688, test_Accuracy: 0.9646\n",
      "Epoch: [ 0] [  420/  468] time: 175.2208, train_loss: 0.05787601, train_accuracy: 0.9766, test_Accuracy: 0.9664\n",
      "Epoch: [ 0] [  421/  468] time: 175.7801, train_loss: 0.12102231, train_accuracy: 0.9609, test_Accuracy: 0.9670\n",
      "Epoch: [ 0] [  422/  468] time: 176.2684, train_loss: 0.13532048, train_accuracy: 0.9688, test_Accuracy: 0.9670\n",
      "Epoch: [ 0] [  423/  468] time: 176.7407, train_loss: 0.06803796, train_accuracy: 0.9844, test_Accuracy: 0.9658\n",
      "Epoch: [ 0] [  424/  468] time: 177.2691, train_loss: 0.17246465, train_accuracy: 0.9609, test_Accuracy: 0.9649\n",
      "Epoch: [ 0] [  425/  468] time: 177.6796, train_loss: 0.10736057, train_accuracy: 0.9766, test_Accuracy: 0.9631\n",
      "Epoch: [ 0] [  426/  468] time: 178.0861, train_loss: 0.08942219, train_accuracy: 0.9531, test_Accuracy: 0.9648\n",
      "Epoch: [ 0] [  427/  468] time: 178.4530, train_loss: 0.03502883, train_accuracy: 1.0000, test_Accuracy: 0.9675\n",
      "Epoch: [ 0] [  428/  468] time: 178.9562, train_loss: 0.05006492, train_accuracy: 0.9844, test_Accuracy: 0.9690\n",
      "Epoch: [ 0] [  429/  468] time: 179.4373, train_loss: 0.12455621, train_accuracy: 0.9766, test_Accuracy: 0.9687\n",
      "Epoch: [ 0] [  430/  468] time: 179.8041, train_loss: 0.12623709, train_accuracy: 0.9688, test_Accuracy: 0.9692\n",
      "Epoch: [ 0] [  431/  468] time: 180.1965, train_loss: 0.06443386, train_accuracy: 0.9844, test_Accuracy: 0.9696\n",
      "Epoch: [ 0] [  432/  468] time: 180.5520, train_loss: 0.07489470, train_accuracy: 0.9922, test_Accuracy: 0.9697\n",
      "Epoch: [ 0] [  433/  468] time: 180.8999, train_loss: 0.02792163, train_accuracy: 1.0000, test_Accuracy: 0.9679\n",
      "Epoch: [ 0] [  434/  468] time: 181.3300, train_loss: 0.05986711, train_accuracy: 0.9766, test_Accuracy: 0.9656\n",
      "Epoch: [ 0] [  435/  468] time: 181.7259, train_loss: 0.08904810, train_accuracy: 0.9766, test_Accuracy: 0.9632\n",
      "Epoch: [ 0] [  436/  468] time: 182.1843, train_loss: 0.05017616, train_accuracy: 0.9844, test_Accuracy: 0.9622\n",
      "Epoch: [ 0] [  437/  468] time: 182.5707, train_loss: 0.16177763, train_accuracy: 0.9531, test_Accuracy: 0.9600\n",
      "Epoch: [ 0] [  438/  468] time: 182.9580, train_loss: 0.18416043, train_accuracy: 0.9453, test_Accuracy: 0.9642\n",
      "Epoch: [ 0] [  439/  468] time: 183.4283, train_loss: 0.12629704, train_accuracy: 0.9766, test_Accuracy: 0.9681\n",
      "Epoch: [ 0] [  440/  468] time: 183.8425, train_loss: 0.07923482, train_accuracy: 0.9844, test_Accuracy: 0.9700\n",
      "Epoch: [ 0] [  441/  468] time: 184.3020, train_loss: 0.08678934, train_accuracy: 0.9844, test_Accuracy: 0.9710\n",
      "Epoch: [ 0] [  442/  468] time: 184.7151, train_loss: 0.05787509, train_accuracy: 0.9844, test_Accuracy: 0.9706\n",
      "Epoch: [ 0] [  443/  468] time: 185.0601, train_loss: 0.04142454, train_accuracy: 1.0000, test_Accuracy: 0.9710\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [ 0] [  444/  468] time: 185.4022, train_loss: 0.05185466, train_accuracy: 0.9766, test_Accuracy: 0.9708\n",
      "Epoch: [ 0] [  445/  468] time: 185.7486, train_loss: 0.06486334, train_accuracy: 0.9766, test_Accuracy: 0.9701\n",
      "Epoch: [ 0] [  446/  468] time: 186.0980, train_loss: 0.03737441, train_accuracy: 0.9922, test_Accuracy: 0.9688\n",
      "Epoch: [ 0] [  447/  468] time: 186.4601, train_loss: 0.08123229, train_accuracy: 0.9688, test_Accuracy: 0.9675\n",
      "Epoch: [ 0] [  448/  468] time: 186.8806, train_loss: 0.09248833, train_accuracy: 0.9766, test_Accuracy: 0.9664\n",
      "Epoch: [ 0] [  449/  468] time: 187.3616, train_loss: 0.11323323, train_accuracy: 0.9766, test_Accuracy: 0.9685\n",
      "Epoch: [ 0] [  450/  468] time: 187.7185, train_loss: 0.16114613, train_accuracy: 0.9531, test_Accuracy: 0.9692\n",
      "Epoch: [ 0] [  451/  468] time: 188.1005, train_loss: 0.02924548, train_accuracy: 0.9922, test_Accuracy: 0.9689\n",
      "Epoch: [ 0] [  452/  468] time: 188.5595, train_loss: 0.07950802, train_accuracy: 0.9766, test_Accuracy: 0.9682\n",
      "Epoch: [ 0] [  453/  468] time: 188.9718, train_loss: 0.07212029, train_accuracy: 0.9844, test_Accuracy: 0.9666\n",
      "Epoch: [ 0] [  454/  468] time: 189.3879, train_loss: 0.06318218, train_accuracy: 0.9922, test_Accuracy: 0.9654\n",
      "Epoch: [ 0] [  455/  468] time: 189.7931, train_loss: 0.10723884, train_accuracy: 0.9609, test_Accuracy: 0.9661\n",
      "Epoch: [ 0] [  456/  468] time: 190.2377, train_loss: 0.08044103, train_accuracy: 0.9688, test_Accuracy: 0.9684\n",
      "Epoch: [ 0] [  457/  468] time: 190.6913, train_loss: 0.09724182, train_accuracy: 0.9688, test_Accuracy: 0.9696\n",
      "Epoch: [ 0] [  458/  468] time: 191.1041, train_loss: 0.04339262, train_accuracy: 0.9922, test_Accuracy: 0.9692\n",
      "Epoch: [ 0] [  459/  468] time: 191.6266, train_loss: 0.06980290, train_accuracy: 0.9844, test_Accuracy: 0.9666\n",
      "Epoch: [ 0] [  460/  468] time: 191.9678, train_loss: 0.05971711, train_accuracy: 0.9844, test_Accuracy: 0.9628\n",
      "Epoch: [ 0] [  461/  468] time: 192.3265, train_loss: 0.04852306, train_accuracy: 0.9844, test_Accuracy: 0.9608\n",
      "Epoch: [ 0] [  462/  468] time: 192.7128, train_loss: 0.07233200, train_accuracy: 0.9844, test_Accuracy: 0.9607\n",
      "Epoch: [ 0] [  463/  468] time: 193.1800, train_loss: 0.05109987, train_accuracy: 0.9844, test_Accuracy: 0.9592\n",
      "Epoch: [ 0] [  464/  468] time: 193.6716, train_loss: 0.07670274, train_accuracy: 0.9844, test_Accuracy: 0.9628\n",
      "Epoch: [ 0] [  465/  468] time: 194.1168, train_loss: 0.09968638, train_accuracy: 0.9766, test_Accuracy: 0.9657\n",
      "Epoch: [ 0] [  466/  468] time: 194.6892, train_loss: 0.12496883, train_accuracy: 0.9609, test_Accuracy: 0.9662\n",
      "Epoch: [ 0] [  467/  468] time: 195.0608, train_loss: 0.02832513, train_accuracy: 0.9922, test_Accuracy: 0.9631\n"
     ]
    }
   ],
   "source": [
    "if train_flag :\n",
    "\n",
    "    checkpoint = tf.train.Checkpoint(dnn=network)\n",
    "\n",
    "    # create writer for tensorboard\n",
    "    summary_writer = tf.summary.create_file_writer(logdir=logs_dir)\n",
    "    start_time = time()\n",
    "\n",
    "    # restore check-point if it exits\n",
    "    could_load, checkpoint_counter = load(network, checkpoint_dir)    \n",
    "\n",
    "    if could_load:\n",
    "        start_epoch = (int)(checkpoint_counter / training_iterations)        \n",
    "        counter = checkpoint_counter        \n",
    "        print(\" [*] Load SUCCESS\")\n",
    "    else:\n",
    "        start_epoch = 0\n",
    "        start_iteration = 0\n",
    "        counter = 0\n",
    "        print(\" [!] Load failed...\")\n",
    "    \n",
    "    # train phase\n",
    "    with summary_writer.as_default():  # for tensorboard\n",
    "        for epoch in range(start_epoch, training_epochs):\n",
    "            for idx, (train_input, train_label) in enumerate(train_dataset):            \n",
    "                grads = grad(network, train_input, train_label)\n",
    "                optimizer.apply_gradients(grads_and_vars=zip(grads, network.variables))\n",
    "\n",
    "                train_loss = loss_fn(network, train_input, train_label)\n",
    "                train_accuracy = accuracy_fn(network, train_input, train_label)\n",
    "                \n",
    "                for test_input, test_label in test_dataset:                \n",
    "                    test_accuracy = accuracy_fn(network, test_input, test_label)\n",
    "\n",
    "                tf.summary.scalar(name='train_loss', data=train_loss, step=counter)\n",
    "                tf.summary.scalar(name='train_accuracy', data=train_accuracy, step=counter)\n",
    "                tf.summary.scalar(name='test_accuracy', data=test_accuracy, step=counter)\n",
    "\n",
    "                print(\n",
    "                    \"Epoch: [%2d] [%5d/%5d] time: %4.4f, train_loss: %.8f, train_accuracy: %.4f, test_Accuracy: %.4f\" \\\n",
    "                    % (epoch, idx, training_iterations, time() - start_time, train_loss, train_accuracy,\n",
    "                       test_accuracy))\n",
    "                counter += 1                \n",
    "        checkpoint.save(file_prefix=checkpoint_prefix + '-{}'.format(counter))\n",
    "        \n",
    "# test phase      \n",
    "else :\n",
    "    _, _ = load(network, checkpoint_dir)\n",
    "    for test_input, test_label in test_dataset:    \n",
    "        test_accuracy = accuracy_fn(network, test_input, test_label)\n",
    "\n",
    "    print(\"test_Accuracy: %.4f\" % (test_accuracy))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Accuracy : 96.31%\n",
    "### 함수를 바꾸는것보다 더 확실히 정확도가 향상되었음."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}