Lab10/Lab10-2-2 minkiKang.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 모두를 위한 딥러닝 시즌2 - TensorFlow\n",
    "\n",
    "## LAB10-2-2 MNIST NN DEEP Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.5.0\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras.utils import to_categorical\n",
    "from tensorflow.keras.datasets import mnist\n",
    "from time import time\n",
    "import os\n",
    "print(tf.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load(model, checkpoint_dir):\n",
    "    print(\" [*] Reading checkpoints...\")\n",
    "\n",
    "    ckpt = tf.train.get_checkpoint_state(checkpoint_dir)\n",
    "    if ckpt :\n",
    "        ckpt_name = os.path.basename(ckpt.model_checkpoint_path)\n",
    "        checkpoint = tf.train.Checkpoint(dnn=model)\n",
    "        checkpoint.restore(save_path=os.path.join(checkpoint_dir, ckpt_name))\n",
    "        counter = int(ckpt_name.split('-')[1])\n",
    "        print(\" [*] Success to read {}\".format(ckpt_name))\n",
    "        return True, counter\n",
    "    else:\n",
    "        print(\" [*] Failed to find a checkpoint\")\n",
    "        return False, 0\n",
    "\n",
    "def check_folder(dir):\n",
    "    if not os.path.exists(dir):\n",
    "        os.makedirs(dir)\n",
    "    return dir"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 데이터 로드 & 전처리"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_mnist() :\n",
    "    (train_data, train_labels), (test_data, test_labels) = mnist.load_data()\n",
    "    train_data = np.expand_dims(train_data, axis=-1) \n",
    "    test_data = np.expand_dims(test_data, axis=-1)\n",
    "\n",
    "    train_data, test_data = normalize(train_data, test_data)\n",
    "\n",
    "    train_labels = to_categorical(train_labels, 10)\n",
    "    test_labels = to_categorical(test_labels, 10)\n",
    "\n",
    "    return train_data, train_labels, test_data, test_labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def normalize(train_data, test_data):\n",
    "    train_data = train_data.astype(np.float32) / 255.0\n",
    "    test_data = test_data.astype(np.float32) / 255.0\n",
    "\n",
    "    return train_data, test_data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### LOSS , accuracy, grad function 정의"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def loss_fn(model, images, labels):\n",
    "    logits = model(images, training=True)\n",
    "    loss = tf.reduce_mean(tf.keras.losses.categorical_crossentropy(y_pred=logits, y_true=labels, from_logits=True))\n",
    "    return loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def accuracy_fn(model, images, labels):\n",
    "    logits = model(images, training=False)\n",
    "    prediction = tf.equal(tf.argmax(logits, -1), tf.argmax(labels, -1))\n",
    "    accuracy = tf.reduce_mean(tf.cast(prediction, tf.float32))\n",
    "    return accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def grad(model, images, labels):\n",
    "    with tf.GradientTape() as tape:\n",
    "        loss = loss_fn(model, images, labels)\n",
    "    return tape.gradient(loss, model.variables)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model 함수"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def flatten() :\n",
    "    return tf.keras.layers.Flatten()\n",
    "\n",
    "def dense(label_dim, weight_init) :\n",
    "    return tf.keras.layers.Dense(units=label_dim, use_bias=True, kernel_initializer=weight_init)\n",
    "\n",
    "def relu() :\n",
    "    return tf.keras.layers.Activation(tf.keras.activations.relu)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model 만들기\n",
    "1. class 버젼"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "class create_model_class(tf.keras.Model):\n",
    "    def __init__(self, label_dim):\n",
    "        super(create_model_class, self).__init__()\n",
    "        weight_init = tf.keras.initializers.glorot_uniform()\n",
    "\n",
    "        self.model = tf.keras.Sequential()\n",
    "        self.model.add(flatten())\n",
    "\n",
    "        for i in range(4):\n",
    "            self.model.add(dense(512, weight_init))\n",
    "            self.model.add(relu())\n",
    "\n",
    "        self.model.add(dense(label_dim, weight_init))\n",
    "\n",
    "    def call(self, x, training=None, mask=None):\n",
    "\n",
    "        x = self.model(x)\n",
    "\n",
    "        return x"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model 만들기\n",
    "2. function 버젼"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_model_function(label_dim) :\n",
    "    weight_init = tf.keras.initializers.glorot_uniform()\n",
    "\n",
    "    model = tf.keras.Sequential()\n",
    "    model.add(flatten())\n",
    "\n",
    "    for i in range(4) :\n",
    "        model.add(dense(512, weight_init))\n",
    "        model.add(relu())\n",
    "\n",
    "    model.add(dense(label_dim, weight_init))\n",
    "\n",
    "    return model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### DATA & 하이퍼 파라미터 정의"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_x, train_y, test_x, test_y = load_mnist()\n",
    "\n",
    "learning_rate = 0.001\n",
    "batch_size = 128\n",
    "\n",
    "training_epochs = 1\n",
    "training_iterations = len(train_x) // batch_size\n",
    "\n",
    "label_dim = 10\n",
    "\n",
    "train_flag = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dataset = tf.data.Dataset.from_tensor_slices((train_x, train_y)).\\\n",
    "    shuffle(buffer_size=100000).\\\n",
    "    prefetch(buffer_size=batch_size).\\\n",
    "    batch(batch_size, drop_remainder=True)\n",
    "\n",
    "test_dataset = tf.data.Dataset.from_tensor_slices((test_x, test_y)).\\\n",
    "    shuffle(buffer_size=100000).\\\n",
    "    prefetch(buffer_size=len(test_x)).\\\n",
    "    batch(len(test_x))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model, 옵티마이저, writer 정의"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "network = create_model_function(label_dim)\n",
    "\n",
    "optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate)\n",
    "\n",
    "checkpoint_dir = 'checkpoints'\n",
    "logs_dir = 'logs'\n",
    "\n",
    "model_dir = 'nn_deep'\n",
    "\n",
    "checkpoint_dir = os.path.join(checkpoint_dir, model_dir)\n",
    "check_folder(checkpoint_dir)\n",
    "checkpoint_prefix = os.path.join(checkpoint_dir, model_dir)\n",
    "logs_dir = os.path.join(logs_dir, model_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " [*] Reading checkpoints...\n",
      " [*] Failed to find a checkpoint\n",
      " [!] Load failed...\n",
      "Epoch: [ 0] [    0/  468] time: 1.6626, train_loss: 2.02639771, train_accuracy: 0.4766, test_Accuracy: 0.3097\n",
      "Epoch: [ 0] [    1/  468] time: 2.1444, train_loss: 1.85179687, train_accuracy: 0.4922, test_Accuracy: 0.3891\n",
      "Epoch: [ 0] [    2/  468] time: 2.7401, train_loss: 1.61427808, train_accuracy: 0.6328, test_Accuracy: 0.5164\n",
      "Epoch: [ 0] [    3/  468] time: 3.2681, train_loss: 1.48496461, train_accuracy: 0.6172, test_Accuracy: 0.5812\n",
      "Epoch: [ 0] [    4/  468] time: 3.8439, train_loss: 1.19562840, train_accuracy: 0.7031, test_Accuracy: 0.6224\n",
      "Epoch: [ 0] [    5/  468] time: 4.4926, train_loss: 0.95590538, train_accuracy: 0.7891, test_Accuracy: 0.6928\n",
      "Epoch: [ 0] [    6/  468] time: 4.9825, train_loss: 0.81262755, train_accuracy: 0.7812, test_Accuracy: 0.7349\n",
      "Epoch: [ 0] [    7/  468] time: 5.4972, train_loss: 0.74732697, train_accuracy: 0.7812, test_Accuracy: 0.7482\n",
      "Epoch: [ 0] [    8/  468] time: 6.0717, train_loss: 0.55612296, train_accuracy: 0.8359, test_Accuracy: 0.8047\n",
      "Epoch: [ 0] [    9/  468] time: 6.5305, train_loss: 0.62555486, train_accuracy: 0.7656, test_Accuracy: 0.7113\n",
      "Epoch: [ 0] [   10/  468] time: 6.9858, train_loss: 0.55878711, train_accuracy: 0.8203, test_Accuracy: 0.7812\n",
      "Epoch: [ 0] [   11/  468] time: 7.5622, train_loss: 0.51816869, train_accuracy: 0.8359, test_Accuracy: 0.7872\n",
      "Epoch: [ 0] [   12/  468] time: 7.9249, train_loss: 0.55879605, train_accuracy: 0.8281, test_Accuracy: 0.8205\n",
      "Epoch: [ 0] [   13/  468] time: 8.3144, train_loss: 0.47275525, train_accuracy: 0.8516, test_Accuracy: 0.8122\n",
      "Epoch: [ 0] [   14/  468] time: 8.8032, train_loss: 0.44336277, train_accuracy: 0.8359, test_Accuracy: 0.8251\n",
      "Epoch: [ 0] [   15/  468] time: 9.3789, train_loss: 0.46166545, train_accuracy: 0.8828, test_Accuracy: 0.8560\n",
      "Epoch: [ 0] [   16/  468] time: 9.9773, train_loss: 0.48363435, train_accuracy: 0.8359, test_Accuracy: 0.8339\n",
      "Epoch: [ 0] [   17/  468] time: 10.5533, train_loss: 0.36785790, train_accuracy: 0.8516, test_Accuracy: 0.8393\n",
      "Epoch: [ 0] [   18/  468] time: 11.1485, train_loss: 0.47649470, train_accuracy: 0.8828, test_Accuracy: 0.8682\n",
      "Epoch: [ 0] [   19/  468] time: 11.6414, train_loss: 0.52649915, train_accuracy: 0.8516, test_Accuracy: 0.8520\n",
      "Epoch: [ 0] [   20/  468] time: 12.1913, train_loss: 0.56360430, train_accuracy: 0.8203, test_Accuracy: 0.8410\n",
      "Epoch: [ 0] [   21/  468] time: 12.5446, train_loss: 0.28209257, train_accuracy: 0.8828, test_Accuracy: 0.8406\n",
      "Epoch: [ 0] [   22/  468] time: 13.0200, train_loss: 0.46821979, train_accuracy: 0.8516, test_Accuracy: 0.8715\n",
      "Epoch: [ 0] [   23/  468] time: 13.6446, train_loss: 0.43653408, train_accuracy: 0.8750, test_Accuracy: 0.8822\n",
      "Epoch: [ 0] [   24/  468] time: 14.1811, train_loss: 0.33116782, train_accuracy: 0.9219, test_Accuracy: 0.8845\n",
      "Epoch: [ 0] [   25/  468] time: 14.7417, train_loss: 0.48529553, train_accuracy: 0.8672, test_Accuracy: 0.8823\n",
      "Epoch: [ 0] [   26/  468] time: 15.2233, train_loss: 0.29929322, train_accuracy: 0.8984, test_Accuracy: 0.8931\n",
      "Epoch: [ 0] [   27/  468] time: 15.6559, train_loss: 0.42372787, train_accuracy: 0.8672, test_Accuracy: 0.8915\n",
      "Epoch: [ 0] [   28/  468] time: 16.0166, train_loss: 0.40326977, train_accuracy: 0.8281, test_Accuracy: 0.8984\n",
      "Epoch: [ 0] [   29/  468] time: 16.4834, train_loss: 0.42478439, train_accuracy: 0.8672, test_Accuracy: 0.8975\n",
      "Epoch: [ 0] [   30/  468] time: 17.1049, train_loss: 0.23474216, train_accuracy: 0.9375, test_Accuracy: 0.8904\n",
      "Epoch: [ 0] [   31/  468] time: 17.6333, train_loss: 0.26802653, train_accuracy: 0.9375, test_Accuracy: 0.8823\n",
      "Epoch: [ 0] [   32/  468] time: 18.0693, train_loss: 0.26872170, train_accuracy: 0.9219, test_Accuracy: 0.8850\n",
      "Epoch: [ 0] [   33/  468] time: 18.5846, train_loss: 0.31947058, train_accuracy: 0.8984, test_Accuracy: 0.8880\n",
      "Epoch: [ 0] [   34/  468] time: 19.1355, train_loss: 0.39374378, train_accuracy: 0.8516, test_Accuracy: 0.8819\n",
      "Epoch: [ 0] [   35/  468] time: 19.7041, train_loss: 0.31495565, train_accuracy: 0.8984, test_Accuracy: 0.8836\n",
      "Epoch: [ 0] [   36/  468] time: 20.2556, train_loss: 0.41316575, train_accuracy: 0.9141, test_Accuracy: 0.8951\n",
      "Epoch: [ 0] [   37/  468] time: 20.7693, train_loss: 0.34646225, train_accuracy: 0.9062, test_Accuracy: 0.9082\n",
      "Epoch: [ 0] [   38/  468] time: 21.2455, train_loss: 0.29582563, train_accuracy: 0.8828, test_Accuracy: 0.9019\n",
      "Epoch: [ 0] [   39/  468] time: 21.6929, train_loss: 0.32248154, train_accuracy: 0.8984, test_Accuracy: 0.8849\n",
      "Epoch: [ 0] [   40/  468] time: 22.2799, train_loss: 0.39314300, train_accuracy: 0.8984, test_Accuracy: 0.8933\n",
      "Epoch: [ 0] [   41/  468] time: 22.7268, train_loss: 0.32158220, train_accuracy: 0.9219, test_Accuracy: 0.8966\n",
      "Epoch: [ 0] [   42/  468] time: 23.2446, train_loss: 0.29621947, train_accuracy: 0.9062, test_Accuracy: 0.9045\n",
      "Epoch: [ 0] [   43/  468] time: 23.7000, train_loss: 0.33137825, train_accuracy: 0.8906, test_Accuracy: 0.9046\n",
      "Epoch: [ 0] [   44/  468] time: 24.2173, train_loss: 0.24017844, train_accuracy: 0.9219, test_Accuracy: 0.9024\n",
      "Epoch: [ 0] [   45/  468] time: 24.7638, train_loss: 0.27373642, train_accuracy: 0.9062, test_Accuracy: 0.9027\n",
      "Epoch: [ 0] [   46/  468] time: 25.3084, train_loss: 0.20909405, train_accuracy: 0.9453, test_Accuracy: 0.9017\n",
      "Epoch: [ 0] [   47/  468] time: 25.8061, train_loss: 0.47610539, train_accuracy: 0.8672, test_Accuracy: 0.9129\n",
      "Epoch: [ 0] [   48/  468] time: 26.2838, train_loss: 0.25820667, train_accuracy: 0.9297, test_Accuracy: 0.9157\n",
      "Epoch: [ 0] [   49/  468] time: 26.7748, train_loss: 0.30084544, train_accuracy: 0.8984, test_Accuracy: 0.9123\n",
      "Epoch: [ 0] [   50/  468] time: 27.2735, train_loss: 0.24420170, train_accuracy: 0.9297, test_Accuracy: 0.9008\n",
      "Epoch: [ 0] [   51/  468] time: 27.7428, train_loss: 0.34485096, train_accuracy: 0.9219, test_Accuracy: 0.8950\n",
      "Epoch: [ 0] [   52/  468] time: 28.2285, train_loss: 0.25762808, train_accuracy: 0.9219, test_Accuracy: 0.9020\n",
      "Epoch: [ 0] [   53/  468] time: 28.7521, train_loss: 0.36474735, train_accuracy: 0.9062, test_Accuracy: 0.9097\n",
      "Epoch: [ 0] [   54/  468] time: 29.2728, train_loss: 0.27423772, train_accuracy: 0.9375, test_Accuracy: 0.9167\n",
      "Epoch: [ 0] [   55/  468] time: 29.8641, train_loss: 0.23773149, train_accuracy: 0.9453, test_Accuracy: 0.9215\n",
      "Epoch: [ 0] [   56/  468] time: 30.3846, train_loss: 0.24193463, train_accuracy: 0.9219, test_Accuracy: 0.9206\n",
      "Epoch: [ 0] [   57/  468] time: 30.9383, train_loss: 0.21922940, train_accuracy: 0.9609, test_Accuracy: 0.9143\n",
      "Epoch: [ 0] [   58/  468] time: 31.4764, train_loss: 0.17469695, train_accuracy: 0.9453, test_Accuracy: 0.9145\n",
      "Epoch: [ 0] [   59/  468] time: 31.9448, train_loss: 0.22478482, train_accuracy: 0.9375, test_Accuracy: 0.9157\n",
      "Epoch: [ 0] [   60/  468] time: 32.4614, train_loss: 0.32465959, train_accuracy: 0.8984, test_Accuracy: 0.9200\n",
      "Epoch: [ 0] [   61/  468] time: 33.0325, train_loss: 0.22454751, train_accuracy: 0.9453, test_Accuracy: 0.9207\n",
      "Epoch: [ 0] [   62/  468] time: 33.5587, train_loss: 0.17863050, train_accuracy: 0.9219, test_Accuracy: 0.9202\n",
      "Epoch: [ 0] [   63/  468] time: 34.0474, train_loss: 0.27775407, train_accuracy: 0.9062, test_Accuracy: 0.9181\n",
      "Epoch: [ 0] [   64/  468] time: 34.5095, train_loss: 0.18581118, train_accuracy: 0.9297, test_Accuracy: 0.9171\n",
      "Epoch: [ 0] [   65/  468] time: 35.0789, train_loss: 0.20897689, train_accuracy: 0.9219, test_Accuracy: 0.9167\n",
      "Epoch: [ 0] [   66/  468] time: 35.6376, train_loss: 0.24953313, train_accuracy: 0.9219, test_Accuracy: 0.9153\n",
      "Epoch: [ 0] [   67/  468] time: 36.2383, train_loss: 0.18526343, train_accuracy: 0.9453, test_Accuracy: 0.9197\n",
      "Epoch: [ 0] [   68/  468] time: 36.7300, train_loss: 0.27959126, train_accuracy: 0.8906, test_Accuracy: 0.9228\n",
      "Epoch: [ 0] [   69/  468] time: 37.1750, train_loss: 0.24824589, train_accuracy: 0.9219, test_Accuracy: 0.9220\n",
      "Epoch: [ 0] [   70/  468] time: 37.7131, train_loss: 0.27421695, train_accuracy: 0.9297, test_Accuracy: 0.9248\n",
      "Epoch: [ 0] [   71/  468] time: 38.2223, train_loss: 0.33436385, train_accuracy: 0.9062, test_Accuracy: 0.9275\n",
      "Epoch: [ 0] [   72/  468] time: 38.7751, train_loss: 0.16566004, train_accuracy: 0.9453, test_Accuracy: 0.9299\n",
      "Epoch: [ 0] [   73/  468] time: 39.2247, train_loss: 0.08854404, train_accuracy: 0.9766, test_Accuracy: 0.9277\n",
      "Epoch: [ 0] [   74/  468] time: 39.8007, train_loss: 0.20632297, train_accuracy: 0.9609, test_Accuracy: 0.9253\n",
      "Epoch: [ 0] [   75/  468] time: 40.2821, train_loss: 0.22123028, train_accuracy: 0.9453, test_Accuracy: 0.9265\n",
      "Epoch: [ 0] [   76/  468] time: 40.7341, train_loss: 0.16541186, train_accuracy: 0.9375, test_Accuracy: 0.9329\n",
      "Epoch: [ 0] [   77/  468] time: 41.3223, train_loss: 0.13434994, train_accuracy: 0.9688, test_Accuracy: 0.9368\n",
      "Epoch: [ 0] [   78/  468] time: 41.6774, train_loss: 0.19572967, train_accuracy: 0.9453, test_Accuracy: 0.9363\n",
      "Epoch: [ 0] [   79/  468] time: 42.0906, train_loss: 0.24479514, train_accuracy: 0.9609, test_Accuracy: 0.9328\n",
      "Epoch: [ 0] [   80/  468] time: 42.4913, train_loss: 0.20637056, train_accuracy: 0.9453, test_Accuracy: 0.9268\n",
      "Epoch: [ 0] [   81/  468] time: 42.8891, train_loss: 0.25985003, train_accuracy: 0.9219, test_Accuracy: 0.9268\n",
      "Epoch: [ 0] [   82/  468] time: 43.3014, train_loss: 0.19212638, train_accuracy: 0.9453, test_Accuracy: 0.9318\n",
      "Epoch: [ 0] [   83/  468] time: 43.7209, train_loss: 0.21706152, train_accuracy: 0.9453, test_Accuracy: 0.9357\n",
      "Epoch: [ 0] [   84/  468] time: 44.1504, train_loss: 0.19181693, train_accuracy: 0.9453, test_Accuracy: 0.9381\n",
      "Epoch: [ 0] [   85/  468] time: 44.5922, train_loss: 0.07276830, train_accuracy: 0.9844, test_Accuracy: 0.9353\n",
      "Epoch: [ 0] [   86/  468] time: 45.0286, train_loss: 0.25332052, train_accuracy: 0.9219, test_Accuracy: 0.9270\n",
      "Epoch: [ 0] [   87/  468] time: 45.5948, train_loss: 0.18429098, train_accuracy: 0.9453, test_Accuracy: 0.9203\n",
      "Epoch: [ 0] [   88/  468] time: 46.1100, train_loss: 0.28095728, train_accuracy: 0.9297, test_Accuracy: 0.9263\n",
      "Epoch: [ 0] [   89/  468] time: 46.5583, train_loss: 0.17593798, train_accuracy: 0.9531, test_Accuracy: 0.9314\n",
      "Epoch: [ 0] [   90/  468] time: 47.0217, train_loss: 0.17549412, train_accuracy: 0.9453, test_Accuracy: 0.9306\n",
      "Epoch: [ 0] [   91/  468] time: 47.5793, train_loss: 0.16543156, train_accuracy: 0.9453, test_Accuracy: 0.9301\n",
      "Epoch: [ 0] [   92/  468] time: 48.0537, train_loss: 0.10252437, train_accuracy: 0.9688, test_Accuracy: 0.9304\n",
      "Epoch: [ 0] [   93/  468] time: 48.5759, train_loss: 0.14957981, train_accuracy: 0.9453, test_Accuracy: 0.9330\n",
      "Epoch: [ 0] [   94/  468] time: 49.0227, train_loss: 0.15656196, train_accuracy: 0.9688, test_Accuracy: 0.9352\n",
      "Epoch: [ 0] [   95/  468] time: 49.4987, train_loss: 0.19620517, train_accuracy: 0.9219, test_Accuracy: 0.9342\n",
      "Epoch: [ 0] [   96/  468] time: 49.9884, train_loss: 0.09707466, train_accuracy: 0.9766, test_Accuracy: 0.9337\n",
      "Epoch: [ 0] [   97/  468] time: 50.4864, train_loss: 0.18174826, train_accuracy: 0.9375, test_Accuracy: 0.9338\n",
      "Epoch: [ 0] [   98/  468] time: 50.9811, train_loss: 0.23113410, train_accuracy: 0.9453, test_Accuracy: 0.9310\n",
      "Epoch: [ 0] [   99/  468] time: 51.4602, train_loss: 0.15953958, train_accuracy: 0.9609, test_Accuracy: 0.9283\n",
      "Epoch: [ 0] [  100/  468] time: 51.9393, train_loss: 0.25143701, train_accuracy: 0.8906, test_Accuracy: 0.9298\n",
      "Epoch: [ 0] [  101/  468] time: 52.4503, train_loss: 0.14243069, train_accuracy: 0.9688, test_Accuracy: 0.9323\n",
      "Epoch: [ 0] [  102/  468] time: 52.8837, train_loss: 0.17070998, train_accuracy: 0.9453, test_Accuracy: 0.9288\n",
      "Epoch: [ 0] [  103/  468] time: 53.4409, train_loss: 0.28306150, train_accuracy: 0.9141, test_Accuracy: 0.9294\n",
      "Epoch: [ 0] [  104/  468] time: 53.9303, train_loss: 0.11761062, train_accuracy: 0.9688, test_Accuracy: 0.9331\n",
      "Epoch: [ 0] [  105/  468] time: 54.4660, train_loss: 0.12100567, train_accuracy: 0.9688, test_Accuracy: 0.9349\n",
      "Epoch: [ 0] [  106/  468] time: 54.9562, train_loss: 0.15262938, train_accuracy: 0.9531, test_Accuracy: 0.9355\n",
      "Epoch: [ 0] [  107/  468] time: 55.4999, train_loss: 0.21575266, train_accuracy: 0.9062, test_Accuracy: 0.9367\n",
      "Epoch: [ 0] [  108/  468] time: 55.9190, train_loss: 0.10616966, train_accuracy: 0.9609, test_Accuracy: 0.9372\n",
      "Epoch: [ 0] [  109/  468] time: 56.4432, train_loss: 0.20420285, train_accuracy: 0.9219, test_Accuracy: 0.9396\n",
      "Epoch: [ 0] [  110/  468] time: 56.9541, train_loss: 0.12896767, train_accuracy: 0.9531, test_Accuracy: 0.9345\n",
      "Epoch: [ 0] [  111/  468] time: 57.4240, train_loss: 0.14872409, train_accuracy: 0.9531, test_Accuracy: 0.9319\n",
      "Epoch: [ 0] [  112/  468] time: 57.9285, train_loss: 0.09571786, train_accuracy: 0.9844, test_Accuracy: 0.9274\n",
      "Epoch: [ 0] [  113/  468] time: 58.3267, train_loss: 0.20800206, train_accuracy: 0.9141, test_Accuracy: 0.9296\n",
      "Epoch: [ 0] [  114/  468] time: 58.7581, train_loss: 0.26310375, train_accuracy: 0.9141, test_Accuracy: 0.9386\n",
      "Epoch: [ 0] [  115/  468] time: 59.1673, train_loss: 0.09160641, train_accuracy: 0.9688, test_Accuracy: 0.9413\n",
      "Epoch: [ 0] [  116/  468] time: 59.7541, train_loss: 0.18140523, train_accuracy: 0.9453, test_Accuracy: 0.9407\n",
      "Epoch: [ 0] [  117/  468] time: 60.1539, train_loss: 0.17428556, train_accuracy: 0.9531, test_Accuracy: 0.9375\n",
      "Epoch: [ 0] [  118/  468] time: 60.6481, train_loss: 0.14260423, train_accuracy: 0.9609, test_Accuracy: 0.9376\n",
      "Epoch: [ 0] [  119/  468] time: 61.2057, train_loss: 0.08641519, train_accuracy: 0.9688, test_Accuracy: 0.9375\n",
      "Epoch: [ 0] [  120/  468] time: 61.7767, train_loss: 0.18291007, train_accuracy: 0.9453, test_Accuracy: 0.9409\n",
      "Epoch: [ 0] [  121/  468] time: 62.1531, train_loss: 0.20721330, train_accuracy: 0.9453, test_Accuracy: 0.9414\n",
      "Epoch: [ 0] [  122/  468] time: 62.6033, train_loss: 0.06603124, train_accuracy: 0.9844, test_Accuracy: 0.9424\n",
      "Epoch: [ 0] [  123/  468] time: 63.0654, train_loss: 0.10769351, train_accuracy: 0.9688, test_Accuracy: 0.9416\n",
      "Epoch: [ 0] [  124/  468] time: 63.5561, train_loss: 0.13465367, train_accuracy: 0.9453, test_Accuracy: 0.9426\n",
      "Epoch: [ 0] [  125/  468] time: 64.0521, train_loss: 0.26930261, train_accuracy: 0.9297, test_Accuracy: 0.9362\n",
      "Epoch: [ 0] [  126/  468] time: 64.5120, train_loss: 0.15670753, train_accuracy: 0.9453, test_Accuracy: 0.9339\n",
      "Epoch: [ 0] [  127/  468] time: 64.9937, train_loss: 0.11526508, train_accuracy: 0.9609, test_Accuracy: 0.9377\n",
      "Epoch: [ 0] [  128/  468] time: 65.4752, train_loss: 0.31078115, train_accuracy: 0.8984, test_Accuracy: 0.9454\n",
      "Epoch: [ 0] [  129/  468] time: 65.9320, train_loss: 0.10374214, train_accuracy: 0.9766, test_Accuracy: 0.9473\n",
      "Epoch: [ 0] [  130/  468] time: 66.3566, train_loss: 0.11234278, train_accuracy: 0.9531, test_Accuracy: 0.9437\n",
      "Epoch: [ 0] [  131/  468] time: 66.8337, train_loss: 0.17962554, train_accuracy: 0.9453, test_Accuracy: 0.9421\n",
      "Epoch: [ 0] [  132/  468] time: 67.2562, train_loss: 0.15697023, train_accuracy: 0.9453, test_Accuracy: 0.9420\n",
      "Epoch: [ 0] [  133/  468] time: 67.6408, train_loss: 0.18682334, train_accuracy: 0.9375, test_Accuracy: 0.9460\n",
      "Epoch: [ 0] [  134/  468] time: 68.1020, train_loss: 0.11871973, train_accuracy: 0.9688, test_Accuracy: 0.9497\n",
      "Epoch: [ 0] [  135/  468] time: 68.5030, train_loss: 0.13312697, train_accuracy: 0.9688, test_Accuracy: 0.9498\n",
      "Epoch: [ 0] [  136/  468] time: 68.8840, train_loss: 0.10547537, train_accuracy: 0.9688, test_Accuracy: 0.9484\n",
      "Epoch: [ 0] [  137/  468] time: 69.2893, train_loss: 0.09522498, train_accuracy: 0.9766, test_Accuracy: 0.9479\n",
      "Epoch: [ 0] [  138/  468] time: 69.8222, train_loss: 0.21709588, train_accuracy: 0.9297, test_Accuracy: 0.9484\n",
      "Epoch: [ 0] [  139/  468] time: 70.2679, train_loss: 0.06215958, train_accuracy: 0.9844, test_Accuracy: 0.9451\n",
      "Epoch: [ 0] [  140/  468] time: 70.7063, train_loss: 0.14972806, train_accuracy: 0.9531, test_Accuracy: 0.9390\n",
      "Epoch: [ 0] [  141/  468] time: 71.1680, train_loss: 0.15265463, train_accuracy: 0.9453, test_Accuracy: 0.9356\n",
      "Epoch: [ 0] [  142/  468] time: 71.6328, train_loss: 0.15953714, train_accuracy: 0.9453, test_Accuracy: 0.9372\n",
      "Epoch: [ 0] [  143/  468] time: 72.1132, train_loss: 0.21228217, train_accuracy: 0.9375, test_Accuracy: 0.9465\n",
      "Epoch: [ 0] [  144/  468] time: 72.5354, train_loss: 0.08856539, train_accuracy: 0.9766, test_Accuracy: 0.9509\n",
      "Epoch: [ 0] [  145/  468] time: 72.9223, train_loss: 0.08485565, train_accuracy: 0.9766, test_Accuracy: 0.9526\n",
      "Epoch: [ 0] [  146/  468] time: 73.3808, train_loss: 0.11440787, train_accuracy: 0.9609, test_Accuracy: 0.9490\n",
      "Epoch: [ 0] [  147/  468] time: 73.8394, train_loss: 0.13930257, train_accuracy: 0.9453, test_Accuracy: 0.9446\n",
      "Epoch: [ 0] [  148/  468] time: 74.2855, train_loss: 0.19703487, train_accuracy: 0.9375, test_Accuracy: 0.9403\n",
      "Epoch: [ 0] [  149/  468] time: 74.7363, train_loss: 0.12516175, train_accuracy: 0.9609, test_Accuracy: 0.9420\n",
      "Epoch: [ 0] [  150/  468] time: 75.1621, train_loss: 0.15567058, train_accuracy: 0.9609, test_Accuracy: 0.9449\n",
      "Epoch: [ 0] [  151/  468] time: 75.6497, train_loss: 0.21557216, train_accuracy: 0.9453, test_Accuracy: 0.9492\n",
      "Epoch: [ 0] [  152/  468] time: 76.1670, train_loss: 0.20405090, train_accuracy: 0.9609, test_Accuracy: 0.9504\n",
      "Epoch: [ 0] [  153/  468] time: 76.6135, train_loss: 0.09544660, train_accuracy: 0.9844, test_Accuracy: 0.9462\n",
      "Epoch: [ 0] [  154/  468] time: 77.0670, train_loss: 0.26624221, train_accuracy: 0.9297, test_Accuracy: 0.9408\n",
      "Epoch: [ 0] [  155/  468] time: 77.5738, train_loss: 0.13618138, train_accuracy: 0.9609, test_Accuracy: 0.9377\n",
      "Epoch: [ 0] [  156/  468] time: 78.0964, train_loss: 0.09341370, train_accuracy: 0.9609, test_Accuracy: 0.9343\n",
      "Epoch: [ 0] [  157/  468] time: 78.5988, train_loss: 0.18977840, train_accuracy: 0.9375, test_Accuracy: 0.9379\n",
      "Epoch: [ 0] [  158/  468] time: 79.1096, train_loss: 0.14068428, train_accuracy: 0.9609, test_Accuracy: 0.9428\n",
      "Epoch: [ 0] [  159/  468] time: 79.6189, train_loss: 0.10572978, train_accuracy: 0.9688, test_Accuracy: 0.9478\n",
      "Epoch: [ 0] [  160/  468] time: 80.1707, train_loss: 0.14488509, train_accuracy: 0.9688, test_Accuracy: 0.9442\n",
      "Epoch: [ 0] [  161/  468] time: 80.6756, train_loss: 0.19359486, train_accuracy: 0.9453, test_Accuracy: 0.9346\n",
      "Epoch: [ 0] [  162/  468] time: 81.1706, train_loss: 0.16717061, train_accuracy: 0.9531, test_Accuracy: 0.9284\n",
      "Epoch: [ 0] [  163/  468] time: 81.6256, train_loss: 0.17823580, train_accuracy: 0.9531, test_Accuracy: 0.9235\n",
      "Epoch: [ 0] [  164/  468] time: 82.0854, train_loss: 0.13417992, train_accuracy: 0.9609, test_Accuracy: 0.9252\n",
      "Epoch: [ 0] [  165/  468] time: 82.5886, train_loss: 0.21483167, train_accuracy: 0.9453, test_Accuracy: 0.9373\n",
      "Epoch: [ 0] [  166/  468] time: 82.9952, train_loss: 0.09693559, train_accuracy: 0.9766, test_Accuracy: 0.9516\n",
      "Epoch: [ 0] [  167/  468] time: 83.5321, train_loss: 0.11200704, train_accuracy: 0.9766, test_Accuracy: 0.9533\n",
      "Epoch: [ 0] [  168/  468] time: 84.0294, train_loss: 0.16293444, train_accuracy: 0.9531, test_Accuracy: 0.9488\n",
      "Epoch: [ 0] [  169/  468] time: 84.5392, train_loss: 0.14251783, train_accuracy: 0.9609, test_Accuracy: 0.9411\n",
      "Epoch: [ 0] [  170/  468] time: 84.9886, train_loss: 0.13997811, train_accuracy: 0.9609, test_Accuracy: 0.9376\n",
      "Epoch: [ 0] [  171/  468] time: 85.4682, train_loss: 0.07564577, train_accuracy: 0.9922, test_Accuracy: 0.9337\n",
      "Epoch: [ 0] [  172/  468] time: 85.9095, train_loss: 0.19320256, train_accuracy: 0.9297, test_Accuracy: 0.9370\n",
      "Epoch: [ 0] [  173/  468] time: 86.4403, train_loss: 0.18585260, train_accuracy: 0.9297, test_Accuracy: 0.9443\n",
      "Epoch: [ 0] [  174/  468] time: 86.8570, train_loss: 0.24217319, train_accuracy: 0.9297, test_Accuracy: 0.9518\n",
      "Epoch: [ 0] [  175/  468] time: 87.3254, train_loss: 0.10565606, train_accuracy: 0.9766, test_Accuracy: 0.9557\n",
      "Epoch: [ 0] [  176/  468] time: 87.7888, train_loss: 0.15257345, train_accuracy: 0.9531, test_Accuracy: 0.9522\n",
      "Epoch: [ 0] [  177/  468] time: 88.2806, train_loss: 0.12326809, train_accuracy: 0.9531, test_Accuracy: 0.9489\n",
      "Epoch: [ 0] [  178/  468] time: 88.7069, train_loss: 0.14796320, train_accuracy: 0.9453, test_Accuracy: 0.9466\n",
      "Epoch: [ 0] [  179/  468] time: 89.0935, train_loss: 0.15619451, train_accuracy: 0.9609, test_Accuracy: 0.9450\n",
      "Epoch: [ 0] [  180/  468] time: 89.4889, train_loss: 0.20216419, train_accuracy: 0.9297, test_Accuracy: 0.9478\n",
      "Epoch: [ 0] [  181/  468] time: 89.9766, train_loss: 0.13750333, train_accuracy: 0.9531, test_Accuracy: 0.9489\n",
      "Epoch: [ 0] [  182/  468] time: 90.3607, train_loss: 0.11788044, train_accuracy: 0.9688, test_Accuracy: 0.9510\n",
      "Epoch: [ 0] [  183/  468] time: 90.8389, train_loss: 0.10918991, train_accuracy: 0.9766, test_Accuracy: 0.9518\n",
      "Epoch: [ 0] [  184/  468] time: 91.2314, train_loss: 0.16732740, train_accuracy: 0.9531, test_Accuracy: 0.9510\n",
      "Epoch: [ 0] [  185/  468] time: 91.7730, train_loss: 0.20805159, train_accuracy: 0.9375, test_Accuracy: 0.9498\n",
      "Epoch: [ 0] [  186/  468] time: 92.2746, train_loss: 0.20044281, train_accuracy: 0.9688, test_Accuracy: 0.9496\n",
      "Epoch: [ 0] [  187/  468] time: 92.7013, train_loss: 0.14032070, train_accuracy: 0.9688, test_Accuracy: 0.9560\n",
      "Epoch: [ 0] [  188/  468] time: 93.0867, train_loss: 0.11706147, train_accuracy: 0.9688, test_Accuracy: 0.9584\n",
      "Epoch: [ 0] [  189/  468] time: 93.6756, train_loss: 0.09424214, train_accuracy: 0.9766, test_Accuracy: 0.9567\n",
      "Epoch: [ 0] [  190/  468] time: 94.2499, train_loss: 0.21171261, train_accuracy: 0.9297, test_Accuracy: 0.9524\n",
      "Epoch: [ 0] [  191/  468] time: 94.7978, train_loss: 0.08968289, train_accuracy: 0.9609, test_Accuracy: 0.9488\n",
      "Epoch: [ 0] [  192/  468] time: 95.3350, train_loss: 0.13656214, train_accuracy: 0.9688, test_Accuracy: 0.9487\n",
      "Epoch: [ 0] [  193/  468] time: 95.8639, train_loss: 0.15716638, train_accuracy: 0.9375, test_Accuracy: 0.9502\n",
      "Epoch: [ 0] [  194/  468] time: 96.4123, train_loss: 0.10131617, train_accuracy: 0.9609, test_Accuracy: 0.9512\n",
      "Epoch: [ 0] [  195/  468] time: 96.9038, train_loss: 0.15211324, train_accuracy: 0.9688, test_Accuracy: 0.9549\n",
      "Epoch: [ 0] [  196/  468] time: 97.2975, train_loss: 0.10975751, train_accuracy: 0.9688, test_Accuracy: 0.9546\n",
      "Epoch: [ 0] [  197/  468] time: 97.7119, train_loss: 0.07044784, train_accuracy: 0.9766, test_Accuracy: 0.9549\n",
      "Epoch: [ 0] [  198/  468] time: 98.1327, train_loss: 0.19315219, train_accuracy: 0.9453, test_Accuracy: 0.9516\n",
      "Epoch: [ 0] [  199/  468] time: 98.4982, train_loss: 0.08790835, train_accuracy: 0.9766, test_Accuracy: 0.9489\n",
      "Epoch: [ 0] [  200/  468] time: 98.8898, train_loss: 0.15220618, train_accuracy: 0.9531, test_Accuracy: 0.9492\n",
      "Epoch: [ 0] [  201/  468] time: 99.3701, train_loss: 0.18772852, train_accuracy: 0.9453, test_Accuracy: 0.9539\n",
      "Epoch: [ 0] [  202/  468] time: 99.8479, train_loss: 0.10918977, train_accuracy: 0.9688, test_Accuracy: 0.9584\n",
      "Epoch: [ 0] [  203/  468] time: 100.3138, train_loss: 0.18111539, train_accuracy: 0.9531, test_Accuracy: 0.9584\n",
      "Epoch: [ 0] [  204/  468] time: 100.6615, train_loss: 0.13775209, train_accuracy: 0.9531, test_Accuracy: 0.9570\n",
      "Epoch: [ 0] [  205/  468] time: 101.1444, train_loss: 0.10390475, train_accuracy: 0.9688, test_Accuracy: 0.9549\n",
      "Epoch: [ 0] [  206/  468] time: 101.5237, train_loss: 0.14733343, train_accuracy: 0.9531, test_Accuracy: 0.9539\n",
      "Epoch: [ 0] [  207/  468] time: 101.9067, train_loss: 0.12875423, train_accuracy: 0.9531, test_Accuracy: 0.9535\n",
      "Epoch: [ 0] [  208/  468] time: 102.3816, train_loss: 0.23895371, train_accuracy: 0.9297, test_Accuracy: 0.9551\n",
      "Epoch: [ 0] [  209/  468] time: 102.8422, train_loss: 0.10875640, train_accuracy: 0.9688, test_Accuracy: 0.9559\n",
      "Epoch: [ 0] [  210/  468] time: 103.2715, train_loss: 0.16076890, train_accuracy: 0.9531, test_Accuracy: 0.9534\n",
      "Epoch: [ 0] [  211/  468] time: 103.8568, train_loss: 0.11883750, train_accuracy: 0.9688, test_Accuracy: 0.9526\n",
      "Epoch: [ 0] [  212/  468] time: 104.4125, train_loss: 0.03256597, train_accuracy: 1.0000, test_Accuracy: 0.9519\n",
      "Epoch: [ 0] [  213/  468] time: 104.9203, train_loss: 0.13071319, train_accuracy: 0.9609, test_Accuracy: 0.9568\n",
      "Epoch: [ 0] [  214/  468] time: 105.3808, train_loss: 0.15905881, train_accuracy: 0.9297, test_Accuracy: 0.9594\n",
      "Epoch: [ 0] [  215/  468] time: 105.9514, train_loss: 0.15941530, train_accuracy: 0.9453, test_Accuracy: 0.9602\n",
      "Epoch: [ 0] [  216/  468] time: 106.4718, train_loss: 0.18701205, train_accuracy: 0.9688, test_Accuracy: 0.9593\n",
      "Epoch: [ 0] [  217/  468] time: 107.0382, train_loss: 0.15463577, train_accuracy: 0.9531, test_Accuracy: 0.9602\n",
      "Epoch: [ 0] [  218/  468] time: 107.5808, train_loss: 0.06363159, train_accuracy: 0.9844, test_Accuracy: 0.9599\n",
      "Epoch: [ 0] [  219/  468] time: 108.0531, train_loss: 0.13673180, train_accuracy: 0.9688, test_Accuracy: 0.9580\n",
      "Epoch: [ 0] [  220/  468] time: 108.5092, train_loss: 0.11168933, train_accuracy: 0.9766, test_Accuracy: 0.9562\n",
      "Epoch: [ 0] [  221/  468] time: 109.0308, train_loss: 0.07854626, train_accuracy: 0.9766, test_Accuracy: 0.9577\n",
      "Epoch: [ 0] [  222/  468] time: 109.5345, train_loss: 0.11549052, train_accuracy: 0.9766, test_Accuracy: 0.9584\n",
      "Epoch: [ 0] [  223/  468] time: 110.1182, train_loss: 0.10351031, train_accuracy: 0.9531, test_Accuracy: 0.9576\n",
      "Epoch: [ 0] [  224/  468] time: 110.6585, train_loss: 0.08925643, train_accuracy: 0.9688, test_Accuracy: 0.9571\n",
      "Epoch: [ 0] [  225/  468] time: 111.2062, train_loss: 0.07295242, train_accuracy: 0.9688, test_Accuracy: 0.9553\n",
      "Epoch: [ 0] [  226/  468] time: 111.7220, train_loss: 0.12013385, train_accuracy: 0.9531, test_Accuracy: 0.9535\n",
      "Epoch: [ 0] [  227/  468] time: 112.2109, train_loss: 0.12738264, train_accuracy: 0.9609, test_Accuracy: 0.9532\n",
      "Epoch: [ 0] [  228/  468] time: 112.8246, train_loss: 0.08318602, train_accuracy: 0.9844, test_Accuracy: 0.9504\n",
      "Epoch: [ 0] [  229/  468] time: 113.3951, train_loss: 0.16382928, train_accuracy: 0.9531, test_Accuracy: 0.9539\n",
      "Epoch: [ 0] [  230/  468] time: 113.9070, train_loss: 0.09406640, train_accuracy: 0.9688, test_Accuracy: 0.9569\n",
      "Epoch: [ 0] [  231/  468] time: 114.4345, train_loss: 0.09047431, train_accuracy: 0.9766, test_Accuracy: 0.9604\n",
      "Epoch: [ 0] [  232/  468] time: 114.9860, train_loss: 0.07429829, train_accuracy: 0.9844, test_Accuracy: 0.9611\n",
      "Epoch: [ 0] [  233/  468] time: 115.4632, train_loss: 0.05181652, train_accuracy: 0.9844, test_Accuracy: 0.9598\n",
      "Epoch: [ 0] [  234/  468] time: 116.0631, train_loss: 0.13344121, train_accuracy: 0.9609, test_Accuracy: 0.9575\n",
      "Epoch: [ 0] [  235/  468] time: 116.6109, train_loss: 0.07013915, train_accuracy: 0.9844, test_Accuracy: 0.9555\n",
      "Epoch: [ 0] [  236/  468] time: 117.1264, train_loss: 0.10349147, train_accuracy: 0.9609, test_Accuracy: 0.9560\n",
      "Epoch: [ 0] [  237/  468] time: 117.6505, train_loss: 0.12534714, train_accuracy: 0.9609, test_Accuracy: 0.9561\n",
      "Epoch: [ 0] [  238/  468] time: 118.0343, train_loss: 0.10523778, train_accuracy: 0.9609, test_Accuracy: 0.9570\n",
      "Epoch: [ 0] [  239/  468] time: 118.5111, train_loss: 0.09366937, train_accuracy: 0.9766, test_Accuracy: 0.9582\n",
      "Epoch: [ 0] [  240/  468] time: 118.9816, train_loss: 0.11613838, train_accuracy: 0.9688, test_Accuracy: 0.9581\n",
      "Epoch: [ 0] [  241/  468] time: 119.5640, train_loss: 0.08847392, train_accuracy: 0.9844, test_Accuracy: 0.9573\n",
      "Epoch: [ 0] [  242/  468] time: 120.0890, train_loss: 0.13431859, train_accuracy: 0.9688, test_Accuracy: 0.9580\n",
      "Epoch: [ 0] [  243/  468] time: 120.5956, train_loss: 0.06584116, train_accuracy: 0.9766, test_Accuracy: 0.9595\n",
      "Epoch: [ 0] [  244/  468] time: 121.0003, train_loss: 0.10372606, train_accuracy: 0.9531, test_Accuracy: 0.9586\n",
      "Epoch: [ 0] [  245/  468] time: 121.5673, train_loss: 0.08700268, train_accuracy: 0.9609, test_Accuracy: 0.9585\n",
      "Epoch: [ 0] [  246/  468] time: 122.0485, train_loss: 0.13492505, train_accuracy: 0.9609, test_Accuracy: 0.9568\n",
      "Epoch: [ 0] [  247/  468] time: 122.4435, train_loss: 0.13766763, train_accuracy: 0.9766, test_Accuracy: 0.9580\n",
      "Epoch: [ 0] [  248/  468] time: 122.9937, train_loss: 0.14369251, train_accuracy: 0.9609, test_Accuracy: 0.9613\n",
      "Epoch: [ 0] [  249/  468] time: 123.5109, train_loss: 0.06677584, train_accuracy: 0.9766, test_Accuracy: 0.9614\n",
      "Epoch: [ 0] [  250/  468] time: 124.0629, train_loss: 0.12480135, train_accuracy: 0.9609, test_Accuracy: 0.9622\n",
      "Epoch: [ 0] [  251/  468] time: 124.5154, train_loss: 0.12598708, train_accuracy: 0.9609, test_Accuracy: 0.9600\n",
      "Epoch: [ 0] [  252/  468] time: 125.0549, train_loss: 0.10854317, train_accuracy: 0.9609, test_Accuracy: 0.9562\n",
      "Epoch: [ 0] [  253/  468] time: 125.3899, train_loss: 0.09536682, train_accuracy: 0.9688, test_Accuracy: 0.9532\n",
      "Epoch: [ 0] [  254/  468] time: 125.8402, train_loss: 0.17433292, train_accuracy: 0.9453, test_Accuracy: 0.9562\n",
      "Epoch: [ 0] [  255/  468] time: 126.2736, train_loss: 0.19942771, train_accuracy: 0.9375, test_Accuracy: 0.9598\n",
      "Epoch: [ 0] [  256/  468] time: 126.7237, train_loss: 0.05842169, train_accuracy: 0.9922, test_Accuracy: 0.9575\n",
      "Epoch: [ 0] [  257/  468] time: 127.2017, train_loss: 0.05593536, train_accuracy: 0.9688, test_Accuracy: 0.9570\n",
      "Epoch: [ 0] [  258/  468] time: 127.6593, train_loss: 0.12906188, train_accuracy: 0.9609, test_Accuracy: 0.9531\n",
      "Epoch: [ 0] [  259/  468] time: 128.1367, train_loss: 0.14079911, train_accuracy: 0.9688, test_Accuracy: 0.9490\n",
      "Epoch: [ 0] [  260/  468] time: 128.6433, train_loss: 0.24534732, train_accuracy: 0.9531, test_Accuracy: 0.9480\n",
      "Epoch: [ 0] [  261/  468] time: 129.1281, train_loss: 0.07624574, train_accuracy: 0.9688, test_Accuracy: 0.9449\n",
      "Epoch: [ 0] [  262/  468] time: 129.6333, train_loss: 0.05679135, train_accuracy: 0.9922, test_Accuracy: 0.9444\n",
      "Epoch: [ 0] [  263/  468] time: 130.1479, train_loss: 0.07729102, train_accuracy: 0.9766, test_Accuracy: 0.9476\n",
      "Epoch: [ 0] [  264/  468] time: 130.6433, train_loss: 0.10086500, train_accuracy: 0.9609, test_Accuracy: 0.9526\n",
      "Epoch: [ 0] [  265/  468] time: 131.0579, train_loss: 0.20249274, train_accuracy: 0.9297, test_Accuracy: 0.9590\n",
      "Epoch: [ 0] [  266/  468] time: 131.4764, train_loss: 0.10278238, train_accuracy: 0.9766, test_Accuracy: 0.9588\n",
      "Epoch: [ 0] [  267/  468] time: 131.9762, train_loss: 0.06869310, train_accuracy: 0.9766, test_Accuracy: 0.9562\n",
      "Epoch: [ 0] [  268/  468] time: 132.4828, train_loss: 0.14326423, train_accuracy: 0.9688, test_Accuracy: 0.9523\n",
      "Epoch: [ 0] [  269/  468] time: 132.9268, train_loss: 0.07822350, train_accuracy: 0.9766, test_Accuracy: 0.9510\n",
      "Epoch: [ 0] [  270/  468] time: 133.3525, train_loss: 0.08391584, train_accuracy: 0.9844, test_Accuracy: 0.9509\n",
      "Epoch: [ 0] [  271/  468] time: 133.8243, train_loss: 0.20937578, train_accuracy: 0.9453, test_Accuracy: 0.9524\n",
      "Epoch: [ 0] [  272/  468] time: 134.3411, train_loss: 0.13187924, train_accuracy: 0.9453, test_Accuracy: 0.9570\n",
      "Epoch: [ 0] [  273/  468] time: 134.8386, train_loss: 0.16130568, train_accuracy: 0.9375, test_Accuracy: 0.9571\n",
      "Epoch: [ 0] [  274/  468] time: 135.2708, train_loss: 0.13380635, train_accuracy: 0.9609, test_Accuracy: 0.9578\n",
      "Epoch: [ 0] [  275/  468] time: 135.7686, train_loss: 0.13820516, train_accuracy: 0.9453, test_Accuracy: 0.9592\n",
      "Epoch: [ 0] [  276/  468] time: 136.2678, train_loss: 0.07824568, train_accuracy: 0.9844, test_Accuracy: 0.9576\n",
      "Epoch: [ 0] [  277/  468] time: 136.7862, train_loss: 0.09841959, train_accuracy: 0.9688, test_Accuracy: 0.9581\n",
      "Epoch: [ 0] [  278/  468] time: 137.2826, train_loss: 0.13875963, train_accuracy: 0.9766, test_Accuracy: 0.9563\n",
      "Epoch: [ 0] [  279/  468] time: 137.7915, train_loss: 0.13243146, train_accuracy: 0.9766, test_Accuracy: 0.9556\n",
      "Epoch: [ 0] [  280/  468] time: 138.3114, train_loss: 0.09185062, train_accuracy: 0.9688, test_Accuracy: 0.9523\n",
      "Epoch: [ 0] [  281/  468] time: 138.8097, train_loss: 0.18448126, train_accuracy: 0.9531, test_Accuracy: 0.9485\n",
      "Epoch: [ 0] [  282/  468] time: 139.3116, train_loss: 0.20752075, train_accuracy: 0.9453, test_Accuracy: 0.9497\n",
      "Epoch: [ 0] [  283/  468] time: 139.7715, train_loss: 0.10761496, train_accuracy: 0.9609, test_Accuracy: 0.9471\n",
      "Epoch: [ 0] [  284/  468] time: 140.2254, train_loss: 0.17214467, train_accuracy: 0.9688, test_Accuracy: 0.9508\n",
      "Epoch: [ 0] [  285/  468] time: 140.7549, train_loss: 0.08937956, train_accuracy: 0.9844, test_Accuracy: 0.9581\n",
      "Epoch: [ 0] [  286/  468] time: 141.2559, train_loss: 0.07203329, train_accuracy: 0.9766, test_Accuracy: 0.9604\n",
      "Epoch: [ 0] [  287/  468] time: 141.7888, train_loss: 0.10896029, train_accuracy: 0.9688, test_Accuracy: 0.9617\n",
      "Epoch: [ 0] [  288/  468] time: 142.2994, train_loss: 0.11979379, train_accuracy: 0.9531, test_Accuracy: 0.9614\n",
      "Epoch: [ 0] [  289/  468] time: 142.7777, train_loss: 0.07454057, train_accuracy: 0.9766, test_Accuracy: 0.9598\n",
      "Epoch: [ 0] [  290/  468] time: 143.2892, train_loss: 0.13548425, train_accuracy: 0.9688, test_Accuracy: 0.9572\n",
      "Epoch: [ 0] [  291/  468] time: 143.8324, train_loss: 0.21618071, train_accuracy: 0.9375, test_Accuracy: 0.9544\n",
      "Epoch: [ 0] [  292/  468] time: 144.3690, train_loss: 0.05089604, train_accuracy: 0.9922, test_Accuracy: 0.9526\n",
      "Epoch: [ 0] [  293/  468] time: 144.9884, train_loss: 0.11390664, train_accuracy: 0.9531, test_Accuracy: 0.9532\n",
      "Epoch: [ 0] [  294/  468] time: 145.5399, train_loss: 0.14481798, train_accuracy: 0.9531, test_Accuracy: 0.9571\n",
      "Epoch: [ 0] [  295/  468] time: 146.0426, train_loss: 0.10959156, train_accuracy: 0.9766, test_Accuracy: 0.9609\n",
      "Epoch: [ 0] [  296/  468] time: 146.5527, train_loss: 0.15527706, train_accuracy: 0.9609, test_Accuracy: 0.9640\n",
      "Epoch: [ 0] [  297/  468] time: 147.0569, train_loss: 0.10796995, train_accuracy: 0.9766, test_Accuracy: 0.9652\n",
      "Epoch: [ 0] [  298/  468] time: 147.5385, train_loss: 0.07667607, train_accuracy: 0.9766, test_Accuracy: 0.9653\n",
      "Epoch: [ 0] [  299/  468] time: 148.1115, train_loss: 0.16321103, train_accuracy: 0.9609, test_Accuracy: 0.9639\n",
      "Epoch: [ 0] [  300/  468] time: 148.5931, train_loss: 0.07968758, train_accuracy: 0.9688, test_Accuracy: 0.9628\n",
      "Epoch: [ 0] [  301/  468] time: 149.1467, train_loss: 0.04815003, train_accuracy: 0.9844, test_Accuracy: 0.9608\n",
      "Epoch: [ 0] [  302/  468] time: 149.6794, train_loss: 0.08423892, train_accuracy: 0.9766, test_Accuracy: 0.9608\n",
      "Epoch: [ 0] [  303/  468] time: 150.1878, train_loss: 0.07661533, train_accuracy: 0.9922, test_Accuracy: 0.9622\n",
      "Epoch: [ 0] [  304/  468] time: 150.7072, train_loss: 0.11491346, train_accuracy: 0.9688, test_Accuracy: 0.9631\n",
      "Epoch: [ 0] [  305/  468] time: 151.1960, train_loss: 0.18115199, train_accuracy: 0.9531, test_Accuracy: 0.9627\n",
      "Epoch: [ 0] [  306/  468] time: 151.7647, train_loss: 0.11604664, train_accuracy: 0.9531, test_Accuracy: 0.9635\n",
      "Epoch: [ 0] [  307/  468] time: 152.2148, train_loss: 0.21169463, train_accuracy: 0.9531, test_Accuracy: 0.9650\n",
      "Epoch: [ 0] [  308/  468] time: 152.6573, train_loss: 0.06390033, train_accuracy: 0.9922, test_Accuracy: 0.9652\n",
      "Epoch: [ 0] [  309/  468] time: 153.1450, train_loss: 0.09901649, train_accuracy: 0.9609, test_Accuracy: 0.9656\n",
      "Epoch: [ 0] [  310/  468] time: 153.5913, train_loss: 0.09563054, train_accuracy: 0.9766, test_Accuracy: 0.9650\n",
      "Epoch: [ 0] [  311/  468] time: 154.0725, train_loss: 0.13976775, train_accuracy: 0.9609, test_Accuracy: 0.9653\n",
      "Epoch: [ 0] [  312/  468] time: 154.5628, train_loss: 0.13345507, train_accuracy: 0.9609, test_Accuracy: 0.9666\n",
      "Epoch: [ 0] [  313/  468] time: 154.9902, train_loss: 0.10509095, train_accuracy: 0.9609, test_Accuracy: 0.9670\n",
      "Epoch: [ 0] [  314/  468] time: 155.4512, train_loss: 0.09616749, train_accuracy: 0.9609, test_Accuracy: 0.9658\n",
      "Epoch: [ 0] [  315/  468] time: 155.9285, train_loss: 0.12113697, train_accuracy: 0.9531, test_Accuracy: 0.9653\n",
      "Epoch: [ 0] [  316/  468] time: 156.4081, train_loss: 0.08903537, train_accuracy: 0.9609, test_Accuracy: 0.9662\n",
      "Epoch: [ 0] [  317/  468] time: 156.8824, train_loss: 0.07556509, train_accuracy: 0.9766, test_Accuracy: 0.9661\n",
      "Epoch: [ 0] [  318/  468] time: 157.3230, train_loss: 0.06975540, train_accuracy: 0.9922, test_Accuracy: 0.9668\n",
      "Epoch: [ 0] [  319/  468] time: 157.7804, train_loss: 0.08938858, train_accuracy: 0.9922, test_Accuracy: 0.9669\n",
      "Epoch: [ 0] [  320/  468] time: 158.2943, train_loss: 0.08076423, train_accuracy: 0.9844, test_Accuracy: 0.9659\n",
      "Epoch: [ 0] [  321/  468] time: 158.7761, train_loss: 0.07457527, train_accuracy: 0.9844, test_Accuracy: 0.9663\n",
      "Epoch: [ 0] [  322/  468] time: 159.1764, train_loss: 0.14272714, train_accuracy: 0.9375, test_Accuracy: 0.9664\n",
      "Epoch: [ 0] [  323/  468] time: 159.6665, train_loss: 0.06620903, train_accuracy: 0.9766, test_Accuracy: 0.9664\n",
      "Epoch: [ 0] [  324/  468] time: 160.1376, train_loss: 0.07228123, train_accuracy: 0.9922, test_Accuracy: 0.9649\n",
      "Epoch: [ 0] [  325/  468] time: 160.5882, train_loss: 0.10870206, train_accuracy: 0.9609, test_Accuracy: 0.9647\n",
      "Epoch: [ 0] [  326/  468] time: 161.0902, train_loss: 0.08943989, train_accuracy: 0.9844, test_Accuracy: 0.9652\n",
      "Epoch: [ 0] [  327/  468] time: 161.6518, train_loss: 0.04063260, train_accuracy: 0.9844, test_Accuracy: 0.9653\n",
      "Epoch: [ 0] [  328/  468] time: 162.1236, train_loss: 0.15450275, train_accuracy: 0.9688, test_Accuracy: 0.9667\n",
      "Epoch: [ 0] [  329/  468] time: 162.5817, train_loss: 0.10198423, train_accuracy: 0.9688, test_Accuracy: 0.9664\n",
      "Epoch: [ 0] [  330/  468] time: 163.0238, train_loss: 0.09875754, train_accuracy: 0.9688, test_Accuracy: 0.9648\n",
      "Epoch: [ 0] [  331/  468] time: 163.5092, train_loss: 0.05249163, train_accuracy: 0.9844, test_Accuracy: 0.9636\n",
      "Epoch: [ 0] [  332/  468] time: 163.9469, train_loss: 0.09266815, train_accuracy: 0.9766, test_Accuracy: 0.9623\n",
      "Epoch: [ 0] [  333/  468] time: 164.3906, train_loss: 0.07783536, train_accuracy: 0.9766, test_Accuracy: 0.9615\n",
      "Epoch: [ 0] [  334/  468] time: 164.8348, train_loss: 0.10333064, train_accuracy: 0.9609, test_Accuracy: 0.9621\n",
      "Epoch: [ 0] [  335/  468] time: 165.2365, train_loss: 0.15599325, train_accuracy: 0.9609, test_Accuracy: 0.9597\n",
      "Epoch: [ 0] [  336/  468] time: 165.7807, train_loss: 0.16567373, train_accuracy: 0.9453, test_Accuracy: 0.9579\n",
      "Epoch: [ 0] [  337/  468] time: 166.1692, train_loss: 0.12523817, train_accuracy: 0.9609, test_Accuracy: 0.9581\n",
      "Epoch: [ 0] [  338/  468] time: 166.5953, train_loss: 0.07381719, train_accuracy: 0.9766, test_Accuracy: 0.9605\n",
      "Epoch: [ 0] [  339/  468] time: 167.0794, train_loss: 0.17842041, train_accuracy: 0.9531, test_Accuracy: 0.9601\n",
      "Epoch: [ 0] [  340/  468] time: 167.5818, train_loss: 0.07820989, train_accuracy: 0.9922, test_Accuracy: 0.9581\n",
      "Epoch: [ 0] [  341/  468] time: 168.0680, train_loss: 0.09068450, train_accuracy: 0.9609, test_Accuracy: 0.9585\n",
      "Epoch: [ 0] [  342/  468] time: 168.5583, train_loss: 0.05371711, train_accuracy: 0.9766, test_Accuracy: 0.9597\n",
      "Epoch: [ 0] [  343/  468] time: 169.0492, train_loss: 0.20581302, train_accuracy: 0.9609, test_Accuracy: 0.9599\n",
      "Epoch: [ 0] [  344/  468] time: 169.5398, train_loss: 0.07219683, train_accuracy: 0.9766, test_Accuracy: 0.9602\n",
      "Epoch: [ 0] [  345/  468] time: 170.0246, train_loss: 0.14545532, train_accuracy: 0.9609, test_Accuracy: 0.9615\n",
      "Epoch: [ 0] [  346/  468] time: 170.5095, train_loss: 0.12121032, train_accuracy: 0.9766, test_Accuracy: 0.9609\n",
      "Epoch: [ 0] [  347/  468] time: 170.9673, train_loss: 0.05750932, train_accuracy: 0.9766, test_Accuracy: 0.9603\n",
      "Epoch: [ 0] [  348/  468] time: 171.4761, train_loss: 0.08804124, train_accuracy: 0.9766, test_Accuracy: 0.9606\n",
      "Epoch: [ 0] [  349/  468] time: 171.9350, train_loss: 0.14209306, train_accuracy: 0.9531, test_Accuracy: 0.9593\n",
      "Epoch: [ 0] [  350/  468] time: 172.4655, train_loss: 0.10426049, train_accuracy: 0.9609, test_Accuracy: 0.9608\n",
      "Epoch: [ 0] [  351/  468] time: 172.9739, train_loss: 0.04817408, train_accuracy: 0.9844, test_Accuracy: 0.9614\n",
      "Epoch: [ 0] [  352/  468] time: 173.4479, train_loss: 0.08029583, train_accuracy: 0.9922, test_Accuracy: 0.9626\n",
      "Epoch: [ 0] [  353/  468] time: 173.9688, train_loss: 0.06987655, train_accuracy: 0.9766, test_Accuracy: 0.9618\n",
      "Epoch: [ 0] [  354/  468] time: 174.5623, train_loss: 0.06811203, train_accuracy: 0.9844, test_Accuracy: 0.9622\n",
      "Epoch: [ 0] [  355/  468] time: 174.9807, train_loss: 0.06579322, train_accuracy: 0.9844, test_Accuracy: 0.9613\n",
      "Epoch: [ 0] [  356/  468] time: 175.4712, train_loss: 0.08748046, train_accuracy: 0.9766, test_Accuracy: 0.9634\n",
      "Epoch: [ 0] [  357/  468] time: 175.9578, train_loss: 0.06409794, train_accuracy: 0.9922, test_Accuracy: 0.9643\n",
      "Epoch: [ 0] [  358/  468] time: 176.4216, train_loss: 0.04918201, train_accuracy: 0.9766, test_Accuracy: 0.9638\n",
      "Epoch: [ 0] [  359/  468] time: 176.9388, train_loss: 0.11766171, train_accuracy: 0.9531, test_Accuracy: 0.9620\n",
      "Epoch: [ 0] [  360/  468] time: 177.3734, train_loss: 0.11518982, train_accuracy: 0.9766, test_Accuracy: 0.9602\n",
      "Epoch: [ 0] [  361/  468] time: 177.7688, train_loss: 0.08535288, train_accuracy: 0.9766, test_Accuracy: 0.9611\n",
      "Epoch: [ 0] [  362/  468] time: 178.2573, train_loss: 0.10160240, train_accuracy: 0.9766, test_Accuracy: 0.9619\n",
      "Epoch: [ 0] [  363/  468] time: 178.6925, train_loss: 0.06396087, train_accuracy: 0.9922, test_Accuracy: 0.9626\n",
      "Epoch: [ 0] [  364/  468] time: 179.1316, train_loss: 0.11248076, train_accuracy: 0.9766, test_Accuracy: 0.9593\n",
      "Epoch: [ 0] [  365/  468] time: 179.5425, train_loss: 0.10999109, train_accuracy: 0.9688, test_Accuracy: 0.9581\n",
      "Epoch: [ 0] [  366/  468] time: 180.1032, train_loss: 0.14236179, train_accuracy: 0.9531, test_Accuracy: 0.9599\n",
      "Epoch: [ 0] [  367/  468] time: 180.6343, train_loss: 0.08522084, train_accuracy: 0.9609, test_Accuracy: 0.9608\n",
      "Epoch: [ 0] [  368/  468] time: 181.0266, train_loss: 0.12522858, train_accuracy: 0.9766, test_Accuracy: 0.9613\n",
      "Epoch: [ 0] [  369/  468] time: 181.4819, train_loss: 0.10768862, train_accuracy: 0.9609, test_Accuracy: 0.9630\n",
      "Epoch: [ 0] [  370/  468] time: 181.9797, train_loss: 0.09083509, train_accuracy: 0.9766, test_Accuracy: 0.9649\n",
      "Epoch: [ 0] [  371/  468] time: 182.5354, train_loss: 0.06952055, train_accuracy: 0.9766, test_Accuracy: 0.9670\n",
      "Epoch: [ 0] [  372/  468] time: 183.0117, train_loss: 0.09763034, train_accuracy: 0.9844, test_Accuracy: 0.9639\n",
      "Epoch: [ 0] [  373/  468] time: 183.5113, train_loss: 0.12039956, train_accuracy: 0.9688, test_Accuracy: 0.9636\n",
      "Epoch: [ 0] [  374/  468] time: 184.0039, train_loss: 0.06519896, train_accuracy: 0.9766, test_Accuracy: 0.9638\n",
      "Epoch: [ 0] [  375/  468] time: 184.3666, train_loss: 0.05298217, train_accuracy: 0.9766, test_Accuracy: 0.9650\n",
      "Epoch: [ 0] [  376/  468] time: 184.7686, train_loss: 0.10668980, train_accuracy: 0.9688, test_Accuracy: 0.9649\n",
      "Epoch: [ 0] [  377/  468] time: 185.1921, train_loss: 0.05182985, train_accuracy: 0.9844, test_Accuracy: 0.9641\n",
      "Epoch: [ 0] [  378/  468] time: 185.7979, train_loss: 0.04911999, train_accuracy: 0.9922, test_Accuracy: 0.9624\n",
      "Epoch: [ 0] [  379/  468] time: 186.2744, train_loss: 0.08599630, train_accuracy: 0.9609, test_Accuracy: 0.9636\n",
      "Epoch: [ 0] [  380/  468] time: 186.8454, train_loss: 0.05964197, train_accuracy: 0.9766, test_Accuracy: 0.9627\n",
      "Epoch: [ 0] [  381/  468] time: 187.3178, train_loss: 0.11337121, train_accuracy: 0.9766, test_Accuracy: 0.9630\n",
      "Epoch: [ 0] [  382/  468] time: 187.7543, train_loss: 0.05388638, train_accuracy: 0.9844, test_Accuracy: 0.9638\n",
      "Epoch: [ 0] [  383/  468] time: 188.2363, train_loss: 0.03951590, train_accuracy: 1.0000, test_Accuracy: 0.9659\n",
      "Epoch: [ 0] [  384/  468] time: 188.7884, train_loss: 0.12264882, train_accuracy: 0.9688, test_Accuracy: 0.9661\n",
      "Epoch: [ 0] [  385/  468] time: 189.2957, train_loss: 0.07112828, train_accuracy: 0.9922, test_Accuracy: 0.9662\n",
      "Epoch: [ 0] [  386/  468] time: 189.7432, train_loss: 0.09321328, train_accuracy: 0.9766, test_Accuracy: 0.9663\n",
      "Epoch: [ 0] [  387/  468] time: 190.3380, train_loss: 0.06531389, train_accuracy: 0.9688, test_Accuracy: 0.9673\n",
      "Epoch: [ 0] [  388/  468] time: 190.8303, train_loss: 0.09537575, train_accuracy: 0.9766, test_Accuracy: 0.9673\n",
      "Epoch: [ 0] [  389/  468] time: 191.2610, train_loss: 0.05305265, train_accuracy: 0.9922, test_Accuracy: 0.9688\n",
      "Epoch: [ 0] [  390/  468] time: 191.7457, train_loss: 0.05313455, train_accuracy: 0.9844, test_Accuracy: 0.9687\n",
      "Epoch: [ 0] [  391/  468] time: 192.2302, train_loss: 0.07365111, train_accuracy: 0.9922, test_Accuracy: 0.9681\n",
      "Epoch: [ 0] [  392/  468] time: 192.8584, train_loss: 0.10744260, train_accuracy: 0.9609, test_Accuracy: 0.9667\n",
      "Epoch: [ 0] [  393/  468] time: 193.3374, train_loss: 0.09262305, train_accuracy: 0.9531, test_Accuracy: 0.9653\n",
      "Epoch: [ 0] [  394/  468] time: 193.8316, train_loss: 0.09068259, train_accuracy: 0.9688, test_Accuracy: 0.9632\n",
      "Epoch: [ 0] [  395/  468] time: 194.3173, train_loss: 0.07569441, train_accuracy: 0.9844, test_Accuracy: 0.9622\n",
      "Epoch: [ 0] [  396/  468] time: 194.6684, train_loss: 0.07717681, train_accuracy: 0.9688, test_Accuracy: 0.9604\n",
      "Epoch: [ 0] [  397/  468] time: 195.0612, train_loss: 0.12641411, train_accuracy: 0.9609, test_Accuracy: 0.9631\n",
      "Epoch: [ 0] [  398/  468] time: 195.4854, train_loss: 0.13288854, train_accuracy: 0.9766, test_Accuracy: 0.9669\n",
      "Epoch: [ 0] [  399/  468] time: 196.0346, train_loss: 0.06365856, train_accuracy: 0.9844, test_Accuracy: 0.9673\n",
      "Epoch: [ 0] [  400/  468] time: 196.5560, train_loss: 0.07478695, train_accuracy: 0.9688, test_Accuracy: 0.9662\n",
      "Epoch: [ 0] [  401/  468] time: 197.1000, train_loss: 0.03879867, train_accuracy: 0.9922, test_Accuracy: 0.9643\n",
      "Epoch: [ 0] [  402/  468] time: 197.6326, train_loss: 0.04770987, train_accuracy: 0.9922, test_Accuracy: 0.9630\n",
      "Epoch: [ 0] [  403/  468] time: 198.2152, train_loss: 0.09065982, train_accuracy: 0.9766, test_Accuracy: 0.9603\n",
      "Epoch: [ 0] [  404/  468] time: 198.6042, train_loss: 0.08995209, train_accuracy: 0.9766, test_Accuracy: 0.9599\n",
      "Epoch: [ 0] [  405/  468] time: 199.1589, train_loss: 0.09596381, train_accuracy: 0.9688, test_Accuracy: 0.9595\n",
      "Epoch: [ 0] [  406/  468] time: 199.6885, train_loss: 0.09692593, train_accuracy: 0.9766, test_Accuracy: 0.9600\n",
      "Epoch: [ 0] [  407/  468] time: 200.1627, train_loss: 0.06726599, train_accuracy: 0.9844, test_Accuracy: 0.9613\n",
      "Epoch: [ 0] [  408/  468] time: 200.7094, train_loss: 0.07326011, train_accuracy: 0.9922, test_Accuracy: 0.9612\n",
      "Epoch: [ 0] [  409/  468] time: 201.2182, train_loss: 0.05938032, train_accuracy: 0.9844, test_Accuracy: 0.9621\n",
      "Epoch: [ 0] [  410/  468] time: 201.7878, train_loss: 0.05227538, train_accuracy: 0.9922, test_Accuracy: 0.9613\n",
      "Epoch: [ 0] [  411/  468] time: 202.1585, train_loss: 0.10120234, train_accuracy: 0.9609, test_Accuracy: 0.9597\n",
      "Epoch: [ 0] [  412/  468] time: 202.5654, train_loss: 0.05173219, train_accuracy: 0.9922, test_Accuracy: 0.9571\n",
      "Epoch: [ 0] [  413/  468] time: 203.0785, train_loss: 0.07106582, train_accuracy: 0.9688, test_Accuracy: 0.9556\n",
      "Epoch: [ 0] [  414/  468] time: 203.6206, train_loss: 0.14081536, train_accuracy: 0.9688, test_Accuracy: 0.9590\n",
      "Epoch: [ 0] [  415/  468] time: 203.9842, train_loss: 0.09438284, train_accuracy: 0.9688, test_Accuracy: 0.9600\n",
      "Epoch: [ 0] [  416/  468] time: 204.5126, train_loss: 0.05975757, train_accuracy: 0.9844, test_Accuracy: 0.9624\n",
      "Epoch: [ 0] [  417/  468] time: 205.0066, train_loss: 0.10020308, train_accuracy: 0.9844, test_Accuracy: 0.9604\n",
      "Epoch: [ 0] [  418/  468] time: 205.5831, train_loss: 0.19013505, train_accuracy: 0.9766, test_Accuracy: 0.9570\n",
      "Epoch: [ 0] [  419/  468] time: 206.1154, train_loss: 0.06297561, train_accuracy: 0.9922, test_Accuracy: 0.9575\n",
      "Epoch: [ 0] [  420/  468] time: 206.6342, train_loss: 0.03766482, train_accuracy: 1.0000, test_Accuracy: 0.9593\n",
      "Epoch: [ 0] [  421/  468] time: 207.1317, train_loss: 0.07488678, train_accuracy: 0.9609, test_Accuracy: 0.9606\n",
      "Epoch: [ 0] [  422/  468] time: 207.7751, train_loss: 0.10966524, train_accuracy: 0.9531, test_Accuracy: 0.9614\n",
      "Epoch: [ 0] [  423/  468] time: 208.1433, train_loss: 0.15475777, train_accuracy: 0.9453, test_Accuracy: 0.9627\n",
      "Epoch: [ 0] [  424/  468] time: 208.6533, train_loss: 0.08174133, train_accuracy: 0.9688, test_Accuracy: 0.9613\n",
      "Epoch: [ 0] [  425/  468] time: 209.0904, train_loss: 0.11885649, train_accuracy: 0.9609, test_Accuracy: 0.9619\n",
      "Epoch: [ 0] [  426/  468] time: 209.6372, train_loss: 0.08362469, train_accuracy: 0.9766, test_Accuracy: 0.9642\n",
      "Epoch: [ 0] [  427/  468] time: 210.0845, train_loss: 0.08254836, train_accuracy: 0.9844, test_Accuracy: 0.9641\n",
      "Epoch: [ 0] [  428/  468] time: 210.5504, train_loss: 0.05750975, train_accuracy: 0.9844, test_Accuracy: 0.9647\n",
      "Epoch: [ 0] [  429/  468] time: 211.0864, train_loss: 0.02324268, train_accuracy: 1.0000, test_Accuracy: 0.9620\n",
      "Epoch: [ 0] [  430/  468] time: 211.4298, train_loss: 0.21974379, train_accuracy: 0.9609, test_Accuracy: 0.9616\n",
      "Epoch: [ 0] [  431/  468] time: 211.8039, train_loss: 0.06039507, train_accuracy: 0.9844, test_Accuracy: 0.9623\n",
      "Epoch: [ 0] [  432/  468] time: 212.1451, train_loss: 0.13213399, train_accuracy: 0.9766, test_Accuracy: 0.9650\n",
      "Epoch: [ 0] [  433/  468] time: 212.6706, train_loss: 0.14800003, train_accuracy: 0.9531, test_Accuracy: 0.9648\n",
      "Epoch: [ 0] [  434/  468] time: 213.1266, train_loss: 0.12224413, train_accuracy: 0.9609, test_Accuracy: 0.9659\n",
      "Epoch: [ 0] [  435/  468] time: 213.5686, train_loss: 0.11420291, train_accuracy: 0.9609, test_Accuracy: 0.9667\n",
      "Epoch: [ 0] [  436/  468] time: 214.0901, train_loss: 0.02818738, train_accuracy: 1.0000, test_Accuracy: 0.9669\n",
      "Epoch: [ 0] [  437/  468] time: 214.6492, train_loss: 0.14086199, train_accuracy: 0.9453, test_Accuracy: 0.9663\n",
      "Epoch: [ 0] [  438/  468] time: 215.2052, train_loss: 0.03030792, train_accuracy: 1.0000, test_Accuracy: 0.9655\n",
      "Epoch: [ 0] [  439/  468] time: 215.6043, train_loss: 0.07708180, train_accuracy: 0.9766, test_Accuracy: 0.9651\n",
      "Epoch: [ 0] [  440/  468] time: 215.9836, train_loss: 0.08538762, train_accuracy: 0.9766, test_Accuracy: 0.9642\n",
      "Epoch: [ 0] [  441/  468] time: 216.4139, train_loss: 0.06684645, train_accuracy: 0.9688, test_Accuracy: 0.9649\n",
      "Epoch: [ 0] [  442/  468] time: 216.9514, train_loss: 0.07187739, train_accuracy: 0.9766, test_Accuracy: 0.9643\n",
      "Epoch: [ 0] [  443/  468] time: 217.5087, train_loss: 0.08631504, train_accuracy: 0.9922, test_Accuracy: 0.9644\n",
      "Epoch: [ 0] [  444/  468] time: 218.0735, train_loss: 0.04345548, train_accuracy: 0.9844, test_Accuracy: 0.9644\n",
      "Epoch: [ 0] [  445/  468] time: 218.5046, train_loss: 0.05508425, train_accuracy: 0.9844, test_Accuracy: 0.9652\n",
      "Epoch: [ 0] [  446/  468] time: 218.9680, train_loss: 0.03669357, train_accuracy: 0.9922, test_Accuracy: 0.9650\n",
      "Epoch: [ 0] [  447/  468] time: 219.5622, train_loss: 0.07323974, train_accuracy: 0.9766, test_Accuracy: 0.9663\n",
      "Epoch: [ 0] [  448/  468] time: 219.9201, train_loss: 0.12712598, train_accuracy: 0.9766, test_Accuracy: 0.9681\n",
      "Epoch: [ 0] [  449/  468] time: 220.2791, train_loss: 0.09639876, train_accuracy: 0.9688, test_Accuracy: 0.9686\n",
      "Epoch: [ 0] [  450/  468] time: 220.7274, train_loss: 0.06392962, train_accuracy: 0.9844, test_Accuracy: 0.9668\n",
      "Epoch: [ 0] [  451/  468] time: 221.2406, train_loss: 0.08238812, train_accuracy: 0.9844, test_Accuracy: 0.9631\n",
      "Epoch: [ 0] [  452/  468] time: 221.7587, train_loss: 0.10314344, train_accuracy: 0.9609, test_Accuracy: 0.9612\n",
      "Epoch: [ 0] [  453/  468] time: 222.2610, train_loss: 0.08817288, train_accuracy: 0.9766, test_Accuracy: 0.9603\n",
      "Epoch: [ 0] [  454/  468] time: 222.7896, train_loss: 0.10606150, train_accuracy: 0.9688, test_Accuracy: 0.9627\n",
      "Epoch: [ 0] [  455/  468] time: 223.2728, train_loss: 0.10213736, train_accuracy: 0.9688, test_Accuracy: 0.9651\n",
      "Epoch: [ 0] [  456/  468] time: 223.6847, train_loss: 0.10425242, train_accuracy: 0.9844, test_Accuracy: 0.9660\n",
      "Epoch: [ 0] [  457/  468] time: 224.1717, train_loss: 0.02254441, train_accuracy: 1.0000, test_Accuracy: 0.9644\n",
      "Epoch: [ 0] [  458/  468] time: 224.5805, train_loss: 0.06146629, train_accuracy: 0.9844, test_Accuracy: 0.9649\n",
      "Epoch: [ 0] [  459/  468] time: 225.1000, train_loss: 0.12219218, train_accuracy: 0.9531, test_Accuracy: 0.9639\n",
      "Epoch: [ 0] [  460/  468] time: 225.6661, train_loss: 0.15192850, train_accuracy: 0.9609, test_Accuracy: 0.9646\n",
      "Epoch: [ 0] [  461/  468] time: 226.1576, train_loss: 0.10514589, train_accuracy: 0.9688, test_Accuracy: 0.9612\n",
      "Epoch: [ 0] [  462/  468] time: 226.6413, train_loss: 0.07298227, train_accuracy: 0.9766, test_Accuracy: 0.9560\n",
      "Epoch: [ 0] [  463/  468] time: 227.0962, train_loss: 0.11569086, train_accuracy: 0.9766, test_Accuracy: 0.9560\n",
      "Epoch: [ 0] [  464/  468] time: 227.6442, train_loss: 0.11265893, train_accuracy: 0.9609, test_Accuracy: 0.9603\n",
      "Epoch: [ 0] [  465/  468] time: 228.1038, train_loss: 0.09734915, train_accuracy: 0.9531, test_Accuracy: 0.9633\n",
      "Epoch: [ 0] [  466/  468] time: 228.5395, train_loss: 0.09016009, train_accuracy: 0.9766, test_Accuracy: 0.9662\n",
      "Epoch: [ 0] [  467/  468] time: 229.0625, train_loss: 0.12086271, train_accuracy: 0.9688, test_Accuracy: 0.9691\n"
     ]
    }
   ],
   "source": [
    "if train_flag :\n",
    "\n",
    "    checkpoint = tf.train.Checkpoint(dnn=network)\n",
    "\n",
    "    summary_writer = tf.summary.create_file_writer(logdir=logs_dir)\n",
    "    start_time = time()\n",
    "\n",
    "    could_load, checkpoint_counter = load(network, checkpoint_dir)    \n",
    "\n",
    "    if could_load:\n",
    "        start_epoch = (int)(checkpoint_counter / training_iterations)        \n",
    "        counter = checkpoint_counter        \n",
    "        print(\" [*] Load SUCCESS\")\n",
    "    else:\n",
    "        start_epoch = 0\n",
    "        start_iteration = 0\n",
    "        counter = 0\n",
    "        print(\" [!] Load failed...\")\n",
    "    \n",
    "    with summary_writer.as_default():  # for tensorboard\n",
    "        for epoch in range(start_epoch, training_epochs):\n",
    "            for idx, (train_input, train_label) in enumerate(train_dataset):            \n",
    "                grads = grad(network, train_input, train_label)\n",
    "                optimizer.apply_gradients(grads_and_vars=zip(grads, network.variables))\n",
    "\n",
    "                train_loss = loss_fn(network, train_input, train_label)\n",
    "                train_accuracy = accuracy_fn(network, train_input, train_label)\n",
    "                \n",
    "                for test_input, test_label in test_dataset:                \n",
    "                    test_accuracy = accuracy_fn(network, test_input, test_label)\n",
    "\n",
    "                tf.summary.scalar(name='train_loss', data=train_loss, step=counter)\n",
    "                tf.summary.scalar(name='train_accuracy', data=train_accuracy, step=counter)\n",
    "                tf.summary.scalar(name='test_accuracy', data=test_accuracy, step=counter)\n",
    "\n",
    "                print(\n",
    "                    \"Epoch: [%2d] [%5d/%5d] time: %4.4f, train_loss: %.8f, train_accuracy: %.4f, test_Accuracy: %.4f\" \\\n",
    "                    % (epoch, idx, training_iterations, time() - start_time, train_loss, train_accuracy,\n",
    "                       test_accuracy))\n",
    "                counter += 1                \n",
    "        checkpoint.save(file_prefix=checkpoint_prefix + '-{}'.format(counter))\n",
    "        \n",
    "else :\n",
    "    _, _ = load(network, checkpoint_dir)\n",
    "    for test_input, test_label in test_dataset:    \n",
    "        test_accuracy = accuracy_fn(network, test_input, test_label)\n",
    "\n",
    "    print(\"test_Accuracy: %.4f\" % (test_accuracy))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}