Lab10/Lab10_4_LEEJOOYOUNG.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Lab10-4_LEEJOOYOUNG.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XHrvpG818F5x",
        "outputId": "5d88393f-2c75-4510-c9a2-d02a08784f7d"
      },
      "source": [
        "import tensorflow as tf\n",
        "import numpy as np\n",
        "from tensorflow.keras.utils import to_categorical\n",
        "from tensorflow.keras.datasets import mnist\n",
        "from time import time\n",
        "import os\n",
        "print(tf.__version__)"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2.6.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-xhfbchO9MOu"
      },
      "source": [
        "def load(model, checkpoint_dir):\n",
        "    print(\" [*] Reading checkpoints...\")\n",
        "\n",
        "    ckpt = tf.train.get_checkpoint_state(checkpoint_dir)\n",
        "    if ckpt :\n",
        "        ckpt_name = os.path.basename(ckpt.model_checkpoint_path)\n",
        "        checkpoint = tf.train.Checkpoint(dnn=model)\n",
        "        checkpoint.restore(save_path=os.path.join(checkpoint_dir, ckpt_name))\n",
        "        counter = int(ckpt_name.split('-')[1])\n",
        "        print(\" [*] Success to read {}\".format(ckpt_name))\n",
        "        return True, counter\n",
        "    else:\n",
        "        print(\" [*] Failed to find a checkpoint\")\n",
        "        return False, 0\n",
        "\n",
        "def check_folder(dir):\n",
        "    if not os.path.exists(dir):\n",
        "        os.makedirs(dir)\n",
        "    return dir"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cBgfK4sx9SkN"
      },
      "source": [
        "def load_mnist() :\n",
        "    (train_data, train_labels), (test_data, test_labels) = mnist.load_data()\n",
        "    train_data = np.expand_dims(train_data, axis=-1) # [N, 28, 28] -> [N, 28, 28, 1]\n",
        "    test_data = np.expand_dims(test_data, axis=-1) # [N, 28, 28] -> [N, 28, 28, 1]\n",
        "\n",
        "    train_data, test_data = normalize(train_data, test_data)\n",
        "\n",
        "    train_labels = to_categorical(train_labels, 10) # [N,] -> [N, 10]\n",
        "    test_labels = to_categorical(test_labels, 10) # [N,] -> [N, 10]\n",
        "\n",
        "    return train_data, train_labels, test_data, test_labels\n",
        "\n",
        "def normalize(train_data, test_data):\n",
        "    train_data = train_data.astype(np.float32) / 255.0\n",
        "    test_data = test_data.astype(np.float32) / 255.0\n",
        "\n",
        "    return train_data, test_data"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "j7R1ivxF9UDY"
      },
      "source": [
        "def loss_fn(model, images, labels):\n",
        "    logits = model(images, training=True)\n",
        "    loss = tf.reduce_mean(tf.keras.losses.categorical_crossentropy(y_pred=logits, y_true=labels, \n",
        "                                                                   from_logits=True))\n",
        "    return loss\n",
        "\n",
        "def accuracy_fn(model, images, labels):\n",
        "    logits = model(images, training=False)\n",
        "    prediction = tf.equal(tf.argmax(logits, -1), tf.argmax(labels, -1))\n",
        "    accuracy = tf.reduce_mean(tf.cast(prediction, tf.float32))\n",
        "    return accuracy\n",
        "\n",
        "def grad(model, images, labels):\n",
        "    with tf.GradientTape() as tape:\n",
        "        loss = loss_fn(model, images, labels)\n",
        "    return tape.gradient(loss, model.trainable_variables)"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "T2Ors8gT9bJW"
      },
      "source": [
        "def flatten() :\n",
        "    return tf.keras.layers.Flatten()\n",
        "\n",
        "def dense(label_dim, weight_init) :\n",
        "    return tf.keras.layers.Dense(units=label_dim, use_bias=True, kernel_initializer=weight_init)\n",
        "\n",
        "def relu() :\n",
        "    return tf.keras.layers.Activation(tf.keras.activations.relu)\n",
        "\n",
        "def batch_norm() :\n",
        "    return tf.keras.layers.BatchNormalization()"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kgYzfDrw9e5r"
      },
      "source": [
        "class create_model_class(tf.keras.Model):\n",
        "    def __init__(self, label_dim):\n",
        "        super(create_model_class, self).__init__()\n",
        "        weight_init = tf.keras.initializers.glorot_uniform()\n",
        "\n",
        "        self.model = tf.keras.Sequential()\n",
        "        self.model.add(flatten())\n",
        "\n",
        "        for i in range(4):\n",
        "            self.model.add(dense(512, weight_init))\n",
        "            self.model.add(batch_norm())\n",
        "            self.model.add(relu())\n",
        "\n",
        "        self.model.add(dense(label_dim, weight_init))\n",
        "\n",
        "    def call(self, x, training=None, mask=None):\n",
        "\n",
        "        x = self.model(x)\n",
        "\n",
        "        return x"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "f-Sh9bdH9kRa"
      },
      "source": [
        "def create_model_function(label_dim) :\n",
        "    weight_init = tf.keras.initializers.glorot_uniform()\n",
        "\n",
        "    model = tf.keras.Sequential()\n",
        "    model.add(flatten())\n",
        "\n",
        "    for i in range(4) :\n",
        "        model.add(dense(512, weight_init))\n",
        "        model.add(batch_norm())\n",
        "        model.add(relu())\n",
        "\n",
        "    model.add(dense(label_dim, weight_init))\n",
        "\n",
        "    return model"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FJBqDoN99mYC",
        "outputId": "378b9567-4197-4b6f-ddc9-60f5155c651a"
      },
      "source": [
        "\"\"\" dataset \"\"\"\n",
        "train_x, train_y, test_x, test_y = load_mnist()\n",
        "\n",
        "\"\"\" parameters \"\"\"\n",
        "learning_rate = 0.001\n",
        "batch_size = 128\n",
        "\n",
        "training_epochs = 1\n",
        "training_iterations = len(train_x) // batch_size\n",
        "\n",
        "label_dim = 10\n",
        "\n",
        "train_flag = True\n",
        "\n",
        "\"\"\" Graph Input using Dataset API \"\"\"\n",
        "train_dataset = tf.data.Dataset.from_tensor_slices((train_x, train_y)).\\\n",
        "    shuffle(buffer_size=100000).\\\n",
        "    prefetch(buffer_size=batch_size).\\\n",
        "    batch(batch_size, drop_remainder=True)\n",
        "\n",
        "test_dataset = tf.data.Dataset.from_tensor_slices((test_x, test_y)).\\\n",
        "    shuffle(buffer_size=100000).\\\n",
        "    prefetch(buffer_size=len(test_x)).\\\n",
        "    batch(len(test_x))"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/mnist.npz\n",
            "11493376/11490434 [==============================] - 0s 0us/step\n",
            "11501568/11490434 [==============================] - 0s 0us/step\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "s_ga65GU9tju"
      },
      "source": [
        "\"\"\" Model \"\"\"\n",
        "network = create_model_function(label_dim)\n",
        "\n",
        "\"\"\" Training \"\"\"\n",
        "optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate)\n",
        "\n",
        "\"\"\" Writer \"\"\"\n",
        "checkpoint_dir = 'checkpoints'\n",
        "logs_dir = 'logs'\n",
        "\n",
        "model_dir = 'nn_batchnorm'\n",
        "\n",
        "checkpoint_dir = os.path.join(checkpoint_dir, model_dir)\n",
        "check_folder(checkpoint_dir)\n",
        "checkpoint_prefix = os.path.join(checkpoint_dir, model_dir)\n",
        "logs_dir = os.path.join(logs_dir, model_dir)\n"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lxCu7j2E9yEs",
        "outputId": "cee249af-eeb8-4fb1-94f3-316a6cf0f934"
      },
      "source": [
        "if train_flag :\n",
        "\n",
        "    checkpoint = tf.train.Checkpoint(dnn=network)\n",
        "\n",
        "    # create writer for tensorboard\n",
        "    summary_writer = tf.summary.create_file_writer(logdir=logs_dir)\n",
        "    start_time = time()\n",
        "\n",
        "    # restore check-point if it exits\n",
        "    could_load, checkpoint_counter = load(network, checkpoint_dir)    \n",
        "\n",
        "    if could_load:\n",
        "        start_epoch = (int)(checkpoint_counter / training_iterations)        \n",
        "        counter = checkpoint_counter        \n",
        "        print(\" [*] Load SUCCESS\")\n",
        "    else:\n",
        "        start_epoch = 0\n",
        "        start_iteration = 0\n",
        "        counter = 0\n",
        "        print(\" [!] Load failed...\")\n",
        "    \n",
        "    # train phase\n",
        "    with summary_writer.as_default():  # for tensorboard\n",
        "        for epoch in range(start_epoch, training_epochs):\n",
        "            for idx, (train_input, train_label) in enumerate(train_dataset):            \n",
        "                grads = grad(network, train_input, train_label)\n",
        "                optimizer.apply_gradients(grads_and_vars=zip(grads, network.trainable_variables))\n",
        "\n",
        "                train_loss = loss_fn(network, train_input, train_label)\n",
        "                train_accuracy = accuracy_fn(network, train_input, train_label)\n",
        "                \n",
        "                for test_input, test_label in test_dataset:                \n",
        "                    test_accuracy = accuracy_fn(network, test_input, test_label)\n",
        "\n",
        "                tf.summary.scalar(name='train_loss', data=train_loss, step=counter)\n",
        "                tf.summary.scalar(name='train_accuracy', data=train_accuracy, step=counter)\n",
        "                tf.summary.scalar(name='test_accuracy', data=test_accuracy, step=counter)\n",
        "\n",
        "                print(\n",
        "                    \"Epoch: [%2d] [%5d/%5d] time: %4.4f, train_loss: %.8f, train_accuracy: %.4f, test_Accuracy: %.4f\" \\\n",
        "                    % (epoch, idx, training_iterations, time() - start_time, train_loss, train_accuracy,\n",
        "                       test_accuracy))\n",
        "                counter += 1                \n",
        "        checkpoint.save(file_prefix=checkpoint_prefix + '-{}'.format(counter))\n",
        "        \n",
        "# test phase      \n",
        "else :\n",
        "    _, _ = load(network, checkpoint_dir)\n",
        "    for test_input, test_label in test_dataset:    \n",
        "        test_accuracy = accuracy_fn(network, test_input, test_label)\n",
        "\n",
        "    print(\"test_Accuracy: %.4f\" % (test_accuracy))"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            " [*] Reading checkpoints...\n",
            " [*] Failed to find a checkpoint\n",
            " [!] Load failed...\n",
            "Epoch: [ 0] [    0/  468] time: 1.4922, train_loss: 0.34087646, train_accuracy: 0.2812, test_Accuracy: 0.2059\n",
            "Epoch: [ 0] [    1/  468] time: 2.1623, train_loss: 0.33622277, train_accuracy: 0.3984, test_Accuracy: 0.2757\n",
            "Epoch: [ 0] [    2/  468] time: 2.8689, train_loss: 0.35907507, train_accuracy: 0.3828, test_Accuracy: 0.3735\n",
            "Epoch: [ 0] [    3/  468] time: 3.6171, train_loss: 0.25274342, train_accuracy: 0.5547, test_Accuracy: 0.4710\n",
            "Epoch: [ 0] [    4/  468] time: 4.2956, train_loss: 0.35523099, train_accuracy: 0.5547, test_Accuracy: 0.5747\n",
            "Epoch: [ 0] [    5/  468] time: 4.9852, train_loss: 0.29102072, train_accuracy: 0.6562, test_Accuracy: 0.6568\n",
            "Epoch: [ 0] [    6/  468] time: 5.7610, train_loss: 0.22269259, train_accuracy: 0.8203, test_Accuracy: 0.7190\n",
            "Epoch: [ 0] [    7/  468] time: 6.5211, train_loss: 0.29748505, train_accuracy: 0.7578, test_Accuracy: 0.7614\n",
            "Epoch: [ 0] [    8/  468] time: 7.2085, train_loss: 0.31462491, train_accuracy: 0.7656, test_Accuracy: 0.7811\n",
            "Epoch: [ 0] [    9/  468] time: 7.8826, train_loss: 0.27131569, train_accuracy: 0.7422, test_Accuracy: 0.7947\n",
            "Epoch: [ 0] [   10/  468] time: 8.6266, train_loss: 0.16369550, train_accuracy: 0.8125, test_Accuracy: 0.8101\n",
            "Epoch: [ 0] [   11/  468] time: 9.3654, train_loss: 0.24771929, train_accuracy: 0.8594, test_Accuracy: 0.8123\n",
            "Epoch: [ 0] [   12/  468] time: 10.1203, train_loss: 0.28495014, train_accuracy: 0.7969, test_Accuracy: 0.8117\n",
            "Epoch: [ 0] [   13/  468] time: 10.7947, train_loss: 0.23032774, train_accuracy: 0.8516, test_Accuracy: 0.8172\n",
            "Epoch: [ 0] [   14/  468] time: 11.4722, train_loss: 0.27094963, train_accuracy: 0.8203, test_Accuracy: 0.8209\n",
            "Epoch: [ 0] [   15/  468] time: 12.2308, train_loss: 0.15198734, train_accuracy: 0.8828, test_Accuracy: 0.8261\n",
            "Epoch: [ 0] [   16/  468] time: 12.9248, train_loss: 0.24406159, train_accuracy: 0.8516, test_Accuracy: 0.8263\n",
            "Epoch: [ 0] [   17/  468] time: 13.6728, train_loss: 0.21643522, train_accuracy: 0.8359, test_Accuracy: 0.8278\n",
            "Epoch: [ 0] [   18/  468] time: 14.3512, train_loss: 0.24062890, train_accuracy: 0.8047, test_Accuracy: 0.8303\n",
            "Epoch: [ 0] [   19/  468] time: 15.0182, train_loss: 0.14326192, train_accuracy: 0.8984, test_Accuracy: 0.8328\n",
            "Epoch: [ 0] [   20/  468] time: 15.6881, train_loss: 0.22844274, train_accuracy: 0.8750, test_Accuracy: 0.8368\n",
            "Epoch: [ 0] [   21/  468] time: 16.3859, train_loss: 0.20379390, train_accuracy: 0.8828, test_Accuracy: 0.8473\n",
            "Epoch: [ 0] [   22/  468] time: 17.0555, train_loss: 0.20652321, train_accuracy: 0.8672, test_Accuracy: 0.8604\n",
            "Epoch: [ 0] [   23/  468] time: 17.7471, train_loss: 0.27435288, train_accuracy: 0.8516, test_Accuracy: 0.8732\n",
            "Epoch: [ 0] [   24/  468] time: 18.5019, train_loss: 0.23315024, train_accuracy: 0.8516, test_Accuracy: 0.8814\n",
            "Epoch: [ 0] [   25/  468] time: 19.1662, train_loss: 0.23858239, train_accuracy: 0.8750, test_Accuracy: 0.8881\n",
            "Epoch: [ 0] [   26/  468] time: 19.9053, train_loss: 0.19798076, train_accuracy: 0.8516, test_Accuracy: 0.8936\n",
            "Epoch: [ 0] [   27/  468] time: 20.5200, train_loss: 0.21566465, train_accuracy: 0.8984, test_Accuracy: 0.8969\n",
            "Epoch: [ 0] [   28/  468] time: 21.1478, train_loss: 0.17196637, train_accuracy: 0.9141, test_Accuracy: 0.8995\n",
            "Epoch: [ 0] [   29/  468] time: 21.7637, train_loss: 0.20560455, train_accuracy: 0.8984, test_Accuracy: 0.9015\n",
            "Epoch: [ 0] [   30/  468] time: 22.5163, train_loss: 0.16681904, train_accuracy: 0.9219, test_Accuracy: 0.9025\n",
            "Epoch: [ 0] [   31/  468] time: 23.1917, train_loss: 0.18229705, train_accuracy: 0.8984, test_Accuracy: 0.9061\n",
            "Epoch: [ 0] [   32/  468] time: 23.8662, train_loss: 0.18008786, train_accuracy: 0.9141, test_Accuracy: 0.9048\n",
            "Epoch: [ 0] [   33/  468] time: 24.6113, train_loss: 0.12440440, train_accuracy: 0.9453, test_Accuracy: 0.9044\n",
            "Epoch: [ 0] [   34/  468] time: 25.3444, train_loss: 0.13532025, train_accuracy: 0.9141, test_Accuracy: 0.9047\n",
            "Epoch: [ 0] [   35/  468] time: 25.9536, train_loss: 0.14780462, train_accuracy: 0.9141, test_Accuracy: 0.9059\n",
            "Epoch: [ 0] [   36/  468] time: 26.5919, train_loss: 0.13949530, train_accuracy: 0.8984, test_Accuracy: 0.9057\n",
            "Epoch: [ 0] [   37/  468] time: 27.2874, train_loss: 0.18540728, train_accuracy: 0.8984, test_Accuracy: 0.9052\n",
            "Epoch: [ 0] [   38/  468] time: 27.9672, train_loss: 0.08979709, train_accuracy: 0.9531, test_Accuracy: 0.9005\n",
            "Epoch: [ 0] [   39/  468] time: 28.6388, train_loss: 0.17721355, train_accuracy: 0.9453, test_Accuracy: 0.8957\n",
            "Epoch: [ 0] [   40/  468] time: 29.3183, train_loss: 0.24860783, train_accuracy: 0.8672, test_Accuracy: 0.8936\n",
            "Epoch: [ 0] [   41/  468] time: 30.0666, train_loss: 0.30111951, train_accuracy: 0.8828, test_Accuracy: 0.8915\n",
            "Epoch: [ 0] [   42/  468] time: 30.7485, train_loss: 0.19755577, train_accuracy: 0.8828, test_Accuracy: 0.8886\n",
            "Epoch: [ 0] [   43/  468] time: 31.4951, train_loss: 0.13268441, train_accuracy: 0.8594, test_Accuracy: 0.8857\n",
            "Epoch: [ 0] [   44/  468] time: 32.2336, train_loss: 0.21517073, train_accuracy: 0.9062, test_Accuracy: 0.8859\n",
            "Epoch: [ 0] [   45/  468] time: 32.8885, train_loss: 0.20221369, train_accuracy: 0.8828, test_Accuracy: 0.8859\n",
            "Epoch: [ 0] [   46/  468] time: 33.5652, train_loss: 0.14255555, train_accuracy: 0.9531, test_Accuracy: 0.8849\n",
            "Epoch: [ 0] [   47/  468] time: 34.1651, train_loss: 0.21195361, train_accuracy: 0.8750, test_Accuracy: 0.8851\n",
            "Epoch: [ 0] [   48/  468] time: 34.8227, train_loss: 0.11155048, train_accuracy: 0.9375, test_Accuracy: 0.8863\n",
            "Epoch: [ 0] [   49/  468] time: 35.4993, train_loss: 0.24054486, train_accuracy: 0.8750, test_Accuracy: 0.8882\n",
            "Epoch: [ 0] [   50/  468] time: 36.1769, train_loss: 0.13554230, train_accuracy: 0.8672, test_Accuracy: 0.8890\n",
            "Epoch: [ 0] [   51/  468] time: 36.8575, train_loss: 0.22115788, train_accuracy: 0.8828, test_Accuracy: 0.8906\n",
            "Epoch: [ 0] [   52/  468] time: 37.5972, train_loss: 0.15859167, train_accuracy: 0.8828, test_Accuracy: 0.8972\n",
            "Epoch: [ 0] [   53/  468] time: 38.2654, train_loss: 0.21127269, train_accuracy: 0.8750, test_Accuracy: 0.9007\n",
            "Epoch: [ 0] [   54/  468] time: 39.0183, train_loss: 0.13695231, train_accuracy: 0.8828, test_Accuracy: 0.9005\n",
            "Epoch: [ 0] [   55/  468] time: 39.6650, train_loss: 0.13724206, train_accuracy: 0.9297, test_Accuracy: 0.8986\n",
            "Epoch: [ 0] [   56/  468] time: 40.4050, train_loss: 0.10505854, train_accuracy: 0.9219, test_Accuracy: 0.8948\n",
            "Epoch: [ 0] [   57/  468] time: 41.0666, train_loss: 0.17859396, train_accuracy: 0.9453, test_Accuracy: 0.8923\n",
            "Epoch: [ 0] [   58/  468] time: 41.8059, train_loss: 0.13974741, train_accuracy: 0.9141, test_Accuracy: 0.8906\n",
            "Epoch: [ 0] [   59/  468] time: 42.4145, train_loss: 0.15127982, train_accuracy: 0.8984, test_Accuracy: 0.8921\n",
            "Epoch: [ 0] [   60/  468] time: 43.1482, train_loss: 0.15035746, train_accuracy: 0.9297, test_Accuracy: 0.8923\n",
            "Epoch: [ 0] [   61/  468] time: 43.8263, train_loss: 0.06779736, train_accuracy: 0.9297, test_Accuracy: 0.8961\n",
            "Epoch: [ 0] [   62/  468] time: 44.5178, train_loss: 0.14362055, train_accuracy: 0.9219, test_Accuracy: 0.9009\n",
            "Epoch: [ 0] [   63/  468] time: 45.1850, train_loss: 0.15130359, train_accuracy: 0.9062, test_Accuracy: 0.9075\n",
            "Epoch: [ 0] [   64/  468] time: 45.8601, train_loss: 0.14118117, train_accuracy: 0.9141, test_Accuracy: 0.9142\n",
            "Epoch: [ 0] [   65/  468] time: 46.5249, train_loss: 0.15744209, train_accuracy: 0.9297, test_Accuracy: 0.9161\n",
            "Epoch: [ 0] [   66/  468] time: 47.2659, train_loss: 0.16528007, train_accuracy: 0.9141, test_Accuracy: 0.9196\n",
            "Epoch: [ 0] [   67/  468] time: 48.0000, train_loss: 0.10806244, train_accuracy: 0.9297, test_Accuracy: 0.9235\n",
            "Epoch: [ 0] [   68/  468] time: 48.7433, train_loss: 0.11749358, train_accuracy: 0.9453, test_Accuracy: 0.9335\n",
            "Epoch: [ 0] [   69/  468] time: 49.4880, train_loss: 0.12223582, train_accuracy: 0.9375, test_Accuracy: 0.9377\n",
            "Epoch: [ 0] [   70/  468] time: 50.1532, train_loss: 0.11695754, train_accuracy: 0.9531, test_Accuracy: 0.9415\n",
            "Epoch: [ 0] [   71/  468] time: 50.8259, train_loss: 0.14325559, train_accuracy: 0.9531, test_Accuracy: 0.9425\n",
            "Epoch: [ 0] [   72/  468] time: 51.5751, train_loss: 0.14950624, train_accuracy: 0.9531, test_Accuracy: 0.9423\n",
            "Epoch: [ 0] [   73/  468] time: 52.2217, train_loss: 0.14394361, train_accuracy: 0.9453, test_Accuracy: 0.9418\n",
            "Epoch: [ 0] [   74/  468] time: 52.9575, train_loss: 0.06571437, train_accuracy: 0.9766, test_Accuracy: 0.9397\n",
            "Epoch: [ 0] [   75/  468] time: 53.6984, train_loss: 0.20078598, train_accuracy: 0.9375, test_Accuracy: 0.9402\n",
            "Epoch: [ 0] [   76/  468] time: 54.4412, train_loss: 0.14461811, train_accuracy: 0.9297, test_Accuracy: 0.9385\n",
            "Epoch: [ 0] [   77/  468] time: 55.1144, train_loss: 0.21818173, train_accuracy: 0.9375, test_Accuracy: 0.9396\n",
            "Epoch: [ 0] [   78/  468] time: 55.7717, train_loss: 0.16867271, train_accuracy: 0.9297, test_Accuracy: 0.9374\n",
            "Epoch: [ 0] [   79/  468] time: 56.4550, train_loss: 0.10308158, train_accuracy: 0.9453, test_Accuracy: 0.9360\n",
            "Epoch: [ 0] [   80/  468] time: 57.1983, train_loss: 0.20692658, train_accuracy: 0.8906, test_Accuracy: 0.9346\n",
            "Epoch: [ 0] [   81/  468] time: 57.8678, train_loss: 0.11428045, train_accuracy: 0.9609, test_Accuracy: 0.9328\n",
            "Epoch: [ 0] [   82/  468] time: 58.6202, train_loss: 0.12856057, train_accuracy: 0.9297, test_Accuracy: 0.9314\n",
            "Epoch: [ 0] [   83/  468] time: 59.3638, train_loss: 0.11982824, train_accuracy: 0.9297, test_Accuracy: 0.9311\n",
            "Epoch: [ 0] [   84/  468] time: 60.1230, train_loss: 0.20395678, train_accuracy: 0.9297, test_Accuracy: 0.9336\n",
            "Epoch: [ 0] [   85/  468] time: 60.7872, train_loss: 0.11462699, train_accuracy: 0.9297, test_Accuracy: 0.9340\n",
            "Epoch: [ 0] [   86/  468] time: 61.4664, train_loss: 0.13686582, train_accuracy: 0.9219, test_Accuracy: 0.9362\n",
            "Epoch: [ 0] [   87/  468] time: 62.1488, train_loss: 0.12473951, train_accuracy: 0.9375, test_Accuracy: 0.9375\n",
            "Epoch: [ 0] [   88/  468] time: 62.8151, train_loss: 0.14282534, train_accuracy: 0.9688, test_Accuracy: 0.9394\n",
            "Epoch: [ 0] [   89/  468] time: 63.4836, train_loss: 0.11961786, train_accuracy: 0.9453, test_Accuracy: 0.9397\n",
            "Epoch: [ 0] [   90/  468] time: 64.2256, train_loss: 0.12728480, train_accuracy: 0.9453, test_Accuracy: 0.9430\n",
            "Epoch: [ 0] [   91/  468] time: 64.9745, train_loss: 0.12120359, train_accuracy: 0.9375, test_Accuracy: 0.9422\n",
            "Epoch: [ 0] [   92/  468] time: 65.6585, train_loss: 0.09818202, train_accuracy: 0.9531, test_Accuracy: 0.9423\n",
            "Epoch: [ 0] [   93/  468] time: 66.3475, train_loss: 0.19496188, train_accuracy: 0.9297, test_Accuracy: 0.9435\n",
            "Epoch: [ 0] [   94/  468] time: 67.0965, train_loss: 0.19075508, train_accuracy: 0.9609, test_Accuracy: 0.9474\n",
            "Epoch: [ 0] [   95/  468] time: 67.8430, train_loss: 0.11500349, train_accuracy: 0.9766, test_Accuracy: 0.9487\n",
            "Epoch: [ 0] [   96/  468] time: 68.5847, train_loss: 0.13502084, train_accuracy: 0.9219, test_Accuracy: 0.9498\n",
            "Epoch: [ 0] [   97/  468] time: 69.2696, train_loss: 0.08364615, train_accuracy: 0.9766, test_Accuracy: 0.9503\n",
            "Epoch: [ 0] [   98/  468] time: 69.9061, train_loss: 0.09873230, train_accuracy: 0.9844, test_Accuracy: 0.9506\n",
            "Epoch: [ 0] [   99/  468] time: 70.5410, train_loss: 0.14306647, train_accuracy: 0.9609, test_Accuracy: 0.9505\n",
            "Epoch: [ 0] [  100/  468] time: 71.2924, train_loss: 0.06850358, train_accuracy: 0.9844, test_Accuracy: 0.9480\n",
            "Epoch: [ 0] [  101/  468] time: 71.9214, train_loss: 0.14143229, train_accuracy: 0.9531, test_Accuracy: 0.9467\n",
            "Epoch: [ 0] [  102/  468] time: 72.6622, train_loss: 0.13802427, train_accuracy: 0.9219, test_Accuracy: 0.9456\n",
            "Epoch: [ 0] [  103/  468] time: 73.2744, train_loss: 0.14081080, train_accuracy: 0.9531, test_Accuracy: 0.9453\n",
            "Epoch: [ 0] [  104/  468] time: 74.0201, train_loss: 0.08794636, train_accuracy: 0.9609, test_Accuracy: 0.9445\n",
            "Epoch: [ 0] [  105/  468] time: 74.6447, train_loss: 0.12873091, train_accuracy: 0.9609, test_Accuracy: 0.9439\n",
            "Epoch: [ 0] [  106/  468] time: 75.2494, train_loss: 0.13931973, train_accuracy: 0.9531, test_Accuracy: 0.9459\n",
            "Epoch: [ 0] [  107/  468] time: 75.9131, train_loss: 0.09596814, train_accuracy: 0.9766, test_Accuracy: 0.9487\n",
            "Epoch: [ 0] [  108/  468] time: 76.6623, train_loss: 0.10595605, train_accuracy: 0.9766, test_Accuracy: 0.9497\n",
            "Epoch: [ 0] [  109/  468] time: 77.3243, train_loss: 0.13190371, train_accuracy: 0.9688, test_Accuracy: 0.9509\n",
            "Epoch: [ 0] [  110/  468] time: 77.9992, train_loss: 0.08998665, train_accuracy: 0.9766, test_Accuracy: 0.9513\n",
            "Epoch: [ 0] [  111/  468] time: 78.7545, train_loss: 0.15729664, train_accuracy: 0.9375, test_Accuracy: 0.9516\n",
            "Epoch: [ 0] [  112/  468] time: 79.4246, train_loss: 0.09437274, train_accuracy: 0.9609, test_Accuracy: 0.9516\n",
            "Epoch: [ 0] [  113/  468] time: 80.0967, train_loss: 0.11915361, train_accuracy: 0.9688, test_Accuracy: 0.9514\n",
            "Epoch: [ 0] [  114/  468] time: 80.8374, train_loss: 0.12812297, train_accuracy: 0.9531, test_Accuracy: 0.9521\n",
            "Epoch: [ 0] [  115/  468] time: 81.5849, train_loss: 0.12050264, train_accuracy: 0.9844, test_Accuracy: 0.9529\n",
            "Epoch: [ 0] [  116/  468] time: 82.2317, train_loss: 0.07976846, train_accuracy: 0.9844, test_Accuracy: 0.9527\n",
            "Epoch: [ 0] [  117/  468] time: 82.9759, train_loss: 0.09446061, train_accuracy: 0.9844, test_Accuracy: 0.9518\n",
            "Epoch: [ 0] [  118/  468] time: 83.7266, train_loss: 0.17455581, train_accuracy: 0.9375, test_Accuracy: 0.9526\n",
            "Epoch: [ 0] [  119/  468] time: 84.4767, train_loss: 0.16156477, train_accuracy: 0.9453, test_Accuracy: 0.9523\n",
            "Epoch: [ 0] [  120/  468] time: 85.2389, train_loss: 0.15555908, train_accuracy: 0.9453, test_Accuracy: 0.9533\n",
            "Epoch: [ 0] [  121/  468] time: 85.9091, train_loss: 0.11821809, train_accuracy: 0.9688, test_Accuracy: 0.9523\n",
            "Epoch: [ 0] [  122/  468] time: 86.5784, train_loss: 0.08012991, train_accuracy: 0.9844, test_Accuracy: 0.9522\n",
            "Epoch: [ 0] [  123/  468] time: 87.3317, train_loss: 0.07499237, train_accuracy: 0.9531, test_Accuracy: 0.9515\n",
            "Epoch: [ 0] [  124/  468] time: 88.0660, train_loss: 0.04484522, train_accuracy: 0.9844, test_Accuracy: 0.9512\n",
            "Epoch: [ 0] [  125/  468] time: 88.6795, train_loss: 0.17599745, train_accuracy: 0.9531, test_Accuracy: 0.9509\n",
            "Epoch: [ 0] [  126/  468] time: 89.3386, train_loss: 0.06628513, train_accuracy: 0.9766, test_Accuracy: 0.9505\n",
            "Epoch: [ 0] [  127/  468] time: 90.0075, train_loss: 0.06535866, train_accuracy: 0.9844, test_Accuracy: 0.9512\n",
            "Epoch: [ 0] [  128/  468] time: 90.6478, train_loss: 0.22088921, train_accuracy: 0.9297, test_Accuracy: 0.9521\n",
            "Epoch: [ 0] [  129/  468] time: 91.2649, train_loss: 0.04479603, train_accuracy: 0.9922, test_Accuracy: 0.9534\n",
            "Epoch: [ 0] [  130/  468] time: 91.9980, train_loss: 0.14540076, train_accuracy: 0.9609, test_Accuracy: 0.9539\n",
            "Epoch: [ 0] [  131/  468] time: 92.7412, train_loss: 0.08009681, train_accuracy: 0.9844, test_Accuracy: 0.9540\n",
            "Epoch: [ 0] [  132/  468] time: 93.3853, train_loss: 0.11466032, train_accuracy: 0.9609, test_Accuracy: 0.9557\n",
            "Epoch: [ 0] [  133/  468] time: 94.1336, train_loss: 0.11466727, train_accuracy: 0.9688, test_Accuracy: 0.9569\n",
            "Epoch: [ 0] [  134/  468] time: 94.8742, train_loss: 0.09186100, train_accuracy: 0.9531, test_Accuracy: 0.9570\n",
            "Epoch: [ 0] [  135/  468] time: 95.5512, train_loss: 0.04084699, train_accuracy: 0.9922, test_Accuracy: 0.9565\n",
            "Epoch: [ 0] [  136/  468] time: 96.2382, train_loss: 0.12391727, train_accuracy: 0.9766, test_Accuracy: 0.9579\n",
            "Epoch: [ 0] [  137/  468] time: 96.9349, train_loss: 0.12530889, train_accuracy: 0.9531, test_Accuracy: 0.9571\n",
            "Epoch: [ 0] [  138/  468] time: 97.5715, train_loss: 0.12828618, train_accuracy: 0.9531, test_Accuracy: 0.9571\n",
            "Epoch: [ 0] [  139/  468] time: 98.3164, train_loss: 0.11618511, train_accuracy: 0.9766, test_Accuracy: 0.9575\n",
            "Epoch: [ 0] [  140/  468] time: 99.0628, train_loss: 0.08791558, train_accuracy: 0.9922, test_Accuracy: 0.9581\n",
            "Epoch: [ 0] [  141/  468] time: 99.8004, train_loss: 0.10446776, train_accuracy: 0.9688, test_Accuracy: 0.9572\n",
            "Epoch: [ 0] [  142/  468] time: 100.4261, train_loss: 0.13270962, train_accuracy: 0.9531, test_Accuracy: 0.9576\n",
            "Epoch: [ 0] [  143/  468] time: 101.0694, train_loss: 0.08345971, train_accuracy: 0.9609, test_Accuracy: 0.9575\n",
            "Epoch: [ 0] [  144/  468] time: 101.6969, train_loss: 0.09442410, train_accuracy: 0.9453, test_Accuracy: 0.9576\n",
            "Epoch: [ 0] [  145/  468] time: 102.4398, train_loss: 0.12089553, train_accuracy: 0.9375, test_Accuracy: 0.9577\n",
            "Epoch: [ 0] [  146/  468] time: 103.1833, train_loss: 0.08993629, train_accuracy: 0.9688, test_Accuracy: 0.9570\n",
            "Epoch: [ 0] [  147/  468] time: 103.9368, train_loss: 0.08612606, train_accuracy: 0.9609, test_Accuracy: 0.9572\n",
            "Epoch: [ 0] [  148/  468] time: 104.6975, train_loss: 0.06166322, train_accuracy: 0.9609, test_Accuracy: 0.9585\n",
            "Epoch: [ 0] [  149/  468] time: 105.4530, train_loss: 0.07606661, train_accuracy: 0.9766, test_Accuracy: 0.9589\n",
            "Epoch: [ 0] [  150/  468] time: 106.1992, train_loss: 0.04942837, train_accuracy: 0.9766, test_Accuracy: 0.9596\n",
            "Epoch: [ 0] [  151/  468] time: 106.9447, train_loss: 0.09763527, train_accuracy: 0.9766, test_Accuracy: 0.9591\n",
            "Epoch: [ 0] [  152/  468] time: 107.7006, train_loss: 0.07535519, train_accuracy: 0.9609, test_Accuracy: 0.9597\n",
            "Epoch: [ 0] [  153/  468] time: 108.3351, train_loss: 0.12208393, train_accuracy: 0.9375, test_Accuracy: 0.9606\n",
            "Epoch: [ 0] [  154/  468] time: 109.0816, train_loss: 0.06415323, train_accuracy: 0.9688, test_Accuracy: 0.9610\n",
            "Epoch: [ 0] [  155/  468] time: 109.7009, train_loss: 0.05622802, train_accuracy: 0.9844, test_Accuracy: 0.9615\n",
            "Epoch: [ 0] [  156/  468] time: 110.4433, train_loss: 0.04149893, train_accuracy: 0.9922, test_Accuracy: 0.9608\n",
            "Epoch: [ 0] [  157/  468] time: 111.1182, train_loss: 0.09921250, train_accuracy: 0.9375, test_Accuracy: 0.9599\n",
            "Epoch: [ 0] [  158/  468] time: 111.8686, train_loss: 0.06053497, train_accuracy: 0.9688, test_Accuracy: 0.9601\n",
            "Epoch: [ 0] [  159/  468] time: 112.5328, train_loss: 0.12418255, train_accuracy: 0.9609, test_Accuracy: 0.9590\n",
            "Epoch: [ 0] [  160/  468] time: 113.1690, train_loss: 0.07356776, train_accuracy: 0.9766, test_Accuracy: 0.9571\n",
            "Epoch: [ 0] [  161/  468] time: 113.9094, train_loss: 0.14896849, train_accuracy: 0.9531, test_Accuracy: 0.9546\n",
            "Epoch: [ 0] [  162/  468] time: 114.6577, train_loss: 0.07053968, train_accuracy: 0.9766, test_Accuracy: 0.9533\n",
            "Epoch: [ 0] [  163/  468] time: 115.2915, train_loss: 0.04627125, train_accuracy: 0.9766, test_Accuracy: 0.9517\n",
            "Epoch: [ 0] [  164/  468] time: 116.0304, train_loss: 0.09626639, train_accuracy: 0.9609, test_Accuracy: 0.9511\n",
            "Epoch: [ 0] [  165/  468] time: 116.6651, train_loss: 0.13806267, train_accuracy: 0.9688, test_Accuracy: 0.9508\n",
            "Epoch: [ 0] [  166/  468] time: 117.4102, train_loss: 0.09556813, train_accuracy: 0.9766, test_Accuracy: 0.9515\n",
            "Epoch: [ 0] [  167/  468] time: 118.0426, train_loss: 0.08761613, train_accuracy: 0.9609, test_Accuracy: 0.9511\n",
            "Epoch: [ 0] [  168/  468] time: 118.6873, train_loss: 0.09537635, train_accuracy: 0.9688, test_Accuracy: 0.9513\n",
            "Epoch: [ 0] [  169/  468] time: 119.3592, train_loss: 0.09298088, train_accuracy: 0.9688, test_Accuracy: 0.9516\n",
            "Epoch: [ 0] [  170/  468] time: 120.1047, train_loss: 0.07632789, train_accuracy: 0.9688, test_Accuracy: 0.9521\n",
            "Epoch: [ 0] [  171/  468] time: 120.7710, train_loss: 0.14453515, train_accuracy: 0.9453, test_Accuracy: 0.9541\n",
            "Epoch: [ 0] [  172/  468] time: 121.4023, train_loss: 0.04462467, train_accuracy: 1.0000, test_Accuracy: 0.9544\n",
            "Epoch: [ 0] [  173/  468] time: 122.1426, train_loss: 0.07701250, train_accuracy: 0.9844, test_Accuracy: 0.9551\n",
            "Epoch: [ 0] [  174/  468] time: 122.8995, train_loss: 0.03869622, train_accuracy: 0.9922, test_Accuracy: 0.9560\n",
            "Epoch: [ 0] [  175/  468] time: 123.6397, train_loss: 0.10131508, train_accuracy: 0.9531, test_Accuracy: 0.9578\n",
            "Epoch: [ 0] [  176/  468] time: 124.3241, train_loss: 0.12505019, train_accuracy: 0.9766, test_Accuracy: 0.9576\n",
            "Epoch: [ 0] [  177/  468] time: 125.0769, train_loss: 0.08955982, train_accuracy: 0.9609, test_Accuracy: 0.9577\n",
            "Epoch: [ 0] [  178/  468] time: 125.7204, train_loss: 0.13943644, train_accuracy: 0.9453, test_Accuracy: 0.9568\n",
            "Epoch: [ 0] [  179/  468] time: 126.4706, train_loss: 0.12432645, train_accuracy: 0.9609, test_Accuracy: 0.9565\n",
            "Epoch: [ 0] [  180/  468] time: 127.1400, train_loss: 0.11825350, train_accuracy: 0.9531, test_Accuracy: 0.9548\n",
            "Epoch: [ 0] [  181/  468] time: 127.8139, train_loss: 0.10066186, train_accuracy: 0.9609, test_Accuracy: 0.9548\n",
            "Epoch: [ 0] [  182/  468] time: 128.4407, train_loss: 0.12377238, train_accuracy: 0.9922, test_Accuracy: 0.9536\n",
            "Epoch: [ 0] [  183/  468] time: 129.1109, train_loss: 0.16576678, train_accuracy: 0.9375, test_Accuracy: 0.9550\n",
            "Epoch: [ 0] [  184/  468] time: 129.8569, train_loss: 0.08773467, train_accuracy: 0.9609, test_Accuracy: 0.9552\n",
            "Epoch: [ 0] [  185/  468] time: 130.5137, train_loss: 0.10606005, train_accuracy: 0.9609, test_Accuracy: 0.9548\n",
            "Epoch: [ 0] [  186/  468] time: 131.1977, train_loss: 0.16895756, train_accuracy: 0.9766, test_Accuracy: 0.9555\n",
            "Epoch: [ 0] [  187/  468] time: 131.8802, train_loss: 0.04686454, train_accuracy: 0.9844, test_Accuracy: 0.9555\n",
            "Epoch: [ 0] [  188/  468] time: 132.6281, train_loss: 0.07523189, train_accuracy: 0.9688, test_Accuracy: 0.9542\n",
            "Epoch: [ 0] [  189/  468] time: 133.2949, train_loss: 0.06746840, train_accuracy: 0.9688, test_Accuracy: 0.9524\n",
            "Epoch: [ 0] [  190/  468] time: 134.0390, train_loss: 0.04313744, train_accuracy: 0.9688, test_Accuracy: 0.9505\n",
            "Epoch: [ 0] [  191/  468] time: 134.7181, train_loss: 0.11359596, train_accuracy: 0.9766, test_Accuracy: 0.9513\n",
            "Epoch: [ 0] [  192/  468] time: 135.4618, train_loss: 0.18822755, train_accuracy: 0.9297, test_Accuracy: 0.9499\n",
            "Epoch: [ 0] [  193/  468] time: 136.1191, train_loss: 0.08857410, train_accuracy: 0.9688, test_Accuracy: 0.9498\n",
            "Epoch: [ 0] [  194/  468] time: 136.8545, train_loss: 0.11883880, train_accuracy: 0.9375, test_Accuracy: 0.9486\n",
            "Epoch: [ 0] [  195/  468] time: 137.4717, train_loss: 0.05964291, train_accuracy: 0.9688, test_Accuracy: 0.9491\n",
            "Epoch: [ 0] [  196/  468] time: 138.1289, train_loss: 0.10003819, train_accuracy: 0.9531, test_Accuracy: 0.9488\n",
            "Epoch: [ 0] [  197/  468] time: 138.8061, train_loss: 0.13581535, train_accuracy: 0.9453, test_Accuracy: 0.9498\n",
            "Epoch: [ 0] [  198/  468] time: 139.4706, train_loss: 0.17545317, train_accuracy: 0.9375, test_Accuracy: 0.9506\n",
            "Epoch: [ 0] [  199/  468] time: 140.1660, train_loss: 0.06351183, train_accuracy: 0.9844, test_Accuracy: 0.9510\n",
            "Epoch: [ 0] [  200/  468] time: 140.8442, train_loss: 0.06469075, train_accuracy: 0.9844, test_Accuracy: 0.9496\n",
            "Epoch: [ 0] [  201/  468] time: 141.5117, train_loss: 0.08830344, train_accuracy: 0.9766, test_Accuracy: 0.9474\n",
            "Epoch: [ 0] [  202/  468] time: 142.1887, train_loss: 0.09475251, train_accuracy: 0.9688, test_Accuracy: 0.9462\n",
            "Epoch: [ 0] [  203/  468] time: 142.9333, train_loss: 0.08287494, train_accuracy: 0.9844, test_Accuracy: 0.9465\n",
            "Epoch: [ 0] [  204/  468] time: 143.6807, train_loss: 0.08136009, train_accuracy: 0.9688, test_Accuracy: 0.9495\n",
            "Epoch: [ 0] [  205/  468] time: 144.3289, train_loss: 0.11589992, train_accuracy: 0.9453, test_Accuracy: 0.9535\n",
            "Epoch: [ 0] [  206/  468] time: 145.0763, train_loss: 0.11887980, train_accuracy: 0.9531, test_Accuracy: 0.9556\n",
            "Epoch: [ 0] [  207/  468] time: 145.8278, train_loss: 0.06350051, train_accuracy: 0.9688, test_Accuracy: 0.9565\n",
            "Epoch: [ 0] [  208/  468] time: 146.5794, train_loss: 0.03739024, train_accuracy: 0.9844, test_Accuracy: 0.9574\n",
            "Epoch: [ 0] [  209/  468] time: 147.3224, train_loss: 0.18058729, train_accuracy: 0.9375, test_Accuracy: 0.9585\n",
            "Epoch: [ 0] [  210/  468] time: 148.0781, train_loss: 0.09165629, train_accuracy: 0.9375, test_Accuracy: 0.9602\n",
            "Epoch: [ 0] [  211/  468] time: 148.7587, train_loss: 0.13519408, train_accuracy: 0.9531, test_Accuracy: 0.9611\n",
            "Epoch: [ 0] [  212/  468] time: 149.4282, train_loss: 0.11988050, train_accuracy: 0.9844, test_Accuracy: 0.9610\n",
            "Epoch: [ 0] [  213/  468] time: 150.0659, train_loss: 0.08846228, train_accuracy: 0.9766, test_Accuracy: 0.9591\n",
            "Epoch: [ 0] [  214/  468] time: 150.8089, train_loss: 0.14220919, train_accuracy: 0.9688, test_Accuracy: 0.9593\n",
            "Epoch: [ 0] [  215/  468] time: 151.4667, train_loss: 0.16595641, train_accuracy: 0.9688, test_Accuracy: 0.9593\n",
            "Epoch: [ 0] [  216/  468] time: 152.1008, train_loss: 0.16134390, train_accuracy: 0.9688, test_Accuracy: 0.9593\n",
            "Epoch: [ 0] [  217/  468] time: 152.7200, train_loss: 0.13530719, train_accuracy: 0.9531, test_Accuracy: 0.9575\n",
            "Epoch: [ 0] [  218/  468] time: 153.4556, train_loss: 0.09387431, train_accuracy: 0.9688, test_Accuracy: 0.9565\n",
            "Epoch: [ 0] [  219/  468] time: 154.2008, train_loss: 0.18347234, train_accuracy: 0.9453, test_Accuracy: 0.9560\n",
            "Epoch: [ 0] [  220/  468] time: 154.9455, train_loss: 0.07210445, train_accuracy: 0.9766, test_Accuracy: 0.9546\n",
            "Epoch: [ 0] [  221/  468] time: 155.5746, train_loss: 0.10618576, train_accuracy: 0.9609, test_Accuracy: 0.9545\n",
            "Epoch: [ 0] [  222/  468] time: 156.3153, train_loss: 0.10266945, train_accuracy: 0.9844, test_Accuracy: 0.9534\n",
            "Epoch: [ 0] [  223/  468] time: 157.0551, train_loss: 0.06813940, train_accuracy: 0.9609, test_Accuracy: 0.9541\n",
            "Epoch: [ 0] [  224/  468] time: 157.6777, train_loss: 0.04781335, train_accuracy: 0.9844, test_Accuracy: 0.9538\n",
            "Epoch: [ 0] [  225/  468] time: 158.3129, train_loss: 0.10243800, train_accuracy: 0.9531, test_Accuracy: 0.9550\n",
            "Epoch: [ 0] [  226/  468] time: 159.0555, train_loss: 0.09705076, train_accuracy: 0.9688, test_Accuracy: 0.9563\n",
            "Epoch: [ 0] [  227/  468] time: 159.7973, train_loss: 0.04970855, train_accuracy: 0.9688, test_Accuracy: 0.9562\n",
            "Epoch: [ 0] [  228/  468] time: 160.4729, train_loss: 0.06664523, train_accuracy: 0.9688, test_Accuracy: 0.9568\n",
            "Epoch: [ 0] [  229/  468] time: 161.2055, train_loss: 0.10499369, train_accuracy: 0.9688, test_Accuracy: 0.9568\n",
            "Epoch: [ 0] [  230/  468] time: 161.9449, train_loss: 0.14578354, train_accuracy: 0.9219, test_Accuracy: 0.9573\n",
            "Epoch: [ 0] [  231/  468] time: 162.6905, train_loss: 0.13223341, train_accuracy: 0.9688, test_Accuracy: 0.9577\n",
            "Epoch: [ 0] [  232/  468] time: 163.3621, train_loss: 0.07727061, train_accuracy: 0.9766, test_Accuracy: 0.9578\n",
            "Epoch: [ 0] [  233/  468] time: 164.1040, train_loss: 0.08497406, train_accuracy: 0.9688, test_Accuracy: 0.9583\n",
            "Epoch: [ 0] [  234/  468] time: 164.7731, train_loss: 0.08193813, train_accuracy: 0.9609, test_Accuracy: 0.9577\n",
            "Epoch: [ 0] [  235/  468] time: 165.4308, train_loss: 0.11240948, train_accuracy: 0.9688, test_Accuracy: 0.9577\n",
            "Epoch: [ 0] [  236/  468] time: 166.1778, train_loss: 0.09681049, train_accuracy: 0.9844, test_Accuracy: 0.9580\n",
            "Epoch: [ 0] [  237/  468] time: 166.8430, train_loss: 0.05795818, train_accuracy: 0.9844, test_Accuracy: 0.9585\n",
            "Epoch: [ 0] [  238/  468] time: 167.5801, train_loss: 0.10145252, train_accuracy: 0.9531, test_Accuracy: 0.9584\n",
            "Epoch: [ 0] [  239/  468] time: 168.2005, train_loss: 0.12629370, train_accuracy: 0.9766, test_Accuracy: 0.9593\n",
            "Epoch: [ 0] [  240/  468] time: 168.8272, train_loss: 0.05712172, train_accuracy: 0.9766, test_Accuracy: 0.9601\n",
            "Epoch: [ 0] [  241/  468] time: 169.5627, train_loss: 0.09449584, train_accuracy: 0.9766, test_Accuracy: 0.9598\n",
            "Epoch: [ 0] [  242/  468] time: 170.1852, train_loss: 0.06256720, train_accuracy: 0.9844, test_Accuracy: 0.9597\n",
            "Epoch: [ 0] [  243/  468] time: 170.8187, train_loss: 0.03766853, train_accuracy: 0.9844, test_Accuracy: 0.9603\n",
            "Epoch: [ 0] [  244/  468] time: 171.5690, train_loss: 0.11474220, train_accuracy: 0.9609, test_Accuracy: 0.9607\n",
            "Epoch: [ 0] [  245/  468] time: 172.2046, train_loss: 0.13415980, train_accuracy: 0.9766, test_Accuracy: 0.9608\n",
            "Epoch: [ 0] [  246/  468] time: 172.8461, train_loss: 0.03461092, train_accuracy: 0.9844, test_Accuracy: 0.9606\n",
            "Epoch: [ 0] [  247/  468] time: 173.5176, train_loss: 0.09285277, train_accuracy: 0.9688, test_Accuracy: 0.9609\n",
            "Epoch: [ 0] [  248/  468] time: 174.1587, train_loss: 0.08921657, train_accuracy: 0.9844, test_Accuracy: 0.9605\n",
            "Epoch: [ 0] [  249/  468] time: 174.7782, train_loss: 0.02840612, train_accuracy: 0.9844, test_Accuracy: 0.9611\n",
            "Epoch: [ 0] [  250/  468] time: 175.4583, train_loss: 0.13564101, train_accuracy: 0.9609, test_Accuracy: 0.9616\n",
            "Epoch: [ 0] [  251/  468] time: 176.1356, train_loss: 0.03402847, train_accuracy: 0.9766, test_Accuracy: 0.9630\n",
            "Epoch: [ 0] [  252/  468] time: 176.8743, train_loss: 0.04266751, train_accuracy: 0.9922, test_Accuracy: 0.9638\n",
            "Epoch: [ 0] [  253/  468] time: 177.5448, train_loss: 0.11293546, train_accuracy: 0.9688, test_Accuracy: 0.9642\n",
            "Epoch: [ 0] [  254/  468] time: 178.2958, train_loss: 0.04808186, train_accuracy: 0.9766, test_Accuracy: 0.9642\n",
            "Epoch: [ 0] [  255/  468] time: 179.0413, train_loss: 0.05819987, train_accuracy: 0.9844, test_Accuracy: 0.9638\n",
            "Epoch: [ 0] [  256/  468] time: 179.7841, train_loss: 0.11830330, train_accuracy: 0.9453, test_Accuracy: 0.9633\n",
            "Epoch: [ 0] [  257/  468] time: 180.5252, train_loss: 0.07553585, train_accuracy: 0.9766, test_Accuracy: 0.9629\n",
            "Epoch: [ 0] [  258/  468] time: 181.2680, train_loss: 0.07535645, train_accuracy: 0.9844, test_Accuracy: 0.9631\n",
            "Epoch: [ 0] [  259/  468] time: 181.8949, train_loss: 0.05779982, train_accuracy: 0.9766, test_Accuracy: 0.9622\n",
            "Epoch: [ 0] [  260/  468] time: 182.5733, train_loss: 0.04447388, train_accuracy: 0.9766, test_Accuracy: 0.9617\n",
            "Epoch: [ 0] [  261/  468] time: 183.3229, train_loss: 0.05573411, train_accuracy: 0.9922, test_Accuracy: 0.9619\n",
            "Epoch: [ 0] [  262/  468] time: 184.0130, train_loss: 0.08573030, train_accuracy: 0.9766, test_Accuracy: 0.9625\n",
            "Epoch: [ 0] [  263/  468] time: 184.6771, train_loss: 0.05516659, train_accuracy: 0.9688, test_Accuracy: 0.9631\n",
            "Epoch: [ 0] [  264/  468] time: 185.4229, train_loss: 0.11827797, train_accuracy: 0.9688, test_Accuracy: 0.9626\n",
            "Epoch: [ 0] [  265/  468] time: 186.0902, train_loss: 0.11301574, train_accuracy: 0.9844, test_Accuracy: 0.9627\n",
            "Epoch: [ 0] [  266/  468] time: 186.8322, train_loss: 0.04914570, train_accuracy: 0.9922, test_Accuracy: 0.9634\n",
            "Epoch: [ 0] [  267/  468] time: 187.4475, train_loss: 0.10276583, train_accuracy: 0.9609, test_Accuracy: 0.9631\n",
            "Epoch: [ 0] [  268/  468] time: 188.0814, train_loss: 0.07236592, train_accuracy: 0.9766, test_Accuracy: 0.9626\n",
            "Epoch: [ 0] [  269/  468] time: 188.8308, train_loss: 0.05282829, train_accuracy: 0.9766, test_Accuracy: 0.9615\n",
            "Epoch: [ 0] [  270/  468] time: 189.5784, train_loss: 0.02945679, train_accuracy: 1.0000, test_Accuracy: 0.9606\n",
            "Epoch: [ 0] [  271/  468] time: 190.2692, train_loss: 0.09921709, train_accuracy: 0.9688, test_Accuracy: 0.9603\n",
            "Epoch: [ 0] [  272/  468] time: 190.9472, train_loss: 0.04100208, train_accuracy: 1.0000, test_Accuracy: 0.9602\n",
            "Epoch: [ 0] [  273/  468] time: 191.6905, train_loss: 0.07235612, train_accuracy: 0.9766, test_Accuracy: 0.9589\n",
            "Epoch: [ 0] [  274/  468] time: 192.3295, train_loss: 0.08488569, train_accuracy: 0.9844, test_Accuracy: 0.9576\n",
            "Epoch: [ 0] [  275/  468] time: 192.9924, train_loss: 0.12689900, train_accuracy: 0.9609, test_Accuracy: 0.9557\n",
            "Epoch: [ 0] [  276/  468] time: 193.6567, train_loss: 0.12448763, train_accuracy: 0.9297, test_Accuracy: 0.9539\n",
            "Epoch: [ 0] [  277/  468] time: 194.3999, train_loss: 0.06941958, train_accuracy: 0.9609, test_Accuracy: 0.9544\n",
            "Epoch: [ 0] [  278/  468] time: 195.1397, train_loss: 0.09856745, train_accuracy: 0.9688, test_Accuracy: 0.9526\n",
            "Epoch: [ 0] [  279/  468] time: 195.7993, train_loss: 0.06302150, train_accuracy: 0.9766, test_Accuracy: 0.9524\n",
            "Epoch: [ 0] [  280/  468] time: 196.4723, train_loss: 0.15107054, train_accuracy: 0.9297, test_Accuracy: 0.9542\n",
            "Epoch: [ 0] [  281/  468] time: 197.1397, train_loss: 0.09926963, train_accuracy: 0.9531, test_Accuracy: 0.9541\n",
            "Epoch: [ 0] [  282/  468] time: 197.8880, train_loss: 0.08793402, train_accuracy: 0.9766, test_Accuracy: 0.9540\n",
            "Epoch: [ 0] [  283/  468] time: 198.5186, train_loss: 0.13923842, train_accuracy: 0.9531, test_Accuracy: 0.9547\n",
            "Epoch: [ 0] [  284/  468] time: 199.1923, train_loss: 0.05745948, train_accuracy: 0.9766, test_Accuracy: 0.9550\n",
            "Epoch: [ 0] [  285/  468] time: 199.9322, train_loss: 0.09909777, train_accuracy: 0.9453, test_Accuracy: 0.9553\n",
            "Epoch: [ 0] [  286/  468] time: 200.6894, train_loss: 0.10054936, train_accuracy: 0.9609, test_Accuracy: 0.9544\n",
            "Epoch: [ 0] [  287/  468] time: 201.3528, train_loss: 0.07729978, train_accuracy: 0.9531, test_Accuracy: 0.9553\n",
            "Epoch: [ 0] [  288/  468] time: 202.0919, train_loss: 0.05970315, train_accuracy: 0.9844, test_Accuracy: 0.9558\n",
            "Epoch: [ 0] [  289/  468] time: 202.8324, train_loss: 0.06892509, train_accuracy: 0.9844, test_Accuracy: 0.9581\n",
            "Epoch: [ 0] [  290/  468] time: 203.5737, train_loss: 0.04799563, train_accuracy: 0.9922, test_Accuracy: 0.9600\n",
            "Epoch: [ 0] [  291/  468] time: 204.3115, train_loss: 0.09171627, train_accuracy: 0.9609, test_Accuracy: 0.9598\n",
            "Epoch: [ 0] [  292/  468] time: 205.0585, train_loss: 0.05587333, train_accuracy: 0.9922, test_Accuracy: 0.9605\n",
            "Epoch: [ 0] [  293/  468] time: 205.7283, train_loss: 0.08600174, train_accuracy: 0.9688, test_Accuracy: 0.9618\n",
            "Epoch: [ 0] [  294/  468] time: 206.4026, train_loss: 0.03704553, train_accuracy: 0.9844, test_Accuracy: 0.9621\n",
            "Epoch: [ 0] [  295/  468] time: 207.0792, train_loss: 0.08066796, train_accuracy: 0.9844, test_Accuracy: 0.9621\n",
            "Epoch: [ 0] [  296/  468] time: 207.8396, train_loss: 0.08511615, train_accuracy: 0.9688, test_Accuracy: 0.9628\n",
            "Epoch: [ 0] [  297/  468] time: 208.5818, train_loss: 0.10663725, train_accuracy: 0.9766, test_Accuracy: 0.9634\n",
            "Epoch: [ 0] [  298/  468] time: 209.2531, train_loss: 0.06885622, train_accuracy: 0.9766, test_Accuracy: 0.9638\n",
            "Epoch: [ 0] [  299/  468] time: 209.9176, train_loss: 0.14483027, train_accuracy: 0.9688, test_Accuracy: 0.9638\n",
            "Epoch: [ 0] [  300/  468] time: 210.5850, train_loss: 0.07286505, train_accuracy: 0.9453, test_Accuracy: 0.9633\n",
            "Epoch: [ 0] [  301/  468] time: 211.3295, train_loss: 0.06653087, train_accuracy: 0.9844, test_Accuracy: 0.9646\n",
            "Epoch: [ 0] [  302/  468] time: 212.0852, train_loss: 0.05684543, train_accuracy: 0.9609, test_Accuracy: 0.9640\n",
            "Epoch: [ 0] [  303/  468] time: 212.7541, train_loss: 0.08358268, train_accuracy: 0.9766, test_Accuracy: 0.9642\n",
            "Epoch: [ 0] [  304/  468] time: 213.4191, train_loss: 0.05686998, train_accuracy: 0.9688, test_Accuracy: 0.9655\n",
            "Epoch: [ 0] [  305/  468] time: 214.0829, train_loss: 0.02823447, train_accuracy: 0.9844, test_Accuracy: 0.9654\n",
            "Epoch: [ 0] [  306/  468] time: 214.8200, train_loss: 0.06968554, train_accuracy: 0.9688, test_Accuracy: 0.9648\n",
            "Epoch: [ 0] [  307/  468] time: 215.5602, train_loss: 0.05736699, train_accuracy: 0.9688, test_Accuracy: 0.9640\n",
            "Epoch: [ 0] [  308/  468] time: 216.3062, train_loss: 0.05426777, train_accuracy: 0.9766, test_Accuracy: 0.9645\n",
            "Epoch: [ 0] [  309/  468] time: 216.9255, train_loss: 0.07574755, train_accuracy: 0.9766, test_Accuracy: 0.9646\n",
            "Epoch: [ 0] [  310/  468] time: 217.6580, train_loss: 0.05186618, train_accuracy: 0.9609, test_Accuracy: 0.9651\n",
            "Epoch: [ 0] [  311/  468] time: 218.3206, train_loss: 0.09062847, train_accuracy: 0.9766, test_Accuracy: 0.9658\n",
            "Epoch: [ 0] [  312/  468] time: 218.9923, train_loss: 0.06336565, train_accuracy: 0.9922, test_Accuracy: 0.9650\n",
            "Epoch: [ 0] [  313/  468] time: 219.6127, train_loss: 0.06899326, train_accuracy: 0.9609, test_Accuracy: 0.9644\n",
            "Epoch: [ 0] [  314/  468] time: 220.3432, train_loss: 0.07177494, train_accuracy: 0.9609, test_Accuracy: 0.9637\n",
            "Epoch: [ 0] [  315/  468] time: 220.9593, train_loss: 0.07773833, train_accuracy: 0.9531, test_Accuracy: 0.9631\n",
            "Epoch: [ 0] [  316/  468] time: 221.7042, train_loss: 0.16786249, train_accuracy: 0.9688, test_Accuracy: 0.9618\n",
            "Epoch: [ 0] [  317/  468] time: 222.3453, train_loss: 0.15591136, train_accuracy: 0.9844, test_Accuracy: 0.9648\n",
            "Epoch: [ 0] [  318/  468] time: 223.0889, train_loss: 0.07291107, train_accuracy: 0.9844, test_Accuracy: 0.9652\n",
            "Epoch: [ 0] [  319/  468] time: 223.8333, train_loss: 0.11695637, train_accuracy: 0.9688, test_Accuracy: 0.9648\n",
            "Epoch: [ 0] [  320/  468] time: 224.4470, train_loss: 0.06827264, train_accuracy: 0.9766, test_Accuracy: 0.9639\n",
            "Epoch: [ 0] [  321/  468] time: 225.0772, train_loss: 0.06096172, train_accuracy: 0.9844, test_Accuracy: 0.9640\n",
            "Epoch: [ 0] [  322/  468] time: 225.8137, train_loss: 0.04377569, train_accuracy: 0.9844, test_Accuracy: 0.9637\n",
            "Epoch: [ 0] [  323/  468] time: 226.4412, train_loss: 0.06627193, train_accuracy: 0.9453, test_Accuracy: 0.9636\n",
            "Epoch: [ 0] [  324/  468] time: 227.1831, train_loss: 0.12033045, train_accuracy: 0.9609, test_Accuracy: 0.9627\n",
            "Epoch: [ 0] [  325/  468] time: 227.8490, train_loss: 0.08868603, train_accuracy: 0.9688, test_Accuracy: 0.9643\n",
            "Epoch: [ 0] [  326/  468] time: 228.5914, train_loss: 0.14698832, train_accuracy: 0.9531, test_Accuracy: 0.9652\n",
            "Epoch: [ 0] [  327/  468] time: 229.3306, train_loss: 0.03478075, train_accuracy: 0.9844, test_Accuracy: 0.9651\n",
            "Epoch: [ 0] [  328/  468] time: 230.0142, train_loss: 0.13582823, train_accuracy: 0.9531, test_Accuracy: 0.9665\n",
            "Epoch: [ 0] [  329/  468] time: 230.6790, train_loss: 0.14542857, train_accuracy: 0.9609, test_Accuracy: 0.9674\n",
            "Epoch: [ 0] [  330/  468] time: 231.4226, train_loss: 0.09993280, train_accuracy: 0.9766, test_Accuracy: 0.9679\n",
            "Epoch: [ 0] [  331/  468] time: 232.1752, train_loss: 0.03882554, train_accuracy: 0.9922, test_Accuracy: 0.9666\n",
            "Epoch: [ 0] [  332/  468] time: 232.8550, train_loss: 0.10274595, train_accuracy: 0.9609, test_Accuracy: 0.9650\n",
            "Epoch: [ 0] [  333/  468] time: 233.5933, train_loss: 0.10300394, train_accuracy: 0.9531, test_Accuracy: 0.9632\n",
            "Epoch: [ 0] [  334/  468] time: 234.3479, train_loss: 0.07337222, train_accuracy: 0.9688, test_Accuracy: 0.9630\n",
            "Epoch: [ 0] [  335/  468] time: 235.0288, train_loss: 0.08416995, train_accuracy: 0.9766, test_Accuracy: 0.9619\n",
            "Epoch: [ 0] [  336/  468] time: 235.6828, train_loss: 0.04560837, train_accuracy: 0.9766, test_Accuracy: 0.9620\n",
            "Epoch: [ 0] [  337/  468] time: 236.3457, train_loss: 0.11732292, train_accuracy: 0.9766, test_Accuracy: 0.9620\n",
            "Epoch: [ 0] [  338/  468] time: 237.0832, train_loss: 0.09034499, train_accuracy: 0.9609, test_Accuracy: 0.9604\n",
            "Epoch: [ 0] [  339/  468] time: 237.8256, train_loss: 0.11098936, train_accuracy: 0.9609, test_Accuracy: 0.9591\n",
            "Epoch: [ 0] [  340/  468] time: 238.5756, train_loss: 0.05801640, train_accuracy: 0.9844, test_Accuracy: 0.9579\n",
            "Epoch: [ 0] [  341/  468] time: 239.3121, train_loss: 0.10447611, train_accuracy: 0.9531, test_Accuracy: 0.9562\n",
            "Epoch: [ 0] [  342/  468] time: 239.9923, train_loss: 0.06201025, train_accuracy: 0.9453, test_Accuracy: 0.9557\n",
            "Epoch: [ 0] [  343/  468] time: 240.7340, train_loss: 0.14761272, train_accuracy: 0.9688, test_Accuracy: 0.9576\n",
            "Epoch: [ 0] [  344/  468] time: 241.3630, train_loss: 0.05603119, train_accuracy: 0.9766, test_Accuracy: 0.9577\n",
            "Epoch: [ 0] [  345/  468] time: 242.0399, train_loss: 0.10013738, train_accuracy: 0.9453, test_Accuracy: 0.9605\n",
            "Epoch: [ 0] [  346/  468] time: 242.7793, train_loss: 0.10366782, train_accuracy: 0.9766, test_Accuracy: 0.9622\n",
            "Epoch: [ 0] [  347/  468] time: 243.5346, train_loss: 0.05487631, train_accuracy: 0.9453, test_Accuracy: 0.9628\n",
            "Epoch: [ 0] [  348/  468] time: 244.2057, train_loss: 0.01687732, train_accuracy: 0.9922, test_Accuracy: 0.9622\n",
            "Epoch: [ 0] [  349/  468] time: 244.9532, train_loss: 0.06831311, train_accuracy: 0.9688, test_Accuracy: 0.9606\n",
            "Epoch: [ 0] [  350/  468] time: 245.6516, train_loss: 0.06126579, train_accuracy: 0.9922, test_Accuracy: 0.9598\n",
            "Epoch: [ 0] [  351/  468] time: 246.3301, train_loss: 0.08695572, train_accuracy: 0.9688, test_Accuracy: 0.9604\n",
            "Epoch: [ 0] [  352/  468] time: 247.0734, train_loss: 0.08784202, train_accuracy: 0.9688, test_Accuracy: 0.9589\n",
            "Epoch: [ 0] [  353/  468] time: 247.7211, train_loss: 0.17839855, train_accuracy: 0.9688, test_Accuracy: 0.9583\n",
            "Epoch: [ 0] [  354/  468] time: 248.4542, train_loss: 0.07156616, train_accuracy: 0.9766, test_Accuracy: 0.9559\n",
            "Epoch: [ 0] [  355/  468] time: 249.1277, train_loss: 0.05514354, train_accuracy: 0.9688, test_Accuracy: 0.9543\n",
            "Epoch: [ 0] [  356/  468] time: 249.7719, train_loss: 0.05237000, train_accuracy: 0.9688, test_Accuracy: 0.9526\n",
            "Epoch: [ 0] [  357/  468] time: 250.4117, train_loss: 0.12686922, train_accuracy: 0.9609, test_Accuracy: 0.9534\n",
            "Epoch: [ 0] [  358/  468] time: 251.0721, train_loss: 0.07754615, train_accuracy: 0.9688, test_Accuracy: 0.9550\n",
            "Epoch: [ 0] [  359/  468] time: 251.7325, train_loss: 0.06273104, train_accuracy: 0.9688, test_Accuracy: 0.9561\n",
            "Epoch: [ 0] [  360/  468] time: 252.4846, train_loss: 0.06314588, train_accuracy: 0.9844, test_Accuracy: 0.9563\n",
            "Epoch: [ 0] [  361/  468] time: 253.1575, train_loss: 0.04176087, train_accuracy: 0.9766, test_Accuracy: 0.9570\n",
            "Epoch: [ 0] [  362/  468] time: 253.8282, train_loss: 0.02410441, train_accuracy: 1.0000, test_Accuracy: 0.9582\n",
            "Epoch: [ 0] [  363/  468] time: 254.4391, train_loss: 0.14824106, train_accuracy: 0.9688, test_Accuracy: 0.9597\n",
            "Epoch: [ 0] [  364/  468] time: 255.0813, train_loss: 0.09596209, train_accuracy: 0.9688, test_Accuracy: 0.9606\n",
            "Epoch: [ 0] [  365/  468] time: 255.7564, train_loss: 0.09425541, train_accuracy: 0.9688, test_Accuracy: 0.9609\n",
            "Epoch: [ 0] [  366/  468] time: 256.5143, train_loss: 0.07670599, train_accuracy: 0.9688, test_Accuracy: 0.9611\n",
            "Epoch: [ 0] [  367/  468] time: 257.2629, train_loss: 0.05267069, train_accuracy: 0.9844, test_Accuracy: 0.9620\n",
            "Epoch: [ 0] [  368/  468] time: 257.8970, train_loss: 0.06232598, train_accuracy: 0.9766, test_Accuracy: 0.9623\n",
            "Epoch: [ 0] [  369/  468] time: 258.5672, train_loss: 0.07796472, train_accuracy: 0.9766, test_Accuracy: 0.9630\n",
            "Epoch: [ 0] [  370/  468] time: 259.3148, train_loss: 0.03349102, train_accuracy: 0.9922, test_Accuracy: 0.9629\n",
            "Epoch: [ 0] [  371/  468] time: 260.0499, train_loss: 0.06558519, train_accuracy: 0.9844, test_Accuracy: 0.9619\n",
            "Epoch: [ 0] [  372/  468] time: 260.7347, train_loss: 0.07635913, train_accuracy: 0.9844, test_Accuracy: 0.9618\n",
            "Epoch: [ 0] [  373/  468] time: 261.4794, train_loss: 0.10263433, train_accuracy: 0.9531, test_Accuracy: 0.9614\n",
            "Epoch: [ 0] [  374/  468] time: 262.2180, train_loss: 0.02931494, train_accuracy: 0.9844, test_Accuracy: 0.9598\n",
            "Epoch: [ 0] [  375/  468] time: 262.9009, train_loss: 0.04241060, train_accuracy: 0.9844, test_Accuracy: 0.9589\n",
            "Epoch: [ 0] [  376/  468] time: 263.5697, train_loss: 0.12699732, train_accuracy: 0.9609, test_Accuracy: 0.9596\n",
            "Epoch: [ 0] [  377/  468] time: 264.3219, train_loss: 0.06256867, train_accuracy: 0.9766, test_Accuracy: 0.9597\n",
            "Epoch: [ 0] [  378/  468] time: 265.0619, train_loss: 0.12254509, train_accuracy: 0.9609, test_Accuracy: 0.9600\n",
            "Epoch: [ 0] [  379/  468] time: 265.7337, train_loss: 0.04991883, train_accuracy: 0.9688, test_Accuracy: 0.9604\n",
            "Epoch: [ 0] [  380/  468] time: 266.4776, train_loss: 0.04387522, train_accuracy: 0.9922, test_Accuracy: 0.9605\n",
            "Epoch: [ 0] [  381/  468] time: 267.2137, train_loss: 0.03481524, train_accuracy: 0.9844, test_Accuracy: 0.9607\n",
            "Epoch: [ 0] [  382/  468] time: 267.8363, train_loss: 0.04482479, train_accuracy: 0.9922, test_Accuracy: 0.9607\n",
            "Epoch: [ 0] [  383/  468] time: 268.4982, train_loss: 0.05997343, train_accuracy: 0.9766, test_Accuracy: 0.9608\n",
            "Epoch: [ 0] [  384/  468] time: 269.2411, train_loss: 0.10192887, train_accuracy: 0.9531, test_Accuracy: 0.9611\n",
            "Epoch: [ 0] [  385/  468] time: 269.9026, train_loss: 0.09088077, train_accuracy: 0.9531, test_Accuracy: 0.9597\n",
            "Epoch: [ 0] [  386/  468] time: 270.6431, train_loss: 0.02982936, train_accuracy: 0.9766, test_Accuracy: 0.9599\n",
            "Epoch: [ 0] [  387/  468] time: 271.3078, train_loss: 0.06263188, train_accuracy: 0.9766, test_Accuracy: 0.9603\n",
            "Epoch: [ 0] [  388/  468] time: 271.9823, train_loss: 0.09408250, train_accuracy: 0.9688, test_Accuracy: 0.9590\n",
            "Epoch: [ 0] [  389/  468] time: 272.7268, train_loss: 0.06897146, train_accuracy: 0.9609, test_Accuracy: 0.9587\n",
            "Epoch: [ 0] [  390/  468] time: 273.3902, train_loss: 0.02270490, train_accuracy: 0.9844, test_Accuracy: 0.9594\n",
            "Epoch: [ 0] [  391/  468] time: 274.0556, train_loss: 0.11490811, train_accuracy: 0.9766, test_Accuracy: 0.9605\n",
            "Epoch: [ 0] [  392/  468] time: 274.6852, train_loss: 0.03516009, train_accuracy: 0.9844, test_Accuracy: 0.9614\n",
            "Epoch: [ 0] [  393/  468] time: 275.4233, train_loss: 0.02201876, train_accuracy: 0.9922, test_Accuracy: 0.9620\n",
            "Epoch: [ 0] [  394/  468] time: 276.1016, train_loss: 0.07587326, train_accuracy: 0.9531, test_Accuracy: 0.9636\n",
            "Epoch: [ 0] [  395/  468] time: 276.8443, train_loss: 0.03808227, train_accuracy: 0.9844, test_Accuracy: 0.9649\n",
            "Epoch: [ 0] [  396/  468] time: 277.4712, train_loss: 0.04802373, train_accuracy: 0.9766, test_Accuracy: 0.9664\n",
            "Epoch: [ 0] [  397/  468] time: 278.2254, train_loss: 0.06391805, train_accuracy: 0.9922, test_Accuracy: 0.9679\n",
            "Epoch: [ 0] [  398/  468] time: 278.9289, train_loss: 0.05982482, train_accuracy: 0.9844, test_Accuracy: 0.9683\n",
            "Epoch: [ 0] [  399/  468] time: 279.6691, train_loss: 0.08498625, train_accuracy: 0.9766, test_Accuracy: 0.9683\n",
            "Epoch: [ 0] [  400/  468] time: 280.4068, train_loss: 0.04456414, train_accuracy: 0.9922, test_Accuracy: 0.9678\n",
            "Epoch: [ 0] [  401/  468] time: 281.0782, train_loss: 0.08380320, train_accuracy: 0.9688, test_Accuracy: 0.9681\n",
            "Epoch: [ 0] [  402/  468] time: 281.8204, train_loss: 0.03141372, train_accuracy: 0.9844, test_Accuracy: 0.9680\n",
            "Epoch: [ 0] [  403/  468] time: 282.5637, train_loss: 0.06967758, train_accuracy: 0.9766, test_Accuracy: 0.9677\n",
            "Epoch: [ 0] [  404/  468] time: 283.3090, train_loss: 0.05205380, train_accuracy: 0.9844, test_Accuracy: 0.9677\n",
            "Epoch: [ 0] [  405/  468] time: 283.9724, train_loss: 0.07280837, train_accuracy: 0.9844, test_Accuracy: 0.9685\n",
            "Epoch: [ 0] [  406/  468] time: 284.6452, train_loss: 0.08859706, train_accuracy: 0.9766, test_Accuracy: 0.9684\n",
            "Epoch: [ 0] [  407/  468] time: 285.3932, train_loss: 0.06978225, train_accuracy: 0.9609, test_Accuracy: 0.9678\n",
            "Epoch: [ 0] [  408/  468] time: 286.1353, train_loss: 0.04559850, train_accuracy: 0.9922, test_Accuracy: 0.9678\n",
            "Epoch: [ 0] [  409/  468] time: 286.8095, train_loss: 0.07928218, train_accuracy: 0.9766, test_Accuracy: 0.9677\n",
            "Epoch: [ 0] [  410/  468] time: 287.4682, train_loss: 0.06755982, train_accuracy: 0.9922, test_Accuracy: 0.9664\n",
            "Epoch: [ 0] [  411/  468] time: 288.0840, train_loss: 0.10922308, train_accuracy: 0.9844, test_Accuracy: 0.9661\n",
            "Epoch: [ 0] [  412/  468] time: 288.7177, train_loss: 0.04697005, train_accuracy: 0.9844, test_Accuracy: 0.9663\n",
            "Epoch: [ 0] [  413/  468] time: 289.4676, train_loss: 0.11618210, train_accuracy: 0.9531, test_Accuracy: 0.9677\n",
            "Epoch: [ 0] [  414/  468] time: 290.0928, train_loss: 0.21079770, train_accuracy: 0.9531, test_Accuracy: 0.9690\n",
            "Epoch: [ 0] [  415/  468] time: 290.6975, train_loss: 0.09612483, train_accuracy: 0.9844, test_Accuracy: 0.9702\n",
            "Epoch: [ 0] [  416/  468] time: 291.3129, train_loss: 0.07165047, train_accuracy: 0.9688, test_Accuracy: 0.9701\n",
            "Epoch: [ 0] [  417/  468] time: 292.0511, train_loss: 0.02702759, train_accuracy: 0.9922, test_Accuracy: 0.9701\n",
            "Epoch: [ 0] [  418/  468] time: 292.6823, train_loss: 0.04332881, train_accuracy: 0.9844, test_Accuracy: 0.9698\n",
            "Epoch: [ 0] [  419/  468] time: 293.3050, train_loss: 0.04459889, train_accuracy: 0.9922, test_Accuracy: 0.9709\n",
            "Epoch: [ 0] [  420/  468] time: 293.9228, train_loss: 0.10430688, train_accuracy: 0.9609, test_Accuracy: 0.9705\n",
            "Epoch: [ 0] [  421/  468] time: 294.5247, train_loss: 0.04238994, train_accuracy: 1.0000, test_Accuracy: 0.9702\n",
            "Epoch: [ 0] [  422/  468] time: 295.1968, train_loss: 0.04038174, train_accuracy: 0.9922, test_Accuracy: 0.9691\n",
            "Epoch: [ 0] [  423/  468] time: 295.9402, train_loss: 0.05678328, train_accuracy: 0.9688, test_Accuracy: 0.9685\n",
            "Epoch: [ 0] [  424/  468] time: 296.6812, train_loss: 0.09340195, train_accuracy: 0.9688, test_Accuracy: 0.9684\n",
            "Epoch: [ 0] [  425/  468] time: 297.3688, train_loss: 0.07458950, train_accuracy: 0.9609, test_Accuracy: 0.9685\n",
            "Epoch: [ 0] [  426/  468] time: 298.0486, train_loss: 0.07121639, train_accuracy: 0.9844, test_Accuracy: 0.9690\n",
            "Epoch: [ 0] [  427/  468] time: 298.7940, train_loss: 0.17317916, train_accuracy: 0.9609, test_Accuracy: 0.9693\n",
            "Epoch: [ 0] [  428/  468] time: 299.4575, train_loss: 0.10463291, train_accuracy: 0.9688, test_Accuracy: 0.9695\n",
            "Epoch: [ 0] [  429/  468] time: 300.1313, train_loss: 0.11298224, train_accuracy: 0.9766, test_Accuracy: 0.9698\n",
            "Epoch: [ 0] [  430/  468] time: 300.7878, train_loss: 0.07615146, train_accuracy: 0.9844, test_Accuracy: 0.9712\n",
            "Epoch: [ 0] [  431/  468] time: 301.5299, train_loss: 0.08109213, train_accuracy: 0.9766, test_Accuracy: 0.9719\n",
            "Epoch: [ 0] [  432/  468] time: 302.2756, train_loss: 0.05651399, train_accuracy: 0.9844, test_Accuracy: 0.9712\n",
            "Epoch: [ 0] [  433/  468] time: 302.8968, train_loss: 0.01399978, train_accuracy: 1.0000, test_Accuracy: 0.9716\n",
            "Epoch: [ 0] [  434/  468] time: 303.5176, train_loss: 0.11941283, train_accuracy: 0.9844, test_Accuracy: 0.9715\n",
            "Epoch: [ 0] [  435/  468] time: 304.1432, train_loss: 0.14366055, train_accuracy: 0.9297, test_Accuracy: 0.9706\n",
            "Epoch: [ 0] [  436/  468] time: 304.7606, train_loss: 0.03865509, train_accuracy: 0.9766, test_Accuracy: 0.9666\n",
            "Epoch: [ 0] [  437/  468] time: 305.4026, train_loss: 0.08767806, train_accuracy: 0.9844, test_Accuracy: 0.9644\n",
            "Epoch: [ 0] [  438/  468] time: 306.1412, train_loss: 0.06890166, train_accuracy: 0.9766, test_Accuracy: 0.9630\n",
            "Epoch: [ 0] [  439/  468] time: 306.8058, train_loss: 0.14379269, train_accuracy: 0.9609, test_Accuracy: 0.9644\n",
            "Epoch: [ 0] [  440/  468] time: 307.4846, train_loss: 0.09185573, train_accuracy: 0.9844, test_Accuracy: 0.9658\n",
            "Epoch: [ 0] [  441/  468] time: 308.1632, train_loss: 0.05589677, train_accuracy: 0.9766, test_Accuracy: 0.9675\n",
            "Epoch: [ 0] [  442/  468] time: 308.9062, train_loss: 0.04658825, train_accuracy: 0.9922, test_Accuracy: 0.9669\n",
            "Epoch: [ 0] [  443/  468] time: 309.5830, train_loss: 0.08755887, train_accuracy: 0.9688, test_Accuracy: 0.9647\n",
            "Epoch: [ 0] [  444/  468] time: 310.3436, train_loss: 0.08248455, train_accuracy: 0.9766, test_Accuracy: 0.9651\n",
            "Epoch: [ 0] [  445/  468] time: 311.0102, train_loss: 0.11273047, train_accuracy: 0.9609, test_Accuracy: 0.9668\n",
            "Epoch: [ 0] [  446/  468] time: 311.6905, train_loss: 0.09711893, train_accuracy: 0.9766, test_Accuracy: 0.9673\n",
            "Epoch: [ 0] [  447/  468] time: 312.4409, train_loss: 0.06055810, train_accuracy: 0.9766, test_Accuracy: 0.9678\n",
            "Epoch: [ 0] [  448/  468] time: 313.1028, train_loss: 0.03874324, train_accuracy: 0.9844, test_Accuracy: 0.9673\n",
            "Epoch: [ 0] [  449/  468] time: 313.7762, train_loss: 0.10250041, train_accuracy: 0.9609, test_Accuracy: 0.9680\n",
            "Epoch: [ 0] [  450/  468] time: 314.4163, train_loss: 0.04721479, train_accuracy: 0.9766, test_Accuracy: 0.9666\n",
            "Epoch: [ 0] [  451/  468] time: 315.0521, train_loss: 0.06668361, train_accuracy: 0.9922, test_Accuracy: 0.9660\n",
            "Epoch: [ 0] [  452/  468] time: 315.7201, train_loss: 0.07184092, train_accuracy: 0.9766, test_Accuracy: 0.9657\n",
            "Epoch: [ 0] [  453/  468] time: 316.4632, train_loss: 0.03705634, train_accuracy: 0.9844, test_Accuracy: 0.9651\n",
            "Epoch: [ 0] [  454/  468] time: 317.2088, train_loss: 0.05322885, train_accuracy: 0.9688, test_Accuracy: 0.9643\n",
            "Epoch: [ 0] [  455/  468] time: 317.8782, train_loss: 0.04211002, train_accuracy: 0.9688, test_Accuracy: 0.9625\n",
            "Epoch: [ 0] [  456/  468] time: 318.6259, train_loss: 0.04623078, train_accuracy: 0.9531, test_Accuracy: 0.9609\n",
            "Epoch: [ 0] [  457/  468] time: 319.2989, train_loss: 0.12520319, train_accuracy: 0.9609, test_Accuracy: 0.9606\n",
            "Epoch: [ 0] [  458/  468] time: 319.9177, train_loss: 0.07784185, train_accuracy: 0.9688, test_Accuracy: 0.9606\n",
            "Epoch: [ 0] [  459/  468] time: 320.6612, train_loss: 0.09831143, train_accuracy: 0.9375, test_Accuracy: 0.9621\n",
            "Epoch: [ 0] [  460/  468] time: 321.3304, train_loss: 0.05781651, train_accuracy: 0.9688, test_Accuracy: 0.9630\n",
            "Epoch: [ 0] [  461/  468] time: 322.0785, train_loss: 0.03622368, train_accuracy: 0.9844, test_Accuracy: 0.9653\n",
            "Epoch: [ 0] [  462/  468] time: 322.7549, train_loss: 0.07595763, train_accuracy: 0.9844, test_Accuracy: 0.9664\n",
            "Epoch: [ 0] [  463/  468] time: 323.4354, train_loss: 0.07228923, train_accuracy: 0.9609, test_Accuracy: 0.9685\n",
            "Epoch: [ 0] [  464/  468] time: 324.0998, train_loss: 0.07135811, train_accuracy: 0.9844, test_Accuracy: 0.9693\n",
            "Epoch: [ 0] [  465/  468] time: 324.7674, train_loss: 0.07425825, train_accuracy: 0.9766, test_Accuracy: 0.9695\n",
            "Epoch: [ 0] [  466/  468] time: 325.5189, train_loss: 0.06411918, train_accuracy: 0.9922, test_Accuracy: 0.9704\n",
            "Epoch: [ 0] [  467/  468] time: 326.2021, train_loss: 0.05176440, train_accuracy: 0.9766, test_Accuracy: 0.9700\n"
          ]
        }
      ]
    }
  ]
}