Lab10/Lab10_2_2_LEEJOOYOUNG.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Lab10-2-2.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hI_kE9BF6srB",
        "outputId": "ca9e4200-1830-45b0-ac0f-97a762f093f8"
      },
      "source": [
        "import tensorflow as tf\n",
        "import numpy as np\n",
        "from tensorflow.keras.utils import to_categorical\n",
        "from tensorflow.keras.datasets import mnist\n",
        "from time import time\n",
        "import os\n",
        "print(tf.__version__)"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2.6.0\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FI5gDm1O7RHj"
      },
      "source": [
        "def load(model, checkpoint_dir):\n",
        "    print(\" [*] Reading checkpoints...\")\n",
        "\n",
        "    ckpt = tf.train.get_checkpoint_state(checkpoint_dir)\n",
        "    if ckpt :\n",
        "        ckpt_name = os.path.basename(ckpt.model_checkpoint_path)\n",
        "        checkpoint = tf.train.Checkpoint(dnn=model)\n",
        "        checkpoint.restore(save_path=os.path.join(checkpoint_dir, ckpt_name))\n",
        "        counter = int(ckpt_name.split('-')[1])\n",
        "        print(\" [*] Success to read {}\".format(ckpt_name))\n",
        "        return True, counter\n",
        "    else:\n",
        "        print(\" [*] Failed to find a checkpoint\")\n",
        "        return False, 0\n",
        "\n",
        "def check_folder(dir):\n",
        "    if not os.path.exists(dir):\n",
        "        os.makedirs(dir)\n",
        "    return dir"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UwHbbFMO7Skz"
      },
      "source": [
        "def load_mnist() :\n",
        "    (train_data, train_labels), (test_data, test_labels) = mnist.load_data()\n",
        "    train_data = np.expand_dims(train_data, axis=-1) # [N, 28, 28] -> [N, 28, 28, 1]\n",
        "    test_data = np.expand_dims(test_data, axis=-1) # [N, 28, 28] -> [N, 28, 28, 1]\n",
        "\n",
        "    train_data, test_data = normalize(train_data, test_data)\n",
        "\n",
        "    train_labels = to_categorical(train_labels, 10) # [N,] -> [N, 10]\n",
        "    test_labels = to_categorical(test_labels, 10) # [N,] -> [N, 10]\n",
        "\n",
        "    return train_data, train_labels, test_data, test_labels\n",
        "\n",
        "def normalize(train_data, test_data):\n",
        "    train_data = train_data.astype(np.float32) / 255.0\n",
        "    test_data = test_data.astype(np.float32) / 255.0\n",
        "\n",
        "    return train_data, test_data"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MiMcC78g7Ynu"
      },
      "source": [
        "def loss_fn(model, images, labels):\n",
        "    logits = model(images, training=True)\n",
        "    loss = tf.reduce_mean(tf.keras.losses.categorical_crossentropy(y_pred=logits, y_true=labels, \n",
        "                                                                   from_logits=True))\n",
        "    return loss\n",
        "\n",
        "def accuracy_fn(model, images, labels):\n",
        "    logits = model(images, training=False)\n",
        "    prediction = tf.equal(tf.argmax(logits, -1), tf.argmax(labels, -1))\n",
        "    accuracy = tf.reduce_mean(tf.cast(prediction, tf.float32))\n",
        "    return accuracy\n",
        "\n",
        "def grad(model, images, labels):\n",
        "    with tf.GradientTape() as tape:\n",
        "        loss = loss_fn(model, images, labels)\n",
        "    return tape.gradient(loss, model.variables)"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oaF4gRoN7brF"
      },
      "source": [
        "def flatten() :\n",
        "    return tf.keras.layers.Flatten()\n",
        "\n",
        "def dense(label_dim, weight_init) :\n",
        "    return tf.keras.layers.Dense(units=label_dim, use_bias=True, kernel_initializer=weight_init)\n",
        "\n",
        "def relu() :\n",
        "    return tf.keras.layers.Activation(tf.keras.activations.relu)"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "g0WohR7x7dGH"
      },
      "source": [
        "class create_model_class(tf.keras.Model):\n",
        "    def __init__(self, label_dim):\n",
        "        super(create_model_class, self).__init__()\n",
        "        weight_init = tf.keras.initializers.RandomNormal()\n",
        "\n",
        "        self.model = tf.keras.Sequential()\n",
        "        self.model.add(flatten())\n",
        "\n",
        "        for i in range(2):\n",
        "            self.model.add(dense(256, weight_init))\n",
        "            self.model.add(relu())\n",
        "\n",
        "        self.model.add(dense(label_dim, weight_init))\n",
        "\n",
        "    def call(self, x, training=None, mask=None):\n",
        "        x = self.model(x)\n",
        "\n",
        "        return x"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5ZIGz4F97lgB"
      },
      "source": [
        "def create_model_function(label_dim) :\n",
        "    weight_init = tf.keras.initializers.RandomNormal()\n",
        "\n",
        "    model = tf.keras.Sequential()\n",
        "    model.add(flatten())\n",
        "\n",
        "    for i in range(2) :\n",
        "        model.add(dense(256, weight_init))\n",
        "        model.add(relu())\n",
        "\n",
        "    model.add(dense(label_dim, weight_init))\n",
        "\n",
        "    return model"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6CdCx8zt7n-c",
        "outputId": "4c669f7c-5a71-4d2f-8fe8-db250944ee66"
      },
      "source": [
        "\"\"\" dataset \"\"\"\n",
        "train_x, train_y, test_x, test_y = load_mnist()\n",
        "\n",
        "\"\"\" parameters \"\"\"\n",
        "learning_rate = 0.001\n",
        "batch_size = 128\n",
        "\n",
        "training_epochs = 1\n",
        "training_iterations = len(train_x) // batch_size\n",
        "\n",
        "label_dim = 10\n",
        "\n",
        "train_flag = True\n",
        "\n",
        "\"\"\" Graph Input using Dataset API \"\"\"\n",
        "train_dataset = tf.data.Dataset.from_tensor_slices((train_x, train_y)).\\\n",
        "    shuffle(buffer_size=100000).\\\n",
        "    prefetch(buffer_size=batch_size).\\\n",
        "    batch(batch_size, drop_remainder=True)\n",
        "\n",
        "test_dataset = tf.data.Dataset.from_tensor_slices((test_x, test_y)).\\\n",
        "    shuffle(buffer_size=100000).\\\n",
        "    prefetch(buffer_size=len(test_x)).\\\n",
        "    batch(len(test_x))"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/mnist.npz\n",
            "11493376/11490434 [==============================] - 0s 0us/step\n",
            "11501568/11490434 [==============================] - 0s 0us/step\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FZpnzMn_7v7U"
      },
      "source": [
        "\"\"\" Model \"\"\"\n",
        "network = create_model_function(label_dim)\n",
        "\n",
        "\"\"\" Training \"\"\"\n",
        "optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate)\n",
        "\n",
        "\"\"\" Writer \"\"\"\n",
        "checkpoint_dir = 'checkpoints'\n",
        "logs_dir = 'logs'\n",
        "\n",
        "model_dir = 'nn_relu'\n",
        "\n",
        "checkpoint_dir = os.path.join(checkpoint_dir, model_dir)\n",
        "check_folder(checkpoint_dir)\n",
        "checkpoint_prefix = os.path.join(checkpoint_dir, model_dir)\n",
        "logs_dir = os.path.join(logs_dir, model_dir)"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "AtSdr-lJ7yqe",
        "outputId": "8a67782c-6810-4fff-df9f-2fa103b39c8d"
      },
      "source": [
        "if train_flag :\n",
        "\n",
        "    checkpoint = tf.train.Checkpoint(dnn=network)\n",
        "\n",
        "    # create writer for tensorboard\n",
        "    summary_writer = tf.summary.create_file_writer(logdir=logs_dir)\n",
        "    start_time = time()\n",
        "\n",
        "    # restore check-point if it exits\n",
        "    could_load, checkpoint_counter = load(network, checkpoint_dir)    \n",
        "\n",
        "    if could_load:\n",
        "        start_epoch = (int)(checkpoint_counter / training_iterations)        \n",
        "        counter = checkpoint_counter        \n",
        "        print(\" [*] Load SUCCESS\")\n",
        "    else:\n",
        "        start_epoch = 0\n",
        "        start_iteration = 0\n",
        "        counter = 0\n",
        "        print(\" [!] Load failed...\")\n",
        "    \n",
        "    # train phase\n",
        "    with summary_writer.as_default():  # for tensorboard\n",
        "        for epoch in range(start_epoch, training_epochs):\n",
        "            for idx, (train_input, train_label) in enumerate(train_dataset):                \n",
        "                grads = grad(network, train_input, train_label)\n",
        "                optimizer.apply_gradients(grads_and_vars=zip(grads, network.variables))\n",
        "\n",
        "                train_loss = loss_fn(network, train_input, train_label)\n",
        "                train_accuracy = accuracy_fn(network, train_input, train_label)\n",
        "\n",
        "                for test_input, test_label in test_dataset:                \n",
        "                    test_accuracy = accuracy_fn(network, test_input, test_label)\n",
        "\n",
        "                tf.summary.scalar(name='train_loss', data=train_loss, step=counter)\n",
        "                tf.summary.scalar(name='train_accuracy', data=train_accuracy, step=counter)\n",
        "                tf.summary.scalar(name='test_accuracy', data=test_accuracy, step=counter)\n",
        "\n",
        "                print(\n",
        "                    \"Epoch: [%2d] [%5d/%5d] time: %4.4f, train_loss: %.8f, train_accuracy: %.4f, test_Accuracy: %.4f\" \\\n",
        "                    % (epoch, idx, training_iterations, time() - start_time, train_loss, train_accuracy,\n",
        "                       test_accuracy))\n",
        "                counter += 1\n",
        "        checkpoint.save(file_prefix=checkpoint_prefix + '-{}'.format(counter))\n",
        "        \n",
        "# test phase      \n",
        "else :\n",
        "    _, _ = load(network, checkpoint_dir)\n",
        "    for test_input, test_label in test_dataset:    \n",
        "        test_accuracy = accuracy_fn(network, test_input, test_label)\n",
        "\n",
        "    print(\"test_Accuracy: %.4f\" % (test_accuracy))"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            " [*] Reading checkpoints...\n",
            " [*] Failed to find a checkpoint\n",
            " [!] Load failed...\n",
            "Epoch: [ 0] [    0/  468] time: 1.0373, train_loss: 2.18119764, train_accuracy: 0.4297, test_Accuracy: 0.2648\n",
            "Epoch: [ 0] [    1/  468] time: 1.2379, train_loss: 2.13245988, train_accuracy: 0.5547, test_Accuracy: 0.4362\n",
            "Epoch: [ 0] [    2/  468] time: 1.4406, train_loss: 2.08957648, train_accuracy: 0.6016, test_Accuracy: 0.5442\n",
            "Epoch: [ 0] [    3/  468] time: 1.6373, train_loss: 1.99649334, train_accuracy: 0.6406, test_Accuracy: 0.6036\n",
            "Epoch: [ 0] [    4/  468] time: 1.8338, train_loss: 1.95566130, train_accuracy: 0.6875, test_Accuracy: 0.6305\n",
            "Epoch: [ 0] [    5/  468] time: 2.1888, train_loss: 1.82035983, train_accuracy: 0.6875, test_Accuracy: 0.6456\n",
            "Epoch: [ 0] [    6/  468] time: 2.5379, train_loss: 1.75892782, train_accuracy: 0.6719, test_Accuracy: 0.6642\n",
            "Epoch: [ 0] [    7/  468] time: 2.8890, train_loss: 1.73388934, train_accuracy: 0.6094, test_Accuracy: 0.6926\n",
            "Epoch: [ 0] [    8/  468] time: 3.2589, train_loss: 1.57453799, train_accuracy: 0.7031, test_Accuracy: 0.7298\n",
            "Epoch: [ 0] [    9/  468] time: 3.4653, train_loss: 1.46861148, train_accuracy: 0.7969, test_Accuracy: 0.7625\n",
            "Epoch: [ 0] [   10/  468] time: 3.6704, train_loss: 1.22606826, train_accuracy: 0.8438, test_Accuracy: 0.7869\n",
            "Epoch: [ 0] [   11/  468] time: 3.8624, train_loss: 1.20015001, train_accuracy: 0.7734, test_Accuracy: 0.8055\n",
            "Epoch: [ 0] [   12/  468] time: 4.0614, train_loss: 1.13884544, train_accuracy: 0.7656, test_Accuracy: 0.8100\n",
            "Epoch: [ 0] [   13/  468] time: 4.2558, train_loss: 0.95027137, train_accuracy: 0.8359, test_Accuracy: 0.8125\n",
            "Epoch: [ 0] [   14/  468] time: 4.4649, train_loss: 0.91892755, train_accuracy: 0.8047, test_Accuracy: 0.8120\n",
            "Epoch: [ 0] [   15/  468] time: 4.6636, train_loss: 0.88092279, train_accuracy: 0.7734, test_Accuracy: 0.8037\n",
            "Epoch: [ 0] [   16/  468] time: 5.0220, train_loss: 0.73643506, train_accuracy: 0.8203, test_Accuracy: 0.7916\n",
            "Epoch: [ 0] [   17/  468] time: 5.2196, train_loss: 0.70280373, train_accuracy: 0.8516, test_Accuracy: 0.8043\n",
            "Epoch: [ 0] [   18/  468] time: 5.5766, train_loss: 0.65763962, train_accuracy: 0.8438, test_Accuracy: 0.8268\n",
            "Epoch: [ 0] [   19/  468] time: 5.7743, train_loss: 0.59763277, train_accuracy: 0.8594, test_Accuracy: 0.8398\n",
            "Epoch: [ 0] [   20/  468] time: 5.9721, train_loss: 0.63314235, train_accuracy: 0.8672, test_Accuracy: 0.8403\n",
            "Epoch: [ 0] [   21/  468] time: 6.1668, train_loss: 0.63858110, train_accuracy: 0.8047, test_Accuracy: 0.8417\n",
            "Epoch: [ 0] [   22/  468] time: 6.3669, train_loss: 0.61476487, train_accuracy: 0.8594, test_Accuracy: 0.8445\n",
            "Epoch: [ 0] [   23/  468] time: 6.7224, train_loss: 0.49531505, train_accuracy: 0.8359, test_Accuracy: 0.8527\n",
            "Epoch: [ 0] [   24/  468] time: 6.9217, train_loss: 0.55559474, train_accuracy: 0.8047, test_Accuracy: 0.8567\n",
            "Epoch: [ 0] [   25/  468] time: 7.1260, train_loss: 0.50493181, train_accuracy: 0.8516, test_Accuracy: 0.8548\n",
            "Epoch: [ 0] [   26/  468] time: 7.3189, train_loss: 0.48762763, train_accuracy: 0.8438, test_Accuracy: 0.8499\n",
            "Epoch: [ 0] [   27/  468] time: 7.5322, train_loss: 0.40270734, train_accuracy: 0.8516, test_Accuracy: 0.8507\n",
            "Epoch: [ 0] [   28/  468] time: 7.8864, train_loss: 0.47446096, train_accuracy: 0.8438, test_Accuracy: 0.8619\n",
            "Epoch: [ 0] [   29/  468] time: 8.0870, train_loss: 0.39766270, train_accuracy: 0.9062, test_Accuracy: 0.8598\n",
            "Epoch: [ 0] [   30/  468] time: 8.2869, train_loss: 0.42599982, train_accuracy: 0.8672, test_Accuracy: 0.8612\n",
            "Epoch: [ 0] [   31/  468] time: 8.5042, train_loss: 0.38517013, train_accuracy: 0.8828, test_Accuracy: 0.8555\n",
            "Epoch: [ 0] [   32/  468] time: 8.7110, train_loss: 0.50273108, train_accuracy: 0.8594, test_Accuracy: 0.8604\n",
            "Epoch: [ 0] [   33/  468] time: 8.9070, train_loss: 0.42108229, train_accuracy: 0.8594, test_Accuracy: 0.8670\n",
            "Epoch: [ 0] [   34/  468] time: 9.1166, train_loss: 0.51350826, train_accuracy: 0.8594, test_Accuracy: 0.8768\n",
            "Epoch: [ 0] [   35/  468] time: 9.3124, train_loss: 0.37920988, train_accuracy: 0.8828, test_Accuracy: 0.8767\n",
            "Epoch: [ 0] [   36/  468] time: 9.5189, train_loss: 0.38456279, train_accuracy: 0.8828, test_Accuracy: 0.8823\n",
            "Epoch: [ 0] [   37/  468] time: 9.8728, train_loss: 0.40504390, train_accuracy: 0.8828, test_Accuracy: 0.8843\n",
            "Epoch: [ 0] [   38/  468] time: 10.0718, train_loss: 0.56770694, train_accuracy: 0.8125, test_Accuracy: 0.8831\n",
            "Epoch: [ 0] [   39/  468] time: 10.2774, train_loss: 0.25702941, train_accuracy: 0.9297, test_Accuracy: 0.8799\n",
            "Epoch: [ 0] [   40/  468] time: 10.4741, train_loss: 0.40628111, train_accuracy: 0.8828, test_Accuracy: 0.8767\n",
            "Epoch: [ 0] [   41/  468] time: 10.6834, train_loss: 0.50711447, train_accuracy: 0.8516, test_Accuracy: 0.8745\n",
            "Epoch: [ 0] [   42/  468] time: 10.8797, train_loss: 0.30885956, train_accuracy: 0.8984, test_Accuracy: 0.8775\n",
            "Epoch: [ 0] [   43/  468] time: 11.0796, train_loss: 0.32910019, train_accuracy: 0.8906, test_Accuracy: 0.8897\n",
            "Epoch: [ 0] [   44/  468] time: 11.2787, train_loss: 0.50192398, train_accuracy: 0.8672, test_Accuracy: 0.8949\n",
            "Epoch: [ 0] [   45/  468] time: 11.6390, train_loss: 0.33514971, train_accuracy: 0.8672, test_Accuracy: 0.8919\n",
            "Epoch: [ 0] [   46/  468] time: 11.8389, train_loss: 0.22055987, train_accuracy: 0.9219, test_Accuracy: 0.8877\n",
            "Epoch: [ 0] [   47/  468] time: 12.0413, train_loss: 0.35507873, train_accuracy: 0.9062, test_Accuracy: 0.8808\n",
            "Epoch: [ 0] [   48/  468] time: 12.3902, train_loss: 0.31757385, train_accuracy: 0.8828, test_Accuracy: 0.8879\n",
            "Epoch: [ 0] [   49/  468] time: 12.7457, train_loss: 0.29829630, train_accuracy: 0.9219, test_Accuracy: 0.8940\n",
            "Epoch: [ 0] [   50/  468] time: 12.9485, train_loss: 0.38829744, train_accuracy: 0.8750, test_Accuracy: 0.8966\n",
            "Epoch: [ 0] [   51/  468] time: 13.1516, train_loss: 0.38624185, train_accuracy: 0.8984, test_Accuracy: 0.8966\n",
            "Epoch: [ 0] [   52/  468] time: 13.3536, train_loss: 0.36001620, train_accuracy: 0.9062, test_Accuracy: 0.8911\n",
            "Epoch: [ 0] [   53/  468] time: 13.7036, train_loss: 0.46365523, train_accuracy: 0.8438, test_Accuracy: 0.8931\n",
            "Epoch: [ 0] [   54/  468] time: 13.9035, train_loss: 0.35394007, train_accuracy: 0.9141, test_Accuracy: 0.8997\n",
            "Epoch: [ 0] [   55/  468] time: 14.2559, train_loss: 0.41869935, train_accuracy: 0.8828, test_Accuracy: 0.8983\n",
            "Epoch: [ 0] [   56/  468] time: 14.4615, train_loss: 0.35473567, train_accuracy: 0.8750, test_Accuracy: 0.8979\n",
            "Epoch: [ 0] [   57/  468] time: 14.6756, train_loss: 0.17508811, train_accuracy: 0.9609, test_Accuracy: 0.8957\n",
            "Epoch: [ 0] [   58/  468] time: 14.8760, train_loss: 0.38545191, train_accuracy: 0.8906, test_Accuracy: 0.8917\n",
            "Epoch: [ 0] [   59/  468] time: 15.0822, train_loss: 0.38542977, train_accuracy: 0.8828, test_Accuracy: 0.8911\n",
            "Epoch: [ 0] [   60/  468] time: 15.2782, train_loss: 0.39730793, train_accuracy: 0.8906, test_Accuracy: 0.8930\n",
            "Epoch: [ 0] [   61/  468] time: 15.4781, train_loss: 0.35219148, train_accuracy: 0.9141, test_Accuracy: 0.8983\n",
            "Epoch: [ 0] [   62/  468] time: 15.8338, train_loss: 0.37628439, train_accuracy: 0.8906, test_Accuracy: 0.9001\n",
            "Epoch: [ 0] [   63/  468] time: 16.0318, train_loss: 0.39483038, train_accuracy: 0.8828, test_Accuracy: 0.9018\n",
            "Epoch: [ 0] [   64/  468] time: 16.2369, train_loss: 0.34175384, train_accuracy: 0.9062, test_Accuracy: 0.9013\n",
            "Epoch: [ 0] [   65/  468] time: 16.4461, train_loss: 0.29357094, train_accuracy: 0.9375, test_Accuracy: 0.9010\n",
            "Epoch: [ 0] [   66/  468] time: 16.8002, train_loss: 0.44358554, train_accuracy: 0.8984, test_Accuracy: 0.8984\n",
            "Epoch: [ 0] [   67/  468] time: 17.0166, train_loss: 0.39497709, train_accuracy: 0.8594, test_Accuracy: 0.8996\n",
            "Epoch: [ 0] [   68/  468] time: 17.3662, train_loss: 0.37217087, train_accuracy: 0.9141, test_Accuracy: 0.9024\n",
            "Epoch: [ 0] [   69/  468] time: 17.5634, train_loss: 0.30009678, train_accuracy: 0.9219, test_Accuracy: 0.9033\n",
            "Epoch: [ 0] [   70/  468] time: 17.7747, train_loss: 0.32521510, train_accuracy: 0.8906, test_Accuracy: 0.9038\n",
            "Epoch: [ 0] [   71/  468] time: 17.9848, train_loss: 0.19710393, train_accuracy: 0.9531, test_Accuracy: 0.9024\n",
            "Epoch: [ 0] [   72/  468] time: 18.1958, train_loss: 0.27426538, train_accuracy: 0.9453, test_Accuracy: 0.8998\n",
            "Epoch: [ 0] [   73/  468] time: 18.5522, train_loss: 0.28120008, train_accuracy: 0.9219, test_Accuracy: 0.8991\n",
            "Epoch: [ 0] [   74/  468] time: 18.9111, train_loss: 0.31633410, train_accuracy: 0.9141, test_Accuracy: 0.9009\n",
            "Epoch: [ 0] [   75/  468] time: 19.1156, train_loss: 0.42591083, train_accuracy: 0.8984, test_Accuracy: 0.9058\n",
            "Epoch: [ 0] [   76/  468] time: 19.3113, train_loss: 0.34514648, train_accuracy: 0.8906, test_Accuracy: 0.9088\n",
            "Epoch: [ 0] [   77/  468] time: 19.5125, train_loss: 0.29860735, train_accuracy: 0.9375, test_Accuracy: 0.9074\n",
            "Epoch: [ 0] [   78/  468] time: 19.7245, train_loss: 0.37387145, train_accuracy: 0.8906, test_Accuracy: 0.9044\n",
            "Epoch: [ 0] [   79/  468] time: 19.9349, train_loss: 0.40007162, train_accuracy: 0.8906, test_Accuracy: 0.9032\n",
            "Epoch: [ 0] [   80/  468] time: 20.1325, train_loss: 0.24494953, train_accuracy: 0.9375, test_Accuracy: 0.9048\n",
            "Epoch: [ 0] [   81/  468] time: 20.3363, train_loss: 0.20904434, train_accuracy: 0.9219, test_Accuracy: 0.9063\n",
            "Epoch: [ 0] [   82/  468] time: 20.6939, train_loss: 0.31368160, train_accuracy: 0.8906, test_Accuracy: 0.9043\n",
            "Epoch: [ 0] [   83/  468] time: 21.0426, train_loss: 0.19062144, train_accuracy: 0.9688, test_Accuracy: 0.9065\n",
            "Epoch: [ 0] [   84/  468] time: 21.2396, train_loss: 0.23348728, train_accuracy: 0.9219, test_Accuracy: 0.9088\n",
            "Epoch: [ 0] [   85/  468] time: 21.4392, train_loss: 0.35680294, train_accuracy: 0.8828, test_Accuracy: 0.9116\n",
            "Epoch: [ 0] [   86/  468] time: 21.6392, train_loss: 0.26703334, train_accuracy: 0.9141, test_Accuracy: 0.9131\n",
            "Epoch: [ 0] [   87/  468] time: 21.9973, train_loss: 0.20309898, train_accuracy: 0.9453, test_Accuracy: 0.9105\n",
            "Epoch: [ 0] [   88/  468] time: 22.1965, train_loss: 0.28473073, train_accuracy: 0.9219, test_Accuracy: 0.9072\n",
            "Epoch: [ 0] [   89/  468] time: 22.3983, train_loss: 0.38099748, train_accuracy: 0.8750, test_Accuracy: 0.9045\n",
            "Epoch: [ 0] [   90/  468] time: 22.6061, train_loss: 0.26220554, train_accuracy: 0.9141, test_Accuracy: 0.9043\n",
            "Epoch: [ 0] [   91/  468] time: 22.8183, train_loss: 0.31206208, train_accuracy: 0.9062, test_Accuracy: 0.9081\n",
            "Epoch: [ 0] [   92/  468] time: 23.0111, train_loss: 0.34299055, train_accuracy: 0.8594, test_Accuracy: 0.9106\n",
            "Epoch: [ 0] [   93/  468] time: 23.2113, train_loss: 0.32460994, train_accuracy: 0.8906, test_Accuracy: 0.9070\n",
            "Epoch: [ 0] [   94/  468] time: 23.5619, train_loss: 0.25094497, train_accuracy: 0.9219, test_Accuracy: 0.9031\n",
            "Epoch: [ 0] [   95/  468] time: 23.7822, train_loss: 0.21214235, train_accuracy: 0.9453, test_Accuracy: 0.9035\n",
            "Epoch: [ 0] [   96/  468] time: 23.9775, train_loss: 0.45253503, train_accuracy: 0.8906, test_Accuracy: 0.9058\n",
            "Epoch: [ 0] [   97/  468] time: 24.1767, train_loss: 0.27972996, train_accuracy: 0.9062, test_Accuracy: 0.9118\n",
            "Epoch: [ 0] [   98/  468] time: 24.3721, train_loss: 0.38095760, train_accuracy: 0.8906, test_Accuracy: 0.9142\n",
            "Epoch: [ 0] [   99/  468] time: 24.5684, train_loss: 0.36082995, train_accuracy: 0.9375, test_Accuracy: 0.9121\n",
            "Epoch: [ 0] [  100/  468] time: 24.7851, train_loss: 0.27222228, train_accuracy: 0.9062, test_Accuracy: 0.9096\n",
            "Epoch: [ 0] [  101/  468] time: 24.9886, train_loss: 0.36114064, train_accuracy: 0.8594, test_Accuracy: 0.9120\n",
            "Epoch: [ 0] [  102/  468] time: 25.1823, train_loss: 0.34436989, train_accuracy: 0.9062, test_Accuracy: 0.9195\n",
            "Epoch: [ 0] [  103/  468] time: 25.3843, train_loss: 0.25268173, train_accuracy: 0.9297, test_Accuracy: 0.9223\n",
            "Epoch: [ 0] [  104/  468] time: 25.5842, train_loss: 0.17364514, train_accuracy: 0.9453, test_Accuracy: 0.9216\n",
            "Epoch: [ 0] [  105/  468] time: 25.7963, train_loss: 0.18922246, train_accuracy: 0.9453, test_Accuracy: 0.9200\n",
            "Epoch: [ 0] [  106/  468] time: 26.1477, train_loss: 0.30535901, train_accuracy: 0.9453, test_Accuracy: 0.9165\n",
            "Epoch: [ 0] [  107/  468] time: 26.3439, train_loss: 0.29319394, train_accuracy: 0.9297, test_Accuracy: 0.9143\n",
            "Epoch: [ 0] [  108/  468] time: 26.5460, train_loss: 0.34177184, train_accuracy: 0.9141, test_Accuracy: 0.9126\n",
            "Epoch: [ 0] [  109/  468] time: 26.9049, train_loss: 0.17041962, train_accuracy: 0.9531, test_Accuracy: 0.9139\n",
            "Epoch: [ 0] [  110/  468] time: 27.0955, train_loss: 0.33787793, train_accuracy: 0.8984, test_Accuracy: 0.9181\n",
            "Epoch: [ 0] [  111/  468] time: 27.3014, train_loss: 0.36079901, train_accuracy: 0.8828, test_Accuracy: 0.9202\n",
            "Epoch: [ 0] [  112/  468] time: 27.5011, train_loss: 0.24954019, train_accuracy: 0.9062, test_Accuracy: 0.9227\n",
            "Epoch: [ 0] [  113/  468] time: 27.7138, train_loss: 0.19893843, train_accuracy: 0.9453, test_Accuracy: 0.9227\n",
            "Epoch: [ 0] [  114/  468] time: 27.9181, train_loss: 0.36800978, train_accuracy: 0.8828, test_Accuracy: 0.9217\n",
            "Epoch: [ 0] [  115/  468] time: 28.1186, train_loss: 0.23524213, train_accuracy: 0.9219, test_Accuracy: 0.9214\n",
            "Epoch: [ 0] [  116/  468] time: 28.3186, train_loss: 0.22228819, train_accuracy: 0.9531, test_Accuracy: 0.9201\n",
            "Epoch: [ 0] [  117/  468] time: 28.6669, train_loss: 0.30105987, train_accuracy: 0.9141, test_Accuracy: 0.9214\n",
            "Epoch: [ 0] [  118/  468] time: 28.8667, train_loss: 0.27024725, train_accuracy: 0.8906, test_Accuracy: 0.9233\n",
            "Epoch: [ 0] [  119/  468] time: 29.0730, train_loss: 0.23572710, train_accuracy: 0.9375, test_Accuracy: 0.9259\n",
            "Epoch: [ 0] [  120/  468] time: 29.4294, train_loss: 0.16648906, train_accuracy: 0.9609, test_Accuracy: 0.9284\n",
            "Epoch: [ 0] [  121/  468] time: 29.7871, train_loss: 0.27161065, train_accuracy: 0.9062, test_Accuracy: 0.9289\n",
            "Epoch: [ 0] [  122/  468] time: 29.9877, train_loss: 0.20715918, train_accuracy: 0.9453, test_Accuracy: 0.9267\n",
            "Epoch: [ 0] [  123/  468] time: 30.1895, train_loss: 0.21742308, train_accuracy: 0.9375, test_Accuracy: 0.9237\n",
            "Epoch: [ 0] [  124/  468] time: 30.3911, train_loss: 0.26780346, train_accuracy: 0.8984, test_Accuracy: 0.9259\n",
            "Epoch: [ 0] [  125/  468] time: 30.5838, train_loss: 0.23385067, train_accuracy: 0.9531, test_Accuracy: 0.9277\n",
            "Epoch: [ 0] [  126/  468] time: 30.7829, train_loss: 0.23092154, train_accuracy: 0.9141, test_Accuracy: 0.9283\n",
            "Epoch: [ 0] [  127/  468] time: 30.9937, train_loss: 0.22935039, train_accuracy: 0.9375, test_Accuracy: 0.9280\n",
            "Epoch: [ 0] [  128/  468] time: 31.1877, train_loss: 0.10015763, train_accuracy: 0.9766, test_Accuracy: 0.9263\n",
            "Epoch: [ 0] [  129/  468] time: 31.3959, train_loss: 0.16321355, train_accuracy: 0.9609, test_Accuracy: 0.9252\n",
            "Epoch: [ 0] [  130/  468] time: 31.7469, train_loss: 0.24263622, train_accuracy: 0.9375, test_Accuracy: 0.9269\n",
            "Epoch: [ 0] [  131/  468] time: 31.9526, train_loss: 0.20028403, train_accuracy: 0.9375, test_Accuracy: 0.9281\n",
            "Epoch: [ 0] [  132/  468] time: 32.1494, train_loss: 0.35915574, train_accuracy: 0.9062, test_Accuracy: 0.9292\n",
            "Epoch: [ 0] [  133/  468] time: 32.3524, train_loss: 0.24777068, train_accuracy: 0.9297, test_Accuracy: 0.9314\n",
            "Epoch: [ 0] [  134/  468] time: 32.7040, train_loss: 0.26905483, train_accuracy: 0.9141, test_Accuracy: 0.9310\n",
            "Epoch: [ 0] [  135/  468] time: 32.9138, train_loss: 0.28067160, train_accuracy: 0.9297, test_Accuracy: 0.9295\n",
            "Epoch: [ 0] [  136/  468] time: 33.1144, train_loss: 0.26356980, train_accuracy: 0.9297, test_Accuracy: 0.9281\n",
            "Epoch: [ 0] [  137/  468] time: 33.3124, train_loss: 0.26558238, train_accuracy: 0.9297, test_Accuracy: 0.9283\n",
            "Epoch: [ 0] [  138/  468] time: 33.5206, train_loss: 0.25801802, train_accuracy: 0.9141, test_Accuracy: 0.9295\n",
            "Epoch: [ 0] [  139/  468] time: 33.7145, train_loss: 0.33857739, train_accuracy: 0.9141, test_Accuracy: 0.9289\n",
            "Epoch: [ 0] [  140/  468] time: 33.9255, train_loss: 0.27730629, train_accuracy: 0.9297, test_Accuracy: 0.9256\n",
            "Epoch: [ 0] [  141/  468] time: 34.1201, train_loss: 0.27062675, train_accuracy: 0.9062, test_Accuracy: 0.9240\n",
            "Epoch: [ 0] [  142/  468] time: 34.3182, train_loss: 0.23906280, train_accuracy: 0.8906, test_Accuracy: 0.9226\n",
            "Epoch: [ 0] [  143/  468] time: 34.5124, train_loss: 0.27345204, train_accuracy: 0.8750, test_Accuracy: 0.9230\n",
            "Epoch: [ 0] [  144/  468] time: 34.7154, train_loss: 0.35542333, train_accuracy: 0.8750, test_Accuracy: 0.9244\n",
            "Epoch: [ 0] [  145/  468] time: 34.9271, train_loss: 0.20618857, train_accuracy: 0.9375, test_Accuracy: 0.9261\n",
            "Epoch: [ 0] [  146/  468] time: 35.1239, train_loss: 0.20314276, train_accuracy: 0.9453, test_Accuracy: 0.9287\n",
            "Epoch: [ 0] [  147/  468] time: 35.3323, train_loss: 0.22918829, train_accuracy: 0.9141, test_Accuracy: 0.9306\n",
            "Epoch: [ 0] [  148/  468] time: 35.5351, train_loss: 0.10933823, train_accuracy: 0.9688, test_Accuracy: 0.9309\n",
            "Epoch: [ 0] [  149/  468] time: 35.7490, train_loss: 0.23171511, train_accuracy: 0.9375, test_Accuracy: 0.9323\n",
            "Epoch: [ 0] [  150/  468] time: 35.9623, train_loss: 0.14955164, train_accuracy: 0.9609, test_Accuracy: 0.9316\n",
            "Epoch: [ 0] [  151/  468] time: 36.1624, train_loss: 0.40724573, train_accuracy: 0.8672, test_Accuracy: 0.9314\n",
            "Epoch: [ 0] [  152/  468] time: 36.3609, train_loss: 0.34028035, train_accuracy: 0.9297, test_Accuracy: 0.9319\n",
            "Epoch: [ 0] [  153/  468] time: 36.5660, train_loss: 0.09203871, train_accuracy: 0.9766, test_Accuracy: 0.9304\n",
            "Epoch: [ 0] [  154/  468] time: 36.7622, train_loss: 0.15230089, train_accuracy: 0.9609, test_Accuracy: 0.9311\n",
            "Epoch: [ 0] [  155/  468] time: 36.9659, train_loss: 0.28619519, train_accuracy: 0.9062, test_Accuracy: 0.9299\n",
            "Epoch: [ 0] [  156/  468] time: 37.1644, train_loss: 0.21842575, train_accuracy: 0.9609, test_Accuracy: 0.9290\n",
            "Epoch: [ 0] [  157/  468] time: 37.5237, train_loss: 0.30348039, train_accuracy: 0.9297, test_Accuracy: 0.9294\n",
            "Epoch: [ 0] [  158/  468] time: 37.7326, train_loss: 0.21460617, train_accuracy: 0.9219, test_Accuracy: 0.9314\n",
            "Epoch: [ 0] [  159/  468] time: 38.0821, train_loss: 0.22563711, train_accuracy: 0.9375, test_Accuracy: 0.9309\n",
            "Epoch: [ 0] [  160/  468] time: 38.2827, train_loss: 0.20527692, train_accuracy: 0.9453, test_Accuracy: 0.9313\n",
            "Epoch: [ 0] [  161/  468] time: 38.4840, train_loss: 0.19835868, train_accuracy: 0.9531, test_Accuracy: 0.9325\n",
            "Epoch: [ 0] [  162/  468] time: 38.6911, train_loss: 0.24145736, train_accuracy: 0.9141, test_Accuracy: 0.9336\n",
            "Epoch: [ 0] [  163/  468] time: 38.8953, train_loss: 0.19999373, train_accuracy: 0.9609, test_Accuracy: 0.9343\n",
            "Epoch: [ 0] [  164/  468] time: 39.2503, train_loss: 0.25462544, train_accuracy: 0.8984, test_Accuracy: 0.9348\n",
            "Epoch: [ 0] [  165/  468] time: 39.4500, train_loss: 0.36476061, train_accuracy: 0.9062, test_Accuracy: 0.9347\n",
            "Epoch: [ 0] [  166/  468] time: 39.8094, train_loss: 0.14404355, train_accuracy: 0.9688, test_Accuracy: 0.9314\n",
            "Epoch: [ 0] [  167/  468] time: 40.1714, train_loss: 0.41983879, train_accuracy: 0.8828, test_Accuracy: 0.9308\n",
            "Epoch: [ 0] [  168/  468] time: 40.3683, train_loss: 0.33820546, train_accuracy: 0.9062, test_Accuracy: 0.9319\n",
            "Epoch: [ 0] [  169/  468] time: 40.5699, train_loss: 0.21468714, train_accuracy: 0.9609, test_Accuracy: 0.9308\n",
            "Epoch: [ 0] [  170/  468] time: 40.7794, train_loss: 0.24304107, train_accuracy: 0.9531, test_Accuracy: 0.9324\n",
            "Epoch: [ 0] [  171/  468] time: 40.9790, train_loss: 0.17484950, train_accuracy: 0.9453, test_Accuracy: 0.9327\n",
            "Epoch: [ 0] [  172/  468] time: 41.1775, train_loss: 0.23881084, train_accuracy: 0.9141, test_Accuracy: 0.9352\n",
            "Epoch: [ 0] [  173/  468] time: 41.3698, train_loss: 0.17613250, train_accuracy: 0.9375, test_Accuracy: 0.9360\n",
            "Epoch: [ 0] [  174/  468] time: 41.5812, train_loss: 0.33166495, train_accuracy: 0.8828, test_Accuracy: 0.9376\n",
            "Epoch: [ 0] [  175/  468] time: 41.7892, train_loss: 0.22351339, train_accuracy: 0.9453, test_Accuracy: 0.9388\n",
            "Epoch: [ 0] [  176/  468] time: 41.9821, train_loss: 0.17158368, train_accuracy: 0.9766, test_Accuracy: 0.9378\n",
            "Epoch: [ 0] [  177/  468] time: 42.1831, train_loss: 0.21578342, train_accuracy: 0.9453, test_Accuracy: 0.9336\n",
            "Epoch: [ 0] [  178/  468] time: 42.3710, train_loss: 0.25661457, train_accuracy: 0.9297, test_Accuracy: 0.9326\n",
            "Epoch: [ 0] [  179/  468] time: 42.5846, train_loss: 0.30320168, train_accuracy: 0.9219, test_Accuracy: 0.9301\n",
            "Epoch: [ 0] [  180/  468] time: 42.9391, train_loss: 0.29344714, train_accuracy: 0.9141, test_Accuracy: 0.9285\n",
            "Epoch: [ 0] [  181/  468] time: 43.1370, train_loss: 0.20858940, train_accuracy: 0.9453, test_Accuracy: 0.9299\n",
            "Epoch: [ 0] [  182/  468] time: 43.3352, train_loss: 0.16251880, train_accuracy: 0.9453, test_Accuracy: 0.9334\n",
            "Epoch: [ 0] [  183/  468] time: 43.5374, train_loss: 0.21231551, train_accuracy: 0.9531, test_Accuracy: 0.9381\n",
            "Epoch: [ 0] [  184/  468] time: 43.7363, train_loss: 0.26830238, train_accuracy: 0.9141, test_Accuracy: 0.9393\n",
            "Epoch: [ 0] [  185/  468] time: 43.9307, train_loss: 0.20037705, train_accuracy: 0.9375, test_Accuracy: 0.9397\n",
            "Epoch: [ 0] [  186/  468] time: 44.1251, train_loss: 0.13489956, train_accuracy: 0.9766, test_Accuracy: 0.9386\n",
            "Epoch: [ 0] [  187/  468] time: 44.3257, train_loss: 0.21426734, train_accuracy: 0.9531, test_Accuracy: 0.9379\n",
            "Epoch: [ 0] [  188/  468] time: 44.5222, train_loss: 0.26904911, train_accuracy: 0.9219, test_Accuracy: 0.9378\n",
            "Epoch: [ 0] [  189/  468] time: 44.8808, train_loss: 0.24563076, train_accuracy: 0.9297, test_Accuracy: 0.9385\n",
            "Epoch: [ 0] [  190/  468] time: 45.0794, train_loss: 0.11797885, train_accuracy: 0.9688, test_Accuracy: 0.9392\n",
            "Epoch: [ 0] [  191/  468] time: 45.2810, train_loss: 0.21883035, train_accuracy: 0.9297, test_Accuracy: 0.9397\n",
            "Epoch: [ 0] [  192/  468] time: 45.4776, train_loss: 0.19133443, train_accuracy: 0.9375, test_Accuracy: 0.9412\n",
            "Epoch: [ 0] [  193/  468] time: 45.8338, train_loss: 0.22703719, train_accuracy: 0.9297, test_Accuracy: 0.9413\n",
            "Epoch: [ 0] [  194/  468] time: 46.0253, train_loss: 0.17826346, train_accuracy: 0.9609, test_Accuracy: 0.9399\n",
            "Epoch: [ 0] [  195/  468] time: 46.3814, train_loss: 0.18536794, train_accuracy: 0.9375, test_Accuracy: 0.9390\n",
            "Epoch: [ 0] [  196/  468] time: 46.5803, train_loss: 0.23130764, train_accuracy: 0.9375, test_Accuracy: 0.9396\n",
            "Epoch: [ 0] [  197/  468] time: 46.7741, train_loss: 0.16358280, train_accuracy: 0.9531, test_Accuracy: 0.9391\n",
            "Epoch: [ 0] [  198/  468] time: 46.9631, train_loss: 0.12025744, train_accuracy: 0.9688, test_Accuracy: 0.9378\n",
            "Epoch: [ 0] [  199/  468] time: 47.1680, train_loss: 0.54191899, train_accuracy: 0.8906, test_Accuracy: 0.9358\n",
            "Epoch: [ 0] [  200/  468] time: 47.3495, train_loss: 0.23037454, train_accuracy: 0.9609, test_Accuracy: 0.9351\n",
            "Epoch: [ 0] [  201/  468] time: 47.5489, train_loss: 0.18560076, train_accuracy: 0.9297, test_Accuracy: 0.9343\n",
            "Epoch: [ 0] [  202/  468] time: 47.7422, train_loss: 0.17720722, train_accuracy: 0.9453, test_Accuracy: 0.9368\n",
            "Epoch: [ 0] [  203/  468] time: 47.9397, train_loss: 0.16329557, train_accuracy: 0.9688, test_Accuracy: 0.9391\n",
            "Epoch: [ 0] [  204/  468] time: 48.1241, train_loss: 0.18258950, train_accuracy: 0.9531, test_Accuracy: 0.9408\n",
            "Epoch: [ 0] [  205/  468] time: 48.3260, train_loss: 0.15759605, train_accuracy: 0.9609, test_Accuracy: 0.9393\n",
            "Epoch: [ 0] [  206/  468] time: 48.5167, train_loss: 0.27126673, train_accuracy: 0.9219, test_Accuracy: 0.9384\n",
            "Epoch: [ 0] [  207/  468] time: 48.7138, train_loss: 0.12875283, train_accuracy: 0.9609, test_Accuracy: 0.9391\n",
            "Epoch: [ 0] [  208/  468] time: 48.9021, train_loss: 0.16561449, train_accuracy: 0.9609, test_Accuracy: 0.9394\n",
            "Epoch: [ 0] [  209/  468] time: 49.0989, train_loss: 0.14778741, train_accuracy: 0.9609, test_Accuracy: 0.9414\n",
            "Epoch: [ 0] [  210/  468] time: 49.2931, train_loss: 0.12075479, train_accuracy: 0.9766, test_Accuracy: 0.9428\n",
            "Epoch: [ 0] [  211/  468] time: 49.6573, train_loss: 0.15237069, train_accuracy: 0.9609, test_Accuracy: 0.9425\n",
            "Epoch: [ 0] [  212/  468] time: 49.8476, train_loss: 0.25444973, train_accuracy: 0.9219, test_Accuracy: 0.9425\n",
            "Epoch: [ 0] [  213/  468] time: 50.0385, train_loss: 0.19959944, train_accuracy: 0.9297, test_Accuracy: 0.9415\n",
            "Epoch: [ 0] [  214/  468] time: 50.2319, train_loss: 0.25845253, train_accuracy: 0.9219, test_Accuracy: 0.9407\n",
            "Epoch: [ 0] [  215/  468] time: 50.4347, train_loss: 0.21140051, train_accuracy: 0.9375, test_Accuracy: 0.9374\n",
            "Epoch: [ 0] [  216/  468] time: 50.6287, train_loss: 0.19519223, train_accuracy: 0.9453, test_Accuracy: 0.9343\n",
            "Epoch: [ 0] [  217/  468] time: 50.8197, train_loss: 0.14796498, train_accuracy: 0.9688, test_Accuracy: 0.9328\n",
            "Epoch: [ 0] [  218/  468] time: 51.1686, train_loss: 0.16304557, train_accuracy: 0.9531, test_Accuracy: 0.9314\n",
            "Epoch: [ 0] [  219/  468] time: 51.3715, train_loss: 0.19198507, train_accuracy: 0.9531, test_Accuracy: 0.9339\n",
            "Epoch: [ 0] [  220/  468] time: 51.5713, train_loss: 0.19503795, train_accuracy: 0.9375, test_Accuracy: 0.9368\n",
            "Epoch: [ 0] [  221/  468] time: 51.7659, train_loss: 0.24049647, train_accuracy: 0.9141, test_Accuracy: 0.9406\n",
            "Epoch: [ 0] [  222/  468] time: 51.9546, train_loss: 0.24572511, train_accuracy: 0.9453, test_Accuracy: 0.9424\n",
            "Epoch: [ 0] [  223/  468] time: 52.1470, train_loss: 0.24674949, train_accuracy: 0.9219, test_Accuracy: 0.9455\n",
            "Epoch: [ 0] [  224/  468] time: 52.3584, train_loss: 0.10856426, train_accuracy: 0.9688, test_Accuracy: 0.9462\n",
            "Epoch: [ 0] [  225/  468] time: 52.5472, train_loss: 0.23031698, train_accuracy: 0.9531, test_Accuracy: 0.9460\n",
            "Epoch: [ 0] [  226/  468] time: 52.7497, train_loss: 0.20189777, train_accuracy: 0.9297, test_Accuracy: 0.9447\n",
            "Epoch: [ 0] [  227/  468] time: 52.9397, train_loss: 0.16634218, train_accuracy: 0.9453, test_Accuracy: 0.9444\n",
            "Epoch: [ 0] [  228/  468] time: 53.1297, train_loss: 0.19124410, train_accuracy: 0.9297, test_Accuracy: 0.9424\n",
            "Epoch: [ 0] [  229/  468] time: 53.3220, train_loss: 0.20163786, train_accuracy: 0.9531, test_Accuracy: 0.9409\n",
            "Epoch: [ 0] [  230/  468] time: 53.5226, train_loss: 0.14566417, train_accuracy: 0.9609, test_Accuracy: 0.9413\n",
            "Epoch: [ 0] [  231/  468] time: 53.7228, train_loss: 0.22366790, train_accuracy: 0.9375, test_Accuracy: 0.9410\n",
            "Epoch: [ 0] [  232/  468] time: 53.9230, train_loss: 0.24509311, train_accuracy: 0.9297, test_Accuracy: 0.9426\n",
            "Epoch: [ 0] [  233/  468] time: 54.1217, train_loss: 0.22716710, train_accuracy: 0.9141, test_Accuracy: 0.9439\n",
            "Epoch: [ 0] [  234/  468] time: 54.3309, train_loss: 0.14592163, train_accuracy: 0.9531, test_Accuracy: 0.9433\n",
            "Epoch: [ 0] [  235/  468] time: 54.5309, train_loss: 0.13860069, train_accuracy: 0.9531, test_Accuracy: 0.9446\n",
            "Epoch: [ 0] [  236/  468] time: 54.7347, train_loss: 0.15845029, train_accuracy: 0.9609, test_Accuracy: 0.9453\n",
            "Epoch: [ 0] [  237/  468] time: 55.0876, train_loss: 0.24918795, train_accuracy: 0.9297, test_Accuracy: 0.9450\n",
            "Epoch: [ 0] [  238/  468] time: 55.2918, train_loss: 0.19676825, train_accuracy: 0.9609, test_Accuracy: 0.9454\n",
            "Epoch: [ 0] [  239/  468] time: 55.4954, train_loss: 0.10184433, train_accuracy: 0.9609, test_Accuracy: 0.9445\n",
            "Epoch: [ 0] [  240/  468] time: 55.6918, train_loss: 0.30666733, train_accuracy: 0.8984, test_Accuracy: 0.9465\n",
            "Epoch: [ 0] [  241/  468] time: 56.0521, train_loss: 0.18964806, train_accuracy: 0.9375, test_Accuracy: 0.9466\n",
            "Epoch: [ 0] [  242/  468] time: 56.2471, train_loss: 0.12954876, train_accuracy: 0.9688, test_Accuracy: 0.9463\n",
            "Epoch: [ 0] [  243/  468] time: 56.4610, train_loss: 0.20168477, train_accuracy: 0.9375, test_Accuracy: 0.9458\n",
            "Epoch: [ 0] [  244/  468] time: 56.6539, train_loss: 0.12219647, train_accuracy: 0.9688, test_Accuracy: 0.9457\n",
            "Epoch: [ 0] [  245/  468] time: 56.8553, train_loss: 0.14477745, train_accuracy: 0.9609, test_Accuracy: 0.9455\n",
            "Epoch: [ 0] [  246/  468] time: 57.0488, train_loss: 0.22398567, train_accuracy: 0.9375, test_Accuracy: 0.9459\n",
            "Epoch: [ 0] [  247/  468] time: 57.2454, train_loss: 0.18158641, train_accuracy: 0.9453, test_Accuracy: 0.9455\n",
            "Epoch: [ 0] [  248/  468] time: 57.4409, train_loss: 0.11468869, train_accuracy: 0.9688, test_Accuracy: 0.9448\n",
            "Epoch: [ 0] [  249/  468] time: 57.8019, train_loss: 0.09159727, train_accuracy: 0.9844, test_Accuracy: 0.9419\n",
            "Epoch: [ 0] [  250/  468] time: 57.9968, train_loss: 0.16967610, train_accuracy: 0.9609, test_Accuracy: 0.9422\n",
            "Epoch: [ 0] [  251/  468] time: 58.1976, train_loss: 0.22373383, train_accuracy: 0.9375, test_Accuracy: 0.9452\n",
            "Epoch: [ 0] [  252/  468] time: 58.5561, train_loss: 0.15344837, train_accuracy: 0.9609, test_Accuracy: 0.9466\n",
            "Epoch: [ 0] [  253/  468] time: 58.9078, train_loss: 0.13368006, train_accuracy: 0.9609, test_Accuracy: 0.9474\n",
            "Epoch: [ 0] [  254/  468] time: 59.1082, train_loss: 0.26633078, train_accuracy: 0.9297, test_Accuracy: 0.9476\n",
            "Epoch: [ 0] [  255/  468] time: 59.3105, train_loss: 0.37251803, train_accuracy: 0.8984, test_Accuracy: 0.9462\n",
            "Epoch: [ 0] [  256/  468] time: 59.5082, train_loss: 0.16606408, train_accuracy: 0.9453, test_Accuracy: 0.9471\n",
            "Epoch: [ 0] [  257/  468] time: 59.8633, train_loss: 0.13745987, train_accuracy: 0.9688, test_Accuracy: 0.9480\n",
            "Epoch: [ 0] [  258/  468] time: 60.0580, train_loss: 0.13105826, train_accuracy: 0.9844, test_Accuracy: 0.9489\n",
            "Epoch: [ 0] [  259/  468] time: 60.4112, train_loss: 0.19634224, train_accuracy: 0.9453, test_Accuracy: 0.9492\n",
            "Epoch: [ 0] [  260/  468] time: 60.6079, train_loss: 0.12120249, train_accuracy: 0.9531, test_Accuracy: 0.9482\n",
            "Epoch: [ 0] [  261/  468] time: 60.8102, train_loss: 0.19614628, train_accuracy: 0.9297, test_Accuracy: 0.9470\n",
            "Epoch: [ 0] [  262/  468] time: 61.0060, train_loss: 0.13122839, train_accuracy: 0.9609, test_Accuracy: 0.9463\n",
            "Epoch: [ 0] [  263/  468] time: 61.3620, train_loss: 0.12424435, train_accuracy: 0.9609, test_Accuracy: 0.9432\n",
            "Epoch: [ 0] [  264/  468] time: 61.5609, train_loss: 0.12954532, train_accuracy: 0.9688, test_Accuracy: 0.9432\n",
            "Epoch: [ 0] [  265/  468] time: 61.7620, train_loss: 0.24370559, train_accuracy: 0.9375, test_Accuracy: 0.9445\n",
            "Epoch: [ 0] [  266/  468] time: 61.9724, train_loss: 0.16335239, train_accuracy: 0.9453, test_Accuracy: 0.9475\n",
            "Epoch: [ 0] [  267/  468] time: 62.3219, train_loss: 0.15714067, train_accuracy: 0.9531, test_Accuracy: 0.9491\n",
            "Epoch: [ 0] [  268/  468] time: 62.5344, train_loss: 0.15425184, train_accuracy: 0.9609, test_Accuracy: 0.9493\n",
            "Epoch: [ 0] [  269/  468] time: 62.7331, train_loss: 0.10729146, train_accuracy: 0.9531, test_Accuracy: 0.9499\n",
            "Epoch: [ 0] [  270/  468] time: 62.9251, train_loss: 0.11577629, train_accuracy: 0.9688, test_Accuracy: 0.9494\n",
            "Epoch: [ 0] [  271/  468] time: 63.1109, train_loss: 0.08986691, train_accuracy: 0.9766, test_Accuracy: 0.9487\n",
            "Epoch: [ 0] [  272/  468] time: 63.2975, train_loss: 0.16999274, train_accuracy: 0.9375, test_Accuracy: 0.9486\n",
            "Epoch: [ 0] [  273/  468] time: 63.5051, train_loss: 0.20064744, train_accuracy: 0.9531, test_Accuracy: 0.9478\n",
            "Epoch: [ 0] [  274/  468] time: 63.6930, train_loss: 0.14039804, train_accuracy: 0.9453, test_Accuracy: 0.9470\n",
            "Epoch: [ 0] [  275/  468] time: 63.8776, train_loss: 0.16203843, train_accuracy: 0.9688, test_Accuracy: 0.9455\n",
            "Epoch: [ 0] [  276/  468] time: 64.0665, train_loss: 0.18882538, train_accuracy: 0.9375, test_Accuracy: 0.9452\n",
            "Epoch: [ 0] [  277/  468] time: 64.2497, train_loss: 0.18291049, train_accuracy: 0.9453, test_Accuracy: 0.9445\n",
            "Epoch: [ 0] [  278/  468] time: 64.5982, train_loss: 0.11019374, train_accuracy: 0.9609, test_Accuracy: 0.9438\n",
            "Epoch: [ 0] [  279/  468] time: 64.7880, train_loss: 0.21269655, train_accuracy: 0.9609, test_Accuracy: 0.9446\n",
            "Epoch: [ 0] [  280/  468] time: 64.9917, train_loss: 0.33046907, train_accuracy: 0.9297, test_Accuracy: 0.9472\n",
            "Epoch: [ 0] [  281/  468] time: 65.1845, train_loss: 0.17431751, train_accuracy: 0.9688, test_Accuracy: 0.9504\n",
            "Epoch: [ 0] [  282/  468] time: 65.3822, train_loss: 0.12285687, train_accuracy: 0.9609, test_Accuracy: 0.9517\n",
            "Epoch: [ 0] [  283/  468] time: 65.7374, train_loss: 0.16039096, train_accuracy: 0.9609, test_Accuracy: 0.9503\n",
            "Epoch: [ 0] [  284/  468] time: 65.9316, train_loss: 0.06866141, train_accuracy: 0.9766, test_Accuracy: 0.9502\n",
            "Epoch: [ 0] [  285/  468] time: 66.1283, train_loss: 0.16029677, train_accuracy: 0.9531, test_Accuracy: 0.9479\n",
            "Epoch: [ 0] [  286/  468] time: 66.3158, train_loss: 0.19472136, train_accuracy: 0.9609, test_Accuracy: 0.9468\n",
            "Epoch: [ 0] [  287/  468] time: 66.5188, train_loss: 0.10793626, train_accuracy: 0.9688, test_Accuracy: 0.9457\n",
            "Epoch: [ 0] [  288/  468] time: 66.7076, train_loss: 0.11146885, train_accuracy: 0.9609, test_Accuracy: 0.9460\n",
            "Epoch: [ 0] [  289/  468] time: 66.8975, train_loss: 0.17149672, train_accuracy: 0.9531, test_Accuracy: 0.9459\n",
            "Epoch: [ 0] [  290/  468] time: 67.0842, train_loss: 0.18346345, train_accuracy: 0.9297, test_Accuracy: 0.9484\n",
            "Epoch: [ 0] [  291/  468] time: 67.2739, train_loss: 0.23659110, train_accuracy: 0.9453, test_Accuracy: 0.9498\n",
            "Epoch: [ 0] [  292/  468] time: 67.4634, train_loss: 0.17700340, train_accuracy: 0.9375, test_Accuracy: 0.9510\n",
            "Epoch: [ 0] [  293/  468] time: 67.6622, train_loss: 0.12493277, train_accuracy: 0.9844, test_Accuracy: 0.9500\n",
            "Epoch: [ 0] [  294/  468] time: 67.8498, train_loss: 0.17178077, train_accuracy: 0.9453, test_Accuracy: 0.9495\n",
            "Epoch: [ 0] [  295/  468] time: 68.0421, train_loss: 0.13297080, train_accuracy: 0.9688, test_Accuracy: 0.9492\n",
            "Epoch: [ 0] [  296/  468] time: 68.2304, train_loss: 0.14439955, train_accuracy: 0.9766, test_Accuracy: 0.9495\n",
            "Epoch: [ 0] [  297/  468] time: 68.4261, train_loss: 0.14163110, train_accuracy: 0.9453, test_Accuracy: 0.9498\n",
            "Epoch: [ 0] [  298/  468] time: 68.6238, train_loss: 0.18341656, train_accuracy: 0.9531, test_Accuracy: 0.9501\n",
            "Epoch: [ 0] [  299/  468] time: 68.8141, train_loss: 0.16497272, train_accuracy: 0.9453, test_Accuracy: 0.9515\n",
            "Epoch: [ 0] [  300/  468] time: 69.0019, train_loss: 0.16024135, train_accuracy: 0.9375, test_Accuracy: 0.9526\n",
            "Epoch: [ 0] [  301/  468] time: 69.1959, train_loss: 0.16833526, train_accuracy: 0.9531, test_Accuracy: 0.9519\n",
            "Epoch: [ 0] [  302/  468] time: 69.3847, train_loss: 0.09265627, train_accuracy: 0.9766, test_Accuracy: 0.9515\n",
            "Epoch: [ 0] [  303/  468] time: 69.7404, train_loss: 0.17246702, train_accuracy: 0.9453, test_Accuracy: 0.9502\n",
            "Epoch: [ 0] [  304/  468] time: 69.9310, train_loss: 0.18147521, train_accuracy: 0.9453, test_Accuracy: 0.9496\n",
            "Epoch: [ 0] [  305/  468] time: 70.1207, train_loss: 0.18227027, train_accuracy: 0.9688, test_Accuracy: 0.9492\n",
            "Epoch: [ 0] [  306/  468] time: 70.3101, train_loss: 0.10889849, train_accuracy: 0.9688, test_Accuracy: 0.9482\n",
            "Epoch: [ 0] [  307/  468] time: 70.5095, train_loss: 0.11161105, train_accuracy: 0.9688, test_Accuracy: 0.9497\n",
            "Epoch: [ 0] [  308/  468] time: 70.6960, train_loss: 0.28521535, train_accuracy: 0.9375, test_Accuracy: 0.9506\n",
            "Epoch: [ 0] [  309/  468] time: 70.8815, train_loss: 0.18480811, train_accuracy: 0.9531, test_Accuracy: 0.9507\n",
            "Epoch: [ 0] [  310/  468] time: 71.0674, train_loss: 0.22356245, train_accuracy: 0.9375, test_Accuracy: 0.9511\n",
            "Epoch: [ 0] [  311/  468] time: 71.2633, train_loss: 0.18388826, train_accuracy: 0.9453, test_Accuracy: 0.9513\n",
            "Epoch: [ 0] [  312/  468] time: 71.4505, train_loss: 0.13594082, train_accuracy: 0.9531, test_Accuracy: 0.9516\n",
            "Epoch: [ 0] [  313/  468] time: 71.6574, train_loss: 0.25545257, train_accuracy: 0.9297, test_Accuracy: 0.9520\n",
            "Epoch: [ 0] [  314/  468] time: 72.0147, train_loss: 0.13460600, train_accuracy: 0.9609, test_Accuracy: 0.9528\n",
            "Epoch: [ 0] [  315/  468] time: 72.2071, train_loss: 0.11569995, train_accuracy: 0.9766, test_Accuracy: 0.9520\n",
            "Epoch: [ 0] [  316/  468] time: 72.4016, train_loss: 0.17665289, train_accuracy: 0.9453, test_Accuracy: 0.9514\n",
            "Epoch: [ 0] [  317/  468] time: 72.6000, train_loss: 0.13831559, train_accuracy: 0.9531, test_Accuracy: 0.9532\n",
            "Epoch: [ 0] [  318/  468] time: 72.7912, train_loss: 0.16077182, train_accuracy: 0.9531, test_Accuracy: 0.9540\n",
            "Epoch: [ 0] [  319/  468] time: 72.9837, train_loss: 0.25715643, train_accuracy: 0.9297, test_Accuracy: 0.9536\n",
            "Epoch: [ 0] [  320/  468] time: 73.1793, train_loss: 0.12931010, train_accuracy: 0.9688, test_Accuracy: 0.9529\n",
            "Epoch: [ 0] [  321/  468] time: 73.3640, train_loss: 0.16332529, train_accuracy: 0.9453, test_Accuracy: 0.9503\n",
            "Epoch: [ 0] [  322/  468] time: 73.5712, train_loss: 0.14143114, train_accuracy: 0.9453, test_Accuracy: 0.9497\n",
            "Epoch: [ 0] [  323/  468] time: 73.9229, train_loss: 0.17050290, train_accuracy: 0.9219, test_Accuracy: 0.9483\n",
            "Epoch: [ 0] [  324/  468] time: 74.1138, train_loss: 0.14549476, train_accuracy: 0.9609, test_Accuracy: 0.9479\n",
            "Epoch: [ 0] [  325/  468] time: 74.3105, train_loss: 0.22842637, train_accuracy: 0.9297, test_Accuracy: 0.9492\n",
            "Epoch: [ 0] [  326/  468] time: 74.5097, train_loss: 0.15273429, train_accuracy: 0.9531, test_Accuracy: 0.9510\n",
            "Epoch: [ 0] [  327/  468] time: 74.7113, train_loss: 0.14977252, train_accuracy: 0.9609, test_Accuracy: 0.9527\n",
            "Epoch: [ 0] [  328/  468] time: 74.9071, train_loss: 0.20205909, train_accuracy: 0.9297, test_Accuracy: 0.9526\n",
            "Epoch: [ 0] [  329/  468] time: 75.0996, train_loss: 0.26640826, train_accuracy: 0.9453, test_Accuracy: 0.9516\n",
            "Epoch: [ 0] [  330/  468] time: 75.2842, train_loss: 0.13287801, train_accuracy: 0.9609, test_Accuracy: 0.9487\n",
            "Epoch: [ 0] [  331/  468] time: 75.4872, train_loss: 0.19223559, train_accuracy: 0.9297, test_Accuracy: 0.9498\n",
            "Epoch: [ 0] [  332/  468] time: 75.6809, train_loss: 0.23115361, train_accuracy: 0.9141, test_Accuracy: 0.9516\n",
            "Epoch: [ 0] [  333/  468] time: 75.8822, train_loss: 0.12588531, train_accuracy: 0.9688, test_Accuracy: 0.9536\n",
            "Epoch: [ 0] [  334/  468] time: 76.0821, train_loss: 0.12533402, train_accuracy: 0.9688, test_Accuracy: 0.9552\n",
            "Epoch: [ 0] [  335/  468] time: 76.2785, train_loss: 0.11091565, train_accuracy: 0.9766, test_Accuracy: 0.9545\n",
            "Epoch: [ 0] [  336/  468] time: 76.6241, train_loss: 0.11544790, train_accuracy: 0.9766, test_Accuracy: 0.9515\n",
            "Epoch: [ 0] [  337/  468] time: 76.8237, train_loss: 0.15232426, train_accuracy: 0.9531, test_Accuracy: 0.9506\n",
            "Epoch: [ 0] [  338/  468] time: 77.0153, train_loss: 0.17382947, train_accuracy: 0.9375, test_Accuracy: 0.9499\n",
            "Epoch: [ 0] [  339/  468] time: 77.2036, train_loss: 0.14838211, train_accuracy: 0.9453, test_Accuracy: 0.9505\n",
            "Epoch: [ 0] [  340/  468] time: 77.5630, train_loss: 0.22004083, train_accuracy: 0.9375, test_Accuracy: 0.9534\n",
            "Epoch: [ 0] [  341/  468] time: 77.7613, train_loss: 0.15303978, train_accuracy: 0.9609, test_Accuracy: 0.9561\n",
            "Epoch: [ 0] [  342/  468] time: 77.9566, train_loss: 0.12403130, train_accuracy: 0.9453, test_Accuracy: 0.9554\n",
            "Epoch: [ 0] [  343/  468] time: 78.1456, train_loss: 0.12745371, train_accuracy: 0.9609, test_Accuracy: 0.9531\n",
            "Epoch: [ 0] [  344/  468] time: 78.3398, train_loss: 0.14655122, train_accuracy: 0.9688, test_Accuracy: 0.9525\n",
            "Epoch: [ 0] [  345/  468] time: 78.5337, train_loss: 0.16450219, train_accuracy: 0.9453, test_Accuracy: 0.9512\n",
            "Epoch: [ 0] [  346/  468] time: 78.7428, train_loss: 0.09229749, train_accuracy: 0.9688, test_Accuracy: 0.9507\n",
            "Epoch: [ 0] [  347/  468] time: 78.9403, train_loss: 0.12727781, train_accuracy: 0.9766, test_Accuracy: 0.9513\n",
            "Epoch: [ 0] [  348/  468] time: 79.1383, train_loss: 0.14561564, train_accuracy: 0.9531, test_Accuracy: 0.9538\n",
            "Epoch: [ 0] [  349/  468] time: 79.3249, train_loss: 0.08075662, train_accuracy: 0.9688, test_Accuracy: 0.9532\n",
            "Epoch: [ 0] [  350/  468] time: 79.6774, train_loss: 0.14473948, train_accuracy: 0.9609, test_Accuracy: 0.9535\n",
            "Epoch: [ 0] [  351/  468] time: 79.8882, train_loss: 0.14517178, train_accuracy: 0.9609, test_Accuracy: 0.9528\n",
            "Epoch: [ 0] [  352/  468] time: 80.0851, train_loss: 0.11972251, train_accuracy: 0.9609, test_Accuracy: 0.9523\n",
            "Epoch: [ 0] [  353/  468] time: 80.2898, train_loss: 0.10543171, train_accuracy: 0.9688, test_Accuracy: 0.9528\n",
            "Epoch: [ 0] [  354/  468] time: 80.6433, train_loss: 0.15440059, train_accuracy: 0.9375, test_Accuracy: 0.9527\n",
            "Epoch: [ 0] [  355/  468] time: 80.8423, train_loss: 0.17074886, train_accuracy: 0.9375, test_Accuracy: 0.9520\n",
            "Epoch: [ 0] [  356/  468] time: 81.0381, train_loss: 0.17305416, train_accuracy: 0.9141, test_Accuracy: 0.9521\n",
            "Epoch: [ 0] [  357/  468] time: 81.2285, train_loss: 0.11347363, train_accuracy: 0.9688, test_Accuracy: 0.9532\n",
            "Epoch: [ 0] [  358/  468] time: 81.4250, train_loss: 0.07104510, train_accuracy: 0.9766, test_Accuracy: 0.9534\n",
            "Epoch: [ 0] [  359/  468] time: 81.6252, train_loss: 0.12955797, train_accuracy: 0.9531, test_Accuracy: 0.9531\n",
            "Epoch: [ 0] [  360/  468] time: 81.9887, train_loss: 0.14775781, train_accuracy: 0.9609, test_Accuracy: 0.9548\n",
            "Epoch: [ 0] [  361/  468] time: 82.1861, train_loss: 0.08497985, train_accuracy: 0.9844, test_Accuracy: 0.9557\n",
            "Epoch: [ 0] [  362/  468] time: 82.3861, train_loss: 0.09871525, train_accuracy: 0.9844, test_Accuracy: 0.9556\n",
            "Epoch: [ 0] [  363/  468] time: 82.5777, train_loss: 0.13489401, train_accuracy: 0.9531, test_Accuracy: 0.9556\n",
            "Epoch: [ 0] [  364/  468] time: 82.7680, train_loss: 0.22996955, train_accuracy: 0.9453, test_Accuracy: 0.9553\n",
            "Epoch: [ 0] [  365/  468] time: 82.9569, train_loss: 0.08296573, train_accuracy: 0.9844, test_Accuracy: 0.9546\n",
            "Epoch: [ 0] [  366/  468] time: 83.1401, train_loss: 0.14850561, train_accuracy: 0.9453, test_Accuracy: 0.9547\n",
            "Epoch: [ 0] [  367/  468] time: 83.3255, train_loss: 0.20966059, train_accuracy: 0.9531, test_Accuracy: 0.9542\n",
            "Epoch: [ 0] [  368/  468] time: 83.5257, train_loss: 0.06282321, train_accuracy: 0.9922, test_Accuracy: 0.9542\n",
            "Epoch: [ 0] [  369/  468] time: 83.7256, train_loss: 0.16235471, train_accuracy: 0.9297, test_Accuracy: 0.9538\n",
            "Epoch: [ 0] [  370/  468] time: 83.9276, train_loss: 0.16143195, train_accuracy: 0.9688, test_Accuracy: 0.9524\n",
            "Epoch: [ 0] [  371/  468] time: 84.1175, train_loss: 0.08827943, train_accuracy: 0.9844, test_Accuracy: 0.9532\n",
            "Epoch: [ 0] [  372/  468] time: 84.3110, train_loss: 0.20980304, train_accuracy: 0.9531, test_Accuracy: 0.9546\n",
            "Epoch: [ 0] [  373/  468] time: 84.4973, train_loss: 0.15194476, train_accuracy: 0.9375, test_Accuracy: 0.9559\n",
            "Epoch: [ 0] [  374/  468] time: 84.7128, train_loss: 0.14644390, train_accuracy: 0.9531, test_Accuracy: 0.9562\n",
            "Epoch: [ 0] [  375/  468] time: 84.9060, train_loss: 0.10804452, train_accuracy: 0.9609, test_Accuracy: 0.9546\n",
            "Epoch: [ 0] [  376/  468] time: 85.0990, train_loss: 0.20532997, train_accuracy: 0.9453, test_Accuracy: 0.9536\n",
            "Epoch: [ 0] [  377/  468] time: 85.2909, train_loss: 0.18043962, train_accuracy: 0.9531, test_Accuracy: 0.9536\n",
            "Epoch: [ 0] [  378/  468] time: 85.4867, train_loss: 0.17273924, train_accuracy: 0.9453, test_Accuracy: 0.9549\n",
            "Epoch: [ 0] [  379/  468] time: 85.8346, train_loss: 0.16641219, train_accuracy: 0.9453, test_Accuracy: 0.9549\n",
            "Epoch: [ 0] [  380/  468] time: 86.0272, train_loss: 0.09647084, train_accuracy: 0.9688, test_Accuracy: 0.9548\n",
            "Epoch: [ 0] [  381/  468] time: 86.2171, train_loss: 0.12804329, train_accuracy: 0.9609, test_Accuracy: 0.9532\n",
            "Epoch: [ 0] [  382/  468] time: 86.4060, train_loss: 0.15353811, train_accuracy: 0.9453, test_Accuracy: 0.9525\n",
            "Epoch: [ 0] [  383/  468] time: 86.7564, train_loss: 0.10326729, train_accuracy: 0.9688, test_Accuracy: 0.9536\n",
            "Epoch: [ 0] [  384/  468] time: 86.9373, train_loss: 0.09828775, train_accuracy: 0.9844, test_Accuracy: 0.9544\n",
            "Epoch: [ 0] [  385/  468] time: 87.1330, train_loss: 0.12260038, train_accuracy: 0.9688, test_Accuracy: 0.9553\n",
            "Epoch: [ 0] [  386/  468] time: 87.3163, train_loss: 0.17227525, train_accuracy: 0.9453, test_Accuracy: 0.9567\n",
            "Epoch: [ 0] [  387/  468] time: 87.5148, train_loss: 0.06848363, train_accuracy: 0.9688, test_Accuracy: 0.9573\n",
            "Epoch: [ 0] [  388/  468] time: 87.7023, train_loss: 0.16868937, train_accuracy: 0.9609, test_Accuracy: 0.9589\n",
            "Epoch: [ 0] [  389/  468] time: 87.9000, train_loss: 0.20756200, train_accuracy: 0.9375, test_Accuracy: 0.9594\n",
            "Epoch: [ 0] [  390/  468] time: 88.0881, train_loss: 0.14826301, train_accuracy: 0.9688, test_Accuracy: 0.9594\n",
            "Epoch: [ 0] [  391/  468] time: 88.4387, train_loss: 0.10557389, train_accuracy: 0.9609, test_Accuracy: 0.9591\n",
            "Epoch: [ 0] [  392/  468] time: 88.6346, train_loss: 0.18055253, train_accuracy: 0.9375, test_Accuracy: 0.9586\n",
            "Epoch: [ 0] [  393/  468] time: 88.8357, train_loss: 0.17081174, train_accuracy: 0.9531, test_Accuracy: 0.9576\n",
            "Epoch: [ 0] [  394/  468] time: 89.1840, train_loss: 0.05516512, train_accuracy: 0.9844, test_Accuracy: 0.9573\n",
            "Epoch: [ 0] [  395/  468] time: 89.3750, train_loss: 0.16747206, train_accuracy: 0.9688, test_Accuracy: 0.9569\n",
            "Epoch: [ 0] [  396/  468] time: 89.5632, train_loss: 0.16209686, train_accuracy: 0.9609, test_Accuracy: 0.9563\n",
            "Epoch: [ 0] [  397/  468] time: 89.7532, train_loss: 0.10956740, train_accuracy: 0.9688, test_Accuracy: 0.9563\n",
            "Epoch: [ 0] [  398/  468] time: 89.9494, train_loss: 0.07565559, train_accuracy: 0.9766, test_Accuracy: 0.9565\n",
            "Epoch: [ 0] [  399/  468] time: 90.1400, train_loss: 0.07326689, train_accuracy: 0.9766, test_Accuracy: 0.9564\n",
            "Epoch: [ 0] [  400/  468] time: 90.3361, train_loss: 0.13853799, train_accuracy: 0.9688, test_Accuracy: 0.9572\n",
            "Epoch: [ 0] [  401/  468] time: 90.5293, train_loss: 0.13033792, train_accuracy: 0.9531, test_Accuracy: 0.9565\n",
            "Epoch: [ 0] [  402/  468] time: 90.7252, train_loss: 0.10649778, train_accuracy: 0.9844, test_Accuracy: 0.9559\n",
            "Epoch: [ 0] [  403/  468] time: 90.9232, train_loss: 0.16306268, train_accuracy: 0.9375, test_Accuracy: 0.9550\n",
            "Epoch: [ 0] [  404/  468] time: 91.1131, train_loss: 0.04766908, train_accuracy: 0.9844, test_Accuracy: 0.9550\n",
            "Epoch: [ 0] [  405/  468] time: 91.2965, train_loss: 0.20043096, train_accuracy: 0.9453, test_Accuracy: 0.9551\n",
            "Epoch: [ 0] [  406/  468] time: 91.5007, train_loss: 0.19850975, train_accuracy: 0.9453, test_Accuracy: 0.9572\n",
            "Epoch: [ 0] [  407/  468] time: 91.6911, train_loss: 0.14665151, train_accuracy: 0.9453, test_Accuracy: 0.9590\n",
            "Epoch: [ 0] [  408/  468] time: 92.0459, train_loss: 0.08704306, train_accuracy: 0.9766, test_Accuracy: 0.9587\n",
            "Epoch: [ 0] [  409/  468] time: 92.2256, train_loss: 0.13952279, train_accuracy: 0.9609, test_Accuracy: 0.9592\n",
            "Epoch: [ 0] [  410/  468] time: 92.4189, train_loss: 0.25503004, train_accuracy: 0.9219, test_Accuracy: 0.9595\n",
            "Epoch: [ 0] [  411/  468] time: 92.6061, train_loss: 0.12871796, train_accuracy: 0.9531, test_Accuracy: 0.9596\n",
            "Epoch: [ 0] [  412/  468] time: 92.7987, train_loss: 0.21816918, train_accuracy: 0.9688, test_Accuracy: 0.9598\n",
            "Epoch: [ 0] [  413/  468] time: 92.9886, train_loss: 0.15919405, train_accuracy: 0.9609, test_Accuracy: 0.9604\n",
            "Epoch: [ 0] [  414/  468] time: 93.1804, train_loss: 0.15993625, train_accuracy: 0.9531, test_Accuracy: 0.9599\n",
            "Epoch: [ 0] [  415/  468] time: 93.3631, train_loss: 0.27642351, train_accuracy: 0.9375, test_Accuracy: 0.9587\n",
            "Epoch: [ 0] [  416/  468] time: 93.5520, train_loss: 0.16182265, train_accuracy: 0.9531, test_Accuracy: 0.9591\n",
            "Epoch: [ 0] [  417/  468] time: 93.7499, train_loss: 0.09810001, train_accuracy: 0.9688, test_Accuracy: 0.9586\n",
            "Epoch: [ 0] [  418/  468] time: 94.1035, train_loss: 0.15348272, train_accuracy: 0.9609, test_Accuracy: 0.9582\n",
            "Epoch: [ 0] [  419/  468] time: 94.2978, train_loss: 0.25718442, train_accuracy: 0.9375, test_Accuracy: 0.9593\n",
            "Epoch: [ 0] [  420/  468] time: 94.4971, train_loss: 0.11974021, train_accuracy: 0.9531, test_Accuracy: 0.9596\n",
            "Epoch: [ 0] [  421/  468] time: 94.7066, train_loss: 0.11679922, train_accuracy: 0.9688, test_Accuracy: 0.9606\n",
            "Epoch: [ 0] [  422/  468] time: 95.0570, train_loss: 0.13426624, train_accuracy: 0.9609, test_Accuracy: 0.9596\n",
            "Epoch: [ 0] [  423/  468] time: 95.2490, train_loss: 0.13932794, train_accuracy: 0.9531, test_Accuracy: 0.9586\n",
            "Epoch: [ 0] [  424/  468] time: 95.4450, train_loss: 0.17005695, train_accuracy: 0.9609, test_Accuracy: 0.9586\n",
            "Epoch: [ 0] [  425/  468] time: 95.7930, train_loss: 0.08676535, train_accuracy: 0.9688, test_Accuracy: 0.9580\n",
            "Epoch: [ 0] [  426/  468] time: 96.0128, train_loss: 0.04727232, train_accuracy: 1.0000, test_Accuracy: 0.9571\n",
            "Epoch: [ 0] [  427/  468] time: 96.2039, train_loss: 0.12141159, train_accuracy: 0.9609, test_Accuracy: 0.9576\n",
            "Epoch: [ 0] [  428/  468] time: 96.3936, train_loss: 0.11779778, train_accuracy: 0.9609, test_Accuracy: 0.9588\n",
            "Epoch: [ 0] [  429/  468] time: 96.5834, train_loss: 0.14789790, train_accuracy: 0.9531, test_Accuracy: 0.9582\n",
            "Epoch: [ 0] [  430/  468] time: 96.7804, train_loss: 0.15213442, train_accuracy: 0.9688, test_Accuracy: 0.9590\n",
            "Epoch: [ 0] [  431/  468] time: 96.9787, train_loss: 0.10728937, train_accuracy: 0.9609, test_Accuracy: 0.9595\n",
            "Epoch: [ 0] [  432/  468] time: 97.1705, train_loss: 0.07545023, train_accuracy: 0.9844, test_Accuracy: 0.9588\n",
            "Epoch: [ 0] [  433/  468] time: 97.3575, train_loss: 0.11160864, train_accuracy: 0.9609, test_Accuracy: 0.9583\n",
            "Epoch: [ 0] [  434/  468] time: 97.7113, train_loss: 0.18283811, train_accuracy: 0.9609, test_Accuracy: 0.9592\n",
            "Epoch: [ 0] [  435/  468] time: 97.9028, train_loss: 0.17558323, train_accuracy: 0.9219, test_Accuracy: 0.9596\n",
            "Epoch: [ 0] [  436/  468] time: 98.1001, train_loss: 0.10458015, train_accuracy: 0.9922, test_Accuracy: 0.9611\n",
            "Epoch: [ 0] [  437/  468] time: 98.2888, train_loss: 0.10108720, train_accuracy: 0.9766, test_Accuracy: 0.9619\n",
            "Epoch: [ 0] [  438/  468] time: 98.4786, train_loss: 0.10398708, train_accuracy: 0.9688, test_Accuracy: 0.9629\n",
            "Epoch: [ 0] [  439/  468] time: 98.6676, train_loss: 0.11307621, train_accuracy: 0.9609, test_Accuracy: 0.9611\n",
            "Epoch: [ 0] [  440/  468] time: 98.8624, train_loss: 0.10390867, train_accuracy: 0.9766, test_Accuracy: 0.9601\n",
            "Epoch: [ 0] [  441/  468] time: 99.2114, train_loss: 0.10982545, train_accuracy: 0.9688, test_Accuracy: 0.9584\n",
            "Epoch: [ 0] [  442/  468] time: 99.3951, train_loss: 0.25442562, train_accuracy: 0.9141, test_Accuracy: 0.9578\n",
            "Epoch: [ 0] [  443/  468] time: 99.5856, train_loss: 0.24164498, train_accuracy: 0.9219, test_Accuracy: 0.9580\n",
            "Epoch: [ 0] [  444/  468] time: 99.7814, train_loss: 0.08455709, train_accuracy: 0.9609, test_Accuracy: 0.9581\n",
            "Epoch: [ 0] [  445/  468] time: 99.9777, train_loss: 0.11357150, train_accuracy: 0.9609, test_Accuracy: 0.9588\n",
            "Epoch: [ 0] [  446/  468] time: 100.1689, train_loss: 0.12809958, train_accuracy: 0.9844, test_Accuracy: 0.9595\n",
            "Epoch: [ 0] [  447/  468] time: 100.3560, train_loss: 0.19667855, train_accuracy: 0.9531, test_Accuracy: 0.9610\n",
            "Epoch: [ 0] [  448/  468] time: 100.5430, train_loss: 0.14378905, train_accuracy: 0.9609, test_Accuracy: 0.9611\n",
            "Epoch: [ 0] [  449/  468] time: 100.7283, train_loss: 0.17206550, train_accuracy: 0.9453, test_Accuracy: 0.9610\n",
            "Epoch: [ 0] [  450/  468] time: 100.9109, train_loss: 0.16717149, train_accuracy: 0.9375, test_Accuracy: 0.9606\n",
            "Epoch: [ 0] [  451/  468] time: 101.1085, train_loss: 0.11222360, train_accuracy: 0.9609, test_Accuracy: 0.9604\n",
            "Epoch: [ 0] [  452/  468] time: 101.2909, train_loss: 0.21140468, train_accuracy: 0.9375, test_Accuracy: 0.9604\n",
            "Epoch: [ 0] [  453/  468] time: 101.4871, train_loss: 0.16702491, train_accuracy: 0.9453, test_Accuracy: 0.9609\n",
            "Epoch: [ 0] [  454/  468] time: 101.6731, train_loss: 0.12920466, train_accuracy: 0.9453, test_Accuracy: 0.9610\n",
            "Epoch: [ 0] [  455/  468] time: 101.8752, train_loss: 0.11329127, train_accuracy: 0.9609, test_Accuracy: 0.9603\n",
            "Epoch: [ 0] [  456/  468] time: 102.0635, train_loss: 0.11581185, train_accuracy: 0.9766, test_Accuracy: 0.9595\n",
            "Epoch: [ 0] [  457/  468] time: 102.2523, train_loss: 0.18544388, train_accuracy: 0.9375, test_Accuracy: 0.9587\n",
            "Epoch: [ 0] [  458/  468] time: 102.4403, train_loss: 0.09999312, train_accuracy: 0.9766, test_Accuracy: 0.9597\n",
            "Epoch: [ 0] [  459/  468] time: 102.6446, train_loss: 0.17145224, train_accuracy: 0.9531, test_Accuracy: 0.9566\n",
            "Epoch: [ 0] [  460/  468] time: 102.8333, train_loss: 0.09869817, train_accuracy: 0.9766, test_Accuracy: 0.9566\n",
            "Epoch: [ 0] [  461/  468] time: 103.0347, train_loss: 0.21109366, train_accuracy: 0.9297, test_Accuracy: 0.9584\n",
            "Epoch: [ 0] [  462/  468] time: 103.2171, train_loss: 0.22498786, train_accuracy: 0.9375, test_Accuracy: 0.9609\n",
            "Epoch: [ 0] [  463/  468] time: 103.4102, train_loss: 0.06805339, train_accuracy: 0.9766, test_Accuracy: 0.9631\n",
            "Epoch: [ 0] [  464/  468] time: 103.5934, train_loss: 0.15310064, train_accuracy: 0.9297, test_Accuracy: 0.9627\n",
            "Epoch: [ 0] [  465/  468] time: 103.7827, train_loss: 0.17223093, train_accuracy: 0.9609, test_Accuracy: 0.9628\n",
            "Epoch: [ 0] [  466/  468] time: 103.9724, train_loss: 0.18402384, train_accuracy: 0.9297, test_Accuracy: 0.9621\n",
            "Epoch: [ 0] [  467/  468] time: 104.1693, train_loss: 0.21822125, train_accuracy: 0.9531, test_Accuracy: 0.9615\n"
          ]
        }
      ]
    }
  ]
}