material/deep_learning/custom_data_generator.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "custom_data_generator.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "pH--ADDvhXQM"
      },
      "source": [
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "import tensorflow as tf\n",
        "from tensorflow import keras\n",
        "from tensorflow.keras import optimizers\n",
        "from tensorflow.keras.layers import Input, Dense, Flatten\n",
        "\n",
        "import time"
      ],
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iY37KleXmoaY"
      },
      "source": [
        "# 데이터 준비"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5BEmcWKsmj6V"
      },
      "source": [
        "## 데이터 다운로드"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "46F5aLlSlz6G"
      },
      "source": [
        "(raw_train_x, raw_train_y), (raw_test_x, raw_test_y) = tf.keras.datasets.mnist.load_data()"
      ],
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZGtQOq_Pl_Hp",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c4b4bb82-9ed8-48bb-84f0-fe02dcdc6759"
      },
      "source": [
        "print(raw_train_x.shape)\n",
        "print(raw_train_y.shape)\n",
        "print(raw_test_x.shape)\n",
        "print(raw_test_y.shape)"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(60000, 28, 28)\n",
            "(60000,)\n",
            "(10000, 28, 28)\n",
            "(10000,)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "i = int(len(raw_train_x)*0.8)\n",
        "raw_train_x, raw_valid_x = raw_train_x[:i], raw_train_x[i:]\n",
        "raw_train_y, raw_valid_y = raw_train_y[:i], raw_train_y[i:]"
      ],
      "metadata": {
        "id": "Ju9LaPD4IE2v"
      },
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(raw_train_x.shape)\n",
        "print(raw_train_y.shape)\n",
        "print(raw_valid_x.shape)\n",
        "print(raw_valid_y.shape)\n",
        "print(raw_test_x.shape)\n",
        "print(raw_test_y.shape)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XQxpNbaXIU47",
        "outputId": "e2e45937-855a-46a5-cb8e-e1e5151d0a8d"
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(48000, 28, 28)\n",
            "(48000,)\n",
            "(12000, 28, 28)\n",
            "(12000,)\n",
            "(10000, 28, 28)\n",
            "(10000,)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "BATCH_SIZE = 32"
      ],
      "metadata": {
        "id": "Ibz0ZxDoIbUe"
      },
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "import math\n",
        "from tensorflow.keras.utils import Sequence\n",
        "from abc import *\n",
        "import traceback\n",
        "\n",
        "class DataGenerator(Sequence):\n",
        "\n",
        "  def __init__(self, org_image, org_label, batch_size=BATCH_SIZE, shuffle=True):\n",
        "    self.org_image = org_image/255.\n",
        "    self.org_label = org_label\n",
        "    self.batch_size = batch_size\n",
        "    self.shuffle=shuffle\n",
        "    self.indices = np.arange(len(self.org_image))\n",
        "    self._shuffle()\n",
        "\n",
        "  def __len__(self):\n",
        "    return math.ceil(len(self.org_image) / self.batch_size)\n",
        "\n",
        "  def __getitem__(self, idx):\n",
        "    indices = self.indices[idx*self.batch_size : (idx+1)*self.batch_size]\n",
        "    batch_x = [self.org_image[i] for i in indices]\n",
        "    batch_y = [self.org_label[i] for i in indices]\n",
        "    return np.array(batch_x), np.array(batch_y)\n",
        "\n",
        "  def _shuffle(self):\n",
        "    if not self.shuffle: return\n",
        "    np.random.shuffle(self.indices)\n",
        "\n",
        "  def on_epoch_end(self):\n",
        "    self._shuffle()"
      ],
      "metadata": {
        "id": "ublVbb0_HX-5"
      },
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "train_data_generator = DataGenerator(raw_train_x, raw_train_y)\n",
        "valid_data_generator = DataGenerator(raw_valid_x, raw_valid_y)\n",
        "test_data_generator = DataGenerator(raw_test_x, raw_test_y)"
      ],
      "metadata": {
        "id": "ILA7A9fLH1gC"
      },
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CoIUBiMQNt7J",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 301
        },
        "outputId": "b4993772-86b6-47f8-e641-495b79283658"
      },
      "source": [
        "batch_x, batch_y = train_data_generator.__getitem__(0)\n",
        "\n",
        "plt.imshow(batch_x[0], cmap=plt.cm.binary)\n",
        "print(batch_y[0])"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "32\n",
            "4\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAANy0lEQVR4nO3dYaxU9ZnH8d9vXXmhNSpwcyWCS7eSELNmaXODm4iNm8aixER5gSkvCmvMUiMoNRrXuC+q74yubTTZkNyupHSjmCogvFCsNUZsYhqvyioCu6BCuATlKjGlvhDRZ1/cQ3PVO2euc86ZGXm+n+RmZs4z55zHE3+cmfOfmb8jQgBOf3/T6wYAdAdhB5Ig7EAShB1IgrADSfxtN3c2c+bMmDt3bjd3CaRy4MABffjhh56sVinstq+W9LCkMyT9V0TcX/b8uXPnamRkpMouAZQYGhpqWev4ZbztMyT9p6RrJF0iabntSzrdHoBmVXnPvlDS/oh4NyJOSHpC0nX1tAWgblXCfqGkQxMejxbLvsT2KtsjtkfGxsYq7A5AFY1fjY+I4YgYioihgYGBpncHoIUqYT8sac6Ex7OLZQD6UJWwvyppnu3v2p4m6SeSttXTFoC6dTz0FhEnba+R9JzGh97WR8TbtXUGoFaVxtkj4hlJz9TUC4AG8XFZIAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkqg0iyvQS3fddVdp/ZFHHmlZ27p1a+m6ixcv7qinflYp7LYPSDou6XNJJyNiqI6mANSvjjP7P0fEhzVsB0CDeM8OJFE17CHp97Zfs71qsifYXmV7xPbI2NhYxd0B6FTVsC+KiB9IukbSats//OoTImI4IoYiYmhgYKDi7gB0qlLYI+JwcXtU0hZJC+toCkD9Og677bNtn3PqvqQfS9pVV2MA6lXlavygpC22T23n8YjYXktX+JLNmzeX1jdu3Niydtttt5Wue8UVV3TUUzeMjo6W1tetW1da//TTT1vWXnzxxdJ1GWefICLelfSPNfYCoEEMvQFJEHYgCcIOJEHYgSQIO5AEX3HtA3v37i2t33HHHaX1gwcPtqw9/fTTpeu2+6rnkiVLSutNeuihh0rrn3zySWl92rRpLWuXXXZZRz19m3FmB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkGGfvgrKvWkrS7bffXlovG0dvZ/78+aX1q666quNtV3X8+PHS+qZNmypt/9xzz21ZW7p0aaVtfxtxZgeSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJBhn74J248nbtzf3C9xr1qwprZ955pmN7budhx9+uLR+6NChStufPXt2pfVPN5zZgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJxtm7oN04ejHtdcfmzZvXsrZs2bJK227Sc889V1qvelxuvPHGSuufbtqe2W2vt33U9q4Jy6bbft72vuL2/GbbBFDVVF7G/0bS1V9ZdrekFyJinqQXiscA+ljbsEfEDknHvrL4OkkbivsbJF1fc18AatbpBbrBiDhS3H9f0mCrJ9peZXvE9sjY2FiHuwNQVeWr8RERkqKkPhwRQxExNDAwUHV3ADrUadg/sD1Lkorbo/W1BKAJnYZ9m6SVxf2Vksrn/QXQc23H2W1vlHSlpJm2RyX9QtL9kn5n+yZJByXd0GST/e6jjz4qrd9yyy2N7v/yyy9vWZs+fXqj+26nbA71Kr+HL0kzZsword96662Vtn+6aRv2iFjeovSjmnsB0CA+LgskQdiBJAg7kARhB5Ig7EASfMW1Bk899VRpvd1PSbf7Kme7IaYHH3ywtN5LW7ZsaVkbHR2ttO1FixZVWj8bzuxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kATj7FO0Z8+elrW1a9dW2vb4j/20dumll5bW243D99Ljjz/estbuv7udsq/24us4swNJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEoyzT9HWra1/Gv/EiROVtt3u++zXXnttab3se+GzZ8/uqKdT9u7dW1ofHh4urT/77LMta0zJ3F2c2YEkCDuQBGEHkiDsQBKEHUiCsANJEHYgCcbZp+jjjz/u2b7vvPPO0vp9993XsnbzzTeXrrtv377S+ssvv1xabzddNfpH2zO77fW2j9reNWHZvbYP295Z/C1ptk0AVU3lZfxvJF09yfJfRcSC4u+ZetsCULe2YY+IHZKOdaEXAA2qcoFuje03i5f557d6ku1Vtkdsj4yNjVXYHYAqOg37Oknfk7RA0hFJD7V6YkQMR8RQRAwNDAx0uDsAVXUU9oj4ICI+j4gvJP1a0sJ62wJQt47CbnvWhIdLJe1q9VwA/aHtOLvtjZKulDTT9qikX0i60vYCSSHpgKSfNdhjX1i9enXL2iuvvFK67o4dO+pu50vK5n9/4IEHStet+p3yJrX7Pfxp06Z1qZPTQ9uwR8TySRY/2kAvABrEx2WBJAg7kARhB5Ig7EAShB1Igq+4TtGcOXNa1l566aXSdZ988snS+vbt20vru3fvLq2XTSf9xRdflK5bNmzXtHZTNs+fP7+0fs4559TZzmmPMzuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJME4excsW7asUr2dY8da/0TgZ599VrruO++8U1pft25daf2xxx4rrZeNpbf7eu3ixYtL6/hmOLMDSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKMs58Gpk+f3vG6g4ODpfVt27Z1vO12zjrrrNL6ihUrGtt3RpzZgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJxtlR6o033mhs25s3by6tX3TRRY3tO6O2Z3bbc2y/aHu37bdtry2WT7f9vO19xe35zbcLoFNTeRl/UtIdEXGJpH+StNr2JZLulvRCRMyT9ELxGECfahv2iDgSEa8X949L2iPpQknXSdpQPG2DpOubahJAdd/oAp3tuZK+L+lPkgYj4khRel/SpB+ytr3K9ojtkbGxsQqtAqhiymG3/R1JmyT9PCL+PLEW478qOOkvC0bEcEQMRcTQwMBApWYBdG5KYbd9psaD/lhEnLqE+oHtWUV9lqSjzbQIoA5th948/nu/j0raExG/nFDaJmmlpPuL262NdIhGnTx5srT+3nvvldbbTbtc9nPR5513Xum6qNdUxtkvl/RTSW/Z3lksu0fjIf+d7ZskHZR0QzMtAqhD27BHxB8ltfrn+Uf1tgOgKXxcFkiCsANJEHYgCcIOJEHYgST4imty27dvL63v37+/tN5u2uWZM2e2rC1cuLB0XdSLMzuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJME4e3K7d+9udPuLFi1qdPuYOs7sQBKEHUiCsANJEHYgCcIOJEHYgSQIO5AE4+zJXXDBBaX1GTNmlNYvvvji0voTTzzxjXtCMzizA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EASU5mffY6k30oalBSShiPiYdv3SvpXSWPFU++JiGeaahTNWLFiRaU6vj2m8qGak5LuiIjXbZ8j6TXbzxe1X0XEfzTXHoC6TGV+9iOSjhT3j9veI+nCphsDUK9v9J7d9lxJ35f0p2LRGttv2l5v+/wW66yyPWJ7ZGxsbLKnAOiCKYfd9nckbZL084j4s6R1kr4naYHGz/wPTbZeRAxHxFBEDA0MDNTQMoBOTCnsts/UeNAfi4jNkhQRH0TE5xHxhaRfS2KWPqCPtQ27x6fpfFTSnoj45YTlsyY8bamkXfW3B6AuU7kaf7mkn0p6y/bOYtk9kpbbXqDx4bgDkn7WSIcAajGVq/F/lDTZJNyMqQPfInyCDkiCsANJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kIQjons7s8ckHZywaKakD7vWwDfTr731a18SvXWqzt7+LiIm/f23rob9azu3RyJiqGcNlOjX3vq1L4neOtWt3ngZDyRB2IEkeh324R7vv0y/9tavfUn01qmu9NbT9+wAuqfXZ3YAXULYgSR6EnbbV9v+X9v7bd/dix5asX3A9lu2d9oe6XEv620ftb1rwrLptp+3va+4nXSOvR71dq/tw8Wx22l7SY96m2P7Rdu7bb9te22xvKfHrqSvrhy3rr9nt32GpP+TdJWkUUmvSloeEbu72kgLtg9IGoqInn8Aw/YPJf1F0m8j4h+KZQ9IOhYR9xf/UJ4fEf/WJ73dK+kvvZ7Gu5itaNbEacYlXS/pX9TDY1fS1w3qwnHrxZl9oaT9EfFuRJyQ9ISk63rQR9+LiB2Sjn1l8XWSNhT3N2j8f5aua9FbX4iIIxHxenH/uKRT04z39NiV9NUVvQj7hZIOTXg8qv6a7z0k/d72a7ZX9bqZSQxGxJHi/vuSBnvZzCTaTuPdTV+ZZrxvjl0n059XxQW6r1sUET+QdI2k1cXL1b4U4+/B+mnsdErTeHfLJNOM/1Uvj12n059X1YuwH5Y0Z8Lj2cWyvhARh4vbo5K2qP+mov7g1Ay6xe3RHvfzV/00jfdk04yrD45dL6c/70XYX5U0z/Z3bU+T9BNJ23rQx9fYPru4cCLbZ0v6sfpvKuptklYW91dK2trDXr6kX6bxbjXNuHp87Ho+/XlEdP1P0hKNX5F/R9K/96KHFn39vaT/Kf7e7nVvkjZq/GXdZxq/tnGTpBmSXpC0T9IfJE3vo97+W9Jbkt7UeLBm9ai3RRp/if6mpJ3F35JeH7uSvrpy3Pi4LJAEF+iAJAg7kARhB5Ig7EAShB1IgrADSRB2IIn/B0fzK/Yimhx/AAAAAElFTkSuQmCC\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gZ-hK0S5xIl7",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ca8221f9-0246-42c3-b72c-1c916a314f77"
      },
      "source": [
        "\n",
        "model = keras.Sequential()\n",
        "model.add(Input((28,28)))\n",
        "model.add(Flatten())\n",
        "model.add(Dense(10, activation='relu'))\n",
        "model.add(Dense(10, activation='relu'))\n",
        "model.add(Dense(10, activation='softmax'))\n",
        "\n",
        "model.compile(optimizer=\"adam\", loss=\"sparse_categorical_crossentropy\", metrics=[\"accuracy\"])\n",
        "model.summary()\n",
        "\n",
        "model.fit(train_data_generator, validation_data=valid_data_generator, epochs=5, verbose=1)\n",
        "\n",
        "\n",
        "loss, acc = model.evaluate(test_data_generator)\n",
        "print(\"loss=\",loss)\n",
        "print(\"acc=\",acc)\n",
        "\n",
        "y_ = model.predict(test_data_generator)\n",
        "print(y_.shape)\n",
        "\n",
        "\n",
        "batch_x, batch_y = test_data_generator.__getitem__(0)\n",
        "y_ = model.predict(batch_x)\n",
        "predicted = np.argmax(y_, axis=1)\n",
        "\n",
        "print(predicted)"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"sequential_2\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " flatten_2 (Flatten)         (None, 784)               0         \n",
            "                                                                 \n",
            " dense_6 (Dense)             (None, 10)                7850      \n",
            "                                                                 \n",
            " dense_7 (Dense)             (None, 10)                110       \n",
            "                                                                 \n",
            " dense_8 (Dense)             (None, 10)                110       \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 8,070\n",
            "Trainable params: 8,070\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "Epoch 1/5\n",
            "1500/1500 [==============================] - 5s 3ms/step - loss: 0.6116 - accuracy: 0.8158 - val_loss: 0.3462 - val_accuracy: 0.9018\n",
            "Epoch 2/5\n",
            "1500/1500 [==============================] - 5s 3ms/step - loss: 0.3320 - accuracy: 0.9052 - val_loss: 0.2814 - val_accuracy: 0.9216\n",
            "Epoch 3/5\n",
            "1500/1500 [==============================] - 5s 3ms/step - loss: 0.2911 - accuracy: 0.9174 - val_loss: 0.2582 - val_accuracy: 0.9274\n",
            "Epoch 4/5\n",
            "1500/1500 [==============================] - 5s 3ms/step - loss: 0.2701 - accuracy: 0.9238 - val_loss: 0.2532 - val_accuracy: 0.9310\n",
            "Epoch 5/5\n",
            "1500/1500 [==============================] - 5s 3ms/step - loss: 0.2555 - accuracy: 0.9281 - val_loss: 0.2468 - val_accuracy: 0.9320\n",
            "313/313 [==============================] - 1s 2ms/step - loss: 0.2637 - accuracy: 0.9255\n",
            "loss= 0.26371678709983826\n",
            "acc= 0.9254999756813049\n",
            "(10000, 10)\n",
            "[7 3 4 0 9 5 1 2 2 6 0 8 4 9 3 8 3 0 0 2 5 0 8 9 1 9 8 2 6 1 6 4]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "fCu4-FZXC_Gj"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}