StyleGAN2_ADA_training.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "StyleGAN2-ADA_training.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "sbWQ38fBoo4h",
        "g0oZRRs2KO5A",
        "ruo5KFL4yw0j",
        "Aj-3t7dM98dj",
        "iL6_7wnZKo3w",
        "uDtZ9CVSBvkZ",
        "BWxFpc-QkJOH",
        "UEzOo7gtBNvE",
        "imBVQY2bQyFN",
        "lUKiyWY22dzf"
      ],
      "machine_shape": "hm"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8UDVO2FGKIQw"
      },
      "source": [
        "# StyleGAN2-ADA: training a model from scratch\n",
        "\n",
        "References:\n",
        "- this notebook: https://github.com/woctezuma/steam-stylegan2-ada\n",
        "- the original StyleGAN2-ADA repository: https://github.com/NVlabs/stylegan2-ada\n",
        "- my fork of StyleGAN2-ADA: https://github.com/woctezuma/stylegan2-ada"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VgwgYzlkyCrU"
      },
      "source": [
        "## Mount Google Drive"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GmokzZyCKA5o",
        "outputId": "5ebc60c7-16b0-4de3-89b6-909b60bf89c8",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "%pip install Google-Colab-Transfer"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting Google-Colab-Transfer\n",
            "  Downloading https://files.pythonhosted.org/packages/a0/90/76fc38bcad442018977ed0e4e663473ef56a4d15395b2aa09055e8c49185/Google_Colab_Transfer-0.1.6-py3-none-any.whl\n",
            "Installing collected packages: Google-Colab-Transfer\n",
            "Successfully installed Google-Colab-Transfer-0.1.6\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PXvJm3hCKWGf",
        "outputId": "45bcddd0-1c2a-4a60-cf79-50aea2085a3e",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "import colab_transfer\n",
        "\n",
        "gd = colab_transfer.get_path_to_home_of_google_drive()\n",
        "lm = colab_transfer.get_path_to_home_of_local_machine()\n",
        "\n",
        "colab_transfer.mount_google_drive()"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive/\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sbWQ38fBoo4h"
      },
      "source": [
        "## Switch to Tensorflow 1.x"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tJiCQbhy51Nt"
      },
      "source": [
        "> TensorFlow 2.x is not supported.\n",
        "\n",
        "Reference: https://github.com/NVlabs/stylegan2-ada#requirements\n",
        "\n",
        "> Colab uses TensorFlow 2.x by default, though you can switch to 1.x by the method shown below.\n",
        "\n",
        "Reference: https://colab.research.google.com/notebooks/tensorflow_version.ipynb"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kHL4_p1xnoDY",
        "outputId": "37fa4004-bef1-4af0-9b47-54815a5b894c",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "%tensorflow_version 1.x"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "TensorFlow 1.x selected.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "g0oZRRs2KO5A"
      },
      "source": [
        "## Install my fork of StyleGAN2-ADA\n",
        "\n",
        "Switch to the branch `google-colab`."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sKYZ49zf41Sv",
        "outputId": "9daddd15-45b6-45f3-f60e-457813639312",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "%cd /content/\n",
        "\n",
        "%rm -rf stylegan2-ada/\n",
        "!git clone https://github.com/woctezuma/stylegan2-ada.git\n",
        "\n",
        "%cd stylegan2-ada/\n",
        "!git checkout google-colab\n",
        "\n",
        "%cd /content/"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content\n",
            "Cloning into 'stylegan2-ada'...\n",
            "remote: Enumerating objects: 17, done.\u001b[K\n",
            "remote: Counting objects: 100% (17/17), done.\u001b[K\n",
            "remote: Compressing objects: 100% (15/15), done.\u001b[K\n",
            "remote: Total 112 (delta 7), reused 10 (delta 2), pack-reused 95\u001b[K\n",
            "Receiving objects: 100% (112/112), 1.39 MiB | 29.06 MiB/s, done.\n",
            "Resolving deltas: 100% (48/48), done.\n",
            "/content/stylegan2-ada\n",
            "Branch 'google-colab' set up to track remote branch 'google-colab' from 'origin'.\n",
            "Switched to a new branch 'google-colab'\n",
            "/content\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ruo5KFL4yw0j"
      },
      "source": [
        "## Prepare datasets (once)\n",
        "\n",
        "Reference: https://github.com/NVlabs/stylegan2-ada#preparing-datasets"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "o6ujXByp9gWm"
      },
      "source": [
        "folder_name = 'datasets/steam/'"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Aj-3t7dM98dj"
      },
      "source": [
        "### Import image data from Google Drive"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OwMz7FvWboio"
      },
      "source": [
        "Data consists of 14,000 vertical Steam banners, resized from 300x450 to 256x256 resolution."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "56Vt8uJ9I78P"
      },
      "source": [
        "archive_name = 'resized_vertical_steam_banners_14k.tar'"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UyTYYYpU8mie"
      },
      "source": [
        "# colab_transfer.copy_file(file_name=archive_name,\n",
        "#                          source = gd + 'data/')"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "04SYpwjy8_5G"
      },
      "source": [
        "# !echo tar -xf {archive_name}\n",
        "# !tar -xf {archive_name}"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iL6_7wnZKo3w"
      },
      "source": [
        "### Prepare datasets for StyleGAN2"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "K1Qd8VPH9djO"
      },
      "source": [
        "custom_dataset = '/content/' + folder_name\n",
        "custom_images = '/content/data/resized_vertical_steam_banners'"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "whUYxCDgK66P"
      },
      "source": [
        "# !python stylegan2-ada/dataset_tool.py create_from_images {custom_dataset} {custom_images}"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uDtZ9CVSBvkZ"
      },
      "source": [
        "### Archive prepared datasets to Google Drive"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5b5PxndUQjvp"
      },
      "source": [
        "# colab_transfer.copy_folder(folder_name,\n",
        "#                            source=lm,\n",
        "#                            destination=gd)"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "K-gghiqKRddO"
      },
      "source": [
        "## Import previously prepared datasets from Google Drive"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8SE22FIqBStI",
        "outputId": "9f241d12-9c50-458c-f778-411061f3f309",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "colab_transfer.copy_folder(folder_name)"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Copying /content/drive/My Drive/datasets/steam/ to /content/datasets/steam/\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RWpIjP7hKzJZ"
      },
      "source": [
        "## Train new networks"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "McdnLndylAxa"
      },
      "source": [
        "### Computation time"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bzU6-RNSQ8kT"
      },
      "source": [
        "A tick requires 4 minutes of computation time, so we save a snapshot every 6 ticks (`--snap=6`) in order to save every 24 minutes.\n",
        "\n",
        "Processing 1 kimg requires 1 min of computation time, so:\n",
        "-   the total training time for **5 Mimg** should be 3.5 days of computation,\n",
        "-   the total training time for **25 Mimg** should be 2.5 weeks of computation."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BWxFpc-QkJOH"
      },
      "source": [
        "### Documentation for transfer learning"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Qjlohd3uNeEY"
      },
      "source": [
        "```\n",
        "transfer learning:\n",
        "  --resume RESUME       Resume from network pickle (default: noresume)\n",
        "  --freezed INT         Freeze-D (default: 0 discriminator layers)\n",
        "```\n",
        "```\n",
        "transfer learning source networks (--resume):\n",
        "  ffhq256        FFHQ trained at 256x256 resolution.\n",
        "  ffhq512        FFHQ trained at 512x512 resolution.\n",
        "  ffhq1024       FFHQ trained at 1024x1024 resolution.\n",
        "  celebahq256    CelebA-HQ trained at 256x256 resolution.\n",
        "  lsundog256     LSUN Dog trained at 256x256 resolution.\n",
        "  <path or URL>  Custom network pickle.\n",
        "```"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UEzOo7gtBNvE"
      },
      "source": [
        "### Training recommendations"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sQfYzLRvBSD3"
      },
      "source": [
        "-   about `--gamma`:\n",
        "> The training configuration can be further customized with additional arguments. Common examples:\n",
        "> * `--gamma=10` overrides R1 gamma. We strongly recommend trying out at least a few different values for each new dataset.\n",
        "\n",
        "-   about `--cfg`:\n",
        "> Please note that `--cfg=auto` only serves as a reasonable first guess for the hyperparameters &mdash; it does not necessarily lead to optimal results for a given dataset. For example, `--cfg=stylegan2` yields considerably better FID  for FFHQ-140k at 1024x1024 than illustrated above. We recommend trying out at least a few different values of `--gamma` for each new dataset.\n",
        "\n",
        "-   about `--metrics`:\n",
        "> By default, `train.py` will automatically compute FID for each network pickle. We strongly recommend inspecting `metric-fid50k_full.txt` at regular intervals to monitor the training progress. When desired, the automatic computation can be disabled with `--metrics none` to speed up the training.\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "imBVQY2bQyFN"
      },
      "source": [
        "### Training settings"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VbIhukS9ZK4x"
      },
      "source": [
        "Resolution is automatically determined from the dataset.\n",
        "Other settings are chosen as follows:\n",
        "- horizontal mirroring, via `--mirror=true` (instead of default to False),\n",
        "- no EMA rampup, via `--cfg=auto_no_ramp`,\n",
        "- cfg map = 8 (instead of 2), via `--cfg_map=8`,\n",
        "- freezed D (the first 10 layers, as in the article for resolution 256), via `--freezed=10`.\n",
        "\n",
        "References:\n",
        "-   https://github.com/NVlabs/stylegan2-ada#training-new-networks\n",
        "-   https://raw.githubusercontent.com/NVlabs/stylegan2-ada/main/docs/train-help.txt"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "apnOxk7UZFIL"
      },
      "source": [
        "### Train\n",
        "\n",
        "NB: to ensure your Colab session stays connected, follow: https://stackoverflow.com/a/58275370/"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lUKiyWY22dzf"
      },
      "source": [
        "#### Initial run (transfer learning)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_yKuIVhicPyS"
      },
      "source": [
        "Start with transfer learning from a model trained on a **diverse** dataset (`lsundog256`).\n",
        "\n",
        "Caveat: the upstream model was trained with cfg map = 8."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "f8TWmeqExkda"
      },
      "source": [
        "# !python stylegan2-ada/train.py \\\n",
        "#  --outdir='/content/drive/My Drive/training-runs' \\\n",
        "#  --snap=6 \\\n",
        "#  --data='/content/datasets/steam' \\\n",
        "#  --mirror=true \\\n",
        "#  --metrics=none \\\n",
        "#  --cfg=auto_no_ramp \\\n",
        "#  --cfg_map=8 \\\n",
        "#  --kimg=5000 \\\n",
        "#  --resume=lsundog256 \\\n",
        "#  --freezed=10 \\\n",
        " "
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2FGozskE2gGM"
      },
      "source": [
        "#### Following runs (resume from latest snapshot)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SvLmB4OrcSHS"
      },
      "source": [
        "Then,  **automatically** resume from the latest pickle snapshot.\n",
        "\n",
        "Caveat: you will have to **manually** specify the augmentation strength, as found in `log.txt`."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7s7cdAcU2l3x",
        "outputId": "3576be3f-da00-44c3-d19c-6e9c98f7f8b1",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "!python stylegan2-ada/train.py \\\n",
        " --outdir='/content/drive/My Drive/training-runs' \\\n",
        " --snap=6 \\\n",
        " --data='/content/datasets/steam' \\\n",
        " --mirror=true \\\n",
        " --metrics=none \\\n",
        " --cfg=auto_no_ramp \\\n",
        " --cfg_map=8 \\\n",
        " --kimg=5000 \\\n",
        " --resume=latest \\\n",
        " --p=0.713 \\\n",
        " --freezed=10 \\\n",
        " "
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tcmalloc: large alloc 4294967296 bytes == 0x8250000 @  0x7f9e8d0c3001 0x7f9e8a2ec765 0x7f9e8a350bb0 0x7f9e8a352a4f 0x7f9e8a3e9048 0x50a4a5 0x50cc96 0x507be4 0x508ec2 0x594a01 0x549e8f 0x5515c1 0x5a9dac 0x50a433 0x50cc96 0x507be4 0x588e5c 0x59fd0e 0x50d256 0x507be4 0x588e5c 0x59fd0e 0x50d256 0x507be4 0x588e5c 0x59fd0e 0x50d256 0x5095c8 0x50a2fd 0x50beb4 0x507be4\n",
            "tcmalloc: large alloc 4294967296 bytes == 0x7f9cbac20000 @  0x7f9e8d0c11e7 0x7f9e8a2ec5e1 0x7f9e8a350c78 0x7f9e8a350f37 0x7f9e8a3e8f28 0x50a4a5 0x50cc96 0x507be4 0x509900 0x50a2fd 0x50cc96 0x507be4 0x509900 0x50a2fd 0x50cc96 0x507be4 0x509900 0x50a2fd 0x50cc96 0x5095c8 0x50a2fd 0x50beb4 0x507be4 0x509900 0x50a2fd 0x50beb4 0x507be4 0x509900 0x50a2fd 0x50cc96 0x507be4\n",
            "tcmalloc: large alloc 4294967296 bytes == 0x7f9bb9c1e000 @  0x7f9e8d0c11e7 0x7f9e8a2ec5e1 0x7f9e8a350c78 0x7f9e8a350f37 0x7f9e483560c5 0x7f9e47cd9902 0x7f9e47cd9eb2 0x7f9e47c92c3e 0x50a12f 0x50beb4 0x5095c8 0x50a2fd 0x50beb4 0x507be4 0x588c8b 0x59fd0e 0x50d256 0x507be4 0x509900 0x50a2fd 0x50beb4 0x507be4 0x509900 0x50a2fd 0x50beb4 0x5095c8 0x50a2fd 0x50beb4 0x507be4 0x508ec2 0x594a01\n",
            "\n",
            "Training options:\n",
            "{\n",
            "  \"G_args\": {\n",
            "    \"func_name\": \"training.networks.G_main\",\n",
            "    \"fmap_base\": 8192,\n",
            "    \"fmap_max\": 512,\n",
            "    \"mapping_layers\": 8,\n",
            "    \"num_fp16_res\": 4,\n",
            "    \"conv_clamp\": 256\n",
            "  },\n",
            "  \"D_args\": {\n",
            "    \"func_name\": \"training.networks.D_main\",\n",
            "    \"mbstd_group_size\": 4,\n",
            "    \"fmap_base\": 8192,\n",
            "    \"fmap_max\": 512,\n",
            "    \"num_fp16_res\": 4,\n",
            "    \"conv_clamp\": 256,\n",
            "    \"freeze_layers\": 10\n",
            "  },\n",
            "  \"G_opt_args\": {\n",
            "    \"beta1\": 0.0,\n",
            "    \"beta2\": 0.99,\n",
            "    \"learning_rate\": 0.0025\n",
            "  },\n",
            "  \"D_opt_args\": {\n",
            "    \"beta1\": 0.0,\n",
            "    \"beta2\": 0.99,\n",
            "    \"learning_rate\": 0.0025\n",
            "  },\n",
            "  \"loss_args\": {\n",
            "    \"func_name\": \"training.loss.stylegan2\",\n",
            "    \"r1_gamma\": 0.8192\n",
            "  },\n",
            "  \"augment_args\": {\n",
            "    \"class_name\": \"training.augment.AdaptiveAugment\",\n",
            "    \"tune_heuristic\": \"rt\",\n",
            "    \"tune_target\": 0.6,\n",
            "    \"initial_strength\": 0.713,\n",
            "    \"apply_func\": \"training.augment.augment_pipeline\",\n",
            "    \"apply_args\": {\n",
            "      \"xflip\": 1,\n",
            "      \"rotate90\": 1,\n",
            "      \"xint\": 1,\n",
            "      \"scale\": 1,\n",
            "      \"rotate\": 1,\n",
            "      \"aniso\": 1,\n",
            "      \"xfrac\": 1,\n",
            "      \"brightness\": 1,\n",
            "      \"contrast\": 1,\n",
            "      \"lumaflip\": 1,\n",
            "      \"hue\": 1,\n",
            "      \"saturation\": 1\n",
            "    },\n",
            "    \"tune_kimg\": 100\n",
            "  },\n",
            "  \"num_gpus\": 1,\n",
            "  \"image_snapshot_ticks\": 6,\n",
            "  \"network_snapshot_ticks\": 6,\n",
            "  \"train_dataset_args\": {\n",
            "    \"path\": \"/content/datasets/steam\",\n",
            "    \"max_label_size\": 0,\n",
            "    \"resolution\": 256,\n",
            "    \"mirror_augment\": true\n",
            "  },\n",
            "  \"metric_arg_list\": [],\n",
            "  \"metric_dataset_args\": {\n",
            "    \"path\": \"/content/datasets/steam\",\n",
            "    \"max_label_size\": 0,\n",
            "    \"resolution\": 256,\n",
            "    \"mirror_augment\": true\n",
            "  },\n",
            "  \"total_kimg\": 5000,\n",
            "  \"minibatch_size\": 16,\n",
            "  \"minibatch_gpu\": 16,\n",
            "  \"G_smoothing_kimg\": 5.0,\n",
            "  \"G_smoothing_rampup\": null,\n",
            "  \"resume_pkl\": \"latest\",\n",
            "  \"run_dir\": \"/content/drive/My Drive/training-runs/00025-steam-mirror-auto_no_ramp1-kimg5000-p0.713-resumecustom-freezed10\"\n",
            "}\n",
            "\n",
            "Output directory:  /content/drive/My Drive/training-runs/00025-steam-mirror-auto_no_ramp1-kimg5000-p0.713-resumecustom-freezed10\n",
            "Training data:     /content/datasets/steam\n",
            "Training length:   5000 kimg\n",
            "Resolution:        256\n",
            "Number of GPUs:    1\n",
            "\n",
            "Creating output directory...\n",
            "Loading training set...\n",
            "tcmalloc: large alloc 4294967296 bytes == 0x7f9aaa428000 @  0x7f9e8d0c3001 0x7f9e8a2ec765 0x7f9e8a350bb0 0x7f9e8a352a4f 0x7f9e8a3e9048 0x50a4a5 0x50cc96 0x507be4 0x508ec2 0x594a01 0x549e8f 0x5515c1 0x5a9dac 0x50a433 0x50cc96 0x507be4 0x588e5c 0x59fd0e 0x50d256 0x507be4 0x588e5c 0x59fd0e 0x50d256 0x507be4 0x588e5c 0x59fd0e 0x50d256 0x5095c8 0x50a2fd 0x50beb4 0x507be4\n",
            "tcmalloc: large alloc 4294967296 bytes == 0x7f99a9c28000 @  0x7f9e8d0c11e7 0x7f9e8a2ec5e1 0x7f9e8a350c78 0x7f9e8a350f37 0x7f9e8a3e8f28 0x50a4a5 0x50cc96 0x507be4 0x509900 0x50a2fd 0x50cc96 0x507be4 0x509900 0x50a2fd 0x50cc96 0x507be4 0x509900 0x50a2fd 0x50cc96 0x5095c8 0x50a2fd 0x50beb4 0x507be4 0x509900 0x50a2fd 0x50beb4 0x507be4 0x509900 0x50a2fd 0x50cc96 0x507be4\n",
            "tcmalloc: large alloc 4294967296 bytes == 0x7f99a9c28000 @  0x7f9e8d0c11e7 0x7f9e8a2ec5e1 0x7f9e8a350c78 0x7f9e8a350f37 0x7f9e483560c5 0x7f9e47cd9902 0x7f9e47cd9eb2 0x7f9e47c92c3e 0x50a12f 0x50beb4 0x5095c8 0x50a2fd 0x50beb4 0x507be4 0x588c8b 0x59fd0e 0x50d256 0x507be4 0x509900 0x50a2fd 0x50beb4 0x507be4 0x509900 0x50a2fd 0x50beb4 0x5095c8 0x50a2fd 0x50beb4 0x507be4 0x508ec2 0x594a01\n",
            "Image shape: [3, 256, 256]\n",
            "Label shape: [0]\n",
            "\n",
            "Constructing networks...\n",
            "Setting up TensorFlow plugin \"fused_bias_act.cu\": Compiling... Loading... Done.\n",
            "Setting up TensorFlow plugin \"upfirdn_2d.cu\": Compiling... Loading... Done.\n",
            "Resuming from kimg = 4615.0\n",
            "Resuming from \"/content/drive/My Drive/training-runs/00024-steam-mirror-auto_no_ramp1-kimg5000-p0.677-resumecustom-freezed10/network-snapshot-004615.pkl\"\n",
            "\n",
            "G                             Params    OutputShape         WeightShape     \n",
            "---                           ---       ---                 ---             \n",
            "latents_in                    -         (?, 512)            -               \n",
            "labels_in                     -         (?, 0)              -               \n",
            "G_mapping/Normalize           -         (?, 512)            -               \n",
            "G_mapping/Dense0              262656    (?, 512)            (512, 512)      \n",
            "G_mapping/Dense1              262656    (?, 512)            (512, 512)      \n",
            "G_mapping/Dense2              262656    (?, 512)            (512, 512)      \n",
            "G_mapping/Dense3              262656    (?, 512)            (512, 512)      \n",
            "G_mapping/Dense4              262656    (?, 512)            (512, 512)      \n",
            "G_mapping/Dense5              262656    (?, 512)            (512, 512)      \n",
            "G_mapping/Dense6              262656    (?, 512)            (512, 512)      \n",
            "G_mapping/Dense7              262656    (?, 512)            (512, 512)      \n",
            "G_mapping/Broadcast           -         (?, 14, 512)        -               \n",
            "dlatent_avg                   -         (512,)              -               \n",
            "Truncation/Lerp               -         (?, 14, 512)        -               \n",
            "G_synthesis/4x4/Const         8192      (?, 512, 4, 4)      (1, 512, 4, 4)  \n",
            "G_synthesis/4x4/Conv          2622465   (?, 512, 4, 4)      (3, 3, 512, 512)\n",
            "G_synthesis/4x4/ToRGB         264195    (?, 3, 4, 4)        (1, 1, 512, 3)  \n",
            "G_synthesis/8x8/Conv0_up      2622465   (?, 512, 8, 8)      (3, 3, 512, 512)\n",
            "G_synthesis/8x8/Conv1         2622465   (?, 512, 8, 8)      (3, 3, 512, 512)\n",
            "G_synthesis/8x8/Upsample      -         (?, 3, 8, 8)        -               \n",
            "G_synthesis/8x8/ToRGB         264195    (?, 3, 8, 8)        (1, 1, 512, 3)  \n",
            "G_synthesis/16x16/Conv0_up    2622465   (?, 512, 16, 16)    (3, 3, 512, 512)\n",
            "G_synthesis/16x16/Conv1       2622465   (?, 512, 16, 16)    (3, 3, 512, 512)\n",
            "G_synthesis/16x16/Upsample    -         (?, 3, 16, 16)      -               \n",
            "G_synthesis/16x16/ToRGB       264195    (?, 3, 16, 16)      (1, 1, 512, 3)  \n",
            "G_synthesis/32x32/Conv0_up    2622465   (?, 512, 32, 32)    (3, 3, 512, 512)\n",
            "G_synthesis/32x32/Conv1       2622465   (?, 512, 32, 32)    (3, 3, 512, 512)\n",
            "G_synthesis/32x32/Upsample    -         (?, 3, 32, 32)      -               \n",
            "G_synthesis/32x32/ToRGB       264195    (?, 3, 32, 32)      (1, 1, 512, 3)  \n",
            "G_synthesis/64x64/Conv0_up    1442561   (?, 256, 64, 64)    (3, 3, 512, 256)\n",
            "G_synthesis/64x64/Conv1       721409    (?, 256, 64, 64)    (3, 3, 256, 256)\n",
            "G_synthesis/64x64/Upsample    -         (?, 3, 64, 64)      -               \n",
            "G_synthesis/64x64/ToRGB       132099    (?, 3, 64, 64)      (1, 1, 256, 3)  \n",
            "G_synthesis/128x128/Conv0_up  426369    (?, 128, 128, 128)  (3, 3, 256, 128)\n",
            "G_synthesis/128x128/Conv1     213249    (?, 128, 128, 128)  (3, 3, 128, 128)\n",
            "G_synthesis/128x128/Upsample  -         (?, 3, 128, 128)    -               \n",
            "G_synthesis/128x128/ToRGB     66051     (?, 3, 128, 128)    (1, 1, 128, 3)  \n",
            "G_synthesis/256x256/Conv0_up  139457    (?, 64, 256, 256)   (3, 3, 128, 64) \n",
            "G_synthesis/256x256/Conv1     69761     (?, 64, 256, 256)   (3, 3, 64, 64)  \n",
            "G_synthesis/256x256/Upsample  -         (?, 3, 256, 256)    -               \n",
            "G_synthesis/256x256/ToRGB     33027     (?, 3, 256, 256)    (1, 1, 64, 3)   \n",
            "---                           ---       ---                 ---             \n",
            "Total                         24767458                                      \n",
            "\n",
            "\n",
            "D                    Params    OutputShape         WeightShape     \n",
            "---                  ---       ---                 ---             \n",
            "images_in            -         (?, 3, 256, 256)    -               \n",
            "labels_in            -         (?, 0)              -               \n",
            "256x256/FromRGB      -         (?, 64, 256, 256)   -               \n",
            "256x256/Conv0        -         (?, 64, 256, 256)   -               \n",
            "256x256/Conv1_down   -         (?, 128, 128, 128)  -               \n",
            "256x256/Skip         -         (?, 128, 128, 128)  -               \n",
            "128x128/Conv0        -         (?, 128, 128, 128)  -               \n",
            "128x128/Conv1_down   -         (?, 256, 64, 64)    -               \n",
            "128x128/Skip         -         (?, 256, 64, 64)    -               \n",
            "64x64/Conv0          -         (?, 256, 64, 64)    -               \n",
            "64x64/Conv1_down     -         (?, 512, 32, 32)    -               \n",
            "64x64/Skip           -         (?, 512, 32, 32)    -               \n",
            "32x32/Conv0          2359808   (?, 512, 32, 32)    (3, 3, 512, 512)\n",
            "32x32/Conv1_down     2359808   (?, 512, 16, 16)    (3, 3, 512, 512)\n",
            "32x32/Skip           262144    (?, 512, 16, 16)    (1, 1, 512, 512)\n",
            "16x16/Conv0          2359808   (?, 512, 16, 16)    (3, 3, 512, 512)\n",
            "16x16/Conv1_down     2359808   (?, 512, 8, 8)      (3, 3, 512, 512)\n",
            "16x16/Skip           262144    (?, 512, 8, 8)      (1, 1, 512, 512)\n",
            "8x8/Conv0            2359808   (?, 512, 8, 8)      (3, 3, 512, 512)\n",
            "8x8/Conv1_down       2359808   (?, 512, 4, 4)      (3, 3, 512, 512)\n",
            "8x8/Skip             262144    (?, 512, 4, 4)      (1, 1, 512, 512)\n",
            "4x4/MinibatchStddev  -         (?, 513, 4, 4)      -               \n",
            "4x4/Conv             2364416   (?, 512, 4, 4)      (3, 3, 513, 512)\n",
            "4x4/Dense0           4194816   (?, 512)            (8192, 512)     \n",
            "Output               513       (?, 1)              (512, 1)        \n",
            "---                  ---       ---                 ---             \n",
            "Total                21505025                                      \n",
            "\n",
            "Exporting sample images...\n",
            "2020-11-04 08:48:34.432357: W tensorflow/core/framework/cpu_allocator_impl.cc:81] Allocation of 94371840 exceeds 10% of system memory.\n",
            "Replicating networks across 1 GPUs...\n",
            "Initializing augmentations...\n",
            "Setting up optimizers...\n",
            "Constructing training graph...\n",
            "Finalizing training ops...\n",
            "Initializing metrics...\n",
            "Training for 5000 kimg...\n",
            "\n",
            "tick 0     kimg 4615.1   time 2m 18s       sec/tick 35.3    sec/kimg 551.50  maintenance 102.4  gpumem 7.3   augment 0.712\n",
            "tick 1     kimg 4619.1   time 6m 33s       sec/tick 238.0   sec/kimg 59.03   maintenance 17.3   gpumem 7.3   augment 0.712\n",
            "tick 2     kimg 4623.1   time 10m 31s      sec/tick 238.2   sec/kimg 59.08   maintenance 0.0    gpumem 7.3   augment 0.715\n",
            "tick 3     kimg 4627.2   time 14m 29s      sec/tick 237.8   sec/kimg 58.98   maintenance 0.0    gpumem 7.3   augment 0.719\n",
            "tick 4     kimg 4631.2   time 18m 27s      sec/tick 237.9   sec/kimg 59.01   maintenance 0.0    gpumem 7.3   augment 0.726\n",
            "tick 5     kimg 4635.2   time 22m 24s      sec/tick 237.6   sec/kimg 58.92   maintenance 0.0    gpumem 7.3   augment 0.725\n",
            "tick 6     kimg 4639.3   time 26m 22s      sec/tick 237.9   sec/kimg 58.99   maintenance 0.0    gpumem 7.3   augment 0.728\n",
            "tick 7     kimg 4643.3   time 30m 27s      sec/tick 237.8   sec/kimg 58.99   maintenance 6.8    gpumem 7.3   augment 0.731\n",
            "tick 8     kimg 4647.3   time 34m 25s      sec/tick 238.0   sec/kimg 59.02   maintenance 0.0    gpumem 7.3   augment 0.732\n",
            "tick 9     kimg 4651.4   time 38m 23s      sec/tick 237.6   sec/kimg 58.93   maintenance 0.0    gpumem 7.3   augment 0.727\n",
            "tick 10    kimg 4655.4   time 42m 20s      sec/tick 237.7   sec/kimg 58.94   maintenance 0.0    gpumem 7.3   augment 0.726\n",
            "tick 11    kimg 4659.4   time 46m 18s      sec/tick 237.7   sec/kimg 58.96   maintenance 0.0    gpumem 7.3   augment 0.721\n",
            "tick 12    kimg 4663.4   time 50m 16s      sec/tick 237.9   sec/kimg 59.00   maintenance 0.0    gpumem 7.3   augment 0.723\n",
            "tick 13    kimg 4667.5   time 54m 20s      sec/tick 237.5   sec/kimg 58.90   maintenance 6.5    gpumem 7.3   augment 0.723\n",
            "tick 14    kimg 4671.5   time 58m 18s      sec/tick 237.7   sec/kimg 58.95   maintenance 0.0    gpumem 7.3   augment 0.716\n",
            "tick 15    kimg 4675.5   time 1h 02m 15s   sec/tick 237.9   sec/kimg 59.01   maintenance 0.0    gpumem 7.3   augment 0.725\n",
            "tick 16    kimg 4679.6   time 1h 06m 13s   sec/tick 238.0   sec/kimg 59.02   maintenance 0.0    gpumem 7.3   augment 0.725\n",
            "tick 17    kimg 4683.6   time 1h 10m 11s   sec/tick 237.7   sec/kimg 58.94   maintenance 0.0    gpumem 7.3   augment 0.731\n",
            "tick 18    kimg 4687.6   time 1h 14m 09s   sec/tick 238.2   sec/kimg 59.07   maintenance 0.0    gpumem 7.3   augment 0.738\n",
            "tick 19    kimg 4691.7   time 1h 18m 14s   sec/tick 238.4   sec/kimg 59.14   maintenance 6.4    gpumem 7.3   augment 0.735\n",
            "tick 20    kimg 4695.7   time 1h 22m 12s   sec/tick 238.0   sec/kimg 59.03   maintenance 0.0    gpumem 7.3   augment 0.741\n",
            "tick 21    kimg 4699.7   time 1h 26m 10s   sec/tick 237.5   sec/kimg 58.91   maintenance 0.0    gpumem 7.3   augment 0.736\n",
            "tick 22    kimg 4703.8   time 1h 30m 08s   sec/tick 237.9   sec/kimg 59.01   maintenance 0.0    gpumem 7.3   augment 0.721\n",
            "tick 23    kimg 4707.8   time 1h 34m 05s   sec/tick 237.6   sec/kimg 58.93   maintenance 0.0    gpumem 7.3   augment 0.725\n",
            "tick 24    kimg 4711.8   time 1h 38m 03s   sec/tick 237.6   sec/kimg 58.93   maintenance 0.0    gpumem 7.3   augment 0.733\n",
            "tick 25    kimg 4715.9   time 1h 42m 06s   sec/tick 237.5   sec/kimg 58.90   maintenance 6.1    gpumem 7.3   augment 0.733\n",
            "tick 26    kimg 4719.9   time 1h 46m 05s   sec/tick 238.3   sec/kimg 59.11   maintenance 0.0    gpumem 7.3   augment 0.738\n",
            "tick 27    kimg 4723.9   time 1h 50m 03s   sec/tick 237.9   sec/kimg 59.01   maintenance 0.0    gpumem 7.3   augment 0.740\n",
            "tick 28    kimg 4728.0   time 1h 54m 01s   sec/tick 237.9   sec/kimg 58.99   maintenance 0.0    gpumem 7.3   augment 0.726\n",
            "tick 29    kimg 4732.0   time 1h 57m 58s   sec/tick 237.8   sec/kimg 58.97   maintenance 0.0    gpumem 7.3   augment 0.728\n",
            "tick 30    kimg 4736.0   time 2h 01m 56s   sec/tick 238.0   sec/kimg 59.02   maintenance 0.0    gpumem 7.3   augment 0.724\n",
            "tick 31    kimg 4740.1   time 2h 06m 00s   sec/tick 237.9   sec/kimg 59.00   maintenance 6.2    gpumem 7.3   augment 0.728\n",
            "tick 32    kimg 4744.1   time 2h 09m 58s   sec/tick 237.9   sec/kimg 58.99   maintenance 0.0    gpumem 7.3   augment 0.720\n",
            "tick 33    kimg 4748.1   time 2h 13m 56s   sec/tick 237.5   sec/kimg 58.90   maintenance 0.0    gpumem 7.3   augment 0.721\n",
            "tick 34    kimg 4752.2   time 2h 17m 54s   sec/tick 237.9   sec/kimg 59.00   maintenance 0.0    gpumem 7.3   augment 0.719\n",
            "tick 35    kimg 4756.2   time 2h 21m 51s   sec/tick 237.6   sec/kimg 58.92   maintenance 0.0    gpumem 7.3   augment 0.723\n",
            "tick 36    kimg 4760.2   time 2h 25m 49s   sec/tick 237.9   sec/kimg 59.01   maintenance 0.0    gpumem 7.3   augment 0.737\n",
            "tick 37    kimg 4764.2   time 2h 29m 53s   sec/tick 237.7   sec/kimg 58.96   maintenance 6.2    gpumem 7.3   augment 0.733\n",
            "tick 38    kimg 4768.3   time 2h 33m 51s   sec/tick 237.8   sec/kimg 58.99   maintenance 0.0    gpumem 7.3   augment 0.737\n",
            "tick 39    kimg 4772.3   time 2h 37m 49s   sec/tick 238.0   sec/kimg 59.04   maintenance 0.0    gpumem 7.3   augment 0.732\n",
            "tick 40    kimg 4776.3   time 2h 41m 47s   sec/tick 238.1   sec/kimg 59.05   maintenance 0.0    gpumem 7.3   augment 0.732\n",
            "tick 41    kimg 4780.4   time 2h 45m 45s   sec/tick 237.8   sec/kimg 58.98   maintenance 0.0    gpumem 7.3   augment 0.739\n",
            "tick 42    kimg 4784.4   time 2h 49m 43s   sec/tick 237.8   sec/kimg 58.98   maintenance 0.0    gpumem 7.3   augment 0.735\n",
            "tick 43    kimg 4788.4   time 2h 53m 47s   sec/tick 238.1   sec/kimg 59.04   maintenance 6.2    gpumem 7.3   augment 0.731\n",
            "tick 44    kimg 4792.5   time 2h 57m 45s   sec/tick 237.7   sec/kimg 58.96   maintenance 0.0    gpumem 7.3   augment 0.734\n",
            "tick 45    kimg 4796.5   time 3h 01m 42s   sec/tick 237.6   sec/kimg 58.92   maintenance 0.0    gpumem 7.3   augment 0.736\n",
            "tick 46    kimg 4800.5   time 3h 05m 40s   sec/tick 238.0   sec/kimg 59.04   maintenance 0.0    gpumem 7.3   augment 0.733\n",
            "tick 47    kimg 4804.6   time 3h 09m 38s   sec/tick 237.9   sec/kimg 59.01   maintenance 0.0    gpumem 7.3   augment 0.736\n",
            "tick 48    kimg 4808.6   time 3h 13m 37s   sec/tick 238.1   sec/kimg 59.05   maintenance 0.0    gpumem 7.3   augment 0.728\n",
            "tick 49    kimg 4812.6   time 3h 17m 40s   sec/tick 237.4   sec/kimg 58.89   maintenance 6.2    gpumem 7.3   augment 0.725\n",
            "tick 50    kimg 4816.7   time 3h 21m 38s   sec/tick 237.8   sec/kimg 58.99   maintenance 0.0    gpumem 7.3   augment 0.721\n",
            "tick 51    kimg 4820.7   time 3h 25m 36s   sec/tick 237.8   sec/kimg 58.98   maintenance 0.0    gpumem 7.3   augment 0.721\n",
            "tick 52    kimg 4824.7   time 3h 29m 33s   sec/tick 237.5   sec/kimg 58.91   maintenance 0.0    gpumem 7.3   augment 0.717\n",
            "tick 53    kimg 4828.8   time 3h 33m 31s   sec/tick 237.4   sec/kimg 58.87   maintenance 0.0    gpumem 7.3   augment 0.717\n",
            "tick 54    kimg 4832.8   time 3h 37m 29s   sec/tick 237.8   sec/kimg 58.98   maintenance 0.0    gpumem 7.3   augment 0.721\n",
            "tick 55    kimg 4836.8   time 3h 41m 32s   sec/tick 237.8   sec/kimg 58.97   maintenance 6.2    gpumem 7.3   augment 0.723\n",
            "tick 56    kimg 4840.9   time 3h 45m 30s   sec/tick 237.6   sec/kimg 58.92   maintenance 0.0    gpumem 7.3   augment 0.721\n",
            "tick 57    kimg 4844.9   time 3h 49m 27s   sec/tick 237.4   sec/kimg 58.87   maintenance 0.0    gpumem 7.3   augment 0.726\n",
            "tick 58    kimg 4848.9   time 3h 53m 25s   sec/tick 237.6   sec/kimg 58.92   maintenance 0.0    gpumem 7.3   augment 0.729\n",
            "tick 59    kimg 4853.0   time 3h 57m 23s   sec/tick 237.6   sec/kimg 58.94   maintenance 0.0    gpumem 7.3   augment 0.728\n",
            "tick 60    kimg 4857.0   time 4h 01m 20s   sec/tick 237.8   sec/kimg 58.97   maintenance 0.0    gpumem 7.3   augment 0.730\n",
            "tick 61    kimg 4861.0   time 4h 05m 24s   sec/tick 237.6   sec/kimg 58.92   maintenance 6.1    gpumem 7.3   augment 0.732\n",
            "tick 62    kimg 4865.0   time 4h 09m 22s   sec/tick 237.5   sec/kimg 58.92   maintenance 0.0    gpumem 7.3   augment 0.725\n",
            "tick 63    kimg 4869.1   time 4h 13m 19s   sec/tick 237.6   sec/kimg 58.93   maintenance 0.0    gpumem 7.3   augment 0.725\n",
            "tick 64    kimg 4873.1   time 4h 17m 17s   sec/tick 237.6   sec/kimg 58.93   maintenance 0.0    gpumem 7.3   augment 0.726\n",
            "tick 65    kimg 4877.1   time 4h 21m 14s   sec/tick 237.1   sec/kimg 58.81   maintenance 0.0    gpumem 7.3   augment 0.723\n",
            "tick 66    kimg 4881.2   time 4h 25m 11s   sec/tick 237.4   sec/kimg 58.87   maintenance 0.0    gpumem 7.3   augment 0.723\n",
            "tick 67    kimg 4885.2   time 4h 29m 15s   sec/tick 237.6   sec/kimg 58.92   maintenance 6.1    gpumem 7.3   augment 0.725\n",
            "tick 68    kimg 4889.2   time 4h 33m 13s   sec/tick 237.6   sec/kimg 58.92   maintenance 0.0    gpumem 7.3   augment 0.732\n",
            "tick 69    kimg 4893.3   time 4h 37m 10s   sec/tick 237.4   sec/kimg 58.88   maintenance 0.0    gpumem 7.3   augment 0.731\n",
            "tick 70    kimg 4897.3   time 4h 41m 08s   sec/tick 237.6   sec/kimg 58.93   maintenance 0.0    gpumem 7.3   augment 0.735\n",
            "tick 71    kimg 4901.3   time 4h 45m 06s   sec/tick 237.7   sec/kimg 58.97   maintenance 0.0    gpumem 7.3   augment 0.736\n",
            "tick 72    kimg 4905.4   time 4h 49m 03s   sec/tick 237.8   sec/kimg 58.98   maintenance 0.0    gpumem 7.3   augment 0.737\n",
            "tick 73    kimg 4909.4   time 4h 53m 07s   sec/tick 237.7   sec/kimg 58.96   maintenance 6.2    gpumem 7.3   augment 0.739\n",
            "tick 74    kimg 4913.4   time 4h 57m 05s   sec/tick 237.7   sec/kimg 58.96   maintenance 0.0    gpumem 7.3   augment 0.737\n",
            "tick 75    kimg 4917.5   time 5h 01m 03s   sec/tick 238.1   sec/kimg 59.04   maintenance 0.0    gpumem 7.3   augment 0.742\n",
            "tick 76    kimg 4921.5   time 5h 05m 01s   sec/tick 238.0   sec/kimg 59.03   maintenance 0.0    gpumem 7.3   augment 0.741\n",
            "tick 77    kimg 4925.5   time 5h 08m 59s   sec/tick 237.6   sec/kimg 58.94   maintenance 0.0    gpumem 7.3   augment 0.745\n",
            "tick 78    kimg 4929.6   time 5h 12m 57s   sec/tick 238.0   sec/kimg 59.03   maintenance 0.0    gpumem 7.3   augment 0.747\n",
            "tick 79    kimg 4933.6   time 5h 17m 01s   sec/tick 238.1   sec/kimg 59.04   maintenance 6.4    gpumem 7.3   augment 0.753\n",
            "tick 80    kimg 4937.6   time 5h 20m 59s   sec/tick 238.1   sec/kimg 59.04   maintenance 0.0    gpumem 7.3   augment 0.749\n",
            "tick 81    kimg 4941.7   time 5h 24m 57s   sec/tick 237.8   sec/kimg 58.97   maintenance 0.0    gpumem 7.3   augment 0.746\n",
            "tick 82    kimg 4945.7   time 5h 28m 55s   sec/tick 237.9   sec/kimg 59.01   maintenance 0.0    gpumem 7.3   augment 0.743\n",
            "tick 83    kimg 4949.7   time 5h 32m 53s   sec/tick 238.0   sec/kimg 59.02   maintenance 0.0    gpumem 7.3   augment 0.741\n",
            "tick 84    kimg 4953.8   time 5h 36m 51s   sec/tick 237.6   sec/kimg 58.94   maintenance 0.0    gpumem 7.3   augment 0.730\n",
            "tick 85    kimg 4957.8   time 5h 40m 54s   sec/tick 237.6   sec/kimg 58.93   maintenance 6.2    gpumem 7.3   augment 0.742\n",
            "tick 86    kimg 4961.8   time 5h 44m 52s   sec/tick 237.8   sec/kimg 58.98   maintenance 0.0    gpumem 7.3   augment 0.738\n",
            "tick 87    kimg 4965.8   time 5h 48m 50s   sec/tick 237.7   sec/kimg 58.97   maintenance 0.0    gpumem 7.3   augment 0.739\n",
            "tick 88    kimg 4969.9   time 5h 52m 48s   sec/tick 237.8   sec/kimg 58.99   maintenance 0.0    gpumem 7.3   augment 0.737\n",
            "tick 89    kimg 4973.9   time 5h 56m 45s   sec/tick 237.4   sec/kimg 58.88   maintenance 0.0    gpumem 7.3   augment 0.736\n",
            "tick 90    kimg 4977.9   time 6h 00m 43s   sec/tick 237.6   sec/kimg 58.92   maintenance 0.0    gpumem 7.3   augment 0.741\n",
            "tick 91    kimg 4982.0   time 6h 04m 47s   sec/tick 237.8   sec/kimg 58.97   maintenance 6.2    gpumem 7.3   augment 0.740\n",
            "tick 92    kimg 4986.0   time 6h 08m 44s   sec/tick 237.8   sec/kimg 58.97   maintenance 0.0    gpumem 7.3   augment 0.737\n",
            "tick 93    kimg 4990.0   time 6h 12m 42s   sec/tick 237.3   sec/kimg 58.86   maintenance 0.0    gpumem 7.3   augment 0.737\n",
            "tick 94    kimg 4994.1   time 6h 16m 39s   sec/tick 237.6   sec/kimg 58.92   maintenance 0.0    gpumem 7.3   augment 0.735\n",
            "tick 95    kimg 4998.1   time 6h 20m 37s   sec/tick 237.7   sec/kimg 58.95   maintenance 0.0    gpumem 7.3   augment 0.733\n",
            "tick 96    kimg 5000.0   time 6h 22m 30s   sec/tick 112.9   sec/kimg 58.81   maintenance 0.0    gpumem 7.3   augment 0.736\n",
            "\n",
            "Exiting...\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}