FTN.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Untitled3.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "rNxDDd6duTRQ",
        "colab_type": "code",
        "cellView": "both",
        "outputId": "56cd688d-cd25-4348-bdb9-ae10de950e6f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 125
        }
      },
      "source": [
        "#@title Imports\n",
        "import tensorflow as tf\n",
        "from PIL import Image\n",
        "import numpy as np\n",
        "import IPython\n",
        "from matplotlib import image\n",
        "from matplotlib import pyplot as plt\n",
        "import pickle\n",
        "import scipy.io\n",
        "import imageio\n",
        "import datetime, os\n",
        "from google.colab import drive\n",
        "drive.mount('/content/gdrive', force_remount=True)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3Aietf%3Awg%3Aoauth%3A2.0%3Aoob&scope=email%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdocs.test%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive.photos.readonly%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fpeopleapi.readonly&response_type=code\n",
            "\n",
            "Enter your authorization code:\n",
            "··········\n",
            "Mounted at /content/gdrive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7xutiwXxuXKB",
        "colab_type": "code",
        "cellView": "both",
        "colab": {}
      },
      "source": [
        "#@markdown Parameters\n",
        "ngf = 64 # Number of filters in first layer of generator \n",
        "ndf = 64 # Number of filters in first layer of discriminator \n",
        "stddev = 0.02 # std of Gaussian distribution\n",
        "batch_size = 1 # batch_size \n",
        "pool_size = 50 # pool_size \n",
        "img_width = 64 # Imput image will of width 256 \n",
        "img_height = 64 # Input image will be of height 256 \n",
        "img_depth = 3 # RGB format\n",
        "to_restore = False\n",
        "vgg_layers = scipy.io.loadmat('/content/gdrive/My Drive/MSc ML/0091/vgg19.mat')[\"layers\"]\n",
        "check_dir = '/content/gdrive/My Drive/MSc ML/0091/64x64/checkpoint/'"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GguBpDdyD6qh",
        "colab_type": "code",
        "outputId": "138435aa-6fe6-474b-cc0d-6def4e120f91",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 87
        }
      },
      "source": [
        "!pip install -q tf-nightly-2.0-preview\n",
        "# Load the TensorBoard notebook extension\n",
        "%load_ext tensorboard"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\u001b[K     |████████████████████████████████| 89.7MB 34.9MB/s \n",
            "\u001b[K     |████████████████████████████████| 3.8MB 32.1MB/s \n",
            "\u001b[K     |████████████████████████████████| 460kB 40.5MB/s \n",
            "\u001b[31mERROR: tensorflow 2.0.0a0 has requirement tb-nightly<1.14.0a20190302,>=1.14.0a20190301, but you'll have tb-nightly 1.15.0a20190903 which is incompatible.\u001b[0m\n",
            "\u001b[?25h"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cIxNWmZUuaCY",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def _weights(layer_idx):\n",
        "    W = vgg_layers[0][layer_idx][0][0][2][0][0]\n",
        "    b = vgg_layers[0][layer_idx][0][0][2][0][1]\n",
        "    return W, b.reshape(b.size)\n",
        "  \n",
        "  \n",
        "def conv2d_relu(prev_layer, layer_idx, layer_name):\n",
        "    with tf.variable_scope(layer_name):\n",
        "        # load parameters\n",
        "        W, b = _weights(layer_idx)\n",
        "        # initialize parameters\n",
        "        W = tf.constant(W, name=\"weights\")\n",
        "        b = tf.constant(b, name=\"bias\")\n",
        "        # convolution \n",
        "        conv2d = tf.nn.conv2d(input=prev_layer,\n",
        "                              filter=W,\n",
        "                              strides=[1, 1, 1, 1],\n",
        "                              padding=\"SAME\")\n",
        "        # activation\n",
        "        out = tf.nn.relu(conv2d + b)\n",
        "    return out\n",
        "    \n",
        "    \n",
        "def avgpool(prev_layer, layer_name):\n",
        "    with tf.variable_scope(layer_name):\n",
        "        # average pooling\n",
        "        out = tf.nn.avg_pool(value=prev_layer,\n",
        "                             ksize=[1, 2, 2, 1],\n",
        "                             strides=[1, 2, 2, 1],\n",
        "                             padding=\"SAME\")\n",
        "    return out\n",
        "  \n",
        "  \n",
        "def _gram_matrix(F, N, M):\n",
        "    F = tf.reshape(F, (M, N))\n",
        "    return tf.matmul(tf.transpose(F), F)\n",
        "  \n",
        "  \n",
        "def _single_style_loss(a, g):\n",
        "    N = a.shape[3]\n",
        "    M = a.shape[1] * a.shape[2]\n",
        "    A = _gram_matrix(a, N, M)\n",
        "    G = _gram_matrix(g, N, M)\n",
        "    return tf.reduce_sum(tf.squared_difference(G,A))/((2 * tf.cast(N,tf.float32) * tf.cast(M,tf.float32)) ** 2)\n",
        "  \n",
        "  \n",
        "def _content_loss(P, F):\n",
        "    return tf.reduce_mean(tf.square(F - P)) / 4.0\n",
        "  \n",
        "  \n",
        "def general_conv2d(inputconv, num_features=64, \n",
        "                   window_height=7, window_width=7, \n",
        "                   stride_height=1, stride_width=1, \n",
        "                   padding_size=1, name=\"conv2d\"):\n",
        "    with tf.variable_scope(name):\n",
        "        paddings = tf.constant([[0, 0], [padding_size, padding_size,], [padding_size, padding_size], [0, 0]])\n",
        "        inputconv_ = tf.pad(inputconv, paddings, \"REFLECT\")\n",
        "        conv = tf.contrib.layers.conv2d(inputconv_, num_features, [window_height, window_width], \n",
        "                                        [stride_height, stride_width], padding='VALID', activation_fn=None, \n",
        "                                        weights_initializer=tf.truncated_normal_initializer(stddev=stddev),\n",
        "                                        biases_initializer=tf.constant_initializer(0.0))\n",
        "        return conv"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FG3ZluSGub2R",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def feature_extractor(input_image, name=\"extractor\"):\n",
        "    with tf.variable_scope(name):\n",
        "        conv1_1 = conv2d_relu(input_image, 0, \"conv1_1\")\n",
        "        conv1_2 = conv2d_relu(conv1_1, 2, \"conv1_2\")\n",
        "        avgpool1 = avgpool(conv1_2, \"avgpool1\")\n",
        "        conv2_1 = conv2d_relu(avgpool1, 5, \"conv2_1\")\n",
        "        conv2_2 = conv2d_relu(conv2_1, 7, \"conv2_2\")\n",
        "        avgpool2 = avgpool(conv2_2, \"avgpool2\")\n",
        "        conv3_1 = conv2d_relu(avgpool2, 10, \"conv3_1\")\n",
        "        conv3_2 = conv2d_relu(conv3_1, 12, \"conv3_2\")\n",
        "        conv3_3 = conv2d_relu(conv3_2, 14, \"conv3_3\")\n",
        "        conv3_4 = conv2d_relu(conv3_3, 16, \"conv3_4\")\n",
        "        avgpool3 = avgpool(conv3_4, \"avgpool3\")\n",
        "        conv4_1 = conv2d_relu(avgpool3, 19, \"conv4_1\")\n",
        "        conv4_2 = conv2d_relu(conv4_1, 21, \"conv4_2\")\n",
        "        conv4_3 = conv2d_relu(conv4_2, 23, \"conv4_3\")\n",
        "        conv4_4 = conv2d_relu(conv4_3, 25, \"conv4_4\")\n",
        "        avgpool4 = avgpool(conv4_4, \"avgpool4\")\n",
        "        conv5_1 = conv2d_relu(avgpool4, 28, \"conv5_1\")\n",
        "        return conv1_1, conv2_1, conv3_1, conv4_1, conv3_2, conv5_1"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2t9-uWrBud0p",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def feature_translator(input_image, name=\"translator\"):\n",
        "    with tf.variable_scope(name):\n",
        "        conv1_1 = conv2d_relu(input_image, 0, \"conv1_1\")\n",
        "        conv1_2 = conv2d_relu(conv1_1, 2, \"conv1_2\")\n",
        "        avgpool1 = avgpool(conv1_2, \"avgpool1\")\n",
        "        conv2_1 = conv2d_relu(avgpool1, 5, \"conv2_1\")\n",
        "        conv2_2 = conv2d_relu(conv2_1, 7, \"conv2_2\")\n",
        "        avgpool2 = avgpool(conv2_2, \"avgpool2\")\n",
        "        conv3_1 = conv2d_relu(avgpool2, 10, \"conv3_1\")\n",
        "        conv3_2 = conv2d_relu(conv3_1, 12, \"conv3_2\")\n",
        "        conv3_3 = conv2d_relu(conv3_2, 14, \"conv3_3\")\n",
        "        conv3_4 = conv2d_relu(conv3_3, 16, \"conv3_4\")\n",
        "        avgpool3 = avgpool(conv3_4, \"avgpool3\")\n",
        "        conv4_1 = conv2d_relu(avgpool3, 19, \"conv4_1\")\n",
        "        conv4_2 = conv2d_relu(conv4_1, 21, \"conv4_2\")\n",
        "        conv4_3 = conv2d_relu(conv4_2, 23, \"conv4_3\")\n",
        "        conv4_4 = conv2d_relu(conv4_3, 25, \"conv4_4\")\n",
        "        avgpool4 = avgpool(conv4_4, \"avgpool4\")\n",
        "        conv5_1 = conv2d_relu(avgpool4, 28, \"conv5_1\")\n",
        "        \n",
        "        conv1_1_1 = tf.nn.leaky_relu(tf.contrib.layers.instance_norm(tf.layers.dense(inputs=conv1_1, units=64, name=\"conv1_1_1\")))\n",
        "        conv1_1_2 = tf.nn.leaky_relu(tf.contrib.layers.instance_norm(tf.layers.dense(inputs=conv1_1_1, units=64, name=\"conv1_1_2\")))\n",
        "        conv1_1_ = tf.contrib.layers.instance_norm(tf.layers.dense(inputs=conv1_1_2, units=64, name=\"conv1_1_3\"))\n",
        "        \n",
        "        conv2_1_1 = tf.nn.leaky_relu(tf.contrib.layers.instance_norm(tf.layers.dense(inputs=conv2_1, units=128, name=\"conv2_1_1\")))\n",
        "        conv2_1_2 = tf.nn.leaky_relu(tf.contrib.layers.instance_norm(tf.layers.dense(inputs=conv2_1_1, units=128, name=\"conv2_1_2\")))\n",
        "        conv2_1_ = tf.contrib.layers.instance_norm(tf.layers.dense(inputs=conv2_1_2, units=128, name=\"conv2_1_3\"))\n",
        "        \n",
        "        conv3_1_1 = tf.nn.leaky_relu(tf.contrib.layers.instance_norm(tf.layers.dense(inputs=conv3_1, units=256, name=\"conv3_1_1\")))\n",
        "        conv3_1_2 = tf.nn.leaky_relu(tf.contrib.layers.instance_norm(tf.layers.dense(inputs=conv3_1_1, units=256, name=\"conv3_1_2\")))\n",
        "        conv3_1_ = tf.contrib.layers.instance_norm(tf.layers.dense(inputs=conv3_1_2, units=256, name=\"conv3_1_3\"))\n",
        "        \n",
        "        conv4_1_1 = tf.nn.leaky_relu(tf.contrib.layers.instance_norm(tf.layers.dense(inputs=conv4_1, units=512, name=\"conv4_1_1\")))\n",
        "        conv4_1_2 = tf.nn.leaky_relu(tf.contrib.layers.instance_norm(tf.layers.dense(inputs=conv4_1_1, units=512, name=\"conv4_1_2\")))\n",
        "        conv4_1_ = tf.contrib.layers.instance_norm(tf.layers.dense(inputs=conv4_1_2, units=512, name=\"conv4_1_3\"))\n",
        "        \n",
        "        conv4_2_1 = tf.nn.leaky_relu(tf.contrib.layers.instance_norm(tf.layers.dense(inputs=conv3_2, units=256, name=\"conv4_2_1\")))\n",
        "        conv4_2_2 = tf.nn.leaky_relu(tf.contrib.layers.instance_norm(tf.layers.dense(inputs=conv4_2_1, units=256, name=\"conv4_2_2\")))\n",
        "        conv4_2_ = tf.contrib.layers.instance_norm(tf.layers.dense(inputs=conv4_2_2, units=256, name=\"conv4_2_3\"))\n",
        "        \n",
        "        conv5_1_1 = tf.nn.leaky_relu(tf.contrib.layers.instance_norm(tf.layers.dense(inputs=conv5_1, units=512, name=\"conv5_1_1\")))\n",
        "        conv5_1_2 = tf.nn.leaky_relu(tf.contrib.layers.instance_norm(tf.layers.dense(inputs=conv5_1_1, units=512, name=\"conv5_1_2\")))\n",
        "        conv5_1_ = tf.contrib.layers.instance_norm(tf.layers.dense(inputs=conv5_1_2, units=512, name=\"conv5_1_3\"))\n",
        "        \n",
        "        return conv1_1_, conv2_1_, conv3_1_, conv4_1_, conv4_2_, conv5_1_     "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "U6_ASi2DDeI2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def feature_translator(input_image, name=\"translator\"):\n",
        "    with tf.variable_scope(name):\n",
        "        conv1_1 = conv2d_relu(input_image, 0, \"conv1_1\")\n",
        "        conv1_2 = conv2d_relu(conv1_1, 2, \"conv1_2\")\n",
        "        avgpool1 = avgpool(conv1_2, \"avgpool1\")\n",
        "        conv2_1 = conv2d_relu(avgpool1, 5, \"conv2_1\")\n",
        "        conv2_2 = conv2d_relu(conv2_1, 7, \"conv2_2\")\n",
        "        avgpool2 = avgpool(conv2_2, \"avgpool2\")\n",
        "        conv3_1 = conv2d_relu(avgpool2, 10, \"conv3_1\")\n",
        "        conv3_2 = conv2d_relu(conv3_1, 12, \"conv3_2\")\n",
        "        conv3_3 = conv2d_relu(conv3_2, 14, \"conv3_3\")\n",
        "        conv3_4 = conv2d_relu(conv3_3, 16, \"conv3_4\")\n",
        "        avgpool3 = avgpool(conv3_4, \"avgpool3\")\n",
        "        conv4_1 = conv2d_relu(avgpool3, 19, \"conv4_1\")\n",
        "        conv4_2 = conv2d_relu(conv4_1, 21, \"conv4_2\")\n",
        "        conv4_3 = conv2d_relu(conv4_2, 23, \"conv4_3\")\n",
        "        conv4_4 = conv2d_relu(conv4_3, 25, \"conv4_4\")\n",
        "        avgpool4 = avgpool(conv4_4, \"avgpool4\")\n",
        "        conv5_1 = conv2d_relu(avgpool4, 28, \"conv5_1\")\n",
        "        \n",
        "        conv1_1_1 = tf.nn.leaky_relu(tf.contrib.layers.instance_norm(general_conv2d(conv1_1, 64, 3, 3, 1, 1, 1, name=\"conv1_1_1\")))\n",
        "        conv1_1_2 = tf.nn.leaky_relu(tf.contrib.layers.instance_norm(general_conv2d(conv1_1_1, 64, 3, 3, 1, 1, 1, name=\"conv1_1_2\")))\n",
        "        conv1_1_ = tf.contrib.layers.instance_norm(general_conv2d(conv1_1_2, 64, 3, 3, 1, 1, 1, name=\"conv1_1_\"))\n",
        "        \n",
        "        conv2_1_1 = tf.nn.leaky_relu(tf.contrib.layers.instance_norm(general_conv2d(conv2_1, 128, 3, 3, 1, 1, 1, name=\"conv2_1_1\")))\n",
        "        conv2_1_2 = tf.nn.leaky_relu(tf.contrib.layers.instance_norm(general_conv2d(conv2_1_1, 128, 3, 3, 1, 1, 1, name=\"conv2_1_2\")))\n",
        "        conv2_1_ = tf.contrib.layers.instance_norm(general_conv2d(conv2_1_2, 128, 3, 3, 1, 1, 1, name=\"conv2_1_\"))\n",
        "        \n",
        "        conv3_1_1 = tf.nn.leaky_relu(tf.contrib.layers.instance_norm(general_conv2d(conv3_1, 256, 3, 3, 1, 1, 1, name=\"conv3_1_1\")))\n",
        "        conv3_1_2 = tf.nn.leaky_relu(tf.contrib.layers.instance_norm(general_conv2d(conv3_1_1, 256, 3, 3, 1, 1, 1, name=\"conv3_1_2\")))\n",
        "        conv3_1_ = tf.contrib.layers.instance_norm(general_conv2d(conv3_1_2, 256, 3, 3, 1, 1, 1, name=\"conv3_1_\"))\n",
        "        \n",
        "        conv4_1_1 = tf.nn.leaky_relu(tf.contrib.layers.instance_norm(general_conv2d(conv4_1, 512, 3, 3, 1, 1, 1, name=\"conv4_1_1\")))\n",
        "        conv4_1_2 = tf.nn.leaky_relu(tf.contrib.layers.instance_norm(general_conv2d(conv4_1_1, 512, 3, 3, 1, 1, 1, name=\"conv4_1_2\")))\n",
        "        conv4_1_ = tf.contrib.layers.instance_norm(general_conv2d(conv4_1_2, 512, 3, 3, 1, 1, 1, name=\"conv4_1_\"))\n",
        "        \n",
        "        conv3_2_1 = tf.nn.leaky_relu(tf.contrib.layers.instance_norm(general_conv2d(conv3_2, 256, 3, 3, 1, 1, 1, name=\"conv3_2_1\")))\n",
        "        conv3_2_2 = tf.nn.leaky_relu(tf.contrib.layers.instance_norm(general_conv2d(conv3_2_1, 256, 3, 3, 1, 1, 1, name=\"conv3_2_2\")))\n",
        "        conv3_2_ = tf.contrib.layers.instance_norm(general_conv2d(conv3_2_2, 256, 3, 3, 1, 1, 1, name=\"conv3_2_\"))\n",
        "        \n",
        "        conv5_1_1 = tf.nn.leaky_relu(tf.contrib.layers.instance_norm(general_conv2d(conv5_1, 512, 3, 3, 1, 1, 1, name=\"conv5_1_1\")))\n",
        "        conv5_1_2 = tf.nn.leaky_relu(tf.contrib.layers.instance_norm(general_conv2d(conv5_1_1, 512, 3, 3, 1, 1, 1, name=\"conv5_1_2\")))\n",
        "        conv5_1_ = tf.contrib.layers.instance_norm(general_conv2d(conv5_1_2, 512, 3, 3, 1, 1, 1, name=\"conv5_1_\"))\n",
        "        \n",
        "        return conv1_1_, conv2_1_, conv3_1_, conv4_1_, conv3_2_, conv5_1_     "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QkPMvA07ufqf",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "tf.reset_default_graph()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "g1hgzJNTuh9Z",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "input_A = tf.placeholder(tf.float32, [batch_size, img_width, img_height, img_depth], name=\"input_A\")\n",
        "input_B = tf.placeholder(tf.float32, [batch_size, img_width, img_height, img_depth], name=\"input_B\")\n",
        "\n",
        "conv1_1_B, conv2_1_B, conv3_1_B, conv4_1_B, conv4_2_B, conv5_1_B = feature_translator(input_A, name=\"translatorAtoB\")\n",
        "conv1_1B, conv2_1B, conv3_1B, conv4_1B, conv4_2B, conv5_1B = feature_extractor(input_B, name=\"extractor\")\n",
        "\n",
        "conv1_1_A, conv2_1_A, conv3_1_A, conv4_1_A, conv4_2_A, conv5_1_A = feature_translator(input_B, name=\"translatorBtoA\")\n",
        "conv1_1A, conv2_1A, conv3_1A, conv4_1A, conv4_2A, conv5_1A = feature_extractor(input_A, name=\"extractor\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Gdwm475cujaS",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# loss\n",
        "with tf.name_scope('loss_AtoB'):\n",
        "    loss_AtoB_1 = _single_style_loss(conv1_1_B,conv1_1B)\n",
        "    loss_AtoB_2 = _single_style_loss(conv2_1_B,conv2_1B)\n",
        "    loss_AtoB_3 = _single_style_loss(conv3_1_B,conv3_1B)\n",
        "    loss_AtoB_4 = _single_style_loss(conv4_1_B,conv4_1B)\n",
        "    loss_AtoB_5 = _single_style_loss(conv5_1_B,conv5_1B)\n",
        "    loss_AtoB_c = _content_loss(conv4_2_B,conv4_2B)\n",
        "    loss_AtoB = tf.reduce_mean(0.01*(0.5*loss_AtoB_1+loss_AtoB_2+1.5*loss_AtoB_3+3.0*loss_AtoB_4+4.0*loss_AtoB_5)+1000*loss_AtoB_c)\n",
        "    tf.summary.scalar('loss_AtoB', loss_AtoB)\n",
        "\n",
        "with tf.name_scope('loss_BtoA'):\n",
        "    loss_BtoA_1 = _single_style_loss(conv1_1_A,conv1_1A)\n",
        "    loss_BtoA_2 = _single_style_loss(conv2_1_A,conv2_1A)\n",
        "    loss_BtoA_3 = _single_style_loss(conv3_1_A,conv3_1A)\n",
        "    loss_BtoA_4 = _single_style_loss(conv4_1_A,conv4_1A)\n",
        "    loss_BtoA_5 = _single_style_loss(conv5_1_A,conv5_1A)\n",
        "    loss_BtoA_c = _content_loss(conv4_2_A,conv4_2A)\n",
        "    loss_BtoA = tf.reduce_mean(0.01*(0.5*loss_BtoA_1+loss_BtoA_2+1.5*loss_BtoA_3+3.0*loss_BtoA_4+4.0*loss_BtoA_5)+1000*loss_BtoA_c)\n",
        "    tf.summary.scalar('loss_BtoA', loss_BtoA)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "I9E1sowvulIb",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# optimizer\n",
        "model_vars = tf.trainable_variables()\n",
        "AtoB_vars = [var for var in model_vars if 'translatorAtoB' in var.name]\n",
        "BtoA_vars = [var for var in model_vars if 'translatorBtoA' in var.name]\n",
        "\n",
        "lr = tf.placeholder(tf.float32, shape=[], name=\"lr\")\n",
        "optimizer = tf.train.AdamOptimizer(learning_rate=lr)\n",
        "AtoB_trainer = optimizer.minimize(loss_AtoB, var_list=AtoB_vars)\n",
        "BtoA_trainer = optimizer.minimize(loss_BtoA, var_list=BtoA_vars)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6D4VqQPEum65",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "with open('/content/gdrive/My Drive/MSc ML/0091/64x64/Portrait2', 'rb') as f:\n",
        "    A_input = pickle.load(f)\n",
        "with open('/content/gdrive/My Drive/MSc ML/0091/64x64/Photo2', 'rb') as f:\n",
        "    B_input = pickle.load(f)    \n",
        "    \n",
        "A_input = np.asarray(A_input)\n",
        "A_input = A_input.astype('float32')\n",
        "B_input = np.asarray(B_input)\n",
        "B_input = B_input.astype('float32')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TeK_ij-NcANV",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "with open('/content/gdrive/My Drive/MSc ML/0091/64x64/Painting', 'rb') as f:\n",
        "    C_input = pickle.load(f)    \n",
        "    \n",
        "C_input = np.asarray(C_input)\n",
        "C_input = C_input.astype('float32')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rVTtPYaQuopA",
        "colab_type": "code",
        "outputId": "aabfba3e-1636-4d7a-f0e4-20c68ffcb529",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "merged = tf.summary.merge_all()\n",
        "saver = tf.train.Saver()\n",
        "loss_A2B_list = []\n",
        "loss_B2A_list = []\n",
        "with tf.Session() as sess:\n",
        "    sess.run(tf.global_variables_initializer())\n",
        "    writer = tf.summary.FileWriter('/content/gdrive/My Drive/MSc ML/0091/64x64/logs', sess.graph)\n",
        "    \n",
        "    if to_restore:\n",
        "        # chkpt_fname = tf.train.latest_checkpoint(check_dir)\n",
        "        saver.restore(sess, '/content/gdrive/My Drive/MSc ML/0091/64x64/checkpoint/featuretrans-1400')\n",
        "      \n",
        "    for epoch in range(1001):\n",
        "        print('epoch' + str(epoch))\n",
        "        # Define the learning rate schedule. The learning rate is kept constant upto 100 epochs and then linearly decayed to 0.\n",
        "        if(epoch < 900) :\n",
        "            # learning rate = 2\n",
        "            curr_lr = 2\n",
        "        else:\n",
        "            curr_lr = 2 - 2*(epoch-900)/100\n",
        "            \n",
        "        loss_A2B = 0 \n",
        "        loss_B2A = 0\n",
        "    \n",
        "        # Running the training loop for all batches\n",
        "        for ptr in range(0, 350):\n",
        "            \n",
        "            # Train generator G_A->B\n",
        "            _, __, loss_AtoB_temp, loss_BtoA_temp, summary = sess.run([AtoB_trainer, BtoA_trainer, loss_AtoB, loss_BtoA, merged], \n",
        "                                                             feed_dict={input_A:A_input[ptr], input_B:B_input[ptr], \n",
        "                                                                        lr:curr_lr})\n",
        "\n",
        "            loss_A2B += loss_AtoB_temp/350\n",
        "            loss_B2A += loss_BtoA_temp/350\n",
        "            \n",
        "            writer.add_summary(summary, epoch * 350 + ptr)\n",
        "\n",
        "        print(' loss AtoB' + str(loss_A2B))\n",
        "        print(' loss BtoA' + str(loss_B2A))\n",
        "        tf.summary.scalar('loss AtoB', loss_A2B)\n",
        "        tf.summary.scalar('loss BtoA', loss_B2A)\n",
        "        loss_A2B_list.append(loss_A2B)\n",
        "        loss_B2A_list.append(loss_B2A)\n",
        "        with open('/content/gdrive/My Drive/MSc ML/0091/64x64/Loss/A2B', 'wb') as f:\n",
        "            pickle.dump(loss_A2B_list, f)\n",
        "        with open('/content/gdrive/My Drive/MSc ML/0091/64x64/Loss/B2A', 'wb') as f:\n",
        "            pickle.dump(loss_B2A_list, f)\n",
        "            \n",
        "                \n",
        "        # save training images\n",
        "        if epoch%200==0:\n",
        "            saver.save(sess, check_dir + \"featuretrans\", global_step=epoch) # 1000-0.01-3\n",
        "\n",
        "    writer.close()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "epoch0\n",
            " loss AtoB99978144.79428561\n",
            " loss BtoA45865250.505714275\n",
            "epoch1\n",
            " loss AtoB59291216.73142858\n",
            " loss BtoA29057281.19428572\n",
            "epoch2\n",
            " loss AtoB56894098.08\n",
            " loss BtoA27892029.779999997\n",
            "epoch3\n",
            " loss AtoB54108148.25714287\n",
            " loss BtoA26315630.494285718\n",
            "epoch4\n",
            " loss AtoB51861737.211428575\n",
            " loss BtoA25247248.005714312\n",
            "epoch5\n",
            " loss AtoB50097605.782857165\n",
            " loss BtoA24404903.822857134\n",
            "epoch6\n",
            " loss AtoB48284037.977142826\n",
            " loss BtoA23646403.082857143\n",
            "epoch7\n",
            " loss AtoB46780819.48000001\n",
            " loss BtoA22892653.207142863\n",
            "epoch8\n",
            " loss AtoB45286825.19428571\n",
            " loss BtoA21982886.288571417\n",
            "epoch9\n",
            " loss AtoB43965382.451428555\n",
            " loss BtoA21146204.78142856\n",
            "epoch10\n",
            " loss AtoB42747437.84000005\n",
            " loss BtoA20411335.59857143\n",
            "epoch11\n",
            " loss AtoB41523386.7885714\n",
            " loss BtoA19767511.42857143\n",
            "epoch12\n",
            " loss AtoB40439232.88571429\n",
            " loss BtoA19186254.950000003\n",
            "epoch13\n",
            " loss AtoB39462723.291428566\n",
            " loss BtoA18644283.217142846\n",
            "epoch14\n",
            " loss AtoB38603719.268571414\n",
            " loss BtoA18160946.222857136\n",
            "epoch15\n",
            " loss AtoB37792050.265714295\n",
            " loss BtoA17739798.39571429\n",
            "epoch16\n",
            " loss AtoB37055474.43428568\n",
            " loss BtoA17398694.561428566\n",
            "epoch17\n",
            " loss AtoB36337358.491428584\n",
            " loss BtoA17150601.061428566\n",
            "epoch18\n",
            " loss AtoB35671225.02857144\n",
            " loss BtoA16932604.710000005\n",
            "epoch19\n",
            " loss AtoB35061802.717142865\n",
            " loss BtoA16922558.99142855\n",
            "epoch20\n",
            " loss AtoB34517596.35714286\n",
            " loss BtoA16778286.795714285\n",
            "epoch21\n",
            " loss AtoB34043482.3\n",
            " loss BtoA16471784.28142857\n",
            "epoch22\n",
            " loss AtoB33653775.505714275\n",
            " loss BtoA16036068.97857143\n",
            "epoch23\n",
            " loss AtoB33374634.95142853\n",
            " loss BtoA15654135.679999996\n",
            "epoch24\n",
            " loss AtoB33317692.022857144\n",
            " loss BtoA15321548.254285723\n",
            "epoch25\n",
            " loss AtoB33404335.131428562\n",
            " loss BtoA15056161.358571438\n",
            "epoch26\n",
            " loss AtoB33006485.059999987\n",
            " loss BtoA14847150.058571432\n",
            "epoch27\n",
            " loss AtoB32494441.377142847\n",
            " loss BtoA14670438.148571441\n",
            "epoch28\n",
            " loss AtoB31965787.380000006\n",
            " loss BtoA14522162.927142868\n",
            "epoch29\n",
            " loss AtoB31409615.68571426\n",
            " loss BtoA14418805.87285714\n",
            "epoch30\n",
            " loss AtoB30860601.008571435\n",
            " loss BtoA14357161.13000001\n",
            "epoch31\n",
            " loss AtoB30340479.431428567\n",
            " loss BtoA14332649.34857144\n",
            "epoch32\n",
            " loss AtoB29858960.53428571\n",
            " loss BtoA14590183.29285715\n",
            "epoch33\n",
            " loss AtoB29410736.971428566\n",
            " loss BtoA14612274.624285718\n",
            "epoch34\n",
            " loss AtoB28995825.88285713\n",
            " loss BtoA14031173.582857141\n",
            "epoch35\n",
            " loss AtoB28636181.50000001\n",
            " loss BtoA13605453.345714292\n",
            "epoch36\n",
            " loss AtoB28301719.199999977\n",
            " loss BtoA13354975.67999999\n",
            "epoch37\n",
            " loss AtoB28056413.382857144\n",
            " loss BtoA13175926.497142863\n",
            "epoch38\n",
            " loss AtoB28014370.11428571\n",
            " loss BtoA13081397.47285714\n",
            "epoch39\n",
            " loss AtoB28149043.46571427\n",
            " loss BtoA13069006.32714285\n",
            "epoch40\n",
            " loss AtoB28149760.325714286\n",
            " loss BtoA13208700.792857135\n",
            "epoch41\n",
            " loss AtoB28014979.465714294\n",
            " loss BtoA13539054.574285718\n",
            "epoch42\n",
            " loss AtoB27581298.99142857\n",
            " loss BtoA13347233.465714293\n",
            "epoch43\n",
            " loss AtoB27642479.851428576\n",
            " loss BtoA13139391.352857145\n",
            "epoch44\n",
            " loss AtoB27848211.139999997\n",
            " loss BtoA12890592.91142857\n",
            "epoch45\n",
            " loss AtoB27986327.277142856\n",
            " loss BtoA12790944.932857145\n",
            "epoch46\n",
            " loss AtoB27960364.268571425\n",
            " loss BtoA12780084.264285719\n",
            "epoch47\n",
            " loss AtoB27609060.40857143\n",
            " loss BtoA12756795.02\n",
            "epoch48\n",
            " loss AtoB27273693.802857164\n",
            " loss BtoA12672227.167142857\n",
            "epoch49\n",
            " loss AtoB26889844.362857122\n",
            " loss BtoA12691251.214285713\n",
            "epoch50\n",
            " loss AtoB26555432.2142857\n",
            " loss BtoA12581764.904285712\n",
            "epoch51\n",
            " loss AtoB26351054.24285715\n",
            " loss BtoA12497323.334285721\n",
            "epoch52\n",
            " loss AtoB26191093.897142872\n",
            " loss BtoA12409112.538571417\n",
            "epoch53\n",
            " loss AtoB26060855.04285714\n",
            " loss BtoA12270089.91571428\n",
            "epoch54\n",
            " loss AtoB26021821.851428583\n",
            " loss BtoA12192544.28571429\n",
            "epoch55\n",
            " loss AtoB26062340.508571416\n",
            " loss BtoA12120468.531428583\n",
            "epoch56\n",
            " loss AtoB26075243.87714284\n",
            " loss BtoA12134923.291428577\n",
            "epoch57\n",
            " loss AtoB26061660.93999999\n",
            " loss BtoA12146595.538571421\n",
            "epoch58\n",
            " loss AtoB26132966.62285715\n",
            " loss BtoA12141150.672857147\n",
            "epoch59\n",
            " loss AtoB26118616.234285705\n",
            " loss BtoA12081875.014285713\n",
            "epoch60\n",
            " loss AtoB25994733.38285714\n",
            " loss BtoA12010054.581428578\n",
            "epoch61\n",
            " loss AtoB25838013.59428572\n",
            " loss BtoA11987120.135714287\n",
            "epoch62\n",
            " loss AtoB25521203.594285715\n",
            " loss BtoA12043806.164285721\n",
            "epoch63\n",
            " loss AtoB25160974.64285715\n",
            " loss BtoA12143034.447142849\n",
            "epoch64\n",
            " loss AtoB24920016.33428573\n",
            " loss BtoA12254428.421428574\n",
            "epoch65\n",
            " loss AtoB24853079.81142858\n",
            " loss BtoA12263965.195714293\n",
            "epoch66\n",
            " loss AtoB24757748.41142858\n",
            " loss BtoA12212346.119999994\n",
            "epoch67\n",
            " loss AtoB24659115.734285723\n",
            " loss BtoA12069154.857142849\n",
            "epoch68\n",
            " loss AtoB24713729.211428557\n",
            " loss BtoA11879440.231428582\n",
            "epoch69\n",
            " loss AtoB24836874.81714285\n",
            " loss BtoA11767110.962857142\n",
            "epoch70\n",
            " loss AtoB24697264.80571429\n",
            " loss BtoA11651047.481428575\n",
            "epoch71\n",
            " loss AtoB24763339.648571398\n",
            " loss BtoA11552998.818571419\n",
            "epoch72\n",
            " loss AtoB24632991.391428586\n",
            " loss BtoA11481325.44142856\n",
            "epoch73\n",
            " loss AtoB24702269.231428567\n",
            " loss BtoA11437137.371428581\n",
            "epoch74\n",
            " loss AtoB24614139.477142848\n",
            " loss BtoA11421495.475714294\n",
            "epoch75\n",
            " loss AtoB24498156.679999996\n",
            " loss BtoA11422762.472857142\n",
            "epoch76\n",
            " loss AtoB24554072.405714285\n",
            " loss BtoA11518685.981428564\n",
            "epoch77\n",
            " loss AtoB24435476.514285725\n",
            " loss BtoA11566668.805714276\n",
            "epoch78\n",
            " loss AtoB24317044.508571416\n",
            " loss BtoA11562654.148571419\n",
            "epoch79\n",
            " loss AtoB23987171.265714277\n",
            " loss BtoA11606113.931428578\n",
            "epoch80\n",
            " loss AtoB23765858.988571417\n",
            " loss BtoA11694832.845714284\n",
            "epoch81\n",
            " loss AtoB23536812.30285715\n",
            " loss BtoA11650059.557142857\n",
            "epoch82\n",
            " loss AtoB23413458.08000001\n",
            " loss BtoA11475216.948571427\n",
            "epoch83\n",
            " loss AtoB23446540.197142832\n",
            " loss BtoA11327991.682857156\n",
            "epoch84\n",
            " loss AtoB23335784.757142887\n",
            " loss BtoA11273598.727142856\n",
            "epoch85\n",
            " loss AtoB23337760.81999998\n",
            " loss BtoA11145358.720000004\n",
            "epoch86\n",
            " loss AtoB23028654.91142856\n",
            " loss BtoA11039627.948571423\n",
            "epoch87\n",
            " loss AtoB22939238.405714285\n",
            " loss BtoA10984493.292857144\n",
            "epoch88\n",
            " loss AtoB22821355.577142846\n",
            " loss BtoA10953268.242857156\n",
            "epoch89\n",
            " loss AtoB22760785.600000005\n",
            " loss BtoA10948799.122857135\n",
            "epoch90\n",
            " loss AtoB22768008.02000002\n",
            " loss BtoA10974301.81428571\n",
            "epoch91\n",
            " loss AtoB22711093.497142855\n",
            " loss BtoA10983676.861428572\n",
            "epoch92\n",
            " loss AtoB22596462.82571427\n",
            " loss BtoA10977134.467142863\n",
            "epoch93\n",
            " loss AtoB22694770.977142867\n",
            " loss BtoA11034655.194285706\n",
            "epoch94\n",
            " loss AtoB22808608.61142855\n",
            " loss BtoA11074094.337142853\n",
            "epoch95\n",
            " loss AtoB22777717.117142845\n",
            " loss BtoA11099707.634285709\n",
            "epoch96\n",
            " loss AtoB22561261.511428587\n",
            " loss BtoA11113458.902857143\n",
            "epoch97\n",
            " loss AtoB22407252.83714285\n",
            " loss BtoA11070749.064285723\n",
            "epoch98\n",
            " loss AtoB22234968.282857146\n",
            " loss BtoA11018030.337142853\n",
            "epoch99\n",
            " loss AtoB22042656.90285714\n",
            " loss BtoA10861345.271428568\n",
            "epoch100\n",
            " loss AtoB22244169.554285694\n",
            " loss BtoA10786766.692857137\n",
            "epoch101\n",
            " loss AtoB22335647.365714274\n",
            " loss BtoA10699499.248571431\n",
            "epoch102\n",
            " loss AtoB22429799.522857133\n",
            " loss BtoA10680496.44142857\n",
            "epoch103\n",
            " loss AtoB22562948.105714273\n",
            " loss BtoA10692941.125714283\n",
            "epoch104\n",
            " loss AtoB22380216.125714265\n",
            " loss BtoA10687861.20142858\n",
            "epoch105\n",
            " loss AtoB22436025.008571416\n",
            " loss BtoA10622280.031428585\n",
            "epoch106\n",
            " loss AtoB22324744.662857123\n",
            " loss BtoA10608268.779999996\n",
            "epoch107\n",
            " loss AtoB22253490.21142857\n",
            " loss BtoA10531421.982857147\n",
            "epoch108\n",
            " loss AtoB21868865.142857134\n",
            " loss BtoA10495081.257142853\n",
            "epoch109\n",
            " loss AtoB21719757.485714287\n",
            " loss BtoA10518558.887142858\n",
            "epoch110\n",
            " loss AtoB21535683.677142866\n",
            " loss BtoA10523567.705714293\n",
            "epoch111\n",
            " loss AtoB21616339.99142856\n",
            " loss BtoA10514524.421428576\n",
            "epoch112\n",
            " loss AtoB21724040.028571423\n",
            " loss BtoA10548591.859999998\n",
            "epoch113\n",
            " loss AtoB21762743.374285705\n",
            " loss BtoA10596881.604285713\n",
            "epoch114\n",
            " loss AtoB21758756.171428595\n",
            " loss BtoA10578667.477142848\n",
            "epoch115\n",
            " loss AtoB21746691.945714302\n",
            " loss BtoA10490780.66714286\n",
            "epoch116\n",
            " loss AtoB21707216.40857145\n",
            " loss BtoA10424264.790000008\n",
            "epoch117\n",
            " loss AtoB21606739.40571429\n",
            " loss BtoA10437067.14571429\n",
            "epoch118\n",
            " loss AtoB21431278.69142857\n",
            " loss BtoA10430859.315714285\n",
            "epoch119\n",
            " loss AtoB21243418.979999986\n",
            " loss BtoA10402387.700000007\n",
            "epoch120\n",
            " loss AtoB21122702.911428556\n",
            " loss BtoA10426745.278571425\n",
            "epoch121\n",
            " loss AtoB21249644.411428574\n",
            " loss BtoA10412920.430000003\n",
            "epoch122\n",
            " loss AtoB21461695.21999999\n",
            " loss BtoA10497617.730000008\n",
            "epoch123\n",
            " loss AtoB21346997.831428573\n",
            " loss BtoA10462808.448571421\n",
            "epoch124\n",
            " loss AtoB21390267.10857145\n",
            " loss BtoA10477965.488571418\n",
            "epoch125\n",
            " loss AtoB21279683.700000014\n",
            " loss BtoA10394534.224285724\n",
            "epoch126\n",
            " loss AtoB21109374.04857144\n",
            " loss BtoA10308218.858571434\n",
            "epoch127\n",
            " loss AtoB21025569.77142855\n",
            " loss BtoA10240251.521428581\n",
            "epoch128\n",
            " loss AtoB21203493.617142845\n",
            " loss BtoA10205982.84285715\n",
            "epoch129\n",
            " loss AtoB21262266.211428583\n",
            " loss BtoA10180250.700000005\n",
            "epoch130\n",
            " loss AtoB21084041.145714283\n",
            " loss BtoA10172425.07999999\n",
            "epoch131\n",
            " loss AtoB21005486.285714276\n",
            " loss BtoA10212152.374285717\n",
            "epoch132\n",
            " loss AtoB20913531.50285713\n",
            " loss BtoA10214328.24428572\n",
            "epoch133\n",
            " loss AtoB20890285.285714276\n",
            " loss BtoA10207732.541428572\n",
            "epoch134\n",
            " loss AtoB20870480.011428583\n",
            " loss BtoA10203100.032857154\n",
            "epoch135\n",
            " loss AtoB20755742.14285714\n",
            " loss BtoA10235998.69571429\n",
            "epoch136\n",
            " loss AtoB20686129.240000002\n",
            " loss BtoA10156656.862857131\n",
            "epoch137\n",
            " loss AtoB20642177.805714283\n",
            " loss BtoA10102740.031428564\n",
            "epoch138\n",
            " loss AtoB20581948.554285713\n",
            " loss BtoA10044603.034285713\n",
            "epoch139\n",
            " loss AtoB20643133.517142866\n",
            " loss BtoA9966345.732857138\n",
            "epoch140\n",
            " loss AtoB20566727.542857137\n",
            " loss BtoA9917631.460000008\n",
            "epoch141\n",
            " loss AtoB20590283.422857136\n",
            " loss BtoA9899739.929999994\n",
            "epoch142\n",
            " loss AtoB20618276.194285713\n",
            " loss BtoA9986238.278571427\n",
            "epoch143\n",
            " loss AtoB20688254.54857142\n",
            " loss BtoA10100940.08\n",
            "epoch144\n",
            " loss AtoB20665426.828571446\n",
            " loss BtoA10047454.808571434\n",
            "epoch145\n",
            " loss AtoB20797744.397142865\n",
            " loss BtoA9934118.715714294\n",
            "epoch146\n",
            " loss AtoB20699215.39714285\n",
            " loss BtoA9917691.652857153\n",
            "epoch147\n",
            " loss AtoB20791773.35428573\n",
            " loss BtoA9896620.257142853\n",
            "epoch148\n",
            " loss AtoB20688051.302857157\n",
            " loss BtoA9904111.619999994\n",
            "epoch149\n",
            " loss AtoB20783425.91428573\n",
            " loss BtoA9974643.897142868\n",
            "epoch150\n",
            " loss AtoB20802467.67428569\n",
            " loss BtoA9906957.297142856\n",
            "epoch151\n",
            " loss AtoB20521584.474285737\n",
            " loss BtoA9899837.947142862\n",
            "epoch152\n",
            " loss AtoB20619730.834285703\n",
            " loss BtoA9860022.581428573\n",
            "epoch153\n",
            " loss AtoB20366346.425714284\n",
            " loss BtoA9863963.028571429\n",
            "epoch154\n",
            " loss AtoB20210662.26857143\n",
            " loss BtoA9855609.919999996\n",
            "epoch155\n",
            " loss AtoB20141538.27714285\n",
            " loss BtoA9858908.584285708\n",
            "epoch156\n",
            " loss AtoB20127168.46857143\n",
            " loss BtoA9928126.99\n",
            "epoch157\n",
            " loss AtoB20157505.614285715\n",
            " loss BtoA9855951.76428571\n",
            "epoch158\n",
            " loss AtoB20234049.494285706\n",
            " loss BtoA9861765.992857138\n",
            "epoch159\n",
            " loss AtoB20138429.299999993\n",
            " loss BtoA9804866.294285713\n",
            "epoch160\n",
            " loss AtoB20024404.302857157\n",
            " loss BtoA9763563.915714288\n",
            "epoch161\n",
            " loss AtoB20047123.697142866\n",
            " loss BtoA9789379.84714286\n",
            "epoch162\n",
            " loss AtoB20049421.611428577\n",
            " loss BtoA9818286.707142858\n",
            "epoch163\n",
            " loss AtoB20020467.022857137\n",
            " loss BtoA9742987.885714287\n",
            "epoch164\n",
            " loss AtoB20015408.242857136\n",
            " loss BtoA9712621.089999994\n",
            "epoch165\n",
            " loss AtoB20107253.000000007\n",
            " loss BtoA9708956.389999999\n",
            "epoch166\n",
            " loss AtoB20138022.98857142\n",
            " loss BtoA9716222.151428575\n",
            "epoch167\n",
            " loss AtoB20033123.642857157\n",
            " loss BtoA9732196.31571429\n",
            "epoch168\n",
            " loss AtoB20009377.662857138\n",
            " loss BtoA9734864.624285728\n",
            "epoch169\n",
            " loss AtoB19952676.48857143\n",
            " loss BtoA9804705.721428562\n",
            "epoch170\n",
            " loss AtoB19832449.480000004\n",
            " loss BtoA9755159.528571432\n",
            "epoch171\n",
            " loss AtoB19787120.61142857\n",
            " loss BtoA9779768.534285717\n",
            "epoch172\n",
            " loss AtoB19812955.02857142\n",
            " loss BtoA9722399.50285714\n",
            "epoch173\n",
            " loss AtoB19771186.61714285\n",
            " loss BtoA9681201.77571428\n",
            "epoch174\n",
            " loss AtoB19689649.442857143\n",
            " loss BtoA9688661.115714286\n",
            "epoch175\n",
            " loss AtoB19610352.414285738\n",
            " loss BtoA9676526.225714292\n",
            "epoch176\n",
            " loss AtoB19578627.25428571\n",
            " loss BtoA9644475.801428579\n",
            "epoch177\n",
            " loss AtoB19715411.422857143\n",
            " loss BtoA9593697.241428576\n",
            "epoch178\n",
            " loss AtoB19643057.885714304\n",
            " loss BtoA9623849.98\n",
            "epoch179\n",
            " loss AtoB19839683.051428575\n",
            " loss BtoA9608950.127142867\n",
            "epoch180\n",
            " loss AtoB19789358.245714277\n",
            " loss BtoA9651649.594285712\n",
            "epoch181\n",
            " loss AtoB19647869.774285726\n",
            " loss BtoA9735851.778571425\n",
            "epoch182\n",
            " loss AtoB19477878.47714286\n",
            " loss BtoA9718238.892857144\n",
            "epoch183\n",
            " loss AtoB19414252.080000006\n",
            " loss BtoA9630791.93285714\n",
            "epoch184\n",
            " loss AtoB19395851.14571429\n",
            " loss BtoA9627772.718571432\n",
            "epoch185\n",
            " loss AtoB19521276.462857157\n",
            " loss BtoA9581570.615714276\n",
            "epoch186\n",
            " loss AtoB19568250.29142857\n",
            " loss BtoA9596168.782857142\n",
            "epoch187\n",
            " loss AtoB19514673.900000006\n",
            " loss BtoA9555375.678571427\n",
            "epoch188\n",
            " loss AtoB19606831.282857146\n",
            " loss BtoA9554499.988571422\n",
            "epoch189\n",
            " loss AtoB19530190.74571429\n",
            " loss BtoA9555312.071428563\n",
            "epoch190\n",
            " loss AtoB19416296.088571433\n",
            " loss BtoA9569764.585714282\n",
            "epoch191\n",
            " loss AtoB19541398.62285716\n",
            " loss BtoA9553167.541428575\n",
            "epoch192\n",
            " loss AtoB19613951.61428572\n",
            " loss BtoA9514738.221428575\n",
            "epoch193\n",
            " loss AtoB19697423.571428586\n",
            " loss BtoA9459435.008571422\n",
            "epoch194\n",
            " loss AtoB19734024.79142855\n",
            " loss BtoA9431885.217142856\n",
            "epoch195\n",
            " loss AtoB19685197.54285713\n",
            " loss BtoA9468975.602857146\n",
            "epoch196\n",
            " loss AtoB19533085.96571428\n",
            " loss BtoA9483404.387142852\n",
            "epoch197\n",
            " loss AtoB19361561.977142844\n",
            " loss BtoA9490052.68428571\n",
            "epoch198\n",
            " loss AtoB19303876.90285715\n",
            " loss BtoA9484762.06428572\n",
            "epoch199\n",
            " loss AtoB19212350.402857136\n",
            " loss BtoA9438262.534285717\n",
            "epoch200\n",
            " loss AtoB19145964.98\n",
            " loss BtoA9407219.091428574\n",
            "epoch201\n",
            " loss AtoB19192177.802857134\n",
            " loss BtoA9405011.70285714\n",
            "epoch202\n",
            " loss AtoB19264873.93714286\n",
            " loss BtoA9297514.598571429\n",
            "epoch203\n",
            " loss AtoB19257128.137142856\n",
            " loss BtoA9255139.878571434\n",
            "epoch204\n",
            " loss AtoB19134434.757142857\n",
            " loss BtoA9233177.782857144\n",
            "epoch205\n",
            " loss AtoB19154862.46857142\n",
            " loss BtoA9304455.127142861\n",
            "epoch206\n",
            " loss AtoB19061112.257142857\n",
            " loss BtoA9378773.80142857\n",
            "epoch207\n",
            " loss AtoB19003232.154285714\n",
            " loss BtoA9376543.389999999\n",
            "epoch208\n",
            " loss AtoB18983917.437142868\n",
            " loss BtoA9341598.904285712\n",
            "epoch209\n",
            " loss AtoB18973620.445714295\n",
            " loss BtoA9365156.067142854\n",
            "epoch210\n",
            " loss AtoB19157597.222857144\n",
            " loss BtoA9401742.818571428\n",
            "epoch211\n",
            " loss AtoB19223801.511428565\n",
            " loss BtoA9351790.784285711\n",
            "epoch212\n",
            " loss AtoB19106141.828571443\n",
            " loss BtoA9322482.105714288\n",
            "epoch213\n",
            " loss AtoB19095264.662857138\n",
            " loss BtoA9326705.752857145\n",
            "epoch214\n",
            " loss AtoB19026594.951428566\n",
            " loss BtoA9283564.808571428\n",
            "epoch215\n",
            " loss AtoB19024761.53999998\n",
            " loss BtoA9272632.151428571\n",
            "epoch216\n",
            " loss AtoB18978341.054285698\n",
            " loss BtoA9317278.44142857\n",
            "epoch217\n",
            " loss AtoB18869094.69428573\n",
            " loss BtoA9294231.607142854\n",
            "epoch218\n",
            " loss AtoB18978716.299999997\n",
            " loss BtoA9291468.431428572\n",
            "epoch219\n",
            " loss AtoB19070854.33714285\n",
            " loss BtoA9291560.315714294\n",
            "epoch220\n",
            " loss AtoB19121281.16857142\n",
            " loss BtoA9268780.802857151\n",
            "epoch221\n",
            " loss AtoB19113923.982857134\n",
            " loss BtoA9273183.22857142\n",
            "epoch222\n",
            " loss AtoB18873158.699999984\n",
            " loss BtoA9269058.774285719\n",
            "epoch223\n",
            " loss AtoB18782212.33428571\n",
            " loss BtoA9244756.944285717\n",
            "epoch224\n",
            " loss AtoB18749722.248571422\n",
            " loss BtoA9258102.757142855\n",
            "epoch225\n",
            " loss AtoB18824206.254285708\n",
            " loss BtoA9225113.844285715\n",
            "epoch226\n",
            " loss AtoB18917944.271428585\n",
            " loss BtoA9198006.534285713\n",
            "epoch227\n",
            " loss AtoB18886735.62857143\n",
            " loss BtoA9165546.571428569\n",
            "epoch228\n",
            " loss AtoB18763255.108571433\n",
            " loss BtoA9200601.721428573\n",
            "epoch229\n",
            " loss AtoB18789972.671428565\n",
            " loss BtoA9190043.858571427\n",
            "epoch230\n",
            " loss AtoB18820711.428571433\n",
            " loss BtoA9176727.570000002\n",
            "epoch231\n",
            " loss AtoB18690430.88\n",
            " loss BtoA9134918.122857146\n",
            "epoch232\n",
            " loss AtoB18705750.668571427\n",
            " loss BtoA9119283.160000006\n",
            "epoch233\n",
            " loss AtoB18751750.67714284\n",
            " loss BtoA9102624.334285716\n",
            "epoch234\n",
            " loss AtoB18666395.114285704\n",
            " loss BtoA9117154.564285716\n",
            "epoch235\n",
            " loss AtoB18700943.397142872\n",
            " loss BtoA9154057.45285714\n",
            "epoch236\n",
            " loss AtoB18834474.279999986\n",
            " loss BtoA9163118.255714294\n",
            "epoch237\n",
            " loss AtoB18805191.040000003\n",
            " loss BtoA9137107.01428571\n",
            "epoch238\n",
            " loss AtoB18875712.760000005\n",
            " loss BtoA9120880.958571421\n",
            "epoch239\n",
            " loss AtoB18862315.76857142\n",
            " loss BtoA9134772.801428562\n",
            "epoch240\n",
            " loss AtoB18859209.897142857\n",
            " loss BtoA9125370.4\n",
            "epoch241\n",
            " loss AtoB18675883.694285695\n",
            " loss BtoA9176055.182857143\n",
            "epoch242\n",
            " loss AtoB18580598.25428571\n",
            " loss BtoA9184727.197142854\n",
            "epoch243\n",
            " loss AtoB18586509.965714287\n",
            " loss BtoA9269777.088571431\n",
            "epoch244\n",
            " loss AtoB18577006.78857142\n",
            " loss BtoA9177134.811428571\n",
            "epoch245\n",
            " loss AtoB18526328.37428571\n",
            " loss BtoA9093702.42571429\n",
            "epoch246\n",
            " loss AtoB18492766.897142854\n",
            " loss BtoA9082476.450000003\n",
            "epoch247\n",
            " loss AtoB18646042.55142857\n",
            " loss BtoA9069601.981428567\n",
            "epoch248\n",
            " loss AtoB18658299.63142856\n",
            " loss BtoA9032800.657142853\n",
            "epoch249\n",
            " loss AtoB18548235.997142855\n",
            " loss BtoA9068379.851428572\n",
            "epoch250\n",
            " loss AtoB18435517.3942857\n",
            " loss BtoA9077441.09142857\n",
            "epoch251\n",
            " loss AtoB18367415.394285712\n",
            " loss BtoA9081852.518571429\n",
            "epoch252\n",
            " loss AtoB18415416.937142853\n",
            " loss BtoA9084972.964285724\n",
            "epoch253\n",
            " loss AtoB18421273.240000002\n",
            " loss BtoA9106343.858571427\n",
            "epoch254\n",
            " loss AtoB18450848.431428567\n",
            " loss BtoA9118381.692857152\n",
            "epoch255\n",
            " loss AtoB18319829.774285704\n",
            " loss BtoA9091891.938571425\n",
            "epoch256\n",
            " loss AtoB18412365.182857137\n",
            " loss BtoA9106049.891428558\n",
            "epoch257\n",
            " loss AtoB18393442.8485714\n",
            " loss BtoA9101871.727142861\n",
            "epoch258\n",
            " loss AtoB18407583.625714287\n",
            " loss BtoA9047900.17857143\n",
            "epoch259\n",
            " loss AtoB18469363.2742857\n",
            " loss BtoA9024500.168571422\n",
            "epoch260\n",
            " loss AtoB18469565.365714286\n",
            " loss BtoA9005659.418571427\n",
            "epoch261\n",
            " loss AtoB18484195.714285716\n",
            " loss BtoA8999786.201428575\n",
            "epoch262\n",
            " loss AtoB18349455.182857137\n",
            " loss BtoA8981403.908571428\n",
            "epoch263\n",
            " loss AtoB18234934.285714284\n",
            " loss BtoA9002501.33714285\n",
            "epoch264\n",
            " loss AtoB18227300.448571414\n",
            " loss BtoA9004709.428571425\n",
            "epoch265\n",
            " loss AtoB18192351.31142857\n",
            " loss BtoA9003719.114285719\n",
            "epoch266\n",
            " loss AtoB18314113.13714286\n",
            " loss BtoA9030078.714285718\n",
            "epoch267\n",
            " loss AtoB18398487.359999985\n",
            " loss BtoA9006913.677142862\n",
            "epoch268\n",
            " loss AtoB18364277.517142847\n",
            " loss BtoA8993924.927142855\n",
            "epoch269\n",
            " loss AtoB18284444.45714287\n",
            " loss BtoA8993033.087142864\n",
            "epoch270\n",
            " loss AtoB18228678.165714283\n",
            " loss BtoA8963438.135714293\n",
            "epoch271\n",
            " loss AtoB18159984.157142848\n",
            " loss BtoA8946839.322857143\n",
            "epoch272\n",
            " loss AtoB18173533.365714286\n",
            " loss BtoA8988687.78\n",
            "epoch273\n",
            " loss AtoB18274644.357142866\n",
            " loss BtoA8920548.03571429\n",
            "epoch274\n",
            " loss AtoB18319464.219999995\n",
            " loss BtoA8916615.199999997\n",
            "epoch275\n",
            " loss AtoB18182344.40857143\n",
            " loss BtoA8921865.155714288\n",
            "epoch276\n",
            " loss AtoB18124109.368571423\n",
            " loss BtoA8966228.167142864\n",
            "epoch277\n",
            " loss AtoB18072474.248571433\n",
            " loss BtoA8923897.137142856\n",
            "epoch278\n",
            " loss AtoB18112632.168571424\n",
            " loss BtoA8887944.182857152\n",
            "epoch279\n",
            " loss AtoB18104014.04571429\n",
            " loss BtoA8857925.85428571\n",
            "epoch280\n",
            " loss AtoB18116448.73428571\n",
            " loss BtoA8868765.08142857\n",
            "epoch281\n",
            " loss AtoB18127963.86857143\n",
            " loss BtoA8912609.74857143\n",
            "epoch282\n",
            " loss AtoB18237105.95714286\n",
            " loss BtoA8952755.035714287\n",
            "epoch283\n",
            " loss AtoB18374615.21999999\n",
            " loss BtoA8908346.207142858\n",
            "epoch284\n",
            " loss AtoB18342788.045714293\n",
            " loss BtoA8843955.902857153\n",
            "epoch285\n",
            " loss AtoB18206256.94571428\n",
            " loss BtoA8843409.239999998\n",
            "epoch286\n",
            " loss AtoB18131645.55428571\n",
            " loss BtoA8815150.505714279\n",
            "epoch287\n",
            " loss AtoB18073993.411428563\n",
            " loss BtoA8788238.938571425\n",
            "epoch288\n",
            " loss AtoB17982209.53142857\n",
            " loss BtoA8783340.468571434\n",
            "epoch289\n",
            " loss AtoB18058752.51142857\n",
            " loss BtoA8812189.968571432\n",
            "epoch290\n",
            " loss AtoB17908768.262857147\n",
            " loss BtoA8829517.857142858\n",
            "epoch291\n",
            " loss AtoB17944081.037142865\n",
            " loss BtoA8852769.141428564\n",
            "epoch292\n",
            " loss AtoB18023519.431428578\n",
            " loss BtoA8908907.934285723\n",
            "epoch293\n",
            " loss AtoB17899256.874285717\n",
            " loss BtoA8881513.39142857\n",
            "epoch294\n",
            " loss AtoB17973488.345714282\n",
            " loss BtoA8790717.619999997\n",
            "epoch295\n",
            " loss AtoB17985152.962857146\n",
            " loss BtoA8806931.461428564\n",
            "epoch296\n",
            " loss AtoB17992569.054285705\n",
            " loss BtoA8789829.949999997\n",
            "epoch297\n",
            " loss AtoB18032989.722857147\n",
            " loss BtoA8777339.948571425\n",
            "epoch298\n",
            " loss AtoB17922703.608571436\n",
            " loss BtoA8761740.99142857\n",
            "epoch299\n",
            " loss AtoB17933956.457142852\n",
            " loss BtoA8811825.391428571\n",
            "epoch300\n",
            " loss AtoB17888852.24571431\n",
            " loss BtoA8788706.451428564\n",
            "epoch301\n",
            " loss AtoB17874738.79142857\n",
            " loss BtoA8797031.59142857\n",
            "epoch302\n",
            " loss AtoB17916468.14857143\n",
            " loss BtoA8825631.278571427\n",
            "epoch303\n",
            " loss AtoB18021213.617142864\n",
            " loss BtoA8808018.805714289\n",
            "epoch304\n",
            " loss AtoB18032181.46571429\n",
            " loss BtoA8771040.812857136\n",
            "epoch305\n",
            " loss AtoB17969628.454285704\n",
            " loss BtoA8789500.100000005\n",
            "epoch306\n",
            " loss AtoB17919473.197142847\n",
            " loss BtoA8808784.338571424\n",
            "epoch307\n",
            " loss AtoB17804045.874285705\n",
            " loss BtoA8864596.702857135\n",
            "epoch308\n",
            " loss AtoB17802991.285714276\n",
            " loss BtoA8824056.47142857\n",
            "epoch309\n",
            " loss AtoB17793558.7857143\n",
            " loss BtoA8766542.890000002\n",
            "epoch310\n",
            " loss AtoB17817819.642857146\n",
            " loss BtoA8707226.88571428\n",
            "epoch311\n",
            " loss AtoB17904137.265714288\n",
            " loss BtoA8681570.65857143\n",
            "epoch312\n",
            " loss AtoB18002038.454285704\n",
            " loss BtoA8726226.837142855\n",
            "epoch313\n",
            " loss AtoB18022231.08\n",
            " loss BtoA8767231.785714284\n",
            "epoch314\n",
            " loss AtoB17908201.78\n",
            " loss BtoA8760898.04\n",
            "epoch315\n",
            " loss AtoB17813722.00857143\n",
            " loss BtoA8723513.790000007\n",
            "epoch316\n",
            " loss AtoB17806213.008571442\n",
            " loss BtoA8790639.690000003\n",
            "epoch317\n",
            " loss AtoB17855619.611428574\n",
            " loss BtoA8748730.13000001\n",
            "epoch318\n",
            " loss AtoB17843645.131428562\n",
            " loss BtoA8704001.004285716\n",
            "epoch319\n",
            " loss AtoB17839619.868571416\n",
            " loss BtoA8672554.329999996\n",
            "epoch320\n",
            " loss AtoB17810848.685714286\n",
            " loss BtoA8694331.665714284\n",
            "epoch321\n",
            " loss AtoB17732619.488571424\n",
            " loss BtoA8681360.211428575\n",
            "epoch322\n",
            " loss AtoB17677240.071428563\n",
            " loss BtoA8713752.829999996\n",
            "epoch323\n",
            " loss AtoB17635081.75428572\n",
            " loss BtoA8710345.891428573\n",
            "epoch324\n",
            " loss AtoB17682612.782857157\n",
            " loss BtoA8702237.25\n",
            "epoch325\n",
            " loss AtoB17680109.81428573\n",
            " loss BtoA8684541.732857136\n",
            "epoch326\n",
            " loss AtoB17575033.962857135\n",
            " loss BtoA8654129.36142857\n",
            "epoch327\n",
            " loss AtoB17664261.574285727\n",
            " loss BtoA8608473.878571417\n",
            "epoch328\n",
            " loss AtoB17636370.128571432\n",
            " loss BtoA8607092.287857145\n",
            "epoch329\n",
            " loss AtoB17770051.859999996\n",
            " loss BtoA8635015.521428574\n",
            "epoch330\n",
            " loss AtoB17882266.114285726\n",
            " loss BtoA8626052.24071429\n",
            "epoch331\n",
            " loss AtoB17937278.388571437\n",
            " loss BtoA8658195.120000001\n",
            "epoch332\n",
            " loss AtoB17842720.159999996\n",
            " loss BtoA8717007.00571428\n",
            "epoch333\n",
            " loss AtoB17653903.23714286\n",
            " loss BtoA8692209.961428571\n",
            "epoch334\n",
            " loss AtoB17540171.98\n",
            " loss BtoA8660944.04571428\n",
            "epoch335\n",
            " loss AtoB17567592.52571428\n",
            " loss BtoA8608852.965714283\n",
            "epoch336\n",
            " loss AtoB17633551.64285715\n",
            " loss BtoA8579607.060000008\n",
            "epoch337\n",
            " loss AtoB17645288.571428575\n",
            " loss BtoA8550502.459999999\n",
            "epoch338\n",
            " loss AtoB17558054.03999999\n",
            " loss BtoA8594077.472857142\n",
            "epoch339\n",
            " loss AtoB17606730.317142855\n",
            " loss BtoA8594709.655714292\n",
            "epoch340\n",
            " loss AtoB17719401.785714287\n",
            " loss BtoA8611910.095714282\n",
            "epoch341\n",
            " loss AtoB17755582.82571429\n",
            " loss BtoA8631244.877142854\n",
            "epoch342\n",
            " loss AtoB17647512.031428564\n",
            " loss BtoA8623538.327142853\n",
            "epoch343\n",
            " loss AtoB17582164.98\n",
            " loss BtoA8649119.158571435\n",
            "epoch344\n",
            " loss AtoB17488881.577142853\n",
            " loss BtoA8644299.498571428\n",
            "epoch345\n",
            " loss AtoB17548839.479999993\n",
            " loss BtoA8663726.261428569\n",
            "epoch346\n",
            " loss AtoB17649693.28571429\n",
            " loss BtoA8641695.63142857\n",
            "epoch347\n",
            " loss AtoB17684844.814285707\n",
            " loss BtoA8580999.704285711\n",
            "epoch348\n",
            " loss AtoB17640475.67428572\n",
            " loss BtoA8576867.651428575\n",
            "epoch349\n",
            " loss AtoB17534687.025714286\n",
            " loss BtoA8576641.182142857\n",
            "epoch350\n",
            " loss AtoB17558053.528571438\n",
            " loss BtoA8576567.476428572\n",
            "epoch351\n",
            " loss AtoB17474029.64857143\n",
            " loss BtoA8504115.640000002\n",
            "epoch352\n",
            " loss AtoB17443196.571428575\n",
            " loss BtoA8511889.998571424\n",
            "epoch353\n",
            " loss AtoB17359596.95714286\n",
            " loss BtoA8576460.858571427\n",
            "epoch354\n",
            " loss AtoB17374098.340000007\n",
            " loss BtoA8607138.07214286\n",
            "epoch355\n",
            " loss AtoB17503776.33142858\n",
            " loss BtoA8644415.76\n",
            "epoch356\n",
            " loss AtoB17527702.399999995\n",
            " loss BtoA8663117.154285721\n",
            "epoch357\n",
            " loss AtoB17558712.311428573\n",
            " loss BtoA8683746.044285713\n",
            "epoch358\n",
            " loss AtoB17585700.95428572\n",
            " loss BtoA8652037.308571432\n",
            "epoch359\n",
            " loss AtoB17572485.859999992\n",
            " loss BtoA8627473.38714286\n",
            "epoch360\n",
            " loss AtoB17497275.211428575\n",
            " loss BtoA8637682.100000003\n",
            "epoch361\n",
            " loss AtoB17471043.711428568\n",
            " loss BtoA8643046.959999995\n",
            "epoch362\n",
            " loss AtoB17468068.33142857\n",
            " loss BtoA8620603.080000002\n",
            "epoch363\n",
            " loss AtoB17410812.239999995\n",
            " loss BtoA8592731.080714282\n",
            "epoch364\n",
            " loss AtoB17414787.842857163\n",
            " loss BtoA8524400.96142857\n",
            "epoch365\n",
            " loss AtoB17398973.86\n",
            " loss BtoA8495032.284285719\n",
            "epoch366\n",
            " loss AtoB17348865.534285717\n",
            " loss BtoA8490953.799999999\n",
            "epoch367\n",
            " loss AtoB17333940.137142856\n",
            " loss BtoA8544992.257142857\n",
            "epoch368\n",
            " loss AtoB17361346.82000001\n",
            " loss BtoA8558491.115714286\n",
            "epoch369\n",
            " loss AtoB17430352.348571435\n",
            " loss BtoA8565508.112857142\n",
            "epoch370\n",
            " loss AtoB17437363.622857153\n",
            " loss BtoA8535697.010000002\n",
            "epoch371\n",
            " loss AtoB17486470.41428572\n",
            " loss BtoA8536176.491428569\n",
            "epoch372\n",
            " loss AtoB17545612.602857143\n",
            " loss BtoA8574727.559999999\n",
            "epoch373\n",
            " loss AtoB17530273.159999993\n",
            " loss BtoA8576597.905714283\n",
            "epoch374\n",
            " loss AtoB17521113.228571426\n",
            " loss BtoA8564837.447857143\n",
            "epoch375\n",
            " loss AtoB17423870.548571426\n",
            " loss BtoA8521558.265714286\n",
            "epoch376\n",
            " loss AtoB17343559.80571427\n",
            " loss BtoA8496809.428571435\n",
            "epoch377\n",
            " loss AtoB17284640.842857145\n",
            " loss BtoA8469870.232857145\n",
            "epoch378\n",
            " loss AtoB17304655.16571428\n",
            " loss BtoA8536199.890000004\n",
            "epoch379\n",
            " loss AtoB17233548.894285716\n",
            " loss BtoA8571209.545714287\n",
            "epoch380\n",
            " loss AtoB17286723.08571427\n",
            " loss BtoA8591988.732857142\n",
            "epoch381\n",
            " loss AtoB17370054.988571428\n",
            " loss BtoA8569108.19428572\n",
            "epoch382\n",
            " loss AtoB17429782.51714285\n",
            " loss BtoA8563302.84857143\n",
            "epoch383\n",
            " loss AtoB17487934.174285714\n",
            " loss BtoA8563143.250000002\n",
            "epoch384\n",
            " loss AtoB17451455.225714285\n",
            " loss BtoA8557526.898571426\n",
            "epoch385\n",
            " loss AtoB17459327.517142862\n",
            " loss BtoA8547255.355714286\n",
            "epoch386\n",
            " loss AtoB17354868.894285724\n",
            " loss BtoA8514044.018571433\n",
            "epoch387\n",
            " loss AtoB17265504.460000005\n",
            " loss BtoA8487057.364285711\n",
            "epoch388\n",
            " loss AtoB17231497.105714284\n",
            " loss BtoA8478569.42142857\n",
            "epoch389\n",
            " loss AtoB17183335.542857144\n",
            " loss BtoA8476979.197142856\n",
            "epoch390\n",
            " loss AtoB17262083.37714286\n",
            " loss BtoA8520099.637142861\n",
            "epoch391\n",
            " loss AtoB17304949.05142857\n",
            " loss BtoA8517607.289999997\n",
            "epoch392\n",
            " loss AtoB17238151.77428572\n",
            " loss BtoA8580937.248571424\n",
            "epoch393\n",
            " loss AtoB17163394.671428587\n",
            " loss BtoA8595262.462857135\n",
            "epoch394\n",
            " loss AtoB17219657.291428562\n",
            " loss BtoA8567932.231428565\n",
            "epoch395\n",
            " loss AtoB17202402.702857133\n",
            " loss BtoA8520269.325714292\n",
            "epoch396\n",
            " loss AtoB17151012.945714287\n",
            " loss BtoA8531301.015714286\n",
            "epoch397\n",
            " loss AtoB17126447.27428572\n",
            " loss BtoA8482462.229999999\n",
            "epoch398\n",
            " loss AtoB17136926.42571429\n",
            " loss BtoA8486453.235714287\n",
            "epoch399\n",
            " loss AtoB17111222.49428572\n",
            " loss BtoA8497652.51\n",
            "epoch400\n",
            " loss AtoB17072507.199999996\n",
            " loss BtoA8548738.966428574\n",
            "epoch401\n",
            " loss AtoB17068696.21999998\n",
            " loss BtoA8581899.28285715\n",
            "epoch402\n",
            " loss AtoB17113717.774285715\n",
            " loss BtoA8524202.361428572\n",
            "epoch403\n",
            " loss AtoB17218290.957142856\n",
            " loss BtoA8489760.195714282\n",
            "epoch404\n",
            " loss AtoB17201218.782857142\n",
            " loss BtoA8446532.765\n",
            "epoch405\n",
            " loss AtoB17099315.922857158\n",
            " loss BtoA8466141.808571437\n",
            "epoch406\n",
            " loss AtoB16985648.70571428\n",
            " loss BtoA8428463.92785714\n",
            "epoch407\n",
            " loss AtoB16871844.639999993\n",
            " loss BtoA8438569.638571434\n",
            "epoch408\n",
            " loss AtoB16845289.81142858\n",
            " loss BtoA8491084.820000004\n",
            "epoch409\n",
            " loss AtoB16832057.22285714\n",
            " loss BtoA8493931.015714284\n",
            "epoch410\n",
            " loss AtoB16878657.277142856\n",
            " loss BtoA8526331.397857139\n",
            "epoch411\n",
            " loss AtoB17020441.74285715\n",
            " loss BtoA8523786.052142847\n",
            "epoch412\n",
            " loss AtoB17084418.07999999\n",
            " loss BtoA8489316.395714287\n",
            "epoch413\n",
            " loss AtoB17100930.79142856\n",
            " loss BtoA8454163.894285718\n",
            "epoch414\n",
            " loss AtoB17082894.794285715\n",
            " loss BtoA8413040.087142862\n",
            "epoch415\n",
            " loss AtoB17097392.54\n",
            " loss BtoA8414523.235714281\n",
            "epoch416\n",
            " loss AtoB17047527.360000003\n",
            " loss BtoA8440083.55571428\n",
            "epoch417\n",
            " loss AtoB16958829.079999994\n",
            " loss BtoA8478771.350714287\n",
            "epoch418\n",
            " loss AtoB16939493.845714286\n",
            " loss BtoA8537325.810714288\n",
            "epoch419\n",
            " loss AtoB16890499.040000003\n",
            " loss BtoA8577522.684285715\n",
            "epoch420\n",
            " loss AtoB16893944.134285707\n",
            " loss BtoA8537181.82571428\n",
            "epoch421\n",
            " loss AtoB17068601.754285708\n",
            " loss BtoA8536259.561428567\n",
            "epoch422\n",
            " loss AtoB17121386.74857144\n",
            " loss BtoA8463877.314285712\n",
            "epoch423\n",
            " loss AtoB17132511.242857143\n",
            " loss BtoA8407447.145714281\n",
            "epoch424\n",
            " loss AtoB17149474.60000002\n",
            " loss BtoA8390262.715714285\n",
            "epoch425\n",
            " loss AtoB17089911.98571428\n",
            " loss BtoA8396935.300000004\n",
            "epoch426\n",
            " loss AtoB16981702.36\n",
            " loss BtoA8408894.231428573\n",
            "epoch427\n",
            " loss AtoB16851322.682857152\n",
            " loss BtoA8401882.429999998\n",
            "epoch428\n",
            " loss AtoB16805693.31142858\n",
            " loss BtoA8406847.381428573\n",
            "epoch429\n",
            " loss AtoB16788163.242857132\n",
            " loss BtoA8401799.882142855\n",
            "epoch430\n",
            " loss AtoB16963148.728571437\n",
            " loss BtoA8386449.365000005\n",
            "epoch431\n",
            " loss AtoB16987860.860000003\n",
            " loss BtoA8359955.21214285\n",
            "epoch432\n",
            " loss AtoB16963463.022857156\n",
            " loss BtoA8351919.845714281\n",
            "epoch433\n",
            " loss AtoB17022555.359999992\n",
            " loss BtoA8351006.809999999\n",
            "epoch434\n",
            " loss AtoB17018048.65714285\n",
            " loss BtoA8405419.241428569\n",
            "epoch435\n",
            " loss AtoB17001089.828571424\n",
            " loss BtoA8495035.634285714\n",
            "epoch436\n",
            " loss AtoB16992333.948571436\n",
            " loss BtoA8457657.17857143\n",
            "epoch437\n",
            " loss AtoB16929620.919999987\n",
            " loss BtoA8393155.285714285\n",
            "epoch438\n",
            " loss AtoB16974265.211428564\n",
            " loss BtoA8357840.635000003\n",
            "epoch439\n",
            " loss AtoB16991618.33428572\n",
            " loss BtoA8322945.771428566\n",
            "epoch440\n",
            " loss AtoB16969062.714285735\n",
            " loss BtoA8327876.55857143\n",
            "epoch441\n",
            " loss AtoB16941371.00857143\n",
            " loss BtoA8336635.785714289\n",
            "epoch442\n",
            " loss AtoB16950678.948571432\n",
            " loss BtoA8376068.635714281\n",
            "epoch443\n",
            " loss AtoB16848629.459999993\n",
            " loss BtoA8406655.74\n",
            "epoch444\n",
            " loss AtoB16863206.551428583\n",
            " loss BtoA8405815.730714288\n",
            "epoch445\n",
            " loss AtoB16874718.94571428\n",
            " loss BtoA8403596.924285714\n",
            "epoch446\n",
            " loss AtoB16886630.902857132\n",
            " loss BtoA8395288.849285714\n",
            "epoch447\n",
            " loss AtoB16862434.14285713\n",
            " loss BtoA8379846.795714288\n",
            "epoch448\n",
            " loss AtoB16822139.988571428\n",
            " loss BtoA8394977.185714286\n",
            "epoch449\n",
            " loss AtoB16841423.434285715\n",
            " loss BtoA8368624.292857149\n",
            "epoch450\n",
            " loss AtoB16869747.191428583\n",
            " loss BtoA8385760.045714282\n",
            "epoch451\n",
            " loss AtoB16853714.28285714\n",
            " loss BtoA8371710.762857147\n",
            "epoch452\n",
            " loss AtoB16900548.78285713\n",
            " loss BtoA8362272.096428567\n",
            "epoch453\n",
            " loss AtoB17004157.965714287\n",
            " loss BtoA8332061.549999995\n",
            "epoch454\n",
            " loss AtoB17088986.74285714\n",
            " loss BtoA8335913.713571426\n",
            "epoch455\n",
            " loss AtoB17020365.83142858\n",
            " loss BtoA8319389.461428573\n",
            "epoch456\n",
            " loss AtoB16933476.048571438\n",
            " loss BtoA8306048.287142853\n",
            "epoch457\n",
            " loss AtoB16928473.38571429\n",
            " loss BtoA8311440.610000006\n",
            "epoch458\n",
            " loss AtoB16817168.514285713\n",
            " loss BtoA8341685.936428569\n",
            "epoch459\n",
            " loss AtoB16755406.880000003\n",
            " loss BtoA8400809.244285714\n",
            "epoch460\n",
            " loss AtoB16781227.47428571\n",
            " loss BtoA8339711.73714286\n",
            "epoch461\n",
            " loss AtoB16807220.714285728\n",
            " loss BtoA8302604.05571428\n",
            "epoch462\n",
            " loss AtoB16878008.405714292\n",
            " loss BtoA8316197.142857141\n",
            "epoch463\n",
            " loss AtoB16964239.725714296\n",
            " loss BtoA8318099.430714288\n",
            "epoch464\n",
            " loss AtoB16875566.51142856\n",
            " loss BtoA8293741.35714286\n",
            "epoch465\n",
            " loss AtoB16756131.362857139\n",
            " loss BtoA8282546.707142854\n",
            "epoch466\n",
            " loss AtoB16689994.894285716\n",
            " loss BtoA8279911.717857147\n",
            "epoch467\n",
            " loss AtoB16656404.177142851\n",
            " loss BtoA8264221.292857138\n",
            "epoch468\n",
            " loss AtoB16726838.380000027\n",
            " loss BtoA8256777.097142849\n",
            "epoch469\n",
            " loss AtoB16772640.95142858\n",
            " loss BtoA8296196.8678571405\n",
            "epoch470\n",
            " loss AtoB16760951.991428563\n",
            " loss BtoA8329157.31357143\n",
            "epoch471\n",
            " loss AtoB16703009.268571422\n",
            " loss BtoA8354593.97714286\n",
            "epoch472\n",
            " loss AtoB16682242.014285725\n",
            " loss BtoA8373547.138571428\n",
            "epoch473\n",
            " loss AtoB16661064.637142858\n",
            " loss BtoA8359471.738571421\n",
            "epoch474\n",
            " loss AtoB16665774.825714279\n",
            " loss BtoA8334718.6099999985\n",
            "epoch475\n",
            " loss AtoB16704400.465714281\n",
            " loss BtoA8285413.382857143\n",
            "epoch476\n",
            " loss AtoB16744168.59142857\n",
            " loss BtoA8252638.376428571\n",
            "epoch477\n",
            " loss AtoB16761931.237142853\n",
            " loss BtoA8233734.248571428\n",
            "epoch478\n",
            " loss AtoB16793350.680000003\n",
            " loss BtoA8243981.725714284\n",
            "epoch479\n",
            " loss AtoB16906122.38\n",
            " loss BtoA8259487.351428566\n",
            "epoch480\n",
            " loss AtoB16745395.940000001\n",
            " loss BtoA8262131.7442857195\n",
            "epoch481\n",
            " loss AtoB16769943.27428571\n",
            " loss BtoA8275011.532857142\n",
            "epoch482\n",
            " loss AtoB16930846.948571436\n",
            " loss BtoA8257571.534285715\n",
            "epoch483\n",
            " loss AtoB16974820.414285723\n",
            " loss BtoA8234550.681428574\n",
            "epoch484\n",
            " loss AtoB16892699.211428568\n",
            " loss BtoA8223848.506428568\n",
            "epoch485\n",
            " loss AtoB16857335.57714286\n",
            " loss BtoA8232419.842857141\n",
            "epoch486\n",
            " loss AtoB16773226.37142856\n",
            " loss BtoA8242890.137857143\n",
            "epoch487\n",
            " loss AtoB16735387.651428565\n",
            " loss BtoA8276098.715714284\n",
            "epoch488\n",
            " loss AtoB16665717.959999988\n",
            " loss BtoA8292244.166428569\n",
            "epoch489\n",
            " loss AtoB16566303.545714285\n",
            " loss BtoA8267694.3942857105\n",
            "epoch490\n",
            " loss AtoB16594690.41428572\n",
            " loss BtoA8247509.922857154\n",
            "epoch491\n",
            " loss AtoB16657817.888571417\n",
            " loss BtoA8276012.489285714\n",
            "epoch492\n",
            " loss AtoB16579804.702857146\n",
            " loss BtoA8248835.632857147\n",
            "epoch493\n",
            " loss AtoB16630273.84\n",
            " loss BtoA8233178.228571428\n",
            "epoch494\n",
            " loss AtoB16651447.908571443\n",
            " loss BtoA8201209.565714285\n",
            "epoch495\n",
            " loss AtoB16585793.408571435\n",
            " loss BtoA8185587.179999992\n",
            "epoch496\n",
            " loss AtoB16548410.837142851\n",
            " loss BtoA8178458.8771428615\n",
            "epoch497\n",
            " loss AtoB16547670.57428571\n",
            " loss BtoA8199523.399285708\n",
            "epoch498\n",
            " loss AtoB16672316.557142839\n",
            " loss BtoA8257595.872857144\n",
            "epoch499\n",
            " loss AtoB16692274.371428572\n",
            " loss BtoA8296022.044285717\n",
            "epoch500\n",
            " loss AtoB16676836.66\n",
            " loss BtoA8309682.609999996\n",
            "epoch501\n",
            " loss AtoB16586844.75714286\n",
            " loss BtoA8291457.335714291\n",
            "epoch502\n",
            " loss AtoB16512526.234285725\n",
            " loss BtoA8243982.022142851\n",
            "epoch503\n",
            " loss AtoB16519081.345714282\n",
            " loss BtoA8202015.92857143\n",
            "epoch504\n",
            " loss AtoB16546571.965714287\n",
            " loss BtoA8178864.204285715\n",
            "epoch505\n",
            " loss AtoB16619785.248571437\n",
            " loss BtoA8190582.791428572\n",
            "epoch506\n",
            " loss AtoB16669941.42857142\n",
            " loss BtoA8173822.067857147\n",
            "epoch507\n",
            " loss AtoB16653684.837142855\n",
            " loss BtoA8193952.419999996\n",
            "epoch508\n",
            " loss AtoB16678124.442857137\n",
            " loss BtoA8184690.812142853\n",
            "epoch509\n",
            " loss AtoB16644301.034285715\n",
            " loss BtoA8223441.868571435\n",
            "epoch510\n",
            " loss AtoB16589763.797142852\n",
            " loss BtoA8228674.087142852\n",
            "epoch511\n",
            " loss AtoB16672568.931428568\n",
            " loss BtoA8250261.29428572\n",
            "epoch512\n",
            " loss AtoB16771573.46857143\n",
            " loss BtoA8206865.980000001\n",
            "epoch513\n",
            " loss AtoB16773831.648571437\n",
            " loss BtoA8184284.738571428\n",
            "epoch514\n",
            " loss AtoB16788443.942857143\n",
            " loss BtoA8154786.305000001\n",
            "epoch515\n",
            " loss AtoB16723589.457142847\n",
            " loss BtoA8164614.599999994\n",
            "epoch516\n",
            " loss AtoB16617909.617142858\n",
            " loss BtoA8170981.642857141\n",
            "epoch517\n",
            " loss AtoB16502602.280000001\n",
            " loss BtoA8177826.565714282\n",
            "epoch518\n",
            " loss AtoB16510747.980000002\n",
            " loss BtoA8202129.562142859\n",
            "epoch519\n",
            " loss AtoB16546310.45714285\n",
            " loss BtoA8239240.047142863\n",
            "epoch520\n",
            " loss AtoB16554663.585714294\n",
            " loss BtoA8231452.097142857\n",
            "epoch521\n",
            " loss AtoB16605478.582857136\n",
            " loss BtoA8224448.817142861\n",
            "epoch522\n",
            " loss AtoB16517116.620000007\n",
            " loss BtoA8231735.364999998\n",
            "epoch523\n",
            " loss AtoB16423747.877142865\n",
            " loss BtoA8211466.134285717\n",
            "epoch524\n",
            " loss AtoB16401639.614285715\n",
            " loss BtoA8192242.284999999\n",
            "epoch525\n",
            " loss AtoB16488566.21142857\n",
            " loss BtoA8209159.472857143\n",
            "epoch526\n",
            " loss AtoB16513846.008571425\n",
            " loss BtoA8204394.792857146\n",
            "epoch527\n",
            " loss AtoB16523120.737142868\n",
            " loss BtoA8205195.743571425\n",
            "epoch528\n",
            " loss AtoB16548200.437142853\n",
            " loss BtoA8190426.014285724\n",
            "epoch529\n",
            " loss AtoB16449586.02285714\n",
            " loss BtoA8197556.487142858\n",
            "epoch530\n",
            " loss AtoB16480497.279999992\n",
            " loss BtoA8208489.412857144\n",
            "epoch531\n",
            " loss AtoB16510671.399999995\n",
            " loss BtoA8249301.331428573\n",
            "epoch532\n",
            " loss AtoB16486452.76857143\n",
            " loss BtoA8273286.373571424\n",
            "epoch533\n",
            " loss AtoB16422695.717142876\n",
            " loss BtoA8218104.168571428\n",
            "epoch534\n",
            " loss AtoB16383467.574285699\n",
            " loss BtoA8152583.2492857175\n",
            "epoch535\n",
            " loss AtoB16419697.454285724\n",
            " loss BtoA8139661.780000003\n",
            "epoch536\n",
            " loss AtoB16462634.342857148\n",
            " loss BtoA8154116.921428578\n",
            "epoch537\n",
            " loss AtoB16469131.171428574\n",
            " loss BtoA8148991.220714287\n",
            "epoch538\n",
            " loss AtoB16382228.24285714\n",
            " loss BtoA8130530.590714282\n",
            "epoch539\n",
            " loss AtoB16327273.571428562\n",
            " loss BtoA8162030.95428572\n",
            "epoch540\n",
            " loss AtoB16339046.000000002\n",
            " loss BtoA8159692.625714288\n",
            "epoch541\n",
            " loss AtoB16452950.368571436\n",
            " loss BtoA8149425.347142856\n",
            "epoch542\n",
            " loss AtoB16623045.988571433\n",
            " loss BtoA8172802.244285712\n",
            "epoch543\n",
            " loss AtoB16530960.839999983\n",
            " loss BtoA8138435.41857143\n",
            "epoch544\n",
            " loss AtoB16373778.78285714\n",
            " loss BtoA8127460.398571425\n",
            "epoch545\n",
            " loss AtoB16282590.654285723\n",
            " loss BtoA8118779.921428572\n",
            "epoch546\n",
            " loss AtoB16288325.268571433\n",
            " loss BtoA8110769.224285713\n",
            "epoch547\n",
            " loss AtoB16339633.07428571\n",
            " loss BtoA8125572.897857143\n",
            "epoch548\n",
            " loss AtoB16382115.642857153\n",
            " loss BtoA8129477.404999997\n",
            "epoch549\n",
            " loss AtoB16480455.60857143\n",
            " loss BtoA8130158.828571428\n",
            "epoch550\n",
            " loss AtoB16464276.017142866\n",
            " loss BtoA8116160.654285716\n",
            "epoch551\n",
            " loss AtoB16384912.791428568\n",
            " loss BtoA8110288.605714288\n",
            "epoch552\n",
            " loss AtoB16301424.03714285\n",
            " loss BtoA8139424.358571434\n",
            "epoch553\n",
            " loss AtoB16257363.134285726\n",
            " loss BtoA8152375.7885714285\n",
            "epoch554\n",
            " loss AtoB16253053.065714285\n",
            " loss BtoA8131976.2757142885\n",
            "epoch555\n",
            " loss AtoB16279273.08\n",
            " loss BtoA8136429.540000003\n",
            "epoch556\n",
            " loss AtoB16288577.411428561\n",
            " loss BtoA8126818.797142862\n",
            "epoch557\n",
            " loss AtoB16300344.988571433\n",
            " loss BtoA8130561.184999995\n",
            "epoch558\n",
            " loss AtoB16367430.00857143\n",
            " loss BtoA8157976.792857133\n",
            "epoch559\n",
            " loss AtoB16387899.928571435\n",
            " loss BtoA8158376.318571425\n",
            "epoch560\n",
            " loss AtoB16389813.96857142\n",
            " loss BtoA8109735.411428568\n",
            "epoch561\n",
            " loss AtoB16415646.948571442\n",
            " loss BtoA8126548.768571435\n",
            "epoch562\n",
            " loss AtoB16381649.402857149\n",
            " loss BtoA8148999.187142863\n",
            "epoch563\n",
            " loss AtoB16333909.60857143\n",
            " loss BtoA8130568.800714286\n",
            "epoch564\n",
            " loss AtoB16345988.228571434\n",
            " loss BtoA8135085.7614285685\n",
            "epoch565\n",
            " loss AtoB16329624.011428572\n",
            " loss BtoA8098881.995714283\n",
            "epoch566\n",
            " loss AtoB16352725.562857151\n",
            " loss BtoA8084934.259999992\n",
            "epoch567\n",
            " loss AtoB16375530.882857151\n",
            " loss BtoA8093058.039999998\n",
            "epoch568\n",
            " loss AtoB16393596.759999998\n",
            " loss BtoA8117024.255714289\n",
            "epoch569\n",
            " loss AtoB16366899.168571418\n",
            " loss BtoA8172907.352857142\n",
            "epoch570\n",
            " loss AtoB16368233.542857157\n",
            " loss BtoA8226351.545\n",
            "epoch571\n",
            " loss AtoB16356557.582857145\n",
            " loss BtoA8199030.635714284\n",
            "epoch572\n",
            " loss AtoB16326903.517142862\n",
            " loss BtoA8204926.86142857\n",
            "epoch573\n",
            " loss AtoB16361755.511428574\n",
            " loss BtoA8193575.731428566\n",
            "epoch574\n",
            " loss AtoB16372578.799999986\n",
            " loss BtoA8146754.0850000065\n",
            "epoch575\n",
            " loss AtoB16350732.254285717\n",
            " loss BtoA8115197.148571427\n",
            "epoch576\n",
            " loss AtoB16425329.854285704\n",
            " loss BtoA8122905.700714291\n",
            "epoch577\n",
            " loss AtoB16417591.514285706\n",
            " loss BtoA8085481.63357143\n",
            "epoch578\n",
            " loss AtoB16424602.691428572\n",
            " loss BtoA8092865.814285711\n",
            "epoch579\n",
            " loss AtoB16348889.69142856\n",
            " loss BtoA8106779.533571428\n",
            "epoch580\n",
            " loss AtoB16288842.054285726\n",
            " loss BtoA8140752.841428573\n",
            "epoch581\n",
            " loss AtoB16273241.800000008\n",
            " loss BtoA8165100.170000001\n",
            "epoch582\n",
            " loss AtoB16354832.560000004\n",
            " loss BtoA8159714.337142858\n",
            "epoch583\n",
            " loss AtoB16353380.491428565\n",
            " loss BtoA8154891.690000002\n",
            "epoch584\n",
            " loss AtoB16423800.065714292\n",
            " loss BtoA8118478.992857146\n",
            "epoch585\n",
            " loss AtoB16368513.094285723\n",
            " loss BtoA8113126.187142848\n",
            "epoch586\n",
            " loss AtoB16350209.76857142\n",
            " loss BtoA8101903.494285717\n",
            "epoch587\n",
            " loss AtoB16390204.808571441\n",
            " loss BtoA8127524.312857142\n",
            "epoch588\n",
            " loss AtoB16360821.79142857\n",
            " loss BtoA8154539.302142862\n",
            "epoch589\n",
            " loss AtoB16292799.682857135\n",
            " loss BtoA8178031.584285714\n",
            "epoch590\n",
            " loss AtoB16294640.608571425\n",
            " loss BtoA8216597.128571425\n",
            "epoch591\n",
            " loss AtoB16332096.177142859\n",
            " loss BtoA8215833.334285711\n",
            "epoch592\n",
            " loss AtoB16333627.21428572\n",
            " loss BtoA8161271.547142856\n",
            "epoch593\n",
            " loss AtoB16429252.079999989\n",
            " loss BtoA8148980.161428573\n",
            "epoch594\n",
            " loss AtoB16449543.454285724\n",
            " loss BtoA8111248.769285722\n",
            "epoch595\n",
            " loss AtoB16435657.865714286\n",
            " loss BtoA8097212.92571429\n",
            "epoch596\n",
            " loss AtoB16358307.282857146\n",
            " loss BtoA8082584.7671428565\n",
            "epoch597\n",
            " loss AtoB16401734.97714286\n",
            " loss BtoA8105958.069285719\n",
            "epoch598\n",
            " loss AtoB16409256.094285717\n",
            " loss BtoA8109083.317857143\n",
            "epoch599\n",
            " loss AtoB16285776.64\n",
            " loss BtoA8089658.628571427\n",
            "epoch600\n",
            " loss AtoB16260758.494285712\n",
            " loss BtoA8090777.234285714\n",
            "epoch601\n",
            " loss AtoB16211126.811428567\n",
            " loss BtoA8087112.322857142\n",
            "epoch602\n",
            " loss AtoB16191075.062857138\n",
            " loss BtoA8096845.234285711\n",
            "epoch603\n",
            " loss AtoB16189890.557142863\n",
            " loss BtoA8086737.855714289\n",
            "epoch604\n",
            " loss AtoB16259319.780000001\n",
            " loss BtoA8080923.3971428545\n",
            "epoch605\n",
            " loss AtoB16313223.76285716\n",
            " loss BtoA8078002.591428578\n",
            "epoch606\n",
            " loss AtoB16259803.562857132\n",
            " loss BtoA8086942.369999997\n",
            "epoch607\n",
            " loss AtoB16303309.922857149\n",
            " loss BtoA8098469.28214285\n",
            "epoch608\n",
            " loss AtoB16301349.714285716\n",
            " loss BtoA8091081.871428574\n",
            "epoch609\n",
            " loss AtoB16251687.457142854\n",
            " loss BtoA8112210.872857144\n",
            "epoch610\n",
            " loss AtoB16235812.834285712\n",
            " loss BtoA8067740.044285713\n",
            "epoch611\n",
            " loss AtoB16236728.554285716\n",
            " loss BtoA8077261.499285711\n",
            "epoch612\n",
            " loss AtoB16286009.265714297\n",
            " loss BtoA8047377.3221428525\n",
            "epoch613\n",
            " loss AtoB16254132.951428572\n",
            " loss BtoA8028662.605714288\n",
            "epoch614\n",
            " loss AtoB16205768.594285725\n",
            " loss BtoA8020463.86142857\n",
            "epoch615\n",
            " loss AtoB16177742.577142855\n",
            " loss BtoA8042510.18142857\n",
            "epoch616\n",
            " loss AtoB16213222.568571435\n",
            " loss BtoA8070290.7914285725\n",
            "epoch617\n",
            " loss AtoB16227590.937142862\n",
            " loss BtoA8077939.681428566\n",
            "epoch618\n",
            " loss AtoB16344484.294285722\n",
            " loss BtoA8126715.513571436\n",
            "epoch619\n",
            " loss AtoB16299858.17142858\n",
            " loss BtoA8091110.81285714\n",
            "epoch620\n",
            " loss AtoB16261503.640000002\n",
            " loss BtoA8052207.409999997\n",
            "epoch621\n",
            " loss AtoB16228573.997142864\n",
            " loss BtoA8019361.812142849\n",
            "epoch622\n",
            " loss AtoB16190651.220000006\n",
            " loss BtoA8011527.60642858\n",
            "epoch623\n",
            " loss AtoB16145693.954285711\n",
            " loss BtoA8006145.042142855\n",
            "epoch624\n",
            " loss AtoB16179536.457142865\n",
            " loss BtoA8019252.090714284\n",
            "epoch625\n",
            " loss AtoB16279855.751428585\n",
            " loss BtoA8043098.060714289\n",
            "epoch626\n",
            " loss AtoB16303368.220000006\n",
            " loss BtoA8053225.049999997\n",
            "epoch627\n",
            " loss AtoB16247394.485714296\n",
            " loss BtoA8047228.787142863\n",
            "epoch628\n",
            " loss AtoB16222352.548571419\n",
            " loss BtoA8053460.009999999\n",
            "epoch629\n",
            " loss AtoB16152422.95142857\n",
            " loss BtoA8072372.842857143\n",
            "epoch630\n",
            " loss AtoB16097344.082857143\n",
            " loss BtoA8061704.411428568\n",
            "epoch631\n",
            " loss AtoB16109090.280000007\n",
            " loss BtoA8050337.868571432\n",
            "epoch632\n",
            " loss AtoB16178844.351428572\n",
            " loss BtoA8056993.128571433\n",
            "epoch633\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-19-44a7f4bb7faa>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     29\u001b[0m             _, __, loss_AtoB_temp, loss_BtoA_temp, summary = sess.run([AtoB_trainer, BtoA_trainer, loss_AtoB, loss_BtoA, merged], \n\u001b[1;32m     30\u001b[0m                                                              feed_dict={input_A:A_input[ptr], input_B:B_input[ptr], \n\u001b[0;32m---> 31\u001b[0;31m                                                                         lr:curr_lr})\n\u001b[0m\u001b[1;32m     32\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     33\u001b[0m             \u001b[0mloss_A2B\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mloss_AtoB_temp\u001b[0m\u001b[0;34m/\u001b[0m\u001b[0;36m350\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    948\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    949\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 950\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    951\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    952\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1171\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mhandle\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mfeed_dict_tensor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1172\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m-> 1173\u001b[0;31m                              feed_dict_tensor, options, run_metadata)\n\u001b[0m\u001b[1;32m   1174\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1175\u001b[0m       \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1348\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1349\u001b[0m       return self._do_call(_run_fn, feeds, fetches, targets, options,\n\u001b[0;32m-> 1350\u001b[0;31m                            run_metadata)\n\u001b[0m\u001b[1;32m   1351\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1352\u001b[0m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_prun_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeeds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetches\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m   1354\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1355\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1356\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1357\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1358\u001b[0m       \u001b[0mmessage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[0;34m(feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[1;32m   1339\u001b[0m       \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_extend_graph\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1340\u001b[0m       return self._call_tf_sessionrun(\n\u001b[0;32m-> 1341\u001b[0;31m           options, feed_dict, fetch_list, target_list, run_metadata)\n\u001b[0m\u001b[1;32m   1342\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1343\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_prun_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.6/dist-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_call_tf_sessionrun\u001b[0;34m(self, options, feed_dict, fetch_list, target_list, run_metadata)\u001b[0m\n\u001b[1;32m   1427\u001b[0m     return tf_session.TF_SessionRun_wrapper(\n\u001b[1;32m   1428\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptions\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1429\u001b[0;31m         run_metadata)\n\u001b[0m\u001b[1;32m   1430\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1431\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_call_tf_sessionprun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KvneHqf0AYmy",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "with open('/content/gdrive/My Drive/MSc ML/0091/64x64/Loss/A2B', 'rb') as f:\n",
        "    loss_B2A = pickle.load(f)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mrDSqk0ZusvW",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class VGG(object):\n",
        "    def __init__(self, input_img):\n",
        "        # load vgg19\n",
        "        self.vgg_layers = scipy.io.loadmat('/content/gdrive/My Drive/MSc ML/0091/vgg19.mat')[\"layers\"]\n",
        "        self.input_img = input_img\n",
        "        # mean of vgg19\n",
        "        self.mean_pixels = np.array([123.68, 116.779, 103.939]).reshape((1, 1, 1, 3))\n",
        "\n",
        "    def _weights(self, layer_idx, expected_layer_name):\n",
        "        \"\"\"\n",
        "        get pre-trained weights of each layer\n",
        "        \n",
        "        :param layer_idx: layer id in VGG\n",
        "        :param expected_layer_name: layer name\n",
        "        :return: pre-trained weight W and b\n",
        "        \"\"\"\n",
        "        W = self.vgg_layers[0][layer_idx][0][0][2][0][0]\n",
        "        b = self.vgg_layers[0][layer_idx][0][0][2][0][1]\n",
        "        # name of current layer\n",
        "        layer_name = self.vgg_layers[0][layer_idx][0][0][0][0]\n",
        "        assert layer_name == expected_layer_name, print(\"Layer name error!\")\n",
        "\n",
        "        return W, b.reshape(b.size)\n",
        "\n",
        "    def conv2d_relu(self, prev_layer, layer_idx, layer_name):\n",
        "        \"\"\"\n",
        "        :param prev_layer: previous layer\n",
        "        :param layer_idx: layer id in VGG\n",
        "        :param layer_name: layer name\n",
        "        \"\"\"\n",
        "        with tf.variable_scope(layer_name):\n",
        "            # load parameters\n",
        "            W, b = self._weights(layer_idx, layer_name)\n",
        "            # initialize parameters\n",
        "            W = tf.constant(W, name=\"weights\")\n",
        "            b = tf.constant(b, name=\"bias\")\n",
        "            # convolution \n",
        "            conv2d = tf.nn.conv2d(input=prev_layer,\n",
        "                                  filter=W,\n",
        "                                  strides=[1, 1, 1, 1],\n",
        "                                  padding=\"SAME\")\n",
        "            # activation\n",
        "            out = tf.nn.relu(conv2d + b)\n",
        "        setattr(self, layer_name, out)\n",
        "\n",
        "    def avgpool(self, prev_layer, layer_name):\n",
        "        \"\"\"\n",
        "        average pooling层（这里参考了原论文中提到了avg-pooling比max-pooling效果好，所以采用avg-pooling）\n",
        "        \n",
        "        :param prev_layer: 前一层网络（卷积层）\n",
        "        :param layer_name: 当前layer命名\n",
        "        \"\"\"\n",
        "        with tf.variable_scope(layer_name):\n",
        "            # average pooling\n",
        "            out = tf.nn.avg_pool(value=prev_layer,\n",
        "                                 ksize=[1, 2, 2, 1],\n",
        "                                 strides=[1, 2, 2, 1],\n",
        "                                 padding=\"SAME\")\n",
        "\n",
        "        setattr(self, layer_name, out)\n",
        "\n",
        "    def load(self):\n",
        "        \"\"\"\n",
        "        加载pre-trained的数据\n",
        "        \"\"\"\n",
        "        self.conv2d_relu(self.input_img, 0, \"conv1_1\")\n",
        "        self.conv2d_relu(self.conv1_1, 2, \"conv1_2\")\n",
        "        self.avgpool(self.conv1_2, \"avgpool1\")\n",
        "        self.conv2d_relu(self.avgpool1, 5, \"conv2_1\")\n",
        "        self.conv2d_relu(self.conv2_1, 7, \"conv2_2\")\n",
        "        self.avgpool(self.conv2_2, \"avgpool2\")\n",
        "        self.conv2d_relu(self.avgpool2, 10, \"conv3_1\")\n",
        "        self.conv2d_relu(self.conv3_1, 12, \"conv3_2\")\n",
        "        self.conv2d_relu(self.conv3_2, 14, \"conv3_3\")\n",
        "        self.conv2d_relu(self.conv3_3, 16, \"conv3_4\")\n",
        "        self.avgpool(self.conv3_4, \"avgpool3\")\n",
        "        self.conv2d_relu(self.avgpool3, 19, \"conv4_1\")\n",
        "        self.conv2d_relu(self.conv4_1, 21, \"conv4_2\")\n",
        "        self.conv2d_relu(self.conv4_2, 23, \"conv4_3\")\n",
        "        self.conv2d_relu(self.conv4_3, 25, \"conv4_4\")\n",
        "        self.avgpool(self.conv4_4, \"avgpool4\")\n",
        "        self.conv2d_relu(self.avgpool4, 28, \"conv5_1\")\n",
        "        self.conv2d_relu(self.conv5_1, 30, \"conv5_2\")\n",
        "        self.conv2d_relu(self.conv5_2, 32, \"conv5_3\")\n",
        "        self.conv2d_relu(self.conv5_3, 34, \"conv5_4\")\n",
        "        self.avgpool(self.conv5_4, \"avgpool5\")"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7y87CSyjuu58",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class LatentTransfer(object):\n",
        "    def __init__(self, initial_img, conv1_1_, conv2_1_, conv3_1_, conv4_1_, conv5_1_, conv4_2_, img_width, img_height, ptr):\n",
        "\n",
        "        self.conv1_1_ = conv1_1_\n",
        "        self.conv2_1_ = conv2_1_\n",
        "        self.conv3_1_ = conv3_1_\n",
        "        self.conv4_1_ = conv4_1_\n",
        "        self.conv4_2_ = conv4_2_\n",
        "        self.conv5_1_ = conv5_1_\n",
        "        self.img_width = img_width\n",
        "        self.img_height = img_height\n",
        "        self.ptr = ptr\n",
        "        self.initial_img = initial_img\n",
        "        \n",
        "        # feature layer\n",
        "        self.content_layer = \"conv3_2\"\n",
        "        self.style_layers = [\"conv1_1\", \"conv2_1\", \"conv3_1\", \"conv4_1\", \"conv5_1\"]\n",
        "\n",
        "        # weights of content loss and style loss\n",
        "        self.content_w = 1000\n",
        "        self.style_w = 0.01\n",
        "\n",
        "        # weights of style layers\n",
        "        self.style_layer_w = [0.5, 1.0, 1.5, 3.0, 4.0]\n",
        "\n",
        "        # global step and learning rate\n",
        "        self.gstep = tf.Variable(0, dtype=tf.int32, trainable=False, name=\"global_step\")  # global step\n",
        "        self.lr = 2.0\n",
        "\n",
        "\n",
        "    def create_input(self):\n",
        "        with tf.variable_scope(\"input\"):\n",
        "            self.input_img = tf.get_variable(\"in_img\", \n",
        "                                             shape=([1, self.img_height, self.img_width, 3]),\n",
        "                                             dtype=tf.float32,\n",
        "                                             initializer=tf.zeros_initializer())\n",
        "\n",
        "    def load_vgg(self):\n",
        "        self.vgg = VGG(self.input_img)\n",
        "        self.vgg.load()\n",
        "\n",
        "    def _content_loss(self, P, F):\n",
        "        \"\"\"\n",
        "        :param P: feature map of content image\n",
        "        :param F: feature map of generated image\n",
        "        \"\"\"\n",
        "        self.content_loss = tf.reduce_sum(tf.square(F - P)) / (4.0 * tf.cast(P.shape[0],tf.float32) * tf.cast(P.shape[1],tf.float32) * tf.cast(P.shape[2],tf.float32))\n",
        "        \n",
        "    def _gram_matrix(self, F, N, M):\n",
        "        \"\"\"\n",
        "        :param F: feature map\n",
        "        :param N: third dimension of feature map\n",
        "        :param M: first x second dimension of feature map\n",
        "        :return: Gram Matrix of F\n",
        "        \"\"\"\n",
        "        F = tf.reshape(F, (M, N))\n",
        "\n",
        "        return tf.matmul(tf.transpose(F), F)\n",
        "\n",
        "    def _single_style_loss(self, a, g):\n",
        "        \"\"\"\n",
        "        :param a: feature map of style image\n",
        "        :param g: feature map of generated image\n",
        "        :return: style loss\n",
        "        \"\"\"\n",
        "        N = a.shape[3]\n",
        "        M = a.shape[1] * a.shape[2]\n",
        "\n",
        "        # Gram Matrix of feature maps\n",
        "        A = self._gram_matrix(a, N, M)\n",
        "        G = self._gram_matrix(g, N, M)\n",
        "\n",
        "        return tf.reduce_sum(tf.square(G - A)) / ((2 * tf.cast(N,tf.float32) * tf.cast(M,tf.float32)) ** 2)\n",
        "\n",
        "    def _style_loss(self):\n",
        "        # style layer: conv1_1, conv2_1, conv3_1, conv4_1, conv5_1\n",
        "        E_1 = self._single_style_loss(self.conv1_1_, getattr(self.vgg, self.style_layers[0]))\n",
        "        E_2 = self._single_style_loss(self.conv2_1_, getattr(self.vgg, self.style_layers[1]))\n",
        "        E_3 = self._single_style_loss(self.conv3_1_, getattr(self.vgg, self.style_layers[2]))\n",
        "        E_4 = self._single_style_loss(self.conv4_1_, getattr(self.vgg, self.style_layers[3]))\n",
        "        E_5 = self._single_style_loss(self.conv5_1_, getattr(self.vgg, self.style_layers[4]))\n",
        "        # weighted sum\n",
        "        self.style_loss = 0.5*E_1+1.0*E_2+1.5*E_3+3.0*E_4+4.0*E_5\n",
        "\n",
        "    def losses(self):\n",
        "        \"\"\"\n",
        "        total loss\n",
        "        \"\"\"\n",
        "        with tf.variable_scope(\"losses\"):\n",
        "            # contents loss\n",
        "            with tf.Session() as sess:\n",
        "                gen_img_content = getattr(self.vgg, self.content_layer)\n",
        "            self._content_loss(self.conv4_2_, gen_img_content)\n",
        "\n",
        "            # style loss \n",
        "            self._style_loss()\n",
        "            \n",
        "            # total loss\n",
        "            self.total_loss = self.content_w * self.content_loss + self.style_w * self.style_loss\n",
        "\n",
        "    def optimize(self):\n",
        "        self.optimizer = tf.train.AdamOptimizer(self.lr).minimize(self.total_loss, global_step=self.gstep)\n",
        "\n",
        "    def create_summary(self):\n",
        "        with tf.name_scope(\"summary\"):\n",
        "            tf.summary.scalar(\"contents_loss\", self.content_loss)\n",
        "            tf.summary.scalar(\"style_loss\", self.style_loss)\n",
        "            tf.summary.scalar(\"total_loss\", self.total_loss)\n",
        "            self.summary_op = tf.summary.merge_all()\n",
        "\n",
        "    def build(self):\n",
        "        self.create_input()\n",
        "        self.load_vgg()\n",
        "        self.losses()\n",
        "        self.optimize()\n",
        "        self.create_summary()\n",
        "\n",
        "    def train(self, epoches=300):\n",
        "        skip_step = 1\n",
        "        with tf.Session() as sess:\n",
        "            sess.run(tf.global_variables_initializer())\n",
        "            writer = tf.summary.FileWriter(\"graphs/style_transfer\", sess.graph)\n",
        "            \n",
        "            sess.run(self.input_img.assign(self.initial_img))\n",
        "\n",
        "            saver = tf.train.Saver()\n",
        "            ckpt = tf.train.get_checkpoint_state('/content/gdrive/My Drive/MSc ML/0091/model.ckpt')\n",
        "            if ckpt and ckpt.model_checkpoint_path:\n",
        "                print(\"You have pre-trained model, if you do not want to use this, please delete the existing one.\")\n",
        "                saver.restore(sess, ckpt.model_checkpoint_path)\n",
        "\n",
        "            initial_step = self.gstep.eval()\n",
        "\n",
        "            for epoch in range(initial_step, epoches):\n",
        "                # 前面几轮每隔10个epoch生成一张图片\n",
        "                if epoch >= 5 and epoch < 20:\n",
        "                    skip_step = 10\n",
        "                # 后面每隔20个epoch生成一张图片\n",
        "                elif epoch >= 20:\n",
        "                    skip_step = 20\n",
        "                \n",
        "                sess.run(self.optimizer)\n",
        "                \n",
        "            gen_image, total_loss, summary = sess.run([self.input_img,\n",
        "                                                       self.total_loss,\n",
        "                                                       self.summary_op])\n",
        "            # 对生成的图片逆向mean-center，即在每个channel上加上mean\n",
        "            gen_image = gen_image + self.vgg.mean_pixels \n",
        "            writer.add_summary(summary, global_step=epoch)\n",
        "\n",
        "            filename = \"/content/gdrive/My Drive/MSc ML/0091/64x64/output10/\" + str(self.ptr) + \".png\"\n",
        "            imageio.imwrite(filename, gen_image.reshape(gen_image.shape[1], gen_image.shape[2], gen_image.shape[3]))\n",
        "\n",
        "            print(\"Step {}\\n   Sum: {:5.1f}\".format(epoch + 1, np.sum(gen_image)))\n",
        "            print(\"   Loss: {:5.1f}\".format(total_loss))\n",
        "                        "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "w9su0TbZ6ttP",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "mean = np.mean(A_input,0)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8PGKs1VQUegu",
        "colab_type": "code",
        "outputId": "5e136490-60e4-456f-a7d1-4cc51adc845d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 747
        }
      },
      "source": [
        "# B to A\n",
        "for ptr in range(12,22):\n",
        "    tf.reset_default_graph()\n",
        "\n",
        "    input_A = tf.placeholder(tf.float32, [batch_size, img_width, img_height, img_depth], name=\"input_A\")\n",
        "    input_B = tf.placeholder(tf.float32, [batch_size, img_width, img_height, img_depth], name=\"input_B\")\n",
        "\n",
        "    conv1_1_B, conv2_1_B, conv3_1_B, conv4_1_B, conv4_2_B, conv5_1_B = feature_translator(input_A, name=\"translatorAtoB\")\n",
        "    conv1_1B, conv2_1B, conv3_1B, conv4_1B, conv4_2B, conv5_1B = feature_extractor(input_B, name=\"extractor\")\n",
        "\n",
        "    conv1_1_A, conv2_1_A, conv3_1_A, conv4_1_A, conv4_2_A, conv5_1_A = feature_translator(input_B, name=\"translatorBtoA\")\n",
        "    conv1_1A, conv2_1A, conv3_1A, conv4_1A, conv4_2A, conv5_1A = feature_extractor(input_A, name=\"extractor\")\n",
        "\n",
        "\n",
        "    saver = tf.train.Saver()\n",
        "    with tf.Session() as sess:\n",
        "        sess.run(tf.global_variables_initializer())\n",
        "        saver.restore(sess, '/content/gdrive/My Drive/MSc ML/0091/64x64/checkpoint/dense-1000/featuretrans-1000')\n",
        "        conv1_1_A_temp, conv2_1_A_temp, conv3_1_A_temp, conv4_1_A_temp, conv4_2_A_temp, conv5_1_A_temp = sess.run([conv1_1_A, conv2_1_A, conv3_1_A, conv4_1_A, conv4_2_A, conv5_1_A], \n",
        "                                                                                                                  feed_dict={input_B:B_input[ptr]})\n",
        "\n",
        "\n",
        "    # style transfer\n",
        "    style_transfer = LatentTransfer(np.random.normal(127.5,10.0,(1,64,64,3)), conv1_1_A_temp, conv2_1_A_temp, conv3_1_A_temp, conv4_1_A_temp, conv5_1_A_temp, conv4_2_A_temp, img_width, img_height, ptr)\n",
        "    style_transfer.build()\n",
        "    style_transfer.train(100)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "W0905 20:00:11.516843 140237731604352 util.py:61] Lossy conversion from float64 to uint8. Range [104.69343392992019, 382.51709716796876]. Convert image to uint8 prior to saving to suppress this warning.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Step 100\n",
            "   Sum: 2735359.4\n",
            "   Loss: 222030656.0\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "W0905 20:00:28.901626 140237731604352 util.py:61] Lossy conversion from float64 to uint8. Range [126.86334120178222, 370.55771606445313]. Convert image to uint8 prior to saving to suppress this warning.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Step 100\n",
            "   Sum: 2765734.0\n",
            "   Loss: 179247616.0\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "W0905 20:00:45.305198 140237731604352 util.py:61] Lossy conversion from float64 to uint8. Range [133.97930990600585, 383.47550268554687]. Convert image to uint8 prior to saving to suppress this warning.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Step 100\n",
            "   Sum: 2811913.7\n",
            "   Loss: 201449312.0\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "W0905 20:01:02.454618 140237731604352 util.py:61] Lossy conversion from float64 to uint8. Range [141.05530630493163, 359.6902817382812]. Convert image to uint8 prior to saving to suppress this warning.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Step 100\n",
            "   Sum: 2875297.9\n",
            "   Loss: 223738176.0\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "W0905 20:01:19.046793 140237731604352 util.py:61] Lossy conversion from float64 to uint8. Range [130.14670263671874, 365.6444317626953]. Convert image to uint8 prior to saving to suppress this warning.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Step 100\n",
            "   Sum: 2796804.4\n",
            "   Loss: 204630560.0\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "W0905 20:01:36.273123 140237731604352 util.py:61] Lossy conversion from float64 to uint8. Range [118.23410974884032, 348.7937668457031]. Convert image to uint8 prior to saving to suppress this warning.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Step 100\n",
            "   Sum: 3030995.0\n",
            "   Loss: 346092480.0\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "W0905 20:01:53.396337 140237731604352 util.py:61] Lossy conversion from float64 to uint8. Range [114.94311319732665, 367.49178283691407]. Convert image to uint8 prior to saving to suppress this warning.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Step 100\n",
            "   Sum: 2778815.2\n",
            "   Loss: 208126080.0\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "W0905 20:02:10.013514 140237731604352 util.py:61] Lossy conversion from float64 to uint8. Range [120.121954788208, 363.6588360595703]. Convert image to uint8 prior to saving to suppress this warning.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Step 100\n",
            "   Sum: 2794467.8\n",
            "   Loss: 211573712.0\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "W0905 20:02:27.157706 140237731604352 util.py:61] Lossy conversion from float64 to uint8. Range [102.76862686157227, 371.09790502929684]. Convert image to uint8 prior to saving to suppress this warning.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Step 100\n",
            "   Sum: 2958105.1\n",
            "   Loss: 224787424.0\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "W0905 20:02:44.350972 140237731604352 util.py:61] Lossy conversion from float64 to uint8. Range [121.78834789276124, 381.92566650390626]. Convert image to uint8 prior to saving to suppress this warning.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Step 100\n",
            "   Sum: 2789086.0\n",
            "   Loss: 238896640.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PUhzjrffL5J_",
        "colab_type": "code",
        "outputId": "29cdea0c-dca3-422a-d7bf-1b563bb32cea",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 747
        }
      },
      "source": [
        "for ptr in range(12, 22):\n",
        "    tf.reset_default_graph()\n",
        "\n",
        "    input_A = tf.placeholder(tf.float32, [batch_size, img_width, img_height, img_depth], name=\"input_A\")\n",
        "    input_B = tf.placeholder(tf.float32, [batch_size, img_width, img_height, img_depth], name=\"input_B\")\n",
        "\n",
        "    conv1_1_B, conv2_1_B, conv3_1_B, conv4_1_B, conv4_2_B, conv5_1_B = feature_translator(input_A, name=\"translatorAtoB\")\n",
        "    conv1_1B, conv2_1B, conv3_1B, conv4_1B, conv4_2B, conv5_1B = feature_extractor(input_B, name=\"extractor\")\n",
        "\n",
        "    conv1_1_A, conv2_1_A, conv3_1_A, conv4_1_A, conv4_2_A, conv5_1_A = feature_translator(input_B, name=\"translatorBtoA\")\n",
        "    conv1_1A, conv2_1A, conv3_1A, conv4_1A, conv4_2A, conv5_1A = feature_extractor(input_A, name=\"extractor\")\n",
        "\n",
        "\n",
        "    saver = tf.train.Saver()\n",
        "    with tf.Session() as sess:\n",
        "        sess.run(tf.global_variables_initializer())\n",
        "        saver.restore(sess, '/content/gdrive/My Drive/MSc ML/0091/64x64/checkpoint/dense-1000/featuretrans-1000')\n",
        "        conv1_1_A_temp, conv2_1_A_temp, conv3_1_A_temp, conv4_1_A_temp, conv4_2_A_temp, conv5_1_A_temp = sess.run([conv1_1_B, conv2_1_B, conv3_1_B, conv4_1_B, conv4_2_B, conv5_1_B], \n",
        "                                                                                                                  feed_dict={input_A:A_input[ptr]})\n",
        "\n",
        "\n",
        "    # style transfer\n",
        "    style_transfer = LatentTransfer(np.random.normal(127.5,10.0,(1,64,64,3)), conv1_1_A_temp, conv2_1_A_temp, conv3_1_A_temp, conv4_1_A_temp, conv5_1_A_temp, conv4_2_A_temp, img_width, img_height, ptr)\n",
        "    style_transfer.build()\n",
        "    style_transfer.train(100)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "W0905 20:07:24.640275 140237731604352 util.py:61] Lossy conversion from float64 to uint8. Range [78.85172544860839, 409.48209448242187]. Convert image to uint8 prior to saving to suppress this warning.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Step 100\n",
            "   Sum: 2814768.5\n",
            "   Loss: 423641056.0\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "W0905 20:07:41.857168 140237731604352 util.py:61] Lossy conversion from float64 to uint8. Range [89.26175962829589, 397.9545056152344]. Convert image to uint8 prior to saving to suppress this warning.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Step 100\n",
            "   Sum: 2859920.4\n",
            "   Loss: 375310976.0\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "W0905 20:07:59.002156 140237731604352 util.py:61] Lossy conversion from float64 to uint8. Range [112.26409040832519, 412.2032849121094]. Convert image to uint8 prior to saving to suppress this warning.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Step 100\n",
            "   Sum: 2923825.7\n",
            "   Loss: 388798784.0\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "W0905 20:08:15.737011 140237731604352 util.py:61] Lossy conversion from float64 to uint8. Range [104.22045748376846, 381.2018505859375]. Convert image to uint8 prior to saving to suppress this warning.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Step 100\n",
            "   Sum: 2962867.7\n",
            "   Loss: 426796224.0\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "W0905 20:08:32.783720 140237731604352 util.py:61] Lossy conversion from float64 to uint8. Range [98.57282720947265, 409.45056616210934]. Convert image to uint8 prior to saving to suppress this warning.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Step 100\n",
            "   Sum: 2903592.7\n",
            "   Loss: 416371456.0\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "W0905 20:08:49.154825 140237731604352 util.py:61] Lossy conversion from float64 to uint8. Range [55.01946340942382, 368.8085858154297]. Convert image to uint8 prior to saving to suppress this warning.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Step 100\n",
            "   Sum: 3128614.6\n",
            "   Loss: 538019264.0\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "W0905 20:09:06.230504 140237731604352 util.py:61] Lossy conversion from float64 to uint8. Range [108.78278385543823, 387.64765234375]. Convert image to uint8 prior to saving to suppress this warning.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Step 100\n",
            "   Sum: 2875860.0\n",
            "   Loss: 393314912.0\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "W0905 20:09:23.143379 140237731604352 util.py:61] Lossy conversion from float64 to uint8. Range [99.05548702621459, 390.6495739746094]. Convert image to uint8 prior to saving to suppress this warning.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Step 100\n",
            "   Sum: 2887669.1\n",
            "   Loss: 412234048.0\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "W0905 20:09:39.957492 140237731604352 util.py:61] Lossy conversion from float64 to uint8. Range [103.4981403543949, 385.5727917480469]. Convert image to uint8 prior to saving to suppress this warning.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Step 100\n",
            "   Sum: 3029126.8\n",
            "   Loss: 425245408.0\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "W0905 20:09:57.164739 140237731604352 util.py:61] Lossy conversion from float64 to uint8. Range [95.15247695922852, 398.9484020996094]. Convert image to uint8 prior to saving to suppress this warning.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Step 100\n",
            "   Sum: 2835536.4\n",
            "   Loss: 474607840.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sluwNprmL2u-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}