Keypoint_cnn.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Keypoint-cnn.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EuNN96CjP7Sa"
      },
      "source": [
        "Key.Net: Keypoint Detection by Handcrafted and Learned CNN Filters\r\n",
        "\r\n",
        "\r\n",
        "---\r\n",
        "\r\n",
        "\r\n",
        "**Assistant Professor :**\r\n",
        "* Dr. Fadaeieslam\r\n",
        "\r\n",
        "**By :**\r\n",
        "* Amir Shokri\r\n",
        "* Farshad Asgharzade\r\n",
        "* Alireza Gholamnia\r\n",
        "\r\n",
        "**Emails :**\r\n",
        "* amirsh.nll@gmail.com\r\n",
        "* Farshad_asgharzade@hotmail.com\r\n",
        "* gholamniareza@gmail.com"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ee5GXd587V6Z",
        "outputId": "47c697f1-54de-478f-dbd8-1d780e63b55a"
      },
      "source": [
        "from google.colab import drive\r\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.activity.readonly&response_type=code\n",
            "\n",
            "Enter your authorization code:\n",
            "4/1AY0e-g6hscTjujPaSWUvQOZP9WiVYDlXHREIvYTcpxuOcIes7KlIB4a7vMs\n",
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7iTTFpyP4bfX"
      },
      "source": [
        "import drive.MyDrive.keypoint\r\n",
        "import drive.MyDrive.keypoint.keyNet"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lMAbXsq6N3KV",
        "outputId": "c725f417-4a13-4874-e844-ff64994e2f15"
      },
      "source": [
        "!pip install tensorflow==1.14"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting tensorflow==1.14\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/de/f0/96fb2e0412ae9692dbf400e5b04432885f677ad6241c088ccc5fe7724d69/tensorflow-1.14.0-cp36-cp36m-manylinux1_x86_64.whl (109.2MB)\n",
            "\u001b[K     |████████████████████████████████| 109.2MB 51kB/s \n",
            "\u001b[?25hRequirement already satisfied: grpcio>=1.8.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.14) (1.32.0)\n",
            "Requirement already satisfied: protobuf>=3.6.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.14) (3.12.4)\n",
            "Requirement already satisfied: keras-preprocessing>=1.0.5 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.14) (1.1.2)\n",
            "Requirement already satisfied: gast>=0.2.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.14) (0.3.3)\n",
            "Collecting keras-applications>=1.0.6\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/71/e3/19762fdfc62877ae9102edf6342d71b28fbfd9dea3d2f96a882ce099b03f/Keras_Applications-1.0.8-py3-none-any.whl (50kB)\n",
            "\u001b[K     |████████████████████████████████| 51kB 6.7MB/s \n",
            "\u001b[?25hRequirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.14) (1.1.0)\n",
            "Requirement already satisfied: wrapt>=1.11.1 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.14) (1.12.1)\n",
            "Requirement already satisfied: wheel>=0.26 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.14) (0.36.2)\n",
            "Collecting tensorflow-estimator<1.15.0rc0,>=1.14.0rc0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/3c/d5/21860a5b11caf0678fbc8319341b0ae21a07156911132e0e71bffed0510d/tensorflow_estimator-1.14.0-py2.py3-none-any.whl (488kB)\n",
            "\u001b[K     |████████████████████████████████| 491kB 55.5MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy<2.0,>=1.14.5 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.14) (1.19.5)\n",
            "Requirement already satisfied: six>=1.10.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.14) (1.15.0)\n",
            "Requirement already satisfied: astor>=0.6.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.14) (0.8.1)\n",
            "Requirement already satisfied: google-pasta>=0.1.6 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.14) (0.2.0)\n",
            "Collecting tensorboard<1.15.0,>=1.14.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/91/2d/2ed263449a078cd9c8a9ba50ebd50123adf1f8cfbea1492f9084169b89d9/tensorboard-1.14.0-py3-none-any.whl (3.1MB)\n",
            "\u001b[K     |████████████████████████████████| 3.2MB 41.1MB/s \n",
            "\u001b[?25hRequirement already satisfied: absl-py>=0.7.0 in /usr/local/lib/python3.6/dist-packages (from tensorflow==1.14) (0.10.0)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from protobuf>=3.6.1->tensorflow==1.14) (51.1.2)\n",
            "Requirement already satisfied: h5py in /usr/local/lib/python3.6/dist-packages (from keras-applications>=1.0.6->tensorflow==1.14) (2.10.0)\n",
            "Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.6/dist-packages (from tensorboard<1.15.0,>=1.14.0->tensorflow==1.14) (1.0.1)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.6/dist-packages (from tensorboard<1.15.0,>=1.14.0->tensorflow==1.14) (3.3.3)\n",
            "Requirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.6/dist-packages (from markdown>=2.6.8->tensorboard<1.15.0,>=1.14.0->tensorflow==1.14) (3.3.0)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.6/dist-packages (from importlib-metadata; python_version < \"3.8\"->markdown>=2.6.8->tensorboard<1.15.0,>=1.14.0->tensorflow==1.14) (3.4.0)\n",
            "Requirement already satisfied: typing-extensions>=3.6.4; python_version < \"3.8\" in /usr/local/lib/python3.6/dist-packages (from importlib-metadata; python_version < \"3.8\"->markdown>=2.6.8->tensorboard<1.15.0,>=1.14.0->tensorflow==1.14) (3.7.4.3)\n",
            "Installing collected packages: keras-applications, tensorflow-estimator, tensorboard, tensorflow\n",
            "  Found existing installation: tensorflow-estimator 2.4.0\n",
            "    Uninstalling tensorflow-estimator-2.4.0:\n",
            "      Successfully uninstalled tensorflow-estimator-2.4.0\n",
            "  Found existing installation: tensorboard 2.4.0\n",
            "    Uninstalling tensorboard-2.4.0:\n",
            "      Successfully uninstalled tensorboard-2.4.0\n",
            "  Found existing installation: tensorflow 2.4.0\n",
            "    Uninstalling tensorflow-2.4.0:\n",
            "      Successfully uninstalled tensorflow-2.4.0\n",
            "Successfully installed keras-applications-1.0.8 tensorboard-1.14.0 tensorflow-1.14.0 tensorflow-estimator-1.14.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9kLmJyvPE2kv",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "06a8163e-3a75-481a-83f0-c759ba811ddf"
      },
      "source": [
        "!pip install ipykernel"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: ipykernel in /usr/local/lib/python3.6/dist-packages (4.10.1)\n",
            "Requirement already satisfied: ipython>=4.0.0 in /usr/local/lib/python3.6/dist-packages (from ipykernel) (5.5.0)\n",
            "Requirement already satisfied: traitlets>=4.1.0 in /usr/local/lib/python3.6/dist-packages (from ipykernel) (4.3.3)\n",
            "Requirement already satisfied: tornado>=4.0 in /usr/local/lib/python3.6/dist-packages (from ipykernel) (5.1.1)\n",
            "Requirement already satisfied: jupyter-client in /usr/local/lib/python3.6/dist-packages (from ipykernel) (5.3.5)\n",
            "Requirement already satisfied: setuptools>=18.5 in /usr/local/lib/python3.6/dist-packages (from ipython>=4.0.0->ipykernel) (51.1.2)\n",
            "Requirement already satisfied: decorator in /usr/local/lib/python3.6/dist-packages (from ipython>=4.0.0->ipykernel) (4.4.2)\n",
            "Requirement already satisfied: pexpect; sys_platform != \"win32\" in /usr/local/lib/python3.6/dist-packages (from ipython>=4.0.0->ipykernel) (4.8.0)\n",
            "Requirement already satisfied: prompt-toolkit<2.0.0,>=1.0.4 in /usr/local/lib/python3.6/dist-packages (from ipython>=4.0.0->ipykernel) (1.0.18)\n",
            "Requirement already satisfied: simplegeneric>0.8 in /usr/local/lib/python3.6/dist-packages (from ipython>=4.0.0->ipykernel) (0.8.1)\n",
            "Requirement already satisfied: pygments in /usr/local/lib/python3.6/dist-packages (from ipython>=4.0.0->ipykernel) (2.6.1)\n",
            "Requirement already satisfied: pickleshare in /usr/local/lib/python3.6/dist-packages (from ipython>=4.0.0->ipykernel) (0.7.5)\n",
            "Requirement already satisfied: ipython-genutils in /usr/local/lib/python3.6/dist-packages (from traitlets>=4.1.0->ipykernel) (0.2.0)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from traitlets>=4.1.0->ipykernel) (1.15.0)\n",
            "Requirement already satisfied: pyzmq>=13 in /usr/local/lib/python3.6/dist-packages (from jupyter-client->ipykernel) (20.0.0)\n",
            "Requirement already satisfied: jupyter-core>=4.6.0 in /usr/local/lib/python3.6/dist-packages (from jupyter-client->ipykernel) (4.7.0)\n",
            "Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.6/dist-packages (from jupyter-client->ipykernel) (2.8.1)\n",
            "Requirement already satisfied: ptyprocess>=0.5 in /usr/local/lib/python3.6/dist-packages (from pexpect; sys_platform != \"win32\"->ipython>=4.0.0->ipykernel) (0.7.0)\n",
            "Requirement already satisfied: wcwidth in /usr/local/lib/python3.6/dist-packages (from prompt-toolkit<2.0.0,>=1.0.4->ipython>=4.0.0->ipykernel) (0.2.5)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8b45sfxC65Ru",
        "outputId": "9b8956a9-1c93-4dc9-be8d-1f49bc4e6e80"
      },
      "source": [
        "pip install --user gast==0.2.2"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting gast==0.2.2\n",
            "  Downloading https://files.pythonhosted.org/packages/4e/35/11749bf99b2d4e3cceb4d55ca22590b0d7c2c62b9de38ac4a4a7f4687421/gast-0.2.2.tar.gz\n",
            "Building wheels for collected packages: gast\n",
            "  Building wheel for gast (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for gast: filename=gast-0.2.2-cp36-none-any.whl size=7540 sha256=f3cd1dda55b92ee33cb37d7c1dd89477c7690c3cfca57fbb3e2934d89c250cbc\n",
            "  Stored in directory: /root/.cache/pip/wheels/5c/2e/7e/a1d4d4fcebe6c381f378ce7743a3ced3699feb89bcfbdadadd\n",
            "Successfully built gast\n",
            "\u001b[31mERROR: tensorflow-probability 0.12.1 has requirement gast>=0.3.2, but you'll have gast 0.2.2 which is incompatible.\u001b[0m\n",
            "Installing collected packages: gast\n",
            "Successfully installed gast-0.2.2\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zGpwZGq2gS-H",
        "outputId": "ca711ee5-75c6-412b-dff3-214802997c91"
      },
      "source": [
        "!wget http://icvl.ee.ic.ac.uk/vbalnt/hpatches/hpatches-sequences-release.tar.gz"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2021-01-19 21:39:00--  http://icvl.ee.ic.ac.uk/vbalnt/hpatches/hpatches-sequences-release.tar.gz\n",
            "Resolving icvl.ee.ic.ac.uk (icvl.ee.ic.ac.uk)... 155.198.116.158\n",
            "Connecting to icvl.ee.ic.ac.uk (icvl.ee.ic.ac.uk)|155.198.116.158|:80... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 1279780913 (1.2G) [application/x-gzip]\n",
            "Saving to: ‘hpatches-sequences-release.tar.gz’\n",
            "\n",
            "hpatches-sequences- 100%[===================>]   1.19G  20.5MB/s    in 61s     \n",
            "\n",
            "2021-01-19 21:40:02 (20.1 MB/s) - ‘hpatches-sequences-release.tar.gz’ saved [1279780913/1279780913]\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rKlf4N67xfkT"
      },
      "source": [
        "!tar -xvf  '/content/hpatches-sequences-release.tar.gz' -C '/content/'"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "z8pSQha9N2_1"
      },
      "source": [
        ""
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "22qvExEtBcTm"
      },
      "source": [
        "**train_network.py**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YGzLmYSRBeFJ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "4c0f4dc1-073d-484d-f939-b36a63318713"
      },
      "source": [
        "import os, argparse, math, cv2, sys, time\r\n",
        "import numpy as np\r\n",
        "from tqdm import tqdm\r\n",
        "import tensorflow as tf\r\n",
        "from drive.MyDrive.keypoint.keyNet.model.keynet_architecture import keynet\r\n",
        "from drive.MyDrive.keypoint.keyNet.loss.score_loss_function import msip_loss_function\r\n",
        "import drive.MyDrive.keypoint.keyNet.aux.tools as aux\r\n",
        "import drive.MyDrive.keypoint.HSequences_bench.tools.geometry_tools as geo_tools\r\n",
        "import drive.MyDrive.keypoint.HSequences_bench.tools.repeatability_tools as rep_tools\r\n",
        "import drive.MyDrive.keypoint.keyNet\r\n",
        "from drive.MyDrive.keypoint.keyNet.datasets.tf_dataset import tf_dataset as tf_dataset\r\n",
        "from contextlib import contextmanager\r\n",
        "from argparse import ArgumentParser\r\n",
        "import skimage"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:516: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
            "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
            "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:517: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
            "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
            "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:518: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
            "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
            "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:519: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
            "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
            "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:520: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
            "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
            "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/dtypes.py:525: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
            "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n",
            "/usr/local/lib/python3.6/dist-packages/tensorboard/compat/tensorflow_stub/dtypes.py:541: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
            "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
            "/usr/local/lib/python3.6/dist-packages/tensorboard/compat/tensorflow_stub/dtypes.py:542: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
            "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
            "/usr/local/lib/python3.6/dist-packages/tensorboard/compat/tensorflow_stub/dtypes.py:543: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
            "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
            "/usr/local/lib/python3.6/dist-packages/tensorboard/compat/tensorflow_stub/dtypes.py:544: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
            "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
            "/usr/local/lib/python3.6/dist-packages/tensorboard/compat/tensorflow_stub/dtypes.py:545: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
            "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
            "/usr/local/lib/python3.6/dist-packages/tensorboard/compat/tensorflow_stub/dtypes.py:550: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
            "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/compat/v2_compat.py:61: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "non-resource variables are not supported in the long term\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VPMYOPm7DUNi"
      },
      "source": [
        "os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gFwZgHSFDYhZ"
      },
      "source": [
        "@contextmanager\r\n",
        "def suppress_stdout():\r\n",
        "    with open(os.devnull, \"w\") as devnull:\r\n",
        "        old_stdout = sys.stdout\r\n",
        "        sys.stdout = devnull\r\n",
        "        try:\r\n",
        "            yield\r\n",
        "        finally:\r\n",
        "            sys.stdout = old_stdout"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pN1EoQhGDa2R"
      },
      "source": [
        "def save_log(str, file):\r\n",
        "    print(str)\r\n",
        "    file.write(str+'\\n')\r\n",
        "    file.flush()"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "u36UhoKZDdDc"
      },
      "source": [
        "try:\r\n",
        "    # python 3.4+ should use builtin unittest.mock not mock package\r\n",
        "    from unittest.mock import patch\r\n",
        "except ImportError:\r\n",
        "    from mock import patch"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "587EYXd_DqjS"
      },
      "source": [
        "def train_keynet_architecture():\r\n",
        "\r\n",
        "    parser = argparse.ArgumentParser(description='Train Key.Net Architecture')\r\n",
        "\r\n",
        "    parser.add_argument('--data-dir', type=str, default='drive/MyDrive/keypoint/keyNet/ImageNet',\r\n",
        "                        help='The root path to the data from which the synthetic dataset will be created.')\r\n",
        "\r\n",
        "    parser.add_argument('--tfrecord-dir', type=str, default='keyNet/tfrecords',\r\n",
        "                        help='The path to save the generated tfrecords.')\r\n",
        "\r\n",
        "    parser.add_argument('--weights-dir', type=str, default='keyNet/weights',\r\n",
        "                        help='The path to save the Key.Net weights.')\r\n",
        "\r\n",
        "    parser.add_argument('--write-summary', type=bool, default=False,\r\n",
        "                        help='Set to True if you desire to save the summary of the training.')\r\n",
        "\r\n",
        "    parser.add_argument('--network-version', type=str, default='KeyNet_default',\r\n",
        "                        help='The Key.Net network version name')\r\n",
        "\r\n",
        "    parser.add_argument('--num-epochs', type=int, default=20,\r\n",
        "                        help='Number of epochs for training.')\r\n",
        "\r\n",
        "    parser.add_argument('--epochs-val', type=int, default=5,\r\n",
        "                        help='Set the number of training epochs between repeteability checks on the validation set.')\r\n",
        "\r\n",
        "    parser.add_argument('--batch-size', type=int, default=32,\r\n",
        "                        help='The batch size for training.')\r\n",
        "\r\n",
        "    parser.add_argument('--init-initial-learning-rate', type=float, default=1e-3,\r\n",
        "                        help='The init initial learning rate value.')\r\n",
        "\r\n",
        "    parser.add_argument('--weights-decay', type=float, default=1e-5,\r\n",
        "                        help='The weight decay value.')\r\n",
        "\r\n",
        "    parser.add_argument('--num-epochs-before-decay', type=int, default=10,\r\n",
        "                        help='The number of epochs before decay.')\r\n",
        "\r\n",
        "    parser.add_argument('--learning-rate-decay-factor', type=float, default=0.7,\r\n",
        "                        help='The learning rate decay factor.')\r\n",
        "\r\n",
        "    parser.add_argument('--random-seed', type=int, default=12345,\r\n",
        "                        help='The random seed value for TensorFlow and Numpy.')\r\n",
        "\r\n",
        "    parser.add_argument('--resume-training', type=bool, default=False,\r\n",
        "                        help='Set True if resume training is desired.')\r\n",
        "\r\n",
        "    parser.add_argument('--num-filters', type=int, default=8,\r\n",
        "                        help='The number of filters in each learnable block.')\r\n",
        "\r\n",
        "    parser.add_argument('--num-learnable-blocks', type=int, default=3,\r\n",
        "                        help='The number of learnable blocks after handcrafted block.')\r\n",
        "\r\n",
        "    parser.add_argument('--num-levels-within-net', type=int, default=3,\r\n",
        "                        help='The number of pyramid levels inside the architecture.')\r\n",
        "\r\n",
        "    parser.add_argument('--factor-scaling-pyramid', type=float, default=1.2,\r\n",
        "                        help='The scale factor between the multi-scale pyramid levels in the architecture.')\r\n",
        "\r\n",
        "    parser.add_argument('--conv-kernel-size', type=int, default=5,\r\n",
        "                        help='The size of the convolutional filters in each of the learnable blocks.')\r\n",
        "\r\n",
        "    parser.add_argument('--nms-size', type=int, default=15,\r\n",
        "                        help='The NMS size for computing the validation repeatability.')\r\n",
        "\r\n",
        "    parser.add_argument('--border-size', type=int, default=15,\r\n",
        "                        help='The number of pixels to remove from the borders to compute the repeatability.')\r\n",
        "\r\n",
        "    parser.add_argument('--max-angle', type=int, default=45,\r\n",
        "                        help='The max angle value for generating a synthetic view to train Key.Net.')\r\n",
        "\r\n",
        "    parser.add_argument('--max-scale', type=int, default=2.0,\r\n",
        "                        help='The max scale value for generating a synthetic view to train Key.Net.')\r\n",
        "\r\n",
        "    parser.add_argument('--max-shearing', type=int, default=0.8,\r\n",
        "                        help='The max shearing value for generating a synthetic view to train Key.Net.')\r\n",
        "\r\n",
        "    parser.add_argument('--patch-size', type=int, default=192,\r\n",
        "                        help='The patch size of the generated dataset.')\r\n",
        "\r\n",
        "    parser.add_argument('--weight-coordinates', type=bool, default=True,\r\n",
        "                        help='Weighting coordinates by their scores.')\r\n",
        "\r\n",
        "    parser.add_argument('--is-debugging', type=bool, default=False,\r\n",
        "                        help='Set variable to True if you desire to train network on a smaller dataset.')\r\n",
        "\r\n",
        "    parser.add_argument('--gpu-memory-fraction', type=float, default=0.9,\r\n",
        "                        help='The fraction of GPU used by the script.')\r\n",
        "\r\n",
        "    parser.add_argument('--gpu-visible-devices', type=str, default=\"0\",\r\n",
        "                        help='Set CUDA_VISIBLE_DEVICES variable.')\r\n",
        "    \r\n",
        "    #parser.parse_args(['--data-dir', '/path/to/ImageNet', '--tfrecord-dir', 'keyNet/tfrecords/' , '--weights-dir', 'keyNet/weights' , '--write-summary', False , '--network-version', 'KeyNet_default', '--num-epochs', 25 , '--epochs-val', 3 , '--batch-size', 32 , '--init-initial-learning-rate', 1e-3 , '--weights-decay', 1e-5 , '--num-epochs-before-decay', 10 , '--learning-rate-decay-factor', 0.7 , '--random-seed', 12345 , '--resume-training', False , '--num-filters', 8 , '--num-learnable-blocks', 3 , '--num-levels-within-net', 3 , '--factor-scaling-pyramid', 1.2 , '--conv-kernel-size', 5 , '--nms-size', 15 , '--border-size', 15 , '--max-angle', 45 , '--max-scale', 2 , '--max-shearing', 1 , '--patch-size', 192 , '--weight-coordinates', True , '--is-debugging', False , '--gpu-memory-fraction', 0.9 , '--gpu-visible-devices', \"0\"])\r\n",
        "\r\n",
        "    parser.parse_args(['--data-dir', 'drive/MyDrive/keypoint/keyNet/ImageNet'])\r\n",
        "    parser.parse_args(['--tfrecord-dir', 'keyNet/tfrecords/' ])\r\n",
        "    parser.parse_args(['--weights-dir', 'drive/MyDrive/keypoint/keyNet/weights/' ])\r\n",
        "    parser.parse_args(['--write-summary', False ])\r\n",
        "    parser.parse_args(['--network-version', 'KeyNet_default'])\r\n",
        "    parser.parse_args(['--num-epochs', '20' ])\r\n",
        "    parser.parse_args(['--epochs-val', '5' ])\r\n",
        "    parser.parse_args(['--batch-size', '32' ])\r\n",
        "    parser.parse_args(['--init-initial-learning-rate', '1e-3' ])\r\n",
        "    parser.parse_args(['--weights-decay', '1e-5' ])\r\n",
        "    parser.parse_args(['--num-epochs-before-decay', '10' ])\r\n",
        "    parser.parse_args(['--learning-rate-decay-factor', '0.7' ])\r\n",
        "    parser.parse_args(['--random-seed', '12345' ])\r\n",
        "    parser.parse_args(['--resume-training', 'False' ])\r\n",
        "    parser.parse_args(['--num-filters', '8' ])\r\n",
        "    parser.parse_args(['--num-learnable-blocks', '3' ])\r\n",
        "    parser.parse_args(['--num-levels-within-net', '3' ])\r\n",
        "    parser.parse_args(['--factor-scaling-pyramid', '1.2' ])\r\n",
        "    parser.parse_args(['--conv-kernel-size', '5' ])\r\n",
        "    parser.parse_args(['--nms-size', '15' ])\r\n",
        "    parser.parse_args(['--border-size', '15' ])\r\n",
        "    parser.parse_args(['--max-angle', '45' ])\r\n",
        "    parser.parse_args(['--max-scale', '2' ])\r\n",
        "    parser.parse_args(['--max-shearing', '1' ])\r\n",
        "    parser.parse_args(['--patch-size', '192' ])\r\n",
        "    parser.parse_args(['--weight-coordinates', 'True' ])\r\n",
        "    parser.parse_args(['--is-debugging', 'False' ])\r\n",
        "    parser.parse_args(['--gpu-memory-fraction', '0.9' ])\r\n",
        "    parser.parse_args(['--gpu-visible-devices', \"0\" ])\r\n",
        "    import sys\r\n",
        "    sys.argv=['']\r\n",
        "    del sys\r\n",
        "    args = parser.parse_args()\r\n",
        "\r\n",
        "    aux.check_directory('logs')\r\n",
        "    log_file = open('logs/'+args.network_version + \".txt\", \"w+\")\r\n",
        "\r\n",
        "    # Set CUDA GPU environment\r\n",
        "    os.environ[\"CUDA_VISIBLE_DEVICES\"] = args.gpu_visible_devices\r\n",
        "\r\n",
        "    version_network_name = args.network_version\r\n",
        "\r\n",
        "    # Check directories\r\n",
        "    aux.check_directory('drive/MyDrive/keypoint/keyNet/data/')\r\n",
        "    aux.check_directory(\"drive/MyDrive/keypoint/\" + args.weights_dir)\r\n",
        "    aux.check_directory(\"drive/MyDrive/keypoint/\" + args.weights_dir + '/' + version_network_name)\r\n",
        "    aux.check_directory(\"drive/MyDrive/keypoint/\" + args.weights_dir + '/' + version_network_name + '_best')\r\n",
        "    aux.check_directory(\"drive/MyDrive/keypoint/\" + args.tfrecord_dir)\r\n",
        "\r\n",
        "    #aux.nll_check_tensorboard_directory()\r\n",
        "\r\n",
        "    # Set random seeds\r\n",
        "    tf.set_random_seed(args.random_seed)\r\n",
        "    np.random.seed(args.random_seed)\r\n",
        "\r\n",
        "    print('Start training Key.Net Architecture: ' + version_network_name)\r\n",
        "\r\n",
        "    def check_val_rep(num_points=25):\r\n",
        "        total_rep_avg = []\r\n",
        "        num_examples = dataset_class.get_num_patches(True)\r\n",
        "        fetches = [src_score_maps_activation, dst_score_maps_activation]\r\n",
        "\r\n",
        "        for _ in tqdm(range(num_examples)):\r\n",
        "            images_batch, images_dst_batch, h_src_2_dst_batch, h_dst_2_src_batch = sess.run(next_val_batch)\r\n",
        "\r\n",
        "            feed_dict = {\r\n",
        "                input_network_src: images_batch,\r\n",
        "                input_network_dst: images_dst_batch,\r\n",
        "                h_src_2_dst: h_src_2_dst_batch,\r\n",
        "                h_dst_2_src: h_dst_2_src_batch,\r\n",
        "                phase_train: False,\r\n",
        "                dimension_image: np.array(\r\n",
        "                    [images_batch.shape[0], images_batch.shape[1], images_batch.shape[2]], dtype=np.int32),\r\n",
        "                dimension_image_dst: np.array(\r\n",
        "                    [images_dst_batch.shape[0], images_dst_batch.shape[1], images_dst_batch.shape[2]], dtype=np.int32),\r\n",
        "            }\r\n",
        "\r\n",
        "            src_scores, dst_scores = sess.run(fetches, feed_dict=feed_dict)\r\n",
        "\r\n",
        "            # Apply NMS\r\n",
        "            src_scores = rep_tools.apply_nms(src_scores[0, :, :, 0], args.nms_size)\r\n",
        "            dst_scores = rep_tools.apply_nms(dst_scores[0, :, :, 0], args.nms_size)\r\n",
        "\r\n",
        "            hom = geo_tools.prepare_homography(h_dst_2_src_batch[0])\r\n",
        "            mask_src, mask_dst = geo_tools.create_common_region_masks(hom, images_batch[0].shape, images_dst_batch[0].shape)\r\n",
        "\r\n",
        "            src_scores = np.multiply(src_scores, mask_src)\r\n",
        "            dst_scores = np.multiply(dst_scores, mask_dst)\r\n",
        "\r\n",
        "            src_pts = geo_tools.get_point_coordinates(src_scores, num_points=num_points, order_coord='xysr')\r\n",
        "            dst_pts = geo_tools.get_point_coordinates(dst_scores, num_points=num_points, order_coord='xysr')\r\n",
        "\r\n",
        "            dst_to_src_pts = geo_tools.apply_homography_to_points(dst_pts, hom)\r\n",
        "\r\n",
        "            repeatability_results = rep_tools.compute_repeatability(src_pts, dst_to_src_pts)\r\n",
        "\r\n",
        "            total_rep_avg.append(repeatability_results['rep_single_scale'])\r\n",
        "        return np.asarray(total_rep_avg).mean()\r\n",
        "\r\n",
        "    def train_epoch():\r\n",
        "\r\n",
        "        total_loss_avg = []\r\n",
        "        num_examples = dataset_class.get_num_patches()\r\n",
        "\r\n",
        "        for step in tqdm(range(int(math.ceil(num_examples / args.batch_size))+1)):\r\n",
        "\r\n",
        "            images_batch, images_dst_batch, h_src_2_dst_batch, h_dst_2_src_batch = sess.run(next_batch)\r\n",
        "\r\n",
        "            feed_dict = {\r\n",
        "                input_network_src: images_batch,\r\n",
        "                input_network_dst: images_dst_batch,\r\n",
        "                input_border_mask: aux.remove_borders(np.ones_like(images_batch), 16),\r\n",
        "                h_src_2_dst: h_src_2_dst_batch,\r\n",
        "                h_dst_2_src: h_dst_2_src_batch,\r\n",
        "                phase_train: True,\r\n",
        "                dimension_image: np.array([images_batch.shape[0], images_batch.shape[1], images_batch.shape[2]], dtype=np.int32),\r\n",
        "                dimension_image_dst: np.array([images_dst_batch.shape[0], images_dst_batch.shape[1], images_dst_batch.shape[2]],dtype=np.int32),\r\n",
        "                }\r\n",
        "\r\n",
        "            fetches = [train_op, loss_net, global_step, merged_summary]\r\n",
        "            _, loss, global_step_count, summary = sess.run(fetches, feed_dict=feed_dict)\r\n",
        "\r\n",
        "            if args.write_summary:\r\n",
        "                train_writer.add_summary(summary, global_step_count)\r\n",
        "\r\n",
        "            total_loss_avg.append(loss)\r\n",
        "\r\n",
        "            if step % 50 == 0:\r\n",
        "\r\n",
        "                feed_dict = {\r\n",
        "                    input_network_src: np.reshape(images_batch[0, :, :, :], (1, images_batch.shape[1], images_batch.shape[2], images_batch.shape[3])),\r\n",
        "                    input_network_dst: np.reshape(images_dst_batch[0, :, :, :], (1, images_dst_batch.shape[1], images_dst_batch.shape[2], images_dst_batch.shape[3])),\r\n",
        "                    phase_train: False,\r\n",
        "                    dimension_image: np.array([1, images_batch.shape[1], images_batch.shape[2]],dtype=np.int32),\r\n",
        "                    dimension_image_dst: np.array([1, images_dst_batch.shape[1], images_dst_batch.shape[2]],dtype=np.int32),\r\n",
        "                }\r\n",
        "\r\n",
        "                fetches = [src_score_maps_activation, dst_score_maps_activation]\r\n",
        "                deep_src, deep_dst = sess.run(fetches, feed_dict=feed_dict)\r\n",
        "\r\n",
        "                deep_src = aux.remove_borders(deep_src, 16)\r\n",
        "                deep_dst = aux.remove_borders(deep_dst, 16)\r\n",
        "\r\n",
        "                cv2.imwrite('drive/MyDrive/keypoint/keyNet/data/image_dst_' + version_network_name + '.png', 255 * images_dst_batch[0,:,:,0])\r\n",
        "                cv2.imwrite('drive/MyDrive/keypoint/keyNet/data/KeyNet_dst_' + version_network_name + '.png', 255 * deep_dst[0,:,:, 0] / deep_dst[0,:,:,0].max())\r\n",
        "                cv2.imwrite('drive/MyDrive/keypoint/keyNet/data/image_src_' + version_network_name + '.png', 255 * images_batch[0,:,:,0])\r\n",
        "                cv2.imwrite('drive/MyDrive/keypoint/keyNet/data/KeyNet_src_' + version_network_name + '.png', 255 * deep_src[0,:,:, 0] / deep_src[0,:,:, 0].max())\r\n",
        "\r\n",
        "        return np.asarray(total_loss_avg).mean()\r\n",
        "\r\n",
        "    with tf.Graph().as_default():\r\n",
        "\r\n",
        "        with tf.name_scope('inputs'):\r\n",
        "\r\n",
        "            # Define the input tensor shape\r\n",
        "            tensor_input_shape = (None, None, None, 1)\r\n",
        "            tensor_homography_shape = (None, 8)\r\n",
        "\r\n",
        "            # Define Placeholders\r\n",
        "            input_network_src = tf.placeholder(dtype=tf.float32, shape=tensor_input_shape, name='input_network_src')\r\n",
        "            input_network_dst = tf.placeholder(dtype=tf.float32, shape=tensor_input_shape, name='input_network_dst')\r\n",
        "            input_border_mask = tf.placeholder(dtype=tf.float32, shape=tensor_input_shape, name='input_border_mask')\r\n",
        "            h_src_2_dst = tf.placeholder(dtype=tf.float32, shape=tensor_homography_shape, name='H_scr_2_dst')\r\n",
        "            h_dst_2_src = tf.placeholder(dtype=tf.float32, shape=tensor_homography_shape, name='H_dst_2_src')\r\n",
        "            dimension_image = tf.placeholder(dtype=tf.int32, shape=(3,), name='dimension_image')\r\n",
        "            dimension_image_dst = tf.placeholder(dtype=tf.int32, shape=(3,), name='dimension_image_dst')\r\n",
        "            phase_train = tf.placeholder(tf.bool, name='phase_train')\r\n",
        "\r\n",
        "        with tf.name_scope('model_deep_detector'):\r\n",
        "\r\n",
        "            MSIP_sizes = [8, 16, 24, 32, 40]\r\n",
        "            MSIP_factor_loss = [256.0, 64.0, 16.0, 4.0, 1.0]\r\n",
        "\r\n",
        "            deep_architecture = keynet(args, MSIP_sizes)\r\n",
        "\r\n",
        "            src_score_maps = deep_architecture.model(input_network_src, phase_train, dimension_image, reuse=False)\r\n",
        "            dst_score_maps = deep_architecture.model(input_network_dst, phase_train, dimension_image_dst, reuse=True)\r\n",
        "\r\n",
        "            kernels = deep_architecture.get_kernels()\r\n",
        "\r\n",
        "        # Create Dataset\r\n",
        "        dataset_class = tf_dataset(args.data_dir, \"drive/MyDrive/keypoint/\" + args.tfrecord_dir, 32, args.batch_size,\r\n",
        "                                   args.max_angle, args.max_scale, args.max_shearing, args.random_seed, args.is_debugging)\r\n",
        "        train_dataset = dataset_class.create_dataset_object()\r\n",
        "        dataset_it = train_dataset.make_one_shot_iterator()\r\n",
        "        next_batch = dataset_it.get_next()\r\n",
        "        val_dataset = dataset_class.create_dataset_object(is_val=True)\r\n",
        "        dataset_val_it = val_dataset.make_one_shot_iterator()\r\n",
        "        next_val_batch = dataset_val_it.get_next()\r\n",
        "\r\n",
        "        # Learning Settings\r\n",
        "        num_batches_per_epoch = dataset_class.get_num_patches() / args.batch_size\r\n",
        "        num_steps_per_epoch = num_batches_per_epoch  # Because one step is one batch processed\r\n",
        "        decay_steps = int(args.num_epochs_before_decay * num_steps_per_epoch)\r\n",
        "\r\n",
        "        global_step = tf.train.get_or_create_global_step()\r\n",
        "\r\n",
        "        lr = tf.train.exponential_decay(\r\n",
        "            learning_rate   = args.init_initial_learning_rate,\r\n",
        "            global_step     = global_step,\r\n",
        "            decay_steps     = decay_steps,\r\n",
        "            decay_rate      = args.learning_rate_decay_factor,\r\n",
        "            staircase       = True)\r\n",
        "\r\n",
        "        optimizer = tf.train.AdamOptimizer(learning_rate=lr)\r\n",
        "\r\n",
        "        src_score_maps_activation = src_score_maps['output']\r\n",
        "        dst_score_maps_activation = dst_score_maps['output']\r\n",
        "\r\n",
        "        # Loss Function\r\n",
        "        MSIP_elements = {}\r\n",
        "        loss_net = 0.0\r\n",
        "\r\n",
        "        for MSIP_idx in range(len(MSIP_sizes)):\r\n",
        "            MSIP_loss, loss_elements = msip_loss_function(input_network_src, src_score_maps, dst_score_maps,\r\n",
        "                                                                      MSIP_sizes[MSIP_idx], kernels, h_src_2_dst, h_dst_2_src,\r\n",
        "                                                                      args.weight_coordinates, 32, input_border_mask)\r\n",
        "            MSIP_level_name = \"MSIP_ws_{}\".format(MSIP_sizes[MSIP_idx])\r\n",
        "            MSIP_elements[MSIP_level_name] = loss_elements\r\n",
        "            tf.summary.scalar(MSIP_level_name, MSIP_loss)\r\n",
        "            tf.losses.add_loss(MSIP_factor_loss[MSIP_idx] * MSIP_loss)\r\n",
        "            loss_net += MSIP_factor_loss[MSIP_idx] * MSIP_loss\r\n",
        "\r\n",
        "        total_loss = tf.losses.get_total_loss(add_regularization_losses=False)\r\n",
        "        train_op = tf.contrib.training.create_train_op(total_loss, optimizer)\r\n",
        "        merged_summary = tf.summary.merge_all()\r\n",
        "\r\n",
        "        # Restore Variables\r\n",
        "        if args.resume_training:\r\n",
        "            checkpoint_file_path = os.path.join(args.weights_dir, version_network_name)\r\n",
        "            variables_to_restore = tf.contrib.framework.get_variables_to_restore()\r\n",
        "            if os.listdir(checkpoint_file_path):\r\n",
        "                init_assign_op, init_feed_dict = tf.contrib.framework.assign_from_checkpoint(\r\n",
        "                    tf.train.latest_checkpoint(checkpoint_file_path), variables_to_restore)\r\n",
        "\r\n",
        "        # GPU Usage\r\n",
        "        config = tf.ConfigProto()\r\n",
        "        config.gpu_options.per_process_gpu_memory_fraction = args.gpu_memory_fraction\r\n",
        "\r\n",
        "        with tf.Session(config=config) as sess:\r\n",
        "\r\n",
        "            count = 0\r\n",
        "            max_counts = 3\r\n",
        "\r\n",
        "            sess.run(tf.global_variables_initializer())\r\n",
        "\r\n",
        "            saver = tf.train.Saver()\r\n",
        "            saver_best = tf.train.Saver()\r\n",
        "            \r\n",
        "            if args.write_summary:\r\n",
        "                train_writer = tf.summary.FileWriter('drive/MyDrive/keypoint/keyNet/logs_network/' + version_network_name + '/train ', sess.graph)\r\n",
        "\r\n",
        "            if args.resume_training and os.listdir(checkpoint_file_path):\r\n",
        "                sess.run(init_assign_op, init_feed_dict)\r\n",
        "                keynet_rep_best = check_val_rep()\r\n",
        "            else:\r\n",
        "                keynet_rep_best = 0.0\r\n",
        "\r\n",
        "            print('Start training . . .')\r\n",
        "\r\n",
        "            for epoch in range(0, args.num_epochs):\r\n",
        "\r\n",
        "                start_time = time.time()\r\n",
        "                loss = train_epoch()\r\n",
        "\r\n",
        "                aux.check_directory(\"drive/MyDrive/keypoint/\" + args.weights_dir + '/' + version_network_name + '/')\r\n",
        "                saver.save(sess, \"drive/MyDrive/keypoint/\" + args.weights_dir + '/' + version_network_name + '/model-', global_step)\r\n",
        "\r\n",
        "                if epoch % args.epochs_val == 0:\r\n",
        "                    with suppress_stdout():\r\n",
        "                        keynet_rep_val = check_val_rep()\r\n",
        "                    save_log('\\nRepeatability Validation: {:.3f}.'.format(keynet_rep_val), log_file)\r\n",
        "                else:\r\n",
        "                    keynet_rep_val = 0\r\n",
        "\r\n",
        "                # Control the early stopping\r\n",
        "                if epoch == 0:\r\n",
        "                    loss_best = loss\r\n",
        "                else:\r\n",
        "                    if keynet_rep_best < keynet_rep_val:\r\n",
        "                        keynet_rep_best = keynet_rep_val\r\n",
        "                        saver_best.save(sess, \"drive/MyDrive/keypoint/\" + args.weights_dir + '/' + version_network_name + '_best' + '/model-', global_step)\r\n",
        "                        count = 0\r\n",
        "                    elif keynet_rep_val > 0:\r\n",
        "                        if loss_best > loss:\r\n",
        "                            loss_best = loss\r\n",
        "                        else:\r\n",
        "                            count += 1\r\n",
        "\r\n",
        "                time_elapsed = time.time() - start_time\r\n",
        "\r\n",
        "                save_log('\\nEpoch ' + str(epoch) + '. Loss: ' + str(loss) + '. Time per epoch: ' + str(time_elapsed), log_file)\r\n",
        "                if keynet_rep_val > 0:\r\n",
        "                    print('Repeatability Val: {:.3f}\\n'.format(keynet_rep_val))\r\n",
        "                else:\r\n",
        "                    print('')\r\n",
        "\r\n",
        "                if count > max_counts:\r\n",
        "                    break\r\n",
        "\r\n",
        "            save_log('\\nRepeatability Val: {:.3f}. Best iteration'.format(keynet_rep_best), log_file)\r\n",
        "            log_file.close()\r\n",
        "            print('End training')"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "U5DN1Mv9BQBV"
      },
      "source": [
        "**extract_multiscale_features.py**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "077U39nZ8xXj"
      },
      "source": [
        "import os, sys, cv2\r\n",
        "from os import path, mkdir\r\n",
        "import argparse\r\n",
        "import drive.MyDrive.keypoint.keyNet.aux.tools as aux\r\n",
        "from skimage.transform import pyramid_gaussian\r\n",
        "import drive.MyDrive.keypoint.HSequences_bench.tools.geometry_tools as geo_tools\r\n",
        "import drive.MyDrive.keypoint.HSequences_bench.tools.repeatability_tools as rep_tools\r\n",
        "from drive.MyDrive.keypoint.keyNet.model.keynet_architecture import *\r\n",
        "import drive.MyDrive.keypoint.keyNet.aux.desc_aux_function as loss_desc\r\n",
        "from drive.MyDrive.keypoint.keyNet.model.hardnet_pytorch import *\r\n",
        "from drive.MyDrive.keypoint.keyNet.datasets.dataset_utils import read_bw_image\r\n",
        "import torch"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rARIrvOYBGj3"
      },
      "source": [
        "os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'\r\n"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Dltc4fQWBIEA"
      },
      "source": [
        "def check_directory(dir):\r\n",
        "    if not path.isdir(dir):\r\n",
        "        mkdir(dir)"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9qmF8HEZBKue"
      },
      "source": [
        "def create_result_dir(path):\r\n",
        "    directories = path.split('/')\r\n",
        "    tmp = ''\r\n",
        "    for idx, dir in enumerate(directories):\r\n",
        "        tmp += (dir + '/')\r\n",
        "        if idx == len(directories)-1:\r\n",
        "            continue\r\n",
        "        check_directory(tmp)"
      ],
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lKTVs01nBM-Q"
      },
      "source": [
        "def extract_multiscale_features():\r\n",
        "\r\n",
        "    parser = argparse.ArgumentParser(description='HSequences Extract Features')\r\n",
        "\r\n",
        "    parser.add_argument('--list-images', type=str, default='drive/MyDrive/keypoint/test_im/image.txt', help='File containing the image paths for extracting features.')\r\n",
        "    parser.parse_args(['--list-images', 'drive/MyDrive/keypoint/test_im/image.txt'])\r\n",
        "\r\n",
        "    parser.add_argument('--results-dir', type=str, default='extracted_features/',\r\n",
        "                        help='The output path to save the extracted keypoint.')\r\n",
        "\r\n",
        "    parser.add_argument('--network-version', type=str, default='KeyNet_default',\r\n",
        "                        help='The Key.Net network version name')\r\n",
        "\r\n",
        "    parser.add_argument('--checkpoint-det-dir', type=str, default='keyNet/pretrained_nets/KeyNet_default',\r\n",
        "                        help='The path to the checkpoint file to load the detector weights.')\r\n",
        "\r\n",
        "    parser.add_argument('--pytorch-hardnet-dir', type=str, default='keyNet/pretrained_nets/HardNet++.pth',\r\n",
        "                        help='The path to the checkpoint file to load the HardNet descriptor weights.')\r\n",
        "\r\n",
        "    # Detector Settings\r\n",
        "\r\n",
        "    parser.add_argument('--num-filters', type=int, default=8,\r\n",
        "                        help='The number of filters in each learnable block.')\r\n",
        "\r\n",
        "    parser.add_argument('--num-learnable-blocks', type=int, default=3,\r\n",
        "                        help='The number of learnable blocks after handcrafted block.')\r\n",
        "\r\n",
        "    parser.add_argument('--num-levels-within-net', type=int, default=3,\r\n",
        "                        help='The number of pyramid levels inside the architecture.')\r\n",
        "\r\n",
        "    parser.add_argument('--factor-scaling-pyramid', type=float, default=1.2,\r\n",
        "                        help='The scale factor between the multi-scale pyramid levels in the architecture.')\r\n",
        "\r\n",
        "    parser.add_argument('--conv-kernel-size', type=int, default=5,\r\n",
        "                        help='The size of the convolutional filters in each of the learnable blocks.')\r\n",
        "\r\n",
        "    # Multi-Scale Extractor Settings\r\n",
        "\r\n",
        "    parser.add_argument('--extract-MS', type=bool, default=True,\r\n",
        "                        help='Set to True if you want to extract multi-scale features.')\r\n",
        "\r\n",
        "    parser.add_argument('--num-points', type=int, default=1500,\r\n",
        "                        help='The number of desired features to extract.')\r\n",
        "\r\n",
        "    parser.add_argument('--nms-size', type=int, default=15,\r\n",
        "                        help='The NMS size for computing the validation repeatability.')\r\n",
        "\r\n",
        "    parser.add_argument('--border-size', type=int, default=15,\r\n",
        "                        help='The number of pixels to remove from the borders to compute the repeatability.')\r\n",
        "\r\n",
        "    parser.add_argument('--order-coord', type=str, default='xysr',\r\n",
        "                        help='The coordinate order that follows the extracted points. Use yxsr or xysr.')\r\n",
        "\r\n",
        "    parser.add_argument('--random-seed', type=int, default=12345,\r\n",
        "                        help='The random seed value for TensorFlow and Numpy.')\r\n",
        "\r\n",
        "    parser.add_argument('--pyramid_levels', type=int, default=5,\r\n",
        "                        help='The number of downsample levels in the pyramid.')\r\n",
        "\r\n",
        "    parser.add_argument('--upsampled-levels', type=int, default=1,\r\n",
        "                        help='The number of upsample levels in the pyramid.')\r\n",
        "\r\n",
        "    parser.add_argument('--scale-factor-levels', type=float, default=1.41,\r\n",
        "                        help='The scale factor between the pyramid levels.')\r\n",
        "\r\n",
        "    parser.add_argument('--scale-factor', type=float, default=2.,\r\n",
        "                        help='The scale factor to extract patches before descriptor.')\r\n",
        "\r\n",
        "    # GPU Settings\r\n",
        "\r\n",
        "    parser.add_argument('--gpu-memory-fraction', type=float, default=0.9,\r\n",
        "                        help='The fraction of GPU used by the script.')\r\n",
        "\r\n",
        "    parser.add_argument('--gpu-visible-devices', type=str, default=\"0\",\r\n",
        "                        help='Set CUDA_VISIBLE_DEVICES variable.')\r\n",
        "    \r\n",
        "    parser.parse_args(['--results-dir', 'drive/MyDrive/keypoint/test_im/'])\r\n",
        "    parser.parse_args(['--network-version', 'KeyNet_default'])\r\n",
        "    parser.parse_args(['--checkpoint-det-dir', 'keyNet/pretrained_nets/KeyNet_default'])\r\n",
        "    parser.parse_args(['--pytorch-hardnet-dir', 'keyNet/pretrained_nets/HardNet++.pth'])\r\n",
        "    parser.parse_args(['--num-filters', '8'])\r\n",
        "    parser.parse_args(['--num-learnable-blocks', '3'])\r\n",
        "    parser.parse_args(['--num-levels-within-net', '3'])\r\n",
        "    parser.parse_args(['--factor-scaling-pyramid', '1.2'])\r\n",
        "    parser.parse_args(['--conv-kernel-size', '5'])\r\n",
        "    parser.parse_args(['--extract-MS', 'True'])\r\n",
        "    parser.parse_args(['--num-points', '1500'])\r\n",
        "    parser.parse_args(['--nms-size', '15'])\r\n",
        "    parser.parse_args(['--border-size', '15'])\r\n",
        "    parser.parse_args(['--order-coord', 'xysr'])\r\n",
        "    parser.parse_args(['--random-seed', '12345'])\r\n",
        "    parser.parse_args(['--pyramid_levels', '5'])\r\n",
        "    parser.parse_args(['--upsampled-levels', '1'])\r\n",
        "    parser.parse_args(['--scale-factor-levels', '1.41'])\r\n",
        "    parser.parse_args(['--scale-factor', '2.'])\r\n",
        "    parser.parse_args(['--gpu-memory-fraction', '0.9'])\r\n",
        "    parser.parse_args(['--gpu-visible-devices', '0'])\r\n",
        "\r\n",
        "    args = parser.parse_known_args()[0]\r\n",
        "\r\n",
        "    # remove verbose bits from tf\r\n",
        "    os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'\r\n",
        "    tf.logging.set_verbosity(tf.logging.ERROR)\r\n",
        "\r\n",
        "    # Set CUDA GPU environment\r\n",
        "    os.environ[\"CUDA_VISIBLE_DEVICES\"] = args.gpu_visible_devices\r\n",
        "\r\n",
        "    version_network_name = args.network_version\r\n",
        "\r\n",
        "    if not args.extract_MS:\r\n",
        "        args.pyramid_levels = 0\r\n",
        "        args.upsampled_levels = 0\r\n",
        "\r\n",
        "    print('Extract features for : ' + version_network_name)\r\n",
        "\r\n",
        "    aux.check_directory(args.results_dir)\r\n",
        "    aux.check_directory(os.path.join(args.results_dir, version_network_name))\r\n",
        "\r\n",
        "    def extract_features(image):\r\n",
        "\r\n",
        "        pyramid = pyramid_gaussian(image, max_layer=args.pyramid_levels, downscale=args.scale_factor_levels)\r\n",
        "\r\n",
        "        score_maps = {}\r\n",
        "        for (j, resized) in enumerate(pyramid):\r\n",
        "            im = resized.reshape(1, resized.shape[0], resized.shape[1], 1)\r\n",
        "\r\n",
        "            feed_dict = {\r\n",
        "                input_network: im,\r\n",
        "                phase_train: False,\r\n",
        "                dimension_image: np.array([1, im.shape[1], im.shape[2]], dtype=np.int32),\r\n",
        "            }\r\n",
        "\r\n",
        "            im_scores = sess.run(maps, feed_dict=feed_dict)\r\n",
        "\r\n",
        "            im_scores = geo_tools.remove_borders(im_scores, borders=args.border_size)\r\n",
        "            score_maps['map_' + str(j + 1 + args.upsampled_levels)] = im_scores[0, :, :, 0]\r\n",
        "\r\n",
        "        if args.upsampled_levels:\r\n",
        "            for j in range(args.upsampled_levels):\r\n",
        "                factor = args.scale_factor_levels ** (args.upsampled_levels - j)\r\n",
        "                up_image = cv2.resize(image, (0, 0), fx=factor, fy=factor)\r\n",
        "\r\n",
        "                im = np.reshape(up_image, (1, up_image.shape[0], up_image.shape[1], 1))\r\n",
        "\r\n",
        "                feed_dict = {\r\n",
        "                    input_network: im,\r\n",
        "                    phase_train: False,\r\n",
        "                    dimension_image: np.array([1, im.shape[1], im.shape[2]], dtype=np.int32),\r\n",
        "                }\r\n",
        "\r\n",
        "                im_scores = sess.run(maps, feed_dict=feed_dict)\r\n",
        "\r\n",
        "                im_scores = geo_tools.remove_borders(im_scores, borders=args.border_size)\r\n",
        "                score_maps['map_' + str(j + 1)] = im_scores[0, :, :, 0]\r\n",
        "\r\n",
        "        im_pts = []\r\n",
        "        for idx_level in range(levels):\r\n",
        "\r\n",
        "            scale_value = (args.scale_factor_levels ** (idx_level - args.upsampled_levels))\r\n",
        "            scale_factor = 1. / scale_value\r\n",
        "\r\n",
        "            h_scale = np.asarray([[scale_factor, 0., 0.], [0., scale_factor, 0.], [0., 0., 1.]])\r\n",
        "            h_scale_inv = np.linalg.inv(h_scale)\r\n",
        "            h_scale_inv = h_scale_inv / h_scale_inv[2, 2]\r\n",
        "\r\n",
        "            num_points_level = point_level[idx_level]\r\n",
        "            if idx_level > 0:\r\n",
        "                res_points = int(np.asarray([point_level[a] for a in range(0, idx_level + 1)]).sum() - len(im_pts))\r\n",
        "                num_points_level = res_points\r\n",
        "\r\n",
        "            im_scores = rep_tools.apply_nms(score_maps['map_' + str(idx_level + 1)], args.nms_size)\r\n",
        "            im_pts_tmp = geo_tools.get_point_coordinates(im_scores, num_points=num_points_level, order_coord='xysr')\r\n",
        "\r\n",
        "            im_pts_tmp = geo_tools.apply_homography_to_points(im_pts_tmp, h_scale_inv)\r\n",
        "\r\n",
        "            if not idx_level:\r\n",
        "                im_pts = im_pts_tmp\r\n",
        "            else:\r\n",
        "                im_pts = np.concatenate((im_pts, im_pts_tmp), axis=0)\r\n",
        "\r\n",
        "        if args.order_coord == 'yxsr':\r\n",
        "            im_pts = np.asarray(list(map(lambda x: [x[1], x[0], x[2], x[3]], im_pts)))\r\n",
        "\r\n",
        "        im_pts = im_pts[(-1 * im_pts[:, 3]).argsort()]\r\n",
        "        im_pts = im_pts[:args.num_points]\r\n",
        "\r\n",
        "        # Extract descriptor from features\r\n",
        "        descriptors = []\r\n",
        "        im = image.reshape(1, image.shape[0], image.shape[1], 1)\r\n",
        "        for idx_desc_batch in range(int(len(im_pts) / 250 + 1)):\r\n",
        "            points_batch = im_pts[idx_desc_batch * 250: (idx_desc_batch + 1) * 250]\r\n",
        "\r\n",
        "            if not len(points_batch):\r\n",
        "                break\r\n",
        "\r\n",
        "            feed_dict = {\r\n",
        "                input_network: im,\r\n",
        "                phase_train: False,\r\n",
        "                kpts_coord: points_batch[:, :2],\r\n",
        "                kpts_scale: args.scale_factor * points_batch[:, 2],\r\n",
        "                kpts_batch: np.zeros(len(points_batch)),\r\n",
        "                dimension_image: np.array([1, im.shape[1], im.shape[2]], dtype=np.int32),\r\n",
        "            }\r\n",
        "\r\n",
        "            patch_batch = sess.run(input_patches, feed_dict=feed_dict)\r\n",
        "            patch_batch = np.reshape(patch_batch, (patch_batch.shape[0], 1, 32, 32))\r\n",
        "            data_a = torch.from_numpy(patch_batch)\r\n",
        "            data_a = data_a.cuda()\r\n",
        "            data_a = Variable(data_a)\r\n",
        "            with torch.no_grad():\r\n",
        "                out_a = model(data_a)\r\n",
        "            desc_batch = out_a.data.cpu().numpy().reshape(-1, 128)\r\n",
        "            if idx_desc_batch == 0:\r\n",
        "                descriptors = desc_batch\r\n",
        "            else:\r\n",
        "                descriptors = np.concatenate([descriptors, desc_batch], axis=0)\r\n",
        "\r\n",
        "        return im_pts, descriptors\r\n",
        "\r\n",
        "    with tf.Graph().as_default():\r\n",
        "\r\n",
        "        tf.set_random_seed(args.random_seed)\r\n",
        "\r\n",
        "        with tf.name_scope('inputs'):\r\n",
        "\r\n",
        "            # Define the input tensor shape\r\n",
        "            tensor_input_shape = (None, None, None, 1)\r\n",
        "\r\n",
        "            input_network = tf.placeholder(dtype=tf.float32, shape=tensor_input_shape, name='input_network')\r\n",
        "            dimension_image = tf.placeholder(dtype=tf.int32, shape=(3,), name='dimension_image')\r\n",
        "            kpts_coord = tf.placeholder(dtype=tf.float32, shape=(None, 2), name='kpts_coord')\r\n",
        "            kpts_batch = tf.placeholder(dtype=tf.int32, shape=(None,), name='kpts_batch')\r\n",
        "            kpts_scale = tf.placeholder(dtype=tf.float32, name='kpts_scale')\r\n",
        "            phase_train = tf.placeholder(tf.bool, name='phase_train')\r\n",
        "\r\n",
        "        with tf.name_scope('model_deep_detector'):\r\n",
        "\r\n",
        "            deep_architecture = keynet(args)\r\n",
        "            output_network = deep_architecture.model(input_network, phase_train, dimension_image, reuse=False)\r\n",
        "            maps = tf.nn.relu(output_network['output'])\r\n",
        "\r\n",
        "        # Extract Patches from inputs:\r\n",
        "        input_patches = loss_desc.build_patch_extraction(kpts_coord, kpts_batch, input_network, kpts_scale=kpts_scale)\r\n",
        "\r\n",
        "        # Define Pytorch HardNet\r\n",
        "        model = HardNet()\r\n",
        "        checkpoint = torch.load(args.pytorch_hardnet_dir)\r\n",
        "        model.load_state_dict(checkpoint['state_dict'])\r\n",
        "        model.eval()\r\n",
        "        model.cuda()\r\n",
        "\r\n",
        "        # Define variables\r\n",
        "        detect_var = [v for v in tf.trainable_variables(scope='model_deep_detector')]\r\n",
        "\r\n",
        "        if os.listdir(args.checkpoint_det_dir):\r\n",
        "            init_assign_op_det, init_feed_dict_det = tf.contrib.framework.assign_from_checkpoint(\r\n",
        "                tf.train.latest_checkpoint(args.checkpoint_det_dir), detect_var)\r\n",
        "\r\n",
        "        point_level = []\r\n",
        "        tmp = 0.0\r\n",
        "        factor_points = (args.scale_factor_levels ** 2)\r\n",
        "        levels = args.pyramid_levels + args.upsampled_levels + 1\r\n",
        "        for idx_level in range(levels):\r\n",
        "            tmp += factor_points ** (-1 * (idx_level - args.upsampled_levels))\r\n",
        "            point_level.append(args.num_points * factor_points ** (-1 * (idx_level - args.upsampled_levels)))\r\n",
        "\r\n",
        "        point_level = np.asarray(list(map(lambda x: int(x / tmp), point_level)))\r\n",
        "\r\n",
        "        # GPU Usage\r\n",
        "        config = tf.ConfigProto()\r\n",
        "        config.gpu_options.per_process_gpu_memory_fraction = args.gpu_memory_fraction\r\n",
        "        config.gpu_options.allow_growth = True\r\n",
        "\r\n",
        "        with tf.Session(config=config) as sess:\r\n",
        "            sess.run(tf.global_variables_initializer())\r\n",
        "\r\n",
        "            if os.listdir(args.checkpoint_det_dir):\r\n",
        "                sess.run(init_assign_op_det, init_feed_dict_det)\r\n",
        "\r\n",
        "            # read image and extract keypoints and descriptors\r\n",
        "            f = open(args.list_images, \"r\")\r\n",
        "            for path_to_image in f:\r\n",
        "                path = path_to_image.split('\\n')[0]\r\n",
        "\r\n",
        "                if not os.path.exists(path):\r\n",
        "                    print('[ERROR]: File {0} not found!'.format(path))\r\n",
        "                    return\r\n",
        "\r\n",
        "                create_result_dir(os.path.join(args.results_dir, version_network_name, path))\r\n",
        "\r\n",
        "                im = read_bw_image(path)\r\n",
        "\r\n",
        "                im = im.astype(float) / im.max()\r\n",
        "\r\n",
        "                im_pts, descriptors = extract_features(im)\r\n",
        "\r\n",
        "                file_name = os.path.join(args.results_dir, version_network_name, path)+'.kpt'\r\n",
        "                np.save(file_name, im_pts)\r\n",
        "\r\n",
        "                file_name = os.path.join(args.results_dir, version_network_name, path)+'.dsc'\r\n",
        "                np.save(file_name, descriptors)"
      ],
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0to1G-XxBXLV"
      },
      "source": [
        "**hsequeces_bench.py**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SSt-NKIDD6V_"
      },
      "source": [
        "import os\r\n",
        "import argparse\r\n",
        "import numpy as np\r\n",
        "import pickle\r\n",
        "from tqdm import tqdm\r\n",
        "import drive.MyDrive.keypoint.HSequences_bench.tools.aux_tools as aux\r\n",
        "import drive.MyDrive.keypoint.HSequences_bench.tools.geometry_tools as geo_tools\r\n",
        "import drive.MyDrive.keypoint.HSequences_bench.tools.repeatability_tools as rep_tools\r\n",
        "import drive.MyDrive.keypoint.HSequences_bench.tools.matching_tools as match_tools\r\n",
        "from drive.MyDrive.keypoint.HSequences_bench.tools.HSequences_reader import HSequences_dataset\r\n",
        "from drive.MyDrive.keypoint.HSequences_bench.tools.opencv_matcher import OpencvBruteForceMatcher"
      ],
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zzq9RjYpEJJB"
      },
      "source": [
        "def hsequences_metrics():\r\n",
        "\r\n",
        "    parser = argparse.ArgumentParser(description='HSequences Compute Repeatability')\r\n",
        "\r\n",
        "    parser.add_argument('--data-dir', type=str, default='keypoint/hpatches-sequences-release/',\r\n",
        "                        help='The root path to HSequences dataset.')\r\n",
        "\r\n",
        "    parser.add_argument('--results-bench-dir', type=str, default='drive/MyDrive/keypoint/HSequences_bench/results/',\r\n",
        "                        help='The output path to save the results.')\r\n",
        "\r\n",
        "    parser.add_argument('--detector-name', type=str, default='KeyNet_default',\r\n",
        "                        help='The name of the detector to compute metrics.')\r\n",
        "\r\n",
        "    parser.add_argument('--results-dir', type=str, default='drive/MyDrive/keypoint/extracted_features/',\r\n",
        "                        help='The path to the extracted points.')\r\n",
        "\r\n",
        "    parser.add_argument('--split', type=str, default='view',\r\n",
        "                        help='The name of the HPatches (HSequences) split. Use full, debug_view, debug_illum, view or illum.')\r\n",
        "\r\n",
        "    parser.add_argument('--split-path', type=str, default='/content/drive/MyDrive/keypoint/HSequences_bench/tools/splits.json',\r\n",
        "                        help='The path to the split json file.')\r\n",
        "\r\n",
        "    parser.add_argument('--top-k-points', type=int, default=1000,\r\n",
        "                        help='The number of top points to use for evaluation. Set to None to use all points')\r\n",
        "\r\n",
        "    parser.add_argument('--overlap', type=float, default=0.6,\r\n",
        "                        help='The overlap threshold for a correspondence to be considered correct.')\r\n",
        "\r\n",
        "    parser.add_argument('--pixel-threshold', type=int, default=5,\r\n",
        "                        help='The distance of pixels for a matching correspondence to be considered correct.')\r\n",
        "\r\n",
        "    parser.add_argument('--dst-to-src-evaluation', type=bool, default=True,\r\n",
        "                        help='Order to apply homography to points. Use True for dst to src, False otherwise.')\r\n",
        "\r\n",
        "    parser.add_argument('--order-coord', type=str, default='xysr',\r\n",
        "                        help='The coordinate order that follows the extracted points. Use either xysr or yxsr.')\r\n",
        "    \r\n",
        "    parser.parse_args(['--data-dir', 'drive/MyDrive/keypoint/hpatches-sequences-release/'])\r\n",
        "    parser.parse_args(['--results-bench-dir', 'drive/MyDrive/keypoint/HSequences_bench/'])\r\n",
        "    parser.parse_args(['--detector-name', 'KeyNet_default'])\r\n",
        "    parser.parse_args(['--results-dir', 'drive/MyDrive/keypoint/xtracted_features/'])\r\n",
        "    parser.parse_args(['--split', 'view'])\r\n",
        "    parser.parse_args(['--split-path', '/content/drive/MyDrive/keypoint/HSequences_bench/tools/splits.json'])\r\n",
        "    parser.parse_args(['--top-k-points', '1000'])\r\n",
        "    parser.parse_args(['--overlap', '0.6'])\r\n",
        "    parser.parse_args(['--pixel-threshold', '5'])\r\n",
        "    parser.parse_args(['--dst-to-src-evaluation', 'True'])\r\n",
        "    parser.parse_args(['--order-coord', 'xysr'])\r\n",
        "\r\n",
        "    args = parser.parse_args()\r\n",
        "\r\n",
        "    print(args.detector_name + ': ' + args.split)\r\n",
        "\r\n",
        "    # create the dataloader\r\n",
        "    data_loader = HSequences_dataset(args.data_dir, args.split, args.split_path)\r\n",
        "    results = aux.create_overlapping_results(args.detector_name, args.overlap)\r\n",
        "\r\n",
        "    # matching method\r\n",
        "    matcher = OpencvBruteForceMatcher('l2')\r\n",
        "\r\n",
        "    count_seq = 0\r\n",
        "\r\n",
        "    # load data and compute the keypoints\r\n",
        "    for sample_id, sample_data in enumerate(data_loader.extract_hsequences()):\r\n",
        "\r\n",
        "        sequence = sample_data['sequence_name']\r\n",
        "\r\n",
        "        count_seq += 1\r\n",
        "        image_src = sample_data['im_src']\r\n",
        "        images_dst = sample_data['images_dst']\r\n",
        "        h_src_2_dst = sample_data['h_src_2_dst']\r\n",
        "        h_dst_2_src = sample_data['h_dst_2_src']\r\n",
        "\r\n",
        "        print('\\nComputing ' + sequence + ' sequence {0} / {1} \\n'.format(count_seq, len(data_loader.sequences)))\r\n",
        "\r\n",
        "        for idx_im in tqdm(range(len(images_dst))):\r\n",
        "\r\n",
        "            # create the mask to filter out the points outside of the common areas\r\n",
        "            mask_src, mask_dst = geo_tools.create_common_region_masks(h_dst_2_src[idx_im], image_src.shape, images_dst[idx_im].shape)\r\n",
        "\r\n",
        "            # compute the files paths\r\n",
        "            src_pts_filename = os.path.join(args.results_dir, args.detector_name,\r\n",
        "                            'hpatches-sequences-release', '{}/1.ppm.kpt.npy'.format(sample_data['sequence_name']))\r\n",
        "            src_dsc_filename = os.path.join(args.results_dir, args.detector_name,\r\n",
        "                            'hpatches-sequences-release', '{}/1.ppm.dsc.npy'.format(sample_data['sequence_name']))\r\n",
        "            dst_pts_filename = os.path.join(args.results_dir, args.detector_name,\r\n",
        "                            'hpatches-sequences-release', '{}/{}.ppm.kpt.npy'.format(sample_data['sequence_name'], idx_im+2))\r\n",
        "            dst_dsc_filename = os.path.join(args.results_dir, args.detector_name,\r\n",
        "                            'hpatches-sequences-release', '{}/{}.ppm.dsc.npy'.format(sample_data['sequence_name'], idx_im+2))\r\n",
        "\r\n",
        "            if not os.path.isfile(src_pts_filename):\r\n",
        "                print(\"Could not find the file: \" + src_pts_filename)\r\n",
        "                return False\r\n",
        "\r\n",
        "            if not os.path.isfile(src_dsc_filename):\r\n",
        "                print(\"Could not find the file: \" + src_dsc_filename)\r\n",
        "                return False\r\n",
        "\r\n",
        "            if not os.path.isfile(dst_pts_filename):\r\n",
        "                print(\"Could not find the file: \" + dst_pts_filename)\r\n",
        "                return False\r\n",
        "\r\n",
        "            if not os.path.isfile(dst_dsc_filename):\r\n",
        "                print(\"Could not find the file: \" + dst_dsc_filename)\r\n",
        "                return False\r\n",
        "\r\n",
        "            # load the points\r\n",
        "            src_pts = np.load(src_pts_filename)\r\n",
        "            src_dsc = np.load(src_dsc_filename)\r\n",
        "\r\n",
        "            dst_pts = np.load(dst_pts_filename)\r\n",
        "            dst_dsc = np.load(dst_dsc_filename)\r\n",
        "\r\n",
        "            if args.order_coord == 'xysr':\r\n",
        "                src_pts = np.asarray(list(map(lambda x: [x[1], x[0], x[2], x[3]], src_pts)))\r\n",
        "                dst_pts = np.asarray(list(map(lambda x: [x[1], x[0], x[2], x[3]], dst_pts)))\r\n",
        "\r\n",
        "            # Check Common Points\r\n",
        "            src_idx = rep_tools.check_common_points(src_pts, mask_src)\r\n",
        "            src_pts = src_pts[src_idx]\r\n",
        "            src_dsc = src_dsc[src_idx]\r\n",
        "\r\n",
        "            dst_idx = rep_tools.check_common_points(dst_pts, mask_dst)\r\n",
        "            dst_pts = dst_pts[dst_idx]\r\n",
        "            dst_dsc = dst_dsc[dst_idx]\r\n",
        "\r\n",
        "            # Select top K points\r\n",
        "            if args.top_k_points:\r\n",
        "                src_idx = rep_tools.select_top_k(src_pts, args.top_k_points)\r\n",
        "                src_pts = src_pts[src_idx]\r\n",
        "                src_dsc = src_dsc[src_idx]\r\n",
        "\r\n",
        "                dst_idx = rep_tools.select_top_k(dst_pts, args.top_k_points)\r\n",
        "                dst_pts = dst_pts[dst_idx]\r\n",
        "                dst_dsc = dst_dsc[dst_idx]\r\n",
        "\r\n",
        "            src_pts = np.asarray(list(map(lambda x: [x[1], x[0], x[2], x[3]], src_pts)))\r\n",
        "            dst_pts = np.asarray(list(map(lambda x: [x[1], x[0], x[2], x[3]], dst_pts)))\r\n",
        "\r\n",
        "            src_to_dst_pts = geo_tools.apply_homography_to_points(\r\n",
        "                src_pts, h_src_2_dst[idx_im])\r\n",
        "\r\n",
        "            dst_to_src_pts = geo_tools.apply_homography_to_points(\r\n",
        "                dst_pts, h_dst_2_src[idx_im])\r\n",
        "\r\n",
        "            if args.dst_to_src_evaluation:\r\n",
        "                points_src = src_pts\r\n",
        "                points_dst = dst_to_src_pts\r\n",
        "            else:\r\n",
        "                points_src = src_to_dst_pts\r\n",
        "                points_dst = dst_pts\r\n",
        "\r\n",
        "            # compute repeatability\r\n",
        "            repeatability_results = rep_tools.compute_repeatability(points_src, points_dst, overlap_err=1-args.overlap,\r\n",
        "                                                                    dist_match_thresh=args.pixel_threshold)\r\n",
        "\r\n",
        "            # match descriptors\r\n",
        "            matches = matcher.match(src_dsc, dst_dsc)\r\n",
        "            matches_np = aux.convert_opencv_matches_to_numpy(matches)\r\n",
        "\r\n",
        "            matches_inv = matcher.match(dst_dsc, src_dsc)\r\n",
        "            matches_inv_np = aux.convert_opencv_matches_to_numpy(matches_inv)\r\n",
        "\r\n",
        "            mask = matches_np[:, 0] == matches_inv_np[matches_np[:, 1], 1]\r\n",
        "            matches_np = matches_np[mask]\r\n",
        "\r\n",
        "            match_score, match_score_corr, num_matches = {}, {}, {}\r\n",
        "\r\n",
        "            # compute matching based on pixel distance\r\n",
        "            for th_i in range(1, 11):\r\n",
        "                match_score_i, match_score_corr_i, num_matches_i = match_tools.compute_matching_based_distance(points_src, points_dst, matches_np,\r\n",
        "                                                                       repeatability_results['total_num_points'],\r\n",
        "                                                                       pixel_threshold=th_i,\r\n",
        "                                                                       possible_matches=repeatability_results['possible_matches'])\r\n",
        "                match_score[str(th_i)] = match_score_i\r\n",
        "                match_score_corr[str(th_i)] = match_score_corr_i\r\n",
        "                num_matches[str(th_i)] = num_matches_i\r\n",
        "\r\n",
        "            mma = np.mean([match_score[str(idx)] for idx in match_score])\r\n",
        "\r\n",
        "            results['rep_single_scale'].append(\r\n",
        "                repeatability_results['rep_single_scale'])\r\n",
        "            results['rep_multi_scale'].append(\r\n",
        "                repeatability_results['rep_multi_scale'])\r\n",
        "            results['num_points_single_scale'].append(\r\n",
        "                repeatability_results['num_points_single_scale'])\r\n",
        "            results['num_points_multi_scale'].append(\r\n",
        "                repeatability_results['num_points_multi_scale'])\r\n",
        "            results['error_overlap_single_scale'].append(\r\n",
        "                repeatability_results['error_overlap_single_scale'])\r\n",
        "            results['error_overlap_multi_scale'].append(\r\n",
        "                repeatability_results['error_overlap_multi_scale'])\r\n",
        "\r\n",
        "            results['mma'].append(match_score[str(args.pixel_threshold)])\r\n",
        "            results['mma_corr'].append(match_score_corr[str(args.pixel_threshold)])\r\n",
        "            results['num_matches'].append(num_matches[str(args.pixel_threshold)])\r\n",
        "            results['num_mutual_corresp'].append(len(matches_np))\r\n",
        "            results['avg_mma'].append(mma)\r\n",
        "            results['num_features'].append(repeatability_results['total_num_points'])\r\n",
        "\r\n",
        "    # average the results\r\n",
        "    rep_single = np.array(results['rep_single_scale']).mean()\r\n",
        "    rep_multi = np.array(results['rep_multi_scale']).mean()\r\n",
        "    error_overlap_s = np.array(results['error_overlap_single_scale']).mean()\r\n",
        "    error_overlap_m = np.array(results['error_overlap_multi_scale']).mean()\r\n",
        "    mma = np.array(results['mma']).mean()\r\n",
        "    mma_corr = np.array(results['mma_corr']).mean()\r\n",
        "    num_matches = np.array(results['num_matches']).mean()\r\n",
        "    num_mutual_corresp = np.array(results['num_mutual_corresp']).mean()\r\n",
        "    avg_mma = np.array(results['avg_mma']).mean()\r\n",
        "    num_features = np.array(results['num_features']).mean()\r\n",
        "\r\n",
        "    # Matching Score: Matching Score taking into account all features that have been\r\n",
        "    # detected in any of the two images.\r\n",
        "    # Matching Score (possible matches): Matching Score only taking into account those features that have been\r\n",
        "    # detected in both images.\r\n",
        "    # MMA Score is computed based on the Matching Score (all detected features)\r\n",
        "\r\n",
        "    print('\\n## Overlap @{0}:\\n \\\r\n",
        "           #### Rep. Multi: {1:.4f}\\n \\\r\n",
        "           #### Rep. Single: {2:.4f}\\n \\\r\n",
        "           #### Overlap Multi: {3:.4f}\\n \\\r\n",
        "           #### Overlap Single: {4:.4f}\\n \\\r\n",
        "           #### MMA: {5:.4f}\\n \\\r\n",
        "           #### MMA (possible matches): {6:.4f}\\n \\\r\n",
        "           #### Num matches: {7:.4f}\\n \\\r\n",
        "           #### Num Mutual Correspondences: {8:.4f}\\n \\\r\n",
        "           #### Avg. over Threshold MMA: {9:.4f}\\n \\\r\n",
        "           #### Num Feats: {10:.4f}'.format(\r\n",
        "        args.overlap, rep_multi, rep_single, error_overlap_s, error_overlap_m, mma,\r\n",
        "        mma_corr, num_matches, num_mutual_corresp, avg_mma, num_features))\r\n",
        "\r\n",
        "    # Store data (serialize)\r\n",
        "    output_file_path = os.path.join(args.results_bench_dir, '{0}_{1}.pickle'\r\n",
        "        .format(args.detector_name, args.split))\r\n",
        "    with open(output_file_path, 'wb') as handle:\r\n",
        "            pickle.dump(results, handle, protocol=pickle.HIGHEST_PROTOCOL)"
      ],
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wQwLJFs6EQPb"
      },
      "source": [
        "**RUN **"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oE9Sae4uETSR"
      },
      "source": [
        "train_keynet_architecture()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XcwvrkPHIxSs"
      },
      "source": [
        "extract_multiscale_features()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "opbCjLf8MllJ"
      },
      "source": [
        "hsequences_metrics()"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}