SMART2021_AT_Prediction_Task_v6.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "SMART2021_AT Prediction Task_v6.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyMaBWnyaGxhySv3elNDggul",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/chaeyoonyunakim/smart-2021-AT_Answer_Type_Prediction/blob/main/SMART2021_AT_Prediction_Task_v6.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Q1YCTaqmYYfV"
      },
      "source": [
        "## 0. Environment setup"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tMErWmmfkxqp",
        "outputId": "aaf0b5f7-da92-41dd-fd44-6011512a7c3f"
      },
      "source": [
        "# mount google drive\n",
        "from google.colab import drive\n",
        "\n",
        "# authorization\n",
        "drive.mount('/content/drive')\n",
        "\n",
        "# locate dataset folder\n",
        "%ls -l '/content/drive/My Drive/2021_INM363_SMART/task1_dbpedia_train.json'"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n",
            "-rw------- 1 root root 9263124 Sep 29 08:07 '/content/drive/My Drive/2021_INM363_SMART/task1_dbpedia_train.json'\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KqGDQCAWfPtF"
      },
      "source": [
        "# set working directory\n",
        "import os\n",
        "os.chdir(path = \"/content/drive/My Drive/2021_INM363_SMART/\")"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "C1KBaWJJk4Qg"
      },
      "source": [
        "# import basics\n",
        "import pandas as pd\n",
        "import json\n",
        "import numpy as np\n",
        "import pickle\n",
        "seed = 20211001\n",
        "import regex as re"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qghbo96ZlCTN",
        "outputId": "323e9994-6764-4d63-d202-1a55912c10e4"
      },
      "source": [
        "# import nlp relevants\n",
        "import nltk\n",
        "from nltk.tokenize import word_tokenize\n",
        "from collections import defaultdict\n",
        "from nltk.corpus import stopwords\n",
        "nltk.download('stopwords')\n",
        "nltk.download('punkt')\n",
        "set(stopwords.words('english'))\n",
        "from nltk.stem import PorterStemmer\n",
        "from nltk.stem import WordNetLemmatizer\n",
        "nltk.download('wordnet')\n",
        "\n",
        "# for bag-of-words (bow)\n",
        "from sklearn import feature_extraction, model_selection, naive_bayes, pipeline, manifold, preprocessing"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Package stopwords is already up-to-date!\n",
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Package punkt is already up-to-date!\n",
            "[nltk_data] Downloading package wordnet to /root/nltk_data...\n",
            "[nltk_data]   Package wordnet is already up-to-date!\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VGJtj4fPlFid"
      },
      "source": [
        "# import scikit-learn tools for modelling and evaluation\n",
        "from sklearn.model_selection import train_test_split"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ApUkA-SzlD3c"
      },
      "source": [
        "# import algos\n",
        "from sklearn import svm\n",
        "from sklearn.svm import SVC\n",
        "from sklearn.neural_network import MLPClassifier\n",
        "from sklearn.linear_model import LogisticRegression"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HUs8fZVzZnq2"
      },
      "source": [
        "## 1. Data Loading & Manipulation"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qRL8r9it-bla"
      },
      "source": [
        "***Filter ''o'' in question out***"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "C1W4YPu2wBEo"
      },
      "source": [
        "ap_re = re.compile(\"(^\\')(.*)(\\'$)\")\n",
        "\n",
        "def remove_appostrophe(x):\n",
        "  matches = ap_re.findall(x)\n",
        "  if len(matches) == 1:\n",
        "    return matches[0][1]\n",
        "  else:\n",
        "    return x"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "F7bJ0m9W-XH7"
      },
      "source": [
        "***Load Dataset*** "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XWVZEsmFwBLW"
      },
      "source": [
        "def load_data(path, train = True):\n",
        "  data = pd.read_json(path)\n",
        "  \n",
        "  # map na strings to nan\n",
        "  data.loc[:, \"question\"].replace(\"n/a\", np.nan, inplace = True)\n",
        "  \n",
        "  # drop na in data\n",
        "  if train:\n",
        "    data.dropna(subset=['id', 'question', 'category'], inplace=True)\n",
        "  else:\n",
        "    data.dropna(subset=['id', 'question'], inplace=True)\n",
        "\n",
        "  # remove apostrophes from the start and end of str\n",
        "  data.loc[:, \"question\"] = data[\"question\"].map(lambda x: remove_appostrophe(x))\n",
        "\n",
        "  # for the training data remove rows that have no types\n",
        "  if train:\n",
        "    data = data[data[\"type\"].map(lambda x : len(x) != 0)]\n",
        "\n",
        "  return data"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sw9SNMabwBRI"
      },
      "source": [
        "# load dataset\n",
        "train_data2020 = load_data('/content/drive/My Drive/2021_INM363_SMART/smarttask_dbpedia_train.json')\n",
        "test_data2020 = load_data('/content/drive/My Drive/2021_INM363_SMART/smarttask_dbpedia_test.json', train = False)\n",
        "\n",
        "train_data2021 = load_data('/content/drive/My Drive/2021_INM363_SMART/task1_dbpedia_train.json')\n",
        "test_data2021 = load_data('/content/drive/My Drive/2021_INM363_SMART/task1_dbpedia_test.json', train = False)"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0t3x_V5ogknH",
        "outputId": "457c70d5-5c88-478d-8cde-3d29c6b04542"
      },
      "source": [
        "# check data size\n",
        "train_data2020.shape, test_data2020.shape, train_data2021.shape, test_data2021.shape"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((17482, 4), (4378, 4), (36670, 4), (9104, 2))"
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RsgzXPEz-2ry"
      },
      "source": [
        "***Merge two training dataset***"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9fzeOQLKrifd"
      },
      "source": [
        "# unify the id format\n",
        "train_data2020.loc[:, \"id\"] = train_data2020[\"id\"].map(lambda x: x.split('_')[1])"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "e_dmL_aNrimT"
      },
      "source": [
        "# concat two dataset\n",
        "all_data = pd.concat([train_data2020, train_data2021], axis = 0)"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dzfd0-sSsAAk"
      },
      "source": [
        "# pre processing to remove duplicates in type\n",
        "# transform array to string\n",
        "all_data.loc[:, \"type_str\"] = all_data[\"type\"].map(lambda x: \",\".join(x))"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WQXhkU89_EVf"
      },
      "source": [
        "# remove duplicates when question, category, and type are same\n",
        "merged_data = all_data.drop_duplicates(subset=[\"question\", \"category\", \"type_str\"])"
      ],
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "d0khrcFZZqsS"
      },
      "source": [
        "***Tabular representation of the dataset***"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 136
        },
        "id": "mknRYgOTripL",
        "outputId": "f1116b4f-fac7-4fc1-ec2b-553dc75f910f"
      },
      "source": [
        "merged_data.head(3)"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>question</th>\n",
              "      <th>category</th>\n",
              "      <th>type</th>\n",
              "      <th>type_str</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1177</td>\n",
              "      <td>Was Jacqueline Kennedy Onassis a follower of M...</td>\n",
              "      <td>boolean</td>\n",
              "      <td>[boolean]</td>\n",
              "      <td>boolean</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>14427</td>\n",
              "      <td>What is the name of the opera based on Twelfth...</td>\n",
              "      <td>resource</td>\n",
              "      <td>[dbo:Opera, dbo:MusicalWork, dbo:Work]</td>\n",
              "      <td>dbo:Opera,dbo:MusicalWork,dbo:Work</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>16615</td>\n",
              "      <td>When did Lena Horne receive the Grammy Award f...</td>\n",
              "      <td>literal</td>\n",
              "      <td>[date]</td>\n",
              "      <td>date</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "      id  ...                            type_str\n",
              "0   1177  ...                             boolean\n",
              "1  14427  ...  dbo:Opera,dbo:MusicalWork,dbo:Work\n",
              "2  16615  ...                                date\n",
              "\n",
              "[3 rows x 5 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 195
        },
        "id": "kvZtVjKP6eC6",
        "outputId": "f837e882-801b-43fc-9ec3-6b10c9133de0"
      },
      "source": [
        "merged_data = merged_data.drop(['type_str'], axis =1)\n",
        "merged_data.head()"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>question</th>\n",
              "      <th>category</th>\n",
              "      <th>type</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1177</td>\n",
              "      <td>Was Jacqueline Kennedy Onassis a follower of M...</td>\n",
              "      <td>boolean</td>\n",
              "      <td>[boolean]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>14427</td>\n",
              "      <td>What is the name of the opera based on Twelfth...</td>\n",
              "      <td>resource</td>\n",
              "      <td>[dbo:Opera, dbo:MusicalWork, dbo:Work]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>16615</td>\n",
              "      <td>When did Lena Horne receive the Grammy Award f...</td>\n",
              "      <td>literal</td>\n",
              "      <td>[date]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>23480</td>\n",
              "      <td>Do Prince Harry and Prince William have the sa...</td>\n",
              "      <td>boolean</td>\n",
              "      <td>[boolean]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>3681</td>\n",
              "      <td>What is the subsidiary company working for Leo...</td>\n",
              "      <td>resource</td>\n",
              "      <td>[dbo:EducationalInstitution, dbo:Organisation,...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "      id  ...                                               type\n",
              "0   1177  ...                                          [boolean]\n",
              "1  14427  ...             [dbo:Opera, dbo:MusicalWork, dbo:Work]\n",
              "2  16615  ...                                             [date]\n",
              "3  23480  ...                                          [boolean]\n",
              "4   3681  ...  [dbo:EducationalInstitution, dbo:Organisation,...\n",
              "\n",
              "[5 rows x 4 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GQKlBsKeMkK9",
        "outputId": "b9d1eeae-fb18-4cca-cb40-10a49774f99f"
      },
      "source": [
        "# check data size\n",
        "all_data.shape, merged_data.shape"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "((54152, 5), (39556, 4))"
            ]
          },
          "metadata": {},
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uEofX29lcEwf"
      },
      "source": [
        "***Check unique number of types***"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 136
        },
        "id": "W2lKz1i-begi",
        "outputId": "98b3375e-2234-4f51-ba5c-7b8cb36d9cbd"
      },
      "source": [
        "df = pd.DataFrame()\n",
        "df['type1'] = merged_data.type.str[0]\n",
        "df['type2'] = merged_data.type.str[1]\n",
        "df['type3'] = merged_data.type.str[2]\n",
        "df['type4'] = merged_data.type.str[3]\n",
        "df['type5'] = merged_data.type.str[4]\n",
        "df['type6'] = merged_data.type.str[5]\n",
        "df['type7'] = merged_data.type.str[6]\n",
        "df['type8'] = merged_data.type.str[7]\n",
        "df['type9'] = merged_data.type.str[8]\n",
        "df['type10'] = merged_data.type.str[9]\n",
        "df.head(3)"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>type1</th>\n",
              "      <th>type2</th>\n",
              "      <th>type3</th>\n",
              "      <th>type4</th>\n",
              "      <th>type5</th>\n",
              "      <th>type6</th>\n",
              "      <th>type7</th>\n",
              "      <th>type8</th>\n",
              "      <th>type9</th>\n",
              "      <th>type10</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>boolean</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>dbo:Opera</td>\n",
              "      <td>dbo:MusicalWork</td>\n",
              "      <td>dbo:Work</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>date</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "       type1            type2     type3 type4  ... type7 type8 type9 type10\n",
              "0    boolean              NaN       NaN   NaN  ...   NaN   NaN   NaN    NaN\n",
              "1  dbo:Opera  dbo:MusicalWork  dbo:Work   NaN  ...   NaN   NaN   NaN    NaN\n",
              "2       date              NaN       NaN   NaN  ...   NaN   NaN   NaN    NaN\n",
              "\n",
              "[3 rows x 10 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GZrlCWihbC9X",
        "outputId": "f43bef7e-dcb8-4be2-baf0-09918587496b"
      },
      "source": [
        "print('the number of type1:',len(df['type1'].unique()))\n",
        "print('the number of type2:',len(df['type2'].unique())) \n",
        "print('the number of type3:',len(df['type3'].unique())) \n",
        "print('the number of type4:',len(df['type4'].unique())) \n",
        "print('the number of type5:',len(df['type5'].unique())) \n",
        "print('the number of type6:',len(df['type6'].unique())) \n",
        "print('the number of type7:',len(df['type7'].unique())) \n",
        "print('the number of type8:',len(df['type8'].unique())) \n",
        "print('the number of type9:',len(df['type9'].unique()))\n",
        "print('the number of type10:',len(df['type10'].unique())) \n",
        "df['type9'].unique()"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "the number of type1: 302\n",
            "the number of type2: 173\n",
            "the number of type3: 134\n",
            "the number of type4: 130\n",
            "the number of type5: 87\n",
            "the number of type6: 58\n",
            "the number of type7: 37\n",
            "the number of type8: 37\n",
            "the number of type9: 30\n",
            "the number of type10: 23\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([nan, 'dbo:Location', 'dbo:Cleric', 'dbo:Group', 'dbo:Stream',\n",
              "       'dbo:Region', 'dbo:MilitaryUnit', 'dbo:PopulatedPlace', 'dbo:Band',\n",
              "       'dbo:Settlement', 'dbo:Village', 'dbo:Country',\n",
              "       'dbo:AdministrativeRegion', 'dbo:River', 'dbo:City', 'dbo:Place',\n",
              "       'dbo:Agent', 'dbo:MusicalArtist', 'dbo:MusicGenre', 'dbo:Station',\n",
              "       'dbo:Town', 'dbo:NaturalPlace', 'dbo:RollerCoaster', 'dbo:Work',\n",
              "       'dbo:Ship', 'dbo:CityDistrict', 'dbo:Mammal', 'dbo:BodyOfWater',\n",
              "       'dbo:Airport', 'dbo:Broadcaster'], dtype=object)"
            ]
          },
          "metadata": {},
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Tc3FaIg_tHHC"
      },
      "source": [
        "## 2. Preprocessing"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "G7LfNufdhaCV"
      },
      "source": [
        "***Extract, Transform, Load (ETL)***"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Epe56ypal-qk"
      },
      "source": [
        "class ETL:\n",
        "\n",
        "###############################################################################\n",
        "#   Global Variables     \n",
        "###############################################################################\n",
        "\n",
        "  # text normalization - stemming, lemmatization, stopwords\n",
        "  ps = PorterStemmer()\n",
        "  wordnet_lemmatizer = WordNetLemmatizer() \n",
        "  s_words = stopwords.words()\n",
        "\n",
        "  # set default feature_extraction parameters\n",
        "  count_vectorizer = None \n",
        "  inv_count_vectorizer_vocab = None\n",
        "  tfidf_vectorizer = None\n",
        "  inv_tfidf_vectorizer_vocab = None\n",
        "\n",
        "  # category maps\n",
        "  category_map = {\"boolean\": 0, \"resource\": 1, \"literal\": 2}\n",
        "  inv_category_map = {}\n",
        "\n",
        "  for label, ind in category_map.items():\n",
        "    inv_category_map[ind] = label\n",
        "\n",
        "  # literal maps\n",
        "  literal_map = {\"date\": 0, \"string\": 1, \"number\": 2}\n",
        "  inv_literal_map = {}\n",
        "\n",
        "  for label, ind in literal_map.items():\n",
        "    inv_literal_map[ind] = label  \n",
        "\n",
        "  # resource maps\n",
        "  type_maps = {}\n",
        "  invtype_maps = {}\n",
        "\n",
        "\n",
        "###############################################################################\n",
        "#   Main\n",
        "###############################################################################\n",
        "\n",
        "  def __init__(self, path_to_type_maps = None, path_to_vectorizers = None):\n",
        "\n",
        "    # load type maps if requested\n",
        "    if path_to_type_maps != None:\n",
        "      base_dir = path_to_type_maps\n",
        "      paths = [fp for fp in os.listdir(base_dir) if \"type\" in fp]\n",
        "\n",
        "      for fp in paths:\n",
        "        with open(os.path.join(base_dir, fp), \"r\") as input_file:\n",
        "          type_name = fp.split(\"_\")[0]\n",
        "          self.invtype_maps[type_name] = {}\n",
        "          self.type_maps[type_name] = json.load(input_file)[type_name]\n",
        "          for ontology, ind in self.type_maps[type_name].items():\n",
        "            self.invtype_maps[type_name][ind] = ontology\n",
        "\n",
        "    # load data vectorizers if requested\n",
        "    if path_to_vectorizers != None:\n",
        "      base_dir = path_to_vectorizers\n",
        "      paths = [fp for fp in os.listdir(base_dir) if \"vectorizer\" in fp]\n",
        "\n",
        "      for fp in paths:\n",
        "        vectorizer_name = fp.split(\"_\")[0]\n",
        "        with open(os.path.join(base_dir, fp), \"rb\") as input_file:\n",
        "          if vectorizer_name == \"count\":\n",
        "            self.count_vectorizer = pickle.load(input_file)\n",
        "          elif vectorizer_name == \"tfidf\":\n",
        "            self.tfidf_vectorizer = pickle.load(input_file)\n",
        "          else:\n",
        "            NotImplementedError\n",
        "\n",
        "\n",
        "  # split training dataset to exclude validation dataset\n",
        "  # set train:val = 8:2\n",
        "  def split_data(self, data, val_size = 0.2):\n",
        "    df_train, df_test = model_selection.train_test_split(data, test_size = val_size, random_state = seed)\n",
        "    return df_train, df_test\n",
        "\n",
        "\n",
        "  # normalization of question sentences\n",
        "  def _norm_sent(self, sent, rm_stopwords = False, stemming = True, lemmatization = False):\n",
        "    # tokenize - sentence to word\n",
        "    words = word_tokenize(sent)\n",
        "    # take if all characters in the string are alphabets and then decapitalize\n",
        "    sent = [w.lower() for w in words if w.isalpha()] \n",
        "\n",
        "    # remove stopwords\n",
        "    if rm_stopwords:\n",
        "      sent = [w for w in sent if w not in self.s_words]    \n",
        "\n",
        "    # apply lemmatization \n",
        "    if lemmatization:\n",
        "      sent = [self.wordnet_lemmatizer.lemmatize(w, pos = \"n\") for w in sent]\n",
        "      sent = [self.wordnet_lemmatizer.lemmatize(w, pos = \"v\") for w in sent]\n",
        "      sent = [self.wordnet_lemmatizer.lemmatize(w, pos = (\"a\")) for w in sent]\n",
        "\n",
        "    # apply stemming \n",
        "    if stemming:\n",
        "      sent = [self.ps.stem(w) for w in sent]\n",
        "\n",
        "    sent = \" \".join(sent)\n",
        "    return sent  \n",
        "\n",
        "\n",
        "  # add a new column to show how question parsing has done through normalization above\n",
        "  # for Tabular representation of the dataset\n",
        "  def norm_data(self, data):   \n",
        "    data.loc[:, \"question_processed\"] = data[\"question\"].apply(lambda x: self._norm_sent(x, rm_stopwords = False, lemmatization = True, stemming = True))\n",
        "    return data\n",
        "\n",
        "\n",
        "\n",
        "  # vectorization - fit vectorizer to training data\n",
        "  def bow_fit(self, corpus, type = \"tf\", max_features = 10000, ngram_range = (1,2)):\n",
        "\n",
        "    if type == \"tf\":\n",
        "      self.count_vectorizer = feature_extraction.text.CountVectorizer(max_features = max_features, ngram_range = ngram_range)\n",
        "      self.count_vectorizer.fit(corpus[\"question_processed\"])\n",
        "\n",
        "      # create a reverse mapping for the vocab\n",
        "      self.inv_count_vectorizer_vocab = {}\n",
        "      for label, ind in self.count_vectorizer.vocabulary_.items():\n",
        "        self.inv_count_vectorizer_vocab[ind] = label\n",
        "\n",
        "    elif type == \"tfidf\": \n",
        "      self.tfidf_vectorizer = feature_extraction.text.TfidfVectorizer(max_features = max_features, ngram_range = ngram_range)\n",
        "      self.tfidf_vectorizer.fit(corpus[\"question_processed\"])\n",
        "      \n",
        "      # create a reverse mapping for the vocab\n",
        "      self.inv_tfidf_vectorizer_vocab = {}\n",
        "      for label, ind in self.tfidf_vectorizer.vocabulary_.items():\n",
        "        self.inv_tfidf_vectorizer_vocab[ind] = label\n",
        "\n",
        "    else:\n",
        "      return NotImplementedError\n",
        "\n",
        "\n",
        "  # transformation\n",
        "  def bow_transform(self, data, type = \"tf\"):\n",
        "    if type == \"tf\":\n",
        "      return self.count_vectorizer.transform(data[\"question_processed\"])\n",
        "    elif type == \"tfidf\":\n",
        "      return self.tfidf_vectorizer.transform(data[\"question_processed\"])\n",
        "    else:\n",
        "      return NotImplementedError\n",
        "\n",
        "\n",
        "  # category maps\n",
        "  def category_to_int(self, data):\n",
        "    return data.category.map(lambda x: self.category_map[x])\n",
        "\n",
        "  # literal maps\n",
        "  def literal_to_int(self, data):\n",
        "    return data.type.map(lambda x: self.literal_map[x[0]])\n",
        "\n",
        "\n",
        "  # distribute type by ontology class and encode missing if none\n",
        "  def type_to_int(self, data, type_no):\n",
        "    return data.type.map(\n",
        "        lambda x: self.type_maps[f\"type{type_no}\"][x[type_no - 1]] \n",
        "        if len(x) >= type_no \n",
        "        else self.type_maps[f\"type{type_no}\"][\"missing\"]\n",
        "        )\n",
        "\n",
        "\n",
        "  # resource maps\n",
        "  def add_type_maps(self, train_data, depth = 11, save = True, path = \"resource_types/\"):\n",
        "\n",
        "    levels = range(1, depth)\n",
        "    \n",
        "    if save:\n",
        "      os.makedirs(path, exist_ok = True)\n",
        "  \n",
        "    for l in levels:\n",
        "      type_name = f\"type{l}\"\n",
        "      self.type_maps[type_name] = {}\n",
        "      self.invtype_maps[type_name] = {}\n",
        "      ind = 0\n",
        "      temp_df = train_data[train_data[\"category\"] == \"resource\"][\"type\"].map(lambda x: x[l-1] if len(x) >= l else \"missing\").to_frame(type_name)\n",
        "      for ontology in temp_df[type_name]:\n",
        "        if (ontology not in self.type_maps[type_name]) and (ontology != \"missing\"):\n",
        "          self.type_maps[type_name][ontology] = ind \n",
        "          self.invtype_maps[type_name][ind] = ontology\n",
        "          ind += 1\n",
        "      if save:\n",
        "        with open(os.path.join(\"resource_types\", f\"type{l}_map.json\"), \"w\") as outfile:\n",
        "          temp_json_obj = json.dump(self.type_maps, outfile)\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "  # save output\n",
        "  def save_vectorizers(self, path):\n",
        "\n",
        "    # make sure directory exists\n",
        "    os.makedirs(exist_ok= True, name=path)\n",
        "\n",
        "    if self.count_vectorizer != None:\n",
        "      with open(os.path.join(path, \"count_vectorizer.pkl\"), \"wb\") as count_file:\n",
        "        pickle.dump(self.count_vectorizer, count_file)\n",
        "    if self.tfidf_vectorizer != None:\n",
        "      with open(os.path.join(path, \"tfidf_vectorizer.pkl\"), \"wb\") as tfidf_file:\n",
        "        pickle.dump(self.tfidf_vectorizer, tfidf_file)"
      ],
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zRRJ_AlIwcCS"
      },
      "source": [
        "etl = ETL()"
      ],
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eKbfRa2a80BP"
      },
      "source": [
        "# split dataset\n",
        "# for validation 8:2\n",
        "# df_train, df_val = etl.split_data(merged_data)"
      ],
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MVWgYwQS80ZZ"
      },
      "source": [
        "# text normalization\n",
        "# df_train = etl.norm_data(df_train)   # training set without val\n",
        "df_train = etl.norm_data(merged_data) # training set including val (total)"
      ],
      "execution_count": 23,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_YJAsGNPC9Dv",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 402
        },
        "outputId": "b78adef0-268a-4eaf-f9a4-f21cf6d62b11"
      },
      "source": [
        "df_train[['question', 'question_processed']]"
      ],
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>question</th>\n",
              "      <th>question_processed</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Was Jacqueline Kennedy Onassis a follower of M...</td>\n",
              "      <td>wa jacquelin kennedi onassi a follow of melkit...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>What is the name of the opera based on Twelfth...</td>\n",
              "      <td>what be the name of the opera base on twelfth ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>When did Lena Horne receive the Grammy Award f...</td>\n",
              "      <td>when do lena horn receiv the grammi award for ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Do Prince Harry and Prince William have the sa...</td>\n",
              "      <td>do princ harri and princ william have the same...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>What is the subsidiary company working for Leo...</td>\n",
              "      <td>what be the subsidiari compani work for leonar...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>36665</th>\n",
              "      <td>what kinds of music is played by season's end</td>\n",
              "      <td>what kind of music be play by season end</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>36666</th>\n",
              "      <td>which asteroid group is 6753 fursenko a member...</td>\n",
              "      <td>which asteroid group be fursenko a member of</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>36667</th>\n",
              "      <td>What language is azhakiya ravanan filmed in?</td>\n",
              "      <td>what languag be azhakiya ravanan film in</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>36668</th>\n",
              "      <td>which position did herby fortunat play in foot...</td>\n",
              "      <td>which posit do herbi fortunat play in footbal</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>36669</th>\n",
              "      <td>who is a person that was born in sao paulo</td>\n",
              "      <td>who be a person that wa bear in sao paulo</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>39556 rows × 2 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                question                                 question_processed\n",
              "0      Was Jacqueline Kennedy Onassis a follower of M...  wa jacquelin kennedi onassi a follow of melkit...\n",
              "1      What is the name of the opera based on Twelfth...  what be the name of the opera base on twelfth ...\n",
              "2      When did Lena Horne receive the Grammy Award f...  when do lena horn receiv the grammi award for ...\n",
              "3      Do Prince Harry and Prince William have the sa...  do princ harri and princ william have the same...\n",
              "4      What is the subsidiary company working for Leo...  what be the subsidiari compani work for leonar...\n",
              "...                                                  ...                                                ...\n",
              "36665      what kinds of music is played by season's end           what kind of music be play by season end\n",
              "36666  which asteroid group is 6753 fursenko a member...       which asteroid group be fursenko a member of\n",
              "36667       What language is azhakiya ravanan filmed in?           what languag be azhakiya ravanan film in\n",
              "36668  which position did herby fortunat play in foot...      which posit do herbi fortunat play in footbal\n",
              "36669         who is a person that was born in sao paulo          who be a person that wa bear in sao paulo\n",
              "\n",
              "[39556 rows x 2 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ku3cAUNXuzUG"
      },
      "source": [
        "# vectorization - bag of words model\n",
        "etl.bow_fit(corpus = df_train, type = \"tf\")\n",
        "etl.save_vectorizers(path=\"sklearn_objects\")"
      ],
      "execution_count": 25,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VpGAtqqku-Mb"
      },
      "source": [
        "## 3. Category Prediction Task"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hNrKgk6Cu9b8"
      },
      "source": [
        "# set category prediction dataset\n",
        "X_train_category = etl.bow_transform(df_train)\n",
        "y_train_category = etl.category_to_int(df_train)"
      ],
      "execution_count": 26,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "c-2QymUbxPNo",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "6a0f48c4-f7ba-4cc6-cb2f-09eb5d83c2eb"
      },
      "source": [
        "# model for category classification\n",
        "clf_category = LogisticRegression(\n",
        "    random_state=seed, penalty = 'elasticnet', solver = 'saga',\n",
        "    l1_ratio = 0.2, n_jobs = -1, verbose = 2, max_iter = 200)\\\n",
        "    .fit(X_train_category, y_train_category)"
      ],
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[Parallel(n_jobs=-1)]: Using backend ThreadingBackend with 2 concurrent workers.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "max_iter reached after 820 seconds\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/linear_model/_sag.py:330: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
            "  \"the coef_ did not converge\", ConvergenceWarning)\n",
            "[Parallel(n_jobs=-1)]: Done   1 out of   1 | elapsed: 13.7min finished\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XhcXhB1491gk",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d558c5e5-af47-4ab6-9fa9-74289739c496"
      },
      "source": [
        "clf_category.score(X_train_category, y_train_category)"
      ],
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.9836687228233391"
            ]
          },
          "metadata": {},
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "v6t3q77QvuOE"
      },
      "source": [
        "## 4-1. Type Prediction Task - Literal"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FjARPmJfv84w"
      },
      "source": [
        "***3 different model in total: 1 for category, 1 for literal, 1 for resource***"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vi_yxC3C5nrx",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "069da16f-5ec4-4f6d-827e-c6d2925e6dfb"
      },
      "source": [
        "# model for literal classification\n",
        "# get which rows are for literal only  \n",
        "train_literal_rows = (df_train[\"category\"] == \"literal\").values\n",
        "y_train_literal = etl.literal_to_int(df_train[train_literal_rows])\n",
        "\n",
        "clf_literal = LogisticRegression(\n",
        "    random_state=seed, penalty = 'elasticnet', solver = 'saga',\n",
        "    l1_ratio = 0.5, n_jobs = -1, verbose = 2, max_iter = 200)\\\n",
        "    .fit(X_train_category[train_literal_rows, :], y_train_literal)"
      ],
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[Parallel(n_jobs=-1)]: Using backend ThreadingBackend with 2 concurrent workers.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "max_iter reached after 17 seconds\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/linear_model/_sag.py:330: ConvergenceWarning: The max_iter was reached which means the coef_ did not converge\n",
            "  \"the coef_ did not converge\", ConvergenceWarning)\n",
            "[Parallel(n_jobs=-1)]: Done   1 out of   1 | elapsed:   16.5s finished\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ihsyVY143UPg",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "58a08e5e-e735-4f4a-8ce3-5da83c0ddcc0"
      },
      "source": [
        "clf_literal.score(X_train_category[train_literal_rows], y_train_literal)"
      ],
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.9790128254955305"
            ]
          },
          "metadata": {},
          "execution_count": 30
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Hr-4DSgGGYoF"
      },
      "source": [
        "## 4-2. Type Prediction Task - Resource\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1bLD1-vEVYWj"
      },
      "source": [
        "***Identify types of resources in the train data***"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SZe_y3EeIYvz"
      },
      "source": [
        "etl.add_type_maps(df_train)"
      ],
      "execution_count": 31,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EZU7YplswisE",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "fca95a97-a79a-45e0-84ae-6ee2cbdf33bc"
      },
      "source": [
        "etl.type_maps.keys()"
      ],
      "execution_count": 32,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "dict_keys(['type1', 'type2', 'type3', 'type4', 'type5', 'type6', 'type7', 'type8', 'type9', 'type10'])"
            ]
          },
          "metadata": {},
          "execution_count": 32
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TPYDqPkqw6KD",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "5ac8bcd2-32fd-4f0a-caec-e18fcc4fe7dd"
      },
      "source": [
        "etl.type_maps.values()"
      ],
      "execution_count": 33,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "dict_values([{'dbo:Opera': 0, 'dbo:EducationalInstitution': 1, 'dbo:State': 2, 'dbo:Country': 3, 'dbo:AcademicSubject': 4, 'dbo:Name': 5, 'dbo:Organisation': 6, 'dbo:Person': 7, 'dbo:WrittenWork': 8, 'dbo:EthnicGroup': 9, 'dbo:Museum': 10, 'dbo:Writer': 11, 'dbo:AmericanFootballPlayer': 12, 'dbo:ChemicalCompound': 13, 'dbo:OfficeHolder': 14, 'dbo:Magazine': 15, 'dbo:Award': 16, 'dbo:Animal': 17, 'dbo:Activity': 18, 'dbo:NobelPrize': 19, 'dbo:Single': 20, 'dbo:Work': 21, 'dbo:MetroStation': 22, 'dbo:Film': 23, 'dbo:Profession': 24, 'dbo:City': 25, 'dbo:Deity': 26, 'dbo:Galaxy': 27, 'dbo:Company': 28, 'dbo:Gene': 29, 'dbo:Contest': 30, 'dbo:TelevisionEpisode': 31, 'dbo:University': 32, 'dbo:MusicalArtist': 33, 'dbo:Scientist': 34, 'dbo:PoliticalParty': 35, 'dbo:MusicGenre': 36, 'dbo:Disease': 37, 'dbo:Taxon': 38, 'dbo:Book': 39, 'dbo:Settlement': 40, 'dbo:GovernmentAgency': 41, 'dbo:Ship': 42, 'dbo:Drug': 43, 'dbo:Island': 44, 'dbo:Mountain': 45, 'dbo:Village': 46, 'dbo:River': 47, 'dbo:Media': 48, 'dbo:Sport': 49, 'dbo:LegalCase': 50, 'dbo:HistoricalRegion': 51, 'dbo:MilitaryConflict': 52, 'dbo:SportsTeam': 53, 'dbo:TelevisionStation': 54, 'dbo:Archive': 55, 'dbo:Asteroid': 56, 'dbo:GivenName': 57, 'dbo:Software': 58, 'dbo:Monastery': 59, 'dbo:TelevisionShow': 60, 'dbo:MilitaryUnit': 61, 'dbo:Archipelago': 62, 'dbo:Saint': 63, 'dbo:RailwayLine': 64, 'dbo:Non-ProfitOrganisation': 65, 'dbo:Cemetery': 66, 'dbo:Tournament': 67, 'dbo:Aircraft': 68, 'dbo:SoccerPlayer': 69, 'dbo:Royalty': 70, 'dbo:Building': 71, 'dbo:Bridge': 72, 'dbo:Band': 73, 'dbo:Lake': 74, 'dbo:Family': 75, 'dbo:Spacecraft': 76, 'dbo:Church': 77, 'dbo:BoardGame': 78, 'dbo:Constellation': 79, 'dbo:Play': 80, 'dbo:Region': 81, 'dbo:VideoGame': 82, 'dbo:Castle': 83, 'dbo:BasketballPlayer': 84, 'dbo:InformationAppliance': 85, 'dbo:Election': 86, 'dbo:Protein': 87, 'dbo:Weapon': 88, 'dbo:ProgrammingLanguage': 89, 'dbo:Ideology': 90, 'dbo:BasketballTeam': 91, 'dbo:Album': 92, 'dbo:RailwayStation': 93, 'dbo:RecordLabel': 94, 'dbo:AdultActor': 95, 'dbo:Vein': 96, 'dbo:President': 97, 'dbo:SpaceMission': 98, 'dbo:Event': 99, 'dbo:SpaceStation': 100, 'dbo:Game': 101, 'dbo:Airport': 102, 'dbo:Parliament': 103, 'dbo:MedicalSpecialty': 104, 'dbo:Currency': 105, 'dbo:Planet': 106, 'dbo:FormulaOneRacer': 107, 'dbo:SportsClub': 108, 'dbo:RailwayTunnel': 109, 'dbo:WineRegion': 110, 'dbo:Newspaper': 111, 'dbo:Town': 112, 'dbo:HorseRace': 113, 'dbo:Manga': 114, 'dbo:Skyscraper': 115, 'dbo:BaseballTeam': 116, 'dbo:NobleFamily': 117, 'dbo:SportsLeague': 118, 'dbo:Novel': 119, 'dbo:Mammal': 120, 'dbo:FigureSkater': 121, 'dbo:NaturalRegion': 122, 'dbo:CyclingRace': 123, 'dbo:Hotel': 124, 'dbo:Road': 125, 'dbo:Stadium': 126, 'dbo:NationalAnthem': 127, 'dbo:Musical': 128, 'dbo:HockeyTeam': 129, 'dbo:Legislature': 130, 'dbo:MartialArtist': 131, 'dbo:Dam': 132, 'dbo:AcademicJournal': 133, 'dbo:MountainRange': 134, 'dbo:AnatomicalStructure': 135, 'dbo:Ocean': 136, 'dbo:SoccerClub': 137, 'dbo:FictionalCharacter': 138, 'dbo:Airline': 139, 'dbo:Language': 140, 'dbo:Locomotive': 141, 'dbo:Beverage': 142, 'dbo:MusicalWork': 143, 'dbo:Monarch': 144, 'dbo:PowerStation': 145, 'dbo:MountainPass': 146, 'dbo:AmericanFootballTeam': 147, 'dbo:Letter': 148, 'dbo:Mineral': 149, 'dbo:Cat': 150, 'dbo:CollegeCoach': 151, 'dbo:Boxer': 152, 'dbo:Architect': 153, 'dbo:SkiResort': 154, 'dbo:Philosopher': 155, 'dbo:Valley': 156, 'dbo:PopulatedPlace': 157, 'dbo:Senator': 158, 'dbo:Factory': 159, 'dbo:MilitaryPerson': 160, 'dbo:Gymnast': 161, 'dbo:Library': 162, 'dbo:Port': 163, 'dbo:Volcano': 164, 'dbo:Swimmer': 165, 'dbo:School': 166, 'dbo:Astronaut': 167, 'dbo:Food': 168, 'dbo:Venue': 169, 'dbo:Artist': 170, 'dbo:Politician': 171, 'dbo:Rocket': 172, 'dbo:Cave': 173, 'dbo:SportsManager': 174, 'dbo:GeologicalPeriod': 175, 'dbo:Judge': 176, 'dbo:AdministrativeRegion': 177, 'dbo:Square': 178, 'dbo:Monument': 179, 'dbo:Comedian': 180, 'dbo:SoccerManager': 181, 'dbo:Mollusca': 182, 'dbo:ComicsCreator': 183, 'dbo:Prison': 184, 'dbo:BeautyQueen': 185, 'dbo:Train': 186, 'dbo:Anime': 187, 'dbo:Enzyme': 188, 'dbo:Muscle': 189, 'dbo:Holiday': 190, 'dbo:RadioProgram': 191, 'dbo:BaseballLeague': 192, 'dbo:Ambassador': 193, 'dbo:Governor': 194, 'dbo:ComicsCharacter': 195, 'dbo:GolfCourse': 196, 'dbo:ProtectedArea': 197, 'dbo:PublicService': 198, 'dbo:Street': 199, 'dbo:LaunchPad': 200, 'dbo:RugbyPlayer': 201, 'dbo:Mosque': 202, 'dbo:Bird': 203, 'dbo:Broadcaster': 204, 'dbo:Plant': 205, 'dbo:BasketballLeague': 206, 'dbo:Tax': 207, 'dbo:Artery': 208, 'dbo:Bacteria': 209, 'dbo:Bank': 210, 'dbo:Restaurant': 211, 'dbo:PersonFunction': 212, 'dbo:TennisPlayer': 213, 'dbo:Bone': 214, 'dbo:ReligiousBuilding': 215, 'dbo:Publisher': 216, 'dbo:Motorcycle': 217, 'dbo:AutomobileEngine': 218, 'dbo:Poem': 219, 'dbo:Lighthouse': 220, 'dbo:IceHockeyLeague': 221, 'dbo:Tower': 222, 'dbo:HistoricBuilding': 223, 'dbo:Zoo': 224, 'dbo:Nerve': 225, 'dbo:HollywoodCartoon': 226, 'dbo:ArchitecturalStructure': 227, 'dbo:MusicFestival': 228, 'dbo:Brain': 229, 'dbo:Insect': 230, 'dbo:Song': 231, 'dbo:Agent': 232, 'dbo:ChessPlayer': 233, 'dbo:CultivatedVariety': 234, 'dbo:IceHockeyPlayer': 235, 'dbo:Hospital': 236, 'dbo:Flag': 237, 'dbo:FloweringPlant': 238, 'dbo:WorldHeritageSite': 239, 'dbo:HistoricalPeriod': 240, 'dbo:PublicTransitSystem': 241, 'dbo:DartsPlayer': 242, 'dbo:NaturalPlace': 243, 'dbo:Meeting': 244, 'dbo:BaseballPlayer': 245, 'dbo:AmericanFootballLeague': 246, 'dbo:TennisTournament': 247, 'dbo:Place': 248, 'dbo:Noble': 249, 'dbo:Sea': 250, 'dbo:Grape': 251, 'dbo:FilmFestival': 252, 'dbo:Continent': 253, 'dbo:PlayboyPlaymate': 254, 'dbo:BroadcastNetwork': 255, 'dbo:Priest': 256, 'dbo:HotSpring': 257, 'dbo:Actor': 258, 'dbo:MemberOfParliament': 259, 'dbo:Glacier': 260, 'dbo:Wrestler': 261, 'dbo:Beer': 262, 'dbo:CardGame': 263, 'dbo:Brewery': 264, 'dbo:RaceTrack': 265, 'dbo:FootballLeagueSeason': 266, 'dbo:Genre': 267, 'dbo:SocietalEvent': 268, 'dbo:SportsEvent': 269, 'dbo:MythologicalFigure': 270, 'dbo:Colour': 271, 'dbo:College': 272, 'dbo:Reptile': 273, 'dbo:PeriodicalLiterature': 274, 'dbo:Automobile': 275, 'dbo:OlympicEvent': 276, 'dbo:TelevisionSeason': 277, 'dbo:CanadianFootballTeam': 278, 'dbo:Eukaryote': 279, 'dbo:Artwork': 280, 'dbo:Star': 281, 'dbo:ArtificialSatellite': 282, 'dbo:Athlete': 283, 'dbo:MeanOfTransportation': 284, 'dbo:SoccerTournament': 285, 'dbo:Arachnid': 286, 'dbo:Fish': 287, 'dbo:CyclingTeam': 288, 'dbo:Comic': 289, 'dbo:Device': 290, 'dbo:ComicStrip': 291, 'dbo:SportFacility': 292, 'dbo:Surname': 293, 'dbo:Criminal': 294, 'dbo:GolfTournament': 295, 'dbo:Convention': 296, 'dbo:RadioStation': 297}, {'dbo:MusicalWork': 0, 'dbo:Organisation': 1, 'dbo:PopulatedPlace': 2, 'dbo:State': 3, 'dbo:TopicalConcept': 4, 'dbo:Agent': 5, 'dbo:Work': 6, 'dbo:Castle': 7, 'dbo:Person': 8, 'dbo:GridironFootballPlayer': 9, 'dbo:ChemicalSubstance': 10, 'dbo:President': 11, 'dbo:PeriodicalLiterature': 12, 'dbo:Eukaryote': 13, 'dbo:Award': 14, 'dbo:Station': 15, 'dbo:PersonFunction': 16, 'dbo:City': 17, 'dbo:Settlement': 18, 'dbo:CelestialBody': 19, 'dbo:Biomolecule': 20, 'dbo:Competition': 21, 'dbo:EducationalInstitution': 22, 'dbo:Artist': 23, 'dbo:Band': 24, 'dbo:Genre': 25, 'dbo:TelevisionShow': 26, 'dbo:WrittenWork': 27, 'dbo:MeanOfTransportation': 28, 'dbo:NaturalPlace': 29, 'dbo:Stream': 30, 'dbo:Game': 31, 'dbo:Case': 32, 'dbo:Region': 33, 'dbo:SocietalEvent': 34, 'dbo:BroadcastNetwork': 35, 'dbo:CollectionOfValuables': 36, 'dbo:Name': 37, 'dbo:ReligiousBuilding': 38, 'dbo:Cleric': 39, 'dbo:RouteOfTransportation': 40, 'dbo:Place': 41, 'dbo:SportsEvent': 42, 'dbo:Athlete': 43, 'dbo:OfficeHolder': 44, 'dbo:ArchitecturalStructure': 45, 'dbo:Country': 46, 'dbo:FormulaOneTeam': 47, 'dbo:Group': 48, 'dbo:Continent': 49, 'dbo:Film': 50, 'dbo:Sea': 51, 'dbo:Software': 52, 'dbo:Island': 53, 'dbo:Building': 54, 'dbo:Device': 55, 'dbo:Language': 56, 'dbo:SoccerClub': 57, 'dbo:Company': 58, 'dbo:Actor': 59, 'dbo:AnatomicalStructure': 60, 'dbo:Politician': 61, 'dbo:Activity': 62, 'dbo:Infrastructure': 63, 'dbo:RacingDriver': 64, 'dbo:PublicTransitSystem': 65, 'dbo:Comic': 66, 'dbo:SportsTeam': 67, 'dbo:Family': 68, 'dbo:Book': 69, 'dbo:Animal': 70, 'dbo:Race': 71, 'dbo:SportFacility': 72, 'dbo:Sport': 73, 'dbo:BodyOfWater': 74, 'dbo:BaseballTeam': 75, 'dbo:Deity': 76, 'dbo:AmericanFootballTeam': 77, 'dbo:MilitaryPerson': 78, 'dbo:Food': 79, 'dbo:Mammal': 80, 'dbo:Coach': 81, 'dbo:SkiArea': 82, 'dbo:Royalty': 83, 'dbo:SoccerPlayer': 84, 'dbo:TimePeriod': 85, 'dbo:Philosopher': 86, 'dbo:SportsManager': 87, 'dbo:SportsClub': 88, 'dbo:Cartoon': 89, 'dbo:SportsLeague': 90, 'dbo:Theatre': 91, 'dbo:FictionalCharacter': 92, 'dbo:Museum': 93, 'dbo:BasketballTeam': 94, 'dbo:SoccerLeague': 95, 'dbo:Species': 96, 'dbo:Engine': 97, 'dbo:Broadcaster': 98, 'dbo:Stadium': 99, 'dbo:Tower': 100, 'dbo:WinterSportPlayer': 101, 'dbo:Mountain': 102, 'dbo:TelevisionEpisode': 103, 'dbo:Writer': 104, 'dbo:Volcano': 105, 'dbo:PowerStation': 106, 'dbo:Insect': 107, 'dbo:ComicsCreator': 108, 'dbo:Plant': 109, 'dbo:Skyscraper': 110, 'dbo:School': 111, 'dbo:Airline': 112, 'dbo:Non-ProfitOrganisation': 113, 'dbo:Noble': 114, 'dbo:Church': 115, 'dbo:Monument': 116, 'dbo:Tournament': 117, 'dbo:Location': 118, 'dbo:River': 119, 'dbo:Town': 120, 'dbo:FloweringPlant': 121, 'dbo:RailwayStation': 122, 'dbo:MountainRange': 123, 'dbo:Beverage': 124, 'dbo:PoliticalParty': 125, 'dbo:MusicalArtist': 126, 'dbo:Pope': 127, 'dbo:MusicGenre': 128, 'dbo:AmericanFootballPlayer': 129, 'dbo:MilitaryConflict': 130, 'dbo:Saint': 131, 'dbo:Disease': 132, 'dbo:Grape': 133, 'dbo:SoccerTournament': 134, 'dbo:Website': 135, 'dbo:EthnicGroup': 136, 'dbo:Scientist': 137, 'dbo:FootballMatch': 138, 'dbo:Olympics': 139, 'dbo:TelevisionStation': 140, 'dbo:Event': 141, 'dbo:Album': 142, 'dbo:MilitaryUnit': 143, 'dbo:Election': 144, 'dbo:Song': 145, 'dbo:RugbyClub': 146, 'dbo:University': 147, 'dbo:ProgrammingLanguage': 148, 'dbo:RadioStation': 149, 'dbo:Library': 150, 'dbo:InformationAppliance': 151, 'dbo:VideoGame': 152, 'dbo:Musical': 153, 'dbo:Criminal': 154, 'dbo:Drug': 155, 'dbo:Religious': 156, 'dbo:Anime': 157, 'dbo:Publisher': 158, 'dbo:HollywoodCartoon': 159, 'dbo:Cardinal': 160, 'dbo:GovernmentAgency': 161, 'dbo:Architect': 162, 'dbo:PlayboyPlaymate': 163, 'dbo:Legislature': 164, 'dbo:HorseRace': 165, 'dbo:RecordLabel': 166, 'dbo:FormulaOneRacer': 167, 'dbo:Venue': 168, 'dbo:Star': 169, 'dbo:Automobile': 170, 'dbo:Presenter': 171}, {'dbo:Work': 0, 'dbo:Agent': 1, 'dbo:Place': 2, 'dbo:PopulatedPlace': 3, 'dbo:Building': 4, 'dbo:Athlete': 5, 'dbo:Politician': 6, 'dbo:WrittenWork': 7, 'dbo:Species': 8, 'dbo:Infrastructure': 9, 'dbo:Settlement': 10, 'dbo:Event': 11, 'dbo:Organisation': 12, 'dbo:Person': 13, 'dbo:TopicalConcept': 14, 'dbo:BodyOfWater': 15, 'dbo:Activity': 16, 'dbo:UnitOfWork': 17, 'dbo:State': 18, 'dbo:Broadcaster': 19, 'dbo:Location': 20, 'dbo:SocietalEvent': 21, 'dbo:SportsTeam': 22, 'dbo:AdministrativeRegion': 23, 'dbo:ArchitecturalStructure': 24, 'dbo:Artist': 25, 'dbo:OfficeHolder': 26, 'dbo:MotorsportRacer': 27, 'dbo:Plant': 28, 'dbo:SportsEvent': 29, 'dbo:Island': 30, 'dbo:ProtectedArea': 31, 'dbo:Group': 32, 'dbo:NaturalPlace': 33, 'dbo:Eukaryote': 34, 'dbo:River': 35, 'dbo:Animal': 36, 'dbo:SportFacility': 37, 'dbo:Cemetery': 38, 'dbo:Country': 39, 'dbo:MilitaryPerson': 40, 'dbo:President': 41, 'dbo:SoccerManager': 42, 'dbo:Venue': 43, 'dbo:ReligiousBuilding': 44, 'dbo:SportsLeague': 45, 'dbo:Company': 46, 'dbo:SoccerPlayer': 47, 'dbo:PublicTransitSystem': 48, 'dbo:Device': 49, 'dbo:MusicalArtist': 50, 'dbo:EducationalInstitution': 51, 'dbo:Band': 52, 'dbo:SoccerClub': 53, 'dbo:Station': 54, 'dbo:Volcano': 55, 'dbo:Food': 56, 'dbo:SportsManager': 57, 'dbo:FictionalCharacter': 58, 'dbo:City': 59, 'dbo:Scientist': 60, 'dbo:MusicGenre': 61, 'dbo:Writer': 62, 'dbo:Continent': 63, 'dbo:Sport': 64, 'dbo:BaseballTeam': 65, 'dbo:HistoricBuilding': 66, 'dbo:CityDistrict': 67, 'dbo:Mountain': 68, 'dbo:GridironFootballPlayer': 69, 'dbo:TelevisionShow': 70, 'dbo:SoapCharacter': 71, 'dbo:Sea': 72, 'dbo:RecordLabel': 73, 'dbo:MotorsportSeason': 74, 'dbo:HistoricPlace': 75, 'dbo:PersonFunction': 76, 'dbo:Model': 77, 'dbo:PeriodicalLiterature': 78, 'dbo:HumanGene': 79, 'dbo:Olympics': 80, 'dbo:Film': 81, 'dbo:ConcentrationCamp': 82, 'dbo:School': 83, 'dbo:TelevisionStation': 84, 'dbo:Royalty': 85, 'dbo:Bank': 86, 'dbo:AmericanFootballLeague': 87, 'dbo:MilitaryUnit': 88, 'dbo:RaceHorse': 89, 'dbo:AmericanFootballTeam': 90, 'dbo:BeautyQueen': 91, 'dbo:IceHockeyPlayer': 92, 'dbo:SoccerClubSeason': 93, 'dbo:Planet': 94, 'dbo:Bodybuilder': 95, 'dbo:Song': 96, 'dbo:Architect': 97, 'dbo:Constellation': 98, 'dbo:VideoGame': 99, 'dbo:RugbyPlayer': 100, 'dbo:Cartoon': 101, 'dbo:Star': 102, 'dbo:Pope': 103, 'dbo:Park': 104, 'dbo:Disease': 105, 'dbo:University': 106, 'dbo:CollegeCoach': 107, 'dbo:Book': 108, 'dbo:Astronaut': 109, 'dbo:Chef': 110, 'dbo:Economist': 111, 'dbo:Bridge': 112, 'dbo:MusicalWork': 113, 'dbo:Noble': 114, 'dbo:Manga': 115, 'dbo:MilitaryConflict': 116, 'dbo:GovernmentalAdministrativeRegion': 117, 'dbo:ChristianBishop': 118, 'dbo:Medician': 119, 'dbo:Saint': 120, 'dbo:PoliticalParty': 121, 'dbo:Cleric': 122, 'dbo:Album': 123, 'dbo:RacingDriver': 124, 'dbo:Insect': 125, 'dbo:Award': 126, 'dbo:Play': 127, 'dbo:RugbyClub': 128, 'dbo:WomensTennisAssociationTournament': 129, 'dbo:Senator': 130, 'dbo:EthnicGroup': 131, 'dbo:HorseTrainer': 132}, {'dbo:Location': 0, 'dbo:Place': 1, 'dbo:ArchitecturalStructure': 2, 'dbo:Person': 3, 'dbo:Work': 4, 'dbo:PopulatedPlace': 5, 'dbo:Agent': 6, 'dbo:NaturalPlace': 7, 'dbo:Organisation': 8, 'dbo:Event': 9, 'dbo:Athlete': 10, 'dbo:Eukaryote': 11, 'dbo:SocietalEvent': 12, 'dbo:CollegeCoach': 13, 'dbo:Species': 14, 'dbo:Building': 15, 'dbo:Bird': 16, 'dbo:Governor': 17, 'dbo:Settlement': 18, 'dbo:TelevisionShow': 19, 'dbo:Airport': 20, 'dbo:Band': 21, 'dbo:Insect': 22, 'dbo:Infrastructure': 23, 'dbo:Hospital': 24, 'dbo:Country': 25, 'dbo:SportsTeam': 26, 'dbo:OfficeHolder': 27, 'dbo:Entomologist': 28, 'dbo:University': 29, 'dbo:SportsClub': 30, 'dbo:AmericanFootballTeam': 31, 'dbo:Region': 32, 'dbo:SoccerPlayer': 33, 'dbo:Station': 34, 'dbo:Broadcaster': 35, 'dbo:Continent': 36, 'dbo:Town': 37, 'dbo:Cleric': 38, 'dbo:WorldHeritageSite': 39, 'dbo:Island': 40, 'dbo:Village': 41, 'dbo:City': 42, 'dbo:BasketballPlayer': 43, 'dbo:BaseballPlayer': 44, 'dbo:BodyOfWater': 45, 'dbo:Comedian': 46, 'dbo:MusicalArtist': 47, 'dbo:Bridge': 48, 'dbo:ProtectedArea': 49, 'dbo:CityDistrict': 50, 'dbo:Lake': 51, 'dbo:Group': 52, 'dbo:Road': 53, 'dbo:Airline': 54, 'dbo:Protein': 55, 'dbo:SportsEvent': 56, 'dbo:SoccerClub': 57, 'dbo:Museum': 58, 'dbo:Single': 59, 'dbo:MartialArtist': 60, 'dbo:HistoricPlace': 61, 'dbo:IceHockeyPlayer': 62, 'dbo:SportsLeague': 63, 'dbo:Castle': 64, 'dbo:TennisPlayer': 65, 'dbo:HandballTeam': 66, 'dbo:BaseballTeam': 67, 'dbo:GolfPlayer': 68, 'dbo:CelestialBody': 69, 'dbo:Bay': 70, 'dbo:Bank': 71, 'dbo:Genre': 72, 'dbo:Crater': 73, 'dbo:SoccerManager': 74, 'dbo:Wrestler': 75, 'dbo:MotorsportRacer': 76, 'dbo:Boxer': 77, 'dbo:MusicGenre': 78, 'dbo:Astronaut': 79, 'dbo:ComicsCreator': 80, 'dbo:SportsManager': 81, 'dbo:Company': 82, 'dbo:MusicalWork': 83, 'dbo:Scientist': 84, 'dbo:ComicStrip': 85, 'dbo:Coach': 86, 'dbo:Royalty': 87, 'dbo:PersonFunction': 88, 'dbo:RecordLabel': 89, 'dbo:Swimmer': 90, 'dbo:Cricketer': 91, 'dbo:Noble': 92, 'dbo:Book': 93, 'dbo:ScreenWriter': 94, 'dbo:AustralianFootballTeam': 95, 'dbo:RailwayLine': 96, 'dbo:WrittenWork': 97, 'dbo:Congressman': 98, 'dbo:Municipality': 99, 'dbo:Politician': 100, 'dbo:Painter': 101, 'dbo:ChessPlayer': 102, 'dbo:TelevisionStation': 103, 'dbo:President': 104, 'dbo:TableTennisPlayer': 105, 'dbo:Mayor': 106, 'dbo:AmusementParkAttraction': 107, 'dbo:NascarDriver': 108, 'dbo:RacingDriver': 109, 'dbo:MemberOfParliament': 110, 'dbo:PrimeMinister': 111, 'dbo:Poem': 112, 'dbo:Jockey': 113, 'dbo:MilitaryPerson': 114, 'dbo:FashionDesigner': 115, 'dbo:HorseTrainer': 116, 'dbo:MilitaryStructure': 117, 'dbo:Historian': 118, 'dbo:Automobile': 119, 'dbo:Skier': 120, 'dbo:Artist': 121, 'dbo:Monarch': 122, 'dbo:Cyclist': 123, 'dbo:FictionalCharacter': 124, 'dbo:River': 125, 'dbo:RadioHost': 126, 'dbo:PoliticalParty': 127, 'dbo:GridironFootballPlayer': 128}, {'dbo:Location': 0, 'dbo:Place': 1, 'dbo:Agent': 2, 'dbo:Settlement': 3, 'dbo:Person': 4, 'dbo:Species': 5, 'dbo:Event': 6, 'dbo:ArchitecturalStructure': 7, 'dbo:Insect': 8, 'dbo:PopulatedPlace': 9, 'dbo:Work': 10, 'dbo:Infrastructure': 11, 'dbo:Eukaryote': 12, 'dbo:Country': 13, 'dbo:Building': 14, 'dbo:Writer': 15, 'dbo:Venue': 16, 'dbo:Region': 17, 'dbo:Prison': 18, 'dbo:HistoricPlace': 19, 'dbo:Town': 20, 'dbo:ReligiousBuilding': 21, 'dbo:GridironFootballPlayer': 22, 'dbo:River': 23, 'dbo:FormulaOneTeam': 24, 'dbo:Village': 25, 'dbo:Film': 26, 'dbo:OfficeHolder': 27, 'dbo:University': 28, 'dbo:SportsClub': 29, 'dbo:PublicTransitSystem': 30, 'dbo:MusicalArtist': 31, 'dbo:Tournament': 32, 'dbo:NaturalPlace': 33, 'dbo:Museum': 34, 'dbo:Lake': 35, 'dbo:Band': 36, 'dbo:Road': 37, 'dbo:Organisation': 38, 'dbo:Biomolecule': 39, 'dbo:SocietalEvent': 40, 'dbo:AmericanFootballTeam': 41, 'dbo:City': 42, 'dbo:WorldHeritageSite': 43, 'dbo:Island': 44, 'dbo:Broadcaster': 45, 'dbo:AdministrativeRegion': 46, 'dbo:MountainRange': 47, 'dbo:WinterSportPlayer': 48, 'dbo:AnatomicalStructure': 49, 'dbo:SportsManager': 50, 'dbo:School': 51, 'dbo:Hotel': 52, 'dbo:Single': 53, 'dbo:TopicalConcept': 54, 'dbo:MotorcycleRider': 55, 'dbo:Group': 56, 'dbo:SoccerPlayer': 57, 'dbo:Cleric': 58, 'dbo:HistoricBuilding': 59, 'dbo:Album': 60, 'dbo:Congressman': 61, 'dbo:ComicsCreator': 62, 'dbo:MemberOfParliament': 63, 'dbo:Governor': 64, 'dbo:WrittenWork': 65, 'dbo:BaseballPlayer': 66, 'dbo:SkiArea': 67, 'dbo:Hospital': 68, 'dbo:Royalty': 69, 'dbo:Stadium': 70, 'dbo:Company': 71, 'dbo:Mammal': 72, 'dbo:RouteOfTransportation': 73, 'dbo:Coach': 74, 'dbo:SoccerClub': 75, 'dbo:MilitaryPerson': 76, 'dbo:MotorsportRacer': 77, 'dbo:CollegeCoach': 78, 'dbo:Station': 79, 'dbo:Magazine': 80, 'dbo:Scientist': 81, 'dbo:RaceHorse': 82, 'dbo:AdultActor': 83, 'dbo:ClassicalMusicArtist': 84, 'dbo:Castle': 85}, {'dbo:Location': 0, 'dbo:PopulatedPlace': 1, 'dbo:Agent': 2, 'dbo:Place': 3, 'dbo:Eukaryote': 4, 'dbo:ArchitecturalStructure': 5, 'dbo:Species': 6, 'dbo:Continent': 7, 'dbo:AdministrativeRegion': 8, 'dbo:Royalty': 9, 'dbo:Stadium': 10, 'dbo:Group': 11, 'dbo:CollegeCoach': 12, 'dbo:Country': 13, 'dbo:Stream': 14, 'dbo:MusicalArtist': 15, 'dbo:Museum': 16, 'dbo:Village': 17, 'dbo:Broadcaster': 18, 'dbo:NaturalPlace': 19, 'dbo:Legislature': 20, 'dbo:Band': 21, 'dbo:EthnicGroup': 22, 'dbo:River': 23, 'dbo:RouteOfTransportation': 24, 'dbo:BodyOfWater': 25, 'dbo:Region': 26, 'dbo:Infrastructure': 27, 'dbo:PoliticalParty': 28, 'dbo:Gene': 29, 'dbo:Diocese': 30, 'dbo:Dam': 31, 'dbo:Settlement': 32, 'dbo:SportsClub': 33, 'dbo:Island': 34, 'dbo:MountainRange': 35, 'dbo:City': 36, 'dbo:EducationalInstitution': 37, 'dbo:University': 38, 'dbo:MusicGenre': 39, 'dbo:SpeedwayRider': 40, 'dbo:Company': 41, 'dbo:Person': 42, 'dbo:Politician': 43, 'dbo:Work': 44, 'dbo:Organisation': 45, 'dbo:SportsManager': 46, 'dbo:Insect': 47, 'dbo:RailwayTunnel': 48, 'dbo:NascarDriver': 49, 'dbo:Artist': 50, 'dbo:MotorsportRacer': 51, 'dbo:WrittenWork': 52, 'dbo:GridironFootballPlayer': 53, 'dbo:Newspaper': 54, 'dbo:Economist': 55, 'dbo:Town': 56}, {'dbo:Place': 0, 'dbo:Location': 1, 'dbo:Species': 2, 'dbo:PopulatedPlace': 3, 'dbo:Region': 4, 'dbo:Artist': 5, 'dbo:Town': 6, 'dbo:Settlement': 7, 'dbo:Village': 8, 'dbo:Country': 9, 'dbo:Agent': 10, 'dbo:Organisation': 11, 'dbo:NaturalPlace': 12, 'dbo:RouteOfTransportation': 13, 'dbo:CityDistrict': 14, 'dbo:AdministrativeRegion': 15, 'dbo:ChemicalCompound': 16, 'dbo:City': 17, 'dbo:GovernmentAgency': 18, 'dbo:Band': 19, 'dbo:MusicalArtist': 20, 'dbo:BodyOfWater': 21, 'dbo:Legislature': 22, 'dbo:ArchitecturalStructure': 23, 'dbo:Book': 24, 'dbo:Mammal': 25, 'dbo:Ship': 26, 'dbo:RecordLabel': 27, 'dbo:Scientist': 28, 'dbo:RadioStation': 29, 'dbo:MusicFestival': 30, 'dbo:PeriodicalLiterature': 31, 'dbo:School': 32, 'dbo:Person': 33, 'dbo:WrittenWork': 34, 'dbo:NascarDriver': 35}, {'dbo:Location': 0, 'dbo:Place': 1, 'dbo:ChristianBishop': 2, 'dbo:Island': 3, 'dbo:Country': 4, 'dbo:Agent': 5, 'dbo:River': 6, 'dbo:Settlement': 7, 'dbo:City': 8, 'dbo:Enzyme': 9, 'dbo:Group': 10, 'dbo:Town': 11, 'dbo:AdministrativeRegion': 12, 'dbo:PopulatedPlace': 13, 'dbo:NaturalPlace': 14, 'dbo:Continent': 15, 'dbo:RecordLabel': 16, 'dbo:Band': 17, 'dbo:Building': 18, 'dbo:ArchitecturalStructure': 19, 'dbo:BodyOfWater': 20, 'dbo:Organisation': 21, 'dbo:MusicalArtist': 22, 'dbo:AmusementParkAttraction': 23, 'dbo:Film': 24, 'dbo:HistoricPlace': 25, 'dbo:FormerMunicipality': 26, 'dbo:Road': 27, 'dbo:Insect': 28, 'dbo:MeanOfTransportation': 29, 'dbo:Village': 30, 'dbo:Diocese': 31, 'dbo:Region': 32, 'dbo:Work': 33, 'dbo:Person': 34, 'dbo:PeriodicalLiterature': 35}, {'dbo:Location': 0, 'dbo:Cleric': 1, 'dbo:Group': 2, 'dbo:Stream': 3, 'dbo:Region': 4, 'dbo:MilitaryUnit': 5, 'dbo:PopulatedPlace': 6, 'dbo:Band': 7, 'dbo:Settlement': 8, 'dbo:Village': 9, 'dbo:Country': 10, 'dbo:AdministrativeRegion': 11, 'dbo:River': 12, 'dbo:City': 13, 'dbo:Place': 14, 'dbo:Agent': 15, 'dbo:MusicalArtist': 16, 'dbo:MusicGenre': 17, 'dbo:Station': 18, 'dbo:Town': 19, 'dbo:NaturalPlace': 20, 'dbo:RollerCoaster': 21, 'dbo:Work': 22, 'dbo:Ship': 23, 'dbo:CityDistrict': 24, 'dbo:Mammal': 25, 'dbo:BodyOfWater': 26, 'dbo:Airport': 27, 'dbo:Broadcaster': 28}, {'dbo:Politician': 0, 'dbo:PopulatedPlace': 1, 'dbo:Person': 2, 'dbo:Organisation': 3, 'dbo:Settlement': 4, 'dbo:Stream': 5, 'dbo:Country': 6, 'dbo:AdministrativeRegion': 7, 'dbo:Location': 8, 'dbo:Place': 9, 'dbo:Agent': 10, 'dbo:Region': 11, 'dbo:Town': 12, 'dbo:Infrastructure': 13, 'dbo:River': 14, 'dbo:Film': 15, 'dbo:Insect': 16, 'dbo:EthnicGroup': 17, 'dbo:NaturalPlace': 18, 'dbo:Village': 19, 'dbo:Island': 20, 'dbo:Entomologist': 21}])"
            ]
          },
          "metadata": {},
          "execution_count": 33
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PorG_IvOK4_N",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "9e110789-1f11-4859-f10f-d75f9a576428"
      },
      "source": [
        "resource_models = []\n",
        "\n",
        "for l in range(1, 11):\n",
        "  # model for resource classification\n",
        "  # get which rows are for resource only\n",
        "  # must only include rows that have the type at the level \n",
        "  # of classification \n",
        "  train_resource_rows = ((df_train[\"category\"] == \"resource\") & (df_train[\"type\"].map(lambda x: len(x)) >= l)).values\n",
        "  y_train_type = etl.type_to_int(df_train[train_resource_rows], type_no=l)\n",
        "\n",
        "  clf_type = MLPClassifier(\n",
        "      random_state=seed, max_iter=10, hidden_layer_sizes=(1000, 500, 300), verbose = 2).\\\n",
        "    fit(X_train_category[train_resource_rows], y_train_type)\n",
        "\n",
        "  resource_models.append(clf_type)"
      ],
      "execution_count": 34,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Iteration 1, loss = 2.14002143\n",
            "Iteration 2, loss = 1.03116504\n",
            "Iteration 3, loss = 0.62297222\n",
            "Iteration 4, loss = 0.37859177\n",
            "Iteration 5, loss = 0.25528809\n",
            "Iteration 6, loss = 0.19085766\n",
            "Iteration 7, loss = 0.15787654\n",
            "Iteration 8, loss = 0.13812974\n",
            "Iteration 9, loss = 0.12472322\n",
            "Iteration 10, loss = 0.11284379\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/neural_network/_multilayer_perceptron.py:571: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (10) reached and the optimization hasn't converged yet.\n",
            "  % self.max_iter, ConvergenceWarning)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Iteration 1, loss = 2.08878833\n",
            "Iteration 2, loss = 0.99633147\n",
            "Iteration 3, loss = 0.60163255\n",
            "Iteration 4, loss = 0.38697955\n",
            "Iteration 5, loss = 0.27508625\n",
            "Iteration 6, loss = 0.21432184\n",
            "Iteration 7, loss = 0.18018812\n",
            "Iteration 8, loss = 0.15607034\n",
            "Iteration 9, loss = 0.14235186\n",
            "Iteration 10, loss = 0.13592778\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/neural_network/_multilayer_perceptron.py:571: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (10) reached and the optimization hasn't converged yet.\n",
            "  % self.max_iter, ConvergenceWarning)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Iteration 1, loss = 2.13147989\n",
            "Iteration 2, loss = 1.16462734\n",
            "Iteration 3, loss = 0.77946541\n",
            "Iteration 4, loss = 0.54746403\n",
            "Iteration 5, loss = 0.41258115\n",
            "Iteration 6, loss = 0.32841607\n",
            "Iteration 7, loss = 0.27502682\n",
            "Iteration 8, loss = 0.24973910\n",
            "Iteration 9, loss = 0.22341338\n",
            "Iteration 10, loss = 0.21124127\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/neural_network/_multilayer_perceptron.py:571: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (10) reached and the optimization hasn't converged yet.\n",
            "  % self.max_iter, ConvergenceWarning)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Iteration 1, loss = 2.37021883\n",
            "Iteration 2, loss = 1.34517080\n",
            "Iteration 3, loss = 0.91403112\n",
            "Iteration 4, loss = 0.66878606\n",
            "Iteration 5, loss = 0.50619872\n",
            "Iteration 6, loss = 0.40519070\n",
            "Iteration 7, loss = 0.33815534\n",
            "Iteration 8, loss = 0.30452070\n",
            "Iteration 9, loss = 0.29280518\n",
            "Iteration 10, loss = 0.25941069\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/neural_network/_multilayer_perceptron.py:571: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (10) reached and the optimization hasn't converged yet.\n",
            "  % self.max_iter, ConvergenceWarning)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Iteration 1, loss = 2.05832300\n",
            "Iteration 2, loss = 1.21112987\n",
            "Iteration 3, loss = 0.86577162\n",
            "Iteration 4, loss = 0.62948037\n",
            "Iteration 5, loss = 0.46834557\n",
            "Iteration 6, loss = 0.36960250\n",
            "Iteration 7, loss = 0.32460028\n",
            "Iteration 8, loss = 0.27729944\n",
            "Iteration 9, loss = 0.25256042\n",
            "Iteration 10, loss = 0.23549953\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/neural_network/_multilayer_perceptron.py:571: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (10) reached and the optimization hasn't converged yet.\n",
            "  % self.max_iter, ConvergenceWarning)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Iteration 1, loss = 2.55437747\n",
            "Iteration 2, loss = 1.51324069\n",
            "Iteration 3, loss = 1.15626311\n",
            "Iteration 4, loss = 0.88904609\n",
            "Iteration 5, loss = 0.69730961\n",
            "Iteration 6, loss = 0.57936781\n",
            "Iteration 7, loss = 0.50096976\n",
            "Iteration 8, loss = 0.44672973\n",
            "Iteration 9, loss = 0.40457990\n",
            "Iteration 10, loss = 0.36945666\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/neural_network/_multilayer_perceptron.py:571: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (10) reached and the optimization hasn't converged yet.\n",
            "  % self.max_iter, ConvergenceWarning)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Iteration 1, loss = 2.41720268\n",
            "Iteration 2, loss = 1.59153601\n",
            "Iteration 3, loss = 1.36829417\n",
            "Iteration 4, loss = 1.18619546\n",
            "Iteration 5, loss = 0.99075170\n",
            "Iteration 6, loss = 0.79234502\n",
            "Iteration 7, loss = 0.66969677\n",
            "Iteration 8, loss = 0.57947964\n",
            "Iteration 9, loss = 0.52753791\n",
            "Iteration 10, loss = 0.47509443\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/neural_network/_multilayer_perceptron.py:571: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (10) reached and the optimization hasn't converged yet.\n",
            "  % self.max_iter, ConvergenceWarning)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Iteration 1, loss = 3.19969455\n",
            "Iteration 2, loss = 2.32865191\n",
            "Iteration 3, loss = 2.02787055\n",
            "Iteration 4, loss = 1.84222573\n",
            "Iteration 5, loss = 1.64849474\n",
            "Iteration 6, loss = 1.43307125\n",
            "Iteration 7, loss = 1.21924283\n",
            "Iteration 8, loss = 1.03329275\n",
            "Iteration 9, loss = 0.87530654\n",
            "Iteration 10, loss = 0.76774408\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/neural_network/_multilayer_perceptron.py:571: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (10) reached and the optimization hasn't converged yet.\n",
            "  % self.max_iter, ConvergenceWarning)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Iteration 1, loss = 2.94434100\n",
            "Iteration 2, loss = 2.17768995\n",
            "Iteration 3, loss = 1.89854230\n",
            "Iteration 4, loss = 1.70453867\n",
            "Iteration 5, loss = 1.51012518\n",
            "Iteration 6, loss = 1.31420127\n",
            "Iteration 7, loss = 1.11859841\n",
            "Iteration 8, loss = 0.94808144\n",
            "Iteration 9, loss = 0.79683240\n",
            "Iteration 10, loss = 0.68191041\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/neural_network/_multilayer_perceptron.py:571: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (10) reached and the optimization hasn't converged yet.\n",
            "  % self.max_iter, ConvergenceWarning)\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Iteration 1, loss = 2.84029671\n",
            "Iteration 2, loss = 2.01547160\n",
            "Iteration 3, loss = 1.70651877\n",
            "Iteration 4, loss = 1.56188614\n",
            "Iteration 5, loss = 1.39651607\n",
            "Iteration 6, loss = 1.24598573\n",
            "Iteration 7, loss = 1.11727590\n",
            "Iteration 8, loss = 0.98955061\n",
            "Iteration 9, loss = 0.86532694\n",
            "Iteration 10, loss = 0.73887018\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/neural_network/_multilayer_perceptron.py:571: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (10) reached and the optimization hasn't converged yet.\n",
            "  % self.max_iter, ConvergenceWarning)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8fxgGgeOMM9W",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "9cda173d-dd91-4066-c8c2-1830e9d12e7c"
      },
      "source": [
        "clf_type.score(X_train_category[train_resource_rows], y_train_type) "
      ],
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.7934362934362934"
            ]
          },
          "metadata": {},
          "execution_count": 35
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wwcZ38iu_YYH"
      },
      "source": [
        "## 4. Save models "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EJnfBXgkfyUr"
      },
      "source": [
        "with open(os.path.join(\"sklearn_objects\", \"category_model.pkl\"), \"wb\") as mdl_file:\n",
        "  pickle.dump(clf_category, mdl_file)"
      ],
      "execution_count": 36,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "X_QaOf94RtQs"
      },
      "source": [
        "with open(os.path.join(\"sklearn_objects\", \"literal_model.pkl\"), \"wb\") as mdl_file:\n",
        "  pickle.dump(clf_literal, mdl_file)"
      ],
      "execution_count": 37,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ASFCv0nkaphR"
      },
      "source": [
        "for l in range(1,11):\n",
        "  with open(os.path.join(\"sklearn_objects\", f\"resource_level_{l}_model.pkl\"), \"wb\") as mdl_file:\n",
        "    pickle.dump(resource_models[l-1], mdl_file)"
      ],
      "execution_count": 38,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pol2B42Cw2ck"
      },
      "source": [
        "## 5. Results & Evaluation"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QV76HT3oUoxa"
      },
      "source": [
        "***Load Pre-trained models***"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZmutptNCTqbc"
      },
      "source": [
        "with open(\"sklearn_objects/category_model.pkl\", \"rb\") as clf_cat_file:\n",
        "  clf_category = pickle.load(clf_cat_file)"
      ],
      "execution_count": 39,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lxMJq2xsBOwL"
      },
      "source": [
        "with open(\"sklearn_objects/literal_model.pkl\", \"rb\") as clf_lit_file:\n",
        "  clf_literal = pickle.load(clf_lit_file)"
      ],
      "execution_count": 40,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-Jx_Y4PaBQ0q"
      },
      "source": [
        "resource_models = []\n",
        "for l in range(1,11):\n",
        "  with open(f\"sklearn_objects/resource_level_{l}_model.pkl\", \"rb\") as res_mdl:\n",
        "    resource_models.append(pickle.load(res_mdl))"
      ],
      "execution_count": 41,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZP9mCDz0BTeD"
      },
      "source": [
        "***Load processing class***"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PYLTHE0gUll3"
      },
      "source": [
        "etl = ETL(path_to_type_maps=\"resource_types\", path_to_vectorizers=\"sklearn_objects\")"
      ],
      "execution_count": 42,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "orIFdQaDN9ba"
      },
      "source": [
        "class ModelEvaluation:\n",
        "  def __init__(self, etl_inst, cat_model, lit_model, res_models):\n",
        "    self.etl_inst = etl_inst\n",
        "    self.cat_model = cat_model\n",
        "    self.lit_model = lit_model\n",
        "    self.res_models = res_models\n",
        "  \n",
        "  # X is a df\n",
        "  def get_predictions(self, X, bow_type = \"tf\"):\n",
        "\n",
        "    X = X.copy()\n",
        "\n",
        "    X.reset_index(inplace = True, drop = True)\n",
        "    \n",
        "    X_norm = self.etl_inst.norm_data(X)\n",
        "    X_vec = self.etl_inst.bow_transform(X_norm, type = bow_type)\n",
        "\n",
        "    bool_int = self.etl_inst.category_map[\"boolean\"]\n",
        "    literal_int = self.etl_inst.category_map[\"literal\"]\n",
        "    resource_int = self.etl_inst.category_map[\"resource\"]\n",
        "\n",
        "    cat_pred = self.cat_model.predict(X_vec)\n",
        "\n",
        "    ind_bool = cat_pred == bool_int\n",
        "    ind_literal = cat_pred == literal_int\n",
        "    ind_resource = cat_pred == resource_int\n",
        "\n",
        "    if len(ind_bool) > 0:\n",
        "      X.loc[ind_bool, \"cat_prediction\"] = \"boolean\"\n",
        "      X.loc[ind_bool, \"type_prediction\"] = pd.Series(\n",
        "          cat_pred[ind_bool], name = \"type_prediction\")\\\n",
        "          .map(lambda x: [\"boolean\"]).values\n",
        "\n",
        "    if len(ind_literal) > 0:\n",
        "      X.loc[ind_literal, \"cat_prediction\"] = \"literal\"\n",
        "      literal_pred = self.lit_model.predict(X_vec[ind_literal])\n",
        "      X.loc[ind_literal, \"type_prediction\"] = pd.Series(\n",
        "          literal_pred, name = \"type_prediction\")\\\n",
        "          .map(lambda x: [self.etl_inst.inv_literal_map[x]]).values\n",
        "\n",
        "    if len(ind_resource) > 0:\n",
        "      resource_preds = []\n",
        "      for ind, type_model in enumerate(self.res_models):\n",
        "          resource_preds.append(\n",
        "            pd.Series(\n",
        "                type_model.predict(X_vec[ind_resource]), name = f\"type_{ind}\").\\\n",
        "                map(lambda x: self.etl_inst.invtype_maps[f\"type{ind+1}\"][x])\n",
        "                )\n",
        "      resource_preds = pd.Series(pd.concat(resource_preds, axis = 1).values.tolist(), name = \"type_prediction\")\n",
        "      X.loc[ind_resource, \"type_prediction\"] = resource_preds.values\n",
        "      X.loc[ind_resource, \"cat_prediction\"] = \"resource\"\n",
        "      \n",
        "      return X\n",
        "\n",
        "  def output_predictions(self):\n",
        "    return NotImplementedError"
      ],
      "execution_count": 43,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "D6BLutJ8jA-M"
      },
      "source": [
        "me = ModelEvaluation(etl, clf_category, clf_literal, resource_models)"
      ],
      "execution_count": 44,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6b_rDe2EKOry"
      },
      "source": [
        "***Validate***"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "00lDYJIE0pit"
      },
      "source": [
        "# out_val = me.get_predictions(df_val)\n",
        "\n",
        "# true_output = out_val.loc[:, [\"id\", \"question\", \"category\", \"type\"]]\n",
        "# true_output_dict = [pred for ind, pred in true_output.to_dict(orient = \"index\").items()]\n",
        "\n",
        "# system_output = out_val.loc[:, [\"id\", \"cat_prediction\", \"type_prediction\"]]\n",
        "# system_output.columns = [\"id\", \"category\", \"type\"]\n",
        "# system_output_dict = [pred for ind, pred in system_output.to_dict(orient = \"index\").items()]"
      ],
      "execution_count": 45,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "k9raG_KIyB1x"
      },
      "source": [
        "***Run Evaluation***"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dtiKCQsPxH_V"
      },
      "source": [
        "# os.makedirs(\"system_output/\", exist_ok = True)\n",
        "# with open(os.path.join(\"system_output\", \"ground_truth_json.json\"), \"w\") as gfile:\n",
        "#   json.dump(true_output_dict, gfile)\n",
        "\n",
        "# with open(os.path.join(\"system_output\", \"system_output_json.json\"), \"w\") as sfile:\n",
        "#   json.dump(system_output_dict, sfile)"
      ],
      "execution_count": 46,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zPusetmQx_T7"
      },
      "source": [
        "# !python evaluate.py --type_hierarchy_tsv dbpedia_types.tsv  \\\n",
        "#  --ground_truth_json system_output/ground_truth_json.json \\\n",
        "#  --system_output_json system_output/system_output_json.json"
      ],
      "execution_count": 47,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "plQTw8kzKQmC"
      },
      "source": [
        "***Save Test Output***"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IvHRpUYDKHm6"
      },
      "source": [
        "out_test = me.get_predictions(test_data2021)\n",
        "\n",
        "#true_output = out_test.loc[:, [\"id\", \"question\"]]\n",
        "#true_output_dict = [pred for ind, pred in true_output.to_dict(orient = \"index\").items()]\n",
        "\n",
        "system_output = out_test.loc[:, [\"id\", \"cat_prediction\", \"type_prediction\"]]\n",
        "system_output.columns = [\"id\", \"category\", \"type\"]\n",
        "system_output_dict = [pred for ind, pred in system_output.to_dict(orient = \"index\").items()]"
      ],
      "execution_count": 48,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mZ4WgtQq2k1g"
      },
      "source": [
        "os.makedirs(\"system_output/\", exist_ok = True)\n",
        "# with open(os.path.join(\"system_output\", \"ground_truth_json.json\"), \"w\") as gfile:\n",
        "#   json.dump(true_output_dict, gfile)\n",
        "\n",
        "with open(os.path.join(\"system_output\", \"system_output_json.json\"), \"w\") as sfile:\n",
        "  json.dump(system_output_dict, sfile)"
      ],
      "execution_count": 49,
      "outputs": []
    }
  ]
}