Retrofitting using Master datasets.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "whole-training",
   "metadata": {},
   "source": [
    "In this notebook, we perform the retrofitting and evaluate the original, retrofitted embeddings, similarity values and capture all the results."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "dynamic-generator",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import scipy.stats as stats\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "from tqdm.notebook import tqdm\n",
    "from itertools import combinations\n",
    "from math import comb\n",
    "from time import time\n",
    "from datetime import date\n",
    "import warnings\n",
    "from joblib import Parallel, delayed\n",
    "import json\n",
    "import copy\n",
    "from collections import defaultdict\n",
    "\n",
    "from sklearn.model_selection import StratifiedKFold, KFold\n",
    "from sklearn.pipeline import make_pipeline\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.svm import SVC, SVR\n",
    "from sklearn.metrics import accuracy_score, mean_squared_error, classification_report, plot_confusion_matrix, confusion_matrix\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.preprocessing import normalize\n",
    "\n",
    "warnings.filterwarnings('ignore')\n",
    "# pd.set_option('mode.chained_assignment', None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "amazing-cleveland",
   "metadata": {},
   "outputs": [],
   "source": [
    "# evaluation datasets\n",
    "WORDSIM_DF = '../data/evaluation/wordsim353_with_r3.csv'\n",
    "WORDSIM_OLD_FINAL_FILE = \"../data/evaluation/wordsim_old.csv\"\n",
    "DBPEDIA_MC_30_FINAL_FILE = \"../data/evaluation/mc-30_DBpedia.csv\"\n",
    "DBPEDIA_RG_65_FINAL_FILE = \"../data/evaluation/rg-65_DBpedia.csv\"\n",
    "\n",
    "CONCEPTNET_FILE = \"../data/evaluation/kgtk_conceptnet_final.csv\"\n",
    "WIKI_CS_FILE = '../data/evaluation/wikidata-cs_final.csv'\n",
    "\n",
    "INPUT_EMB_FOLDER_PATH = '../data/embeddings/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "improved-robinson",
   "metadata": {},
   "outputs": [],
   "source": [
    "# basis\n",
    "P279_CHILD_PAR_DISTILBERT_COSSIM_FILE = \"../data/basis/P279_ChildPar.all-distilroberta-v1.csv\"\n",
    "P279_SIBLINGS_DISTILBERT_COSSIM_FILE = \"../data/basis/P279_Siblings.all-distilroberta-v1.csv\"\n",
    "\n",
    "P279_CHILD_PAR_CLASSSIM_FILE = \"../data/basis/P279_ChildPar.classSim.csv\"\n",
    "P279_SIBLINGS_CLASSSIM_FILE = \"../data/basis/P279_Siblings.classSim.csv\"\n",
    "\n",
    "PROBASE_FINAL_FILE = '../data/basis/intermediate_files/probase_WQnodes_subset_and_sim.csv'\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "fallen-turner",
   "metadata": {},
   "outputs": [],
   "source": [
    "# similarity scores files\n",
    "\n",
    "WORDSIM_CLASS_SIM_FILE = '../data/embeddings/wordsim_class_sim.csv'\n",
    "WORDSIM_JC_SIM_FILE = '../data/embeddings/wordsim_jc_sim.csv'\n",
    "WORDSIM_TOP_SIM_FILE = '../data/embeddings/wordsim_top_sim.csv'\n",
    "\n",
    "WORDSIM_OLD_CLASS_SIM_FILE = '../data/embeddings/wordsim_old_class_sim.csv'\n",
    "WORDSIM_OLD_JC_SIM_FILE = '../data/embeddings/wordsim_old_jc_sim.csv'\n",
    "WORDSIM_OLD_TOP_SIM_FILE = '../data/embeddings/wordsim_old_top_sim.csv'\n",
    "\n",
    "DBPEDIA_MC_30_CLASS_SIM_FILE = '../data/embeddings/dbpedia_mc_30_class_sim.csv'\n",
    "DBPEDIA_MC_30_JC_SIM_FILE = '../data/embeddings/dbpedia_mc_30_jc_sim.csv'\n",
    "DBPEDIA_MC_30_TOP_SIM_FILE = '../data/embeddings/dbpedia_mc_30_top_sim.csv'\n",
    "\n",
    "DBPEDIA_RG_65_CLASS_SIM_FILE = '../data/embeddings/dbpedia_rg_65_class_sim.csv'\n",
    "DBPEDIA_RG_65_JC_SIM_FILE = '../data/embeddings/dbpedia_rg_65_jc_sim.csv'\n",
    "DBPEDIA_RG_65_TOP_SIM_FILE = '../data/embeddings/dbpedia_rg_65_top_sim.csv'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "balanced-conspiracy",
   "metadata": {},
   "source": [
    "# Retrofitting Pre-Req Class Definitions"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "theoretical-perception",
   "metadata": {},
   "source": [
    "## Utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "arbitrary-palace",
   "metadata": {},
   "outputs": [],
   "source": [
    "class Utils:\n",
    "    \"\"\"\n",
    "    This contains all the utility functions needed by any part of retrofitting\n",
    "    \"\"\"\n",
    "    _today = date.today()\n",
    "    today_date = _today.strftime(\"%b_%d_%Y\")\n",
    "    LABELS = ['I','M','U']\n",
    "    \n",
    "    @classmethod\n",
    "    def normalize(cls, embed_dict):\n",
    "        '''Normalize the embeddings'''\n",
    "        for key, val in embed_dict.items():\n",
    "            temp = np.array([float(val1) for val1 in val])\n",
    "            temp2 = temp**2\n",
    "            embed_dict[key] = temp / np.sqrt((temp2.sum() + 1e-6))\n",
    "        return embed_dict\n",
    "    \n",
    "    @classmethod\n",
    "    def fetch_embeddings(cls, df):\n",
    "        '''Fetch embeddings from dataframe'''\n",
    "        embed_dict = {}\n",
    "        for _, row in df.iterrows():\n",
    "            embed_dict[row.node] = row.value\n",
    "        return normalize(embed_dict)\n",
    "    \n",
    "    @classmethod\n",
    "    def fill_coverage(cls, embed_dict, embed_name):\n",
    "        '''\n",
    "        In many embedding files, the embeddings don't cover all the nodes of the evaluation benchmarks.\n",
    "        Thus, for these missing nodes, we just initialize them with zeros.\n",
    "        '''\n",
    "        wordsim_df = pd.read_csv(WORDSIM_DF)\n",
    "#         wiki_cs_df = pd.read_csv(WIKICS_DF)\n",
    "#         concept_net_df = pd.read_csv(CONCEPTNET_DF)\n",
    "        \n",
    "        compulsory_coverage_set = set(\n",
    "                        wordsim_df['word1_kg_id'].to_list() \n",
    "                        + wordsim_df['word2_kg_id'].to_list()\n",
    "                        + evalD.dbpedia_mc_30_df['word1_kg_id'].to_list()\n",
    "                        + evalD.dbpedia_mc_30_df['word2_kg_id'].to_list()\n",
    "                        + evalD.dbpedia_rg_65_df['word1_kg_id'].to_list()\n",
    "                        + evalD.dbpedia_rg_65_df['word2_kg_id'].to_list())\n",
    "#                         + wiki_cs_df['word1_kg_id'].to_list() \n",
    "#                         + wiki_cs_df['word2_kg_id'].to_list()\n",
    "#                         + concept_net_df['word1_kg_id'].to_list()\n",
    "#                         + concept_net_df['word2_kg_id'].to_list())\n",
    "        \n",
    "        embed_size = len(embed_dict[next(iter(embed_dict))])\n",
    "        count = 0\n",
    "        for word in compulsory_coverage_set:\n",
    "            if word not in embed_dict:\n",
    "                embed_dict[word] = np.zeros((embed_size))\n",
    "                count += 1\n",
    "        print(f\"Added {count} corrections to {embed_name}\")\n",
    "        return embed_dict\n",
    "    \n",
    "    @classmethod\n",
    "    def check_coverage(cls, embed_dict):\n",
    "        '''Check coverage of embeddinga dict wrt Wordsim nodes'''\n",
    "        wordsim_df = pd.read_csv(WORDSIM_DF)\n",
    "        \n",
    "        compulsory_coverage_set = set(list(zip(wordsim_df['word1_kg_id'].to_list(), wordsim_df['word2_kg_id'].to_list())))\n",
    "        embed_size = len(embed_dict[next(iter(embed_dict))])\n",
    "        count = 0\n",
    "        for word1, word2 in compulsory_coverage_set:\n",
    "            if word1 not in embed_dict or word2 not in embed_dict:\n",
    "                count += 1\n",
    "        return (len(wordsim_df) - count)\n",
    "    \n",
    "    @classmethod\n",
    "    def check_eval_coverage(cls, embed_dict, eval_df):\n",
    "        '''Check coverage of embeddings dict wrt evaluation benchmark dataset provided'''\n",
    "        \n",
    "        compulsory_coverage_set = set(eval_df['word1_kg_id'].to_list() + eval_df['word2_kg_id'].to_list())\n",
    "        embed_size = len(embed_dict[next(iter(embed_dict))])\n",
    "        count = 0\n",
    "        for word in compulsory_coverage_set:\n",
    "            if word not in embed_dict:\n",
    "                count += 1\n",
    "        return count\n",
    "    \n",
    "    @classmethod\n",
    "    def find_missing_words(cls, embed_dict):\n",
    "        '''\n",
    "        Find the nodes in the evaluation benchmark dataset which we do not have an embedding in the \n",
    "        embedding dict provided\n",
    "        '''\n",
    "        \n",
    "        compulsory_coverage_set = set((\n",
    "            inp.wordsim_df['word1_kg_id'].to_list()) + (inp.wordsim_df['word2_kg_id'].to_list()\n",
    "       ))\n",
    "        embed_size = len(embed_dict[next(iter(embed_dict))])\n",
    "        missing_words = []\n",
    "        for word in compulsory_coverage_set:\n",
    "            if word not in embed_dict or not(embed_dict[word].any()):\n",
    "                missing_words.append(word)\n",
    "        return missing_words\n",
    "    \n",
    "    @classmethod\n",
    "    def determine_distances(cls, embed_dict, new_embed_dict):\n",
    "        '''Determine euclidean distance between the old and new embeddings provided'''\n",
    "        dist = []\n",
    "        for word in embed_dict.keys():\n",
    "            dist.append(euclidean_distances([embed_dict[word]], [new_embed_dict[word]])[0][0])\n",
    "        return dist\n",
    "    \n",
    "    @classmethod\n",
    "    def serialize_embedding_dict(cls, embed_dict):\n",
    "        '''Serialize the embeddings in the dict for saving'''\n",
    "        for key2 in embed_dict.keys():\n",
    "            embed_dict[key2] = embed_dict[key2].tolist() if type(embed_dict[key2]) != list else embed_dict[key2]\n",
    "        return embed_dict\n",
    "    \n",
    "    @classmethod\n",
    "    def deserialize_embedding_dict(cls, embed_dict):\n",
    "        '''Deserialize the embeddings in the dict after loading'''\n",
    "        for key2 in embed_dict.keys():\n",
    "            embed_dict[key2] = np.array(embed_dict[key2])\n",
    "        return embed_dict\n",
    "    \n",
    "    @classmethod\n",
    "    def label_samples(cls, score):\n",
    "        '''Given a score value (1-4), assign the label depending on the pre-decided ranges'''\n",
    "        return 'I' if score <= 1.75 else 'U' if score >= 3.5 else 'M'\n",
    "    \n",
    "    @classmethod\n",
    "    def alt_label_samples(cls, score, quartiles):\n",
    "        '''Given a score value (ideally 1-4) and the quartile ranges, assign the label'''\n",
    "        return ['Q'+str(i+1) for i in range(len(quartiles) - 1) if quartiles[i] <= score < quartiles[i+1]][0]\n",
    "    \n",
    "    @classmethod\n",
    "    def alt2_label_samples(cls, row, quartiles):\n",
    "        '''Given a score value (ideally 1-4) and the quartile ranges, assign the label'''\n",
    "        return [i for i, quartile in (quartiles.items()) if (row.word1_kg_id, row.word2_kg_id) in quartile][0]\n",
    "    \n",
    "    @classmethod\n",
    "    def determine_cos_sim(cls, emb1, emb2):\n",
    "        '''Determine cosine similarity value given the 2 embeddings'''\n",
    "        return cosine_similarity(\n",
    "                np.array(emb1).reshape(1,-1), \n",
    "                np.array(emb2).reshape(1,-1)\n",
    "            )[0][0]\n",
    "    \n",
    "    @classmethod\n",
    "    def plot_confusion_matrix(cls, conf_matrix, title):\n",
    "        '''Plot the confusion matrix'''\n",
    "        plt.close()\n",
    "        sns.heatmap(conf_matrix, xticklabels=Utils.LABELS, yticklabels=Utils.LABELS, annot=True)\n",
    "        plt.xlabel('Predicted')\n",
    "        plt.ylabel('Actual')\n",
    "        plt.title(title+' Confusion Matrix')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "urban-pennsylvania",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Code for checking variable sizes in Jupyter notebook. NOT WORKING\n",
    "# import sys\n",
    "# def sizeof_fmt(num, suffix='B'):\n",
    "#     ''' by Fred Cirera,  https://stackoverflow.com/a/1094933/1870254, modified'''\n",
    "#     for unit in ['','Ki','Mi','Gi','Ti','Pi','Ei','Zi']:\n",
    "#         if abs(num) < 1024.0:\n",
    "#             return \"%3.1f %s%s\" % (num, unit, suffix)\n",
    "#         num /= 1024.0\n",
    "#     return \"%.1f %s%s\" % (num, 'Yi', suffix)\n",
    "\n",
    "#     for name, size in sorted(((name, sys.getsizeof(value)) for name, value in locals().items()),\n",
    "#                              key= lambda x: -x[1])[:10]:\n",
    "#         print(\"{:>30}: {:>8}\".format(name, sizeof_fmt(size)))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "equal-papua",
   "metadata": {},
   "source": [
    "## Inputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "excess-mambo",
   "metadata": {},
   "outputs": [],
   "source": [
    "class InputEmbeddings:\n",
    "    \"\"\"\n",
    "    This will hold all the input embeddings:\n",
    "        'text_7_props', 'complex', 'transe', 'abstract_first_sent', 'labels', 'labels_n_desc',\n",
    "        'has_h', 'has_s' (depending on boolean value of `has_embeddings_include`)\n",
    "    Instance variables:\n",
    "        - embed_dict_master - holds all qnode to embedding mappings as a dictionary\n",
    "        - emb_list - list of all keys of the above dictionary\n",
    "        - embedding_lengths - dictionary of the embedding lengths of each of the keys stored\n",
    "    \"\"\"\n",
    "    def __init__(self, has_embeddings_include: bool = True):\n",
    "        self.embed_dict_master = {}\n",
    "        self.emb_list = ['text_7_props', 'complex', 'transe', 'abstract_first_sent', 'labels', 'labels_n_desc']\n",
    "        if has_embeddings_include:\n",
    "            self.emb_list += ['has_h', 'has_s']\n",
    "        self.embedding_lengths = {}\n",
    "        \n",
    "        # Load each set of embeddings\n",
    "        for emb_key in tqdm(self.emb_list, desc='Input Embeddings', leave=False):\n",
    "            self.embed_dict_master[emb_key] = self.fetch_embedding(emb_key)\n",
    "            \n",
    "        # Generate Embedding Stats\n",
    "        self.fetch_embedding_stats()\n",
    "        print(\"Fetched all input embeddings\")\n",
    "\n",
    "    def fetch_embedding(self, emb_key):\n",
    "        '''Load set of embeddings, deserialize them and fill coverage for this set'''\n",
    "        emb = Utils.deserialize_embedding_dict(\n",
    "                json.load(open(INPUT_EMB_FOLDER_PATH+emb_key+'_orig_embedding_dict.json'))\n",
    "            )\n",
    "        \n",
    "        print(f\"OG Coverage of {emb_key}: {Utils.check_coverage(emb)}\")\n",
    "        return Utils.fill_coverage(\n",
    "                emb, emb_key\n",
    "            )\n",
    "\n",
    "    def fetch_embedding_stats(self):\n",
    "        '''Print Embedding sets stats'''\n",
    "        for emb_name in self.embed_dict_master.keys():\n",
    "            self.embedding_lengths[emb_name] = len(next(iter(self.embed_dict_master[emb_name].values())))\n",
    "            print(f\"Embedding: {emb_name}, Size: {len(self.embed_dict_master[emb_name].keys())}, Length: {self.embedding_lengths[emb_name]}\")\n",
    "\n",
    "class ReducedInputEmbeddings:\n",
    "    '''\n",
    "    This will hold the input embeddings whose lengths have been reduced to a fixed size: `final_embed_len`\n",
    "    '''\n",
    "    def __init__(self, embed_dict_master, final_embed_len):\n",
    "        '''\n",
    "        Use PCA to change dimensionality of each set of embeddings\n",
    "        Inputs:\n",
    "            - embed_dict_master: dict - set of embeddings loaded in `InputEmbeddings`\n",
    "            - final_embed_len: int - final length of embeddings needed\n",
    "        '''\n",
    "        self.embed_dict_master = copy.deepcopy(embed_dict_master)\n",
    "        self.final_embed_len = final_embed_len\n",
    "        for key in tqdm(self.embed_dict_master.keys()):\n",
    "#             tsne = TSNE(final_embed_len, verbose=1, method='exact') NOT WORKING. Hence, used PCA alternatively. Kept just for documentation that this one didn't work\n",
    "            tfmr = PCA(final_embed_len)\n",
    "            tfmr_proj = tfmr.fit_transform(pd.DataFrame(list(self.embed_dict_master[key].values())))\n",
    "            tfmr_proj = normalize(tfmr_proj, axis=0)\n",
    "            for w_key, emb in zip(self.embed_dict_master[key].keys(), tfmr_proj):\n",
    "                self.embed_dict_master[key][w_key] = emb        \n",
    "                \n",
    "    def generate_concatenated_embedding_dict(self, key_comb):\n",
    "        '''\n",
    "        Using a global list of nodes, concatenate embeddings wherever present else\n",
    "            concatenate with zeros (matching embedding length) wherever absent\n",
    "        '''\n",
    "        embedDict = defaultdict(list)\n",
    "        masterKeySet = set()\n",
    "        for key in key_comb:\n",
    "            for qnode in self.embed_dict_master[key]:\n",
    "                masterKeySet.add(qnode)\n",
    "        for qnode in masterKeySet:\n",
    "            for key in key_comb:\n",
    "                if qnode in self.embed_dict_master[key]:\n",
    "                    embedDict[qnode] = embedDict[qnode] + (self.embed_dict_master[key][qnode].tolist())\n",
    "                else:\n",
    "#                     print(\"Hit missing elem branch for concatenation\")\n",
    "                    embedDict[qnode] = embedDict[qnode] + [0]*self.final_embed_len\n",
    "            embedDict[qnode] = np.array(embedDict[qnode])\n",
    "        return dict(embedDict)\n",
    "\n",
    "class InputScoreTables:\n",
    "    '''\n",
    "    This will hold the table with scores. For class, JC, topSim, we do not have embeddings \n",
    "        but we directly have the similarity scores fetched from the API. \n",
    "        For other embeddings, we identify these similarity scores and store them in the tables format.\n",
    "        This caching helps accelerate the analysis\n",
    "        Thus, we store these similarity scores tables in a separate object.\n",
    "    '''\n",
    "    def __init__(self, embed_dict_master, exception_cols, eval_file, new_embed_dict_master=None, new_embed_suffix='_bert_child_par_1_weighted'):\n",
    "        '''\n",
    "        Inputs:\n",
    "            - embed_dict_master: dict - Loaded sets of embeddings for generating score tables\n",
    "            - exception_cols: list - list of embedding sets to be skipped while computing average similarity scores\n",
    "            - eval_file: string - the evaluation benchmark dataset to be used for creating score tables\n",
    "            - new_embed_dict_master: dict - if present, add similarity scores wrt these with a suffix of `_retrofitted` in the score table keys\n",
    "            - new_embed_suffix: string - used for fetching the sets of new embeddings from `new_embed_dict_master`\n",
    "        '''\n",
    "        self.input_score_tables = {}\n",
    "        self.new_embed_suffix = new_embed_suffix\n",
    "        print(f\"Fetching {eval_file} wordsim score tables and eval file\")\n",
    "        if eval_file == 'wordsim_new':\n",
    "            self.input_score_tables['classSim'] = pd.read_csv(WORDSIM_CLASS_SIM_FILE)\n",
    "            self.input_score_tables['JC'] = pd.read_csv(WORDSIM_JC_SIM_FILE)\n",
    "            self.input_score_tables['topSim'] = pd.read_csv(WORDSIM_TOP_SIM_FILE)\n",
    "            self.wordsim = evalD.wordsim_df.copy()\n",
    "        elif eval_file == 'wordsim_old':\n",
    "            self.input_score_tables['classSim'] = pd.read_csv(WORDSIM_OLD_CLASS_SIM_FILE)\n",
    "            self.input_score_tables['JC'] = pd.read_csv(WORDSIM_OLD_JC_SIM_FILE)\n",
    "            self.input_score_tables['topSim'] = pd.read_csv(WORDSIM_OLD_TOP_SIM_FILE)\n",
    "            self.wordsim = evalD.old_wordsim_df.copy()\n",
    "        elif eval_file == 'dbpedia_mc_30':\n",
    "            self.input_score_tables['classSim'] = pd.read_csv(DBPEDIA_MC_30_CLASS_SIM_FILE)\n",
    "            self.input_score_tables['JC'] = pd.read_csv(DBPEDIA_MC_30_JC_SIM_FILE)\n",
    "            self.input_score_tables['topSim'] = pd.read_csv(DBPEDIA_MC_30_TOP_SIM_FILE)\n",
    "            self.wordsim = evalD.dbpedia_mc_30_df.copy()\n",
    "        elif eval_file == 'dbpedia_rg_65':\n",
    "            self.input_score_tables['classSim'] = pd.read_csv(DBPEDIA_RG_65_CLASS_SIM_FILE)\n",
    "            self.input_score_tables['JC'] = pd.read_csv(DBPEDIA_RG_65_JC_SIM_FILE)\n",
    "            self.input_score_tables['topSim'] = pd.read_csv(DBPEDIA_RG_65_TOP_SIM_FILE)\n",
    "            self.wordsim = evalD.dbpedia_rg_65_df.copy()\n",
    "            \n",
    "        if embed_dict_master is not None:\n",
    "            for emb in embed_dict_master:\n",
    "#                 print(f\"Emb: {emb}\")\n",
    "                self.input_score_tables[emb] = self.construct_wsim_tab(embed_dict_master[emb])\n",
    "                if new_embed_dict_master is not None:\n",
    "                    self.input_score_tables[emb+'_retrofitted'] = self.construct_wsim_tab(new_embed_dict_master[emb + self.new_embed_suffix])\n",
    "        self.input_score_tables['average'] = self.get_averaged_dict(exception_cols, False)\n",
    "        if new_embed_dict_master is not None:\n",
    "            self.input_score_tables['average_retrofitted'] = self.get_averaged_dict(exception_cols, True)\n",
    "    \n",
    "    def construct_wsim_tab(self, embed_dict):\n",
    "        '''Construct similarity scores table based on the evaluation benchmark dataset using the embedding dict provided'''\n",
    "        \n",
    "        eval_dataset = self.wordsim.copy()\n",
    "\n",
    "        eval_dataset['embedding_cos_sim'] = eval_dataset.apply(lambda p: Utils.determine_cos_sim(embed_dict[p['word1_kg_id']], embed_dict[p['word2_kg_id']]) \n",
    "                                                   if p['word1_kg_id'] in embed_dict and p['word2_kg_id'] in embed_dict and embed_dict[p['word1_kg_id']].sum() != 0 and embed_dict[p['word2_kg_id']].sum() != 0\n",
    "                                                   else None, axis=1)\n",
    "        eval_dataset['embedding_na'] = eval_dataset['embedding_cos_sim'].isna()\n",
    "#         print(f\"Coverage: {len(eval_dataset) - eval_dataset['embedding_cos_sim'].isna().sum()}\")\n",
    "        eval_dataset['embedding_cos_sim'].fillna(eval_dataset['embedding_cos_sim'].mean(skipna=True), inplace=True)\n",
    "        \n",
    "        # Scale abs value of cosine similarities to 1,4 strictly\n",
    "        eval_dataset['embedding_cos_sim'] = eval_dataset['embedding_cos_sim'].apply(lambda p: 4 - 3 * abs(p))\n",
    "        \n",
    "        return eval_dataset\n",
    "        \n",
    "    def get_pairwise_dict(self, tab_key):\n",
    "        '''Transformation function to generate dict of (node1, node2): cosSim values from the score table'''\n",
    "        if tab_key not in self.input_score_tables:\n",
    "            raise \"Key not present in table\"\n",
    "        return {(row['word1_kg_id'], row['word2_kg_id']): row['embedding_cos_sim'] for _, row in self.input_score_tables[tab_key].iterrows()}\n",
    "    \n",
    "    def get_averaged_dict(self, exception_cols, new_embed=False):\n",
    "        '''\n",
    "        Generate dict of averaged embeddings while skipping the sets mentioned in `exception_cols`\n",
    "        If new_embed is False, embedding sets NOT having `_retrofitted` suffix and the ones not mentioned in `exception_cols`\n",
    "            are used for generating the average cosSim values\n",
    "        If new_embed is True, embedding sets having `_retrofitted` suffix and the ones not mentioned in `exception_cols`\n",
    "            are used for generating the average cosSim values\n",
    "        Inputs:\n",
    "            - exception_cols: list - list of embedding sets to be skipped while computing average similarity scores\n",
    "            - new_embed: bool - indicator to decide whether to use `..._retrofitted` embedding sets or not while computing average\n",
    "        Returns:\n",
    "            evaluation benchmark dataset with embedding cosSim scores averaged from desired columns\n",
    "        '''\n",
    "        final_dict = defaultdict(list)\n",
    "        col_list = []\n",
    "        try:\n",
    "            for tab_key in set(self.input_score_tables.keys()) - exception_cols:\n",
    "                if new_embed and not(tab_key.endswith('retrofitted')):\n",
    "                    continue\n",
    "                elif not(new_embed) and tab_key.endswith('retrofitted'):\n",
    "                    continue\n",
    "                if any([tab_key.startswith(col1) for col1 in exception_cols]):\n",
    "                    continue\n",
    "                if tab_key.startswith('average'):\n",
    "                    continue\n",
    "                col_list.append(tab_key)\n",
    "                for _, row in self.input_score_tables[tab_key].iterrows():\n",
    "                    if row['embedding_na'] == False:\n",
    "                        final_dict[(row['word1_kg_id'], row['word2_kg_id'])].append(row['embedding_cos_sim'])\n",
    "                    else:\n",
    "    #                     print('na embedding was present, hence skipped')\n",
    "                        pass\n",
    "        except Exception as exc:\n",
    "            print(tab_key)\n",
    "            raise exc\n",
    "        for key in final_dict:\n",
    "            final_dict[key] = np.mean(np.array(final_dict[key]))\n",
    "        print(f\"Returning averaged scores from {len(col_list)} algorithms - {col_list}\")\n",
    "        eval_dataset = self.wordsim.copy()\n",
    "\n",
    "        eval_dataset['embedding_cos_sim'] = eval_dataset.apply(lambda p: final_dict[(p['word1_kg_id'], p['word2_kg_id'])], axis=1)\n",
    "        eval_dataset['embedding_na'] = eval_dataset['embedding_cos_sim'].isna()\n",
    "        \n",
    "        return eval_dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ceramic-secretary",
   "metadata": {},
   "source": [
    "## NeighborDatasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "detected-division",
   "metadata": {},
   "outputs": [],
   "source": [
    "class NeighborDatasets:\n",
    "    \"\"\"\n",
    "    Instance variables:\n",
    "        - neighbors_dict_master - holds all qnode to neighbor qnode mappings as a dictionary\n",
    "        - basis_list - list of all keys of the above dictionary\n",
    "    \"\"\"\n",
    "    \n",
    "    def __init__(self, class_datasets_fetch: bool = False, probase_datasets_fetch: bool = True):\n",
    "        self.neighbors_dict_master = {}\n",
    "        \n",
    "        pbar = tqdm(desc='Neighbor Datasets', leave=False, total = \n",
    "                    3\n",
    "                    + (3 if class_datasets_fetch else 0) \n",
    "                    + (1 if probase_datasets_fetch else 0) \n",
    "                   )\n",
    "        \n",
    "        bert_P279_child_par_df = pd.read_csv(P279_CHILD_PAR_DISTILBERT_COSSIM_FILE)\n",
    "        bert_P279_siblings_df = pd.read_csv(P279_SIBLINGS_DISTILBERT_COSSIM_FILE)\n",
    "        \n",
    "        self.neighbors_dict_master['bert_child_par'] = self.fetch_neighbours(bert_P279_child_par_df)\n",
    "        pbar.update(1)\n",
    "        self.neighbors_dict_master['bert_siblings'] = self.fetch_neighbours(bert_P279_siblings_df)\n",
    "        pbar.update(1)\n",
    "        self.neighbors_dict_master['bert_all'] = self.fetch_neighbours(pd.concat([\n",
    "                bert_P279_child_par_df, bert_P279_siblings_df\n",
    "            ]))\n",
    "        pbar.update(1)\n",
    "            \n",
    "        if class_datasets_fetch:\n",
    "            class_P279_child_par_df = pd.read_csv(P279_CHILD_PAR_CLASSSIM_FILE)\n",
    "            class_P279_child_par_df['similarity_value'] = class_P279_child_par_df['classSim']\n",
    "            \n",
    "            class_P279_siblings_df = pd.read_csv(P279_SIBLINGS_CLASSSIM_FILE)\n",
    "            class_P279_siblings_df['similarity_value'] = class_P279_siblings_df['classSim']\n",
    "            \n",
    "            self.neighbors_dict_master['class_child_par'] = self.fetch_neighbours(class_P279_child_par_df)\n",
    "            pbar.update(1)\n",
    "            self.neighbors_dict_master['class_siblings'] = self.fetch_neighbours(class_P279_siblings_df)\n",
    "            pbar.update(1)\n",
    "            self.neighbors_dict_master['class_all'] = self.fetch_neighbours(pd.concat([\n",
    "                    class_P279_child_par_df, class_P279_siblings_df\n",
    "                ]))\n",
    "            pbar.update(1)\n",
    "\n",
    "        if probase_datasets_fetch:\n",
    "            probase_df = self.process_probase(PROBASE_FINAL_FILE)\n",
    "            \n",
    "            self.neighbors_dict_master['probase'] = self.fetch_neighbours(probase_df)\n",
    "            pbar.update(1)\n",
    "#             self.neighbors_dict_master['probase+bert_all'] = self.fetch_neighbours(pd.concat([\n",
    "#                     bert_P279_child_par_df, bert_P279_siblings_df, probase_df\n",
    "#                 ]))\n",
    "#             pbar.update(1)\n",
    "        \n",
    "        self.basis_list = list(self.neighbors_dict_master.keys())\n",
    "        \n",
    "        pbar.close()\n",
    "\n",
    "        print(f\"Fetched neighbour datasets: {self.basis_list}\")\n",
    "    \n",
    "    def process_probase(self, probase_file_path):\n",
    "        '''Process probase dataset to shift the cosSim values to generate significant retrofitting amongst neighouring nodes'''\n",
    "        probase_df = pd.read_csv(probase_file_path)\n",
    "#         probase_df = probase_df.rename(columns={'n1_final_qnode': 'node1', 'n2_final_qnode': 'node2', 'sim': 'similarity_value'})\n",
    "        probase_df['similarity_value'] = 0.5 + 0.5 * probase_df['similarity_value']\n",
    "        \n",
    "        return probase_df\n",
    "        \n",
    "    def fetch_neighbours(self, df):\n",
    "        '''Generate neighbouring nodes dictionary of the form: node: [neighbouring nodes] from the dataset `df` provided'''\n",
    "        neighbours_dict = {}\n",
    "        for _, row in df.iterrows():\n",
    "            if row.node1 not in neighbours_dict:\n",
    "                neighbours_dict[row.node1] = []\n",
    "            neighbours_dict[row.node1].append((row.node2, row.similarity_value))\n",
    "\n",
    "            if row.node2 not in neighbours_dict:\n",
    "                neighbours_dict[row.node2] = []\n",
    "            neighbours_dict[row.node2].append((row.node1, row.similarity_value))\n",
    "#         print(max([len(neigh) for neigh in neighbours_dict.values()]))\n",
    "        \n",
    "        return neighbours_dict"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "architectural-service",
   "metadata": {},
   "source": [
    "## EvaluationDatasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "through-address",
   "metadata": {},
   "outputs": [],
   "source": [
    "class EvaluationDatasets:\n",
    "    '''\n",
    "    This will hold the evaluation benchmark datasets:\n",
    "    Instance variables:\n",
    "        - wordsim_df: Wordsim 353 Annotated dataset\n",
    "        - old_wordsim_df: Wordsim 353 old dataset\n",
    "        - dbpedia_mc_30_df: DBPedia MC 30\n",
    "        - dbpedia_rg_65_df: DBPedia RG 65\n",
    "        - coverage: set of nodes from all evaluation benchmark tables stored\n",
    "    '''\n",
    "    def __init__(self):\n",
    "        self.wordsim_df = pd.read_csv(WORDSIM_DF)\n",
    "        self.wordsim_df['category'] = self.wordsim_df.Avg.apply(Utils.label_samples)\n",
    "        self.fetch_distribution_stats(\"Wordsim-353\", self.wordsim_df)\n",
    "        \n",
    "        self.old_wordsim_df = pd.read_csv(WORDSIM_OLD_FINAL_FILE)\n",
    "        self.wordsim_df['category'] = self.wordsim_df.Avg.apply(Utils.label_samples)\n",
    "        self.fetch_distribution_stats(\"Wordsim-353 OLD\", self.old_wordsim_df)\n",
    "        \n",
    "        self.dbpedia_mc_30_df = pd.read_csv(DBPEDIA_MC_30_FINAL_FILE)\n",
    "        self.dbpedia_mc_30_df['category'] = self.dbpedia_mc_30_df.Avg.apply(Utils.label_samples)\n",
    "        self.fetch_distribution_stats(\"DBPedia MC 30\", self.dbpedia_mc_30_df)\n",
    "        \n",
    "        self.dbpedia_rg_65_df = pd.read_csv(DBPEDIA_RG_65_FINAL_FILE)\n",
    "        self.dbpedia_rg_65_df['category'] = self.dbpedia_rg_65_df.Avg.apply(Utils.label_samples)\n",
    "        self.fetch_distribution_stats(\"DBPedia RG 65\", self.dbpedia_rg_65_df)\n",
    "        \n",
    "#         self.wiki_cs_df = pd.read_csv('../data/wikidata-cs_categorized.csv')\n",
    "#         self.fetch_distribution_stats(\"Wikidata CS\", self.wiki_cs_df)\n",
    "        \n",
    "#         self.concept_net_df = pd.read_csv('../data/kgtk_conceptnet_evaluation.csv')\n",
    "#         self.fetch_distribution_stats(\"Concept Net\", self.concept_net_df)\n",
    "        \n",
    "        self.get_coverage_nodes()\n",
    "        \n",
    "    def fetch_distribution_stats(self, name, dataset):\n",
    "        '''Print dataset stats'''\n",
    "        print(f\"Dataset: {name}\")\n",
    "        print(dataset.category.value_counts())\n",
    "    \n",
    "    def get_coverage_nodes(self):\n",
    "        '''Store set of nodes in `self.coverage`'''\n",
    "        self.coverage = set(\n",
    "                        self.wordsim_df['word1_kg_id'].to_list() \n",
    "                        + self.wordsim_df['word2_kg_id'].to_list() \n",
    "                        + self.dbpedia_mc_30_df['word1_kg_id'].to_list()\n",
    "                        + self.dbpedia_mc_30_df['word2_kg_id'].to_list()\n",
    "                        + self.dbpedia_rg_65_df['word1_kg_id'].to_list()\n",
    "                        + self.dbpedia_rg_65_df['word2_kg_id'].to_list())\n",
    "#                         + self.wiki_cs_df['word1_kg_id'].to_list() \n",
    "#                         + self.wiki_cs_df['word2_kg_id'].to_list()\n",
    "#                         + self.concept_net_df['word1_kg_id'].to_list()\n",
    "#                         + self.concept_net_df['word2_kg_id'].to_list())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "refined-growth",
   "metadata": {},
   "source": [
    "## ResultMetrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "necessary-naples",
   "metadata": {},
   "outputs": [],
   "source": [
    "class ResultMetrics:\n",
    "    \n",
    "    @classmethod\n",
    "    def compute_classification_results(cls,\n",
    "            embed_dict, \n",
    "            eval_dataset,\n",
    "            get_output_values: bool = False,\n",
    "            old_accuracy = None\n",
    "            ):\n",
    "        \"\"\"\n",
    "        Inputs:\n",
    "            - embed_dict - dictionary of qnodes with node embeddings as its values\n",
    "            - eval_dataset - evaluation dataset as pandas dataframe that must have the \n",
    "                following columns for this function to work correctly:\n",
    "                * word1_kg_id - Qnode of node1 in the evaluation pair\n",
    "                * word2_kg_id - Qnode of node2 in the evaluation pair\n",
    "                * category - Category of the evaluation pair. One of the labels: I/U/M\n",
    "        Outputs:\n",
    "            - response_dict - Returns a dictionary with the following keys:\n",
    "                * covered_pairs - Indicates the number of pairs of the evaluation dataset that the \n",
    "                    embedding dictionary can cover\n",
    "                \n",
    "        \"\"\"\n",
    "        response_dict = {}\n",
    "        \n",
    "        eval_dataset = eval_dataset.copy()\n",
    "\n",
    "        missing_words_set = set(\n",
    "            eval_dataset[eval_dataset.word1_kg_id.apply(lambda p: p not in embed_dict)].word1_kg_id.to_list() \n",
    "            + eval_dataset[eval_dataset.word2_kg_id.apply(lambda p: p not in embed_dict)].word2_kg_id.to_list()\n",
    "        )\n",
    "        \n",
    "        response_dict['covered_pairs'] = len(eval_dataset)\n",
    "\n",
    "        eval_dataset['embedding_cos_sim'] = eval_dataset.apply(lambda p: Utils.determine_cos_sim(embed_dict[p['word1_kg_id']], embed_dict[p['word2_kg_id']]) \n",
    "                                                   if p['word1_kg_id'] in embed_dict and p['word2_kg_id'] in embed_dict \n",
    "                                                   else None, axis=1)\n",
    "        \n",
    "        eval_dataset['embedding_cos_sim'].fillna(eval_dataset['embedding_cos_sim'].mean(skipna=True), inplace=True)\n",
    "        \n",
    "        # Scale abs value of cosine similarities to 1,4 strictly\n",
    "        eval_dataset['embedding_cos_sim'] = eval_dataset['embedding_cos_sim'].apply(lambda p: 4 - 3 * abs(p))\n",
    "\n",
    "        response_dict['accuracy'] = 100 * accuracy_score(\n",
    "                eval_dataset['category'],\n",
    "                eval_dataset['embedding_cos_sim'].apply(Utils.label_samples)\n",
    "            )\n",
    "    \n",
    "        response_dict['classification_report'] = classification_report(\n",
    "                eval_dataset['category'], \n",
    "                eval_dataset['embedding_cos_sim'].apply(Utils.label_samples), \n",
    "                output_dict=True\n",
    "            )\n",
    "\n",
    "        response_dict['conf_matrix'] = confusion_matrix(\n",
    "                eval_dataset['category'], \n",
    "                eval_dataset['embedding_cos_sim'].apply(Utils.label_samples), \n",
    "                labels=Utils.LABELS\n",
    "            )\n",
    "        if 'Avg' in eval_dataset.columns:\n",
    "            response_dict['KT'] = stats.kendalltau(eval_dataset['Avg'], eval_dataset['embedding_cos_sim']).correlation\n",
    "            response_dict['SR'] = stats.spearmanr(eval_dataset['Avg'], eval_dataset['embedding_cos_sim']).correlation\n",
    "            response_dict['RMSE'] = mean_squared_error(eval_dataset['Avg'], eval_dataset['embedding_cos_sim'], squared=False)\n",
    "        else:\n",
    "            response_dict['KT'] = None\n",
    "            response_dict['SR'] = None\n",
    "            response_dict['RMSE'] = None\n",
    "        \n",
    "        if old_accuracy is not None:\n",
    "            response_dict['increase_acc'] = response_dict['accuracy'] - old_accuracy\n",
    "        else:\n",
    "            response_dict['increase_acc'] = None\n",
    "        \n",
    "        if get_output_values:\n",
    "            response_dict['preds'] = eval_dataset['embedding_cos_sim'].apply(Utils.label_samples)\n",
    "\n",
    "        return response_dict, \\\n",
    "                (response_dict['covered_pairs'],  \\\n",
    "                 response_dict['accuracy'], \\\n",
    "                 response_dict['increase_acc'], \\\n",
    "                 \n",
    "                 response_dict['classification_report']['I']['precision'],  \\\n",
    "                 response_dict['classification_report']['I']['recall'],  \\\n",
    "                 response_dict['classification_report']['I']['f1-score'],   \\\n",
    "                 \n",
    "                 response_dict['classification_report']['M']['precision'],  \\\n",
    "                 response_dict['classification_report']['M']['recall'],  \\\n",
    "                 response_dict['classification_report']['M']['f1-score'], \\\n",
    "                 \n",
    "                 response_dict['classification_report']['U']['precision'],  \\\n",
    "                 response_dict['classification_report']['U']['recall'],  \\\n",
    "                 response_dict['classification_report']['U']['f1-score'], \\\n",
    "                \n",
    "                 response_dict['KT'], \\\n",
    "                 response_dict['SR'],\n",
    "                 response_dict['RMSE'])\n",
    "    \n",
    "    @classmethod\n",
    "    def fetch_best_result_for_emb(cls, results_df, emb_col, target_col, iter_col, highest: bool = True):\n",
    "        '''\n",
    "        Given a set of iterations for a given `emb_col` parameter, retain the row and values at iteration 0\n",
    "            and the best row and values in terms of value in `target_col`. \n",
    "        Here, the iteration column is indicated by `iter_col`\n",
    "        Inputs:\n",
    "            - results_df: Pandas DataFrame - containing the results\n",
    "            - emb_col: string - Embedding column used as the pivot\n",
    "            - target_col: string - Target column used to identify the best row and values\n",
    "            - iter_col: string - Iteration column used to indicate iteration number\n",
    "            - highest: bool - Pick highest value of `target_col` as best value if True else pick lowest value as best value\n",
    "        Returns:\n",
    "            Pandas DataFrame containing best results\n",
    "        '''\n",
    "        opt_value = {}\n",
    "        for _, row in results_df.iterrows():\n",
    "            if row[emb_col] not in opt_value:\n",
    "                opt_value[row[emb_col]] = {'opt_metric': float('-inf') if highest else float('inf'),\n",
    "                                     'opt_row': [], 'old_row': []}\n",
    "            if row[iter_col] == 0:\n",
    "                opt_value[row[emb_col]]['old_row'] = row\n",
    "            else:\n",
    "                if (highest and row[target_col] > opt_value[row[emb_col]]['opt_metric']) \\\n",
    "                        or (not(highest) and row[target_col] < opt_value[row[emb_col]]['opt_metric']):\n",
    "                    opt_value[row[emb_col]]['opt_metric'] = row[target_col]\n",
    "                    opt_value[row[emb_col]]['opt_row'] = row\n",
    "        best_results = []\n",
    "        for emb_key in opt_value:\n",
    "            best_results.append(opt_value[emb_key]['old_row'])\n",
    "            best_results.append(opt_value[emb_key]['opt_row'])\n",
    "        return pd.DataFrame(best_results, columns = results_df.columns)\n",
    "    \n",
    "    @classmethod\n",
    "    def compute_classification_n_regression_stats(cls, ist, suffix, standard_labels=True):\n",
    "        '''\n",
    "        Compute unsupervised classification, supervised 10-fold cross-validation based classification, regression results\n",
    "            and store results at `../data/retrofitting/score_table_algorithms_results.SUFFIX.DATE.csv`\n",
    "        Inputs:\n",
    "            - ist: InputScoreTables instance\n",
    "            - suffix: string - used in filename\n",
    "            - standard_labels: bool - Use standard category labels if True else use quantile labels\n",
    "        '''\n",
    "        if not(standard_labels):\n",
    "            quantiles = evalD.wordsim_df.Avg.quantile([0, 0.25, 0.5, 0.75, 1]).to_list()\n",
    "            quantiles[0], quantiles[-1] = float('-inf'), float('inf')\n",
    "            print(f\"Quantiles being used by wordsim: {quantiles}\")\n",
    "            wordsim_cats = ist.wordsim.Avg.apply(Utils.alt_label_samples, args=(quantiles,))\n",
    "        else:\n",
    "            wordsim_cats = ist.wordsim.category\n",
    "        \n",
    "        eval_df = ist.wordsim.copy()\n",
    "        if not(standard_labels):\n",
    "            eval_df['quartile'] = eval_df.Avg.apply(Utils.alt_label_samples, args=(quantiles,))\n",
    "            \n",
    "        results = []\n",
    "        \n",
    "        # For each table in score tables\n",
    "        for tab_key in ist.input_score_tables:\n",
    "            cosSimPreds_df = ist.input_score_tables[tab_key]\n",
    "            response_dict = {}\n",
    "            if standard_labels:\n",
    "                preds = cosSimPreds_df['embedding_cos_sim'].apply(Utils.label_samples)\n",
    "            else:\n",
    "                temp_wordsim_df = cosSimPreds_df.sort_values(by=['embedding_cos_sim', 'word1_kg_id', 'word2_kg_id'])\n",
    "                quantiles_emb = cosSimPreds_df['embedding_cos_sim'].quantile([0, 0.25, 0.5, 0.75, 1]).to_list()\n",
    "                quantiles_emb[0], quantiles_emb[-1] = float('-inf'), float('inf')\n",
    "                print(f\"Quantiles being used by {tab_key}: {quantiles_emb}\")\n",
    "                preds = cosSimPreds_df['embedding_cos_sim'].apply(Utils.alt_label_samples, args=(quantiles_emb,))\n",
    "            eval_df[tab_key] = cosSimPreds_df['embedding_cos_sim']\n",
    "            eval_df[tab_key+'_cat'] = preds\n",
    "            response_dict['accuracy'] = 100 * accuracy_score(\n",
    "                    wordsim_cats,\n",
    "                    preds\n",
    "                )\n",
    "\n",
    "            response_dict['classification_report'] = classification_report(\n",
    "                    wordsim_cats,\n",
    "                    preds, \n",
    "                    output_dict=True\n",
    "                )\n",
    "\n",
    "            response_dict['KT'] = stats.kendalltau(cosSimPreds_df['Avg'], cosSimPreds_df['embedding_cos_sim']).correlation\n",
    "            response_dict['SR'] = stats.spearmanr(cosSimPreds_df['Avg'], cosSimPreds_df['embedding_cos_sim']).correlation\n",
    "            response_dict['RMSE'] = mean_squared_error(cosSimPreds_df['Avg'], cosSimPreds_df['embedding_cos_sim'], squared=False)\n",
    "            \n",
    "            # SVR related stats\n",
    "            temp_dict = {'basis': '', 'emb': tab_key, 'weightedness': True, \n",
    "                'iter_num': 0, 'weight_case': None, 'svm_input': 'score'}\n",
    "            svr_res = SVMProcedures.execute_supervised_scenario(\n",
    "                ist.wordsim, temp_dict, ist.get_pairwise_dict(tab_key), \n",
    "                {},num_of_splits=10,\n",
    "                comb_mode=False, SVC_or_SVR='SVR', score_table_mode=True\n",
    "            )\n",
    "            \n",
    "            if standard_labels:\n",
    "                results.append([tab_key, response_dict['accuracy'], response_dict['classification_report']['macro avg']['precision'], \\\n",
    "                            response_dict['classification_report']['macro avg']['recall'], \\\n",
    "                            response_dict['classification_report']['macro avg']['f1-score'], \\\n",
    "                                \\\n",
    "                            response_dict['classification_report']['I']['precision'], \\\n",
    "                            response_dict['classification_report']['I']['recall'], \\\n",
    "                            response_dict['classification_report']['I']['f1-score'], \\\n",
    "                                \\\n",
    "                            response_dict['classification_report']['M']['precision'], \\\n",
    "                            response_dict['classification_report']['M']['recall'], \\\n",
    "                            response_dict['classification_report']['M']['f1-score'], \\\n",
    "                                \\\n",
    "                            response_dict['classification_report']['U']['precision'], \\\n",
    "                            response_dict['classification_report']['U']['recall'], \\\n",
    "                            response_dict['classification_report']['U']['f1-score'], \\\n",
    "                                \\\n",
    "                            response_dict['KT'], response_dict['SR'], response_dict['RMSE'], svr_res[-2], svr_res[-1], svr_res[-3]])\n",
    "            else:\n",
    "                results.append([tab_key, response_dict['accuracy'], response_dict['classification_report']['macro avg']['precision'], \\\n",
    "                            response_dict['classification_report']['macro avg']['recall'], \\\n",
    "                            response_dict['classification_report']['macro avg']['f1-score'], \\\n",
    "                                \\\n",
    "                            response_dict['classification_report']['Q1']['precision'], \\\n",
    "                            response_dict['classification_report']['Q1']['recall'], \\\n",
    "                            response_dict['classification_report']['Q1']['f1-score'], \\\n",
    "                                \\\n",
    "                            response_dict['classification_report']['Q2']['precision'], \\\n",
    "                            response_dict['classification_report']['Q2']['recall'], \\\n",
    "                            response_dict['classification_report']['Q2']['f1-score'], \\\n",
    "                                \\\n",
    "                            response_dict['classification_report']['Q3']['precision'], \\\n",
    "                            response_dict['classification_report']['Q3']['recall'], \\\n",
    "                            response_dict['classification_report']['Q3']['f1-score'], \\\n",
    "                                \\\n",
    "                            response_dict['classification_report']['Q4']['precision'], \\\n",
    "                            response_dict['classification_report']['Q4']['recall'], \\\n",
    "                            response_dict['classification_report']['Q4']['f1-score'], \\\n",
    "                                \\\n",
    "                            response_dict['KT'], response_dict['SR'], response_dict['RMSE'], svr_res[-2], svr_res[-1], svr_res[-3]])\n",
    "        if standard_labels:\n",
    "            res_df = pd.DataFrame(results, columns=['algorithm', 'accuracy', 'P', 'R', 'F1', 'I P', 'I R', 'I F1', 'M P', 'M R', 'M F1', 'U P', 'U R', 'U F1', 'Kendall Tau', 'Spearman Rank', 'RMSE', 'SVR Kendall Tau', 'SVR Spearman Rank', 'SVR RMSE'])\n",
    "        else:\n",
    "            res_df = pd.DataFrame(results, columns=['algorithm', 'accuracy', 'P', 'R', 'F1', 'Q1 P', 'Q1 R', 'Q1 F1', 'Q2 P', 'Q2 R', 'Q2 F1', 'Q3 P', 'Q3 R', 'Q3 F1', 'Q4 P', 'Q4 R', 'Q4 F1', 'Kendall Tau', 'Spearman Rank', 'RMSE', 'SVR Kendall Tau', 'SVR Spearman Rank', 'SVR RMSE'])\n",
    "        res_df.to_csv('../data/retrofitting/score_table_algorithms_results.' + suffix + '.' + Utils.today_date + '.csv', index=False)\n",
    "        return res_df, eval_df\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "arranged-cosmetic",
   "metadata": {},
   "source": [
    "## RetrofittingProcedures"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "prescription-mainstream",
   "metadata": {},
   "outputs": [],
   "source": [
    "class RetrofittingProcedures:\n",
    "    \n",
    "    np_label_samples = np.vectorize(Utils.label_samples)\n",
    "    \n",
    "    @classmethod\n",
    "    def retrofit(cls,embed_dict, neighbors_dict, weight_case, weight_assignment=False):\n",
    "        '''\n",
    "        Retrofit the embeddings in the `embed_dict` given the pairs in the `neighbors_dict`\n",
    "        Inputs:\n",
    "            - embed_dict: dict - set of embeddings\n",
    "            - neighbors_dict: dict - dictionary of lists of neighbors to a node and their similarity scores\n",
    "                Format: node1: [(node2, cosSim), ...]\n",
    "            - weight_case: float - indicating the significance to be placed on the neighboring embeddings\n",
    "            - weight_assignment: bool - indicates if weights (cosSim values of neighbors) need to be considered for retrofitting\n",
    "        '''\n",
    "        new_embed_dict = {}\n",
    "        for word in embed_dict.keys():\n",
    "            if word in neighbors_dict:\n",
    "                neighbs = neighbors_dict[word]\n",
    "                neighbs = list(filter(lambda p: p[0] in embed_dict, neighbs))\n",
    "                if len(neighbs) == 0:\n",
    "                    new_embed_dict[word] = embed_dict[word]\n",
    "                    continue\n",
    "                if weight_assignment:\n",
    "                    sum_of_sims = sum([neighb[1] for neighb in neighbs])\n",
    "                    sum_of_embs = sum([embed_dict[neighb[0]] * float(neighb[1]) for neighb in neighbs])\n",
    "                else:\n",
    "                    sum_of_sims = sum([1 for neighb in neighbs])\n",
    "                    sum_of_embs = sum([embed_dict[neighb[0]] for neighb in neighbs])\n",
    "\n",
    "                if weight_case == 1:\n",
    "                    new_embed_dict[word] = (embed_dict[word] * (len(neighbs)) + sum_of_embs) / ((len(neighbs)) + sum_of_sims)\n",
    "                elif weight_case == 2:\n",
    "                    new_embed_dict[word] = (embed_dict[word] * (len(neighbs))**2 + sum_of_embs) / ((len(neighbs))**2 + sum_of_sims)\n",
    "                elif weight_case == 0.5:\n",
    "                    new_embed_dict[word] = (embed_dict[word] * (len(neighbs))**0.5 + sum_of_embs) / ((len(neighbs))**0.5 + sum_of_sims)\n",
    "                else:\n",
    "                    raise\n",
    "            else:\n",
    "                new_embed_dict[word] = embed_dict[word]\n",
    "        return new_embed_dict\n",
    "    \n",
    "    @classmethod\n",
    "    def execute_all_unsupervised_scenarios(cls,\n",
    "                emb_list, basis_list, \n",
    "                embed_dict_master, neigh_dict_master,\n",
    "                eval_dataset,\n",
    "                scenario_name: str,\n",
    "                num_of_iterations: int = 2, \n",
    "                weightedness_list: list = [True],\n",
    "                weight_cases_list: list = [1],\n",
    "                get_output_values: bool = False,\n",
    "                prev_new_embed_dict_master = None\n",
    "            ):\n",
    "        '''\n",
    "        For each case of basis, embedding, weight case, iteration, compute the unsupervised classification results.\n",
    "        For each iteration that's considered, the embeddings are retrofitted wrt same neighbors dict. \n",
    "        Thus, as the iterations proceed, the embeddings with neighboring nodes are pulled closer and closer together.\n",
    "        Inputs:\n",
    "            - emb_list: list of strings - list of embedding sets to be considered\n",
    "            - basis_list: list of strings - list of basis dictionaries to be considered\n",
    "            - embed_dict_master: dict - sets of all embeddings\n",
    "            - neigh_dict_master: dict - sets of all neighboring dictionaries\n",
    "            - eval_dataset: Pandas DataFrame - evaluation benchmark dataset on which performance is to be evaluated\n",
    "            - scenario_name: string - indicating all details about the basis, embedding, weight case, iteration\n",
    "            - num_of_iterations: int - indicating the max no. of iterations to be retrofitted\n",
    "            - weightedness_list: list = [True] - indicates if the weights in the neighboring dicts need to be considered for analysis\n",
    "            - weight_cases_list: list = [1] - indicates the list of weight cases (1, 0.5, 2) to be considered for analysis\n",
    "            - get_output_values: bool = False - indicates if the results' intermediate values need to be returned from a function call\n",
    "            - prev_new_embed_dict_master: dict - New embeddings sets. This is used if this analysis is to be re-executed with few additional cases.\n",
    "                Having this cache helps preventing re-executions and saving time.\n",
    "        '''\n",
    "        \n",
    "        new_embed_dict_master = {}\n",
    "        responses_dict_master = {}\n",
    "        results = []\n",
    "        \n",
    "        for basis in tqdm(basis_list, desc='Basis', leave=False):\n",
    "            for emb in tqdm(emb_list, desc='Embedding', leave=False):\n",
    "                for weightedness in weightedness_list:\n",
    "                    for weight_case in tqdm(weight_cases_list, desc='Weight Case', leave=False):\n",
    "                        # Base Reference Initializations and Calculations\n",
    "                        embed_dict = embed_dict_master[emb]\n",
    "                        responses_dict, result_values = ResultMetrics.compute_classification_results(\n",
    "                            embed_dict, eval_dataset, get_output_values=get_output_values, old_accuracy=None)\n",
    "                        results.append([emb, basis, weight_case, weightedness, 0, 'base', *result_values, 0])\n",
    "                        old_accuracy = responses_dict['accuracy']\n",
    "                        \n",
    "                        for iter_num in tqdm(range(1,num_of_iterations+1), desc='Iteration', leave=False):\n",
    "                            start_time = time()\n",
    "                            \n",
    "                            case_name = emb + '_' + basis + '_' + str(weight_case) + ('_weighted' if weightedness else '_unweighted')\n",
    "                            \n",
    "                            if prev_new_embed_dict_master is not None and case_name in prev_new_embed_dict_master:\n",
    "                                new_embed_dict = prev_new_embed_dict_master[case_name]\n",
    "                            else:\n",
    "                                new_embed_dict = cls.retrofit(embed_dict, neigh_dict_master[basis], weight_case, weightedness)\n",
    "                            \n",
    "                            responses_dict, result_values = ResultMetrics.compute_classification_results(\n",
    "                                new_embed_dict, eval_dataset, get_output_values=get_output_values, old_accuracy=old_accuracy)\n",
    "                            \n",
    "                            results.append([emb, basis, weight_case, weightedness, iter_num, case_name, \\\n",
    "                                                *result_values, \\\n",
    "                                                time() - start_time\n",
    "                                            ])\n",
    "                            \n",
    "                            new_embed_dict_master[case_name] = embed_dict = new_embed_dict\n",
    "                            responses_dict_master[case_name] = responses_dict\n",
    "                            \n",
    "        resultsDF = pd.DataFrame(results, columns=['Embedding', 'Basis', 'Weight Case', 'Weightedness', \n",
    "                                                   'Iteration Num', 'Case Name', \\\n",
    "                                                   'No. of Pairs Covered', 'Accuracy', 'Increase in Accuracy', \\\n",
    "                                                   'I Precision', 'I Recall', 'I F1-Score', \\\n",
    "                                                   'M Precision', 'M Recall', 'M F1-Score', \\\n",
    "                                                   'U Precision', 'U Recall', 'U F1-Score',\n",
    "                                                   'KT Correlation', 'SpearmanR Correlation', 'RMSE', \\\n",
    "                                                   'Time to Retrofit'])\n",
    "        resultsDF.to_csv('../data/retrofitting/retro_unsup_results.' + scenario_name + '.'+ Utils.today_date +'.csv', index=False)\n",
    "#         best_results_df = ResultMetrics.fetch_best_result_for_emb(resultsDF, 'Embedding', 'Accuracy', 'Iteration Num', highest=True)\n",
    "#         best_results_df.to_csv('../data/retrofitting/retro_unsup_results.' + scenario_name + '.'+ Utils.today_date +'.best.csv', index=False)\n",
    "        \n",
    "#         cls.save_needed_embeddings(new_embed_dict_master)\n",
    "        \n",
    "        return new_embed_dict_master, responses_dict_master\n",
    "    \n",
    "    @classmethod\n",
    "    def save_all_embeddings(cls, new_embed_dict_master):\n",
    "        '''Save all the retrofitted embeddings from `new_embed_dict_master` to the specified path'''\n",
    "        for case_name in new_embed_dict_master:\n",
    "            json.dump(Utils.serialize_embedding_dict(new_embed_dict_master[case_name]), open(INPUT_EMB_FOLDER_PATH + 'new_embeddings/' + case_name + '.' + Utils.today_date + '.json', 'w'))\n",
    "    \n",
    "    @classmethod\n",
    "    def save_needed_embeddings(cls, new_embed_dict_master):\n",
    "        '''\n",
    "        Retain only the embeddings of those nodes that would be covered by the evaluation benchmark dataset.\n",
    "        Ideally, if you are retrofitting the embeddings, you should not do this, since this might distort the retrofitting process.\n",
    "        '''\n",
    "        for case_name in new_embed_dict_master:\n",
    "            temp = {key: new_embed_dict_master[case_name][key] for key in new_embed_dict_master[case_name] if key in evalD.coverage}\n",
    "            new_embed_dict_master[case_name] = temp"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "attempted-implementation",
   "metadata": {},
   "source": [
    "## SVMProcedures"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "missing-facial",
   "metadata": {},
   "outputs": [],
   "source": [
    "class SVMProcedures:\n",
    "    @classmethod\n",
    "    def execute_supervised_scenario(cls,\n",
    "                eval_dataset, case, embed_dict_master, new_embed_dict_master, \n",
    "                num_of_splits = 10, \n",
    "                comb_mode: bool = False, SVC_or_SVR: str = 'SVC', \n",
    "                score_table_mode: bool = False\n",
    "            ):\n",
    "        '''\n",
    "        Execute the 10-fold cross-validation SVR/SVC model\n",
    "        Inputs:\n",
    "            - eval_dataset: Pandas DataFrame - the evaluation benchmark dataset\n",
    "            - case: dict - contains details about the scenario - embedding, basis, weight case, iteration num\n",
    "            - embed_dict_master: dict - sets of embeddings\n",
    "            - new_embed_dict_master: dict - sets of new embeddings\n",
    "            - num_of_splits: int (default 10) - indicating k-fold cross validation\n",
    "            - comb_mode: bool (default False) - \n",
    "                If True concatenate the 2 embeddings in each row of the benchmark dataset for SVM evaluation \n",
    "                else use the similarity score wrt the 2 embeddings in each row of the dataset for SVM evaluation\n",
    "            - SVC_or_SVR: string (SVC/SVR) - indicate whether you want classification or regression results\n",
    "            - score_table_mode: bool (default False) - \n",
    "                If False, works using the standard embeddings from embed_dict_master (For each set of embeddings, we have (node): emb)\n",
    "                else uses the scores from score table generated embed_dict_master (For each set of embeddings, we have (node1, node2): sim score)\n",
    "        '''\n",
    "        \n",
    "        X = []        \n",
    "        \n",
    "        ################\n",
    "        # 2 Approaches based on argument: `comb_mode`\n",
    "        ################\n",
    "        \n",
    "        if comb_mode: ########## COMBINATION MODE CODE ####################\n",
    "            case_name = \" & \".join(case['emb']) + '_' + case['basis'] + '_' + str(case['weight_case']) + ('_weighted' if case['weightedness'] else '_unweighted')\n",
    "            \n",
    "            for _, row in eval_dataset.iterrows():\n",
    "                if case['svm_input'] == 'emb':\n",
    "                    tempX = []\n",
    "                    for individual_emb in case['emb']:\n",
    "                        ind_case_name = individual_emb + '_' + case['basis'] + '_' + str(case['weight_case']) + ('_weighted' if case['weightedness'] else '_unweighted')\n",
    "                        if case['iter_num'] != 0 and ind_case_name not in new_embed_dict_master:\n",
    "                            return case_name, case, None\n",
    "                        if case['iter_num'] == 0:\n",
    "                            if score_table_mode:\n",
    "                                raise \"Not yet implemented\"\n",
    "                            tempX += embed_dict_master[individual_emb][row['word1_kg_id']].tolist() + embedDictMaster[individual_emb][row['word2_kg_id']].tolist()\n",
    "                        else:\n",
    "                            tempX += new_embed_dict_master[ind_case_name][row['word1_kg_id']].tolist() + newEmbedDictMaster[ind_case_name][row['word2_kg_id']].tolist()\n",
    "                    X.append(tempX)\n",
    "                else:\n",
    "                    tempX = []\n",
    "                    for individual_emb in case['emb']:\n",
    "                        ind_case_name = individual_emb + '_' + case['basis'] + '_' + str(case['weight_case']) + ('_weighted' if case['weightedness'] else '_unweighted')\n",
    "                        if case['iter_num'] != 0 and ind_case_name not in new_embed_dict_master:\n",
    "                            return case_name, case, None\n",
    "                        if case['iter_num'] == 0:\n",
    "                            if score_table_mode:\n",
    "                                raise \"Not yet implemented\"\n",
    "                            tempX.append(abs(Utils.determine_cos_sim(\n",
    "                                embed_dict_master[individual_emb][row['word1_kg_id']], \n",
    "                                embed_dict_master[individual_emb][row['word2_kg_id']]\n",
    "                            )))\n",
    "                        else:\n",
    "                            tempX.append(abs(Utils.determine_cos_sim(\n",
    "                                new_embed_dict_master[ind_case_name][row['word1_kg_id']],\n",
    "                                new_embed_dict_master[ind_case_name][row['word2_kg_id']]\n",
    "                            )))\n",
    "                    X.append(tempX)\n",
    "\n",
    "        else: ########## NON-COMBINATION MODE CODE ####################\n",
    "            try:\n",
    "                case_name = case['emb'] + '_' + case['basis'] + '_' + str(case['weight_case']) + ('_weighted' if case['weightedness'] else '_unweighted')\n",
    "                if case['iter_num'] != 0 and case_name not in new_embed_dict_master:\n",
    "                    return case_name, case, None\n",
    "                for _, row in eval_dataset.iterrows():\n",
    "                    if case['svm_input'] == 'emb':\n",
    "                        if case['iter_num'] == 0:\n",
    "                            if score_table_mode:\n",
    "                                raise \"Not yet implemented\"\n",
    "                            X.append(embed_dict_master[case['emb']][row['word1_kg_id']].tolist() + embed_dict_master[case['emb']][row['word2_kg_id']].tolist())\n",
    "                        else:\n",
    "                            X.append(new_embed_dict_master[case_name][row['word1_kg_id']].tolist() + new_embed_dict_master[case_name][row['word2_kg_id']].tolist())\n",
    "                    else:\n",
    "                        if case['iter_num'] == 0:\n",
    "                            if score_table_mode:\n",
    "                                X.append(embed_dict_master[(row['word1_kg_id'], row['word2_kg_id'])])\n",
    "                            else:\n",
    "                                X.append(abs(Utils.determine_cos_sim(\n",
    "                                        embed_dict_master[case['emb']][row['word1_kg_id']], \n",
    "                                        embed_dict_master[case['emb']][row['word2_kg_id']]\n",
    "                                    )))\n",
    "                        else:\n",
    "                            X.append(abs(Utils.determine_cos_sim(\n",
    "                                    new_embed_dict_master[case_name][row['word1_kg_id']],\n",
    "                                    new_embed_dict_master[case_name][row['word2_kg_id']]\n",
    "                                )))\n",
    "            except Exception as err:\n",
    "                print(case_name)\n",
    "                raise err\n",
    "                    \n",
    "        X = pd.DataFrame(X)\n",
    "        \n",
    "        ################\n",
    "        # 2 Approaches based on argument: `SVC_or_SVR`\n",
    "        ################\n",
    "        \n",
    "        # Target split depending on SVC or SVM\n",
    "        if SVC_or_SVR == 'SVC':\n",
    "            Y = eval_dataset['category']\n",
    "        elif SVC_or_SVR == 'SVR':\n",
    "            if 'Avg' not in eval_dataset.columns:\n",
    "                raise ValueError(\"Avg column not present in the provided eval_dataset\")\n",
    "            Y = (eval_dataset['Avg'] - 1) / 3\n",
    "        else:\n",
    "            raise ValueError(\"Invalid SVC_or_SVR provided\")\n",
    "        \n",
    "        if SVC_or_SVR == 'SVC':\n",
    "            skf = StratifiedKFold(n_splits=num_of_splits, random_state=19, shuffle=True)\n",
    "            X_train_splits, X_test_splits, Y_train_splits, Y_test_splits = [], [], [], []\n",
    "            for train_index, test_index in skf.split(X, Y):\n",
    "                X_train_splits.append(X.iloc[train_index])\n",
    "                X_test_splits.append(X.iloc[test_index])\n",
    "                Y_train_splits.append(Y.iloc[train_index])\n",
    "                Y_test_splits.append(Y.iloc[test_index])\n",
    "        elif SVC_or_SVR == 'SVR':\n",
    "            skf = KFold(n_splits=num_of_splits, random_state=19, shuffle=True)\n",
    "            X_train_splits, X_test_splits, Y_train_splits, Y_test_splits = [], [], [], []\n",
    "            for train_index, test_index in skf.split(X, Y):\n",
    "                X_train_splits.append(X.iloc[train_index])\n",
    "                X_test_splits.append(X.iloc[test_index])\n",
    "                Y_train_splits.append(Y.iloc[train_index])\n",
    "                Y_test_splits.append(Y.iloc[test_index])\n",
    "\n",
    "        preds = []\n",
    "        \n",
    "        # Classifier/Regressor training depending on SVC or SVM\n",
    "        if SVC_or_SVR == 'SVC':\n",
    "            for X_train1, Y_train1, X_test1, Y_test1 in zip(X_train_splits, Y_train_splits, X_test_splits, Y_test_splits):\n",
    "                clf = make_pipeline(StandardScaler(), SVC(gamma='auto', random_state=100, max_iter=100))\n",
    "                clf.fit(X_train1, Y_train1)\n",
    "                preds.append(clf.predict(X_test1))\n",
    "                \n",
    "            acc, f1_score = 0, 0\n",
    "            for pred, Y_test1 in zip(preds, Y_test_splits):\n",
    "                acc += accuracy_score(pred, Y_test1)\n",
    "                f1_score += classification_report(\n",
    "                    Y_test1,\n",
    "                    pred, \n",
    "                    output_dict=True\n",
    "                )['macro avg']['f1-score']\n",
    "\n",
    "            return case_name, *list(case.values()), acc/num_of_splits, f1_score/num_of_splits\n",
    "        \n",
    "        elif SVC_or_SVR == 'SVR':\n",
    "            for X_train1, Y_train1, X_test1, Y_test1 in zip(X_train_splits, Y_train_splits, X_test_splits, Y_test_splits):\n",
    "                clf = make_pipeline(StandardScaler(), SVR(gamma='auto', max_iter=100))\n",
    "                clf.fit(X_train1, Y_train1)\n",
    "                preds.append(clf.predict(X_test1))\n",
    "            \n",
    "            acc = 0\n",
    "            ktCorr = 0\n",
    "            spearmanR = 0\n",
    "            for pred, Y_test1 in zip(preds, Y_test_splits):\n",
    "                acc += mean_squared_error(pred * 3 + 1, Y_test1 * 3 + 1, squared=False)\n",
    "                ktCorr += stats.kendalltau(Y_test1 * 3 + 1, pred * 3 + 1).correlation\n",
    "                spearmanR += stats.spearmanr(Y_test1 * 3 + 1, pred * 3 + 1).correlation\n",
    "                \n",
    "            return case_name, *list(case.values()), acc/num_of_splits, ktCorr/num_of_splits, spearmanR/num_of_splits\n",
    "    \n",
    "    @classmethod\n",
    "    def execute_all_supervised_scenarios(cls,\n",
    "                emb_list, basis_list, embed_dict_master, new_embed_dict_master, \n",
    "                eval_dataset, \n",
    "                scenario_name: str,\n",
    "                num_of_splits = 10,\n",
    "                comb_mode: bool = False, SVC_or_SVR: str = 'SVC', \n",
    "                num_of_iterations = 2,\n",
    "                num_of_jobs = 1\n",
    "            ):\n",
    "        '''\n",
    "        Main invoker function calling `execute_supervised_scenario` for all scenarios as specified\n",
    "        This uses multithreading for parallel execution of scenarios\n",
    "        Inputs:\n",
    "            - emb_list: list of strings - list of embedding sets to be considered\n",
    "            - basis_list: list of strings - list of basis dictionaries to be considered\n",
    "            - embed_dict_master: dict - sets of all embeddings\n",
    "            - neigh_dict_master: dict - sets of all neighboring dictionaries\n",
    "            - eval_dataset: Pandas DataFrame - evaluation benchmark dataset on which performance is to be evaluated\n",
    "            - scenario_name: string - indicating all details about the basis, embedding, weight case, iteration\n",
    "            - num_of_splits: int (default 10) - indicates the k in k-fold cross validation\n",
    "            - comb_mode: bool (default False) - \n",
    "                If True concatenate the 2 embeddings in each row of the benchmark dataset for SVM evaluation \n",
    "                else use the similarity score wrt the 2 embeddings in each row of the dataset for SVM evaluation\n",
    "            - SVC_or_SVR: string (SVC/SVR) - indicate whether you want classification or regression results\n",
    "            - num_of_iterations: int (default 2) - max no. of iterations to be considered for supervised training\n",
    "            - num_of_jobs: int (default 1) - no. of executions to be done in parallel\n",
    "        '''\n",
    "        \n",
    "        if not(comb_mode):\n",
    "            svm_cases_list = []\n",
    "            for basis in basis_list:\n",
    "                for emb in emb_list:\n",
    "                    for weightedness in [True]:\n",
    "                        for iter_num in range(0,num_of_iterations+1):\n",
    "                            for weight_case in [1]:\n",
    "                                for svm_input in ['score']:\n",
    "                                    temp_dict = {'basis': basis, 'emb': emb, 'weightedness': weightedness, \n",
    "                                                'iter_num': iter_num, 'weight_case': weight_case, 'svm_input': svm_input}\n",
    "                                    svm_cases_list.append(temp_dict) \n",
    "        else:\n",
    "            svm_cases_list = []\n",
    "            for basis in basis_list:\n",
    "                for emb in emb_list:\n",
    "                    for weightedness in [True]:\n",
    "                        for iter_num in range(0,num_of_iterations+1):\n",
    "                            for weight_case in [1]:\n",
    "                                for svm_input in ['score']:\n",
    "                                    for i in range(1,len(emb_list)+1):\n",
    "                                        for emb_comb in combinations(emb_list, i):\n",
    "                                            temp_dict = {'basis': basis, 'emb': emb_comb, 'weightedness': weightedness, \n",
    "                                                        'iter_num': iter_num, 'weight_case': weight_case, 'svm_input': svm_input}\n",
    "                                            svm_cases_list.append(temp_dict) \n",
    "\n",
    "        results = Parallel(n_jobs=num_of_jobs)(delayed(cls.execute_supervised_scenario)(\n",
    "                eval_dataset, caseDict, embed_dict_master, \n",
    "                new_embed_dict_master,num_of_splits,\n",
    "                comb_mode, SVC_or_SVR\n",
    "            ) for caseDict in tqdm(svm_cases_list))\n",
    "        \n",
    "        if SVC_or_SVR == 'SVC':\n",
    "            results_df = pd.DataFrame(results, columns=['Case Name','Basis','Embedding','Weightedness', 'Iteration Num', 'Weight Case', 'Technique','Accuracy', 'F1'])\n",
    "#             best_results_df = ResultMetrics.fetch_best_result_for_emb(results_df, 'Embedding', 'Accuracy', 'Iteration Num', highest=True)\n",
    "#             best_results_df.to_csv('../data/retrofitting/retro_SVC_results.' + scenario_name + '.'+ Utils.today_date +'.best.csv', index=False)\n",
    "        else:\n",
    "            results_df = pd.DataFrame(results, columns=['Case Name','Basis','Embedding','Weightedness', 'Iteration Num', 'Weight Case', 'Technique','MSE', 'KT Correlation', 'SR Correlation'])\n",
    "#             best_results_df = ResultMetrics.fetch_best_result_for_emb(results_df, 'Embedding', 'MSE', 'Iteration Num', highest=False)\n",
    "#             best_results_df.to_csv('../data/retrofitting/retro_SVR_results.' + scenario_name + '.'+ Utils.today_date +'.best.csv', index=False)\n",
    "            \n",
    "        results_df.to_csv('../data/retrofitting/retro_SVM_results.' + scenario_name + '.'+ Utils.today_date +'.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "informed-voluntary",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "retained-frederick",
   "metadata": {},
   "source": [
    "# The Master Controller"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "south-strand",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataset: Wordsim-353\n",
      "M    220\n",
      "U    103\n",
      "I     11\n",
      "Name: category, dtype: int64\n",
      "Dataset: Wordsim-353 OLD\n",
      "M    280\n",
      "U     44\n",
      "I     25\n",
      "Name: category, dtype: int64\n",
      "Dataset: DBPedia MC 30\n",
      "M    11\n",
      "U     4\n",
      "I     1\n",
      "Name: category, dtype: int64\n",
      "Dataset: DBPedia RG 65\n",
      "M    26\n",
      "U     6\n",
      "I     2\n",
      "Name: category, dtype: int64\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Input Embeddings:   0%|          | 0/8 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "OG Coverage of text_7_props: 334\n",
      "Added 0 corrections to text_7_props\n",
      "OG Coverage of complex: 334\n",
      "Added 0 corrections to complex\n",
      "OG Coverage of transe: 334\n",
      "Added 0 corrections to transe\n",
      "OG Coverage of abstract_first_sent: 334\n",
      "Added 0 corrections to abstract_first_sent\n",
      "OG Coverage of labels: 334\n",
      "Added 0 corrections to labels\n",
      "OG Coverage of labels_n_desc: 334\n",
      "Added 0 corrections to labels_n_desc\n",
      "OG Coverage of has_h: 327\n",
      "Added 11 corrections to has_h\n",
      "OG Coverage of has_s: 226\n",
      "Added 96 corrections to has_s\n",
      "Embedding: text_7_props, Size: 241696, Length: 1024\n",
      "Embedding: complex, Size: 241698, Length: 100\n",
      "Embedding: transe, Size: 241698, Length: 100\n",
      "Embedding: abstract_first_sent, Size: 241698, Length: 768\n",
      "Embedding: labels, Size: 241698, Length: 768\n",
      "Embedding: labels_n_desc, Size: 241698, Length: 768\n",
      "Embedding: has_h, Size: 166212, Length: 200\n",
      "Embedding: has_s, Size: 117089, Length: 200\n",
      "Fetched all input embeddings\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Neighbor Datasets:   0%|          | 0/4 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fetched neighbour datasets: ['bert_child_par', 'bert_siblings', 'bert_all', 'probase']\n",
      "CPU times: user 10min 14s, sys: 47.4 s, total: 11min 1s\n",
      "Wall time: 11min 28s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "new_embed_dict_master, responses_dict_master = {}, {}\n",
    "\n",
    "# # Load all supporting files\n",
    "evalD = EvaluationDatasets()\n",
    "inp = InputEmbeddings()\n",
    "basis = NeighborDatasets()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "id": "prepared-browse",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "435e91b7c612454c9bd52535b81a6d89",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/8 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "inp_tsne = ReducedInputEmbeddings(inp.embed_dict_master, 100)\n",
    "conc_emb_dict = inp_tsne.generate_concatenated_embedding_dict(list(set(inp.embed_dict_master.keys()) - set(['labels', 'labels_n_desc'])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "id": "geological-class",
   "metadata": {},
   "outputs": [],
   "source": [
    "json.dump(Utils.serialize_embedding_dict(conc_emb_dict), open(INPUT_EMB_FOLDER_PATH + 'concatenated_orig_embedding_dict.json', 'w'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "id": "circular-monte",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'covered_pairs': 349,\n",
       " 'accuracy': 42.693409742120345,\n",
       " 'classification_report': {'I': {'precision': 0.8888888888888888,\n",
       "   'recall': 0.32,\n",
       "   'f1-score': 0.47058823529411764,\n",
       "   'support': 25},\n",
       "  'M': {'precision': 0.8347107438016529,\n",
       "   'recall': 0.3607142857142857,\n",
       "   'f1-score': 0.5037406483790524,\n",
       "   'support': 280},\n",
       "  'U': {'precision': 0.182648401826484,\n",
       "   'recall': 0.9090909090909091,\n",
       "   'f1-score': 0.30418250950570336,\n",
       "   'support': 44},\n",
       "  'accuracy': 0.4269340974212034,\n",
       "  'macro avg': {'precision': 0.6354160115056753,\n",
       "   'recall': 0.5299350649350649,\n",
       "   'f1-score': 0.4261704643929578,\n",
       "   'support': 349},\n",
       "  'weighted avg': {'precision': 0.756383266954299,\n",
       "   'recall': 0.4269340974212034,\n",
       "   'f1-score': 0.47620664139466634,\n",
       "   'support': 349}},\n",
       " 'conf_matrix': array([[  8,  16,   1],\n",
       "        [  1, 101, 178],\n",
       "        [  0,   4,  40]]),\n",
       " 'KT': 0.4387071263112683,\n",
       " 'SR': 0.6180061306656681,\n",
       " 'RMSE': 0.8573774932552949,\n",
       " 'increase_acc': None}"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "responses_dict, _ = ResultMetrics.compute_classification_results(\n",
    "                            conc_emb_dict, evalD.old_wordsim_df, get_output_values=False, old_accuracy=None)\n",
    "responses_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "id": "freelance-daughter",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'covered_pairs': 334,\n",
       " 'accuracy': 53.293413173652695,\n",
       " 'classification_report': {'I': {'precision': 0.0,\n",
       "   'recall': 0.0,\n",
       "   'f1-score': 0.0,\n",
       "   'support': 11},\n",
       "  'M': {'precision': 0.782608695652174,\n",
       "   'recall': 0.4090909090909091,\n",
       "   'f1-score': 0.537313432835821,\n",
       "   'support': 220},\n",
       "  'U': {'precision': 0.4018264840182648,\n",
       "   'recall': 0.8543689320388349,\n",
       "   'f1-score': 0.5465838509316769,\n",
       "   'support': 103},\n",
       "  'accuracy': 0.5329341317365269,\n",
       "  'macro avg': {'precision': 0.3948117265568129,\n",
       "   'recall': 0.42115328037658134,\n",
       "   'f1-score': 0.3612990945891659,\n",
       "   'support': 334},\n",
       "  'weighted avg': {'precision': 0.6394073080759268,\n",
       "   'recall': 0.5329341317365269,\n",
       "   'f1-score': 0.5224763229636028,\n",
       "   'support': 334}},\n",
       " 'conf_matrix': array([[  0,  10,   1],\n",
       "        [  0,  90, 130],\n",
       "        [  0,  15,  88]]),\n",
       " 'KT': 0.43650705554014996,\n",
       " 'SR': 0.5871457253915232,\n",
       " 'RMSE': 0.706830860671417,\n",
       " 'increase_acc': None}"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "responses_dict, _ = ResultMetrics.compute_classification_results(\n",
    "                            conc_emb_dict, evalD.wordsim_df, get_output_values=False, old_accuracy=None)\n",
    "responses_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "id": "norwegian-custom",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'covered_pairs': 16,\n",
       " 'accuracy': 31.25,\n",
       " 'classification_report': {'I': {'precision': 0.0,\n",
       "   'recall': 0.0,\n",
       "   'f1-score': 0.0,\n",
       "   'support': 1},\n",
       "  'M': {'precision': 0.5,\n",
       "   'recall': 0.2727272727272727,\n",
       "   'f1-score': 0.3529411764705882,\n",
       "   'support': 11},\n",
       "  'U': {'precision': 0.2,\n",
       "   'recall': 0.5,\n",
       "   'f1-score': 0.28571428571428575,\n",
       "   'support': 4},\n",
       "  'accuracy': 0.3125,\n",
       "  'macro avg': {'precision': 0.2333333333333333,\n",
       "   'recall': 0.25757575757575757,\n",
       "   'f1-score': 0.21288515406162464,\n",
       "   'support': 16},\n",
       "  'weighted avg': {'precision': 0.39375,\n",
       "   'recall': 0.3125,\n",
       "   'f1-score': 0.31407563025210083,\n",
       "   'support': 16}},\n",
       " 'conf_matrix': array([[0, 1, 0],\n",
       "        [0, 3, 8],\n",
       "        [0, 2, 2]]),\n",
       " 'KT': 0.25,\n",
       " 'SR': 0.4088235294117647,\n",
       " 'RMSE': 0.9704384065378439,\n",
       " 'increase_acc': None}"
      ]
     },
     "execution_count": 125,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "responses_dict, _ = ResultMetrics.compute_classification_results(\n",
    "                            conc_emb_dict, evalD.dbpedia_mc_30_df, get_output_values=False, old_accuracy=None)\n",
    "responses_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "id": "freelance-pepper",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'covered_pairs': 34,\n",
       " 'accuracy': 44.11764705882353,\n",
       " 'classification_report': {'I': {'precision': 0.0,\n",
       "   'recall': 0.0,\n",
       "   'f1-score': 0.0,\n",
       "   'support': 2},\n",
       "  'M': {'precision': 0.7692307692307693,\n",
       "   'recall': 0.38461538461538464,\n",
       "   'f1-score': 0.5128205128205128,\n",
       "   'support': 26},\n",
       "  'U': {'precision': 0.23809523809523808,\n",
       "   'recall': 0.8333333333333334,\n",
       "   'f1-score': 0.37037037037037035,\n",
       "   'support': 6},\n",
       "  'accuracy': 0.4411764705882353,\n",
       "  'macro avg': {'precision': 0.33577533577533575,\n",
       "   'recall': 0.40598290598290604,\n",
       "   'f1-score': 0.2943969610636277,\n",
       "   'support': 34},\n",
       "  'weighted avg': {'precision': 0.6302521008403361,\n",
       "   'recall': 0.4411764705882353,\n",
       "   'f1-score': 0.457516339869281,\n",
       "   'support': 34}},\n",
       " 'conf_matrix': array([[ 0,  2,  0],\n",
       "        [ 0, 10, 16],\n",
       "        [ 0,  1,  5]]),\n",
       " 'KT': 0.3038438086632597,\n",
       " 'SR': 0.43203179862117314,\n",
       " 'RMSE': 0.8817551071220816,\n",
       " 'increase_acc': None}"
      ]
     },
     "execution_count": 126,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "responses_dict, _ = ResultMetrics.compute_classification_results(\n",
    "                            conc_emb_dict, evalD.dbpedia_rg_65_df, get_output_values=False, old_accuracy=None)\n",
    "responses_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "alleged-depth",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "banned-handling",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "flexible-hudson",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a8d976845dd74434b9bfd7c87ae1ac62",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "subset_keys = ['abstract_first_sent', 'has_h']\n",
    "temp_dict = {}\n",
    "for key in subset_keys:\n",
    "    temp_dict[key] = copy.deepcopy(inp.embed_dict_master[key])\n",
    "inp_tsne = ReducedInputEmbeddings(temp_dict, 100)\n",
    "conc_emb_dict = inp_tsne.generate_concatenated_embedding_dict(subset_keys)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "limited-composer",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'covered_pairs': 349,\n",
       " 'accuracy': 60.458452722063036,\n",
       " 'classification_report': {'I': {'precision': 0.7272727272727273,\n",
       "   'recall': 0.32,\n",
       "   'f1-score': 0.4444444444444444,\n",
       "   'support': 25},\n",
       "  'M': {'precision': 0.8858695652173914,\n",
       "   'recall': 0.5821428571428572,\n",
       "   'f1-score': 0.7025862068965518,\n",
       "   'support': 280},\n",
       "  'U': {'precision': 0.2597402597402597,\n",
       "   'recall': 0.9090909090909091,\n",
       "   'f1-score': 0.40404040404040403,\n",
       "   'support': 44},\n",
       "  'accuracy': 0.6045845272206304,\n",
       "  'macro avg': {'precision': 0.6242941840767928,\n",
       "   'recall': 0.6037445887445888,\n",
       "   'f1-score': 0.5170236851271334,\n",
       "   'support': 349},\n",
       "  'weighted avg': {'precision': 0.7955698219806854,\n",
       "   'recall': 0.6045845272206304,\n",
       "   'f1-score': 0.6464556642404682,\n",
       "   'support': 349}},\n",
       " 'conf_matrix': array([[  8,  17,   0],\n",
       "        [  3, 163, 114],\n",
       "        [  0,   4,  40]]),\n",
       " 'KT': 0.449925164929108,\n",
       " 'SR': 0.6314889524026878,\n",
       " 'RMSE': 0.7098851741322992,\n",
       " 'increase_acc': None}"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "responses_dict, _ = ResultMetrics.compute_classification_results(\n",
    "                            conc_emb_dict, evalD.old_wordsim_df, get_output_values=False, old_accuracy=None)\n",
    "responses_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "junior-candy",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'covered_pairs': 334,\n",
       " 'accuracy': 67.96407185628742,\n",
       " 'classification_report': {'I': {'precision': 0.0,\n",
       "   'recall': 0.0,\n",
       "   'f1-score': 0.0,\n",
       "   'support': 11},\n",
       "  'M': {'precision': 0.8202247191011236,\n",
       "   'recall': 0.6636363636363637,\n",
       "   'f1-score': 0.7336683417085428,\n",
       "   'support': 220},\n",
       "  'U': {'precision': 0.525974025974026,\n",
       "   'recall': 0.7864077669902912,\n",
       "   'f1-score': 0.6303501945525293,\n",
       "   'support': 103},\n",
       "  'accuracy': 0.6796407185628742,\n",
       "  'macro avg': {'precision': 0.44873291502504986,\n",
       "   'recall': 0.48334804354221833,\n",
       "   'f1-score': 0.4546728454203574,\n",
       "   'support': 334},\n",
       "  'weighted avg': {'precision': 0.7024693499328499,\n",
       "   'recall': 0.6796407185628742,\n",
       "   'f1-score': 0.6776440275891913,\n",
       "   'support': 334}},\n",
       " 'conf_matrix': array([[  0,  10,   1],\n",
       "        [  2, 146,  72],\n",
       "        [  0,  22,  81]]),\n",
       " 'KT': 0.48777318852355533,\n",
       " 'SR': 0.6543303283611083,\n",
       " 'RMSE': 0.5724958150747688,\n",
       " 'increase_acc': None}"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "responses_dict, _ = ResultMetrics.compute_classification_results(\n",
    "                            conc_emb_dict, evalD.wordsim_df, get_output_values=False, old_accuracy=None)\n",
    "responses_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "adapted-butter",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'covered_pairs': 16,\n",
       " 'accuracy': 62.5,\n",
       " 'classification_report': {'I': {'precision': 0.0,\n",
       "   'recall': 0.0,\n",
       "   'f1-score': 0.0,\n",
       "   'support': 1},\n",
       "  'M': {'precision': 0.7272727272727273,\n",
       "   'recall': 0.7272727272727273,\n",
       "   'f1-score': 0.7272727272727273,\n",
       "   'support': 11},\n",
       "  'U': {'precision': 0.4,\n",
       "   'recall': 0.5,\n",
       "   'f1-score': 0.4444444444444445,\n",
       "   'support': 4},\n",
       "  'accuracy': 0.625,\n",
       "  'macro avg': {'precision': 0.3757575757575758,\n",
       "   'recall': 0.4090909090909091,\n",
       "   'f1-score': 0.39057239057239057,\n",
       "   'support': 16},\n",
       "  'weighted avg': {'precision': 0.6,\n",
       "   'recall': 0.625,\n",
       "   'f1-score': 0.6111111111111112,\n",
       "   'support': 16}},\n",
       " 'conf_matrix': array([[0, 1, 0],\n",
       "        [0, 8, 3],\n",
       "        [0, 2, 2]]),\n",
       " 'KT': 0.45,\n",
       " 'SR': 0.5852941176470587,\n",
       " 'RMSE': 0.7296263062335091,\n",
       " 'increase_acc': None}"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "responses_dict, _ = ResultMetrics.compute_classification_results(\n",
    "                            conc_emb_dict, evalD.dbpedia_mc_30_df, get_output_values=False, old_accuracy=None)\n",
    "responses_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "scientific-reservation",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'covered_pairs': 34,\n",
       " 'accuracy': 64.70588235294117,\n",
       " 'classification_report': {'I': {'precision': 0.0,\n",
       "   'recall': 0.0,\n",
       "   'f1-score': 0.0,\n",
       "   'support': 2},\n",
       "  'M': {'precision': 0.8181818181818182,\n",
       "   'recall': 0.6923076923076923,\n",
       "   'f1-score': 0.7500000000000001,\n",
       "   'support': 26},\n",
       "  'U': {'precision': 0.3333333333333333,\n",
       "   'recall': 0.6666666666666666,\n",
       "   'f1-score': 0.4444444444444444,\n",
       "   'support': 6},\n",
       "  'accuracy': 0.6470588235294118,\n",
       "  'macro avg': {'precision': 0.38383838383838387,\n",
       "   'recall': 0.452991452991453,\n",
       "   'f1-score': 0.3981481481481482,\n",
       "   'support': 34},\n",
       "  'weighted avg': {'precision': 0.6844919786096257,\n",
       "   'recall': 0.6470588235294118,\n",
       "   'f1-score': 0.6519607843137256,\n",
       "   'support': 34}},\n",
       " 'conf_matrix': array([[ 0,  2,  0],\n",
       "        [ 0, 18,  8],\n",
       "        [ 0,  2,  4]]),\n",
       " 'KT': 0.40750816691307773,\n",
       " 'SR': 0.5159318543137886,\n",
       " 'RMSE': 0.7176324851002853,\n",
       " 'increase_acc': None}"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "responses_dict, _ = ResultMetrics.compute_classification_results(\n",
    "                            conc_emb_dict, evalD.dbpedia_rg_65_df, get_output_values=False, old_accuracy=None)\n",
    "responses_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "removed-charity",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "governmental-yeast",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# res_df[res_df.abstract_first_sent >= 3.9]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "id": "neither-ambassador",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fetching wordsim_new wordsim score tables and eval file\n",
      "Returning averaged scores from 9 algorithms - ['has_h', 'complex', 'classSim', 'topSim', 'transe', 'has_s', 'JC', 'text_7_props', 'abstract_first_sent']\n",
      "Quantiles being used by wordsim: [-inf, 2.8, 3.0, 3.6, inf]\n",
      "Quantiles being used by classSim: [-inf, 3.3613822255703725, 3.809400978521772, 3.957763635353713, inf]\n",
      "Quantiles being used by JC: [-inf, 2.8351504372328686, 3.503981920452832, 3.7887034134721307, inf]\n",
      "Quantiles being used by topSim: [-inf, 2.446182786605355, 2.783429730161016, 2.9719271938105174, inf]\n",
      "Quantiles being used by text_7_props: [-inf, 1.9650170825691686, 2.2566934546159736, 2.559627663334526, inf]\n",
      "Quantiles being used by complex: [-inf, 2.3927659996609845, 2.599180941250367, 2.8733883637473148, inf]\n",
      "Quantiles being used by transe: [-inf, 2.617148987716268, 3.003437976104153, 3.330058848001383, inf]\n",
      "Quantiles being used by abstract_first_sent: [-inf, 2.947749962401879, 3.3795752703868414, 3.68130819302516, inf]\n",
      "Quantiles being used by labels: [-inf, 3.1803031793989787, 3.358407141292364, 3.4953677794287143, inf]\n",
      "Quantiles being used by labels_n_desc: [-inf, 3.10791697124227, 3.492018772048467, 3.713840109637744, inf]\n",
      "Quantiles being used by has_h: [-inf, 2.4939370764800715, 2.7830003954956934, 3.1044453710327633, inf]\n",
      "Quantiles being used by has_s: [-inf, 3.6532584882855805, 3.766277049131519, 3.8578288730135064, inf]\n",
      "Quantiles being used by average: [-inf, 2.7909952054316762, 3.0593728660061075, 3.2363275209135374, inf]\n",
      "Fetching wordsim_new wordsim score tables and eval file\n",
      "Returning averaged scores from 9 algorithms - ['has_h', 'complex', 'classSim', 'topSim', 'transe', 'has_s', 'JC', 'text_7_props', 'abstract_first_sent']\n",
      "Fetching wordsim_old wordsim score tables and eval file\n",
      "Returning averaged scores from 9 algorithms - ['has_h', 'complex', 'classSim', 'topSim', 'transe', 'has_s', 'JC', 'text_7_props', 'abstract_first_sent']\n",
      "Fetching dbpedia_mc_30 wordsim score tables and eval file\n",
      "Returning averaged scores from 9 algorithms - ['has_h', 'complex', 'classSim', 'topSim', 'transe', 'has_s', 'JC', 'text_7_props', 'abstract_first_sent']\n",
      "Fetching dbpedia_rg_65 wordsim score tables and eval file\n",
      "Returning averaged scores from 9 algorithms - ['has_h', 'complex', 'classSim', 'topSim', 'transe', 'has_s', 'JC', 'text_7_props', 'abstract_first_sent']\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Basis:   0%|          | 0/4 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Embedding:   0%|          | 0/8 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Embedding:   0%|          | 0/8 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Embedding:   0%|          | 0/8 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Embedding:   0%|          | 0/8 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Analysed wordsim_ind\n",
      "Fetching wordsim_old wordsim score tables and eval file\n",
      "Returning averaged scores from 9 algorithms - ['has_h', 'classSim', 'topSim', 'text_7_props', 'complex', 'transe', 'has_s', 'JC', 'abstract_first_sent']\n",
      "Returning averaged scores from 6 algorithms - ['complex_retrofitted', 'text_7_props_retrofitted', 'has_h_retrofitted', 'transe_retrofitted', 'abstract_first_sent_retrofitted', 'has_s_retrofitted']\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e7480ac3f6aa4af1864bb3065b47c72d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/96 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Analysed SVC_Wordsim\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a4abc0d4a3394d5f8945064c5567105f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/96 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Analysed SVR_Wordsim\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Basis:   0%|          | 0/4 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Embedding:   0%|          | 0/8 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Embedding:   0%|          | 0/8 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Embedding:   0%|          | 0/8 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Embedding:   0%|          | 0/8 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Analysed dbpedia_mc_30_ind\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f31cd314bd7b43b597fba463afa427b8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/96 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Analysed SVR_dbpedia_mc_30\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Basis:   0%|          | 0/4 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Embedding:   0%|          | 0/8 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Embedding:   0%|          | 0/8 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Embedding:   0%|          | 0/8 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Embedding:   0%|          | 0/8 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Weight Case:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Analysed dbpedia_rg_65_ind\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a940ace7cb8d47fcb4593e41cacdcc24",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/96 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Analysed SVR_dbpedia_rg_65\n",
      "CPU times: user 7min, sys: 36 s, total: 7min 36s\n",
      "Wall time: 7min 35s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "start_master_time = time()\n",
    "\n",
    "ist = InputScoreTables(inp.embed_dict_master, set(['labels', 'labels_n_desc']), eval_file='wordsim_new')\n",
    "_,res_df = ResultMetrics.compute_classification_n_regression_stats(ist, 'wsim_quantiles', standard_labels=False)\n",
    "res_df.to_csv('../data/retrofitting/wordsim_quantile_analysis.'+ Utils.today_date +'.csv', index=False)\n",
    "\n",
    "ist = InputScoreTables(inp.embed_dict_master, set(['labels', 'labels_n_desc']), eval_file='wordsim_new')\n",
    "_,res_df = ResultMetrics.compute_classification_n_regression_stats(ist, 'wsim_orig', standard_labels=True)\n",
    "res_df.to_csv('../data/retrofitting/wordsim_all_algo_scores.'+ Utils.today_date +'.csv', index=False)\n",
    "\n",
    "ist = InputScoreTables(inp.embed_dict_master, set(['labels', 'labels_n_desc']), eval_file='wordsim_old')\n",
    "_,res_df = ResultMetrics.compute_classification_n_regression_stats(ist, 'wsim_old', standard_labels=True)\n",
    "res_df.to_csv('../data/retrofitting/wordsim_all_algo_scores.wsim_old.'+ Utils.today_date +'.csv', index=False)\n",
    "\n",
    "ist = InputScoreTables(inp.embed_dict_master, set(['labels', 'labels_n_desc']), eval_file='dbpedia_mc_30')\n",
    "_,res_df = ResultMetrics.compute_classification_n_regression_stats(ist, 'dbpedia_mc_30', standard_labels=True)\n",
    "res_df.to_csv('../data/retrofitting/dbpedia_mc_30_all_algo_scores.'+ Utils.today_date +'.csv', index=False)\n",
    "\n",
    "ist = InputScoreTables(inp.embed_dict_master, set(['labels', 'labels_n_desc']), eval_file='dbpedia_rg_65')\n",
    "_,res_df = ResultMetrics.compute_classification_n_regression_stats(ist, 'dbpedia_rg_65', standard_labels=True)\n",
    "res_df.to_csv('../data/retrofitting/dbpedia_rg_65_all_algo_scores.'+ Utils.today_date +'.csv', index=False)\n",
    "\n",
    "\n",
    "# Wordsim executions\n",
    "new_embed_dict_master['wordsim'], responses_dict_master['wordsim'] = RetrofittingProcedures.execute_all_unsupervised_scenarios(inp.emb_list, basis.basis_list, inp.embed_dict_master, basis.neighbors_dict_master, \n",
    "                                                                            evalD.wordsim_df, \"wordsim_ind\")\n",
    "\n",
    "print(\"Analysed wordsim_ind\")\n",
    "\n",
    "ist = InputScoreTables(inp.embed_dict_master, set(['labels', 'labels_n_desc']), eval_file='wordsim_new', new_embed_dict_master=new_embed_dict_master['wordsim'])\n",
    "_,res_df = ResultMetrics.compute_classification_n_regression_stats(ist, 'wsim_orig_retro', standard_labels=True)\n",
    "res_df.to_csv('../data/retrofitting/wordsim_all_algo_scores.wsim_orig_retro.'+ Utils.today_date +'.csv', index=False)\n",
    "\n",
    "SVMProcedures.execute_all_supervised_scenarios(inp.emb_list, basis.basis_list, inp.embed_dict_master, \n",
    "                                            new_embed_dict_master['wordsim'], \n",
    "                                            evalD.wordsim_df, \"SVC_Wordsim\",\n",
    "                                            comb_mode = False, SVC_or_SVR = 'SVC')\n",
    "print(\"Analysed SVC_Wordsim\")\n",
    "\n",
    "SVMProcedures.execute_all_supervised_scenarios(inp.emb_list, basis.basis_list, inp.embed_dict_master, \n",
    "                                            new_embed_dict_master['wordsim'], \n",
    "                                            evalD.wordsim_df, \"SVR_Wordsim\",\n",
    "                                            comb_mode = False, SVC_or_SVR = 'SVR')\n",
    "print(\"Analysed SVR_Wordsim\")\n",
    "\n",
    "# RetrofittingProcedures.save_all_embeddings(new_embed_dict_master['wordsim'])\n",
    "\n",
    "# SVMProcedures.execute_all_supervised_scenarios(inp.emb_list, basis.basis_list, inp.embed_dict_master, \n",
    "#                                             new_embed_dict_master['wordsim'], \n",
    "#                                             evalD.wordsim_df, \"SVR_Wordsim\",\n",
    "#                                             comb_mode = True, SVC_or_SVR = 'SVR')\n",
    "# print(\"Analysed SVR_Wordsim combinatrics\")\n",
    "\n",
    "# new_embed_dict_master, responses_dict_master = {}, {}\n",
    "\n",
    "\n",
    "\n",
    "new_embed_dict_master['dbpedia_mc_30'], responses_dict_master['dbpedia_mc_30'] = RetrofittingProcedures.execute_all_unsupervised_scenarios(inp.emb_list, basis.basis_list, inp.embed_dict_master, basis.neighbors_dict_master, \n",
    "                                                                            evalD.dbpedia_mc_30_df, \"dbpedia_mc_30_ind\", prev_new_embed_dict_master=new_embed_dict_master['wordsim'])\n",
    "print(\"Analysed dbpedia_mc_30_ind\")\n",
    "SVMProcedures.execute_all_supervised_scenarios(inp.emb_list, basis.basis_list, inp.embed_dict_master, \n",
    "                                            new_embed_dict_master['dbpedia_mc_30'], \n",
    "                                            evalD.dbpedia_mc_30_df, \"SVR_dbpedia_mc_30\",\n",
    "                                            comb_mode = False, SVC_or_SVR = 'SVR')\n",
    "print(\"Analysed SVR_dbpedia_mc_30\")\n",
    "\n",
    "new_embed_dict_master['dbpedia_rg_65'], responses_dict_master['dbpedia_rg_65'] = RetrofittingProcedures.execute_all_unsupervised_scenarios(inp.emb_list, basis.basis_list, inp.embed_dict_master, basis.neighbors_dict_master, \n",
    "                                                                            evalD.dbpedia_rg_65_df, \"dbpedia_rg_65\", prev_new_embed_dict_master=new_embed_dict_master['wordsim'])\n",
    "print(\"Analysed dbpedia_rg_65_ind\")\n",
    "SVMProcedures.execute_all_supervised_scenarios(inp.emb_list, basis.basis_list, inp.embed_dict_master, \n",
    "                                            new_embed_dict_master['dbpedia_rg_65'], \n",
    "                                            evalD.dbpedia_rg_65_df, \"SVR_dbpedia_rg_65\",\n",
    "                                            comb_mode = False, SVC_or_SVR = 'SVR')\n",
    "print(\"Analysed SVR_dbpedia_rg_65\")\n",
    "                                                                                                                               \n",
    "                                                                                                                               \n",
    "# # Wiki CS executions\n",
    "# new_embed_dict_master['wiki_cs'], responses_dict_master['wiki_cs'] = RetrofittingProcedures.execute_all_unsupervised_scenarios(inp.emb_list, basis.basis_list, inp.embed_dict_master, basis.neighbors_dict_master, \n",
    "#                                                           evalD.wiki_cs_df, \"wiki_cs_ind\")\n",
    "# print(\"Analysed wiki_cs_ind\")\n",
    "\n",
    "# SVMProcedures.execute_all_supervised_scenarios(inp.emb_list, basis.basis_list, inp.embed_dict_master,\n",
    "#                                             new_embed_dict_master['wiki_cs'],\n",
    "#                                             evalD.wordsim_df, \"SVC_Wiki_CS\",\n",
    "#                                             comb_mode = False, SVC_or_SVR = 'SVC')\n",
    "# print(\"Analysed SVC_Wiki_CS\")\n",
    "\n",
    "                                                                                                                               \n",
    "                                                                                                                               \n",
    "                                                                                                                               \n",
    "# # Conceptnet executions\n",
    "# new_embed_dict_master['conceptnet'], responses_dict_master['conceptnet'] = RetrofittingProcedures.execute_all_unsupervised_scenarios(inp.emb_list, basis.basis_list, inp.embed_dict_master, basis.neighbors_dict_master, \n",
    "#                                                           evalD.concept_net_df, \"concept_net_ind\")\n",
    "# print(\"Analysed concept_net_ind\")\n",
    "\n",
    "# SVMProcedures.execute_all_supervised_scenarios(inp.emb_list, basis.basis_list, inp.embed_dict_master,\n",
    "#                                             new_embed_dict_master['conceptnet'],\n",
    "#                                             evalD.wordsim_df, \"SVC_Conceptnet\",\n",
    "#                                             comb_mode = False, SVC_or_SVR = 'SVC')\n",
    "# print(\"Analysed SVC_Conceptnet\")\n",
    "\n",
    "# print(f\"Time taken for end-to-end execution: {time() - start_master_time}s\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "id": "critical-leader",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fetching wordsim_new wordsim score tables and eval file\n",
      "Returning averaged scores from 9 algorithms - ['has_h', 'classSim', 'topSim', 'text_7_props', 'complex', 'transe', 'has_s', 'JC', 'abstract_first_sent']\n",
      "Returning averaged scores from 6 algorithms - ['complex_retrofitted', 'text_7_props_retrofitted', 'has_h_retrofitted', 'transe_retrofitted', 'abstract_first_sent_retrofitted', 'has_s_retrofitted']\n"
     ]
    }
   ],
   "source": [
    "ist = InputScoreTables(inp.embed_dict_master, set(['labels', 'labels_n_desc']), eval_file='wordsim_new', new_embed_dict_master=new_embed_dict_master['wordsim'])\n",
    "_,res_df = ResultMetrics.compute_classification_n_regression_stats(ist, 'wsim_orig_retro', standard_labels=True)\n",
    "res_df.to_csv('../data/retrofitting/wordsim_all_algo_scores.wsim_orig_retro.'+ Utils.today_date +'.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "id": "secure-mainstream",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['text_7_props_bert_child_par_1_weighted', 'complex_bert_child_par_1_weighted', 'transe_bert_child_par_1_weighted', 'abstract_first_sent_bert_child_par_1_weighted', 'labels_bert_child_par_1_weighted', 'labels_n_desc_bert_child_par_1_weighted', 'has_h_bert_child_par_1_weighted', 'has_s_bert_child_par_1_weighted', 'text_7_props_bert_siblings_1_weighted', 'complex_bert_siblings_1_weighted', 'transe_bert_siblings_1_weighted', 'abstract_first_sent_bert_siblings_1_weighted', 'labels_bert_siblings_1_weighted', 'labels_n_desc_bert_siblings_1_weighted', 'has_h_bert_siblings_1_weighted', 'has_s_bert_siblings_1_weighted', 'text_7_props_bert_all_1_weighted', 'complex_bert_all_1_weighted', 'transe_bert_all_1_weighted', 'abstract_first_sent_bert_all_1_weighted', 'labels_bert_all_1_weighted', 'labels_n_desc_bert_all_1_weighted', 'has_h_bert_all_1_weighted', 'has_s_bert_all_1_weighted', 'text_7_props_probase_1_weighted', 'complex_probase_1_weighted', 'transe_probase_1_weighted', 'abstract_first_sent_probase_1_weighted', 'labels_probase_1_weighted', 'labels_n_desc_probase_1_weighted', 'has_h_probase_1_weighted', 'has_s_probase_1_weighted'])"
      ]
     },
     "execution_count": 159,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_embed_dict_master['wordsim'].keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "id": "incident-upset",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fetching wordsim_new wordsim score tables and eval file\n",
      "Returning averaged scores from 9 algorithms - ['has_h', 'classSim', 'topSim', 'text_7_props', 'complex', 'transe', 'has_s', 'JC', 'abstract_first_sent']\n",
      "Returning averaged scores from 6 algorithms - ['complex_retrofitted', 'text_7_props_retrofitted', 'has_h_retrofitted', 'transe_retrofitted', 'abstract_first_sent_retrofitted', 'has_s_retrofitted']\n",
      "Fetching wordsim_new wordsim score tables and eval file\n",
      "Returning averaged scores from 9 algorithms - ['has_h', 'classSim', 'topSim', 'text_7_props', 'complex', 'transe', 'has_s', 'JC', 'abstract_first_sent']\n",
      "Returning averaged scores from 6 algorithms - ['complex_retrofitted', 'text_7_props_retrofitted', 'has_h_retrofitted', 'transe_retrofitted', 'abstract_first_sent_retrofitted', 'has_s_retrofitted']\n",
      "Fetching wordsim_new wordsim score tables and eval file\n",
      "Returning averaged scores from 9 algorithms - ['has_h', 'classSim', 'topSim', 'text_7_props', 'complex', 'transe', 'has_s', 'JC', 'abstract_first_sent']\n",
      "Returning averaged scores from 6 algorithms - ['complex_retrofitted', 'text_7_props_retrofitted', 'has_h_retrofitted', 'transe_retrofitted', 'abstract_first_sent_retrofitted', 'has_s_retrofitted']\n",
      "Fetching wordsim_new wordsim score tables and eval file\n",
      "Returning averaged scores from 9 algorithms - ['has_h', 'classSim', 'topSim', 'text_7_props', 'complex', 'transe', 'has_s', 'JC', 'abstract_first_sent']\n",
      "Returning averaged scores from 6 algorithms - ['complex_retrofitted', 'text_7_props_retrofitted', 'has_h_retrofitted', 'transe_retrofitted', 'abstract_first_sent_retrofitted', 'has_s_retrofitted']\n"
     ]
    }
   ],
   "source": [
    "ist = InputScoreTables(inp.embed_dict_master, set(['labels', 'labels_n_desc']), eval_file='wordsim_new', new_embed_dict_master=new_embed_dict_master['wordsim'], new_embed_suffix='_bert_all_1_weighted')\n",
    "_,res_df = ResultMetrics.compute_classification_n_regression_stats(ist, 'wsim_orig_retro.bert_all', standard_labels=True)\n",
    "res_df.to_csv('../data/retrofitting/wordsim_all_algo_scores.wsim_orig_retro.bert_all.'+ Utils.today_date +'.csv', index=False)\n",
    "\n",
    "ist = InputScoreTables(inp.embed_dict_master, set(['labels', 'labels_n_desc']), eval_file='wordsim_new', new_embed_dict_master=new_embed_dict_master['wordsim'], new_embed_suffix='_bert_siblings_1_weighted')\n",
    "_,res_df = ResultMetrics.compute_classification_n_regression_stats(ist, 'wsim_orig_retro.bert_siblings', standard_labels=True)\n",
    "res_df.to_csv('../data/retrofitting/wordsim_all_algo_scores.wsim_orig_retro.bert_siblings.'+ Utils.today_date +'.csv', index=False)\n",
    "\n",
    "ist = InputScoreTables(inp.embed_dict_master, set(['labels', 'labels_n_desc']), eval_file='wordsim_new', new_embed_dict_master=new_embed_dict_master['wordsim'], new_embed_suffix='_bert_child_par_1_weighted')\n",
    "_,res_df = ResultMetrics.compute_classification_n_regression_stats(ist, 'wsim_orig_retro.bert_child_par', standard_labels=True)\n",
    "res_df.to_csv('../data/retrofitting/wordsim_all_algo_scores.wsim_orig_retro.bert_child_par.'+ Utils.today_date +'.csv', index=False)\n",
    "\n",
    "ist = InputScoreTables(inp.embed_dict_master, set(['labels', 'labels_n_desc']), eval_file='wordsim_new', new_embed_dict_master=new_embed_dict_master['wordsim'], new_embed_suffix='_probase_1_weighted')\n",
    "_,res_df = ResultMetrics.compute_classification_n_regression_stats(ist, 'wsim_orig_retro.probase', standard_labels=True)\n",
    "res_df.to_csv('../data/retrofitting/wordsim_all_algo_scores.wsim_orig_retro.probase.'+ Utils.today_date +'.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "id": "baking-tenant",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fetching dbpedia_rg_65 wordsim score tables and eval file\n",
      "Returning averaged scores from 9 algorithms - ['has_h', 'classSim', 'topSim', 'text_7_props', 'complex', 'transe', 'has_s', 'JC', 'abstract_first_sent']\n",
      "Returning averaged scores from 6 algorithms - ['complex_retrofitted', 'text_7_props_retrofitted', 'has_h_retrofitted', 'transe_retrofitted', 'abstract_first_sent_retrofitted', 'has_s_retrofitted']\n",
      "Fetching dbpedia_mc_30 wordsim score tables and eval file\n",
      "Returning averaged scores from 9 algorithms - ['has_h', 'classSim', 'topSim', 'text_7_props', 'complex', 'transe', 'has_s', 'JC', 'abstract_first_sent']\n",
      "Returning averaged scores from 6 algorithms - ['complex_retrofitted', 'text_7_props_retrofitted', 'has_h_retrofitted', 'transe_retrofitted', 'abstract_first_sent_retrofitted', 'has_s_retrofitted']\n"
     ]
    }
   ],
   "source": [
    "ist = InputScoreTables(inp.embed_dict_master, set(['labels', 'labels_n_desc']), eval_file='dbpedia_rg_65', new_embed_dict_master=new_embed_dict_master['wordsim'])\n",
    "_,res_df = ResultMetrics.compute_classification_n_regression_stats(ist, 'dbpedia_rg_65_retro', standard_labels=True)\n",
    "res_df.to_csv('../data/retrofitting/wordsim_all_algo_scores.dbpedia_rg_65_retro.'+ Utils.today_date +'.csv', index=False)\n",
    "\n",
    "ist = InputScoreTables(inp.embed_dict_master, set(['labels', 'labels_n_desc']), eval_file='dbpedia_mc_30', new_embed_dict_master=new_embed_dict_master['wordsim'])\n",
    "_,res_df = ResultMetrics.compute_classification_n_regression_stats(ist, 'dbpedia_mc_30_retro', standard_labels=True)\n",
    "res_df.to_csv('../data/retrofitting/wordsim_all_algo_scores.dbpedia_mc_30_retro.'+ Utils.today_date +'.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "id": "signed-birmingham",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fetching wordsim_new wordsim score tables and eval file\n",
      "Returning averaged scores from 9 algorithms - ['has_h', 'classSim', 'topSim', 'text_7_props', 'complex', 'transe', 'has_s', 'JC', 'abstract_first_sent']\n",
      "Returning averaged scores from 6 algorithms - ['complex_retrofitted', 'text_7_props_retrofitted', 'has_h_retrofitted', 'transe_retrofitted', 'abstract_first_sent_retrofitted', 'has_s_retrofitted']\n",
      "Quantiles being used by wordsim: [-inf, 2.8, 3.0, 3.6, inf]\n",
      "Quantiles being used by classSim: [-inf, 3.3613822255703725, 3.809400978521772, 3.957763635353713, inf]\n",
      "Quantiles being used by JC: [-inf, 2.8351504372328686, 3.503981920452832, 3.7887034134721307, inf]\n",
      "Quantiles being used by topSim: [-inf, 2.446182786605355, 2.783429730161016, 2.9719271938105174, inf]\n",
      "Quantiles being used by text_7_props: [-inf, 1.9650170825691686, 2.2566934546159736, 2.559627663334526, inf]\n",
      "Quantiles being used by text_7_props_retrofitted: [-inf, 1.762183866514523, 2.0080365311430546, 2.3180250624316017, inf]\n",
      "Quantiles being used by complex: [-inf, 2.3927659996609845, 2.599180941250367, 2.8733883637473148, inf]\n",
      "Quantiles being used by complex_retrofitted: [-inf, 2.0536754340024217, 2.260695707272224, 2.5128493479950333, inf]\n",
      "Quantiles being used by transe: [-inf, 2.617148987716268, 3.003437976104153, 3.330058848001383, inf]\n",
      "Quantiles being used by transe_retrofitted: [-inf, 2.329112837122091, 2.7299604350430027, 3.087581721829718, inf]\n",
      "Quantiles being used by abstract_first_sent: [-inf, 2.947749962401879, 3.3795752703868414, 3.68130819302516, inf]\n",
      "Quantiles being used by abstract_first_sent_retrofitted: [-inf, 2.7416095897589052, 3.2230888519232512, 3.5847932460158303, inf]\n",
      "Quantiles being used by labels: [-inf, 3.1803031793989787, 3.358407141292364, 3.4953677794287143, inf]\n",
      "Quantiles being used by labels_retrofitted: [-inf, 2.7732952298158873, 2.9891637765017074, 3.132839862624656, inf]\n",
      "Quantiles being used by labels_n_desc: [-inf, 3.10791697124227, 3.492018772048467, 3.713840109637744, inf]\n",
      "Quantiles being used by labels_n_desc_retrofitted: [-inf, 2.779412137291035, 3.2304132203248543, 3.5072003299844803, inf]\n",
      "Quantiles being used by has_h: [-inf, 2.4939370764800715, 2.7830003954956934, 3.1044453710327633, inf]\n",
      "Quantiles being used by has_h_retrofitted: [-inf, 2.1919012589955873, 2.5292776775207564, 2.8190277530395202, inf]\n",
      "Quantiles being used by has_s: [-inf, 3.6532584882855805, 3.766277049131519, 3.8578288730135064, inf]\n",
      "Quantiles being used by has_s_retrofitted: [-inf, 2.6833420313220886, 2.9690031368318417, 3.3151083336253895, inf]\n",
      "Quantiles being used by average: [-inf, 2.7909952054316762, 3.0593728660061075, 3.236327520913537, inf]\n",
      "Quantiles being used by average_retrofitted: [-inf, 2.385035646251107, 2.6243786902507553, 2.8364598567486903, inf]\n"
     ]
    }
   ],
   "source": [
    "ist = InputScoreTables(inp.embed_dict_master, set(['labels', 'labels_n_desc']), eval_file='wordsim_new', new_embed_dict_master=new_embed_dict_master['wordsim'], new_embed_suffix='_bert_child_par_1_weighted')\n",
    "_,res_df = ResultMetrics.compute_classification_n_regression_stats(ist, 'wsim_quantiles.retrofit', standard_labels=False)\n",
    "res_df.to_csv('../data/retrofitting/wordsim_quantile_analysis.retrofit.'+ Utils.today_date +'.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "id": "affecting-catholic",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "60ed67791d46446ca6017c0250039563",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/96 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Analysed SVC_Wordsim\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d96c018f742b4194a0bef0a70f9822af",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/96 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Analysed SVC_dbpedia_mc_30\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d563ea703dd445bfa3947464ce30ffcd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/96 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Analysed SVC_dbpedia_rg_65\n"
     ]
    }
   ],
   "source": [
    "SVMProcedures.execute_all_supervised_scenarios(inp.emb_list, basis.basis_list, inp.embed_dict_master, \n",
    "                                            new_embed_dict_master['wordsim'], \n",
    "                                            evalD.wordsim_df, \"SVC_Wordsim\",\n",
    "                                            comb_mode = False, SVC_or_SVR = 'SVC')\n",
    "print(\"Analysed SVC_Wordsim\")\n",
    "\n",
    "SVMProcedures.execute_all_supervised_scenarios(inp.emb_list, basis.basis_list, inp.embed_dict_master, \n",
    "                                            new_embed_dict_master['dbpedia_mc_30'], \n",
    "                                            evalD.dbpedia_mc_30_df, \"SVC_dbpedia_mc_30\",\n",
    "                                            comb_mode = False, SVC_or_SVR = 'SVC')\n",
    "print(\"Analysed SVC_dbpedia_mc_30\")\n",
    "\n",
    "SVMProcedures.execute_all_supervised_scenarios(inp.emb_list, basis.basis_list, inp.embed_dict_master, \n",
    "                                            new_embed_dict_master['dbpedia_rg_65'], \n",
    "                                            evalD.dbpedia_rg_65_df, \"SVC_dbpedia_rg_65\",\n",
    "                                            comb_mode = False, SVC_or_SVR = 'SVC')\n",
    "print(\"Analysed SVC_dbpedia_rg_65\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "champion-memory",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "hundred-virus",
   "metadata": {},
   "source": [
    "# Evaluation Datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "limited-hormone",
   "metadata": {},
   "outputs": [],
   "source": [
    "bioDF = pd.read_csv('../data/pedersen2007measures_table1.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "alien-happiness",
   "metadata": {},
   "outputs": [],
   "source": [
    "bioDF.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "parental-event",
   "metadata": {},
   "outputs": [],
   "source": [
    "bioDFNodesSet = set(bioDF.Term1_kg_id.to_list() + bioDF.Term2_kg_id.to_list())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "wicked-proportion",
   "metadata": {},
   "outputs": [],
   "source": [
    "P279childParNodesSet = set(p279WordSimSeededDF_wabs_text.node1.to_list() + p279WordSimSeededDF_wabs_text.node2.to_list())\n",
    "P279siblingsNodesSet = set(p279Seeded_SiblingsDF3_wabs_text.node1.to_list() + p279Seeded_SiblingsDF3_wabs_text.node2.to_list())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "pediatric-tomato",
   "metadata": {},
   "outputs": [],
   "source": [
    "sum(bioDF.Term1_kg_id.apply(lambda p: p in P279childParNodesSet or p in P279siblingsNodesSet)), \\\n",
    "sum(bioDF.Term2_kg_id.apply(lambda p: p in P279childParNodesSet or p in P279siblingsNodesSet))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "transparent-diversity",
   "metadata": {},
   "outputs": [],
   "source": [
    "probaseNodesSet = set(probDF_Qnodes_DF_WQnodes1_subset.node1.to_list() + probDF_Qnodes_DF_WQnodes1_subset.node2.to_list())\n",
    "\n",
    "sum(bioDF.Term1_kg_id.apply(lambda p: p in probaseNodesSet)), \\\n",
    "sum(bioDF.Term2_kg_id.apply(lambda p: p in probaseNodesSet))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "uniform-civilian",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "voluntary-logic",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "opened-baker",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "supposed-charm",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "kgtkEnv",
   "language": "python",
   "name": "kgtkenv"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "215px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}