models.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "os.environ[\"CUDA_DEVICE_ORDER\"] = \"PCI_BUS_ID\"\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"5,6,7,8\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pathlib import Path\n",
    "import string\n",
    "from typing import Literal, Any\n",
    "\n",
    "import pandas as pd\n",
    "import requests\n",
    "from selfcheckgpt.modeling_selfcheck import SelfCheckNLI\n",
    "from sentence_transformers import SentenceTransformer\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "import torch\n",
    "import torch.nn.functional as F\n",
    "import tqdm.notebook as tqdm\n",
    "from transformers import pipeline, AutoTokenizer, AutoModelForSequenceClassification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "DeviceType = Literal['cpu', 'cuda']\n",
    "Samples = list[Literal['train', 'val', 'test']]\n",
    "Tasks = list[Literal['agnostic', 'aware']]\n",
    "Model = Literal[\n",
    "    'FacebookAI/roberta-large-mnli',\n",
    "    'deberta-selfchecknli',\n",
    "    'google/t5_xxl_true_nli_mixture',\n",
    "    'sentence-transformers/nli-roberta-large',\n",
    "    'microsoft/deberta-base-mnli', \n",
    "    'microsoft/deberta-large-mnli', \n",
    "    'microsoft/deberta-xlarge-mnli',\n",
    "    'microsoft/deberta-v2-xlarge-mnli',\n",
    "    'microsoft/deberta-v2-xxlarge-mnli',\n",
    "    'openchat',\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "MODEL: Model = 'deberta-selfchecknli'\n",
    "DEVICE: DeviceType = 'cuda'\n",
    "SAMPLES: Samples = ['val', 'test']\n",
    "TASKS: Tasks = ['agnostic', 'aware']\n",
    "INVERSE_NLI = False\n",
    "\n",
    "INPUT_DATA_PATH = './data/'\n",
    "OUTPUT_DATA_PATH = './output/'\n",
    "REFERENCE_PATHS, SUBMISSION_PATHS, RESULT_PATHS = {}, {}, {}\n",
    "for sample in SAMPLES:\n",
    "    REFERENCE_PATHS[sample] = f'{INPUT_DATA_PATH}{sample}/'\n",
    "    SUBMISSION_PATHS[sample] = f\"{OUTPUT_DATA_PATH}{MODEL.replace('/', '-')}{'_inverse' if INVERSE_NLI else ''}/{sample}/\"\n",
    "    RESULT_PATHS[sample] = f'{SUBMISSION_PATHS[sample]}{sample}.score.txt'\n",
    "\n",
    "# For OpneChat model only\n",
    "TEMPERATURE = 0\n",
    "MAX_TOKENS = 5\n",
    "REQUEST_NUMBER = 1\n",
    "INSTRACTION = \"Is the Sentence supported by the Context above? Answer using ONLY yes or no.\"\n",
    "TEMPLATE = \"Context: {context}\\n\\nSentence: {sentence}\\n\\nSystem: {instruction}\\n\\n\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "for sample in SAMPLES:\n",
    "    Path(SUBMISSION_PATHS[sample]).mkdir(parents=True, exist_ok=True)\n",
    "\n",
    "class UserError(Exception): pass\n",
    "\n",
    "assert DEVICE in ['cpu', 'cuda'], 'Wrong device'\n",
    "assert all(sample in ['train', 'val', 'test'] for sample in SAMPLES), 'Wrong sample'\n",
    "assert all(task in ['agnostic', 'aware'] for task in TASKS), 'Wrong task'\n",
    "assert MODEL in [\n",
    "    'FacebookAI/roberta-large-mnli',\n",
    "    'deberta-selfchecknli',\n",
    "    'google/t5_xxl_true_nli_mixture',\n",
    "    'sentence-transformers/nli-roberta-large',\n",
    "    'microsoft/deberta-base-mnli', \n",
    "    'microsoft/deberta-large-mnli', \n",
    "    'microsoft/deberta-xlarge-mnli',\n",
    "    'microsoft/deberta-v2-xlarge-mnli',\n",
    "    'microsoft/deberta-v2-xxlarge-mnli',\n",
    "    'openchat',\n",
    "    ], 'Wrong model'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load data\n",
    "data = {}\n",
    "for sample in SAMPLES:\n",
    "    for task in TASKS:\n",
    "        file_path = INPUT_DATA_PATH + sample + '/' + sample + '.model-' + task + '.json'\n",
    "        data[sample + '-' + task] = pd.read_json(file_path)\n",
    "\n",
    "# prepocess data\n",
    "if 'train-aware' in data.keys():\n",
    "    data['train-aware'].at[10519, 'src'] = 'None'\n",
    "if 'train-agnostic' in data.keys():\n",
    "    data['train-agnostic'].drop(columns=['model'], inplace=True)\n",
    "if 'valid-agnostic' in data.keys():\n",
    "    data['valid-agnostic'].drop(columns=['model'], inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Predict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_texts(datapoint: pd.Series):\n",
    "    task = str(datapoint['task'])\n",
    "    src = str(datapoint['src'])\n",
    "    hyp = str(datapoint['hyp'])\n",
    "    tgt = str(datapoint['tgt'])\n",
    "    if task == 'DM':\n",
    "        premise = tgt\n",
    "    elif task == 'MT':\n",
    "        premise = tgt  # may be src, but in another language\n",
    "    elif task =='PG':\n",
    "        premise = src  # may be either tgt or src\n",
    "    else:\n",
    "        raise UserError('Task should be either DM, MT or PG.')\n",
    "    \n",
    "    return premise, hyp\n",
    "\n",
    "\n",
    "def get_model_object() -> Any:\n",
    "    if MODEL == 'deberta-selfchecknli':\n",
    "        model_object = SelfCheckNLI(device=torch.device(DEVICE))\n",
    "    elif MODEL == 'google/t5_xxl_true_nli_mixture':\n",
    "        model_object = pipeline(\"text2text-generation\", model=MODEL, device_map=\"auto\")\n",
    "    elif MODEL == 'sentence-transformers/nli-roberta-large':\n",
    "        model_object = SentenceTransformer(MODEL, device=DEVICE)\n",
    "    elif MODEL == 'FacebookAI/roberta-large-mnli':\n",
    "        model_object = torch.hub.load('pytorch/fairseq', 'roberta.large.mnli')\n",
    "        model_object.eval()\n",
    "        if DEVICE == 'cuda': model_object.cuda()\n",
    "    elif MODEL in [\n",
    "        'microsoft/deberta-base-mnli', 'microsoft/deberta-large-mnli',\n",
    "        'microsoft/deberta-xlarge-mnli', 'microsoft/deberta-v2-xlarge-mnli',\n",
    "        'microsoft/deberta-v2-xxlarge-mnli',\n",
    "        ]:\n",
    "        global tokenizer\n",
    "        model_object = AutoModelForSequenceClassification.from_pretrained(MODEL)\n",
    "        tokenizer = AutoTokenizer.from_pretrained(MODEL)\n",
    "    elif MODEL == 'openchat':\n",
    "        model_object = None\n",
    "    else:\n",
    "        raise UserError('Wrong method or model')\n",
    "    \n",
    "    return model_object\n",
    "\n",
    "\n",
    "def predict(premise: str, hyp: str, model: Any) -> tuple[float, float | None]:\n",
    "    entailment_proba = None\n",
    "    if MODEL == 'deberta-selfchecknli':\n",
    "        contradiction_proba = model.predict(sentences=[hyp], sampled_passages=[premise])[0]\n",
    "    elif MODEL == 'FacebookAI/roberta-large-mnli':\n",
    "        tokens = model.encode(premise, hyp)\n",
    "        result = model.predict('mnli', tokens)\n",
    "        probs = torch.exp(result.detach()[0]).tolist()\n",
    "        contradiction_proba, entailment_proba = probs[0], probs[2]\n",
    "    elif MODEL == 'google/t5_xxl_true_nli_mixture':\n",
    "        template = \"premise: {premise} hypothesis: {hypothesis}\"\n",
    "        prompt = template.format(premise=premise, hypothesis=hyp)\n",
    "        output = model([prompt])\n",
    "        contradiction_proba = float(1 - int(output[0]['generated_text']))\n",
    "    elif MODEL == 'sentence-transformers/nli-roberta-large':\n",
    "        embeddings = model.encode([hyp, premise])\n",
    "        contradiction_proba = 1 - cosine_similarity(embeddings, dense_output=False)[0][1]\n",
    "    elif MODEL in [\n",
    "        'microsoft/deberta-base-mnli', 'microsoft/deberta-large-mnli',\n",
    "        'microsoft/deberta-xlarge-mnli', 'microsoft/deberta-v2-xlarge-mnli',\n",
    "        'microsoft/deberta-v2-xxlarge-mnli',\n",
    "        ]:\n",
    "        inputs = tokenizer(premise, hyp, return_tensors=\"pt\", padding=True, truncation=True)\n",
    "        with torch.no_grad():\n",
    "            outputs = model(**inputs)\n",
    "        probabilities = F.softmax(outputs.logits, dim=1)\n",
    "        contradiction_proba = probabilities[0][0].item()\n",
    "        entailment_proba = probabilities[0][2].item()\n",
    "    elif MODEL == 'openchat':\n",
    "        contradiction_proba = 0.0\n",
    "        for _ in range(REQUEST_NUMBER):\n",
    "            url = \"http://localhost:18888/v1/chat/completions\"\n",
    "            prompt = TEMPLATE.format(context=premise, sentence=hyp, instruction=INSTRACTION)\n",
    "            data = {\n",
    "                \"temperature\": TEMPERATURE,\n",
    "                \"max_tokens\": MAX_TOKENS,\n",
    "                \"model\": \"openchat_3.5\",\n",
    "                \"messages\": [{\"role\": \"user\", \"content\": prompt}],\n",
    "            }\n",
    "            response = requests.post(url, json=data).json()\n",
    "            answer = response['choices'][0]['message']['content']\n",
    "            answer = answer.strip(string.punctuation + string.whitespace).lower()\n",
    "            if answer.startswith(\"yes\"):\n",
    "                pass\n",
    "            elif answer.startswith(\"no\"):\n",
    "                contradiction_proba += 1.0\n",
    "            else:\n",
    "                contradiction_proba += 0.5\n",
    "        contradiction_proba /= REQUEST_NUMBER\n",
    "    else:\n",
    "        raise UserError('Wrong model')\n",
    "\n",
    "    return contradiction_proba, entailment_proba\n",
    "\n",
    "\n",
    "def predict_df(data: pd.DataFrame, sample: str, output_path: str, model: Any, round_probs=False):\n",
    "    # check output path\n",
    "    print(output_path)\n",
    "    if os.path.exists(output_path):\n",
    "        raise UserError('The file already exists.')\n",
    "    \n",
    "    with open(output_path, 'a', encoding='utf-8') as fp:\n",
    "        first_iter = True\n",
    "        test_sample = sample == 'test'\n",
    "        predictions = []\n",
    "        for _, row in tqdm.tqdm(data.iterrows(), total=len(data)):\n",
    "            # get premise and hypothesis\n",
    "            premise, hyp = get_texts(row)\n",
    "            if INVERSE_NLI:\n",
    "                premise, hyp = hyp, premise  # Swap them\n",
    "    \n",
    "            # predict\n",
    "            contradiction_proba, entailment_proba = predict(premise, hyp, model)\n",
    "            \n",
    "            # postprocess probability\n",
    "            if round_probs:  # round to 0.2 step\n",
    "                contradiction_proba = round(contradiction_proba * 5) / 5\n",
    "                if entailment_proba is not None:\n",
    "                    entailment_proba = round(entailment_proba * 5) / 5 \n",
    "\n",
    "            # form and write header\n",
    "            if first_iter:\n",
    "                header = 'id,' if test_sample else ''\n",
    "                header += 'p(Contr)'\n",
    "                if entailment_proba is not None:\n",
    "                    header += ',p(Entl)'\n",
    "                header += '\\n'\n",
    "                fp.write(header)\n",
    "                first_iter = False\n",
    "            \n",
    "            # form probability to string\n",
    "            data_string = f\"{row['id']},\" if test_sample else ''\n",
    "            data_string += f'{contradiction_proba:.6f}'\n",
    "            if entailment_proba is not None:\n",
    "                data_string += f',{entailment_proba:.6f}'\n",
    "            data_string += '\\n'\n",
    "\n",
    "            # write last 50 preds to file\n",
    "            predictions.append(data_string)\n",
    "            if len(predictions) % 50 == 0:\n",
    "                fp.writelines(predictions)\n",
    "                predictions = []\n",
    "\n",
    "        if predictions:\n",
    "            fp.writelines(predictions)\n",
    "            predictions = []\n",
    "\n",
    "\n",
    "def predict_dfs(data: dict[str, pd.DataFrame]):\n",
    "    model = get_model_object()\n",
    "    for sample in SAMPLES:\n",
    "        for task in TASKS:\n",
    "            output_file_path = f'{SUBMISSION_PATHS[sample]}{sample}.model-{task}.csv'\n",
    "            predict_df(data[sample + '-' + task], sample, output_file_path, model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "224a309f2ca34799a81acc079c759047",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "tokenizer_config.json:   0%|          | 0.00/400 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "216caaa875174d08868389e47512dbb9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "spm.model:   0%|          | 0.00/2.46M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6819a54a384e4678a9990b16b7fd0c82",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "added_tokens.json:   0%|          | 0.00/23.0 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7da654abe31d48f7ae4bc00d30e1ccc8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "special_tokens_map.json:   0%|          | 0.00/173 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9932139308c4470894282329b83e3362",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "config.json:   0%|          | 0.00/883 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "78a7bd8d97994db2b2402972f3a69416",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "pytorch_model.bin:   0%|          | 0.00/1.74G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/vscode/.local/lib/python3.11/site-packages/torch/_utils.py:831: UserWarning: TypedStorage is deprecated. It will be removed in the future and UntypedStorage will be the only storage class. This should only matter to you if you are using storages directly.  To access UntypedStorage directly, use tensor.untyped_storage() instead of tensor.storage()\n",
      "  return self.fget.__get__(instance, owner)()\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SelfCheck-NLI initialized to device cuda\n",
      "./output/deberta-selfchecknli/val/val.model-agnostic.csv\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b2db6aff275446abac2d970efe873e35",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/499 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "./output/deberta-selfchecknli/val/val.model-aware.csv\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f9660e9abec3481d80d506bef6dd6dd1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/501 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "./output/deberta-selfchecknli/test/test.model-agnostic.csv\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e55c99798f714ee5b35543e987409d31",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1500 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "./output/deberta-selfchecknli/test/test.model-aware.csv\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "95329ecc5d3b45bab35f18cca0ffb341",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1500 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "predict_dfs(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>p(Contr)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.002097</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.643224</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.707190</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.008859</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.004337</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>494</th>\n",
       "      <td>0.000138</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495</th>\n",
       "      <td>0.000263</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>496</th>\n",
       "      <td>0.000291</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>497</th>\n",
       "      <td>0.000053</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>498</th>\n",
       "      <td>0.000145</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>499 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     p(Contr)\n",
       "0    0.002097\n",
       "1    0.643224\n",
       "2    0.707190\n",
       "3    0.008859\n",
       "4    0.004337\n",
       "..        ...\n",
       "494  0.000138\n",
       "495  0.000263\n",
       "496  0.000291\n",
       "497  0.000053\n",
       "498  0.000145\n",
       "\n",
       "[499 rows x 1 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>p(Contr)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.736973</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.014666</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.058523</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.001738</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.986732</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>496</th>\n",
       "      <td>0.997198</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>497</th>\n",
       "      <td>0.000630</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>498</th>\n",
       "      <td>0.751895</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>499</th>\n",
       "      <td>0.976152</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>500</th>\n",
       "      <td>0.988687</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>501 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     p(Contr)\n",
       "0    0.736973\n",
       "1    0.014666\n",
       "2    0.058523\n",
       "3    0.001738\n",
       "4    0.986732\n",
       "..        ...\n",
       "496  0.997198\n",
       "497  0.000630\n",
       "498  0.751895\n",
       "499  0.976152\n",
       "500  0.988687\n",
       "\n",
       "[501 rows x 1 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>p(Contr)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0.591021</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>0.603800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>0.000358</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6</td>\n",
       "      <td>0.990409</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7</td>\n",
       "      <td>0.059645</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1495</th>\n",
       "      <td>2992</td>\n",
       "      <td>0.000220</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1496</th>\n",
       "      <td>2993</td>\n",
       "      <td>0.996383</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1497</th>\n",
       "      <td>2994</td>\n",
       "      <td>0.017649</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1498</th>\n",
       "      <td>2996</td>\n",
       "      <td>0.000135</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1499</th>\n",
       "      <td>2999</td>\n",
       "      <td>0.000592</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1500 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        id  p(Contr)\n",
       "0        1  0.591021\n",
       "1        2  0.603800\n",
       "2        3  0.000358\n",
       "3        6  0.990409\n",
       "4        7  0.059645\n",
       "...    ...       ...\n",
       "1495  2992  0.000220\n",
       "1496  2993  0.996383\n",
       "1497  2994  0.017649\n",
       "1498  2996  0.000135\n",
       "1499  2999  0.000592\n",
       "\n",
       "[1500 rows x 2 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>p(Contr)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0.001624</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4</td>\n",
       "      <td>0.002131</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5</td>\n",
       "      <td>0.003519</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>9</td>\n",
       "      <td>0.347478</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>11</td>\n",
       "      <td>0.000133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1495</th>\n",
       "      <td>2988</td>\n",
       "      <td>0.010280</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1496</th>\n",
       "      <td>2991</td>\n",
       "      <td>0.001613</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1497</th>\n",
       "      <td>2995</td>\n",
       "      <td>0.000221</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1498</th>\n",
       "      <td>2997</td>\n",
       "      <td>0.000605</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1499</th>\n",
       "      <td>2998</td>\n",
       "      <td>0.204039</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1500 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        id  p(Contr)\n",
       "0        0  0.001624\n",
       "1        4  0.002131\n",
       "2        5  0.003519\n",
       "3        9  0.347478\n",
       "4       11  0.000133\n",
       "...    ...       ...\n",
       "1495  2988  0.010280\n",
       "1496  2991  0.001613\n",
       "1497  2995  0.000221\n",
       "1498  2997  0.000605\n",
       "1499  2998  0.204039\n",
       "\n",
       "[1500 rows x 2 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Show predictions\n",
    "df = {}\n",
    "for sample in SAMPLES:\n",
    "    for task in TASKS:\n",
    "        path = f'{SUBMISSION_PATHS[sample]}{sample}.model-{task}.csv'\n",
    "        df[sample + '-' + task] = pd.read_csv(path)\n",
    "        display(df[sample + '-' + task])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Evaluate\n",
    "## Convert .csv to .json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def norm_probs(a, b):\n",
    "    if (sum_ := a + b) > 0:\n",
    "        a, b = min(a / sum_, 1.0), min(b / sum_, 1.0)\n",
    "    else:\n",
    "        a, b = 0.5, 0.5\n",
    "    return a, b\n",
    "\n",
    "\n",
    "# Create .json from .csv\n",
    "df = {}\n",
    "for sample in SAMPLES:\n",
    "    for task in TASKS:\n",
    "        csv_path = f'{SUBMISSION_PATHS[sample]}{sample}.model-{task}.csv'\n",
    "        json_path = f'{SUBMISSION_PATHS[sample]}{sample}.model-{task}.json'\n",
    "        df[task] = pd.read_csv(csv_path).reset_index(drop=True)\n",
    "        if 'p(Entl)' in df[task]:\n",
    "            df[task]['p(Hallucination)'] = df[task].apply(lambda x: norm_probs(x['p(Contr)'], x['p(Entl)'])[0], axis=1)\n",
    "            df[task].drop(columns=['p(Entl)'], inplace=True)\n",
    "        else:\n",
    "            df[task]['p(Hallucination)'] = df[task]['p(Contr)']\n",
    "        df[task].drop(columns=['p(Contr)'], inplace=True)\n",
    "        df[task]['label'] = df[task].apply(lambda x: 'Hallucination' if x['p(Hallucination)'] > 0.5 else 'Not Hallucination', axis=1)\n",
    "        df[task].to_json(json_path, orient='records')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Check file format"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all clear!\n",
      "all clear!\n"
     ]
    }
   ],
   "source": [
    "for sample in SAMPLES:\n",
    "    if sample == 'val':\n",
    "        !python ./check_output.py {SUBMISSION_PATHS[sample]} --is_val\n",
    "    elif sample == 'test':\n",
    "        !python ./check_output.py {SUBMISSION_PATHS[sample]}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "val\n",
      "aware_acc:0.716566866267465\n",
      "aware_rho:0.6714934562884112\n",
      "agnostic_acc:0.751503006012024\n",
      "agnostic_rho:0.6994542370904473\n",
      "\n",
      "test\n",
      "agnostic_acc:0.7533333333333333\n",
      "agnostic_rho:0.6829601168067542\n",
      "aware_acc:0.7586666666666667\n",
      "aware_rho:0.6831961651299768\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for sample in SAMPLES:\n",
    "    if sample == 'val':\n",
    "        !python ./score.py {SUBMISSION_PATHS[sample]} {REFERENCE_PATHS[sample]} {RESULT_PATHS[sample]} --is_val\n",
    "    elif sample == 'test':\n",
    "        !python ./score.py {SUBMISSION_PATHS[sample]} {REFERENCE_PATHS[sample]} {RESULT_PATHS[sample]}\n",
    "\n",
    "    if os.path.isfile(RESULT_PATHS[sample]):\n",
    "        with open(RESULT_PATHS[sample]) as fp:\n",
    "            print(sample)\n",
    "            print(fp.read())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Delete .json files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for sample in SAMPLES:\n",
    "    for task in TASKS:\n",
    "        json_path = f'{SUBMISSION_PATHS[sample]}{sample}.model-{task}.json'\n",
    "        if os.path.isfile(json_path):\n",
    "            os.remove(json_path)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}