data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pathlib import Path\n",
    "\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "DATA_PATH = Path('./data/')\n",
    "SAMPLES = ['train', 'val', 'test']\n",
    "TRACKS = ['agnostic', 'aware']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load data\n",
    "df = {}\n",
    "for set_ in SAMPLES:\n",
    "    df[set_] = {}\n",
    "    for track in TRACKS:\n",
    "        df[set_][track] = pd.read_json(DATA_PATH / set_  / (set_ + '.model-' + track + '.json'))\n",
    "df_trial = pd.read_json(DATA_PATH / 'trial' / 'trial.json')\n",
    "\n",
    "# prepocess data\n",
    "df['train']['aware'].at[10519, 'src'] = 'None'\n",
    "df['train']['agnostic'].drop(columns=['model'], inplace=True)\n",
    "df['val']['agnostic'].drop(columns=['model'], inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Trial data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 80 entries, 0 to 79\n",
      "Data columns (total 9 columns):\n",
      " #   Column            Non-Null Count  Dtype  \n",
      "---  ------            --------------  -----  \n",
      " 0   hyp               80 non-null     object \n",
      " 1   ref               80 non-null     object \n",
      " 2   src               80 non-null     object \n",
      " 3   tgt               80 non-null     object \n",
      " 4   model             80 non-null     object \n",
      " 5   task              80 non-null     object \n",
      " 6   labels            80 non-null     object \n",
      " 7   label             80 non-null     object \n",
      " 8   p(Hallucination)  80 non-null     float64\n",
      "dtypes: float64(1), object(8)\n",
      "memory usage: 5.8+ KB\n"
     ]
    }
   ],
   "source": [
    "df_trial.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>hyp</th>\n",
       "      <th>ref</th>\n",
       "      <th>src</th>\n",
       "      <th>tgt</th>\n",
       "      <th>model</th>\n",
       "      <th>task</th>\n",
       "      <th>labels</th>\n",
       "      <th>label</th>\n",
       "      <th>p(Hallucination)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>A district of Kowloon, China.</td>\n",
       "      <td>tgt</td>\n",
       "      <td>The City &lt;define&gt; Chiuchow &lt;/define&gt; is Kowloo...</td>\n",
       "      <td>The Chaoshan region where the Teochew dialect ...</td>\n",
       "      <td></td>\n",
       "      <td>DM</td>\n",
       "      <td>[Hallucination, Hallucination, Hallucination]</td>\n",
       "      <td>Hallucination</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>(Wicca) A witchdoctor.</td>\n",
       "      <td>tgt</td>\n",
       "      <td>“ I went to see the witchdoctor , the &lt;define&gt;...</td>\n",
       "      <td>(South Africa) A traditional tribal diviner or...</td>\n",
       "      <td></td>\n",
       "      <td>DM</td>\n",
       "      <td>[Hallucination, Hallucination, Not Hallucination]</td>\n",
       "      <td>Hallucination</td>\n",
       "      <td>0.666667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>(nautical) A halyard.</td>\n",
       "      <td>tgt</td>\n",
       "      <td>Haul away , keeping strain on both parts of th...</td>\n",
       "      <td>(nautical) A staff that carries a flag or penn...</td>\n",
       "      <td></td>\n",
       "      <td>DM</td>\n",
       "      <td>[Hallucination, Not Hallucination, Not Halluci...</td>\n",
       "      <td>Not Hallucination</td>\n",
       "      <td>0.333333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>The quality of being rational.</td>\n",
       "      <td>tgt</td>\n",
       "      <td>And what difference is there between the credi...</td>\n",
       "      <td>(archaic) The quality or state of being rational.</td>\n",
       "      <td></td>\n",
       "      <td>DM</td>\n",
       "      <td>[Not Hallucination, Not Hallucination, Not Hal...</td>\n",
       "      <td>Not Hallucination</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>(uncountable) The study of trees.</td>\n",
       "      <td>tgt</td>\n",
       "      <td>It is now generally supposed that the forbidde...</td>\n",
       "      <td>The worship of trees.</td>\n",
       "      <td></td>\n",
       "      <td>DM</td>\n",
       "      <td>[Hallucination, Hallucination, Hallucination]</td>\n",
       "      <td>Hallucination</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                 hyp  ref  \\\n",
       "0      A district of Kowloon, China.  tgt   \n",
       "1             (Wicca) A witchdoctor.  tgt   \n",
       "2              (nautical) A halyard.  tgt   \n",
       "3     The quality of being rational.  tgt   \n",
       "4  (uncountable) The study of trees.  tgt   \n",
       "\n",
       "                                                 src  \\\n",
       "0  The City <define> Chiuchow </define> is Kowloo...   \n",
       "1  “ I went to see the witchdoctor , the <define>...   \n",
       "2  Haul away , keeping strain on both parts of th...   \n",
       "3  And what difference is there between the credi...   \n",
       "4  It is now generally supposed that the forbidde...   \n",
       "\n",
       "                                                 tgt model task  \\\n",
       "0  The Chaoshan region where the Teochew dialect ...         DM   \n",
       "1  (South Africa) A traditional tribal diviner or...         DM   \n",
       "2  (nautical) A staff that carries a flag or penn...         DM   \n",
       "3  (archaic) The quality or state of being rational.         DM   \n",
       "4                              The worship of trees.         DM   \n",
       "\n",
       "                                              labels              label  \\\n",
       "0      [Hallucination, Hallucination, Hallucination]      Hallucination   \n",
       "1  [Hallucination, Hallucination, Not Hallucination]      Hallucination   \n",
       "2  [Hallucination, Not Hallucination, Not Halluci...  Not Hallucination   \n",
       "3  [Not Hallucination, Not Hallucination, Not Hal...  Not Hallucination   \n",
       "4      [Hallucination, Hallucination, Hallucination]      Hallucination   \n",
       "\n",
       "   p(Hallucination)  \n",
       "0          1.000000  \n",
       "1          0.666667  \n",
       "2          0.333333  \n",
       "3          0.000000  \n",
       "4          1.000000  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_trial.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "task  ref     model                      \n",
       "DM    tgt                                    36\n",
       "MT    either                                 35\n",
       "PG    either  tuner007/pegasus_paraphrase     9\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_trial[['task', 'ref', 'model']].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train data (agnostic)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 30000 entries, 0 to 29999\n",
      "Data columns (total 5 columns):\n",
      " #   Column  Non-Null Count  Dtype \n",
      "---  ------  --------------  ----- \n",
      " 0   hyp     30000 non-null  object\n",
      " 1   tgt     30000 non-null  object\n",
      " 2   src     30000 non-null  object\n",
      " 3   ref     30000 non-null  object\n",
      " 4   task    30000 non-null  object\n",
      "dtypes: object(5)\n",
      "memory usage: 1.1+ MB\n"
     ]
    }
   ],
   "source": [
    "df['train']['agnostic'].info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>hyp</th>\n",
       "      <th>tgt</th>\n",
       "      <th>src</th>\n",
       "      <th>ref</th>\n",
       "      <th>task</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Don't worry, it's only temporary.</td>\n",
       "      <td>Don't worry. It's only temporary.</td>\n",
       "      <td>Не волнуйся. Это только временно.</td>\n",
       "      <td>either</td>\n",
       "      <td>MT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Tom is never where he should be.</td>\n",
       "      <td>Tom is never where he's supposed to be.</td>\n",
       "      <td>Тома никогда нет там, где он должен быть.</td>\n",
       "      <td>either</td>\n",
       "      <td>MT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>It's hard for me to work with Tom.</td>\n",
       "      <td>I have trouble working with Tom.</td>\n",
       "      <td>Мне сложно работать с Томом.</td>\n",
       "      <td>either</td>\n",
       "      <td>MT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Water, please.</td>\n",
       "      <td>I'd like some water.</td>\n",
       "      <td>Воду, пожалуйста.</td>\n",
       "      <td>either</td>\n",
       "      <td>MT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>I didn't expect Tom to betray me.</td>\n",
       "      <td>I didn't think that Tom would betray me.</td>\n",
       "      <td>Я не ожидал, что Том предаст меня.</td>\n",
       "      <td>either</td>\n",
       "      <td>MT</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                  hyp  \\\n",
       "0   Don't worry, it's only temporary.   \n",
       "1    Tom is never where he should be.   \n",
       "2  It's hard for me to work with Tom.   \n",
       "3                      Water, please.   \n",
       "4   I didn't expect Tom to betray me.   \n",
       "\n",
       "                                        tgt  \\\n",
       "0         Don't worry. It's only temporary.   \n",
       "1   Tom is never where he's supposed to be.   \n",
       "2          I have trouble working with Tom.   \n",
       "3                      I'd like some water.   \n",
       "4  I didn't think that Tom would betray me.   \n",
       "\n",
       "                                         src     ref task  \n",
       "0          Не волнуйся. Это только временно.  either   MT  \n",
       "1  Тома никогда нет там, где он должен быть.  either   MT  \n",
       "2               Мне сложно работать с Томом.  either   MT  \n",
       "3                          Воду, пожалуйста.  either   MT  \n",
       "4         Я не ожидал, что Том предаст меня.  either   MT  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['train']['agnostic'].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "task  ref   \n",
       "DM    tgt       10000\n",
       "MT    either    10000\n",
       "PG    src       10000\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['train']['agnostic'][['task', 'ref']].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train data (aware)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 30000 entries, 0 to 29999\n",
      "Data columns (total 6 columns):\n",
      " #   Column  Non-Null Count  Dtype \n",
      "---  ------  --------------  ----- \n",
      " 0   hyp     30000 non-null  object\n",
      " 1   tgt     30000 non-null  object\n",
      " 2   src     30000 non-null  object\n",
      " 3   ref     30000 non-null  object\n",
      " 4   task    30000 non-null  object\n",
      " 5   model   30000 non-null  object\n",
      "dtypes: object(6)\n",
      "memory usage: 1.4+ MB\n"
     ]
    }
   ],
   "source": [
    "df['train']['aware'].info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>hyp</th>\n",
       "      <th>tgt</th>\n",
       "      <th>src</th>\n",
       "      <th>ref</th>\n",
       "      <th>task</th>\n",
       "      <th>model</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Of or pertaining to the language of a particul...</td>\n",
       "      <td>Of or pertaining to everyday language , as opp...</td>\n",
       "      <td>There are blacktips , silvertips , bronze whal...</td>\n",
       "      <td>tgt</td>\n",
       "      <td>DM</td>\n",
       "      <td>ltg/flan-t5-definition-en-base</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Not coercive ; not involving coercion</td>\n",
       "      <td>Not coercive ; free of coercion</td>\n",
       "      <td>Mr. obama signed executive orders requiring al...</td>\n",
       "      <td>tgt</td>\n",
       "      <td>DM</td>\n",
       "      <td>ltg/flan-t5-definition-en-base</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>To express or express by words ; to express by...</td>\n",
       "      <td>To depict or portray .</td>\n",
       "      <td>Disloyal ? / the word is too good to paint out...</td>\n",
       "      <td>tgt</td>\n",
       "      <td>DM</td>\n",
       "      <td>ltg/flan-t5-definition-en-base</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Having the power to authoritatively speak or w...</td>\n",
       "      <td>Having a commanding style .</td>\n",
       "      <td>He instructed us in that booming , authoritati...</td>\n",
       "      <td>tgt</td>\n",
       "      <td>DM</td>\n",
       "      <td>ltg/flan-t5-definition-en-base</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Without a scot .</td>\n",
       "      <td>Without consequences or penalties , to go free...</td>\n",
       "      <td>To get off scot-free . What is the meaning of ...</td>\n",
       "      <td>tgt</td>\n",
       "      <td>DM</td>\n",
       "      <td>ltg/flan-t5-definition-en-base</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 hyp  \\\n",
       "0  Of or pertaining to the language of a particul...   \n",
       "1              Not coercive ; not involving coercion   \n",
       "2  To express or express by words ; to express by...   \n",
       "3  Having the power to authoritatively speak or w...   \n",
       "4                                   Without a scot .   \n",
       "\n",
       "                                                 tgt  \\\n",
       "0  Of or pertaining to everyday language , as opp...   \n",
       "1                    Not coercive ; free of coercion   \n",
       "2                             To depict or portray .   \n",
       "3                        Having a commanding style .   \n",
       "4  Without consequences or penalties , to go free...   \n",
       "\n",
       "                                                 src  ref task  \\\n",
       "0  There are blacktips , silvertips , bronze whal...  tgt   DM   \n",
       "1  Mr. obama signed executive orders requiring al...  tgt   DM   \n",
       "2  Disloyal ? / the word is too good to paint out...  tgt   DM   \n",
       "3  He instructed us in that booming , authoritati...  tgt   DM   \n",
       "4  To get off scot-free . What is the meaning of ...  tgt   DM   \n",
       "\n",
       "                            model  \n",
       "0  ltg/flan-t5-definition-en-base  \n",
       "1  ltg/flan-t5-definition-en-base  \n",
       "2  ltg/flan-t5-definition-en-base  \n",
       "3  ltg/flan-t5-definition-en-base  \n",
       "4  ltg/flan-t5-definition-en-base  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['train']['aware'].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "task  ref     model                           \n",
       "DM    tgt     ltg/flan-t5-definition-en-base      10000\n",
       "MT    either  facebook/nllb-200-distilled-600M    10000\n",
       "PG    src     tuner007/pegasus_paraphrase         10000\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['train']['aware'][['task', 'ref', 'model']].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Validation data (agnostic)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 499 entries, 0 to 498\n",
      "Data columns (total 8 columns):\n",
      " #   Column            Non-Null Count  Dtype  \n",
      "---  ------            --------------  -----  \n",
      " 0   hyp               499 non-null    object \n",
      " 1   ref               499 non-null    object \n",
      " 2   src               499 non-null    object \n",
      " 3   tgt               499 non-null    object \n",
      " 4   task              499 non-null    object \n",
      " 5   labels            499 non-null    object \n",
      " 6   label             499 non-null    object \n",
      " 7   p(Hallucination)  499 non-null    float64\n",
      "dtypes: float64(1), object(7)\n",
      "memory usage: 31.3+ KB\n"
     ]
    }
   ],
   "source": [
    "df['val']['agnostic'].info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>hyp</th>\n",
       "      <th>ref</th>\n",
       "      <th>src</th>\n",
       "      <th>tgt</th>\n",
       "      <th>task</th>\n",
       "      <th>labels</th>\n",
       "      <th>label</th>\n",
       "      <th>p(Hallucination)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Resembling or characteristic of a weasel.</td>\n",
       "      <td>tgt</td>\n",
       "      <td>The writer had just entered into his eighteent...</td>\n",
       "      <td>Resembling a weasel (in appearance).</td>\n",
       "      <td>DM</td>\n",
       "      <td>[Hallucination, Not Hallucination, Not Halluci...</td>\n",
       "      <td>Not Hallucination</td>\n",
       "      <td>0.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Alternative form of sheath knife</td>\n",
       "      <td>tgt</td>\n",
       "      <td>Sailors ' and fishermen 's &lt;define&gt; sheath - k...</td>\n",
       "      <td>.</td>\n",
       "      <td>DM</td>\n",
       "      <td>[Hallucination, Hallucination, Hallucination, ...</td>\n",
       "      <td>Hallucination</td>\n",
       "      <td>0.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>(obsolete) A short period of time.</td>\n",
       "      <td>tgt</td>\n",
       "      <td>As to age , Bead could not form any clear impr...</td>\n",
       "      <td>(poetic) An instant, a short moment.</td>\n",
       "      <td>DM</td>\n",
       "      <td>[Not Hallucination, Not Hallucination, Not Hal...</td>\n",
       "      <td>Not Hallucination</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>(slang) An incel.</td>\n",
       "      <td>tgt</td>\n",
       "      <td>Because redpillers are usually normies or &lt;def...</td>\n",
       "      <td>(incel, _, slang) A man of a slightly lower ra...</td>\n",
       "      <td>DM</td>\n",
       "      <td>[Not Hallucination, Not Hallucination, Halluci...</td>\n",
       "      <td>Not Hallucination</td>\n",
       "      <td>0.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>An island in Lienchiang County, Taiwan.</td>\n",
       "      <td>tgt</td>\n",
       "      <td>On the second day of massive live - fire drill...</td>\n",
       "      <td>An island in Dongyin, Lienchiang, Taiwan, in t...</td>\n",
       "      <td>DM</td>\n",
       "      <td>[Not Hallucination, Not Hallucination, Not Hal...</td>\n",
       "      <td>Not Hallucination</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                         hyp  ref  \\\n",
       "0  Resembling or characteristic of a weasel.  tgt   \n",
       "1           Alternative form of sheath knife  tgt   \n",
       "2         (obsolete) A short period of time.  tgt   \n",
       "3                          (slang) An incel.  tgt   \n",
       "4    An island in Lienchiang County, Taiwan.  tgt   \n",
       "\n",
       "                                                 src  \\\n",
       "0  The writer had just entered into his eighteent...   \n",
       "1  Sailors ' and fishermen 's <define> sheath - k...   \n",
       "2  As to age , Bead could not form any clear impr...   \n",
       "3  Because redpillers are usually normies or <def...   \n",
       "4  On the second day of massive live - fire drill...   \n",
       "\n",
       "                                                 tgt task  \\\n",
       "0               Resembling a weasel (in appearance).   DM   \n",
       "1                                                  .   DM   \n",
       "2               (poetic) An instant, a short moment.   DM   \n",
       "3  (incel, _, slang) A man of a slightly lower ra...   DM   \n",
       "4  An island in Dongyin, Lienchiang, Taiwan, in t...   DM   \n",
       "\n",
       "                                              labels              label  \\\n",
       "0  [Hallucination, Not Hallucination, Not Halluci...  Not Hallucination   \n",
       "1  [Hallucination, Hallucination, Hallucination, ...      Hallucination   \n",
       "2  [Not Hallucination, Not Hallucination, Not Hal...  Not Hallucination   \n",
       "3  [Not Hallucination, Not Hallucination, Halluci...  Not Hallucination   \n",
       "4  [Not Hallucination, Not Hallucination, Not Hal...  Not Hallucination   \n",
       "\n",
       "   p(Hallucination)  \n",
       "0               0.2  \n",
       "1               0.8  \n",
       "2               0.0  \n",
       "3               0.2  \n",
       "4               0.0  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['val']['agnostic'].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "task  ref   \n",
       "DM    tgt       187\n",
       "MT    either    187\n",
       "PG    either    125\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['val']['agnostic'][['task', 'ref']].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Validation data (aware)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 501 entries, 0 to 500\n",
      "Data columns (total 9 columns):\n",
      " #   Column            Non-Null Count  Dtype  \n",
      "---  ------            --------------  -----  \n",
      " 0   hyp               501 non-null    object \n",
      " 1   ref               501 non-null    object \n",
      " 2   src               501 non-null    object \n",
      " 3   tgt               501 non-null    object \n",
      " 4   model             501 non-null    object \n",
      " 5   task              501 non-null    object \n",
      " 6   labels            501 non-null    object \n",
      " 7   label             501 non-null    object \n",
      " 8   p(Hallucination)  501 non-null    float64\n",
      "dtypes: float64(1), object(8)\n",
      "memory usage: 35.4+ KB\n"
     ]
    }
   ],
   "source": [
    "df['val']['aware'].info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>hyp</th>\n",
       "      <th>ref</th>\n",
       "      <th>src</th>\n",
       "      <th>tgt</th>\n",
       "      <th>model</th>\n",
       "      <th>task</th>\n",
       "      <th>labels</th>\n",
       "      <th>label</th>\n",
       "      <th>p(Hallucination)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>A sloping top .</td>\n",
       "      <td>tgt</td>\n",
       "      <td>The sides of the casket were covered with heav...</td>\n",
       "      <td>A decorative feature that sits on top of somet...</td>\n",
       "      <td>ltg/flan-t5-definition-en-base</td>\n",
       "      <td>DM</td>\n",
       "      <td>[Not Hallucination, Hallucination, Not Halluci...</td>\n",
       "      <td>Hallucination</td>\n",
       "      <td>0.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>To react too much .</td>\n",
       "      <td>tgt</td>\n",
       "      <td>Please try not to overreact if she drives badl...</td>\n",
       "      <td>To react too much or too intensely .</td>\n",
       "      <td>ltg/flan-t5-definition-en-base</td>\n",
       "      <td>DM</td>\n",
       "      <td>[Not Hallucination, Not Hallucination, Not Hal...</td>\n",
       "      <td>Not Hallucination</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>The process of spoiling ; the state of being s...</td>\n",
       "      <td>tgt</td>\n",
       "      <td>To prevent spoilage , store in a cool , dry pl...</td>\n",
       "      <td>The process of spoiling .</td>\n",
       "      <td>ltg/flan-t5-definition-en-base</td>\n",
       "      <td>DM</td>\n",
       "      <td>[Hallucination, Not Hallucination, Hallucinati...</td>\n",
       "      <td>Hallucination</td>\n",
       "      <td>0.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>To arrange in a particular way .</td>\n",
       "      <td>tgt</td>\n",
       "      <td>The way the opposition has framed the argument...</td>\n",
       "      <td>To construct in words so as to establish a con...</td>\n",
       "      <td>ltg/flan-t5-definition-en-base</td>\n",
       "      <td>DM</td>\n",
       "      <td>[Hallucination, Not Hallucination, Not Halluci...</td>\n",
       "      <td>Hallucination</td>\n",
       "      <td>0.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>A feeling of concern ; a feeling of anxiety .</td>\n",
       "      <td>tgt</td>\n",
       "      <td>To mix with thy concernments i desist . What i...</td>\n",
       "      <td>That in which one is concerned or interested ;...</td>\n",
       "      <td>ltg/flan-t5-definition-en-base</td>\n",
       "      <td>DM</td>\n",
       "      <td>[Not Hallucination, Hallucination, Hallucinati...</td>\n",
       "      <td>Hallucination</td>\n",
       "      <td>0.6</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 hyp  ref  \\\n",
       "0                                    A sloping top .  tgt   \n",
       "1                                To react too much .  tgt   \n",
       "2  The process of spoiling ; the state of being s...  tgt   \n",
       "3                   To arrange in a particular way .  tgt   \n",
       "4      A feeling of concern ; a feeling of anxiety .  tgt   \n",
       "\n",
       "                                                 src  \\\n",
       "0  The sides of the casket were covered with heav...   \n",
       "1  Please try not to overreact if she drives badl...   \n",
       "2  To prevent spoilage , store in a cool , dry pl...   \n",
       "3  The way the opposition has framed the argument...   \n",
       "4  To mix with thy concernments i desist . What i...   \n",
       "\n",
       "                                                 tgt  \\\n",
       "0  A decorative feature that sits on top of somet...   \n",
       "1               To react too much or too intensely .   \n",
       "2                          The process of spoiling .   \n",
       "3  To construct in words so as to establish a con...   \n",
       "4  That in which one is concerned or interested ;...   \n",
       "\n",
       "                            model task  \\\n",
       "0  ltg/flan-t5-definition-en-base   DM   \n",
       "1  ltg/flan-t5-definition-en-base   DM   \n",
       "2  ltg/flan-t5-definition-en-base   DM   \n",
       "3  ltg/flan-t5-definition-en-base   DM   \n",
       "4  ltg/flan-t5-definition-en-base   DM   \n",
       "\n",
       "                                              labels              label  \\\n",
       "0  [Not Hallucination, Hallucination, Not Halluci...      Hallucination   \n",
       "1  [Not Hallucination, Not Hallucination, Not Hal...  Not Hallucination   \n",
       "2  [Hallucination, Not Hallucination, Hallucinati...      Hallucination   \n",
       "3  [Hallucination, Not Hallucination, Not Halluci...      Hallucination   \n",
       "4  [Not Hallucination, Hallucination, Hallucinati...      Hallucination   \n",
       "\n",
       "   p(Hallucination)  \n",
       "0               0.6  \n",
       "1               0.0  \n",
       "2               0.6  \n",
       "3               0.6  \n",
       "4               0.6  "
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['val']['aware'].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "task  ref     model                           \n",
       "DM    tgt     ltg/flan-t5-definition-en-base      188\n",
       "MT    either  facebook/nllb-200-distilled-600M    188\n",
       "PG    src     tuner007/pegasus_paraphrase         125\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['val']['aware'][['task', 'ref', 'model']].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Test data (agnostic)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 1500 entries, 0 to 1499\n",
      "Data columns (total 8 columns):\n",
      " #   Column            Non-Null Count  Dtype  \n",
      "---  ------            --------------  -----  \n",
      " 0   id                1500 non-null   int64  \n",
      " 1   src               1500 non-null   object \n",
      " 2   tgt               1500 non-null   object \n",
      " 3   hyp               1500 non-null   object \n",
      " 4   task              1500 non-null   object \n",
      " 5   labels            1500 non-null   object \n",
      " 6   label             1500 non-null   object \n",
      " 7   p(Hallucination)  1500 non-null   float64\n",
      "dtypes: float64(1), int64(1), object(6)\n",
      "memory usage: 93.9+ KB\n"
     ]
    }
   ],
   "source": [
    "df['test']['agnostic'].info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>src</th>\n",
       "      <th>tgt</th>\n",
       "      <th>hyp</th>\n",
       "      <th>task</th>\n",
       "      <th>labels</th>\n",
       "      <th>label</th>\n",
       "      <th>p(Hallucination)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Ты удивишься, если я скажу, что на самом деле ...</td>\n",
       "      <td>Would you be surprised if I told you my name i...</td>\n",
       "      <td>You're gonna be surprised if I say my real nam...</td>\n",
       "      <td>MT</td>\n",
       "      <td>[Not Hallucination, Not Hallucination, Not Hal...</td>\n",
       "      <td>Not Hallucination</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>Еды будет полно.</td>\n",
       "      <td>There will be plenty of food.</td>\n",
       "      <td>The food will be full.</td>\n",
       "      <td>MT</td>\n",
       "      <td>[Hallucination, Not Hallucination, Hallucinati...</td>\n",
       "      <td>Hallucination</td>\n",
       "      <td>0.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>Думаете, Том будет меня ждать?</td>\n",
       "      <td>Do you think that Tom will wait for me?</td>\n",
       "      <td>You think Tom's gonna wait for me?</td>\n",
       "      <td>MT</td>\n",
       "      <td>[Not Hallucination, Not Hallucination, Not Hal...</td>\n",
       "      <td>Not Hallucination</td>\n",
       "      <td>0.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6</td>\n",
       "      <td>Два брата довольно разные.</td>\n",
       "      <td>The two brothers are pretty different.</td>\n",
       "      <td>There's a lot of friends.</td>\n",
       "      <td>MT</td>\n",
       "      <td>[Hallucination, Hallucination, Hallucination, ...</td>\n",
       "      <td>Hallucination</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>7</td>\n",
       "      <td>&lt;define&gt; Infradiaphragmatic &lt;/define&gt; intra- a...</td>\n",
       "      <td>(medicine) Below the diaphragm.</td>\n",
       "      <td>(anatomy) Relating to the diaphragm.</td>\n",
       "      <td>DM</td>\n",
       "      <td>[Hallucination, Hallucination, Hallucination, ...</td>\n",
       "      <td>Hallucination</td>\n",
       "      <td>0.8</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id                                                src  \\\n",
       "0   1  Ты удивишься, если я скажу, что на самом деле ...   \n",
       "1   2                                   Еды будет полно.   \n",
       "2   3                     Думаете, Том будет меня ждать?   \n",
       "3   6                         Два брата довольно разные.   \n",
       "4   7  <define> Infradiaphragmatic </define> intra- a...   \n",
       "\n",
       "                                                 tgt  \\\n",
       "0  Would you be surprised if I told you my name i...   \n",
       "1                      There will be plenty of food.   \n",
       "2            Do you think that Tom will wait for me?   \n",
       "3             The two brothers are pretty different.   \n",
       "4                    (medicine) Below the diaphragm.   \n",
       "\n",
       "                                                 hyp task  \\\n",
       "0  You're gonna be surprised if I say my real nam...   MT   \n",
       "1                             The food will be full.   MT   \n",
       "2                 You think Tom's gonna wait for me?   MT   \n",
       "3                          There's a lot of friends.   MT   \n",
       "4               (anatomy) Relating to the diaphragm.   DM   \n",
       "\n",
       "                                              labels              label  \\\n",
       "0  [Not Hallucination, Not Hallucination, Not Hal...  Not Hallucination   \n",
       "1  [Hallucination, Not Hallucination, Hallucinati...      Hallucination   \n",
       "2  [Not Hallucination, Not Hallucination, Not Hal...  Not Hallucination   \n",
       "3  [Hallucination, Hallucination, Hallucination, ...      Hallucination   \n",
       "4  [Hallucination, Hallucination, Hallucination, ...      Hallucination   \n",
       "\n",
       "   p(Hallucination)  \n",
       "0               0.0  \n",
       "1               0.8  \n",
       "2               0.2  \n",
       "3               1.0  \n",
       "4               0.8  "
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['test']['agnostic'].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "task\n",
       "DM      563\n",
       "MT      562\n",
       "PG      375\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['test']['agnostic'][['task']].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Test data (aware)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 1500 entries, 0 to 1499\n",
      "Data columns (total 8 columns):\n",
      " #   Column            Non-Null Count  Dtype  \n",
      "---  ------            --------------  -----  \n",
      " 0   id                1500 non-null   int64  \n",
      " 1   src               1500 non-null   object \n",
      " 2   tgt               1500 non-null   object \n",
      " 3   hyp               1500 non-null   object \n",
      " 4   task              1500 non-null   object \n",
      " 5   labels            1500 non-null   object \n",
      " 6   label             1500 non-null   object \n",
      " 7   p(Hallucination)  1500 non-null   float64\n",
      "dtypes: float64(1), int64(1), object(6)\n",
      "memory usage: 93.9+ KB\n"
     ]
    }
   ],
   "source": [
    "df['test']['aware'].info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>src</th>\n",
       "      <th>tgt</th>\n",
       "      <th>hyp</th>\n",
       "      <th>task</th>\n",
       "      <th>labels</th>\n",
       "      <th>label</th>\n",
       "      <th>p(Hallucination)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>Sadly , at the end of his life , Luther became...</td>\n",
       "      <td>Crazy person ; crackpot or lunatic .</td>\n",
       "      <td>A crazy person .</td>\n",
       "      <td>DM</td>\n",
       "      <td>[Not Hallucination, Not Hallucination, Not Hal...</td>\n",
       "      <td>Not Hallucination</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4</td>\n",
       "      <td>इतिहासकार लोगन FBI के पिछला नीति के बहुत आलोचन...</td>\n",
       "      <td>Historians have criticized past FBI policies f...</td>\n",
       "      <td>Historians have strongly criticized the FBI's ...</td>\n",
       "      <td>MT</td>\n",
       "      <td>[Not Hallucination, Not Hallucination, Not Hal...</td>\n",
       "      <td>Not Hallucination</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5</td>\n",
       "      <td>Àwọn erekùsù ìlà oòrùn adúláwọ̀ wà ní òkun Ind...</td>\n",
       "      <td>The East African Islands are in the Indian Oce...</td>\n",
       "      <td>The eastern islands of the Indian Ocean are lo...</td>\n",
       "      <td>MT</td>\n",
       "      <td>[Not Hallucination, Hallucination, Hallucinati...</td>\n",
       "      <td>Hallucination</td>\n",
       "      <td>0.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>9</td>\n",
       "      <td>Smith was first introduced into society at the...</td>\n",
       "      <td>High society .</td>\n",
       "      <td>The social group of a person , especially a wo...</td>\n",
       "      <td>DM</td>\n",
       "      <td>[Hallucination, Hallucination, Hallucination, ...</td>\n",
       "      <td>Hallucination</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>11</td>\n",
       "      <td>Ngini mamakai roket raksasa malabihi 100 kaki ...</td>\n",
       "      <td>It takes a giant rocket over a 100 feet high t...</td>\n",
       "      <td>It uses a giant rocket over 100 feet high to l...</td>\n",
       "      <td>MT</td>\n",
       "      <td>[Hallucination, Not Hallucination, Not Halluci...</td>\n",
       "      <td>Not Hallucination</td>\n",
       "      <td>0.4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id                                                src  \\\n",
       "0   0  Sadly , at the end of his life , Luther became...   \n",
       "1   4  इतिहासकार लोगन FBI के पिछला नीति के बहुत आलोचन...   \n",
       "2   5  Àwọn erekùsù ìlà oòrùn adúláwọ̀ wà ní òkun Ind...   \n",
       "3   9  Smith was first introduced into society at the...   \n",
       "4  11  Ngini mamakai roket raksasa malabihi 100 kaki ...   \n",
       "\n",
       "                                                 tgt  \\\n",
       "0               Crazy person ; crackpot or lunatic .   \n",
       "1  Historians have criticized past FBI policies f...   \n",
       "2  The East African Islands are in the Indian Oce...   \n",
       "3                                     High society .   \n",
       "4  It takes a giant rocket over a 100 feet high t...   \n",
       "\n",
       "                                                 hyp task  \\\n",
       "0                                   A crazy person .   DM   \n",
       "1  Historians have strongly criticized the FBI's ...   MT   \n",
       "2  The eastern islands of the Indian Ocean are lo...   MT   \n",
       "3  The social group of a person , especially a wo...   DM   \n",
       "4  It uses a giant rocket over 100 feet high to l...   MT   \n",
       "\n",
       "                                              labels              label  \\\n",
       "0  [Not Hallucination, Not Hallucination, Not Hal...  Not Hallucination   \n",
       "1  [Not Hallucination, Not Hallucination, Not Hal...  Not Hallucination   \n",
       "2  [Not Hallucination, Hallucination, Hallucinati...      Hallucination   \n",
       "3  [Hallucination, Hallucination, Hallucination, ...      Hallucination   \n",
       "4  [Hallucination, Not Hallucination, Not Halluci...  Not Hallucination   \n",
       "\n",
       "   p(Hallucination)  \n",
       "0               0.0  \n",
       "1               0.0  \n",
       "2               0.6  \n",
       "3               1.0  \n",
       "4               0.4  "
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['test']['aware'].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "task\n",
       "MT      563\n",
       "DM      562\n",
       "PG      375\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['test']['aware'][['task']].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Some check"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "i = -1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0                     Не волнуйся. Это только временно.\n",
       "1             Тома никогда нет там, где он должен быть.\n",
       "2                          Мне сложно работать с Томом.\n",
       "3                                     Воду, пожалуйста.\n",
       "4                    Я не ожидал, что Том предаст меня.\n",
       "5                     Всё не так плохо, как думает Том.\n",
       "6                                        Не усложняйте.\n",
       "7     Почему вы не можете принять меня таким, какой ...\n",
       "8                                   Ненавижу это место.\n",
       "9             Том находится там в качестве добровольца.\n",
       "10                                 Можно мне тоже одну?\n",
       "11              Ты в жизни не такой, как на фотографии.\n",
       "12          Она начала осваиваться в своем новом офисе.\n",
       "13                               У меня сейчас нет дел.\n",
       "14                Экватор делит землю на два полушария.\n",
       "15                       Я хотел бы стать христианином.\n",
       "16                           Все будут тебе завидовать.\n",
       "17                        Том купил себе новое пианино.\n",
       "18                      Вы знали, что у Тома три брата?\n",
       "19                  Не давайте Тому прыгать на кровати.\n",
       "20                     Впредь обещаю быть пунктуальным.\n",
       "21                            Поделите это между собой.\n",
       "22                                   У Тома есть выбор.\n",
       "23      Мы надеемся, что нам не понадобится это делать.\n",
       "24                                 Как ты его уговорил?\n",
       "25              Том видел Мэри, но ничего ей не сказал.\n",
       "26                        В зиме мне не нравится холод.\n",
       "27                   Том познакомился с Мэри год назад.\n",
       "28                Мне нужна не белая, а цветная бумага.\n",
       "29               Том покупает одежду в торговом центре.\n",
       "Name: src, dtype: object"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "i += 1\n",
    "step = 30\n",
    "df['train']['agnostic'].loc[df['train']['agnostic']['task'] == 'MT', 'src'].iloc[step*i:step*(i+1)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}