XLM-RoBERTa_training_with_human_data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "7c775dc7-34a2-417f-baa2-af0d30eb8485",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-04-23 11:32:05.941500: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
      "To enable the following instructions: AVX2 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2024-04-23 11:32:26.089420: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n"
     ]
    }
   ],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "import transformers\n",
    "from transformers import pipeline\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import torch\n",
    "from transformers import XLMRobertaForTokenClassification, XLMRobertaTokenizerFast, TrainingArguments, Trainer\n",
    "from transformers import DataCollatorForTokenClassification\n",
    "from transformers import AutoModelForTokenClassification\n",
    "from transformers import AutoTokenizer\n",
    "from torch.utils.data import TensorDataset, DataLoader\n",
    "from torch.nn.utils.rnn import pad_sequence\n",
    "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from datasets import Dataset, load_dataset\n",
    "from datasets import load_metric\n",
    "import pandas as pd\n",
    "import csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "463a78d5-1d06-402e-a17c-a4e40224d233",
   "metadata": {},
   "outputs": [],
   "source": [
    "list_of_training_data = [\"\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "a2434322-2a3c-43e0-86d5-80088d53001c",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_file_path = \"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "e8ffaad6-96ac-44e4-8b04-744136486abe",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "len output: 47978\n",
      "len output:test: 41201\n"
     ]
    }
   ],
   "source": [
    "# provided by supervisor and modified by student\n",
    "output = []\n",
    "output_test = []\n",
    "\n",
    "for train_file in list_of_training_data:\n",
    "    with open(train_file, 'r') as csvfile:\n",
    "        reader = csv.reader(csvfile, delimiter=\"\\t\")\n",
    "        output += list(reader)\n",
    "        \n",
    "with open(test_file_path, 'r') as csvfile:\n",
    "    reader2 = csv.reader(csvfile, delimiter=\"\\t\")\n",
    "    output_test = list(reader2)\n",
    "     \n",
    "\n",
    "output = [row for row in output]\n",
    "output_test = [row for row in output_test]\n",
    "\n",
    "print(\"len output:\", len(output))\n",
    "print(\"len output:test:\", len(output_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "b770d5ec-a799-4a74-aa1e-4cbbd9e465a6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# provided by supervisor and modified by student\n",
    "total = []\n",
    "temporary = []\n",
    "      \n",
    "for entry in output:\n",
    "    if len(entry) == 2:\n",
    "        text = entry[0]\n",
    "        label = entry[1]\n",
    "        if label == '0':\n",
    "            label = 'O'\n",
    "        if label in ['SMOKING', 'DISEASE', 'Smoking', 'Disease', 'label']:\n",
    "            label = 'O'\n",
    "        if text not in ['SEP', 'CLS']:\n",
    "            temporary.append([text, label])\n",
    "    if len(entry) < 2:\n",
    "        total.append(temporary)\n",
    "        temporary = []\n",
    "        \n",
    "total_test = []\n",
    "temporary_test = []\n",
    "\n",
    "for entry in output_test:\n",
    "    if len(entry) == 2:\n",
    "        text = entry[0]\n",
    "        label = entry[1]\n",
    "        if label in ['0', 'BREAST']:\n",
    "            label = 'O'\n",
    "        if label in ['FAMILY']:\n",
    "            label = 'O'\n",
    "        if text not in ['SEP', 'CLS']:\n",
    "            temporary_test.append([text, label])\n",
    "    if len(entry) < 2:\n",
    "        total_test.append(temporary_test)\n",
    "        temporary_test = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "8a75cd83-5050-4ed3-a6a2-8b595b65750c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# provided by supervisor\n",
    "labels_ = []\n",
    "labels_test_ = []\n",
    "\n",
    "for x in total:\n",
    "    for y in x:\n",
    "        label_ = y[1]\n",
    "        labels_.append(label_)\n",
    "        \n",
    "for x in total_test:\n",
    "    for y in x:\n",
    "        label_test = y[1]\n",
    "        labels_test_.append(label_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "cd36c28a-672b-4d12-924f-26356671f75d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# provided by supervisor\n",
    "nplabels = np.array(labels_)\n",
    "nplabels_test = np.array(labels_test_)\n",
    "\n",
    "unique_values = np.unique(nplabels, return_counts=True)\n",
    "unique_values_test = np.unique(nplabels_test, return_counts=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "19a092e6-7f20-4a77-844f-9acb6f508e84",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train:\n",
      "DRUG - 243\n",
      "O - 45031\n",
      "PROCEDURE - 2204\n",
      "\n",
      "test:\n",
      "DRUG - 656\n",
      "O - 39061\n",
      "PROCEDURE - 1184\n"
     ]
    }
   ],
   "source": [
    "# provided by supervisor\n",
    "counts_ = unique_values[1]\n",
    "names_ = unique_values[0]\n",
    "\n",
    "counts_test_ = unique_values_test[1]\n",
    "names_test_ = unique_values_test[0]\n",
    "\n",
    "print(\"train:\")\n",
    "for k in range(len(counts_)):\n",
    "    print(names_[k], \"-\", counts_[k])\n",
    "\n",
    "print()\n",
    "print(\"test:\")\n",
    "for k in range(len(counts_test_)):\n",
    "    print(names_test_[k], \"-\", counts_test_[k])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "fc599e8b-5dfc-4375-ba98-b2c1ad99f7fb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['DRUG', 'O', 'PROCEDURE']\n",
      "['DRUG', 'O', 'PROCEDURE']\n"
     ]
    }
   ],
   "source": [
    "# provided by supervisor\n",
    "label_list = list(names_)\n",
    "label_list_test = list(names_test_)\n",
    "\n",
    "label2id = {label: i for i, label in enumerate(label_list)}\n",
    "id2label = {i: label for i, label in enumerate(label_list)}\n",
    "\n",
    "label2id_test = {label: i for i, label in enumerate(label_list_test)}\n",
    "id2label_test = {i: label for i, label in enumerate(label_list_test)}\n",
    "\n",
    "print(label_list)\n",
    "print(label_list_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "05d2cb6e-6022-4541-8851-46e985efd731",
   "metadata": {},
   "outputs": [],
   "source": [
    "# provided by supervisor and modified by student\n",
    "split_85 = int(len(total) * 0.85)\n",
    "validation_total = total[split_85:]\n",
    "training_total = total[:split_85]\n",
    "test_total = total_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "3ce04858-71f9-4e6d-a1d3-1c22093adaf4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# provided by supervisor\n",
    "from datasets import Dataset\n",
    "import pandas as pd\n",
    "\n",
    "def get_all_tokens_and_ner_tags(texts):\n",
    "    return pd.concat([get_tokens_and_ner_tags(texts)]).reset_index().drop('index', axis=1)\n",
    "    \n",
    "def get_tokens_and_ner_tags(texts):\n",
    "    all_tokens = []\n",
    "    all_entities = []\n",
    "    for row in texts:\n",
    "        tokens = [x[0] for x in row]\n",
    "        entities = [x[1] for x in row]\n",
    "        \n",
    "        all_tokens.append(tokens)\n",
    "        all_entities.append(entities)\n",
    "    return pd.DataFrame({'tokens': all_tokens, 'ner_tags': all_entities})\n",
    "\n",
    "def create_sliding_windows(train_dataset, max_length=512, stride=512):\n",
    "    windowed_dataset = []\n",
    "    for data in train_dataset:\n",
    "        tokens = data[\"tokens\"]\n",
    "        ner_tags = data[\"ner_tags\"]\n",
    "        for i in range(0, len(tokens), stride):\n",
    "            window_tokens = tokens[i:i+max_length]\n",
    "            window_ner_tags = ner_tags[i:i+max_length]\n",
    "            windowed_data = {\"tokens\": window_tokens, \"ner_tags\": window_ner_tags}\n",
    "            windowed_dataset.append(windowed_data)\n",
    "    windowss=pd.DataFrame(windowed_dataset)\n",
    "    return Dataset.from_pandas(windowss)\n",
    "\n",
    "def get_un_token_dataset(train_directory, test_directory, validation_directory):\n",
    "    train_df = get_all_tokens_and_ner_tags(train_directory)\n",
    "    test_df = get_all_tokens_and_ner_tags(test_directory)\n",
    "    validation_df = get_all_tokens_and_ner_tags(validation_directory)\n",
    "    \n",
    "    train_dataset = Dataset.from_pandas(train_df)\n",
    "    test_dataset = Dataset.from_pandas(test_df)\n",
    "    validation_dataset = Dataset.from_pandas(validation_df)\n",
    "    \n",
    "    # Apply sliding window on train_dataset\n",
    "    train_dataset = create_sliding_windows(train_dataset)\n",
    "    test_dataset = create_sliding_windows(test_dataset)\n",
    "    validation_dataset = create_sliding_windows(validation_dataset)\n",
    "    \n",
    "    return (train_dataset, test_dataset, validation_dataset)\n",
    "\n",
    "train_dataset, test_dataset, validation_dataset = get_un_token_dataset(training_total, test_total, validation_total)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "adde54ad-466f-489e-acbf-adc6e9de44c3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "425\n",
      "300\n",
      "75\n"
     ]
    }
   ],
   "source": [
    "# provided by supervisor\n",
    "print(len(train_dataset))\n",
    "print(len(test_dataset))\n",
    "print(len(validation_dataset))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c6280ef9-3df0-4a4c-ab4e-d720fa2008a0",
   "metadata": {},
   "source": [
    "## BERT model with GPT annotations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "a7478886-44b3-43b8-9b4e-baccf436db31",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at xlm-roberta-base were not used when initializing XLMRobertaForTokenClassification: ['lm_head.bias', 'lm_head.layer_norm.bias', 'lm_head.dense.bias', 'lm_head.layer_norm.weight', 'lm_head.dense.weight']\n",
      "- This IS expected if you are initializing XLMRobertaForTokenClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing XLMRobertaForTokenClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of XLMRobertaForTokenClassification were not initialized from the model checkpoint at xlm-roberta-base and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "source": [
    "# provided by supervisor\n",
    "model = XLMRobertaForTokenClassification.from_pretrained('xlm-roberta-base', num_labels=len(label_list), id2label=id2label, label2id=label2id)\n",
    "tokenizer = XLMRobertaTokenizerFast.from_pretrained('xlm-roberta-base', max_length=512)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "3128fc02-9188-4c84-ba7a-92dff54ea272",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'DRUG': 0, 'O': 1, 'PROCEDURE': 2}\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "30daf89b00e8443696034c279fb8a244",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/425 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "87d0cfbde4414855a7b47e513c26ecfa",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/300 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dd3e5c1780b3441bac6303dbba58212f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/75 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# provided by supervisor\n",
    "task = \"ner\"\n",
    "\n",
    "def tokenize_and_align_labels(examples):\n",
    "    label_all_tokens = True\n",
    "    tokenized_inputs = tokenizer(list(examples[\"tokens\"]), truncation=True, is_split_into_words=True)\n",
    "\n",
    "    labels = []\n",
    "    for i, label in enumerate(examples[f\"{task}_tags\"]):\n",
    "        word_ids = tokenized_inputs.word_ids(batch_index=i)\n",
    "        previous_word_idx = None\n",
    "        label_ids = []\n",
    "        for word_idx in word_ids:\n",
    "            if word_idx is None:\n",
    "                label_ids.append(-100)\n",
    "            elif label[word_idx] == '0':\n",
    "                label_ids.append(0)\n",
    "            elif word_idx != previous_word_idx:\n",
    "                label_ids.append(label2id[label[word_idx]])\n",
    "            else:\n",
    "                label_ids.append(label2id[label[word_idx]] if label_all_tokens else -100)\n",
    "            previous_word_idx = word_idx\n",
    "        labels.append(label_ids)\n",
    "        \n",
    "    tokenized_inputs[\"labels\"] = labels\n",
    "    return tokenized_inputs\n",
    "\n",
    "print(label2id)\n",
    "\n",
    "train_tokenized_datasets = train_dataset.map(tokenize_and_align_labels, batched=True)\n",
    "test_tokenized_datasets = test_dataset.map(tokenize_and_align_labels, batched=True)\n",
    "validation_tokenized_datasets = validation_dataset.map(tokenize_and_align_labels, batched=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "6547eb54-a7dc-45f3-95c4-80a3ee1c1e8c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Detected kernel version 3.10.0, which is below the recommended minimum of 5.5.0; this can cause the process to hang. It is recommended to upgrade the kernel to the minimum version or higher.\n",
      "You're using a XLMRobertaTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='216' max='216' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [216/216 05:25, Epoch 8/8]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Recall</th>\n",
       "      <th>F1</th>\n",
       "      <th>Accuracy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.265281</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.904668</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.189764</td>\n",
       "      <td>0.310976</td>\n",
       "      <td>0.395349</td>\n",
       "      <td>0.348123</td>\n",
       "      <td>0.936860</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.178163</td>\n",
       "      <td>0.278226</td>\n",
       "      <td>0.534884</td>\n",
       "      <td>0.366048</td>\n",
       "      <td>0.925577</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.161002</td>\n",
       "      <td>0.460123</td>\n",
       "      <td>0.581395</td>\n",
       "      <td>0.513699</td>\n",
       "      <td>0.944416</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.160546</td>\n",
       "      <td>0.538462</td>\n",
       "      <td>0.651163</td>\n",
       "      <td>0.589474</td>\n",
       "      <td>0.949281</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.161559</td>\n",
       "      <td>0.505747</td>\n",
       "      <td>0.682171</td>\n",
       "      <td>0.580858</td>\n",
       "      <td>0.948453</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.160036</td>\n",
       "      <td>0.540373</td>\n",
       "      <td>0.674419</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>0.950730</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>No log</td>\n",
       "      <td>0.163369</td>\n",
       "      <td>0.543750</td>\n",
       "      <td>0.674419</td>\n",
       "      <td>0.602076</td>\n",
       "      <td>0.951454</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# provided by supervisor\n",
    "batch_size = 16\n",
    "\n",
    "args = TrainingArguments(\n",
    "    f\"test-{task}\",\n",
    "    evaluation_strategy = \"epoch\",\n",
    "    learning_rate=2e-5,\n",
    "    per_device_train_batch_size=batch_size,\n",
    "    per_device_eval_batch_size=batch_size,\n",
    "    num_train_epochs=8,\n",
    "    weight_decay=0.01\n",
    ")\n",
    "\n",
    "data_collator = DataCollatorForTokenClassification(tokenizer)\n",
    "\n",
    "\n",
    "metric = load_metric(\"seqeval\")\n",
    "results_collection = []\n",
    "\n",
    "\n",
    "def compute_metrics(p):\n",
    "    predictions, labels = p\n",
    "    predictions = np.argmax(predictions, axis=2)\n",
    "\n",
    "    true_predictions = [[label_list[p] for (p, l) in zip(prediction, label) if l != -100] for prediction, label in zip(predictions, labels)]\n",
    "    true_labels = [[label_list[l] for (p, l) in zip(prediction, label) if l != -100] for prediction, label in zip(predictions, labels)]\n",
    "    \n",
    "\n",
    "    results = metric.compute(predictions=true_predictions, references=true_labels)\n",
    "    results_collection.append(results)\n",
    "    return {\"precision\": results[\"overall_precision\"], \"recall\": results[\"overall_recall\"], \"f1\": results[\"overall_f1\"], \"accuracy\": results[\"overall_accuracy\"]}\n",
    "    \n",
    "\n",
    "\n",
    "trainer = Trainer(\n",
    "    model,\n",
    "    args,\n",
    "    train_dataset=train_tokenized_datasets,\n",
    "    eval_dataset=validation_tokenized_datasets,\n",
    "    data_collator=data_collator,\n",
    "    tokenizer=tokenizer,\n",
    "    compute_metrics=compute_metrics\n",
    ")\n",
    "\n",
    "trainer.train()\n",
    "trainer.evaluate()\n",
    "trainer.save_model('xlmr_finetune_human_annot.model')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "878348c2-b35f-43b8-97d3-6415b454fe6f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'ROCEDURE': {'precision': 0.2521489971346705,\n",
       "  'recall': 0.15602836879432624,\n",
       "  'f1': 0.19277108433734938,\n",
       "  'number': 564},\n",
       " 'RUG': {'precision': 0.3082191780821918,\n",
       "  'recall': 0.2631578947368421,\n",
       "  'f1': 0.2839116719242902,\n",
       "  'number': 342},\n",
       " 'overall_precision': 0.27769110764430577,\n",
       " 'overall_recall': 0.19646799116997793,\n",
       " 'overall_f1': 0.23012281835811246,\n",
       " 'overall_accuracy': 0.9533766392940312}"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# provided by supervisor\n",
    "predictions, labels, _ = trainer.predict(test_tokenized_datasets)\n",
    "predictions = np.argmax(predictions, axis=2)\n",
    "\n",
    "# Remove ignored index (special tokens)\n",
    "true_predictions = [\n",
    "    [label_list[p] for (p, l) in zip(prediction, label) if l != -100]\n",
    "    for prediction, label in zip(predictions, labels)\n",
    "]\n",
    "true_labels = [\n",
    "    [label_list_test[l] for (p, l) in zip(prediction, label) if l != -100]\n",
    "    for prediction, label in zip(predictions, labels)\n",
    "]\n",
    "\n",
    "results = metric.compute(predictions=true_predictions, references=true_labels)\n",
    "results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6347f9c8-9d85-4f8c-b76b-0c9f24bd0e6a",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.19"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}