SciBERTForTokenCLS.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "5f4f74d3-5fef-4786-ab6b-c7296f240615",
   "metadata": {},
   "source": [
    "# **Fine-tuning SciBERT for named-entity recognition**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "90e028e0-376b-4f1f-9aa3-54cb80b5115e",
   "metadata": {},
   "outputs": [],
   "source": [
    "#!pip install seqeval"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8ad8c0d4-3187-47ef-9c1d-9c251c1a2571",
   "metadata": {},
   "outputs": [],
   "source": [
    "#!pip install tensorflow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "859f362e-cabc-47e0-bd92-84ce117b6870",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/shjiang/miniconda3/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n",
      "2023-09-25 17:05:47.069223: I tensorflow/core/util/port.cc:110] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n",
      "2023-09-25 17:05:47.121021: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
      "To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2023-09-25 17:05:48.145018: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cuda\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.metrics import accuracy_score\n",
    "import torch\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "from transformers import pipeline, AutoModelForTokenClassification, get_cosine_schedule_with_warmup, AutoTokenizer\n",
    "from seqeval.metrics import classification_report\n",
    "import math\n",
    "import os\n",
    "from torch import cuda\n",
    "device = 'cuda' if cuda.is_available() else 'cpu'\n",
    "print(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "654de60e-abe4-4bec-8010-5e1cb4549d6e",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mon Sep 25 17:05:49 2023       \n",
      "+-----------------------------------------------------------------------------+\n",
      "| NVIDIA-SMI 510.47.03    Driver Version: 510.47.03    CUDA Version: 11.6     |\n",
      "|-------------------------------+----------------------+----------------------+\n",
      "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
      "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
      "|                               |                      |               MIG M. |\n",
      "|===============================+======================+======================|\n",
      "|   0  Tesla V100-PCIE...  Off  | 00000000:3B:00.0 Off |                    0 |\n",
      "| N/A   61C    P0   204W / 250W |  27183MiB / 32768MiB |     75%      Default |\n",
      "|                               |                      |                  N/A |\n",
      "+-------------------------------+----------------------+----------------------+\n",
      "|   1  Tesla V100-PCIE...  Off  | 00000000:AF:00.0 Off |                    0 |\n",
      "| N/A   68C    P0   185W / 250W |   2261MiB / 32768MiB |     70%      Default |\n",
      "|                               |                      |                  N/A |\n",
      "+-------------------------------+----------------------+----------------------+\n",
      "                                                                               \n",
      "+-----------------------------------------------------------------------------+\n",
      "| Processes:                                                                  |\n",
      "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n",
      "|        ID   ID                                                   Usage      |\n",
      "|=============================================================================|\n",
      "|    0   N/A  N/A    122228      C   python                           2257MiB |\n",
      "|    0   N/A  N/A    130333      C   julia                           24922MiB |\n",
      "|    1   N/A  N/A    122496      C   python                           2257MiB |\n",
      "+-----------------------------------------------------------------------------+\n"
     ]
    }
   ],
   "source": [
    "!nvidia-smi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "1c235a2e-b218-433d-8394-49e64398cc29",
   "metadata": {},
   "outputs": [],
   "source": [
    "TRAIN_BATCH_SIZE = 16\n",
    "VALID_BATCH_SIZE = 16\n",
    "EPOCHS = 5#3#20\n",
    "LEARNING_RATE = 5e-5 #1e-05\n",
    "MAX_GRAD_NORM = 10\n",
    "MAX_LEN = 256\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d79babb2-90d0-4e0d-a65b-0e4e6af351a9",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true
   },
   "source": [
    "# Preprocessing the dataset\n",
    "\n",
    "Named entity recognition (NER) uses a specific annotation scheme, which is defined (at least for European languages) at the word level. An annotation scheme that is widely used is called IOB-tagging, which stands for Inside-Outside-Beginning. Each tag indicates whether the corresponding word is inside, outside or at the beginning of a specific named entity. The reason this is used is because named entities usually comprise more than 1 word.\n",
    "\n",
    "Let's have a look at an example. If you have a sentence like \"Barack Obama was born in Hawaï\", then the corresponding tags would be [B-PERS, I-PERS, O, O, O, B-GEO]. B-PERS means that the word \"Barack\" is the beginning of a person, I-PERS means that the word \"Obama\" is inside a person, \"O\" means that the word \"was\" is outside a named entity, and so on. So one typically has as many tags as there are words in a sentence."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "fd49a5c3-b9fc-4d40-867a-9b9175ae13f8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "13692"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#df = pd.read_csv(\"cleaned_plain-text_labeled_term+combined_no_ref_no_cit_def_same_len_only.csv\", delimiter=',')\n",
    "df = pd.read_csv(\"cleaned_plain-text_labeled_term+combined_no_ref_no_cit_def_same_len_only_must_conatin_B.csv\", delimiter=',')\n",
    "len(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dec0d7b3-6c92-4225-afff-f2c4ca3c7df2",
   "metadata": {},
   "outputs": [],
   "source": [
    "all_data = df[['plain_text_def','labeled_def']].copy()\n",
    "\n",
    "all_data.rename(columns={\"plain_text_def\": \"sentence\", \"labeled_def\": \"word_labels\" }, inplace=True)\n",
    "\n",
    "all_data['word_labels'] = all_data['word_labels'].str.replace('I_MATH_TERM','I-MATH_TERM')\n",
    "all_data['word_labels'] = all_data['word_labels'].str.replace('B_MATH_TERM','B-MATH_TERM')\n",
    "\n",
    "\n",
    "data = all_data[:8192] #make a small sample first\n",
    "data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9e9a1c11-92c4-4ade-bf5d-a57cff09e7ab",
   "metadata": {},
   "outputs": [],
   "source": [
    "gen_data = all_data[8192:9216]\n",
    "gen_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f5461baf-e0d0-4ed2-a1f9-d2391079897c",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "e3761565-81a4-4b6f-95d7-5b67156d53f4",
   "metadata": {},
   "source": [
    "# Preparing the dataset and dataloader\n",
    "\n",
    "Now that our data is preprocessed, we can turn it into PyTorch tensors such that we can provide it to the model. Let's start by defining some key variables that will be used later on in the training/evaluation process:\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "324c5cdb-31c3-4135-b3a4-db3a15bc5f2d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def tokenize_and_preserve_labels(sentence, text_labels, tokenizer):\n",
    "    \"\"\"\n",
    "    Word piece tokenization makes it difficult to match word labels\n",
    "    back up with individual word pieces. This function tokenizes each\n",
    "    word one at a time so that it is easier to preserve the correct\n",
    "    label for each subword. It is, of course, a bit slower in processing\n",
    "    time, but it will help our model achieve higher accuracy.\n",
    "    \"\"\"\n",
    "\n",
    "    tokenized_sentence = []\n",
    "    labels = []\n",
    "\n",
    "    for word, label in zip(sentence.split(), text_labels.split()):\n",
    "\n",
    "        # Tokenize the word and count # of subwords the word is broken into\n",
    "        tokenized_word = tokenizer.tokenize(word)\n",
    "        n_subwords = len(tokenized_word)\n",
    "\n",
    "        # Add the tokenized word to the final tokenized word list\n",
    "        tokenized_sentence.extend(tokenized_word)\n",
    "\n",
    "        # Add the same label to the new list of labels `n_subwords` times\n",
    "        labels.extend([label] * n_subwords)\n",
    "\n",
    "    return tokenized_sentence, labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "1b750e01-db44-4825-9eb2-6f6cdbbc73b2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'B-MATH_TERM': 0, 'I-MATH_TERM': 1, 'O': 2}"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labels = ['B-MATH_TERM', 'I-MATH_TERM', 'O']\n",
    "\n",
    "label2id = { label : labels.index(label) for label in labels}\n",
    "\n",
    "id2label = { labels.index(label) : label for label in labels}\n",
    "\n",
    "label2id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "09be9b4a-0479-42b7-8048-268ba11ef9db",
   "metadata": {},
   "outputs": [],
   "source": [
    "class dataset(Dataset):\n",
    "    def __init__(self, dataframe, tokenizer, max_len):\n",
    "        self.len = len(dataframe)\n",
    "        self.data = dataframe\n",
    "        self.tokenizer = tokenizer\n",
    "        self.max_len = max_len\n",
    "        \n",
    "    def __getitem__(self, index):\n",
    "        # step 1: tokenize (and adapt corresponding labels)\n",
    "        sentence = self.data.sentence[index]  \n",
    "        word_labels = self.data.word_labels[index]  \n",
    "        tokenized_sentence, labels = tokenize_and_preserve_labels(sentence, word_labels, self.tokenizer)\n",
    "        \n",
    "        # step 2: add special tokens (and corresponding labels)\n",
    "        tokenized_sentence = [\"<s> \"] + tokenized_sentence + [\" </s>\"] # add special tokens of Roberta\n",
    "        labels.insert(0, \"O\") # add outside label for [CLS] token\n",
    "        labels.append(\"O\") # add outside label for [SEP] token\n",
    "\n",
    "        # step 3: truncating/padding\n",
    "        maxlen = self.max_len\n",
    "\n",
    "        if (len(tokenized_sentence) > maxlen):\n",
    "          # truncate\n",
    "          tokenized_sentence = tokenized_sentence[:maxlen]\n",
    "          labels = labels[:maxlen]\n",
    "        else:\n",
    "          # pad\n",
    "          tokenized_sentence = tokenized_sentence + ['<pad>'for _ in range(maxlen - len(tokenized_sentence))]\n",
    "          labels = labels + [\"O\" for _ in range(maxlen - len(labels))]\n",
    "\n",
    "        # step 4: obtain the attention mask\n",
    "        attn_mask = [1 if tok != '<pad>' else 0 for tok in tokenized_sentence] #modifié selon https://huggingface.co/docs/transformers/v4.21.1/en/model_doc/camembert\n",
    "        \n",
    "        # step 5: convert tokens to input ids\n",
    "        ids = self.tokenizer.convert_tokens_to_ids(tokenized_sentence)\n",
    "\n",
    "        label_ids = [label2id[label] for label in labels]\n",
    "        # the following line is deprecated\n",
    "        #label_ids = [label if label != 0 else -100 for label in label_ids]\n",
    "        \n",
    "        return {\n",
    "              'ids': torch.tensor(ids, dtype=torch.long),\n",
    "              'mask': torch.tensor(attn_mask, dtype=torch.long),\n",
    "              #'token_type_ids': torch.tensor(token_ids, dtype=torch.long),\n",
    "              'targets': torch.tensor(label_ids, dtype=torch.long)\n",
    "        } \n",
    "    \n",
    "    def __len__(self):\n",
    "        return self.len"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ec45ff33-0dba-406d-9923-faefb5257a97",
   "metadata": {},
   "outputs": [],
   "source": [
    "#splite dataset and load for the first time\n",
    "\"\"\"train_size = 0.8\n",
    "train_dataset = data.sample(frac=train_size,random_state=200)\n",
    "val_dataset = data.drop(train_dataset.index).reset_index(drop=True)\n",
    "train_dataset = train_dataset.reset_index(drop=True)\n",
    "\n",
    "#save the data sets\n",
    "train_dataset.to_csv('data/train.csv', index=False)\n",
    "val_dataset.to_csv('data/val.csv', index=False)\n",
    "gen_data.to_csv('data/test.csv', index=False)\n",
    "\n",
    "print(\"FULL TrainigDataset: {}\".format(data.shape))\n",
    "print(\"TRAIN Dataset: {}\".format(train_dataset.shape))\n",
    "print(\"VALIDATION Dataset: {}\".format(val_dataset.shape))\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c9ea79fd-1ee7-4a4d-be0b-4514a90cb99e",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"training_set = dataset(train_dataset, tokenizer, MAX_LEN)\n",
    "validation_set = dataset(val_dataset, tokenizer, MAX_LEN)\n",
    "test_generalizability_set = dataset(gen_data, tokenizer, MAX_LEN)\"\"\"\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0235941b-d79f-49fa-adb5-c43261cade29",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true
   },
   "source": [
    "## verify tokenization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "1a00be62-d42f-47c3-9a47-e996565e6d5d",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Downloading (…)lve/main/config.json: 100%|█████████████████████████████████████████████████████████████████████████████████| 385/385 [00:00<00:00, 2.40MB/s]\n",
      "Downloading (…)solve/main/vocab.txt: 100%|███████████████████████████████████████████████████████████████████████████████| 228k/228k [00:00<00:00, 1.41MB/s]\n"
     ]
    }
   ],
   "source": [
    "tokenizer = AutoTokenizer.from_pretrained(\"allenai/scibert_scivocab_uncased\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "8ed4cf78-72b0-428e-ac97-280a980a355d",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Downloading pytorch_model.bin: 100%|█████████████████████████████████████████████████████████████████████████████████████| 442M/442M [00:05<00:00, 86.4MB/s]\n",
      "Some weights of the model checkpoint at allenai/scibert_scivocab_uncased were not used when initializing BertForTokenClassification: ['cls.seq_relationship.bias', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.decoder.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.dense.bias', 'cls.seq_relationship.weight', 'cls.predictions.decoder.weight']\n",
      "- This IS expected if you are initializing BertForTokenClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertForTokenClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of BertForTokenClassification were not initialized from the model checkpoint at allenai/scibert_scivocab_uncased and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "BertForTokenClassification(\n",
       "  (bert): BertModel(\n",
       "    (embeddings): BertEmbeddings(\n",
       "      (word_embeddings): Embedding(31090, 768, padding_idx=0)\n",
       "      (position_embeddings): Embedding(512, 768)\n",
       "      (token_type_embeddings): Embedding(2, 768)\n",
       "      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "      (dropout): Dropout(p=0.1, inplace=False)\n",
       "    )\n",
       "    (encoder): BertEncoder(\n",
       "      (layer): ModuleList(\n",
       "        (0-11): 12 x BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (intermediate_act_fn): GELUActivation()\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (dropout): Dropout(p=0.1, inplace=False)\n",
       "  (classifier): Linear(in_features=768, out_features=3, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model = AutoModelForTokenClassification.from_pretrained(\"allenai/scibert_scivocab_uncased\",\n",
    "                                                        num_labels=len(id2label),\n",
    "                                                        id2label=id2label,\n",
    "                                                        label2id=label2id)\n",
    "model.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "304859c2-61cd-4574-a786-4652c4536907",
   "metadata": {},
   "outputs": [],
   "source": [
    "validation_set = dataset(pd.read_csv('data/val.csv'), tokenizer, MAX_LEN)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "441c5831-0188-4d4b-b737-f2ae455ec207",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'ids': tensor([  101,   106,  1338,  5945,  3346,   165,   106,  2147,   145,   106,\n",
       "           422,   105,   546,   422,   582,   106,   165,   106,  3427,   610,\n",
       "           422,   137,   105,   862,   170,  7273,   106,   101,   260,   170,\n",
       "          1901,   170, 30194,  1342,   244,   422,   158,  3661,   165,   106,\n",
       "          2605,   263,  5829,   191,   106,  6643,   260,   170,  1901,   147,\n",
       "          1342,   244,   422,   158,  3661,   422,   582,   260,   170,  1901,\n",
       "           862,   275,  1342,   158,   422,   170,  3661,   422,   334,   165,\n",
       "          6433,   190,  2646,   147, 19971,   131,   111,   158,   137,   170,\n",
       "          8855,   205,   906,   843,  5810,   610,   171,   422,   185,  2977,\n",
       "           106,  5945,  2605,   105,  7273,   145,   171,   546,   862,   170,\n",
       "          7273,   106,   101,   171, 30132, 30194,   170,  7273,   171, 30132,\n",
       "           214, 11364,   140,   130,  3555,   145,   171,  4627,   158,   422,\n",
       "           171,  4627,   170,   546,   121,   171,   147,  5631,   121,   105,\n",
       "          7273,   145,   171,   546,   145,   159,   546,   168,   693,   159,\n",
       "          1027,   170,  7273,   106,   101,   171, 30132,   543,   137,   617,\n",
       "           543,   105,   145,  1342,   244,   422,   158,  3661,  4627,   170,\n",
       "          7273,   145,   101,  4627,   106,   101,   546,   145,   159,   546,\n",
       "           546,   275,   158,   422,   582,   101,  4627,   106,   101,   862,\n",
       "           106,   101,   260,   170,  1901,  2402,   106,   101,  1342,   171,\n",
       "          4627,   158,   422,   171,  4627,   170,  3661,   165,   111,  2605,\n",
       "           334,   165,   111,  4770,   191,   106,   137,  5086,   158,   422,\n",
       "           170,   147,   171,  4627,   158,   422,   171,  4627,   170,  1222,\n",
       "           205,   101,   101,   101,   101,   101,   101,   101,   101,   101,\n",
       "           101,   101,   101,   101,   101,   101,   101,   101,   101,   101,\n",
       "           101,   101,   101,   101,   101,   101,   101,   101,   101,   101,\n",
       "           101,   101,   101,   101,   101,   101]),\n",
       " 'mask': tensor([1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "         1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n",
       "         1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]),\n",
       " 'targets': tensor([2, 2, 0, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,\n",
       "         2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,\n",
       "         2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,\n",
       "         2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,\n",
       "         2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,\n",
       "         2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,\n",
       "         2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,\n",
       "         2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,\n",
       "         2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,\n",
       "         2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,\n",
       "         2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2])}"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "validation_set[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "a7982e26-c359-4d93-a293-251545e9bdec",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[UNK]            O\n",
      "for              O\n",
      "k                O\n",
      "##≥              O\n",
      "2                O\n",
      ",                O\n",
      "a                O\n",
      "k                B-MATH_TERM\n",
      "-                B-MATH_TERM\n",
      "tensor           B-MATH_TERM\n",
      "with             O\n",
      "entries          O\n",
      "in               O\n",
      "is               O\n",
      "a                O\n",
      "function         O\n",
      "t                O\n",
      ":                O\n",
      "{                O\n",
      "1                O\n",
      ",                O\n",
      ".                O\n",
      ".                O\n",
      ".                O\n",
      ",                O\n",
      "d                O\n",
      "}                O\n",
      "^                O\n",
      "[UNK]            O\n",
      ".                O\n",
      "we               O\n",
      "refer            O\n",
      "to               O\n",
      "the              O\n",
      "number           O\n",
      "k                O\n",
      "as               O\n",
      "the              O\n",
      "order            O\n",
      "of               O\n",
      "the              O\n",
      "tensor           O\n",
      "t                O\n",
      ".                O\n",
      "we               O\n",
      "denote           O\n",
      "by               O\n",
      "t                O\n",
      "_                O\n",
      "i                O\n"
     ]
    }
   ],
   "source": [
    "# print the first 50 tokens and corresponding labels\n",
    "for token, label in zip(tokenizer.convert_ids_to_tokens(validation_set[15][\"ids\"][:50]), validation_set[15][\"targets\"][:50]):\n",
    "  print('{0:15}  {1}'.format(token, id2label[label.item()]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "19cbcaa9-5be6-4513-9ad5-7465eb667fc7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[UNK]            O\n",
      "the              O\n",
      "mixed            B-MATH_TERM\n",
      "volume           I-MATH_TERM\n",
      "(                O\n",
      "p                O\n",
      "_                O\n",
      "1                O\n",
      ",                O\n",
      "[UNK]            O\n",
      ",                O\n",
      "p                O\n",
      "_                O\n",
      "n                O\n",
      ")                O\n",
      "is               O\n",
      "the              O\n",
      "coefficient      O\n",
      "of               O\n",
      "the              O\n",
      "monomial         O\n",
      "_                O\n",
      "1                O\n",
      "[UNK]            O\n",
      "_                O\n",
      "n                O\n",
      "in               O\n",
      "the              O\n",
      "polynomial       O\n",
      "(                O\n"
     ]
    }
   ],
   "source": [
    "# print the first 50 tokens and corresponding labels\n",
    "for token, label in zip(tokenizer.convert_ids_to_tokens(validation_set[49][\"ids\"][:30]), validation_set[49][\"targets\"][:30]):\n",
    "  print('{0:15}  {1}'.format(token, id2label[label.item()]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "4dfa6f5c-d8a6-46e9-973f-89b9e63bd899",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "intial loss = 0.9741547703742981\n"
     ]
    }
   ],
   "source": [
    "# 3 labels: -ln(1/3) = 1.09861228867\n",
    "ids = validation_set[0][\"ids\"].unsqueeze(0)\n",
    "mask = validation_set[0][\"mask\"].unsqueeze(0)\n",
    "targets = validation_set[0][\"targets\"].unsqueeze(0)\n",
    "\n",
    "ids = ids.to(device)#, dtype = torch.long)\n",
    "mask = mask.to(device)#, dtype = torch.long)\n",
    "targets = targets.to(device)#, dtype = torch.long)\n",
    "model.to(device)\n",
    "outputs = model(input_ids=ids, attention_mask=mask, labels=targets)\n",
    "initial_loss = outputs[0]\n",
    "\n",
    "print(f\"intial loss = {initial_loss.item()}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "78edbdb5-869f-4f4e-ad04-216bd6e83cba",
   "metadata": {},
   "source": [
    "# Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "0e097121-0aa8-4e82-9aa0-261a782a273a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Defining the training function on the 80% of the dataset for tuning the bert model\n",
    "def train(model, training_loader, optimizer, scheduler=None):\n",
    "    tr_loss, tr_accuracy = 0, 0\n",
    "    nb_tr_examples, nb_tr_steps = 0, 0\n",
    "    tr_preds, tr_labels = [], []\n",
    "    # put model in training mode\n",
    "    model.train()\n",
    "    \n",
    "    for idx, batch in enumerate(training_loader):\n",
    "        \n",
    "        ids = batch['ids'].to(device, dtype = torch.long)\n",
    "        mask = batch['mask'].to(device, dtype = torch.long)\n",
    "        targets = batch['targets'].to(device, dtype = torch.long)\n",
    "\n",
    "        outputs = model(input_ids=ids, attention_mask=mask, labels=targets)\n",
    "        loss, tr_logits = outputs.loss, outputs.logits\n",
    "        '''\n",
    "        loss, tr_logits  = model(input_ids=ids, attention_mask=mask, labels=targets)#temporary modification for transformer 3'''\n",
    "        \n",
    "        tr_loss += loss.item()\n",
    "\n",
    "        nb_tr_steps += 1\n",
    "        nb_tr_examples += targets.size(0)\n",
    "        \n",
    "        #if idx % 100==0:\n",
    "        #    loss_step = tr_loss/nb_tr_steps\n",
    "        #    print(f\"Training loss per 100 training steps: {loss_step}\")\n",
    "           \n",
    "        # compute training accuracy\n",
    "        flattened_targets = targets.view(-1) # shape (batch_size * seq_len,)\n",
    "        active_logits = tr_logits.view(-1, model.num_labels) # shape (batch_size * seq_len, num_labels)\n",
    "        flattened_predictions = torch.argmax(active_logits, axis=1) # shape (batch_size * seq_len,)\n",
    "        # now, use mask to determine where we should compare predictions with targets (includes [CLS] and [SEP] token predictions)\n",
    "        active_accuracy = mask.view(-1) == 1 # active accuracy is also of shape (batch_size * seq_len,)\n",
    "        targets = torch.masked_select(flattened_targets, active_accuracy)\n",
    "        predictions = torch.masked_select(flattened_predictions, active_accuracy)\n",
    "        \n",
    "        tr_preds.extend(predictions)\n",
    "        tr_labels.extend(targets)\n",
    "        \n",
    "        tmp_tr_accuracy = accuracy_score(targets.cpu().numpy(), predictions.cpu().numpy())\n",
    "        tr_accuracy += tmp_tr_accuracy\n",
    "    \n",
    "        # gradient clipping\n",
    "        torch.nn.utils.clip_grad_norm_(\n",
    "            parameters=model.parameters(), max_norm=MAX_GRAD_NORM\n",
    "        )\n",
    "        \n",
    "        # backward pass\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        #scheduler.step()\n",
    "\n",
    "    epoch_loss = tr_loss / nb_tr_steps\n",
    "    tr_accuracy = tr_accuracy / nb_tr_steps\n",
    "    #print(f\"Trained {nb_tr_steps} steps\")\n",
    "    print(f\"Training loss epoch: {epoch_loss}\")\n",
    "    print(f\"Training accuracy epoch: {tr_accuracy}\")\n",
    "    \n",
    "\n",
    "def valid(model, validation_loader):\n",
    "    # put model in evaluation mode\n",
    "    model.eval()\n",
    "    \n",
    "    eval_loss, eval_accuracy = 0, 0\n",
    "    nb_eval_examples, nb_eval_steps = 0, 0\n",
    "    eval_preds, eval_labels = [], []\n",
    "    \n",
    "    with torch.no_grad():\n",
    "        for idx, batch in enumerate(validation_loader):\n",
    "            \n",
    "            ids = batch['ids'].to(device, dtype = torch.long)\n",
    "            mask = batch['mask'].to(device, dtype = torch.long)\n",
    "            targets = batch['targets'].to(device, dtype = torch.long)\n",
    "            \n",
    "           \n",
    "            outputs = model(input_ids=ids, attention_mask=mask, labels=targets)\n",
    "            loss, eval_logits = outputs.loss, outputs.logits\n",
    "            \n",
    "            eval_loss += loss.item()\n",
    "\n",
    "            nb_eval_steps += 1\n",
    "            nb_eval_examples += targets.size(0)\n",
    "        \n",
    "            #if idx % 100==0:\n",
    "            #    loss_step = eval_loss/nb_eval_steps\n",
    "            #    print(f\"Validation loss per 100 evaluation steps: {loss_step}\")\n",
    "              \n",
    "            # compute evaluation accuracy\n",
    "            flattened_targets = targets.view(-1) # shape (batch_size * seq_len,)\n",
    "            active_logits = eval_logits.view(-1, model.num_labels) # shape (batch_size * seq_len, num_labels)\n",
    "            flattened_predictions = torch.argmax(active_logits, axis=1) # shape (batch_size * seq_len,)\n",
    "            # now, use mask to determine where we should compare predictions with targets (includes [CLS] and [SEP] token predictions)\n",
    "            active_accuracy = mask.view(-1) == 1 # active accuracy is also of shape (batch_size * seq_len,)\n",
    "            targets = torch.masked_select(flattened_targets, active_accuracy)\n",
    "            predictions = torch.masked_select(flattened_predictions, active_accuracy)\n",
    "            \n",
    "            eval_labels.extend(targets)\n",
    "            eval_preds.extend(predictions)\n",
    "            \n",
    "            tmp_eval_accuracy = accuracy_score(targets.cpu().numpy(), predictions.cpu().numpy())\n",
    "            eval_accuracy += tmp_eval_accuracy\n",
    "    \n",
    "    #print(eval_labels)\n",
    "    #print(eval_preds)\n",
    "\n",
    "    labels = [id2label[id.item()] for id in eval_labels]\n",
    "    predictions = [id2label[id.item()] for id in eval_preds]\n",
    "\n",
    "    #print(labels)\n",
    "    #print(predictions)\n",
    "    \n",
    "    eval_loss = eval_loss / nb_eval_steps\n",
    "    eval_accuracy = eval_accuracy / nb_eval_steps\n",
    "    print(f\"Validation Loss: {eval_loss}\")\n",
    "    print(f\"Validation Accuracy: {eval_accuracy}\")\n",
    "\n",
    "    return labels, predictions\n",
    "\n",
    "def print_reports_to_csv(test_results, model_name, LEARNING_RATE, EPOCHS, trainset_num, report_type):\n",
    "    test_reports = []\n",
    "    for res in test_results:\n",
    "        report = classification_report([res['labels']], [res['predictions']], output_dict=True)\n",
    "        flattened_report = {str(k+'_'+v_k) : v_v for k,v in report.items() for v_k, v_v in v.items()  }\n",
    "        flattened_report['trainset_size'] = res['trainset_size']\n",
    "        flattened_report['model'] = res['model']\n",
    "        flattened_report['trainset_num'] = trainset_num\n",
    "        test_reports.append(flattened_report)\n",
    "    \n",
    "    df_test_reports = pd.DataFrame(test_reports)\n",
    "    if '/' in model_name:\n",
    "        model_name =  model_name.split('/')[1] \n",
    "    test_report_name = 'finetuning_results/'+report_type+'_'+ model_name + '_' + str(LEARNING_RATE) + '_16_' + str(EPOCHS) + '.csv'\n",
    "    df_test_reports.to_csv(test_report_name, mode='a', header=not os.path.exists(test_report_name),index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "e1d013d7-590a-4f18-b6d4-257566929bde",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TRAIN Dataset: (2048, 3)\n",
      "tranining steps: 641\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at allenai/scibert_scivocab_uncased were not used when initializing BertForTokenClassification: ['cls.predictions.transform.LayerNorm.bias', 'cls.predictions.decoder.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.decoder.weight', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.bias', 'cls.seq_relationship.bias', 'cls.seq_relationship.weight']\n",
      "- This IS expected if you are initializing BertForTokenClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertForTokenClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of BertForTokenClassification were not initialized from the model checkpoint at allenai/scibert_scivocab_uncased and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training epoch: 1\n",
      "Training loss epoch: 0.04844611239241203\n",
      "Training accuracy epoch: 0.9678432912651765\n",
      "Training epoch: 2\n",
      "Training loss epoch: 0.020968455166439526\n",
      "Training accuracy epoch: 0.983703942963952\n",
      "Training epoch: 3\n",
      "Training loss epoch: 0.013675421245352481\n",
      "Training accuracy epoch: 0.989592676644147\n",
      "Training epoch: 4\n",
      "Training loss epoch: 0.009473891661400557\n",
      "Training accuracy epoch: 0.9926497862402861\n",
      "Training epoch: 5\n",
      "Training loss epoch: 0.007030413659322221\n",
      "Training accuracy epoch: 0.9944624602169134\n",
      "Validation Loss: 0.04776241526573519\n",
      "Validation Accuracy: 0.9761473652797359\n",
      "Validation Loss: 0.03815797183779068\n",
      "Validation Accuracy: 0.9796851690607118\n",
      "TRAIN Dataset: (2048, 3)\n",
      "tranining steps: 641\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at allenai/scibert_scivocab_uncased were not used when initializing BertForTokenClassification: ['cls.predictions.transform.LayerNorm.bias', 'cls.predictions.decoder.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.decoder.weight', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.bias', 'cls.seq_relationship.bias', 'cls.seq_relationship.weight']\n",
      "- This IS expected if you are initializing BertForTokenClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertForTokenClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of BertForTokenClassification were not initialized from the model checkpoint at allenai/scibert_scivocab_uncased and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training epoch: 1\n",
      "Training loss epoch: 0.058820389705942944\n",
      "Training accuracy epoch: 0.9604216556715948\n",
      "Training epoch: 2\n",
      "Training loss epoch: 0.022941595834709005\n",
      "Training accuracy epoch: 0.9819973653900124\n",
      "Training epoch: 3\n",
      "Training loss epoch: 0.015313429965317482\n",
      "Training accuracy epoch: 0.9884186284275218\n",
      "Training epoch: 4\n",
      "Training loss epoch: 0.010788217494337005\n",
      "Training accuracy epoch: 0.9913672457265829\n",
      "Training epoch: 5\n",
      "Training loss epoch: 0.0070726598669352825\n",
      "Training accuracy epoch: 0.9945000259237932\n",
      "Validation Loss: 0.03503436975393303\n",
      "Validation Accuracy: 0.9805387904235249\n",
      "Validation Loss: 0.03286657855642261\n",
      "Validation Accuracy: 0.9806898786380154\n",
      "TRAIN Dataset: (2048, 3)\n",
      "tranining steps: 641\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at allenai/scibert_scivocab_uncased were not used when initializing BertForTokenClassification: ['cls.predictions.transform.LayerNorm.bias', 'cls.predictions.decoder.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.decoder.weight', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.bias', 'cls.seq_relationship.bias', 'cls.seq_relationship.weight']\n",
      "- This IS expected if you are initializing BertForTokenClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertForTokenClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of BertForTokenClassification were not initialized from the model checkpoint at allenai/scibert_scivocab_uncased and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training epoch: 1\n",
      "Training loss epoch: 0.05709518063667929\n",
      "Training accuracy epoch: 0.9628066551075274\n",
      "Training epoch: 2\n",
      "Training loss epoch: 0.02273244834577781\n",
      "Training accuracy epoch: 0.9827136953432788\n",
      "Training epoch: 3\n",
      "Training loss epoch: 0.0149981589820527\n",
      "Training accuracy epoch: 0.9885659663556894\n",
      "Training epoch: 4\n",
      "Training loss epoch: 0.009880044912279118\n",
      "Training accuracy epoch: 0.9923664779001723\n",
      "Training epoch: 5\n",
      "Training loss epoch: 0.006981975878261437\n",
      "Training accuracy epoch: 0.9947536690724958\n",
      "Validation Loss: 0.03910925377254622\n",
      "Validation Accuracy: 0.9794335863488893\n",
      "Validation Loss: 0.03597838876885362\n",
      "Validation Accuracy: 0.9800388700828142\n",
      "TRAIN Dataset: (2048, 3)\n",
      "tranining steps: 641\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at allenai/scibert_scivocab_uncased were not used when initializing BertForTokenClassification: ['cls.predictions.transform.LayerNorm.bias', 'cls.predictions.decoder.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.decoder.weight', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.bias', 'cls.seq_relationship.bias', 'cls.seq_relationship.weight']\n",
      "- This IS expected if you are initializing BertForTokenClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertForTokenClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of BertForTokenClassification were not initialized from the model checkpoint at allenai/scibert_scivocab_uncased and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training epoch: 1\n",
      "Training loss epoch: 0.0630019766103942\n",
      "Training accuracy epoch: 0.9615181089850545\n",
      "Training epoch: 2\n",
      "Training loss epoch: 0.022886105827637948\n",
      "Training accuracy epoch: 0.9824739555090163\n",
      "Training epoch: 3\n",
      "Training loss epoch: 0.015224765629682224\n",
      "Training accuracy epoch: 0.9880308409452386\n",
      "Training epoch: 4\n",
      "Training loss epoch: 0.010829122173163341\n",
      "Training accuracy epoch: 0.9915705122457537\n",
      "Training epoch: 5\n",
      "Training loss epoch: 0.008314699834954808\n",
      "Training accuracy epoch: 0.9933993202920588\n",
      "Validation Loss: 0.03772085444129343\n",
      "Validation Accuracy: 0.9773526047375296\n",
      "Validation Loss: 0.03185143136943225\n",
      "Validation Accuracy: 0.9804900197974016\n",
      "TRAIN Dataset: (2048, 3)\n",
      "tranining steps: 641\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at allenai/scibert_scivocab_uncased were not used when initializing BertForTokenClassification: ['cls.predictions.transform.LayerNorm.bias', 'cls.predictions.decoder.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.decoder.weight', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.bias', 'cls.seq_relationship.bias', 'cls.seq_relationship.weight']\n",
      "- This IS expected if you are initializing BertForTokenClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertForTokenClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of BertForTokenClassification were not initialized from the model checkpoint at allenai/scibert_scivocab_uncased and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training epoch: 1\n",
      "Training loss epoch: 0.05177912908766302\n",
      "Training accuracy epoch: 0.9649832322569791\n",
      "Training epoch: 2\n",
      "Training loss epoch: 0.02202035732989316\n",
      "Training accuracy epoch: 0.9833181066146075\n",
      "Training epoch: 3\n",
      "Training loss epoch: 0.014089848886214895\n",
      "Training accuracy epoch: 0.9892196498537408\n",
      "Training epoch: 4\n",
      "Training loss epoch: 0.009550886954457383\n",
      "Training accuracy epoch: 0.9925368256280244\n",
      "Training epoch: 5\n",
      "Training loss epoch: 0.007977945720995194\n",
      "Training accuracy epoch: 0.9937084144487606\n",
      "Validation Loss: 0.036355055167186484\n",
      "Validation Accuracy: 0.979898937442305\n",
      "Validation Loss: 0.032848001588718034\n",
      "Validation Accuracy: 0.9805130933975811\n",
      "TRAIN Dataset: (2048, 3)\n",
      "tranining steps: 641\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at allenai/scibert_scivocab_uncased were not used when initializing BertForTokenClassification: ['cls.predictions.transform.LayerNorm.bias', 'cls.predictions.decoder.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.decoder.weight', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.bias', 'cls.seq_relationship.bias', 'cls.seq_relationship.weight']\n",
      "- This IS expected if you are initializing BertForTokenClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertForTokenClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of BertForTokenClassification were not initialized from the model checkpoint at allenai/scibert_scivocab_uncased and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training epoch: 1\n",
      "Training loss epoch: 0.05095025615446502\n",
      "Training accuracy epoch: 0.9673619925936193\n",
      "Training epoch: 2\n",
      "Training loss epoch: 0.022006324008543743\n",
      "Training accuracy epoch: 0.9830956955920991\n",
      "Training epoch: 3\n",
      "Training loss epoch: 0.014493742663034936\n",
      "Training accuracy epoch: 0.9887617493141294\n",
      "Training epoch: 4\n",
      "Training loss epoch: 0.009856853849669278\n",
      "Training accuracy epoch: 0.9920178227729073\n",
      "Training epoch: 5\n",
      "Training loss epoch: 0.007324524503928842\n",
      "Training accuracy epoch: 0.9942374152947985\n",
      "Validation Loss: 0.034457067805754985\n",
      "Validation Accuracy: 0.9795227737966286\n",
      "Validation Loss: 0.03263825182511937\n",
      "Validation Accuracy: 0.9804562243486246\n",
      "TRAIN Dataset: (2048, 3)\n",
      "tranining steps: 641\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at allenai/scibert_scivocab_uncased were not used when initializing BertForTokenClassification: ['cls.predictions.transform.LayerNorm.bias', 'cls.predictions.decoder.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.decoder.weight', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.bias', 'cls.seq_relationship.bias', 'cls.seq_relationship.weight']\n",
      "- This IS expected if you are initializing BertForTokenClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertForTokenClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of BertForTokenClassification were not initialized from the model checkpoint at allenai/scibert_scivocab_uncased and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training epoch: 1\n",
      "Training loss epoch: 0.057803555915597826\n",
      "Training accuracy epoch: 0.9615648613080969\n",
      "Training epoch: 2\n",
      "Training loss epoch: 0.0228957394574536\n",
      "Training accuracy epoch: 0.9821086822990395\n",
      "Training epoch: 3\n",
      "Training loss epoch: 0.015904439012956573\n",
      "Training accuracy epoch: 0.9876584556254121\n",
      "Training epoch: 4\n",
      "Training loss epoch: 0.011088230294262758\n",
      "Training accuracy epoch: 0.9914407640341134\n",
      "Training epoch: 5\n",
      "Training loss epoch: 0.008046644165915495\n",
      "Training accuracy epoch: 0.9937124612596587\n",
      "Validation Loss: 0.03618266040907253\n",
      "Validation Accuracy: 0.979701159652315\n",
      "Validation Loss: 0.032646034931531176\n",
      "Validation Accuracy: 0.9811104876726261\n",
      "TRAIN Dataset: (2048, 3)\n",
      "tranining steps: 641\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at allenai/scibert_scivocab_uncased were not used when initializing BertForTokenClassification: ['cls.predictions.transform.LayerNorm.bias', 'cls.predictions.decoder.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.decoder.weight', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.bias', 'cls.seq_relationship.bias', 'cls.seq_relationship.weight']\n",
      "- This IS expected if you are initializing BertForTokenClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertForTokenClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of BertForTokenClassification were not initialized from the model checkpoint at allenai/scibert_scivocab_uncased and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training epoch: 1\n",
      "Training loss epoch: 0.05547658056457294\n",
      "Training accuracy epoch: 0.9650963314917158\n",
      "Training epoch: 2\n",
      "Training loss epoch: 0.022578924130357336\n",
      "Training accuracy epoch: 0.9830054567975999\n",
      "Training epoch: 3\n",
      "Training loss epoch: 0.015817198356671724\n",
      "Training accuracy epoch: 0.9879951033618254\n",
      "Training epoch: 4\n",
      "Training loss epoch: 0.010915683931671083\n",
      "Training accuracy epoch: 0.9915968420862908\n",
      "Training epoch: 5\n",
      "Training loss epoch: 0.008225162714552425\n",
      "Training accuracy epoch: 0.9935388494161135\n",
      "Validation Loss: 0.03418073483707407\n",
      "Validation Accuracy: 0.9788793191784099\n",
      "Validation Loss: 0.030708343372680247\n",
      "Validation Accuracy: 0.9808486514973249\n",
      "TRAIN Dataset: (2048, 3)\n",
      "tranining steps: 641\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at allenai/scibert_scivocab_uncased were not used when initializing BertForTokenClassification: ['cls.predictions.transform.LayerNorm.bias', 'cls.predictions.decoder.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.decoder.weight', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.bias', 'cls.seq_relationship.bias', 'cls.seq_relationship.weight']\n",
      "- This IS expected if you are initializing BertForTokenClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertForTokenClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of BertForTokenClassification were not initialized from the model checkpoint at allenai/scibert_scivocab_uncased and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training epoch: 1\n",
      "Training loss epoch: 0.05991003054805333\n",
      "Training accuracy epoch: 0.9610757670212086\n",
      "Training epoch: 2\n",
      "Training loss epoch: 0.022810118756751763\n",
      "Training accuracy epoch: 0.9827479355799866\n",
      "Training epoch: 3\n",
      "Training loss epoch: 0.015706655492977006\n",
      "Training accuracy epoch: 0.9874988343417955\n",
      "Training epoch: 4\n",
      "Training loss epoch: 0.011018938130291644\n",
      "Training accuracy epoch: 0.9912275344645863\n",
      "Training epoch: 5\n",
      "Training loss epoch: 0.008157872465744731\n",
      "Training accuracy epoch: 0.9938825397650449\n",
      "Validation Loss: 0.03432344572170626\n",
      "Validation Accuracy: 0.9793299507366534\n",
      "Validation Loss: 0.035128644143696874\n",
      "Validation Accuracy: 0.9799662017034344\n",
      "CPU times: user 33min 51s, sys: 12min 36s, total: 46min 28s\n",
      "Wall time: 46min 44s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "train_params = {'batch_size': TRAIN_BATCH_SIZE,\n",
    "                'shuffle': True,\n",
    "            'num_workers': 0\n",
    "            }\n",
    "\n",
    "val_params = {'batch_size': VALID_BATCH_SIZE,\n",
    "                'shuffle': True,\n",
    "                'num_workers': 0\n",
    "             }\n",
    "for trainset_num in range(2,11): \n",
    "\n",
    "    train_file_name = 'data/10-fold/train_499_'+str(trainset_num)+'.csv'#'data/train.csv'\n",
    "    val_file_name = 'data/10-fold/val_499_'+str(trainset_num)+'.csv'#'data/val.csv'\n",
    "    \n",
    "    for model_name in ['allenai/scibert_scivocab_uncased']:\n",
    "        tokenizer = AutoTokenizer.from_pretrained(model_name, from_tf=False, model_max_length=MAX_LEN)\n",
    "        \n",
    "        test_generalizability_set = dataset(pd.read_csv('data/test_GPT+labels.csv'), tokenizer, MAX_LEN)\n",
    "        \n",
    "        validation_set = dataset(pd.read_csv(val_file_name), tokenizer, MAX_LEN)\n",
    "        df_training_set = pd.read_csv(train_file_name)\n",
    "        \n",
    "        val_results = []\n",
    "        test_results = []\n",
    "        \n",
    "        validation_loader = DataLoader(validation_set, **val_params)\n",
    "        test_gen_loader = DataLoader(test_generalizability_set, **val_params)\n",
    "        \n",
    "        for trainsetsize in [2048]:  #[64,128,256,512,1024,2048,4096,8192,11401] are already done\n",
    "            training_set = dataset(df_training_set[:trainsetsize], tokenizer, MAX_LEN)\n",
    "        \n",
    "            print(\"TRAIN Dataset: {}\".format(training_set.data.shape))\n",
    "            #train_params['batch_size'] =  int( trainsetsize / 32) if (trainsetsize < 1024) else 16\n",
    "            training_loader = DataLoader(training_set, **train_params)\n",
    "        \n",
    "        \n",
    "            num_training_steps = int(training_loader.dataset.len / train_params['batch_size'] * EPOCHS)\n",
    "            print(f'tranining steps: {num_training_steps+1}')\n",
    "        \n",
    "            #Shrey uses TF model\n",
    "            model = AutoModelForTokenClassification.from_pretrained(model_name,\n",
    "                                                                    from_tf=False,\n",
    "                                                                    num_labels=len(id2label),\n",
    "                                                                    id2label=id2label,\n",
    "                                                                    label2id=label2id)\n",
    "            model.to(device)\n",
    "        \n",
    "            optimizer = torch.optim.Adam(params=model.parameters(), lr=LEARNING_RATE)\n",
    "            #scheduler = get_cosine_schedule_with_warmup(optimizer = optimizer, num_warmup_steps = 50, num_training_steps=num_training_steps)\n",
    "            for epoch in range(EPOCHS):\n",
    "            #for epoch in range(flex_epoch_nb): \n",
    "                print(f\"Training epoch: {epoch + 1}\")\n",
    "                train(model, training_loader, optimizer)\n",
    "                #valid(model, validation_loader)\n",
    "                #valid(model, test_gen_loader)\n",
    "            labels, predictions = valid(model, validation_loader)     \n",
    "            val_results.append({'trainset_size': trainsetsize, 'model': model_name, 'labels': labels, 'predictions': predictions})\n",
    "        \n",
    "            #test generalizablity\n",
    "            labels, predictions = valid(model, test_gen_loader)\n",
    "            test_results.append({'trainset_size': trainsetsize, 'model': model_name, 'labels': labels, 'predictions': predictions})\n",
    "            ner_model_name = 'ner_model/'+model_name+ '_ft_' + str(EPOCHS) + 'ep_train_size_'+str(trainsetsize) + '_trainset_'+str(trainset_num)\n",
    "            model.save_pretrained(ner_model_name)\n",
    "            tokenizer.save_pretrained(ner_model_name)\n",
    "            # gpt_aligned_eval(model, tokenizer, ner_model_name) # too slow!\n",
    "        \n",
    "        print_reports_to_csv(val_results, model_name, LEARNING_RATE, EPOCHS, trainset_num, 'validation')\n",
    "        print_reports_to_csv(test_results, model_name, LEARNING_RATE, EPOCHS, trainset_num, 'generalizability')\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "1d870653-2d78-40b8-a747-69a12b3b34f8",
   "metadata": {},
   "outputs": [],
   "source": [
    "def gpt_aligned_eval(model, tokenizer, model_name):\n",
    "    df_test_data = pd.read_csv('data/test_GPT+labels.csv')\n",
    "    eval_list = []\n",
    "    if '/' in model_name:\n",
    "            model_name =  model_name.split('/')[1] \n",
    "    pipe = pipeline(task=\"token-classification\", model=model.to('cpu'), tokenizer=tokenizer, aggregation_strategy=\"first\")\n",
    "    \n",
    "    for index, row in df_test_data.iterrows():\n",
    "    \n",
    "        #let's remove repeated terms, keeping 1616 unique out of 1660 terms\n",
    "        expected_list = set(row['plain_text_term'].split(';'))\n",
    "        while '' in expected_list:\n",
    "            expected_list.remove('')\n",
    "    \n",
    "        extracted_list = pipe(row['sentence'])\n",
    "        extracted_list = [x['word'].strip() for x in extracted_list] \n",
    "        while '' in extracted_list:\n",
    "            extracted_list.remove('')\n",
    "        \n",
    "        num_TP = 0\n",
    "        num_too_long = 0\n",
    "        num_cut_off = 0\n",
    "        num_split_term = 0\n",
    "        TP_list = []\n",
    "        ST_list = [] \n",
    "        \n",
    "        for expected in expected_list:\n",
    "            for extracted in extracted_list:\n",
    "                \n",
    "                if extracted.casefold() == expected.casefold():\n",
    "                    num_TP = num_TP + 1\n",
    "                    TP_list.append(expected)\n",
    "                elif extracted.casefold() in expected.casefold():\n",
    "                    num_cut_off = num_cut_off + 1\n",
    "                elif expected.casefold() in extracted.casefold():\n",
    "                    num_too_long = num_too_long + 1\n",
    "                    \n",
    "            expected_no_space = expected.replace(\" \",\"\")\n",
    "            extracted_no_space = (\"\".join(extracted_list)).replace(\" \",\"\")\n",
    "            if expected_no_space.casefold() in extracted_no_space.casefold(): # including TPs\n",
    "                num_split_term = num_split_term + 1\n",
    "                ST_list.append(expected)\n",
    "        \n",
    "        num_TP = num_TP - (len(TP_list) - len(set(TP_list)))\n",
    "        num_split_term = num_split_term - (len(ST_list) -len(set(ST_list)))\n",
    "        \n",
    "        eval_list.append({'True Term Num' : len(expected_list),\n",
    "                            'Extracted Term Num': len(extracted_list),\n",
    "                            'TP': num_TP,\n",
    "                            'Cut Off': num_cut_off,\n",
    "                            'Too Long': num_too_long,\n",
    "                            'Split Term': num_split_term,\n",
    "                            'extracted': '###'.join(extracted_list)})\n",
    "    df_eval = pd.DataFrame(eval_list)\n",
    "    df_eval['expected'] = df_test_data['plain_text_term']\n",
    "    eval_report_name = 'GPT_results/ft_'+model_name+'_first_eval.csv'\n",
    "    df_eval.to_csv(eval_report_name,index=False)\n",
    "\n",
    "    #print eval\n",
    "    print(f'ner model name: {model_name}')\n",
    "    num_T = df_eval['True Term Num'].sum()\n",
    "    print(\"True Term Num: \" + str(num_T))\n",
    "    num_Ex = df_eval['Extracted Term Num'].sum()\n",
    "    print(\"Extracted Term Num: \" + str(num_Ex))\n",
    "    print(\"True positive: \" + str(df_eval['TP'].sum()))\n",
    "    num_ST = df_eval['Split Term'].sum()\n",
    "    print(\"True positive + split terms: \" + str(num_ST))\n",
    "    print(\"Too Long: \" + str(df_eval['Too Long'].sum()))\n",
    "    print(\"Cut Off: \" + str(df_eval['Cut Off'].sum()))\n",
    "    print(\"precision /correct rate: \" + str(num_ST / num_Ex))\n",
    "    print(\"recall: \" + str(num_ST / num_T))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "0105ddef-86fb-47fb-a49a-55b1c42e1566",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/shjiang/miniconda3/lib/python3.11/site-packages/transformers/pipelines/token_classification.py:393: UserWarning: Tokenizer does not support real words, using fallback heuristic\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ner model name: roberta-base_ft_5ep_train_size_1024_trainset_1\n",
      "True Term Num: 1616\n",
      "Extracted Term Num: 2067\n",
      "True positive: 893\n",
      "True positive + split terms: 1100\n",
      "Too Long: 224\n",
      "Cut Off: 580\n",
      "precision /correct rate: 0.5321722302854378\n",
      "recall: 0.6806930693069307\n",
      "CPU times: user 24min 33s, sys: 3.53 s, total: 24min 36s\n",
      "Wall time: 3min 5s\n"
     ]
    }
   ],
   "source": [
    "#%time gpt_aligned_eval(model, tokenizer, ner_model_name) # Wall time: 3min 5s with gpu005"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "669f63a0-21d5-481e-b335-f5c09f73f15e",
   "metadata": {},
   "source": [
    "# Saving the model for future use"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a6929c5f-7639-454b-941a-00ce78a73cd9",
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save_pretrained('ner_model/'+model_name+ '_ft_' + str(EPOCHS) + 'ep')\n",
    "tokenizer.save_pretrained('ner_model/'+model_name+ '_ft_' + str(EPOCHS) + 'ep')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9d64e2c5-2281-495b-a2be-770608b03d04",
   "metadata": {},
   "source": [
    "# Inference\n",
    "\n",
    "The fun part is when we can quickly test the model on new, unseen sentences. Here, we use the prediction of the first word piece of every word. Note that the function we used to train our model (tokenze_and_preserve_labels) propagated the label to all subsequent word pieces (so you could for example also perform a majority vote on the predicted labels of all word pieces of a word).\n",
    "\n",
    "In other words, the code below does not take into account when predictions of different word pieces that belong to the same word do not match."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "e85f9e9e-fc65-4f43-a0bc-5af88e6b41a6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'entity_group': 'MATH_TERM',\n",
       "  'score': 0.9680205,\n",
       "  'word': ' B',\n",
       "  'start': 4,\n",
       "  'end': 5},\n",
       " {'entity_group': 'MATH_TERM',\n",
       "  'score': 0.9650901,\n",
       "  'word': 'etti poset',\n",
       "  'start': 5,\n",
       "  'end': 15}]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#model = AutoModelForTokenClassification.from_pretrained('NER_model_4/model_out/')\n",
    "pipe = pipeline(task=\"token-classification\", model=model.to('cpu'), tokenizer=tokenizer, aggregation_strategy=\"simple\")\n",
    "pipe(\"The Betti poset of a poset P is the subposet consisting of all homologically contributing elements, B(P)={q∈ P  | _i(Δ_q) ≠ 0  i}.\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "ba62b57b-bb24-4795-9b18-54f892dedd7d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'entity_group': 'MATH_TERM',\n",
       "  'score': 0.903038,\n",
       "  'word': ' trivial normal holonomy',\n",
       "  'start': 40,\n",
       "  'end': 63}]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pipe(\"A subskeleton (Γ_0,α_0,θ_0)⊆(Γ,α,θ) has trivial normal holonomy if the holonomy map K_γ^⊥ is trivial for all loops γ⊂Γ_0.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "61d632a9-cb25-440c-9d4a-e9fc9c9519bc",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_eval.to_csv('GPT_results/ft_roberta-base_ft_3ep_train_size_11366'+'_eval.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "311d7c31-c0dc-4d8f-8546-fafeae7bbdb4",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}