GPT_prompting_and_eval.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "c510bc17-c866-4be8-91b0-ed3c253525d5",
   "metadata": {},
   "outputs": [],
   "source": [
    "#!pip uninstall openai -y\n",
    "#!pip install openai==0.27.2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "9a7ce25b-748f-40de-ba4e-d9fe2d15f3bb",
   "metadata": {},
   "outputs": [],
   "source": [
    "import openai\n",
    "import torch\n",
    "import time\n",
    "import os\n",
    "import csv\n",
    "import json\n",
    "import re\n",
    "import random\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from collections import defaultdict \n",
    "#! pip install nltk\n",
    "from nltk.tokenize import TreebankWordTokenizer\n",
    "\n",
    "from seqeval.metrics import classification_report"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8d4e6196-a253-4b85-9110-89ef58f51d5a",
   "metadata": {},
   "source": [
    "# Cleaning the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "033eea1b-383d-485b-a590-9dd69bc497b4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# provided by supervisor\n",
    "data_file_path = ''\n",
    "\n",
    "texts = []\n",
    "\n",
    "# Open the TSV file using a context manager\n",
    "with open(data_file_path, 'r', newline='') as tsv_file:\n",
    "    # Create a CSV reader object with tab delimiter\n",
    "    tsv_reader = csv.reader(tsv_file, delimiter='\\t')\n",
    "    \n",
    "    # Iterate over each row in the TSV file\n",
    "    for row in tsv_reader:\n",
    "        # Each row is a list where elements are separated by tabs\n",
    "        texts.append(row)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "9a690de3-f09a-4434-8611-5b7155072f63",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "500\n"
     ]
    }
   ],
   "source": [
    "# provided by supervisor\n",
    "sep_texts = []\n",
    "temp = []\n",
    "\n",
    "for enum, i in enumerate(texts[1:]):\n",
    "    if len(i) < 2:\n",
    "        sep_texts.append(temp)\n",
    "        temp = []\n",
    "    else:\n",
    "        temp.append(i)\n",
    "\n",
    "print(len(sep_texts))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "55e5c510-6099-45a8-840c-fc1d32da9dfa",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "330\n",
      "170\n"
     ]
    }
   ],
   "source": [
    "# provided by supervisor\n",
    "cleaned_anamneesid = []\n",
    "cleaned_protseduurid = []\n",
    "\n",
    "for text in sep_texts:\n",
    "    text_type = \"\"\n",
    "    parsing = False\n",
    "    temp = []\n",
    "    for token in text:\n",
    "        if text_type == \"\": # determine whether it is anamnees or protseduur\n",
    "            if \"anamnees\" in token[0]:\n",
    "                text_type = \"a\"\n",
    "            if \"protseduur\" in token[0]:\n",
    "                text_type = \"p\"\n",
    "        \n",
    "        if parsing:\n",
    "            temp.append(token)\n",
    "        if '-' in token[0]: # start adding tokens after separator token is seen\n",
    "            parsing = True\n",
    "    \n",
    "    if text_type == 'p':\n",
    "        cleaned_protseduurid.append(temp)\n",
    "    elif text_type == 'a':\n",
    "        cleaned_anamneesid.append(temp)\n",
    "    else:\n",
    "        print(\"didnt match\")\n",
    "\n",
    "print(len(cleaned_anamneesid))\n",
    "print(len(cleaned_protseduurid))\n",
    "\n",
    "# remove first element that is a comma\n",
    "final_anamneesid = [i[1:] for i in cleaned_anamneesid]\n",
    "final_protseduurid = [i[1:] for i in cleaned_protseduurid]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "8d06db63-efa5-4fc0-9d1f-024a0ceb6a7d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "- tuli kaitsevä\n"
     ]
    }
   ],
   "source": [
    "# all of the texts in one list\n",
    "human_annotated = []\n",
    "sentences = []\n",
    "sent_count = 0\n",
    "for sentence in final_anamneesid:\n",
    "    temp = \" \".join([el[0] for el in sentence])\n",
    "    sentences.append((sent_count, temp))\n",
    "    human_annotated.append(sentence)\n",
    "    sent_count += 1\n",
    "\n",
    "for sentence in final_protseduurid:\n",
    "    temp = \" \".join([el[0] for el in sentence])\n",
    "    sentences.append((sent_count, temp))\n",
    "    human_annotated.append(sentence)\n",
    "    sent_count += 1\n",
    "print(sentences[0][0])\n",
    "print(sentences[0][1][:15]) # see if first symbols seem correct"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "59b49da3-7c5a-41f8-b462-7762d54a8431",
   "metadata": {},
   "source": [
    "# Looking at the classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "9f596b88-202e-4578-bdd2-bd8fd2273488",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'O': 28511, 'DISEASE': 451, 'SMOKING': 58, 'DRUG': 236, 'PROCEDURE': 787}\n",
      "{'O': 11425, 'DISEASE': 256, 'SMOKING': 0, 'DRUG': 7, 'PROCEDURE': 1246}\n"
     ]
    }
   ],
   "source": [
    "anamnees_tokens = {'O': 0, 'DISEASE': 0, 'SMOKING': 0, 'DRUG': 0, 'PROCEDURE': 0}\n",
    "protseduur_tokens = {'O': 0, 'DISEASE': 0, 'SMOKING': 0, 'DRUG': 0, 'PROCEDURE': 0}\n",
    "\n",
    "for text in final_anamneesid:\n",
    "    for token in text:\n",
    "        anamnees_tokens[token[1]] = anamnees_tokens[token[1]] + 1\n",
    "\n",
    "for text in final_protseduurid:\n",
    "    for token in text:\n",
    "        protseduur_tokens[token[1]] = protseduur_tokens[token[1]] + 1\n",
    "\n",
    "print(anamnees_tokens)\n",
    "print(protseduur_tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "d612d6ba-08c6-424c-98ac-7faad5e5dba4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# in this thesis SMOKING class is not used, it is counted as O\n",
    "anamnees_tokens['O'] = anamnees_tokens['O'] + anamnees_tokens['SMOKING']\n",
    "anamnees_tokens.pop('SMOKING')\n",
    "\n",
    "protseduur_tokens['O'] = protseduur_tokens['O'] + protseduur_tokens['SMOKING']\n",
    "protseduur_tokens.pop('SMOKING')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "bd93876a-a932-43c5-bbd4-20aa06458e05",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'O': 39994, 'DISEASE': 707, 'DRUG': 243, 'PROCEDURE': 2033}\n",
      "Tokens 42977\n"
     ]
    }
   ],
   "source": [
    "# both text types are counted as one in this thesis\n",
    "sum_tokens = {}\n",
    "for key in anamnees_tokens:\n",
    "    sum_tokens[key] = anamnees_tokens[key] + protseduur_tokens[key]\n",
    "print(sum_tokens)\n",
    "\n",
    "token_count = 0\n",
    "for key, value in sum_tokens.items():\n",
    "    token_count += value\n",
    "print(\"Tokens\", token_count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "119e7b02-88b9-4fc1-ab5c-cc295d5825a8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# statistics on how long the entities are in words\n",
    "entity_counts = defaultdict(int)\n",
    "for sent in human_annotated:\n",
    "    current_entity = None\n",
    "    word_count = 0\n",
    "    for word in sent:\n",
    "        tag = word[1]\n",
    "        if tag == current_entity:\n",
    "            current_entity = tag\n",
    "            word_count += 1\n",
    "        elif tag != \"SMOKING\" and tag != \"O\":\n",
    "            if current_entity is not None:\n",
    "                entity_counts[current_entity + \"-\" + str(word_count)] += 1\n",
    "            current_entity = tag\n",
    "            word_count = 1\n",
    "        else:# tag O or SMOKING (we don't count SMOKING)\n",
    "            if current_entity is not None:\n",
    "                # If we were counting an entity, save the count\n",
    "                entity_counts[current_entity + \"-\" + str(word_count)] += 1\n",
    "                current_entity = None\n",
    "                word_count = 0\n",
    "    \n",
    "    if current_entity is not None:\n",
    "        entity_counts[current_entity + \"-\" + str(word_count)] += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "335f1155-d73b-4574-93c2-ea827845739e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "defaultdict(int,\n",
       "            {'DRUG-1': 197,\n",
       "             'DISEASE-3': 32,\n",
       "             'PROCEDURE-2': 155,\n",
       "             'PROCEDURE-5': 40,\n",
       "             'PROCEDURE-4': 54,\n",
       "             'PROCEDURE-3': 142,\n",
       "             'DISEASE-2': 83,\n",
       "             'DISEASE-4': 19,\n",
       "             'DISEASE-5': 11,\n",
       "             'DRUG-2': 20,\n",
       "             'DISEASE-1': 160,\n",
       "             'PROCEDURE-1': 167,\n",
       "             'PROCEDURE-6': 29,\n",
       "             'PROCEDURE-7': 16,\n",
       "             'PROCEDURE-10': 12,\n",
       "             'DRUG-3': 2,\n",
       "             'DISEASE-8': 7,\n",
       "             'PROCEDURE-17': 1,\n",
       "             'PROCEDURE-11': 8,\n",
       "             'DISEASE-7': 4,\n",
       "             'DISEASE-11': 2,\n",
       "             'PROCEDURE-12': 7,\n",
       "             'PROCEDURE-9': 7,\n",
       "             'PROCEDURE-8': 5,\n",
       "             'DISEASE-17': 1,\n",
       "             'DISEASE-13': 1,\n",
       "             'DISEASE-6': 3,\n",
       "             'PROCEDURE-16': 1})"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# template is <entity name>-<length in words>\n",
    "# for example \"lung cancer\" is one disease entity that consists of two words and therefore goes under DISEASE-2\n",
    "entity_counts"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0c82c726-f788-43ae-a25f-bc011efdf51b",
   "metadata": {},
   "source": [
    "# Prompts with GPT"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "87dbce32-f831-4c82-9c35-6aab59619d80",
   "metadata": {},
   "source": [
    "## Setting up GPT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "93880353-5fca-4049-8874-df99ff345b06",
   "metadata": {},
   "outputs": [],
   "source": [
    "openai.api_type = \"\"\n",
    "openai.api_key = \"\"\n",
    "openai.api_base = \"\"\n",
    "openai.api_version = \"\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7a9500be-a73c-42b0-b340-618483b59b22",
   "metadata": {},
   "source": [
    "## Creating prompts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "353886e0-1ba9-46dc-b415-0e5fdcf7f78e",
   "metadata": {},
   "outputs": [],
   "source": [
    "base_prompt_1 = \"In the text below, give the list of: \"\n",
    "base_prompt_2 = \". Words need to be in exactly the same format as in input text. Format the output in JSON with the following keys: \"\n",
    "base_prompt_3 = \". Text below: \"\n",
    "# with base_prompt_3 if you use \"TEXT:\" or \"INPUT:\" then it counts this as a JSON key"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1784c46e-9492-4d16-bc5e-4a6fc787886a",
   "metadata": {},
   "source": [
    "There are 7 prompts and 3 temperatures which means that alltogether there are 21 prompts. Combining temperature and base_prompts is done in \"Parsing responses together\"."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "ab304816-f23d-4879-b540-351d21e1828b",
   "metadata": {},
   "outputs": [],
   "source": [
    "base_prompts = [\n",
    "    base_prompt_1 + \"drug named entity\" + base_prompt_2 + \"DRUG for drug named entity\" + base_prompt_3,\n",
    "    base_prompt_1 + \"procedure named entity\" + base_prompt_2 + \"PROCEDURE for procedure named entity\" + base_prompt_3,\n",
    "    base_prompt_1 + \"disease named entity\" + base_prompt_2 + \"DISEASE for disease named entity\" + base_prompt_3,\n",
    "    \n",
    "    base_prompt_1 + \"drug named entity, procedure named entity\" + base_prompt_2 + \"DRUG for drug named entity, PROCEDURE for procedure named entity\" + base_prompt_3,\n",
    "    base_prompt_1 + \"drug named entity, disease named entity\" + base_prompt_2 + \"DRUG for drug named entity, DISEASE for disease named entity\" + base_prompt_3,\n",
    "    base_prompt_1 + \"disease named entity, procedure named entity\" + base_prompt_2 + \"DISEASE for disease named entity, PROCEDURE for procedure named entity\" + base_prompt_3,\n",
    "    \n",
    "    base_prompt_1 + \"drug named entity, disease named entity, procedure named entity\" + base_prompt_2 + \"DRUG for drug named entity, DISEASE for disease named entity, PROCEDURE for procedure named entity\" + base_prompt_3\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "c28df474-a885-4de4-85f9-0d2be4a5074a",
   "metadata": {},
   "outputs": [],
   "source": [
    "log_names = [\n",
    "    \"drug_temperature_\", \n",
    "    \"procedure_temperature_\",\n",
    "    \"disease_temperature_\",\n",
    "\n",
    "    \"drug_procedure_temperature_\",\n",
    "    \"drug_disease_temperature_\",\n",
    "    \"disease_procedure_temperature_\",\n",
    "\n",
    "    \"drug_disease_procedure_temperature_\"\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "ef72ef73-8353-410d-bb33-87704b1737c6",
   "metadata": {},
   "outputs": [],
   "source": [
    "entity_names = [\n",
    "    [\"DRUG\"],\n",
    "    [\"PROCEDURE\"],\n",
    "    [\"DISEASE\"],\n",
    "\n",
    "    [\"DRUG\", \"PROCEDURE\"],\n",
    "    [\"DRUG\", \"DISEASE\"],\n",
    "    [\"DISEASE\", \"PROCEDURE\"],\n",
    "\n",
    "    [\"DRUG\", \"DISEASE\", \"PROCEDURE\"]\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "d647ed85-0090-461c-b2c8-8c87b1c60b8a",
   "metadata": {},
   "outputs": [],
   "source": [
    "temperatures = [0, 0.5, 1]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "decb3e3a-e17f-403d-8e85-4190e1efc712",
   "metadata": {},
   "source": [
    "## Running prompts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "dcca0b0c-ac96-4b98-8282-2093ec706701",
   "metadata": {},
   "outputs": [],
   "source": [
    "# provided by supervisor and modified by student\n",
    "def ask_openai(prompt: str, prompt_temperature: int):\n",
    "    try:\n",
    "        response = openai.ChatCompletion.create(\n",
    "            deployment_id = \"\",\n",
    "            model = \"gpt-35-turbo\",\n",
    "            temperature = prompt_temperature,\n",
    "            messages=[{\"role\": \"user\", \"content\": prompt}]\n",
    "        )\n",
    "    except Exception as e:\n",
    "        time.sleep(1) # sometimes the error is rate limit related and waiting a second can help\n",
    "        print(\"Error in ask_openai\")\n",
    "        print(e)\n",
    "        response = openai.ChatCompletion.create(\n",
    "            deployment_id = \"\",\n",
    "            model = \"gpt-35-turbo\",\n",
    "            temperature = prompt_temperature,\n",
    "            messages=[{\"role\": \"user\", \"content\": prompt}]\n",
    "        )\n",
    "    return response['choices'][0]['message']['content'], prompt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "2b5a7f64-2161-4319-a755-acb11cf21c9b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# provided by supervisor and modified by student\n",
    "def run_prompts(prompt: str, sents, prompt_temperature):\n",
    "    prompts = []\n",
    "    responses = []\n",
    "    went_through = []\n",
    "    \n",
    "    for enum, sent in sents:\n",
    "        try:\n",
    "            result1, result2 = ask_openai(prompt + \"\\n\" + \"\\\"\" + sent + \"\\\"\", prompt_temperature)\n",
    "            prompts.append(result2)\n",
    "            responses.append((enum, result1))\n",
    "            went_through.append(enum)\n",
    "        except Exception as e:\n",
    "            print(\"Error in run_prompts\")\n",
    "            print(e)\n",
    "    \n",
    "    print(\"With prompt (\" + str(prompt_temperature) + \" temperature): \" + prompt + str(len(went_through)) + \" went through\")\n",
    "    return prompts, responses"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "84671bf4-bc0c-4d2f-a26c-b90dcd48d48b",
   "metadata": {},
   "source": [
    "## Parsing prompts"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "513b8c05-a8f2-4ad6-bf27-7723a78d26a2",
   "metadata": {},
   "source": [
    "### Help functions for parsing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "bd37fefb-4862-430c-8873-f4fdb8dd8e03",
   "metadata": {},
   "outputs": [],
   "source": [
    "# provided by supervisor and modified by student\n",
    "def parse_prompt(clinical_text: str, response: str, entities):\n",
    "    results = []\n",
    "    json_data = json.loads(response)\n",
    "    \n",
    "    for entity in entities:\n",
    "        if entity in json_data:\n",
    "            if isinstance(json_data[entity], str): # when JSON element is a string not a list\n",
    "                json_data[entity] = [json_data[entity]]\n",
    "            elif not isinstance(json_data[entity], list): # when JSON element is something other than a string or a list\n",
    "                json_data[entity] = []\n",
    "                \n",
    "            for finding in json_data[entity]:\n",
    "                if finding in ['none', 'None']:\n",
    "                    pass\n",
    "                \n",
    "                pattern = re.compile(finding.lower())\n",
    "                \n",
    "                for match in pattern.finditer(clinical_text.lower()):\n",
    "                    start = match.start()\n",
    "                    end = match.end()\n",
    "                    #print(start, end, finding, entity)\n",
    "                    dict_result = {'entity_type': entity, 'start_idx': start, 'end_idx': end, 'text': clinical_text[start:end]}\n",
    "                    if dict_result not in results:\n",
    "                        results.append(dict_result)\n",
    "                    \n",
    "    return results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "4a030d1a-ef4f-4f81-aad0-c1f26570ba09",
   "metadata": {},
   "outputs": [],
   "source": [
    "# provided by supervisor\n",
    "def parsed_results_to_train(parsed_results, clinical_results, entities):\n",
    "    tokens_and_tags = []\n",
    "    res = list(TreebankWordTokenizer().span_tokenize(clinical_results))\n",
    "    dic = {k:v for k,v in zip(res, [clinical_results[i:j] for i, j in res])}\n",
    "    \n",
    "    for entry in dic:\n",
    "        token = dic[entry]\n",
    "        tags = []\n",
    "        \n",
    "        # print(entry, dic[entry])\n",
    "        tokenized_span_start = entry[0]\n",
    "        tokenized_span_end = entry[1]\n",
    "        \n",
    "        for parse in parsed_results:\n",
    "            parse_span_start = parse['start_idx']\n",
    "            parse_span_end = parse['end_idx']\n",
    "            parse_entity = parse['entity_type']\n",
    "            \n",
    "            if len(range(max(parse_span_start, tokenized_span_start), min(parse_span_end, tokenized_span_end))) > 0:\n",
    "                if parse_entity not in tags:\n",
    "                    tags.append(parse_entity)\n",
    "        \n",
    "        if len(tags) == 0:\n",
    "            tags.append('O')\n",
    "        \n",
    "        tokens_and_tags.append((token, tags))\n",
    "        \n",
    "    return tokens_and_tags"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "16e0b17f-448c-4fec-b5ae-b2a42a85b9fb",
   "metadata": {},
   "outputs": [],
   "source": [
    "# provided by supervisor and modified by student\n",
    "def save_list_to_file(int_list, filename):\n",
    "    with open(filename, 'a') as file:\n",
    "        for num in int_list:\n",
    "            file.write(str(num) + '\\n')\n",
    "\n",
    "def load_list_from_file(filename):\n",
    "    int_list = []\n",
    "    with open(filename, 'r') as file:\n",
    "        for line in file:\n",
    "            int_list.append(int(line.strip()))\n",
    "    return int_list\n",
    "\n",
    "def save_responses_to_file(responses, filename):\n",
    "    with open(filename, 'a') as file:\n",
    "        for enum, response in responses:\n",
    "            file.write(str(enum) + str(response) + '\\n')\n",
    "\n",
    "error_file_path = \"all_errors.txt\"\n",
    "def save_errors_to_file(err, prompt_name):\n",
    "    with open(error_file_path, 'a') as file:\n",
    "        file.write(str(err) + prompt_name + '\\n')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f8bea8eb-3c30-47b5-b56f-4c2154cc5c3d",
   "metadata": {},
   "source": [
    "### Parsing responses together"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "81356631-bcf7-4457-9949-97889b547400",
   "metadata": {},
   "outputs": [],
   "source": [
    "# provided by supervisor and modified by student\n",
    "def parsing_responses(responses, sents, log_name, entities):\n",
    "    successful_texts = []\n",
    "    parsed_answers = []\n",
    "    passed_ids = []\n",
    "    \n",
    "    save_responses_to_file(responses, log_name + \"_responses.txt\")# saving them into a file incase we want to further look into why something is not correct\n",
    "    \n",
    "    for res in responses:\n",
    "        row_number = res[0]\n",
    "        ans_ = res[1]\n",
    "        og_text = sents[row_number][1]\n",
    "        try:\n",
    "            parsed_answer = parse_prompt(og_text, ans_[ans_.find('{'):ans_.find('}')+1], entities)\n",
    "            parsed_answers.append(parsed_answer)\n",
    "            successful_texts.append(og_text)\n",
    "            passed_ids.append(row_number)\n",
    "        except Exception as e:\n",
    "            print(\"error in parsing_responses function\")\n",
    "            save_errors_to_file(str(e) + \"\\n in datarow \" + str(row_number) + \", model answer \" + ans_, log_name)\n",
    "            print(\"----------\")\n",
    "            print(e)\n",
    "            print()\n",
    "            print(og_text)\n",
    "            print()\n",
    "            print(ans_)\n",
    "            print(\"-------\")\n",
    "            pass\n",
    "    \n",
    "    print(\"Parsed:\", len(successful_texts), 'out of', len(responses))\n",
    "\n",
    "    save_list_to_file(passed_ids, log_name + \"_passed_ids.txt\")\n",
    "    passed_ids = load_list_from_file(log_name + \"_passed_ids.txt\")\n",
    "    \n",
    "    results_for_csv = []\n",
    "    for text_, results_ in zip(successful_texts, parsed_answers):\n",
    "        train_format_ = parsed_results_to_train(results_, text_, entities)\n",
    "        results_for_csv.append(train_format_)\n",
    "\n",
    "    with open(log_name + \".tsv\", 'a', newline='') as tsvfile:\n",
    "        writer = csv.writer(tsvfile, delimiter='\\t', lineterminator='\\n')\n",
    "        for res_ in results_for_csv:\n",
    "            for line in res_:\n",
    "                writer.writerow([line[0], line[1][0]])\n",
    "            writer.writerow([])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "200ba697-8c68-4ffe-a63a-412fc295c9b5",
   "metadata": {},
   "outputs": [],
   "source": [
    "outputdata_folder_path = \"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "a52ce06e-7abf-47db-aaf4-af703e87d766",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\ntest_sentences = sentences\\n\\nfor i in range(0, 1): # usually would be len(base_prompts)\\n    base_prompt = base_prompts[i]\\n    log_name_base = outputdata_folder_path + log_names[i]\\n    entities = entity_names[i]\\n    \\n    for temperature in temperatures:\\n        log_name = log_name_base + str(temperature)\\n        prompts, responses = run_prompts(base_prompt, test_sentences, temperature)\\n        parsing_responses(responses, test_sentences, log_name, entities)\\n    print()\\n'"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# uncomment when you want to get answers from openai and parse them\n",
    "'''\n",
    "test_sentences = sentences\n",
    "\n",
    "for i in range(0, 1): # usually would be len(base_prompts)\n",
    "    base_prompt = base_prompts[i]\n",
    "    log_name_base = outputdata_folder_path + log_names[i]\n",
    "    entities = entity_names[i]\n",
    "    \n",
    "    for temperature in temperatures:\n",
    "        log_name = log_name_base + str(temperature)\n",
    "        prompts, responses = run_prompts(base_prompt, test_sentences, temperature)\n",
    "        parsing_responses(responses, test_sentences, log_name, entities)\n",
    "    print()\n",
    "'''"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "947e75e7-8277-44df-bf58-5d6a0d4639cd",
   "metadata": {},
   "source": [
    "### Getting human and gpt annotations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "7748055b-4908-44fa-82ec-82460664f8de",
   "metadata": {},
   "outputs": [],
   "source": [
    "# provided by supervisor and modified by student\n",
    "def get_annotations(log_name):\n",
    "    gpt_annotations = []\n",
    "    with open(log_name + \".tsv\", 'r') as tsvfile:\n",
    "        reader = csv.reader(tsvfile, delimiter='\\t')\n",
    "        temp = []\n",
    "        for line in reader:\n",
    "            if len(line) < 2:\n",
    "                gpt_annotations.append(temp)\n",
    "                temp = []\n",
    "            else:\n",
    "                temp.append(line)\n",
    "                \n",
    "    comparable_human_annotations = []\n",
    "    passed_ids = load_list_from_file(log_name + \"_passed_ids.txt\")\n",
    "    failed_ids = []\n",
    "    \n",
    "    cnt = 0\n",
    "    unequal = []\n",
    "    for i in range(len(human_annotated)):\n",
    "        if i in passed_ids:\n",
    "            if len(human_annotated[i]) != len(gpt_annotations[cnt]):\n",
    "                #print(i)\n",
    "                unequal.append(cnt)\n",
    "            else:\n",
    "                comparable_human_annotations.append(human_annotated[i])\n",
    "            cnt +=1\n",
    "        else:\n",
    "            failed_ids.append(i)\n",
    "\n",
    "    for uneq in unequal:\n",
    "        #print(gpt_annotations[uneq])\n",
    "        gpt_annotations.pop(uneq)\n",
    "        failed_ids.append(uneq)\n",
    "    \n",
    "    human_only_labels = []\n",
    "    gpt_only_labels = []\n",
    "\n",
    "    for text in comparable_human_annotations:\n",
    "        temp = []\n",
    "        for pair in text:\n",
    "            # uncomment this when you want to exclude punctuation\n",
    "            #if pair[0] in '!\"#$%&\\'()*+,-./:;<=>?@[\\\\]^_`{|}~':\n",
    "                #temp.append(\"O\")\n",
    "            #elif pair[1] == \"SMOKING\":\n",
    "                #temp.append(\"O\")\n",
    "            #else:\n",
    "                #temp.append(pair[1])\n",
    "            # comment out this if-else when you use the if-else statements above\n",
    "            if pair[1] == \"SMOKING\":\n",
    "                temp.append(\"O\")\n",
    "            else:\n",
    "                temp.append(pair[1])\n",
    "        human_only_labels.append(temp)\n",
    "        \n",
    "    for text in gpt_annotations:\n",
    "        temp = []\n",
    "        for pair in text:\n",
    "            # uncomment this when you want to exclude punctuation\n",
    "            #if pair[0] in '!\"#$%&\\'()*+,-./:;<=>?@[\\\\]^_`{|}~':\n",
    "                #temp.append(\"O\")\n",
    "            #else:\n",
    "                #temp.append(pair[1])\n",
    "            # comment out row below when you use the if-else statements above\n",
    "            temp.append(pair[1])\n",
    "        gpt_only_labels.append(temp)\n",
    "\n",
    "    unequal = []\n",
    "    for i in range(len(human_only_labels)):\n",
    "        if len(human_only_labels[i]) != len(gpt_only_labels[i]):\n",
    "            unequal.append(i)\n",
    "            failed_ids.append(i)\n",
    "    \n",
    "    print(\"Failed to map:\", failed_ids)\n",
    "    \n",
    "    for uneq in unequal:\n",
    "        gpt_only_labels.pop(uneq)\n",
    "        human_only_labels.pop(uneq)\n",
    "    return human_only_labels, gpt_only_labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "46fea70c-ad51-483a-ae7c-44d0db9b2d9c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Failed to map: [31, 49, 327, 372, 374, 411, 354, 491, 490]\n",
      "drug_temperature_0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/gpfs/space/home/kukkvero/.conda/envs/gpt-for-data/lib/python3.8/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: DRUG seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/gpfs/space/home/kukkvero/.conda/envs/gpt-for-data/lib/python3.8/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: DISEASE seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/gpfs/space/home/kukkvero/.conda/envs/gpt-for-data/lib/python3.8/site-packages/seqeval/metrics/sequence_labeling.py:171: UserWarning: PROCEDURE seems not to be NE tag.\n",
      "  warnings.warn('{} seems not to be NE tag.'.format(chunk))\n",
      "/gpfs/space/home/kukkvero/.conda/envs/gpt-for-data/lib/python3.8/site-packages/seqeval/metrics/v1.py:57: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "      ISEASE       0.00      0.00      0.00       335\n",
      "    ROCEDURE       0.00      0.00      0.00       724\n",
      "         RUG       0.15      0.86      0.26       219\n",
      "\n",
      "   micro avg       0.15      0.15      0.15      1278\n",
      "   macro avg       0.05      0.29      0.09      1278\n",
      "weighted avg       0.03      0.15      0.04      1278\n",
      "\n",
      "Failed to map: [31, 327, 372, 429, 430, 431, 432, 433, 463, 465, 466, 467, 468, 471, 472, 477, 478, 479, 480, 481, 482, 483, 484, 485, 486, 487, 488, 489, 490, 491, 493, 494, 355, 465, 464]\n",
      "drug_temperature_0.5\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "      ISEASE       0.00      0.00      0.00       317\n",
      "    ROCEDURE       0.00      0.00      0.00       672\n",
      "         RUG       0.17      0.89      0.28       218\n",
      "\n",
      "   micro avg       0.17      0.16      0.16      1207\n",
      "   macro avg       0.06      0.30      0.09      1207\n",
      "weighted avg       0.03      0.16      0.05      1207\n",
      "\n",
      "Failed to map: [31, 139, 222, 296, 363, 372, 379, 380, 406, 407, 433, 434, 460, 353, 484, 483]\n",
      "drug_temperature_1\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "      ISEASE       0.00      0.00      0.00       320\n",
      "    ROCEDURE       0.00      0.00      0.00       698\n",
      "         RUG       0.16      0.85      0.27       212\n",
      "\n",
      "   micro avg       0.16      0.15      0.15      1230\n",
      "   macro avg       0.05      0.28      0.09      1230\n",
      "weighted avg       0.03      0.15      0.05      1230\n",
      "\n",
      "Failed to map: [18, 31, 137, 166, 189, 196, 236, 247, 270, 282, 372, 392, 425, 347, 484, 483]\n",
      "procedure_temperature_0\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "      ISEASE       0.00      0.00      0.00       329\n",
      "    ROCEDURE       0.15      0.39      0.22       704\n",
      "         RUG       0.00      0.00      0.00       204\n",
      "\n",
      "   micro avg       0.15      0.22      0.18      1237\n",
      "   macro avg       0.05      0.13      0.07      1237\n",
      "weighted avg       0.08      0.22      0.12      1237\n",
      "\n",
      "Failed to map: [14, 18, 31, 50, 71, 115, 137, 166, 189, 247, 270, 372, 392, 346, 484, 483]\n",
      "procedure_temperature_0.5\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "      ISEASE       0.00      0.00      0.00       323\n",
      "    ROCEDURE       0.15      0.36      0.21       707\n",
      "         RUG       0.00      0.00      0.00       197\n",
      "\n",
      "   micro avg       0.15      0.21      0.17      1227\n",
      "   macro avg       0.05      0.12      0.07      1227\n",
      "weighted avg       0.08      0.21      0.12      1227\n",
      "\n",
      "Failed to map: [31, 50, 113, 129, 166, 270, 294, 372, 350, 489, 488]\n",
      "procedure_temperature_1\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "      ISEASE       0.00      0.00      0.00       336\n",
      "    ROCEDURE       0.13      0.32      0.18       736\n",
      "         RUG       0.00      0.00      0.00       213\n",
      "\n",
      "   micro avg       0.13      0.19      0.15      1285\n",
      "   macro avg       0.04      0.11      0.06      1285\n",
      "weighted avg       0.07      0.19      0.10      1285\n",
      "\n",
      "Failed to map: [31, 320, 322, 354, 494, 493]\n",
      "disease_temperature_0\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "      ISEASE       0.17      0.58      0.26       339\n",
      "    ROCEDURE       0.00      0.00      0.00       741\n",
      "         RUG       0.00      0.00      0.00       218\n",
      "\n",
      "   micro avg       0.17      0.15      0.16      1298\n",
      "   macro avg       0.06      0.19      0.09      1298\n",
      "weighted avg       0.04      0.15      0.07      1298\n",
      "\n",
      "Failed to map: [31, 270, 325, 334, 335, 336, 337, 339, 340, 348, 488, 487]\n",
      "disease_temperature_0.5\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "      ISEASE       0.17      0.56      0.25       331\n",
      "    ROCEDURE       0.00      0.00      0.00       717\n",
      "         RUG       0.00      0.00      0.00       218\n",
      "\n",
      "   micro avg       0.17      0.15      0.16      1266\n",
      "   macro avg       0.06      0.19      0.08      1266\n",
      "weighted avg       0.04      0.15      0.07      1266\n",
      "\n",
      "Failed to map: [101, 103, 104, 105, 319, 320, 322, 325, 326, 329, 405, 406, 407, 408, 409, 410, 347, 481, 480]\n",
      "disease_temperature_1\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "      ISEASE       0.16      0.54      0.25       336\n",
      "    ROCEDURE       0.00      0.00      0.00       729\n",
      "         RUG       0.00      0.00      0.00       216\n",
      "\n",
      "   micro avg       0.16      0.14      0.15      1281\n",
      "   macro avg       0.05      0.18      0.08      1281\n",
      "weighted avg       0.04      0.14      0.06      1281\n",
      "\n",
      "Failed to map: [327, 353, 392, 355, 494, 493]\n",
      "drug_procedure_temperature_0\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "      ISEASE       0.00      0.00      0.00       336\n",
      "    ROCEDURE       0.22      0.30      0.26       708\n",
      "         RUG       0.43      0.81      0.57       219\n",
      "\n",
      "   micro avg       0.29      0.31      0.30      1263\n",
      "   macro avg       0.22      0.37      0.27      1263\n",
      "weighted avg       0.20      0.31      0.24      1263\n",
      "\n",
      "Failed to map: [327, 356, 496, 495]\n",
      "drug_procedure_temperature_0.5\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "      ISEASE       0.00      0.00      0.00       339\n",
      "    ROCEDURE       0.24      0.27      0.26       741\n",
      "         RUG       0.46      0.79      0.58       219\n",
      "\n",
      "   micro avg       0.31      0.29      0.30      1299\n",
      "   macro avg       0.23      0.36      0.28      1299\n",
      "weighted avg       0.22      0.29      0.24      1299\n",
      "\n",
      "Failed to map: [270, 272, 355, 495, 494]\n",
      "drug_procedure_temperature_1\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "      ISEASE       0.00      0.00      0.00       333\n",
      "    ROCEDURE       0.26      0.25      0.26       741\n",
      "         RUG       0.39      0.77      0.52       218\n",
      "\n",
      "   micro avg       0.31      0.28      0.29      1292\n",
      "   macro avg       0.22      0.34      0.26      1292\n",
      "weighted avg       0.22      0.28      0.24      1292\n",
      "\n",
      "Failed to map: [314, 316, 317, 318, 319, 322, 392, 429, 430, 431, 351, 487, 486]\n",
      "drug_disease_temperature_0\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "      ISEASE       0.17      0.54      0.26       336\n",
      "    ROCEDURE       0.00      0.00      0.00       723\n",
      "         RUG       0.53      0.86      0.65       217\n",
      "\n",
      "   micro avg       0.26      0.29      0.27      1276\n",
      "   macro avg       0.23      0.47      0.30      1276\n",
      "weighted avg       0.13      0.29      0.18      1276\n",
      "\n",
      "Failed to map: [223, 250, 306, 309, 315, 316, 317, 320, 321, 322, 323, 401, 402, 403, 404, 405, 406, 407, 409, 346, 478, 477]\n",
      "drug_disease_temperature_0.5\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "      ISEASE       0.19      0.54      0.28       335\n",
      "    ROCEDURE       0.00      0.00      0.00       716\n",
      "         RUG       0.43      0.84      0.57       217\n",
      "\n",
      "   micro avg       0.26      0.29      0.27      1268\n",
      "   macro avg       0.21      0.46      0.28      1268\n",
      "weighted avg       0.12      0.29      0.17      1268\n",
      "\n",
      "Failed to map: [33, 306, 307, 308, 317, 318, 320, 321, 322, 323, 324, 325, 422, 345, 484, 483]\n",
      "drug_disease_temperature_1\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "      ISEASE       0.18      0.57      0.28       336\n",
      "    ROCEDURE       0.00      0.00      0.00       731\n",
      "         RUG       0.41      0.81      0.54       215\n",
      "\n",
      "   micro avg       0.25      0.28      0.27      1282\n",
      "   macro avg       0.20      0.46      0.27      1282\n",
      "weighted avg       0.12      0.28      0.16      1282\n",
      "\n",
      "Failed to map: [357, 497, 496]\n",
      "disease_procedure_temperature_0\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "      ISEASE       0.25      0.57      0.34       341\n",
      "    ROCEDURE       0.23      0.29      0.26       742\n",
      "         RUG       0.00      0.00      0.00       219\n",
      "\n",
      "   micro avg       0.24      0.31      0.27      1302\n",
      "   macro avg       0.16      0.29      0.20      1302\n",
      "weighted avg       0.20      0.31      0.24      1302\n",
      "\n",
      "Failed to map: [18, 168, 463, 355, 494, 493]\n",
      "disease_procedure_temperature_0.5\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "      ISEASE       0.23      0.57      0.33       329\n",
      "    ROCEDURE       0.22      0.28      0.25       718\n",
      "         RUG       0.00      0.00      0.00       210\n",
      "\n",
      "   micro avg       0.23      0.31      0.26      1257\n",
      "   macro avg       0.15      0.28      0.19      1257\n",
      "weighted avg       0.19      0.31      0.23      1257\n",
      "\n",
      "Failed to map: [18, 230, 306, 315, 316, 317, 322, 323, 324, 348, 488, 487]\n",
      "disease_procedure_temperature_1\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "      ISEASE       0.19      0.41      0.26       329\n",
      "    ROCEDURE       0.25      0.28      0.27       724\n",
      "         RUG       0.00      0.00      0.00       211\n",
      "\n",
      "   micro avg       0.22      0.27      0.24      1264\n",
      "   macro avg       0.15      0.23      0.18      1264\n",
      "weighted avg       0.19      0.27      0.22      1264\n",
      "\n",
      "Failed to map: [392, 425, 357, 495, 494]\n",
      "drug_disease_procedure_temperature_0\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "      ISEASE       0.30      0.46      0.36       340\n",
      "    ROCEDURE       0.25      0.25      0.25       725\n",
      "         RUG       0.55      0.80      0.65       219\n",
      "\n",
      "   micro avg       0.33      0.40      0.36      1284\n",
      "   macro avg       0.37      0.51      0.42      1284\n",
      "weighted avg       0.32      0.40      0.35      1284\n",
      "\n",
      "Failed to map: [392, 489, 357, 495, 494]\n",
      "drug_disease_procedure_temperature_0.5\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "      ISEASE       0.26      0.46      0.34       336\n",
      "    ROCEDURE       0.23      0.27      0.25       720\n",
      "         RUG       0.56      0.79      0.65       219\n",
      "\n",
      "   micro avg       0.30      0.41      0.35      1275\n",
      "   macro avg       0.35      0.51      0.41      1275\n",
      "weighted avg       0.30      0.41      0.34      1275\n",
      "\n",
      "Failed to map: [0, 16, 487, 497, 355]\n",
      "drug_disease_procedure_temperature_1\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "      ISEASE       0.23      0.42      0.30       340\n",
      "    ROCEDURE       0.28      0.26      0.27       740\n",
      "         RUG       0.57      0.84      0.68       217\n",
      "\n",
      "   micro avg       0.32      0.40      0.35      1297\n",
      "   macro avg       0.36      0.51      0.42      1297\n",
      "weighted avg       0.31      0.40      0.35      1297\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for base_log_name in log_names:\n",
    "    for temperature in temperatures:\n",
    "        log_name = base_log_name + str(temperature)\n",
    "        human_labels, gpt_labels = get_annotations(outputdata_folder_path + log_name)\n",
    "        print(log_name)\n",
    "        #print(len(human_labels))\n",
    "        #print(len(gpt_labels))\n",
    "        try:\n",
    "            print(classification_report(human_labels, gpt_labels))\n",
    "        except Exception as e:\n",
    "            print(e)\n",
    "            pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "66e71fd2-e1a0-4dd9-a8ab-d4041b8a79cb",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.19"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}