gpt4_eval.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "from dotenv import load_dotenv\n",
    "import pandas as pd\n",
    "\n",
    "import openai"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "load_dotenv() # Loads the environment variables\n",
    "OPENAI_API_KEY = os.environ.get(\"OPENAI_API_KEY\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"outputs/summaries1.csv\", usecols=[\"input_excerpts\", \"summaries_llm\", \"summaries_inhouse\", \"tag_name\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SummEvaluation:\n",
    "    def __init__(self):\n",
    "        pass\n",
    "\n",
    "    def get_geval_score(\n",
    "        self,\n",
    "        criteria: str,\n",
    "        steps: str,\n",
    "        document: str,\n",
    "        summary: str,\n",
    "        metric_name: str\n",
    "    ):\n",
    "        prompt = self.prompt.format(\n",
    "            criteria=criteria,\n",
    "            steps=steps,\n",
    "            metric_name=metric_name,\n",
    "            document=document,\n",
    "            summary=summary,\n",
    "        )\n",
    "        response = openai.ChatCompletion.create(\n",
    "            model=\"gpt-4\",\n",
    "            messages=[{\"role\": \"user\", \"content\": prompt}],\n",
    "            temperature=0,\n",
    "            max_tokens=5,\n",
    "            top_p=1,\n",
    "            frequency_penalty=0,\n",
    "            presence_penalty=0,\n",
    "        )\n",
    "        return response.choices[0].message.content.strip()\n",
    "\n",
    "    def eval_summ_using_prompt(self, df):\n",
    "        self.prompt = \"\"\"\n",
    "            You will be given one summary written for an article. Your task is to rate the summary on one metric.\n",
    "            Please make sure you read and understand these instructions very carefully. \n",
    "            Please keep this document open while reviewing, and refer to it as needed.\n",
    "\n",
    "            Evaluation Criteria:\n",
    "\n",
    "            {criteria}\n",
    "\n",
    "            Evaluation Steps:\n",
    "\n",
    "            {steps}\n",
    "\n",
    "            Example:\n",
    "\n",
    "            Source Text:\n",
    "\n",
    "            {document}\n",
    "\n",
    "            Summary:\n",
    "\n",
    "            {summary}\n",
    "\n",
    "            Evaluation Form (scores ONLY):\n",
    "\n",
    "            - {metric_name}\n",
    "        \"\"\"\n",
    "\n",
    "        # Metric 1: Relevance\n",
    "\n",
    "        RELEVANCY_SCORE_CRITERIA = \"\"\"\n",
    "        Relevance(1-5) - selection of important content from the source. \\\n",
    "        The summary should include only important information from the source document. \\\n",
    "        Annotators were instructed to penalize summaries which contained redundancies and excess information.\n",
    "        \"\"\"\n",
    "\n",
    "        RELEVANCY_SCORE_STEPS = \"\"\"\n",
    "        1. Read the summary and the source document carefully.\n",
    "        2. Compare the summary to the source document and identify the main points of the article.\n",
    "        3. Assess how well the summary covers the main points of the article, and how much irrelevant or redundant information it contains.\n",
    "        4. Assign a relevance score from 1 to 5.\n",
    "        \"\"\"\n",
    "\n",
    "        # Metric 2: Coherence\n",
    "\n",
    "        COHERENCE_SCORE_CRITERIA = \"\"\"\n",
    "        Coherence(1-5) - the collective quality of all sentences. \\\n",
    "        We align this dimension with the DUC quality question of structure and coherence \\\n",
    "        whereby \"the summary should be well-structured and well-organized. \\\n",
    "        The summary should not just be a heap of related information, but should build from sentence to a\\\n",
    "        coherent body of information about a topic.\"\n",
    "        \"\"\"\n",
    "\n",
    "        COHERENCE_SCORE_STEPS = \"\"\"\n",
    "        1. Read the article carefully and identify the main topic and key points.\n",
    "        2. Read the summary and compare it to the article. Check if the summary covers the main topic and key points of the article,\n",
    "        and if it presents them in a clear and logical order.\n",
    "        3. Assign a score for coherence on a scale of 1 to 5, where 1 is the lowest and 5 is the highest based on the Evaluation Criteria.\n",
    "        \"\"\"\n",
    "\n",
    "        # Metric 3: Consistency\n",
    "\n",
    "        CONSISTENCY_SCORE_CRITERIA = \"\"\"\n",
    "        Consistency(1-5) - the factual alignment between the summary and the summarized source. \\\n",
    "        A factually consistent summary contains only statements that are entailed by the source document. \\\n",
    "        Annotators were also asked to penalize summaries that contained hallucinated facts.\n",
    "        \"\"\"\n",
    "\n",
    "        CONSISTENCY_SCORE_STEPS = \"\"\"\n",
    "        1. Read the article carefully and identify the main facts and details it presents.\n",
    "        2. Read the summary and compare it to the article. Check if the summary contains any factual errors that are not supported by the article.\n",
    "        3. Assign a score for consistency based on the Evaluation Criteria.\n",
    "        \"\"\"\n",
    "\n",
    "        # Metric 4: Fluency\n",
    "\n",
    "        FLUENCY_SCORE_CRITERIA = \"\"\"\n",
    "        Fluency(1-3): the quality of the summary in terms of grammar, spelling, punctuation, word choice, and sentence structure.\n",
    "        1: Poor. The summary has many errors that make it hard to understand or sound unnatural.\n",
    "        2: Fair. The summary has some errors that affect the clarity or smoothness of the text, but the main points are still comprehensible.\n",
    "        3: Good. The summary has few or no errors and is easy to read and follow.\n",
    "        \"\"\"\n",
    "\n",
    "        FLUENCY_SCORE_STEPS = \"\"\"\n",
    "        Read the summary and evaluate its fluency based on the given criteria. Assign a fluency score from 1 to 3.\n",
    "        \"\"\"\n",
    "\n",
    "        self.evaluation_metrics = {\n",
    "            \"relevance\": (RELEVANCY_SCORE_CRITERIA, RELEVANCY_SCORE_STEPS),\n",
    "            \"coherence\": (COHERENCE_SCORE_CRITERIA, COHERENCE_SCORE_STEPS),\n",
    "            \"consistency\": (CONSISTENCY_SCORE_CRITERIA, CONSISTENCY_SCORE_STEPS),\n",
    "            \"fluency\": (FLUENCY_SCORE_CRITERIA, FLUENCY_SCORE_STEPS)\n",
    "        }\n",
    "        results = {}\n",
    "        for eval_type in self.evaluation_metrics.keys():\n",
    "            results.update({\n",
    "                f\"{eval_type}_llm\": [],\n",
    "                f\"{eval_type}_inhouse\": []\n",
    "            })\n",
    "        for eval_type, (criteria, steps) in self.evaluation_metrics.items():\n",
    "            results[f\"{eval_type}_llm\"] = df.apply(lambda x: self.get_geval_score(criteria, steps, x[\"input_excerpts\"], x[\"summaries_llm\"], eval_type), axis=1).tolist()\n",
    "            results[f\"{eval_type}_inhouse\"] = df.apply(lambda x: self.get_geval_score(criteria, steps, x[\"input_excerpts\"], x[\"summaries_inhouse\"], eval_type), axis=1).tolist()\n",
    "        \n",
    "        return results\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>input_excerpts</th>\n",
       "      <th>summaries_llm</th>\n",
       "      <th>summaries_inhouse</th>\n",
       "      <th>tag_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Current needs include WASH, shelter, health, a...</td>\n",
       "      <td>The earthquake in Morocco has caused significa...</td>\n",
       "      <td>Moroccan authorities are continuing their effo...</td>\n",
       "      <td>protection</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Current needs include WASH, shelter, health, a...</td>\n",
       "      <td>The earthquake in Morocco has caused extensive...</td>\n",
       "      <td>The devastating earthquake in Morocco has left...</td>\n",
       "      <td>shelter</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>According to an Unosat analysis performed on 1...</td>\n",
       "      <td>According to an Unosat analysis on 10/9/2023, ...</td>\n",
       "      <td>The earthquake that struck Morocco last week h...</td>\n",
       "      <td>education</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Current needs include WASH, shelter, health, a...</td>\n",
       "      <td>Caritas Morocco continues to provide aid to th...</td>\n",
       "      <td>The earthquake in Morocco, which killed at lea...</td>\n",
       "      <td>food</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>The average standard of living of households i...</td>\n",
       "      <td>The average standard of living of households i...</td>\n",
       "      <td>The Moroccan government has announced that it ...</td>\n",
       "      <td>livelihoods</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>They are among 60 firefighters from 14 UK fire...</td>\n",
       "      <td>The earthquake in Morocco has caused significa...</td>\n",
       "      <td>The devastating earthquake that struck Morocco...</td>\n",
       "      <td>health</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>They are among 60 firefighters from 14 UK fire...</td>\n",
       "      <td>The 6.8 magnitude earthquake in Morocco's High...</td>\n",
       "      <td>Emergency supplies are being sought to help th...</td>\n",
       "      <td>cross</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>All international airports and ports are funct...</td>\n",
       "      <td>All international airports and ports are funct...</td>\n",
       "      <td>The Moroccan government has confirmed that it ...</td>\n",
       "      <td>logistics</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Through these completed activities, AFH has ma...</td>\n",
       "      <td>Through completed activities, AFH has made a m...</td>\n",
       "      <td>The British Farmers' Fund (AFH) has completed ...</td>\n",
       "      <td>nutrition</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>In recent years, the Moroccan government and d...</td>\n",
       "      <td>Summary 1: This article discusses aid programs...</td>\n",
       "      <td>Moroccan aid agency CARE Maroc has launched a ...</td>\n",
       "      <td>agriculture</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Current needs include WASH, shelter, health, a...</td>\n",
       "      <td>The earthquake in Morocco has caused significa...</td>\n",
       "      <td>The Moroccan army is providing emergency shelt...</td>\n",
       "      <td>wash</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       input_excerpts  \\\n",
       "0   Current needs include WASH, shelter, health, a...   \n",
       "1   Current needs include WASH, shelter, health, a...   \n",
       "2   According to an Unosat analysis performed on 1...   \n",
       "3   Current needs include WASH, shelter, health, a...   \n",
       "4   The average standard of living of households i...   \n",
       "5   They are among 60 firefighters from 14 UK fire...   \n",
       "6   They are among 60 firefighters from 14 UK fire...   \n",
       "7   All international airports and ports are funct...   \n",
       "8   Through these completed activities, AFH has ma...   \n",
       "9   In recent years, the Moroccan government and d...   \n",
       "10  Current needs include WASH, shelter, health, a...   \n",
       "\n",
       "                                        summaries_llm  \\\n",
       "0   The earthquake in Morocco has caused significa...   \n",
       "1   The earthquake in Morocco has caused extensive...   \n",
       "2   According to an Unosat analysis on 10/9/2023, ...   \n",
       "3   Caritas Morocco continues to provide aid to th...   \n",
       "4   The average standard of living of households i...   \n",
       "5   The earthquake in Morocco has caused significa...   \n",
       "6   The 6.8 magnitude earthquake in Morocco's High...   \n",
       "7   All international airports and ports are funct...   \n",
       "8   Through completed activities, AFH has made a m...   \n",
       "9   Summary 1: This article discusses aid programs...   \n",
       "10  The earthquake in Morocco has caused significa...   \n",
       "\n",
       "                                    summaries_inhouse     tag_name  \n",
       "0   Moroccan authorities are continuing their effo...   protection  \n",
       "1   The devastating earthquake in Morocco has left...      shelter  \n",
       "2   The earthquake that struck Morocco last week h...    education  \n",
       "3   The earthquake in Morocco, which killed at lea...         food  \n",
       "4   The Moroccan government has announced that it ...  livelihoods  \n",
       "5   The devastating earthquake that struck Morocco...       health  \n",
       "6   Emergency supplies are being sought to help th...        cross  \n",
       "7   The Moroccan government has confirmed that it ...    logistics  \n",
       "8   The British Farmers' Fund (AFH) has completed ...    nutrition  \n",
       "9   Moroccan aid agency CARE Maroc has launched a ...  agriculture  \n",
       "10  The Moroccan army is providing emergency shelt...         wash  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "summ_prompt = SummEvaluation()\n",
    "results = summ_prompt.eval_summ_using_prompt(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_scores = pd.DataFrame.from_dict(results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_merged = pd.concat([df, df_scores], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>input_excerpts</th>\n",
       "      <th>summaries_llm</th>\n",
       "      <th>summaries_inhouse</th>\n",
       "      <th>tag_name</th>\n",
       "      <th>relevance_llm</th>\n",
       "      <th>relevance_inhouse</th>\n",
       "      <th>coherence_llm</th>\n",
       "      <th>coherence_inhouse</th>\n",
       "      <th>consistency_llm</th>\n",
       "      <th>consistency_inhouse</th>\n",
       "      <th>fluency_llm</th>\n",
       "      <th>fluency_inhouse</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Current needs include WASH, shelter, health, a...</td>\n",
       "      <td>The earthquake in Morocco has caused significa...</td>\n",
       "      <td>Moroccan authorities are continuing their effo...</td>\n",
       "      <td>protection</td>\n",
       "      <td>5</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Current needs include WASH, shelter, health, a...</td>\n",
       "      <td>The earthquake in Morocco has caused extensive...</td>\n",
       "      <td>The devastating earthquake in Morocco has left...</td>\n",
       "      <td>shelter</td>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>According to an Unosat analysis performed on 1...</td>\n",
       "      <td>According to an Unosat analysis on 10/9/2023, ...</td>\n",
       "      <td>The earthquake that struck Morocco last week h...</td>\n",
       "      <td>education</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Current needs include WASH, shelter, health, a...</td>\n",
       "      <td>Caritas Morocco continues to provide aid to th...</td>\n",
       "      <td>The earthquake in Morocco, which killed at lea...</td>\n",
       "      <td>food</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>The average standard of living of households i...</td>\n",
       "      <td>The average standard of living of households i...</td>\n",
       "      <td>The Moroccan government has announced that it ...</td>\n",
       "      <td>livelihoods</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>They are among 60 firefighters from 14 UK fire...</td>\n",
       "      <td>The earthquake in Morocco has caused significa...</td>\n",
       "      <td>The devastating earthquake that struck Morocco...</td>\n",
       "      <td>health</td>\n",
       "      <td>5</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>They are among 60 firefighters from 14 UK fire...</td>\n",
       "      <td>The 6.8 magnitude earthquake in Morocco's High...</td>\n",
       "      <td>Emergency supplies are being sought to help th...</td>\n",
       "      <td>cross</td>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>All international airports and ports are funct...</td>\n",
       "      <td>All international airports and ports are funct...</td>\n",
       "      <td>The Moroccan government has confirmed that it ...</td>\n",
       "      <td>logistics</td>\n",
       "      <td>5</td>\n",
       "      <td>3.5</td>\n",
       "      <td>4.5</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>3.5</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Through these completed activities, AFH has ma...</td>\n",
       "      <td>Through completed activities, AFH has made a m...</td>\n",
       "      <td>The British Farmers' Fund (AFH) has completed ...</td>\n",
       "      <td>nutrition</td>\n",
       "      <td>4.5</td>\n",
       "      <td>1</td>\n",
       "      <td>4.5</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>In recent years, the Moroccan government and d...</td>\n",
       "      <td>Summary 1: This article discusses aid programs...</td>\n",
       "      <td>Moroccan aid agency CARE Maroc has launched a ...</td>\n",
       "      <td>agriculture</td>\n",
       "      <td>Summary 1:</td>\n",
       "      <td>3</td>\n",
       "      <td>Summary 1:</td>\n",
       "      <td>3</td>\n",
       "      <td>Summary 1:</td>\n",
       "      <td>1</td>\n",
       "      <td>Summary 1:</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Current needs include WASH, shelter, health, a...</td>\n",
       "      <td>The earthquake in Morocco has caused significa...</td>\n",
       "      <td>The Moroccan army is providing emergency shelt...</td>\n",
       "      <td>wash</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       input_excerpts  \\\n",
       "0   Current needs include WASH, shelter, health, a...   \n",
       "1   Current needs include WASH, shelter, health, a...   \n",
       "2   According to an Unosat analysis performed on 1...   \n",
       "3   Current needs include WASH, shelter, health, a...   \n",
       "4   The average standard of living of households i...   \n",
       "5   They are among 60 firefighters from 14 UK fire...   \n",
       "6   They are among 60 firefighters from 14 UK fire...   \n",
       "7   All international airports and ports are funct...   \n",
       "8   Through these completed activities, AFH has ma...   \n",
       "9   In recent years, the Moroccan government and d...   \n",
       "10  Current needs include WASH, shelter, health, a...   \n",
       "\n",
       "                                        summaries_llm  \\\n",
       "0   The earthquake in Morocco has caused significa...   \n",
       "1   The earthquake in Morocco has caused extensive...   \n",
       "2   According to an Unosat analysis on 10/9/2023, ...   \n",
       "3   Caritas Morocco continues to provide aid to th...   \n",
       "4   The average standard of living of households i...   \n",
       "5   The earthquake in Morocco has caused significa...   \n",
       "6   The 6.8 magnitude earthquake in Morocco's High...   \n",
       "7   All international airports and ports are funct...   \n",
       "8   Through completed activities, AFH has made a m...   \n",
       "9   Summary 1: This article discusses aid programs...   \n",
       "10  The earthquake in Morocco has caused significa...   \n",
       "\n",
       "                                    summaries_inhouse     tag_name  \\\n",
       "0   Moroccan authorities are continuing their effo...   protection   \n",
       "1   The devastating earthquake in Morocco has left...      shelter   \n",
       "2   The earthquake that struck Morocco last week h...    education   \n",
       "3   The earthquake in Morocco, which killed at lea...         food   \n",
       "4   The Moroccan government has announced that it ...  livelihoods   \n",
       "5   The devastating earthquake that struck Morocco...       health   \n",
       "6   Emergency supplies are being sought to help th...        cross   \n",
       "7   The Moroccan government has confirmed that it ...    logistics   \n",
       "8   The British Farmers' Fund (AFH) has completed ...    nutrition   \n",
       "9   Moroccan aid agency CARE Maroc has launched a ...  agriculture   \n",
       "10  The Moroccan army is providing emergency shelt...         wash   \n",
       "\n",
       "   relevance_llm relevance_inhouse coherence_llm coherence_inhouse  \\\n",
       "0              5                 3             5                 2   \n",
       "1              5                 2             5                 2   \n",
       "2              5                 4             5                 3   \n",
       "3              5                 1             5                 1   \n",
       "4              5                 1             5                 1   \n",
       "5              5                 3             5                 3   \n",
       "6              5                 2             5                 1   \n",
       "7              5               3.5           4.5                 3   \n",
       "8            4.5                 1           4.5                 2   \n",
       "9     Summary 1:                 3    Summary 1:                 3   \n",
       "10             5                 1             5                 1   \n",
       "\n",
       "   consistency_llm consistency_inhouse fluency_llm fluency_inhouse  \n",
       "0                5                   5           3               3  \n",
       "1                5                   1           3               2  \n",
       "2                5                   5           3               3  \n",
       "3                5                   1           3               2  \n",
       "4                5                   1           3               2  \n",
       "5                5                   5           3               3  \n",
       "6                5                   1           3               2  \n",
       "7                5                 3.5           3               2  \n",
       "8                5                   1           3               2  \n",
       "9       Summary 1:                   1  Summary 1:               3  \n",
       "10               5                   1           3               2  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_merged"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_merged.to_csv(\"outputs/summaries1_results.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}