Metrics.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import paths\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "manual_annotations = json.load(open(paths.MELART_ANNOTATIONS_PATH))\n",
    "automatic_annotations = json.load(open(paths.MELART_AUTO_ANNOTATIONS_PATH))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "FP: Q568847 6 {'qid': 'Q302', 'text': 'Christ', 'start': 105, 'end': 111}\n",
      "FP: Q568847 12 {'qid': 'Q302', 'text': 'Christ', 'start': 119, 'end': 125}\n",
      "FP: Q568847 19 {'qid': 'Q345', 'text': 'Mary', 'start': 97, 'end': 101}\n",
      "FP: Q1516449 9 {'qid': 'Q8441', 'text': 'man', 'start': 98, 'end': 101}\n",
      "FP: Q698015 3 {'qid': 'Q7560', 'text': 'mother', 'start': 29, 'end': 35}\n",
      "FP: Q323753 4 {'qid': 'Q162691', 'text': 'Assumption of the Virgin', 'start': 4, 'end': 28}\n",
      "FP: Q323753 5 {'qid': 'Q345', 'text': 'Mary', 'start': 12, 'end': 16}\n",
      "FP: Q3464081 2 {'qid': 'Q4817', 'text': 'pillar', 'start': 35, 'end': 41}\n",
      "FP: Q919294 4 {'qid': 'Q345', 'text': 'Maria', 'start': 38, 'end': 43}\n",
      "FP: Q1477693 9 {'qid': 'Q7560', 'text': 'mother', 'start': 104, 'end': 110}\n",
      "FP: Q1477693 19 {'qid': 'Q7560', 'text': 'mother', 'start': 165, 'end': 171}\n",
      "FP: Q1477693 20 {'qid': 'Q7560', 'text': 'mother', 'start': 110, 'end': 116}\n",
      "FP: Q1170194 5 {'qid': 'Q19546', 'text': 'pope', 'start': 4, 'end': 8}\n",
      "FP: Q680128 0 {'qid': 'Q302', 'text': 'Jesus', 'start': 17, 'end': 22}\n",
      "FP: Q680128 3 {'qid': 'Q8441', 'text': 'men', 'start': 77, 'end': 80}\n",
      "FP: Q680128 7 {'qid': 'Q302', 'text': 'Christ', 'start': 0, 'end': 6}\n",
      "FP: Q3605055 4 {'qid': 'Q58701', 'text': 'Adam and Eve', 'start': 0, 'end': 12}\n",
      "FP: Q924655 3 {'qid': 'Q31966', 'text': 'Mark', 'start': 3, 'end': 7}\n",
      "FP: Q2627056 2 {'qid': 'Q12791', 'text': 'sword', 'start': 4, 'end': 9}\n",
      "FP: Q2627056 3 {'qid': 'Q12791', 'text': 'sword', 'start': 235, 'end': 240}\n",
      "FP: Q2627056 13 {'qid': 'Q13472331', 'text': 'David and Goliath', 'start': 77, 'end': 94}\n",
      "FP: Q2270778 0 {'qid': 'Q130704', 'text': 'Peter denying Jesus', 'start': 11, 'end': 30}\n",
      "FP: Q2270778 1 {'qid': 'Q33923', 'text': 'Saint Peter', 'start': 14, 'end': 25}\n",
      "FP: Q3699475 10 {'qid': 'Q144', 'text': 'dog', 'start': 49, 'end': 52}\n",
      "FP: Q3699475 12 {'qid': 'Q144', 'text': 'dog', 'start': 35, 'end': 38}\n",
      "FP: Q1752990 0 {'qid': 'Q8441', 'text': 'man', 'start': 24, 'end': 27}\n",
      "FP: Q1752990 1 {'qid': 'Q8441', 'text': 'men', 'start': 26, 'end': 29}\n",
      "FP: Q1752990 1 {'qid': 'Q8441', 'text': 'man', 'start': 101, 'end': 104}\n",
      "FP: Q1752990 2 {'qid': 'Q8441', 'text': 'man', 'start': 10, 'end': 13}\n",
      "FP: Q1752990 3 {'qid': 'Q8441', 'text': 'man', 'start': 14, 'end': 17}\n",
      "FP: Q1752990 3 {'qid': 'Q8441', 'text': 'man', 'start': 48, 'end': 51}\n",
      "FP: Q1752990 4 {'qid': 'Q8441', 'text': 'man', 'start': 11, 'end': 14}\n",
      "FP: Q1752990 5 {'qid': 'Q8441', 'text': 'men', 'start': 19, 'end': 22}\n",
      "FP: Q1752990 8 {'qid': 'Q218041', 'text': 'poison hemlock', 'start': 149, 'end': 163}\n",
      "FP: Q3399482 0 {'qid': 'Q15615042', 'text': 'Amédée de Pastoret', 'start': 52, 'end': 70}\n",
      "FP: Q636537 0 {'qid': 'Q2687', 'text': 'July 13', 'start': 68, 'end': 75}\n",
      "FP: Q2619037 0 {'qid': 'Q131514', 'text': 'sofa', 'start': 200, 'end': 204}\n",
      "FP: Q2619037 0 {'qid': 'Q200539', 'text': 'dress', 'start': 229, 'end': 234}\n",
      "FP: Q507120 9 {'qid': 'Q573', 'text': 'day', 'start': 17, 'end': 20}\n",
      "FP: Q3837491 2 {'qid': 'Q10884', 'text': 'trees', 'start': 11, 'end': 16}\n",
      "FP: Q3837491 6 {'qid': 'Q10884', 'text': 'trees', 'start': 99, 'end': 104}\n",
      "FP: Q3837491 17 {'qid': 'Q8441', 'text': 'man', 'start': 79, 'end': 82}\n",
      "FP: Q2622172 1 {'qid': 'Q200539', 'text': 'dress', 'start': 45, 'end': 50}\n",
      "FP: Q2622172 2 {'qid': 'Q8436', 'text': 'family', 'start': 59, 'end': 65}\n",
      "FP: Q1219263 0 {'qid': 'Q140', 'text': 'lion', 'start': 35, 'end': 39}\n",
      "FP: Q1219263 2 {'qid': 'Q140', 'text': 'lion', 'start': 2, 'end': 6}\n",
      "FP: Q3207251 0 {'qid': 'Q209554', 'text': 'flute', 'start': 57, 'end': 62}\n",
      "FP: Q3207251 1 {'qid': 'Q209554', 'text': 'flute', 'start': 129, 'end': 134}\n",
      "TP: 453\n",
      "FP: 48\n",
      "FN: 192\n",
      "Precision: 0.9041916167664671\n",
      "Recall: 0.7023255813953488\n",
      "F1: 0.7905759162303665\n",
      "Wrong entity: 12\n"
     ]
    }
   ],
   "source": [
    "tp=0\n",
    "fp=0\n",
    "fn=0\n",
    "wrong_entity=0\n",
    "for qid,obj in manual_annotations.items():\n",
    "    if obj[\"split\"]==\"test\":\n",
    "        obj_auto = automatic_annotations[qid]\n",
    "        manual_matches = obj[\"visual_el_matches\"] + obj[\"contextual_el_matches\"]\n",
    "        auto_matches = obj_auto[\"visual_el_matches\"] + obj_auto[\"contextual_el_matches\"]\n",
    "        for i,sentence_matches in enumerate(manual_matches):\n",
    "            for match in sentence_matches:\n",
    "                is_tp=False\n",
    "                for auto_match in auto_matches[i]:\n",
    "                    if (match[\"start\"]==auto_match[\"start\"] or match[\"end\"]==auto_match[\"end\"]) and match[\"qid\"]!=auto_match[\"qid\"]:\n",
    "                        wrong_entity+=1\n",
    "                    if match[\"qid\"]==auto_match[\"qid\"] and (match[\"start\"]==auto_match[\"start\"] or match[\"end\"]==auto_match[\"end\"]):\n",
    "                        is_tp=True\n",
    "                        break\n",
    "                if is_tp:\n",
    "                    tp+=1\n",
    "                else:\n",
    "                    fn+=1\n",
    "            for match in auto_matches[i]:\n",
    "                is_fp=True\n",
    "                for manual_match in sentence_matches:\n",
    "                    if match[\"qid\"]==manual_match[\"qid\"] and (match[\"start\"]==manual_match[\"start\"] or match[\"end\"]==manual_match[\"end\"]):\n",
    "                        is_fp=False\n",
    "                        break\n",
    "                if is_fp:\n",
    "                    fp+=1\n",
    "                    print(\"FP:\",qid,i,match)\n",
    "print(\"TP:\",tp)\n",
    "print(\"FP:\",fp)\n",
    "print(\"FN:\",fn)\n",
    "precision = tp/(tp+fp)\n",
    "recall = tp/(tp+fn)\n",
    "f1 = 2*precision*recall/(precision+recall)\n",
    "print(\"Precision:\",precision)\n",
    "print(\"Recall:\",recall)\n",
    "print(\"F1:\",f1)\n",
    "print(\"Wrong entity:\",wrong_entity)\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1616"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(manual_annotations.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Visual matches: 2916\n",
      "Contextual matches: 3669\n",
      "Visual matches sentences: 2118\n",
      "Contextual matches sentences: 2716\n",
      "Original visual sentences: 5469\n",
      "Original contextual sentences: 10970\n"
     ]
    }
   ],
   "source": [
    "visual_matches_count_sentences=0\n",
    "contextual_matches_count_sentences=0\n",
    "visual_matches_count=0\n",
    "contextual_matches_count=0\n",
    "original_visual_sentences=0\n",
    "original_contextual_sentences=0\n",
    "for qid,obj in manual_annotations.items():\n",
    "    obj_auto = obj\n",
    "    visual_matches = obj_auto[\"visual_el_matches\"]\n",
    "    for sentence_matches in visual_matches:\n",
    "        original_visual_sentences+=1\n",
    "        if len(sentence_matches)>0:\n",
    "            visual_matches_count_sentences+=1\n",
    "            visual_matches_count+=len(sentence_matches)\n",
    "    contextual_matches = obj_auto[\"contextual_el_matches\"]\n",
    "    for sentence_matches in contextual_matches:\n",
    "        original_contextual_sentences+=1\n",
    "        if len(sentence_matches)>0:\n",
    "            contextual_matches_count_sentences+=1\n",
    "            contextual_matches_count+=len(sentence_matches)\n",
    "print(\"Visual matches:\",visual_matches_count)\n",
    "print(\"Contextual matches:\",contextual_matches_count)\n",
    "print(\"Visual matches sentences:\",visual_matches_count_sentences)\n",
    "print(\"Contextual matches sentences:\",contextual_matches_count_sentences)\n",
    "print(\"Original visual sentences:\",original_visual_sentences)\n",
    "print(\"Original contextual sentences:\",original_contextual_sentences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6444"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "visual_matches_count+contextual_matches_count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.32514445744266013"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "original_visual_sentences/(original_visual_sentences+original_contextual_sentences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.44124448181627074"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "visual_matches_count_sentences/(visual_matches_count_sentences+contextual_matches_count_sentences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}