eval_best_model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "3e3c45b9",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import pickle\n",
    "import numpy as np\n",
    "from sklearn.metrics import precision_score, recall_score, f1_score, confusion_matrix\n",
    "from operator import truediv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "414e3862",
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.set_option('display.max_columns', None)\n",
    "pd.set_option('display.max_colwidth', None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "e87084fb",
   "metadata": {},
   "outputs": [],
   "source": [
    "results_hq1 = pd.read_pickle(\"/mnt/data/Users/A-PROOF/Cecilia_S/models/fine_tuned/best_model/results_hq1.pkl\")\n",
    "results_hq2 = pd.read_pickle(\"/mnt/data/Users/A-PROOF/Cecilia_S/models/fine_tuned/best_model/results_hq2.pkl\")\n",
    "results_hq3 = pd.read_pickle(\"/mnt/data/Users/A-PROOF/Cecilia_S/models/fine_tuned/best_model/results_hq3.pkl\")\n",
    "results_lq1 = pd.read_pickle(\"/mnt/data/Users/A-PROOF/Cecilia_S/models/fine_tuned/best_model/results_lq1.pkl\")\n",
    "results_lq2 = pd.read_pickle(\"/mnt/data/Users/A-PROOF/Cecilia_S/models/fine_tuned/best_model/results_lq2.pkl\")\n",
    "results_lq3 = pd.read_pickle(\"/mnt/data/Users/A-PROOF/Cecilia_S/models/fine_tuned/best_model/results_lq3.pkl\")\n",
    "results_lqhq3 = pd.read_pickle(\"/mnt/data/Users/A-PROOF/Cecilia_S/models/fine_tuned/best_model/results_lqhq3.pkl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "a885544c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>O</th>\n",
       "      <th>ADM</th>\n",
       "      <th>ATT</th>\n",
       "      <th>BER</th>\n",
       "      <th>ENR</th>\n",
       "      <th>ETN</th>\n",
       "      <th>FAC</th>\n",
       "      <th>INS</th>\n",
       "      <th>MBW</th>\n",
       "      <th>STM</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Precision</th>\n",
       "      <td>0.791667</td>\n",
       "      <td>0.625455</td>\n",
       "      <td>0.888889</td>\n",
       "      <td>0.375000</td>\n",
       "      <td>0.621951</td>\n",
       "      <td>0.419689</td>\n",
       "      <td>0.397129</td>\n",
       "      <td>0.352113</td>\n",
       "      <td>0.513158</td>\n",
       "      <td>0.648276</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Recall</th>\n",
       "      <td>0.102888</td>\n",
       "      <td>0.945055</td>\n",
       "      <td>0.888889</td>\n",
       "      <td>0.857143</td>\n",
       "      <td>0.944444</td>\n",
       "      <td>0.964286</td>\n",
       "      <td>0.976471</td>\n",
       "      <td>0.694444</td>\n",
       "      <td>0.975000</td>\n",
       "      <td>0.969072</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>F1 score</th>\n",
       "      <td>0.182109</td>\n",
       "      <td>0.752735</td>\n",
       "      <td>0.888889</td>\n",
       "      <td>0.521739</td>\n",
       "      <td>0.750000</td>\n",
       "      <td>0.584838</td>\n",
       "      <td>0.564626</td>\n",
       "      <td>0.467290</td>\n",
       "      <td>0.672414</td>\n",
       "      <td>0.776860</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  O       ADM       ATT       BER       ENR       ETN  \\\n",
       "Precision  0.791667  0.625455  0.888889  0.375000  0.621951  0.419689   \n",
       "Recall     0.102888  0.945055  0.888889  0.857143  0.944444  0.964286   \n",
       "F1 score   0.182109  0.752735  0.888889  0.521739  0.750000  0.584838   \n",
       "\n",
       "                FAC       INS       MBW       STM  \n",
       "Precision  0.397129  0.352113  0.513158  0.648276  \n",
       "Recall     0.976471  0.694444  0.975000  0.969072  \n",
       "F1 score   0.564626  0.467290  0.672414  0.776860  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results_hq1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "7e9181f2",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "results = [results_hq1, results_hq2, results_hq3, results_lq1, results_lq2, results_lq3, results_lqhq3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "e1b44958",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                  O       ADM       ATT       BER       ENR       ETN  \\\n",
      "Precision  0.791667  0.625455  0.888889  0.375000  0.621951  0.419689   \n",
      "Recall     0.102888  0.945055  0.888889  0.857143  0.944444  0.964286   \n",
      "F1 score   0.182109  0.752735  0.888889  0.521739  0.750000  0.584838   \n",
      "\n",
      "                FAC       INS       MBW       STM  \n",
      "Precision  0.397129  0.352113  0.513158  0.648276  \n",
      "Recall     0.976471  0.694444  0.975000  0.969072  \n",
      "F1 score   0.564626  0.467290  0.672414  0.776860  \n",
      "------------\n",
      "                  O       ADM       ATT       BER       ENR       ETN  \\\n",
      "Precision  0.743243  0.643382  0.888889  0.352941  0.571429  0.415385   \n",
      "Recall     0.099278  0.961538  0.888889  0.857143  0.888889  0.964286   \n",
      "F1 score   0.175159  0.770925  0.888889  0.500000  0.695652  0.580645   \n",
      "\n",
      "                FAC       INS       MBW       STM  \n",
      "Precision  0.415385  0.320513  0.520000  0.630872  \n",
      "Recall     0.952941  0.694444  0.975000  0.969072  \n",
      "F1 score   0.578571  0.438596  0.678261  0.764228  \n",
      "------------\n",
      "                  O       ADM       ATT       BER       ENR       ETN  \\\n",
      "Precision  0.756098  0.637037  0.727273  0.352941  0.644737  0.423280   \n",
      "Recall     0.111913  0.945055  0.888889  0.857143  0.907407  0.952381   \n",
      "F1 score   0.194969  0.761062  0.800000  0.500000  0.753846  0.586081   \n",
      "\n",
      "                FAC       INS       MBW       STM  \n",
      "Precision  0.411168  0.313953  0.527027  0.643836  \n",
      "Recall     0.952941  0.750000  0.975000  0.969072  \n",
      "F1 score   0.574468  0.442623  0.684211  0.773663  \n",
      "------------\n",
      "                  O       ADM       ATT       BER       ENR       ETN  \\\n",
      "Precision  0.657895  0.638132  0.800000  0.538462  0.672727  0.416667   \n",
      "Recall     0.180505  0.901099  0.888889  1.000000  0.685185  0.952381   \n",
      "F1 score   0.283286  0.747153  0.842105  0.700000  0.678899  0.579710   \n",
      "\n",
      "                FAC       INS       MBW       STM  \n",
      "Precision  0.426230  0.348485  0.565217  0.635762  \n",
      "Recall     0.917647  0.638889  0.975000  0.989691  \n",
      "F1 score   0.582090  0.450980  0.715596  0.774194  \n",
      "------------\n",
      "                  O       ADM       ATT       BER       ENR       ETN  \\\n",
      "Precision  0.703125  0.631783  0.727273  0.466667  0.686567  0.417526   \n",
      "Recall     0.162455  0.895604  0.888889  1.000000  0.851852  0.964286   \n",
      "F1 score   0.263930  0.740909  0.800000  0.636364  0.760331  0.582734   \n",
      "\n",
      "                FAC       INS       MBW       STM  \n",
      "Precision  0.403061  0.370968  0.542857  0.653061  \n",
      "Recall     0.929412  0.638889  0.950000  0.989691  \n",
      "F1 score   0.562278  0.469388  0.690909  0.786885  \n",
      "------------\n",
      "                  O       ADM       ATT       BER       ENR       ETN  \\\n",
      "Precision  0.637306  0.631373  0.888889  0.500000  0.666667  0.415385   \n",
      "Recall     0.222022  0.884615  0.888889  0.857143  0.851852  0.964286   \n",
      "F1 score   0.329317  0.736842  0.888889  0.631579  0.747967  0.580645   \n",
      "\n",
      "                FAC       INS       MBW       STM  \n",
      "Precision  0.411765  0.411765  0.569231  0.643836  \n",
      "Recall     0.741176  0.583333  0.925000  0.969072  \n",
      "F1 score   0.529412  0.482759  0.704762  0.773663  \n",
      "------------\n",
      "                  O       ADM       ATT       BER       ENR       ETN  \\\n",
      "Precision  0.615000  0.634921  1.000000  0.428571  0.677966  0.416667   \n",
      "Recall     0.222022  0.879121  0.888889  0.857143  0.740741  0.952381   \n",
      "F1 score   0.326260  0.737327  0.941176  0.571429  0.707965  0.579710   \n",
      "\n",
      "                FAC       INS       MBW       STM  \n",
      "Precision  0.393939  0.392157  0.571429  0.645833  \n",
      "Recall     0.764706  0.555556  0.900000  0.958763  \n",
      "F1 score   0.520000  0.459770  0.699029  0.771784  \n",
      "------------\n"
     ]
    }
   ],
   "source": [
    "for df in results:\n",
    "    print(df)\n",
    "    print(\"------------\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "b4799d76",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "avg recall: 0.8317692134087642\n",
      "median recall: 0.9447496947496947\n",
      "avg recall: 0.8251480574947646\n",
      "median recall: 0.9209150326797385\n",
      "avg recall: 0.8309801749694815\n",
      "median recall: 0.9262311762311762\n",
      "avg recall: 0.8129286012078285\n",
      "median recall: 0.9093729799612152\n",
      "avg recall: 0.8271077099521318\n",
      "median recall: 0.9125080801551391\n",
      "avg recall: 0.7887388326304539\n",
      "median recall: 0.8708791208791209\n",
      "avg recall: 0.7719320303430572\n",
      "median recall: 0.8681318681318682\n"
     ]
    }
   ],
   "source": [
    "domains = ['O', 'ADM', 'ATT', 'BER', 'ENR', 'ETN', 'FAC', 'INS', 'MBW', 'STM']\n",
    "\n",
    "for df in results:\n",
    "    recall = 0\n",
    "    all_recalls = []\n",
    "    for domain in domains:\n",
    "        recall += df[domain]['Recall'].tolist()\n",
    "        all_recalls.append(df[domain]['Recall'].tolist())\n",
    "    all_recalls = sorted(all_recalls)\n",
    "    median = (all_recalls[4] + all_recalls[5]) / 2\n",
    "    avg_recall = recall / 10\n",
    "    print(\"avg recall:\", avg_recall)\n",
    "    print(\"median recall:\", median)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "e4cb6fbf",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "avg precision: 0.5633326059158297\n",
      "median precision: 0.5675545571245186\n",
      "avg precision: 0.5502038767475995\n",
      "median precision: 0.5457142857142857\n",
      "avg precision: 0.5437349411669481\n",
      "median precision: 0.582032032032032\n",
      "avg precision: 0.5699575847807318\n",
      "median precision: 0.6004894903541607\n",
      "avg precision: 0.5602887610823027\n",
      "median precision: 0.5873200442967885\n",
      "avg precision: 0.5776214216875475\n",
      "median precision: 0.6003016591251885\n",
      "avg precision: 0.5776482993300041\n",
      "median precision: 0.5932142857142857\n"
     ]
    }
   ],
   "source": [
    "domains = ['O', 'ADM', 'ATT', 'BER', 'ENR', 'ETN', 'FAC', 'INS', 'MBW', 'STM']\n",
    "\n",
    "for df in results:\n",
    "    precision = 0\n",
    "    all_precisions = []\n",
    "    for domain in domains:\n",
    "        precision += df[domain]['Precision'].tolist()\n",
    "        all_precisions.append(df[domain]['Precision'].tolist())\n",
    "    all_precisions = sorted(all_precisions)\n",
    "    median = (all_precisions[4] + all_precisions[5]) / 2\n",
    "    avg_precision = precision / 10\n",
    "    print(\"avg precision:\", avg_precision)\n",
    "    print(\"median precision:\", median)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "3a9379dd",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Highest precision score: 1.0 in                   O       ADM       ATT       BER       ENR       ETN  \\\n",
      "Precision  0.615000  0.634921  1.000000  0.428571  0.677966  0.416667   \n",
      "Recall     0.222022  0.879121  0.888889  0.857143  0.740741  0.952381   \n",
      "F1 score   0.326260  0.737327  0.941176  0.571429  0.707965  0.579710   \n",
      "\n",
      "                FAC       INS       MBW       STM  \n",
      "Precision  0.393939  0.392157  0.571429  0.645833  \n",
      "Recall     0.764706  0.555556  0.900000  0.958763  \n",
      "F1 score   0.520000  0.459770  0.699029  0.771784  \n",
      "------------\n",
      "Highest recall score: 1.0 in                   O       ADM       ATT       BER       ENR       ETN  \\\n",
      "Precision  0.657895  0.638132  0.800000  0.538462  0.672727  0.416667   \n",
      "Recall     0.180505  0.901099  0.888889  1.000000  0.685185  0.952381   \n",
      "F1 score   0.283286  0.747153  0.842105  0.700000  0.678899  0.579710   \n",
      "\n",
      "                FAC       INS       MBW       STM  \n",
      "Precision  0.426230  0.348485  0.565217  0.635762  \n",
      "Recall     0.917647  0.638889  0.975000  0.989691  \n",
      "F1 score   0.582090  0.450980  0.715596  0.774194  \n",
      "------------\n",
      "Highest F1 score: 0.9411764705882353 in                   O       ADM       ATT       BER       ENR       ETN  \\\n",
      "Precision  0.615000  0.634921  1.000000  0.428571  0.677966  0.416667   \n",
      "Recall     0.222022  0.879121  0.888889  0.857143  0.740741  0.952381   \n",
      "F1 score   0.326260  0.737327  0.941176  0.571429  0.707965  0.579710   \n",
      "\n",
      "                FAC       INS       MBW       STM  \n",
      "Precision  0.393939  0.392157  0.571429  0.645833  \n",
      "Recall     0.764706  0.555556  0.900000  0.958763  \n",
      "F1 score   0.520000  0.459770  0.699029  0.771784  \n"
     ]
    }
   ],
   "source": [
    "domains = ['O', 'ADM', 'ATT', 'BER', 'ENR', 'ETN', 'FAC', 'INS', 'MBW', 'STM']\n",
    "max_prec = 0\n",
    "max_rec = 0\n",
    "max_f1 = 0\n",
    "max_prec_df = \"\"\n",
    "max_rec_df = \"\"\n",
    "max_f1_df = \"\"\n",
    "\n",
    "for df in results:\n",
    "    for domain in domains:\n",
    "        if df[domain]['Precision'].tolist() > max_prec:\n",
    "            max_prec = df[domain]['Precision'].tolist()\n",
    "            max_prec_df = df\n",
    "        if df[domain]['Recall'].tolist() > max_rec:\n",
    "            max_rec = df[domain]['Recall'].tolist()\n",
    "            max_rec_df = df\n",
    "        if df[domain]['F1 score'].tolist() > max_f1:\n",
    "            max_f1 = df[domain]['F1 score'].tolist()\n",
    "            max_f1_df = df\n",
    "                \n",
    "print(\"Highest precision score:\", max_prec, \"in\", max_prec_df)\n",
    "print(\"------------\")\n",
    "print(\"Highest recall score:\", max_rec, \"in\", max_rec_df)\n",
    "print(\"------------\")\n",
    "print(\"Highest F1 score:\", max_f1, \"in\", max_f1_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2e03e32d",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}