founta_basile_comparison.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "fb8de06f-6ad3-4732-b42c-92bce13e437e",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Xformers is not installed correctly. If you want to use memorry_efficient_attention to accelerate training use the following command to install Xformers\n",
      "pip install xformers.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dd27add27bee4f4a83923912788b1b88",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "[False, True]"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# loading model\n",
    "from evaluate_detexd_roberta import predict_delicate\n",
    "from transformers import pipeline\n",
    "\n",
    "classifier = pipeline(\"text-classification\", model=\"grammarly/detexd-roberta-base\", device=0, batch_size=8)\n",
    "predict_delicate(classifier, ['hello', 'fuck'])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "df776287-5b37-4c1b-ae07-eeda27f45ffe",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Founta dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e83f72ed-7786-4f27-9e80-551b1e5355e5",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Data preparation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "e821afef-26e9-4a4b-be24-cc9898f1e713",
   "metadata": {},
   "outputs": [],
   "source": [
    "# you can find the original data here: https://zenodo.org/record/3706866#.Y35QVOzMIZ9\n",
    "\n",
    "import pandas as pd\n",
    "\n",
    "df = pd.read_table('founta_data/hatespeech_text_label_vote_RESTRICTED_100K.csv',\n",
    "                   names=['text', 'label', 'votes'], skiprows=2)\n",
    "# skipping broken lines"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "d11bac4c-195c-402a-bd7b-afad8fcb4056",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "label\n",
       "abusive    27150\n",
       "hateful     4965\n",
       "normal     53851\n",
       "spam       14029\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# the dataset got bigger 80k -> 100k\n",
    "# there's no information whant changed\n",
    "# the rows are shuffled and ids removed, so we can not restore original dataset\n",
    "df.label.value_counts().sort_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "90314b13-b41e-40ac-8491-5814b2a60c08",
   "metadata": {},
   "outputs": [],
   "source": [
    "# removing spam since there are delicate texts there that are not filtered like porn\n",
    "df = df[df.label != 'spam']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "401d46ce-fbfa-4755-ae52-5418a884451e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "label\n",
       "abusive    27150\n",
       "hateful     4965\n",
       "normal     53851\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.label.value_counts().sort_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "201931ae-5125-4d3c-b98e-2145603f28ac",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a59551635ef44e6f88201a54e4603af9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/85966 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import datasets\n",
    "from transformers.pipelines.pt_utils import KeyDataset\n",
    "\n",
    "preds = predict_delicate(classifier, KeyDataset(datasets.Dataset.from_pandas(df), 'text'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "2448e7ea-7b68-4592-aed0-c1c5d9653cdf",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['real'] = df.label != 'normal'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "6e81e541-4f67-4206-8110-062af9114ea7",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['pred'] = preds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "c9721c15-4467-4863-8b4e-4ea6f9050726",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv('founta_data/preds.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f72ee91d-8d54-4160-b3f7-6c879e9fa115",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Data analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "131c549d-2075-4f2d-9a97-f49a1660dc4b",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('founta_data/preds.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "911bebf5-234d-47f2-a17a-9043acee212b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.3735779261568527"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.real.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "ec02caa6-1499-4792-9961-672f61d43059",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.7626132876614572, 0.6655145570605636, 0.7107630401888895, None)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import precision_recall_fscore_support\n",
    "founta_results = precision_recall_fscore_support(y_true=df.real, y_pred=df.pred, average='binary')\n",
    "founta_results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "84ec8005-d29b-4897-be0c-4f9a96bed8a0",
   "metadata": {},
   "outputs": [],
   "source": [
    "# FPs\n",
    "print('\\n------\\n'.join(df[df.pred & ~df.real].sample(10, random_state=23543).text))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f6a82b08-0228-41e8-9e6d-da7051dbef21",
   "metadata": {},
   "outputs": [],
   "source": [
    "# FNs\n",
    "print('\\n------\\n'.join(df[~df.pred & df.real].sample(10, random_state=23543).text))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c6cf0a37-9282-49da-9a78-068bd49538f5",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Basile dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2d910004-46ca-4f7d-a83b-7df3c71a64ad",
   "metadata": {},
   "source": [
    "### Data preparation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "3c70b4d8-a4c9-429e-8710-d06b82755216",
   "metadata": {},
   "outputs": [],
   "source": [
    "# you can find the original data here: http://hatespeech.di.unito.it/hateval.html\n",
    "# or here: https://github.com/alisonrib17/SemEval2019-Task5/tree/master/English\n",
    "\n",
    "df = pd.read_table('basile_data/test_en.tsv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "3a68241a-5622-4bc7-bc36-0565f80800e1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.4834224598930481"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.text.str.contains('bitch', case=False).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "6dd694d0-12d3-477d-85c3-80b790e11652",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b08c5a154b7b4260850f2349afd2d7da",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/2805 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "df['pred'] = predict_delicate(classifier, KeyDataset(datasets.Dataset.from_pandas(df), 'text'))\n",
    "df['real'] = df.HS == 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "f6d9a8f1-0aaa-4caa-a364-8ef52db22e62",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv('basile_data/preds.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "19dac446-fca9-4ee0-9bdb-775345c4ec9e",
   "metadata": {},
   "source": [
    "### Data analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "680216af-44d5-4c13-b3ef-4ee2ef49ac85",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('basile_data/preds.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "e69dae9e-e246-45af-a75a-212b158a2d59",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.4206773618538324"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.real.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "e880347d-a337-4f68-8c92-5b709f72d62c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7875222816399287"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.pred.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "839b5c43-4fd1-475f-97af-2ef5dfc47eee",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.47532820280669985, 0.8898305084745762, 0.6196518146946001, None)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "basile_results = precision_recall_fscore_support(y_true=df.real, y_pred=df.pred, average='binary')\n",
    "basile_results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2d710427-903d-4336-b163-bf7d7082ca8e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# FPs\n",
    "print('\\n------\\n'.join(df[df.pred & ~df.real].sample(10, random_state=23543).text))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b32cdadb-2db6-442a-bbb0-4032eb79480b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# FNs\n",
    "print('\\n------\\n'.join(df[~df.pred & df.real].sample(10, random_state=23543).text))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "e91f25b9-a0f4-41d1-8291-4dd2c56657c5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style type=\"text/css\">\n",
       "</style>\n",
       "<table id=\"T_1f79a\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th class=\"blank level0\" >&nbsp;</th>\n",
       "      <th id=\"T_1f79a_level0_col0\" class=\"col_heading level0 col0\" >precision</th>\n",
       "      <th id=\"T_1f79a_level0_col1\" class=\"col_heading level0 col1\" >recall</th>\n",
       "      <th id=\"T_1f79a_level0_col2\" class=\"col_heading level0 col2\" >f-score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th id=\"T_1f79a_level0_row0\" class=\"row_heading level0 row0\" >Founta</th>\n",
       "      <td id=\"T_1f79a_row0_col0\" class=\"data row0 col0\" >76.3%</td>\n",
       "      <td id=\"T_1f79a_row0_col1\" class=\"data row0 col1\" >66.6%</td>\n",
       "      <td id=\"T_1f79a_row0_col2\" class=\"data row0 col2\" >71.1%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th id=\"T_1f79a_level0_row1\" class=\"row_heading level0 row1\" >Basile</th>\n",
       "      <td id=\"T_1f79a_row1_col0\" class=\"data row1 col0\" >47.5%</td>\n",
       "      <td id=\"T_1f79a_row1_col1\" class=\"data row1 col1\" >89.0%</td>\n",
       "      <td id=\"T_1f79a_row1_col2\" class=\"data row1 col2\" >62.0%</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7f68206e0d90>"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame([founta_results, basile_results],\n",
    "             columns=['precision', 'recall', 'f-score', '_'],\n",
    "             index=['Founta', 'Basile']\n",
    "            ).iloc[:, :3].style.format('{:.1%}'.format)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3143de6c-8a48-4bfc-b5c7-4f4205c5a556",
   "metadata": {},
   "source": [
    "### Restoring precision and recall"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "d8f4f226-b7bb-43f1-b649-b1f978db1fbb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "precision: 56.1%\n",
      "recall: 77.3%\n",
      "f1: 65.0%\n"
     ]
    }
   ],
   "source": [
    "def positive_class_metrics(positive_class_percent, macro_f1, accuracy):\n",
    "    def f1_macro_fn(tp):\n",
    "        tn = accuracy - tp\n",
    "        fp = negative_class_percent - tn\n",
    "        fn = positive_class_percent - tp\n",
    "        p1 = tp / (tp + fp + 1e-100)\n",
    "        r1 = tp / (tp + fn + 1e-100)\n",
    "        p0 = tn / (tn + fn + 1e-100)\n",
    "        r0 = tn / (tn + fp + 1e-100)\n",
    "        f1_fn = p0 * r0 / (p0 + r0 + 1e-100) + p1 * r1 / (p1 + r1 + 1e-100)\n",
    "        return f1_fn, p1, r1\n",
    "\n",
    "    negative_class_percent = 1 - positive_class_percent\n",
    "    n = 1000\n",
    "    diff, tp_opt = min((abs(f1_macro_fn(tp / n)[0] - macro_f1), tp / n) for tp in range(n + 1))\n",
    "    _, p1, r1 = f1_macro_fn(tp_opt)\n",
    "    f1 = 2 / (1 / p1 + 1 / r1)\n",
    "    print(f'precision: {p1:.1%}')\n",
    "    print(f'recall: {r1:.1%}')\n",
    "    print(f'f1: {f1:.1%}')\n",
    "\n",
    "\n",
    "positive_class_metrics(\n",
    "    positive_class_percent=0.4206773618538324,\n",
    "    accuracy=0.65,\n",
    "    macro_f1=0.651)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}