02_evaluate_ner_test.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Re-evaluation\n",
    "Perform two evaluations:\n",
    "1. Strict morpheme evaluation\n",
    "1. Token evaluation (morpheme labels are extended to the token level heuristically)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The autoreload extension is already loaded. To reload it, use:\n",
      "  %reload_ext autoreload\n"
     ]
    }
   ],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-13T07:20:24.034028Z",
     "start_time": "2019-03-13T07:20:22.687626Z"
    }
   },
   "outputs": [],
   "source": [
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-13T07:20:24.382406Z",
     "start_time": "2019-03-13T07:20:24.037019Z"
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-13T07:20:27.996727Z",
     "start_time": "2019-03-13T07:20:24.385088Z"
    }
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "sns.set_context('paper')\n",
    "sns.set_style('white')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append('/home/nlp/danb/NER')\n",
    "\n",
    "import bclm\n",
    "import ne_evaluate_mentions as nem"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_biose_count(path, sent_id_shift=1):\n",
    "    sents = nem.read_file_sents(path, fix_multi_tag=False, sent_id_shift=sent_id_shift)\n",
    "    bc = []\n",
    "    for i, sent in sents.iteritems():\n",
    "        for j, (tok, bio) in enumerate(sent):\n",
    "            bc.append([i, j+1, tok, bio, len(bio.split('^'))])\n",
    "\n",
    "    bc = pd.DataFrame(bc, columns=['sent_id', 'token_id', 'token_str', \n",
    "                                   'biose', 'biose_count'])\n",
    "    return bc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "import networkx as nx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_valid_edges(lattices, bc,\n",
    "                    non_o_only=True, keep_all_if_no_valid=True):\n",
    "    valid_edges = []\n",
    "    for (i, df), (_, biose, biose_count) in zip(lattices.groupby(['sent_id', 'token_id']), \n",
    "                                                bc[['biose', 'biose_count']].itertuples()):\n",
    "        el = df[['ID1', 'ID2']].rename(columns={'ID1': 'source', 'ID2': 'target'})\n",
    "        #min_node = [n for n,v in G.nodes(data=True) if v['since'] == 'December 2008'][0]\n",
    "\n",
    "        g = nx.from_pandas_edgelist(el, create_using=nx.DiGraph)\n",
    "        min_node = el.source.min()\n",
    "        max_node = el.target.max()\n",
    "        #print(min_node,max_node)\n",
    "        #print(biose_count)\n",
    "        if non_o_only and not '-' in biose:\n",
    "            vp = list(nx.all_simple_paths(g, min_node, max_node))\n",
    "        else:\n",
    "            vp = [path for path in nx.all_simple_paths(g, min_node, max_node, cutoff=biose_count+1) if len(path)==biose_count+1]\n",
    "        if keep_all_if_no_valid and len(vp)==0:\n",
    "             vp = nx.all_simple_paths(g, min_node, max_node)\n",
    "        for path in vp:\n",
    "            for source, target in zip(path[:-1], path[1:]):\n",
    "                valid_edges.append((i[0], i[1], source, target))\n",
    "                \n",
    "    return valid_edges"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def to_lattices(df, path, cols = ['ID1', 'ID2', 'form', 'lemma', 'upostag', 'xpostag', 'feats', 'token_id']):\n",
    "    with open(path, 'w', encoding='utf8') as of:\n",
    "        for _, sent in df.groupby('sent_id'):\n",
    "            for _, row in sent[cols].iterrows():\n",
    "                of.write('\\t'.join(row.astype(str).tolist())+'\\n')\n",
    "            of.write('\\n')\n",
    "            \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def prune_lattices(lattices_path, ner_pred_path, output_path, keep_all_if_no_valid=True):\n",
    "    lat = bclm.read_lattices(lattices_path)\n",
    "    bc = get_biose_count(ner_pred_path, sent_id_shift=1)\n",
    "    valid_edges = get_valid_edges(lat, bc, non_o_only=False, keep_all_if_no_valid=keep_all_if_no_valid)\n",
    "    cols = ['sent_id', 'token_id', 'ID1', 'ID2']\n",
    "    pruned_lat = lat[lat[cols].apply(lambda x: tuple(x), axis=1).isin(valid_edges)]\n",
    "    to_lattices(pruned_lat, output_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluate Segmentation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "dropped = [5438, 5444, 5445, 5446, 5448, 5449, 5450, 5451, 5453, 5459]\n",
    "spdf = bclm.read_dataframe('spmrl')\n",
    "spdf = spdf[(~spdf.sent_id.isin(dropped))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/nlp/danb/anaconda3/envs/tf/lib/python3.7/site-packages/ipykernel_launcher.py:6: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  \n"
     ]
    }
   ],
   "source": [
    "dev_gold = bclm.read_dataframe('spmrl', subset='dev')\n",
    "test_gold = spdf[spdf.set=='test']\n",
    "test_sent_id_map = (test_gold.groupby('sent_id').size()\n",
    "                    .reset_index().drop(0, axis=1).reset_index()\n",
    "                    .assign(index=lambda x: x+1).set_index('sent_id')['index'])\n",
    "test_gold['sent_id'] = test_gold.sent_id.map(test_sent_id_map)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "cols = ['sent_id', 'token_id', 'form']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/nlp/danb/anaconda3/envs/tf/lib/python3.7/site-packages/bclm/evaluations.py:48: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  gold_df['upostag'] = gold_df.upostag.str.replace('_','-')\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "16828 gold tokens/morphems, 16857 predicted, 15399 correct.\n",
      "Precision: 91.35\n",
      "Recall:    91.51\n",
      "F1:        91.43\n",
      "FP ex.: [(1, 1, 'הכל', 'VB'), (1, 3, 'עם', 'IN'), (1, 17, 'החל', 'VB'), (1, 20, 'כלה', 'NN'), (2, 1, 'אומר', 'NNT')]\n",
      "FN ex.: [(1, 1, 'הכל', 'NN'), (1, 3, 'עמ', 'IN'), (1, 17, 'החל', 'IN'), (1, 20, 'כלה', 'IN'), (2, 1, 'אומר', 'BN')]\n",
      "16828 gold tokens/morphems, 16857 predicted, 16421 correct.\n",
      "Precision: 97.41\n",
      "Recall:    97.58\n",
      "F1:        97.5\n",
      "FP ex.: [(1, 3, 'עם'), (2, 3, 'ה'), (2, 3, 'תק\"ם'), (4, 10, 'לנו'), (4, 10, 'ש')]\n",
      "FN ex.: [(1, 3, 'עמ'), (2, 3, 'התק\"ם'), (4, 10, 'אנחנו'), (4, 10, 'של'), (6, 9, 'O')]\n",
      "16828 gold tokens/morphems, 16842 predicted, 15394 correct.\n",
      "Precision: 91.4\n",
      "Recall:    91.48\n",
      "F1:        91.44\n",
      "FP ex.: [(1, 1, 'הכל', 'VB'), (1, 3, 'עם', 'IN'), (1, 17, 'החל', 'VB'), (1, 20, 'כלה', 'NN'), (2, 1, 'אומר', 'NNT')]\n",
      "FN ex.: [(1, 1, 'הכל', 'NN'), (1, 3, 'עמ', 'IN'), (1, 17, 'החל', 'IN'), (1, 20, 'כלה', 'IN'), (2, 1, 'אומר', 'BN')]\n",
      "16828 gold tokens/morphems, 16842 predicted, 16406 correct.\n",
      "Precision: 97.41\n",
      "Recall:    97.49\n",
      "F1:        97.45\n",
      "FP ex.: [(1, 3, 'עם'), (2, 3, 'ה'), (2, 3, 'תק\"ם'), (4, 10, 'לנו'), (4, 10, 'ש')]\n",
      "FN ex.: [(1, 3, 'עמ'), (2, 3, 'התק\"ם'), (4, 10, 'אנחנו'), (4, 10, 'של'), (6, 9, 'O')]\n",
      "16828 gold tokens/morphems, 16846 predicted, 15406 correct.\n",
      "Precision: 91.45\n",
      "Recall:    91.55\n",
      "F1:        91.5\n",
      "FP ex.: [(1, 1, 'הכל', 'VB'), (1, 3, 'עם', 'IN'), (1, 17, 'החל', 'VB'), (1, 20, 'כלה', 'NN'), (2, 1, 'אומר', 'NNT')]\n",
      "FN ex.: [(1, 1, 'הכל', 'NN'), (1, 3, 'עמ', 'IN'), (1, 17, 'החל', 'IN'), (1, 20, 'כלה', 'IN'), (2, 1, 'אומר', 'BN')]\n",
      "16828 gold tokens/morphems, 16846 predicted, 16418 correct.\n",
      "Precision: 97.46\n",
      "Recall:    97.56\n",
      "F1:        97.51\n",
      "FP ex.: [(1, 3, 'עם'), (2, 3, 'ה'), (2, 3, 'תק\"ם'), (4, 10, 'לנו'), (4, 10, 'ש')]\n",
      "FN ex.: [(1, 3, 'עמ'), (2, 3, 'התק\"ם'), (4, 10, 'אנחנו'), (4, 10, 'של'), (6, 9, 'O')]\n",
      "16828 gold tokens/morphems, 16842 predicted, 15381 correct.\n",
      "Precision: 91.33\n",
      "Recall:    91.4\n",
      "F1:        91.36\n",
      "FP ex.: [(1, 1, 'הכל', 'VB'), (1, 3, 'עם', 'IN'), (1, 17, 'החל', 'VB'), (1, 20, 'כלה', 'NN'), (2, 1, 'אומר', 'NNT')]\n",
      "FN ex.: [(1, 1, 'הכל', 'NN'), (1, 3, 'עמ', 'IN'), (1, 17, 'החל', 'IN'), (1, 20, 'כלה', 'IN'), (2, 1, 'אומר', 'BN')]\n",
      "16828 gold tokens/morphems, 16842 predicted, 16413 correct.\n",
      "Precision: 97.45\n",
      "Recall:    97.53\n",
      "F1:        97.49\n",
      "FP ex.: [(1, 3, 'עם'), (2, 3, 'ה'), (2, 3, 'תק\"ם'), (4, 10, 'לנו'), (4, 10, 'ש')]\n",
      "FN ex.: [(1, 3, 'עמ'), (2, 3, 'התק\"ם'), (4, 10, 'אנחנו'), (4, 10, 'של'), (6, 9, 'O')]\n",
      "16828 gold tokens/morphems, 16872 predicted, 15400 correct.\n",
      "Precision: 91.28\n",
      "Recall:    91.51\n",
      "F1:        91.39\n",
      "FP ex.: [(1, 1, 'הכל', 'VB'), (1, 3, 'עם', 'IN'), (1, 17, 'החל', 'VB'), (1, 20, 'כלה', 'NN'), (2, 1, 'אומר', 'NNT')]\n",
      "FN ex.: [(1, 1, 'הכל', 'NN'), (1, 3, 'עמ', 'IN'), (1, 17, 'החל', 'IN'), (1, 20, 'כלה', 'IN'), (2, 1, 'אומר', 'BN')]\n",
      "16828 gold tokens/morphems, 16872 predicted, 16423 correct.\n",
      "Precision: 97.34\n",
      "Recall:    97.59\n",
      "F1:        97.47\n",
      "FP ex.: [(1, 3, 'עם'), (2, 3, 'ה'), (2, 3, 'תק\"ם'), (4, 10, 'לנו'), (4, 10, 'ש')]\n",
      "FN ex.: [(1, 3, 'עמ'), (2, 3, 'התק\"ם'), (4, 10, 'אנחנו'), (4, 10, 'של'), (6, 9, 'O')]\n",
      "16828 gold tokens/morphems, 16842 predicted, 15403 correct.\n",
      "Precision: 91.46\n",
      "Recall:    91.53\n",
      "F1:        91.49\n",
      "FP ex.: [(1, 1, 'הכל', 'VB'), (1, 3, 'עם', 'IN'), (1, 17, 'החל', 'VB'), (1, 20, 'כלה', 'NN'), (2, 1, 'אומר', 'NNT')]\n",
      "FN ex.: [(1, 1, 'הכל', 'NN'), (1, 3, 'עמ', 'IN'), (1, 17, 'החל', 'IN'), (1, 20, 'כלה', 'IN'), (2, 1, 'אומר', 'BN')]\n",
      "16828 gold tokens/morphems, 16842 predicted, 16427 correct.\n",
      "Precision: 97.54\n",
      "Recall:    97.62\n",
      "F1:        97.58\n",
      "FP ex.: [(1, 3, 'עם'), (2, 3, 'ה'), (2, 3, 'תק\"ם'), (4, 10, 'לנו'), (4, 10, 'ש')]\n",
      "FN ex.: [(1, 3, 'עמ'), (2, 3, 'התק\"ם'), (4, 10, 'אנחנו'), (4, 10, 'של'), (6, 9, 'O')]\n",
      "16828 gold tokens/morphems, 16853 predicted, 15403 correct.\n",
      "Precision: 91.4\n",
      "Recall:    91.53\n",
      "F1:        91.46\n",
      "FP ex.: [(1, 1, 'הכל', 'VB'), (1, 3, 'עם', 'IN'), (1, 17, 'החל', 'VB'), (1, 20, 'כלה', 'NN'), (2, 1, 'אומר', 'NNT')]\n",
      "FN ex.: [(1, 1, 'הכל', 'NN'), (1, 3, 'עמ', 'IN'), (1, 17, 'החל', 'IN'), (1, 20, 'כלה', 'IN'), (2, 1, 'אומר', 'BN')]\n",
      "16828 gold tokens/morphems, 16853 predicted, 16415 correct.\n",
      "Precision: 97.4\n",
      "Recall:    97.55\n",
      "F1:        97.47\n",
      "FP ex.: [(1, 3, 'עם'), (2, 3, 'ה'), (2, 3, 'תק\"ם'), (4, 10, 'לנו'), (4, 10, 'ש')]\n",
      "FN ex.: [(1, 3, 'עמ'), (2, 3, 'התק\"ם'), (4, 10, 'אנחנו'), (4, 10, 'של'), (6, 9, 'O')]\n",
      "16828 gold tokens/morphems, 16875 predicted, 15404 correct.\n",
      "Precision: 91.28\n",
      "Recall:    91.54\n",
      "F1:        91.41\n",
      "FP ex.: [(1, 1, 'הכל', 'VB'), (1, 3, 'עם', 'IN'), (1, 17, 'החל', 'VB'), (1, 20, 'כלה', 'NN'), (2, 1, 'אומר', 'NNT')]\n",
      "FN ex.: [(1, 1, 'הכל', 'NN'), (1, 3, 'עמ', 'IN'), (1, 17, 'החל', 'IN'), (1, 20, 'כלה', 'IN'), (2, 1, 'אומר', 'BN')]\n",
      "16828 gold tokens/morphems, 16875 predicted, 16430 correct.\n",
      "Precision: 97.36\n",
      "Recall:    97.63\n",
      "F1:        97.5\n",
      "FP ex.: [(1, 3, 'עם'), (2, 3, 'ה'), (2, 3, 'תק\"ם'), (4, 10, 'לנו'), (4, 10, 'ש')]\n",
      "FN ex.: [(1, 3, 'עמ'), (2, 3, 'התק\"ם'), (4, 10, 'אנחנו'), (4, 10, 'של'), (6, 9, 'O')]\n",
      "16828 gold tokens/morphems, 16854 predicted, 15402 correct.\n",
      "Precision: 91.38\n",
      "Recall:    91.53\n",
      "F1:        91.46\n",
      "FP ex.: [(1, 1, 'הכל', 'VB'), (1, 3, 'עם', 'IN'), (1, 17, 'החל', 'VB'), (1, 20, 'כלה', 'NN'), (2, 1, 'אומר', 'NNT')]\n",
      "FN ex.: [(1, 1, 'הכל', 'NN'), (1, 3, 'עמ', 'IN'), (1, 17, 'החל', 'IN'), (1, 20, 'כלה', 'IN'), (2, 1, 'אומר', 'BN')]\n",
      "16828 gold tokens/morphems, 16854 predicted, 16420 correct.\n",
      "Precision: 97.42\n",
      "Recall:    97.58\n",
      "F1:        97.5\n",
      "FP ex.: [(1, 3, 'עם'), (2, 3, 'ה'), (2, 3, 'תק\"ם'), (4, 10, 'לנו'), (4, 10, 'ש')]\n",
      "FN ex.: [(1, 3, 'עמ'), (2, 3, 'התק\"ם'), (4, 10, 'אנחנו'), (4, 10, 'של'), (6, 9, 'O')]\n",
      "16828 gold tokens/morphems, 16838 predicted, 15384 correct.\n",
      "Precision: 91.36\n",
      "Recall:    91.42\n",
      "F1:        91.39\n",
      "FP ex.: [(1, 1, 'הכל', 'VB'), (1, 3, 'עם', 'IN'), (1, 17, 'החל', 'VB'), (1, 20, 'כלה', 'NN'), (2, 1, 'אומר', 'NNT')]\n",
      "FN ex.: [(1, 1, 'הכל', 'NN'), (1, 3, 'עמ', 'IN'), (1, 17, 'החל', 'IN'), (1, 20, 'כלה', 'IN'), (2, 1, 'אומר', 'BN')]\n",
      "16828 gold tokens/morphems, 16838 predicted, 16413 correct.\n",
      "Precision: 97.48\n",
      "Recall:    97.53\n",
      "F1:        97.5\n",
      "FP ex.: [(1, 3, 'עם'), (2, 3, 'ה'), (2, 3, 'תק\"ם'), (4, 10, 'לנו'), (4, 10, 'ש')]\n",
      "FN ex.: [(1, 3, 'עמ'), (2, 3, 'התק\"ם'), (4, 10, 'אנחנו'), (4, 10, 'של'), (6, 9, 'O')]\n"
     ]
    }
   ],
   "source": [
    "res = []\n",
    "for folder in os.scandir('output/predict'):\n",
    "    if 'multi' in folder.name and not '.ipynb' in folder.name:\n",
    "        test_lfo = bclm.read_yap_output(treebank_set=None, tokens_path=bclm.TREEBANK_TOKEN_PATHS['test'], \n",
    "                                             dep_path=os.path.join(folder, 'test_pruned.conll'),\n",
    "                                             map_path=os.path.join(folder, 'test_pruned.map'))\n",
    "        p,r,f_sp = bclm.evaluate_dfs(test_gold, test_lfo)\n",
    "        \n",
    "        p,r,f_so = bclm.evaluate_dfs(test_gold, test_lfo, cols=cols)\n",
    "        \n",
    "        res.append((folder.name, f_sp, f_so))\n",
    "#         for file in os.scandir(folder):\n",
    "#             if '.bmes' in file.name and not '.ipynb' in file.name:\n",
    "#                 if 'dev' in file.name:\n",
    "#                     prune_lattices(bclm.LATTICES_PATHS['dev'], \n",
    "#                        file.path,\n",
    "#                        os.path.join(folder.path, 'dev_pruned.lat'))\n",
    "#                 elif 'test' in file.name:\n",
    "#                     prune_lattices(bclm.LATTICES_PATHS['test'], \n",
    "#                        file.path,\n",
    "#                        os.path.join(folder.path, 'test_pruned.lat'))\n",
    "                    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>f_seg_pos</th>\n",
       "      <th>f_seg_only</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>multi_54360</td>\n",
       "      <td>91.429420</td>\n",
       "      <td>97.497402</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>multi_44184</td>\n",
       "      <td>91.440451</td>\n",
       "      <td>97.451737</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>multi_20423</td>\n",
       "      <td>91.500861</td>\n",
       "      <td>97.511433</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>multi_80520</td>\n",
       "      <td>91.363231</td>\n",
       "      <td>97.493317</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>multi_27916</td>\n",
       "      <td>91.394659</td>\n",
       "      <td>97.465875</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>multi_63795</td>\n",
       "      <td>91.493911</td>\n",
       "      <td>97.576478</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>multi_30528</td>\n",
       "      <td>91.464030</td>\n",
       "      <td>97.473353</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>multi_78160</td>\n",
       "      <td>91.410260</td>\n",
       "      <td>97.498739</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>multi_12345</td>\n",
       "      <td>91.455377</td>\n",
       "      <td>97.500148</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>multi_95148</td>\n",
       "      <td>91.391909</td>\n",
       "      <td>97.504901</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         model  f_seg_pos  f_seg_only\n",
       "0  multi_54360  91.429420   97.497402\n",
       "1  multi_44184  91.440451   97.451737\n",
       "2  multi_20423  91.500861   97.511433\n",
       "3  multi_80520  91.363231   97.493317\n",
       "4  multi_27916  91.394659   97.465875\n",
       "5  multi_63795  91.493911   97.576478\n",
       "6  multi_30528  91.464030   97.473353\n",
       "7  multi_78160  91.410260   97.498739\n",
       "8  multi_12345  91.455377   97.500148\n",
       "9  multi_95148  91.391909   97.504901"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "seg_res_df = pd.DataFrame(res, columns=['model', 'f_seg_pos', 'f_seg_only'])\n",
    "seg_res_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "f_seg_pos     91.434411\n",
       "f_seg_only    97.497338\n",
       "dtype: float64"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "seg_res_df.mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Align Multitok"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "def soft_merge_bio_labels(multitok_sents, tokmorph_sents, verbose=False):\n",
    "    new_sents = []\n",
    "    for (i, mt_sent), (sent_id, mor_sent) in zip(multitok_sents.iteritems(), tokmorph_sents.iteritems()):\n",
    "        new_sent = []\n",
    "        for (form, bio), (token_id, token_str, forms) in zip(mt_sent, mor_sent):\n",
    "            forms = forms.split('^')\n",
    "            bio = bio.split('^')\n",
    "            if len(forms) == len(bio):\n",
    "                new_forms = (1, list(zip(forms,bio)))\n",
    "            elif len(forms)>len(bio):\n",
    "                dif = len(forms) - len(bio)\n",
    "                new_forms = (2, list(zip(forms[:dif],['O']*dif)) + list(zip(forms[::-1], bio[::-1]))[::-1])\n",
    "                if verbose:\n",
    "                    print(new_forms)\n",
    "            else:\n",
    "                new_forms = (3, list(zip(forms[::-1], bio[::-1]))[::-1])\n",
    "                if verbose:\n",
    "                    print(new_forms)\n",
    "            new_sent.extend(new_forms[1])\n",
    "        new_sents.append(new_sent)\n",
    "    return new_sents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def align_multitok(ner_pred_path, tokens_path, conll_path, map_path, output_path):\n",
    "    x = nem.read_file_sents(ner_pred_path, fix_multi_tag=False)\n",
    "    prun_yo = bclm.read_yap_output(treebank_set=None, tokens_path=tokens_path, dep_path=conll_path, map_path=map_path)\n",
    "    prun_yo = bclm.get_token_df(prun_yo, fields=['form'])\n",
    "    prun_sents = bclm.get_sentences_list(prun_yo, fields=['token_id', 'token_str', 'form'])\n",
    "    new_sents = soft_merge_bio_labels(x, prun_sents, verbose=False)\n",
    "\n",
    "    with open(output_path, 'w') as of:\n",
    "        for sent in new_sents:\n",
    "            for form, bio in sent:\n",
    "                of.write(form+' '+bio+'\\n')\n",
    "            of.write('\\n')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "decode_sets = {\n",
    "    'token': {\n",
    "        'dev': '../NER/data/for_ncrf/morph_gold_dev.bmes',\n",
    "        'test': '../NER/data/for_ncrf/morph_gold_test.bmes',\n",
    "    },\n",
    "    'multitok': {\n",
    "        'dev': '../NER/data/for_ncrf/morph_gold_dev.bmes',\n",
    "        'test': '../NER/data/for_ncrf/morph_gold_test.bmes',\n",
    "    }\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.75, 0.7628755364806867, 0.7563829787234043)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "align_multitok('output/predict/multi_44184/token_gold_test_dummy_o.bmes', \n",
    "               bclm.TREEBANK_TOKEN_PATHS['test'], \n",
    "               'output/predict/multi_44184/test_pruned.conll',\n",
    "               'output/predict/multi_44184/test_pruned.map',\n",
    "               'output/predict/multi_44184/morph_pruned_test.bmes'\n",
    "              )\n",
    "p, r, f = nem.evaluate_files(decode_sets['multitok']['test'], 'output/predict/multi_44184/morph_pruned_test.bmes', str_join_char='')\n",
    "p,r,f"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.7789473684210526, 0.7939914163090128, 0.7863974495217853)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nem.evaluate_files('../NER/data/for_ncrf/token_gold_test_fix.bmes', \n",
    "                   'output/predict/multi_44184/token_gold_test_dummy_o.bmes', str_join_char='')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "multi_54360\n",
      "multi_44184\n",
      "multi_20423\n",
      "multi_80520\n",
      "multi_27916\n",
      "multi_63795\n",
      "multi_30528\n",
      "multi_78160\n",
      "multi_12345\n",
      "multi_95148\n"
     ]
    }
   ],
   "source": [
    "res = []\n",
    "for folder in os.scandir('output/predict'):\n",
    "    if os.path.isdir(folder) and 'multi' in folder.name and not '.ipynb_checkpoints' in folder.name:\n",
    "        print (folder.name)\n",
    "\n",
    "        pruned_ner_path=os.path.join(folder.path, 'morph_pruned_test.bmes')\n",
    "        \n",
    "        align_multitok(os.path.join(folder.path, 'token_gold_test_dummy_o.bmes'), \n",
    "                       bclm.TREEBANK_TOKEN_PATHS['test'], \n",
    "                       os.path.join(folder.path, 'test_pruned.conll'),\n",
    "                       os.path.join(folder.path, 'test_pruned.map'),\n",
    "                       pruned_ner_path\n",
    "                      )\n",
    "        p, r, f = nem.evaluate_files(decode_sets['multitok']['test'], pruned_ner_path, str_join_char='')\n",
    "        res.append((folder.name, p, r, f))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('multi_54360', 0.7497371188222923, 0.7650214592274678, 0.7573021773765267),\n",
       " ('multi_44184', 0.75, 0.7628755364806867, 0.7563829787234043),\n",
       " ('multi_20423', 0.75, 0.7660944206008584, 0.7579617834394905),\n",
       " ('multi_80520', 0.7368972746331237, 0.7542918454935622, 0.7454931071049842),\n",
       " ('multi_27916', 0.7466666666666667, 0.7811158798283262, 0.7635028841111694),\n",
       " ('multi_63795', 0.7357512953367875, 0.7618025751072961, 0.7485503426462835),\n",
       " ('multi_30528', 0.7427685950413223, 0.7714592274678111, 0.7568421052631579),\n",
       " ('multi_78160', 0.7515856236786469, 0.7628755364806867, 0.7571884984025559),\n",
       " ('multi_12345', 0.7306122448979592, 0.7682403433476395, 0.7489539748953974),\n",
       " ('multi_95148', 0.7569296375266524, 0.7618025751072961, 0.7593582887700534)]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>f</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>multi_54360</td>\n",
       "      <td>0.749737</td>\n",
       "      <td>0.765021</td>\n",
       "      <td>0.757302</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>multi_44184</td>\n",
       "      <td>0.750000</td>\n",
       "      <td>0.762876</td>\n",
       "      <td>0.756383</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>multi_20423</td>\n",
       "      <td>0.750000</td>\n",
       "      <td>0.766094</td>\n",
       "      <td>0.757962</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>multi_80520</td>\n",
       "      <td>0.736897</td>\n",
       "      <td>0.754292</td>\n",
       "      <td>0.745493</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>multi_27916</td>\n",
       "      <td>0.746667</td>\n",
       "      <td>0.781116</td>\n",
       "      <td>0.763503</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>multi_63795</td>\n",
       "      <td>0.735751</td>\n",
       "      <td>0.761803</td>\n",
       "      <td>0.748550</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>multi_30528</td>\n",
       "      <td>0.742769</td>\n",
       "      <td>0.771459</td>\n",
       "      <td>0.756842</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>multi_78160</td>\n",
       "      <td>0.751586</td>\n",
       "      <td>0.762876</td>\n",
       "      <td>0.757188</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>multi_12345</td>\n",
       "      <td>0.730612</td>\n",
       "      <td>0.768240</td>\n",
       "      <td>0.748954</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>multi_95148</td>\n",
       "      <td>0.756930</td>\n",
       "      <td>0.761803</td>\n",
       "      <td>0.759358</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         model  precision    recall         f\n",
       "0  multi_54360   0.749737  0.765021  0.757302\n",
       "1  multi_44184   0.750000  0.762876  0.756383\n",
       "2  multi_20423   0.750000  0.766094  0.757962\n",
       "3  multi_80520   0.736897  0.754292  0.745493\n",
       "4  multi_27916   0.746667  0.781116  0.763503\n",
       "5  multi_63795   0.735751  0.761803  0.748550\n",
       "6  multi_30528   0.742769  0.771459  0.756842\n",
       "7  multi_78160   0.751586  0.762876  0.757188\n",
       "8  multi_12345   0.730612  0.768240  0.748954\n",
       "9  multi_95148   0.756930  0.761803  0.759358"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ne_morph_df = pd.DataFrame(res, columns=['model', 'precision', 'recall', 'f'])\n",
    "ne_morph_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "706\n",
      "706\n",
      "706\n",
      "706\n",
      "706\n",
      "706\n",
      "706\n",
      "706\n",
      "706\n",
      "706\n"
     ]
    }
   ],
   "source": [
    "def biose_to_o(in_path, out_path):\n",
    "    sents = 0\n",
    "    with open(out_path, 'w', encoding='utf8') as of:\n",
    "        for line in open(in_path, 'r'):\n",
    "            if line=='\\n':\n",
    "                of.write(line)\n",
    "                sents+=1\n",
    "            else:\n",
    "                line = line.strip()\n",
    "                word, tag = line.split()\n",
    "                tag = 'O'\n",
    "                of.write(word+' '+tag+'\\n')\n",
    "            \n",
    "    print (sents)\n",
    "    \n",
    "for folder in os.scandir('output/predict'):\n",
    "    if os.path.isdir(folder) and 'multi' in folder.name and not '.ipynb_checkpoints' in folder.name:\n",
    "        pruned_ner_path=os.path.join(folder.path, 'morph_pruned_test.bmes')\n",
    "        biose_to_o(pruned_ner_path, pruned_ner_path.replace('.bmes', '.bioul'))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "def jsonl_to_biose(in_path, out_path, bioul_to_biose=True):\n",
    "    sents = 0\n",
    "    with open(out_path, 'w', encoding='utf8') as of:\n",
    "        for line in open(in_path, 'r'):\n",
    "            sent = json.loads(line)\n",
    "            for word, tag in zip(sent['words'], sent['tags']):\n",
    "                if bioul_to_biose:\n",
    "                    tag = tag.replace('L-', 'E-').replace('U-', 'S-')\n",
    "                of.write(word+' '+tag+'\\n')\n",
    "            of.write('\\n')\n",
    "            sents+=1\n",
    "    print (sents)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "706\n",
      "706\n",
      "706\n",
      "706\n",
      "706\n",
      "706\n",
      "706\n",
      "706\n",
      "706\n",
      "706\n"
     ]
    }
   ],
   "source": [
    "for folder in os.scandir('output/predict'):\n",
    "    if os.path.isdir(folder) and 'morph' in folder.name and not '.ipynb_checkpoints' in folder.name:\n",
    "        pruned_ner_path=os.path.join(folder.path, 'morph_pruned_test.json')\n",
    "        jsonl_to_biose(pruned_ner_path, pruned_ner_path.replace('.json', '.bmes'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## SINGLE + MULTI"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>p</th>\n",
       "      <th>r</th>\n",
       "      <th>f</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>eval_unit</th>\n",
       "      <th>variant</th>\n",
       "      <th>prediction</th>\n",
       "      <th>align</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">morph</th>\n",
       "      <th rowspan=\"3\" valign=\"top\">morph</th>\n",
       "      <th>gold</th>\n",
       "      <th>-</th>\n",
       "      <td>0.760813</td>\n",
       "      <td>0.796888</td>\n",
       "      <td>0.778388</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hybrid</th>\n",
       "      <th>-</th>\n",
       "      <td>0.734401</td>\n",
       "      <td>0.761588</td>\n",
       "      <td>0.747710</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>yap</th>\n",
       "      <th>-</th>\n",
       "      <td>0.661856</td>\n",
       "      <td>0.673391</td>\n",
       "      <td>0.667525</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">token</th>\n",
       "      <th>multi</th>\n",
       "      <th>tokens</th>\n",
       "      <th>-</th>\n",
       "      <td>0.774972</td>\n",
       "      <td>0.797747</td>\n",
       "      <td>0.786160</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>single</th>\n",
       "      <th>tokens</th>\n",
       "      <th>-</th>\n",
       "      <td>0.781663</td>\n",
       "      <td>0.814914</td>\n",
       "      <td>0.797819</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           p         r         f\n",
       "eval_unit variant prediction align                              \n",
       "morph     morph   gold       -      0.760813  0.796888  0.778388\n",
       "                  hybrid     -      0.734401  0.761588  0.747710\n",
       "                  yap        -      0.661856  0.673391  0.667525\n",
       "token     multi   tokens     -      0.774972  0.797747  0.786160\n",
       "          single  tokens     -      0.781663  0.814914  0.797819"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res = []\n",
    "for folder in os.scandir('output/predict'):\n",
    "    if '.ipynb' in folder.name:\n",
    "        continue\n",
    "        \n",
    "    variant, seed = folder.name.split('_')\n",
    "    \n",
    "    if 'single' in folder.name:    \n",
    "        p,r,f = nem.evaluate_files('../NER/data/for_ncrf/token_gold_test_fix.bmes', \n",
    "                                   os.path.join(folder.path,'token_gold_test_fix.bmes'), str_join_char='')\n",
    "        res.append(('test', 'token', variant, 'tokens', '-', seed, p, r, f))\n",
    "\n",
    "        \n",
    "    if 'multi' in folder.name:\n",
    "        p,r,f = nem.evaluate_files('../NER/data/for_ncrf/token_gold_test_fix.bmes', \n",
    "                                   os.path.join(folder.path,'token_gold_test_dummy_o.bmes'), str_join_char='')\n",
    "        res.append(('test', 'token', variant, 'tokens', '-', seed, p, r, f))\n",
    "        \n",
    "    if 'morph' in folder.name:\n",
    "        p,r,f = nem.evaluate_files('../NER/data/for_ncrf/morph_gold_test.bmes', \n",
    "                                   os.path.join(folder.path,'morph_gold_test.bmes'), str_join_char='')\n",
    "        res.append(('test', 'morph', variant, 'gold', '-', seed, p, r, f))\n",
    "        p,r,f = nem.evaluate_files('../NER/data/for_ncrf/morph_gold_test.bmes', \n",
    "                                   os.path.join(folder.path,'morph_yap_test.bmes'), str_join_char='')\n",
    "        res.append(('test', 'morph', variant, 'yap', '-', seed, p, r, f))\n",
    "        p,r,f = nem.evaluate_files('../NER/data/for_ncrf/morph_gold_test.bmes', \n",
    "                                   os.path.join(folder.path,'morph_pruned_test.bmes'), str_join_char='')\n",
    "        res.append(('test', 'morph', variant, 'hybrid', '-', seed, p, r, f))\n",
    "        \n",
    "        \n",
    "    \n",
    "    \n",
    "\n",
    "ne_df = pd.DataFrame(res, columns=['set', 'eval_unit', 'variant', 'prediction', 'align', 'seed', 'p', 'r', 'f'])\n",
    "\n",
    "ne_df.groupby(['eval_unit','variant', 'prediction', 'align'])[['p', 'r', 'f']].mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "eval_unit  variant  prediction  align\n",
       "morph      morph    gold        -        0.784453\n",
       "                    yap         -        0.673834\n",
       "token      multi    tokens      -        0.787413\n",
       "           single   tokens      -        0.794716\n",
       "Name: f, dtype: float64"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res = []\n",
    "for folder in os.scandir('output/predict_char'):\n",
    "    if '.ipynb' in folder.name:\n",
    "        continue\n",
    "        \n",
    "    variant, seed = folder.name.split('_')\n",
    "    \n",
    "    if 'single' in folder.name:    \n",
    "        p,r,f = nem.evaluate_files('../NER/data/for_ncrf/token_gold_test_fix.bmes', \n",
    "                                   os.path.join(folder.path,'token_gold_test_fix.bmes'), str_join_char='')\n",
    "        res.append(('test', 'token', variant, 'tokens', '-', seed, p, r, f))\n",
    "\n",
    "        \n",
    "    if 'multi' in folder.name:\n",
    "        p,r,f = nem.evaluate_files('../NER/data/for_ncrf/token_gold_test_fix.bmes', \n",
    "                                   os.path.join(folder.path,'token_gold_test_dummy_o.bmes'), str_join_char='')\n",
    "        res.append(('test', 'token', variant, 'tokens', '-', seed, p, r, f))\n",
    "        \n",
    "    if 'morph' in folder.name:\n",
    "        p,r,f = nem.evaluate_files('../NER/data/for_ncrf/morph_gold_test.bmes', \n",
    "                                   os.path.join(folder.path,'morph_gold_test.bmes'), str_join_char='')\n",
    "        res.append(('test', 'morph', variant, 'gold', '-', seed, p, r, f))\n",
    "        p,r,f = nem.evaluate_files('../NER/data/for_ncrf/morph_gold_test.bmes', \n",
    "                                   os.path.join(folder.path,'morph_yap_test.bmes'), str_join_char='')\n",
    "        res.append(('test', 'morph', variant, 'yap', '-', seed, p, r, f))\n",
    "        #p,r,f = nem.evaluate_files('../NER/data/for_ncrf/morph_gold_test.bmes', \n",
    "        #                           os.path.join(folder.path,'morph_pruned_test.bmes'), str_join_char='')\n",
    "        #res.append(('test', 'morph', variant, 'hybrid', '-', seed, p, r, f))\n",
    "        \n",
    "        \n",
    "    \n",
    "    \n",
    "\n",
    "ne_c_df = pd.DataFrame(res, columns=['set', 'eval_unit', 'variant', 'prediction', 'align', 'seed', 'p', 'r', 'f'])\n",
    "\n",
    "ne_c_df.groupby(['eval_unit','variant', 'prediction', 'align']).f.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Add Alignments"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Token Level Eval"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "\n",
    "o_re = re.compile('^O+$') \n",
    "s_re = re.compile('^O*SO*$|^O*BI*EO*$')\n",
    "b_re = re.compile('^O*BI*$')\n",
    "i_re = re.compile('^I+$')\n",
    "e_re = re.compile('^I*EO*$')\n",
    "def get_fixed_for_valid_biose(bio_seq):\n",
    "    if o_re.match(bio_seq):\n",
    "        return 'O'\n",
    "    if s_re.match(bio_seq):\n",
    "        return 'S'\n",
    "    if b_re.match(bio_seq):\n",
    "        return 'B'\n",
    "    if i_re.match(bio_seq):\n",
    "        return 'I'\n",
    "    if e_re.match(bio_seq):\n",
    "        return 'E'\n",
    "    raise ValueError\n",
    "    \n",
    "\n",
    "def get_fixed_for_invalid_biose(parts):\n",
    "    bio = 'O'\n",
    "    if 'S' in parts:\n",
    "        bio = 'S'\n",
    "    elif 'B' in parts and 'E' in parts:\n",
    "        bio='S'\n",
    "    elif 'E' in parts:\n",
    "        bio = 'E'\n",
    "    elif 'B' in parts:\n",
    "        bio = 'B'\n",
    "    elif 'I' in parts:\n",
    "        bio = 'I'\n",
    "    return bio\n",
    "\n",
    "valid_bio_re = re.compile('^O*BI*$|^O*BI*EO*$|^I+$|^I*EO*$|^O*SO*$')\n",
    "\n",
    "from functools import lru_cache\n",
    "\n",
    "\n",
    "def validate_biose_sequence(full_bio_seq):\n",
    "    #print(full_bio_seq)\n",
    "    bio_seq, type_seq = zip(*[('O', None) if b=='O' else b.split('-') for b in full_bio_seq])\n",
    "    bio_seq = ''.join(bio_seq)\n",
    "    valid_bio = valid_bio_re.match(bio_seq)\n",
    "    type_seq = list(filter(lambda x: x is not None, type_seq))\n",
    "    type_seq_set = set(type_seq)\n",
    "\n",
    "    if valid_bio:\n",
    "        fixed_bio = get_fixed_for_valid_biose(bio_seq)\n",
    "        if fixed_bio!='O':\n",
    "            fixed_bio += '-' + type_seq[0]\n",
    "            \n",
    "    else:\n",
    "        #take the first BIOSE tag which is not O:\n",
    "        #fixed_bio = list(filter(lambda x: x!='O', full_bio_seq))[0]\n",
    "        #rough BIOSE and first category:\n",
    "        fixed_bio = get_fixed_for_invalid_biose(bio_seq)\n",
    "        if fixed_bio!='O':\n",
    "            fixed_bio += '-' + type_seq[0]\n",
    "        \n",
    "    return valid_bio is not None, len(type_seq_set)<=1, fixed_bio\n",
    "\n",
    "\n",
    "@lru_cache(1000)\n",
    "def get_fixed_bio_sequence(full_bio_seq):\n",
    "    return validate_biose_sequence(full_bio_seq)[2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/nlp/danb/anaconda3/envs/tf/lib/python3.7/site-packages/ipykernel_launcher.py:6: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  \n"
     ]
    }
   ],
   "source": [
    "dropped = [5438, 5444, 5445, 5446, 5448, 5449, 5450, 5451, 5453, 5459]\n",
    "spdf = bclm.read_dataframe('spmrl')\n",
    "spdf = spdf[(~spdf.sent_id.isin(dropped))]\n",
    "dev_gold = spdf[spdf.set=='dev']\n",
    "test_gold = spdf[spdf.set=='test']\n",
    "test_gold['sent_id'] = test_gold.sent_id.rank(method='dense').astype(int)\n",
    "dev_yap = bclm.read_yap_output(treebank_set='dev')\n",
    "test_yap = bclm.read_yap_output(treebank_set='test')\n",
    "dev_gold_sents = bclm.get_sentences_list(dev_gold, fields=['token_id', 'token_str'])\n",
    "test_gold_sents = bclm.get_sentences_list(test_gold, fields=['token_id', 'token_str'])\n",
    "dev_yap_sents = bclm.get_sentences_list(dev_yap, fields=['token_id', 'token_str'])\n",
    "test_yap_sents = bclm.get_sentences_list(test_yap, fields=['token_id', 'token_str'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/nlp/danb/anaconda3/envs/tf/lib/python3.7/site-packages/pandas/core/frame.py:3509: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self[k1] = value[k2]\n"
     ]
    }
   ],
   "source": [
    "dev_gold_tok = (bclm.get_token_df(dev_gold, biose=['biose_layer0'])\n",
    "                .rename(columns={'biose_layer0': 'fixed_bio'}))\n",
    "test_gold_tok = (bclm.get_token_df(test_gold, biose=['biose_layer0'])\n",
    "                .rename(columns={'biose_layer0': 'fixed_bio'}))\n",
    "test_gold_tok['sent_id'] = test_gold_tok.sent_id.rank(method='dense').astype(int)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sent_id</th>\n",
       "      <th>token_id</th>\n",
       "      <th>token_str</th>\n",
       "      <th>bio</th>\n",
       "      <th>fixed_bio</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>הכל</td>\n",
       "      <td>O^O</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>נושאים</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>עמם</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>את</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>כישלונות</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>הקליטה</td>\n",
       "      <td>O^O</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>בעליות</td>\n",
       "      <td>O^O^O</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1</td>\n",
       "      <td>8</td>\n",
       "      <td>הקודמות</td>\n",
       "      <td>O^O</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1</td>\n",
       "      <td>9</td>\n",
       "      <td>,</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>1</td>\n",
       "      <td>10</td>\n",
       "      <td>את</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>1</td>\n",
       "      <td>11</td>\n",
       "      <td>מטען</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "      <td>הדימויים</td>\n",
       "      <td>O^O</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>השליליים</td>\n",
       "      <td>O^O</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>1</td>\n",
       "      <td>14</td>\n",
       "      <td>שנתקלו</td>\n",
       "      <td>O^O</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>1</td>\n",
       "      <td>15</td>\n",
       "      <td>בהם</td>\n",
       "      <td>O^O^O</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>1</td>\n",
       "      <td>16</td>\n",
       "      <td>,</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>1</td>\n",
       "      <td>17</td>\n",
       "      <td>החל</td>\n",
       "      <td>O^O</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>1</td>\n",
       "      <td>18</td>\n",
       "      <td>ממיליונרים</td>\n",
       "      <td>O^O</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>1</td>\n",
       "      <td>19</td>\n",
       "      <td>,</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>1</td>\n",
       "      <td>20</td>\n",
       "      <td>וכלה</td>\n",
       "      <td>O^O</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    sent_id  token_id   token_str    bio fixed_bio\n",
       "0         1         1         הכל    O^O         O\n",
       "1         1         2      נושאים      O         O\n",
       "2         1         3         עמם      O         O\n",
       "3         1         4          את      O         O\n",
       "4         1         5    כישלונות      O         O\n",
       "5         1         6      הקליטה    O^O         O\n",
       "6         1         7      בעליות  O^O^O         O\n",
       "7         1         8     הקודמות    O^O         O\n",
       "8         1         9           ,      O         O\n",
       "9         1        10          את      O         O\n",
       "10        1        11        מטען      O         O\n",
       "11        1        12    הדימויים    O^O         O\n",
       "12        1        13    השליליים    O^O         O\n",
       "13        1        14      שנתקלו    O^O         O\n",
       "14        1        15         בהם  O^O^O         O\n",
       "15        1        16           ,      O         O\n",
       "16        1        17         החל    O^O         O\n",
       "17        1        18  ממיליונרים    O^O         O\n",
       "18        1        19           ,      O         O\n",
       "19        1        20        וכלה    O^O         O"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path = 'output/predict/morph_12345/morph_yap_test.bmes'\n",
    "def get_fixed_tok(path, orig_sents=dev_yap_sents):\n",
    "    x = nem.read_file_sents(path, fix_multi_tag=False)\n",
    "    new_sents = []\n",
    "    for (i, ner_sent), (sent_id, yap_sent) in zip(x.iteritems(), orig_sents.iteritems()):\n",
    "        for (form, bio), (token_id, token_str) in zip(ner_sent, yap_sent):\n",
    "            new_sents.append((sent_id, token_id, token_str, form, bio))\n",
    "    new_sents = pd.DataFrame(new_sents, columns=['sent_id', 'token_id', 'token_str', 'form', 'bio'])\n",
    "    new_toks = bclm.get_token_df(new_sents, fields=['bio'])\n",
    "    new_toks['fixed_bio'] = new_toks.bio.apply(lambda x: get_fixed_bio_sequence(tuple(x.split('^'))))\n",
    "    return new_toks\n",
    "\n",
    "new_toks = get_fixed_tok(path, orig_sents=test_yap_sents)\n",
    "new_toks.head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sents_from_df(df, sent_id_col='sent_id', \n",
    "                  group_cols=['token_str'], \n",
    "                  val_cols=['fixed_bio']):\n",
    "    sents = bclm.get_sentences_list(df, fields=group_cols+val_cols)\n",
    "    return sents\n",
    "\n",
    "def evaluate_dataframes(gold_df, pred_df, fix_multi_tag_pred=True, truncate=None, ignore_cat=False, str_join_char=' '):\n",
    "    gold_sents = sents_from_df(gold_df)\n",
    "    pred_sents = sents_from_df(pred_df)\n",
    "    gold_mentions = nem.sents_to_mentions(gold_sents, truncate=truncate, ignore_cat=ignore_cat, str_join_char=str_join_char)\n",
    "    pred_mentions = nem.sents_to_mentions(pred_sents, truncate=truncate, ignore_cat=ignore_cat, str_join_char=str_join_char)\n",
    "    return nem.evaluate_mentions(gold_mentions, pred_mentions, verbose=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "sent_id\n",
       "1      [[הכל, O], [נושאים, O], [עמם, O], [את, O], [כי...\n",
       "2      [[אומר, O], [מזכיר, O], [התק\"ם, S-ORG], [,, O]...\n",
       "3      [[לא, O], [ייתכן, O], [שעולה, O], [יבוא, O], [...\n",
       "4      [[לא, O], [ייתכן, O], [שהוא, O], [יירד, O], [מ...\n",
       "5      [[לכן, O], [קבענו, O], [עיקרון, O], [שצריכה, O...\n",
       "                             ...                        \n",
       "702    [[האנטיפסטו, O], [של, O], [מאכלי, O], [ים, O],...\n",
       "703    [[אף, O], [שהמנה, O], [היתה, O], [טעימה, O], [...\n",
       "704    [[כמו, O], [שיעולים, O], [,, O], [שלשול, O], [...\n",
       "705    [[הוריהם, O], [האמינו, O], [ברפואה, O], [המודר...\n",
       "706    [[העצה, O], [היתה, O], [טובה, O], [אך, O], [לא...\n",
       "Length: 706, dtype: object"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sents_from_df(new_toks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.7568421052631579, 0.7714592274678111, 0.7640807651434643)"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluate_dataframes(test_gold_tok, new_toks, str_join_char='')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Hybrid\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "out_folder = '../NER/data/tokens_for_ncrf'\n",
    "dev_out = os.path.join(out_folder, 'dev_tokens.txt')\n",
    "test_out = os.path.join(out_folder, 'test_tokens.txt')\n",
    "token_paths = {'dev': dev_out, 'test': test_out}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "@lru_cache(512)\n",
    "def get_prun_yo(ds, dep_path, map_path):\n",
    "\n",
    "    \n",
    "    prun_yo = bclm.read_yap_output(treebank_set=None,\n",
    "                               tokens_path=token_paths[ds],\n",
    "                               dep_path=dep_path,\n",
    "                               map_path=map_path,\n",
    "                                )\n",
    "    return prun_yo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.7664609053497943, 0.7993562231759657, 0.7825630252100841)"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dep_path='output/predict/multi_12345/test_pruned.conll'\n",
    "map_path='output/predict/multi_12345/test_pruned.map'\n",
    "\n",
    "prun_yo = get_prun_yo('test', dep_path, map_path)\n",
    "prun_sents = bclm.get_sentences_list(prun_yo, fields=['token_id', 'token_str'])\n",
    "path =  'output/predict/morph_12345/morph_pruned_test.bmes'\n",
    "new_toks = get_fixed_tok(path, orig_sents=prun_sents)\n",
    "evaluate_dataframes(test_gold_tok, new_toks, str_join_char='')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.7438271604938271, 0.7757510729613734, 0.7594537815126049)"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nem.evaluate_files('../NER/data/for_ncrf/morph_gold_test.bmes',\n",
    "                  path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Run on all pruned"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "@lru_cache(512)\n",
    "def get_sent_list(ds, dp, mp):\n",
    "    prun_yo = get_prun_yo(ds, dp, mp)\n",
    "    return bclm.get_sentences_list(prun_yo, fields=['token_id', 'token_str'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "align_tok_res = []\n",
    "for folder in os.scandir('output/predict'):\n",
    "    if 'morph' in folder.name and not '.ipynb' in folder.name:\n",
    "        ## test \n",
    "        variant, seed = folder.name.split('_')\n",
    "        file = os.path.join(folder.path,'morph_pruned_test.bmes')\n",
    "        multi_folder = folder.path.replace('morph_', 'multi_')\n",
    "        dep_path = os.path.join(multi_folder, 'test_pruned.conll')\n",
    "        map_path = os.path.join(multi_folder, 'test_pruned.map')\n",
    "        out_path = os.path.join(folder.path, 'morph_pruned_test_align_tokens.bmes')\n",
    "        \n",
    "        prun_sents = get_sent_list('test',dep_path , map_path)\n",
    "        new_toks = get_fixed_tok(file, orig_sents=prun_sents)\n",
    "        \n",
    "        if not os.path.exists(out_path):\n",
    "            new_sents = bclm.get_sentences_list(new_toks, fields=['token_str', 'fixed_bio'])\n",
    "            with open(out_path, 'w') as of:\n",
    "                for i, sent in new_sents.iteritems():\n",
    "                    for tok, bio in sent:\n",
    "                        of.write(tok+' '+bio+'\\n')\n",
    "                    of.write('\\n')\n",
    "                    \n",
    "        p, r, f = evaluate_dataframes(test_gold_tok, new_toks, str_join_char='')\n",
    "\n",
    "        align_tok_res.append(('test', 'token', 'morph', 'hybrid', 'tokens', seed, p, r, f))\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "eval_unit  variant  prediction  align \n",
       "token      morph    hybrid      tokens    0.77026\n",
       "Name: f, dtype: float64"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "at_df = pd.DataFrame(align_tok_res, columns=['set', 'eval_unit', 'variant', 'prediction', 'align', 'seed', 'p', 'r', 'f'])\n",
    "\n",
    "at_df.groupby(['eval_unit','variant', 'prediction', 'align']).f.mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Run all gold and YAP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "align_tok_res_yg = []\n",
    "for folder in os.scandir('output/predict'):\n",
    "    if 'morph' in folder.name and not '.ipynb' in folder.name:\n",
    "        ## test \n",
    "        ## - gold\n",
    "        variant, seed = folder.name.split('_')\n",
    "        file = os.path.join(folder.path,'morph_gold_test.bmes')\n",
    "        out_path = os.path.join(folder.path, 'morph_gold_test_align_tokens.bmes')\n",
    "        \n",
    "        new_toks = get_fixed_tok(file, orig_sents=test_gold_sents)\n",
    "        \n",
    "        if not os.path.exists(out_path):\n",
    "            new_sents = bclm.get_sentences_list(new_toks, fields=['token_str', 'fixed_bio'])\n",
    "            with open(out_path, 'w') as of:\n",
    "                for i, sent in new_sents.iteritems():\n",
    "                    for tok, bio in sent:\n",
    "                        of.write(tok+' '+bio+'\\n')\n",
    "                    of.write('\\n')\n",
    "                    \n",
    "        p, r, f = evaluate_dataframes(test_gold_tok, new_toks, str_join_char='')\n",
    "\n",
    "        align_tok_res_yg.append(('test', 'token', 'morph', 'gold', 'tokens', seed, p, r, f))\n",
    "        \n",
    "        ## - yap\n",
    "        variant, seed = folder.name.split('_')\n",
    "        file = os.path.join(folder.path,'morph_yap_test.bmes')\n",
    "        out_path = os.path.join(folder.path, 'morph_yap_test_align_tokens.bmes')\n",
    "        \n",
    "        new_toks = get_fixed_tok(file, orig_sents=test_yap_sents)\n",
    "        \n",
    "        if not os.path.exists(out_path):\n",
    "            new_sents = bclm.get_sentences_list(new_toks, fields=['token_str', 'fixed_bio'])\n",
    "            with open(out_path, 'w') as of:\n",
    "                for i, sent in new_sents.iteritems():\n",
    "                    for tok, bio in sent:\n",
    "                        of.write(tok+' '+bio+'\\n')\n",
    "                    of.write('\\n')\n",
    "        p, r, f = evaluate_dataframes(test_gold_tok, new_toks, str_join_char='')\n",
    "\n",
    "        align_tok_res_yg.append(('test', 'token', 'morph', 'yap', 'tokens', seed, p, r, f))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>p</th>\n",
       "      <th>r</th>\n",
       "      <th>f</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>eval_unit</th>\n",
       "      <th>variant</th>\n",
       "      <th>prediction</th>\n",
       "      <th>align</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">token</th>\n",
       "      <th rowspan=\"3\" valign=\"top\">morph</th>\n",
       "      <th>gold</th>\n",
       "      <th>tokens</th>\n",
       "      <td>0.764682</td>\n",
       "      <td>0.800536</td>\n",
       "      <td>0.782153</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hybrid</th>\n",
       "      <th>tokens</th>\n",
       "      <td>0.756655</td>\n",
       "      <td>0.784442</td>\n",
       "      <td>0.770260</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>yap</th>\n",
       "      <th>tokens</th>\n",
       "      <td>0.737297</td>\n",
       "      <td>0.750107</td>\n",
       "      <td>0.743593</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            p         r         f\n",
       "eval_unit variant prediction align                               \n",
       "token     morph   gold       tokens  0.764682  0.800536  0.782153\n",
       "                  hybrid     tokens  0.756655  0.784442  0.770260\n",
       "                  yap        tokens  0.737297  0.750107  0.743593"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "at_df = pd.DataFrame(align_tok_res+align_tok_res_yg, columns=['set', 'eval_unit', 'variant', 'prediction', 'align', 'seed', 'p', 'r', 'f'])\n",
    "\n",
    "at_df.groupby(['eval_unit','variant', 'prediction', 'align'])[['p', 'r', 'f']].mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Morpheme Level Eval"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Token Multi"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Hybrid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "multi_54360\n",
      "multi_44184\n",
      "multi_20423\n",
      "multi_80520\n",
      "multi_27916\n",
      "multi_63795\n",
      "multi_30528\n",
      "multi_78160\n",
      "multi_12345\n",
      "multi_95148\n"
     ]
    }
   ],
   "source": [
    "align_morph_res_hyb = []\n",
    "for folder in os.scandir('output/predict'):\n",
    "    if os.path.isdir(folder) and 'multi' in folder.name and not '.ipynb_checkpoints' in folder.name:\n",
    "        print (folder.name)\n",
    "\n",
    "        pruned_ner_path=os.path.join(folder.path, 'morph_pruned_test.bmes')\n",
    "        \n",
    "        p, r, f = nem.evaluate_files(decode_sets['multitok']['test'], pruned_ner_path, str_join_char='')\n",
    "        align_morph_res_hyb.append(('test', 'morph', 'multi', 'tokens', 'hybrid', seed, p, r, f))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>p</th>\n",
       "      <th>r</th>\n",
       "      <th>f</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>eval_unit</th>\n",
       "      <th>variant</th>\n",
       "      <th>prediction</th>\n",
       "      <th>align</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>morph</th>\n",
       "      <th>multi</th>\n",
       "      <th>tokens</th>\n",
       "      <th>hybrid</th>\n",
       "      <td>0.745095</td>\n",
       "      <td>0.765558</td>\n",
       "      <td>0.755154</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">token</th>\n",
       "      <th rowspan=\"3\" valign=\"top\">morph</th>\n",
       "      <th>gold</th>\n",
       "      <th>tokens</th>\n",
       "      <td>0.764682</td>\n",
       "      <td>0.800536</td>\n",
       "      <td>0.782153</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hybrid</th>\n",
       "      <th>tokens</th>\n",
       "      <td>0.756655</td>\n",
       "      <td>0.784442</td>\n",
       "      <td>0.770260</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>yap</th>\n",
       "      <th>tokens</th>\n",
       "      <td>0.737297</td>\n",
       "      <td>0.750107</td>\n",
       "      <td>0.743593</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            p         r         f\n",
       "eval_unit variant prediction align                               \n",
       "morph     multi   tokens     hybrid  0.745095  0.765558  0.755154\n",
       "token     morph   gold       tokens  0.764682  0.800536  0.782153\n",
       "                  hybrid     tokens  0.756655  0.784442  0.770260\n",
       "                  yap        tokens  0.737297  0.750107  0.743593"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "at_df = pd.DataFrame(align_tok_res+align_tok_res_yg+align_morph_res_hyb, columns=['set', 'eval_unit', 'variant', 'prediction', 'align', 'seed', 'p', 'r', 'f'])\n",
    "\n",
    "at_df.groupby(['eval_unit','variant', 'prediction', 'align'])[['p', 'r', 'f']].mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### YAP + GOLD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "def align_multitok_yg(ner_pred_path, prun_sents, output_path):\n",
    "    x = nem.read_file_sents(ner_pred_path, fix_multi_tag=False)\n",
    "\n",
    "    new_sents = soft_merge_bio_labels(x, prun_sents, verbose=False)\n",
    "\n",
    "    with open(output_path, 'w') as of:\n",
    "        for sent in new_sents:\n",
    "            for form, bio in sent:\n",
    "                of.write(form+' '+bio+'\\n')\n",
    "            of.write('\\n')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "gold_morph = {'dev': dev_gold, 'test': test_gold}\n",
    "def get_sents_for_mult(treebank_set, gold=False, pred_set=None, \n",
    "                       dep_path=None, map_path=None):\n",
    "    if treebank_set is None:\n",
    "        prun_yo = get_prun_yo(pred_set, dep_path, map_path)\n",
    "    else:\n",
    "        if not gold:\n",
    "            prun_yo = bclm.read_yap_output(treebank_set=treebank_set)\n",
    "        else:\n",
    "            prun_yo = gold_morph[treebank_set]\n",
    "    prun_yo = bclm.get_token_df(prun_yo, fields=['form'])\n",
    "    prun_sents = bclm.get_sentences_list(prun_yo, fields=['token_id', 'token_str', 'form'])\n",
    "    return prun_sents\n",
    "\n",
    "dev_yap_sents_m = get_sents_for_mult('dev')\n",
    "test_yap_sents_m = get_sents_for_mult('test')\n",
    "dev_gold_sents_m = get_sents_for_mult('dev', gold=True)\n",
    "test_gold_sents_m = get_sents_for_mult('test', gold=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "multi_54360\n",
      "multi_44184\n",
      "multi_20423\n",
      "multi_80520\n",
      "multi_27916\n",
      "multi_63795\n",
      "multi_30528\n",
      "multi_78160\n",
      "multi_12345\n",
      "multi_95148\n"
     ]
    }
   ],
   "source": [
    "align_morph_res_yap = []\n",
    "for folder in os.scandir('output/predict'):\n",
    "    if os.path.isdir(folder) and 'multi' in folder.name and not '.ipynb_checkpoints' in folder.name:\n",
    "        print (folder.name)\n",
    "\n",
    "        yap_ner_path=os.path.join(folder.path, 'morph_yap_test.bmes')\n",
    "        \n",
    "        align_multitok_yg(os.path.join(folder.path, 'token_gold_test_dummy_o.bmes'), \n",
    "                           test_yap_sents_m,\n",
    "                           yap_ner_path\n",
    "                          )\n",
    "        p, r, f = nem.evaluate_files(decode_sets['multitok']['test'], yap_ner_path, str_join_char='')\n",
    "        align_morph_res_yap.append(('test', 'morph', 'multi', 'tokens', 'yap', seed, p, r, f))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "multi_54360\n",
      "multi_44184\n",
      "multi_20423\n",
      "multi_80520\n",
      "multi_27916\n",
      "multi_63795\n",
      "multi_30528\n",
      "multi_78160\n",
      "multi_12345\n",
      "multi_95148\n"
     ]
    }
   ],
   "source": [
    "align_morph_res_gold = []\n",
    "for folder in os.scandir('output/predict'):\n",
    "    if os.path.isdir(folder) and 'multi' in folder.name and not '.ipynb_checkpoints' in folder.name:\n",
    "        print (folder.name)\n",
    "\n",
    "        gold_ner_path=os.path.join(folder.path, 'morph_gold_test.bmes')\n",
    "        \n",
    "        align_multitok_yg(os.path.join(folder.path, 'token_gold_test_dummy_o.bmes'), \n",
    "                           test_gold_sents_m,\n",
    "                           gold_ner_path\n",
    "                          )\n",
    "        p, r, f = nem.evaluate_files(decode_sets['multitok']['test'], gold_ner_path, str_join_char='')\n",
    "        align_morph_res_gold.append(('test', 'morph', 'multi', 'tokens', 'gold', seed, p, r, f))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>p</th>\n",
       "      <th>r</th>\n",
       "      <th>f</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>eval_unit</th>\n",
       "      <th>variant</th>\n",
       "      <th>prediction</th>\n",
       "      <th>align</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">morph</th>\n",
       "      <th rowspan=\"3\" valign=\"top\">multi</th>\n",
       "      <th rowspan=\"3\" valign=\"top\">tokens</th>\n",
       "      <th>gold</th>\n",
       "      <td>0.770651</td>\n",
       "      <td>0.788841</td>\n",
       "      <td>0.779604</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hybrid</th>\n",
       "      <td>0.745095</td>\n",
       "      <td>0.765558</td>\n",
       "      <td>0.755154</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>yap</th>\n",
       "      <td>0.704023</td>\n",
       "      <td>0.701824</td>\n",
       "      <td>0.702888</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">token</th>\n",
       "      <th rowspan=\"3\" valign=\"top\">morph</th>\n",
       "      <th>gold</th>\n",
       "      <th>tokens</th>\n",
       "      <td>0.764682</td>\n",
       "      <td>0.800536</td>\n",
       "      <td>0.782153</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hybrid</th>\n",
       "      <th>tokens</th>\n",
       "      <td>0.756655</td>\n",
       "      <td>0.784442</td>\n",
       "      <td>0.770260</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>yap</th>\n",
       "      <th>tokens</th>\n",
       "      <td>0.737297</td>\n",
       "      <td>0.750107</td>\n",
       "      <td>0.743593</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            p         r         f\n",
       "eval_unit variant prediction align                               \n",
       "morph     multi   tokens     gold    0.770651  0.788841  0.779604\n",
       "                             hybrid  0.745095  0.765558  0.755154\n",
       "                             yap     0.704023  0.701824  0.702888\n",
       "token     morph   gold       tokens  0.764682  0.800536  0.782153\n",
       "                  hybrid     tokens  0.756655  0.784442  0.770260\n",
       "                  yap        tokens  0.737297  0.750107  0.743593"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "at_df = pd.DataFrame(align_tok_res+align_tok_res_yg+align_morph_res_hyb+align_morph_res_yap+align_morph_res_gold, columns=['set', 'eval_unit', 'variant', 'prediction', 'align', 'seed', 'p', 'r', 'f'])\n",
    "\n",
    "at_df.groupby(['eval_unit','variant', 'prediction', 'align'])[['p', 'r', 'f']].mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>eval_unit</th>\n",
       "      <th>variant</th>\n",
       "      <th>prediction</th>\n",
       "      <th>align</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">morph</th>\n",
       "      <th rowspan=\"3\" valign=\"top\">multi</th>\n",
       "      <th rowspan=\"3\" valign=\"top\">tokens</th>\n",
       "      <th>gold</th>\n",
       "      <td>$77.96 ± 0.3$</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hybrid</th>\n",
       "      <td>$75.52 ± 0.3$</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>yap</th>\n",
       "      <td>$70.29 ± 0.3$</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">token</th>\n",
       "      <th rowspan=\"3\" valign=\"top\">morph</th>\n",
       "      <th>gold</th>\n",
       "      <th>tokens</th>\n",
       "      <td>$78.22 ± 0.9$</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hybrid</th>\n",
       "      <th>tokens</th>\n",
       "      <td>$77.03 ± 0.8$</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>yap</th>\n",
       "      <th>tokens</th>\n",
       "      <td>$74.36 ± 0.8$</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              mean\n",
       "eval_unit variant prediction align                \n",
       "morph     multi   tokens     gold    $77.96 ± 0.3$\n",
       "                             hybrid  $75.52 ± 0.3$\n",
       "                             yap     $70.29 ± 0.3$\n",
       "token     morph   gold       tokens  $78.22 ± 0.9$\n",
       "                  hybrid     tokens  $77.03 ± 0.8$\n",
       "                  yap        tokens  $74.36 ± 0.8$"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "at_df = pd.DataFrame(align_tok_res+align_tok_res_yg+align_morph_res_hyb+align_morph_res_yap+align_morph_res_gold, columns=['set', 'eval_unit', 'variant', 'prediction', 'align', 'seed', 'p', 'r', 'f'])\n",
    "\n",
    "(at_df.groupby(['eval_unit','variant', 'prediction', 'align']).f.agg(['mean', 'std']).mul(100).round(2)\n",
    "         .assign(mean = lambda x: '$'+x['mean'].apply('{:,.2f}'.format).astype(str)+' ± '+ (1.96*(x['std']/np.sqrt(10))).round(1).astype(str)+'$')[['mean']])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>eval_unit</th>\n",
       "      <th>variant</th>\n",
       "      <th>prediction</th>\n",
       "      <th>align</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">morph</th>\n",
       "      <th rowspan=\"3\" valign=\"top\">morph</th>\n",
       "      <th>gold</th>\n",
       "      <th>-</th>\n",
       "      <td>$77.84 ± 0.9$</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hybrid</th>\n",
       "      <th>-</th>\n",
       "      <td>$74.77 ± 0.9$</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>yap</th>\n",
       "      <th>-</th>\n",
       "      <td>$66.75 ± 0.8$</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">token</th>\n",
       "      <th>multi</th>\n",
       "      <th>tokens</th>\n",
       "      <th>-</th>\n",
       "      <td>$78.62 ± 0.4$</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>single</th>\n",
       "      <th>tokens</th>\n",
       "      <th>-</th>\n",
       "      <td>$79.78 ± 0.7$</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                             mean\n",
       "eval_unit variant prediction align               \n",
       "morph     morph   gold       -      $77.84 ± 0.9$\n",
       "                  hybrid     -      $74.77 ± 0.9$\n",
       "                  yap        -      $66.75 ± 0.8$\n",
       "token     multi   tokens     -      $78.62 ± 0.4$\n",
       "          single  tokens     -      $79.78 ± 0.7$"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(ne_df.groupby(['eval_unit','variant', 'prediction', 'align']).f.agg(['mean', 'std']).mul(100).round(2)\n",
    "         .assign(mean = lambda x: '$'+x['mean'].apply('{:,.2f}'.format).astype(str)+' ± '+ (1.96*(x['std']/np.sqrt(10))).round(1).astype(str)+'$')[['mean']])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Token Single"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "tf",
   "language": "python",
   "name": "tf"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}