01_evaluate_ner.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Re-evaluation\n",
    "Perform two evaluations:\n",
    "1. Strict morpheme evaluation\n",
    "1. Token evaluation (morpheme labels are extended to the token level heuristically)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-13T07:20:24.034028Z",
     "start_time": "2019-03-13T07:20:22.687626Z"
    }
   },
   "outputs": [],
   "source": [
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-13T07:20:24.382406Z",
     "start_time": "2019-03-13T07:20:24.037019Z"
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-03-13T07:20:27.996727Z",
     "start_time": "2019-03-13T07:20:24.385088Z"
    }
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "sns.set_context('paper')\n",
    "sns.set_style('white')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create BIOSE files\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "500\n"
     ]
    }
   ],
   "source": [
    "import json\n",
    "\n",
    "def jsonl_to_biose(in_path, out_path, bioul_to_biose=True):\n",
    "    sents = 0\n",
    "    with open(out_path, 'w', encoding='utf8') as of:\n",
    "        for line in open(in_path, 'r'):\n",
    "            sent = json.loads(line)\n",
    "            for word, tag in zip(sent['words'], sent['tags']):\n",
    "                if bioul_to_biose:\n",
    "                    tag = tag.replace('L-', 'E-').replace('U-', 'S-')\n",
    "                of.write(word+' '+tag+'\\n')\n",
    "            of.write('\\n')\n",
    "            sents+=1\n",
    "    print (sents)\n",
    "jsonl_to_biose('output/predict/multi_12345/token_gold_dev_dummy_o.json', 'output/predict/multi_12345/token_gold_dev.bmes')\n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "500\n",
      "706\n",
      "500\n",
      "706\n",
      "500\n",
      "706\n",
      "500\n",
      "706\n",
      "500\n",
      "706\n",
      "500\n",
      "706\n",
      "500\n",
      "706\n",
      "500\n",
      "706\n",
      "500\n",
      "706\n",
      "500\n",
      "706\n",
      "500\n",
      "706\n",
      "500\n",
      "706\n",
      "500\n",
      "706\n",
      "500\n",
      "706\n",
      "500\n",
      "706\n",
      "500\n",
      "706\n",
      "500\n",
      "706\n",
      "500\n",
      "706\n",
      "500\n",
      "706\n",
      "500\n",
      "706\n",
      "500\n",
      "706\n",
      "500\n",
      "706\n",
      "500\n",
      "706\n",
      "500\n",
      "706\n",
      "500\n",
      "706\n",
      "500\n",
      "706\n",
      "500\n",
      "706\n",
      "500\n",
      "706\n",
      "500\n",
      "706\n",
      "500\n",
      "706\n",
      "500\n",
      "706\n",
      "500\n",
      "706\n",
      "500\n",
      "706\n",
      "500\n",
      "706\n",
      "500\n",
      "706\n",
      "500\n",
      "706\n",
      "500\n",
      "706\n",
      "500\n",
      "706\n",
      "500\n",
      "706\n",
      "500\n",
      "706\n"
     ]
    }
   ],
   "source": [
    "for folder in os.scandir('output/predict'):\n",
    "    if not '.ipynb' in folder.name:\n",
    "        for file in os.scandir(folder):\n",
    "            if '.json' in file.name and not '.ipynb' in file.name:\n",
    "                jsonl_to_biose(file.path, file.path.replace('.json', '.bmes'))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "500\n",
      "706\n",
      "500\n",
      "706\n",
      "500\n",
      "706\n",
      "500\n",
      "706\n",
      "500\n",
      "706\n",
      "500\n",
      "706\n",
      "500\n",
      "706\n",
      "500\n",
      "706\n",
      "500\n",
      "706\n",
      "500\n",
      "706\n",
      "500\n",
      "706\n",
      "500\n",
      "706\n",
      "500\n",
      "706\n",
      "500\n",
      "706\n",
      "500\n",
      "706\n",
      "500\n",
      "706\n",
      "500\n",
      "706\n",
      "500\n",
      "706\n",
      "500\n",
      "706\n",
      "500\n",
      "706\n",
      "500\n",
      "706\n",
      "500\n",
      "706\n",
      "500\n",
      "706\n",
      "500\n",
      "706\n",
      "500\n",
      "706\n",
      "500\n",
      "706\n",
      "500\n",
      "706\n",
      "500\n",
      "706\n",
      "500\n",
      "706\n",
      "500\n",
      "706\n",
      "500\n",
      "706\n",
      "500\n",
      "706\n",
      "500\n",
      "706\n",
      "500\n",
      "706\n",
      "500\n",
      "706\n",
      "500\n",
      "706\n",
      "500\n",
      "706\n",
      "500\n",
      "706\n",
      "500\n",
      "706\n",
      "500\n",
      "706\n"
     ]
    }
   ],
   "source": [
    "for folder in os.scandir('output/predict_char'):\n",
    "    if not '.ipynb' in folder.name:\n",
    "        for file in os.scandir(folder):\n",
    "            if '.json' in file.name and not '.ipynb' in file.name:\n",
    "                jsonl_to_biose(file.path, file.path.replace('.json', '.bmes'))\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tasks\n",
    "1. Strict morpheme evaluation\n",
    "   1. morph - as they are now\n",
    "   1. token - evaluate token mentions against gold morpheme mentions\n",
    "   1. multi - a) token vs gold morph b) yap/pruned vs gold morph\n",
    "1. Token evaluation \n",
    "   1. morph - extend heuristically and evaluate against gold token mentions\n",
    "   1. token - as it is now\n",
    "   1. multi - as it is now"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append('/home/nlp/danb/NER')\n",
    "\n",
    "import bclm\n",
    "import ne_evaluate_mentions as nem"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_biose_count(path, sent_id_shift=1):\n",
    "    sents = nem.read_file_sents(path, fix_multi_tag=False, sent_id_shift=sent_id_shift)\n",
    "    bc = []\n",
    "    for i, sent in sents.iteritems():\n",
    "        for j, (tok, bio) in enumerate(sent):\n",
    "            bc.append([i, j+1, tok, bio, len(bio.split('^'))])\n",
    "\n",
    "    bc = pd.DataFrame(bc, columns=['sent_id', 'token_id', 'token_str', \n",
    "                                   'biose', 'biose_count'])\n",
    "    return bc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "import networkx as nx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_valid_edges(lattices, bc,\n",
    "                    non_o_only=True, keep_all_if_no_valid=True):\n",
    "    valid_edges = []\n",
    "    for (i, df), (_, biose, biose_count) in zip(lattices.groupby(['sent_id', 'token_id']), \n",
    "                                                bc[['biose', 'biose_count']].itertuples()):\n",
    "        el = df[['ID1', 'ID2']].rename(columns={'ID1': 'source', 'ID2': 'target'})\n",
    "        #min_node = [n for n,v in G.nodes(data=True) if v['since'] == 'December 2008'][0]\n",
    "\n",
    "        g = nx.from_pandas_edgelist(el, create_using=nx.DiGraph)\n",
    "        min_node = el.source.min()\n",
    "        max_node = el.target.max()\n",
    "        #print(min_node,max_node)\n",
    "        #print(biose_count)\n",
    "        if non_o_only and not '-' in biose:\n",
    "            vp = list(nx.all_simple_paths(g, min_node, max_node))\n",
    "        else:\n",
    "            vp = [path for path in nx.all_simple_paths(g, min_node, max_node, cutoff=biose_count+1) if len(path)==biose_count+1]\n",
    "        if keep_all_if_no_valid and len(vp)==0:\n",
    "             vp = nx.all_simple_paths(g, min_node, max_node)\n",
    "        for path in vp:\n",
    "            for source, target in zip(path[:-1], path[1:]):\n",
    "                valid_edges.append((i[0], i[1], source, target))\n",
    "                \n",
    "    return valid_edges"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def to_lattices(df, path, cols = ['ID1', 'ID2', 'form', 'lemma', 'upostag', 'xpostag', 'feats', 'token_id']):\n",
    "    with open(path, 'w', encoding='utf8') as of:\n",
    "        for _, sent in df.groupby('sent_id'):\n",
    "            for _, row in sent[cols].iterrows():\n",
    "                of.write('\\t'.join(row.astype(str).tolist())+'\\n')\n",
    "            of.write('\\n')\n",
    "            \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "def prune_lattices(lattices_path, ner_pred_path, output_path, keep_all_if_no_valid=True):\n",
    "    lat = bclm.read_lattices(lattices_path)\n",
    "    bc = get_biose_count(ner_pred_path, sent_id_shift=1)\n",
    "    valid_edges = get_valid_edges(lat, bc, non_o_only=False, keep_all_if_no_valid=keep_all_if_no_valid)\n",
    "    cols = ['sent_id', 'token_id', 'ID1', 'ID2']\n",
    "    pruned_lat = lat[lat[cols].apply(lambda x: tuple(x), axis=1).isin(valid_edges)]\n",
    "    to_lattices(pruned_lat, output_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "prune_lattices(bclm.LATTICES_PATHS['dev'], \n",
    "               'output/predict/multi_12345/token_gold_dev_dummy_o.bmes',\n",
    "               'output/predict/multi_12345/pruned.lat')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "for folder in os.scandir('output/predict'):\n",
    "    if 'multi' in folder.name and not '.ipynb' in folder.name:\n",
    "        for file in os.scandir(folder):\n",
    "            if '.bmes' in file.name and not '.ipynb' in file.name:\n",
    "                if 'dev' in file.name:\n",
    "                    prune_lattices(bclm.LATTICES_PATHS['dev'], \n",
    "                       file.path,\n",
    "                       os.path.join(folder.path, 'dev_pruned.lat'))\n",
    "                elif 'test' in file.name:\n",
    "                    prune_lattices(bclm.LATTICES_PATHS['test'], \n",
    "                       file.path,\n",
    "                       os.path.join(folder.path, 'test_pruned.lat'))\n",
    "                    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Run YAP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "yap_path = '/home/nlp/danb/yapproj/src/yap/yap'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "!export GOPATH=/home/nlp/danb/yapproj"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/nlp/danb/yapproj/src/yap/yap - invoke yap as a standalone app or as an api server\n",
      "\n",
      "Commands:\n",
      "\n",
      "    api         start api server\n",
      "    dep         runs dependency training/parsing\n",
      "    hebma       run lexicon-based morphological analyzer on raw input\n",
      "    joint       runs joint morpho-syntactic training and parsing\n",
      "    ma          run data-driven morphological analyzer on raw input\n",
      "    md          runs standalone morphological disambiguation training and parsing\n",
      "\n",
      "Use \"/home/nlp/danb/yapproj/src/yap/yap help <command>\" for more information about a command.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "!{yap_path}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "for folder in os.scandir('output/predict'):\n",
    "    if 'multi' in folder.name and not '.ipynb' in folder.name:\n",
    "        for file in os.scandir(folder):\n",
    "            if '.lat' in file.name and not '.ipynb' in file.name:\n",
    "                base_out = '.'.join(file.name.split('.')[:-1])\n",
    "                seg_out, map_out, conll_out = [os.path.join(folder.path, base_out+suf)\n",
    "                                               for suf in ['.seg', '.map', '.conll']]\n",
    "                if not os.path.exists(seg_out):\n",
    "                    !{yap_path} joint -in {file.path} -os {seg_out} -om {map_out} -oc {conll_out}\n",
    "                    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluate Segmentation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "dropped = [5438, 5444, 5445, 5446, 5448, 5449, 5450, 5451, 5453, 5459]\n",
    "spdf = bclm.read_dataframe('spmrl')\n",
    "spdf = spdf[(~spdf.sent_id.isin(dropped))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/nlp/danb/anaconda3/envs/tf/lib/python3.7/site-packages/ipykernel_launcher.py:6: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  \n"
     ]
    }
   ],
   "source": [
    "dev_gold = bclm.read_dataframe('spmrl', subset='dev')\n",
    "test_gold = spdf[spdf.set=='test']\n",
    "test_sent_id_map = (test_gold.groupby('sent_id').size()\n",
    "                    .reset_index().drop(0, axis=1).reset_index()\n",
    "                    .assign(index=lambda x: x+1).set_index('sent_id')['index'])\n",
    "test_gold['sent_id'] = test_gold.sent_id.map(test_sent_id_map)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "cols = ['sent_id', 'token_id', 'form']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "11301 gold tokens/morphems, 11290 predicted, 10533 correct.\n",
      "Precision: 93.29\n",
      "Recall:    93.2\n",
      "F1:        93.25\n",
      "FP ex.: [(2, 11, 'דנה', 'BN'), (3, 4, 'ח\"כ', 'NNT'), (3, 28, 'הם', 'S-PRN'), (5, 13, 'ה', 'DEF'), (6, 24, 'ה', 'REL')]\n",
      "FN ex.: [(2, 11, 'דנה', 'VB'), (3, 4, 'ח\"כ', 'NN'), (3, 28, 'המ', 'S-PRN'), (6, 24, 'ה', 'DEF'), (8, 1, 'מרגלית', 'NNP')]\n",
      "11301 gold tokens/morphems, 11290 predicted, 11034 correct.\n",
      "Precision: 97.73\n",
      "Recall:    97.64\n",
      "F1:        97.68\n",
      "FP ex.: [(3, 28, 'הם'), (5, 13, 'ה'), (8, 7, 'ה'), (8, 7, 'יתרי'), (8, 9, 'ה')]\n",
      "FN ex.: [(3, 28, 'המ'), (8, 7, 'היתרי'), (17, 11, 'מחפיר'), (18, 8, 'ה'), (23, 8, 'העסקת')]\n",
      "11301 gold tokens/morphems, 11275 predicted, 10534 correct.\n",
      "Precision: 93.43\n",
      "Recall:    93.21\n",
      "F1:        93.32\n",
      "FP ex.: [(2, 11, 'דנה', 'BN'), (3, 4, 'ח\"כ', 'NNT'), (3, 28, 'הם', 'S-PRN'), (5, 13, 'ה', 'DEF'), (6, 24, 'ה', 'REL')]\n",
      "FN ex.: [(2, 11, 'דנה', 'VB'), (3, 4, 'ח\"כ', 'NN'), (3, 28, 'המ', 'S-PRN'), (6, 24, 'ה', 'DEF'), (8, 1, 'מרגלית', 'NNP')]\n",
      "11301 gold tokens/morphems, 11275 predicted, 11038 correct.\n",
      "Precision: 97.9\n",
      "Recall:    97.67\n",
      "F1:        97.79\n",
      "FP ex.: [(3, 28, 'הם'), (5, 13, 'ה'), (8, 9, 'ה'), (12, 6, 'ה'), (17, 11, 'חפיר')]\n",
      "FN ex.: [(3, 28, 'המ'), (17, 11, 'מחפיר'), (18, 8, 'ה'), (25, 3, 'מאז'), (26, 4, 'ב')]\n",
      "11301 gold tokens/morphems, 11287 predicted, 10536 correct.\n",
      "Precision: 93.35\n",
      "Recall:    93.23\n",
      "F1:        93.29\n",
      "FP ex.: [(2, 11, 'דנה', 'BN'), (3, 4, 'ח\"כ', 'NNT'), (3, 28, 'הם', 'S-PRN'), (5, 13, 'ה', 'DEF'), (6, 24, 'ה', 'REL')]\n",
      "FN ex.: [(2, 11, 'דנה', 'VB'), (3, 4, 'ח\"כ', 'NN'), (3, 28, 'המ', 'S-PRN'), (6, 24, 'ה', 'DEF'), (8, 1, 'מרגלית', 'NNP')]\n",
      "11301 gold tokens/morphems, 11287 predicted, 11036 correct.\n",
      "Precision: 97.78\n",
      "Recall:    97.66\n",
      "F1:        97.72\n",
      "FP ex.: [(3, 28, 'הם'), (5, 13, 'ה'), (8, 7, 'ה'), (8, 7, 'יתרי'), (8, 9, 'ה')]\n",
      "FN ex.: [(3, 28, 'המ'), (8, 7, 'היתרי'), (17, 11, 'מחפיר'), (18, 8, 'ה'), (23, 8, 'העסקת')]\n",
      "11301 gold tokens/morphems, 11276 predicted, 10533 correct.\n",
      "Precision: 93.41\n",
      "Recall:    93.2\n",
      "F1:        93.31\n",
      "FP ex.: [(2, 11, 'דנה', 'BN'), (3, 4, 'ח\"כ', 'NNT'), (3, 28, 'הם', 'S-PRN'), (5, 13, 'ה', 'DEF'), (6, 25, 'במקום', 'IN')]\n",
      "FN ex.: [(2, 11, 'דנה', 'VB'), (3, 4, 'ח\"כ', 'NN'), (3, 28, 'המ', 'S-PRN'), (6, 25, 'ב', 'PREPOSITION'), (6, 25, 'מקום', 'NNT')]\n",
      "11301 gold tokens/morphems, 11276 predicted, 11025 correct.\n",
      "Precision: 97.77\n",
      "Recall:    97.56\n",
      "F1:        97.67\n",
      "FP ex.: [(3, 28, 'הם'), (5, 13, 'ה'), (6, 25, 'במקום'), (8, 7, 'ה'), (8, 7, 'יתרי')]\n",
      "FN ex.: [(3, 28, 'המ'), (6, 25, 'ב'), (6, 25, 'מקום'), (8, 7, 'היתרי'), (17, 11, 'מחפיר')]\n",
      "11301 gold tokens/morphems, 11296 predicted, 10547 correct.\n",
      "Precision: 93.37\n",
      "Recall:    93.33\n",
      "F1:        93.35\n",
      "FP ex.: [(2, 11, 'דנה', 'BN'), (3, 4, 'ח\"כ', 'NNT'), (3, 28, 'הם', 'S-PRN'), (5, 13, 'ה', 'DEF'), (6, 25, 'במקום', 'IN')]\n",
      "FN ex.: [(2, 11, 'דנה', 'VB'), (3, 4, 'ח\"כ', 'NN'), (3, 28, 'המ', 'S-PRN'), (6, 25, 'ב', 'PREPOSITION'), (6, 25, 'מקום', 'NNT')]\n",
      "11301 gold tokens/morphems, 11296 predicted, 11054 correct.\n",
      "Precision: 97.86\n",
      "Recall:    97.81\n",
      "F1:        97.84\n",
      "FP ex.: [(3, 28, 'הם'), (5, 13, 'ה'), (6, 25, 'במקום'), (8, 9, 'ה'), (12, 6, 'ה')]\n",
      "FN ex.: [(3, 28, 'המ'), (6, 25, 'ב'), (6, 25, 'מקום'), (17, 11, 'מחפיר'), (18, 8, 'ה')]\n",
      "11301 gold tokens/morphems, 11278 predicted, 10518 correct.\n",
      "Precision: 93.26\n",
      "Recall:    93.07\n",
      "F1:        93.17\n",
      "FP ex.: [(2, 11, 'דנה', 'BN'), (3, 4, 'ח\"כ', 'NNT'), (3, 28, 'הם', 'S-PRN'), (5, 13, 'ה', 'DEF'), (6, 24, 'ה', 'REL')]\n",
      "FN ex.: [(2, 11, 'דנה', 'VB'), (3, 4, 'ח\"כ', 'NN'), (3, 28, 'המ', 'S-PRN'), (6, 24, 'ה', 'DEF'), (8, 1, 'מרגלית', 'NNP')]\n",
      "11301 gold tokens/morphems, 11278 predicted, 11021 correct.\n",
      "Precision: 97.72\n",
      "Recall:    97.52\n",
      "F1:        97.62\n",
      "FP ex.: [(3, 28, 'הם'), (5, 13, 'ה'), (8, 9, 'ה'), (12, 6, 'ה'), (17, 11, 'חפיר')]\n",
      "FN ex.: [(3, 28, 'המ'), (17, 11, 'מחפיר'), (18, 8, 'ה'), (24, 7, 'ה'), (24, 7, 'קרב')]\n",
      "11301 gold tokens/morphems, 11284 predicted, 10534 correct.\n",
      "Precision: 93.35\n",
      "Recall:    93.21\n",
      "F1:        93.28\n",
      "FP ex.: [(2, 11, 'דנה', 'BN'), (3, 4, 'ח\"כ', 'NNT'), (3, 28, 'הם', 'S-PRN'), (5, 13, 'ה', 'DEF'), (6, 24, 'ה', 'REL')]\n",
      "FN ex.: [(2, 11, 'דנה', 'VB'), (3, 4, 'ח\"כ', 'NN'), (3, 28, 'המ', 'S-PRN'), (6, 24, 'ה', 'DEF'), (8, 1, 'מרגלית', 'NNP')]\n",
      "11301 gold tokens/morphems, 11284 predicted, 11033 correct.\n",
      "Precision: 97.78\n",
      "Recall:    97.63\n",
      "F1:        97.7\n",
      "FP ex.: [(3, 28, 'הם'), (5, 13, 'ה'), (8, 9, 'ה'), (12, 6, 'ה'), (17, 11, 'חפיר')]\n",
      "FN ex.: [(3, 28, 'המ'), (17, 11, 'מחפיר'), (18, 8, 'ה'), (24, 24, 'ידי'), (24, 24, 'ל')]\n",
      "11301 gold tokens/morphems, 11306 predicted, 10537 correct.\n",
      "Precision: 93.2\n",
      "Recall:    93.24\n",
      "F1:        93.22\n",
      "FP ex.: [(2, 11, 'דנה', 'BN'), (3, 4, 'ח\"כ', 'NNT'), (3, 28, 'הם', 'S-PRN'), (5, 13, 'ה', 'DEF'), (6, 24, 'ה', 'REL')]\n",
      "FN ex.: [(2, 11, 'דנה', 'VB'), (3, 4, 'ח\"כ', 'NN'), (3, 28, 'המ', 'S-PRN'), (6, 24, 'ה', 'DEF'), (8, 1, 'מרגלית', 'NNP')]\n",
      "11301 gold tokens/morphems, 11306 predicted, 11034 correct.\n",
      "Precision: 97.59\n",
      "Recall:    97.64\n",
      "F1:        97.62\n",
      "FP ex.: [(3, 28, 'הם'), (5, 13, 'ה'), (8, 7, 'ה'), (8, 7, 'יתרי'), (8, 9, 'ה')]\n",
      "FN ex.: [(3, 28, 'המ'), (8, 7, 'היתרי'), (17, 11, 'מחפיר'), (18, 8, 'ה'), (23, 8, 'העסקת')]\n",
      "11301 gold tokens/morphems, 11288 predicted, 10540 correct.\n",
      "Precision: 93.37\n",
      "Recall:    93.27\n",
      "F1:        93.32\n",
      "FP ex.: [(2, 11, 'דנה', 'BN'), (3, 4, 'ח\"כ', 'NNT'), (3, 28, 'הם', 'S-PRN'), (5, 13, 'ה', 'DEF'), (6, 24, 'ה', 'REL')]\n",
      "FN ex.: [(2, 11, 'דנה', 'VB'), (3, 4, 'ח\"כ', 'NN'), (3, 28, 'המ', 'S-PRN'), (6, 24, 'ה', 'DEF'), (8, 1, 'מרגלית', 'NNP')]\n",
      "11301 gold tokens/morphems, 11288 predicted, 11034 correct.\n",
      "Precision: 97.75\n",
      "Recall:    97.64\n",
      "F1:        97.69\n",
      "FP ex.: [(3, 28, 'הם'), (5, 13, 'ה'), (8, 9, 'ה'), (12, 6, 'ה'), (17, 11, 'חפיר')]\n",
      "FN ex.: [(3, 28, 'המ'), (17, 11, 'מחפיר'), (18, 8, 'ה'), (24, 24, 'ידי'), (24, 24, 'ל')]\n",
      "11301 gold tokens/morphems, 11267 predicted, 10530 correct.\n",
      "Precision: 93.46\n",
      "Recall:    93.18\n",
      "F1:        93.32\n",
      "FP ex.: [(2, 11, 'דנה', 'BN'), (3, 4, 'ח\"כ', 'NNT'), (3, 28, 'הם', 'S-PRN'), (5, 13, 'ה', 'DEF'), (6, 25, 'במקום', 'IN')]\n",
      "FN ex.: [(2, 11, 'דנה', 'VB'), (3, 4, 'ח\"כ', 'NN'), (3, 28, 'המ', 'S-PRN'), (6, 25, 'ב', 'PREPOSITION'), (6, 25, 'מקום', 'NNT')]\n",
      "11301 gold tokens/morphems, 11267 predicted, 11025 correct.\n",
      "Precision: 97.85\n",
      "Recall:    97.56\n",
      "F1:        97.7\n",
      "FP ex.: [(3, 28, 'הם'), (5, 13, 'ה'), (6, 25, 'במקום'), (8, 9, 'ה'), (12, 6, 'ה')]\n",
      "FN ex.: [(3, 28, 'המ'), (6, 25, 'ב'), (6, 25, 'מקום'), (17, 11, 'מחפיר'), (18, 8, 'ה')]\n"
     ]
    }
   ],
   "source": [
    "res = []\n",
    "for folder in os.scandir('output/predict'):\n",
    "    if 'multi' in folder.name and not '.ipynb' in folder.name:\n",
    "        dev_lfo = bclm.read_yap_output(treebank_set=None, tokens_path=bclm.TREEBANK_TOKEN_PATHS['dev'], \n",
    "                                             dep_path=os.path.join(folder, 'dev_pruned.conll'),\n",
    "                                             map_path=os.path.join(folder, 'dev_pruned.map'))\n",
    "        p,r,f_sp = bclm.evaluate_dfs(dev_gold, dev_lfo)\n",
    "        \n",
    "        p,r,f_so = bclm.evaluate_dfs(dev_gold, dev_lfo, cols=cols)\n",
    "        \n",
    "        res.append((folder.name, f_sp, f_so))\n",
    "#         for file in os.scandir(folder):\n",
    "#             if '.bmes' in file.name and not '.ipynb' in file.name:\n",
    "#                 if 'dev' in file.name:\n",
    "#                     prune_lattices(bclm.LATTICES_PATHS['dev'], \n",
    "#                        file.path,\n",
    "#                        os.path.join(folder.path, 'dev_pruned.lat'))\n",
    "#                 elif 'test' in file.name:\n",
    "#                     prune_lattices(bclm.LATTICES_PATHS['test'], \n",
    "#                        file.path,\n",
    "#                        os.path.join(folder.path, 'test_pruned.lat'))\n",
    "                    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>f_seg_pos</th>\n",
       "      <th>f_seg_only</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>multi_54360</td>\n",
       "      <td>93.249524</td>\n",
       "      <td>97.684919</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>multi_44184</td>\n",
       "      <td>93.320340</td>\n",
       "      <td>97.785259</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>multi_20423</td>\n",
       "      <td>93.288472</td>\n",
       "      <td>97.715601</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>multi_80520</td>\n",
       "      <td>93.307348</td>\n",
       "      <td>97.665766</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>multi_27916</td>\n",
       "      <td>93.348675</td>\n",
       "      <td>97.835996</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>multi_63795</td>\n",
       "      <td>93.166216</td>\n",
       "      <td>97.621684</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>multi_30528</td>\n",
       "      <td>93.283153</td>\n",
       "      <td>97.702015</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>multi_78160</td>\n",
       "      <td>93.218914</td>\n",
       "      <td>97.615783</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>multi_12345</td>\n",
       "      <td>93.319757</td>\n",
       "      <td>97.693568</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>multi_95148</td>\n",
       "      <td>93.317972</td>\n",
       "      <td>97.704715</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         model  f_seg_pos  f_seg_only\n",
       "0  multi_54360  93.249524   97.684919\n",
       "1  multi_44184  93.320340   97.785259\n",
       "2  multi_20423  93.288472   97.715601\n",
       "3  multi_80520  93.307348   97.665766\n",
       "4  multi_27916  93.348675   97.835996\n",
       "5  multi_63795  93.166216   97.621684\n",
       "6  multi_30528  93.283153   97.702015\n",
       "7  multi_78160  93.218914   97.615783\n",
       "8  multi_12345  93.319757   97.693568\n",
       "9  multi_95148  93.317972   97.704715"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "seg_res_df = pd.DataFrame(res, columns=['model', 'f_seg_pos', 'f_seg_only'])\n",
    "seg_res_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "f_seg_pos     93.282037\n",
       "f_seg_only    97.702530\n",
       "dtype: float64"
      ]
     },
     "execution_count": 165,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "seg_res_df.mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Align Multitok"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "def soft_merge_bio_labels(multitok_sents, tokmorph_sents, verbose=False):\n",
    "    new_sents = []\n",
    "    for (i, mt_sent), (sent_id, mor_sent) in zip(multitok_sents.iteritems(), tokmorph_sents.iteritems()):\n",
    "        new_sent = []\n",
    "        for (form, bio), (token_id, token_str, forms) in zip(mt_sent, mor_sent):\n",
    "            forms = forms.split('^')\n",
    "            bio = bio.split('^')\n",
    "            if len(forms) == len(bio):\n",
    "                new_forms = (1, list(zip(forms,bio)))\n",
    "            elif len(forms)>len(bio):\n",
    "                dif = len(forms) - len(bio)\n",
    "                new_forms = (2, list(zip(forms[:dif],['O']*dif)) + list(zip(forms[::-1], bio[::-1]))[::-1])\n",
    "                if verbose:\n",
    "                    print(new_forms)\n",
    "            else:\n",
    "                new_forms = (3, list(zip(forms[::-1], bio[::-1]))[::-1])\n",
    "                if verbose:\n",
    "                    print(new_forms)\n",
    "            new_sent.extend(new_forms[1])\n",
    "        new_sents.append(new_sent)\n",
    "    return new_sents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "def align_multitok(ner_pred_path, tokens_path, conll_path, map_path, output_path):\n",
    "    x = nem.read_file_sents(ner_pred_path, fix_multi_tag=False)\n",
    "    prun_yo = bclm.read_yap_output(treebank_set=None, tokens_path=tokens_path, dep_path=conll_path, map_path=map_path)\n",
    "    prun_yo = bclm.get_token_df(prun_yo, fields=['form'])\n",
    "    prun_sents = bclm.get_sentences_list(prun_yo, fields=['token_id', 'token_str', 'form'])\n",
    "    new_sents = soft_merge_bio_labels(x, prun_sents, verbose=False)\n",
    "\n",
    "    with open(output_path, 'w') as of:\n",
    "        for sent in new_sents:\n",
    "            for form, bio in sent:\n",
    "                of.write(form+' '+bio+'\\n')\n",
    "            of.write('\\n')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "decode_sets = {\n",
    "    'token': {\n",
    "        'dev': '../NER/data/for_ncrf/morph_gold_dev.bmes',\n",
    "        'test': '../NER/data/for_ncrf/morph_gold_test.bmes',\n",
    "    },\n",
    "    'multitok': {\n",
    "        'dev': '../NER/data/for_ncrf/morph_gold_dev.bmes',\n",
    "        'test': '../NER/data/for_ncrf/morph_gold_test.bmes',\n",
    "    }\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.8050847457627118, 0.7615230460921844, 0.7826982492276005)"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "align_multitok('output/predict/multi_44184/token_gold_dev_dummy_o.bmes', \n",
    "               bclm.TREEBANK_TOKEN_PATHS['dev'], \n",
    "               'output/predict/multi_44184/dev_pruned.conll',\n",
    "               'output/predict/multi_44184/dev_pruned.map',\n",
    "               'output/predict/multi_44184/morph_pruned_dev.bmes'\n",
    "              )\n",
    "p, r, f = nem.evaluate_files(decode_sets['multitok']['dev'], 'output/predict/multi_44184/morph_pruned_dev.bmes', str_join_char='')\n",
    "p,r,f"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.826271186440678, 0.781563126252505, 0.8032955715756952)"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nem.evaluate_files('../NER/data/for_ncrf/token_gold_dev_fix.bmes', \n",
    "                   'output/predict/multi_44184/token_gold_dev_dummy_o.bmes', str_join_char='')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "multi_54360\n",
      "multi_44184\n",
      "multi_20423\n",
      "multi_80520\n",
      "multi_27916\n",
      "multi_63795\n",
      "multi_30528\n",
      "multi_78160\n",
      "multi_12345\n",
      "multi_95148\n"
     ]
    }
   ],
   "source": [
    "res = []\n",
    "for folder in os.scandir('output/predict'):\n",
    "    if os.path.isdir(folder) and 'multi' in folder.name and not '.ipynb_checkpoints' in folder.name:\n",
    "        print (folder.name)\n",
    "\n",
    "        pruned_ner_path=os.path.join(folder.path, 'morph_pruned_dev.bmes')\n",
    "        \n",
    "        align_multitok(os.path.join(folder.path, 'token_gold_dev_dummy_o.bmes'), \n",
    "                       bclm.TREEBANK_TOKEN_PATHS['dev'], \n",
    "                       os.path.join(folder.path, 'dev_pruned.conll'),\n",
    "                       os.path.join(folder.path, 'dev_pruned.map'),\n",
    "                       pruned_ner_path\n",
    "                      )\n",
    "        p, r, f = nem.evaluate_files(decode_sets['multitok']['dev'], pruned_ner_path, str_join_char='')\n",
    "        res.append((folder.name, p, r, f))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('multi_54360', 0.7749469214437368, 0.7314629258517034, 0.7525773195876289),\n",
       " ('multi_44184', 0.8050847457627118, 0.7615230460921844, 0.7826982492276005),\n",
       " ('multi_20423', 0.789587852494577, 0.7294589178356713, 0.7583333333333332),\n",
       " ('multi_80520', 0.778705636743215, 0.7474949899799599, 0.7627811860940694),\n",
       " ('multi_27916', 0.7896995708154506, 0.7374749498997996, 0.7626943005181347),\n",
       " ('multi_63795', 0.7836134453781513, 0.7474949899799599, 0.7651282051282051),\n",
       " ('multi_30528', 0.7906976744186046, 0.749498997995992, 0.7695473251028806),\n",
       " ('multi_78160', 0.8065217391304348, 0.7434869739478958, 0.7737226277372262),\n",
       " ('multi_12345', 0.8021505376344086, 0.7474949899799599, 0.7738589211618256),\n",
       " ('multi_95148', 0.7956521739130434, 0.7334669338677354, 0.7632950990615224)]"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>f</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>multi_54360</td>\n",
       "      <td>0.774947</td>\n",
       "      <td>0.731463</td>\n",
       "      <td>0.752577</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>multi_44184</td>\n",
       "      <td>0.805085</td>\n",
       "      <td>0.761523</td>\n",
       "      <td>0.782698</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>multi_20423</td>\n",
       "      <td>0.789588</td>\n",
       "      <td>0.729459</td>\n",
       "      <td>0.758333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>multi_80520</td>\n",
       "      <td>0.778706</td>\n",
       "      <td>0.747495</td>\n",
       "      <td>0.762781</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>multi_27916</td>\n",
       "      <td>0.789700</td>\n",
       "      <td>0.737475</td>\n",
       "      <td>0.762694</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>multi_63795</td>\n",
       "      <td>0.783613</td>\n",
       "      <td>0.747495</td>\n",
       "      <td>0.765128</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>multi_30528</td>\n",
       "      <td>0.790698</td>\n",
       "      <td>0.749499</td>\n",
       "      <td>0.769547</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>multi_78160</td>\n",
       "      <td>0.806522</td>\n",
       "      <td>0.743487</td>\n",
       "      <td>0.773723</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>multi_12345</td>\n",
       "      <td>0.802151</td>\n",
       "      <td>0.747495</td>\n",
       "      <td>0.773859</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>multi_95148</td>\n",
       "      <td>0.795652</td>\n",
       "      <td>0.733467</td>\n",
       "      <td>0.763295</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         model  precision    recall         f\n",
       "0  multi_54360   0.774947  0.731463  0.752577\n",
       "1  multi_44184   0.805085  0.761523  0.782698\n",
       "2  multi_20423   0.789588  0.729459  0.758333\n",
       "3  multi_80520   0.778706  0.747495  0.762781\n",
       "4  multi_27916   0.789700  0.737475  0.762694\n",
       "5  multi_63795   0.783613  0.747495  0.765128\n",
       "6  multi_30528   0.790698  0.749499  0.769547\n",
       "7  multi_78160   0.806522  0.743487  0.773723\n",
       "8  multi_12345   0.802151  0.747495  0.773859\n",
       "9  multi_95148   0.795652  0.733467  0.763295"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ne_morph_df = pd.DataFrame(res, columns=['model', 'precision', 'recall', 'f'])\n",
    "ne_morph_df"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "def biose_to_bioul(in_path, out_path):\n",
    "    sents = 0\n",
    "    with open(out_path, 'w', encoding='utf8') as of:\n",
    "        for line in open(in_path, 'r'):\n",
    "            if line=='\\n':\n",
    "                of.write(line)\n",
    "                sents+=1\n",
    "            else:\n",
    "                line = line.strip()\n",
    "                word, tag = line.split()\n",
    "                tag = tag.replace('E-', 'L-').replace('S-', 'U-')\n",
    "                of.write(word+' '+tag+'\\n')\n",
    "            \n",
    "    print (sents)\n",
    "    \n",
    "for folder in os.scandir('output/predict'):\n",
    "    if os.path.isdir(folder) and 'multi' in folder.name and not '.ipynb_checkpoints' in folder.name:\n",
    "        pruned_ner_path=os.path.join(folder.path, 'morph_pruned_dev.bmes')\n",
    "        biose_to_bioul(pruned_ner_path, pruned_ner_path.replace('.bmes', '.bioul'))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "500\n",
      "500\n",
      "500\n",
      "500\n",
      "500\n",
      "500\n",
      "500\n",
      "500\n",
      "500\n",
      "500\n"
     ]
    }
   ],
   "source": [
    "def biose_to_o(in_path, out_path):\n",
    "    sents = 0\n",
    "    with open(out_path, 'w', encoding='utf8') as of:\n",
    "        for line in open(in_path, 'r'):\n",
    "            if line=='\\n':\n",
    "                of.write(line)\n",
    "                sents+=1\n",
    "            else:\n",
    "                line = line.strip()\n",
    "                word, tag = line.split()\n",
    "                tag = 'O'\n",
    "                of.write(word+' '+tag+'\\n')\n",
    "            \n",
    "    print (sents)\n",
    "    \n",
    "for folder in os.scandir('output/predict'):\n",
    "    if os.path.isdir(folder) and 'multi' in folder.name and not '.ipynb_checkpoints' in folder.name:\n",
    "        pruned_ner_path=os.path.join(folder.path, 'morph_pruned_dev.bmes')\n",
    "        biose_to_o(pruned_ner_path, pruned_ner_path.replace('.bmes', '.bioul'))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "500\n",
      "500\n",
      "500\n",
      "500\n",
      "500\n",
      "500\n",
      "500\n",
      "500\n",
      "500\n",
      "500\n"
     ]
    }
   ],
   "source": [
    "for folder in os.scandir('output/predict'):\n",
    "    if os.path.isdir(folder) and 'morph' in folder.name and not '.ipynb_checkpoints' in folder.name:\n",
    "        pruned_ner_path=os.path.join(folder.path, 'morph_pruned_dev.json')\n",
    "        jsonl_to_biose(pruned_ner_path, pruned_ner_path.replace('.json', '.bmes'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## SINGLE + MULTI"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>p</th>\n",
       "      <th>r</th>\n",
       "      <th>f</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>eval_unit</th>\n",
       "      <th>variant</th>\n",
       "      <th>prediction</th>\n",
       "      <th>align</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">morph</th>\n",
       "      <th rowspan=\"3\" valign=\"top\">morph</th>\n",
       "      <th>gold</th>\n",
       "      <th>-</th>\n",
       "      <td>0.827206</td>\n",
       "      <td>0.782565</td>\n",
       "      <td>0.804233</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hybrid</th>\n",
       "      <th>-</th>\n",
       "      <td>0.805129</td>\n",
       "      <td>0.766132</td>\n",
       "      <td>0.785106</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>yap</th>\n",
       "      <th>-</th>\n",
       "      <td>0.744964</td>\n",
       "      <td>0.703407</td>\n",
       "      <td>0.723552</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">token</th>\n",
       "      <th>multi</th>\n",
       "      <th>tokens</th>\n",
       "      <th>-</th>\n",
       "      <td>0.807553</td>\n",
       "      <td>0.758116</td>\n",
       "      <td>0.782016</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>single</th>\n",
       "      <th>tokens</th>\n",
       "      <th>-</th>\n",
       "      <td>0.809084</td>\n",
       "      <td>0.783166</td>\n",
       "      <td>0.795844</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           p         r         f\n",
       "eval_unit variant prediction align                              \n",
       "morph     morph   gold       -      0.827206  0.782565  0.804233\n",
       "                  hybrid     -      0.805129  0.766132  0.785106\n",
       "                  yap        -      0.744964  0.703407  0.723552\n",
       "token     multi   tokens     -      0.807553  0.758116  0.782016\n",
       "          single  tokens     -      0.809084  0.783166  0.795844"
      ]
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res = []\n",
    "for folder in os.scandir('output/predict'):\n",
    "    if '.ipynb' in folder.name:\n",
    "        continue\n",
    "        \n",
    "    variant, seed = folder.name.split('_')\n",
    "    \n",
    "    if 'single' in folder.name:    \n",
    "        p,r,f = nem.evaluate_files('../NER/data/for_ncrf/token_gold_dev_fix.bmes', \n",
    "                                   os.path.join(folder.path,'token_gold_dev_fix.bmes'), str_join_char='')\n",
    "        res.append(('dev', 'token', variant, 'tokens', '-', seed, p, r, f))\n",
    "\n",
    "        \n",
    "    if 'multi' in folder.name:\n",
    "        p,r,f = nem.evaluate_files('../NER/data/for_ncrf/token_gold_dev_fix.bmes', \n",
    "                                   os.path.join(folder.path,'token_gold_dev_dummy_o.bmes'), str_join_char='')\n",
    "        res.append(('dev', 'token', variant, 'tokens', '-', seed, p, r, f))\n",
    "        \n",
    "    if 'morph' in folder.name:\n",
    "        p,r,f = nem.evaluate_files('../NER/data/for_ncrf/morph_gold_dev.bmes', \n",
    "                                   os.path.join(folder.path,'morph_gold_dev.bmes'), str_join_char='')\n",
    "        res.append(('dev', 'morph', variant, 'gold', '-', seed, p, r, f))\n",
    "        p,r,f = nem.evaluate_files('../NER/data/for_ncrf/morph_gold_dev.bmes', \n",
    "                                   os.path.join(folder.path,'morph_yap_dev.bmes'), str_join_char='')\n",
    "        res.append(('dev', 'morph', variant, 'yap', '-', seed, p, r, f))\n",
    "        p,r,f = nem.evaluate_files('../NER/data/for_ncrf/morph_gold_dev.bmes', \n",
    "                                   os.path.join(folder.path,'morph_pruned_dev.bmes'), str_join_char='')\n",
    "        res.append(('dev', 'morph', variant, 'hybrid', '-', seed, p, r, f))\n",
    "        \n",
    "        \n",
    "    \n",
    "    \n",
    "\n",
    "ne_df = pd.DataFrame(res, columns=['set', 'eval_unit', 'variant', 'prediction', 'align', 'seed', 'p', 'r', 'f'])\n",
    "\n",
    "ne_df.groupby(['eval_unit','variant', 'prediction', 'align'])[['p', 'r', 'f']].mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "eval_unit  variant  prediction  align\n",
       "morph      morph    gold        -        0.801981\n",
       "                    yap         -        0.721779\n",
       "token      multi    tokens      -        0.781968\n",
       "           single   tokens      -        0.796968\n",
       "Name: f, dtype: float64"
      ]
     },
     "execution_count": 164,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res = []\n",
    "for folder in os.scandir('output/predict_char'):\n",
    "    if '.ipynb' in folder.name:\n",
    "        continue\n",
    "        \n",
    "    variant, seed = folder.name.split('_')\n",
    "    \n",
    "    if 'single' in folder.name:    \n",
    "        p,r,f = nem.evaluate_files('../NER/data/for_ncrf/token_gold_dev_fix.bmes', \n",
    "                                   os.path.join(folder.path,'token_gold_dev_fix.bmes'), str_join_char='')\n",
    "        res.append(('dev', 'token', variant, 'tokens', '-', seed, p, r, f))\n",
    "\n",
    "        \n",
    "    if 'multi' in folder.name:\n",
    "        p,r,f = nem.evaluate_files('../NER/data/for_ncrf/token_gold_dev_fix.bmes', \n",
    "                                   os.path.join(folder.path,'token_gold_dev_dummy_o.bmes'), str_join_char='')\n",
    "        res.append(('dev', 'token', variant, 'tokens', '-', seed, p, r, f))\n",
    "        \n",
    "    if 'morph' in folder.name:\n",
    "        p,r,f = nem.evaluate_files('../NER/data/for_ncrf/morph_gold_dev.bmes', \n",
    "                                   os.path.join(folder.path,'morph_gold_dev.bmes'), str_join_char='')\n",
    "        res.append(('dev', 'morph', variant, 'gold', '-', seed, p, r, f))\n",
    "        p,r,f = nem.evaluate_files('../NER/data/for_ncrf/morph_gold_dev.bmes', \n",
    "                                   os.path.join(folder.path,'morph_yap_dev.bmes'), str_join_char='')\n",
    "        res.append(('dev', 'morph', variant, 'yap', '-', seed, p, r, f))\n",
    "        #p,r,f = nem.evaluate_files('../NER/data/for_ncrf/morph_gold_dev.bmes', \n",
    "        #                           os.path.join(folder.path,'morph_pruned_dev.bmes'), str_join_char='')\n",
    "        #res.append(('dev', 'morph', variant, 'hybrid', '-', seed, p, r, f))\n",
    "        \n",
    "        \n",
    "    \n",
    "    \n",
    "\n",
    "ne_c_df = pd.DataFrame(res, columns=['set', 'eval_unit', 'variant', 'prediction', 'align', 'seed', 'p', 'r', 'f'])\n",
    "\n",
    "ne_c_df.groupby(['eval_unit','variant', 'prediction', 'align']).f.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Add Alignments"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Token Level Eval"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "\n",
    "o_re = re.compile('^O+$') \n",
    "s_re = re.compile('^O*SO*$|^O*BI*EO*$')\n",
    "b_re = re.compile('^O*BI*$')\n",
    "i_re = re.compile('^I+$')\n",
    "e_re = re.compile('^I*EO*$')\n",
    "def get_fixed_for_valid_biose(bio_seq):\n",
    "    if o_re.match(bio_seq):\n",
    "        return 'O'\n",
    "    if s_re.match(bio_seq):\n",
    "        return 'S'\n",
    "    if b_re.match(bio_seq):\n",
    "        return 'B'\n",
    "    if i_re.match(bio_seq):\n",
    "        return 'I'\n",
    "    if e_re.match(bio_seq):\n",
    "        return 'E'\n",
    "    raise ValueError\n",
    "    \n",
    "\n",
    "def get_fixed_for_invalid_biose(parts):\n",
    "    bio = 'O'\n",
    "    if 'S' in parts:\n",
    "        bio = 'S'\n",
    "    elif 'B' in parts and 'E' in parts:\n",
    "        bio='S'\n",
    "    elif 'E' in parts:\n",
    "        bio = 'E'\n",
    "    elif 'B' in parts:\n",
    "        bio = 'B'\n",
    "    elif 'I' in parts:\n",
    "        bio = 'I'\n",
    "    return bio\n",
    "\n",
    "valid_bio_re = re.compile('^O*BI*$|^O*BI*EO*$|^I+$|^I*EO*$|^O*SO*$')\n",
    "\n",
    "from functools import lru_cache\n",
    "\n",
    "\n",
    "def validate_biose_sequence(full_bio_seq):\n",
    "    #print(full_bio_seq)\n",
    "    bio_seq, type_seq = zip(*[('O', None) if b=='O' else b.split('-') for b in full_bio_seq])\n",
    "    bio_seq = ''.join(bio_seq)\n",
    "    valid_bio = valid_bio_re.match(bio_seq)\n",
    "    type_seq = list(filter(lambda x: x is not None, type_seq))\n",
    "    type_seq_set = set(type_seq)\n",
    "\n",
    "    if valid_bio:\n",
    "        fixed_bio = get_fixed_for_valid_biose(bio_seq)\n",
    "        if fixed_bio!='O':\n",
    "            fixed_bio += '-' + type_seq[0]\n",
    "            \n",
    "    else:\n",
    "        #take the first BIOSE tag which is not O:\n",
    "        #fixed_bio = list(filter(lambda x: x!='O', full_bio_seq))[0]\n",
    "        #rough BIOSE and first category:\n",
    "        fixed_bio = get_fixed_for_invalid_biose(bio_seq)\n",
    "        if fixed_bio!='O':\n",
    "            fixed_bio += '-' + type_seq[0]\n",
    "        \n",
    "    return valid_bio is not None, len(type_seq_set)<=1, fixed_bio\n",
    "\n",
    "\n",
    "@lru_cache(1000)\n",
    "def get_fixed_bio_sequence(full_bio_seq):\n",
    "    return validate_biose_sequence(full_bio_seq)[2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/nlp/danb/anaconda3/envs/tf/lib/python3.7/site-packages/ipykernel_launcher.py:6: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  \n"
     ]
    }
   ],
   "source": [
    "dropped = [5438, 5444, 5445, 5446, 5448, 5449, 5450, 5451, 5453, 5459]\n",
    "spdf = bclm.read_dataframe('spmrl')\n",
    "spdf = spdf[(~spdf.sent_id.isin(dropped))]\n",
    "dev_gold = spdf[spdf.set=='dev']\n",
    "test_gold = spdf[spdf.set=='test']\n",
    "test_gold['sent_id'] = test_gold.sent_id.rank(method='dense').astype(int)\n",
    "dev_yap = bclm.read_yap_output(treebank_set='dev')\n",
    "test_yap = bclm.read_yap_output(treebank_set='test')\n",
    "dev_gold_sents = bclm.get_sentences_list(dev_gold, fields=['token_id', 'token_str'])\n",
    "test_gold_sents = bclm.get_sentences_list(test_gold, fields=['token_id', 'token_str'])\n",
    "dev_yap_sents = bclm.get_sentences_list(dev_yap, fields=['token_id', 'token_str'])\n",
    "test_yap_sents = bclm.get_sentences_list(test_yap, fields=['token_id', 'token_str'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/nlp/danb/anaconda3/envs/tf/lib/python3.7/site-packages/pandas/core/frame.py:3509: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self[k1] = value[k2]\n"
     ]
    }
   ],
   "source": [
    "dev_gold_tok = (bclm.get_token_df(dev_gold, biose=['biose_layer0'])\n",
    "                .rename(columns={'biose_layer0': 'fixed_bio'}))\n",
    "test_gold_tok = (bclm.get_token_df(test_gold, biose=['biose_layer0'])\n",
    "                .rename(columns={'biose_layer0': 'fixed_bio'}))\n",
    "test_gold_tok['sent_id'] = test_gold_tok.sent_id.rank(method='dense').astype(int)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sent_id</th>\n",
       "      <th>token_id</th>\n",
       "      <th>token_str</th>\n",
       "      <th>bio</th>\n",
       "      <th>fixed_bio</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>עשרות</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>אנשים</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>מגיעים</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>מתאילנד</td>\n",
       "      <td>O^S-GPE</td>\n",
       "      <td>S-GPE</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>לישראל</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>כשהם</td>\n",
       "      <td>O^O</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>נרשמים</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1</td>\n",
       "      <td>8</td>\n",
       "      <td>כמתנדבים</td>\n",
       "      <td>O^O^O</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1</td>\n",
       "      <td>9</td>\n",
       "      <td>,</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>1</td>\n",
       "      <td>10</td>\n",
       "      <td>אך</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>1</td>\n",
       "      <td>11</td>\n",
       "      <td>למעשה</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "      <td>משמשים</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "      <td>עובדים</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>1</td>\n",
       "      <td>14</td>\n",
       "      <td>שכירים</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>1</td>\n",
       "      <td>15</td>\n",
       "      <td>זולים</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>1</td>\n",
       "      <td>16</td>\n",
       "      <td>.</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>תופעה</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>זו</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>התבררה</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>אתמול</td>\n",
       "      <td>O</td>\n",
       "      <td>O</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    sent_id  token_id token_str      bio fixed_bio\n",
       "0         1         1     עשרות        O         O\n",
       "1         1         2     אנשים        O         O\n",
       "2         1         3    מגיעים        O         O\n",
       "3         1         4   מתאילנד  O^S-GPE     S-GPE\n",
       "4         1         5    לישראל        O         O\n",
       "5         1         6      כשהם      O^O         O\n",
       "6         1         7    נרשמים        O         O\n",
       "7         1         8  כמתנדבים    O^O^O         O\n",
       "8         1         9         ,        O         O\n",
       "9         1        10        אך        O         O\n",
       "10        1        11     למעשה        O         O\n",
       "11        1        12    משמשים        O         O\n",
       "12        1        13    עובדים        O         O\n",
       "13        1        14    שכירים        O         O\n",
       "14        1        15     זולים        O         O\n",
       "15        1        16         .        O         O\n",
       "16        2         1     תופעה        O         O\n",
       "17        2         2        זו        O         O\n",
       "18        2         3    התבררה        O         O\n",
       "19        2         4     אתמול        O         O"
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path = 'output/predict/morph_12345/morph_yap_dev.bmes'\n",
    "def get_fixed_tok(path, orig_sents=dev_yap_sents):\n",
    "    x = nem.read_file_sents(path, fix_multi_tag=False)\n",
    "    new_sents = []\n",
    "    for (i, ner_sent), (sent_id, yap_sent) in zip(x.iteritems(), orig_sents.iteritems()):\n",
    "        for (form, bio), (token_id, token_str) in zip(ner_sent, yap_sent):\n",
    "            new_sents.append((sent_id, token_id, token_str, form, bio))\n",
    "    new_sents = pd.DataFrame(new_sents, columns=['sent_id', 'token_id', 'token_str', 'form', 'bio'])\n",
    "    new_toks = bclm.get_token_df(new_sents, fields=['bio'])\n",
    "    new_toks['fixed_bio'] = new_toks.bio.apply(lambda x: get_fixed_bio_sequence(tuple(x.split('^'))))\n",
    "    return new_toks\n",
    "\n",
    "new_toks = get_fixed_tok(path)\n",
    "new_toks.head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sents_from_df(df, sent_id_col='sent_id', \n",
    "                  group_cols=['token_str'], \n",
    "                  val_cols=['fixed_bio']):\n",
    "    sents = bclm.get_sentences_list(df, fields=group_cols+val_cols)\n",
    "    return sents\n",
    "\n",
    "def evaluate_dataframes(gold_df, pred_df, fix_multi_tag_pred=True, truncate=None, ignore_cat=False, str_join_char=' '):\n",
    "    gold_sents = sents_from_df(gold_df)\n",
    "    pred_sents = sents_from_df(pred_df)\n",
    "    gold_mentions = nem.sents_to_mentions(gold_sents, truncate=truncate, ignore_cat=ignore_cat, str_join_char=str_join_char)\n",
    "    pred_mentions = nem.sents_to_mentions(pred_sents, truncate=truncate, ignore_cat=ignore_cat, str_join_char=str_join_char)\n",
    "    return nem.evaluate_mentions(gold_mentions, pred_mentions, verbose=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "sent_id\n",
       "1      [[עשרות, O], [אנשים, O], [מגיעים, O], [מתאילנד...\n",
       "2      [[תופעה, O], [זו, O], [התבררה, O], [אתמול, O],...\n",
       "3      [[יו\"ר, O], [הוועדה, O], [,, O], [ח\"כ, O], [או...\n",
       "4      [[מצד, O], [אחד, O], [רוצה, O], [האוצר, S-ORG]...\n",
       "5      [[נמיר, S-PER], [הודיעה, O], [כי, O], [תפנה, O...\n",
       "                             ...                        \n",
       "496    [[מוות, O], [לערבים, O], [,, O], [מוות, O], [ל...\n",
       "497    [[קשה, O], [להוציא, O], [את, O], [הארון, O], [...\n",
       "498    [[עכשיו, O], [קוראים, O], [את, O], [הקדיש, O],...\n",
       "499           [[תם, O], [מסע, O], [ההלווייה, O], [., O]]\n",
       "500           [[מתחיל, O], [מסע, O], [הנקמה, O], [., O]]\n",
       "Length: 500, dtype: object"
      ]
     },
     "execution_count": 117,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sents_from_df(new_toks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.7742616033755274, 0.7354709418837675, 0.7543679342240494)"
      ]
     },
     "execution_count": 119,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluate_dataframes(dev_gold_tok, new_toks, str_join_char='')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Hybrid\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [],
   "source": [
    "out_folder = '../NER/data/tokens_for_ncrf'\n",
    "dev_out = os.path.join(out_folder, 'dev_tokens.txt')\n",
    "test_out = os.path.join(out_folder, 'test_tokens.txt')\n",
    "token_paths = {'dev': dev_out, 'test': test_out}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [],
   "source": [
    "@lru_cache(512)\n",
    "def get_prun_yo(ds, dep_path, map_path):\n",
    "\n",
    "    \n",
    "    prun_yo = bclm.read_yap_output(treebank_set=None,\n",
    "                               tokens_path=token_paths[ds],\n",
    "                               dep_path=dep_path,\n",
    "                               map_path=map_path,\n",
    "                                )\n",
    "    return prun_yo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.805439330543933, 0.7715430861723447, 0.7881269191402253)"
      ]
     },
     "execution_count": 127,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dep_path='output/predict/multi_12345/dev_pruned.conll'\n",
    "map_path='output/predict/multi_12345/dev_pruned.map'\n",
    "\n",
    "prun_yo = get_prun_yo('dev', dep_path, map_path)\n",
    "prun_sents = bclm.get_sentences_list(prun_yo, fields=['token_id', 'token_str'])\n",
    "path =  'output/predict/morph_12345/morph_pruned_dev.bmes'\n",
    "new_toks = get_fixed_tok(path, orig_sents=prun_sents)\n",
    "evaluate_dataframes(dev_gold_tok, new_toks, str_join_char='')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.7949790794979079, 0.7615230460921844, 0.7778915046059366)"
      ]
     },
     "execution_count": 128,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nem.evaluate_files('../NER/data/for_ncrf/morph_gold_dev.bmes',\n",
    "                  path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Run on all pruned"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [],
   "source": [
    "@lru_cache(512)\n",
    "def get_sent_list(ds, dp, mp):\n",
    "    prun_yo = get_prun_yo(ds, dp, mp)\n",
    "    return bclm.get_sentences_list(prun_yo, fields=['token_id', 'token_str'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [],
   "source": [
    "align_tok_res = []\n",
    "for folder in os.scandir('output/predict'):\n",
    "    if 'morph' in folder.name and not '.ipynb' in folder.name:\n",
    "        ## dev \n",
    "        variant, seed = folder.name.split('_')\n",
    "        file = os.path.join(folder.path,'morph_pruned_dev.bmes')\n",
    "        multi_folder = folder.path.replace('morph_', 'multi_')\n",
    "        dep_path = os.path.join(multi_folder, 'dev_pruned.conll')\n",
    "        map_path = os.path.join(multi_folder, 'dev_pruned.map')\n",
    "        out_path = os.path.join(folder.path, 'morph_pruned_dev_align_tokens.bmes')\n",
    "        \n",
    "        prun_sents = get_sent_list('dev',dep_path , map_path)\n",
    "        new_toks = get_fixed_tok(file, orig_sents=prun_sents)\n",
    "        \n",
    "        if not os.path.exists(out_path):\n",
    "            new_sents = bclm.get_sentences_list(new_toks, fields=['token_str', 'fixed_bio'])\n",
    "            with open(out_path, 'w') as of:\n",
    "                for i, sent in new_sents.iteritems():\n",
    "                    for tok, bio in sent:\n",
    "                        of.write(tok+' '+bio+'\\n')\n",
    "                    of.write('\\n')\n",
    "                    \n",
    "        p, r, f = evaluate_dataframes(dev_gold_tok, new_toks, str_join_char='')\n",
    "\n",
    "        align_tok_res.append(('dev', 'token', 'morph', 'hybrid', 'tokens', seed, p, r, f))\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "eval_unit  variant  prediction  align \n",
       "token      morph    hybrid      tokens    0.791924\n",
       "Name: f, dtype: float64"
      ]
     },
     "execution_count": 141,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "at_df = pd.DataFrame(align_tok_res, columns=['set', 'eval_unit', 'variant', 'prediction', 'align', 'seed', 'p', 'r', 'f'])\n",
    "\n",
    "at_df.groupby(['eval_unit','variant', 'prediction', 'align']).f.mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Run all gold and YAP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {},
   "outputs": [],
   "source": [
    "align_tok_res_yg = []\n",
    "for folder in os.scandir('output/predict'):\n",
    "    if 'morph' in folder.name and not '.ipynb' in folder.name:\n",
    "        ## dev \n",
    "        ## - gold\n",
    "        variant, seed = folder.name.split('_')\n",
    "        file = os.path.join(folder.path,'morph_gold_dev.bmes')\n",
    "        out_path = os.path.join(folder.path, 'morph_gold_dev_align_tokens.bmes')\n",
    "        \n",
    "        new_toks = get_fixed_tok(file, orig_sents=dev_gold_sents)\n",
    "        \n",
    "        if not os.path.exists(out_path):\n",
    "            new_sents = bclm.get_sentences_list(new_toks, fields=['token_str', 'fixed_bio'])\n",
    "            with open(out_path, 'w') as of:\n",
    "                for i, sent in new_sents.iteritems():\n",
    "                    for tok, bio in sent:\n",
    "                        of.write(tok+' '+bio+'\\n')\n",
    "                    of.write('\\n')\n",
    "                    \n",
    "        p, r, f = evaluate_dataframes(dev_gold_tok, new_toks, str_join_char='')\n",
    "\n",
    "        align_tok_res_yg.append(('dev', 'token', 'morph', 'gold', 'tokens', seed, p, r, f))\n",
    "        \n",
    "        ## - yap\n",
    "        variant, seed = folder.name.split('_')\n",
    "        file = os.path.join(folder.path,'morph_yap_dev.bmes')\n",
    "        out_path = os.path.join(folder.path, 'morph_yap_dev_align_tokens.bmes')\n",
    "        \n",
    "        new_toks = get_fixed_tok(file, orig_sents=dev_yap_sents)\n",
    "        \n",
    "        if not os.path.exists(out_path):\n",
    "            new_sents = bclm.get_sentences_list(new_toks, fields=['token_str', 'fixed_bio'])\n",
    "            with open(out_path, 'w') as of:\n",
    "                for i, sent in new_sents.iteritems():\n",
    "                    for tok, bio in sent:\n",
    "                        of.write(tok+' '+bio+'\\n')\n",
    "                    of.write('\\n')\n",
    "        p, r, f = evaluate_dataframes(dev_gold_tok, new_toks, str_join_char='')\n",
    "\n",
    "        align_tok_res_yg.append(('dev', 'token', 'morph', 'yap', 'tokens', seed, p, r, f))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>p</th>\n",
       "      <th>r</th>\n",
       "      <th>f</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>eval_unit</th>\n",
       "      <th>variant</th>\n",
       "      <th>prediction</th>\n",
       "      <th>align</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">token</th>\n",
       "      <th rowspan=\"3\" valign=\"top\">morph</th>\n",
       "      <th>gold</th>\n",
       "      <th>tokens</th>\n",
       "      <td>0.828991</td>\n",
       "      <td>0.783768</td>\n",
       "      <td>0.805714</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hybrid</th>\n",
       "      <th>tokens</th>\n",
       "      <td>0.812380</td>\n",
       "      <td>0.772545</td>\n",
       "      <td>0.791924</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>yap</th>\n",
       "      <th>tokens</th>\n",
       "      <td>0.785387</td>\n",
       "      <td>0.741082</td>\n",
       "      <td>0.762553</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            p         r         f\n",
       "eval_unit variant prediction align                               \n",
       "token     morph   gold       tokens  0.828991  0.783768  0.805714\n",
       "                  hybrid     tokens  0.812380  0.772545  0.791924\n",
       "                  yap        tokens  0.785387  0.741082  0.762553"
      ]
     },
     "execution_count": 144,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "at_df = pd.DataFrame(align_tok_res+align_tok_res_yg, columns=['set', 'eval_unit', 'variant', 'prediction', 'align', 'seed', 'p', 'r', 'f'])\n",
    "\n",
    "at_df.groupby(['eval_unit','variant', 'prediction', 'align'])[['p', 'r', 'f']].mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Morpheme Level Eval"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Token Multi"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Hybrid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "multi_54360\n",
      "multi_44184\n",
      "multi_20423\n",
      "multi_80520\n",
      "multi_27916\n",
      "multi_63795\n",
      "multi_30528\n",
      "multi_78160\n",
      "multi_12345\n",
      "multi_95148\n"
     ]
    }
   ],
   "source": [
    "align_morph_res_hyb = []\n",
    "for folder in os.scandir('output/predict'):\n",
    "    if os.path.isdir(folder) and 'multi' in folder.name and not '.ipynb_checkpoints' in folder.name:\n",
    "        print (folder.name)\n",
    "\n",
    "        pruned_ner_path=os.path.join(folder.path, 'morph_pruned_dev.bmes')\n",
    "        \n",
    "        p, r, f = nem.evaluate_files(decode_sets['multitok']['dev'], pruned_ner_path, str_join_char='')\n",
    "        align_morph_res_hyb.append(('dev', 'morph', 'multi', 'tokens', 'hybrid', seed, p, r, f))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>p</th>\n",
       "      <th>r</th>\n",
       "      <th>f</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>eval_unit</th>\n",
       "      <th>variant</th>\n",
       "      <th>prediction</th>\n",
       "      <th>align</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>morph</th>\n",
       "      <th>multi</th>\n",
       "      <th>tokens</th>\n",
       "      <th>hybrid</th>\n",
       "      <td>0.791666</td>\n",
       "      <td>0.742886</td>\n",
       "      <td>0.766464</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">token</th>\n",
       "      <th rowspan=\"3\" valign=\"top\">morph</th>\n",
       "      <th>gold</th>\n",
       "      <th>tokens</th>\n",
       "      <td>0.828991</td>\n",
       "      <td>0.783768</td>\n",
       "      <td>0.805714</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hybrid</th>\n",
       "      <th>tokens</th>\n",
       "      <td>0.812380</td>\n",
       "      <td>0.772545</td>\n",
       "      <td>0.791924</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>yap</th>\n",
       "      <th>tokens</th>\n",
       "      <td>0.785387</td>\n",
       "      <td>0.741082</td>\n",
       "      <td>0.762553</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            p         r         f\n",
       "eval_unit variant prediction align                               \n",
       "morph     multi   tokens     hybrid  0.791666  0.742886  0.766464\n",
       "token     morph   gold       tokens  0.828991  0.783768  0.805714\n",
       "                  hybrid     tokens  0.812380  0.772545  0.791924\n",
       "                  yap        tokens  0.785387  0.741082  0.762553"
      ]
     },
     "execution_count": 148,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "at_df = pd.DataFrame(align_tok_res+align_tok_res_yg+align_morph_res_hyb, columns=['set', 'eval_unit', 'variant', 'prediction', 'align', 'seed', 'p', 'r', 'f'])\n",
    "\n",
    "at_df.groupby(['eval_unit','variant', 'prediction', 'align'])[['p', 'r', 'f']].mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### YAP + GOLD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [],
   "source": [
    "def align_multitok_yg(ner_pred_path, prun_sents, output_path):\n",
    "    x = nem.read_file_sents(ner_pred_path, fix_multi_tag=False)\n",
    "\n",
    "    new_sents = soft_merge_bio_labels(x, prun_sents, verbose=False)\n",
    "\n",
    "    with open(output_path, 'w') as of:\n",
    "        for sent in new_sents:\n",
    "            for form, bio in sent:\n",
    "                of.write(form+' '+bio+'\\n')\n",
    "            of.write('\\n')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [],
   "source": [
    "gold_morph = {'dev': dev_gold, 'test': test_gold}\n",
    "def get_sents_for_mult(treebank_set, gold=False, pred_set=None, \n",
    "                       dep_path=None, map_path=None):\n",
    "    if treebank_set is None:\n",
    "        prun_yo = get_prun_yo(pred_set, dep_path, map_path)\n",
    "    else:\n",
    "        if not gold:\n",
    "            prun_yo = bclm.read_yap_output(treebank_set=treebank_set)\n",
    "        else:\n",
    "            prun_yo = gold_morph[treebank_set]\n",
    "    prun_yo = bclm.get_token_df(prun_yo, fields=['form'])\n",
    "    prun_sents = bclm.get_sentences_list(prun_yo, fields=['token_id', 'token_str', 'form'])\n",
    "    return prun_sents\n",
    "\n",
    "dev_yap_sents_m = get_sents_for_mult('dev')\n",
    "test_yap_sents_m = get_sents_for_mult('test')\n",
    "dev_gold_sents_m = get_sents_for_mult('dev', gold=True)\n",
    "test_gold_sents_m = get_sents_for_mult('test', gold=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "multi_54360\n",
      "multi_44184\n",
      "multi_20423\n",
      "multi_80520\n",
      "multi_27916\n",
      "multi_63795\n",
      "multi_30528\n",
      "multi_78160\n",
      "multi_12345\n",
      "multi_95148\n"
     ]
    }
   ],
   "source": [
    "align_morph_res_yap = []\n",
    "for folder in os.scandir('output/predict'):\n",
    "    if os.path.isdir(folder) and 'multi' in folder.name and not '.ipynb_checkpoints' in folder.name:\n",
    "        print (folder.name)\n",
    "\n",
    "        yap_ner_path=os.path.join(folder.path, 'morph_yap_dev.bmes')\n",
    "        \n",
    "        align_multitok_yg(os.path.join(folder.path, 'token_gold_dev_dummy_o.bmes'), \n",
    "                           dev_yap_sents_m,\n",
    "                           yap_ner_path\n",
    "                          )\n",
    "        p, r, f = nem.evaluate_files(decode_sets['multitok']['dev'], yap_ner_path, str_join_char='')\n",
    "        align_morph_res_yap.append(('dev', 'morph', 'multi', 'tokens', 'yap', seed, p, r, f))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "multi_54360\n",
      "multi_44184\n",
      "multi_20423\n",
      "multi_80520\n",
      "multi_27916\n",
      "multi_63795\n",
      "multi_30528\n",
      "multi_78160\n",
      "multi_12345\n",
      "multi_95148\n"
     ]
    }
   ],
   "source": [
    "align_morph_res_gold = []\n",
    "for folder in os.scandir('output/predict'):\n",
    "    if os.path.isdir(folder) and 'multi' in folder.name and not '.ipynb_checkpoints' in folder.name:\n",
    "        print (folder.name)\n",
    "\n",
    "        gold_ner_path=os.path.join(folder.path, 'morph_gold_dev.bmes')\n",
    "        \n",
    "        align_multitok_yg(os.path.join(folder.path, 'token_gold_dev_dummy_o.bmes'), \n",
    "                           dev_gold_sents_m,\n",
    "                           gold_ner_path\n",
    "                          )\n",
    "        p, r, f = nem.evaluate_files(decode_sets['multitok']['dev'], gold_ner_path, str_join_char='')\n",
    "        align_morph_res_gold.append(('dev', 'morph', 'multi', 'tokens', 'gold', seed, p, r, f))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>p</th>\n",
       "      <th>r</th>\n",
       "      <th>f</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>eval_unit</th>\n",
       "      <th>variant</th>\n",
       "      <th>prediction</th>\n",
       "      <th>align</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">morph</th>\n",
       "      <th rowspan=\"3\" valign=\"top\">multi</th>\n",
       "      <th rowspan=\"3\" valign=\"top\">tokens</th>\n",
       "      <th>gold</th>\n",
       "      <td>0.803848</td>\n",
       "      <td>0.752705</td>\n",
       "      <td>0.777399</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hybrid</th>\n",
       "      <td>0.791666</td>\n",
       "      <td>0.742886</td>\n",
       "      <td>0.766464</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>yap</th>\n",
       "      <td>0.755070</td>\n",
       "      <td>0.696593</td>\n",
       "      <td>0.724617</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">token</th>\n",
       "      <th rowspan=\"3\" valign=\"top\">morph</th>\n",
       "      <th>gold</th>\n",
       "      <th>tokens</th>\n",
       "      <td>0.828991</td>\n",
       "      <td>0.783768</td>\n",
       "      <td>0.805714</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hybrid</th>\n",
       "      <th>tokens</th>\n",
       "      <td>0.812380</td>\n",
       "      <td>0.772545</td>\n",
       "      <td>0.791924</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>yap</th>\n",
       "      <th>tokens</th>\n",
       "      <td>0.785387</td>\n",
       "      <td>0.741082</td>\n",
       "      <td>0.762553</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            p         r         f\n",
       "eval_unit variant prediction align                               \n",
       "morph     multi   tokens     gold    0.803848  0.752705  0.777399\n",
       "                             hybrid  0.791666  0.742886  0.766464\n",
       "                             yap     0.755070  0.696593  0.724617\n",
       "token     morph   gold       tokens  0.828991  0.783768  0.805714\n",
       "                  hybrid     tokens  0.812380  0.772545  0.791924\n",
       "                  yap        tokens  0.785387  0.741082  0.762553"
      ]
     },
     "execution_count": 161,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "at_df = pd.DataFrame(align_tok_res+align_tok_res_yg+align_morph_res_hyb+align_morph_res_yap+align_morph_res_gold, columns=['set', 'eval_unit', 'variant', 'prediction', 'align', 'seed', 'p', 'r', 'f'])\n",
    "\n",
    "at_df.groupby(['eval_unit','variant', 'prediction', 'align'])[['p', 'r', 'f']].mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>eval_unit</th>\n",
       "      <th>variant</th>\n",
       "      <th>prediction</th>\n",
       "      <th>align</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">morph</th>\n",
       "      <th rowspan=\"3\" valign=\"top\">multi</th>\n",
       "      <th rowspan=\"3\" valign=\"top\">tokens</th>\n",
       "      <th>gold</th>\n",
       "      <td>$77.74 ± 0.5$</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hybrid</th>\n",
       "      <td>$76.65 ± 0.5$</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>yap</th>\n",
       "      <td>$72.46 ± 0.5$</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">token</th>\n",
       "      <th rowspan=\"3\" valign=\"top\">morph</th>\n",
       "      <th>gold</th>\n",
       "      <th>tokens</th>\n",
       "      <td>$80.57 ± 0.3$</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hybrid</th>\n",
       "      <th>tokens</th>\n",
       "      <td>$79.19 ± 0.4$</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>yap</th>\n",
       "      <th>tokens</th>\n",
       "      <td>$76.26 ± 0.4$</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              mean\n",
       "eval_unit variant prediction align                \n",
       "morph     multi   tokens     gold    $77.74 ± 0.5$\n",
       "                             hybrid  $76.65 ± 0.5$\n",
       "                             yap     $72.46 ± 0.5$\n",
       "token     morph   gold       tokens  $80.57 ± 0.3$\n",
       "                  hybrid     tokens  $79.19 ± 0.4$\n",
       "                  yap        tokens  $76.26 ± 0.4$"
      ]
     },
     "execution_count": 168,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "at_df = pd.DataFrame(align_tok_res+align_tok_res_yg+align_morph_res_hyb+align_morph_res_yap+align_morph_res_gold, columns=['set', 'eval_unit', 'variant', 'prediction', 'align', 'seed', 'p', 'r', 'f'])\n",
    "\n",
    "(at_df.groupby(['eval_unit','variant', 'prediction', 'align']).f.agg(['mean', 'std']).mul(100).round(2)\n",
    "         .assign(mean = lambda x: '$'+x['mean'].apply('{:,.2f}'.format).astype(str)+' ± '+ (1.96*(x['std']/np.sqrt(10))).round(1).astype(str)+'$')[['mean']])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>eval_unit</th>\n",
       "      <th>variant</th>\n",
       "      <th>prediction</th>\n",
       "      <th>align</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">morph</th>\n",
       "      <th rowspan=\"3\" valign=\"top\">morph</th>\n",
       "      <th>gold</th>\n",
       "      <th>-</th>\n",
       "      <td>$80.42 ± 0.3$</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>hybrid</th>\n",
       "      <th>-</th>\n",
       "      <td>$78.51 ± 0.3$</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>yap</th>\n",
       "      <th>-</th>\n",
       "      <td>$72.36 ± 0.4$</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">token</th>\n",
       "      <th>multi</th>\n",
       "      <th>tokens</th>\n",
       "      <th>-</th>\n",
       "      <td>$78.20 ± 0.6$</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>single</th>\n",
       "      <th>tokens</th>\n",
       "      <th>-</th>\n",
       "      <td>$79.58 ± 0.3$</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                             mean\n",
       "eval_unit variant prediction align               \n",
       "morph     morph   gold       -      $80.42 ± 0.3$\n",
       "                  hybrid     -      $78.51 ± 0.3$\n",
       "                  yap        -      $72.36 ± 0.4$\n",
       "token     multi   tokens     -      $78.20 ± 0.6$\n",
       "          single  tokens     -      $79.58 ± 0.3$"
      ]
     },
     "execution_count": 169,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(ne_df.groupby(['eval_unit','variant', 'prediction', 'align']).f.agg(['mean', 'std']).mul(100).round(2)\n",
    "         .assign(mean = lambda x: '$'+x['mean'].apply('{:,.2f}'.format).astype(str)+' ± '+ (1.96*(x['std']/np.sqrt(10))).round(1).astype(str)+'$')[['mean']])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "eval_unit  variant  prediction  align \n",
       "morph      multi    tokens      gold      10\n",
       "                                hybrid    10\n",
       "                                yap       10\n",
       "token      morph    gold        tokens    10\n",
       "                    hybrid      tokens    10\n",
       "                    yap         tokens    10\n",
       "dtype: int64"
      ]
     },
     "execution_count": 166,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "at_df.groupby(['eval_unit','variant', 'prediction', 'align']).size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "tf",
   "language": "python",
   "name": "tf"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}