evaluation/evaluation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import os\n",
    "import nltk\n",
    "import numpy as np\n",
    "import subprocess\n",
    "\n",
    "unseen_domains = ['Artist', 'Politician', 'CelestialBody', 'Athlete', 'MeanOfTransportation']\n",
    "# root path\n",
    "path='evaluation/'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Evaluation of Ordering and Structuring"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "All domains:\n",
      "Task:  ordering\n",
      "Set:  dev\n",
      "Model:  random\n",
      "Accuracy:  0.29\n",
      "----------\n",
      "Task:  ordering\n",
      "Set:  dev\n",
      "Model:  major\n",
      "Accuracy:  0.54\n",
      "----------\n",
      "Task:  ordering\n",
      "Set:  dev\n",
      "Model:  transformer\n",
      "Accuracy:  0.59\n",
      "----------\n",
      "Task:  ordering\n",
      "Set:  dev\n",
      "Model:  rnn\n",
      "Accuracy:  0.63\n",
      "----------\n",
      "Task:  structing\n",
      "Set:  dev\n",
      "Model:  random\n",
      "Accuracy:  0.26\n",
      "----------\n",
      "Task:  structing\n",
      "Set:  dev\n",
      "Model:  major\n",
      "Accuracy:  0.49\n",
      "----------\n",
      "Task:  structing\n",
      "Set:  dev\n",
      "Model:  transformer\n",
      "Accuracy:  0.65\n",
      "----------\n",
      "Task:  structing\n",
      "Set:  dev\n",
      "Model:  rnn\n",
      "Accuracy:  0.67\n",
      "----------\n",
      "Task:  ordering\n",
      "Set:  test\n",
      "Model:  random\n",
      "Accuracy:  0.31\n",
      "----------\n",
      "Task:  ordering\n",
      "Set:  test\n",
      "Model:  major\n",
      "Accuracy:  0.48\n",
      "----------\n",
      "Task:  ordering\n",
      "Set:  test\n",
      "Model:  transformer\n",
      "Accuracy:  0.34\n",
      "----------\n",
      "Task:  ordering\n",
      "Set:  test\n",
      "Model:  rnn\n",
      "Accuracy:  0.35\n",
      "----------\n",
      "Task:  structing\n",
      "Set:  test\n",
      "Model:  random\n",
      "Accuracy:  0.29\n",
      "----------\n",
      "Task:  structing\n",
      "Set:  test\n",
      "Model:  major\n",
      "Accuracy:  0.27\n",
      "----------\n",
      "Task:  structing\n",
      "Set:  test\n",
      "Model:  transformer\n",
      "Accuracy:  0.36\n",
      "----------\n",
      "Task:  structing\n",
      "Set:  test\n",
      "Model:  rnn\n",
      "Accuracy:  0.39\n",
      "----------\n"
     ]
    }
   ],
   "source": [
    "print('All domains:')\n",
    "for _set in ['dev', 'test']:\n",
    "    for task in ['ordering', 'structing']:\n",
    "        gold_path=os.path.join(path, 'data', task, _set + '.json')\n",
    "        gold = json.load(open(gold_path))\n",
    "\n",
    "        for model in ['random', 'major', 'transformer', 'rnn']:\n",
    "            p = os.path.join(path, 'results/steps', task, model, _set + '.out.postprocessed')\n",
    "            with open(p) as f:\n",
    "                y_pred_ = f.read().split('\\n')[:-1]\n",
    "\n",
    "            y_real, y_pred = [], []\n",
    "            for i, g in enumerate(gold):\n",
    "                t = [' '.join(target['output']) for target in g['targets']]\n",
    "                y_real.append(t)\n",
    "                y_pred.append(y_pred_[i].strip())\n",
    "\n",
    "            num, dem = 0.0, 0\n",
    "            for i, y_ in enumerate(y_pred):\n",
    "                y = y_real[i]\n",
    "                if y_.strip() in y:\n",
    "                    num += 1\n",
    "                dem += 1\n",
    "            print('Task: ', task)\n",
    "            print('Set: ', _set)\n",
    "            print('Model: ', model)\n",
    "            print('Accuracy: ', round(num/dem, 2))\n",
    "            print(10 * '-')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Seen domains:\n",
      "Task:  ordering\n",
      "Set:  dev\n",
      "Model:  random\n",
      "Accuracy:  0.29\n",
      "----------\n",
      "Task:  ordering\n",
      "Set:  dev\n",
      "Model:  major\n",
      "Accuracy:  0.54\n",
      "----------\n",
      "Task:  ordering\n",
      "Set:  dev\n",
      "Model:  transformer\n",
      "Accuracy:  0.59\n",
      "----------\n",
      "Task:  ordering\n",
      "Set:  dev\n",
      "Model:  rnn\n",
      "Accuracy:  0.63\n",
      "----------\n",
      "Task:  structing\n",
      "Set:  dev\n",
      "Model:  random\n",
      "Accuracy:  0.26\n",
      "----------\n",
      "Task:  structing\n",
      "Set:  dev\n",
      "Model:  major\n",
      "Accuracy:  0.49\n",
      "----------\n",
      "Task:  structing\n",
      "Set:  dev\n",
      "Model:  transformer\n",
      "Accuracy:  0.65\n",
      "----------\n",
      "Task:  structing\n",
      "Set:  dev\n",
      "Model:  rnn\n",
      "Accuracy:  0.67\n",
      "----------\n",
      "Task:  ordering\n",
      "Set:  test\n",
      "Model:  random\n",
      "Accuracy:  0.29\n",
      "----------\n",
      "Task:  ordering\n",
      "Set:  test\n",
      "Model:  major\n",
      "Accuracy:  0.51\n",
      "----------\n",
      "Task:  ordering\n",
      "Set:  test\n",
      "Model:  transformer\n",
      "Accuracy:  0.56\n",
      "----------\n",
      "Task:  ordering\n",
      "Set:  test\n",
      "Model:  rnn\n",
      "Accuracy:  0.56\n",
      "----------\n",
      "Task:  structing\n",
      "Set:  test\n",
      "Model:  random\n",
      "Accuracy:  0.29\n",
      "----------\n",
      "Task:  structing\n",
      "Set:  test\n",
      "Model:  major\n",
      "Accuracy:  0.45\n",
      "----------\n",
      "Task:  structing\n",
      "Set:  test\n",
      "Model:  transformer\n",
      "Accuracy:  0.59\n",
      "----------\n",
      "Task:  structing\n",
      "Set:  test\n",
      "Model:  rnn\n",
      "Accuracy:  0.63\n",
      "----------\n"
     ]
    }
   ],
   "source": [
    "print('Seen domains:')\n",
    "for _set in ['dev', 'test']:\n",
    "    for task in ['ordering', 'structing']:\n",
    "        gold_path=os.path.join(path, 'data', task, _set + '.json')\n",
    "        gold = json.load(open(gold_path))\n",
    "\n",
    "        for model in ['random', 'major', 'transformer', 'rnn']:\n",
    "            p = os.path.join(path, 'results/steps', task, model, _set + '.out.postprocessed')\n",
    "            with open(p) as f:\n",
    "                y_pred_ = f.read().split('\\n')[:-1]\n",
    "\n",
    "            y_real, y_pred = [], []\n",
    "            for i, g in enumerate(gold):\n",
    "                if g['category'] not in unseen_domains:\n",
    "                    t = [' '.join(target['output']) for target in g['targets']]\n",
    "                    y_real.append(t)\n",
    "                    y_pred.append(y_pred_[i].strip())\n",
    "\n",
    "            num, dem = 0.0, 0\n",
    "            for i, y_ in enumerate(y_pred):\n",
    "                y = y_real[i]\n",
    "                if y_.strip() in y:\n",
    "                    num += 1\n",
    "                dem += 1\n",
    "            print('Task: ', task)\n",
    "            print('Set: ', _set)\n",
    "            print('Model: ', model)\n",
    "            print('Accuracy: ', round(num/dem, 2))\n",
    "            print(10 * '-')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unseen domains:\n",
      "Task:  ordering\n",
      "Set:  dev\n",
      "Model:  random\n",
      "Accuracy:  0\n",
      "----------\n",
      "Task:  ordering\n",
      "Set:  dev\n",
      "Model:  major\n",
      "Accuracy:  0\n",
      "----------\n",
      "Task:  ordering\n",
      "Set:  dev\n",
      "Model:  transformer\n",
      "Accuracy:  0\n",
      "----------\n",
      "Task:  ordering\n",
      "Set:  dev\n",
      "Model:  rnn\n",
      "Accuracy:  0\n",
      "----------\n",
      "Task:  structing\n",
      "Set:  dev\n",
      "Model:  random\n",
      "Accuracy:  0\n",
      "----------\n",
      "Task:  structing\n",
      "Set:  dev\n",
      "Model:  major\n",
      "Accuracy:  0\n",
      "----------\n",
      "Task:  structing\n",
      "Set:  dev\n",
      "Model:  transformer\n",
      "Accuracy:  0\n",
      "----------\n",
      "Task:  structing\n",
      "Set:  dev\n",
      "Model:  rnn\n",
      "Accuracy:  0\n",
      "----------\n",
      "Task:  ordering\n",
      "Set:  test\n",
      "Model:  random\n",
      "Accuracy:  0.35\n",
      "----------\n",
      "Task:  ordering\n",
      "Set:  test\n",
      "Model:  major\n",
      "Accuracy:  0.44\n",
      "----------\n",
      "Task:  ordering\n",
      "Set:  test\n",
      "Model:  transformer\n",
      "Accuracy:  0.09\n",
      "----------\n",
      "Task:  ordering\n",
      "Set:  test\n",
      "Model:  rnn\n",
      "Accuracy:  0.1\n",
      "----------\n",
      "Task:  structing\n",
      "Set:  test\n",
      "Model:  random\n",
      "Accuracy:  0.3\n",
      "----------\n",
      "Task:  structing\n",
      "Set:  test\n",
      "Model:  major\n",
      "Accuracy:  0.06\n",
      "----------\n",
      "Task:  structing\n",
      "Set:  test\n",
      "Model:  transformer\n",
      "Accuracy:  0.12\n",
      "----------\n",
      "Task:  structing\n",
      "Set:  test\n",
      "Model:  rnn\n",
      "Accuracy:  0.13\n",
      "----------\n"
     ]
    }
   ],
   "source": [
    "print('Unseen domains:')\n",
    "for _set in ['dev', 'test']:\n",
    "    for task in ['ordering', 'structing']:\n",
    "        gold_path=os.path.join(path, 'data', task, _set + '.json')\n",
    "        gold = json.load(open(gold_path))\n",
    "\n",
    "        for model in ['random', 'major', 'transformer', 'rnn']:\n",
    "            p = os.path.join(path, 'results/steps', task, model, _set + '.out.postprocessed')\n",
    "            with open(p) as f:\n",
    "                y_pred_ = f.read().split('\\n')[:-1]\n",
    "\n",
    "            y_real, y_pred = [], []\n",
    "            for i, g in enumerate(gold):\n",
    "                if g['category'] in unseen_domains:\n",
    "                    t = [' '.join(target['output']) for target in g['targets']]\n",
    "                    y_real.append(t)\n",
    "\n",
    "                    y_pred.append(y_pred_[i].strip())\n",
    "\n",
    "            num, dem = 0.0, 0\n",
    "            for i, y_ in enumerate(y_pred):\n",
    "                y = y_real[i]\n",
    "                if y_.strip() in y:\n",
    "                    num += 1\n",
    "                dem += 1\n",
    "            print('Task: ', task)\n",
    "            print('Set: ', _set)\n",
    "            print('Model: ', model)\n",
    "            print('Accuracy: ', round(num/dem, 2) if dem > 0 else 0)\n",
    "            print(10 * '-')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Evaluation of Referring Expressions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "All domains:\n",
      "REG Task:\n",
      "Set:  dev\n",
      "Baseline Accuracy:  0.54\n",
      "NeuralREG Accuracy:  0.72\n",
      "----------\n",
      "REG Task:\n",
      "Set:  test\n",
      "Baseline Accuracy:  0.51\n",
      "NeuralREG Accuracy:  0.39\n",
      "----------\n"
     ]
    }
   ],
   "source": [
    "print('All domains:')\n",
    "for _set in ['dev', 'test']:\n",
    "    gold_path=os.path.join(path, 'data', 'reg', _set + '.json')\n",
    "    gold = json.load(open(gold_path))\n",
    "\n",
    "    p = os.path.join(path, 'results/steps/reg', _set + '.out.postprocessed')\n",
    "    with open(p) as f:\n",
    "        y_pred_ = f.read().split('\\n')[:-1]\n",
    "\n",
    "    y_real, y_pred, y_baseline = [], [], []\n",
    "    for i, g in enumerate(gold):\n",
    "        y_real.append(' '.join(g['refex']).strip().lower())\n",
    "        y_pred.append(y_pred_[i].strip().lower())\n",
    "    \n",
    "    num, dem = 0.0, 0\n",
    "    baseline = 0\n",
    "    for i, y_ in enumerate(y_pred):\n",
    "        refex = ' '.join(nltk.word_tokenize(gold[i]['entity'].replace('\\'', ' ').replace('\\\"', ' ').replace('_', ' ')))\n",
    "        y = y_real[i]\n",
    "        if y_.strip() == y:\n",
    "            num += 1\n",
    "        if refex.strip().lower() == y:\n",
    "            baseline += 1\n",
    "        dem += 1\n",
    "    print('REG Task:')\n",
    "    print('Set: ', _set)\n",
    "    print('Baseline Accuracy: ', round(baseline/dem, 2) if dem > 0 else 0)\n",
    "    print('NeuralREG Accuracy: ', round(num/dem, 2) if dem > 0 else 0)\n",
    "    print(10 * '-')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Seen domains:\n",
      "REG Task:\n",
      "Set:  dev\n",
      "Baseline Accuracy:  0.54\n",
      "NeuralREG Accuracy:  0.72\n",
      "----------\n",
      "REG Task:\n",
      "Set:  test\n",
      "Baseline Accuracy:  0.53\n",
      "NeuralREG Accuracy:  0.7\n",
      "----------\n"
     ]
    }
   ],
   "source": [
    "print('Seen domains:')\n",
    "for _set in ['dev', 'test']:\n",
    "    gold_path=os.path.join(path, 'data', 'reg', _set + '.json')\n",
    "    gold = json.load(open(gold_path))\n",
    "\n",
    "    p = os.path.join(path, 'results/steps/reg', _set + '.out.postprocessed')\n",
    "    with open(p) as f:\n",
    "        y_pred_ = f.read().split('\\n')[:-1]\n",
    "\n",
    "    y_real, y_pred, y_baseline = [], [], []\n",
    "    for i, g in enumerate(gold):\n",
    "    #     if g['category'] in unseen_domains:\n",
    "        y_real.append(' '.join(g['refex']).strip().lower())\n",
    "        y_pred.append(y_pred_[i].strip().lower())\n",
    "    \n",
    "    num, dem = 0.0, 0\n",
    "    baseline = 0\n",
    "    for i, y_ in enumerate(y_pred):\n",
    "        if gold[i]['category'] not in unseen_domains:\n",
    "            refex = ' '.join(nltk.word_tokenize(gold[i]['entity'].replace('\\'', ' ').replace('\\\"', ' ').replace('_', ' ')))\n",
    "            y = y_real[i]\n",
    "            if y_.strip() == y:\n",
    "                num += 1\n",
    "            if refex.strip().lower() == y:\n",
    "                baseline += 1\n",
    "            dem += 1\n",
    "    print('REG Task:')\n",
    "    print('Set: ', _set)\n",
    "    print('Baseline Accuracy: ', round(baseline/dem, 2) if dem > 0 else 0)\n",
    "    print('NeuralREG Accuracy: ', round(num/dem, 2) if dem > 0 else 0)\n",
    "    print(10 * '-')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unseen domains:\n",
      "REG Task:\n",
      "Set:  dev\n",
      "Baseline Accuracy:  0\n",
      "NeuralREG Accuracy:  0\n",
      "----------\n",
      "REG Task:\n",
      "Set:  test\n",
      "Baseline Accuracy:  0.5\n",
      "NeuralREG Accuracy:  0.07\n",
      "----------\n"
     ]
    }
   ],
   "source": [
    "print('Unseen domains:')\n",
    "for _set in ['dev', 'test']:\n",
    "    gold_path=os.path.join(path, 'data', 'reg', _set + '.json')\n",
    "    gold = json.load(open(gold_path))\n",
    "\n",
    "    p = os.path.join(path, 'results/steps/reg', _set + '.out.postprocessed')\n",
    "    with open(p) as f:\n",
    "        y_pred_ = f.read().split('\\n')[:-1]\n",
    "\n",
    "    y_real, y_pred, y_baseline = [], [], []\n",
    "    for i, g in enumerate(gold):\n",
    "        y_real.append(' '.join(g['refex']).strip().lower())\n",
    "        y_pred.append(y_pred_[i].strip().lower())\n",
    "    \n",
    "    num, dem = 0.0, 0\n",
    "    baseline = 0\n",
    "    for i, y_ in enumerate(y_pred):\n",
    "        if gold[i]['category'] in unseen_domains:\n",
    "            refex = ' '.join(nltk.word_tokenize(gold[i]['entity'].replace('\\'', ' ').replace('\\\"', ' ').replace('_', ' ')))\n",
    "            y = y_real[i]\n",
    "            if y_.strip() == y:\n",
    "                num += 1\n",
    "            if refex.strip().lower() == y:\n",
    "                baseline += 1\n",
    "            dem += 1\n",
    "    print('REG Task:')\n",
    "    print('Set: ', _set)\n",
    "    print('Baseline Accuracy: ', round(baseline/dem, 2) if dem > 0 else 0)\n",
    "    print('NeuralREG Accuracy: ', round(num/dem, 2) if dem > 0 else 0)\n",
    "    print(10 * '-')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Evaluation of Lexicalization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "All domains:\n",
      "Lexicalization: \n",
      "Set:  dev\n",
      "Model:  random\n",
      "b'BLEU = 40.06, 72.4/46.1/32.0/24.1 (BP=1.000, ratio=1.085, hyp_len=22790, ref_len=21003)\\n'\n",
      "----------\n",
      "Lexicalization: \n",
      "Set:  dev\n",
      "Model:  major\n",
      "b'BLEU = 43.26, 74.8/49.1/35.3/27.0 (BP=1.000, ratio=1.071, hyp_len=22286, ref_len=20813)\\n'\n",
      "----------\n",
      "Lexicalization: \n",
      "Set:  dev\n",
      "Model:  transformer\n",
      "b'BLEU = 48.69, 76.0/54.2/41.3/33.0 (BP=1.000, ratio=1.018, hyp_len=34167, ref_len=33579)\\n'\n",
      "----------\n",
      "Lexicalization: \n",
      "Set:  dev\n",
      "Model:  rnn\n",
      "b'BLEU = 49.71, 77.0/55.1/42.3/34.1 (BP=1.000, ratio=1.006, hyp_len=33998, ref_len=33803)\\n'\n",
      "----------\n",
      "Lexicalization: \n",
      "Set:  test\n",
      "Model:  random\n",
      "b'BLEU = 39.49, 72.6/45.5/31.4/23.4 (BP=1.000, ratio=1.073, hyp_len=29090, ref_len=27122)\\n'\n",
      "----------\n",
      "Lexicalization: \n",
      "Set:  test\n",
      "Model:  major\n",
      "b'BLEU = 44.82, 76.2/50.3/36.9/28.5 (BP=1.000, ratio=1.069, hyp_len=27956, ref_len=26163)\\n'\n",
      "----------\n",
      "Lexicalization: \n",
      "Set:  test\n",
      "Model:  transformer\n",
      "b'BLEU = 38.12, 72.8/46.2/31.6/23.7 (BP=0.956, ratio=0.957, hyp_len=67641, ref_len=70653)\\n'\n",
      "----------\n",
      "Lexicalization: \n",
      "Set:  test\n",
      "Model:  rnn\n",
      "b'BLEU = 37.43, 67.7/43.5/29.8/22.4 (BP=1.000, ratio=1.038, hyp_len=75752, ref_len=73008)\\n'\n",
      "----------\n"
     ]
    }
   ],
   "source": [
    "print('All domains:')\n",
    "for _set in ['dev', 'test']:\n",
    "    for model in ['random', 'major', 'transformer', 'rnn']:\n",
    "        gold_path=os.path.join(path, 'data', 'lexicalization', _set + '.json')\n",
    "        gold = json.load(open(gold_path))\n",
    "\n",
    "        p = os.path.join(path, 'results/steps/lexicalization', model, _set + '.out.postprocessed')\n",
    "        with open(p) as f:\n",
    "            y_pred_ = f.read().split('\\n')[:-1]\n",
    "\n",
    "        y_real, y_pred = [], []\n",
    "        for i, g in enumerate(gold):\n",
    "        #     if g['category'] in unseen_domains:\n",
    "            t = [' '.join(target['output']).lower() for target in g['targets']]\n",
    "            y_real.append(t)\n",
    "            y_pred.append(y_pred_[i].strip().lower())\n",
    "\n",
    "        with open('predictions', 'w') as f:\n",
    "            f.write('\\n'.join(y_pred))\n",
    "\n",
    "        nfiles = max([len(refs) for refs in y_real])\n",
    "        for i in range(nfiles):\n",
    "            with open('reference' + str(i+1), 'w') as f:\n",
    "                for refs in y_real:\n",
    "                    if i < len(refs):\n",
    "                        f.write(refs[i])\n",
    "                    f.write('\\n')\n",
    "\n",
    "        nematus = '/roaming/tcastrof/workspace/nematus/data/multi-bleu.perl'\n",
    "        command = 'perl ' + nematus + ' reference1 reference2 reference3 reference4 reference5 reference6 reference7 reference8 < predictions'\n",
    "        result = subprocess.check_output(command, shell=True)\n",
    "        print('Lexicalization: ')\n",
    "        print('Set: ', _set)\n",
    "        print('Model: ', model)\n",
    "        print(result)\n",
    "        print(10 * '-')\n",
    "\n",
    "        os.remove('reference1')\n",
    "        os.remove('reference2')\n",
    "        os.remove('reference3')\n",
    "        os.remove('reference4')\n",
    "        os.remove('reference5')\n",
    "        os.remove('reference6')\n",
    "        os.remove('reference7')\n",
    "        os.remove('reference8')\n",
    "        os.remove('predictions')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Seen domains:\n",
      "Lexicalization: \n",
      "Set:  dev\n",
      "Model:  random\n",
      "b'BLEU = 40.06, 72.4/46.1/32.0/24.1 (BP=1.000, ratio=1.085, hyp_len=22790, ref_len=21003)\\n'\n",
      "----------\n",
      "Lexicalization: \n",
      "Set:  dev\n",
      "Model:  major\n",
      "b'BLEU = 43.26, 74.8/49.1/35.3/27.0 (BP=1.000, ratio=1.071, hyp_len=22286, ref_len=20813)\\n'\n",
      "----------\n",
      "Lexicalization: \n",
      "Set:  dev\n",
      "Model:  transformer\n",
      "b'BLEU = 48.69, 76.0/54.2/41.3/33.0 (BP=1.000, ratio=1.018, hyp_len=34167, ref_len=33579)\\n'\n",
      "----------\n",
      "Lexicalization: \n",
      "Set:  dev\n",
      "Model:  rnn\n",
      "b'BLEU = 49.71, 77.0/55.1/42.3/34.1 (BP=1.000, ratio=1.006, hyp_len=33998, ref_len=33803)\\n'\n",
      "----------\n",
      "Lexicalization: \n",
      "Set:  test\n",
      "Model:  random\n",
      "b'BLEU = 40.46, 73.1/46.6/32.4/24.2 (BP=1.000, ratio=1.040, hyp_len=24491, ref_len=23559)\\n'\n",
      "----------\n",
      "Lexicalization: \n",
      "Set:  test\n",
      "Model:  major\n",
      "b'BLEU = 45.65, 76.6/51.2/37.8/29.3 (BP=1.000, ratio=1.033, hyp_len=23805, ref_len=23048)\\n'\n",
      "----------\n",
      "Lexicalization: \n",
      "Set:  test\n",
      "Model:  transformer\n",
      "b'BLEU = 48.14, 77.5/54.6/41.4/32.6 (BP=0.985, ratio=0.985, hyp_len=38749, ref_len=39330)\\n'\n",
      "----------\n",
      "Lexicalization: \n",
      "Set:  test\n",
      "Model:  rnn\n",
      "b'BLEU = 49.26, 77.8/55.1/42.0/33.5 (BP=0.994, ratio=0.994, hyp_len=39074, ref_len=39307)\\n'\n",
      "----------\n"
     ]
    }
   ],
   "source": [
    "print('Seen domains:')\n",
    "for _set in ['dev', 'test']:\n",
    "    for model in ['random', 'major', 'transformer', 'rnn']:\n",
    "        gold_path=os.path.join(path, 'data', 'lexicalization', _set + '.json')\n",
    "        gold = json.load(open(gold_path))\n",
    "\n",
    "        p = os.path.join(path, 'results/steps/lexicalization', model, _set + '.out.postprocessed')\n",
    "        with open(p) as f:\n",
    "            y_pred_ = f.read().split('\\n')[:-1]\n",
    "\n",
    "        y_real, y_pred = [], []\n",
    "        for i, g in enumerate(gold):\n",
    "            if g['category'] not in unseen_domains:\n",
    "                t = [' '.join(target['output']).lower() for target in g['targets']]\n",
    "                y_real.append(t)\n",
    "                y_pred.append(y_pred_[i].strip().lower())\n",
    "\n",
    "        with open('predictions', 'w') as f:\n",
    "            f.write('\\n'.join(y_pred))\n",
    "\n",
    "        nfiles = max([len(refs) for refs in y_real])\n",
    "        for i in range(nfiles):\n",
    "            with open('reference' + str(i+1), 'w') as f:\n",
    "                for refs in y_real:\n",
    "                    if i < len(refs):\n",
    "                        f.write(refs[i])\n",
    "                    f.write('\\n')\n",
    "\n",
    "        nematus = '/roaming/tcastrof/workspace/nematus/data/multi-bleu.perl'\n",
    "        command = 'perl ' + nematus + ' reference1 reference2 reference3 reference4 reference5 reference6 reference7 reference8 < predictions'\n",
    "        result = subprocess.check_output(command, shell=True)\n",
    "        print('Lexicalization: ')\n",
    "        print('Set: ', _set)\n",
    "        print('Model: ', model)\n",
    "        print(result)\n",
    "        print(10 * '-')\n",
    "\n",
    "        os.remove('predictions')\n",
    "        os.remove('reference1')\n",
    "        os.remove('reference2')\n",
    "        os.remove('reference3')\n",
    "        os.remove('reference4')\n",
    "        os.remove('reference5')\n",
    "        os.remove('reference6')\n",
    "        os.remove('reference7')\n",
    "        os.remove('reference8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unseen domains:\n",
      "Lexicalization: \n",
      "Set:  test\n",
      "Model:  random\n",
      "b'BLEU = 33.79, 69.8/39.8/25.5/18.4 (BP=1.000, ratio=1.288, hyp_len=4599, ref_len=3572)\\n'\n",
      "----------\n",
      "Lexicalization: \n",
      "Set:  test\n",
      "Model:  major\n",
      "b'BLEU = 39.43, 73.9/44.9/31.4/23.2 (BP=1.000, ratio=1.329, hyp_len=4151, ref_len=3124)\\n'\n",
      "----------\n",
      "Lexicalization: \n",
      "Set:  test\n",
      "Model:  transformer\n",
      "b'BLEU = 24.15, 66.4/34.7/18.2/11.4 (BP=0.919, ratio=0.922, hyp_len=28892, ref_len=31323)\\n'\n",
      "----------\n",
      "Lexicalization: \n",
      "Set:  test\n",
      "Model:  rnn\n",
      "b'BLEU = 23.63, 57.0/31.1/16.8/10.5 (BP=1.000, ratio=1.088, hyp_len=36678, ref_len=33701)\\n'\n",
      "----------\n"
     ]
    }
   ],
   "source": [
    "print('Unseen domains:')\n",
    "for _set in ['dev', 'test']:\n",
    "    for model in ['random', 'major', 'transformer', 'rnn']:\n",
    "        gold_path=os.path.join(path, 'data', 'lexicalization', _set + '.json')\n",
    "        gold = json.load(open(gold_path))\n",
    "\n",
    "        p = os.path.join(path, 'results/steps/lexicalization', model, _set + '.out.postprocessed')\n",
    "        with open(p) as f:\n",
    "            y_pred_ = f.read().split('\\n')[:-1]\n",
    "\n",
    "        y_real, y_pred = [], []\n",
    "        for i, g in enumerate(gold):\n",
    "            if g['category'] in unseen_domains:\n",
    "                t = [' '.join(target['output']).lower() for target in g['targets']]\n",
    "                y_real.append(t)\n",
    "                y_pred.append(y_pred_[i].strip().lower())\n",
    "\n",
    "        with open('predictions', 'w') as f:\n",
    "            f.write('\\n'.join(y_pred))\n",
    "\n",
    "        try:\n",
    "            nfiles = max([len(refs) for refs in y_real])\n",
    "            for i in range(nfiles):\n",
    "                with open('reference' + str(i+1), 'w') as f:\n",
    "                    for refs in y_real:\n",
    "                        if i < len(refs):\n",
    "                            f.write(refs[i])\n",
    "                        f.write('\\n')\n",
    "\n",
    "            nematus = '/roaming/tcastrof/workspace/nematus/data/multi-bleu.perl'\n",
    "            command = 'perl ' + nematus + ' reference1 reference2 reference3 reference4 reference5 reference6 reference7 reference8 < predictions'\n",
    "            result = subprocess.check_output(command, shell=True)\n",
    "            print('Lexicalization: ')\n",
    "            print('Set: ', _set)\n",
    "            print('Model: ', model)\n",
    "            print(result)\n",
    "            print(10 * '-')\n",
    "\n",
    "            os.remove('predictions')\n",
    "            os.remove('reference1')\n",
    "            os.remove('reference2')\n",
    "            os.remove('reference3')\n",
    "            os.remove('reference4')\n",
    "            os.remove('reference5')\n",
    "            os.remove('reference6')\n",
    "            os.remove('reference7')\n",
    "            os.remove('reference8')\n",
    "        except:\n",
    "            pass"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Evaluation of Final Texts (BLEU)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "All domains:\n",
      "Final: \n",
      "Set:  dev\n",
      "Approach: pipeline\n",
      "Model:  rand\n",
      "b'BLEU = 41.01, 75.2/50.1/33.5/22.4 (BP=1.000, ratio=1.132, hyp_len=13392, ref_len=11826)\\n'\n",
      "----------\n",
      "Final: \n",
      "Set:  dev\n",
      "Approach: pipeline\n",
      "Model:  major\n",
      "b'BLEU = 46.76, 79.1/55.9/39.3/27.5 (BP=1.000, ratio=1.039, hyp_len=20297, ref_len=19539)\\n'\n",
      "----------\n",
      "Final: \n",
      "Set:  dev\n",
      "Approach: pipeline\n",
      "Model:  transformer\n",
      "b'BLEU = 57.85, 87.5/67.2/51.2/39.2 (BP=0.987, ratio=0.987, hyp_len=18855, ref_len=19109)\\n'\n",
      "----------\n",
      "Final: \n",
      "Set:  dev\n",
      "Approach: pipeline\n",
      "Model:  rnn\n",
      "b'BLEU = 58.69, 87.1/67.1/51.6/39.8 (BP=0.997, ratio=0.997, hyp_len=19403, ref_len=19464)\\n'\n",
      "----------\n",
      "Final: \n",
      "Set:  dev\n",
      "Approach: end2end\n",
      "Model:  transformer\n",
      "b'BLEU = 55.02, 82.7/62.4/47.8/37.1 (BP=1.000, ratio=1.008, hyp_len=19494, ref_len=19343)\\n'\n",
      "----------\n",
      "Final: \n",
      "Set:  dev\n",
      "Approach: end2end\n",
      "Model:  rnn\n",
      "b'BLEU = 60.19, 85.4/67.6/53.5/42.6 (BP=1.000, ratio=1.000, hyp_len=19389, ref_len=19398)\\n'\n",
      "----------\n",
      "Final: \n",
      "Set:  test\n",
      "Approach: pipeline\n",
      "Model:  rand\n",
      "b'BLEU = 41.68, 76.8/51.1/34.0/22.6 (BP=1.000, ratio=1.133, hyp_len=19903, ref_len=17563)\\n'\n",
      "----------\n",
      "Final: \n",
      "Set:  test\n",
      "Approach: pipeline\n",
      "Model:  major\n",
      "b'BLEU = 43.82, 77.1/52.9/36.3/24.9 (BP=1.000, ratio=1.071, hyp_len=33747, ref_len=31517)\\n'\n",
      "----------\n",
      "Final: \n",
      "Set:  test\n",
      "Approach: pipeline\n",
      "Model:  transformer\n",
      "b'BLEU = 51.68, 83.0/59.5/44.0/32.8 (BP=1.000, ratio=1.026, hyp_len=30942, ref_len=30160)\\n'\n",
      "----------\n",
      "Final: \n",
      "Set:  test\n",
      "Approach: pipeline\n",
      "Model:  rnn\n",
      "b'BLEU = 50.55, 81.3/58.3/43.0/32.1 (BP=1.000, ratio=1.026, hyp_len=32557, ref_len=31718)\\n'\n",
      "----------\n",
      "Final: \n",
      "Set:  test\n",
      "Approach: end2end\n",
      "Model:  transformer\n",
      "b'BLEU = 31.88, 61.1/37.1/26.2/19.1 (BP=0.977, ratio=0.977, hyp_len=38993, ref_len=39898)\\n'\n",
      "----------\n",
      "Final: \n",
      "Set:  test\n",
      "Approach: end2end\n",
      "Model:  rnn\n",
      "b'BLEU = 33.49, 58.1/37.3/27.5/21.1 (BP=1.000, ratio=1.042, hyp_len=42722, ref_len=41009)\\n'\n",
      "----------\n"
     ]
    }
   ],
   "source": [
    "print('All domains:')\n",
    "for _set in ['dev', 'test']:\n",
    "    gold_path=os.path.join(path, 'data', 'end2end', _set + '.json')\n",
    "    gold = json.load(open(gold_path))\n",
    "    for kind in ['pipeline', 'end2end']:\n",
    "        for model in ['rand', 'major', 'transformer', 'rnn']:\n",
    "            if kind == 'end2end' and model in ['rand', 'major']:\n",
    "                continue\n",
    "            else:\n",
    "                p = os.path.join(path, 'results', kind, model, _set + '.out.postprocessed')\n",
    "                with open(p) as f:\n",
    "                    y_pred_ = f.read().split('\\n')[:-1]\n",
    "\n",
    "                y_real, y_pred = [], []\n",
    "                for i, g in enumerate(gold):\n",
    "                #     if g['category'] in unseen_domains:\n",
    "                    targets = [nltk.word_tokenize(' '.join(target['output'])) for target in g['targets']]\n",
    "                    t = [' '.join(target).lower() for target in targets]\n",
    "                    y_real.append(t)\n",
    "                    pred = ' '.join(nltk.word_tokenize(y_pred_[i])).lower()\n",
    "                    y_pred.append(pred)\n",
    "\n",
    "                with open('predictions', 'w') as f:\n",
    "                    f.write('\\n'.join(y_pred))\n",
    "\n",
    "                nfiles = max([len(refs) for refs in y_real])\n",
    "                for i in range(nfiles):\n",
    "                    with open('reference' + str(i+1), 'w') as f:\n",
    "                        for refs in y_real:\n",
    "                            if i < len(refs):\n",
    "                                f.write(refs[i])\n",
    "                            f.write('\\n')\n",
    "\n",
    "                nematus = '/roaming/tcastrof/workspace/nematus/data/multi-bleu.perl'\n",
    "                command = 'perl ' + nematus + ' reference1 reference2 reference3 reference4 reference5 reference6 reference7 reference8 < predictions'\n",
    "                result = subprocess.check_output(command, shell=True)\n",
    "                print('Final: ')\n",
    "                print('Set: ', _set)\n",
    "                print('Approach:', kind)\n",
    "                print('Model: ', model)\n",
    "                print(result)\n",
    "                print(10 * '-')\n",
    "\n",
    "                os.remove('reference1')\n",
    "                os.remove('reference2')\n",
    "                os.remove('reference3')\n",
    "                os.remove('reference4')\n",
    "                os.remove('reference5')\n",
    "                os.remove('reference6')\n",
    "                os.remove('reference7')\n",
    "                os.remove('reference8')\n",
    "                os.remove('predictions')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Seen domains:\n",
      "Final: \n",
      "Set:  dev\n",
      "Approach: pipeline\n",
      "Model:  rand\n",
      "b'BLEU = 41.01, 75.2/50.1/33.5/22.4 (BP=1.000, ratio=1.132, hyp_len=13392, ref_len=11826)\\n'\n",
      "----------\n",
      "Final: \n",
      "Set:  dev\n",
      "Approach: pipeline\n",
      "Model:  major\n",
      "b'BLEU = 46.76, 79.1/55.9/39.3/27.5 (BP=1.000, ratio=1.039, hyp_len=20297, ref_len=19539)\\n'\n",
      "----------\n",
      "Final: \n",
      "Set:  dev\n",
      "Approach: pipeline\n",
      "Model:  transformer\n",
      "b'BLEU = 57.85, 87.5/67.2/51.2/39.2 (BP=0.987, ratio=0.987, hyp_len=18855, ref_len=19109)\\n'\n",
      "----------\n",
      "Final: \n",
      "Set:  dev\n",
      "Approach: pipeline\n",
      "Model:  rnn\n",
      "b'BLEU = 58.69, 87.1/67.1/51.6/39.8 (BP=0.997, ratio=0.997, hyp_len=19403, ref_len=19464)\\n'\n",
      "----------\n",
      "Final: \n",
      "Set:  dev\n",
      "Approach: end2end\n",
      "Model:  transformer\n",
      "b'BLEU = 55.02, 82.7/62.4/47.8/37.1 (BP=1.000, ratio=1.008, hyp_len=19494, ref_len=19343)\\n'\n",
      "----------\n",
      "Final: \n",
      "Set:  dev\n",
      "Approach: end2end\n",
      "Model:  rnn\n",
      "b'BLEU = 60.19, 85.4/67.6/53.5/42.6 (BP=1.000, ratio=1.000, hyp_len=19389, ref_len=19398)\\n'\n",
      "----------\n",
      "Final: \n",
      "Set:  test\n",
      "Approach: pipeline\n",
      "Model:  rand\n",
      "b'BLEU = 41.72, 76.3/50.9/34.0/22.9 (BP=1.000, ratio=1.118, hyp_len=15183, ref_len=13580)\\n'\n",
      "----------\n",
      "Final: \n",
      "Set:  test\n",
      "Approach: pipeline\n",
      "Model:  major\n",
      "b'BLEU = 44.79, 77.0/53.5/37.3/26.2 (BP=1.000, ratio=1.068, hyp_len=24117, ref_len=22572)\\n'\n",
      "----------\n",
      "Final: \n",
      "Set:  test\n",
      "Approach: pipeline\n",
      "Model:  transformer\n",
      "b'BLEU = 56.35, 86.7/65.5/49.4/37.6 (BP=0.989, ratio=0.989, hyp_len=20985, ref_len=21220)\\n'\n",
      "----------\n",
      "Final: \n",
      "Set:  test\n",
      "Approach: pipeline\n",
      "Model:  rnn\n",
      "b'BLEU = 55.75, 86.1/64.6/48.5/36.8 (BP=0.993, ratio=0.993, hyp_len=21523, ref_len=21667)\\n'\n",
      "----------\n",
      "Final: \n",
      "Set:  test\n",
      "Approach: end2end\n",
      "Model:  transformer\n",
      "b'BLEU = 50.79, 80.6/58.7/43.6/32.8 (BP=0.996, ratio=0.996, hyp_len=21854, ref_len=21941)\\n'\n",
      "----------\n",
      "Final: \n",
      "Set:  test\n",
      "Approach: end2end\n",
      "Model:  rnn\n",
      "b'BLEU = 57.20, 83.8/64.8/50.5/39.9 (BP=0.994, ratio=0.994, hyp_len=21815, ref_len=21943)\\n'\n",
      "----------\n"
     ]
    }
   ],
   "source": [
    "print('Seen domains:')\n",
    "for _set in ['dev', 'test']:\n",
    "    gold_path=os.path.join(path, 'data', 'end2end', _set + '.json')\n",
    "    gold = json.load(open(gold_path))\n",
    "    for kind in ['pipeline', 'end2end']:\n",
    "        for model in ['rand', 'major', 'transformer', 'rnn']:\n",
    "            if kind == 'end2end' and model in ['rand', 'major']:\n",
    "                continue\n",
    "            else:\n",
    "                p = os.path.join(path, 'results', kind, model, _set + '.out.postprocessed')\n",
    "                with open(p) as f:\n",
    "                    y_pred_ = f.read().split('\\n')[:-1]\n",
    "\n",
    "                y_real, y_pred = [], []\n",
    "                for i, g in enumerate(gold):\n",
    "                    if g['category'] not in unseen_domains:\n",
    "                        targets = [nltk.word_tokenize(' '.join(target['output'])) for target in g['targets']]\n",
    "                        t = [' '.join(target).lower() for target in targets]\n",
    "                        y_real.append(t)\n",
    "                        pred = ' '.join(nltk.word_tokenize(y_pred_[i])).lower()\n",
    "                        y_pred.append(pred)\n",
    "\n",
    "                with open('predictions', 'w') as f:\n",
    "                    f.write('\\n'.join(y_pred))\n",
    "\n",
    "                nfiles = max([len(refs) for refs in y_real])\n",
    "                for i in range(nfiles):\n",
    "                    with open('reference' + str(i+1), 'w') as f:\n",
    "                        for refs in y_real:\n",
    "                            if i < len(refs):\n",
    "                                f.write(refs[i])\n",
    "                            f.write('\\n')\n",
    "\n",
    "                nematus = '/roaming/tcastrof/workspace/nematus/data/multi-bleu.perl'\n",
    "                command = 'perl ' + nematus + ' reference1 reference2 reference3 reference4 reference5 reference6 reference7 reference8 < predictions'\n",
    "                result = subprocess.check_output(command, shell=True)\n",
    "                print('Final: ')\n",
    "                print('Set: ', _set)\n",
    "                print('Approach:', kind)\n",
    "                print('Model: ', model)\n",
    "                print(result)\n",
    "                print(10 * '-')\n",
    "\n",
    "                os.remove('reference1')\n",
    "                os.remove('reference2')\n",
    "                os.remove('reference3')\n",
    "                os.remove('reference4')\n",
    "                os.remove('reference5')\n",
    "                os.remove('reference6')\n",
    "                os.remove('reference7')\n",
    "                os.remove('reference8')\n",
    "                os.remove('predictions')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unseen domains:\n",
      "Final: \n",
      "Set:  test\n",
      "Approach: pipeline\n",
      "Model:  rand\n",
      "b'BLEU = 41.51, 78.6/51.6/33.9/21.6 (BP=1.000, ratio=1.182, hyp_len=4720, ref_len=3994)\\n'\n",
      "----------\n",
      "Final: \n",
      "Set:  test\n",
      "Approach: pipeline\n",
      "Model:  major\n",
      "b'BLEU = 41.13, 77.3/51.4/33.7/21.4 (BP=1.000, ratio=1.075, hyp_len=9630, ref_len=8956)\\n'\n",
      "----------\n",
      "Final: \n",
      "Set:  test\n",
      "Approach: pipeline\n",
      "Model:  transformer\n",
      "b'BLEU = 38.92, 75.3/46.3/31.5/20.9 (BP=1.000, ratio=1.114, hyp_len=9957, ref_len=8940)\\n'\n",
      "----------\n",
      "Final: \n",
      "Set:  test\n",
      "Approach: pipeline\n",
      "Model:  rnn\n",
      "b'BLEU = 38.55, 71.9/45.5/31.3/21.6 (BP=1.000, ratio=1.098, hyp_len=11034, ref_len=10051)\\n'\n",
      "----------\n",
      "Final: \n",
      "Set:  test\n",
      "Approach: end2end\n",
      "Model:  transformer\n",
      "b'BLEU = 5.88, 36.3/9.3/3.6/1.2 (BP=0.953, ratio=0.954, hyp_len=17139, ref_len=17957)\\n'\n",
      "----------\n",
      "Final: \n",
      "Set:  test\n",
      "Approach: end2end\n",
      "Model:  rnn\n",
      "b'BLEU = 6.24, 31.3/8.7/3.6/1.5 (BP=1.000, ratio=1.097, hyp_len=20907, ref_len=19066)\\n'\n",
      "----------\n"
     ]
    }
   ],
   "source": [
    "print('Unseen domains:')\n",
    "for _set in ['test']:\n",
    "    gold_path=os.path.join(path, 'data', 'end2end', _set + '.json')\n",
    "    gold = json.load(open(gold_path))\n",
    "    for kind in ['pipeline', 'end2end']:\n",
    "        for model in ['rand', 'major', 'transformer', 'rnn']:\n",
    "            if kind == 'end2end' and model in ['rand', 'major']:\n",
    "                continue\n",
    "            else:\n",
    "                p = os.path.join(path, 'results', kind, model, _set + '.out.postprocessed')\n",
    "                with open(p) as f:\n",
    "                    y_pred_ = f.read().split('\\n')[:-1]\n",
    "\n",
    "                y_real, y_pred = [], []\n",
    "                for i, g in enumerate(gold):\n",
    "                    if g['category'] in unseen_domains:\n",
    "                        targets = [nltk.word_tokenize(' '.join(target['output'])) for target in g['targets']]\n",
    "                        t = [' '.join(target).lower() for target in targets]\n",
    "                        y_real.append(t)\n",
    "                        pred = ' '.join(nltk.word_tokenize(y_pred_[i])).lower()\n",
    "                        y_pred.append(pred)\n",
    "\n",
    "                with open('predictions', 'w') as f:\n",
    "                    f.write('\\n'.join(y_pred))\n",
    "\n",
    "                nfiles = max([len(refs) for refs in y_real])\n",
    "                for i in range(nfiles):\n",
    "                    with open('reference' + str(i+1), 'w') as f:\n",
    "                        for refs in y_real:\n",
    "                            if i < len(refs):\n",
    "                                f.write(refs[i])\n",
    "                            f.write('\\n')\n",
    "\n",
    "                nematus = '/roaming/tcastrof/workspace/nematus/data/multi-bleu.perl'\n",
    "                command = 'perl ' + nematus + ' reference1 reference2 reference3 reference4 reference5 reference6 reference7 reference8 < predictions'\n",
    "                result = subprocess.check_output(command, shell=True)\n",
    "                print('Final: ')\n",
    "                print('Set: ', _set)\n",
    "                print('Approach:', kind)\n",
    "                print('Model: ', model)\n",
    "                print(result)\n",
    "                print(10 * '-')\n",
    "\n",
    "                try:\n",
    "                    os.remove('predictions')\n",
    "                    os.remove('reference1')\n",
    "                    os.remove('reference2')\n",
    "                    os.remove('reference3')\n",
    "                    os.remove('reference4')\n",
    "                    os.remove('reference5')\n",
    "                    os.remove('reference6')\n",
    "                    os.remove('reference7')\n",
    "                    os.remove('reference8')\n",
    "                except:\n",
    "                    pass"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Evaluation of Final Texts (METEOR)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "All domains:\n",
      "Final: \n",
      "Set:  dev\n",
      "Approach: pipeline\n",
      "Model:  rand\n",
      "b''\n",
      "----------\n",
      "Final: \n",
      "Set:  dev\n",
      "Approach: pipeline\n",
      "Model:  major\n",
      "b'Final score:            0.41077562572144327'\n",
      "----------\n",
      "Final: \n",
      "Set:  dev\n",
      "Approach: pipeline\n",
      "Model:  transformer\n",
      "b'Final score:            0.4330416533942173'\n",
      "----------\n",
      "Final: \n",
      "Set:  dev\n",
      "Approach: pipeline\n",
      "Model:  rnn\n",
      "b'Final score:            0.44177425982219903'\n",
      "----------\n",
      "Final: \n",
      "Set:  dev\n",
      "Approach: end2end\n",
      "Model:  transformer\n",
      "b'Final score:            0.4134470330967474'\n",
      "----------\n",
      "Final: \n",
      "Set:  dev\n",
      "Approach: end2end\n",
      "Model:  rnn\n",
      "b'Final score:            0.4324570419023144'\n",
      "----------\n",
      "Final: \n",
      "Set:  test\n",
      "Approach: pipeline\n",
      "Model:  rand\n",
      "b'Final score:            0.20375595787202236'\n",
      "----------\n",
      "Final: \n",
      "Set:  test\n",
      "Approach: pipeline\n",
      "Model:  major\n",
      "b'Final score:            0.3279519530149407'\n",
      "----------\n",
      "Final: \n",
      "Set:  test\n",
      "Approach: pipeline\n",
      "Model:  transformer\n",
      "b'Final score:            0.3230984032371677'\n",
      "----------\n",
      "Final: \n",
      "Set:  test\n",
      "Approach: pipeline\n",
      "Model:  rnn\n",
      "b'Final score:            0.3286943151571517'\n",
      "----------\n",
      "Final: \n",
      "Set:  test\n",
      "Approach: end2end\n",
      "Model:  transformer\n",
      "b'Final score:            0.24515982122374186'\n",
      "----------\n",
      "Final: \n",
      "Set:  test\n",
      "Approach: end2end\n",
      "Model:  rnn\n",
      "b'Final score:            0.254004927216645'\n",
      "----------\n"
     ]
    }
   ],
   "source": [
    "print('All domains:')\n",
    "for _set in ['dev', 'test']:\n",
    "    gold_path=os.path.join(path, 'data', 'end2end', _set + '.json')\n",
    "    gold = json.load(open(gold_path))\n",
    "    for kind in ['pipeline', 'end2end']:\n",
    "        for model in ['rand', 'major', 'transformer', 'rnn']:\n",
    "            if kind == 'end2end' and model in ['rand', 'major']:\n",
    "                continue\n",
    "            else:\n",
    "                p = os.path.join(path, 'results', kind, model, _set + '.out.postprocessed')\n",
    "                with open(p) as f:\n",
    "                    y_pred_ = f.read().split('\\n')[:-1]\n",
    "                    \n",
    "                y_real, y_pred = [], []\n",
    "                for i, g in enumerate(gold):\n",
    "#                     if g['category'] in unseen_domains:\n",
    "                    targets = [nltk.word_tokenize(' '.join(target['output'])) for target in g['targets']]\n",
    "                    t = [' '.join(target).lower() for target in targets]\n",
    "                    y_real.append(t)\n",
    "                    pred = ' '.join(nltk.word_tokenize(y_pred_[i])).lower()\n",
    "                    y_pred.append(pred)\n",
    "\n",
    "\n",
    "                with open('predictions', 'w') as f:\n",
    "                    f.write('\\n'.join(y_pred))\n",
    "\n",
    "                with open('reference', 'w') as f:\n",
    "                    for refs in y_real:\n",
    "                        for i in range(8):\n",
    "                            if i < len(refs):\n",
    "                                f.write(refs[i])\n",
    "                            else:\n",
    "                                f.write('')\n",
    "                            f.write('\\n')\n",
    "\n",
    "                java = '/roaming/tcastrof/workspace/java/jre1.8.0_181/bin/java -Xmx2G -jar '\n",
    "                java += '/home/tcastrof/workspace/meteor-1.5/meteor-1.5.jar predictions reference -l en -norm -r 8'\n",
    "                result = subprocess.check_output(java, shell=True)\n",
    "                print('Final: ')\n",
    "                print('Set: ', _set)\n",
    "                print('Approach:', kind)\n",
    "                print('Model: ', model)\n",
    "                print(result.split(b'\\n')[-2])\n",
    "                print(10 * '-')\n",
    "\n",
    "                os.remove('reference')\n",
    "                os.remove('predictions')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Seen domains:\n",
      "Final: \n",
      "Set:  dev\n",
      "Approach: pipeline\n",
      "Model:  rand\n",
      "b''\n",
      "----------\n",
      "Final: \n",
      "Set:  dev\n",
      "Approach: pipeline\n",
      "Model:  major\n",
      "b'Final score:            0.41077562572144327'\n",
      "----------\n",
      "Final: \n",
      "Set:  dev\n",
      "Approach: pipeline\n",
      "Model:  transformer\n",
      "b'Final score:            0.4330416533942173'\n",
      "----------\n",
      "Final: \n",
      "Set:  dev\n",
      "Approach: pipeline\n",
      "Model:  rnn\n",
      "b'Final score:            0.44177425982219903'\n",
      "----------\n",
      "Final: \n",
      "Set:  dev\n",
      "Approach: end2end\n",
      "Model:  transformer\n",
      "b'Final score:            0.4134470330967474'\n",
      "----------\n",
      "Final: \n",
      "Set:  dev\n",
      "Approach: end2end\n",
      "Model:  rnn\n",
      "b'Final score:            0.4324570419023144'\n",
      "----------\n",
      "Final: \n",
      "Set:  test\n",
      "Approach: pipeline\n",
      "Model:  rand\n",
      "b'Final score:            0.27316371692277963'\n",
      "----------\n",
      "Final: \n",
      "Set:  test\n",
      "Approach: pipeline\n",
      "Model:  major\n",
      "b'Final score:            0.4122845738480068'\n",
      "----------\n",
      "Final: \n",
      "Set:  test\n",
      "Approach: pipeline\n",
      "Model:  transformer\n",
      "b'Final score:            0.4144337367285661'\n",
      "----------\n",
      "Final: \n",
      "Set:  test\n",
      "Approach: pipeline\n",
      "Model:  rnn\n",
      "b'Final score:            0.42014805234215746'\n",
      "----------\n",
      "Final: \n",
      "Set:  test\n",
      "Approach: end2end\n",
      "Model:  transformer\n",
      "b'Final score:            0.38962397643540064'\n",
      "----------\n",
      "Final: \n",
      "Set:  test\n",
      "Approach: end2end\n",
      "Model:  rnn\n",
      "b'Final score:            0.41208938673817913'\n",
      "----------\n"
     ]
    }
   ],
   "source": [
    "print('Seen domains:')\n",
    "for _set in ['dev', 'test']:\n",
    "    gold_path=os.path.join(path, 'data', 'end2end', _set + '.json')\n",
    "    gold = json.load(open(gold_path))\n",
    "    for kind in ['pipeline', 'end2end']:\n",
    "        for model in ['rand', 'major', 'transformer', 'rnn']:\n",
    "            if kind == 'end2end' and model in ['rand', 'major']:\n",
    "                continue\n",
    "            else:\n",
    "                p = os.path.join(path, 'results', kind, model, _set + '.out.postprocessed')\n",
    "                with open(p) as f:\n",
    "                    y_pred_ = f.read().split('\\n')[:-1]\n",
    "                    \n",
    "                y_real, y_pred = [], []\n",
    "                for i, g in enumerate(gold):\n",
    "                    if g['category'] not in unseen_domains:\n",
    "                        targets = [nltk.word_tokenize(' '.join(target['output'])) for target in g['targets']]\n",
    "                        t = [' '.join(target).lower() for target in targets]\n",
    "                        y_real.append(t)\n",
    "                        pred = ' '.join(nltk.word_tokenize(y_pred_[i])).lower()\n",
    "                        y_pred.append(pred)\n",
    "\n",
    "\n",
    "                with open('predictions', 'w') as f:\n",
    "                    f.write('\\n'.join(y_pred))\n",
    "\n",
    "                with open('reference', 'w') as f:\n",
    "                    for refs in y_real:\n",
    "                        for i in range(8):\n",
    "                            if i < len(refs):\n",
    "                                f.write(refs[i])\n",
    "                            else:\n",
    "                                f.write('')\n",
    "                            f.write('\\n')\n",
    "\n",
    "                java = '/roaming/tcastrof/workspace/java/jre1.8.0_181/bin/java -Xmx2G -jar '\n",
    "                java += '/home/tcastrof/workspace/meteor-1.5/meteor-1.5.jar predictions reference -l en -norm -r 8'\n",
    "                result = subprocess.check_output(java, shell=True)\n",
    "                print('Final: ')\n",
    "                print('Set: ', _set)\n",
    "                print('Approach:', kind)\n",
    "                print('Model: ', model)\n",
    "                print(result.split(b'\\n')[-2])\n",
    "                print(10 * '-')\n",
    "\n",
    "                os.remove('reference')\n",
    "                os.remove('predictions')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unseen domains:\n",
      "Final: \n",
      "Set:  test\n",
      "Approach: pipeline\n",
      "Model:  rand\n",
      "b''\n",
      "----------\n",
      "Final: \n",
      "Set:  test\n",
      "Approach: pipeline\n",
      "Model:  major\n",
      "b'Final score:            0.2193301666864556'\n",
      "----------\n",
      "Final: \n",
      "Set:  test\n",
      "Approach: pipeline\n",
      "Model:  transformer\n",
      "b'Final score:            0.20948928354092697'\n",
      "----------\n",
      "Final: \n",
      "Set:  test\n",
      "Approach: pipeline\n",
      "Model:  rnn\n",
      "b'Final score:            0.21778297038185643'\n",
      "----------\n",
      "Final: \n",
      "Set:  test\n",
      "Approach: end2end\n",
      "Model:  transformer\n",
      "b'Final score:            0.08846673176674488'\n",
      "----------\n",
      "Final: \n",
      "Set:  test\n",
      "Approach: end2end\n",
      "Model:  rnn\n",
      "b'Final score:            0.08872436586936241'\n",
      "----------\n"
     ]
    }
   ],
   "source": [
    "print('Unseen domains:')\n",
    "for _set in ['test']:\n",
    "    gold_path=os.path.join(path, 'data', 'end2end', _set + '.json')\n",
    "    gold = json.load(open(gold_path))\n",
    "    for kind in ['pipeline', 'end2end']:\n",
    "        for model in ['rand', 'major', 'transformer', 'rnn']:\n",
    "            if kind == 'end2end' and model in ['rand', 'major']:\n",
    "                continue\n",
    "            else:\n",
    "                p = os.path.join(path, 'results', kind, model, _set + '.out.postprocessed')\n",
    "                with open(p) as f:\n",
    "                    y_pred_ = f.read().split('\\n')[:-1]\n",
    "                    \n",
    "                y_real, y_pred = [], []\n",
    "                for i, g in enumerate(gold):\n",
    "                    if g['category'] in unseen_domains:\n",
    "                        targets = [nltk.word_tokenize(' '.join(target['output'])) for target in g['targets']]\n",
    "                        t = [' '.join(target).lower() for target in targets]\n",
    "                        y_real.append(t)\n",
    "                        pred = ' '.join(nltk.word_tokenize(y_pred_[i])).lower()\n",
    "                        y_pred.append(pred)\n",
    "\n",
    "\n",
    "                with open('predictions', 'w') as f:\n",
    "                    f.write('\\n'.join(y_pred))\n",
    "\n",
    "                with open('reference', 'w') as f:\n",
    "                    for refs in y_real:\n",
    "                        for i in range(8):\n",
    "                            if i < len(refs):\n",
    "                                f.write(refs[i])\n",
    "                            else:\n",
    "                                f.write('')\n",
    "                            f.write('\\n')\n",
    "\n",
    "                java = '/roaming/tcastrof/workspace/java/jre1.8.0_181/bin/java -Xmx2G -jar '\n",
    "                java += '/home/tcastrof/workspace/meteor-1.5/meteor-1.5.jar predictions reference -l en -norm -r 8'\n",
    "                result = subprocess.check_output(java, shell=True)\n",
    "                print('Final: ')\n",
    "                print('Set: ', _set)\n",
    "                print('Approach:', kind)\n",
    "                print('Model: ', model)\n",
    "                print(result.split(b'\\n')[-2])\n",
    "                print(10 * '-')\n",
    "\n",
    "                os.remove('reference')\n",
    "                os.remove('predictions')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Evaluation of Final Texts (Fluency and Semantic)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import scipy.stats\n",
    "\n",
    "\n",
    "def mean_confidence_interval(data, confidence=0.95):\n",
    "    a = 1.0 * np.array(data)\n",
    "    n = len(a)\n",
    "    m, se = np.mean(a), scipy.stats.sem(a)\n",
    "    h = se * scipy.stats.t.ppf((1 + confidence) / 2., n-1)\n",
    "    return m, h, m-h, m+h\n",
    "\n",
    "path = 'evaluation/human/grades.json'\n",
    "grades = json.load(open(path))\n",
    "\n",
    "path = 'evaluation/human/participants.json'\n",
    "participants = json.load(open(path))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of participants:  35\n",
      "***Gender:***\n",
      "Counter({'M': 21, 'F': 14})\n",
      "***English Proficiency Level:***\n",
      "Counter({'native': 18, 'fluent': 17})\n",
      "***Age:***\n",
      "32.29\n",
      "\n",
      "\n",
      "All Domains\n",
      "Fluency: \n",
      "rand: 4.55 +-0.27\n",
      "major: 5.0 +-0.23\n",
      "rnn: 5.31 +-0.23\n",
      "transformer: 5.03 +-0.25\n",
      "e2ernn: 4.73 +-0.24\n",
      "e2etransformer: 5.02 +-0.25\n",
      "melbourne: 5.04 +-0.22\n",
      "upfforge: 5.46 +-0.19\n",
      "original: 5.76 +-0.17\n",
      "\n",
      "\n",
      "Semantics: \n",
      "rand: 4.44 +-0.29\n",
      "major: 5.02 +-0.24\n",
      "rnn: 5.21 +-0.23\n",
      "transformer: 4.87 +-0.27\n",
      "e2ernn: 4.47 +-0.26\n",
      "e2etransformer: 4.7 +-0.27\n",
      "melbourne: 4.94 +-0.24\n",
      "upfforge: 5.31 +-0.21\n",
      "original: 5.74 +-0.18\n"
     ]
    }
   ],
   "source": [
    "from collections import Counter\n",
    "grades = json.load(open('evaluation/human/ngrades.json'))\n",
    "participant_ids = set([w['participant_id'] for w in grades])\n",
    "print('Number of participants: ', len(participant_ids))\n",
    "\n",
    "participants = [p for p in participants if p['id'] in participant_ids]\n",
    "\n",
    "print('***Gender:***')\n",
    "print(Counter([p['gender'] for p in participants]))\n",
    "print('***English Proficiency Level:***')\n",
    "print(Counter([p['english_proficiency_level'] for p in participants]))\n",
    "print('***Age:***')\n",
    "print(round(np.mean([int(p['age']) for p in participants]), 2))\n",
    "print('\\n')\n",
    "    \n",
    "print('All Domains')\n",
    "models = set([g['model'] for g in grades])\n",
    "print('Fluency: ')\n",
    "for model in ['rand', 'major', 'rnn', 'transformer', 'e2ernn', 'e2etransformer', 'melbourne', 'upfforge', 'original']:\n",
    "    fluency = [float(g['fluency']) for g in grades if g['model'] == model]# and g['category'] in unseen_domains]\n",
    "    print('{0}: {1} +-{2}'.format(model, round(np.mean(fluency), 2), round(mean_confidence_interval(fluency)[1], 2)))\n",
    "print('\\n')\n",
    "print('Semantics: ')\n",
    "for model in ['rand', 'major', 'rnn', 'transformer', 'e2ernn', 'e2etransformer', 'melbourne', 'upfforge', 'original']:\n",
    "    semantic = [float(g['semantic']) for g in grades if g['model'] == model]# and g['category'] in unseen_domains]\n",
    "    print('{0}: {1} +-{2}'.format(model, round(np.mean(semantic), 2), round(mean_confidence_interval(semantic)[1], 2)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Seen Domains\n",
      "Fluency: \n",
      "rand: 4.79 +-0.31\n",
      "major: 5.25 +-0.24\n",
      "rnn: 5.51 +-0.25\n",
      "transformer: 5.53 +-0.24\n",
      "e2ernn: 5.4 +-0.23\n",
      "e2etransformer: 5.38 +-0.26\n",
      "melbourne: 5.23 +-0.27\n",
      "upfforge: 5.43 +-0.22\n",
      "original: 5.82 +-0.2\n",
      "\n",
      "\n",
      "Semantics: \n",
      "rand: 4.73 +-0.34\n",
      "major: 5.41 +-0.24\n",
      "rnn: 5.48 +-0.25\n",
      "transformer: 5.49 +-0.27\n",
      "e2ernn: 5.21 +-0.25\n",
      "e2etransformer: 5.15 +-0.29\n",
      "melbourne: 5.33 +-0.28\n",
      "upfforge: 5.35 +-0.25\n",
      "original: 5.8 +-0.2\n"
     ]
    }
   ],
   "source": [
    "print('Seen Domains')\n",
    "models = set([g['model'] for g in grades])\n",
    "print('Fluency: ')\n",
    "for model in ['rand', 'major', 'rnn', 'transformer', 'e2ernn', 'e2etransformer', 'melbourne', 'upfforge', 'original']:\n",
    "    fluency = [float(g['fluency']) for g in grades if g['model'] == model and g['category'] not in unseen_domains]\n",
    "    print('{0}: {1} +-{2}'.format(model, round(np.mean(fluency), 2), round(mean_confidence_interval(fluency)[1], 2)))\n",
    "print('\\n')\n",
    "print('Semantics: ')\n",
    "for model in ['rand', 'major', 'rnn', 'transformer', 'e2ernn', 'e2etransformer', 'melbourne', 'upfforge', 'original']:\n",
    "    semantic = [float(g['semantic']) for g in grades if g['model'] == model and g['category'] not in unseen_domains]\n",
    "    print('{0}: {1} +-{2}'.format(model, round(np.mean(semantic), 2), round(mean_confidence_interval(semantic)[1], 2)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unseen Domains\n",
      "Fluency: \n",
      "rand: 4.07 +-0.53\n",
      "major: 4.49 +-0.48\n",
      "rnn: 4.91 +-0.48\n",
      "transformer: 4.05 +-0.5\n",
      "e2ernn: 3.45 +-0.45\n",
      "e2etransformer: 4.32 +-0.5\n",
      "melbourne: 4.65 +-0.39\n",
      "upfforge: 5.51 +-0.35\n",
      "original: 5.63 +-0.32\n",
      "\n",
      "\n",
      "Semantics: \n",
      "rand: 3.86 +-0.54\n",
      "major: 4.25 +-0.49\n",
      "rnn: 4.67 +-0.47\n",
      "transformer: 3.64 +-0.49\n",
      "e2ernn: 3.03 +-0.44\n",
      "e2etransformer: 3.81 +-0.54\n",
      "melbourne: 4.15 +-0.43\n",
      "upfforge: 5.24 +-0.4\n",
      "original: 5.63 +-0.34\n"
     ]
    }
   ],
   "source": [
    "print('Unseen Domains')\n",
    "models = set([g['model'] for g in grades])\n",
    "print('Fluency: ')\n",
    "for model in ['rand', 'major', 'rnn', 'transformer', 'e2ernn', 'e2etransformer', 'melbourne', 'upfforge', 'original']:\n",
    "    fluency = [float(g['fluency']) for g in grades if g['model'] == model and g['category'] in unseen_domains]\n",
    "    print('{0}: {1} +-{2}'.format(model, round(np.mean(fluency), 2), round(mean_confidence_interval(fluency)[1], 2)))\n",
    "print('\\n')\n",
    "print('Semantics: ')\n",
    "for model in ['rand', 'major', 'rnn', 'transformer', 'e2ernn', 'e2etransformer', 'melbourne', 'upfforge', 'original']:\n",
    "    semantic = [float(g['semantic']) for g in grades if g['model'] == model and g['category'] in unseen_domains]\n",
    "    print('{0}: {1} +-{2}'.format(model, round(np.mean(semantic), 2), round(mean_confidence_interval(semantic)[1], 2)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "rand x major : True\n",
      "rand x rnn : True\n",
      "rand x transformer : True\n",
      "rand x e2ernn : False\n",
      "rand x e2etransformer : False\n",
      "rand x melbourne : True\n",
      "rand x upfforge : True\n",
      "rand x original : True\n",
      "major x rand : True\n",
      "major x rnn : False\n",
      "major x transformer : False\n",
      "major x e2ernn : True\n",
      "major x e2etransformer : False\n",
      "major x melbourne : False\n",
      "major x upfforge : False\n",
      "major x original : True\n",
      "rnn x rand : True\n",
      "rnn x major : False\n",
      "rnn x transformer : False\n",
      "rnn x e2ernn : True\n",
      "rnn x e2etransformer : True\n",
      "rnn x melbourne : False\n",
      "rnn x upfforge : False\n",
      "rnn x original : True\n",
      "transformer x rand : True\n",
      "transformer x major : False\n",
      "transformer x rnn : False\n",
      "transformer x e2ernn : True\n",
      "transformer x e2etransformer : False\n",
      "transformer x melbourne : False\n",
      "transformer x upfforge : True\n",
      "transformer x original : True\n",
      "e2ernn x rand : False\n",
      "e2ernn x major : True\n",
      "e2ernn x rnn : True\n",
      "e2ernn x transformer : True\n",
      "e2ernn x e2etransformer : False\n",
      "e2ernn x melbourne : True\n",
      "e2ernn x upfforge : True\n",
      "e2ernn x original : True\n",
      "e2etransformer x rand : False\n",
      "e2etransformer x major : False\n",
      "e2etransformer x rnn : True\n",
      "e2etransformer x transformer : False\n",
      "e2etransformer x e2ernn : False\n",
      "e2etransformer x melbourne : False\n",
      "e2etransformer x upfforge : True\n",
      "e2etransformer x original : True\n",
      "melbourne x rand : True\n",
      "melbourne x major : False\n",
      "melbourne x rnn : False\n",
      "melbourne x transformer : False\n",
      "melbourne x e2ernn : True\n",
      "melbourne x e2etransformer : False\n",
      "melbourne x upfforge : True\n",
      "melbourne x original : True\n",
      "upfforge x rand : True\n",
      "upfforge x major : False\n",
      "upfforge x rnn : False\n",
      "upfforge x transformer : True\n",
      "upfforge x e2ernn : True\n",
      "upfforge x e2etransformer : True\n",
      "upfforge x melbourne : True\n",
      "upfforge x original : True\n",
      "original x rand : True\n",
      "original x major : True\n",
      "original x rnn : True\n",
      "original x transformer : True\n",
      "original x e2ernn : True\n",
      "original x e2etransformer : True\n",
      "original x melbourne : True\n",
      "original x upfforge : True\n"
     ]
    }
   ],
   "source": [
    "from scipy.stats import mannwhitneyu, wilcoxon\n",
    "\n",
    "models = ['rand', 'major', 'rnn', 'transformer', 'e2ernn', 'e2etransformer', 'melbourne', 'upfforge', 'original']\n",
    "for i, model1 in enumerate(models):\n",
    "    for j, model2 in enumerate(models):\n",
    "        if model1 != model2:\n",
    "            fluency1 = [float(g['semantic']) for g in grades if g['model'] == model1]# and g['category'] in unseen_domains]\n",
    "            fluency2 = [float(g['semantic']) for g in grades if g['model'] == model2]# and g['category'] in unseen_domains]\n",
    "            print(model1, 'x', model2, ':', round(mannwhitneyu(fluency1, fluency2)[1], 2) < 0.05)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Evaluation of Annotations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "gold_path='evaluation/questionaire/trials/gold.json'\n",
    "gold = json.load(open(gold_path))\n",
    "\n",
    "path='evaluation/questionaire/annotations.json'\n",
    "annotations = json.load(open(path))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model:  models\\model1.xml\n",
      "Structured followed in 0.35 of the cases\n",
      "More information in 0.53 of the cases\n",
      "Exact number of predicates in 29 out of 75 of the cases (0.39)\n",
      "Reference mistakes in 0.21 of the cases\n",
      "Verb mistakes in 0.05 of the cases\n",
      "Determiner mistakes in 0.03 of the cases\n",
      "Fluency: 5.626666666666667\n",
      "Semantics: 3.52\n",
      "----------\n",
      "Model:  models\\model2.xml\n",
      "Structured followed in 0.33 of the cases\n",
      "More information in 0.41 of the cases\n",
      "Exact number of predicates in 35 out of 75 of the cases (0.47)\n",
      "Reference mistakes in 0.09 of the cases\n",
      "Verb mistakes in 0.03 of the cases\n",
      "Determiner mistakes in 0.0 of the cases\n",
      "Fluency: 4.546666666666667\n",
      "Semantics: 3.973333333333333\n",
      "----------\n",
      "Model:  models\\model3.xml\n",
      "Structured followed in 0.89 of the cases\n",
      "More information in 0.09 of the cases\n",
      "Exact number of predicates in 54 out of 75 of the cases (0.72)\n",
      "Reference mistakes in 0.28 of the cases\n",
      "Verb mistakes in 0.12 of the cases\n",
      "Determiner mistakes in 0.09 of the cases\n",
      "Fluency: 5.746666666666667\n",
      "Semantics: 5.64\n",
      "----------\n",
      "Model:  models\\model5.xml\n",
      "Structured followed in 0.41 of the cases\n",
      "More information in 0.05 of the cases\n",
      "Exact number of predicates in 32 out of 75 of the cases (0.43)\n",
      "Reference mistakes in 0.11 of the cases\n",
      "Verb mistakes in 0.05 of the cases\n",
      "Determiner mistakes in 0.09 of the cases\n",
      "Fluency: 3.7466666666666666\n",
      "Semantics: 3.3466666666666667\n",
      "----------\n",
      "Model:  models\\model7.xml\n",
      "Structured followed in 0.16 of the cases\n",
      "More information in 0.28 of the cases\n",
      "Exact number of predicates in 17 out of 25 of the cases (0.68)\n",
      "Reference mistakes in 0.4 of the cases\n",
      "Verb mistakes in 0.04 of the cases\n",
      "Determiner mistakes in 0.16 of the cases\n",
      "Fluency: 4.2\n",
      "Semantics: 3.92\n",
      "----------\n",
      "Model:  model1.xml\n",
      "Structured followed in 0.33 of the cases\n",
      "More information in 0.36 of the cases\n",
      "Exact number of predicates in 30 out of 75 of the cases (0.4)\n",
      "Reference mistakes in 0.13 of the cases\n",
      "Verb mistakes in 0.03 of the cases\n",
      "Determiner mistakes in 0.0 of the cases\n",
      "Fluency: 6.053333333333334\n",
      "Semantics: 3.6\n",
      "----------\n",
      "Model:  model3.xml\n",
      "Structured followed in 0.8 of the cases\n",
      "More information in 0.08 of the cases\n",
      "Exact number of predicates in 51 out of 75 of the cases (0.68)\n",
      "Reference mistakes in 0.07 of the cases\n",
      "Verb mistakes in 0.05 of the cases\n",
      "Determiner mistakes in 0.0 of the cases\n",
      "Fluency: 6.066666666666666\n",
      "Semantics: 5.253333333333333\n",
      "----------\n",
      "Model:  model4.xml\n",
      "Structured followed in 0.81 of the cases\n",
      "More information in 0.01 of the cases\n",
      "Exact number of predicates in 50 out of 75 of the cases (0.67)\n",
      "Reference mistakes in 0.2 of the cases\n",
      "Verb mistakes in 0.0 of the cases\n",
      "Determiner mistakes in 0.01 of the cases\n",
      "Fluency: 6.013333333333334\n",
      "Semantics: 4.933333333333334\n",
      "----------\n",
      "Model:  model6.xml\n",
      "Structured followed in 0.69 of the cases\n",
      "More information in 0.01 of the cases\n",
      "Exact number of predicates in 56 out of 75 of the cases (0.75)\n",
      "Reference mistakes in 0.01 of the cases\n",
      "Verb mistakes in 0.0 of the cases\n",
      "Determiner mistakes in 0.0 of the cases\n",
      "Fluency: 5.92\n",
      "Semantics: 5.413333333333333\n",
      "----------\n",
      "Model:  model8.xml\n",
      "Structured followed in 0.29 of the cases\n",
      "More information in 0.0 of the cases\n",
      "Exact number of predicates in 68 out of 75 of the cases (0.91)\n",
      "Reference mistakes in 0.0 of the cases\n",
      "Verb mistakes in 0.0 of the cases\n",
      "Determiner mistakes in 0.0 of the cases\n",
      "Fluency: 6.386666666666667\n",
      "Semantics: 6.48\n",
      "----------\n",
      "Model:  models\\model9.xml\n",
      "Structured followed in 0.39 of the cases\n",
      "More information in 0.12 of the cases\n",
      "Exact number of predicates in 74 out of 75 of the cases (0.99)\n",
      "Reference mistakes in 0.08 of the cases\n",
      "Verb mistakes in 0.05 of the cases\n",
      "Determiner mistakes in 0.05 of the cases\n",
      "Fluency: 6.8133333333333335\n",
      "Semantics: 6.866666666666666\n",
      "----------\n"
     ]
    }
   ],
   "source": [
    "for model in annotations:\n",
    "    values = {\n",
    "        'detmistake': 0,\n",
    "        'fluency': [],\n",
    "        'moreinformation': 0,\n",
    "        'numpreds': 0,\n",
    "        'referencemistake': 0,\n",
    "        'semantics': [],\n",
    "        'structurefollowed': 0,\n",
    "        'verbmistake': 0\n",
    "    }\n",
    "    dem = 0\n",
    "    for trial in annotations[model]:\n",
    "        g = [g for g in gold if g['eid'] == trial][0]\n",
    "        category = g['category']\n",
    "        size = g['size']\n",
    "#         if category in unseen_domains:\n",
    "        if annotations[model][trial]['structurefollowed']:\n",
    "            values['structurefollowed'] += 1\n",
    "        if annotations[model][trial]['moreinformation']:\n",
    "            values['moreinformation'] += 1\n",
    "        if annotations[model][trial]['referencemistake']:\n",
    "            values['referencemistake'] += 1\n",
    "        if annotations[model][trial]['verbmistake']:\n",
    "            values['verbmistake'] += 1\n",
    "        if annotations[model][trial]['detmistake']:\n",
    "            values['detmistake'] += 1\n",
    "        if int(size) == int(annotations[model][trial]['numpreds']):\n",
    "            values['numpreds'] += 1\n",
    "        dem += 1\n",
    "        values['fluency'].append(float(annotations[model][trial]['fluency']))\n",
    "        values['semantics'].append(float(annotations[model][trial]['semantics']))\n",
    "    values['structurefollowed'] /= dem\n",
    "    values['moreinformation'] /= dem\n",
    "    values['referencemistake'] /= dem\n",
    "    values['verbmistake'] /= dem\n",
    "    values['detmistake'] /= dem\n",
    "\n",
    "    print('Model: ', model)\n",
    "    print('Structured followed in {0} of the cases'.format(round(values['structurefollowed'], 2)))\n",
    "    print('More information in {0} of the cases'.format(round(values['moreinformation'], 2)))\n",
    "    print('Exact number of predicates in {0} out of {1} of the cases ({2})'.format(values['numpreds'], dem, round(values['numpreds']/dem, 2)))\n",
    "    print('Reference mistakes in {0} of the cases'.format(round(values['referencemistake'], 2)))\n",
    "    print('Verb mistakes in {0} of the cases'.format(round(values['verbmistake'], 2)))\n",
    "    print('Determiner mistakes in {0} of the cases'.format(round(values['detmistake'], 2)))\n",
    "    print('Fluency: {0}'.format(np.mean(values['fluency'])))\n",
    "    print('Semantics: {0}'.format(np.mean(values['semantics'])))\n",
    "    print(10 * '-')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Inter-annotator Agreement"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fluency:\n",
      "\n",
      "Weighted kappa as per NLTK:\t 0.261904761904762 \n",
      "Regular kappa as per NLTK:\t 0.2619047619047618 \n",
      "Krippendorff alpha as per NLTK:\t 0.2596645367412139 \n",
      "===========================================\n",
      "\n",
      "Semantic:\n",
      "\n",
      "Weighted kappa as per NLTK:\t 0.4459338695263628 \n",
      "Regular kappa as per NLTK:\t 0.4459338695263628 \n",
      "Krippendorff alpha as per NLTK:\t 0.44588252876998946 \n",
      "===========================================\n",
      "\n",
      "Predicates:\n",
      "\n",
      "Weighted kappa as per NLTK:\t 0.7042878265155249 \n",
      "Regular kappa as per NLTK:\t 0.7042878265155249 \n",
      "Krippendorff alpha as per NLTK:\t 0.7052170340955771 \n",
      "===========================================\n",
      "\n",
      "Structure Followed:\n",
      "\n",
      "Weighted kappa as per NLTK:\t 0.7796143250688705 \n",
      "Regular kappa as per NLTK:\t 0.7796143250688705 \n",
      "Krippendorff alpha as per NLTK:\t 0.7797016025050654 \n",
      "===========================================\n",
      "\n",
      "Overgeneration:\n",
      "\n",
      "Weighted kappa as per NLTK:\t 0.5279838165879973 \n",
      "Regular kappa as per NLTK:\t 0.5279838165879974 \n",
      "Krippendorff alpha as per NLTK:\t 0.5243181818181818 \n",
      "===========================================\n",
      "\n",
      "Verb mistakes:\n",
      "\n",
      "Weighted kappa as per NLTK:\t 0.6102449888641426 \n",
      "Regular kappa as per NLTK:\t 0.6102449888641432 \n",
      "Krippendorff alpha as per NLTK:\t 0.6079790222888182 \n",
      "===========================================\n",
      "\n",
      "Determiner mistakes:\n",
      "\n",
      "Weighted kappa as per NLTK:\t 0.0 \n",
      "Regular kappa as per NLTK:\t 0.0 \n",
      "Krippendorff alpha as per NLTK:\t -0.027491408934708028 \n",
      "===========================================\n",
      "\n",
      "Reference mistakes:\n",
      "\n",
      "Weighted kappa as per NLTK:\t 0.28251121076233177 \n",
      "Regular kappa as per NLTK:\t 0.282511210762332 \n",
      "Krippendorff alpha as per NLTK:\t 0.25806451612903225 \n",
      "===========================================\n",
      "\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "from nltk.metrics.agreement import AnnotationTask\n",
    "\n",
    "def kappa(obs):\n",
    "    t = AnnotationTask(obs)\n",
    "    print(\"\\nWeighted kappa as per NLTK:\\t\", t.weighted_kappa(),\n",
    "          \"\\nRegular kappa as per NLTK:\\t\", t.kappa(),\n",
    "          \"\\nKrippendorff alpha as per NLTK:\\t\", t.alpha(),\n",
    "          \"\\n===========================================\\n\")\n",
    "\n",
    "path='evaluation/questionaire/observations/'\n",
    "if not os.path.exists(path):\n",
    "    os.mkdir(path)\n",
    "    \n",
    "ann1_model1 = annotations['models\\\\model1.xml']\n",
    "ann2_model1 = annotations['model1.xml']\n",
    "ann1_model3 = annotations['models\\\\model3.xml']\n",
    "ann2_model3 = annotations['model3.xml']\n",
    "\n",
    "obs_fluency = []\n",
    "for trial in ann1_model1:\n",
    "    obs_fluency.append(('ann1', 'model1_'+trial, int(ann1_model1[trial]['fluency'])))\n",
    "for trial in ann1_model3:\n",
    "    obs_fluency.append(('ann1', 'model3_'+trial, int(ann1_model3[trial]['fluency'])))\n",
    "for trial in ann2_model1:\n",
    "    obs_fluency.append(('ann2', 'model1_'+trial, int(ann2_model1[trial]['fluency'])))\n",
    "for trial in ann2_model3:\n",
    "    obs_fluency.append(('ann2', 'model3_'+trial, int(ann2_model3[trial]['fluency'])))\n",
    "\n",
    "print('Fluency:')\n",
    "kappa(obs_fluency)\n",
    "\n",
    "with open(os.path.join(path, 'fluency.csv'), 'w') as f:\n",
    "    for i, o in enumerate(obs_fluency):\n",
    "        obs_fluency[i] = list(obs_fluency[i])\n",
    "        obs_fluency[i][2] = str(obs_fluency[i][2])\n",
    "        obs_fluency[i] = ','.join(obs_fluency[i])\n",
    "    f.write('\\n'.join(obs_fluency))\n",
    "########################################################################################################\n",
    "obs_semantic = []\n",
    "for trial in ann1_model1:\n",
    "    obs_semantic.append(('ann1', 'model1_'+trial, int(ann1_model1[trial]['semantics'])))\n",
    "for trial in ann1_model3:\n",
    "    obs_semantic.append(('ann1', 'model3_'+trial, int(ann1_model3[trial]['semantics'])))\n",
    "for trial in ann2_model1:\n",
    "    obs_semantic.append(('ann2', 'model1_'+trial, int(ann2_model1[trial]['semantics'])))\n",
    "for trial in ann2_model3:\n",
    "    obs_semantic.append(('ann2', 'model3_'+trial, int(ann2_model3[trial]['semantics'])))\n",
    "\n",
    "print('Semantic:')\n",
    "kappa(obs_semantic)\n",
    "\n",
    "with open(os.path.join(path, 'semantic.csv'), 'w') as f:\n",
    "    for i, o in enumerate(obs_semantic):\n",
    "        obs_semantic[i] = list(obs_semantic[i])\n",
    "        obs_semantic[i][2] = str(obs_semantic[i][2])\n",
    "        obs_semantic[i] = ','.join(obs_semantic[i])\n",
    "    f.write('\\n'.join(obs_semantic))\n",
    "########################################################################################################\n",
    "obs_preds = []\n",
    "for trial in ann1_model1:\n",
    "    obs_preds.append(('1', 'model1_'+trial, int(ann1_model1[trial]['numpreds'])))\n",
    "for trial in ann1_model3:\n",
    "    obs_preds.append(('1', 'model3_'+trial, int(ann1_model3[trial]['numpreds'])))\n",
    "for trial in ann2_model1:\n",
    "    obs_preds.append(('2', 'model1_'+trial, int(ann2_model1[trial]['numpreds'])))\n",
    "for trial in ann2_model3:\n",
    "    obs_preds.append(('2', 'model3_'+trial, int(ann2_model3[trial]['numpreds'])))\n",
    "\n",
    "print('Predicates:')\n",
    "kappa(obs_preds)\n",
    "\n",
    "with open(os.path.join(path, 'predicates.csv'), 'w') as f:\n",
    "    for i, o in enumerate(obs_preds):\n",
    "        obs_preds[i] = list(obs_preds[i])\n",
    "        obs_preds[i][2] = str(obs_preds[i][2])\n",
    "        obs_preds[i] = ','.join(obs_preds[i])\n",
    "    f.write('\\n'.join(obs_preds))\n",
    "########################################################################################################\n",
    "obs = []\n",
    "for trial in ann1_model1:\n",
    "    obs.append(('1', 'model1_'+trial, 1 if ann1_model1[trial]['structurefollowed'] else 0))\n",
    "for trial in ann1_model3:\n",
    "    obs.append(('1', 'model3_'+trial, 1 if ann1_model3[trial]['structurefollowed'] else 0))\n",
    "for trial in ann2_model1:\n",
    "    obs.append(('2', 'model1_'+trial, 1 if ann2_model1[trial]['structurefollowed'] else 0))\n",
    "for trial in ann2_model3:\n",
    "    obs.append(('2', 'model3_'+trial, 1 if ann2_model3[trial]['structurefollowed'] else 0))\n",
    "\n",
    "print('Structure Followed:')\n",
    "kappa(obs)\n",
    "\n",
    "with open(os.path.join(path, 'structfollowed.csv'), 'w') as f:\n",
    "    for i, o in enumerate(obs):\n",
    "        obs[i] = list(obs[i])\n",
    "        obs[i][2] = str(obs[i][2])\n",
    "        obs[i] = ','.join(obs[i])\n",
    "    f.write('\\n'.join(obs))\n",
    "########################################################################################################\n",
    "obs = []\n",
    "for trial in ann1_model1:\n",
    "    obs.append(('1', 'model1_'+trial, 1 if ann1_model1[trial]['moreinformation'] else 0))\n",
    "for trial in ann1_model3:\n",
    "    obs.append(('1', 'model3_'+trial, 1 if ann1_model3[trial]['moreinformation'] else 0))\n",
    "for trial in ann2_model1:\n",
    "    obs.append(('2', 'model1_'+trial, 1 if ann2_model1[trial]['moreinformation'] else 0))\n",
    "for trial in ann2_model3:\n",
    "    obs.append(('2', 'model3_'+trial, 1 if ann2_model3[trial]['moreinformation'] else 0))\n",
    "\n",
    "print('Overgeneration:')\n",
    "kappa(obs)\n",
    "\n",
    "with open(os.path.join(path, 'overgeneration.csv'), 'w') as f:\n",
    "    for i, o in enumerate(obs):\n",
    "        obs[i] = list(obs[i])\n",
    "        obs[i][2] = str(obs[i][2])\n",
    "        obs[i] = ','.join(obs[i])\n",
    "    f.write('\\n'.join(obs))\n",
    "########################################################################################################\n",
    "obs = []\n",
    "for trial in ann1_model1:\n",
    "    obs.append(('1', 'model1_'+trial, 1 if ann1_model1[trial]['verbmistake'] else 0))\n",
    "for trial in ann1_model3:\n",
    "    obs.append(('1', 'model3_'+trial, 1 if ann1_model3[trial]['verbmistake'] else 0))\n",
    "for trial in ann2_model1:\n",
    "    obs.append(('2', 'model1_'+trial, 1 if ann2_model1[trial]['verbmistake'] else 0))\n",
    "for trial in ann2_model3:\n",
    "    obs.append(('2', 'model3_'+trial, 1 if ann2_model3[trial]['verbmistake'] else 0))\n",
    "\n",
    "print('Verb mistakes:')\n",
    "kappa(obs)\n",
    "\n",
    "with open(os.path.join(path, 'verbmistakes.csv'), 'w') as f:\n",
    "    for i, o in enumerate(obs):\n",
    "        obs[i] = list(obs[i])\n",
    "        obs[i][2] = str(obs[i][2])\n",
    "        obs[i] = ','.join(obs[i])\n",
    "    f.write('\\n'.join(obs))\n",
    "########################################################################################################\n",
    "obs = []\n",
    "for trial in ann1_model1:\n",
    "    obs.append(('1', 'model1_'+trial, 1 if ann1_model1[trial]['detmistake'] else 0))\n",
    "for trial in ann1_model3:\n",
    "    obs.append(('1', 'model3_'+trial, 1 if ann1_model3[trial]['detmistake'] else 0))\n",
    "for trial in ann2_model1:\n",
    "    obs.append(('2', 'model1_'+trial, 1 if ann2_model1[trial]['detmistake'] else 0))\n",
    "for trial in ann2_model3:\n",
    "    obs.append(('2', 'model3_'+trial, 1 if ann2_model3[trial]['detmistake'] else 0))\n",
    "\n",
    "print('Determiner mistakes:')\n",
    "kappa(obs)\n",
    "\n",
    "with open(os.path.join(path, 'detmistakes.csv'), 'w') as f:\n",
    "    for i, o in enumerate(obs):\n",
    "        obs[i] = list(obs[i])\n",
    "        obs[i][2] = str(obs[i][2])\n",
    "        obs[i] = ','.join(obs[i])\n",
    "    f.write('\\n'.join(obs))\n",
    "########################################################################################################\n",
    "obs = []\n",
    "for trial in ann1_model1:\n",
    "    obs.append(('1', 'model1_'+trial, 1 if ann1_model1[trial]['referencemistake'] else 0))\n",
    "for trial in ann1_model3:\n",
    "    obs.append(('1', 'model3_'+trial, 1 if ann1_model3[trial]['referencemistake'] else 0))\n",
    "for trial in ann2_model1:\n",
    "    obs.append(('2', 'model1_'+trial, 1 if ann2_model1[trial]['referencemistake'] else 0))\n",
    "for trial in ann2_model3:\n",
    "    obs.append(('2', 'model3_'+trial, 1 if ann2_model3[trial]['referencemistake'] else 0))\n",
    "\n",
    "print('Reference mistakes:')\n",
    "kappa(obs)\n",
    "\n",
    "with open(os.path.join(path, 'refmistakes.csv'), 'w') as f:\n",
    "    for i, o in enumerate(obs):\n",
    "        obs[i] = list(obs[i])\n",
    "        obs[i][2] = str(obs[i][2])\n",
    "        obs[i] = ','.join(obs[i])\n",
    "    f.write('\\n'.join(obs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "def split_triples(text):\n",
    "    triples, triple = [], []\n",
    "    for w in text:\n",
    "        if w not in ['<TRIPLE>', '</TRIPLE>']:\n",
    "            triple.append(w)\n",
    "        elif w == '</TRIPLE>':\n",
    "            triples.append(triple)\n",
    "            triple = []\n",
    "    return triples\n",
    "\n",
    "def ordering_analysis(ordering, gold):\n",
    "    for i, entry in enumerate(gold):\n",
    "        triples = split_triples(entry['source'])\n",
    "\n",
    "        num, visited = 0, []\n",
    "        for triple in triples:\n",
    "            for j, predicate in enumerate(ordering[i]):\n",
    "                if predicate == triple[1] and j not in visited:\n",
    "                    num += 1\n",
    "                    visited.append(j)\n",
    "        # How many predicates in the modified tripleset are present in the result?\n",
    "        entry['ordering'] = num\n",
    "    return gold\n",
    "\n",
    "\n",
    "def structing_analysis(structing, gold):\n",
    "    for i, entry in enumerate(self.gold):\n",
    "        triples = split_triples(entry['source'])\n",
    "\n",
    "        num, visited = 0, []\n",
    "        for triple in triples:\n",
    "            for j, predicate in enumerate(structing[i]):\n",
    "                if predicate == triple[1] and j not in visited:\n",
    "                    num += 1\n",
    "                    visited.append(j)\n",
    "        # How many predicates in the modified tripleset are present in the result?\n",
    "        entry['structing'] = num\n",
    "    return gold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 450,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Exact number of predicates in 52 out of 75 of the cases (0.69)\n"
     ]
    }
   ],
   "source": [
    "path = 'results/questionaire/partial.txt'\n",
    "with open(path) as f:\n",
    "    eids = f.read().split()\n",
    "    \n",
    "gold_path='evaluation/questionaire/gold.json'\n",
    "gold = json.load(open(gold_path))\n",
    "\n",
    "p = 'evaluation/results/pipeline/transformer/test.structing.postprocessed'\n",
    "with open(p) as f:\n",
    "    ordering = f.read().split('\\n')[:-1]\n",
    "\n",
    "pos, dem = 0, 0\n",
    "for i, entry in enumerate(gold):\n",
    "    if entry['eid'] in eids:\n",
    "        category = entry['category']\n",
    "#         if category in unseen_domains:\n",
    "        triples = split_triples(entry['source'])\n",
    "        num, visited = 0, []\n",
    "        for triple in triples:\n",
    "            for j, predicate in enumerate(ordering[i].split()):\n",
    "                if predicate == triple[1] and j not in visited:\n",
    "                    num += 1\n",
    "                    visited.append(j)\n",
    "                if len(visited) == len(triples):\n",
    "                    break\n",
    "        if len(triples) == len(visited):\n",
    "            pos += 1\n",
    "        dem += 1\n",
    "\n",
    "print('Exact number of predicates in {0} out of {1} of the cases ({2})'.format(pos, dem, round(pos/dem, 2)))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3.0
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}