run_forged_judge.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import weave\n",
    "from set_env import set_env\n",
    "import nest_asyncio\n",
    "from evalforge.combined_scorer import AssertionScorer\n",
    "from evalforge.evalforge import convert_datapoint_to_example\n",
    "from evalforge.combined_scorer import predict_passthrough\n",
    "import asyncio\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Env set\n"
     ]
    }
   ],
   "source": [
    "set_env(\"OPENAI_API_KEY\")\n",
    "set_env(\"WANDB_API_KEY\")\n",
    "print(\"Env set\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "try:\n",
    "    import IPython\n",
    "    in_jupyter = True\n",
    "except ImportError:\n",
    "    in_jupyter = False\n",
    "if in_jupyter:\n",
    "    nest_asyncio.apply()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Logged in as Weights & Biases user: a-sh0ts.\n",
      "View Weave data at https://wandb.ai/a-sh0ts/forged_test_9967161/weave\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<weave.trace.weave_client.WeaveClient at 0x147cbbb60>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import random\n",
    "weave.init(f\"forged_test_{random.randint(0, 10000000)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "TEST_TASK = \"medical\"\n",
    "if TEST_TASK == \"medical\":\n",
    "    all_data = weave.ref(\"weave:///a-sh0ts/medical_data_results/object/medical_data_annotations:7GcCtWgyPTWtKY48Z7v5VxwCNZXTTTpSMbmubAbyHT8\").get()\n",
    "    data = random.sample(all_data, 10)\n",
    "elif TEST_TASK == \"product\":\n",
    "    pass\n",
    "else:\n",
    "    all_data = data = [\n",
    "        ({\"text\": \"Summarize the impact of climate change on polar bears.\"}, {\"text\": \"Climate change is reducing sea ice, which polar bears rely on for hunting seals.\"}, 1, \"Accurate and relevant.\"),\n",
    "        ({\"text\": \"Explain the process of photosynthesis.\"}, {\"text\": \"Photosynthesis is the process by which plants use sunlight to synthesize foods from carbon dioxide and water.\"}, 1, \"Correct and detailed.\"),\n",
    "        ({\"text\": \"What are the main causes of the American Civil War?\"}, {\"text\": \"The main causes were slavery, states' rights, and economic differences.\"}, 1, \"Concise and accurate.\"),\n",
    "        ({\"text\": \"Describe the symptoms of COVID-19.\"}, {\"text\": \"COVID-19 is caused by a virus that originated in bats.\"}, 0, \"Irrelevant and incorrect.\"),\n",
    "        ({\"text\": \"What is the significance of the Magna Carta?\"}, {\"text\": \"The Magna Carta was a document that limited the power of the king and established certain legal rights.\"}, 1, \"Historically accurate and relevant.\")\n",
    "    ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Ideal\n",
    "# forged_judge = weave.ref(\"weave:///a-sh0ts/evalgen_test_363037/object/final_judge:M6GWs63FKPj8KNNOd56ASr2KwuwNn7YbAAFZdixSA0I\").get()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Copium\n",
    "forged_judge = AssertionScorer()\n",
    "forged_judge.import_assertions(\"forged_judge\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "finalized_task_description = \"The task requires transforming detailed medical dialogues between a doctor and a patient into structured medical notes in a standardized bullet-point list format. The dialogues provide comprehensive information on the patient's medical history, current symptoms, examination findings, familial history, allergies, medications, and future healthcare plans. The output should be a concise, structured list containing the mandatory fields: Chief complaint, History of present illness, Physical examination findings, Symptoms, New medications with dosages, and Follow-up instructions. Missing data should be marked as 'N/A'. The process demands the accurate extraction and summary of relevant medical information while strictly omitting any personally identifiable information (PII) to ensure privacy, substituting terms like 'the patient' for specific identifiers. Notes should maintain consistent punctuation and formatting, with new medications distinctly separated from existing ones. Evaluation focuses on accuracy, completeness, adherence to structure, exclusion of PII, consistent punctuation, and correct labeling of missing data. This task aims to produce anonymized medical notes that support healthcare delivery without compromising patient confidentiality, adhering to relevance, conciseness, and a 150-word limit for summaries if necessary.\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">67</span> examples\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Evaluated \u001b[1;36m1\u001b[0m of \u001b[1;36m67\u001b[0m examples\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">67</span> examples\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Evaluated \u001b[1;36m2\u001b[0m of \u001b[1;36m67\u001b[0m examples\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">3</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">67</span> examples\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Evaluated \u001b[1;36m3\u001b[0m of \u001b[1;36m67\u001b[0m examples\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">67</span> examples\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Evaluated \u001b[1;36m4\u001b[0m of \u001b[1;36m67\u001b[0m examples\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">67</span> examples\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Evaluated \u001b[1;36m5\u001b[0m of \u001b[1;36m67\u001b[0m examples\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">6</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">67</span> examples\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Evaluated \u001b[1;36m6\u001b[0m of \u001b[1;36m67\u001b[0m examples\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">7</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">67</span> examples\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Evaluated \u001b[1;36m7\u001b[0m of \u001b[1;36m67\u001b[0m examples\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">8</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">67</span> examples\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Evaluated \u001b[1;36m8\u001b[0m of \u001b[1;36m67\u001b[0m examples\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">9</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">67</span> examples\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Evaluated \u001b[1;36m9\u001b[0m of \u001b[1;36m67\u001b[0m examples\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">10</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">67</span> examples\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Evaluated \u001b[1;36m10\u001b[0m of \u001b[1;36m67\u001b[0m examples\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">11</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">67</span> examples\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Evaluated \u001b[1;36m11\u001b[0m of \u001b[1;36m67\u001b[0m examples\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">12</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">67</span> examples\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Evaluated \u001b[1;36m12\u001b[0m of \u001b[1;36m67\u001b[0m examples\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">13</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">67</span> examples\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Evaluated \u001b[1;36m13\u001b[0m of \u001b[1;36m67\u001b[0m examples\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">14</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">67</span> examples\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Evaluated \u001b[1;36m14\u001b[0m of \u001b[1;36m67\u001b[0m examples\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">15</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">67</span> examples\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Evaluated \u001b[1;36m15\u001b[0m of \u001b[1;36m67\u001b[0m examples\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">16</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">67</span> examples\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Evaluated \u001b[1;36m16\u001b[0m of \u001b[1;36m67\u001b[0m examples\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">17</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">67</span> examples\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Evaluated \u001b[1;36m17\u001b[0m of \u001b[1;36m67\u001b[0m examples\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">18</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">67</span> examples\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Evaluated \u001b[1;36m18\u001b[0m of \u001b[1;36m67\u001b[0m examples\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">19</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">67</span> examples\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Evaluated \u001b[1;36m19\u001b[0m of \u001b[1;36m67\u001b[0m examples\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">20</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">67</span> examples\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Evaluated \u001b[1;36m20\u001b[0m of \u001b[1;36m67\u001b[0m examples\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">21</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">67</span> examples\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Evaluated \u001b[1;36m21\u001b[0m of \u001b[1;36m67\u001b[0m examples\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">22</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">67</span> examples\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Evaluated \u001b[1;36m22\u001b[0m of \u001b[1;36m67\u001b[0m examples\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">23</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">67</span> examples\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Evaluated \u001b[1;36m23\u001b[0m of \u001b[1;36m67\u001b[0m examples\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">24</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">67</span> examples\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Evaluated \u001b[1;36m24\u001b[0m of \u001b[1;36m67\u001b[0m examples\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">25</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">67</span> examples\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Evaluated \u001b[1;36m25\u001b[0m of \u001b[1;36m67\u001b[0m examples\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">26</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">67</span> examples\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Evaluated \u001b[1;36m26\u001b[0m of \u001b[1;36m67\u001b[0m examples\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">27</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">67</span> examples\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Evaluated \u001b[1;36m27\u001b[0m of \u001b[1;36m67\u001b[0m examples\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">28</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">67</span> examples\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Evaluated \u001b[1;36m28\u001b[0m of \u001b[1;36m67\u001b[0m examples\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">29</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">67</span> examples\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Evaluated \u001b[1;36m29\u001b[0m of \u001b[1;36m67\u001b[0m examples\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">30</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">67</span> examples\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Evaluated \u001b[1;36m30\u001b[0m of \u001b[1;36m67\u001b[0m examples\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">31</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">67</span> examples\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Evaluated \u001b[1;36m31\u001b[0m of \u001b[1;36m67\u001b[0m examples\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">32</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">67</span> examples\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Evaluated \u001b[1;36m32\u001b[0m of \u001b[1;36m67\u001b[0m examples\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">33</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">67</span> examples\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Evaluated \u001b[1;36m33\u001b[0m of \u001b[1;36m67\u001b[0m examples\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">34</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">67</span> examples\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Evaluated \u001b[1;36m34\u001b[0m of \u001b[1;36m67\u001b[0m examples\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">35</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">67</span> examples\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Evaluated \u001b[1;36m35\u001b[0m of \u001b[1;36m67\u001b[0m examples\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">36</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">67</span> examples\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Evaluated \u001b[1;36m36\u001b[0m of \u001b[1;36m67\u001b[0m examples\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">37</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">67</span> examples\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Evaluated \u001b[1;36m37\u001b[0m of \u001b[1;36m67\u001b[0m examples\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">38</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">67</span> examples\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Evaluated \u001b[1;36m38\u001b[0m of \u001b[1;36m67\u001b[0m examples\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">39</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">67</span> examples\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Evaluated \u001b[1;36m39\u001b[0m of \u001b[1;36m67\u001b[0m examples\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">40</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">67</span> examples\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Evaluated \u001b[1;36m40\u001b[0m of \u001b[1;36m67\u001b[0m examples\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">41</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">67</span> examples\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Evaluated \u001b[1;36m41\u001b[0m of \u001b[1;36m67\u001b[0m examples\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">42</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">67</span> examples\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Evaluated \u001b[1;36m42\u001b[0m of \u001b[1;36m67\u001b[0m examples\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">43</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">67</span> examples\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Evaluated \u001b[1;36m43\u001b[0m of \u001b[1;36m67\u001b[0m examples\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">44</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">67</span> examples\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Evaluated \u001b[1;36m44\u001b[0m of \u001b[1;36m67\u001b[0m examples\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">45</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">67</span> examples\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Evaluated \u001b[1;36m45\u001b[0m of \u001b[1;36m67\u001b[0m examples\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">46</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">67</span> examples\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Evaluated \u001b[1;36m46\u001b[0m of \u001b[1;36m67\u001b[0m examples\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">47</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">67</span> examples\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Evaluated \u001b[1;36m47\u001b[0m of \u001b[1;36m67\u001b[0m examples\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">48</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">67</span> examples\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Evaluated \u001b[1;36m48\u001b[0m of \u001b[1;36m67\u001b[0m examples\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">49</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">67</span> examples\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Evaluated \u001b[1;36m49\u001b[0m of \u001b[1;36m67\u001b[0m examples\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">50</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">67</span> examples\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Evaluated \u001b[1;36m50\u001b[0m of \u001b[1;36m67\u001b[0m examples\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">51</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">67</span> examples\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Evaluated \u001b[1;36m51\u001b[0m of \u001b[1;36m67\u001b[0m examples\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">52</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">67</span> examples\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Evaluated \u001b[1;36m52\u001b[0m of \u001b[1;36m67\u001b[0m examples\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">53</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">67</span> examples\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Evaluated \u001b[1;36m53\u001b[0m of \u001b[1;36m67\u001b[0m examples\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">54</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">67</span> examples\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Evaluated \u001b[1;36m54\u001b[0m of \u001b[1;36m67\u001b[0m examples\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">55</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">67</span> examples\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Evaluated \u001b[1;36m55\u001b[0m of \u001b[1;36m67\u001b[0m examples\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">56</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">67</span> examples\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Evaluated \u001b[1;36m56\u001b[0m of \u001b[1;36m67\u001b[0m examples\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">57</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">67</span> examples\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Evaluated \u001b[1;36m57\u001b[0m of \u001b[1;36m67\u001b[0m examples\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">58</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">67</span> examples\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Evaluated \u001b[1;36m58\u001b[0m of \u001b[1;36m67\u001b[0m examples\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">59</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">67</span> examples\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Evaluated \u001b[1;36m59\u001b[0m of \u001b[1;36m67\u001b[0m examples\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">60</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">67</span> examples\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Evaluated \u001b[1;36m60\u001b[0m of \u001b[1;36m67\u001b[0m examples\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">61</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">67</span> examples\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Evaluated \u001b[1;36m61\u001b[0m of \u001b[1;36m67\u001b[0m examples\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">62</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">67</span> examples\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Evaluated \u001b[1;36m62\u001b[0m of \u001b[1;36m67\u001b[0m examples\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">63</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">67</span> examples\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Evaluated \u001b[1;36m63\u001b[0m of \u001b[1;36m67\u001b[0m examples\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">64</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">67</span> examples\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Evaluated \u001b[1;36m64\u001b[0m of \u001b[1;36m67\u001b[0m examples\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">65</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">67</span> examples\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Evaluated \u001b[1;36m65\u001b[0m of \u001b[1;36m67\u001b[0m examples\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">66</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">67</span> examples\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Evaluated \u001b[1;36m66\u001b[0m of \u001b[1;36m67\u001b[0m examples\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">67</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">67</span> examples\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Evaluated \u001b[1;36m67\u001b[0m of \u001b[1;36m67\u001b[0m examples\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluation summary\n",
       "<span style=\"font-weight: bold\">{</span>\n",
       "    <span style=\"color: #008000; text-decoration-color: #008000\">'AssertionScorer'</span>: <span style=\"font-weight: bold\">{</span>\n",
       "        <span style=\"color: #008000; text-decoration-color: #008000\">'Adherence to Structure and Format'</span>: <span style=\"font-weight: bold\">{</span>\n",
       "            <span style=\"color: #008000; text-decoration-color: #008000\">'structure_and_format_llm_check'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'score'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.7910447761194029</span><span style=\"font-weight: bold\">}}</span>\n",
       "        <span style=\"font-weight: bold\">}</span>,\n",
       "        <span style=\"color: #008000; text-decoration-color: #008000\">'Accuracy and Completeness'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'check_accuracy_and_completeness'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'score'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.6417910447761194</span><span style=\"font-weight: bold\">}}}</span>,\n",
       "        <span style=\"color: #008000; text-decoration-color: #008000\">'Adherence to format and completeness'</span>: <span style=\"font-weight: bold\">{</span>\n",
       "            <span style=\"color: #008000; text-decoration-color: #008000\">'test_adheres_to_bullet_point_format'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'score'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.0</span><span style=\"font-weight: bold\">}}</span>,\n",
       "            <span style=\"color: #008000; text-decoration-color: #008000\">'evaluate_structure_and_completeness'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'score'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.9701492537313433</span><span style=\"font-weight: bold\">}}</span>\n",
       "        <span style=\"font-weight: bold\">}</span>,\n",
       "        <span style=\"color: #008000; text-decoration-color: #008000\">'Exclusion of personal identifiable information (PII)'</span>: <span style=\"font-weight: bold\">{</span>\n",
       "            <span style=\"color: #008000; text-decoration-color: #008000\">'evaluate_exclusion_of_pii'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'score'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.8059701492537313</span><span style=\"font-weight: bold\">}}</span>\n",
       "        <span style=\"font-weight: bold\">}</span>,\n",
       "        <span style=\"color: #008000; text-decoration-color: #008000\">'Accuracy and Relevance of Transformed Notes'</span>: <span style=\"font-weight: bold\">{</span>\n",
       "            <span style=\"color: #008000; text-decoration-color: #008000\">'test_accuracy_of_medical_details'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'score'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.0</span><span style=\"font-weight: bold\">}}</span>,\n",
       "            <span style=\"color: #008000; text-decoration-color: #008000\">'relevance_of_transformed_notes'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'score'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.9402985074626866</span><span style=\"font-weight: bold\">}}</span>\n",
       "        <span style=\"font-weight: bold\">}</span>\n",
       "    <span style=\"font-weight: bold\">}</span>,\n",
       "    <span style=\"color: #008000; text-decoration-color: #008000\">'model_latency'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2.1947925482223285</span><span style=\"font-weight: bold\">}</span>\n",
       "<span style=\"font-weight: bold\">}</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "Evaluation summary\n",
       "\u001b[1m{\u001b[0m\n",
       "    \u001b[32m'AssertionScorer'\u001b[0m: \u001b[1m{\u001b[0m\n",
       "        \u001b[32m'Adherence to Structure and Format'\u001b[0m: \u001b[1m{\u001b[0m\n",
       "            \u001b[32m'structure_and_format_llm_check'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'score'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m0.7910447761194029\u001b[0m\u001b[1m}\u001b[0m\u001b[1m}\u001b[0m\n",
       "        \u001b[1m}\u001b[0m,\n",
       "        \u001b[32m'Accuracy and Completeness'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'check_accuracy_and_completeness'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'score'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m0.6417910447761194\u001b[0m\u001b[1m}\u001b[0m\u001b[1m}\u001b[0m\u001b[1m}\u001b[0m,\n",
       "        \u001b[32m'Adherence to format and completeness'\u001b[0m: \u001b[1m{\u001b[0m\n",
       "            \u001b[32m'test_adheres_to_bullet_point_format'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'score'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m0.0\u001b[0m\u001b[1m}\u001b[0m\u001b[1m}\u001b[0m,\n",
       "            \u001b[32m'evaluate_structure_and_completeness'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'score'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m0.9701492537313433\u001b[0m\u001b[1m}\u001b[0m\u001b[1m}\u001b[0m\n",
       "        \u001b[1m}\u001b[0m,\n",
       "        \u001b[32m'Exclusion of personal identifiable information \u001b[0m\u001b[32m(\u001b[0m\u001b[32mPII\u001b[0m\u001b[32m)\u001b[0m\u001b[32m'\u001b[0m: \u001b[1m{\u001b[0m\n",
       "            \u001b[32m'evaluate_exclusion_of_pii'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'score'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m0.8059701492537313\u001b[0m\u001b[1m}\u001b[0m\u001b[1m}\u001b[0m\n",
       "        \u001b[1m}\u001b[0m,\n",
       "        \u001b[32m'Accuracy and Relevance of Transformed Notes'\u001b[0m: \u001b[1m{\u001b[0m\n",
       "            \u001b[32m'test_accuracy_of_medical_details'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'score'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m0.0\u001b[0m\u001b[1m}\u001b[0m\u001b[1m}\u001b[0m,\n",
       "            \u001b[32m'relevance_of_transformed_notes'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'score'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m0.9402985074626866\u001b[0m\u001b[1m}\u001b[0m\u001b[1m}\u001b[0m\n",
       "        \u001b[1m}\u001b[0m\n",
       "    \u001b[1m}\u001b[0m,\n",
       "    \u001b[32m'model_latency'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m2.1947925482223285\u001b[0m\u001b[1m}\u001b[0m\n",
       "\u001b[1m}\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "🍩 https://wandb.ai/a-sh0ts/forged_test_9967161/r/call/0192111c-8cff-7223-bbf7-5321342ebd38\n"
     ]
    }
   ],
   "source": [
    "all_annotation_examples = convert_datapoint_to_example(finalized_task_description, all_data)\n",
    "evaluation = weave.Evaluation(\n",
    "    scorers=[forged_judge],\n",
    "    dataset=all_annotation_examples,\n",
    ")\n",
    "\n",
    "\n",
    "final_judge_results = asyncio.run(evaluation.evaluate(predict_passthrough))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}