RAG_Evaluation.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NxEpyuwr0jNt",
        "outputId": "875aaadf-3f8d-40a8-c556-3ce9ffb56ded"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Cloning into 'edu'...\n",
            "remote: Enumerating objects: 4510, done.\u001b[K\n",
            "remote: Counting objects: 100% (1412/1412), done.\u001b[K\n",
            "remote: Compressing objects: 100% (532/532), done.\u001b[K\n",
            "remote: Total 4510 (delta 1076), reused 1042 (delta 877), pack-reused 3098 (from 1)\u001b[K\n",
            "Receiving objects: 100% (4510/4510), 28.01 MiB | 8.41 MiB/s, done.\n",
            "Resolving deltas: 100% (2471/2471), done.\n",
            "/content/edu/rag-advanced\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m60.8/60.8 kB\u001b[0m \u001b[31m2.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m73.4/73.4 kB\u001b[0m \u001b[31m7.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m50.4/50.4 kB\u001b[0m \u001b[31m4.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m251.9/251.9 kB\u001b[0m \u001b[31m9.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m249.2/249.2 kB\u001b[0m \u001b[31m20.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m162.6/162.6 kB\u001b[0m \u001b[31m16.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m259.0/259.0 kB\u001b[0m \u001b[31m19.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m99.3/99.3 kB\u001b[0m \u001b[31m9.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m51.7/51.7 kB\u001b[0m \u001b[31m5.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m646.7/646.7 kB\u001b[0m \u001b[31m36.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m41.2/41.2 MB\u001b[0m \u001b[31m13.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.2/1.2 MB\u001b[0m \u001b[31m60.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m139.1/139.1 kB\u001b[0m \u001b[31m12.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m586.9/586.9 kB\u001b[0m \u001b[31m42.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.1/3.1 MB\u001b[0m \u001b[31m84.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m76.4/76.4 kB\u001b[0m \u001b[31m7.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m78.0/78.0 kB\u001b[0m \u001b[31m6.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.1/3.1 MB\u001b[0m \u001b[31m83.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.6/1.6 MB\u001b[0m \u001b[31m65.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m310.7/310.7 kB\u001b[0m \u001b[31m28.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m13.0/13.0 MB\u001b[0m \u001b[31m97.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m242.1/242.1 kB\u001b[0m \u001b[31m22.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.7/1.7 MB\u001b[0m \u001b[31m73.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m347.4/347.4 kB\u001b[0m \u001b[31m27.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m66.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m141.9/141.9 kB\u001b[0m \u001b[31m12.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m60.8/60.8 kB\u001b[0m \u001b[31m5.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m12.6/12.6 MB\u001b[0m \u001b[31m81.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.3/2.3 MB\u001b[0m \u001b[31m62.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m207.3/207.3 kB\u001b[0m \u001b[31m18.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m203.2/203.2 kB\u001b[0m \u001b[31m17.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m114.5/114.5 kB\u001b[0m \u001b[31m11.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m45.4/45.4 kB\u001b[0m \u001b[31m3.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m54.5/54.5 kB\u001b[0m \u001b[31m4.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m82.6/82.6 kB\u001b[0m \u001b[31m6.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m388.4/388.4 kB\u001b[0m \u001b[31m31.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m313.8/313.8 kB\u001b[0m \u001b[31m24.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m147.8/147.8 kB\u001b[0m \u001b[31m12.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m74.0/74.0 kB\u001b[0m \u001b[31m6.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m82.1/82.1 kB\u001b[0m \u001b[31m7.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m119.4/119.4 kB\u001b[0m \u001b[31m11.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m62.7/62.7 kB\u001b[0m \u001b[31m5.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.3/58.3 kB\u001b[0m \u001b[31m3.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m26.7/26.7 MB\u001b[0m \u001b[31m22.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.6/5.6 MB\u001b[0m \u001b[31m73.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m134.8/134.8 kB\u001b[0m \u001b[31m11.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m56.8/56.8 kB\u001b[0m \u001b[31m5.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m233.2/233.2 kB\u001b[0m \u001b[31m16.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m128.2/128.2 kB\u001b[0m \u001b[31m10.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m85.0/85.0 kB\u001b[0m \u001b[31m7.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m52.8/52.8 kB\u001b[0m \u001b[31m4.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m565.6/565.6 kB\u001b[0m \u001b[31m32.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m78.6/78.6 kB\u001b[0m \u001b[31m7.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Building wheel for fasttext-langdetect (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for syncer (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for fasttext (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for warc3-wet-clueweb09 (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for cbor (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "gensim 4.3.3 requires scipy<1.14.0,>=1.7.0, but you have scipy 1.14.1 which is incompatible.\u001b[0m\u001b[31m\n",
            "\u001b[0m/content/edu/rag-advanced/notebooks\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[nltk_data] Downloading package wordnet to /root/nltk_data...\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {},
          "execution_count": 1
        }
      ],
      "source": [
        "# @title Setup\n",
        "!git clone https://github.com/wandb/edu.git\n",
        "%cd edu/rag-advanced\n",
        "!pip install -qqq -r requirements.txt\n",
        "%cd notebooks\n",
        "\n",
        "import nltk\n",
        "\n",
        "nltk.download(\"wordnet\")"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import getpass\n",
        "import os\n",
        "\n",
        "os.environ[\"COHERE_API_KEY\"] = getpass.getpass(\"Please enter your COHERE_API_KEY\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "z7nqfkA00kJ6",
        "outputId": "27a838e6-3139-40b6-96c0-183e6c689802"
      },
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Please enter your COHERE_API_KEY··········\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import asyncio\n",
        "\n",
        "import nest_asyncio\n",
        "\n",
        "nest_asyncio.apply()\n",
        "\n",
        "import weave\n",
        "\n",
        "from scripts.utils import display_source"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "r2eQnGxY1RSM",
        "outputId": "d3642bf2-a608-4131-e7b9-1e673ba1260a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "sagemaker.config INFO - Not applying SDK defaults from location: /etc/xdg/sagemaker/config.yaml\n",
            "sagemaker.config INFO - Not applying SDK defaults from location: /root/.config/sagemaker/config.yaml\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "WANDB_PROJECT = \"rag-course\"\n",
        "weave_client = weave.init(WANDB_PROJECT)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 173
        },
        "id": "sbmuuoBx1ZMJ",
        "outputId": "e9b8e224-33e1-498f-c72a-448a74e3e15d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Please login to Weights & Biases (https://wandb.ai/) to continue:\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\u001b[34m\u001b[1mwandb\u001b[0m: Using wandb-core as the SDK backend. Please refer to https://wandb.me/wandb-core for more information.\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ],
            "application/javascript": [
              "\n",
              "        window._wandbApiKey = new Promise((resolve, reject) => {\n",
              "            function loadScript(url) {\n",
              "            return new Promise(function(resolve, reject) {\n",
              "                let newScript = document.createElement(\"script\");\n",
              "                newScript.onerror = reject;\n",
              "                newScript.onload = resolve;\n",
              "                document.body.appendChild(newScript);\n",
              "                newScript.src = url;\n",
              "            });\n",
              "            }\n",
              "            loadScript(\"https://cdn.jsdelivr.net/npm/postmate/build/postmate.min.js\").then(() => {\n",
              "            const iframe = document.createElement('iframe')\n",
              "            iframe.style.cssText = \"width:0;height:0;border:none\"\n",
              "            document.body.appendChild(iframe)\n",
              "            const handshake = new Postmate({\n",
              "                container: iframe,\n",
              "                url: 'https://wandb.ai/authorize'\n",
              "            });\n",
              "            const timeout = setTimeout(() => reject(\"Couldn't auto authenticate\"), 5000)\n",
              "            handshake.then(function(child) {\n",
              "                child.on('authorize', data => {\n",
              "                    clearTimeout(timeout)\n",
              "                    resolve(data)\n",
              "                });\n",
              "            });\n",
              "            })\n",
              "        });\n",
              "    "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\u001b[34m\u001b[1mwandb\u001b[0m: Logging into wandb.ai. (Learn how to deploy a W&B server locally: https://wandb.me/wandb-server)\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: You can find your API key in your browser here: https://wandb.ai/authorize\n",
            "wandb: Paste an API key from your profile and hit enter, or press ctrl+c to quit:"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " ··········\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\u001b[34m\u001b[1mwandb\u001b[0m: Appending key for api.wandb.ai to your netrc file: /root/.netrc\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Logged in as Weights & Biases user: shikhar_dave.\n",
            "View Weave data at https://wandb.ai/shikhar_dave-iit-jodhpur/rag-course/weave\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Easy eval dataset with 20 samples.\n",
        "eval_dataset = weave.ref(\n",
        "    \"weave:///rag-course/dev/object/Dataset:Qj4IFICc2EbdXu5A5UuhkPiWgxM1GvJMIvXEyv1DYnM\"\n",
        ").get()\n",
        "\n",
        "print(\"Number of evaluation samples: \", len(eval_dataset.rows))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wwRmoR561fHI",
        "outputId": "13252c51-c0d0-4768-f34d-3473451a968f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Number of evaluation samples:  20\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "dict(eval_dataset.rows[0])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WnzVMV2H17I0",
        "outputId": "b6688a9f-d1e2-4bd8-eed4-c3da1c5741d1"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'question': 'How can I access the run object from the Lightning WandBLogger function?',\n",
              " 'answer': \"In PyTorch Lightning, the `WandbLogger` is used to log metrics, model weights, and other data to Weights & Biases during training. To access the `wandb.Run` object from within a `LightningModule` when using `WandbLogger`, you can use the `Trainer.logger.experiment` attribute. This attribute provides direct access to the underlying `wandb.Run` object, allowing you to interact with the Weights & Biases API directly.\\n\\nHere's how you can access the `wandb.Run` object using `WandbLogger` in PyTorch Lightning:\\n\\n```python\\nfrom pytorch_lightning import Trainer, LightningModule\\nfrom pytorch_lightning.loggers import WandbLogger\\n\\nclass MyModel(LightningModule):\\n    def training_step(self, batch, batch_idx):\\n        # Your training logic here\\n        loss = ...\\n\\n        # Log metrics\\n        self.log('train_loss', loss)\\n\\n        # Access the wandb.Run object\\n        run = self.trainer.logger.experiment\\n        # Now you can use `run` to interact with wandb API, e.g., to log additional data\\n        run.log({'additional_metric': value})\\n\\n        return loss\\n\\n# Setup the logger\\nwandb_logger = WandbLogger(project='my_project', entity='my_entity')\\n\\n# Setup the trainer\\ntrainer = Trainer(logger=wandb_logger)\\n\\n# Initialize your model\\nmodel = MyModel()\\n\\n# Train the model\\ntrainer.fit(model)\\n```\\n\\nIn this example:\\n- A `WandbLogger` is instantiated and passed to the `Trainer`.\\n- Inside the `LightningModule`, you can access the `wandb.Run` object via `self.trainer.logger.experiment`.\\n- You can then use this `run` object to log additional data or interact with the Weights & Biases API directly.\\n\\nThis approach allows you to leverage the full capabilities of Weights & Biases directly from your training loop in PyTorch Lightning.\",\n",
              " 'contexts': WeaveList([{'content': 'Using PyTorch Lightning\\'s WandbLogger\\nPyTorch Lightning has multiple `WandbLogger` (Pytorch) (Fabric) classes that can be used to seamlessly log metrics, model weights, media and more. Just instantiate the WandbLogger and pass it to Lightning\\'s `Trainer` or `Fabric`.\\nwandb_logger = WandbLogger()\\ntrainer = Trainer(logger=wandb_logger)\\nfabric = L.Fabric(loggers=[wandb_logger])\\nfabric.launch()\\nfabric.log_dict({\\n    \"important_metric\": important_metric\\n})\\nLogger arguments\\nBelow are some of the most used parameters in WandbLogger, see the PyTorch Lightning for a full list and description\\n(Pytorch)\\n(Fabric)\\nParameter\\nDescription\\n`project`\\nDefine what wandb Project to log to\\n`name`\\nGive a name to your wandb run\\n`log_model`\\nLog all models if `log_model=\"all\"` or at end of training if `log_model=True`\\n`save_dir`\\nPath where data is saved\\nLog your hyperparameters\\nclass LitModule(LightningModule):\\n    def __init__(self, *args, **kwarg):\\n        self.save_hyperparameters()\\nwandb_logger.log_hyperparams(\\n    {\\n        \"hyperparameter_1\": hyperparameter_1,\\n        \"hyperparameter_2\": hyperparameter_2,\\n    }\\n)\\nLog additional config parameters\\n# add one parameter\\nwandb_logger.experiment.config[\"key\"] = value\\n# add multiple parameters\\nwandb_logger.experiment.config.update({key1: val1, key2: val2})\\n# use directly wandb module\\nwandb.config[\"key\"] = value\\nwandb.config.update()\\nLog gradients, parameter histogram and model topology\\nYou can pass your model object to `wandblogger.watch()` to monitor your models\\'s gradients and parameters as you train. See the PyTorch Lightning `WandbLogger` documentation', 'source': 'guides/integrations/lightning.md', 'score': 0.9996229999999999, 'relevance': 2}, {'content': 'PyTorch Lightning\\nTry in a Colab Notebook here →\\nPyTorch Lightning provides a lightweight wrapper for organizing your PyTorch code and easily adding advanced features such as distributed training and 16-bit precision. W&B provides a lightweight wrapper for logging your ML experiments. But you don\\'t need to combine the two yourself: Weights & Biases is incorporated directly into the PyTorch Lightning library via the WandbLogger.\\n⚡ Get going lightning-fast with just a few lines.\\nfrom lightning.pytorch.loggers import WandbLogger\\nfrom lightning.pytorch import Trainer\\nwandb_logger = WandbLogger(log_model=\"all\")\\ntrainer = Trainer(logger=wandb_logger)\\n:::info\\nUsing wandb.log(): Please note that the `WandbLogger` logs to W&B using the Trainer\\'s `global_step`. If you are making additional calls to `wandb.log` directly in your code, do not use the `step` argument in `wandb.log()`. \\nInstead, log the Trainer\\'s `global_step` like your other metrics, like so:\\n`wandb.log({\"accuracy\":0.99, \"trainer/global_step\": step})`\\n:::\\nimport lightning as L\\nfrom wandb.integration.lightning.fabric import WandbLogger\\nwandb_logger = WandbLogger(log_model=\"all\")\\nfabric = L.Fabric(loggers=[wandb_logger])\\nfabric.launch()\\nfabric.log_dict({\"important_metric\": important_metric})\\nSign up and Log in to wandb\\na) Sign up for a free account\\nb) Pip install the `wandb` library\\nc) To log in in your training script, you\\'ll need to be signed in to you account at www.wandb.ai, then you will find your API key on the Authorize page.\\nIf you are using Weights and Biases for the first time you might want to check out our quickstart\\npip install wandb\\nwandb login\\n!pip install wandb\\nimport wandb\\nwandb.login()', 'source': 'guides/integrations/lightning.md', 'score': 0.9393594, 'relevance': 2}, {'content': 'Check out interactive examples!\\nYou can follow along in our video tutorial with our tutorial colab here\\nFrequently Asked Questions\\nHow does W&B integrate with Lightning?\\nThe core integration is based on the Lightning loggers API, which lets you write much of your logging code in a framework-agnostic way. `Logger`s are passed to the Lightning Trainer and are triggered based on that API\\'s rich hook-and-callback system. This keeps your research code well-separated from engineering and logging code.\\nWhat does the integration log without any additional code?\\nWe\\'ll save your model checkpoints to W&B, where you can view them or download them for use in future runs. We\\'ll also capture system metrics, like GPU usage and network I/O, environment information, like hardware and OS information, code state (including git commit and diff patch, notebook contents and session history), and anything printed to the standard out.\\nWhat if I really need to use wandb.run in my training setup?\\nYou will have to essentially expand the scope of the variable you need to access yourself. In other words, making sure that the initial conditions are the same on all processes.\\nif os.environ.get(\"LOCAL_RANK\", None) is None:\\n    os.environ[\"WANDB_DIR\"] = wandb.run.dir\\nThen, you can use `os.environ[\"WANDB_DIR\"]` to set up the model checkpoints directory. This way, `wandb.run.dir` can be used by any non-zero rank processes as well.', 'source': 'guides/integrations/lightning.md', 'score': 0.9999046300000001, 'relevance': 1}, {'content': 'With implicit wandb integrations\\nIf you\\'re using a framework integration we support, you can also pass in the callback directly:\\n@wandb_log\\ndef train_model(\\n    train_dataloader_path: components.InputPath(\"dataloader\"),\\n    test_dataloader_path: components.InputPath(\"dataloader\"),\\n    model_path: components.OutputPath(\"pytorch_model\")\\n):\\n    from pytorch_lightning.loggers import WandbLogger\\n    from pytorch_lightning import Trainer\\n    trainer = Trainer(logger=WandbLogger())\\n    ...  # do training', 'source': 'guides/integrations/other/kubeflow-pipelines-kfp.md', 'score': 0.9520419, 'relevance': 1}, {'content': 'Log images, text and more\\nThe `WandbLogger` has `log_image`, `log_text` and `log_table` methods for logging media.\\nYou can also directly call `wandb.log` or `trainer.logger.experiment.log` to log other media types such as Audio, Molecules, Point Clouds, 3D Objects and more.\\n# using tensors, numpy arrays or PIL images\\nwandb_logger.log_image(key=\"samples\", images=[img1, img2])\\n# adding captions\\nwandb_logger.log_image(key=\"samples\", images=[img1, img2], caption=[\"tree\", \"person\"])\\n# using file path\\nwandb_logger.log_image(key=\"samples\", images=[\"img_1.jpg\", \"img_2.jpg\"])\\n# using .log in the trainer\\ntrainer.logger.experiment.log(\\n    {\"samples\": [wandb.Image(img, caption=caption) for (img, caption) in my_images]},\\n    step=current_trainer_global_step,\\n)\\n# data should be a list of lists\\ncolumns = [\"input\", \"label\", \"prediction\"]\\nmy_data = [[\"cheese\", \"english\", \"english\"], [\"fromage\", \"french\", \"spanish\"]]\\n# using columns and data\\nwandb_logger.log_text(key=\"my_samples\", columns=columns, data=my_data)\\n# using a pandas DataFrame\\nwandb_logger.log_text(key=\"my_samples\", dataframe=my_dataframe)\\n# log a W&B Table that has a text caption, an image and audio\\ncolumns = [\"caption\", \"image\", \"sound\"]\\n# data should be a list of lists\\nmy_data = [\\n    [\"cheese\", wandb.Image(img_1), wandb.Audio(snd_1)],\\n    [\"wine\", wandb.Image(img_2), wandb.Audio(snd_2)],\\n]\\n# log the Table\\nwandb_logger.log_table(key=\"my_samples\", columns=columns, data=data)\\nYou can use Lightning\\'s Callbacks system to control when you log to Weights & Biases via the WandbLogger, in this example we log a sample of our validation images and predictions:', 'source': 'guides/integrations/lightning.md', 'score': 0.8296300999999999, 'relevance': 1}, {'content': \"Run\\nView source on GitHub\\nA unit of computation logged by wandb. Typically, this is an ML experiment.\\nRun(\\n    settings: Settings,\\n    config: Optional[Dict[str, Any]] = None,\\n    sweep_config: Optional[Dict[str, Any]] = None,\\n    launch_config: Optional[Dict[str, Any]] = None\\n) -> None\\nCreate a run with `wandb.init()`:\\nimport wandb\\nrun = wandb.init()\\nThere is only ever at most one active `wandb.Run` in any process,\\nand it is accessible as `wandb.run`:\\nimport wandb\\nassert wandb.run is None\\nwandb.init()\\nassert wandb.run is not None\\nanything you log with `wandb.log` will be sent to that run.\\nIf you want to start more runs in the same script or notebook, you'll need to\\nfinish the run that is in-flight. Runs can be finished with `wandb.finish` or\\nby using them in a `with` block:\\nimport wandb\\nwandb.init()\\nwandb.finish()\\nassert wandb.run is None\\nwith wandb.init() as run:\\n    pass  # log data here\\nassert wandb.run is None\", 'source': 'ref/python/run.md', 'score': 0.99928474, 'relevance': 0}, {'content': 'Examples:\\nSet where the run is logged\\nYou can change where the run is logged, just like changing\\nthe organization, repository, and branch in git:\\nimport wandb\\nuser = \"geoff\"\\nproject = \"capsules\"\\ndisplay_name = \"experiment-2021-10-31\"\\nwandb.init(entity=user, project=project, name=display_name)\\nAdd metadata about the run to the config\\nPass a dictionary-style object as the `config` keyword argument to add\\nmetadata, like hyperparameters, to your run.\\nimport wandb\\nconfig = {\"lr\": 3e-4, \"batch_size\": 32}\\nconfig.update({\"architecture\": \"resnet\", \"depth\": 34})\\nwandb.init(config=config)\\nRaises\\n`Error`\\nif some unknown or internal error happened during the run initialization.\\n`AuthenticationError`\\nif the user failed to provide valid credentials.\\n`CommError`\\nif there was a problem communicating with the WandB server.\\n`UsageError`\\nif the user provided invalid arguments.\\n`KeyboardInterrupt`\\nif user interrupts the run.\\nReturns\\nA `Run` object.', 'source': 'ref/python/init.md', 'score': 0.99253935, 'relevance': 0}, {'content': 'Runs\\nA single unit of computation logged by W&B is called a run. You can think of a W&B run as an atomic element of your whole project. You should initiate a new run when you:\\nTrain a model\\nChange a hyperparameter\\nUse a different model\\nLog data or a model as a W&B Artifact\\nDownload a W&B Artifact\\nFor example, during a sweep, W&B explores a hyperparameter search space that you specify. Each new hyperparameter combination created by the sweep is implemented and recorded as a unique run. \\n:::tip\\nSome key things to consider when you create and manage runs:\\n* Anything you log with `wandb.log` is recorded in that run.  For more information on how log objects in W&B, see Log Media and Objects. \\n* Each run is associated to a specific W&B project.\\n* View runs and their properties within the run\\'s project workspace on the W&B App UI.\\n* There is only at most one active wandb.Run in any process,\\nand it is accessible as `wandb.run`.\\n:::\\nCreate a run\\nCreate a W&B run with wandb.init():\\nimport wandb\\nrun = wandb.init()\\nWe recommend you specify a project name and a W&B entity when you create a new run. W&B creates a new project (if the project does not already exist) within the W&B entity you provide. If the project already exists, W&B stores the run in that project.\\nFor example, the following code snippet initializes a run that is stored in a project called `model_registry_example` that is scoped within a `wandbee` entity:\\nimport wandb\\nrun = wandb.init(entity=\"wandbee\", \\\\\\n        project=\"model_registry_example\")\\nW&B prints the name of the run that is created along with a URL path to find out more information about that specific run. \\nFor example, the code snippet above produces this output:', 'source': 'guides/runs/intro.md', 'score': 0.9912548, 'relevance': 0}, {'content': 'Get the command that ran the run\\nEach run captures the command that launched it on the run overview page. To pull this command down from the API, you can run:\\nimport wandb\\napi = wandb.Api()\\nrun = api.run(\"<entity>/<project>/<run_id>\")\\nmeta = json.load(run.file(\"wandb-metadata.json\").download())\\nprogram = [\"python\"] + [meta[\"program\"]] + meta[\"args\"]', 'source': 'guides/track/public-api-guide.md', 'score': 0.9897908999999999, 'relevance': 0}, {'content': 'You can then log the parent_span to W&B like as below. \\nrun = wandb.init(name=\"manual_span_demo\", project=\"wandb_prompts_demo\")\\nrun.log({\"trace\": trace_tree.WBTraceTree(parent_span)})\\nrun.finish()\\nClicking on the W&B Run link generated will take you to a workspace where you can inspect the Trace created.', 'source': 'tutorials/prompts.md', 'score': 0.98942953, 'relevance': 0}])}"
            ]
          },
          "metadata": {},
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Reload the data from Chapter 1\n",
        "chunked_data = weave.ref(\n",
        "    \"weave:///rag-course/rag-course/object/chunked_data:Lt6M8qCUICD1JZTlMYzuDLTVtvFYESxvj3tcAIoPrtE\"\n",
        ").get()\n",
        "chunked_data = weave.ref(\"chunked_data:v0\").get()\n",
        "print(\"Number of chunked data: \", len(chunked_data.rows))\n",
        "chunked_data.rows[:2]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9RushRKT19kk",
        "outputId": "289e525a-ba66-4e4c-e55d-615406f0062e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Number of chunked data:  1073\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[WeaveDict({'content': '--- description: Log and visualize data without a W&B account displayed_sidebar: default --- # Anonymous Mode Are you publishing code that you want anyone to be able to run easily? Use Anonymous Mode to let someone run your code, see a W&B dashboard, and visualize results without needing to create a W&B account first. Allow results to be logged in Anonymous Mode with `wandb.init(`**`anonymous=\"allow\"`**`)` :::info **Publishing a paper?** Please [cite W&B](https://docs.wandb.ai/company/academics#bibtex-citation), and if you have questions about how to make your code accessible while using W&B, reach out to us at support@wandb.com. ::: ### How does someone without an account see results? If someone runs your script and you have to set `anonymous=\"allow\"`: 1. **Auto-create temporary account:** W&B checks for an account that\\'s already signed in. If there\\'s no account, we automatically create a new anonymous account and save that API key for the session. 2. **Log results quickly:** The user can run and re-run the script, and automatically see results show up in the W&B dashboard UI. These unclaimed anonymous runs will be available for 7 days. 3. **Claim data when it\\'s useful**: Once the user finds valuable results in W&B, they can easily click a button in the banner at the top of the page to save their run data to a real account. If they don\\'t claim a run, it will be deleted after 7 days. :::caution **Anonymous run links are sensitive**. These links allow anyone to view and claim the results of an experiment for 7 days, so make sure to only share links with people you trust. If you\\'re trying to share results publicly, but hide the author\\'s identity, please contact us at support@wandb.com to share more about your use case. ::: ### What happens to users with existing accounts? If you set `anonymous=\"allow\"` in', 'metadata': {'source': 'guides/app/features/anon.md', 'raw_tokens': 300, 'cleaned_tokens': 300}, 'cleaned_content': '--- description: Log and visualize data without a W&B account displayed_sidebar: default --- # Anonymous Mode Are you publishing code that you want anyone to be able to run easily? Use Anonymous Mode to let someone run your code, see a W&B dashboard, and visualize results without needing to create a W&B account first. Allow results to be logged in Anonymous Mode with `wandb.init(`**`anonymous=\"allow\"`**`)` :::info **Publishing a paper?** Please [cite W&B](https://docs.wandb.ai/company/academics#bibtex-citation), and if you have questions about how to make your code accessible while using W&B, reach out to us at support@wandb.com. ::: ### How does someone without an account see results? If someone runs your script and you have to set `anonymous=\"allow\"`: 1. **Auto-create temporary account:** W&B checks for an account that\\'s already signed in. If there\\'s no account, we automatically create a new anonymous account and save that API key for the session. 2. **Log results quickly:** The user can run and re-run the script, and automatically see results show up in the W&B dashboard UI. These unclaimed anonymous runs will be available for 7 days. 3. **Claim data when it\\'s useful**: Once the user finds valuable results in W&B, they can easily click a button in the banner at the top of the page to save their run data to a real account. If they don\\'t claim a run, it will be deleted after 7 days. :::caution **Anonymous run links are sensitive**. These links allow anyone to view and claim the results of an experiment for 7 days, so make sure to only share links with people you trust. If you\\'re trying to share results publicly, but hide the author\\'s identity, please contact us at support@wandb.com to share more about your use case. ::: ### What happens to users with existing accounts? If you set `anonymous=\"allow\"` in'}),\n",
              " WeaveDict({'content': 'your script, we will check to make sure there\\'s not an existing account first, before creating an anonymous account. This means that if a W&B user finds your script and runs it, their results will be logged correctly to their account, just like a normal run. ### What are features that aren\\'t available to anonymous users? * **No persistent data**: Runs are only saved for 7 days in an anonymous account. Users can claim anonymous run data by saving it to a real account. ![](@site/static/images/app_ui/anon_mode_no_data.png) * **No artifact logging**: Runs will print a warning on the command line that you can\\'t log an artifact to an anonymous run. ![](@site/static/images/app_ui/anon_example_warning.png) * **No profile or settings pages**: Certain pages aren\\'t available in the UI, because they\\'re only useful for real accounts. ## Example usage [Try the example notebook](http://bit.ly/anon-mode) to see how anonymous mode works. ```python import wandb # Start a run allowing anonymous accounts wandb.init(anonymous=\"allow\") # Log results from your training loop wandb.log({\"acc\": 0.91}) # Mark the run as finished wandb.finish() ```', 'metadata': {'source': 'guides/app/features/anon.md', 'raw_tokens': 170, 'cleaned_tokens': 170}, 'cleaned_content': 'your script, we will check to make sure there\\'s not an existing account first, before creating an anonymous account. This means that if a W&B user finds your script and runs it, their results will be logged correctly to their account, just like a normal run. ### What are features that aren\\'t available to anonymous users? * **No persistent data**: Runs are only saved for 7 days in an anonymous account. Users can claim anonymous run data by saving it to a real account. ![](@site/static/images/app_ui/anon_mode_no_data.png) * **No artifact logging**: Runs will print a warning on the command line that you can\\'t log an artifact to an anonymous run. ![](@site/static/images/app_ui/anon_example_warning.png) * **No profile or settings pages**: Certain pages aren\\'t available in the UI, because they\\'re only useful for real accounts. ## Example usage [Try the example notebook](http://bit.ly/anon-mode) to see how anonymous mode works. ```python import wandb # Start a run allowing anonymous accounts wandb.init(anonymous=\"allow\") # Log results from your training loop wandb.log({\"acc\": 0.91}) # Mark the run as finished wandb.finish() ```'})]"
            ]
          },
          "metadata": {},
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from scripts.retriever import TFIDFRetriever\n",
        "\n",
        "display_source(TFIDFRetriever)\n",
        "\n",
        "retriever = TFIDFRetriever()\n",
        "retriever.index_data(list(map(dict, chunked_data.rows)))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "NXpjkO4J2EVP",
        "outputId": "d9223f94-29be-4b6b-a6a3-47a644d2b1b2"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 1 \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mclass\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;166;226;46;48;2;39;40;34mTFIDFRetriever\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mweave\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m.\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mModel\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[48;2;39;40;34m                                                                            \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 2 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\"\"\u001b[0m\u001b[48;2;39;40;34m                                                                                                       \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 3 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m    A retriever model that uses TF-IDF for indexing and searching documents.\u001b[0m\u001b[48;2;39;40;34m                                  \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 4 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 5 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m    Attributes:\u001b[0m\u001b[48;2;39;40;34m                                                                                               \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 6 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        vectorizer (TfidfVectorizer): The TF-IDF vectorizer.\u001b[0m\u001b[48;2;39;40;34m                                                  \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 7 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        index (list): The indexed data.\u001b[0m\u001b[48;2;39;40;34m                                                                       \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 8 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        data (list): The data to be indexed.\u001b[0m\u001b[48;2;39;40;34m                                                                  \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 9 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m    \"\"\"\u001b[0m\u001b[48;2;39;40;34m                                                                                                       \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m10 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m11 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mvectorizer\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mTfidfVectorizer\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mTfidfVectorizer\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[48;2;39;40;34m                                                           \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m12 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mindex\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mlist\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mNone\u001b[0m\u001b[48;2;39;40;34m                                                                                        \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m13 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mdata\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mlist\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mNone\u001b[0m\u001b[48;2;39;40;34m                                                                                         \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m14 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m15 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mdef\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;166;226;46;48;2;39;40;34mindex_data\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mself\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mdata\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[48;2;39;40;34m                                                                               \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m16 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m        \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\"\"\u001b[0m\u001b[48;2;39;40;34m                                                                                                   \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m17 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        Indexes the provided data using TF-IDF.\u001b[0m\u001b[48;2;39;40;34m                                                               \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m18 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m19 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        Args:\u001b[0m\u001b[48;2;39;40;34m                                                                                                 \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m20 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m            data (list): A list of documents to be indexed. Each document should be a dictionary\u001b[0m\u001b[48;2;39;40;34m              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m21 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m                         containing a key 'cleaned_content' with the text to be indexed.\u001b[0m\u001b[48;2;39;40;34m                      \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m22 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        \"\"\"\u001b[0m\u001b[48;2;39;40;34m                                                                                                   \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m23 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m        \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mself\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m.\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mdata\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mdata\u001b[0m\u001b[48;2;39;40;34m                                                                                      \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m24 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m        \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mdocs\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mdoc\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34mcleaned_content\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mfor\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mdoc\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34min\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mdata\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[48;2;39;40;34m                                                       \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m25 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m        \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mself\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m.\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mindex\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mself\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m.\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mvectorizer\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m.\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mfit_transform\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mdocs\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[48;2;39;40;34m                                                      \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m26 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m27 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;166;226;46;48;2;39;40;34m@weave\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m.\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mop\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[48;2;39;40;34m                                                                                               \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m28 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mdef\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;166;226;46;48;2;39;40;34msearch\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mself\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mquery\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mk\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;174;129;255;48;2;39;40;34m5\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[48;2;39;40;34m                                                                             \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m29 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m        \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\"\"\u001b[0m\u001b[48;2;39;40;34m                                                                                                   \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m30 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        Searches the indexed data for the given query using cosine similarity.\u001b[0m\u001b[48;2;39;40;34m                                \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m31 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m32 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        Args:\u001b[0m\u001b[48;2;39;40;34m                                                                                                 \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m33 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m            query (str): The search query.\u001b[0m\u001b[48;2;39;40;34m                                                                    \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m34 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m            k (int): The number of top results to return. Default is 5.\u001b[0m\u001b[48;2;39;40;34m                                       \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m35 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m36 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        Returns:\u001b[0m\u001b[48;2;39;40;34m                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m37 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m            list: A list of dictionaries containing the source, text, and score of the top-k results.\u001b[0m\u001b[48;2;39;40;34m         \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m38 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        \"\"\"\u001b[0m\u001b[48;2;39;40;34m                                                                                                   \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m39 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m        \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mquery_vec\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mself\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m.\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mvectorizer\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m.\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mtransform\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mquery\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[48;2;39;40;34m                                                        \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m40 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m        \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mcosine_distances\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mcdist\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[48;2;39;40;34m                                                                             \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m41 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m            \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mquery_vec\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m.\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mtodense\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mself\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m.\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mindex\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m.\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mtodense\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mmetric\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34mcosine\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[48;2;39;40;34m                                        \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m42 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m        \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;174;129;255;48;2;39;40;34m0\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[48;2;39;40;34m                                                                                                  \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m43 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m        \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mtop_k_indices\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mcosine_distances\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m.\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34margsort\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mk\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[48;2;39;40;34m                                                        \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m44 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m        \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34moutput\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[48;2;39;40;34m                                                                                           \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m45 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m        \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mfor\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34midx\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34min\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mtop_k_indices\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[48;2;39;40;34m                                                                             \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m46 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m            \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34moutput\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m.\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mappend\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[48;2;39;40;34m                                                                                    \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m47 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m                \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m{\u001b[0m\u001b[48;2;39;40;34m                                                                                             \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m48 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m                    \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34msource\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mself\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m.\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mdata\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34midx\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34mmetadata\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34msource\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[48;2;39;40;34m                                           \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m49 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m                    \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34mtext\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mself\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m.\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mdata\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34midx\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34mcleaned_content\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[48;2;39;40;34m                                                \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m50 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m                    \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34mscore\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;174;129;255;48;2;39;40;34m1\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m-\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mcosine_distances\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34midx\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[48;2;39;40;34m                                                       \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m51 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m                \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m}\u001b[0m\u001b[48;2;39;40;34m                                                                                             \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m52 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m            \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[48;2;39;40;34m                                                                                                 \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m53 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m        \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mreturn\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34moutput\u001b[0m\u001b[48;2;39;40;34m                                                                                         \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m54 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m55 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;166;226;46;48;2;39;40;34m@weave\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m.\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mop\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[48;2;39;40;34m                                                                                               \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m56 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mdef\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;166;226;46;48;2;39;40;34mpredict\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mself\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mquery\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mstr\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mk\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mint\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[48;2;39;40;34m                                                                    \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m57 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m        \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\"\"\u001b[0m\u001b[48;2;39;40;34m                                                                                                   \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m58 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        Predicts the top-k results for the given query.\u001b[0m\u001b[48;2;39;40;34m                                                       \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m59 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m60 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        Args:\u001b[0m\u001b[48;2;39;40;34m                                                                                                 \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m61 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m            query (str): The search query.\u001b[0m\u001b[48;2;39;40;34m                                                                    \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m62 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m            k (int): The number of top results to return.\u001b[0m\u001b[48;2;39;40;34m                                                     \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m63 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m64 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        Returns:\u001b[0m\u001b[48;2;39;40;34m                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m65 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m            list: A list of dictionaries containing the source, text, and score of the top-k results.\u001b[0m\u001b[48;2;39;40;34m         \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m66 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        \"\"\"\u001b[0m\u001b[48;2;39;40;34m                                                                                                   \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m67 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m        \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mreturn\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mself\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m.\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34msearch\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mquery\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mk\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[48;2;39;40;34m                                                                          \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m68 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 1 </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">class</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> </span><span style=\"color: #a6e22e; text-decoration-color: #a6e22e; background-color: #272822\">TFIDFRetriever</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">(weave</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">.</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">Model):</span><span style=\"background-color: #272822\">                                                                            </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 2 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">\"\"\"</span><span style=\"background-color: #272822\">                                                                                                       </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 3 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">    A retriever model that uses TF-IDF for indexing and searching documents.</span><span style=\"background-color: #272822\">                                  </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 4 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 5 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">    Attributes:</span><span style=\"background-color: #272822\">                                                                                               </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 6 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        vectorizer (TfidfVectorizer): The TF-IDF vectorizer.</span><span style=\"background-color: #272822\">                                                  </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 7 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        index (list): The indexed data.</span><span style=\"background-color: #272822\">                                                                       </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 8 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        data (list): The data to be indexed.</span><span style=\"background-color: #272822\">                                                                  </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 9 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">    \"\"\"</span><span style=\"background-color: #272822\">                                                                                                       </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">10 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">11 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    vectorizer: TfidfVectorizer </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> TfidfVectorizer()</span><span style=\"background-color: #272822\">                                                           </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">12 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    index: list </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">None</span><span style=\"background-color: #272822\">                                                                                        </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">13 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    data: list </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">None</span><span style=\"background-color: #272822\">                                                                                         </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">14 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">15 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">def</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> </span><span style=\"color: #a6e22e; text-decoration-color: #a6e22e; background-color: #272822\">index_data</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">(self, data):</span><span style=\"background-color: #272822\">                                                                               </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">16 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">        </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">\"\"\"</span><span style=\"background-color: #272822\">                                                                                                   </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">17 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        Indexes the provided data using TF-IDF.</span><span style=\"background-color: #272822\">                                                               </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">18 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">19 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        Args:</span><span style=\"background-color: #272822\">                                                                                                 </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">20 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">            data (list): A list of documents to be indexed. Each document should be a dictionary</span><span style=\"background-color: #272822\">              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">21 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">                         containing a key 'cleaned_content' with the text to be indexed.</span><span style=\"background-color: #272822\">                      </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">22 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        \"\"\"</span><span style=\"background-color: #272822\">                                                                                                   </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">23 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">        self</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">.</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">data </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> data</span><span style=\"background-color: #272822\">                                                                                      </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">24 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">        docs </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> [doc[</span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">\"cleaned_content\"</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">] </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">for</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> doc </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">in</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> data]</span><span style=\"background-color: #272822\">                                                       </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">25 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">        self</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">.</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">index </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> self</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">.</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">vectorizer</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">.</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">fit_transform(docs)</span><span style=\"background-color: #272822\">                                                      </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">26 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">27 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    </span><span style=\"color: #a6e22e; text-decoration-color: #a6e22e; background-color: #272822\">@weave</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">.</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">op()</span><span style=\"background-color: #272822\">                                                                                               </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">28 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">def</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> </span><span style=\"color: #a6e22e; text-decoration-color: #a6e22e; background-color: #272822\">search</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">(self, query, k</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #ae81ff; text-decoration-color: #ae81ff; background-color: #272822\">5</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">):</span><span style=\"background-color: #272822\">                                                                             </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">29 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">        </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">\"\"\"</span><span style=\"background-color: #272822\">                                                                                                   </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">30 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        Searches the indexed data for the given query using cosine similarity.</span><span style=\"background-color: #272822\">                                </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">31 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">32 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        Args:</span><span style=\"background-color: #272822\">                                                                                                 </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">33 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">            query (str): The search query.</span><span style=\"background-color: #272822\">                                                                    </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">34 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">            k (int): The number of top results to return. Default is 5.</span><span style=\"background-color: #272822\">                                       </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">35 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">36 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        Returns:</span><span style=\"background-color: #272822\">                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">37 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">            list: A list of dictionaries containing the source, text, and score of the top-k results.</span><span style=\"background-color: #272822\">         </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">38 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        \"\"\"</span><span style=\"background-color: #272822\">                                                                                                   </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">39 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">        query_vec </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> self</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">.</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">vectorizer</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">.</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">transform([query])</span><span style=\"background-color: #272822\">                                                        </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">40 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">        cosine_distances </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> cdist(</span><span style=\"background-color: #272822\">                                                                             </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">41 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">            query_vec</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">.</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">todense(), self</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">.</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">index</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">.</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">todense(), metric</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">\"cosine\"</span><span style=\"background-color: #272822\">                                        </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">42 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">        )[</span><span style=\"color: #ae81ff; text-decoration-color: #ae81ff; background-color: #272822\">0</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">]</span><span style=\"background-color: #272822\">                                                                                                  </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">43 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">        top_k_indices </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> cosine_distances</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">.</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">argsort()[:k]</span><span style=\"background-color: #272822\">                                                        </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">44 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">        output </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> []</span><span style=\"background-color: #272822\">                                                                                           </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">45 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">        </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">for</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> idx </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">in</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> top_k_indices:</span><span style=\"background-color: #272822\">                                                                             </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">46 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">            output</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">.</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">append(</span><span style=\"background-color: #272822\">                                                                                    </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">47 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">                {</span><span style=\"background-color: #272822\">                                                                                             </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">48 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">                    </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">\"source\"</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">: self</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">.</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">data[idx][</span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">\"metadata\"</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">][</span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">\"source\"</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">],</span><span style=\"background-color: #272822\">                                           </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">49 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">                    </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">\"text\"</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">: self</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">.</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">data[idx][</span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">\"cleaned_content\"</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">],</span><span style=\"background-color: #272822\">                                                </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">50 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">                    </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">\"score\"</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">: </span><span style=\"color: #ae81ff; text-decoration-color: #ae81ff; background-color: #272822\">1</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">-</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> cosine_distances[idx],</span><span style=\"background-color: #272822\">                                                       </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">51 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">                }</span><span style=\"background-color: #272822\">                                                                                             </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">52 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">            )</span><span style=\"background-color: #272822\">                                                                                                 </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">53 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">        </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">return</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> output</span><span style=\"background-color: #272822\">                                                                                         </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">54 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">55 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    </span><span style=\"color: #a6e22e; text-decoration-color: #a6e22e; background-color: #272822\">@weave</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">.</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">op()</span><span style=\"background-color: #272822\">                                                                                               </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">56 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">def</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> </span><span style=\"color: #a6e22e; text-decoration-color: #a6e22e; background-color: #272822\">predict</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">(self, query: str, k: int):</span><span style=\"background-color: #272822\">                                                                    </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">57 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">        </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">\"\"\"</span><span style=\"background-color: #272822\">                                                                                                   </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">58 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        Predicts the top-k results for the given query.</span><span style=\"background-color: #272822\">                                                       </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">59 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">60 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        Args:</span><span style=\"background-color: #272822\">                                                                                                 </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">61 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">            query (str): The search query.</span><span style=\"background-color: #272822\">                                                                    </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">62 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">            k (int): The number of top results to return.</span><span style=\"background-color: #272822\">                                                     </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">63 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">64 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        Returns:</span><span style=\"background-color: #272822\">                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">65 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">            list: A list of dictionaries containing the source, text, and score of the top-k results.</span><span style=\"background-color: #272822\">         </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">66 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        \"\"\"</span><span style=\"background-color: #272822\">                                                                                                   </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">67 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">        </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">return</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> self</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">.</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">search(query, k)</span><span style=\"background-color: #272822\">                                                                          </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">68 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from scripts.retrieval_metrics import IR_METRICS\n",
        "from scripts.utils import display_source"
      ],
      "metadata": {
        "id": "D-xI7Srg2KMX"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "for scorer in IR_METRICS:\n",
        "    display_source(scorer)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "Cx-BHGSU2cKG",
        "outputId": "6013dc64-ee7e-414d-a8f1-953881a90b4d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 1 \u001b[0m\u001b[38;2;166;226;46;48;2;39;40;34m@weave\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m.\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mop\u001b[0m\u001b[48;2;39;40;34m                                                                                                     \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 2 \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mdef\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;166;226;46;48;2;39;40;34mcompute_hit_rate\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[48;2;39;40;34m                                                                                         \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 3 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mmodel_output\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mList\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mDict\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mstr\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mAny\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mcontexts\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mList\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mDict\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mstr\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mAny\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[48;2;39;40;34m                                        \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 4 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m-\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m>\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mfloat\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[48;2;39;40;34m                                                                                                   \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 5 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\"\"\u001b[0m\u001b[48;2;39;40;34m                                                                                                       \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 6 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m    Calculate the hit rate (precision) for a single query.\u001b[0m\u001b[48;2;39;40;34m                                                    \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 7 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 8 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m    Args:\u001b[0m\u001b[48;2;39;40;34m                                                                                                     \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 9 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        model_output (List[Dict[str, Any]]): The list of retrieved documents from the model.\u001b[0m\u001b[48;2;39;40;34m                  \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m10 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m            Each dictionary contains:\u001b[0m\u001b[48;2;39;40;34m                                                                         \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m11 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m                - 'source': A unique identifier for the document.\u001b[0m\u001b[48;2;39;40;34m                                             \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m12 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m                - 'score': The relevance score of the document.\u001b[0m\u001b[48;2;39;40;34m                                               \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m13 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        contexts (List[Dict[str, Any]]): A list of dictionaries representing the relevant contexts.\u001b[0m\u001b[48;2;39;40;34m           \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m14 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m            Each dictionary contains:\u001b[0m\u001b[48;2;39;40;34m                                                                         \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m15 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m                - 'source': A unique identifier for the relevant document.\u001b[0m\u001b[48;2;39;40;34m                                    \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m16 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m17 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m    Returns:\u001b[0m\u001b[48;2;39;40;34m                                                                                                  \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m18 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        float: The hit rate (precision).\u001b[0m\u001b[48;2;39;40;34m                                                                      \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m19 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m20 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m    The hit rate (precision) measures the proportion of retrieved documents that are relevant.\u001b[0m\u001b[48;2;39;40;34m                \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m21 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m22 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m    This metric is useful for assessing the accuracy of the retrieval system by determining the relevance of t\u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m23 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m    retrieved documents.\u001b[0m\u001b[48;2;39;40;34m                                                                                      \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m24 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m    \"\"\"\u001b[0m\u001b[48;2;39;40;34m                                                                                                       \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m25 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34msearch_results\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mdoc\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34msource\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mfor\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mdoc\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34min\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mmodel_output\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[48;2;39;40;34m                                                  \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m26 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mrelevant_sources\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[48;2;39;40;34m                                                                                      \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m27 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m        \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mcontext\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34msource\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mfor\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mcontext\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34min\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mcontexts\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mif\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mcontext\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34mrelevance\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m!=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;174;129;255;48;2;39;40;34m0\u001b[0m\u001b[48;2;39;40;34m                                \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m28 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[48;2;39;40;34m                                                                                                         \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m29 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m30 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;149;144;119;48;2;39;40;34m# Calculate the number of relevant documents retrieved\u001b[0m\u001b[48;2;39;40;34m                                                    \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m31 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mrelevant_retrieved\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34msum\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[48;2;39;40;34m                                                                                 \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m32 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m        \u001b[0m\u001b[38;2;174;129;255;48;2;39;40;34m1\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mfor\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34msource\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34min\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34msearch_results\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mif\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34msource\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34min\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mrelevant_sources\u001b[0m\u001b[48;2;39;40;34m                                          \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m33 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[48;2;39;40;34m                                                                                                         \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m34 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m35 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;149;144;119;48;2;39;40;34m# Calculate the hit rate (precision)\u001b[0m\u001b[48;2;39;40;34m                                                                      \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m36 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mhit_rate\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mrelevant_retrieved\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m/\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mlen\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34msearch_results\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mif\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34msearch_results\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34melse\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;174;129;255;48;2;39;40;34m0.0\u001b[0m\u001b[48;2;39;40;34m                            \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m37 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m38 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mreturn\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mhit_rate\u001b[0m\u001b[48;2;39;40;34m                                                                                           \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m39 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 1 </span><span style=\"color: #a6e22e; text-decoration-color: #a6e22e; background-color: #272822\">@weave</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">.</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">op</span><span style=\"background-color: #272822\">                                                                                                     </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 2 </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">def</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> </span><span style=\"color: #a6e22e; text-decoration-color: #a6e22e; background-color: #272822\">compute_hit_rate</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">(</span><span style=\"background-color: #272822\">                                                                                         </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 3 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    model_output: List[Dict[str, Any]], contexts: List[Dict[str, Any]]</span><span style=\"background-color: #272822\">                                        </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 4 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">) </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">-&gt;</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> float:</span><span style=\"background-color: #272822\">                                                                                                   </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 5 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">\"\"\"</span><span style=\"background-color: #272822\">                                                                                                       </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 6 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">    Calculate the hit rate (precision) for a single query.</span><span style=\"background-color: #272822\">                                                    </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 7 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 8 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">    Args:</span><span style=\"background-color: #272822\">                                                                                                     </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 9 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        model_output (List[Dict[str, Any]]): The list of retrieved documents from the model.</span><span style=\"background-color: #272822\">                  </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">10 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">            Each dictionary contains:</span><span style=\"background-color: #272822\">                                                                         </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">11 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">                - 'source': A unique identifier for the document.</span><span style=\"background-color: #272822\">                                             </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">12 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">                - 'score': The relevance score of the document.</span><span style=\"background-color: #272822\">                                               </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">13 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        contexts (List[Dict[str, Any]]): A list of dictionaries representing the relevant contexts.</span><span style=\"background-color: #272822\">           </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">14 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">            Each dictionary contains:</span><span style=\"background-color: #272822\">                                                                         </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">15 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">                - 'source': A unique identifier for the relevant document.</span><span style=\"background-color: #272822\">                                    </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">16 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">17 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">    Returns:</span><span style=\"background-color: #272822\">                                                                                                  </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">18 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        float: The hit rate (precision).</span><span style=\"background-color: #272822\">                                                                      </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">19 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">20 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">    The hit rate (precision) measures the proportion of retrieved documents that are relevant.</span><span style=\"background-color: #272822\">                </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">21 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">22 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">    This metric is useful for assessing the accuracy of the retrieval system by determining the relevance of t</span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">23 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">    retrieved documents.</span><span style=\"background-color: #272822\">                                                                                      </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">24 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">    \"\"\"</span><span style=\"background-color: #272822\">                                                                                                       </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">25 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    search_results </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> [doc[</span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">\"source\"</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">] </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">for</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> doc </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">in</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> model_output]</span><span style=\"background-color: #272822\">                                                  </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">26 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    relevant_sources </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> [</span><span style=\"background-color: #272822\">                                                                                      </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">27 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">        context[</span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">\"source\"</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">] </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">for</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> context </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">in</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> contexts </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">if</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> context[</span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">\"relevance\"</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">] </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">!=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> </span><span style=\"color: #ae81ff; text-decoration-color: #ae81ff; background-color: #272822\">0</span><span style=\"background-color: #272822\">                                </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">28 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    ]</span><span style=\"background-color: #272822\">                                                                                                         </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">29 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">30 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    </span><span style=\"color: #959077; text-decoration-color: #959077; background-color: #272822\"># Calculate the number of relevant documents retrieved</span><span style=\"background-color: #272822\">                                                    </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">31 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    relevant_retrieved </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> sum(</span><span style=\"background-color: #272822\">                                                                                 </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">32 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">        </span><span style=\"color: #ae81ff; text-decoration-color: #ae81ff; background-color: #272822\">1</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">for</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> source </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">in</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> search_results </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">if</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> source </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">in</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> relevant_sources</span><span style=\"background-color: #272822\">                                          </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">33 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    )</span><span style=\"background-color: #272822\">                                                                                                         </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">34 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">35 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    </span><span style=\"color: #959077; text-decoration-color: #959077; background-color: #272822\"># Calculate the hit rate (precision)</span><span style=\"background-color: #272822\">                                                                      </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">36 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    hit_rate </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> relevant_retrieved </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">/</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> len(search_results) </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">if</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> search_results </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">else</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> </span><span style=\"color: #ae81ff; text-decoration-color: #ae81ff; background-color: #272822\">0.0</span><span style=\"background-color: #272822\">                            </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">37 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">38 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">return</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> hit_rate</span><span style=\"background-color: #272822\">                                                                                           </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">39 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 1 \u001b[0m\u001b[38;2;166;226;46;48;2;39;40;34m@weave\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m.\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mop\u001b[0m\u001b[48;2;39;40;34m                                                                                                     \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 2 \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mdef\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;166;226;46;48;2;39;40;34mcompute_mrr\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[48;2;39;40;34m                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 3 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mmodel_output\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mList\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mDict\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mstr\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mAny\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mcontexts\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mList\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mDict\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mstr\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mAny\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[48;2;39;40;34m                                        \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 4 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m-\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m>\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mfloat\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[48;2;39;40;34m                                                                                                   \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 5 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\"\"\u001b[0m\u001b[48;2;39;40;34m                                                                                                       \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 6 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m    Calculate the Mean Reciprocal Rank (MRR) for a single query.\u001b[0m\u001b[48;2;39;40;34m                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 7 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 8 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m    Args:\u001b[0m\u001b[48;2;39;40;34m                                                                                                     \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 9 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        model_output (List[Dict[str, Any]]): The list of retrieved documents from the model.\u001b[0m\u001b[48;2;39;40;34m                  \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m10 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m            Each dictionary contains:\u001b[0m\u001b[48;2;39;40;34m                                                                         \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m11 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m                - 'source': A unique identifier for the document.\u001b[0m\u001b[48;2;39;40;34m                                             \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m12 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m                - 'score': The relevance score of the document.\u001b[0m\u001b[48;2;39;40;34m                                               \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m13 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        contexts (List[Dict[str, Any]]): A list of dictionaries representing the relevant contexts.\u001b[0m\u001b[48;2;39;40;34m           \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m14 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m            Each dictionary contains:\u001b[0m\u001b[48;2;39;40;34m                                                                         \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m15 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m                - 'source': A unique identifier for the relevant document.\u001b[0m\u001b[48;2;39;40;34m                                    \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m16 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m17 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m    Returns:\u001b[0m\u001b[48;2;39;40;34m                                                                                                  \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m18 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        float: The MRR score for the given query.\u001b[0m\u001b[48;2;39;40;34m                                                             \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m19 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m20 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m    MRR measures the rank of the first relevant document in the result list.\u001b[0m\u001b[48;2;39;40;34m                                  \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m21 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m22 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m    If no relevant document is found, MRR is 0.\u001b[0m\u001b[48;2;39;40;34m                                                               \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m23 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m24 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m    This metric is useful for evaluating systems where there is typically one relevant document\u001b[0m\u001b[48;2;39;40;34m               \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m25 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m    and the user is interested in finding that document quickly.\u001b[0m\u001b[48;2;39;40;34m                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m26 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m    \"\"\"\u001b[0m\u001b[48;2;39;40;34m                                                                                                       \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m27 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mrelevant_sources\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[48;2;39;40;34m                                                                                      \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m28 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m        \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mcontext\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34msource\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mfor\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mcontext\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34min\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mcontexts\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mif\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mcontext\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34mrelevance\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m!=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;174;129;255;48;2;39;40;34m0\u001b[0m\u001b[48;2;39;40;34m                                \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m29 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[48;2;39;40;34m                                                                                                         \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m30 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m31 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mmrr_score\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;174;129;255;48;2;39;40;34m0\u001b[0m\u001b[48;2;39;40;34m                                                                                             \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m32 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mfor\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mrank\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mresult\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34min\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34menumerate\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mmodel_output\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;174;129;255;48;2;39;40;34m1\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[48;2;39;40;34m                                                           \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m33 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m        \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mif\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mresult\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34msource\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34min\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mrelevant_sources\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[48;2;39;40;34m                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m34 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m            \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mmrr_score\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m+\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;174;129;255;48;2;39;40;34m1\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m/\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mrank\u001b[0m\u001b[48;2;39;40;34m                                                                             \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m35 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m36 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mif\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mmrr_score\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m==\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;174;129;255;48;2;39;40;34m0\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[48;2;39;40;34m                                                                                        \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m37 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m        \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mreturn\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;174;129;255;48;2;39;40;34m0.0\u001b[0m\u001b[48;2;39;40;34m                                                                                            \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m38 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34melse\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[48;2;39;40;34m                                                                                                     \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m39 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m        \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mreturn\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mmrr_score\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m/\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mlen\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mmodel_output\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[48;2;39;40;34m                                                                  \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m40 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 1 </span><span style=\"color: #a6e22e; text-decoration-color: #a6e22e; background-color: #272822\">@weave</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">.</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">op</span><span style=\"background-color: #272822\">                                                                                                     </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 2 </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">def</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> </span><span style=\"color: #a6e22e; text-decoration-color: #a6e22e; background-color: #272822\">compute_mrr</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">(</span><span style=\"background-color: #272822\">                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 3 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    model_output: List[Dict[str, Any]], contexts: List[Dict[str, Any]]</span><span style=\"background-color: #272822\">                                        </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 4 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">) </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">-&gt;</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> float:</span><span style=\"background-color: #272822\">                                                                                                   </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 5 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">\"\"\"</span><span style=\"background-color: #272822\">                                                                                                       </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 6 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">    Calculate the Mean Reciprocal Rank (MRR) for a single query.</span><span style=\"background-color: #272822\">                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 7 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 8 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">    Args:</span><span style=\"background-color: #272822\">                                                                                                     </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 9 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        model_output (List[Dict[str, Any]]): The list of retrieved documents from the model.</span><span style=\"background-color: #272822\">                  </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">10 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">            Each dictionary contains:</span><span style=\"background-color: #272822\">                                                                         </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">11 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">                - 'source': A unique identifier for the document.</span><span style=\"background-color: #272822\">                                             </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">12 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">                - 'score': The relevance score of the document.</span><span style=\"background-color: #272822\">                                               </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">13 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        contexts (List[Dict[str, Any]]): A list of dictionaries representing the relevant contexts.</span><span style=\"background-color: #272822\">           </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">14 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">            Each dictionary contains:</span><span style=\"background-color: #272822\">                                                                         </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">15 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">                - 'source': A unique identifier for the relevant document.</span><span style=\"background-color: #272822\">                                    </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">16 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">17 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">    Returns:</span><span style=\"background-color: #272822\">                                                                                                  </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">18 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        float: The MRR score for the given query.</span><span style=\"background-color: #272822\">                                                             </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">19 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">20 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">    MRR measures the rank of the first relevant document in the result list.</span><span style=\"background-color: #272822\">                                  </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">21 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">22 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">    If no relevant document is found, MRR is 0.</span><span style=\"background-color: #272822\">                                                               </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">23 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">24 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">    This metric is useful for evaluating systems where there is typically one relevant document</span><span style=\"background-color: #272822\">               </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">25 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">    and the user is interested in finding that document quickly.</span><span style=\"background-color: #272822\">                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">26 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">    \"\"\"</span><span style=\"background-color: #272822\">                                                                                                       </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">27 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    relevant_sources </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> [</span><span style=\"background-color: #272822\">                                                                                      </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">28 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">        context[</span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">\"source\"</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">] </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">for</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> context </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">in</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> contexts </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">if</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> context[</span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">\"relevance\"</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">] </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">!=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> </span><span style=\"color: #ae81ff; text-decoration-color: #ae81ff; background-color: #272822\">0</span><span style=\"background-color: #272822\">                                </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">29 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    ]</span><span style=\"background-color: #272822\">                                                                                                         </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">30 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">31 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    mrr_score </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> </span><span style=\"color: #ae81ff; text-decoration-color: #ae81ff; background-color: #272822\">0</span><span style=\"background-color: #272822\">                                                                                             </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">32 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">for</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> rank, result </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">in</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> enumerate(model_output, </span><span style=\"color: #ae81ff; text-decoration-color: #ae81ff; background-color: #272822\">1</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">):</span><span style=\"background-color: #272822\">                                                           </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">33 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">        </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">if</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> result[</span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">\"source\"</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">] </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">in</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> relevant_sources:</span><span style=\"background-color: #272822\">                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">34 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">            mrr_score </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">+=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> </span><span style=\"color: #ae81ff; text-decoration-color: #ae81ff; background-color: #272822\">1</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">/</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> rank</span><span style=\"background-color: #272822\">                                                                             </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">35 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">36 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">if</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> mrr_score </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">==</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> </span><span style=\"color: #ae81ff; text-decoration-color: #ae81ff; background-color: #272822\">0</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">:</span><span style=\"background-color: #272822\">                                                                                        </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">37 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">        </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">return</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> </span><span style=\"color: #ae81ff; text-decoration-color: #ae81ff; background-color: #272822\">0.0</span><span style=\"background-color: #272822\">                                                                                            </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">38 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">else</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">:</span><span style=\"background-color: #272822\">                                                                                                     </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">39 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">        </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">return</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> mrr_score </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">/</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> len(model_output)</span><span style=\"background-color: #272822\">                                                                  </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">40 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 1 \u001b[0m\u001b[38;2;166;226;46;48;2;39;40;34m@weave\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m.\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mop\u001b[0m\u001b[48;2;39;40;34m                                                                                                     \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 2 \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mdef\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;166;226;46;48;2;39;40;34mcompute_ndcg\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[48;2;39;40;34m                                                                                             \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 3 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mmodel_output\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mList\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mDict\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mstr\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mAny\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mcontexts\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mList\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mDict\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mstr\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mAny\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[48;2;39;40;34m                                        \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 4 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m-\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m>\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mfloat\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[48;2;39;40;34m                                                                                                   \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 5 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\"\"\u001b[0m\u001b[48;2;39;40;34m                                                                                                       \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 6 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m    Calculate the Normalized Discounted Cumulative Gain (NDCG) for a single query.\u001b[0m\u001b[48;2;39;40;34m                            \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 7 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 8 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m    Args:\u001b[0m\u001b[48;2;39;40;34m                                                                                                     \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 9 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        model_output (List[Dict[str, Any]]): The list of retrieved documents from the model.\u001b[0m\u001b[48;2;39;40;34m                  \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m10 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m            Each dictionary contains:\u001b[0m\u001b[48;2;39;40;34m                                                                         \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m11 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m                - 'source': A unique identifier for the document.\u001b[0m\u001b[48;2;39;40;34m                                             \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m12 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        contexts (List[Dict[str, Any]]): A list of dictionaries representing the relevant contexts.\u001b[0m\u001b[48;2;39;40;34m           \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m13 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m            Each dictionary contains:\u001b[0m\u001b[48;2;39;40;34m                                                                         \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m14 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m                - 'source': A unique identifier for the relevant document.\u001b[0m\u001b[48;2;39;40;34m                                    \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m15 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m                - 'relevance': The relevance score of the document (0, 1, or 2).\u001b[0m\u001b[48;2;39;40;34m                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m16 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m17 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m    Returns:\u001b[0m\u001b[48;2;39;40;34m                                                                                                  \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m18 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        float: The NDCG score for the given query.\u001b[0m\u001b[48;2;39;40;34m                                                            \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m19 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m    \"\"\"\u001b[0m\u001b[48;2;39;40;34m                                                                                                       \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m20 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mrelevance_map\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m{\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mcontext\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34msource\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mcontext\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34mrelevance\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mfor\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mcontext\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34min\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mcontexts\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m}\u001b[0m\u001b[48;2;39;40;34m                         \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m21 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m22 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mdcg\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;174;129;255;48;2;39;40;34m0.0\u001b[0m\u001b[48;2;39;40;34m                                                                                                 \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m23 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34midcg\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;174;129;255;48;2;39;40;34m0.0\u001b[0m\u001b[48;2;39;40;34m                                                                                                \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m24 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m25 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mfor\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mi\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mresult\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34min\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34menumerate\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mmodel_output\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[48;2;39;40;34m                                                                 \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m26 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m        \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mrel\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mrelevance_map\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m.\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mget\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mresult\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34msource\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;174;129;255;48;2;39;40;34m0\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[48;2;39;40;34m                                                          \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m27 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m        \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mdcg\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m+\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;174;129;255;48;2;39;40;34m2\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m*\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m*\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mrel\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m-\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;174;129;255;48;2;39;40;34m1\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m/\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mnp\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m.\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mlog2\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mi\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m+\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;174;129;255;48;2;39;40;34m2\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[48;2;39;40;34m                                                                  \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m28 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m29 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34msorted_relevances\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34msorted\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[48;2;39;40;34m                                                                               \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m30 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m        \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mcontext\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34mrelevance\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mfor\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mcontext\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34min\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mcontexts\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mreverse\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mTrue\u001b[0m\u001b[48;2;39;40;34m                                          \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m31 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[48;2;39;40;34m                                                                                                         \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m32 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mfor\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mi\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mrel\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34min\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34menumerate\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34msorted_relevances\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[48;2;39;40;34m                                                               \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m33 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m        \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34midcg\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m+\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;174;129;255;48;2;39;40;34m2\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m*\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m*\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mrel\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m-\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;174;129;255;48;2;39;40;34m1\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m/\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mnp\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m.\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mlog2\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mi\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m+\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;174;129;255;48;2;39;40;34m2\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[48;2;39;40;34m                                                                 \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m34 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m35 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mif\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34midcg\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m==\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;174;129;255;48;2;39;40;34m0\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[48;2;39;40;34m                                                                                             \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m36 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m        \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mreturn\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;174;129;255;48;2;39;40;34m0.0\u001b[0m\u001b[48;2;39;40;34m                                                                                            \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m37 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m38 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mndcg\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mdcg\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m/\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34midcg\u001b[0m\u001b[48;2;39;40;34m                                                                                         \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m39 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mreturn\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mndcg\u001b[0m\u001b[48;2;39;40;34m                                                                                               \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m40 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 1 </span><span style=\"color: #a6e22e; text-decoration-color: #a6e22e; background-color: #272822\">@weave</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">.</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">op</span><span style=\"background-color: #272822\">                                                                                                     </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 2 </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">def</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> </span><span style=\"color: #a6e22e; text-decoration-color: #a6e22e; background-color: #272822\">compute_ndcg</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">(</span><span style=\"background-color: #272822\">                                                                                             </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 3 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    model_output: List[Dict[str, Any]], contexts: List[Dict[str, Any]]</span><span style=\"background-color: #272822\">                                        </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 4 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">) </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">-&gt;</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> float:</span><span style=\"background-color: #272822\">                                                                                                   </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 5 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">\"\"\"</span><span style=\"background-color: #272822\">                                                                                                       </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 6 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">    Calculate the Normalized Discounted Cumulative Gain (NDCG) for a single query.</span><span style=\"background-color: #272822\">                            </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 7 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 8 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">    Args:</span><span style=\"background-color: #272822\">                                                                                                     </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 9 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        model_output (List[Dict[str, Any]]): The list of retrieved documents from the model.</span><span style=\"background-color: #272822\">                  </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">10 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">            Each dictionary contains:</span><span style=\"background-color: #272822\">                                                                         </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">11 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">                - 'source': A unique identifier for the document.</span><span style=\"background-color: #272822\">                                             </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">12 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        contexts (List[Dict[str, Any]]): A list of dictionaries representing the relevant contexts.</span><span style=\"background-color: #272822\">           </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">13 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">            Each dictionary contains:</span><span style=\"background-color: #272822\">                                                                         </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">14 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">                - 'source': A unique identifier for the relevant document.</span><span style=\"background-color: #272822\">                                    </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">15 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">                - 'relevance': The relevance score of the document (0, 1, or 2).</span><span style=\"background-color: #272822\">                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">16 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">17 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">    Returns:</span><span style=\"background-color: #272822\">                                                                                                  </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">18 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        float: The NDCG score for the given query.</span><span style=\"background-color: #272822\">                                                            </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">19 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">    \"\"\"</span><span style=\"background-color: #272822\">                                                                                                       </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">20 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    relevance_map </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> {context[</span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">\"source\"</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">]: context[</span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">\"relevance\"</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">] </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">for</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> context </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">in</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> contexts}</span><span style=\"background-color: #272822\">                         </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">21 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">22 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    dcg </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> </span><span style=\"color: #ae81ff; text-decoration-color: #ae81ff; background-color: #272822\">0.0</span><span style=\"background-color: #272822\">                                                                                                 </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">23 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    idcg </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> </span><span style=\"color: #ae81ff; text-decoration-color: #ae81ff; background-color: #272822\">0.0</span><span style=\"background-color: #272822\">                                                                                                </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">24 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">25 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">for</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> i, result </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">in</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> enumerate(model_output):</span><span style=\"background-color: #272822\">                                                                 </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">26 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">        rel </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> relevance_map</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">.</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">get(result[</span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">\"source\"</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">], </span><span style=\"color: #ae81ff; text-decoration-color: #ae81ff; background-color: #272822\">0</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">)</span><span style=\"background-color: #272822\">                                                          </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">27 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">        dcg </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">+=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> (</span><span style=\"color: #ae81ff; text-decoration-color: #ae81ff; background-color: #272822\">2</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">**</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">rel </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">-</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> </span><span style=\"color: #ae81ff; text-decoration-color: #ae81ff; background-color: #272822\">1</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">) </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">/</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> np</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">.</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">log2(i </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">+</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> </span><span style=\"color: #ae81ff; text-decoration-color: #ae81ff; background-color: #272822\">2</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">)</span><span style=\"background-color: #272822\">                                                                  </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">28 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">29 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    sorted_relevances </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> sorted(</span><span style=\"background-color: #272822\">                                                                               </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">30 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">        [context[</span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">\"relevance\"</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">] </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">for</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> context </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">in</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> contexts], reverse</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">True</span><span style=\"background-color: #272822\">                                          </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">31 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    )</span><span style=\"background-color: #272822\">                                                                                                         </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">32 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">for</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> i, rel </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">in</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> enumerate(sorted_relevances):</span><span style=\"background-color: #272822\">                                                               </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">33 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">        idcg </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">+=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> (</span><span style=\"color: #ae81ff; text-decoration-color: #ae81ff; background-color: #272822\">2</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">**</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">rel </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">-</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> </span><span style=\"color: #ae81ff; text-decoration-color: #ae81ff; background-color: #272822\">1</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">) </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">/</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> np</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">.</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">log2(i </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">+</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> </span><span style=\"color: #ae81ff; text-decoration-color: #ae81ff; background-color: #272822\">2</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">)</span><span style=\"background-color: #272822\">                                                                 </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">34 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">35 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">if</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> idcg </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">==</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> </span><span style=\"color: #ae81ff; text-decoration-color: #ae81ff; background-color: #272822\">0</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">:</span><span style=\"background-color: #272822\">                                                                                             </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">36 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">        </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">return</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> </span><span style=\"color: #ae81ff; text-decoration-color: #ae81ff; background-color: #272822\">0.0</span><span style=\"background-color: #272822\">                                                                                            </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">37 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">38 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    ndcg </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> dcg </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">/</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> idcg</span><span style=\"background-color: #272822\">                                                                                         </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">39 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">return</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> ndcg</span><span style=\"background-color: #272822\">                                                                                               </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">40 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 1 \u001b[0m\u001b[38;2;166;226;46;48;2;39;40;34m@weave\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m.\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mop\u001b[0m\u001b[48;2;39;40;34m                                                                                                     \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 2 \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mdef\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;166;226;46;48;2;39;40;34mcompute_map\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[48;2;39;40;34m                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 3 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mmodel_output\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mList\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mDict\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mstr\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mAny\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mcontexts\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mList\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mDict\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mstr\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mAny\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[48;2;39;40;34m                                        \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 4 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m-\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m>\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mfloat\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[48;2;39;40;34m                                                                                                   \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 5 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\"\"\u001b[0m\u001b[48;2;39;40;34m                                                                                                       \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 6 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m    Calculate the Mean Average Precision (MAP) for a single query.\u001b[0m\u001b[48;2;39;40;34m                                            \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 7 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 8 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m    Args:\u001b[0m\u001b[48;2;39;40;34m                                                                                                     \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 9 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        model_output (List[Dict[str, Any]]): The list of retrieved documents from the model.\u001b[0m\u001b[48;2;39;40;34m                  \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m10 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m            Each dictionary contains:\u001b[0m\u001b[48;2;39;40;34m                                                                         \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m11 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m                - 'source': A unique identifier for the document.\u001b[0m\u001b[48;2;39;40;34m                                             \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m12 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m                - 'score': The relevance score of the document.\u001b[0m\u001b[48;2;39;40;34m                                               \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m13 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        contexts (List[Dict[str, Any]]): A list of dictionaries representing the relevant contexts.\u001b[0m\u001b[48;2;39;40;34m           \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m14 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m            Each dictionary contains:\u001b[0m\u001b[48;2;39;40;34m                                                                         \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m15 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m                - 'source': A unique identifier for the relevant document.\u001b[0m\u001b[48;2;39;40;34m                                    \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m16 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m17 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m    Returns:\u001b[0m\u001b[48;2;39;40;34m                                                                                                  \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m18 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        float: The MAP score for the given query.\u001b[0m\u001b[48;2;39;40;34m                                                             \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m19 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m20 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m    MAP provides a single-figure measure of quality across recall levels.\u001b[0m\u001b[48;2;39;40;34m                                     \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m21 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m    For a single query, it's equivalent to the Average Precision (AP).\u001b[0m\u001b[48;2;39;40;34m                                        \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m22 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m23 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m24 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m    Where:\u001b[0m\u001b[48;2;39;40;34m                                                                                                    \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m25 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m    - n is the number of retrieved documents\u001b[0m\u001b[48;2;39;40;34m                                                                  \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m26 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m    - P(k) is the precision at cut-off k in the list\u001b[0m\u001b[48;2;39;40;34m                                                          \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m27 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m    - rel(k) is an indicator function: 1 if the item at rank k is relevant, 0 otherwise\u001b[0m\u001b[48;2;39;40;34m                       \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m28 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m    MAP considers both precision and recall, as well as the ranking of relevant documents.\u001b[0m\u001b[48;2;39;40;34m                    \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m29 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m30 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m    \"\"\"\u001b[0m\u001b[48;2;39;40;34m                                                                                                       \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m31 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mrelevant_sources\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m{\u001b[0m\u001b[48;2;39;40;34m                                                                                      \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m32 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m        \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mcontext\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34msource\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mfor\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mcontext\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34min\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mcontexts\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mif\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mcontext\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34mrelevance\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m!=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;174;129;255;48;2;39;40;34m0\u001b[0m\u001b[48;2;39;40;34m                                \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m33 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m}\u001b[0m\u001b[48;2;39;40;34m                                                                                                         \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m34 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m35 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mnum_relevant\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;174;129;255;48;2;39;40;34m0\u001b[0m\u001b[48;2;39;40;34m                                                                                          \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m36 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34msum_precision\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;174;129;255;48;2;39;40;34m0.0\u001b[0m\u001b[48;2;39;40;34m                                                                                       \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m37 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m38 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mfor\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mi\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mresult\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34min\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34menumerate\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mmodel_output\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[48;2;39;40;34m                                                                 \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m39 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m        \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mif\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mresult\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34msource\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34min\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mrelevant_sources\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[48;2;39;40;34m                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m40 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m            \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mnum_relevant\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m+\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;174;129;255;48;2;39;40;34m1\u001b[0m\u001b[48;2;39;40;34m                                                                                 \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m41 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m            \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34msum_precision\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m+\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mnum_relevant\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m/\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mi\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m+\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;174;129;255;48;2;39;40;34m1\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[48;2;39;40;34m                                                           \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m42 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m43 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mif\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mnum_relevant\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m==\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;174;129;255;48;2;39;40;34m0\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[48;2;39;40;34m                                                                                     \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m44 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m        \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mreturn\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;174;129;255;48;2;39;40;34m0.0\u001b[0m\u001b[48;2;39;40;34m                                                                                            \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m45 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m46 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34maverage_precision\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34msum_precision\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m/\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mlen\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mrelevant_sources\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[48;2;39;40;34m                                                 \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m47 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mreturn\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34maverage_precision\u001b[0m\u001b[48;2;39;40;34m                                                                                  \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m48 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 1 </span><span style=\"color: #a6e22e; text-decoration-color: #a6e22e; background-color: #272822\">@weave</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">.</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">op</span><span style=\"background-color: #272822\">                                                                                                     </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 2 </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">def</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> </span><span style=\"color: #a6e22e; text-decoration-color: #a6e22e; background-color: #272822\">compute_map</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">(</span><span style=\"background-color: #272822\">                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 3 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    model_output: List[Dict[str, Any]], contexts: List[Dict[str, Any]]</span><span style=\"background-color: #272822\">                                        </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 4 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">) </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">-&gt;</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> float:</span><span style=\"background-color: #272822\">                                                                                                   </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 5 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">\"\"\"</span><span style=\"background-color: #272822\">                                                                                                       </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 6 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">    Calculate the Mean Average Precision (MAP) for a single query.</span><span style=\"background-color: #272822\">                                            </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 7 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 8 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">    Args:</span><span style=\"background-color: #272822\">                                                                                                     </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 9 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        model_output (List[Dict[str, Any]]): The list of retrieved documents from the model.</span><span style=\"background-color: #272822\">                  </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">10 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">            Each dictionary contains:</span><span style=\"background-color: #272822\">                                                                         </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">11 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">                - 'source': A unique identifier for the document.</span><span style=\"background-color: #272822\">                                             </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">12 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">                - 'score': The relevance score of the document.</span><span style=\"background-color: #272822\">                                               </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">13 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        contexts (List[Dict[str, Any]]): A list of dictionaries representing the relevant contexts.</span><span style=\"background-color: #272822\">           </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">14 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">            Each dictionary contains:</span><span style=\"background-color: #272822\">                                                                         </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">15 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">                - 'source': A unique identifier for the relevant document.</span><span style=\"background-color: #272822\">                                    </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">16 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">17 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">    Returns:</span><span style=\"background-color: #272822\">                                                                                                  </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">18 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        float: The MAP score for the given query.</span><span style=\"background-color: #272822\">                                                             </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">19 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">20 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">    MAP provides a single-figure measure of quality across recall levels.</span><span style=\"background-color: #272822\">                                     </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">21 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">    For a single query, it's equivalent to the Average Precision (AP).</span><span style=\"background-color: #272822\">                                        </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">22 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">23 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">24 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">    Where:</span><span style=\"background-color: #272822\">                                                                                                    </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">25 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">    - n is the number of retrieved documents</span><span style=\"background-color: #272822\">                                                                  </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">26 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">    - P(k) is the precision at cut-off k in the list</span><span style=\"background-color: #272822\">                                                          </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">27 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">    - rel(k) is an indicator function: 1 if the item at rank k is relevant, 0 otherwise</span><span style=\"background-color: #272822\">                       </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">28 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">    MAP considers both precision and recall, as well as the ranking of relevant documents.</span><span style=\"background-color: #272822\">                    </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">29 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">30 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">    \"\"\"</span><span style=\"background-color: #272822\">                                                                                                       </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">31 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    relevant_sources </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> {</span><span style=\"background-color: #272822\">                                                                                      </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">32 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">        context[</span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">\"source\"</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">] </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">for</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> context </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">in</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> contexts </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">if</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> context[</span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">\"relevance\"</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">] </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">!=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> </span><span style=\"color: #ae81ff; text-decoration-color: #ae81ff; background-color: #272822\">0</span><span style=\"background-color: #272822\">                                </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">33 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    }</span><span style=\"background-color: #272822\">                                                                                                         </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">34 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">35 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    num_relevant </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> </span><span style=\"color: #ae81ff; text-decoration-color: #ae81ff; background-color: #272822\">0</span><span style=\"background-color: #272822\">                                                                                          </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">36 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    sum_precision </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> </span><span style=\"color: #ae81ff; text-decoration-color: #ae81ff; background-color: #272822\">0.0</span><span style=\"background-color: #272822\">                                                                                       </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">37 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">38 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">for</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> i, result </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">in</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> enumerate(model_output):</span><span style=\"background-color: #272822\">                                                                 </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">39 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">        </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">if</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> result[</span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">\"source\"</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">] </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">in</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> relevant_sources:</span><span style=\"background-color: #272822\">                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">40 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">            num_relevant </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">+=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> </span><span style=\"color: #ae81ff; text-decoration-color: #ae81ff; background-color: #272822\">1</span><span style=\"background-color: #272822\">                                                                                 </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">41 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">            sum_precision </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">+=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> num_relevant </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">/</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> (i </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">+</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> </span><span style=\"color: #ae81ff; text-decoration-color: #ae81ff; background-color: #272822\">1</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">)</span><span style=\"background-color: #272822\">                                                           </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">42 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">43 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">if</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> num_relevant </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">==</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> </span><span style=\"color: #ae81ff; text-decoration-color: #ae81ff; background-color: #272822\">0</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">:</span><span style=\"background-color: #272822\">                                                                                     </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">44 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">        </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">return</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> </span><span style=\"color: #ae81ff; text-decoration-color: #ae81ff; background-color: #272822\">0.0</span><span style=\"background-color: #272822\">                                                                                            </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">45 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">46 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    average_precision </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> sum_precision </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">/</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> len(relevant_sources)</span><span style=\"background-color: #272822\">                                                 </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">47 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">return</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> average_precision</span><span style=\"background-color: #272822\">                                                                                  </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">48 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 1 \u001b[0m\u001b[38;2;166;226;46;48;2;39;40;34m@weave\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m.\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mop\u001b[0m\u001b[48;2;39;40;34m                                                                                                     \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 2 \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mdef\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;166;226;46;48;2;39;40;34mcompute_precision\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[48;2;39;40;34m                                                                                        \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 3 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mmodel_output\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mList\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mDict\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mstr\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mAny\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mcontexts\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mList\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mDict\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mstr\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mAny\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[48;2;39;40;34m                                        \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 4 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m-\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m>\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mfloat\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[48;2;39;40;34m                                                                                                   \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 5 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\"\"\u001b[0m\u001b[48;2;39;40;34m                                                                                                       \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 6 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m    Calculate the Precision for a single query.\u001b[0m\u001b[48;2;39;40;34m                                                               \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 7 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 8 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m    Args:\u001b[0m\u001b[48;2;39;40;34m                                                                                                     \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 9 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        model_output (List[Dict[str, Any]]): The list of retrieved documents from the model.\u001b[0m\u001b[48;2;39;40;34m                  \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m10 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m            Each dictionary contains:\u001b[0m\u001b[48;2;39;40;34m                                                                         \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m11 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m                - 'source': A unique identifier for the document.\u001b[0m\u001b[48;2;39;40;34m                                             \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m12 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m                - 'score': The relevance score of the document.\u001b[0m\u001b[48;2;39;40;34m                                               \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m13 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        contexts (List[Dict[str, Any]]): A list of dictionaries representing the relevant contexts.\u001b[0m\u001b[48;2;39;40;34m           \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m14 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m            Each dictionary contains:\u001b[0m\u001b[48;2;39;40;34m                                                                         \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m15 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m                - 'source': A unique identifier for the relevant document.\u001b[0m\u001b[48;2;39;40;34m                                    \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m16 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m17 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m    Returns:\u001b[0m\u001b[48;2;39;40;34m                                                                                                  \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m18 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        float: The Precision score for the given query.\u001b[0m\u001b[48;2;39;40;34m                                                       \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m19 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m20 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m    Precision measures the proportion of retrieved documents that are relevant.\u001b[0m\u001b[48;2;39;40;34m                               \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m21 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m    \"\"\"\u001b[0m\u001b[48;2;39;40;34m                                                                                                       \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m22 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mrelevant_sources\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m{\u001b[0m\u001b[48;2;39;40;34m                                                                                      \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m23 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m        \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mcontext\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34msource\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mfor\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mcontext\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34min\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mcontexts\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mif\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mcontext\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34mrelevance\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m!=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;174;129;255;48;2;39;40;34m0\u001b[0m\u001b[48;2;39;40;34m                                \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m24 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m}\u001b[0m\u001b[48;2;39;40;34m                                                                                                         \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m25 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mretrieved_sources\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m{\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mresult\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34msource\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mfor\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mresult\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34min\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mmodel_output\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m}\u001b[0m\u001b[48;2;39;40;34m                                         \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m26 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m27 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mrelevant_retrieved\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mrelevant_sources\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m&\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mretrieved_sources\u001b[0m\u001b[48;2;39;40;34m                                                 \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m28 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m29 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mprecision\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[48;2;39;40;34m                                                                                             \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m30 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m        \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mlen\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mrelevant_retrieved\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m/\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mlen\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mretrieved_sources\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mif\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mretrieved_sources\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34melse\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;174;129;255;48;2;39;40;34m0.0\u001b[0m\u001b[48;2;39;40;34m                        \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m31 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[48;2;39;40;34m                                                                                                         \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m32 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mreturn\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mprecision\u001b[0m\u001b[48;2;39;40;34m                                                                                          \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m33 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 1 </span><span style=\"color: #a6e22e; text-decoration-color: #a6e22e; background-color: #272822\">@weave</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">.</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">op</span><span style=\"background-color: #272822\">                                                                                                     </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 2 </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">def</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> </span><span style=\"color: #a6e22e; text-decoration-color: #a6e22e; background-color: #272822\">compute_precision</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">(</span><span style=\"background-color: #272822\">                                                                                        </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 3 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    model_output: List[Dict[str, Any]], contexts: List[Dict[str, Any]]</span><span style=\"background-color: #272822\">                                        </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 4 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">) </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">-&gt;</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> float:</span><span style=\"background-color: #272822\">                                                                                                   </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 5 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">\"\"\"</span><span style=\"background-color: #272822\">                                                                                                       </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 6 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">    Calculate the Precision for a single query.</span><span style=\"background-color: #272822\">                                                               </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 7 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 8 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">    Args:</span><span style=\"background-color: #272822\">                                                                                                     </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 9 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        model_output (List[Dict[str, Any]]): The list of retrieved documents from the model.</span><span style=\"background-color: #272822\">                  </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">10 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">            Each dictionary contains:</span><span style=\"background-color: #272822\">                                                                         </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">11 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">                - 'source': A unique identifier for the document.</span><span style=\"background-color: #272822\">                                             </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">12 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">                - 'score': The relevance score of the document.</span><span style=\"background-color: #272822\">                                               </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">13 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        contexts (List[Dict[str, Any]]): A list of dictionaries representing the relevant contexts.</span><span style=\"background-color: #272822\">           </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">14 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">            Each dictionary contains:</span><span style=\"background-color: #272822\">                                                                         </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">15 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">                - 'source': A unique identifier for the relevant document.</span><span style=\"background-color: #272822\">                                    </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">16 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">17 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">    Returns:</span><span style=\"background-color: #272822\">                                                                                                  </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">18 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        float: The Precision score for the given query.</span><span style=\"background-color: #272822\">                                                       </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">19 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">20 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">    Precision measures the proportion of retrieved documents that are relevant.</span><span style=\"background-color: #272822\">                               </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">21 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">    \"\"\"</span><span style=\"background-color: #272822\">                                                                                                       </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">22 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    relevant_sources </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> {</span><span style=\"background-color: #272822\">                                                                                      </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">23 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">        context[</span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">\"source\"</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">] </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">for</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> context </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">in</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> contexts </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">if</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> context[</span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">\"relevance\"</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">] </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">!=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> </span><span style=\"color: #ae81ff; text-decoration-color: #ae81ff; background-color: #272822\">0</span><span style=\"background-color: #272822\">                                </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">24 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    }</span><span style=\"background-color: #272822\">                                                                                                         </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">25 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    retrieved_sources </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> {result[</span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">\"source\"</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">] </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">for</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> result </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">in</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> model_output}</span><span style=\"background-color: #272822\">                                         </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">26 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">27 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    relevant_retrieved </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> relevant_sources </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">&amp;</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> retrieved_sources</span><span style=\"background-color: #272822\">                                                 </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">28 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">29 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    precision </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> (</span><span style=\"background-color: #272822\">                                                                                             </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">30 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">        len(relevant_retrieved) </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">/</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> len(retrieved_sources) </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">if</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> retrieved_sources </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">else</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> </span><span style=\"color: #ae81ff; text-decoration-color: #ae81ff; background-color: #272822\">0.0</span><span style=\"background-color: #272822\">                        </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">31 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    )</span><span style=\"background-color: #272822\">                                                                                                         </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">32 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">return</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> precision</span><span style=\"background-color: #272822\">                                                                                          </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">33 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 1 \u001b[0m\u001b[38;2;166;226;46;48;2;39;40;34m@weave\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m.\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mop\u001b[0m\u001b[48;2;39;40;34m                                                                                                     \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 2 \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mdef\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;166;226;46;48;2;39;40;34mcompute_recall\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[48;2;39;40;34m                                                                                           \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 3 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mmodel_output\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mList\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mDict\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mstr\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mAny\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mcontexts\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mList\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mDict\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mstr\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mAny\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[48;2;39;40;34m                                        \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 4 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m-\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m>\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mfloat\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[48;2;39;40;34m                                                                                                   \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 5 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\"\"\u001b[0m\u001b[48;2;39;40;34m                                                                                                       \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 6 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m    Calculate the Recall for a single query.\u001b[0m\u001b[48;2;39;40;34m                                                                  \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 7 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 8 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m    Args:\u001b[0m\u001b[48;2;39;40;34m                                                                                                     \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 9 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        model_output (List[Dict[str, Any]]): The list of retrieved documents from the model.\u001b[0m\u001b[48;2;39;40;34m                  \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m10 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m            Each dictionary contains:\u001b[0m\u001b[48;2;39;40;34m                                                                         \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m11 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m                - 'source': A unique identifier for the document.\u001b[0m\u001b[48;2;39;40;34m                                             \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m12 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m                - 'score': The relevance score of the document.\u001b[0m\u001b[48;2;39;40;34m                                               \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m13 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        contexts (List[Dict[str, Any]]): A list of dictionaries representing the relevant contexts.\u001b[0m\u001b[48;2;39;40;34m           \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m14 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m            Each dictionary contains:\u001b[0m\u001b[48;2;39;40;34m                                                                         \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m15 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m                - 'source': A unique identifier for the relevant document.\u001b[0m\u001b[48;2;39;40;34m                                    \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m16 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m17 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m    Returns:\u001b[0m\u001b[48;2;39;40;34m                                                                                                  \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m18 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        float: The Recall score for the given query.\u001b[0m\u001b[48;2;39;40;34m                                                          \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m19 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m20 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m    Recall measures the proportion of relevant documents that are retrieved.\u001b[0m\u001b[48;2;39;40;34m                                  \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m21 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m    \"\"\"\u001b[0m\u001b[48;2;39;40;34m                                                                                                       \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m22 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mrelevant_sources\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m{\u001b[0m\u001b[48;2;39;40;34m                                                                                      \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m23 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m        \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mcontext\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34msource\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mfor\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mcontext\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34min\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mcontexts\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mif\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mcontext\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34mrelevance\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m!=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;174;129;255;48;2;39;40;34m0\u001b[0m\u001b[48;2;39;40;34m                                \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m24 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m}\u001b[0m\u001b[48;2;39;40;34m                                                                                                         \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m25 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mretrieved_sources\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m{\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mresult\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34msource\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mfor\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mresult\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34min\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mmodel_output\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m}\u001b[0m\u001b[48;2;39;40;34m                                         \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m26 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m27 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mrelevant_retrieved\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mrelevant_sources\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m&\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mretrieved_sources\u001b[0m\u001b[48;2;39;40;34m                                                 \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m28 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m29 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mrecall\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[48;2;39;40;34m                                                                                                \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m30 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m        \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mlen\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mrelevant_retrieved\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m/\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mlen\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mrelevant_sources\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mif\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mrelevant_sources\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34melse\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;174;129;255;48;2;39;40;34m0.0\u001b[0m\u001b[48;2;39;40;34m                          \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m31 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[48;2;39;40;34m                                                                                                         \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m32 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mreturn\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mrecall\u001b[0m\u001b[48;2;39;40;34m                                                                                             \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m33 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 1 </span><span style=\"color: #a6e22e; text-decoration-color: #a6e22e; background-color: #272822\">@weave</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">.</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">op</span><span style=\"background-color: #272822\">                                                                                                     </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 2 </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">def</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> </span><span style=\"color: #a6e22e; text-decoration-color: #a6e22e; background-color: #272822\">compute_recall</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">(</span><span style=\"background-color: #272822\">                                                                                           </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 3 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    model_output: List[Dict[str, Any]], contexts: List[Dict[str, Any]]</span><span style=\"background-color: #272822\">                                        </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 4 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">) </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">-&gt;</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> float:</span><span style=\"background-color: #272822\">                                                                                                   </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 5 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">\"\"\"</span><span style=\"background-color: #272822\">                                                                                                       </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 6 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">    Calculate the Recall for a single query.</span><span style=\"background-color: #272822\">                                                                  </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 7 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 8 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">    Args:</span><span style=\"background-color: #272822\">                                                                                                     </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 9 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        model_output (List[Dict[str, Any]]): The list of retrieved documents from the model.</span><span style=\"background-color: #272822\">                  </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">10 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">            Each dictionary contains:</span><span style=\"background-color: #272822\">                                                                         </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">11 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">                - 'source': A unique identifier for the document.</span><span style=\"background-color: #272822\">                                             </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">12 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">                - 'score': The relevance score of the document.</span><span style=\"background-color: #272822\">                                               </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">13 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        contexts (List[Dict[str, Any]]): A list of dictionaries representing the relevant contexts.</span><span style=\"background-color: #272822\">           </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">14 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">            Each dictionary contains:</span><span style=\"background-color: #272822\">                                                                         </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">15 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">                - 'source': A unique identifier for the relevant document.</span><span style=\"background-color: #272822\">                                    </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">16 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">17 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">    Returns:</span><span style=\"background-color: #272822\">                                                                                                  </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">18 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        float: The Recall score for the given query.</span><span style=\"background-color: #272822\">                                                          </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">19 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">20 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">    Recall measures the proportion of relevant documents that are retrieved.</span><span style=\"background-color: #272822\">                                  </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">21 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">    \"\"\"</span><span style=\"background-color: #272822\">                                                                                                       </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">22 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    relevant_sources </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> {</span><span style=\"background-color: #272822\">                                                                                      </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">23 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">        context[</span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">\"source\"</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">] </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">for</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> context </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">in</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> contexts </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">if</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> context[</span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">\"relevance\"</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">] </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">!=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> </span><span style=\"color: #ae81ff; text-decoration-color: #ae81ff; background-color: #272822\">0</span><span style=\"background-color: #272822\">                                </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">24 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    }</span><span style=\"background-color: #272822\">                                                                                                         </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">25 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    retrieved_sources </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> {result[</span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">\"source\"</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">] </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">for</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> result </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">in</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> model_output}</span><span style=\"background-color: #272822\">                                         </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">26 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">27 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    relevant_retrieved </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> relevant_sources </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">&amp;</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> retrieved_sources</span><span style=\"background-color: #272822\">                                                 </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">28 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">29 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    recall </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> (</span><span style=\"background-color: #272822\">                                                                                                </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">30 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">        len(relevant_retrieved) </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">/</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> len(relevant_sources) </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">if</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> relevant_sources </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">else</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> </span><span style=\"color: #ae81ff; text-decoration-color: #ae81ff; background-color: #272822\">0.0</span><span style=\"background-color: #272822\">                          </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">31 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    )</span><span style=\"background-color: #272822\">                                                                                                         </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">32 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">return</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> recall</span><span style=\"background-color: #272822\">                                                                                             </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">33 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 1 \u001b[0m\u001b[38;2;166;226;46;48;2;39;40;34m@weave\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m.\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mop\u001b[0m\u001b[48;2;39;40;34m                                                                                                     \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 2 \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mdef\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;166;226;46;48;2;39;40;34mcompute_f1_score\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[48;2;39;40;34m                                                                                         \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 3 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mmodel_output\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mList\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mDict\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mstr\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mAny\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mcontexts\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mList\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mDict\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mstr\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mAny\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[48;2;39;40;34m                                        \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 4 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m-\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m>\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mfloat\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[48;2;39;40;34m                                                                                                   \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 5 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\"\"\u001b[0m\u001b[48;2;39;40;34m                                                                                                       \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 6 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m    Calculate the F1-Score for a single query.\u001b[0m\u001b[48;2;39;40;34m                                                                \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 7 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 8 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m    Args:\u001b[0m\u001b[48;2;39;40;34m                                                                                                     \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 9 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        model_output (List[Dict[str, Any]]): The list of retrieved documents from the model.\u001b[0m\u001b[48;2;39;40;34m                  \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m10 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m            Each dictionary contains:\u001b[0m\u001b[48;2;39;40;34m                                                                         \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m11 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m                - 'source': A unique identifier for the document.\u001b[0m\u001b[48;2;39;40;34m                                             \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m12 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m                - 'score': The relevance score of the document.\u001b[0m\u001b[48;2;39;40;34m                                               \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m13 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        contexts (List[Dict[str, Any]]): A list of dictionaries representing the relevant contexts.\u001b[0m\u001b[48;2;39;40;34m           \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m14 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m            Each dictionary contains:\u001b[0m\u001b[48;2;39;40;34m                                                                         \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m15 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m                - 'source': A unique identifier for the relevant document.\u001b[0m\u001b[48;2;39;40;34m                                    \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m16 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m17 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m    Returns:\u001b[0m\u001b[48;2;39;40;34m                                                                                                  \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m18 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        float: The F1-Score for the given query.\u001b[0m\u001b[48;2;39;40;34m                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m19 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m20 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m    F1-Score is the harmonic mean of Precision and Recall.\u001b[0m\u001b[48;2;39;40;34m                                                    \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m21 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m    \"\"\"\u001b[0m\u001b[48;2;39;40;34m                                                                                                       \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m22 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mprecision\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mcompute_precision\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mmodel_output\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mcontexts\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[48;2;39;40;34m                                                     \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m23 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mrecall\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mcompute_recall\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mmodel_output\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mcontexts\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[48;2;39;40;34m                                                           \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m24 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m25 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mif\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mprecision\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m+\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mrecall\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m==\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;174;129;255;48;2;39;40;34m0\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[48;2;39;40;34m                                                                               \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m26 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m        \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mreturn\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;174;129;255;48;2;39;40;34m0.0\u001b[0m\u001b[48;2;39;40;34m                                                                                            \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m27 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m28 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mf1_score\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;174;129;255;48;2;39;40;34m2\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m*\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mprecision\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m*\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mrecall\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m/\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mprecision\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m+\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mrecall\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[48;2;39;40;34m                                                \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m29 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mreturn\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mf1_score\u001b[0m\u001b[48;2;39;40;34m                                                                                           \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m30 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 1 </span><span style=\"color: #a6e22e; text-decoration-color: #a6e22e; background-color: #272822\">@weave</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">.</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">op</span><span style=\"background-color: #272822\">                                                                                                     </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 2 </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">def</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> </span><span style=\"color: #a6e22e; text-decoration-color: #a6e22e; background-color: #272822\">compute_f1_score</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">(</span><span style=\"background-color: #272822\">                                                                                         </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 3 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    model_output: List[Dict[str, Any]], contexts: List[Dict[str, Any]]</span><span style=\"background-color: #272822\">                                        </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 4 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">) </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">-&gt;</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> float:</span><span style=\"background-color: #272822\">                                                                                                   </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 5 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">\"\"\"</span><span style=\"background-color: #272822\">                                                                                                       </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 6 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">    Calculate the F1-Score for a single query.</span><span style=\"background-color: #272822\">                                                                </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 7 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 8 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">    Args:</span><span style=\"background-color: #272822\">                                                                                                     </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 9 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        model_output (List[Dict[str, Any]]): The list of retrieved documents from the model.</span><span style=\"background-color: #272822\">                  </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">10 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">            Each dictionary contains:</span><span style=\"background-color: #272822\">                                                                         </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">11 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">                - 'source': A unique identifier for the document.</span><span style=\"background-color: #272822\">                                             </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">12 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">                - 'score': The relevance score of the document.</span><span style=\"background-color: #272822\">                                               </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">13 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        contexts (List[Dict[str, Any]]): A list of dictionaries representing the relevant contexts.</span><span style=\"background-color: #272822\">           </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">14 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">            Each dictionary contains:</span><span style=\"background-color: #272822\">                                                                         </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">15 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">                - 'source': A unique identifier for the relevant document.</span><span style=\"background-color: #272822\">                                    </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">16 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">17 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">    Returns:</span><span style=\"background-color: #272822\">                                                                                                  </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">18 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        float: The F1-Score for the given query.</span><span style=\"background-color: #272822\">                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">19 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">20 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">    F1-Score is the harmonic mean of Precision and Recall.</span><span style=\"background-color: #272822\">                                                    </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">21 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">    \"\"\"</span><span style=\"background-color: #272822\">                                                                                                       </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">22 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    precision </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> compute_precision(model_output, contexts)</span><span style=\"background-color: #272822\">                                                     </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">23 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    recall </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> compute_recall(model_output, contexts)</span><span style=\"background-color: #272822\">                                                           </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">24 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">25 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">if</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> precision </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">+</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> recall </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">==</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> </span><span style=\"color: #ae81ff; text-decoration-color: #ae81ff; background-color: #272822\">0</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">:</span><span style=\"background-color: #272822\">                                                                               </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">26 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">        </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">return</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> </span><span style=\"color: #ae81ff; text-decoration-color: #ae81ff; background-color: #272822\">0.0</span><span style=\"background-color: #272822\">                                                                                            </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">27 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">28 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    f1_score </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> </span><span style=\"color: #ae81ff; text-decoration-color: #ae81ff; background-color: #272822\">2</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">*</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> (precision </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">*</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> recall) </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">/</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> (precision </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">+</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> recall)</span><span style=\"background-color: #272822\">                                                </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">29 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">return</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> f1_score</span><span style=\"background-color: #272822\">                                                                                           </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">30 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "retrieval_evaluation = weave.Evaluation(\n",
        "    name=\"Retrieval_Evaluation\",\n",
        "    dataset=eval_dataset,\n",
        "    scorers=IR_METRICS,\n",
        "    preprocess_model_input=lambda x: {\"query\": x[\"question\"], \"k\": 5},\n",
        ")\n",
        "\n",
        "retrieval_scores = asyncio.run(retrieval_evaluation.evaluate(retriever))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 530
        },
        "id": "nDDsXSeW2fvC",
        "outputId": "cc314798-d27b-401d-fe76-ac8f28fdcbc7"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Evaluated \u001b[1;36m1\u001b[0m of \u001b[1;36m20\u001b[0m examples\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">20</span> examples\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Evaluated \u001b[1;36m2\u001b[0m of \u001b[1;36m20\u001b[0m examples\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">20</span> examples\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Evaluated \u001b[1;36m3\u001b[0m of \u001b[1;36m20\u001b[0m examples\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">3</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">20</span> examples\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Evaluated \u001b[1;36m4\u001b[0m of \u001b[1;36m20\u001b[0m examples\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">20</span> examples\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Evaluated \u001b[1;36m5\u001b[0m of \u001b[1;36m20\u001b[0m examples\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">20</span> examples\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Evaluated \u001b[1;36m6\u001b[0m of \u001b[1;36m20\u001b[0m examples\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">6</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">20</span> examples\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Evaluated \u001b[1;36m7\u001b[0m of \u001b[1;36m20\u001b[0m examples\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">7</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">20</span> examples\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Evaluated \u001b[1;36m8\u001b[0m of \u001b[1;36m20\u001b[0m examples\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">8</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">20</span> examples\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Evaluated \u001b[1;36m9\u001b[0m of \u001b[1;36m20\u001b[0m examples\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">9</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">20</span> examples\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Evaluated \u001b[1;36m10\u001b[0m of \u001b[1;36m20\u001b[0m examples\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">10</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">20</span> examples\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Evaluated \u001b[1;36m11\u001b[0m of \u001b[1;36m20\u001b[0m examples\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">11</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">20</span> examples\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Evaluated \u001b[1;36m12\u001b[0m of \u001b[1;36m20\u001b[0m examples\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">12</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">20</span> examples\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Evaluated \u001b[1;36m13\u001b[0m of \u001b[1;36m20\u001b[0m examples\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">13</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">20</span> examples\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Evaluated \u001b[1;36m14\u001b[0m of \u001b[1;36m20\u001b[0m examples\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">14</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">20</span> examples\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Evaluated \u001b[1;36m15\u001b[0m of \u001b[1;36m20\u001b[0m examples\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">15</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">20</span> examples\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Evaluated \u001b[1;36m16\u001b[0m of \u001b[1;36m20\u001b[0m examples\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">16</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">20</span> examples\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Evaluated \u001b[1;36m17\u001b[0m of \u001b[1;36m20\u001b[0m examples\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">17</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">20</span> examples\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Evaluated \u001b[1;36m18\u001b[0m of \u001b[1;36m20\u001b[0m examples\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">18</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">20</span> examples\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Evaluated \u001b[1;36m19\u001b[0m of \u001b[1;36m20\u001b[0m examples\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">19</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">20</span> examples\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Evaluated \u001b[1;36m20\u001b[0m of \u001b[1;36m20\u001b[0m examples\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">20</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">20</span> examples\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Evaluation summary\n",
              "\u001b[1m{\u001b[0m\n",
              "    \u001b[32m'compute_hit_rate'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m0.37\u001b[0m\u001b[1m}\u001b[0m,\n",
              "    \u001b[32m'compute_mrr'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m0.18533333333333332\u001b[0m\u001b[1m}\u001b[0m,\n",
              "    \u001b[32m'compute_ndcg'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m0.2455147810080979\u001b[0m\u001b[1m}\u001b[0m,\n",
              "    \u001b[32m'compute_map'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m0.45282142857142854\u001b[0m\u001b[1m}\u001b[0m,\n",
              "    \u001b[32m'compute_precision'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m0.31500000000000006\u001b[0m\u001b[1m}\u001b[0m,\n",
              "    \u001b[32m'compute_recall'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m0.29464285714285715\u001b[0m\u001b[1m}\u001b[0m,\n",
              "    \u001b[32m'compute_f1_score'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m0.29965007215007217\u001b[0m\u001b[1m}\u001b[0m,\n",
              "    \u001b[32m'model_latency'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m3.6908488869667053\u001b[0m\u001b[1m}\u001b[0m\n",
              "\u001b[1m}\u001b[0m\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluation summary\n",
              "<span style=\"font-weight: bold\">{</span>\n",
              "    <span style=\"color: #008000; text-decoration-color: #008000\">'compute_hit_rate'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.37</span><span style=\"font-weight: bold\">}</span>,\n",
              "    <span style=\"color: #008000; text-decoration-color: #008000\">'compute_mrr'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.18533333333333332</span><span style=\"font-weight: bold\">}</span>,\n",
              "    <span style=\"color: #008000; text-decoration-color: #008000\">'compute_ndcg'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.2455147810080979</span><span style=\"font-weight: bold\">}</span>,\n",
              "    <span style=\"color: #008000; text-decoration-color: #008000\">'compute_map'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.45282142857142854</span><span style=\"font-weight: bold\">}</span>,\n",
              "    <span style=\"color: #008000; text-decoration-color: #008000\">'compute_precision'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.31500000000000006</span><span style=\"font-weight: bold\">}</span>,\n",
              "    <span style=\"color: #008000; text-decoration-color: #008000\">'compute_recall'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.29464285714285715</span><span style=\"font-weight: bold\">}</span>,\n",
              "    <span style=\"color: #008000; text-decoration-color: #008000\">'compute_f1_score'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.29965007215007217</span><span style=\"font-weight: bold\">}</span>,\n",
              "    <span style=\"color: #008000; text-decoration-color: #008000\">'model_latency'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">3.6908488869667053</span><span style=\"font-weight: bold\">}</span>\n",
              "<span style=\"font-weight: bold\">}</span>\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "🍩 https://wandb.ai/shikhar_dave-iit-jodhpur/rag-course/r/call/019275da-6a1f-76b3-8484-27baf231b941\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from scripts.retrieval_metrics import LLM_METRICS\n",
        "\n",
        "for metric in LLM_METRICS:\n",
        "    display_source(metric)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 337
        },
        "id": "kqpSMVtS2l5x",
        "outputId": "4c650557-4b8f-48ec-c9f5-f24cc88e633e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 1 \u001b[0m\u001b[38;2;166;226;46;48;2;39;40;34m@weave\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m.\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mop\u001b[0m\u001b[48;2;39;40;34m                                                                                                     \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 2 \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34masync\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mdef\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;166;226;46;48;2;39;40;34mllm_retrieval_scorer\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[48;2;39;40;34m                                                                               \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 3 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mmodel_output\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mList\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mDict\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mstr\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mAny\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mquestion\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mstr\u001b[0m\u001b[48;2;39;40;34m                                                         \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 4 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m-\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m>\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mDict\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mstr\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mfloat\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[48;2;39;40;34m                                                                                        \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 5 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\"\"\u001b[0m\u001b[48;2;39;40;34m                                                                                                       \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 6 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m    Evaluate the retrieval results using a language model and compute relevance scores.\u001b[0m\u001b[48;2;39;40;34m                       \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 7 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 8 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m    Args:\u001b[0m\u001b[48;2;39;40;34m                                                                                                     \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 9 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        model_output (List[Dict[str, Any]]): The list of retrieved documents from the model.\u001b[0m\u001b[48;2;39;40;34m                  \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m10 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        question (str): The query or question for which the retrieval is being evaluated.\u001b[0m\u001b[48;2;39;40;34m                     \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m11 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m12 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m    Returns:\u001b[0m\u001b[48;2;39;40;34m                                                                                                  \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m13 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        Dict[str, float]: A dictionary containing the mean relevance score and the relevance rank score.\u001b[0m\u001b[48;2;39;40;34m      \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m14 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m    \"\"\"\u001b[0m\u001b[48;2;39;40;34m                                                                                                       \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m15 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mscores\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mawait\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mevaluate_retrieval_with_llm\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mquestion\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mmodel_output\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[48;2;39;40;34m                                        \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m16 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mrelevance_scores\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mitem\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34mrelevance\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mfor\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mitem\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34min\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mscores\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34mfinal_scores\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[48;2;39;40;34m                                 \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m17 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mmean_relevance\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34msum\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mrelevance_scores\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m/\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mlen\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mmodel_output\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[48;2;39;40;34m                                                \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m18 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mrank_score\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mcompute_rank_score\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mrelevance_scores\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[48;2;39;40;34m                                                         \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m19 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mreturn\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m{\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34mrelevance\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mmean_relevance\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34mrelevance_rank_score\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mrank_score\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m}\u001b[0m\u001b[48;2;39;40;34m                                  \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m20 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 1 </span><span style=\"color: #a6e22e; text-decoration-color: #a6e22e; background-color: #272822\">@weave</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">.</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">op</span><span style=\"background-color: #272822\">                                                                                                     </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 2 </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">async</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">def</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> </span><span style=\"color: #a6e22e; text-decoration-color: #a6e22e; background-color: #272822\">llm_retrieval_scorer</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">(</span><span style=\"background-color: #272822\">                                                                               </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 3 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    model_output: List[Dict[str, Any]], question: str</span><span style=\"background-color: #272822\">                                                         </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 4 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">) </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">-&gt;</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> Dict[str, float]:</span><span style=\"background-color: #272822\">                                                                                        </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 5 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">\"\"\"</span><span style=\"background-color: #272822\">                                                                                                       </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 6 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">    Evaluate the retrieval results using a language model and compute relevance scores.</span><span style=\"background-color: #272822\">                       </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 7 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 8 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">    Args:</span><span style=\"background-color: #272822\">                                                                                                     </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 9 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        model_output (List[Dict[str, Any]]): The list of retrieved documents from the model.</span><span style=\"background-color: #272822\">                  </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">10 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        question (str): The query or question for which the retrieval is being evaluated.</span><span style=\"background-color: #272822\">                     </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">11 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">12 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">    Returns:</span><span style=\"background-color: #272822\">                                                                                                  </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">13 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        Dict[str, float]: A dictionary containing the mean relevance score and the relevance rank score.</span><span style=\"background-color: #272822\">      </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">14 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">    \"\"\"</span><span style=\"background-color: #272822\">                                                                                                       </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">15 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    scores </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">await</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> evaluate_retrieval_with_llm(question, model_output)</span><span style=\"background-color: #272822\">                                        </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">16 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    relevance_scores </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> [item[</span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">\"relevance\"</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">] </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">for</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> item </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">in</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> scores[</span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">\"final_scores\"</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">]]</span><span style=\"background-color: #272822\">                                 </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">17 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    mean_relevance </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> sum(relevance_scores) </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">/</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> len(model_output)</span><span style=\"background-color: #272822\">                                                </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">18 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    rank_score </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> compute_rank_score(relevance_scores)</span><span style=\"background-color: #272822\">                                                         </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">19 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">return</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> {</span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">\"relevance\"</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">: mean_relevance, </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">\"relevance_rank_score\"</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">: rank_score}</span><span style=\"background-color: #272822\">                                  </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">20 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "retrieval_evaluation = weave.Evaluation(\n",
        "    name=\"LLM_Judge_Retrieval_Evaluation\",\n",
        "    dataset=eval_dataset,\n",
        "    scorers=LLM_METRICS,\n",
        "    preprocess_model_input=lambda x: {\"query\": x[\"question\"], \"k\": 5},\n",
        ")\n",
        "retrieval_scores = asyncio.run(retrieval_evaluation.evaluate(retriever))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "xn_57zVA4Gxi",
        "outputId": "39260962-b531-406d-acd6-5a10a02278a3"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Predict and score failed\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Predict and score failed\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/flow/eval.py\", line 287, in eval_example\n",
            "    eval_row = await self.predict_and_score(model, example)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/flow/eval.py\", line 224, in predict_and_score\n",
            "    result = await async_call(score_fn, **score_args)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/retrieval_metrics.py\", line 510, in llm_retrieval_scorer\n",
            "    scores = await evaluate_retrieval_with_llm(question, model_output)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/retrieval_metrics.py\", line 473, in evaluate_retrieval_with_llm\n",
            "    return await call_cohere_with_retry(co_client, messages, len(contexts))\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/retrieval_metrics.py\", line 406, in call_cohere_with_retry\n",
            "    response_text = await make_cohere_api_call(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/utils.py\", line 76, in make_cohere_api_call\n",
            "    response = await co_client.chat(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/client.py\", line 111, in _async_wrapped\n",
            "    return await func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/client.py\", line 40, in _async_wrapped\n",
            "    return await method(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/integrations/cohere/cohere_sdk.py\", line 137, in _wrapper\n",
            "    response = await fn(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/v2/client.py\", line 1662, in chat\n",
            "    raise TooManyRequestsError(\n",
            "cohere.errors.too_many_requests_error.TooManyRequestsError: status_code: 429, body: data=None message=\"You are using a Trial key, which is limited to 10 API calls / minute. You can continue to use the Trial key for free or upgrade to a Production key with higher rate limits at 'https://dashboard.cohere.com/api-keys'. Contact us on 'https://discord.gg/XW44jPfYJu' or email us at support@cohere.com with any questions\"\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Predict and score failed\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Predict and score failed\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/flow/eval.py\", line 287, in eval_example\n",
            "    eval_row = await self.predict_and_score(model, example)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/flow/eval.py\", line 224, in predict_and_score\n",
            "    result = await async_call(score_fn, **score_args)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/retrieval_metrics.py\", line 510, in llm_retrieval_scorer\n",
            "    scores = await evaluate_retrieval_with_llm(question, model_output)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/retrieval_metrics.py\", line 473, in evaluate_retrieval_with_llm\n",
            "    return await call_cohere_with_retry(co_client, messages, len(contexts))\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/retrieval_metrics.py\", line 406, in call_cohere_with_retry\n",
            "    response_text = await make_cohere_api_call(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/utils.py\", line 76, in make_cohere_api_call\n",
            "    response = await co_client.chat(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/client.py\", line 111, in _async_wrapped\n",
            "    return await func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/client.py\", line 40, in _async_wrapped\n",
            "    return await method(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/integrations/cohere/cohere_sdk.py\", line 137, in _wrapper\n",
            "    response = await fn(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/v2/client.py\", line 1662, in chat\n",
            "    raise TooManyRequestsError(\n",
            "cohere.errors.too_many_requests_error.TooManyRequestsError: status_code: 429, body: data=None message=\"You are using a Trial key, which is limited to 10 API calls / minute. You can continue to use the Trial key for free or upgrade to a Production key with higher rate limits at 'https://dashboard.cohere.com/api-keys'. Contact us on 'https://discord.gg/XW44jPfYJu' or email us at support@cohere.com with any questions\"\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Predict and score failed\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Predict and score failed\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/flow/eval.py\", line 287, in eval_example\n",
            "    eval_row = await self.predict_and_score(model, example)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/flow/eval.py\", line 224, in predict_and_score\n",
            "    result = await async_call(score_fn, **score_args)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/retrieval_metrics.py\", line 510, in llm_retrieval_scorer\n",
            "    scores = await evaluate_retrieval_with_llm(question, model_output)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/retrieval_metrics.py\", line 473, in evaluate_retrieval_with_llm\n",
            "    return await call_cohere_with_retry(co_client, messages, len(contexts))\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/retrieval_metrics.py\", line 406, in call_cohere_with_retry\n",
            "    response_text = await make_cohere_api_call(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/utils.py\", line 76, in make_cohere_api_call\n",
            "    response = await co_client.chat(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/client.py\", line 111, in _async_wrapped\n",
            "    return await func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/client.py\", line 40, in _async_wrapped\n",
            "    return await method(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/integrations/cohere/cohere_sdk.py\", line 137, in _wrapper\n",
            "    response = await fn(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/v2/client.py\", line 1662, in chat\n",
            "    raise TooManyRequestsError(\n",
            "cohere.errors.too_many_requests_error.TooManyRequestsError: status_code: 429, body: data=None message=\"You are using a Trial key, which is limited to 10 API calls / minute. You can continue to use the Trial key for free or upgrade to a Production key with higher rate limits at 'https://dashboard.cohere.com/api-keys'. Contact us on 'https://discord.gg/XW44jPfYJu' or email us at support@cohere.com with any questions\"\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Evaluated \u001b[1;36m1\u001b[0m of \u001b[1;36m20\u001b[0m examples\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">20</span> examples\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Evaluated \u001b[1;36m2\u001b[0m of \u001b[1;36m20\u001b[0m examples\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">20</span> examples\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Evaluated \u001b[1;36m3\u001b[0m of \u001b[1;36m20\u001b[0m examples\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">3</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">20</span> examples\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Predict and score failed\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Predict and score failed\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/flow/eval.py\", line 287, in eval_example\n",
            "    eval_row = await self.predict_and_score(model, example)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/flow/eval.py\", line 224, in predict_and_score\n",
            "    result = await async_call(score_fn, **score_args)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/retrieval_metrics.py\", line 510, in llm_retrieval_scorer\n",
            "    scores = await evaluate_retrieval_with_llm(question, model_output)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/retrieval_metrics.py\", line 473, in evaluate_retrieval_with_llm\n",
            "    return await call_cohere_with_retry(co_client, messages, len(contexts))\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/retrieval_metrics.py\", line 406, in call_cohere_with_retry\n",
            "    response_text = await make_cohere_api_call(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/utils.py\", line 76, in make_cohere_api_call\n",
            "    response = await co_client.chat(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/client.py\", line 111, in _async_wrapped\n",
            "    return await func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/client.py\", line 40, in _async_wrapped\n",
            "    return await method(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/integrations/cohere/cohere_sdk.py\", line 137, in _wrapper\n",
            "    response = await fn(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/v2/client.py\", line 1662, in chat\n",
            "    raise TooManyRequestsError(\n",
            "cohere.errors.too_many_requests_error.TooManyRequestsError: status_code: 429, body: data=None message=\"You are using a Trial key, which is limited to 10 API calls / minute. You can continue to use the Trial key for free or upgrade to a Production key with higher rate limits at 'https://dashboard.cohere.com/api-keys'. Contact us on 'https://discord.gg/XW44jPfYJu' or email us at support@cohere.com with any questions\"\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Evaluated \u001b[1;36m4\u001b[0m of \u001b[1;36m20\u001b[0m examples\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">20</span> examples\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Predict and score failed\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Predict and score failed\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/flow/eval.py\", line 287, in eval_example\n",
            "    eval_row = await self.predict_and_score(model, example)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/flow/eval.py\", line 224, in predict_and_score\n",
            "    result = await async_call(score_fn, **score_args)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/retrieval_metrics.py\", line 510, in llm_retrieval_scorer\n",
            "    scores = await evaluate_retrieval_with_llm(question, model_output)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/retrieval_metrics.py\", line 473, in evaluate_retrieval_with_llm\n",
            "    return await call_cohere_with_retry(co_client, messages, len(contexts))\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/retrieval_metrics.py\", line 406, in call_cohere_with_retry\n",
            "    response_text = await make_cohere_api_call(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/utils.py\", line 76, in make_cohere_api_call\n",
            "    response = await co_client.chat(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/client.py\", line 111, in _async_wrapped\n",
            "    return await func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/client.py\", line 40, in _async_wrapped\n",
            "    return await method(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/integrations/cohere/cohere_sdk.py\", line 137, in _wrapper\n",
            "    response = await fn(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/v2/client.py\", line 1662, in chat\n",
            "    raise TooManyRequestsError(\n",
            "cohere.errors.too_many_requests_error.TooManyRequestsError: status_code: 429, body: data=None message=\"You are using a Trial key, which is limited to 10 API calls / minute. You can continue to use the Trial key for free or upgrade to a Production key with higher rate limits at 'https://dashboard.cohere.com/api-keys'. Contact us on 'https://discord.gg/XW44jPfYJu' or email us at support@cohere.com with any questions\"\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Predict and score failed\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Predict and score failed\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/flow/eval.py\", line 287, in eval_example\n",
            "    eval_row = await self.predict_and_score(model, example)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/flow/eval.py\", line 224, in predict_and_score\n",
            "    result = await async_call(score_fn, **score_args)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/retrieval_metrics.py\", line 510, in llm_retrieval_scorer\n",
            "    scores = await evaluate_retrieval_with_llm(question, model_output)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/retrieval_metrics.py\", line 473, in evaluate_retrieval_with_llm\n",
            "    return await call_cohere_with_retry(co_client, messages, len(contexts))\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/retrieval_metrics.py\", line 406, in call_cohere_with_retry\n",
            "    response_text = await make_cohere_api_call(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/utils.py\", line 76, in make_cohere_api_call\n",
            "    response = await co_client.chat(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/client.py\", line 111, in _async_wrapped\n",
            "    return await func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/client.py\", line 40, in _async_wrapped\n",
            "    return await method(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/integrations/cohere/cohere_sdk.py\", line 137, in _wrapper\n",
            "    response = await fn(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/v2/client.py\", line 1662, in chat\n",
            "    raise TooManyRequestsError(\n",
            "cohere.errors.too_many_requests_error.TooManyRequestsError: status_code: 429, body: data=None message=\"You are using a Trial key, which is limited to 10 API calls / minute. You can continue to use the Trial key for free or upgrade to a Production key with higher rate limits at 'https://dashboard.cohere.com/api-keys'. Contact us on 'https://discord.gg/XW44jPfYJu' or email us at support@cohere.com with any questions\"\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Predict and score failed\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Predict and score failed\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/flow/eval.py\", line 287, in eval_example\n",
            "    eval_row = await self.predict_and_score(model, example)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/flow/eval.py\", line 224, in predict_and_score\n",
            "    result = await async_call(score_fn, **score_args)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/retrieval_metrics.py\", line 510, in llm_retrieval_scorer\n",
            "    scores = await evaluate_retrieval_with_llm(question, model_output)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/retrieval_metrics.py\", line 473, in evaluate_retrieval_with_llm\n",
            "    return await call_cohere_with_retry(co_client, messages, len(contexts))\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/retrieval_metrics.py\", line 406, in call_cohere_with_retry\n",
            "    response_text = await make_cohere_api_call(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/utils.py\", line 76, in make_cohere_api_call\n",
            "    response = await co_client.chat(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/client.py\", line 111, in _async_wrapped\n",
            "    return await func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/client.py\", line 40, in _async_wrapped\n",
            "    return await method(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/integrations/cohere/cohere_sdk.py\", line 137, in _wrapper\n",
            "    response = await fn(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/v2/client.py\", line 1662, in chat\n",
            "    raise TooManyRequestsError(\n",
            "cohere.errors.too_many_requests_error.TooManyRequestsError: status_code: 429, body: data=None message=\"You are using a Trial key, which is limited to 10 API calls / minute. You can continue to use the Trial key for free or upgrade to a Production key with higher rate limits at 'https://dashboard.cohere.com/api-keys'. Contact us on 'https://discord.gg/XW44jPfYJu' or email us at support@cohere.com with any questions\"\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Predict and score failed\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Predict and score failed\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/flow/eval.py\", line 287, in eval_example\n",
            "    eval_row = await self.predict_and_score(model, example)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/flow/eval.py\", line 224, in predict_and_score\n",
            "    result = await async_call(score_fn, **score_args)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/retrieval_metrics.py\", line 510, in llm_retrieval_scorer\n",
            "    scores = await evaluate_retrieval_with_llm(question, model_output)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/retrieval_metrics.py\", line 473, in evaluate_retrieval_with_llm\n",
            "    return await call_cohere_with_retry(co_client, messages, len(contexts))\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/retrieval_metrics.py\", line 406, in call_cohere_with_retry\n",
            "    response_text = await make_cohere_api_call(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/utils.py\", line 76, in make_cohere_api_call\n",
            "    response = await co_client.chat(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/client.py\", line 111, in _async_wrapped\n",
            "    return await func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/client.py\", line 40, in _async_wrapped\n",
            "    return await method(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/integrations/cohere/cohere_sdk.py\", line 137, in _wrapper\n",
            "    response = await fn(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/v2/client.py\", line 1662, in chat\n",
            "    raise TooManyRequestsError(\n",
            "cohere.errors.too_many_requests_error.TooManyRequestsError: status_code: 429, body: data=None message=\"You are using a Trial key, which is limited to 10 API calls / minute. You can continue to use the Trial key for free or upgrade to a Production key with higher rate limits at 'https://dashboard.cohere.com/api-keys'. Contact us on 'https://discord.gg/XW44jPfYJu' or email us at support@cohere.com with any questions\"\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Evaluated \u001b[1;36m5\u001b[0m of \u001b[1;36m20\u001b[0m examples\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">20</span> examples\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Evaluated \u001b[1;36m6\u001b[0m of \u001b[1;36m20\u001b[0m examples\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">6</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">20</span> examples\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Evaluated \u001b[1;36m7\u001b[0m of \u001b[1;36m20\u001b[0m examples\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">7</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">20</span> examples\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Evaluated \u001b[1;36m8\u001b[0m of \u001b[1;36m20\u001b[0m examples\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">8</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">20</span> examples\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Predict and score failed\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Predict and score failed\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/flow/eval.py\", line 287, in eval_example\n",
            "    eval_row = await self.predict_and_score(model, example)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/flow/eval.py\", line 224, in predict_and_score\n",
            "    result = await async_call(score_fn, **score_args)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/retrieval_metrics.py\", line 510, in llm_retrieval_scorer\n",
            "    scores = await evaluate_retrieval_with_llm(question, model_output)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/retrieval_metrics.py\", line 473, in evaluate_retrieval_with_llm\n",
            "    return await call_cohere_with_retry(co_client, messages, len(contexts))\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/retrieval_metrics.py\", line 406, in call_cohere_with_retry\n",
            "    response_text = await make_cohere_api_call(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/utils.py\", line 76, in make_cohere_api_call\n",
            "    response = await co_client.chat(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/client.py\", line 111, in _async_wrapped\n",
            "    return await func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/client.py\", line 40, in _async_wrapped\n",
            "    return await method(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/integrations/cohere/cohere_sdk.py\", line 137, in _wrapper\n",
            "    response = await fn(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/v2/client.py\", line 1662, in chat\n",
            "    raise TooManyRequestsError(\n",
            "cohere.errors.too_many_requests_error.TooManyRequestsError: status_code: 429, body: data=None message=\"You are using a Trial key, which is limited to 10 API calls / minute. You can continue to use the Trial key for free or upgrade to a Production key with higher rate limits at 'https://dashboard.cohere.com/api-keys'. Contact us on 'https://discord.gg/XW44jPfYJu' or email us at support@cohere.com with any questions\"\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Predict and score failed\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Predict and score failed\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/flow/eval.py\", line 287, in eval_example\n",
            "    eval_row = await self.predict_and_score(model, example)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/flow/eval.py\", line 224, in predict_and_score\n",
            "    result = await async_call(score_fn, **score_args)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/retrieval_metrics.py\", line 510, in llm_retrieval_scorer\n",
            "    scores = await evaluate_retrieval_with_llm(question, model_output)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/retrieval_metrics.py\", line 473, in evaluate_retrieval_with_llm\n",
            "    return await call_cohere_with_retry(co_client, messages, len(contexts))\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/retrieval_metrics.py\", line 406, in call_cohere_with_retry\n",
            "    response_text = await make_cohere_api_call(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/utils.py\", line 76, in make_cohere_api_call\n",
            "    response = await co_client.chat(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/client.py\", line 111, in _async_wrapped\n",
            "    return await func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/client.py\", line 40, in _async_wrapped\n",
            "    return await method(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/integrations/cohere/cohere_sdk.py\", line 137, in _wrapper\n",
            "    response = await fn(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/v2/client.py\", line 1662, in chat\n",
            "    raise TooManyRequestsError(\n",
            "cohere.errors.too_many_requests_error.TooManyRequestsError: status_code: 429, body: data=None message=\"You are using a Trial key, which is limited to 10 API calls / minute. You can continue to use the Trial key for free or upgrade to a Production key with higher rate limits at 'https://dashboard.cohere.com/api-keys'. Contact us on 'https://discord.gg/XW44jPfYJu' or email us at support@cohere.com with any questions\"\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Evaluated \u001b[1;36m9\u001b[0m of \u001b[1;36m20\u001b[0m examples\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">9</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">20</span> examples\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Evaluated \u001b[1;36m10\u001b[0m of \u001b[1;36m20\u001b[0m examples\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">10</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">20</span> examples\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Evaluated \u001b[1;36m11\u001b[0m of \u001b[1;36m20\u001b[0m examples\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">11</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">20</span> examples\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Evaluated \u001b[1;36m12\u001b[0m of \u001b[1;36m20\u001b[0m examples\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">12</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">20</span> examples\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Evaluated \u001b[1;36m13\u001b[0m of \u001b[1;36m20\u001b[0m examples\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">13</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">20</span> examples\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Evaluated \u001b[1;36m14\u001b[0m of \u001b[1;36m20\u001b[0m examples\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">14</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">20</span> examples\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Evaluated \u001b[1;36m15\u001b[0m of \u001b[1;36m20\u001b[0m examples\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">15</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">20</span> examples\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Predict and score failed\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Predict and score failed\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/flow/eval.py\", line 287, in eval_example\n",
            "    eval_row = await self.predict_and_score(model, example)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/flow/eval.py\", line 224, in predict_and_score\n",
            "    result = await async_call(score_fn, **score_args)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/retrieval_metrics.py\", line 510, in llm_retrieval_scorer\n",
            "    scores = await evaluate_retrieval_with_llm(question, model_output)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/retrieval_metrics.py\", line 473, in evaluate_retrieval_with_llm\n",
            "    return await call_cohere_with_retry(co_client, messages, len(contexts))\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/retrieval_metrics.py\", line 406, in call_cohere_with_retry\n",
            "    response_text = await make_cohere_api_call(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/utils.py\", line 76, in make_cohere_api_call\n",
            "    response = await co_client.chat(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/client.py\", line 111, in _async_wrapped\n",
            "    return await func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/client.py\", line 40, in _async_wrapped\n",
            "    return await method(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/integrations/cohere/cohere_sdk.py\", line 137, in _wrapper\n",
            "    response = await fn(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/v2/client.py\", line 1662, in chat\n",
            "    raise TooManyRequestsError(\n",
            "cohere.errors.too_many_requests_error.TooManyRequestsError: status_code: 429, body: data=None message=\"You are using a Trial key, which is limited to 10 API calls / minute. You can continue to use the Trial key for free or upgrade to a Production key with higher rate limits at 'https://dashboard.cohere.com/api-keys'. Contact us on 'https://discord.gg/XW44jPfYJu' or email us at support@cohere.com with any questions\"\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Evaluated \u001b[1;36m16\u001b[0m of \u001b[1;36m20\u001b[0m examples\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">16</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">20</span> examples\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Evaluated \u001b[1;36m17\u001b[0m of \u001b[1;36m20\u001b[0m examples\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">17</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">20</span> examples\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Evaluated \u001b[1;36m18\u001b[0m of \u001b[1;36m20\u001b[0m examples\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">18</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">20</span> examples\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Evaluated \u001b[1;36m19\u001b[0m of \u001b[1;36m20\u001b[0m examples\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">19</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">20</span> examples\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Evaluated \u001b[1;36m20\u001b[0m of \u001b[1;36m20\u001b[0m examples\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">20</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">20</span> examples\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Evaluation summary\n",
              "\u001b[1m{\u001b[0m\n",
              "    \u001b[32m'llm_retrieval_scorer'\u001b[0m: \u001b[1m{\u001b[0m\n",
              "        \u001b[32m'relevance'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m0.6444444444444444\u001b[0m\u001b[1m}\u001b[0m,\n",
              "        \u001b[32m'relevance_rank_score'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m0.6222222222222222\u001b[0m\u001b[1m}\u001b[0m\n",
              "    \u001b[1m}\u001b[0m,\n",
              "    \u001b[32m'model_latency'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'mean'\u001b[0m: \u001b[1;36m3.8790869447920056\u001b[0m\u001b[1m}\u001b[0m\n",
              "\u001b[1m}\u001b[0m\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluation summary\n",
              "<span style=\"font-weight: bold\">{</span>\n",
              "    <span style=\"color: #008000; text-decoration-color: #008000\">'llm_retrieval_scorer'</span>: <span style=\"font-weight: bold\">{</span>\n",
              "        <span style=\"color: #008000; text-decoration-color: #008000\">'relevance'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.6444444444444444</span><span style=\"font-weight: bold\">}</span>,\n",
              "        <span style=\"color: #008000; text-decoration-color: #008000\">'relevance_rank_score'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.6222222222222222</span><span style=\"font-weight: bold\">}</span>\n",
              "    <span style=\"font-weight: bold\">}</span>,\n",
              "    <span style=\"color: #008000; text-decoration-color: #008000\">'model_latency'</span>: <span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'mean'</span>: <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">3.8790869447920056</span><span style=\"font-weight: bold\">}</span>\n",
              "<span style=\"font-weight: bold\">}</span>\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "🍩 https://wandb.ai/shikhar_dave-iit-jodhpur/rag-course/r/call/019275e0-9973-7083-8666-71ec7bd69357\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from scripts.rag_pipeline import SimpleRAGPipeline\n",
        "from scripts.response_generator import SimpleResponseGenerator\n",
        "\n",
        "INITIAL_PROMPT = open(\"prompts/initial_system.txt\", \"r\").read()\n",
        "response_generator = SimpleResponseGenerator(model=\"command-r\", prompt=INITIAL_PROMPT)\n",
        "rag_pipeline = SimpleRAGPipeline(\n",
        "    retriever=retriever, response_generator=response_generator, top_k=5\n",
        ")"
      ],
      "metadata": {
        "id": "s9C6eIMm4I9e"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from scripts.response_metrics import NLP_METRICS\n",
        "\n",
        "for scorer in NLP_METRICS:\n",
        "    display_source(scorer)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "7NiCurGl4Prq",
        "outputId": "5d039bed-ff15-4584-db6a-66b5612150fb"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 1 \u001b[0m\u001b[38;2;166;226;46;48;2;39;40;34m@weave\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m.\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mop\u001b[0m\u001b[48;2;39;40;34m                                                                                                     \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 2 \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mdef\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;166;226;46;48;2;39;40;34mcompute_diff\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mmodel_output\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mstr\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34manswer\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mstr\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m-\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m>\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mfloat\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[48;2;39;40;34m                                                    \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 3 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\"\"\u001b[0m\u001b[48;2;39;40;34m                                                                                                       \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 4 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m    Compute the similarity ratio between the normalized model output and the expected answer.\u001b[0m\u001b[48;2;39;40;34m                 \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 5 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 6 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m    Args:\u001b[0m\u001b[48;2;39;40;34m                                                                                                     \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 7 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        model_output (str): The output generated by the model.\u001b[0m\u001b[48;2;39;40;34m                                                \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 8 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        answer (str): The expected answer.\u001b[0m\u001b[48;2;39;40;34m                                                                    \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 9 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m10 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m    Returns:\u001b[0m\u001b[48;2;39;40;34m                                                                                                  \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m11 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        float: The similarity ratio between the normalized model output and the expected answer.\u001b[0m\u001b[48;2;39;40;34m              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m12 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m    \"\"\"\u001b[0m\u001b[48;2;39;40;34m                                                                                                       \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m13 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mnorm_output\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mnormalize_text\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mmodel_output\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[48;2;39;40;34m                                                                \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m14 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mnorm_answer\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mnormalize_text\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34manswer\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[48;2;39;40;34m                                                                      \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m15 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mreturn\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mdifflib\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m.\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mSequenceMatcher\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mNone\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mnorm_output\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mnorm_answer\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m.\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mratio\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[48;2;39;40;34m                                    \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m16 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 1 </span><span style=\"color: #a6e22e; text-decoration-color: #a6e22e; background-color: #272822\">@weave</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">.</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">op</span><span style=\"background-color: #272822\">                                                                                                     </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 2 </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">def</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> </span><span style=\"color: #a6e22e; text-decoration-color: #a6e22e; background-color: #272822\">compute_diff</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">(model_output: str, answer: str) </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">-&gt;</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> float:</span><span style=\"background-color: #272822\">                                                    </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 3 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">\"\"\"</span><span style=\"background-color: #272822\">                                                                                                       </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 4 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">    Compute the similarity ratio between the normalized model output and the expected answer.</span><span style=\"background-color: #272822\">                 </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 5 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 6 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">    Args:</span><span style=\"background-color: #272822\">                                                                                                     </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 7 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        model_output (str): The output generated by the model.</span><span style=\"background-color: #272822\">                                                </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 8 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        answer (str): The expected answer.</span><span style=\"background-color: #272822\">                                                                    </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 9 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">10 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">    Returns:</span><span style=\"background-color: #272822\">                                                                                                  </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">11 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        float: The similarity ratio between the normalized model output and the expected answer.</span><span style=\"background-color: #272822\">              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">12 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">    \"\"\"</span><span style=\"background-color: #272822\">                                                                                                       </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">13 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    norm_output </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> normalize_text(model_output)</span><span style=\"background-color: #272822\">                                                                </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">14 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    norm_answer </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> normalize_text(answer)</span><span style=\"background-color: #272822\">                                                                      </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">15 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">return</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> difflib</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">.</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">SequenceMatcher(</span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">None</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">, norm_output, norm_answer)</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">.</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">ratio()</span><span style=\"background-color: #272822\">                                    </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">16 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 1 \u001b[0m\u001b[38;2;166;226;46;48;2;39;40;34m@weave\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m.\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mop\u001b[0m\u001b[48;2;39;40;34m                                                                                                     \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 2 \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mdef\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;166;226;46;48;2;39;40;34mcompute_levenshtein\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mmodel_output\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mstr\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34manswer\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mstr\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m-\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m>\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mfloat\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[48;2;39;40;34m                                             \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 3 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\"\"\u001b[0m\u001b[48;2;39;40;34m                                                                                                       \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 4 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m    Compute the Levenshtein ratio between the normalized model output and the answer.\u001b[0m\u001b[48;2;39;40;34m                         \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 5 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 6 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m    Args:\u001b[0m\u001b[48;2;39;40;34m                                                                                                     \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 7 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        model_output (str): The output generated by the model.\u001b[0m\u001b[48;2;39;40;34m                                                \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 8 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        answer (str): The expected answer.\u001b[0m\u001b[48;2;39;40;34m                                                                    \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 9 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m10 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m    Returns:\u001b[0m\u001b[48;2;39;40;34m                                                                                                  \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m11 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        float: The Levenshtein ratio between the normalized model output and the answer.\u001b[0m\u001b[48;2;39;40;34m                      \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m12 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m    \"\"\"\u001b[0m\u001b[48;2;39;40;34m                                                                                                       \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m13 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mnorm_output\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mnormalize_text\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mmodel_output\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[48;2;39;40;34m                                                                \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m14 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mnorm_answer\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mnormalize_text\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34manswer\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[48;2;39;40;34m                                                                      \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m15 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mreturn\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mLevenshtein\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m.\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mratio\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mnorm_output\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mnorm_answer\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[48;2;39;40;34m                                                        \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m16 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 1 </span><span style=\"color: #a6e22e; text-decoration-color: #a6e22e; background-color: #272822\">@weave</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">.</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">op</span><span style=\"background-color: #272822\">                                                                                                     </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 2 </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">def</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> </span><span style=\"color: #a6e22e; text-decoration-color: #a6e22e; background-color: #272822\">compute_levenshtein</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">(model_output: str, answer: str) </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">-&gt;</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> float:</span><span style=\"background-color: #272822\">                                             </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 3 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">\"\"\"</span><span style=\"background-color: #272822\">                                                                                                       </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 4 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">    Compute the Levenshtein ratio between the normalized model output and the answer.</span><span style=\"background-color: #272822\">                         </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 5 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 6 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">    Args:</span><span style=\"background-color: #272822\">                                                                                                     </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 7 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        model_output (str): The output generated by the model.</span><span style=\"background-color: #272822\">                                                </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 8 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        answer (str): The expected answer.</span><span style=\"background-color: #272822\">                                                                    </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 9 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">10 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">    Returns:</span><span style=\"background-color: #272822\">                                                                                                  </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">11 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        float: The Levenshtein ratio between the normalized model output and the answer.</span><span style=\"background-color: #272822\">                      </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">12 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">    \"\"\"</span><span style=\"background-color: #272822\">                                                                                                       </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">13 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    norm_output </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> normalize_text(model_output)</span><span style=\"background-color: #272822\">                                                                </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">14 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    norm_answer </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> normalize_text(answer)</span><span style=\"background-color: #272822\">                                                                      </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">15 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">return</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> Levenshtein</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">.</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">ratio(norm_output, norm_answer)</span><span style=\"background-color: #272822\">                                                        </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">16 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 1 \u001b[0m\u001b[38;2;166;226;46;48;2;39;40;34m@weave\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m.\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mop\u001b[0m\u001b[48;2;39;40;34m                                                                                                     \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 2 \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mdef\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;166;226;46;48;2;39;40;34mcompute_rouge\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mmodel_output\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mstr\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34manswer\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mstr\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m-\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m>\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mfloat\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[48;2;39;40;34m                                                   \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 3 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\"\"\u001b[0m\u001b[48;2;39;40;34m                                                                                                       \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 4 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m    Compute the ROUGE-L F1 score between the normalized model output and the reference answer.\u001b[0m\u001b[48;2;39;40;34m                \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 5 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 6 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m    Args:\u001b[0m\u001b[48;2;39;40;34m                                                                                                     \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 7 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        model_output (str): The model's generated output.\u001b[0m\u001b[48;2;39;40;34m                                                     \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 8 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        answer (str): The reference answer.\u001b[0m\u001b[48;2;39;40;34m                                                                   \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 9 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m10 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m    Returns:\u001b[0m\u001b[48;2;39;40;34m                                                                                                  \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m11 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        float: The ROUGE-L F1 score.\u001b[0m\u001b[48;2;39;40;34m                                                                          \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m12 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m    \"\"\"\u001b[0m\u001b[48;2;39;40;34m                                                                                                       \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m13 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mnorm_output\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mnormalize_text\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mmodel_output\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[48;2;39;40;34m                                                                \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m14 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mnorm_answer\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mnormalize_text\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34manswer\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[48;2;39;40;34m                                                                      \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m15 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mrouge\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mRouge\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mmetrics\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34mrouge-l\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mstats\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34mf\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[48;2;39;40;34m                                                             \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m16 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mscores\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mrouge\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m.\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mget_scores\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mnorm_output\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mnorm_answer\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[48;2;39;40;34m                                                       \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m17 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mreturn\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mscores\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;174;129;255;48;2;39;40;34m0\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34mrouge-l\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34mf\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[48;2;39;40;34m                                                                          \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m18 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 1 </span><span style=\"color: #a6e22e; text-decoration-color: #a6e22e; background-color: #272822\">@weave</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">.</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">op</span><span style=\"background-color: #272822\">                                                                                                     </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 2 </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">def</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> </span><span style=\"color: #a6e22e; text-decoration-color: #a6e22e; background-color: #272822\">compute_rouge</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">(model_output: str, answer: str) </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">-&gt;</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> float:</span><span style=\"background-color: #272822\">                                                   </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 3 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">\"\"\"</span><span style=\"background-color: #272822\">                                                                                                       </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 4 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">    Compute the ROUGE-L F1 score between the normalized model output and the reference answer.</span><span style=\"background-color: #272822\">                </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 5 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 6 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">    Args:</span><span style=\"background-color: #272822\">                                                                                                     </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 7 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        model_output (str): The model's generated output.</span><span style=\"background-color: #272822\">                                                     </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 8 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        answer (str): The reference answer.</span><span style=\"background-color: #272822\">                                                                   </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 9 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">10 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">    Returns:</span><span style=\"background-color: #272822\">                                                                                                  </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">11 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        float: The ROUGE-L F1 score.</span><span style=\"background-color: #272822\">                                                                          </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">12 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">    \"\"\"</span><span style=\"background-color: #272822\">                                                                                                       </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">13 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    norm_output </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> normalize_text(model_output)</span><span style=\"background-color: #272822\">                                                                </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">14 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    norm_answer </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> normalize_text(answer)</span><span style=\"background-color: #272822\">                                                                      </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">15 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    rouge </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> Rouge(metrics</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">[</span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">\"rouge-l\"</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">], stats</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">\"f\"</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">)</span><span style=\"background-color: #272822\">                                                             </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">16 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    scores </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> rouge</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">.</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">get_scores(norm_output, norm_answer)</span><span style=\"background-color: #272822\">                                                       </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">17 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">return</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> scores[</span><span style=\"color: #ae81ff; text-decoration-color: #ae81ff; background-color: #272822\">0</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">][</span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">\"rouge-l\"</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">][</span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">\"f\"</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">]</span><span style=\"background-color: #272822\">                                                                          </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">18 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 1 \u001b[0m\u001b[38;2;166;226;46;48;2;39;40;34m@weave\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m.\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mop\u001b[0m\u001b[48;2;39;40;34m                                                                                                     \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 2 \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mdef\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;166;226;46;48;2;39;40;34mcompute_bleu\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mmodel_output\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mstr\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34manswer\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mstr\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m-\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m>\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mfloat\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[48;2;39;40;34m                                                    \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 3 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\"\"\u001b[0m\u001b[48;2;39;40;34m                                                                                                       \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 4 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m    Compute the BLEU score between the normalized model output and the reference answer.\u001b[0m\u001b[48;2;39;40;34m                      \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 5 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 6 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m    Args:\u001b[0m\u001b[48;2;39;40;34m                                                                                                     \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 7 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        model_output (str): The generated output from the model.\u001b[0m\u001b[48;2;39;40;34m                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 8 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        answer (str): The reference answer.\u001b[0m\u001b[48;2;39;40;34m                                                                   \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 9 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m10 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m    Returns:\u001b[0m\u001b[48;2;39;40;34m                                                                                                  \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m11 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        float: The BLEU score between the normalized model output and the reference answer.\u001b[0m\u001b[48;2;39;40;34m                   \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m12 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m    \"\"\"\u001b[0m\u001b[48;2;39;40;34m                                                                                                       \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m13 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mchencherry\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mSmoothingFunction\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[48;2;39;40;34m                                                                          \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m14 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34msmoothing_function\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mchencherry\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m.\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mmethod2\u001b[0m\u001b[48;2;39;40;34m                                                                   \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m15 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m16 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mnorm_output\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mnormalize_text\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mmodel_output\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[48;2;39;40;34m                                                                \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m17 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mnorm_answer\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mnormalize_text\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34manswer\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[48;2;39;40;34m                                                                      \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m18 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mreference\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mword_tokenize\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mnorm_answer\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[48;2;39;40;34m                                                                    \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m19 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mcandidate\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mword_tokenize\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mnorm_output\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[48;2;39;40;34m                                                                    \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m20 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mscore\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34msentence_bleu\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mreference\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mcandidate\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34msmoothing_function\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34msmoothing_function\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[48;2;39;40;34m                      \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m21 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mreturn\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mscore\u001b[0m\u001b[48;2;39;40;34m                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m22 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 1 </span><span style=\"color: #a6e22e; text-decoration-color: #a6e22e; background-color: #272822\">@weave</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">.</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">op</span><span style=\"background-color: #272822\">                                                                                                     </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 2 </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">def</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> </span><span style=\"color: #a6e22e; text-decoration-color: #a6e22e; background-color: #272822\">compute_bleu</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">(model_output: str, answer: str) </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">-&gt;</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> float:</span><span style=\"background-color: #272822\">                                                    </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 3 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">\"\"\"</span><span style=\"background-color: #272822\">                                                                                                       </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 4 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">    Compute the BLEU score between the normalized model output and the reference answer.</span><span style=\"background-color: #272822\">                      </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 5 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 6 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">    Args:</span><span style=\"background-color: #272822\">                                                                                                     </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 7 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        model_output (str): The generated output from the model.</span><span style=\"background-color: #272822\">                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 8 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        answer (str): The reference answer.</span><span style=\"background-color: #272822\">                                                                   </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 9 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">10 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">    Returns:</span><span style=\"background-color: #272822\">                                                                                                  </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">11 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        float: The BLEU score between the normalized model output and the reference answer.</span><span style=\"background-color: #272822\">                   </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">12 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">    \"\"\"</span><span style=\"background-color: #272822\">                                                                                                       </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">13 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    chencherry </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> SmoothingFunction()</span><span style=\"background-color: #272822\">                                                                          </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">14 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    smoothing_function </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> chencherry</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">.</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">method2</span><span style=\"background-color: #272822\">                                                                   </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">15 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">16 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    norm_output </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> normalize_text(model_output)</span><span style=\"background-color: #272822\">                                                                </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">17 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    norm_answer </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> normalize_text(answer)</span><span style=\"background-color: #272822\">                                                                      </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">18 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    reference </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> word_tokenize(norm_answer)</span><span style=\"background-color: #272822\">                                                                    </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">19 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    candidate </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> word_tokenize(norm_output)</span><span style=\"background-color: #272822\">                                                                    </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">20 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    score </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> sentence_bleu([reference], candidate, smoothing_function</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">smoothing_function)</span><span style=\"background-color: #272822\">                      </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">21 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">return</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> score</span><span style=\"background-color: #272822\">                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">22 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "response_evaluations = weave.Evaluation(\n",
        "    name=\"Response_Evaluation\",\n",
        "    dataset=eval_dataset,\n",
        "    scorers=NLP_METRICS[:-1],\n",
        "    preprocess_model_input=lambda x: {\"query\": x[\"question\"]},\n",
        ")\n",
        "\n",
        "response_scores = asyncio.run(response_evaluations.evaluate(rag_pipeline))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 141
        },
        "id": "6COxfXtD4Rq5",
        "outputId": "e40432ae-b4ce-47c8-d094-8ffc620581e2"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "name 'response_evaluations' is not defined",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-16-01ae018e18fe>\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mresponse_scores\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0masyncio\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresponse_evaluations\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mevaluate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrag_pipeline\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m: name 'response_evaluations' is not defined"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from scripts.response_metrics import LLM_METRICS\n",
        "\n",
        "for metric in LLM_METRICS:\n",
        "    display_source(metric)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 401
        },
        "id": "ckbXNLei4dMx",
        "outputId": "54f710ab-b4d6-4113-bfea-24f387d3c71a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 1 \u001b[0m\u001b[38;2;166;226;46;48;2;39;40;34m@weave\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m.\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mop\u001b[0m\u001b[48;2;39;40;34m                                                                                                     \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 2 \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34masync\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mdef\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;166;226;46;48;2;39;40;34mllm_response_scorer\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[48;2;39;40;34m                                                                                \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 3 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mmodel_output\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mstr\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mquestion\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mstr\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34manswer\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mstr\u001b[0m\u001b[48;2;39;40;34m                                                             \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 4 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m-\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m>\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mDict\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mstr\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mAny\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[48;2;39;40;34m                                                                                          \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 5 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\"\"\u001b[0m\u001b[48;2;39;40;34m                                                                                                       \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 6 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m    Evaluate the correctness of the model output using a language model judge.\u001b[0m\u001b[48;2;39;40;34m                                \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 7 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 8 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m    Args:\u001b[0m\u001b[48;2;39;40;34m                                                                                                     \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 9 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        model_output (str): The output generated by the model.\u001b[0m\u001b[48;2;39;40;34m                                                \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m10 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        question (str): The question posed to the model.\u001b[0m\u001b[48;2;39;40;34m                                                      \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m11 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        answer (str): The reference answer.\u001b[0m\u001b[48;2;39;40;34m                                                                   \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m12 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m13 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m    Returns:\u001b[0m\u001b[48;2;39;40;34m                                                                                                  \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m14 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        Dict[str, Any]: A dictionary containing the final score and a boolean indicating correctness.\u001b[0m\u001b[48;2;39;40;34m         \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m15 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m    \"\"\"\u001b[0m\u001b[48;2;39;40;34m                                                                                                       \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m16 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mevaluation_result\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mawait\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mevaluate_correctness_using_llm_judge\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[48;2;39;40;34m                                           \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m17 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m        \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mquestion\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34manswer\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mmodel_output\u001b[0m\u001b[48;2;39;40;34m                                                                        \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m18 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[48;2;39;40;34m                                                                                                         \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m19 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m20 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mreturn\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m{\u001b[0m\u001b[48;2;39;40;34m                                                                                                  \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m21 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m        \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34mscore\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mevaluation_result\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34mfinal_score\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[48;2;39;40;34m                                                            \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m22 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m        \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34mcorrect\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mevaluation_result\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34mdecision\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m==\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34mcorrect\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[48;2;39;40;34m                                                \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m23 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m}\u001b[0m\u001b[48;2;39;40;34m                                                                                                         \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m24 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 1 </span><span style=\"color: #a6e22e; text-decoration-color: #a6e22e; background-color: #272822\">@weave</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">.</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">op</span><span style=\"background-color: #272822\">                                                                                                     </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 2 </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">async</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">def</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> </span><span style=\"color: #a6e22e; text-decoration-color: #a6e22e; background-color: #272822\">llm_response_scorer</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">(</span><span style=\"background-color: #272822\">                                                                                </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 3 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    model_output: str, question: str, answer: str</span><span style=\"background-color: #272822\">                                                             </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 4 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">) </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">-&gt;</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> Dict[str, Any]:</span><span style=\"background-color: #272822\">                                                                                          </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 5 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">\"\"\"</span><span style=\"background-color: #272822\">                                                                                                       </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 6 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">    Evaluate the correctness of the model output using a language model judge.</span><span style=\"background-color: #272822\">                                </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 7 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 8 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">    Args:</span><span style=\"background-color: #272822\">                                                                                                     </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 9 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        model_output (str): The output generated by the model.</span><span style=\"background-color: #272822\">                                                </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">10 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        question (str): The question posed to the model.</span><span style=\"background-color: #272822\">                                                      </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">11 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        answer (str): The reference answer.</span><span style=\"background-color: #272822\">                                                                   </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">12 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">13 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">    Returns:</span><span style=\"background-color: #272822\">                                                                                                  </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">14 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        Dict[str, Any]: A dictionary containing the final score and a boolean indicating correctness.</span><span style=\"background-color: #272822\">         </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">15 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">    \"\"\"</span><span style=\"background-color: #272822\">                                                                                                       </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">16 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    evaluation_result </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">await</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> evaluate_correctness_using_llm_judge(</span><span style=\"background-color: #272822\">                                           </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">17 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">        question, answer, model_output</span><span style=\"background-color: #272822\">                                                                        </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">18 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    )</span><span style=\"background-color: #272822\">                                                                                                         </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">19 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">20 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">return</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> {</span><span style=\"background-color: #272822\">                                                                                                  </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">21 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">        </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">\"score\"</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">: evaluation_result[</span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">\"final_score\"</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">],</span><span style=\"background-color: #272822\">                                                            </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">22 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">        </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">\"correct\"</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">: evaluation_result[</span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">\"decision\"</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">] </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">==</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">\"correct\"</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">,</span><span style=\"background-color: #272822\">                                                </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">23 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    }</span><span style=\"background-color: #272822\">                                                                                                         </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">24 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "correctness_evaluations = weave.Evaluation(\n",
        "    name=\"Correctness_Evaluation\",\n",
        "    dataset=eval_dataset,\n",
        "    scorers=LLM_METRICS,\n",
        "    preprocess_model_input=lambda x: {\"query\": x[\"question\"]},\n",
        ")\n",
        "\n",
        "response_scores = asyncio.run(correctness_evaluations.evaluate(rag_pipeline))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "HYrKp_ip4dXr",
        "outputId": "fd2e5acd-e54b-4359-cbc5-92299e974600"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "model_output failed\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">model_output failed\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/flow/eval.py\", line 167, in predict_and_score\n",
            "    model_output = await async_call(model_predict, **model_predict_args)\n",
            "  File \"/usr/lib/python3.10/asyncio/threads.py\", line 25, in to_thread\n",
            "    return await loop.run_in_executor(None, func_call)\n",
            "  File \"/usr/lib/python3.10/asyncio/futures.py\", line 285, in __await__\n",
            "    yield self  # This tells Task to wait for completion.\n",
            "  File \"/usr/lib/python3.10/asyncio/tasks.py\", line 304, in __wakeup\n",
            "    future.result()\n",
            "  File \"/usr/lib/python3.10/asyncio/futures.py\", line 201, in result\n",
            "    raise self._exception.with_traceback(self._exception_tb)\n",
            "  File \"/usr/lib/python3.10/concurrent/futures/thread.py\", line 58, in run\n",
            "    result = self.fn(*self.args, **self.kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 482, in wrapper\n",
            "    res, _ = _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 296, in _execute_call\n",
            "    handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 294, in _execute_call\n",
            "    res = func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/rag_pipeline.py\", line 33, in predict\n",
            "    return self.response_generator.predict(query, context)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 482, in wrapper\n",
            "    res, _ = _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 296, in _execute_call\n",
            "    handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 294, in _execute_call\n",
            "    res = func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/response_generator.py\", line 107, in predict\n",
            "    return self.generate_response(query, context)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 482, in wrapper\n",
            "    res, _ = _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 296, in _execute_call\n",
            "    handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 294, in _execute_call\n",
            "    res = func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/response_generator.py\", line 86, in generate_response\n",
            "    response = self.client.chat(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/client.py\", line 103, in _wrapped\n",
            "    return func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/client.py\", line 35, in _wrapped\n",
            "    return method(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 482, in wrapper\n",
            "    res, _ = _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 296, in _execute_call\n",
            "    handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 294, in _execute_call\n",
            "    res = func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/integrations/cohere/cohere_sdk.py\", line 103, in _wrapper\n",
            "    response = fn(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/v2/client.py\", line 602, in chat\n",
            "    raise TooManyRequestsError(\n",
            "cohere.errors.too_many_requests_error.TooManyRequestsError: status_code: 429, body: data=None message=\"You are using a Trial key, which is limited to 10 API calls / minute. You can continue to use the Trial key for free or upgrade to a Production key with higher rate limits at 'https://dashboard.cohere.com/api-keys'. Contact us on 'https://discord.gg/XW44jPfYJu' or email us at support@cohere.com with any questions\"\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "model_output failed\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">model_output failed\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/flow/eval.py\", line 167, in predict_and_score\n",
            "    model_output = await async_call(model_predict, **model_predict_args)\n",
            "  File \"/usr/lib/python3.10/asyncio/threads.py\", line 25, in to_thread\n",
            "    return await loop.run_in_executor(None, func_call)\n",
            "  File \"/usr/lib/python3.10/asyncio/futures.py\", line 285, in __await__\n",
            "    yield self  # This tells Task to wait for completion.\n",
            "  File \"/usr/lib/python3.10/asyncio/tasks.py\", line 304, in __wakeup\n",
            "    future.result()\n",
            "  File \"/usr/lib/python3.10/asyncio/futures.py\", line 201, in result\n",
            "    raise self._exception.with_traceback(self._exception_tb)\n",
            "  File \"/usr/lib/python3.10/concurrent/futures/thread.py\", line 58, in run\n",
            "    result = self.fn(*self.args, **self.kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 482, in wrapper\n",
            "    res, _ = _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 296, in _execute_call\n",
            "    handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 294, in _execute_call\n",
            "    res = func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/rag_pipeline.py\", line 33, in predict\n",
            "    return self.response_generator.predict(query, context)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 482, in wrapper\n",
            "    res, _ = _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 296, in _execute_call\n",
            "    handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 294, in _execute_call\n",
            "    res = func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/response_generator.py\", line 107, in predict\n",
            "    return self.generate_response(query, context)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 482, in wrapper\n",
            "    res, _ = _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 296, in _execute_call\n",
            "    handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 294, in _execute_call\n",
            "    res = func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/response_generator.py\", line 86, in generate_response\n",
            "    response = self.client.chat(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/client.py\", line 103, in _wrapped\n",
            "    return func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/client.py\", line 35, in _wrapped\n",
            "    return method(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 482, in wrapper\n",
            "    res, _ = _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 296, in _execute_call\n",
            "    handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 294, in _execute_call\n",
            "    res = func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/integrations/cohere/cohere_sdk.py\", line 103, in _wrapper\n",
            "    response = fn(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/v2/client.py\", line 602, in chat\n",
            "    raise TooManyRequestsError(\n",
            "cohere.errors.too_many_requests_error.TooManyRequestsError: status_code: 429, body: data=None message=\"You are using a Trial key, which is limited to 10 API calls / minute. You can continue to use the Trial key for free or upgrade to a Production key with higher rate limits at 'https://dashboard.cohere.com/api-keys'. Contact us on 'https://discord.gg/XW44jPfYJu' or email us at support@cohere.com with any questions\"\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "model_output failed\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">model_output failed\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/flow/eval.py\", line 167, in predict_and_score\n",
            "    model_output = await async_call(model_predict, **model_predict_args)\n",
            "  File \"/usr/lib/python3.10/asyncio/threads.py\", line 25, in to_thread\n",
            "    return await loop.run_in_executor(None, func_call)\n",
            "  File \"/usr/lib/python3.10/asyncio/futures.py\", line 285, in __await__\n",
            "    yield self  # This tells Task to wait for completion.\n",
            "  File \"/usr/lib/python3.10/asyncio/tasks.py\", line 304, in __wakeup\n",
            "    future.result()\n",
            "  File \"/usr/lib/python3.10/asyncio/futures.py\", line 201, in result\n",
            "    raise self._exception.with_traceback(self._exception_tb)\n",
            "  File \"/usr/lib/python3.10/concurrent/futures/thread.py\", line 58, in run\n",
            "    result = self.fn(*self.args, **self.kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 482, in wrapper\n",
            "    res, _ = _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 296, in _execute_call\n",
            "    handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 294, in _execute_call\n",
            "    res = func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/rag_pipeline.py\", line 33, in predict\n",
            "    return self.response_generator.predict(query, context)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 482, in wrapper\n",
            "    res, _ = _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 296, in _execute_call\n",
            "    handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 294, in _execute_call\n",
            "    res = func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/response_generator.py\", line 107, in predict\n",
            "    return self.generate_response(query, context)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 482, in wrapper\n",
            "    res, _ = _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 296, in _execute_call\n",
            "    handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 294, in _execute_call\n",
            "    res = func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/response_generator.py\", line 86, in generate_response\n",
            "    response = self.client.chat(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/client.py\", line 103, in _wrapped\n",
            "    return func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/client.py\", line 35, in _wrapped\n",
            "    return method(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 482, in wrapper\n",
            "    res, _ = _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 296, in _execute_call\n",
            "    handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 294, in _execute_call\n",
            "    res = func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/integrations/cohere/cohere_sdk.py\", line 103, in _wrapper\n",
            "    response = fn(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/v2/client.py\", line 602, in chat\n",
            "    raise TooManyRequestsError(\n",
            "cohere.errors.too_many_requests_error.TooManyRequestsError: status_code: 429, body: data=None message=\"You are using a Trial key, which is limited to 10 API calls / minute. You can continue to use the Trial key for free or upgrade to a Production key with higher rate limits at 'https://dashboard.cohere.com/api-keys'. Contact us on 'https://discord.gg/XW44jPfYJu' or email us at support@cohere.com with any questions\"\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "model_output failed\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">model_output failed\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/flow/eval.py\", line 167, in predict_and_score\n",
            "    model_output = await async_call(model_predict, **model_predict_args)\n",
            "  File \"/usr/lib/python3.10/asyncio/threads.py\", line 25, in to_thread\n",
            "    return await loop.run_in_executor(None, func_call)\n",
            "  File \"/usr/lib/python3.10/asyncio/futures.py\", line 285, in __await__\n",
            "    yield self  # This tells Task to wait for completion.\n",
            "  File \"/usr/lib/python3.10/asyncio/tasks.py\", line 304, in __wakeup\n",
            "    future.result()\n",
            "  File \"/usr/lib/python3.10/asyncio/futures.py\", line 201, in result\n",
            "    raise self._exception.with_traceback(self._exception_tb)\n",
            "  File \"/usr/lib/python3.10/concurrent/futures/thread.py\", line 58, in run\n",
            "    result = self.fn(*self.args, **self.kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 482, in wrapper\n",
            "    res, _ = _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 296, in _execute_call\n",
            "    handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 294, in _execute_call\n",
            "    res = func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/rag_pipeline.py\", line 33, in predict\n",
            "    return self.response_generator.predict(query, context)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 482, in wrapper\n",
            "    res, _ = _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 296, in _execute_call\n",
            "    handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 294, in _execute_call\n",
            "    res = func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/response_generator.py\", line 107, in predict\n",
            "    return self.generate_response(query, context)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 482, in wrapper\n",
            "    res, _ = _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 296, in _execute_call\n",
            "    handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 294, in _execute_call\n",
            "    res = func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/response_generator.py\", line 86, in generate_response\n",
            "    response = self.client.chat(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/client.py\", line 103, in _wrapped\n",
            "    return func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/client.py\", line 35, in _wrapped\n",
            "    return method(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 482, in wrapper\n",
            "    res, _ = _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 296, in _execute_call\n",
            "    handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 294, in _execute_call\n",
            "    res = func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/integrations/cohere/cohere_sdk.py\", line 103, in _wrapper\n",
            "    response = fn(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/v2/client.py\", line 602, in chat\n",
            "    raise TooManyRequestsError(\n",
            "cohere.errors.too_many_requests_error.TooManyRequestsError: status_code: 429, body: data=None message=\"You are using a Trial key, which is limited to 10 API calls / minute. You can continue to use the Trial key for free or upgrade to a Production key with higher rate limits at 'https://dashboard.cohere.com/api-keys'. Contact us on 'https://discord.gg/XW44jPfYJu' or email us at support@cohere.com with any questions\"\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "model_output failed\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">model_output failed\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/flow/eval.py\", line 167, in predict_and_score\n",
            "    model_output = await async_call(model_predict, **model_predict_args)\n",
            "  File \"/usr/lib/python3.10/asyncio/threads.py\", line 25, in to_thread\n",
            "    return await loop.run_in_executor(None, func_call)\n",
            "  File \"/usr/lib/python3.10/asyncio/futures.py\", line 285, in __await__\n",
            "    yield self  # This tells Task to wait for completion.\n",
            "  File \"/usr/lib/python3.10/asyncio/tasks.py\", line 304, in __wakeup\n",
            "    future.result()\n",
            "  File \"/usr/lib/python3.10/asyncio/futures.py\", line 201, in result\n",
            "    raise self._exception.with_traceback(self._exception_tb)\n",
            "  File \"/usr/lib/python3.10/concurrent/futures/thread.py\", line 58, in run\n",
            "    result = self.fn(*self.args, **self.kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 482, in wrapper\n",
            "    res, _ = _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 296, in _execute_call\n",
            "    handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 294, in _execute_call\n",
            "    res = func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/rag_pipeline.py\", line 33, in predict\n",
            "    return self.response_generator.predict(query, context)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 482, in wrapper\n",
            "    res, _ = _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 296, in _execute_call\n",
            "    handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 294, in _execute_call\n",
            "    res = func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/response_generator.py\", line 107, in predict\n",
            "    return self.generate_response(query, context)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 482, in wrapper\n",
            "    res, _ = _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 296, in _execute_call\n",
            "    handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 294, in _execute_call\n",
            "    res = func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/response_generator.py\", line 86, in generate_response\n",
            "    response = self.client.chat(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/client.py\", line 103, in _wrapped\n",
            "    return func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/client.py\", line 35, in _wrapped\n",
            "    return method(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 482, in wrapper\n",
            "    res, _ = _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 296, in _execute_call\n",
            "    handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 294, in _execute_call\n",
            "    res = func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/integrations/cohere/cohere_sdk.py\", line 103, in _wrapper\n",
            "    response = fn(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/v2/client.py\", line 602, in chat\n",
            "    raise TooManyRequestsError(\n",
            "cohere.errors.too_many_requests_error.TooManyRequestsError: status_code: 429, body: data=None message=\"You are using a Trial key, which is limited to 10 API calls / minute. You can continue to use the Trial key for free or upgrade to a Production key with higher rate limits at 'https://dashboard.cohere.com/api-keys'. Contact us on 'https://discord.gg/XW44jPfYJu' or email us at support@cohere.com with any questions\"\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "model_output failed\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">model_output failed\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/flow/eval.py\", line 167, in predict_and_score\n",
            "    model_output = await async_call(model_predict, **model_predict_args)\n",
            "  File \"/usr/lib/python3.10/asyncio/threads.py\", line 25, in to_thread\n",
            "    return await loop.run_in_executor(None, func_call)\n",
            "  File \"/usr/lib/python3.10/asyncio/futures.py\", line 285, in __await__\n",
            "    yield self  # This tells Task to wait for completion.\n",
            "  File \"/usr/lib/python3.10/asyncio/tasks.py\", line 304, in __wakeup\n",
            "    future.result()\n",
            "  File \"/usr/lib/python3.10/asyncio/futures.py\", line 201, in result\n",
            "    raise self._exception.with_traceback(self._exception_tb)\n",
            "  File \"/usr/lib/python3.10/concurrent/futures/thread.py\", line 58, in run\n",
            "    result = self.fn(*self.args, **self.kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 482, in wrapper\n",
            "    res, _ = _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 296, in _execute_call\n",
            "    handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 294, in _execute_call\n",
            "    res = func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/rag_pipeline.py\", line 33, in predict\n",
            "    return self.response_generator.predict(query, context)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 482, in wrapper\n",
            "    res, _ = _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 296, in _execute_call\n",
            "    handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 294, in _execute_call\n",
            "    res = func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/response_generator.py\", line 107, in predict\n",
            "    return self.generate_response(query, context)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 482, in wrapper\n",
            "    res, _ = _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 296, in _execute_call\n",
            "    handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 294, in _execute_call\n",
            "    res = func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/response_generator.py\", line 86, in generate_response\n",
            "    response = self.client.chat(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/client.py\", line 103, in _wrapped\n",
            "    return func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/client.py\", line 35, in _wrapped\n",
            "    return method(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 482, in wrapper\n",
            "    res, _ = _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 296, in _execute_call\n",
            "    handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 294, in _execute_call\n",
            "    res = func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/integrations/cohere/cohere_sdk.py\", line 103, in _wrapper\n",
            "    response = fn(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/v2/client.py\", line 602, in chat\n",
            "    raise TooManyRequestsError(\n",
            "cohere.errors.too_many_requests_error.TooManyRequestsError: status_code: 429, body: data=None message=\"You are using a Trial key, which is limited to 10 API calls / minute. You can continue to use the Trial key for free or upgrade to a Production key with higher rate limits at 'https://dashboard.cohere.com/api-keys'. Contact us on 'https://discord.gg/XW44jPfYJu' or email us at support@cohere.com with any questions\"\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "model_output failed\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">model_output failed\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/flow/eval.py\", line 167, in predict_and_score\n",
            "    model_output = await async_call(model_predict, **model_predict_args)\n",
            "  File \"/usr/lib/python3.10/asyncio/threads.py\", line 25, in to_thread\n",
            "    return await loop.run_in_executor(None, func_call)\n",
            "  File \"/usr/lib/python3.10/asyncio/futures.py\", line 285, in __await__\n",
            "    yield self  # This tells Task to wait for completion.\n",
            "  File \"/usr/lib/python3.10/asyncio/tasks.py\", line 304, in __wakeup\n",
            "    future.result()\n",
            "  File \"/usr/lib/python3.10/asyncio/futures.py\", line 201, in result\n",
            "    raise self._exception.with_traceback(self._exception_tb)\n",
            "  File \"/usr/lib/python3.10/concurrent/futures/thread.py\", line 58, in run\n",
            "    result = self.fn(*self.args, **self.kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 482, in wrapper\n",
            "    res, _ = _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 296, in _execute_call\n",
            "    handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 294, in _execute_call\n",
            "    res = func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/rag_pipeline.py\", line 33, in predict\n",
            "    return self.response_generator.predict(query, context)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 482, in wrapper\n",
            "    res, _ = _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 296, in _execute_call\n",
            "    handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 294, in _execute_call\n",
            "    res = func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/response_generator.py\", line 107, in predict\n",
            "    return self.generate_response(query, context)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 482, in wrapper\n",
            "    res, _ = _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 296, in _execute_call\n",
            "    handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 294, in _execute_call\n",
            "    res = func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/response_generator.py\", line 86, in generate_response\n",
            "    response = self.client.chat(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/client.py\", line 103, in _wrapped\n",
            "    return func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/client.py\", line 35, in _wrapped\n",
            "    return method(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 482, in wrapper\n",
            "    res, _ = _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 296, in _execute_call\n",
            "    handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 294, in _execute_call\n",
            "    res = func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/integrations/cohere/cohere_sdk.py\", line 103, in _wrapper\n",
            "    response = fn(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/v2/client.py\", line 602, in chat\n",
            "    raise TooManyRequestsError(\n",
            "cohere.errors.too_many_requests_error.TooManyRequestsError: status_code: 429, body: data=None message=\"You are using a Trial key, which is limited to 10 API calls / minute. You can continue to use the Trial key for free or upgrade to a Production key with higher rate limits at 'https://dashboard.cohere.com/api-keys'. Contact us on 'https://discord.gg/XW44jPfYJu' or email us at support@cohere.com with any questions\"\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "model_output failed\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">model_output failed\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/flow/eval.py\", line 167, in predict_and_score\n",
            "    model_output = await async_call(model_predict, **model_predict_args)\n",
            "  File \"/usr/lib/python3.10/asyncio/threads.py\", line 25, in to_thread\n",
            "    return await loop.run_in_executor(None, func_call)\n",
            "  File \"/usr/lib/python3.10/asyncio/futures.py\", line 285, in __await__\n",
            "    yield self  # This tells Task to wait for completion.\n",
            "  File \"/usr/lib/python3.10/asyncio/tasks.py\", line 304, in __wakeup\n",
            "    future.result()\n",
            "  File \"/usr/lib/python3.10/asyncio/futures.py\", line 201, in result\n",
            "    raise self._exception.with_traceback(self._exception_tb)\n",
            "  File \"/usr/lib/python3.10/concurrent/futures/thread.py\", line 58, in run\n",
            "    result = self.fn(*self.args, **self.kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 482, in wrapper\n",
            "    res, _ = _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 296, in _execute_call\n",
            "    handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 294, in _execute_call\n",
            "    res = func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/rag_pipeline.py\", line 33, in predict\n",
            "    return self.response_generator.predict(query, context)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 482, in wrapper\n",
            "    res, _ = _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 296, in _execute_call\n",
            "    handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 294, in _execute_call\n",
            "    res = func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/response_generator.py\", line 107, in predict\n",
            "    return self.generate_response(query, context)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 482, in wrapper\n",
            "    res, _ = _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 296, in _execute_call\n",
            "    handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 294, in _execute_call\n",
            "    res = func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/response_generator.py\", line 86, in generate_response\n",
            "    response = self.client.chat(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/client.py\", line 103, in _wrapped\n",
            "    return func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/client.py\", line 35, in _wrapped\n",
            "    return method(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 482, in wrapper\n",
            "    res, _ = _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 296, in _execute_call\n",
            "    handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 294, in _execute_call\n",
            "    res = func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/integrations/cohere/cohere_sdk.py\", line 103, in _wrapper\n",
            "    response = fn(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/v2/client.py\", line 602, in chat\n",
            "    raise TooManyRequestsError(\n",
            "cohere.errors.too_many_requests_error.TooManyRequestsError: status_code: 429, body: data=None message=\"You are using a Trial key, which is limited to 10 API calls / minute. You can continue to use the Trial key for free or upgrade to a Production key with higher rate limits at 'https://dashboard.cohere.com/api-keys'. Contact us on 'https://discord.gg/XW44jPfYJu' or email us at support@cohere.com with any questions\"\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "model_output failed\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">model_output failed\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/flow/eval.py\", line 167, in predict_and_score\n",
            "    model_output = await async_call(model_predict, **model_predict_args)\n",
            "  File \"/usr/lib/python3.10/asyncio/threads.py\", line 25, in to_thread\n",
            "    return await loop.run_in_executor(None, func_call)\n",
            "  File \"/usr/lib/python3.10/asyncio/futures.py\", line 285, in __await__\n",
            "    yield self  # This tells Task to wait for completion.\n",
            "  File \"/usr/lib/python3.10/asyncio/tasks.py\", line 304, in __wakeup\n",
            "    future.result()\n",
            "  File \"/usr/lib/python3.10/asyncio/futures.py\", line 201, in result\n",
            "    raise self._exception.with_traceback(self._exception_tb)\n",
            "  File \"/usr/lib/python3.10/concurrent/futures/thread.py\", line 58, in run\n",
            "    result = self.fn(*self.args, **self.kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 482, in wrapper\n",
            "    res, _ = _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 296, in _execute_call\n",
            "    handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 294, in _execute_call\n",
            "    res = func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/rag_pipeline.py\", line 33, in predict\n",
            "    return self.response_generator.predict(query, context)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 482, in wrapper\n",
            "    res, _ = _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 296, in _execute_call\n",
            "    handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 294, in _execute_call\n",
            "    res = func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/response_generator.py\", line 107, in predict\n",
            "    return self.generate_response(query, context)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 482, in wrapper\n",
            "    res, _ = _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 296, in _execute_call\n",
            "    handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 294, in _execute_call\n",
            "    res = func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/response_generator.py\", line 86, in generate_response\n",
            "    response = self.client.chat(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/client.py\", line 103, in _wrapped\n",
            "    return func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/client.py\", line 35, in _wrapped\n",
            "    return method(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 482, in wrapper\n",
            "    res, _ = _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 296, in _execute_call\n",
            "    handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 294, in _execute_call\n",
            "    res = func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/integrations/cohere/cohere_sdk.py\", line 103, in _wrapper\n",
            "    response = fn(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/v2/client.py\", line 602, in chat\n",
            "    raise TooManyRequestsError(\n",
            "cohere.errors.too_many_requests_error.TooManyRequestsError: status_code: 429, body: data=None message=\"You are using a Trial key, which is limited to 10 API calls / minute. You can continue to use the Trial key for free or upgrade to a Production key with higher rate limits at 'https://dashboard.cohere.com/api-keys'. Contact us on 'https://discord.gg/XW44jPfYJu' or email us at support@cohere.com with any questions\"\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "model_output failed\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">model_output failed\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/flow/eval.py\", line 167, in predict_and_score\n",
            "    model_output = await async_call(model_predict, **model_predict_args)\n",
            "  File \"/usr/lib/python3.10/asyncio/threads.py\", line 25, in to_thread\n",
            "    return await loop.run_in_executor(None, func_call)\n",
            "  File \"/usr/lib/python3.10/asyncio/futures.py\", line 285, in __await__\n",
            "    yield self  # This tells Task to wait for completion.\n",
            "  File \"/usr/lib/python3.10/asyncio/tasks.py\", line 304, in __wakeup\n",
            "    future.result()\n",
            "  File \"/usr/lib/python3.10/asyncio/futures.py\", line 201, in result\n",
            "    raise self._exception.with_traceback(self._exception_tb)\n",
            "  File \"/usr/lib/python3.10/concurrent/futures/thread.py\", line 58, in run\n",
            "    result = self.fn(*self.args, **self.kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 482, in wrapper\n",
            "    res, _ = _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 296, in _execute_call\n",
            "    handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 294, in _execute_call\n",
            "    res = func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/rag_pipeline.py\", line 33, in predict\n",
            "    return self.response_generator.predict(query, context)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 482, in wrapper\n",
            "    res, _ = _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 296, in _execute_call\n",
            "    handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 294, in _execute_call\n",
            "    res = func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/response_generator.py\", line 107, in predict\n",
            "    return self.generate_response(query, context)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 482, in wrapper\n",
            "    res, _ = _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 296, in _execute_call\n",
            "    handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 294, in _execute_call\n",
            "    res = func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/response_generator.py\", line 86, in generate_response\n",
            "    response = self.client.chat(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/client.py\", line 103, in _wrapped\n",
            "    return func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/client.py\", line 35, in _wrapped\n",
            "    return method(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 482, in wrapper\n",
            "    res, _ = _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 296, in _execute_call\n",
            "    handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 294, in _execute_call\n",
            "    res = func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/integrations/cohere/cohere_sdk.py\", line 103, in _wrapper\n",
            "    response = fn(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/v2/client.py\", line 602, in chat\n",
            "    raise TooManyRequestsError(\n",
            "cohere.errors.too_many_requests_error.TooManyRequestsError: status_code: 429, body: data=None message=\"You are using a Trial key, which is limited to 10 API calls / minute. You can continue to use the Trial key for free or upgrade to a Production key with higher rate limits at 'https://dashboard.cohere.com/api-keys'. Contact us on 'https://discord.gg/XW44jPfYJu' or email us at support@cohere.com with any questions\"\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Predict and score failed\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Predict and score failed\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/flow/eval.py\", line 287, in eval_example\n",
            "    eval_row = await self.predict_and_score(model, example)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/flow/eval.py\", line 224, in predict_and_score\n",
            "    result = await async_call(score_fn, **score_args)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/response_metrics.py\", line 285, in llm_response_scorer\n",
            "    evaluation_result = await evaluate_correctness_using_llm_judge(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/response_metrics.py\", line 267, in evaluate_correctness_using_llm_judge\n",
            "    return await call_cohere_with_retry(co_client, messages)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/response_metrics.py\", line 204, in call_cohere_with_retry\n",
            "    response_text = await make_cohere_api_call(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/utils.py\", line 76, in make_cohere_api_call\n",
            "    response = await co_client.chat(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/client.py\", line 111, in _async_wrapped\n",
            "    return await func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/client.py\", line 40, in _async_wrapped\n",
            "    return await method(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/integrations/cohere/cohere_sdk.py\", line 137, in _wrapper\n",
            "    response = await fn(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/v2/client.py\", line 1662, in chat\n",
            "    raise TooManyRequestsError(\n",
            "cohere.errors.too_many_requests_error.TooManyRequestsError: status_code: 429, body: data=None message=\"You are using a Trial key, which is limited to 10 API calls / minute. You can continue to use the Trial key for free or upgrade to a Production key with higher rate limits at 'https://dashboard.cohere.com/api-keys'. Contact us on 'https://discord.gg/XW44jPfYJu' or email us at support@cohere.com with any questions\"\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Predict and score failed\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Predict and score failed\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/flow/eval.py\", line 287, in eval_example\n",
            "    eval_row = await self.predict_and_score(model, example)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/flow/eval.py\", line 224, in predict_and_score\n",
            "    result = await async_call(score_fn, **score_args)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/response_metrics.py\", line 285, in llm_response_scorer\n",
            "    evaluation_result = await evaluate_correctness_using_llm_judge(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/response_metrics.py\", line 267, in evaluate_correctness_using_llm_judge\n",
            "    return await call_cohere_with_retry(co_client, messages)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/response_metrics.py\", line 204, in call_cohere_with_retry\n",
            "    response_text = await make_cohere_api_call(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/utils.py\", line 76, in make_cohere_api_call\n",
            "    response = await co_client.chat(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/client.py\", line 111, in _async_wrapped\n",
            "    return await func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/client.py\", line 40, in _async_wrapped\n",
            "    return await method(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/integrations/cohere/cohere_sdk.py\", line 137, in _wrapper\n",
            "    response = await fn(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/v2/client.py\", line 1662, in chat\n",
            "    raise TooManyRequestsError(\n",
            "cohere.errors.too_many_requests_error.TooManyRequestsError: status_code: 429, body: data=None message=\"You are using a Trial key, which is limited to 10 API calls / minute. You can continue to use the Trial key for free or upgrade to a Production key with higher rate limits at 'https://dashboard.cohere.com/api-keys'. Contact us on 'https://discord.gg/XW44jPfYJu' or email us at support@cohere.com with any questions\"\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Predict and score failed\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Predict and score failed\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/flow/eval.py\", line 287, in eval_example\n",
            "    eval_row = await self.predict_and_score(model, example)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/flow/eval.py\", line 224, in predict_and_score\n",
            "    result = await async_call(score_fn, **score_args)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/response_metrics.py\", line 285, in llm_response_scorer\n",
            "    evaluation_result = await evaluate_correctness_using_llm_judge(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/response_metrics.py\", line 267, in evaluate_correctness_using_llm_judge\n",
            "    return await call_cohere_with_retry(co_client, messages)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/response_metrics.py\", line 204, in call_cohere_with_retry\n",
            "    response_text = await make_cohere_api_call(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/utils.py\", line 76, in make_cohere_api_call\n",
            "    response = await co_client.chat(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/client.py\", line 111, in _async_wrapped\n",
            "    return await func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/client.py\", line 40, in _async_wrapped\n",
            "    return await method(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/integrations/cohere/cohere_sdk.py\", line 137, in _wrapper\n",
            "    response = await fn(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/v2/client.py\", line 1662, in chat\n",
            "    raise TooManyRequestsError(\n",
            "cohere.errors.too_many_requests_error.TooManyRequestsError: status_code: 429, body: data=None message=\"You are using a Trial key, which is limited to 10 API calls / minute. You can continue to use the Trial key for free or upgrade to a Production key with higher rate limits at 'https://dashboard.cohere.com/api-keys'. Contact us on 'https://discord.gg/XW44jPfYJu' or email us at support@cohere.com with any questions\"\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Evaluated \u001b[1;36m1\u001b[0m of \u001b[1;36m20\u001b[0m examples\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">20</span> examples\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Evaluated \u001b[1;36m2\u001b[0m of \u001b[1;36m20\u001b[0m examples\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">20</span> examples\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Evaluated \u001b[1;36m3\u001b[0m of \u001b[1;36m20\u001b[0m examples\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">3</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">20</span> examples\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Predict and score failed\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Predict and score failed\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/flow/eval.py\", line 287, in eval_example\n",
            "    eval_row = await self.predict_and_score(model, example)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/flow/eval.py\", line 224, in predict_and_score\n",
            "    result = await async_call(score_fn, **score_args)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/response_metrics.py\", line 285, in llm_response_scorer\n",
            "    evaluation_result = await evaluate_correctness_using_llm_judge(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/response_metrics.py\", line 267, in evaluate_correctness_using_llm_judge\n",
            "    return await call_cohere_with_retry(co_client, messages)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/response_metrics.py\", line 204, in call_cohere_with_retry\n",
            "    response_text = await make_cohere_api_call(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/utils.py\", line 76, in make_cohere_api_call\n",
            "    response = await co_client.chat(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/client.py\", line 111, in _async_wrapped\n",
            "    return await func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/client.py\", line 40, in _async_wrapped\n",
            "    return await method(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/integrations/cohere/cohere_sdk.py\", line 137, in _wrapper\n",
            "    response = await fn(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/v2/client.py\", line 1662, in chat\n",
            "    raise TooManyRequestsError(\n",
            "cohere.errors.too_many_requests_error.TooManyRequestsError: status_code: 429, body: data=None message=\"You are using a Trial key, which is limited to 10 API calls / minute. You can continue to use the Trial key for free or upgrade to a Production key with higher rate limits at 'https://dashboard.cohere.com/api-keys'. Contact us on 'https://discord.gg/XW44jPfYJu' or email us at support@cohere.com with any questions\"\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Evaluated \u001b[1;36m4\u001b[0m of \u001b[1;36m20\u001b[0m examples\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">20</span> examples\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Predict and score failed\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Predict and score failed\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/flow/eval.py\", line 287, in eval_example\n",
            "    eval_row = await self.predict_and_score(model, example)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/flow/eval.py\", line 224, in predict_and_score\n",
            "    result = await async_call(score_fn, **score_args)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/response_metrics.py\", line 285, in llm_response_scorer\n",
            "    evaluation_result = await evaluate_correctness_using_llm_judge(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/response_metrics.py\", line 267, in evaluate_correctness_using_llm_judge\n",
            "    return await call_cohere_with_retry(co_client, messages)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/response_metrics.py\", line 204, in call_cohere_with_retry\n",
            "    response_text = await make_cohere_api_call(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/utils.py\", line 76, in make_cohere_api_call\n",
            "    response = await co_client.chat(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/client.py\", line 111, in _async_wrapped\n",
            "    return await func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/client.py\", line 40, in _async_wrapped\n",
            "    return await method(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/integrations/cohere/cohere_sdk.py\", line 137, in _wrapper\n",
            "    response = await fn(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/v2/client.py\", line 1662, in chat\n",
            "    raise TooManyRequestsError(\n",
            "cohere.errors.too_many_requests_error.TooManyRequestsError: status_code: 429, body: data=None message=\"You are using a Trial key, which is limited to 10 API calls / minute. You can continue to use the Trial key for free or upgrade to a Production key with higher rate limits at 'https://dashboard.cohere.com/api-keys'. Contact us on 'https://discord.gg/XW44jPfYJu' or email us at support@cohere.com with any questions\"\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Predict and score failed\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Predict and score failed\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/flow/eval.py\", line 287, in eval_example\n",
            "    eval_row = await self.predict_and_score(model, example)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/flow/eval.py\", line 224, in predict_and_score\n",
            "    result = await async_call(score_fn, **score_args)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/response_metrics.py\", line 285, in llm_response_scorer\n",
            "    evaluation_result = await evaluate_correctness_using_llm_judge(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/response_metrics.py\", line 267, in evaluate_correctness_using_llm_judge\n",
            "    return await call_cohere_with_retry(co_client, messages)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/response_metrics.py\", line 204, in call_cohere_with_retry\n",
            "    response_text = await make_cohere_api_call(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/utils.py\", line 76, in make_cohere_api_call\n",
            "    response = await co_client.chat(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/client.py\", line 111, in _async_wrapped\n",
            "    return await func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/client.py\", line 40, in _async_wrapped\n",
            "    return await method(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/integrations/cohere/cohere_sdk.py\", line 137, in _wrapper\n",
            "    response = await fn(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/v2/client.py\", line 1662, in chat\n",
            "    raise TooManyRequestsError(\n",
            "cohere.errors.too_many_requests_error.TooManyRequestsError: status_code: 429, body: data=None message=\"You are using a Trial key, which is limited to 10 API calls / minute. You can continue to use the Trial key for free or upgrade to a Production key with higher rate limits at 'https://dashboard.cohere.com/api-keys'. Contact us on 'https://discord.gg/XW44jPfYJu' or email us at support@cohere.com with any questions\"\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Predict and score failed\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Predict and score failed\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/flow/eval.py\", line 287, in eval_example\n",
            "    eval_row = await self.predict_and_score(model, example)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/flow/eval.py\", line 224, in predict_and_score\n",
            "    result = await async_call(score_fn, **score_args)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/response_metrics.py\", line 285, in llm_response_scorer\n",
            "    evaluation_result = await evaluate_correctness_using_llm_judge(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/response_metrics.py\", line 267, in evaluate_correctness_using_llm_judge\n",
            "    return await call_cohere_with_retry(co_client, messages)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/response_metrics.py\", line 204, in call_cohere_with_retry\n",
            "    response_text = await make_cohere_api_call(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/utils.py\", line 76, in make_cohere_api_call\n",
            "    response = await co_client.chat(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/client.py\", line 111, in _async_wrapped\n",
            "    return await func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/client.py\", line 40, in _async_wrapped\n",
            "    return await method(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/integrations/cohere/cohere_sdk.py\", line 137, in _wrapper\n",
            "    response = await fn(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/v2/client.py\", line 1662, in chat\n",
            "    raise TooManyRequestsError(\n",
            "cohere.errors.too_many_requests_error.TooManyRequestsError: status_code: 429, body: data=None message=\"You are using a Trial key, which is limited to 10 API calls / minute. You can continue to use the Trial key for free or upgrade to a Production key with higher rate limits at 'https://dashboard.cohere.com/api-keys'. Contact us on 'https://discord.gg/XW44jPfYJu' or email us at support@cohere.com with any questions\"\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Evaluated \u001b[1;36m5\u001b[0m of \u001b[1;36m20\u001b[0m examples\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">20</span> examples\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Evaluated \u001b[1;36m6\u001b[0m of \u001b[1;36m20\u001b[0m examples\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">6</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">20</span> examples\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Evaluated \u001b[1;36m7\u001b[0m of \u001b[1;36m20\u001b[0m examples\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">7</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">20</span> examples\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Predict and score failed\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Predict and score failed\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/flow/eval.py\", line 287, in eval_example\n",
            "    eval_row = await self.predict_and_score(model, example)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/flow/eval.py\", line 224, in predict_and_score\n",
            "    result = await async_call(score_fn, **score_args)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/response_metrics.py\", line 285, in llm_response_scorer\n",
            "    evaluation_result = await evaluate_correctness_using_llm_judge(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/response_metrics.py\", line 267, in evaluate_correctness_using_llm_judge\n",
            "    return await call_cohere_with_retry(co_client, messages)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/response_metrics.py\", line 204, in call_cohere_with_retry\n",
            "    response_text = await make_cohere_api_call(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/utils.py\", line 76, in make_cohere_api_call\n",
            "    response = await co_client.chat(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/client.py\", line 111, in _async_wrapped\n",
            "    return await func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/client.py\", line 40, in _async_wrapped\n",
            "    return await method(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/integrations/cohere/cohere_sdk.py\", line 137, in _wrapper\n",
            "    response = await fn(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/v2/client.py\", line 1662, in chat\n",
            "    raise TooManyRequestsError(\n",
            "cohere.errors.too_many_requests_error.TooManyRequestsError: status_code: 429, body: data=None message=\"You are using a Trial key, which is limited to 10 API calls / minute. You can continue to use the Trial key for free or upgrade to a Production key with higher rate limits at 'https://dashboard.cohere.com/api-keys'. Contact us on 'https://discord.gg/XW44jPfYJu' or email us at support@cohere.com with any questions\"\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Predict and score failed\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Predict and score failed\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/flow/eval.py\", line 287, in eval_example\n",
            "    eval_row = await self.predict_and_score(model, example)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/flow/eval.py\", line 224, in predict_and_score\n",
            "    result = await async_call(score_fn, **score_args)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/response_metrics.py\", line 285, in llm_response_scorer\n",
            "    evaluation_result = await evaluate_correctness_using_llm_judge(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/response_metrics.py\", line 267, in evaluate_correctness_using_llm_judge\n",
            "    return await call_cohere_with_retry(co_client, messages)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/response_metrics.py\", line 204, in call_cohere_with_retry\n",
            "    response_text = await make_cohere_api_call(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/utils.py\", line 76, in make_cohere_api_call\n",
            "    response = await co_client.chat(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/client.py\", line 111, in _async_wrapped\n",
            "    return await func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/client.py\", line 40, in _async_wrapped\n",
            "    return await method(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/integrations/cohere/cohere_sdk.py\", line 137, in _wrapper\n",
            "    response = await fn(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/v2/client.py\", line 1662, in chat\n",
            "    raise TooManyRequestsError(\n",
            "cohere.errors.too_many_requests_error.TooManyRequestsError: status_code: 429, body: data=None message=\"You are using a Trial key, which is limited to 10 API calls / minute. You can continue to use the Trial key for free or upgrade to a Production key with higher rate limits at 'https://dashboard.cohere.com/api-keys'. Contact us on 'https://discord.gg/XW44jPfYJu' or email us at support@cohere.com with any questions\"\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Predict and score failed\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Predict and score failed\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/flow/eval.py\", line 287, in eval_example\n",
            "    eval_row = await self.predict_and_score(model, example)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/flow/eval.py\", line 224, in predict_and_score\n",
            "    result = await async_call(score_fn, **score_args)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/response_metrics.py\", line 285, in llm_response_scorer\n",
            "    evaluation_result = await evaluate_correctness_using_llm_judge(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/response_metrics.py\", line 267, in evaluate_correctness_using_llm_judge\n",
            "    return await call_cohere_with_retry(co_client, messages)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/response_metrics.py\", line 204, in call_cohere_with_retry\n",
            "    response_text = await make_cohere_api_call(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/utils.py\", line 76, in make_cohere_api_call\n",
            "    response = await co_client.chat(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/client.py\", line 111, in _async_wrapped\n",
            "    return await func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/client.py\", line 40, in _async_wrapped\n",
            "    return await method(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/integrations/cohere/cohere_sdk.py\", line 137, in _wrapper\n",
            "    response = await fn(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/v2/client.py\", line 1662, in chat\n",
            "    raise TooManyRequestsError(\n",
            "cohere.errors.too_many_requests_error.TooManyRequestsError: status_code: 429, body: data=None message=\"You are using a Trial key, which is limited to 10 API calls / minute. You can continue to use the Trial key for free or upgrade to a Production key with higher rate limits at 'https://dashboard.cohere.com/api-keys'. Contact us on 'https://discord.gg/XW44jPfYJu' or email us at support@cohere.com with any questions\"\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Predict and score failed\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Predict and score failed\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/flow/eval.py\", line 287, in eval_example\n",
            "    eval_row = await self.predict_and_score(model, example)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/flow/eval.py\", line 224, in predict_and_score\n",
            "    result = await async_call(score_fn, **score_args)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/response_metrics.py\", line 285, in llm_response_scorer\n",
            "    evaluation_result = await evaluate_correctness_using_llm_judge(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/response_metrics.py\", line 267, in evaluate_correctness_using_llm_judge\n",
            "    return await call_cohere_with_retry(co_client, messages)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/response_metrics.py\", line 204, in call_cohere_with_retry\n",
            "    response_text = await make_cohere_api_call(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/utils.py\", line 76, in make_cohere_api_call\n",
            "    response = await co_client.chat(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/client.py\", line 111, in _async_wrapped\n",
            "    return await func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/client.py\", line 40, in _async_wrapped\n",
            "    return await method(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/integrations/cohere/cohere_sdk.py\", line 137, in _wrapper\n",
            "    response = await fn(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/v2/client.py\", line 1662, in chat\n",
            "    raise TooManyRequestsError(\n",
            "cohere.errors.too_many_requests_error.TooManyRequestsError: status_code: 429, body: data=None message=\"You are using a Trial key, which is limited to 10 API calls / minute. You can continue to use the Trial key for free or upgrade to a Production key with higher rate limits at 'https://dashboard.cohere.com/api-keys'. Contact us on 'https://discord.gg/XW44jPfYJu' or email us at support@cohere.com with any questions\"\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Predict and score failed\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Predict and score failed\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/flow/eval.py\", line 287, in eval_example\n",
            "    eval_row = await self.predict_and_score(model, example)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/flow/eval.py\", line 224, in predict_and_score\n",
            "    result = await async_call(score_fn, **score_args)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/response_metrics.py\", line 285, in llm_response_scorer\n",
            "    evaluation_result = await evaluate_correctness_using_llm_judge(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/response_metrics.py\", line 267, in evaluate_correctness_using_llm_judge\n",
            "    return await call_cohere_with_retry(co_client, messages)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/response_metrics.py\", line 204, in call_cohere_with_retry\n",
            "    response_text = await make_cohere_api_call(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/utils.py\", line 76, in make_cohere_api_call\n",
            "    response = await co_client.chat(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/client.py\", line 111, in _async_wrapped\n",
            "    return await func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/client.py\", line 40, in _async_wrapped\n",
            "    return await method(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/integrations/cohere/cohere_sdk.py\", line 137, in _wrapper\n",
            "    response = await fn(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/v2/client.py\", line 1662, in chat\n",
            "    raise TooManyRequestsError(\n",
            "cohere.errors.too_many_requests_error.TooManyRequestsError: status_code: 429, body: data=None message=\"You are using a Trial key, which is limited to 10 API calls / minute. You can continue to use the Trial key for free or upgrade to a Production key with higher rate limits at 'https://dashboard.cohere.com/api-keys'. Contact us on 'https://discord.gg/XW44jPfYJu' or email us at support@cohere.com with any questions\"\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Predict and score failed\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Predict and score failed\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/flow/eval.py\", line 287, in eval_example\n",
            "    eval_row = await self.predict_and_score(model, example)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/flow/eval.py\", line 224, in predict_and_score\n",
            "    result = await async_call(score_fn, **score_args)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/response_metrics.py\", line 285, in llm_response_scorer\n",
            "    evaluation_result = await evaluate_correctness_using_llm_judge(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/response_metrics.py\", line 267, in evaluate_correctness_using_llm_judge\n",
            "    return await call_cohere_with_retry(co_client, messages)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/response_metrics.py\", line 204, in call_cohere_with_retry\n",
            "    response_text = await make_cohere_api_call(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/utils.py\", line 76, in make_cohere_api_call\n",
            "    response = await co_client.chat(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/client.py\", line 111, in _async_wrapped\n",
            "    return await func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/client.py\", line 40, in _async_wrapped\n",
            "    return await method(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/integrations/cohere/cohere_sdk.py\", line 137, in _wrapper\n",
            "    response = await fn(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/v2/client.py\", line 1662, in chat\n",
            "    raise TooManyRequestsError(\n",
            "cohere.errors.too_many_requests_error.TooManyRequestsError: status_code: 429, body: data=None message=\"You are using a Trial key, which is limited to 10 API calls / minute. You can continue to use the Trial key for free or upgrade to a Production key with higher rate limits at 'https://dashboard.cohere.com/api-keys'. Contact us on 'https://discord.gg/XW44jPfYJu' or email us at support@cohere.com with any questions\"\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Predict and score failed\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Predict and score failed\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/flow/eval.py\", line 287, in eval_example\n",
            "    eval_row = await self.predict_and_score(model, example)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/flow/eval.py\", line 224, in predict_and_score\n",
            "    result = await async_call(score_fn, **score_args)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/response_metrics.py\", line 285, in llm_response_scorer\n",
            "    evaluation_result = await evaluate_correctness_using_llm_judge(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/response_metrics.py\", line 267, in evaluate_correctness_using_llm_judge\n",
            "    return await call_cohere_with_retry(co_client, messages)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/response_metrics.py\", line 204, in call_cohere_with_retry\n",
            "    response_text = await make_cohere_api_call(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/utils.py\", line 76, in make_cohere_api_call\n",
            "    response = await co_client.chat(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/client.py\", line 111, in _async_wrapped\n",
            "    return await func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/client.py\", line 40, in _async_wrapped\n",
            "    return await method(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/integrations/cohere/cohere_sdk.py\", line 137, in _wrapper\n",
            "    response = await fn(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/v2/client.py\", line 1662, in chat\n",
            "    raise TooManyRequestsError(\n",
            "cohere.errors.too_many_requests_error.TooManyRequestsError: status_code: 429, body: data=None message=\"You are using a Trial key, which is limited to 10 API calls / minute. You can continue to use the Trial key for free or upgrade to a Production key with higher rate limits at 'https://dashboard.cohere.com/api-keys'. Contact us on 'https://discord.gg/XW44jPfYJu' or email us at support@cohere.com with any questions\"\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Predict and score failed\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Predict and score failed\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/flow/eval.py\", line 287, in eval_example\n",
            "    eval_row = await self.predict_and_score(model, example)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/flow/eval.py\", line 224, in predict_and_score\n",
            "    result = await async_call(score_fn, **score_args)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/response_metrics.py\", line 285, in llm_response_scorer\n",
            "    evaluation_result = await evaluate_correctness_using_llm_judge(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/response_metrics.py\", line 267, in evaluate_correctness_using_llm_judge\n",
            "    return await call_cohere_with_retry(co_client, messages)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/response_metrics.py\", line 204, in call_cohere_with_retry\n",
            "    response_text = await make_cohere_api_call(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/utils.py\", line 76, in make_cohere_api_call\n",
            "    response = await co_client.chat(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/client.py\", line 111, in _async_wrapped\n",
            "    return await func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/client.py\", line 40, in _async_wrapped\n",
            "    return await method(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/integrations/cohere/cohere_sdk.py\", line 137, in _wrapper\n",
            "    response = await fn(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/v2/client.py\", line 1662, in chat\n",
            "    raise TooManyRequestsError(\n",
            "cohere.errors.too_many_requests_error.TooManyRequestsError: status_code: 429, body: data=None message=\"You are using a Trial key, which is limited to 10 API calls / minute. You can continue to use the Trial key for free or upgrade to a Production key with higher rate limits at 'https://dashboard.cohere.com/api-keys'. Contact us on 'https://discord.gg/XW44jPfYJu' or email us at support@cohere.com with any questions\"\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Predict and score failed\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Predict and score failed\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/flow/eval.py\", line 287, in eval_example\n",
            "    eval_row = await self.predict_and_score(model, example)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/flow/eval.py\", line 224, in predict_and_score\n",
            "    result = await async_call(score_fn, **score_args)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/response_metrics.py\", line 285, in llm_response_scorer\n",
            "    evaluation_result = await evaluate_correctness_using_llm_judge(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/response_metrics.py\", line 267, in evaluate_correctness_using_llm_judge\n",
            "    return await call_cohere_with_retry(co_client, messages)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/response_metrics.py\", line 204, in call_cohere_with_retry\n",
            "    response_text = await make_cohere_api_call(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/utils.py\", line 76, in make_cohere_api_call\n",
            "    response = await co_client.chat(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/client.py\", line 111, in _async_wrapped\n",
            "    return await func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/client.py\", line 40, in _async_wrapped\n",
            "    return await method(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/integrations/cohere/cohere_sdk.py\", line 137, in _wrapper\n",
            "    response = await fn(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/v2/client.py\", line 1662, in chat\n",
            "    raise TooManyRequestsError(\n",
            "cohere.errors.too_many_requests_error.TooManyRequestsError: status_code: 429, body: data=None message=\"You are using a Trial key, which is limited to 10 API calls / minute. You can continue to use the Trial key for free or upgrade to a Production key with higher rate limits at 'https://dashboard.cohere.com/api-keys'. Contact us on 'https://discord.gg/XW44jPfYJu' or email us at support@cohere.com with any questions\"\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Predict and score failed\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Predict and score failed\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/flow/eval.py\", line 287, in eval_example\n",
            "    eval_row = await self.predict_and_score(model, example)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/flow/eval.py\", line 224, in predict_and_score\n",
            "    result = await async_call(score_fn, **score_args)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/response_metrics.py\", line 285, in llm_response_scorer\n",
            "    evaluation_result = await evaluate_correctness_using_llm_judge(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/response_metrics.py\", line 267, in evaluate_correctness_using_llm_judge\n",
            "    return await call_cohere_with_retry(co_client, messages)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/response_metrics.py\", line 204, in call_cohere_with_retry\n",
            "    response_text = await make_cohere_api_call(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/utils.py\", line 76, in make_cohere_api_call\n",
            "    response = await co_client.chat(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/client.py\", line 111, in _async_wrapped\n",
            "    return await func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/client.py\", line 40, in _async_wrapped\n",
            "    return await method(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/integrations/cohere/cohere_sdk.py\", line 137, in _wrapper\n",
            "    response = await fn(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/v2/client.py\", line 1662, in chat\n",
            "    raise TooManyRequestsError(\n",
            "cohere.errors.too_many_requests_error.TooManyRequestsError: status_code: 429, body: data=None message=\"You are using a Trial key, which is limited to 10 API calls / minute. You can continue to use the Trial key for free or upgrade to a Production key with higher rate limits at 'https://dashboard.cohere.com/api-keys'. Contact us on 'https://discord.gg/XW44jPfYJu' or email us at support@cohere.com with any questions\"\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Predict and score failed\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Predict and score failed\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/flow/eval.py\", line 287, in eval_example\n",
            "    eval_row = await self.predict_and_score(model, example)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/flow/eval.py\", line 224, in predict_and_score\n",
            "    result = await async_call(score_fn, **score_args)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/response_metrics.py\", line 285, in llm_response_scorer\n",
            "    evaluation_result = await evaluate_correctness_using_llm_judge(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/response_metrics.py\", line 267, in evaluate_correctness_using_llm_judge\n",
            "    return await call_cohere_with_retry(co_client, messages)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/response_metrics.py\", line 204, in call_cohere_with_retry\n",
            "    response_text = await make_cohere_api_call(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/utils.py\", line 76, in make_cohere_api_call\n",
            "    response = await co_client.chat(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/client.py\", line 111, in _async_wrapped\n",
            "    return await func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/client.py\", line 40, in _async_wrapped\n",
            "    return await method(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/integrations/cohere/cohere_sdk.py\", line 137, in _wrapper\n",
            "    response = await fn(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/v2/client.py\", line 1662, in chat\n",
            "    raise TooManyRequestsError(\n",
            "cohere.errors.too_many_requests_error.TooManyRequestsError: status_code: 429, body: data=None message=\"You are using a Trial key, which is limited to 10 API calls / minute. You can continue to use the Trial key for free or upgrade to a Production key with higher rate limits at 'https://dashboard.cohere.com/api-keys'. Contact us on 'https://discord.gg/XW44jPfYJu' or email us at support@cohere.com with any questions\"\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Predict and score failed\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Predict and score failed\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/flow/eval.py\", line 287, in eval_example\n",
            "    eval_row = await self.predict_and_score(model, example)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/flow/eval.py\", line 224, in predict_and_score\n",
            "    result = await async_call(score_fn, **score_args)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/response_metrics.py\", line 285, in llm_response_scorer\n",
            "    evaluation_result = await evaluate_correctness_using_llm_judge(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/response_metrics.py\", line 267, in evaluate_correctness_using_llm_judge\n",
            "    return await call_cohere_with_retry(co_client, messages)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/response_metrics.py\", line 204, in call_cohere_with_retry\n",
            "    response_text = await make_cohere_api_call(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/utils.py\", line 76, in make_cohere_api_call\n",
            "    response = await co_client.chat(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/client.py\", line 111, in _async_wrapped\n",
            "    return await func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/client.py\", line 40, in _async_wrapped\n",
            "    return await method(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/integrations/cohere/cohere_sdk.py\", line 137, in _wrapper\n",
            "    response = await fn(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/v2/client.py\", line 1662, in chat\n",
            "    raise TooManyRequestsError(\n",
            "cohere.errors.too_many_requests_error.TooManyRequestsError: status_code: 429, body: data=None message=\"You are using a Trial key, which is limited to 10 API calls / minute. You can continue to use the Trial key for free or upgrade to a Production key with higher rate limits at 'https://dashboard.cohere.com/api-keys'. Contact us on 'https://discord.gg/XW44jPfYJu' or email us at support@cohere.com with any questions\"\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Predict and score failed\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Predict and score failed\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/flow/eval.py\", line 287, in eval_example\n",
            "    eval_row = await self.predict_and_score(model, example)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/flow/eval.py\", line 224, in predict_and_score\n",
            "    result = await async_call(score_fn, **score_args)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/response_metrics.py\", line 285, in llm_response_scorer\n",
            "    evaluation_result = await evaluate_correctness_using_llm_judge(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/response_metrics.py\", line 267, in evaluate_correctness_using_llm_judge\n",
            "    return await call_cohere_with_retry(co_client, messages)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/response_metrics.py\", line 204, in call_cohere_with_retry\n",
            "    response_text = await make_cohere_api_call(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/content/edu/rag-advanced/notebooks/scripts/utils.py\", line 76, in make_cohere_api_call\n",
            "    response = await co_client.chat(\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/client.py\", line 111, in _async_wrapped\n",
            "    return await func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/client.py\", line 40, in _async_wrapped\n",
            "    return await method(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 459, in wrapper\n",
            "    res, _ = await _execute_call(wrapper, call, *args, **kwargs)  # type: ignore\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 287, in _call_async\n",
            "    return handle_exception(e)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/trace/op.py\", line 285, in _call_async\n",
            "    res = await func(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/weave/integrations/cohere/cohere_sdk.py\", line 137, in _wrapper\n",
            "    response = await fn(*args, **kwargs)\n",
            "  File \"/usr/local/lib/python3.10/dist-packages/cohere/v2/client.py\", line 1662, in chat\n",
            "    raise TooManyRequestsError(\n",
            "cohere.errors.too_many_requests_error.TooManyRequestsError: status_code: 429, body: data=None message=\"You are using a Trial key, which is limited to 10 API calls / minute. You can continue to use the Trial key for free or upgrade to a Production key with higher rate limits at 'https://dashboard.cohere.com/api-keys'. Contact us on 'https://discord.gg/XW44jPfYJu' or email us at support@cohere.com with any questions\"\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Evaluated \u001b[1;36m8\u001b[0m of \u001b[1;36m20\u001b[0m examples\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">8</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">20</span> examples\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Evaluated \u001b[1;36m9\u001b[0m of \u001b[1;36m20\u001b[0m examples\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">9</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">20</span> examples\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Evaluated \u001b[1;36m10\u001b[0m of \u001b[1;36m20\u001b[0m examples\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">10</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">20</span> examples\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Evaluated \u001b[1;36m11\u001b[0m of \u001b[1;36m20\u001b[0m examples\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">11</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">20</span> examples\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Evaluated \u001b[1;36m12\u001b[0m of \u001b[1;36m20\u001b[0m examples\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">12</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">20</span> examples\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Evaluated \u001b[1;36m13\u001b[0m of \u001b[1;36m20\u001b[0m examples\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">13</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">20</span> examples\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Evaluated \u001b[1;36m14\u001b[0m of \u001b[1;36m20\u001b[0m examples\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">14</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">20</span> examples\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Evaluated \u001b[1;36m15\u001b[0m of \u001b[1;36m20\u001b[0m examples\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">15</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">20</span> examples\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Evaluated \u001b[1;36m16\u001b[0m of \u001b[1;36m20\u001b[0m examples\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">16</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">20</span> examples\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Evaluated \u001b[1;36m17\u001b[0m of \u001b[1;36m20\u001b[0m examples\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">17</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">20</span> examples\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Evaluated \u001b[1;36m18\u001b[0m of \u001b[1;36m20\u001b[0m examples\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">18</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">20</span> examples\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Evaluated \u001b[1;36m19\u001b[0m of \u001b[1;36m20\u001b[0m examples\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">19</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">20</span> examples\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Evaluated \u001b[1;36m20\u001b[0m of \u001b[1;36m20\u001b[0m examples\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluated <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">20</span> of <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">20</span> examples\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Evaluation summary\n",
              "\u001b[1m{\u001b[0m\u001b[32m'llm_response_scorer'\u001b[0m: \u001b[3;35mNone\u001b[0m\u001b[1m}\u001b[0m\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">Evaluation summary\n",
              "<span style=\"font-weight: bold\">{</span><span style=\"color: #008000; text-decoration-color: #008000\">'llm_response_scorer'</span>: <span style=\"color: #800080; text-decoration-color: #800080; font-style: italic\">None</span><span style=\"font-weight: bold\">}</span>\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "🍩 https://wandb.ai/shikhar_dave-iit-jodhpur/rag-course/r/call/019275e2-0d8e-7980-ad56-5a676af15ce1\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "ZZlnPj4u4gOS"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}