Basic_RAG_Cohere_Wandb.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zf6GNVA-Js6y",
        "outputId": "9476f376-608f-4222-e880-4e9f3195adf7"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Cloning into 'edu'...\n",
            "remote: Enumerating objects: 4510, done.\u001b[K\n",
            "remote: Counting objects: 100% (1393/1393), done.\u001b[K\n",
            "remote: Compressing objects: 100% (531/531), done.\u001b[K\n",
            "remote: Total 4510 (delta 1058), reused 1024 (delta 859), pack-reused 3117 (from 1)\u001b[K\n",
            "Receiving objects: 100% (4510/4510), 28.01 MiB | 6.17 MiB/s, done.\n",
            "Resolving deltas: 100% (2472/2472), done.\n",
            "Updating files: 100% (461/461), done.\n",
            "/content/edu/rag-advanced\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m303.0/303.0 kB\u001b[0m \u001b[31m10.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m60.8/60.8 kB\u001b[0m \u001b[31m1.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m73.4/73.4 kB\u001b[0m \u001b[31m4.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m50.4/50.4 kB\u001b[0m \u001b[31m3.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m982.3/982.3 kB\u001b[0m \u001b[31m41.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m249.2/249.2 kB\u001b[0m \u001b[31m15.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m162.6/162.6 kB\u001b[0m \u001b[31m11.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m259.0/259.0 kB\u001b[0m \u001b[31m17.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m99.3/99.3 kB\u001b[0m \u001b[31m7.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m51.7/51.7 kB\u001b[0m \u001b[31m3.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m41.2/41.2 MB\u001b[0m \u001b[31m27.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.2/1.2 MB\u001b[0m \u001b[31m42.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m74.0/74.0 kB\u001b[0m \u001b[31m5.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m139.1/139.1 kB\u001b[0m \u001b[31m9.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m586.9/586.9 kB\u001b[0m \u001b[31m33.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.1/3.1 MB\u001b[0m \u001b[31m71.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m203.2/203.2 kB\u001b[0m \u001b[31m13.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m76.4/76.4 kB\u001b[0m \u001b[31m5.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m78.0/78.0 kB\u001b[0m \u001b[31m5.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m383.7/383.7 kB\u001b[0m \u001b[31m24.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.1/3.1 MB\u001b[0m \u001b[31m71.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.6/1.6 MB\u001b[0m \u001b[31m53.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m310.7/310.7 kB\u001b[0m \u001b[31m19.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m13.0/13.0 MB\u001b[0m \u001b[31m68.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m242.2/242.2 kB\u001b[0m \u001b[31m13.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.7/1.7 MB\u001b[0m \u001b[31m53.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m347.4/347.4 kB\u001b[0m \u001b[31m20.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m44.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m141.9/141.9 kB\u001b[0m \u001b[31m9.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m60.8/60.8 kB\u001b[0m \u001b[31m4.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m12.6/12.6 MB\u001b[0m \u001b[31m75.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.3/2.3 MB\u001b[0m \u001b[31m49.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m207.3/207.3 kB\u001b[0m \u001b[31m12.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m114.5/114.5 kB\u001b[0m \u001b[31m6.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m45.4/45.4 kB\u001b[0m \u001b[31m2.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m325.2/325.2 kB\u001b[0m \u001b[31m15.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m54.5/54.5 kB\u001b[0m \u001b[31m3.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m82.7/82.7 kB\u001b[0m \u001b[31m5.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m388.4/388.4 kB\u001b[0m \u001b[31m23.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m313.8/313.8 kB\u001b[0m \u001b[31m21.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m147.8/147.8 kB\u001b[0m \u001b[31m10.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m82.1/82.1 kB\u001b[0m \u001b[31m5.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m119.4/119.4 kB\u001b[0m \u001b[31m9.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m62.7/62.7 kB\u001b[0m \u001b[31m4.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.3/58.3 kB\u001b[0m \u001b[31m3.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m26.7/26.7 MB\u001b[0m \u001b[31m22.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.6/5.6 MB\u001b[0m \u001b[31m71.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m134.8/134.8 kB\u001b[0m \u001b[31m9.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m56.8/56.8 kB\u001b[0m \u001b[31m3.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m233.2/233.2 kB\u001b[0m \u001b[31m15.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m128.2/128.2 kB\u001b[0m \u001b[31m8.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m85.0/85.0 kB\u001b[0m \u001b[31m5.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m52.8/52.8 kB\u001b[0m \u001b[31m3.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m565.6/565.6 kB\u001b[0m \u001b[31m27.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m78.6/78.6 kB\u001b[0m \u001b[31m5.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Building wheel for fasttext-langdetect (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for PyStemmer (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for syncer (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for fasttext (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for warc3-wet-clueweb09 (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for cbor (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "gensim 4.3.3 requires scipy<1.14.0,>=1.7.0, but you have scipy 1.14.1 which is incompatible.\u001b[0m\u001b[31m\n",
            "\u001b[0m/content/edu/rag-advanced/notebooks\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[nltk_data] Downloading package wordnet to /root/nltk_data...\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {},
          "execution_count": 1
        }
      ],
      "source": [
        "!git clone https://github.com/wandb/edu.git\n",
        "%cd edu/rag-advanced\n",
        "!pip install -qqq -r requirements.txt\n",
        "%cd notebooks\n",
        "\n",
        "import nltk\n",
        "\n",
        "nltk.download(\"wordnet\")"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import getpass\n",
        "import os\n",
        "\n",
        "import wandb\n",
        "\n",
        "os.environ[\"COHERE_API_KEY\"] = getpass.getpass(\"Please enter your COHERE_API_KEY\")\n",
        "wandb.login()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 156
        },
        "id": "hOH8EexUJ9Ij",
        "outputId": "1bc7d560-3e90-49fc-febb-a386a987dba4"
      },
      "execution_count": 3,
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Please enter your COHERE_API_KEY··········\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\u001b[34m\u001b[1mwandb\u001b[0m: Using wandb-core as the SDK backend. Please refer to https://wandb.me/wandb-core for more information.\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ],
            "application/javascript": [
              "\n",
              "        window._wandbApiKey = new Promise((resolve, reject) => {\n",
              "            function loadScript(url) {\n",
              "            return new Promise(function(resolve, reject) {\n",
              "                let newScript = document.createElement(\"script\");\n",
              "                newScript.onerror = reject;\n",
              "                newScript.onload = resolve;\n",
              "                document.body.appendChild(newScript);\n",
              "                newScript.src = url;\n",
              "            });\n",
              "            }\n",
              "            loadScript(\"https://cdn.jsdelivr.net/npm/postmate/build/postmate.min.js\").then(() => {\n",
              "            const iframe = document.createElement('iframe')\n",
              "            iframe.style.cssText = \"width:0;height:0;border:none\"\n",
              "            document.body.appendChild(iframe)\n",
              "            const handshake = new Postmate({\n",
              "                container: iframe,\n",
              "                url: 'https://wandb.ai/authorize'\n",
              "            });\n",
              "            const timeout = setTimeout(() => reject(\"Couldn't auto authenticate\"), 5000)\n",
              "            handshake.then(function(child) {\n",
              "                child.on('authorize', data => {\n",
              "                    clearTimeout(timeout)\n",
              "                    resolve(data)\n",
              "                });\n",
              "            });\n",
              "            })\n",
              "        });\n",
              "    "
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\u001b[34m\u001b[1mwandb\u001b[0m: Logging into wandb.ai. (Learn how to deploy a W&B server locally: https://wandb.me/wandb-server)\n",
            "\u001b[34m\u001b[1mwandb\u001b[0m: You can find your API key in your browser here: https://wandb.ai/authorize\n",
            "wandb: Paste an API key from your profile and hit enter, or press ctrl+c to quit:"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            " ··········\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\u001b[34m\u001b[1mwandb\u001b[0m: Appending key for api.wandb.ai to your netrc file: /root/.netrc\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {},
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import pathlib\n",
        "from typing import List\n",
        "\n",
        "import weave\n",
        "\n",
        "from scripts.utils import display_source"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cNUVkARxRqMK",
        "outputId": "dea7a5f2-4a24-4311-baf3-6c980c1dcfb1"
      },
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "sagemaker.config INFO - Not applying SDK defaults from location: /etc/xdg/sagemaker/config.yaml\n",
            "sagemaker.config INFO - Not applying SDK defaults from location: /root/.config/sagemaker/config.yaml\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "WANDB_PROJECT = \"rag-course\"\n",
        "\n",
        "run = wandb.init(\n",
        "    project=WANDB_PROJECT,\n",
        "    group=\"Chapter 1\",\n",
        ")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 121
        },
        "id": "6r4qYtFUSGEG",
        "outputId": "189d0662-3456-46b1-8c7b-415903565f06"
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mshikhar_dave\u001b[0m (\u001b[33mshikhar_dave-iit-jodhpur\u001b[0m). Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "Tracking run with wandb version 0.18.3"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "Run data is saved locally in <code>/content/edu/rag-advanced/notebooks/wandb/run-20241008_151056-j9g94z6x</code>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "Syncing run <strong><a href='https://wandb.ai/shikhar_dave-iit-jodhpur/rag-course/runs/j9g94z6x' target=\"_blank\">chocolate-sunset-1</a></strong> to <a href='https://wandb.ai/shikhar_dave-iit-jodhpur/rag-course' target=\"_blank\">Weights & Biases</a> (<a href='https://wandb.me/run' target=\"_blank\">docs</a>)<br/>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              " View project at <a href='https://wandb.ai/shikhar_dave-iit-jodhpur/rag-course' target=\"_blank\">https://wandb.ai/shikhar_dave-iit-jodhpur/rag-course</a>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              " View run at <a href='https://wandb.ai/shikhar_dave-iit-jodhpur/rag-course/runs/j9g94z6x' target=\"_blank\">https://wandb.ai/shikhar_dave-iit-jodhpur/rag-course/runs/j9g94z6x</a>"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "documents_artifact = run.use_artifact(\n",
        "    f\"rag-course/dev/wandb_docs:latest\", type=\"dataset\"\n",
        ")\n",
        "data_dir = \"../data/wandb_docs\"\n",
        "\n",
        "docs_dir = documents_artifact.download(data_dir)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XyG-j5gprue2",
        "outputId": "ccdbecf0-2b2a-47c6-f593-fc7baa6e5449"
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "\u001b[34m\u001b[1mwandb\u001b[0m:   406 of 406 files downloaded.  \n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "docs_dir = pathlib.Path(docs_dir)\n",
        "docs_files = sorted(docs_dir.rglob(\"*.md\"))\n",
        "\n",
        "print(f\"Number of files: {len(docs_files)}\\n\")\n",
        "print(\"First 5 files:\\n{files}\".format(files=\"\\n\".join(map(str, docs_files[:5]))))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "flOnDwANsAeB",
        "outputId": "4d2b830d-bcc8-4f69-87a4-7c017fe0fc89"
      },
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Number of files: 406\n",
            "\n",
            "First 5 files:\n",
            "../data/wandb_docs/guides/app/features/anon.md\n",
            "../data/wandb_docs/guides/app/features/custom-charts/intro.md\n",
            "../data/wandb_docs/guides/app/features/custom-charts/walkthrough.md\n",
            "../data/wandb_docs/guides/app/features/intro.md\n",
            "../data/wandb_docs/guides/app/features/notes.md\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(docs_files[0].read_text())"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ouzPthAksJZJ",
        "outputId": "260a18b3-2a86-4dd7-f2d3-119bafcba7c4"
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "---\n",
            "description: Log and visualize data without a W&B account\n",
            "displayed_sidebar: default\n",
            "---\n",
            "\n",
            "# Anonymous Mode\n",
            "\n",
            "Are you publishing code that you want anyone to be able to run easily? Use Anonymous Mode to let someone run your code, see a W&B dashboard, and visualize results without needing to create a W&B account first.\n",
            "\n",
            "Allow results to be logged in Anonymous Mode with `wandb.init(`**`anonymous=\"allow\"`**`)`\n",
            "\n",
            ":::info\n",
            "**Publishing a paper?** Please [cite W&B](https://docs.wandb.ai/company/academics#bibtex-citation), and if you have questions about how to make your code accessible while using W&B, reach out to us at support@wandb.com.\n",
            ":::\n",
            "\n",
            "### How does someone without an account see results?\n",
            "\n",
            "If someone runs your script and you have to set `anonymous=\"allow\"`:\n",
            "\n",
            "1. **Auto-create temporary account:** W&B checks for an account that's already signed in. If there's no account, we automatically create a new anonymous account and save that API key for the session.\n",
            "2. **Log results quickly:** The user can run and re-run the script, and automatically see results show up in the W&B dashboard UI. These unclaimed anonymous runs will be available for 7 days.\n",
            "3. **Claim data when it's useful**: Once the user finds valuable results in W&B, they can easily click a button in the banner at the top of the page to save their run data to a real account. If they don't claim a run, it will be deleted after 7 days.\n",
            "\n",
            ":::caution\n",
            "**Anonymous run links are sensitive**. These links allow anyone to view and claim the results of an experiment for 7 days, so make sure to only share links with people you trust. If you're trying to share results publicly, but hide the author's identity, please contact us at support@wandb.com to share more about your use case.\n",
            ":::\n",
            "\n",
            "### What happens to users with existing accounts?\n",
            "\n",
            "If you set `anonymous=\"allow\"` in your script, we will check to make sure there's not an existing account first, before creating an anonymous account. This means that if a W&B user finds your script and runs it, their results will be logged correctly to their account, just like a normal run.\n",
            "\n",
            "### What are features that aren't available to anonymous users?\n",
            "\n",
            "*   **No persistent data**: Runs are only saved for 7 days in an anonymous account. Users can claim anonymous run data by saving it to a real account.\n",
            "\n",
            "\n",
            "![](@site/static/images/app_ui/anon_mode_no_data.png)\n",
            "\n",
            "*   **No artifact logging**: Runs will print a warning on the command line that you can't log an artifact to an anonymous run.\n",
            "\n",
            "![](@site/static/images/app_ui/anon_example_warning.png)\n",
            "\n",
            "* **No profile or settings pages**: Certain pages aren't available in the UI, because they're only useful for real accounts.\n",
            "\n",
            "## Example usage\n",
            "\n",
            "[Try the example notebook](http://bit.ly/anon-mode) to see how anonymous mode works.\n",
            "\n",
            "```python\n",
            "import wandb\n",
            "\n",
            "# Start a run allowing anonymous accounts\n",
            "wandb.init(anonymous=\"allow\")\n",
            "\n",
            "# Log results from your training loop\n",
            "wandb.log({\"acc\": 0.91})\n",
            "\n",
            "# Mark the run as finished\n",
            "wandb.finish()\n",
            "```\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# We'll store the files as dictionaries with some content and metadata\n",
        "data = []\n",
        "for file in docs_files:\n",
        "    content = file.read_text()\n",
        "    data.append(\n",
        "        {\n",
        "            \"content\": content,\n",
        "            \"metadata\": {\n",
        "                \"source\": str(file.relative_to(docs_dir)),\n",
        "                \"raw_tokens\": len(content.split()),\n",
        "            },\n",
        "        }\n",
        "    )\n",
        "data[:2]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6OhCizonsRur",
        "outputId": "a716d662-a5c4-4986-b101-2e95eaf608b0"
      },
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[{'content': '---\\ndescription: Log and visualize data without a W&B account\\ndisplayed_sidebar: default\\n---\\n\\n# Anonymous Mode\\n\\nAre you publishing code that you want anyone to be able to run easily? Use Anonymous Mode to let someone run your code, see a W&B dashboard, and visualize results without needing to create a W&B account first.\\n\\nAllow results to be logged in Anonymous Mode with `wandb.init(`**`anonymous=\"allow\"`**`)`\\n\\n:::info\\n**Publishing a paper?** Please [cite W&B](https://docs.wandb.ai/company/academics#bibtex-citation), and if you have questions about how to make your code accessible while using W&B, reach out to us at support@wandb.com.\\n:::\\n\\n### How does someone without an account see results?\\n\\nIf someone runs your script and you have to set `anonymous=\"allow\"`:\\n\\n1. **Auto-create temporary account:** W&B checks for an account that\\'s already signed in. If there\\'s no account, we automatically create a new anonymous account and save that API key for the session.\\n2. **Log results quickly:** The user can run and re-run the script, and automatically see results show up in the W&B dashboard UI. These unclaimed anonymous runs will be available for 7 days.\\n3. **Claim data when it\\'s useful**: Once the user finds valuable results in W&B, they can easily click a button in the banner at the top of the page to save their run data to a real account. If they don\\'t claim a run, it will be deleted after 7 days.\\n\\n:::caution\\n**Anonymous run links are sensitive**. These links allow anyone to view and claim the results of an experiment for 7 days, so make sure to only share links with people you trust. If you\\'re trying to share results publicly, but hide the author\\'s identity, please contact us at support@wandb.com to share more about your use case.\\n:::\\n\\n### What happens to users with existing accounts?\\n\\nIf you set `anonymous=\"allow\"` in your script, we will check to make sure there\\'s not an existing account first, before creating an anonymous account. This means that if a W&B user finds your script and runs it, their results will be logged correctly to their account, just like a normal run.\\n\\n### What are features that aren\\'t available to anonymous users?\\n\\n*   **No persistent data**: Runs are only saved for 7 days in an anonymous account. Users can claim anonymous run data by saving it to a real account.\\n\\n\\n![](@site/static/images/app_ui/anon_mode_no_data.png)\\n\\n*   **No artifact logging**: Runs will print a warning on the command line that you can\\'t log an artifact to an anonymous run.\\n\\n![](@site/static/images/app_ui/anon_example_warning.png)\\n\\n* **No profile or settings pages**: Certain pages aren\\'t available in the UI, because they\\'re only useful for real accounts.\\n\\n## Example usage\\n\\n[Try the example notebook](http://bit.ly/anon-mode) to see how anonymous mode works.\\n\\n```python\\nimport wandb\\n\\n# Start a run allowing anonymous accounts\\nwandb.init(anonymous=\"allow\")\\n\\n# Log results from your training loop\\nwandb.log({\"acc\": 0.91})\\n\\n# Mark the run as finished\\nwandb.finish()\\n```\\n',\n",
              "  'metadata': {'source': 'guides/app/features/anon.md', 'raw_tokens': 470}},\n",
              " {'content': '---\\nslug: /guides/app/features/custom-charts\\ndisplayed_sidebar: default\\n---\\n\\nimport Tabs from \\'@theme/Tabs\\';\\nimport TabItem from \\'@theme/TabItem\\';\\n\\n# Custom Charts\\n\\nUse **Custom Charts** to create charts that aren\\'t possible right now in the default UI. Log arbitrary tables of data and visualize them exactly how you want. Control details of fonts, colors, and tooltips with the power of [Vega](https://vega.github.io/vega/).\\n\\n* **What\\'s possible**: Read the[ launch announcement →](https://wandb.ai/wandb/posts/reports/Announcing-the-W-B-Machine-Learning-Visualization-IDE--VmlldzoyNjk3Nzg)\\n* **Code**: Try a live example in a[ hosted notebook →](https://tiny.cc/custom-charts)\\n* **Video**: Watch a quick [walkthrough video →](https://www.youtube.com/watch?v=3-N9OV6bkSM)\\n* **Example**: Quick Keras and Sklearn [demo notebook →](https://colab.research.google.com/drive/1g-gNGokPWM2Qbc8p1Gofud0\\\\_5AoZdoSD?usp=sharing)\\n\\n![Supported charts from vega.github.io/vega](/images/app_ui/supported_charts.png)\\n\\n### How it works\\n\\n1. **Log data**: From your script, log [config](../../../../guides/track/config.md) and summary data as you normally would when running with W&B. To visualize a list of multiple values logged at one specific time, use a custom`wandb.Table`\\n2. **Customize the chart**: Pull in any of this logged data with a [GraphQL](https://graphql.org) query. Visualize the results of your query with [Vega](https://vega.github.io/vega/), a powerful visualization grammar.\\n3. **Log the chart**: Call your own preset from your script with `wandb.plot_table()`.\\n\\n![](/images/app_ui/pr_roc.png)\\n\\n## Log charts from a script\\n\\n### Builtin presets\\n\\nThese presets have builtin `wandb.plot` methods that make it fast to log charts directly from your script and see the exact visualizations you\\'re looking for in the UI.\\n\\n<Tabs\\n  defaultValue=\"line-plot\"\\n  values={[\\n    {label: \\'Line plot\\', value: \\'line-plot\\'},\\n    {label: \\'Scatter plot\\', value: \\'scatter-plot\\'},\\n    {label: \\'Bar chart\\', value: \\'bar-chart\\'},\\n    {label: \\'Histogram\\', value: \\'histogram\\'},\\n    {label: \\'PR curve\\', value: \\'pr-curve\\'},\\n    {label: \\'ROC curve\\', value: \\'roc-curve\\'},\\n  ]}>\\n  <TabItem value=\"line-plot\">\\n\\n`wandb.plot.line()`\\n\\nLog a custom line plot—a list of connected and ordered points (x,y) on arbitrary axes x and y.\\n\\n```python\\ndata = [[x, y] for (x, y) in zip(x_values, y_values)]\\ntable = wandb.Table(data=data, columns=[\"x\", \"y\"])\\nwandb.log(\\n    {\\n        \"my_custom_plot_id\": wandb.plot.line(\\n            table, \"x\", \"y\", title=\"Custom Y vs X Line Plot\"\\n        )\\n    }\\n)\\n```\\n\\nYou can use this to log curves on any two dimensions. Note that if you\\'re plotting two lists of values against each other, the number of values in the lists must match exactly (i.e. each point must have an x and a y).\\n\\n![](/images/app_ui/line_plot.png)\\n\\n[See in the app →](https://wandb.ai/wandb/plots/reports/Custom-Line-Plots--VmlldzoyNjk5NTA)\\n\\n[Run the code →](https://tiny.cc/custom-charts)\\n\\n  </TabItem>\\n  <TabItem value=\"scatter-plot\">\\n\\n`wandb.plot.scatter()`\\n\\nLog a custom scatter plot—a list of points (x, y) on a pair of arbitrary axes x and y.\\n\\n```python\\ndata = [[x, y] for (x, y) in zip(class_x_prediction_scores, class_y_prediction_scores)]\\ntable = wandb.Table(data=data, columns=[\"class_x\", \"class_y\"])\\nwandb.log({\"my_custom_id\": wandb.plot.scatter(table, \"class_x\", \"class_y\")})\\n```\\n\\nYou can use this to log scatter points on any two dimensions. Note that if you\\'re plotting two lists of values against each other, the number of values in the lists must match exactly (i.e. each point must have an x and a y).\\n\\n![](/images/app_ui/demo_scatter_plot.png)\\n\\n[See in the app →](https://wandb.ai/wandb/plots/reports/Custom-Scatter-Plots--VmlldzoyNjk5NDQ)\\n\\n[Run the code →](https://tiny.cc/custom-charts)\\n\\n  </TabItem>\\n  <TabItem value=\"bar-chart\">\\n\\n`wandb.plot.bar()`\\n\\nLog a custom bar chart—a list of labeled values as bars—natively in a few lines:\\n\\n```python\\ndata = [[label, val] for (label, val) in zip(labels, values)]\\ntable = wandb.Table(data=data, columns=[\"label\", \"value\"])\\nwandb.log(\\n    {\\n        \"my_bar_chart_id\": wandb.plot.bar(\\n            table, \"label\", \"value\", title=\"Custom Bar Chart\"\\n        )\\n    }\\n)\\n```\\n\\nYou can use this to log arbitrary bar charts. Note that the number of labels and values in the lists must match exactly (i.e. each data point must have both).\\n\\n![](@site/static/images/app_ui/line_plot_bar_chart.png)\\n\\n[See in the app →](https://wandb.ai/wandb/plots/reports/Custom-Bar-Charts--VmlldzoyNzExNzk)\\n\\n[Run the code →](https://tiny.cc/custom-charts)\\n\\n  </TabItem>\\n  <TabItem value=\"histogram\">\\n\\n`wandb.plot.histogram()`\\n\\nLog a custom histogram—sort list of values into bins by count/frequency of occurrence—natively in a few lines. Let\\'s say I have a list of prediction confidence scores (`scores`) and want to visualize their distribution:\\n\\n```python\\ndata = [[s] for s in scores]\\ntable = wandb.Table(data=data, columns=[\"scores\"])\\nwandb.log({\"my_histogram\": wandb.plot.histogram(table, \"scores\", title=None)})\\n```\\n\\nYou can use this to log arbitrary histograms. Note that `data` is a list of lists, intended to support a 2D array of rows and columns.\\n\\n![](/images/app_ui/demo_custom_chart_histogram.png)\\n\\n[See in the app →](https://wandb.ai/wandb/plots/reports/Custom-Histograms--VmlldzoyNzE0NzM)\\n\\n[Run the code →](https://tiny.cc/custom-charts)\\n\\n  </TabItem>\\n    <TabItem value=\"pr-curve\">\\n\\n`wandb.plot.pr_curve()`\\n\\nCreate a [Precision-Recall curve](https://scikit-learn.org/stable/modules/generated/sklearn.metrics.precision\\\\_recall\\\\_curve.html#sklearn.metrics.precision\\\\_recall\\\\_curve) in one line:\\n\\n```python\\nplot = wandb.plot.pr_curve(ground_truth, predictions, labels=None, classes_to_plot=None)\\n\\nwandb.log({\"pr\": plot})\\n```\\n\\nYou can log this whenever your code has access to:\\n\\n* a model\\'s predicted scores (`predictions`) on a set of examples\\n* the corresponding ground truth labels (`ground_truth`) for those examples\\n* (optionally) a list of the labels/class names (`labels=[\"cat\", \"dog\", \"bird\"...]` if label index 0 means cat, 1 = dog, 2 = bird, etc.)\\n* (optionally) a subset (still in list format) of the labels to visualize in the plot\\n\\n![](/images/app_ui/demo_average_precision_lines.png)\\n\\n\\n[See in the app →](https://wandb.ai/wandb/plots/reports/Plot-Precision-Recall-Curves--VmlldzoyNjk1ODY)\\n\\n[Run the code →](https://colab.research.google.com/drive/1mS8ogA3LcZWOXchfJoMrboW3opY1A8BY?usp=sharing)\\n\\n  </TabItem>\\n  <TabItem value=\"roc-curve\">\\n\\n`wandb.plot.roc_curve()`\\n\\nCreate an [ROC curve](https://scikit-learn.org/stable/modules/generated/sklearn.metrics.roc\\\\_curve.html#sklearn.metrics.roc\\\\_curve) in one line:\\n\\n```python\\nplot = wandb.plot.roc_curve(\\n    ground_truth, predictions, labels=None, classes_to_plot=None\\n)\\n\\nwandb.log({\"roc\": plot})\\n```\\n\\nYou can log this whenever your code has access to:\\n\\n* a model\\'s predicted scores (`predictions`) on a set of examples\\n* the corresponding ground truth labels (`ground_truth`) for those examples\\n* (optionally) a list of the labels/ class names (`labels=[\"cat\", \"dog\", \"bird\"...]` if label index 0 means cat, 1 = dog, 2 = bird, etc.)\\n* (optionally) a subset (still in list format) of these labels to visualize on the plot\\n\\n![](/images/app_ui/demo_custom_chart_roc_curve.png)\\n\\n[See in the app →](https://wandb.ai/wandb/plots/reports/Plot-ROC-Curves--VmlldzoyNjk3MDE)\\n\\n[Run the code →](https://colab.research.google.com/drive/1\\\\_RMppCqsA8XInV\\\\_jhJz32NCZG6Z5t1RO?usp=sharing)\\n\\n  </TabItem>\\n</Tabs>\\n\\n### Custom presets\\n\\nTweak a builtin preset, or create a new preset, then save the chart. Use the chart ID to log data to that custom preset directly from your script.\\n\\n```python\\n# Create a table with the columns to plot\\ntable = wandb.Table(data=data, columns=[\"step\", \"height\"])\\n\\n# Map from the table\\'s columns to the chart\\'s fields\\nfields = {\"x\": \"step\", \"value\": \"height\"}\\n\\n# Use the table to populate the new custom chart preset\\n# To use your own saved chart preset, change the vega_spec_name\\nmy_custom_chart = wandb.plot_table(\\n    vega_spec_name=\"carey/new_chart\",\\n    data_table=table,\\n    fields=fields,\\n)\\n```\\n\\n[Run the code →](https://tiny.cc/custom-charts)\\n\\n![](/images/app_ui/custom_presets.png)\\n\\n## Log data\\n\\nHere are the data types you can log from your script and use in a custom chart:\\n\\n* **Config**: Initial settings of your experiment (your independent variables). This includes any named fields you\\'ve logged as keys to `wandb.config` at the start of your training (e.g. `wandb.config.learning_rate = 0.0001)`\\n* **Summary**: Single values logged during training (your results or dependent variables), e.g. `wandb.log({\"val_acc\" : 0.8})`. If you write to this key multiple times during training via `wandb.log()`, the summary is set to the final value of that key.\\n* **History**: The full time series of the logged scalar is available to the query via the `history` field\\n* **summaryTable**: If you need to log a list of multiple values, use a `wandb.Table()` to save that data, then query it in your custom panel.\\n* **historyTable**: If you need to see the history data, then query `historyTable` in your custom chart panel. Each time you call `wandb.Table()` or log a custom chart, you\\'re creating a new table in history for that step.\\n\\n### How to log a custom table\\n\\nUse `wandb.Table()` to log your data as a 2D array. Typically each row of this table represents one data point, and each column denotes the relevant fields/dimensions for each data point which you\\'d like to plot. As you configure a custom panel, the whole table will be accessible via the named key passed to `wandb.log()`(\"custom\\\\_data\\\\_table\" below), and the individual fields will be accessible via the column names (\"x\", \"y\", and \"z\"). You can log tables at multiple time steps throughout your experiment. The maximum size of each table is 10,000 rows.\\n\\n[Try it in a Google Colab →](https://tiny.cc/custom-charts)\\n\\n```python\\n# Logging a custom table of data\\nmy_custom_data = [[x1, y1, z1], [x2, y2, z2]]\\nwandb.log(\\n    {\"custom_data_table\": wandb.Table(data=my_custom_data, columns=[\"x\", \"y\", \"z\"])}\\n)\\n```\\n\\n## Customize the chart\\n\\nAdd a new custom chart to get started, then edit the query to select data from your visible runs. The query uses [GraphQL](https://graphql.org) to fetch data from the config, summary, and history fields in your runs.\\n\\n![Add a new custom chart, then edit the query](/images/app_ui/customize_chart.gif)\\n\\n### Custom visualizations\\n\\nSelect a **Chart** in the upper right corner to start with a default preset. Next, pick **Chart fields** to map the data you\\'re pulling in from the query to the corresponding fields in your chart. Here\\'s an example of selecting a metric to get from the query, then mapping that into the bar chart fields below.\\n\\n![Creating a custom bar chart showing accuracy across runs in a project](/images/app_ui/demo_make_a_custom_chart_bar_chart.gif)\\n\\n### How to edit Vega\\n\\nClick **Edit** at the top of the panel to go into [Vega](https://vega.github.io/vega/) edit mode. Here you can define a [Vega specification](https://vega.github.io/vega/docs/specification/) that creates an interactive chart in the UI. You can change any aspect of the chart, from the visual style (e.g. change the title, pick a different color scheme, show curves as a series of points instead of as connected lines) to the data itself (use a Vega transform to bin an array of values into a histogram, etc.). The panel preview will update interactively, so you can see the effect of your changes as you edit the Vega spec or query. The [Vega documentation and tutorials ](https://vega.github.io/vega/)are an excellent source of inspiration.\\n\\n**Field references**\\n\\nTo pull data into your chart from W&B, add template strings of the form `\"${field:<field-name>}\"` anywhere in your Vega spec. This will create a dropdown in the **Chart Fields** area on the right side, which users can use to select a query result column to map into Vega.\\n\\nTo set a default value for a field, use this syntax: `\"${field:<field-name>:<placeholder text>}\"`\\n\\n### Saving chart presets\\n\\nApply any changes to a specific visualization panel with the button at the bottom of the modal. Alternatively, you can save the Vega spec to use elsewhere in your project. To save the reusable chart definition, click **Save as** at the top of the Vega editor and give your preset a name.\\n\\n## Articles and guides\\n\\n1. [The W&B Machine Learning Visualization IDE](https://wandb.ai/wandb/posts/reports/The-W-B-Machine-Learning-Visualization-IDE--VmlldzoyNjk3Nzg)\\n2. [Visualizing NLP Attention Based Models](https://wandb.ai/kylegoyette/gradientsandtranslation2/reports/Visualizing-NLP-Attention-Based-Models-Using-Custom-Charts--VmlldzoyNjg2MjM)\\n3. [Visualizing The Effect of Attention on Gradient Flow](https://wandb.ai/kylegoyette/gradientsandtranslation/reports/Visualizing-The-Effect-of-Attention-on-Gradient-Flow-Using-Custom-Charts--VmlldzoyNjg1NDg)\\n4. [Logging arbitrary curves](https://wandb.ai/stacey/presets/reports/Logging-Arbitrary-Curves--VmlldzoyNzQyMzA)\\n\\n## Frequently asked questions\\n\\n### Coming soon\\n\\n* **Polling**: Auto-refresh of data in the chart\\n* **Sampling**: Dynamically adjust the total number of points loaded into the panel for efficiency\\n\\n### Gotchas\\n\\n* Not seeing the data you\\'re expecting in the query as you\\'re editing your chart? It might be because the column you\\'re looking for is not logged in the runs you have selected. Save your chart and go back out to the runs table, and select the runs you\\'d like to visualize with the **eye** icon.\\n\\n### How to show a \"step slider\" in a custom chart?\\n\\nThis can be enabled on the “Other settings” page of the custom chart editor. If you change your query to use a `historyTable` instead of the `summaryTable`, you\\'ll get an option to “Show step selector” in the custom chart editor. This gives you a slider that lets you select the step.\\n\\n<!-- ![Show step slider in a custom chart](/images/app_ui/step_sllider_custon_charts.mov>) -->\\n\\n### How to delete a custom chart preset?\\n\\nYou can do this by going into the custom chart editor. Then click on the currently selected chart type, this will open up a menu with all your presets. Hover the mouse on a preset you want to delete and then click on the Trash icon.\\n\\n![](/images/app_ui/delete_custome_chart_preset.gif)\\n\\n\\n### Common use cases\\n\\n* Customize bar plots with error bars\\n* Show model validation metrics which require custom x-y coordinates (like precision-recall curves)\\n* Overlay data distributions from two different models/experiments as histograms\\n* Show changes in a metric via snapshots at multiple points during training\\n* Create a unique visualization not yet available in W&B (and hopefully share it with the world)\\n',\n",
              "  'metadata': {'source': 'guides/app/features/custom-charts/intro.md',\n",
              "   'raw_tokens': 1904}}]"
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "total_tokens = sum(map(lambda x: x[\"metadata\"][\"raw_tokens\"], data))\n",
        "print(f\"Total Tokens in dataset: {total_tokens}\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dgzWIQU8sejl",
        "outputId": "0ad3c3a2-9c10-4d5f-eaed-77adc170e3c0"
      },
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Total Tokens in dataset: 260530\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "WANDB_PROJECT = \"rag-course\"\n",
        "weave_client = weave.init(WANDB_PROJECT)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "10liDIsfsqXp",
        "outputId": "665a9619-7b58-48ad-facc-440864fbd2ca"
      },
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "WARNING:weave.trace.env:There are different credentials in the netrc file and the environment. Using the environment value.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Logged in as Weights & Biases user: shikhar_dave.\n",
            "View Weave data at https://wandb.ai/shikhar_dave-iit-jodhpur/rag-course/weave\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# build weave dataset\n",
        "raw_data = weave.Dataset(name=\"raw_data\", rows=data)\n",
        "\n",
        "# publish the dataset\n",
        "weave.publish(raw_data)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Uo6W6Ko_t6tk",
        "outputId": "4dcf30cb-c91c-4067-ec83-7f9533504dbd"
      },
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "📦 Published to https://wandb.ai/shikhar_dave-iit-jodhpur/rag-course/weave/objects/raw_data/versions/FtXM1nny6EREQTDUYbI7olXelWIWwvUwlVXYNhXYsPM\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "ObjectRef(entity='shikhar_dave-iit-jodhpur', project='rag-course', name='raw_data', digest='FtXM1nny6EREQTDUYbI7olXelWIWwvUwlVXYNhXYsPM', extra=())"
            ]
          },
          "metadata": {},
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# These are hyperparameters of our ingestion pipeline\n",
        "\n",
        "CHUNK_SIZE = 300\n",
        "CHUNK_OVERLAP = 0\n",
        "\n",
        "\n",
        "def split_into_chunks(\n",
        "    text: str, chunk_size: int = CHUNK_SIZE, chunk_overlap: int = CHUNK_OVERLAP\n",
        ") -> List[str]:\n",
        "    \"\"\"Function to split the text into chunks of a maximum number of tokens\n",
        "    ensure that the chunks are of size CHUNK_SIZE and overlap by chunk_overlap tokens\n",
        "    use the `tokenizer.encode` method to tokenize the text\n",
        "    \"\"\"\n",
        "    tokens = text.split()\n",
        "    chunks = []\n",
        "    start = 0\n",
        "    while start < len(tokens):\n",
        "        end = start + chunk_size\n",
        "        chunk = tokens[start:end]\n",
        "        chunks.append(\" \".join(chunk))\n",
        "        start = end - chunk_overlap\n",
        "    return chunks"
      ],
      "metadata": {
        "id": "F9hB_3f_uBIX"
      },
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# download the `raw_data` Dataset\n",
        "raw_data = weave.ref(\n",
        "    \"weave:///rag-course/rag-course/object/raw_data:nuZosGsP58MXKxhluN2hzvKK9XB8xSxlTuOBmEzWzLo\"\n",
        ").get()\n",
        "raw_data = weave.ref(\"raw_data:v0\").get()\n",
        "\n",
        "# this is how we index into the data\n",
        "print(raw_data.rows[:2])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0IlpWL7VufI7",
        "outputId": "e66623ce-71a9-4c8a-b384-b4df4caeda94"
      },
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[WeaveDict({'content': '---\\ndescription: Log and visualize data without a W&B account\\ndisplayed_sidebar: default\\n---\\n\\n# Anonymous Mode\\n\\nAre you publishing code that you want anyone to be able to run easily? Use Anonymous Mode to let someone run your code, see a W&B dashboard, and visualize results without needing to create a W&B account first.\\n\\nAllow results to be logged in Anonymous Mode with `wandb.init(`**`anonymous=\"allow\"`**`)`\\n\\n:::info\\n**Publishing a paper?** Please [cite W&B](https://docs.wandb.ai/company/academics#bibtex-citation), and if you have questions about how to make your code accessible while using W&B, reach out to us at support@wandb.com.\\n:::\\n\\n### How does someone without an account see results?\\n\\nIf someone runs your script and you have to set `anonymous=\"allow\"`:\\n\\n1. **Auto-create temporary account:** W&B checks for an account that\\'s already signed in. If there\\'s no account, we automatically create a new anonymous account and save that API key for the session.\\n2. **Log results quickly:** The user can run and re-run the script, and automatically see results show up in the W&B dashboard UI. These unclaimed anonymous runs will be available for 7 days.\\n3. **Claim data when it\\'s useful**: Once the user finds valuable results in W&B, they can easily click a button in the banner at the top of the page to save their run data to a real account. If they don\\'t claim a run, it will be deleted after 7 days.\\n\\n:::caution\\n**Anonymous run links are sensitive**. These links allow anyone to view and claim the results of an experiment for 7 days, so make sure to only share links with people you trust. If you\\'re trying to share results publicly, but hide the author\\'s identity, please contact us at support@wandb.com to share more about your use case.\\n:::\\n\\n### What happens to users with existing accounts?\\n\\nIf you set `anonymous=\"allow\"` in your script, we will check to make sure there\\'s not an existing account first, before creating an anonymous account. This means that if a W&B user finds your script and runs it, their results will be logged correctly to their account, just like a normal run.\\n\\n### What are features that aren\\'t available to anonymous users?\\n\\n*   **No persistent data**: Runs are only saved for 7 days in an anonymous account. Users can claim anonymous run data by saving it to a real account.\\n\\n\\n![](@site/static/images/app_ui/anon_mode_no_data.png)\\n\\n*   **No artifact logging**: Runs will print a warning on the command line that you can\\'t log an artifact to an anonymous run.\\n\\n![](@site/static/images/app_ui/anon_example_warning.png)\\n\\n* **No profile or settings pages**: Certain pages aren\\'t available in the UI, because they\\'re only useful for real accounts.\\n\\n## Example usage\\n\\n[Try the example notebook](http://bit.ly/anon-mode) to see how anonymous mode works.\\n\\n```python\\nimport wandb\\n\\n# Start a run allowing anonymous accounts\\nwandb.init(anonymous=\"allow\")\\n\\n# Log results from your training loop\\nwandb.log({\"acc\": 0.91})\\n\\n# Mark the run as finished\\nwandb.finish()\\n```\\n', 'metadata': {'source': 'guides/app/features/anon.md', 'raw_tokens': 470}}), WeaveDict({'content': '---\\nslug: /guides/app/features/custom-charts\\ndisplayed_sidebar: default\\n---\\n\\nimport Tabs from \\'@theme/Tabs\\';\\nimport TabItem from \\'@theme/TabItem\\';\\n\\n# Custom Charts\\n\\nUse **Custom Charts** to create charts that aren\\'t possible right now in the default UI. Log arbitrary tables of data and visualize them exactly how you want. Control details of fonts, colors, and tooltips with the power of [Vega](https://vega.github.io/vega/).\\n\\n* **What\\'s possible**: Read the[ launch announcement →](https://wandb.ai/wandb/posts/reports/Announcing-the-W-B-Machine-Learning-Visualization-IDE--VmlldzoyNjk3Nzg)\\n* **Code**: Try a live example in a[ hosted notebook →](https://tiny.cc/custom-charts)\\n* **Video**: Watch a quick [walkthrough video →](https://www.youtube.com/watch?v=3-N9OV6bkSM)\\n* **Example**: Quick Keras and Sklearn [demo notebook →](https://colab.research.google.com/drive/1g-gNGokPWM2Qbc8p1Gofud0\\\\_5AoZdoSD?usp=sharing)\\n\\n![Supported charts from vega.github.io/vega](/images/app_ui/supported_charts.png)\\n\\n### How it works\\n\\n1. **Log data**: From your script, log [config](../../../../guides/track/config.md) and summary data as you normally would when running with W&B. To visualize a list of multiple values logged at one specific time, use a custom`wandb.Table`\\n2. **Customize the chart**: Pull in any of this logged data with a [GraphQL](https://graphql.org) query. Visualize the results of your query with [Vega](https://vega.github.io/vega/), a powerful visualization grammar.\\n3. **Log the chart**: Call your own preset from your script with `wandb.plot_table()`.\\n\\n![](/images/app_ui/pr_roc.png)\\n\\n## Log charts from a script\\n\\n### Builtin presets\\n\\nThese presets have builtin `wandb.plot` methods that make it fast to log charts directly from your script and see the exact visualizations you\\'re looking for in the UI.\\n\\n<Tabs\\n  defaultValue=\"line-plot\"\\n  values={[\\n    {label: \\'Line plot\\', value: \\'line-plot\\'},\\n    {label: \\'Scatter plot\\', value: \\'scatter-plot\\'},\\n    {label: \\'Bar chart\\', value: \\'bar-chart\\'},\\n    {label: \\'Histogram\\', value: \\'histogram\\'},\\n    {label: \\'PR curve\\', value: \\'pr-curve\\'},\\n    {label: \\'ROC curve\\', value: \\'roc-curve\\'},\\n  ]}>\\n  <TabItem value=\"line-plot\">\\n\\n`wandb.plot.line()`\\n\\nLog a custom line plot—a list of connected and ordered points (x,y) on arbitrary axes x and y.\\n\\n```python\\ndata = [[x, y] for (x, y) in zip(x_values, y_values)]\\ntable = wandb.Table(data=data, columns=[\"x\", \"y\"])\\nwandb.log(\\n    {\\n        \"my_custom_plot_id\": wandb.plot.line(\\n            table, \"x\", \"y\", title=\"Custom Y vs X Line Plot\"\\n        )\\n    }\\n)\\n```\\n\\nYou can use this to log curves on any two dimensions. Note that if you\\'re plotting two lists of values against each other, the number of values in the lists must match exactly (i.e. each point must have an x and a y).\\n\\n![](/images/app_ui/line_plot.png)\\n\\n[See in the app →](https://wandb.ai/wandb/plots/reports/Custom-Line-Plots--VmlldzoyNjk5NTA)\\n\\n[Run the code →](https://tiny.cc/custom-charts)\\n\\n  </TabItem>\\n  <TabItem value=\"scatter-plot\">\\n\\n`wandb.plot.scatter()`\\n\\nLog a custom scatter plot—a list of points (x, y) on a pair of arbitrary axes x and y.\\n\\n```python\\ndata = [[x, y] for (x, y) in zip(class_x_prediction_scores, class_y_prediction_scores)]\\ntable = wandb.Table(data=data, columns=[\"class_x\", \"class_y\"])\\nwandb.log({\"my_custom_id\": wandb.plot.scatter(table, \"class_x\", \"class_y\")})\\n```\\n\\nYou can use this to log scatter points on any two dimensions. Note that if you\\'re plotting two lists of values against each other, the number of values in the lists must match exactly (i.e. each point must have an x and a y).\\n\\n![](/images/app_ui/demo_scatter_plot.png)\\n\\n[See in the app →](https://wandb.ai/wandb/plots/reports/Custom-Scatter-Plots--VmlldzoyNjk5NDQ)\\n\\n[Run the code →](https://tiny.cc/custom-charts)\\n\\n  </TabItem>\\n  <TabItem value=\"bar-chart\">\\n\\n`wandb.plot.bar()`\\n\\nLog a custom bar chart—a list of labeled values as bars—natively in a few lines:\\n\\n```python\\ndata = [[label, val] for (label, val) in zip(labels, values)]\\ntable = wandb.Table(data=data, columns=[\"label\", \"value\"])\\nwandb.log(\\n    {\\n        \"my_bar_chart_id\": wandb.plot.bar(\\n            table, \"label\", \"value\", title=\"Custom Bar Chart\"\\n        )\\n    }\\n)\\n```\\n\\nYou can use this to log arbitrary bar charts. Note that the number of labels and values in the lists must match exactly (i.e. each data point must have both).\\n\\n![](@site/static/images/app_ui/line_plot_bar_chart.png)\\n\\n[See in the app →](https://wandb.ai/wandb/plots/reports/Custom-Bar-Charts--VmlldzoyNzExNzk)\\n\\n[Run the code →](https://tiny.cc/custom-charts)\\n\\n  </TabItem>\\n  <TabItem value=\"histogram\">\\n\\n`wandb.plot.histogram()`\\n\\nLog a custom histogram—sort list of values into bins by count/frequency of occurrence—natively in a few lines. Let\\'s say I have a list of prediction confidence scores (`scores`) and want to visualize their distribution:\\n\\n```python\\ndata = [[s] for s in scores]\\ntable = wandb.Table(data=data, columns=[\"scores\"])\\nwandb.log({\"my_histogram\": wandb.plot.histogram(table, \"scores\", title=None)})\\n```\\n\\nYou can use this to log arbitrary histograms. Note that `data` is a list of lists, intended to support a 2D array of rows and columns.\\n\\n![](/images/app_ui/demo_custom_chart_histogram.png)\\n\\n[See in the app →](https://wandb.ai/wandb/plots/reports/Custom-Histograms--VmlldzoyNzE0NzM)\\n\\n[Run the code →](https://tiny.cc/custom-charts)\\n\\n  </TabItem>\\n    <TabItem value=\"pr-curve\">\\n\\n`wandb.plot.pr_curve()`\\n\\nCreate a [Precision-Recall curve](https://scikit-learn.org/stable/modules/generated/sklearn.metrics.precision\\\\_recall\\\\_curve.html#sklearn.metrics.precision\\\\_recall\\\\_curve) in one line:\\n\\n```python\\nplot = wandb.plot.pr_curve(ground_truth, predictions, labels=None, classes_to_plot=None)\\n\\nwandb.log({\"pr\": plot})\\n```\\n\\nYou can log this whenever your code has access to:\\n\\n* a model\\'s predicted scores (`predictions`) on a set of examples\\n* the corresponding ground truth labels (`ground_truth`) for those examples\\n* (optionally) a list of the labels/class names (`labels=[\"cat\", \"dog\", \"bird\"...]` if label index 0 means cat, 1 = dog, 2 = bird, etc.)\\n* (optionally) a subset (still in list format) of the labels to visualize in the plot\\n\\n![](/images/app_ui/demo_average_precision_lines.png)\\n\\n\\n[See in the app →](https://wandb.ai/wandb/plots/reports/Plot-Precision-Recall-Curves--VmlldzoyNjk1ODY)\\n\\n[Run the code →](https://colab.research.google.com/drive/1mS8ogA3LcZWOXchfJoMrboW3opY1A8BY?usp=sharing)\\n\\n  </TabItem>\\n  <TabItem value=\"roc-curve\">\\n\\n`wandb.plot.roc_curve()`\\n\\nCreate an [ROC curve](https://scikit-learn.org/stable/modules/generated/sklearn.metrics.roc\\\\_curve.html#sklearn.metrics.roc\\\\_curve) in one line:\\n\\n```python\\nplot = wandb.plot.roc_curve(\\n    ground_truth, predictions, labels=None, classes_to_plot=None\\n)\\n\\nwandb.log({\"roc\": plot})\\n```\\n\\nYou can log this whenever your code has access to:\\n\\n* a model\\'s predicted scores (`predictions`) on a set of examples\\n* the corresponding ground truth labels (`ground_truth`) for those examples\\n* (optionally) a list of the labels/ class names (`labels=[\"cat\", \"dog\", \"bird\"...]` if label index 0 means cat, 1 = dog, 2 = bird, etc.)\\n* (optionally) a subset (still in list format) of these labels to visualize on the plot\\n\\n![](/images/app_ui/demo_custom_chart_roc_curve.png)\\n\\n[See in the app →](https://wandb.ai/wandb/plots/reports/Plot-ROC-Curves--VmlldzoyNjk3MDE)\\n\\n[Run the code →](https://colab.research.google.com/drive/1\\\\_RMppCqsA8XInV\\\\_jhJz32NCZG6Z5t1RO?usp=sharing)\\n\\n  </TabItem>\\n</Tabs>\\n\\n### Custom presets\\n\\nTweak a builtin preset, or create a new preset, then save the chart. Use the chart ID to log data to that custom preset directly from your script.\\n\\n```python\\n# Create a table with the columns to plot\\ntable = wandb.Table(data=data, columns=[\"step\", \"height\"])\\n\\n# Map from the table\\'s columns to the chart\\'s fields\\nfields = {\"x\": \"step\", \"value\": \"height\"}\\n\\n# Use the table to populate the new custom chart preset\\n# To use your own saved chart preset, change the vega_spec_name\\nmy_custom_chart = wandb.plot_table(\\n    vega_spec_name=\"carey/new_chart\",\\n    data_table=table,\\n    fields=fields,\\n)\\n```\\n\\n[Run the code →](https://tiny.cc/custom-charts)\\n\\n![](/images/app_ui/custom_presets.png)\\n\\n## Log data\\n\\nHere are the data types you can log from your script and use in a custom chart:\\n\\n* **Config**: Initial settings of your experiment (your independent variables). This includes any named fields you\\'ve logged as keys to `wandb.config` at the start of your training (e.g. `wandb.config.learning_rate = 0.0001)`\\n* **Summary**: Single values logged during training (your results or dependent variables), e.g. `wandb.log({\"val_acc\" : 0.8})`. If you write to this key multiple times during training via `wandb.log()`, the summary is set to the final value of that key.\\n* **History**: The full time series of the logged scalar is available to the query via the `history` field\\n* **summaryTable**: If you need to log a list of multiple values, use a `wandb.Table()` to save that data, then query it in your custom panel.\\n* **historyTable**: If you need to see the history data, then query `historyTable` in your custom chart panel. Each time you call `wandb.Table()` or log a custom chart, you\\'re creating a new table in history for that step.\\n\\n### How to log a custom table\\n\\nUse `wandb.Table()` to log your data as a 2D array. Typically each row of this table represents one data point, and each column denotes the relevant fields/dimensions for each data point which you\\'d like to plot. As you configure a custom panel, the whole table will be accessible via the named key passed to `wandb.log()`(\"custom\\\\_data\\\\_table\" below), and the individual fields will be accessible via the column names (\"x\", \"y\", and \"z\"). You can log tables at multiple time steps throughout your experiment. The maximum size of each table is 10,000 rows.\\n\\n[Try it in a Google Colab →](https://tiny.cc/custom-charts)\\n\\n```python\\n# Logging a custom table of data\\nmy_custom_data = [[x1, y1, z1], [x2, y2, z2]]\\nwandb.log(\\n    {\"custom_data_table\": wandb.Table(data=my_custom_data, columns=[\"x\", \"y\", \"z\"])}\\n)\\n```\\n\\n## Customize the chart\\n\\nAdd a new custom chart to get started, then edit the query to select data from your visible runs. The query uses [GraphQL](https://graphql.org) to fetch data from the config, summary, and history fields in your runs.\\n\\n![Add a new custom chart, then edit the query](/images/app_ui/customize_chart.gif)\\n\\n### Custom visualizations\\n\\nSelect a **Chart** in the upper right corner to start with a default preset. Next, pick **Chart fields** to map the data you\\'re pulling in from the query to the corresponding fields in your chart. Here\\'s an example of selecting a metric to get from the query, then mapping that into the bar chart fields below.\\n\\n![Creating a custom bar chart showing accuracy across runs in a project](/images/app_ui/demo_make_a_custom_chart_bar_chart.gif)\\n\\n### How to edit Vega\\n\\nClick **Edit** at the top of the panel to go into [Vega](https://vega.github.io/vega/) edit mode. Here you can define a [Vega specification](https://vega.github.io/vega/docs/specification/) that creates an interactive chart in the UI. You can change any aspect of the chart, from the visual style (e.g. change the title, pick a different color scheme, show curves as a series of points instead of as connected lines) to the data itself (use a Vega transform to bin an array of values into a histogram, etc.). The panel preview will update interactively, so you can see the effect of your changes as you edit the Vega spec or query. The [Vega documentation and tutorials ](https://vega.github.io/vega/)are an excellent source of inspiration.\\n\\n**Field references**\\n\\nTo pull data into your chart from W&B, add template strings of the form `\"${field:<field-name>}\"` anywhere in your Vega spec. This will create a dropdown in the **Chart Fields** area on the right side, which users can use to select a query result column to map into Vega.\\n\\nTo set a default value for a field, use this syntax: `\"${field:<field-name>:<placeholder text>}\"`\\n\\n### Saving chart presets\\n\\nApply any changes to a specific visualization panel with the button at the bottom of the modal. Alternatively, you can save the Vega spec to use elsewhere in your project. To save the reusable chart definition, click **Save as** at the top of the Vega editor and give your preset a name.\\n\\n## Articles and guides\\n\\n1. [The W&B Machine Learning Visualization IDE](https://wandb.ai/wandb/posts/reports/The-W-B-Machine-Learning-Visualization-IDE--VmlldzoyNjk3Nzg)\\n2. [Visualizing NLP Attention Based Models](https://wandb.ai/kylegoyette/gradientsandtranslation2/reports/Visualizing-NLP-Attention-Based-Models-Using-Custom-Charts--VmlldzoyNjg2MjM)\\n3. [Visualizing The Effect of Attention on Gradient Flow](https://wandb.ai/kylegoyette/gradientsandtranslation/reports/Visualizing-The-Effect-of-Attention-on-Gradient-Flow-Using-Custom-Charts--VmlldzoyNjg1NDg)\\n4. [Logging arbitrary curves](https://wandb.ai/stacey/presets/reports/Logging-Arbitrary-Curves--VmlldzoyNzQyMzA)\\n\\n## Frequently asked questions\\n\\n### Coming soon\\n\\n* **Polling**: Auto-refresh of data in the chart\\n* **Sampling**: Dynamically adjust the total number of points loaded into the panel for efficiency\\n\\n### Gotchas\\n\\n* Not seeing the data you\\'re expecting in the query as you\\'re editing your chart? It might be because the column you\\'re looking for is not logged in the runs you have selected. Save your chart and go back out to the runs table, and select the runs you\\'d like to visualize with the **eye** icon.\\n\\n### How to show a \"step slider\" in a custom chart?\\n\\nThis can be enabled on the “Other settings” page of the custom chart editor. If you change your query to use a `historyTable` instead of the `summaryTable`, you\\'ll get an option to “Show step selector” in the custom chart editor. This gives you a slider that lets you select the step.\\n\\n<!-- ![Show step slider in a custom chart](/images/app_ui/step_sllider_custon_charts.mov>) -->\\n\\n### How to delete a custom chart preset?\\n\\nYou can do this by going into the custom chart editor. Then click on the currently selected chart type, this will open up a menu with all your presets. Hover the mouse on a preset you want to delete and then click on the Trash icon.\\n\\n![](/images/app_ui/delete_custome_chart_preset.gif)\\n\\n\\n### Common use cases\\n\\n* Customize bar plots with error bars\\n* Show model validation metrics which require custom x-y coordinates (like precision-recall curves)\\n* Overlay data distributions from two different models/experiments as histograms\\n* Show changes in a metric via snapshots at multiple points during training\\n* Create a unique visualization not yet available in W&B (and hopefully share it with the world)\\n', 'metadata': {'source': 'guides/app/features/custom-charts/intro.md', 'raw_tokens': 1904}})]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "chunked_data = []\n",
        "for doc in raw_data.rows:\n",
        "    chunks = split_into_chunks(doc[\"content\"])\n",
        "    for chunk in chunks:\n",
        "        chunked_data.append(\n",
        "            {\n",
        "                \"content\": chunk,\n",
        "                \"metadata\": {\n",
        "                    \"source\": doc[\"metadata\"][\"source\"],\n",
        "                    \"raw_tokens\": len(chunk.split()),\n",
        "                },\n",
        "            }\n",
        "        )\n",
        "\n",
        "print(chunked_data[:2])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WYEvrHFLuoJi",
        "outputId": "2c55cdc4-1e52-4bb9-c525-5e3214bc8325"
      },
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[{'content': '--- description: Log and visualize data without a W&B account displayed_sidebar: default --- # Anonymous Mode Are you publishing code that you want anyone to be able to run easily? Use Anonymous Mode to let someone run your code, see a W&B dashboard, and visualize results without needing to create a W&B account first. Allow results to be logged in Anonymous Mode with `wandb.init(`**`anonymous=\"allow\"`**`)` :::info **Publishing a paper?** Please [cite W&B](https://docs.wandb.ai/company/academics#bibtex-citation), and if you have questions about how to make your code accessible while using W&B, reach out to us at support@wandb.com. ::: ### How does someone without an account see results? If someone runs your script and you have to set `anonymous=\"allow\"`: 1. **Auto-create temporary account:** W&B checks for an account that\\'s already signed in. If there\\'s no account, we automatically create a new anonymous account and save that API key for the session. 2. **Log results quickly:** The user can run and re-run the script, and automatically see results show up in the W&B dashboard UI. These unclaimed anonymous runs will be available for 7 days. 3. **Claim data when it\\'s useful**: Once the user finds valuable results in W&B, they can easily click a button in the banner at the top of the page to save their run data to a real account. If they don\\'t claim a run, it will be deleted after 7 days. :::caution **Anonymous run links are sensitive**. These links allow anyone to view and claim the results of an experiment for 7 days, so make sure to only share links with people you trust. If you\\'re trying to share results publicly, but hide the author\\'s identity, please contact us at support@wandb.com to share more about your use case. ::: ### What happens to users with existing accounts? If you set `anonymous=\"allow\"` in', 'metadata': {'source': 'guides/app/features/anon.md', 'raw_tokens': 300}}, {'content': 'your script, we will check to make sure there\\'s not an existing account first, before creating an anonymous account. This means that if a W&B user finds your script and runs it, their results will be logged correctly to their account, just like a normal run. ### What are features that aren\\'t available to anonymous users? * **No persistent data**: Runs are only saved for 7 days in an anonymous account. Users can claim anonymous run data by saving it to a real account. ![](@site/static/images/app_ui/anon_mode_no_data.png) * **No artifact logging**: Runs will print a warning on the command line that you can\\'t log an artifact to an anonymous run. ![](@site/static/images/app_ui/anon_example_warning.png) * **No profile or settings pages**: Certain pages aren\\'t available in the UI, because they\\'re only useful for real accounts. ## Example usage [Try the example notebook](http://bit.ly/anon-mode) to see how anonymous mode works. ```python import wandb # Start a run allowing anonymous accounts wandb.init(anonymous=\"allow\") # Log results from your training loop wandb.log({\"acc\": 0.91}) # Mark the run as finished wandb.finish() ```', 'metadata': {'source': 'guides/app/features/anon.md', 'raw_tokens': 170}}]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from scripts.preprocess import make_text_tokenization_safe\n",
        "display_source(make_text_tokenization_safe)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 513
        },
        "id": "s0kuet_CwuPV",
        "outputId": "3ef93b2b-c2c9-4e1e-8269-ed306f3c28a9"
      },
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 1 \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mdef\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;166;226;46;48;2;39;40;34mmake_text_tokenization_safe\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[48;2;39;40;34m                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 2 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mcontent\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mstr\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mspecial_tokens_set\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mset\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mget_special_tokens_set\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[48;2;39;40;34m                                          \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 3 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m-\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m>\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mstr\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[48;2;39;40;34m                                                                                                     \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 4 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\"\"\u001b[0m\u001b[48;2;39;40;34m                                                                                                       \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 5 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m    Makes the text safe for tokenization by removing special tokens.\u001b[0m\u001b[48;2;39;40;34m                                          \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 6 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 7 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m    Args:\u001b[0m\u001b[48;2;39;40;34m                                                                                                     \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 8 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        content: A string containing the text to be processed.\u001b[0m\u001b[48;2;39;40;34m                                                \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 9 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        special_tokens_set: A set of special tokens to be removed from the text.\u001b[0m\u001b[48;2;39;40;34m                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m10 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m11 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m    Returns:\u001b[0m\u001b[48;2;39;40;34m                                                                                                  \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m12 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        A string with the special tokens removed.\u001b[0m\u001b[48;2;39;40;34m                                                             \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m13 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m    \"\"\"\u001b[0m\u001b[48;2;39;40;34m                                                                                                       \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m14 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m15 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mdef\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;166;226;46;48;2;39;40;34mremove_special_tokens\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mtext\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mstr\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m-\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m>\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mstr\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[48;2;39;40;34m                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m16 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m        \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\"\"\u001b[0m\u001b[48;2;39;40;34m                                                                                                   \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m17 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        Removes special tokens from the given text.\u001b[0m\u001b[48;2;39;40;34m                                                           \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m18 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m19 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        Args:\u001b[0m\u001b[48;2;39;40;34m                                                                                                 \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m20 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m            text: A string representing the text.\u001b[0m\u001b[48;2;39;40;34m                                                             \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m21 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m22 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        Returns:\u001b[0m\u001b[48;2;39;40;34m                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m23 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m            The text with special tokens removed.\u001b[0m\u001b[48;2;39;40;34m                                                             \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m24 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        \"\"\"\u001b[0m\u001b[48;2;39;40;34m                                                                                                   \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m25 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m        \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mfor\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mtoken\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34min\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mspecial_tokens_set\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[48;2;39;40;34m                                                                      \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m26 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m            \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mtext\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mtext\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m.\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mreplace\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mtoken\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[48;2;39;40;34m                                                                    \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m27 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m        \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mreturn\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mtext\u001b[0m\u001b[48;2;39;40;34m                                                                                           \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m28 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m29 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mcleaned_content\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mremove_special_tokens\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mcontent\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[48;2;39;40;34m                                                          \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m30 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mreturn\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mcleaned_content\u001b[0m\u001b[48;2;39;40;34m                                                                                    \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m31 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 1 </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">def</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> </span><span style=\"color: #a6e22e; text-decoration-color: #a6e22e; background-color: #272822\">make_text_tokenization_safe</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">(</span><span style=\"background-color: #272822\">                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 2 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    content: str, special_tokens_set: set </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> get_special_tokens_set()</span><span style=\"background-color: #272822\">                                          </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 3 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">) </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">-&gt;</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> str:</span><span style=\"background-color: #272822\">                                                                                                     </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 4 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">\"\"\"</span><span style=\"background-color: #272822\">                                                                                                       </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 5 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">    Makes the text safe for tokenization by removing special tokens.</span><span style=\"background-color: #272822\">                                          </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 6 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 7 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">    Args:</span><span style=\"background-color: #272822\">                                                                                                     </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 8 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        content: A string containing the text to be processed.</span><span style=\"background-color: #272822\">                                                </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 9 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        special_tokens_set: A set of special tokens to be removed from the text.</span><span style=\"background-color: #272822\">                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">10 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">11 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">    Returns:</span><span style=\"background-color: #272822\">                                                                                                  </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">12 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        A string with the special tokens removed.</span><span style=\"background-color: #272822\">                                                             </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">13 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">    \"\"\"</span><span style=\"background-color: #272822\">                                                                                                       </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">14 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">15 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">def</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> </span><span style=\"color: #a6e22e; text-decoration-color: #a6e22e; background-color: #272822\">remove_special_tokens</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">(text: str) </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">-&gt;</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> str:</span><span style=\"background-color: #272822\">                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">16 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">        </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">\"\"\"</span><span style=\"background-color: #272822\">                                                                                                   </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">17 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        Removes special tokens from the given text.</span><span style=\"background-color: #272822\">                                                           </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">18 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">19 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        Args:</span><span style=\"background-color: #272822\">                                                                                                 </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">20 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">            text: A string representing the text.</span><span style=\"background-color: #272822\">                                                             </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">21 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">22 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        Returns:</span><span style=\"background-color: #272822\">                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">23 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">            The text with special tokens removed.</span><span style=\"background-color: #272822\">                                                             </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">24 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        \"\"\"</span><span style=\"background-color: #272822\">                                                                                                   </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">25 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">        </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">for</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> token </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">in</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> special_tokens_set:</span><span style=\"background-color: #272822\">                                                                      </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">26 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">            text </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> text</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">.</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">replace(token, </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">\"\"</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">)</span><span style=\"background-color: #272822\">                                                                    </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">27 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">        </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">return</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> text</span><span style=\"background-color: #272822\">                                                                                           </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">28 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">29 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    cleaned_content </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> remove_special_tokens(content)</span><span style=\"background-color: #272822\">                                                          </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">30 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">return</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> cleaned_content</span><span style=\"background-color: #272822\">                                                                                    </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">31 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "cleaned_data = []\n",
        "for doc in chunked_data:\n",
        "    cleaned_doc = doc.copy()\n",
        "    cleaned_doc[\"cleaned_content\"] = make_text_tokenization_safe(doc[\"content\"])\n",
        "    cleaned_doc[\"metadata\"][\"cleaned_tokens\"] = len(\n",
        "        cleaned_doc[\"cleaned_content\"].split()\n",
        "    )\n",
        "    cleaned_data.append(cleaned_doc)\n",
        "print(cleaned_data[:2])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PFPLh_c1xqQk",
        "outputId": "9884faaa-3def-4036-9b79-d6a69164c145"
      },
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[{'content': '--- description: Log and visualize data without a W&B account displayed_sidebar: default --- # Anonymous Mode Are you publishing code that you want anyone to be able to run easily? Use Anonymous Mode to let someone run your code, see a W&B dashboard, and visualize results without needing to create a W&B account first. Allow results to be logged in Anonymous Mode with `wandb.init(`**`anonymous=\"allow\"`**`)` :::info **Publishing a paper?** Please [cite W&B](https://docs.wandb.ai/company/academics#bibtex-citation), and if you have questions about how to make your code accessible while using W&B, reach out to us at support@wandb.com. ::: ### How does someone without an account see results? If someone runs your script and you have to set `anonymous=\"allow\"`: 1. **Auto-create temporary account:** W&B checks for an account that\\'s already signed in. If there\\'s no account, we automatically create a new anonymous account and save that API key for the session. 2. **Log results quickly:** The user can run and re-run the script, and automatically see results show up in the W&B dashboard UI. These unclaimed anonymous runs will be available for 7 days. 3. **Claim data when it\\'s useful**: Once the user finds valuable results in W&B, they can easily click a button in the banner at the top of the page to save their run data to a real account. If they don\\'t claim a run, it will be deleted after 7 days. :::caution **Anonymous run links are sensitive**. These links allow anyone to view and claim the results of an experiment for 7 days, so make sure to only share links with people you trust. If you\\'re trying to share results publicly, but hide the author\\'s identity, please contact us at support@wandb.com to share more about your use case. ::: ### What happens to users with existing accounts? If you set `anonymous=\"allow\"` in', 'metadata': {'source': 'guides/app/features/anon.md', 'raw_tokens': 300, 'cleaned_tokens': 300}, 'cleaned_content': '--- description: Log and visualize data without a W&B account displayed_sidebar: default --- # Anonymous Mode Are you publishing code that you want anyone to be able to run easily? Use Anonymous Mode to let someone run your code, see a W&B dashboard, and visualize results without needing to create a W&B account first. Allow results to be logged in Anonymous Mode with `wandb.init(`**`anonymous=\"allow\"`**`)` :::info **Publishing a paper?** Please [cite W&B](https://docs.wandb.ai/company/academics#bibtex-citation), and if you have questions about how to make your code accessible while using W&B, reach out to us at support@wandb.com. ::: ### How does someone without an account see results? If someone runs your script and you have to set `anonymous=\"allow\"`: 1. **Auto-create temporary account:** W&B checks for an account that\\'s already signed in. If there\\'s no account, we automatically create a new anonymous account and save that API key for the session. 2. **Log results quickly:** The user can run and re-run the script, and automatically see results show up in the W&B dashboard UI. These unclaimed anonymous runs will be available for 7 days. 3. **Claim data when it\\'s useful**: Once the user finds valuable results in W&B, they can easily click a button in the banner at the top of the page to save their run data to a real account. If they don\\'t claim a run, it will be deleted after 7 days. :::caution **Anonymous run links are sensitive**. These links allow anyone to view and claim the results of an experiment for 7 days, so make sure to only share links with people you trust. If you\\'re trying to share results publicly, but hide the author\\'s identity, please contact us at support@wandb.com to share more about your use case. ::: ### What happens to users with existing accounts? If you set `anonymous=\"allow\"` in'}, {'content': 'your script, we will check to make sure there\\'s not an existing account first, before creating an anonymous account. This means that if a W&B user finds your script and runs it, their results will be logged correctly to their account, just like a normal run. ### What are features that aren\\'t available to anonymous users? * **No persistent data**: Runs are only saved for 7 days in an anonymous account. Users can claim anonymous run data by saving it to a real account. ![](@site/static/images/app_ui/anon_mode_no_data.png) * **No artifact logging**: Runs will print a warning on the command line that you can\\'t log an artifact to an anonymous run. ![](@site/static/images/app_ui/anon_example_warning.png) * **No profile or settings pages**: Certain pages aren\\'t available in the UI, because they\\'re only useful for real accounts. ## Example usage [Try the example notebook](http://bit.ly/anon-mode) to see how anonymous mode works. ```python import wandb # Start a run allowing anonymous accounts wandb.init(anonymous=\"allow\") # Log results from your training loop wandb.log({\"acc\": 0.91}) # Mark the run as finished wandb.finish() ```', 'metadata': {'source': 'guides/app/features/anon.md', 'raw_tokens': 170, 'cleaned_tokens': 170}, 'cleaned_content': 'your script, we will check to make sure there\\'s not an existing account first, before creating an anonymous account. This means that if a W&B user finds your script and runs it, their results will be logged correctly to their account, just like a normal run. ### What are features that aren\\'t available to anonymous users? * **No persistent data**: Runs are only saved for 7 days in an anonymous account. Users can claim anonymous run data by saving it to a real account. ![](@site/static/images/app_ui/anon_mode_no_data.png) * **No artifact logging**: Runs will print a warning on the command line that you can\\'t log an artifact to an anonymous run. ![](@site/static/images/app_ui/anon_example_warning.png) * **No profile or settings pages**: Certain pages aren\\'t available in the UI, because they\\'re only useful for real accounts. ## Example usage [Try the example notebook](http://bit.ly/anon-mode) to see how anonymous mode works. ```python import wandb # Start a run allowing anonymous accounts wandb.init(anonymous=\"allow\") # Log results from your training loop wandb.log({\"acc\": 0.91}) # Mark the run as finished wandb.finish() ```'}]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "dataset = weave.Dataset(name=\"chunked_data\", rows=cleaned_data)\n",
        "weave.publish(dataset)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gPccDKt8xwea",
        "outputId": "b63e6894-4ac3-47a7-8c8b-10f38dfe2a00"
      },
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "📦 Published to https://wandb.ai/shikhar_dave-iit-jodhpur/rag-course/weave/objects/chunked_data/versions/2QCiHXRTg3HGY1QYBLC7yGWar7TYVmi6YulXCOm7Fp4\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "ObjectRef(entity='shikhar_dave-iit-jodhpur', project='rag-course', name='chunked_data', digest='2QCiHXRTg3HGY1QYBLC7yGWar7TYVmi6YulXCOm7Fp4', extra=())"
            ]
          },
          "metadata": {},
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "chunked_data = weave.ref(\n",
        "    \"weave:///rag-course/rag-course/object/chunked_data:Lt6M8qCUICD1JZTlMYzuDLTVtvFYESxvj3tcAIoPrtE\"\n",
        ").get()\n",
        "chunked_data = weave.ref(\"chunked_data:v0\").get()\n",
        "print(chunked_data.rows[:2])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VF0Z2rPIx1rU",
        "outputId": "6e40e658-ad02-48cd-81a1-08478750b192"
      },
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[WeaveDict({'content': '--- description: Log and visualize data without a W&B account displayed_sidebar: default --- # Anonymous Mode Are you publishing code that you want anyone to be able to run easily? Use Anonymous Mode to let someone run your code, see a W&B dashboard, and visualize results without needing to create a W&B account first. Allow results to be logged in Anonymous Mode with `wandb.init(`**`anonymous=\"allow\"`**`)` :::info **Publishing a paper?** Please [cite W&B](https://docs.wandb.ai/company/academics#bibtex-citation), and if you have questions about how to make your code accessible while using W&B, reach out to us at support@wandb.com. ::: ### How does someone without an account see results? If someone runs your script and you have to set `anonymous=\"allow\"`: 1. **Auto-create temporary account:** W&B checks for an account that\\'s already signed in. If there\\'s no account, we automatically create a new anonymous account and save that API key for the session. 2. **Log results quickly:** The user can run and re-run the script, and automatically see results show up in the W&B dashboard UI. These unclaimed anonymous runs will be available for 7 days. 3. **Claim data when it\\'s useful**: Once the user finds valuable results in W&B, they can easily click a button in the banner at the top of the page to save their run data to a real account. If they don\\'t claim a run, it will be deleted after 7 days. :::caution **Anonymous run links are sensitive**. These links allow anyone to view and claim the results of an experiment for 7 days, so make sure to only share links with people you trust. If you\\'re trying to share results publicly, but hide the author\\'s identity, please contact us at support@wandb.com to share more about your use case. ::: ### What happens to users with existing accounts? If you set `anonymous=\"allow\"` in', 'metadata': {'source': 'guides/app/features/anon.md', 'raw_tokens': 300, 'cleaned_tokens': 300}, 'cleaned_content': '--- description: Log and visualize data without a W&B account displayed_sidebar: default --- # Anonymous Mode Are you publishing code that you want anyone to be able to run easily? Use Anonymous Mode to let someone run your code, see a W&B dashboard, and visualize results without needing to create a W&B account first. Allow results to be logged in Anonymous Mode with `wandb.init(`**`anonymous=\"allow\"`**`)` :::info **Publishing a paper?** Please [cite W&B](https://docs.wandb.ai/company/academics#bibtex-citation), and if you have questions about how to make your code accessible while using W&B, reach out to us at support@wandb.com. ::: ### How does someone without an account see results? If someone runs your script and you have to set `anonymous=\"allow\"`: 1. **Auto-create temporary account:** W&B checks for an account that\\'s already signed in. If there\\'s no account, we automatically create a new anonymous account and save that API key for the session. 2. **Log results quickly:** The user can run and re-run the script, and automatically see results show up in the W&B dashboard UI. These unclaimed anonymous runs will be available for 7 days. 3. **Claim data when it\\'s useful**: Once the user finds valuable results in W&B, they can easily click a button in the banner at the top of the page to save their run data to a real account. If they don\\'t claim a run, it will be deleted after 7 days. :::caution **Anonymous run links are sensitive**. These links allow anyone to view and claim the results of an experiment for 7 days, so make sure to only share links with people you trust. If you\\'re trying to share results publicly, but hide the author\\'s identity, please contact us at support@wandb.com to share more about your use case. ::: ### What happens to users with existing accounts? If you set `anonymous=\"allow\"` in'}), WeaveDict({'content': 'your script, we will check to make sure there\\'s not an existing account first, before creating an anonymous account. This means that if a W&B user finds your script and runs it, their results will be logged correctly to their account, just like a normal run. ### What are features that aren\\'t available to anonymous users? * **No persistent data**: Runs are only saved for 7 days in an anonymous account. Users can claim anonymous run data by saving it to a real account. ![](@site/static/images/app_ui/anon_mode_no_data.png) * **No artifact logging**: Runs will print a warning on the command line that you can\\'t log an artifact to an anonymous run. ![](@site/static/images/app_ui/anon_example_warning.png) * **No profile or settings pages**: Certain pages aren\\'t available in the UI, because they\\'re only useful for real accounts. ## Example usage [Try the example notebook](http://bit.ly/anon-mode) to see how anonymous mode works. ```python import wandb # Start a run allowing anonymous accounts wandb.init(anonymous=\"allow\") # Log results from your training loop wandb.log({\"acc\": 0.91}) # Mark the run as finished wandb.finish() ```', 'metadata': {'source': 'guides/app/features/anon.md', 'raw_tokens': 170, 'cleaned_tokens': 170}, 'cleaned_content': 'your script, we will check to make sure there\\'s not an existing account first, before creating an anonymous account. This means that if a W&B user finds your script and runs it, their results will be logged correctly to their account, just like a normal run. ### What are features that aren\\'t available to anonymous users? * **No persistent data**: Runs are only saved for 7 days in an anonymous account. Users can claim anonymous run data by saving it to a real account. ![](@site/static/images/app_ui/anon_mode_no_data.png) * **No artifact logging**: Runs will print a warning on the command line that you can\\'t log an artifact to an anonymous run. ![](@site/static/images/app_ui/anon_example_warning.png) * **No profile or settings pages**: Certain pages aren\\'t available in the UI, because they\\'re only useful for real accounts. ## Example usage [Try the example notebook](http://bit.ly/anon-mode) to see how anonymous mode works. ```python import wandb # Start a run allowing anonymous accounts wandb.init(anonymous=\"allow\") # Log results from your training loop wandb.log({\"acc\": 0.91}) # Mark the run as finished wandb.finish() ```'})]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from scripts.retriever import TFIDFRetriever\n",
        "display_source(TFIDFRetriever)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "7QmYwqW0yr_X",
        "outputId": "01966cf6-edaf-4988-de7c-4cf477a8919a"
      },
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 1 \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mclass\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;166;226;46;48;2;39;40;34mTFIDFRetriever\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mweave\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m.\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mModel\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[48;2;39;40;34m                                                                            \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 2 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\"\"\u001b[0m\u001b[48;2;39;40;34m                                                                                                       \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 3 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m    A retriever model that uses TF-IDF for indexing and searching documents.\u001b[0m\u001b[48;2;39;40;34m                                  \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 4 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 5 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m    Attributes:\u001b[0m\u001b[48;2;39;40;34m                                                                                               \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 6 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        vectorizer (TfidfVectorizer): The TF-IDF vectorizer.\u001b[0m\u001b[48;2;39;40;34m                                                  \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 7 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        index (list): The indexed data.\u001b[0m\u001b[48;2;39;40;34m                                                                       \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 8 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        data (list): The data to be indexed.\u001b[0m\u001b[48;2;39;40;34m                                                                  \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 9 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m    \"\"\"\u001b[0m\u001b[48;2;39;40;34m                                                                                                       \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m10 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m11 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mvectorizer\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mTfidfVectorizer\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mTfidfVectorizer\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[48;2;39;40;34m                                                           \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m12 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mindex\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mlist\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mNone\u001b[0m\u001b[48;2;39;40;34m                                                                                        \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m13 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mdata\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mlist\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mNone\u001b[0m\u001b[48;2;39;40;34m                                                                                         \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m14 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m15 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mdef\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;166;226;46;48;2;39;40;34mindex_data\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mself\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mdata\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[48;2;39;40;34m                                                                               \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m16 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m        \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\"\"\u001b[0m\u001b[48;2;39;40;34m                                                                                                   \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m17 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        Indexes the provided data using TF-IDF.\u001b[0m\u001b[48;2;39;40;34m                                                               \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m18 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m19 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        Args:\u001b[0m\u001b[48;2;39;40;34m                                                                                                 \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m20 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m            data (list): A list of documents to be indexed. Each document should be a dictionary\u001b[0m\u001b[48;2;39;40;34m              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m21 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m                         containing a key 'cleaned_content' with the text to be indexed.\u001b[0m\u001b[48;2;39;40;34m                      \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m22 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        \"\"\"\u001b[0m\u001b[48;2;39;40;34m                                                                                                   \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m23 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m        \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mself\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m.\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mdata\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mdata\u001b[0m\u001b[48;2;39;40;34m                                                                                      \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m24 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m        \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mdocs\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mdoc\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34mcleaned_content\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mfor\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mdoc\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34min\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mdata\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[48;2;39;40;34m                                                       \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m25 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m        \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mself\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m.\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mindex\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mself\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m.\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mvectorizer\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m.\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mfit_transform\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mdocs\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[48;2;39;40;34m                                                      \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m26 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m27 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;166;226;46;48;2;39;40;34m@weave\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m.\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mop\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[48;2;39;40;34m                                                                                               \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m28 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mdef\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;166;226;46;48;2;39;40;34msearch\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mself\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mquery\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mk\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;174;129;255;48;2;39;40;34m5\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[48;2;39;40;34m                                                                             \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m29 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m        \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\"\"\u001b[0m\u001b[48;2;39;40;34m                                                                                                   \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m30 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        Searches the indexed data for the given query using cosine similarity.\u001b[0m\u001b[48;2;39;40;34m                                \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m31 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m32 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        Args:\u001b[0m\u001b[48;2;39;40;34m                                                                                                 \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m33 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m            query (str): The search query.\u001b[0m\u001b[48;2;39;40;34m                                                                    \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m34 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m            k (int): The number of top results to return. Default is 5.\u001b[0m\u001b[48;2;39;40;34m                                       \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m35 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m36 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        Returns:\u001b[0m\u001b[48;2;39;40;34m                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m37 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m            list: A list of dictionaries containing the source, text, and score of the top-k results.\u001b[0m\u001b[48;2;39;40;34m         \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m38 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        \"\"\"\u001b[0m\u001b[48;2;39;40;34m                                                                                                   \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m39 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m        \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mquery_vec\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mself\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m.\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mvectorizer\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m.\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mtransform\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mquery\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[48;2;39;40;34m                                                        \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m40 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m        \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mcosine_distances\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mcdist\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[48;2;39;40;34m                                                                             \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m41 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m            \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mquery_vec\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m.\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mtodense\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mself\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m.\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mindex\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m.\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mtodense\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mmetric\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34mcosine\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[48;2;39;40;34m                                        \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m42 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m        \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;174;129;255;48;2;39;40;34m0\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[48;2;39;40;34m                                                                                                  \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m43 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m        \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mtop_k_indices\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mcosine_distances\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m.\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34margsort\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mk\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[48;2;39;40;34m                                                        \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m44 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m        \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34moutput\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[48;2;39;40;34m                                                                                           \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m45 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m        \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mfor\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34midx\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34min\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mtop_k_indices\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[48;2;39;40;34m                                                                             \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m46 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m            \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34moutput\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m.\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mappend\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[48;2;39;40;34m                                                                                    \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m47 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m                \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m{\u001b[0m\u001b[48;2;39;40;34m                                                                                             \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m48 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m                    \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34msource\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mself\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m.\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mdata\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34midx\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34mmetadata\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34msource\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[48;2;39;40;34m                                           \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m49 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m                    \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34mtext\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mself\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m.\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mdata\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34midx\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34mcleaned_content\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[48;2;39;40;34m                                                \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m50 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m                    \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34mscore\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;174;129;255;48;2;39;40;34m1\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m-\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mcosine_distances\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34midx\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[48;2;39;40;34m                                                       \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m51 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m                \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m}\u001b[0m\u001b[48;2;39;40;34m                                                                                             \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m52 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m            \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[48;2;39;40;34m                                                                                                 \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m53 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m        \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mreturn\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34moutput\u001b[0m\u001b[48;2;39;40;34m                                                                                         \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m54 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m55 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;166;226;46;48;2;39;40;34m@weave\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m.\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mop\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[48;2;39;40;34m                                                                                               \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m56 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mdef\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;166;226;46;48;2;39;40;34mpredict\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mself\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mquery\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mstr\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mk\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mint\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[48;2;39;40;34m                                                                    \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m57 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m        \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\"\"\u001b[0m\u001b[48;2;39;40;34m                                                                                                   \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m58 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        Predicts the top-k results for the given query.\u001b[0m\u001b[48;2;39;40;34m                                                       \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m59 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m60 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        Args:\u001b[0m\u001b[48;2;39;40;34m                                                                                                 \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m61 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m            query (str): The search query.\u001b[0m\u001b[48;2;39;40;34m                                                                    \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m62 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m            k (int): The number of top results to return.\u001b[0m\u001b[48;2;39;40;34m                                                     \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m63 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m64 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        Returns:\u001b[0m\u001b[48;2;39;40;34m                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m65 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m            list: A list of dictionaries containing the source, text, and score of the top-k results.\u001b[0m\u001b[48;2;39;40;34m         \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m66 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        \"\"\"\u001b[0m\u001b[48;2;39;40;34m                                                                                                   \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m67 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m        \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mreturn\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mself\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m.\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34msearch\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mquery\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mk\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[48;2;39;40;34m                                                                          \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m68 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 1 </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">class</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> </span><span style=\"color: #a6e22e; text-decoration-color: #a6e22e; background-color: #272822\">TFIDFRetriever</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">(weave</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">.</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">Model):</span><span style=\"background-color: #272822\">                                                                            </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 2 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">\"\"\"</span><span style=\"background-color: #272822\">                                                                                                       </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 3 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">    A retriever model that uses TF-IDF for indexing and searching documents.</span><span style=\"background-color: #272822\">                                  </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 4 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 5 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">    Attributes:</span><span style=\"background-color: #272822\">                                                                                               </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 6 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        vectorizer (TfidfVectorizer): The TF-IDF vectorizer.</span><span style=\"background-color: #272822\">                                                  </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 7 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        index (list): The indexed data.</span><span style=\"background-color: #272822\">                                                                       </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 8 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        data (list): The data to be indexed.</span><span style=\"background-color: #272822\">                                                                  </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 9 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">    \"\"\"</span><span style=\"background-color: #272822\">                                                                                                       </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">10 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">11 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    vectorizer: TfidfVectorizer </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> TfidfVectorizer()</span><span style=\"background-color: #272822\">                                                           </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">12 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    index: list </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">None</span><span style=\"background-color: #272822\">                                                                                        </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">13 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    data: list </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">None</span><span style=\"background-color: #272822\">                                                                                         </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">14 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">15 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">def</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> </span><span style=\"color: #a6e22e; text-decoration-color: #a6e22e; background-color: #272822\">index_data</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">(self, data):</span><span style=\"background-color: #272822\">                                                                               </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">16 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">        </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">\"\"\"</span><span style=\"background-color: #272822\">                                                                                                   </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">17 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        Indexes the provided data using TF-IDF.</span><span style=\"background-color: #272822\">                                                               </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">18 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">19 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        Args:</span><span style=\"background-color: #272822\">                                                                                                 </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">20 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">            data (list): A list of documents to be indexed. Each document should be a dictionary</span><span style=\"background-color: #272822\">              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">21 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">                         containing a key 'cleaned_content' with the text to be indexed.</span><span style=\"background-color: #272822\">                      </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">22 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        \"\"\"</span><span style=\"background-color: #272822\">                                                                                                   </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">23 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">        self</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">.</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">data </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> data</span><span style=\"background-color: #272822\">                                                                                      </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">24 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">        docs </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> [doc[</span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">\"cleaned_content\"</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">] </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">for</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> doc </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">in</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> data]</span><span style=\"background-color: #272822\">                                                       </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">25 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">        self</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">.</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">index </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> self</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">.</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">vectorizer</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">.</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">fit_transform(docs)</span><span style=\"background-color: #272822\">                                                      </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">26 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">27 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    </span><span style=\"color: #a6e22e; text-decoration-color: #a6e22e; background-color: #272822\">@weave</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">.</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">op()</span><span style=\"background-color: #272822\">                                                                                               </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">28 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">def</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> </span><span style=\"color: #a6e22e; text-decoration-color: #a6e22e; background-color: #272822\">search</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">(self, query, k</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #ae81ff; text-decoration-color: #ae81ff; background-color: #272822\">5</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">):</span><span style=\"background-color: #272822\">                                                                             </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">29 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">        </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">\"\"\"</span><span style=\"background-color: #272822\">                                                                                                   </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">30 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        Searches the indexed data for the given query using cosine similarity.</span><span style=\"background-color: #272822\">                                </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">31 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">32 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        Args:</span><span style=\"background-color: #272822\">                                                                                                 </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">33 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">            query (str): The search query.</span><span style=\"background-color: #272822\">                                                                    </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">34 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">            k (int): The number of top results to return. Default is 5.</span><span style=\"background-color: #272822\">                                       </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">35 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">36 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        Returns:</span><span style=\"background-color: #272822\">                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">37 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">            list: A list of dictionaries containing the source, text, and score of the top-k results.</span><span style=\"background-color: #272822\">         </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">38 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        \"\"\"</span><span style=\"background-color: #272822\">                                                                                                   </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">39 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">        query_vec </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> self</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">.</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">vectorizer</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">.</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">transform([query])</span><span style=\"background-color: #272822\">                                                        </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">40 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">        cosine_distances </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> cdist(</span><span style=\"background-color: #272822\">                                                                             </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">41 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">            query_vec</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">.</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">todense(), self</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">.</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">index</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">.</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">todense(), metric</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">\"cosine\"</span><span style=\"background-color: #272822\">                                        </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">42 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">        )[</span><span style=\"color: #ae81ff; text-decoration-color: #ae81ff; background-color: #272822\">0</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">]</span><span style=\"background-color: #272822\">                                                                                                  </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">43 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">        top_k_indices </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> cosine_distances</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">.</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">argsort()[:k]</span><span style=\"background-color: #272822\">                                                        </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">44 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">        output </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> []</span><span style=\"background-color: #272822\">                                                                                           </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">45 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">        </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">for</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> idx </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">in</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> top_k_indices:</span><span style=\"background-color: #272822\">                                                                             </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">46 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">            output</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">.</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">append(</span><span style=\"background-color: #272822\">                                                                                    </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">47 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">                {</span><span style=\"background-color: #272822\">                                                                                             </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">48 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">                    </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">\"source\"</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">: self</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">.</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">data[idx][</span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">\"metadata\"</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">][</span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">\"source\"</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">],</span><span style=\"background-color: #272822\">                                           </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">49 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">                    </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">\"text\"</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">: self</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">.</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">data[idx][</span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">\"cleaned_content\"</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">],</span><span style=\"background-color: #272822\">                                                </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">50 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">                    </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">\"score\"</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">: </span><span style=\"color: #ae81ff; text-decoration-color: #ae81ff; background-color: #272822\">1</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">-</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> cosine_distances[idx],</span><span style=\"background-color: #272822\">                                                       </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">51 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">                }</span><span style=\"background-color: #272822\">                                                                                             </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">52 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">            )</span><span style=\"background-color: #272822\">                                                                                                 </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">53 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">        </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">return</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> output</span><span style=\"background-color: #272822\">                                                                                         </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">54 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">55 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    </span><span style=\"color: #a6e22e; text-decoration-color: #a6e22e; background-color: #272822\">@weave</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">.</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">op()</span><span style=\"background-color: #272822\">                                                                                               </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">56 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">def</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> </span><span style=\"color: #a6e22e; text-decoration-color: #a6e22e; background-color: #272822\">predict</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">(self, query: str, k: int):</span><span style=\"background-color: #272822\">                                                                    </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">57 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">        </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">\"\"\"</span><span style=\"background-color: #272822\">                                                                                                   </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">58 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        Predicts the top-k results for the given query.</span><span style=\"background-color: #272822\">                                                       </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">59 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">60 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        Args:</span><span style=\"background-color: #272822\">                                                                                                 </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">61 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">            query (str): The search query.</span><span style=\"background-color: #272822\">                                                                    </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">62 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">            k (int): The number of top results to return.</span><span style=\"background-color: #272822\">                                                     </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">63 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">64 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        Returns:</span><span style=\"background-color: #272822\">                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">65 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">            list: A list of dictionaries containing the source, text, and score of the top-k results.</span><span style=\"background-color: #272822\">         </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">66 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        \"\"\"</span><span style=\"background-color: #272822\">                                                                                                   </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">67 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">        </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">return</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> self</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">.</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">search(query, k)</span><span style=\"background-color: #272822\">                                                                          </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">68 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "retriever = TFIDFRetriever()\n",
        "retriever.index_data(list(map(dict, chunked_data.rows)))"
      ],
      "metadata": {
        "id": "Sa_4d44O2SHJ"
      },
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "query = \"How do I use W&B to log metrics in my training script?\"\n",
        "search_results = retriever.search(query)\n",
        "for result in search_results:\n",
        "    print(result)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0CR8xNz62fIm",
        "outputId": "907cb924-69a5-4d85-c65c-307aa87f8096"
      },
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "🍩 https://wandb.ai/shikhar_dave-iit-jodhpur/rag-course/r/call/01926cdb-09b1-7683-b884-b942050a0769\n",
            "{'source': 'guides/technical-faq/general.md', 'text': '--- displayed_sidebar: default --- # General ### What does `wandb.init` do to my training process? When `wandb.init()` is called from your training script an API call is made to create a run object on our servers. A new process is started to stream and collect metrics, thereby keeping all threads and logic out of your primary process. Your script runs normally and writes to local files, while the separate process streams them to our servers along with system metrics. You can always turn off streaming by running `wandb off` from your training directory, or setting the `WANDB_MODE` environment variable to `offline`. ### Does your tool track or store training data? You can pass a SHA or other unique identifier to `wandb.config.update(...)` to associate a dataset with a training run. W&B does not store any data unless `wandb.save` is called with the local file name. ### What formula do you use for your smoothing algorithm? We use the same exponential moving average formula as TensorBoard. You can find an [explanation here](https://stackoverflow.com/questions/42281844/what-is-the-mathematics-behind-the-smoothing-parameter-in-tensorboards-scalar). ### How do I get the random run name in my script? Call `wandb.run.save()` and then get the name with `wandb.run.name` . ### What is the difference between `.log()` and `.summary`? The summary is the value that shows in the table while the log will save all the values for plotting later. For example, you might want to call `wandb.log` every time the accuracy changes. Usually, you can just use .log. `wandb.log()` will also update the summary value by default unless you have set the summary manually for that metric The scatterplot and parallel coordinate plots will also use the summary value while the line plot plots all of the values set by .log The reason we have both is that some people like to set the summary manually because', 'score': 0.28870928597000667}\n",
            "{'source': 'guides/technical-faq/troubleshooting.md', 'text': '--- displayed_sidebar: default --- import Tabs from \\'@theme/Tabs\\'; import TabItem from \\'@theme/TabItem\\'; # Troubleshooting ### If wandb crashes, will it possibly crash my training run? It is extremely important to us that we never interfere with your training runs. We run wandb in a separate process to make sure that if wandb somehow crashes, your training will continue to run. If the internet goes out, wandb will continue to retry sending data to [wandb.ai](https://wandb.ai). ### Why is a run marked crashed in W&B when it’s training fine locally? This is likely a connection problem — if your server loses internet access and data stops syncing to W&B, we mark the run as crashed after a short period of retrying. ### Does logging block my training? \"Is the logging function lazy? I don\\'t want to be dependent on the network to send the results to your servers and then carry on with my local operations.\" Calling `wandb.log` writes a line to a local file; it does not block any network calls. When you call `wandb.init` we launch a new process on the same machine that listens for filesystem changes and talks to our web service asynchronously from your training process. ### How do I stop wandb from writing to my terminal or my jupyter notebook output? Set the environment variable [`WANDB_SILENT`](../track/environment-variables.md) to `true`. <Tabs defaultValue=\"python\" values={[ {label: \\'Python\\', value: \\'python\\'}, {label: \\'Jupyter Notebook\\', value: \\'notebook\\'}, {label: \\'Command Line\\', value: \\'command-line\\'}, ]}> <TabItem value=\"python\"> ```python os.environ[\"WANDB_SILENT\"] = \"true\" ``` </TabItem> <TabItem value=\"notebook\"> ```python %env WANDB_SILENT=true ``` </TabItem> <TabItem value=\"command-line\"> ```python WANDB_SILENT=true ``` </TabItem> </Tabs> ### How do I kill a job with wandb? Press `Ctrl+D` on your keyboard to stop a script that is instrumented with wandb. ### How do I deal with network issues? If you\\'re seeing SSL or network errors:`wandb:', 'score': 0.25915729952187805}\n",
            "{'source': 'guides/track/log/logging-faqs.md', 'text': '--- description: >- Answers to frequently asked questions about tracking data from machine learning experiments with W&B Experiments. displayed_sidebar: default --- import Tabs from \\'@theme/Tabs\\'; import TabItem from \\'@theme/TabItem\\'; # Logging FAQs <head> <title>Frequently Asked Questions About Logging Data from Experiments</title> </head> ### How can I organize my logged charts and media in the W&B UI? We treat `/` as a separator for organizing logged panels in the W&B UI. By default, the component of the logged item\\'s name before a `/` is used to define a group of panel called a \"Panel Section\". ```python wandb.log({\"val/loss\": 1.1, \"val/acc\": 0.3}) wandb.log({\"train/loss\": 0.1, \"train/acc\": 0.94}) ``` In the [Workspace](../../app/pages/workspaces.md) settings, you can change whether panels are grouped by just the first component or by all components separated by `/`. ### How can I compare images or media across epochs or steps? Each time you log images from a step, we save them to show in the UI. Expand the image panel, and use the step slider to look at images from different steps. This makes it easy to compare how a model\\'s output changes during training. ### What if I want to log some metrics on batches and some metrics only on epochs? If you\\'d like to log certain metrics in every batch and standardize plots, you can log x axis values that you want to plot with your metrics. Then in the custom plots, click edit and select a custom x-axis. ```python wandb.log({\"batch\": batch_idx, \"loss\": 0.3}) wandb.log({\"epoch\": epoch, \"val_acc\": 0.94}) ``` ### How do I log a list of values? <!-- Logging lists directly is not supported. Instead, list-like collections of numerical data are converted to [histograms](../../../ref/python/data-types/histogram.md). To log all of the entries in a list, give a name to each entry in the list and use those names as keys', 'score': 0.25378170864192895}\n",
            "{'source': 'guides/track/log/media.md', 'text': 'table = wandb.Table(columns=columns) table.add_data(\"I love my phone\", \"1\", \"1\") table.add_data(\"My phone sucks\", \"0\", \"-1\") wandb.log({\"examples\": table}) ``` You can also pass a pandas `DataFrame` object. ```python table = wandb.Table(dataframe=my_dataframe) ``` </TabItem> <TabItem value=\"html\"> ```python wandb.log({\"custom_file\": wandb.Html(open(\"some.html\"))}) wandb.log({\"custom_string\": wandb.Html(\\'<a href=\"https://mysite\">Link</a>\\')}) ``` Custom html can be logged at any key, and this exposes an HTML panel on the run page. By default we inject default styles, you can disable default styles by passing `inject=False`. ```python wandb.log({\"custom_file\": wandb.Html(open(\"some.html\"), inject=False)}) ``` </TabItem> </Tabs> ## Frequently Asked Questions ### How can I compare images or media across epochs or steps? Each time you log images from a step, we save them to show in the UI. Expand the image panel, and use the step slider to look at images from different steps. This makes it easy to compare how a model\\'s output changes during training. ### What if I want to integrate W&B into my project, but I don\\'t want to upload any images or media? W&B can be used even for projects that only log scalars — you specify any files or data you\\'d like to upload explicitly. Here\\'s [a quick example in PyTorch](http://wandb.me/pytorch-colab) that does not log images. ### How do I log a PNG? [`wandb.Image`](../../../ref/python/data-types/image.md) converts `numpy` arrays or instances of `PILImage` to PNGs by default. ```python wandb.log({\"example\": wandb.Image(...)}) # Or multiple images wandb.log({\"example\": [wandb.Image(...) for img in images]}) ``` ### How do I log a video? Videos are logged using the [`wandb.Video`](../../../ref/python/data-types/video.md) data type: ```python wandb.log({\"example\": wandb.Video(\"myvideo.mp4\")}) ``` Now you can view videos in the media browser. Go to your project workspace, run workspace, or report and click \"Add visualization\" to add a rich media panel. ### How do I navigate and zoom in point clouds? You can hold control and use the mouse to move around inside the space. ###', 'score': 0.23904511486347035}\n",
            "{'source': 'guides/integrations/huggingface.md', 'text': \"manage model lifecycle, facilitate easy tracking and auditing throughout the ML lifecyle, and [automate](https://docs.wandb.ai/guides/models/automation) downstream actions with webhooks or jobs. See the [Model Registry](../model_registry) documentation for how to link a model Artifact to the Model Registry. ### 5) Visualise evaluation outputs during training Visualing your model outputs during training or evaluation is often essential to really understand how your model is training. By using the callbacks system in the Transformers Trainer, you can log additional helpful data to W&B such as your models' text generation outputs or other predictions to W&B Tables. See the **[Custom logging section](#custom-logging-log-and-view-evaluation-samples-during-training)** below for a full guide on how to log evaluation outupts while training to log to a W&B Table like this: ![Shows a W&B Table with evaluation outputs](/images/integrations/huggingface_eval_tables.png) ### 6) Finish your W&B Run (Notebook only) If your training is encapsulated in a Python script, the W&B run will end when your script finishes. If you are using a Jupyter or Google Colab notebook, you'll need to tell us when you're done with training by calling `wandb.finish()`. ```python trainer.train() # start training and logging to W&B # post-training analysis, testing, other logged code wandb.finish() ``` ### 7) Visualize your results Once you have logged your training results you can explore your results dynamically in the [W&B Dashboard](../track/app.md). It's easy to compare across dozens of runs at once, zoom in on interesting findings, and coax insights out of complex data with flexible, interactive visualizations. ## Advanced features and FAQs ### How do I save the best model? If `load_best_model_at_end=True` is set in the `TrainingArguments` that are passed to the `Trainer`, then W&B will save the best performing model checkpoint to Artifacts. If you'd like to centralize all your best model versions across your team to organize them by ML task, stage them for production,\", 'score': 0.2328486885623663}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from scripts.response_generator import SimpleResponseGenerator\n",
        "display_source(SimpleResponseGenerator)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "MqfD6Fju2vxV",
        "outputId": "73829cc9-16b5-4846-88fe-f702c582bd52"
      },
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 1 \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mclass\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;166;226;46;48;2;39;40;34mSimpleResponseGenerator\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mweave\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m.\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mModel\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[48;2;39;40;34m                                                                   \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 2 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\"\"\u001b[0m\u001b[48;2;39;40;34m                                                                                                       \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 3 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m    A simple response generator model using Cohere's API.\u001b[0m\u001b[48;2;39;40;34m                                                     \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 4 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 5 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m    Attributes:\u001b[0m\u001b[48;2;39;40;34m                                                                                               \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 6 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        model (str): The model name to be used for generating responses.\u001b[0m\u001b[48;2;39;40;34m                                      \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 7 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        prompt (str): The prompt to be used for generating responses.\u001b[0m\u001b[48;2;39;40;34m                                         \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 8 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        client (cohere.ClientV2): The Cohere client for interacting with the Cohere API.\u001b[0m\u001b[48;2;39;40;34m                      \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 9 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m    \"\"\"\u001b[0m\u001b[48;2;39;40;34m                                                                                                       \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m10 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m11 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mmodel\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mstr\u001b[0m\u001b[48;2;39;40;34m                                                                                                \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m12 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mprompt\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mstr\u001b[0m\u001b[48;2;39;40;34m                                                                                               \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m13 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mclient\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mcohere\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m.\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mClientV2\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mNone\u001b[0m\u001b[48;2;39;40;34m                                                                            \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m14 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m15 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mdef\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;166;226;46;48;2;39;40;34m__init__\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mself\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m*\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m*\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mkwargs\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[48;2;39;40;34m                                                                             \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m16 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m        \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\"\"\u001b[0m\u001b[48;2;39;40;34m                                                                                                   \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m17 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        Initialize the SimpleResponseGenerator with the provided keyword arguments.\u001b[0m\u001b[48;2;39;40;34m                           \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m18 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        Sets up the Cohere client using the API key from environment variables.\u001b[0m\u001b[48;2;39;40;34m                               \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m19 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        \"\"\"\u001b[0m\u001b[48;2;39;40;34m                                                                                                   \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m20 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m        \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34msuper\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m.\u001b[0m\u001b[38;2;166;226;46;48;2;39;40;34m__init__\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m*\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m*\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mkwargs\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[48;2;39;40;34m                                                                            \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m21 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m        \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mself\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m.\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mclient\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mcohere\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m.\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mClientV2\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[48;2;39;40;34m                                                                        \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m22 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m            \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mapi_key\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mos\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m.\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34menviron\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34mCOHERE_API_KEY\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[48;2;39;40;34m                                                             \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m23 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m        \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[48;2;39;40;34m                                                                                                     \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m24 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m25 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;166;226;46;48;2;39;40;34m@weave\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m.\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mop\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[48;2;39;40;34m                                                                                               \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m26 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mdef\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;166;226;46;48;2;39;40;34mgenerate_context\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mself\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mcontext\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mList\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mDict\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mstr\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34many\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m-\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m>\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mList\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mDict\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mstr\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34many\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[48;2;39;40;34m                        \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m27 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m        \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\"\"\u001b[0m\u001b[48;2;39;40;34m                                                                                                   \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m28 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        Generate a list of contexts from the provided context list.\u001b[0m\u001b[48;2;39;40;34m                                           \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m29 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m30 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        Args:\u001b[0m\u001b[48;2;39;40;34m                                                                                                 \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m31 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m            context (List[Dict[str, any]]): A list of dictionaries containing context data.\u001b[0m\u001b[48;2;39;40;34m                   \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m32 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m33 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        Returns:\u001b[0m\u001b[48;2;39;40;34m                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m34 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m            List[Dict[str, any]]: A list of dictionaries with 'source' and 'text' keys.\u001b[0m\u001b[48;2;39;40;34m                       \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m35 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        \"\"\"\u001b[0m\u001b[48;2;39;40;34m                                                                                                   \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m36 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m        \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mcontexts\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[48;2;39;40;34m                                                                                          \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m37 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m            \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m{\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34mdata\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m{\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34msource\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mitem\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34msource\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34mtext\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mitem\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34mtext\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m}\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m}\u001b[0m\u001b[48;2;39;40;34m                                        \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m38 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m            \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mfor\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mitem\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34min\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mcontext\u001b[0m\u001b[48;2;39;40;34m                                                                               \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m39 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m        \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[48;2;39;40;34m                                                                                                     \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m40 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m        \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mreturn\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mcontexts\u001b[0m\u001b[48;2;39;40;34m                                                                                       \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m41 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m42 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mdef\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;166;226;46;48;2;39;40;34mcreate_messages\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mself\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mquery\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mstr\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[48;2;39;40;34m                                                                    \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m43 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m        \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\"\"\u001b[0m\u001b[48;2;39;40;34m                                                                                                   \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m44 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        Create a list of messages for the chat model based on the query.\u001b[0m\u001b[48;2;39;40;34m                                      \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m45 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m46 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        Args:\u001b[0m\u001b[48;2;39;40;34m                                                                                                 \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m47 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m            query (str): The user's query.\u001b[0m\u001b[48;2;39;40;34m                                                                    \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m48 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m49 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        Returns:\u001b[0m\u001b[48;2;39;40;34m                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m50 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m            List[Dict[str, any]]: A list of messages formatted for the chat model.\u001b[0m\u001b[48;2;39;40;34m                            \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m51 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        \"\"\"\u001b[0m\u001b[48;2;39;40;34m                                                                                                   \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m52 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m        \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mmessages\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[48;2;39;40;34m                                                                                          \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m53 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m            \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m{\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34mrole\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34msystem\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34mcontent\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mself\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m.\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mprompt\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m}\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[48;2;39;40;34m                                                       \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m54 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m            \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m{\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34mrole\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34muser\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34mcontent\u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mquery\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m}\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[48;2;39;40;34m                                                               \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m55 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m        \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[48;2;39;40;34m                                                                                                     \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m56 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m        \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mreturn\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mmessages\u001b[0m\u001b[48;2;39;40;34m                                                                                       \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m57 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m58 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;166;226;46;48;2;39;40;34m@weave\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m.\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mop\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[48;2;39;40;34m                                                                                               \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m59 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mdef\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;166;226;46;48;2;39;40;34mgenerate_response\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mself\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mquery\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mstr\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mcontext\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mList\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mDict\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mstr\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34many\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m-\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m>\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mstr\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[48;2;39;40;34m                            \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m60 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m        \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\"\"\u001b[0m\u001b[48;2;39;40;34m                                                                                                   \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m61 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        Generate a response from the chat model based on the query and context.\u001b[0m\u001b[48;2;39;40;34m                               \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m62 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m63 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        Args:\u001b[0m\u001b[48;2;39;40;34m                                                                                                 \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m64 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m            query (str): The user's query.\u001b[0m\u001b[48;2;39;40;34m                                                                    \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m65 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m            context (List[Dict[str, any]]): A list of dictionaries containing context data.\u001b[0m\u001b[48;2;39;40;34m                   \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m66 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m67 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        Returns:\u001b[0m\u001b[48;2;39;40;34m                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m68 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m            str: The generated response from the chat model.\u001b[0m\u001b[48;2;39;40;34m                                                  \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m69 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        \"\"\"\u001b[0m\u001b[48;2;39;40;34m                                                                                                   \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m70 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m        \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mdocuments\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mself\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m.\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mgenerate_context\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mcontext\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[48;2;39;40;34m                                                            \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m71 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m        \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mmessages\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mself\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m.\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mcreate_messages\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mquery\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[48;2;39;40;34m                                                                \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m72 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m        \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mresponse\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mself\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m.\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mclient\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m.\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mchat\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[48;2;39;40;34m                                                                          \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m73 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m            \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mmessages\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mmessages\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[48;2;39;40;34m                                                                                \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m74 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m            \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mmodel\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mself\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m.\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mmodel\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[48;2;39;40;34m                                                                                 \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m75 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m            \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mtemperature\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;174;129;255;48;2;39;40;34m0.1\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[48;2;39;40;34m                                                                                  \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m76 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m            \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mmax_tokens\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;174;129;255;48;2;39;40;34m2000\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[48;2;39;40;34m                                                                                  \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m77 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m            \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mdocuments\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mdocuments\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[48;2;39;40;34m                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m78 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m        \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[48;2;39;40;34m                                                                                                     \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m79 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m        \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mreturn\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mresponse\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m.\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mmessage\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m.\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mcontent\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;174;129;255;48;2;39;40;34m0\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m.\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mtext\u001b[0m\u001b[48;2;39;40;34m                                                               \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m80 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m81 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;166;226;46;48;2;39;40;34m@weave\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m.\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mop\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[48;2;39;40;34m                                                                                               \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m82 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mdef\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;166;226;46;48;2;39;40;34mpredict\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mself\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mquery\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mstr\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mcontext\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mList\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mDict\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m[\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mstr\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34many\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m]\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[48;2;39;40;34m                                             \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m83 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m        \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\"\"\u001b[0m\u001b[48;2;39;40;34m                                                                                                   \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m84 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        Predict the response for the given query and context.\u001b[0m\u001b[48;2;39;40;34m                                                 \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m85 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m86 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        Args:\u001b[0m\u001b[48;2;39;40;34m                                                                                                 \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m87 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m            query (str): The user's query.\u001b[0m\u001b[48;2;39;40;34m                                                                    \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m88 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m            context (List[Dict[str, any]]): A list of dictionaries containing context data.\u001b[0m\u001b[48;2;39;40;34m                   \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m89 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m90 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        Returns:\u001b[0m\u001b[48;2;39;40;34m                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m91 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m            str: The predicted response from the chat model.\u001b[0m\u001b[48;2;39;40;34m                                                  \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m92 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        \"\"\"\u001b[0m\u001b[48;2;39;40;34m                                                                                                   \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m93 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m        \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mreturn\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mself\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m.\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mgenerate_response\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mquery\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mcontext\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[48;2;39;40;34m                                                         \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m94 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 1 </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">class</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> </span><span style=\"color: #a6e22e; text-decoration-color: #a6e22e; background-color: #272822\">SimpleResponseGenerator</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">(weave</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">.</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">Model):</span><span style=\"background-color: #272822\">                                                                   </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 2 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">\"\"\"</span><span style=\"background-color: #272822\">                                                                                                       </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 3 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">    A simple response generator model using Cohere's API.</span><span style=\"background-color: #272822\">                                                     </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 4 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 5 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">    Attributes:</span><span style=\"background-color: #272822\">                                                                                               </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 6 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        model (str): The model name to be used for generating responses.</span><span style=\"background-color: #272822\">                                      </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 7 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        prompt (str): The prompt to be used for generating responses.</span><span style=\"background-color: #272822\">                                         </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 8 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        client (cohere.ClientV2): The Cohere client for interacting with the Cohere API.</span><span style=\"background-color: #272822\">                      </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 9 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">    \"\"\"</span><span style=\"background-color: #272822\">                                                                                                       </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">10 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">11 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    model: str</span><span style=\"background-color: #272822\">                                                                                                </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">12 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    prompt: str</span><span style=\"background-color: #272822\">                                                                                               </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">13 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    client: cohere</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">.</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">ClientV2 </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">None</span><span style=\"background-color: #272822\">                                                                            </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">14 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">15 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">def</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> </span><span style=\"color: #a6e22e; text-decoration-color: #a6e22e; background-color: #272822\">__init__</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">(self, </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">**</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">kwargs):</span><span style=\"background-color: #272822\">                                                                             </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">16 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">        </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">\"\"\"</span><span style=\"background-color: #272822\">                                                                                                   </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">17 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        Initialize the SimpleResponseGenerator with the provided keyword arguments.</span><span style=\"background-color: #272822\">                           </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">18 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        Sets up the Cohere client using the API key from environment variables.</span><span style=\"background-color: #272822\">                               </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">19 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        \"\"\"</span><span style=\"background-color: #272822\">                                                                                                   </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">20 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">        super()</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">.</span><span style=\"color: #a6e22e; text-decoration-color: #a6e22e; background-color: #272822\">__init__</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">(</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">**</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">kwargs)</span><span style=\"background-color: #272822\">                                                                            </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">21 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">        self</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">.</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">client </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> cohere</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">.</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">ClientV2(</span><span style=\"background-color: #272822\">                                                                        </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">22 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">            api_key</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">os</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">.</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">environ[</span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">\"COHERE_API_KEY\"</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">],</span><span style=\"background-color: #272822\">                                                             </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">23 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">        )</span><span style=\"background-color: #272822\">                                                                                                     </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">24 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">25 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    </span><span style=\"color: #a6e22e; text-decoration-color: #a6e22e; background-color: #272822\">@weave</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">.</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">op()</span><span style=\"background-color: #272822\">                                                                                               </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">26 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">def</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> </span><span style=\"color: #a6e22e; text-decoration-color: #a6e22e; background-color: #272822\">generate_context</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">(self, context: List[Dict[str, any]]) </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">-&gt;</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> List[Dict[str, any]]:</span><span style=\"background-color: #272822\">                        </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">27 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">        </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">\"\"\"</span><span style=\"background-color: #272822\">                                                                                                   </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">28 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        Generate a list of contexts from the provided context list.</span><span style=\"background-color: #272822\">                                           </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">29 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">30 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        Args:</span><span style=\"background-color: #272822\">                                                                                                 </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">31 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">            context (List[Dict[str, any]]): A list of dictionaries containing context data.</span><span style=\"background-color: #272822\">                   </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">32 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">33 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        Returns:</span><span style=\"background-color: #272822\">                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">34 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">            List[Dict[str, any]]: A list of dictionaries with 'source' and 'text' keys.</span><span style=\"background-color: #272822\">                       </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">35 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        \"\"\"</span><span style=\"background-color: #272822\">                                                                                                   </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">36 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">        contexts </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> [</span><span style=\"background-color: #272822\">                                                                                          </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">37 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">            {</span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">\"data\"</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">: {</span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">\"source\"</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">: item[</span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">\"source\"</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">], </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">\"text\"</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">: item[</span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">\"text\"</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">]}}</span><span style=\"background-color: #272822\">                                        </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">38 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">            </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">for</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> item </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">in</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> context</span><span style=\"background-color: #272822\">                                                                               </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">39 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">        ]</span><span style=\"background-color: #272822\">                                                                                                     </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">40 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">        </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">return</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> contexts</span><span style=\"background-color: #272822\">                                                                                       </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">41 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">42 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">def</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> </span><span style=\"color: #a6e22e; text-decoration-color: #a6e22e; background-color: #272822\">create_messages</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">(self, query: str):</span><span style=\"background-color: #272822\">                                                                    </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">43 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">        </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">\"\"\"</span><span style=\"background-color: #272822\">                                                                                                   </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">44 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        Create a list of messages for the chat model based on the query.</span><span style=\"background-color: #272822\">                                      </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">45 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">46 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        Args:</span><span style=\"background-color: #272822\">                                                                                                 </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">47 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">            query (str): The user's query.</span><span style=\"background-color: #272822\">                                                                    </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">48 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">49 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        Returns:</span><span style=\"background-color: #272822\">                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">50 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">            List[Dict[str, any]]: A list of messages formatted for the chat model.</span><span style=\"background-color: #272822\">                            </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">51 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        \"\"\"</span><span style=\"background-color: #272822\">                                                                                                   </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">52 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">        messages </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> [</span><span style=\"background-color: #272822\">                                                                                          </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">53 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">            {</span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">\"role\"</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">: </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">\"system\"</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">, </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">\"content\"</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">: self</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">.</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">prompt},</span><span style=\"background-color: #272822\">                                                       </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">54 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">            {</span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">\"role\"</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">: </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">\"user\"</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">, </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">\"content\"</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">: query},</span><span style=\"background-color: #272822\">                                                               </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">55 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">        ]</span><span style=\"background-color: #272822\">                                                                                                     </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">56 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">        </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">return</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> messages</span><span style=\"background-color: #272822\">                                                                                       </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">57 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">58 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    </span><span style=\"color: #a6e22e; text-decoration-color: #a6e22e; background-color: #272822\">@weave</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">.</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">op()</span><span style=\"background-color: #272822\">                                                                                               </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">59 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">def</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> </span><span style=\"color: #a6e22e; text-decoration-color: #a6e22e; background-color: #272822\">generate_response</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">(self, query: str, context: List[Dict[str, any]]) </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">-&gt;</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> str:</span><span style=\"background-color: #272822\">                            </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">60 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">        </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">\"\"\"</span><span style=\"background-color: #272822\">                                                                                                   </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">61 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        Generate a response from the chat model based on the query and context.</span><span style=\"background-color: #272822\">                               </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">62 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">63 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        Args:</span><span style=\"background-color: #272822\">                                                                                                 </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">64 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">            query (str): The user's query.</span><span style=\"background-color: #272822\">                                                                    </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">65 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">            context (List[Dict[str, any]]): A list of dictionaries containing context data.</span><span style=\"background-color: #272822\">                   </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">66 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">67 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        Returns:</span><span style=\"background-color: #272822\">                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">68 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">            str: The generated response from the chat model.</span><span style=\"background-color: #272822\">                                                  </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">69 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        \"\"\"</span><span style=\"background-color: #272822\">                                                                                                   </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">70 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">        documents </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> self</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">.</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">generate_context(context)</span><span style=\"background-color: #272822\">                                                            </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">71 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">        messages </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> self</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">.</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">create_messages(query)</span><span style=\"background-color: #272822\">                                                                </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">72 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">        response </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> self</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">.</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">client</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">.</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">chat(</span><span style=\"background-color: #272822\">                                                                          </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">73 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">            messages</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">messages,</span><span style=\"background-color: #272822\">                                                                                </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">74 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">            model</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">self</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">.</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">model,</span><span style=\"background-color: #272822\">                                                                                 </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">75 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">            temperature</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #ae81ff; text-decoration-color: #ae81ff; background-color: #272822\">0.1</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">,</span><span style=\"background-color: #272822\">                                                                                  </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">76 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">            max_tokens</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #ae81ff; text-decoration-color: #ae81ff; background-color: #272822\">2000</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">,</span><span style=\"background-color: #272822\">                                                                                  </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">77 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">            documents</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">documents,</span><span style=\"background-color: #272822\">                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">78 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">        )</span><span style=\"background-color: #272822\">                                                                                                     </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">79 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">        </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">return</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> response</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">.</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">message</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">.</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">content[</span><span style=\"color: #ae81ff; text-decoration-color: #ae81ff; background-color: #272822\">0</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">]</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">.</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">text</span><span style=\"background-color: #272822\">                                                               </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">80 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">81 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    </span><span style=\"color: #a6e22e; text-decoration-color: #a6e22e; background-color: #272822\">@weave</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">.</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">op()</span><span style=\"background-color: #272822\">                                                                                               </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">82 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">def</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> </span><span style=\"color: #a6e22e; text-decoration-color: #a6e22e; background-color: #272822\">predict</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">(self, query: str, context: List[Dict[str, any]]):</span><span style=\"background-color: #272822\">                                             </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">83 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">        </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">\"\"\"</span><span style=\"background-color: #272822\">                                                                                                   </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">84 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        Predict the response for the given query and context.</span><span style=\"background-color: #272822\">                                                 </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">85 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">86 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        Args:</span><span style=\"background-color: #272822\">                                                                                                 </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">87 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">            query (str): The user's query.</span><span style=\"background-color: #272822\">                                                                    </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">88 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">            context (List[Dict[str, any]]): A list of dictionaries containing context data.</span><span style=\"background-color: #272822\">                   </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">89 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">90 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        Returns:</span><span style=\"background-color: #272822\">                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">91 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">            str: The predicted response from the chat model.</span><span style=\"background-color: #272822\">                                                  </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">92 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        \"\"\"</span><span style=\"background-color: #272822\">                                                                                                   </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">93 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">        </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">return</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> self</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">.</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">generate_response(query, context)</span><span style=\"background-color: #272822\">                                                         </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">94 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "INITIAL_PROMPT = open(\"prompts/initial_system.txt\", \"r\").read()\n",
        "print(INITIAL_PROMPT)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pcmWWS4Y3Fhm",
        "outputId": "221cd519-0bf6-41cb-eeb0-9e06d1859631"
      },
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Answer to the following question about W&B. Provide an helful and complete answer based only on the provided documents.\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "response_generator = SimpleResponseGenerator(model=\"command-r\", prompt=INITIAL_PROMPT)\n",
        "answer = response_generator.generate_response(query, search_results)\n",
        "print(answer)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "83PR2UXc5YVi",
        "outputId": "f1a47abb-356c-402b-cba5-f01673edb976"
      },
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "🍩 https://wandb.ai/shikhar_dave-iit-jodhpur/rag-course/r/call/01926ce5-cfa3-71b3-b85a-ce04130d5b89\n",
            "You can use the W&B to log metrics in your training script by following these steps:\n",
            "1. Call `wandb.init()` in your training script. This creates a run object on W&B servers and starts a new process to stream and collect metrics.\n",
            "2. Your script runs normally and writes to local files, while the separate process streams them to W&B servers along with system metrics. \n",
            "3. Log metrics by calling `wandb.log()` at relevant points in your script. For example, you might call this every time the accuracy changes. You can log images, videos, and other media types, as well as scalars.\n",
            "4. To finish your W&B run, call `wandb.finish()` when your training script ends.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from scripts.rag_pipeline import SimpleRAGPipeline\n",
        "\n",
        "display_source(SimpleRAGPipeline)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 465
        },
        "id": "S3m2QQ8b5b8U",
        "outputId": "f6da47a1-94d0-4fd1-890d-d9fe07dc1205"
      },
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 1 \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mclass\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;166;226;46;48;2;39;40;34mSimpleRAGPipeline\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mweave\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m.\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mModel\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[48;2;39;40;34m                                                                         \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 2 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\"\"\u001b[0m\u001b[48;2;39;40;34m                                                                                                       \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 3 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m    A simple RAG (Retrieval-Augmented Generation) pipeline.\u001b[0m\u001b[48;2;39;40;34m                                                   \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 4 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 5 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m    Attributes:\u001b[0m\u001b[48;2;39;40;34m                                                                                               \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 6 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        retriever (weave.Model): The model used for retrieving relevant documents.\u001b[0m\u001b[48;2;39;40;34m                            \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 7 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        response_generator (weave.Model): The model used for generating responses.\u001b[0m\u001b[48;2;39;40;34m                            \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 8 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        top_k (int): The number of top documents to retrieve.\u001b[0m\u001b[48;2;39;40;34m                                                 \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m 9 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m    \"\"\"\u001b[0m\u001b[48;2;39;40;34m                                                                                                       \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m10 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m11 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mretriever\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mweave\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m.\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mModel\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mNone\u001b[0m\u001b[48;2;39;40;34m                                                                             \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m12 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mresponse_generator\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mweave\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m.\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mModel\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mNone\u001b[0m\u001b[48;2;39;40;34m                                                                    \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m13 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mtop_k\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mint\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;174;129;255;48;2;39;40;34m5\u001b[0m\u001b[48;2;39;40;34m                                                                                            \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m14 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m15 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;166;226;46;48;2;39;40;34m@weave\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m.\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mop\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[48;2;39;40;34m                                                                                               \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m16 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m    \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mdef\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;166;226;46;48;2;39;40;34mpredict\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mself\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mquery\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mstr\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m:\u001b[0m\u001b[48;2;39;40;34m                                                                            \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m17 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m        \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m\"\"\"\u001b[0m\u001b[48;2;39;40;34m                                                                                                   \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m18 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        Predicts a response based on the input query.\u001b[0m\u001b[48;2;39;40;34m                                                         \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m19 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m20 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        Args:\u001b[0m\u001b[48;2;39;40;34m                                                                                                 \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m21 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m            query (str): The input query string.\u001b[0m\u001b[48;2;39;40;34m                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m22 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m23 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        Returns:\u001b[0m\u001b[48;2;39;40;34m                                                                                              \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m24 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m            The generated response based on the retrieved context.\u001b[0m\u001b[48;2;39;40;34m                                            \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m25 \u001b[0m\u001b[38;2;230;219;116;48;2;39;40;34m        \"\"\"\u001b[0m\u001b[48;2;39;40;34m                                                                                                   \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m26 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m        \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mcontext\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m=\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mself\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m.\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mretriever\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m.\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mpredict\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mquery\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mself\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m.\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mtop_k\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[48;2;39;40;34m                                                   \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m27 \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m        \u001b[0m\u001b[38;2;102;217;239;48;2;39;40;34mreturn\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mself\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m.\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mresponse_generator\u001b[0m\u001b[38;2;255;70;137;48;2;39;40;34m.\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mpredict\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m(\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mquery\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m,\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m \u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34mcontext\u001b[0m\u001b[38;2;248;248;242;48;2;39;40;34m)\u001b[0m\u001b[48;2;39;40;34m                                                \u001b[0m\n",
              "\u001b[1;38;2;227;227;221;48;2;39;40;34m  \u001b[0m\u001b[38;2;101;102;96;48;2;39;40;34m28 \u001b[0m\u001b[48;2;39;40;34m                                                                                                              \u001b[0m\n"
            ],
            "text/html": [
              "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 1 </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">class</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> </span><span style=\"color: #a6e22e; text-decoration-color: #a6e22e; background-color: #272822\">SimpleRAGPipeline</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">(weave</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">.</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">Model):</span><span style=\"background-color: #272822\">                                                                         </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 2 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">\"\"\"</span><span style=\"background-color: #272822\">                                                                                                       </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 3 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">    A simple RAG (Retrieval-Augmented Generation) pipeline.</span><span style=\"background-color: #272822\">                                                   </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 4 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 5 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">    Attributes:</span><span style=\"background-color: #272822\">                                                                                               </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 6 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        retriever (weave.Model): The model used for retrieving relevant documents.</span><span style=\"background-color: #272822\">                            </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 7 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        response_generator (weave.Model): The model used for generating responses.</span><span style=\"background-color: #272822\">                            </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 8 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        top_k (int): The number of top documents to retrieve.</span><span style=\"background-color: #272822\">                                                 </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\"> 9 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">    \"\"\"</span><span style=\"background-color: #272822\">                                                                                                       </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">10 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">11 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    retriever: weave</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">.</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">Model </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">None</span><span style=\"background-color: #272822\">                                                                             </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">12 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    response_generator: weave</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">.</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">Model </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">None</span><span style=\"background-color: #272822\">                                                                    </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">13 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    top_k: int </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> </span><span style=\"color: #ae81ff; text-decoration-color: #ae81ff; background-color: #272822\">5</span><span style=\"background-color: #272822\">                                                                                            </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">14 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">15 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    </span><span style=\"color: #a6e22e; text-decoration-color: #a6e22e; background-color: #272822\">@weave</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">.</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">op()</span><span style=\"background-color: #272822\">                                                                                               </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">16 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">    </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">def</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> </span><span style=\"color: #a6e22e; text-decoration-color: #a6e22e; background-color: #272822\">predict</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">(self, query: str):</span><span style=\"background-color: #272822\">                                                                            </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">17 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">        </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">\"\"\"</span><span style=\"background-color: #272822\">                                                                                                   </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">18 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        Predicts a response based on the input query.</span><span style=\"background-color: #272822\">                                                         </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">19 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">20 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        Args:</span><span style=\"background-color: #272822\">                                                                                                 </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">21 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">            query (str): The input query string.</span><span style=\"background-color: #272822\">                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">22 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">23 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        Returns:</span><span style=\"background-color: #272822\">                                                                                              </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">24 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">            The generated response based on the retrieved context.</span><span style=\"background-color: #272822\">                                            </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">25 </span><span style=\"color: #e6db74; text-decoration-color: #e6db74; background-color: #272822\">        \"\"\"</span><span style=\"background-color: #272822\">                                                                                                   </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">26 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">        context </span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">=</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> self</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">.</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">retriever</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">.</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">predict(query, self</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">.</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">top_k)</span><span style=\"background-color: #272822\">                                                   </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">27 </span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">        </span><span style=\"color: #66d9ef; text-decoration-color: #66d9ef; background-color: #272822\">return</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\"> self</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">.</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">response_generator</span><span style=\"color: #ff4689; text-decoration-color: #ff4689; background-color: #272822\">.</span><span style=\"color: #f8f8f2; text-decoration-color: #f8f8f2; background-color: #272822\">predict(query, context)</span><span style=\"background-color: #272822\">                                                </span>\n",
              "<span style=\"color: #e3e3dd; text-decoration-color: #e3e3dd; background-color: #272822; font-weight: bold\">  </span><span style=\"color: #656660; text-decoration-color: #656660; background-color: #272822\">28 </span><span style=\"background-color: #272822\">                                                                                                              </span>\n",
              "</pre>\n"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Initialize the response generator\n",
        "response_generator = SimpleResponseGenerator(model=\"command-r\", prompt=INITIAL_PROMPT)\n",
        "\n",
        "# Bring them together as a RAG pipeline\n",
        "rag_pipeline = SimpleRAGPipeline(\n",
        "    retriever=retriever, response_generator=response_generator, top_k=5\n",
        ")"
      ],
      "metadata": {
        "id": "uA6tPEd35shm"
      },
      "execution_count": 27,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "response = rag_pipeline.predict(\"How do I get get started with wandb?\")\n",
        "print(response, sep=\"\\n\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "L_lXOF7V51ch",
        "outputId": "cb0a8f33-9551-43df-9c13-205155bcf0a7"
      },
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "🍩 https://wandb.ai/shikhar_dave-iit-jodhpur/rag-course/r/call/01926ce7-c0ff-7311-b65b-b441b1c60ba0\n",
            "To get started with wandb.init(), you should first call it from your training script – this will create a run object on W&B servers. A new process is then started to stream and collect metrics, keeping all threads and logic out of your primary process. \n",
            "\n",
            "Your script runs as normal, but now writes to local files, while the separate process streams them to the servers along with system metrics. You can turn off streaming by running wandb off from your training directory, or setting the WANDB_MODE environment variable to offline.\n",
            "\n",
            "You can also use wandb.login() in your Jupyter notebook to get started with W&B. After running this command, start a new cell with %%wandb to see live graphs.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "OFp-GRl057Xf"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}