llamaindex_&_mongodb.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "T6cVhgu91ytl"
      },
      "outputs": [],
      "source": [
        "!pip install llama-index\n",
        "\n",
        "!pip install llama-index-vector-stores-mongodb\n",
        "\n",
        "!pip install llama-index-embeddings-openai\n",
        "\n",
        "!pip install pymongo\n",
        "\n",
        "!pip install datasets\n",
        "\n",
        "!pip install pandas\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "sZQLsBWL13AC"
      },
      "outputs": [],
      "source": [
        "%env OPENAI_API_KEY=your_api_key"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "qi0Lr2POZP-h"
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "import uuid  # Optional for random UUIDs\n",
        "import re\n",
        "\n",
        "from datasets import load_dataset\n",
        "\n",
        "dataset = load_dataset(\"aakash0017/It-support-synthetic-data\")\n",
        "\n",
        "# Define regular expression pattern (slightly modified)\n",
        "pattern = r\"(?<=### Human: )(.*?)(?=### Assistant: )\"  # Use non-greedy match\n",
        "\n",
        "# Create empty lists to store questions and answers\n",
        "questions = []\n",
        "answers = []\n",
        "\n",
        "# Extract data using regular expression\n",
        "for row in dataset[\"train\"]:\n",
        "  match = re.search(pattern, row[\"text\"])\n",
        "  if match:\n",
        "    questions.append(match.group(1).strip())  # Strip trailing whitespaces\n",
        "    answers.append(row[\"text\"].split(\"### Assistant: \")[-1])\n",
        "\n",
        "# Create DataFrame with extracted data\n",
        "dataset_df = pd.DataFrame({\"question\": questions, \"answer\": answers})\n",
        "\n",
        "# Generate unique IDs with leading zeros for consistent formatting\n",
        "dataset_df[\"ticket-number\"] = \"IDC\" + dataset_df.apply(lambda row: str(uuid.uuid4())[:8], axis=1).str.zfill(8)\n",
        "\n",
        "# Add text_content column\n",
        "dataset_df[\"text_content\"] = dataset_df[\"question\"] + \" \" + dataset_df[\"answer\"]\n",
        "\n",
        "# Reorder columns to make \"ticket-number\" the first column\n",
        "reordered_cols = [\"ticket-number\", \"question\", \"answer\", \"text_content\"]\n",
        "dataset_df = dataset_df[reordered_cols]\n",
        "dataset_df.head(10)\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "\"\"\" For testing purpose. Takes 10 rows from the data so it takes less time.\n",
        "import pandas as pd\n",
        "import re\n",
        "\n",
        "from datasets import load_dataset\n",
        "\n",
        "# Assuming you have already run the code that creates the DataFrame (df) with questions and answers\n",
        "\n",
        "# Extract only the first 10 rows\n",
        "dataset_df = dataset_df.head(10)\n",
        "\n",
        "dataset_df.head(100)\n",
        "\"\"\""
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 363
        },
        "id": "qEuLPbtVnL9u",
        "outputId": "f032a864-caef-47b6-9758-a67b7f67fa5f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "  ticket-number                                           question  \\\n",
              "0   IDC79659407  What is the newest learning management system ...   \n",
              "1   IDCfc1b32d0  Does Boston University offer a laptop loan pro...   \n",
              "2   IDC5385384b                          What is Blackboard Learn?   \n",
              "3   IDCb586e780     What are the key features of Blackboard Learn?   \n",
              "4   IDC1309c0a1              What to expect from Blackboard Learn?   \n",
              "5   IDCf1f27cca        How do I get started with Blackboard Learn?   \n",
              "6   IDC53714075              What does the IT Help Center provide?   \n",
              "7   IDC87909b25               How do I contact the IT Help Center?   \n",
              "8   IDCbd8115e6       What services does the IT Help Center offer?   \n",
              "9   IDCa2be0617  Do I need a current BU login name to use some ...   \n",
              "\n",
              "                                              answer  \\\n",
              "0                                  Blackboard Learn.   \n",
              "1  Yes, they have a laptop loaner program for act...   \n",
              "2  Blackboard Learn is the primary learning manag...   \n",
              "3  The key features of Blackboard Learn include i...   \n",
              "4  Blackboard Learn is normally available 24/7, e...   \n",
              "5  To get started with Blackboard Learn, faculty ...   \n",
              "6  The IT Help Center provides centralized techno...   \n",
              "7  You can contact the IT Help Center online or b...   \n",
              "8  The IT Help Center offers a variety of service...   \n",
              "9  Yes, some services require a current BU login ...   \n",
              "\n",
              "                                        text_content  \n",
              "0  What is the newest learning management system ...  \n",
              "1  Does Boston University offer a laptop loan pro...  \n",
              "2  What is Blackboard Learn? Blackboard Learn is ...  \n",
              "3  What are the key features of Blackboard Learn?...  \n",
              "4  What to expect from Blackboard Learn? Blackboa...  \n",
              "5  How do I get started with Blackboard Learn? To...  \n",
              "6  What does the IT Help Center provide? The IT H...  \n",
              "7  How do I contact the IT Help Center? You can c...  \n",
              "8  What services does the IT Help Center offer? T...  \n",
              "9  Do I need a current BU login name to use some ...  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-727f3c5e-fdf5-4edd-9263-4b0931f20c98\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>ticket-number</th>\n",
              "      <th>question</th>\n",
              "      <th>answer</th>\n",
              "      <th>text_content</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>IDC79659407</td>\n",
              "      <td>What is the newest learning management system ...</td>\n",
              "      <td>Blackboard Learn.</td>\n",
              "      <td>What is the newest learning management system ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>IDCfc1b32d0</td>\n",
              "      <td>Does Boston University offer a laptop loan pro...</td>\n",
              "      <td>Yes, they have a laptop loaner program for act...</td>\n",
              "      <td>Does Boston University offer a laptop loan pro...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>IDC5385384b</td>\n",
              "      <td>What is Blackboard Learn?</td>\n",
              "      <td>Blackboard Learn is the primary learning manag...</td>\n",
              "      <td>What is Blackboard Learn? Blackboard Learn is ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>IDCb586e780</td>\n",
              "      <td>What are the key features of Blackboard Learn?</td>\n",
              "      <td>The key features of Blackboard Learn include i...</td>\n",
              "      <td>What are the key features of Blackboard Learn?...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>IDC1309c0a1</td>\n",
              "      <td>What to expect from Blackboard Learn?</td>\n",
              "      <td>Blackboard Learn is normally available 24/7, e...</td>\n",
              "      <td>What to expect from Blackboard Learn? Blackboa...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>IDCf1f27cca</td>\n",
              "      <td>How do I get started with Blackboard Learn?</td>\n",
              "      <td>To get started with Blackboard Learn, faculty ...</td>\n",
              "      <td>How do I get started with Blackboard Learn? To...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>IDC53714075</td>\n",
              "      <td>What does the IT Help Center provide?</td>\n",
              "      <td>The IT Help Center provides centralized techno...</td>\n",
              "      <td>What does the IT Help Center provide? The IT H...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>IDC87909b25</td>\n",
              "      <td>How do I contact the IT Help Center?</td>\n",
              "      <td>You can contact the IT Help Center online or b...</td>\n",
              "      <td>How do I contact the IT Help Center? You can c...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>IDCbd8115e6</td>\n",
              "      <td>What services does the IT Help Center offer?</td>\n",
              "      <td>The IT Help Center offers a variety of service...</td>\n",
              "      <td>What services does the IT Help Center offer? T...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>IDCa2be0617</td>\n",
              "      <td>Do I need a current BU login name to use some ...</td>\n",
              "      <td>Yes, some services require a current BU login ...</td>\n",
              "      <td>Do I need a current BU login name to use some ...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-727f3c5e-fdf5-4edd-9263-4b0931f20c98')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-727f3c5e-fdf5-4edd-9263-4b0931f20c98 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-727f3c5e-fdf5-4edd-9263-4b0931f20c98');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-94c40ae2-45b6-478c-a7b9-c2d927ec6e6c\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-94c40ae2-45b6-478c-a7b9-c2d927ec6e6c')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-94c40ae2-45b6-478c-a7b9-c2d927ec6e6c button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "dataset_df",
              "summary": "{\n  \"name\": \"dataset_df\",\n  \"rows\": 10,\n  \"fields\": [\n    {\n      \"column\": \"ticket-number\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 10,\n        \"samples\": [\n          \"IDCbd8115e6\",\n          \"IDCfc1b32d0\",\n          \"IDCf1f27cca\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"question\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 10,\n        \"samples\": [\n          \"What services does the IT Help Center offer?\",\n          \"Does Boston University offer a laptop loan program for students?\",\n          \"How do I get started with Blackboard Learn?\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"answer\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 10,\n        \"samples\": [\n          \"The IT Help Center offers a variety of services, including software support for Mac and Windows operating systems and applications, manufacturer authorized warranty and non-warranty hardware repair, sales (and Software Assurance fulfillment) of Microsoft and Adobe software to BU departments and sales of Microsoft Office to BU individuals, assistance with your BU login name and Kerberos password, and access to educational discounts on recommended computer systems.\",\n          \"Yes, they have a laptop loaner program for actively enrolled BU students.\",\n          \"To get started with Blackboard Learn, faculty can create a new course site or copy a previous semester's site online. Instructors can access Blackboard Learn to start using a course site after it has been created. They can also attend a training session, view online help material, or take advantage of Educational Technology Consulting. Read the Ed Tech Blog to learn how others at BU are using Blackboard. Online Campus contact information is available for those with questions or concerns.\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"text_content\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 10,\n        \"samples\": [\n          \"What services does the IT Help Center offer? The IT Help Center offers a variety of services, including software support for Mac and Windows operating systems and applications, manufacturer authorized warranty and non-warranty hardware repair, sales (and Software Assurance fulfillment) of Microsoft and Adobe software to BU departments and sales of Microsoft Office to BU individuals, assistance with your BU login name and Kerberos password, and access to educational discounts on recommended computer systems.\",\n          \"Does Boston University offer a laptop loan program for students? Yes, they have a laptop loaner program for actively enrolled BU students.\",\n          \"How do I get started with Blackboard Learn? To get started with Blackboard Learn, faculty can create a new course site or copy a previous semester's site online. Instructors can access Blackboard Learn to start using a course site after it has been created. They can also attend a training session, view online help material, or take advantage of Educational Technology Consulting. Read the Ed Tech Blog to learn how others at BU are using Blackboard. Online Campus contact information is available for those with questions or concerns.\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 150
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "6Wxa6oNM25ug"
      },
      "outputs": [],
      "source": [
        "from llama_index.core.settings import Settings\n",
        "\n",
        "from llama_index.llms.openai import OpenAI\n",
        "\n",
        "from llama_index.embeddings.openai import OpenAIEmbedding\n",
        "\n",
        "embed_model=OpenAIEmbedding(model=\"text-embedding-3-small\",dimensions=256)\n",
        "\n",
        "llm = OpenAI(model=\"gpt-4-turbo\")\n",
        "\n",
        "Settings.llm=llm\n",
        "\n",
        "Settings.embed_model=embed_model\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "zI6wf8_Ycm5e"
      },
      "outputs": [],
      "source": [
        "import json\n",
        "from llama_index.core import Document\n",
        "from llama_index.core.schema import MetadataMode\n",
        "\n",
        "# Convert the DataFrame to a JSON string representation\n",
        "documents_json = dataset_df.to_json(orient='records')\n",
        "# Load the JSON string into a Python list of dictionaries\n",
        "documents_list = json.loads(documents_json)\n",
        "\n",
        "llama_documents = []\n",
        "\n",
        "for document in documents_list:\n",
        "\n",
        "  # Value for metadata must be one of (str, int, float, None)\n",
        "  document[\"ticket-number\"] = json.dumps(document[\"ticket-number\"])\n",
        "  document[\"question\"] = json.dumps(document[\"question\"])\n",
        "  document[\"answer\"] = json.dumps(document[\"answer\"])\n",
        "\n",
        "\n",
        "\n",
        "  # Create a Document object with the text and excluded metadata for llm and embedding models\n",
        "  llama_document = Document(\n",
        "      text=document[\"text_content\"],\n",
        "      metadata=document,\n",
        "      excluded_llm_metadata_keys=[\"question\", \"answer\"],\n",
        "      excluded_embed_metadata_keys=[\"question\", \"answer\"],\n",
        "\n",
        "      metadata_template=\"{key}=>{value}\",\n",
        "      text_template=\"Metadata: {metadata_str}\\n-----\\nContent: {content}\",\n",
        "      )\n",
        "\n",
        "  llama_documents.append(llama_document)\n",
        "\n",
        "# Observing an example of what the LLM and Embedding model receive as input\n",
        "print(\n",
        "    \"\\nThe LLM sees this: \\n\",\n",
        "    llama_documents[0].get_content(metadata_mode=MetadataMode.LLM),\n",
        ")\n",
        "print(\n",
        "    \"\\nThe Embedding model sees this: \\n\",\n",
        "    llama_documents[0].get_content(metadata_mode=MetadataMode.EMBED),\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "gfCdCTnl3D0B"
      },
      "outputs": [],
      "source": [
        "from llama_index.core.node_parser import SentenceSplitter\n",
        "\n",
        "parser = SentenceSplitter()\n",
        "nodes = parser.get_nodes_from_documents(llama_documents)\n",
        "\n",
        "for node in nodes:\n",
        "    node_embedding = embed_model.get_text_embedding(\n",
        "        node.get_content(metadata_mode=\"all\")\n",
        "    )\n",
        "    node.embedding = node_embedding\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NxeWc-2WfsdV",
        "outputId": "80450568-2a63-4da9-d6b2-8e1162ffb809"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Connection to MongoDB successful\n"
          ]
        }
      ],
      "source": [
        "#set the uri before running this cell.\n",
        "import pymongo\n",
        "from google.colab import userdata\n",
        "\n",
        "\n",
        "def get_mongo_client(mongo_uri):\n",
        "  \"\"\"Establish connection to the MongoDB.\"\"\"\n",
        "  try:\n",
        "    client = pymongo.MongoClient(mongo_uri)\n",
        "    print(\"Connection to MongoDB successful\")\n",
        "    return client\n",
        "  except pymongo.errors.ConnectionFailure as e:\n",
        "    print(f\"Connection failed: {e}\")\n",
        "    return None\n",
        "\n",
        "mongo_uri = userdata.get('MONGO_URI')\n",
        "if not mongo_uri:\n",
        "  print(\"MONGO_URI not set in environment variables\")\n",
        "\n",
        "mongo_client = get_mongo_client(mongo_uri)\n",
        "\n",
        "DB_NAME=\"itdata\"\n",
        "COLLECTION_NAME=\"it_support_data\"\n",
        "\n",
        "db = mongo_client[DB_NAME]\n",
        "collection = db[COLLECTION_NAME]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pxq8Jo591qA0",
        "outputId": "53df5541-ea58-4be5-9514-6d37a4c939aa"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "DeleteResult({'n': 0, 'electionId': ObjectId('7fffffff000000000000030b'), 'opTime': {'ts': Timestamp(1714584301, 16), 't': 779}, 'ok': 1.0, '$clusterTime': {'clusterTime': Timestamp(1714584301, 16), 'signature': {'hash': b'[O\\xaa\\xe8\\x1dCM.\\xba\\xe4\\x18\\x9b^\\xef=\\xa7\\xb0K|\\x0b', 'keyId': 7311276111621521410}}, 'operationTime': Timestamp(1714584301, 16)}, acknowledged=True)"
            ]
          },
          "metadata": {},
          "execution_count": 185
        }
      ],
      "source": [
        "# To ensure we are working with a fresh collection\n",
        "# delete any existing records in the collection\n",
        "\n",
        "collection.delete_many({})\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "_AZ6X4Cc1y2c"
      },
      "outputs": [],
      "source": [
        "from llama_index.vector_stores.mongodb import MongoDBAtlasVectorSearch\n",
        "\n",
        "vector_store = MongoDBAtlasVectorSearch(mongo_client, db_name=DB_NAME, collection_name=COLLECTION_NAME, index_name=\"vector_index\")\n",
        "vector_store.add(nodes)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "vxHGR-U71z6m"
      },
      "outputs": [],
      "source": [
        "from llama_index.core import VectorStoreIndex, StorageContext\n",
        "index = VectorStoreIndex.from_vector_store(vector_store)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "HznS4NRp2Kkz"
      },
      "outputs": [],
      "source": [
        "\n",
        "import pprint\n",
        "from llama_index.core.response.notebook_utils import display_response\n",
        "\n",
        "query_engine = index.as_query_engine(similarity_top_k=3)\n",
        "query = \"Do I need a current BU login name to use some services??\"\n",
        "response = query_engine.query(query)\n",
        "display_response(response)\n",
        "pprint.pprint(response.source_nodes)\n"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}