cdQA2.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "cdQA2.ipynb",
      "provenance": [],
      "authorship_tag": "ABX9TyMRvuNpm0FPRxw8hORXonPm",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/anirbansaha96/AI-ML-Playground/blob/master/cdQA2.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xls3Fy2CXHsE",
        "colab_type": "text"
      },
      "source": [
        "We implement the cdQA suite by downloading PDF's using URL's"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ls-8oJkLRNbT",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#!pip install cdqa"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0yIIBm8kRQid",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import os\n",
        "import pandas as pd\n",
        "from ast import literal_eval\n",
        "\n",
        "from cdqa.utils.converters import pdf_converter\n",
        "from cdqa.utils.filters import filter_paragraphs\n",
        "from cdqa.pipeline import QAPipeline\n",
        "from cdqa.utils.download import download_model\n",
        "# Download model\n",
        "download_model(model='bert-squad_1.1', dir='./models')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "f0EsjML0SRQM",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def download_pdf():\n",
        "    import os\n",
        "    import wget\n",
        "    directory = './data/pdf/'\n",
        "    models_url = [\n",
        "      'https://docsmsftpdfs.blob.core.windows.net/guides/azure/azure-ops-guide.pdf'\n",
        "    ]\n",
        "\n",
        "    print('\\nDownloading PDF files...')\n",
        "\n",
        "    if not os.path.exists(directory):\n",
        "        os.makedirs(directory)\n",
        "    for url in models_url:\n",
        "        wget.download(url=url, out=directory)\n",
        "\n",
        "download_pdf()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vT9QeqdoSXHT",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df = pdf_converter(directory_path='./data/pdf/')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WT6EJV57SY91",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "cdqa_pipeline = QAPipeline(reader='./models/bert_qa.joblib', max_df=1.0)\n",
        "\n",
        "# Fit Retriever to documents\n",
        "cdqa_pipeline.fit_retriever(df=df)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TBTPt87uSqeL",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 107
        },
        "outputId": "e1867717-f1e7-42da-b288-f905bcf4f78e"
      },
      "source": [
        "query = 'What is the azure portal?'\n",
        "prediction = cdqa_pipeline.predict(query)\n",
        "print('query: {}'.format(query))\n",
        "print('answer: {}'.format(prediction[0]))\n",
        "print('title: {}'.format(prediction[1]))\n",
        "print('paragraph: {}'.format(prediction[2]))"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "query: What is the azure portal?\n",
            "answer: a web-based application\n",
            "title: azure-ops-guide\n",
            "paragraph: Azure portal The Azure portal is a web-based application that can be used to create, manage, and remove Azure resources and services. The Azure portal is located at https://portal.azure.com. It includes a customizable dashboard and tooling for managing Azure resources. It also provides billing and subscription information. For more information, see Microsoft Azure portal overview. \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "16tKMccdS_Eu",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 107
        },
        "outputId": "399f0c12-b2ad-4ff7-d81c-8d79e6d6899d"
      },
      "source": [
        "query = 'What is An Azure Resource Manager template ?'\n",
        "prediction = cdqa_pipeline.predict(query)\n",
        "print('query: {}'.format(query))\n",
        "print('answer: {}'.format(prediction[0]))\n",
        "print('title: {}'.format(prediction[1]))\n",
        "print('paragraph: {}'.format(prediction[2]))"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "query: What is An Azure Resource Manager template ?\n",
            "answer: a JavaScript Object Notation (JSON) file that defines one or more resources to deploy to a resource group\n",
            "title: azure-ops-guide\n",
            "paragraph: Resource Manager templates An Azure Resource Manager template is a JavaScript Object Notation (JSON) file that defines one or more resources to deploy to a resource group. It also defines the dependencies between deployed resources. Resource Manager templates are covered in more detail later in this guide.   https://azure.microsoft.com/en-us/regions/https://azure.microsoft.com/en-us/documentation/articles/azure-portal-overview/https://docs.microsoft.com/en-us/azure/azure-portal/resource-group-portal\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AtnNPYARTDeG",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 107
        },
        "outputId": "60f9c3e1-b7db-41b2-ec7f-ea222051d216"
      },
      "source": [
        "query = 'How can we automate?'\n",
        "prediction = cdqa_pipeline.predict(query)\n",
        "print('query: {}'.format(query))\n",
        "print('answer: {}'.format(prediction[0]))\n",
        "print('title: {}'.format(prediction[1]))\n",
        "print('paragraph: {}'.format(prediction[2]))"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "query: How can we automate?\n",
            "answer: by using PowerShell or the Azure command-line interface (CLI)\n",
            "title: azure-ops-guide\n",
            "paragraph: Automation In addition to creating, managing, and deleting resources by using the Azure portal, you can automate these activities by using PowerShell or the Azure command-line interface (CLI).  \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ljLwQAmeTYR-",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 107
        },
        "outputId": "7395d819-92a4-4ba7-f665-157d957e2ccd"
      },
      "source": [
        "query = 'What is an Azure Resource Group?'\n",
        "prediction = cdqa_pipeline.predict(query)\n",
        "print('query: {}'.format(query))\n",
        "print('answer: {}'.format(prediction[0]))\n",
        "print('title: {}'.format(prediction[1]))\n",
        "print('paragraph: {}'.format(prediction[2]))"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "query: What is an Azure Resource Group?\n",
            "answer: a container that holds related resources for an Azure solution\n",
            "title: azure-ops-guide\n",
            "paragraph: Resource groups An Azure resource group is a container that holds related resources for an Azure solution. The resource group can include all the resources for the solution, or only resources that you want to manage as a group. Azure resource groups are covered in more detail later in this guide. \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MplPZ63HTlk3",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 107
        },
        "outputId": "183398a9-34c8-4983-c986-434677a96535"
      },
      "source": [
        "query = 'How do we use a pay-as-you-go model?'\n",
        "prediction = cdqa_pipeline.predict(query)\n",
        "print('query: {}'.format(query))\n",
        "print('answer: {}'.format(prediction[0]))\n",
        "print('title: {}'.format(prediction[1]))\n",
        "print('paragraph: {}'.format(prediction[2]))"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "query: How do we use a pay-as-you-go model?\n",
            "answer: by attaching a credit card or debit card to the account\n",
            "title: azure-ops-guide\n",
            "paragraph: Pay-as-you-go: With this subscription, you pay for what you use by attaching a credit card or debit card to the account. If you are an organization, you can also be approved for invoicing. \n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}