tutorials/7_Processors.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "\n",
    "import sys\n",
    "sys.path.append('..\\\\code')\n",
    "\n",
    "\n",
    "import os\n",
    "from dotenv import load_dotenv\n",
    "load_dotenv()\n",
    "\n",
    "from IPython.display import display, Markdown, HTML\n",
    "from PIL import Image\n",
    "from doc_utils import *\n",
    "from processor import *\n",
    "\n",
    "\n",
    "\n",
    "def show_img(img_path, width = None):\n",
    "    if width is not None:\n",
    "        display(HTML(f'<img src=\"{img_path}\" width={width}>'))  \n",
    "    else:\n",
    "        display(Image.open(img_path))\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Ingest PDF Document with Document Intelligence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "doc_path = r\"sample_data\\1_London_Brochure.pdf\"\n",
    "\n",
    "ingestion_params_dict = {\n",
    "        \"index_name\" : 'test_pdf_doc_int',\n",
    "        \"delete_existing_output_dir\": True,\n",
    "        \"processing_mode_pdf\" : 'document-intelligence',\n",
    "        \"doc_path\" : doc_path,\n",
    "        'num_threads': 1\n",
    "    }\n",
    "\n",
    "pdf1 = PdfProcessor(ingestion_params_dict)\n",
    "pdf1.ingest_document()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Ingest PDF Document with GPT-4-Vision"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "doc_path = r\"sample_data\\1_London_Brochure.pdf\"\n",
    "\n",
    "ingestion_params_dict = {\n",
    "        \"index_name\" : 'test_pdf_gt4_vision',\n",
    "        \"delete_existing_output_dir\": True,\n",
    "        \"processing_mode_pdf\" : 'gpt-4-vision',\n",
    "        \"doc_path\" : doc_path,\n",
    "        'num_threads': 1\n",
    "    }\n",
    "\n",
    "pdf2 = PdfProcessor(ingestion_params_dict)\n",
    "pdf2.ingest_document()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Ingest PDF Document with a Hybrid approach"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "doc_path = r\"sample_data\\1_London_Brochure.pdf\"\n",
    "\n",
    "ingestion_params_dict = {\n",
    "        \"index_name\" : 'test_pdf_hybrid',\n",
    "        \"delete_existing_output_dir\": True,\n",
    "        \"processing_mode_pdf\" : 'hybrid',\n",
    "        \"doc_path\" : doc_path,\n",
    "        'num_threads': 1\n",
    "    }\n",
    "\n",
    "pdf2 = PdfProcessor(ingestion_params_dict)\n",
    "pdf2.ingest_document()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Ingest Xlsx Document with OpenPyxl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "doc_path = r\"sample_data/11_work_orders_mini.xlsx\"\n",
    "\n",
    "ingestion_params_dict = {\n",
    "        \"index_name\" : 'test_xlsx',\n",
    "        \"delete_existing_output_dir\": True,\n",
    "        \"doc_path\" : doc_path,\n",
    "        'num_threads': 1\n",
    "    }\n",
    "\n",
    "xlsx = XlsxProcessor(ingestion_params_dict)\n",
    "xlsx.ingest_document()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Ingest Docx Document with Python-Docx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "doc_path = r\"sample_data/1_London_Brochure.docx\"\n",
    "\n",
    "ingestion_params_dict = {\n",
    "        \"index_name\" : 'test_docx_pydocx',\n",
    "        \"delete_existing_output_dir\": True,\n",
    "        \"processing_mode_pdf\" : 'py-docx',\n",
    "        \"doc_path\" : doc_path,\n",
    "        'num_threads': 1\n",
    "    }\n",
    "\n",
    "docx1 = DocxProcessor(ingestion_params_dict)\n",
    "docx1.ingest_document()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Ingest Docx Document with Document Intelligence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "doc_path = r\"sample_data/1_London_Brochure.docx\"\n",
    "\n",
    "ingestion_params_dict = {\n",
    "        \"index_name\" : 'test_docx_doc_int',\n",
    "        \"delete_existing_output_dir\": True,\n",
    "        \"processing_mode_pdf\" : 'document-intelligence',\n",
    "        \"doc_path\" : doc_path,\n",
    "        'num_threads': 1\n",
    "    }\n",
    "\n",
    "docx2 = DocxProcessor(ingestion_params_dict)\n",
    "docx2.ingest_document()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "mmdoc",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}