refactor(docling): pipeline options (#21951)

.dockerignore

@@ -118,6 +118,7 @@ machine-learning/feature-store/driver_features/data
 machine-learning/graph-neural-network/dataset
 machine-learning/graph-neural-network/wandb
 machine-learning/hm-autogluon/AutogluonModels
+machine-learning/hm-docling/data
 machine-learning/hm-langchain/applications/*/data
 machine-learning/hm-llama-index/applications/*/data
 machine-learning/hm-mlflow/experiments/*/data

.gitignore

@@ -118,6 +118,7 @@ machine-learning/feature-store/driver_features/data
 machine-learning/graph-neural-network/dataset
 machine-learning/graph-neural-network/wandb
 machine-learning/hm-autogluon/AutogluonModels
+machine-learning/hm-docling/data
 machine-learning/hm-langchain/applications/*/data
 machine-learning/hm-llama-index/applications/*/data
 machine-learning/hm-mlflow/experiments/*/data

.markdownlint-cli2.jsonc

@@ -126,6 +126,7 @@
     "machine-learning/graph-neural-network/dataset",
     "machine-learning/graph-neural-network/wandb",
     "machine-learning/hm-autogluon/AutogluonModels",
+    "machine-learning/hm-docling/data",
     "machine-learning/hm-langchain/applications/*/data",
     "machine-learning/hm-llama-index/applications/*/data",
     "machine-learning/hm-mlflow/experiments/*/data",

.prettierignore

@@ -118,6 +118,7 @@ machine-learning/feature-store/driver_features/data
 machine-learning/graph-neural-network/dataset
 machine-learning/graph-neural-network/wandb
 machine-learning/hm-autogluon/AutogluonModels
+machine-learning/hm-docling/data
 machine-learning/hm-langchain/applications/*/data
 machine-learning/hm-llama-index/applications/*/data
 machine-learning/hm-mlflow/experiments/*/data

.rubocop.yml

@@ -123,6 +123,7 @@ AllCops:
     - 'machine-learning/graph-neural-network/dataset/**/*'
     - 'machine-learning/graph-neural-network/wandb/**/*'
     - 'machine-learning/hm-autogluon/AutogluonModels/**/*'
+    - 'machine-learning/hm-docling/data/**/*'
     - 'machine-learning/hm-langchain/applications/*/data/**/*'
     - 'machine-learning/hm-llama-index/applications/*/data/**/*'
     - 'machine-learning/hm-mlflow/experiments/*/data/**/*'

.ruff.toml

@@ -117,6 +117,7 @@ exclude = [
   "machine-learning/graph-neural-network/dataset",
   "machine-learning/graph-neural-network/wandb",
   "machine-learning/hm-autogluon/AutogluonModels",
+  "machine-learning/hm-docling/data",
   "machine-learning/hm-langchain/applications/*/data",
   "machine-learning/hm-llama-index/applications/*/data",
   "machine-learning/hm-mlflow/experiments/*/data",

.solhintignore

@@ -116,6 +116,7 @@ machine-learning/feature-store/driver_features/data
 machine-learning/graph-neural-network/dataset
 machine-learning/graph-neural-network/wandb
 machine-learning/hm-autogluon/AutogluonModels
+machine-learning/hm-docling/data
 machine-learning/hm-langchain/applications/*/data
 machine-learning/hm-llama-index/applications/*/data
 machine-learning/hm-mlflow/experiments/*/data

.sqlfluffignore

@@ -117,6 +117,7 @@ machine-learning/feature-store/driver_features/data
 machine-learning/graph-neural-network/dataset
 machine-learning/graph-neural-network/wandb
 machine-learning/hm-autogluon/AutogluonModels
+machine-learning/hm-docling/data
 machine-learning/hm-langchain/applications/*/data
 machine-learning/hm-llama-index/applications/*/data
 machine-learning/hm-mlflow/experiments/*/data

.stylelintignore

@@ -118,6 +118,7 @@ machine-learning/feature-store/driver_features/data
 machine-learning/graph-neural-network/dataset
 machine-learning/graph-neural-network/wandb
 machine-learning/hm-autogluon/AutogluonModels
+machine-learning/hm-docling/data
 machine-learning/hm-langchain/applications/*/data
 machine-learning/hm-llama-index/applications/*/data
 machine-learning/hm-mlflow/experiments/*/data

.textlintignore

@@ -118,6 +118,7 @@ machine-learning/feature-store/driver_features/data/**/*
 machine-learning/graph-neural-network/dataset/**/*
 machine-learning/graph-neural-network/wandb/**/*
 machine-learning/hm-autogluon/AutogluonModels/**/*
+machine-learning/hm-docling/data/**/*
 machine-learning/hm-langchain/applications/*/data/**/*
 machine-learning/hm-llama-index/applications/*/data/**/*
 machine-learning/hm-mlflow/experiments/*/data/**/*

.yamllint.yaml

@@ -123,6 +123,7 @@ ignore: |
   machine-learning/graph-neural-network/dataset
   machine-learning/graph-neural-network/wandb
   machine-learning/hm-autogluon/AutogluonModels
+  machine-learning/hm-docling/data
   machine-learning/hm-langchain/applications/*/data
   machine-learning/hm-llama-index/applications/*/data
   machine-learning/hm-mlflow/experiments/*/data

eslint.config.mjs

@@ -130,6 +130,7 @@ export default [
       'machine-learning/graph-neural-network/dataset',
       'machine-learning/graph-neural-network/wandb',
       'machine-learning/hm-autogluon/AutogluonModels',
+      'machine-learning/hm-docling/data',
       'machine-learning/hm-langchain/applications/*/data',
       'machine-learning/hm-llama-index/applications/*/data',
       'machine-learning/hm-mlflow/experiments/*/data',

machine-learning/hm-docling/src/main.py

@@ -1,19 +1,31 @@
 import logging
 from pathlib import Path
 
-from docling.datamodel.pipeline_options import EasyOcrOptions, PdfPipelineOptions
-from docling.document_converter import DocumentConverter
+from docling.datamodel.base_models import InputFormat
+from docling.datamodel.pipeline_options import (
+    EasyOcrOptions,
+    PdfPipelineOptions,
+    TableStructureOptions,
+)
+from docling.document_converter import DocumentConverter, PdfFormatOption
 
 
 def main() -> None:
     data_dir = Path("data")
     pdf_paths = data_dir.glob("**/*.pdf")
 
-    pipeline_options = PdfPipelineOptions()
-    pipeline_options.do_ocr = True
-    pipeline_options.do_table_structure = True
-    pipeline_options.table_structure_options.do_cell_matching = True
-    pipeline_options.ocr_options = EasyOcrOptions(force_full_page_ocr=True)
+    pipeline_options = PdfPipelineOptions(
+        do_ocr=True,
+        do_table_structure=True,
+        table_structure_options=TableStructureOptions(do_cell_matching=True),
+        ocr_options=EasyOcrOptions(),
+    )
+    converter = DocumentConverter(
+        allowed_formats=[InputFormat.PDF],
+        format_options={
+            InputFormat.PDF: PdfFormatOption(pipeline_options=pipeline_options)
+        },
+    )
 
     converter = DocumentConverter()
 

machine-learning/hm-langchain/applications/chat-pdf/src/main.py

@@ -29,6 +29,6 @@ def main(model_path: str, pdf_path: str, question: str) -> None:
     # https://gpt4all.io/index.html
     external_model_path = "data/ggml-model-gpt4all-falcon-q4_0.bin"
     external_pdf_path = "data/my.pdf"
-    external_question = "Could you please summarize this PDF? Thank you!"
+    external_question = "Could you please summarize this PDF?"
 
     main(external_model_path, external_pdf_path, external_question)

machine-learning/hm-llama-index/applications/chat-pdf/src/main.py

@@ -28,8 +28,8 @@ def main() -> None:
         temperature=0.7,
     )
 
-    pdf_path = Path("data/paper.pdf")
-    question = "Could you please summarize this PDF? Thank you!"
+    pdf_path = Path("data/file.pdf")
+    question = "Could you please summarize this PDF?"
     answer = chat_with_pdf(pdf_path, question)
     logging.info(answer)