Initialize pipelines for more doc types

Signed-off-by: Aakanksha Duggal <[email protected]>
instructlab · Jan 20, 2025 · 1484355 · 1484355
1 parent 0b91a3f
commit 1484355
Showing 1 changed file with 3 additions and 1 deletion.
diff --git a/src/instructlab/sdg/utils/chunkers.py b/src/instructlab/sdg/utils/chunkers.py
@@ -155,7 +155,9 @@ def chunk_documents(self) -> List:
         docling_json_paths = list(docling_artifacts_path.glob("*.json"))
         chunks = []
         for json_fp in docling_json_paths:
-            chunks.extend(self._process_parsed_docling_json(json_fp))
+            with json_fp.open("r", encoding="utf-8") as file:
+                data = json.load(file)
+                chunks.extend(self._process_parsed_docling_json(data))
 
         return chunks