Merged from scicom run

NERC-CEH · Nov 21, 2024 · ae2fefe · ae2fefe
2 parents d6c2eed + 7621920
commit ae2fefe
Show file tree

Hide file tree

Showing 4 changed files with 69 additions and 52 deletions.
diff --git a/data/.gitignore b/data/.gitignore
@@ -14,3 +14,4 @@
 /metrics.json
 /eval.png
 /eidc_rag_testset.csv
+/eidc_rag_test_set.csv
diff --git a/dvc.lock b/dvc.lock
@@ -1,7 +1,7 @@
 schema: '2.0'
 stages:
   fetch-metadata:
-    cmd: python scripts/fetch_eidc_metadata.py data/eidc_metadata.json -s 1
+    cmd: python scripts/fetch_eidc_metadata.py data/eidc_metadata.json -s 2000
     deps:
     - path: scripts/fetch_eidc_metadata.py
       hash: md5
@@ -10,8 +10,8 @@ stages:
     outs:
     - path: data/eidc_metadata.json
       hash: md5
-      md5: ee850e1b0b28cd55ad7d7b31c81645db
-      size: 114886
+      md5: 828442e08598fb258894f9d414943330
+      size: 12247809
   prepare:
     cmd: python scripts/extract_metadata.py data/eidc_metadata.json data/extracted_metadata.json
     deps:
@@ -33,135 +33,135 @@ stages:
     deps:
     - path: data/eidc_metadata.json
       hash: md5
-      md5: ee850e1b0b28cd55ad7d7b31c81645db
-      size: 114886
+      md5: 828442e08598fb258894f9d414943330
+      size: 12247809
     - path: scripts/extract_metadata.py
       hash: md5
       md5: e66f21369c5106eaaad4476612c6fb5e
       size: 1313
     outs:
     - path: data/extracted_metadata.json
       hash: md5
-      md5: 6870e7ecdde041bc8b62d2759ab745c3
-      size: 2381
+      md5: f6123510b2b337bc8a2b6a7180e54b36
+      size: 4606527
   chunk-data:
     cmd: python scripts/chunk_data.py -o data/chunked_data.json -c 250 -ol 75 data/extracted_metadata.json
       data/supporting-docs.json -m 250
     deps:
     - path: data/extracted_metadata.json
       hash: md5
-      md5: 6870e7ecdde041bc8b62d2759ab745c3
-      size: 2381
+      md5: f6123510b2b337bc8a2b6a7180e54b36
+      size: 4606527
     - path: data/supporting-docs.json
       hash: md5
-      md5: 12837e5cbf10fbd75c6fa476d6423a41
-      size: 75646
+      md5: 9144618bb329984fcd622811a7eac3bb
+      size: 72280322
     - path: scripts/chunk_data.py
       hash: md5
       md5: 3ad449140b03e1c2904b22a5b401a12e
       size: 2705
     outs:
     - path: data/chunked_data.json
       hash: md5
-      md5: 2bd1ec3c646b46de10f43e87a711ec34
-      size: 2576
+      md5: 39990a40f23e70dc78424ce0bc408983
+      size: 124484286
   create-embeddings:
     cmd: python scripts/create_embeddings.py data/chunked_data.json data/embeddings.json
     deps:
     - path: data/chunked_data.json
       hash: md5
-      md5: 2bd1ec3c646b46de10f43e87a711ec34
-      size: 2576
+      md5: 39990a40f23e70dc78424ce0bc408983
+      size: 124484286
     - path: scripts/create_embeddings.py
       hash: md5
       md5: fa4627c83a65af2e3ea9b2b749f1b29d
       size: 952
     outs:
     - path: data/embeddings.json
       hash: md5
-      md5: 84df39fc14944f3834863c56062f42bb
-      size: 61385
+      md5: 8971ce1f4d4ade1507b9a469656c36d0
+      size: 1706778742
   upload-to-docstore:
     cmd: python scripts/upload_to_docstore.py data/embeddings.json -o data/chroma-data
       -em all-MiniLM-L6-v2 -c eidc-data
     deps:
     - path: data/embeddings.json
       hash: md5
-      md5: 84df39fc14944f3834863c56062f42bb
-      size: 61385
+      md5: 8971ce1f4d4ade1507b9a469656c36d0
+      size: 1706778742
     - path: scripts/upload_to_docstore.py
       hash: md5
-      md5: 7b9433047ff175d5e6af8d6056caf05b
-      size: 1931
+      md5: 645bdeb372bc79fa7a2e3d8a9eac0d4c
+      size: 2330
     outs:
     - path: data/chroma-data
       hash: md5
-      md5: c302823e4ac392340c4dea80eff42d41.dir
-      size: 1872612
-      nfiles: 5
+      md5: 7d158df1ea32a09783259b756f468666.dir
+      size: 1126480472
+      nfiles: 6
   run-rag-pipeline:
-    cmd: python scripts/run_rag_pipeline.py -i data/eidc_rag_testset.csv -o data/evaluation_data.csv
-      -ds data/chroma-data -c eidc-data -m llama3.1
+    cmd: python scripts/run_rag_pipeline.py data/eidc_rag_test_set.csv data/evaluation_data.csv
+      data/chroma-data -c eidc-data
     deps:
     - path: data/chroma-data
       hash: md5
-      md5: c302823e4ac392340c4dea80eff42d41.dir
-      size: 1872612
-      nfiles: 5
-    - path: data/eidc_rag_testset.csv
+      md5: 7d158df1ea32a09783259b756f468666.dir
+      size: 1126480472
+      nfiles: 6
+    - path: data/eidc_rag_test_set.csv
       hash: md5
-      md5: 946861e99a3d1d5c37e48d6c791145ba
-      size: 4572
+      md5: f301e759e74ce5e71b50e04993ec8c88
+      size: 144597
     - path: scripts/run_rag_pipeline.py
       hash: md5
-      md5: a3f803eafc1a73d837a763f04a56924e
-      size: 3937
+      md5: 0be13da9adedc1c0dad4837523893061
+      size: 3869
     outs:
     - path: data/evaluation_data.csv
       hash: md5
-      md5: 7697d47129fe7491dfa15c8795ca29fe
-      size: 3911
+      md5: 61fc8879585c0385277ebdc8a6b82420
+      size: 203253
   generate-testset:
-    cmd: head -n 3 data/synthetic-datasets/eidc_rag_test_sample.csv > data/eidc_rag_testset.csv
+    cmd: cp data/synthetic-datasets/eidc_rag_test_set.csv data/
     outs:
-    - path: data/eidc_rag_testset.csv
+    - path: data/eidc_rag_test_set.csv
       hash: md5
-      md5: 946861e99a3d1d5c37e48d6c791145ba
-      size: 4572
+      md5: f301e759e74ce5e71b50e04993ec8c88
+      size: 144597
   fetch-supporting-docs:
     cmd: python scripts/fetch_supporting_docs.py data/eidc_metadata.json data/supporting-docs.json
     deps:
     - path: data/eidc_metadata.json
       hash: md5
-      md5: ee850e1b0b28cd55ad7d7b31c81645db
-      size: 114886
+      md5: 828442e08598fb258894f9d414943330
+      size: 12247809
     - path: scripts/fetch_supporting_docs.py
       hash: md5
       md5: 02b94a2cc7bff711784cbdec3650b618
       size: 1718
     outs:
     - path: data/supporting-docs.json
       hash: md5
-      md5: 12837e5cbf10fbd75c6fa476d6423a41
-      size: 75646
+      md5: 9144618bb329984fcd622811a7eac3bb
+      size: 72280322
   evaluate:
     cmd: python scripts/evaluate.py data/evaluation_data.csv -m data/metrics.json
       -img data/eval.png
     deps:
     - path: data/evaluation_data.csv
       hash: md5
-      md5: 7697d47129fe7491dfa15c8795ca29fe
-      size: 3911
+      md5: 61fc8879585c0385277ebdc8a6b82420
+      size: 203253
     - path: scripts/evaluate.py
       hash: md5
       md5: 4154acf8e74c1d8bcd0b0da72af038e0
       size: 2728
     outs:
     - path: data/eval.png
       hash: md5
-      md5: ac93331955c478b2a08ae3a4f081e841
-      size: 54427
+      md5: 2a1630782c103077959097db4e06b7d8
+      size: 83362
     - path: data/metrics.json
       hash: md5
-      md5: 9c25b2a92fa4c5fc59fbb2fdae83d0a2
-      size: 225
+      md5: dfdf0d0bf1519ccfa78f95263d63c231
+      size: 285
diff --git a/scripts/run_rag_pipeline.py b/scripts/run_rag_pipeline.py
@@ -1,7 +1,9 @@
 import shutil
 from argparse import ArgumentParser
 from typing import Any, Dict, List, Tuple
-
+__import__("pysqlite3")
+import sys
+sys.modules["sqlite3"] = sys.modules.pop("pysqlite3")
 import pandas as pd
 from haystack import Pipeline
 from haystack.components.builders import PromptBuilder

diff --git a/scripts/upload_to_docstore.py b/scripts/upload_to_docstore.py
@@ -4,8 +4,13 @@
 import uuid
 from argparse import ArgumentParser
 
+__import__("pysqlite3")
+import sys
+
+sys.modules["sqlite3"] = sys.modules.pop("pysqlite3")
 import chromadb
 from chromadb.utils import embedding_functions
+from chromadb.utils.batch_utils import create_batches
 
 
 def main(
@@ -33,7 +38,16 @@ def main(
         collection = client.create_collection(
             name=collection_name, embedding_function=func
         )
-        collection.add(documents=docs, metadatas=metas, embeddings=embs, ids=ids)
+        batches = create_batches(
+            api=client, ids=ids, documents=docs, embeddings=embs, metadatas=metas
+        )
+        for batch in batches:
+            collection.add(
+                documents=batch[3],
+                metadatas=batch[2],
+                embeddings=batch[1],
+                ids=batch[0],
+            )
 
 
 if __name__ == "__main__":