fix: linting

arc53 · Dec 23, 2024 · 41b4c28 · 41b4c28
1 parent 90962ee
commit 41b4c28
Show file tree

Hide file tree

Showing 3 changed files with 3 additions and 5 deletions.
diff --git a/application/parser/chunking.py b/application/parser/chunking.py
@@ -1,5 +1,5 @@
 import re
-from typing import List, Tuple, Union
+from typing import List, Tuple
 import logging
 from application.parser.schema.base import Document
 from application.utils import get_encoding

diff --git a/application/parser/embedding_pipeline.py b/application/parser/embedding_pipeline.py
@@ -61,15 +61,15 @@ def embed_and_store_documents(docs, folder_name, source_id, task_status):
 
     # Process and embed documents
     for idx, doc in tqdm(
-        docs,
+        enumerate(docs),
         desc="Embedding 🦖",
         unit="docs",
         total=total_docs,
         bar_format="{l_bar}{bar}| Time Left: {remaining}",
     ):
         try:
             # Update task status for progress tracking
-            progress = int((idx / total_docs) * 100)
+            progress = int(((idx + 1) / total_docs) * 100)
             task_status.update_state(state="PROGRESS", meta={"current": progress})
 
             # Add document to vector store

diff --git a/application/worker.py b/application/worker.py
@@ -126,7 +126,6 @@ def ingest_worker(
     limit = None
     exclude = True
     sample = False
-    token_check = True
     full_path = os.path.join(directory, user, name_job)
 
     logging.info(f"Ingest file: {full_path}", extra={"user": user, "job": name_job})
@@ -205,7 +204,6 @@ def remote_worker(
     operation_mode="upload",
     doc_id=None,
 ):
-    token_check = True
     full_path = os.path.join(directory, user, name_job)
 
     if not os.path.exists(full_path):