feat: propose a simpler way to get software row_id

evadb/optimizer/statement_to_opr_converter.py

@@ -14,10 +14,7 @@
 # limitations under the License.
 from evadb.catalog.catalog_type import TableType
 from evadb.catalog.catalog_utils import get_table_primary_columns
-from evadb.constants import MAGIC_NUMBER
 from evadb.expression.abstract_expression import AbstractExpression, ExpressionType
-from evadb.expression.arithmetic_expression import ArithmeticExpression
-from evadb.expression.constant_value_expression import ConstantValueExpression
 from evadb.expression.tuple_value_expression import TupleValueExpression
 from evadb.optimizer.operators import (
     LogicalCreate,
@@ -334,27 +331,6 @@ def visit_create_index(self, statement: CreateIndexStatement):
         if statement.udf_func:
             project_exprs = [statement.udf_func.copy()]
 
-        # We need to also store the primary keys of the table in the index to later do
-        # the mapping. Typically, the vector indexes support setting an integer ID with
-        # the embedding. Unfortunately, we cannot support multi-column primary keys for
-        # cases like video table and document table.
-        # Hack: In such cases, we convert them into a unique ID using the following
-        # logic: We assume that the maximum number of files in the table is <=
-        # MAGIC_NUMBER and the number of frames or chunks for each video/document is <=
-        # MAGIC_NUMBER. Based on this assumption, we can safely say that
-        # `_row_id` * MAGIC_NUMBER + `chunk_id` for document table and
-        # `_row_id` * MAGIC_NUMBER + `id`) for video table
-        # `_row_id` * MAGIC_NUMBER + `paragraph`) for PDF table
-        # will be unique.
-        def _build_expression(row_id_expr, id_expr):
-            left = ArithmeticExpression(
-                ExpressionType.ARITHMETIC_MULTIPLY,
-                row_id_expr,
-                ConstantValueExpression(MAGIC_NUMBER),
-            )
-            right = id_expr
-            return ArithmeticExpression(ExpressionType.ARITHMETIC_ADD, left, right)
-
         primary_cols = get_table_primary_columns(catalog_entry)
 
         # primary_col to TupleValueExpressions
@@ -376,14 +352,6 @@ def _build_expression(row_id_expr, id_expr):
             )
 
         unique_col = primary_exprs[0]
-
-        if catalog_entry.table_type in [
-            TableType.VIDEO_DATA,
-            TableType.DOCUMENT_DATA,
-            TableType.PDF_DATA,
-        ]:
-            unique_col = _build_expression(primary_exprs[0], primary_exprs[1])
-
         project_exprs = [unique_col] + project_exprs
 
         logical_project = LogicalProject(project_exprs)

evadb/readers/pdf_reader.py

@@ -35,11 +35,10 @@ def _read(self) -> Iterator[Dict]:
 
         # PAGE ID, PARAGRAPH ID, STRING
         # Maintain a global paragraph number per PDF
-        global_paragraph_no = 0
         for page_no, page in enumerate(doc):
             blocks = page.get_text("dict")["blocks"]
             # iterate through the text blocks
-            for _, b in enumerate(blocks):
+            for paragraph_no, b in enumerate(blocks):
                 # this block contains text
                 if b["type"] == 0:
                     # text found in block
@@ -52,8 +51,7 @@ def _read(self) -> Iterator[Dict]:
                             if span["text"].strip():
                                 block_string += span["text"]
                     yield {
-                        "paragraph": global_paragraph_no,
+                        "paragraph": paragraph_no,
                         "page": page_no,
                         "data": block_string,
                     }
-                    global_paragraph_no += 1

evadb/storage/document_storage_engine.py

@@ -29,8 +29,10 @@ def __init__(self, db: EvaDBDatabase):
     def read(
         self, table: TableCatalogEntry, chunk_params: dict = {}
     ) -> Iterator[Batch]:
+        # Software row_id generated at runtime
+        row_id = 1
         for doc_files in self._rdb_handler.read(self._get_metadata_table(table), 12):
-            for _, (row_id, file_name) in doc_files.iterrows():
+            for _, (_, file_name) in doc_files.iterrows():
                 system_file_name = self._xform_file_url_to_file_name(file_name)
                 doc_file = Path(table.file_url) / system_file_name
                 # setting batch_mem_size = 1, we need fix it
@@ -41,3 +43,4 @@ def read(
                     batch.frames[table.columns[0].name] = row_id
                     batch.frames[table.columns[1].name] = str(file_name)
                     yield batch
+                    row_id += 1

evadb/storage/image_storage_engine.py

@@ -27,8 +27,10 @@ def __init__(self, db: EvaDBDatabase):
         super().__init__(db)
 
     def read(self, table: TableCatalogEntry) -> Iterator[Batch]:
+        # Software row_id generated at runtime
+        row_id = 1
         for image_files in self._rdb_handler.read(self._get_metadata_table(table), 12):
-            for _, (row_id, file_name) in image_files.iterrows():
+            for _, (_, file_name) in image_files.iterrows():
                 system_file_name = self._xform_file_url_to_file_name(file_name)
                 image_file = Path(table.file_url) / system_file_name
                 # setting batch_mem_size = 1, we need fix it
@@ -37,3 +39,4 @@ def read(self, table: TableCatalogEntry) -> Iterator[Batch]:
                     batch.frames[table.columns[0].name] = row_id
                     batch.frames[table.columns[1].name] = str(file_name)
                     yield batch
+                    row_id += 1

evadb/storage/pdf_storage_engine.py

@@ -27,8 +27,10 @@ def __init__(self, db: EvaDBDatabase):
         super().__init__(db)
 
     def read(self, table: TableCatalogEntry) -> Iterator[Batch]:
+        # Software row_id generated at runtime
+        row_id = 1
         for image_files in self._rdb_handler.read(self._get_metadata_table(table), 12):
-            for _, (row_id, file_name) in image_files.iterrows():
+            for _, (_, file_name) in image_files.iterrows():
                 system_file_name = self._xform_file_url_to_file_name(file_name)
                 image_file = Path(table.file_url) / system_file_name
                 # setting batch_mem_size = 1, we need fix it
@@ -37,3 +39,4 @@ def read(self, table: TableCatalogEntry) -> Iterator[Batch]:
                     batch.frames[table.columns[0].name] = row_id
                     batch.frames[table.columns[1].name] = str(file_name)
                     yield batch
+                    row_id += 1

evadb/storage/video_storage_engine.py

@@ -38,8 +38,10 @@ def read(
         read_audio: bool = False,
         read_video: bool = True,
     ) -> Iterator[Batch]:
+        # Software row_id generated at runtime
+        row_id = 1
         for video_files in self._rdb_handler.read(self._get_metadata_table(table), 12):
-            for _, (row_id, video_file_name) in video_files.iterrows():
+            for _, (_, video_file_name) in video_files.iterrows():
                 system_file_name = self._xform_file_url_to_file_name(video_file_name)
                 video_file = Path(table.file_url) / system_file_name
                 # increase batch size when reading audio so that
@@ -59,3 +61,4 @@ def read(
                     batch.frames[table.columns[0].name] = row_id
                     batch.frames[table.columns[1].name] = str(video_file_name)
                     yield batch
+                    row_id += 1