From 568a2a0e2491786f0601048cc36d4ed662d63a17 Mon Sep 17 00:00:00 2001 From: daoleno Date: Fri, 22 Dec 2023 18:18:08 +0800 Subject: [PATCH] fix bigquery query condition --- bq-syncer/sync_parquet.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bq-syncer/sync_parquet.py b/bq-syncer/sync_parquet.py index 5109a37..1b6dffd 100644 --- a/bq-syncer/sync_parquet.py +++ b/bq-syncer/sync_parquet.py @@ -85,12 +85,12 @@ def sync_table(table_item, index, total_tables): # If the parquet files exist, get the maximum timestamp from the latest file parquet_files = sorted( - glob.glob(os.path.join(output_directory, f"{table_id}_*.parquet")) + glob.glob(os.path.join(table_directory, f"{table_id}_*.parquet")) ) if parquet_files: df_old = pl.read_parquet(parquet_files[-1]) # read the latest file - if "datastream_metadata.source_timestamp" in df_old.columns: - last_timestamp = df_old["datastream_metadata.source_timestamp"].max() + if "source_timestamp" in df_old.columns: + last_timestamp = df_old["source_timestamp"].max() # Generate list of fields and their types, preserving the original schema's order. fields = [