Skip to content

Commit

Permalink
Using data.ts for last_processed_ts; not using last_trip_done
Browse files Browse the repository at this point in the history
Fetching the last trip using the queries used to export and delete the data.
Fetched using indexing [-1] and then getting time from 'data.ts' instead of 'data.write_ts.'
  • Loading branch information
Mahadik, Mukul Chandrakant authored and Mahadik, Mukul Chandrakant committed Aug 9, 2024
1 parent b65430d commit 5fb6370
Showing 1 changed file with 3 additions and 15 deletions.
18 changes: 3 additions & 15 deletions emission/storage/pipeline_queries.py
Original file line number Diff line number Diff line change
Expand Up @@ -237,18 +237,12 @@ def get_time_range_for_purge_data(user_id):
tq.timeType = "data.end_ts"
return tq

def mark_purge_data_done(user_id, last_trip_done):
if last_trip_done is None:
def mark_purge_data_done(user_id, last_processed_ts):
if last_processed_ts is None:
mark_stage_done(user_id, ps.PipelineStages.PURGE_TIMESERIES_DATA, None)
else:
mark_stage_done(user_id, ps.PipelineStages.PURGE_TIMESERIES_DATA,
last_trip_done.data.ts + END_FUZZ_AVOID_LTE)
# def mark_purge_data_done(user_id, last_processed_ts):
# if last_processed_ts is None:
# mark_stage_done(user_id, ps.PipelineStages.PURGE_TIMESERIES_DATA, None)
# else:
# mark_stage_done(user_id, ps.PipelineStages.PURGE_TIMESERIES_DATA,
# last_processed_ts + END_FUZZ_AVOID_LTE)
last_processed_ts + END_FUZZ_AVOID_LTE)

def mark_purge_data_failed(user_id):
mark_stage_failed(user_id, ps.PipelineStages.PURGE_TIMESERIES_DATA)
Expand All @@ -264,12 +258,6 @@ def mark_restore_data_done(user_id, last_processed_ts):
else:
mark_stage_done(user_id, ps.PipelineStages.RESTORE_TIMESERIES_DATA,
last_processed_ts + END_FUZZ_AVOID_LTE)
# def mark_restore_data_done(user_id, last_processed_ts):
# if last_processed_ts is None:
# mark_stage_done(user_id, ps.PipelineStages.RESTORE_TIMESERIES_DATA, None)
# else:
# mark_stage_done(user_id, ps.PipelineStages.RESTORE_TIMESERIES_DATA,
# last_processed_ts + END_FUZZ_AVOID_LTE)

def mark_restore_data_failed(user_id):
mark_stage_failed(user_id, ps.PipelineStages.RESTORE_TIMESERIES_DATA)
Expand Down

0 comments on commit 5fb6370

Please sign in to comment.