From 5f50702e5a3bab7f4ca7b5b09faf1ca6c2a9f7b9 Mon Sep 17 00:00:00 2001 From: muhammadali286 Date: Tue, 26 Nov 2024 11:34:17 +0500 Subject: [PATCH 1/4] Added management command to delete old pipeline errors. --- .../commands/delete_old_pipeline_error.py | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 figures/management/commands/delete_old_pipeline_error.py diff --git a/figures/management/commands/delete_old_pipeline_error.py b/figures/management/commands/delete_old_pipeline_error.py new file mode 100644 index 00000000..848e5b35 --- /dev/null +++ b/figures/management/commands/delete_old_pipeline_error.py @@ -0,0 +1,32 @@ +from __future__ import print_function +from __future__ import absolute_import +from textwrap import dedent + +from django.core.management.base import BaseCommand +from django.utils.timezone import now, timedelta +from figures.models import PipelineError + +class Command(BaseCommand): + '''Delete PipelineError records older than a specified number of days (default is 15 days).''' + help = dedent(__doc__).strip() + + def add_arguments(self, parser): + '''Add command arguments.''' + parser.add_argument( + '--days', + type=int, + default=15, + help='Number of days to retain PipelineError records. Defaults to 15.', + ) + + def handle(self, *args, **options): + '''Handle the deletion process.''' + days = options['days'] + cutoff_date = now() - timedelta(days=days) + + print(f"Deleting PipelineError records older than {days} days...") + + deleted_count, _ = PipelineError.objects.filter(created__lt=cutoff_date).delete() + + print(f"Deleted {deleted_count} PipelineError records older than {days} days.") + print('Done.') From d6e0576180caf0060b681bc4b54cdf509b1269af Mon Sep 17 00:00:00 2001 From: muhammadali286 Date: Tue, 26 Nov 2024 14:13:51 +0500 Subject: [PATCH 2/4] Updated command to batch deletion. --- .../commands/delete_old_pipeline_error.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/figures/management/commands/delete_old_pipeline_error.py b/figures/management/commands/delete_old_pipeline_error.py index 848e5b35..c1483078 100644 --- a/figures/management/commands/delete_old_pipeline_error.py +++ b/figures/management/commands/delete_old_pipeline_error.py @@ -24,9 +24,17 @@ def handle(self, *args, **options): days = options['days'] cutoff_date = now() - timedelta(days=days) - print(f"Deleting PipelineError records older than {days} days...") + print(f"Deleting PipelineError records older than {days} days in batches...") - deleted_count, _ = PipelineError.objects.filter(created__lt=cutoff_date).delete() + batch_size = 1000 + total_deleted = 0 + while True: + old_records = PipelineError.objects.filter(created__lt=cutoff_date)[:batch_size] + if not old_records: + break + deleted_count = old_records.delete()[0] + total_deleted += deleted_count + print(f"Deleted {deleted_count} records in this batch. Total deleted: {total_deleted}.") - print(f"Deleted {deleted_count} PipelineError records older than {days} days.") + print(f"Deletion complete. Total {total_deleted} PipelineError records older than {days} days deleted.") print('Done.') From ac564b26ff5229446c9fb1887cd01e7cf5335866 Mon Sep 17 00:00:00 2001 From: muhammadali286 Date: Wed, 27 Nov 2024 10:45:16 +0500 Subject: [PATCH 3/4] Shift to legacy batch deletion --- .../management/commands/delete_old_pipeline_error.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/figures/management/commands/delete_old_pipeline_error.py b/figures/management/commands/delete_old_pipeline_error.py index c1483078..5f18107c 100644 --- a/figures/management/commands/delete_old_pipeline_error.py +++ b/figures/management/commands/delete_old_pipeline_error.py @@ -26,15 +26,17 @@ def handle(self, *args, **options): print(f"Deleting PipelineError records older than {days} days in batches...") - batch_size = 1000 + batch_size = 1000 # Adjust batch size as needed for optimal performance total_deleted = 0 + while True: old_records = PipelineError.objects.filter(created__lt=cutoff_date)[:batch_size] - if not old_records: + if not old_records.exists(): break - deleted_count = old_records.delete()[0] - total_deleted += deleted_count - print(f"Deleted {deleted_count} records in this batch. Total deleted: {total_deleted}.") + ids_to_delete = list(old_records.values_list('id', flat=True)) + PipelineError.objects.filter(id__in=ids_to_delete).delete() + total_deleted += len(ids_to_delete) + print(f"Deleted {len(ids_to_delete)} records in this batch. Total deleted: {total_deleted}.") print(f"Deletion complete. Total {total_deleted} PipelineError records older than {days} days deleted.") print('Done.') From 4a1331ee13dea3d5d432319006a6e3aa6909a53a Mon Sep 17 00:00:00 2001 From: muhammadali286 Date: Wed, 27 Nov 2024 14:24:30 +0500 Subject: [PATCH 4/4] :recycle: --- figures/management/commands/delete_old_pipeline_error.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/figures/management/commands/delete_old_pipeline_error.py b/figures/management/commands/delete_old_pipeline_error.py index 5f18107c..7c2ad0e2 100644 --- a/figures/management/commands/delete_old_pipeline_error.py +++ b/figures/management/commands/delete_old_pipeline_error.py @@ -26,7 +26,7 @@ def handle(self, *args, **options): print(f"Deleting PipelineError records older than {days} days in batches...") - batch_size = 1000 # Adjust batch size as needed for optimal performance + batch_size = 1000 total_deleted = 0 while True: