Revert "feat(api): scheduled stamp data dump"

api/ceramic_cache/api.py

@@ -150,20 +150,17 @@ def cache_stamps(request, payload: List[CacheStampPayload]):
 
         address = get_address_from_did(request.did)
         stamp_objects = []
-        now = get_utc_time()
         for p in payload:
             stamp_object = CeramicCache(
                 address=address,
                 provider=p.provider,
                 stamp=p.stamp,
-                updated_at=now,
             )
             stamp_objects.append(stamp_object)
-
         created = CeramicCache.objects.bulk_create(
             stamp_objects,
             update_conflicts=True,
-            update_fields=["stamp", "updated_at"],
+            update_fields=["stamp"],
             unique_fields=["address", "provider"],
         )
 
@@ -186,15 +183,13 @@ def patch_stamps(request, payload: List[CacheStampPayload]):
         stamp_objects = []
         providers_to_delete = []
         updated = []
-        now = get_utc_time()
 
         for p in payload:
             if p.stamp:
                 stamp_object = CeramicCache(
                     address=address,
                     provider=p.provider,
                     stamp=p.stamp,
-                    updated_at=now,
                 )
                 stamp_objects.append(stamp_object)
             else:
@@ -204,7 +199,7 @@ def patch_stamps(request, payload: List[CacheStampPayload]):
             updated = CeramicCache.objects.bulk_create(
                 stamp_objects,
                 update_conflicts=True,
-                update_fields=["stamp", "updated_at"],
+                update_fields=["stamp"],
                 unique_fields=["address", "provider"],
             )
 
@@ -263,7 +258,6 @@ def cache_stamp(request, payload: CacheStampPayload):
             provider=payload.provider,
             defaults=dict(
                 stamp=payload.stamp,
-                updated_at=get_utc_time(),
             ),
         )
 

api/ceramic_cache/management/commands/dump_stamp_data.py

@@ -8,7 +8,6 @@
 from django.core.management.base import BaseCommand
 from django.core.paginator import Paginator
 from django.utils import timezone
-from tqdm import tqdm
 
 s3 = boto3.client(
     "s3",
@@ -27,57 +26,25 @@ def handle(self, *args, **options):
 
         if not latest_export:
             print("No previous exports found. Exporting all data.")
-            latest_export = StampExports(
-                last_export_ts=datetime.date.fromisoformat("1970-01-01")
+            latest_export = StampExports.objects.create(
+                last_export_ts=timezone.now() - datetime.timedelta(days=7)
             )
 
-        print(f"Getting Stamps updated since {latest_export.last_export_ts}")
-
-        query = (
-            CeramicCache.objects.values("stamp", "updated_at")
-            .order_by("updated_at")
-            .using("read_replica_0")
+        paginator = Paginator(
+            CeramicCache.objects.filter(
+                created_at__gt=latest_export.last_export_ts
+            ).values_list("stamp", flat=True),
+            1000,
         )
 
         # Generate the dump file name
         file_name = f'stamps_{latest_export.last_export_ts.strftime("%Y%m%d_%H%M%S")}_{timezone.now().strftime("%Y%m%d_%H%M%S")}.jsonl'
 
-        last_updated_at = latest_export.last_export_ts
-        chunk_size = 1000
-
-        try:
-            # Write serialized data to the file
-            with open(file_name, "w") as f:
-                with tqdm(
-                    unit="items", unit_scale=None, desc="Exporting stamps"
-                ) as progress_bar:
-                    has_more = True
-                    while has_more:
-                        objects = list(
-                            query.filter(updated_at__gt=last_updated_at)[:chunk_size]
-                        )
-                        if objects:
-                            num_objects = len(objects)
-                            progress_bar.update(num_objects)
-
-                            for cache_obj in objects:
-                                f.write(
-                                    json.dumps({"stamp": cache_obj["stamp"]}) + "\n"
-                                )
-
-                            last_updated_at = cache_obj["updated_at"]
-
-                            # If we get less than the chunk size, we've reached the end
-                            # No need to keep querying which could result in querying forever
-                            if num_objects < chunk_size:
-                                has_more = False
-                        else:
-                            has_more = False
-
-        finally:
-            self.stdout.write(
-                self.style.SUCCESS(f'Last stamp updated at "{last_updated_at}"')
-            )
+        # Write serialized data to the file
+        with open(file_name, "w") as f:
+            for page in paginator.page_range:
+                for stamp in paginator.page(page).object_list:
+                    f.write(json.dumps({"stamp": stamp}) + "\n")
 
         # Upload to S3 bucket
         s3.upload_file(file_name, settings.S3_WEEKLY_BACKUP_BUCKET_NAME, file_name)
@@ -86,7 +53,7 @@ def handle(self, *args, **options):
         os.remove(file_name)
 
         StampExports.objects.create(
-            last_export_ts=last_updated_at, stamp_total=progress_bar.n
+            last_export_ts=timezone.now(), stamp_total=paginator.count
         )
 
         print(f"Data dump completed and uploaded to S3 as {file_name}")

api/ceramic_cache/models.py

@@ -14,16 +14,7 @@ class CeramicCache(models.Model):
         auto_now_add=True,
         blank=True,
         null=True,
-        help_text="This is the timestamp that this DB record was created (it is not necessarily the stamp issuance timestamp)",
-    )
-
-    # NOTE! auto_now is here to make tests easier, but it is not
-    # supported for bulk updates so it should be set explicitly
-    updated_at = models.DateTimeField(
-        blank=False,
-        null=False,
-        auto_now=True,
-        help_text="This is the timestamp that this DB record was updated (it is not necessarily the stamp issuance timestamp)",
+        help_text="This is the timestamp that this DB record was created (it is not necesarily the stamp issuance timestamp)",
     )
 
     class Meta: