From 54c936f38e690972b079706a9dd1007bcf1be682 Mon Sep 17 00:00:00 2001
From: Toni <toni.nurmi@csc.fi>
Date: Mon, 15 Nov 2021 17:21:59 +0200
Subject: [PATCH] hotfix for too many database writes
---
src/metax_api/services/rabbitmq_service.py | 19 +++++++++----
src/metax_api/signals/post_delete.py | 33 +++++++++++-----------
src/metax_api/tests/models/signals.py | 10 -------
3 files changed, 30 insertions(+), 32 deletions(-)
diff --git a/src/metax_api/services/rabbitmq_service.py b/src/metax_api/services/rabbitmq_service.py
index 0e541693..bc2dbe4f 100755
--- a/src/metax_api/services/rabbitmq_service.py
+++ b/src/metax_api/services/rabbitmq_service.py
@@ -118,22 +118,29 @@ def consume_api_errors(self):
channel = connection.channel()
try:
+ errors = []
for method, _, body in channel.consume("metax-apierrors", inactivity_timeout=1):
if method is None and body is None:
channel.cancel()
break
try:
- error = loads(body)
- ApiError.objects.create(identifier=error["identifier"], error=error)
- except DatabaseError as e:
- _logger.error("cannot create API Error. Discarding..")
- _logger.debug(f"error: {e}")
+ error_payload = loads(body)
+ error = ApiError(identifier=error_payload["identifier"], error=error_payload)
+ errors.append(error)
+ except Exception as e:
+ _logger.error(e)
finally:
channel.basic_ack(method.delivery_tag)
+ try:
+ ApiError.objects.bulk_create(errors, batch_size=5000)
+ except DatabaseError as e:
+ _logger.error("cannot create API Error. Discarding..")
+ _logger.error(f"error: {e}")
+
except Exception as e:
_logger.error(e)
finally:
- _logger.debug("All ApiErrors were handled")
+ _logger.info("All ApiErrors were handled")
connection.close()
def init_exchanges(self):
diff --git a/src/metax_api/signals/post_delete.py b/src/metax_api/signals/post_delete.py
index 1ff39f75..3dd1a0f5 100644
--- a/src/metax_api/signals/post_delete.py
+++ b/src/metax_api/signals/post_delete.py
@@ -5,23 +5,24 @@
from django.dispatch import receiver
from django.core.serializers.json import DjangoJSONEncoder
from django.forms.models import model_to_dict
-from ..models import DeletedObject
+from ..models import DeletedObject, CatalogRecord, CatalogRecordV2
_logger = logging.getLogger(__name__)
@receiver(post_delete)
-def deleted_object_receiver(instance, *args, **kwargs):
- try:
- model_type = instance._meta.model.__name__
- if hasattr(instance, '_initial_data["date_created"]'):
- instance._initial_data["date_created"] = instance._initial_data["date_created"].strftime("%m/%d/%Y, %H:%M:%S")
- if hasattr(instance, 'date_created'):
- instance.date_created = instance.date_created.strftime("%m/%d/%Y, %H:%M:%S")
- if hasattr(instance, 'date_modified'):
- instance.date_modified = instance.date_modified.strftime("%m/%d/%Y, %H:%M:%S")
- instance = model_to_dict(instance)
- deleted_object_json = json.dumps(instance, cls=DjangoJSONEncoder)
- DeletedObject.objects.create(model_name=model_type, object_data=deleted_object_json)
- except Exception as e:
- _logger.error("cannot save Deleted Object. Discarding..")
- _logger.debug(f"error: {e}")
\ No newline at end of file
+def deleted_object_receiver(instance, sender, *args, **kwargs):
+ if sender in [CatalogRecord, CatalogRecordV2]:
+ try:
+ model_type = instance._meta.model.__name__
+ if hasattr(instance, '_initial_data["date_created"]'):
+ instance._initial_data["date_created"] = instance._initial_data["date_created"].strftime("%m/%d/%Y, %H:%M:%S")
+ if hasattr(instance, 'date_created'):
+ instance.date_created = instance.date_created.strftime("%m/%d/%Y, %H:%M:%S")
+ if hasattr(instance, 'date_modified'):
+ instance.date_modified = instance.date_modified.strftime("%m/%d/%Y, %H:%M:%S")
+ instance = model_to_dict(instance)
+ deleted_object_json = json.dumps(instance, cls=DjangoJSONEncoder)
+ DeletedObject.objects.create(model_name=model_type, object_data=deleted_object_json)
+ except Exception as e:
+ _logger.error("cannot save Deleted Object. Discarding..")
+ _logger.debug(f"error: {e}")
\ No newline at end of file
diff --git a/src/metax_api/tests/models/signals.py b/src/metax_api/tests/models/signals.py
index 548474ce..b9c11aae 100644
--- a/src/metax_api/tests/models/signals.py
+++ b/src/metax_api/tests/models/signals.py
@@ -33,14 +33,4 @@ def test_deleting_catalog_record_creates_new_deleted_object(self):
self.assertEqual(deleted_object_v2.model_name, "CatalogRecordV2")
self.assertEqual(deleted_object_v2.date_deleted.strftime("%d/%m/%Y"), self.today)
- def test_deleting_file_creates_new_deleted_object(self):
- File.objects.get(pk=1).delete(hard=True)
- deleted_object = DeletedObject.objects.last()
- self.assertEqual(deleted_object.model_name, "File")
- self.assertEqual(deleted_object.date_deleted.strftime("%d/%m/%Y"), self.today)
- def test_deleting_directory_creates_new_deleted_object(self):
- Directory.objects.get(pk=1).delete()
- deleted_object = DeletedObject.objects.last()
- self.assertEqual(deleted_object.model_name, "Directory")
- self.assertEqual(deleted_object.date_deleted.strftime("%d/%m/%Y"), self.today)
\ No newline at end of file