diff --git a/config/settings/base.py b/config/settings/base.py
index eb22f5e08..8bd30f7c3 100644
--- a/config/settings/base.py
+++ b/config/settings/base.py
@@ -837,6 +837,12 @@
# ------------------------------------------------------------------------------
INACTIVE_CONVERSATION_TIMEOUT_IN_MONTHS = env.int("INACTIVE_CONVERSATION_TIMEOUT_IN_MONTHS", 6)
+# Privacy timeouts
+# ------------------------------------------------------------------------------
+INACTIVE_USER_TIMEOUT_IN_MONTHS = env.int("INACTIVE_USER_TIMEOUT_IN_MONTHS", 3 * 12)
+INACTIVE_USER_WARNING_DELAY_IN_DAYS = env.int("INACTIVE_USER_WARNING_DELAY_IN_DAYS", 7)
+
+
# Wagtail
# ------------------------------------------------------------------------------
diff --git a/lemarche/conversations/models.py b/lemarche/conversations/models.py
index 282960831..5f030c68a 100644
--- a/lemarche/conversations/models.py
+++ b/lemarche/conversations/models.py
@@ -309,9 +309,6 @@ def get_template_id(self):
return self.brevo_id
return None
- def create_send_log(self, **kwargs):
- TemplateTransactionalSendLog.objects.create(template_transactional=self, **kwargs)
-
def send_transactional_email(
self,
recipient_email,
@@ -325,6 +322,9 @@ def send_transactional_email(
):
if self.is_active:
args = {
+ "template_transactional": self,
+ "recipient_content_object": recipient_content_object,
+ "parent_content_object": parent_content_object,
"template_id": self.get_template_id,
"recipient_email": recipient_email,
"recipient_name": recipient_name,
@@ -337,12 +337,6 @@ def send_transactional_email(
api_mailjet.send_transactional_email_with_template(**args)
elif self.source == conversation_constants.SOURCE_BREVO:
api_brevo.send_transactional_email_with_template(**args)
- # create log
- self.create_send_log(
- recipient_content_object=recipient_content_object,
- parent_content_object=parent_content_object,
- extra_data={"source": self.source, "args": args}, # "response": result()
- )
class TemplateTransactionalSendLog(models.Model):
diff --git a/lemarche/users/admin.py b/lemarche/users/admin.py
index 9f9f71343..4fe1f333c 100644
--- a/lemarche/users/admin.py
+++ b/lemarche/users/admin.py
@@ -6,7 +6,7 @@
from django.contrib.contenttypes.admin import GenericTabularInline
from django.db import models
from django.urls import reverse
-from django.utils.html import format_html, mark_safe
+from django.utils.html import format_html
from fieldsets_with_inlines import FieldsetsInlineMixin
from lemarche.conversations.models import TemplateTransactionalSendLog
@@ -126,6 +126,31 @@ def queryset(self, request, queryset):
return queryset
+class IsAnonymizedFilter(admin.SimpleListFilter):
+ """Custom admin filter to target users who are anonymized"""
+
+ title = "Est anonymisé"
+ parameter_name = "is_anonymized"
+
+ def lookups(self, request, model_admin):
+ return ("Yes", "Oui"), (None, "Non")
+
+ def queryset(self, request, queryset):
+ value = self.value()
+ if value == "Yes":
+ return queryset.filter(is_anonymized=True)
+ return queryset.filter(is_anonymized=False)
+
+ def choices(self, changelist):
+ """Removed the first yield from the base method to only have 2 choices, defaulting too No"""
+ for lookup, title in self.lookup_choices:
+ yield {
+ "selected": self.value() == lookup,
+ "query_string": changelist.get_query_string({self.parameter_name: lookup}),
+ "display": title,
+ }
+
+
class UserNoteInline(GenericTabularInline):
model = Note
fields = ["text", "author", "created_at", "updated_at"]
@@ -181,6 +206,7 @@ class UserAdmin(FieldsetsInlineMixin, UserAdmin):
HasApiKeyFilter,
"is_staff",
"is_superuser",
+ IsAnonymizedFilter,
]
search_fields = ["id", "email", "first_name", "last_name"]
search_help_text = "Cherche sur les champs : ID, E-mail, Prénom, Nom"
@@ -194,8 +220,6 @@ class UserAdmin(FieldsetsInlineMixin, UserAdmin):
"siae_count_annotated_with_link",
"tender_count_annotated_with_link",
"favorite_list_count_with_link",
- "image_url",
- "image_url_display",
"recipient_transactional_send_logs_count_with_link",
"brevo_contact_id",
"extra_data_display",
@@ -263,25 +287,6 @@ class UserAdmin(FieldsetsInlineMixin, UserAdmin):
"Permissions",
{"classes": ["collapse"], "fields": ("is_active", "is_staff", "is_superuser", "groups")},
),
- (
- "Données C4 Cocorico",
- {
- "classes": ["collapse"],
- "fields": (
- "c4_id",
- "c4_website",
- "c4_siret",
- "c4_naf",
- "c4_phone_prefix",
- "c4_time_zone",
- "c4_phone_verified",
- "c4_email_verified",
- "c4_id_card_verified",
- "image_url",
- "image_url_display",
- ),
- },
- ),
(
"Stats",
{
@@ -383,17 +388,6 @@ def favorite_list_count_with_link(self, user):
favorite_list_count_with_link.short_description = "Nombre de listes de favoris"
favorite_list_count_with_link.admin_order_field = "favorite_list_count"
- def image_url_display(self, user):
- if user.image_url:
- return mark_safe(
- f''
- f'
'
- f""
- )
- return mark_safe("-
")
-
- image_url_display.short_description = "Image"
-
def recipient_transactional_send_logs_count_with_link(self, obj):
url = reverse("admin:conversations_templatetransactionalsendlog_changelist") + f"?user__id__exact={obj.id}"
return format_html(f'{obj.recipient_transactional_send_logs.count()}')
diff --git a/lemarche/users/management/commands/anonymize_old_users.py b/lemarche/users/management/commands/anonymize_old_users.py
new file mode 100644
index 000000000..90f70bb41
--- /dev/null
+++ b/lemarche/users/management/commands/anonymize_old_users.py
@@ -0,0 +1,114 @@
+from dateutil.relativedelta import relativedelta
+from django.conf import settings
+from django.contrib.auth.hashers import UNUSABLE_PASSWORD_PREFIX
+from django.contrib.postgres.functions import RandomUUID
+from django.core.management.base import BaseCommand
+from django.db import transaction
+from django.db.models import F, Value
+from django.db.models.functions import Concat
+from django.template import defaulttags
+from django.utils import timezone
+
+from lemarche.conversations.models import TemplateTransactional
+from lemarche.siaes.models import SiaeUser
+from lemarche.users.models import User
+
+
+class DryRunException(Exception):
+ """To be raised in a dry run mode to abort current transaction"""
+
+ pass
+
+
+class Command(BaseCommand):
+ """Update and anonymize inactive users past a defined inactivity period"""
+
+ def add_arguments(self, parser):
+ parser.add_argument(
+ "--month_timeout",
+ type=int,
+ default=settings.INACTIVE_USER_TIMEOUT_IN_MONTHS,
+ help="Délai en mois à partir duquel les utilisateurs sont considérés inactifs",
+ )
+ parser.add_argument(
+ "--warning_delay",
+ type=int,
+ default=settings.INACTIVE_USER_WARNING_DELAY_IN_DAYS,
+ help="Délai en jours avant la date de suppression pour prevenir les utilisateurs",
+ )
+ parser.add_argument(
+ "--dry_run",
+ type=bool,
+ default=False,
+ help="La commande est exécutée sans que les modifications soient transmises à la base",
+ )
+
+ def handle(self, *args, **options):
+ expiry_date = timezone.now() - relativedelta(months=options["month_timeout"])
+ warning_date = expiry_date + relativedelta(days=options["warning_delay"])
+
+ try:
+ self.anonymize_old_users(expiry_date=expiry_date, dry_run=options["dry_run"])
+ except DryRunException:
+ self.stdout.write("Fin du dry_run d'anonymisation")
+
+ self.warn_users_by_email(expiry_date=expiry_date, warning_date=warning_date, dry_run=options["dry_run"])
+
+ @transaction.atomic
+ def anonymize_old_users(self, expiry_date: timezone.datetime, dry_run: bool):
+ """Update inactive users since x months and strip them from their personal data.
+ email is unique and not nullable, therefore it's replaced with the object id."""
+
+ qs = User.objects.filter(last_login__lte=expiry_date, is_anonymized=False)
+ users_to_update_count = qs.count()
+
+ qs.update(
+ is_active=False, # inactive users are allowed to log in standard login views
+ is_anonymized=True,
+ email=Concat(F("id"), Value("@domain.invalid")),
+ first_name="",
+ last_name="",
+ phone="",
+ api_key=None,
+ api_key_last_updated=None,
+ # https://docs.djangoproject.com/en/5.1/ref/contrib/auth/#django.contrib.auth.models.User.set_unusable_password
+ # Imitating the method but in sql. Prevent password reset attempt
+ # Random string is to avoid chances of impersonation by admins https://code.djangoproject.com/ticket/20079
+ password=Concat(Value(UNUSABLE_PASSWORD_PREFIX), RandomUUID()),
+ )
+ # remove anonymized users in Siaes
+ SiaeUser.objects.filter(user__is_anonymized=True).delete()
+
+ self.stdout.write(f"Utilisateurs anonymisés avec succès ({users_to_update_count} traités)")
+
+ if dry_run: # cancel transaction
+ raise DryRunException
+
+ @transaction.atomic
+ def warn_users_by_email(self, warning_date: timezone.datetime, expiry_date: timezone.datetime, dry_run: bool):
+ email_template = TemplateTransactional.objects.get(code="USER_ANONYMIZATION_WARNING")
+
+ # Users that have already received the mail are excluded
+ users_to_warn = User.objects.filter(last_login__lte=warning_date, is_active=True, is_anonymized=False).exclude(
+ recipient_transactional_send_logs__template_transactional__code=email_template.code,
+ recipient_transactional_send_logs__extra_data__contains={"sent": True},
+ )
+
+ if dry_run:
+ self.stdout.write(
+ f"Fin du dry_run d'avertissement des utilisateurs, {users_to_warn.count()} auraient été avertis"
+ )
+ return # exit before sending emails
+
+ for user in users_to_warn:
+ email_template.send_transactional_email(
+ recipient_email=user.email,
+ recipient_name=user.full_name,
+ variables={
+ "user_full_name": user.full_name,
+ "anonymization_date": defaulttags.date(expiry_date), # natural date
+ },
+ recipient_content_object=user,
+ )
+
+ self.stdout.write(f"Un email d'avertissement a été envoyé à {users_to_warn.count()} utilisateurs")
diff --git a/lemarche/users/migrations/0040_remove_user_c4_email_verified_remove_user_c4_id_and_more.py b/lemarche/users/migrations/0040_remove_user_c4_email_verified_remove_user_c4_id_and_more.py
new file mode 100644
index 000000000..978e745ee
--- /dev/null
+++ b/lemarche/users/migrations/0040_remove_user_c4_email_verified_remove_user_c4_id_and_more.py
@@ -0,0 +1,48 @@
+# Generated by Django 4.2.15 on 2024-12-23 17:02
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+ dependencies = [
+ ("users", "0039_remove_user_user_email_ci_uniqueness_and_more"),
+ ]
+
+ operations = [
+ migrations.RemoveField(
+ model_name="user",
+ name="c4_email_verified",
+ ),
+ migrations.RemoveField(
+ model_name="user",
+ name="c4_id",
+ ),
+ migrations.RemoveField(
+ model_name="user",
+ name="c4_id_card_verified",
+ ),
+ migrations.RemoveField(
+ model_name="user",
+ name="c4_naf",
+ ),
+ migrations.RemoveField(
+ model_name="user",
+ name="c4_phone_prefix",
+ ),
+ migrations.RemoveField(
+ model_name="user",
+ name="c4_phone_verified",
+ ),
+ migrations.RemoveField(
+ model_name="user",
+ name="c4_siret",
+ ),
+ migrations.RemoveField(
+ model_name="user",
+ name="c4_time_zone",
+ ),
+ migrations.RemoveField(
+ model_name="user",
+ name="c4_website",
+ ),
+ ]
diff --git a/lemarche/users/migrations/0041_remove_user_image_name_remove_user_image_url.py b/lemarche/users/migrations/0041_remove_user_image_name_remove_user_image_url.py
new file mode 100644
index 000000000..bbc9d223f
--- /dev/null
+++ b/lemarche/users/migrations/0041_remove_user_image_name_remove_user_image_url.py
@@ -0,0 +1,20 @@
+# Generated by Django 4.2.15 on 2024-12-24 09:04
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+ dependencies = [
+ ("users", "0040_remove_user_c4_email_verified_remove_user_c4_id_and_more"),
+ ]
+
+ operations = [
+ migrations.RemoveField(
+ model_name="user",
+ name="image_name",
+ ),
+ migrations.RemoveField(
+ model_name="user",
+ name="image_url",
+ ),
+ ]
diff --git a/lemarche/users/migrations/0042_email_template_anonymise_user.py b/lemarche/users/migrations/0042_email_template_anonymise_user.py
new file mode 100644
index 000000000..e0e3ccdce
--- /dev/null
+++ b/lemarche/users/migrations/0042_email_template_anonymise_user.py
@@ -0,0 +1,30 @@
+from django.db import migrations
+
+
+def create_template(apps, schema_editor):
+ TemplateTransactional = apps.get_model("conversations", "TemplateTransactional")
+ TemplateTransactional.objects.create(
+ name="Avertissement d'anonymisation de compte utilisateur",
+ code="USER_ANONYMIZATION_WARNING",
+ description="""
+ Bonjour {{ params.user_full_name }}, votre compte va être supprimé le {{ params.anonymization_date }}
+ si vous ne vous connectez pas avant.
+ Bonne journée,
+ L'équipe du marché de l'inclusion
+ """,
+ )
+
+
+def delete_template(apps, schema_editor):
+ TemplateTransactional = apps.get_model("conversations", "TemplateTransactional")
+ TemplateTransactional.objects.get(code="USER_ANONYMIZATION_WARNING").delete()
+
+
+class Migration(migrations.Migration):
+ dependencies = [
+ ("users", "0041_remove_user_image_name_remove_user_image_url"),
+ ]
+
+ operations = [
+ migrations.RunPython(create_template, reverse_code=delete_template),
+ ]
diff --git a/lemarche/users/migrations/0043_user_is_anonymized.py b/lemarche/users/migrations/0043_user_is_anonymized.py
new file mode 100644
index 000000000..6b083c0e9
--- /dev/null
+++ b/lemarche/users/migrations/0043_user_is_anonymized.py
@@ -0,0 +1,17 @@
+# Generated by Django 4.2.15 on 2024-12-26 09:23
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+ dependencies = [
+ ("users", "0042_email_template_anonymise_user"),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name="user",
+ name="is_anonymized",
+ field=models.BooleanField(default=False, verbose_name="L'utilisateur à été anonymisé"),
+ ),
+ ]
diff --git a/lemarche/users/models.py b/lemarche/users/models.py
index 804717f6d..0abd372a8 100644
--- a/lemarche/users/models.py
+++ b/lemarche/users/models.py
@@ -255,19 +255,6 @@ class User(AbstractUser):
default=False,
)
- image_name = models.CharField(verbose_name="Nom de l'image", max_length=255, blank=True)
- image_url = models.URLField(verbose_name="Lien vers l'image", max_length=500, blank=True)
-
- c4_id = models.IntegerField(blank=True, null=True)
- c4_phone_prefix = models.CharField(verbose_name="Indicatif international", max_length=20, blank=True)
- c4_time_zone = models.CharField(verbose_name="Fuseau", max_length=150, blank=True)
- c4_website = models.URLField(verbose_name="Site web", blank=True)
- c4_siret = models.CharField(verbose_name="Siret ou Siren", max_length=14, blank=True)
- c4_naf = models.CharField(verbose_name="Naf", max_length=5, blank=True)
- c4_phone_verified = models.BooleanField(default=False)
- c4_email_verified = models.BooleanField(default=False)
- c4_id_card_verified = models.BooleanField(default=False)
-
# services data
brevo_contact_id = models.PositiveIntegerField("Brevo contact id", blank=True, null=True)
@@ -291,6 +278,7 @@ class User(AbstractUser):
)
# is_active, is_staff, is_superuser
+ is_anonymized = models.BooleanField(verbose_name="L'utilisateur à été anonymisé", default=False)
# date_joined, last_login
created_at = models.DateTimeField(verbose_name="Date de création", default=timezone.now)
diff --git a/lemarche/users/tests.py b/lemarche/users/tests.py
index 335480fdd..d206aafc7 100644
--- a/lemarche/users/tests.py
+++ b/lemarche/users/tests.py
@@ -1,6 +1,17 @@
-from django.test import TestCase
+from datetime import datetime
+from io import StringIO
+from unittest.mock import patch
+
+from dateutil.relativedelta import relativedelta
+from django.core.management import call_command
+from django.core.validators import validate_email
+from django.db.models import F
+from django.test import TestCase, override_settings
+from django.utils import timezone
from lemarche.companies.factories import CompanyFactory
+from lemarche.conversations.constants import SOURCE_BREVO
+from lemarche.conversations.models import TemplateTransactional, TemplateTransactionalSendLog
from lemarche.favorites.factories import FavoriteListFactory
from lemarche.siaes.factories import SiaeFactory
from lemarche.tenders.factories import TenderFactory
@@ -157,3 +168,145 @@ def test_update_related_favorite_list_count_on_save(self):
user.favorite_lists.first().delete()
self.assertEqual(user.favorite_lists.count(), 1)
self.assertEqual(user.favorite_list_count, 1)
+
+
+# To avoid different results when test will be run in the future, we patch
+# and froze timezone.now used in the command
+# Settings are also overriden to avoid changing settings breaking tests
+@patch("django.utils.timezone.now", lambda: datetime(year=2024, month=1, day=1, tzinfo=timezone.utc))
+@override_settings(
+ INACTIVE_USER_TIMEOUT_IN_MONTHS=12,
+ INACTIVE_USER_WARNING_DELAY_IN_DAYS=7,
+)
+class UserAnonymizationTestCase(TestCase):
+ def setUp(self):
+ frozen_now = datetime(year=2024, month=1, day=1, tzinfo=timezone.utc)
+ self.frozen_last_year = frozen_now - relativedelta(years=1)
+ self.frozen_warning_date = self.frozen_last_year + relativedelta(days=7)
+
+ self.std_out = StringIO() # to read output from executed management commands
+
+ siae_1 = SiaeFactory()
+ siae_2 = SiaeFactory()
+
+ UserFactory(first_name="active_user", last_login=frozen_now)
+ UserFactory(
+ last_login=self.frozen_last_year,
+ # personal data
+ email="inactive_user_1@email.com",
+ first_name="inactive_user_1",
+ last_name="doe",
+ phone="06 15 15 15 15",
+ api_key="123456789",
+ api_key_last_updated=frozen_now,
+ siaes=[siae_1],
+ )
+ UserFactory(
+ last_login=self.frozen_last_year,
+ # personal data
+ email="inactive_user_2@email.com",
+ first_name="inactive_user_2",
+ last_name="doe",
+ phone="06 15 15 15 15",
+ api_key="0000000000",
+ api_key_last_updated=frozen_now,
+ siaes=[siae_1, siae_2],
+ )
+ UserFactory(
+ last_login=self.frozen_warning_date,
+ first_name="about_to_be_inactive",
+ )
+ # Set email as active to check if it's really sent
+ TemplateTransactional.objects.all().update(is_active=True, source=SOURCE_BREVO)
+
+ def test_set_inactive_user(self):
+ """Select users that last logged for more than a year and flag them as inactive"""
+ User.objects.filter(last_login__lte=self.frozen_last_year).update(
+ is_active=False, # inactive users should not be allowed to log in
+ email=F("id"),
+ first_name="",
+ last_name="",
+ phone="",
+ )
+ qs = User.objects.filter(last_login__lte=self.frozen_last_year)
+ self.assertQuerySetEqual(qs.order_by("id"), User.objects.filter(is_active=False).order_by("id"))
+
+ anonymized_user = User.objects.filter(is_active=False).first()
+ self.assertEqual(anonymized_user.email, f"{anonymized_user.id}")
+ self.assertFalse(anonymized_user.first_name)
+ self.assertFalse(anonymized_user.last_name)
+ self.assertFalse(anonymized_user.phone)
+
+ # ensure that no error is raised calling save() with a malformed user email
+ anonymized_user.email = "000"
+ anonymized_user.save()
+
+ def test_anonymize_command(self):
+ """Test the admin command 'anonymize_old_users'"""
+
+ call_command("anonymize_old_users", stdout=self.std_out)
+
+ self.assertEqual(User.objects.filter(is_active=False).count(), 2)
+
+ anonymized_user = User.objects.filter(is_active=False).first()
+
+ self.assertEqual(anonymized_user.email, f"{anonymized_user.id}@domain.invalid")
+ validate_email(anonymized_user.email)
+
+ self.assertTrue(anonymized_user.is_anonymized)
+
+ self.assertFalse(anonymized_user.first_name)
+ self.assertFalse(anonymized_user.last_name)
+ self.assertFalse(anonymized_user.phone)
+ self.assertFalse(anonymized_user.siaes.all())
+
+ self.assertIsNone(anonymized_user.api_key)
+ self.assertIsNone(anonymized_user.api_key_last_updated)
+
+ self.assertFalse(anonymized_user.has_usable_password())
+ # from UNUSABLE_PASSWORD_SUFFIX_LENGTH it should be 40, but we're pretty close
+ self.assertEqual(len(anonymized_user.password), 37)
+
+ self.assertIn("Utilisateurs anonymisés avec succès", self.std_out.getvalue())
+
+ def test_warn_command(self):
+ """Test the admin command 'anonymize_old_users' to check if users are warned by email
+ before their account is being removed"""
+
+ call_command("anonymize_old_users", stdout=self.std_out)
+
+ log_qs = TemplateTransactionalSendLog.objects.all()
+ self.assertEqual(
+ log_qs.count(),
+ 1,
+ )
+
+ # Called twice to veryfi that emails are not sent multiple times
+ call_command("anonymize_old_users", stdout=self.std_out)
+ log_qs = TemplateTransactionalSendLog.objects.all()
+ self.assertEqual(
+ log_qs.count(),
+ 1,
+ msg="Warning emails are sent multiple times !",
+ )
+
+ email_log = log_qs.first()
+ self.assertEqual(email_log.recipient_content_object, User.objects.get(first_name="about_to_be_inactive"))
+
+ def test_dryrun_anonymize_command(self):
+ """Ensure that the database is not modified after dryrun"""
+
+ original_qs_count = User.objects.filter(is_active=True).count()
+
+ call_command("anonymize_old_users", dry_run=True, stdout=self.std_out)
+
+ self.assertEqual(original_qs_count, User.objects.filter(is_active=True).count())
+
+ self.assertIn("Utilisateurs anonymisés avec succès (2 traités)", self.std_out.getvalue())
+
+ def test_dryrun_warn_command(self):
+ """Ensure that the database is not modified after dryrun and no email have been sent"""
+
+ call_command("anonymize_old_users", dry_run=True, stdout=self.std_out)
+
+ self.assertFalse(TemplateTransactionalSendLog.objects.all())
diff --git a/lemarche/utils/apis/api_brevo.py b/lemarche/utils/apis/api_brevo.py
index 39c399982..3e62734a2 100644
--- a/lemarche/utils/apis/api_brevo.py
+++ b/lemarche/utils/apis/api_brevo.py
@@ -7,6 +7,7 @@
from huey.contrib.djhuey import task
from sib_api_v3_sdk.rest import ApiException
+from lemarche.conversations.constants import SOURCE_BREVO
from lemarche.tenders import constants as tender_constants
from lemarche.utils.constants import EMAIL_SUBJECT_PREFIX
from lemarche.utils.data import sanitize_to_send_by_email
@@ -348,6 +349,9 @@ def get_all_users_from_list(
@task()
def send_transactional_email_with_template(
+ template_transactional,
+ recipient_content_object,
+ parent_content_object,
template_id: int,
recipient_email: str,
recipient_name: str,
@@ -368,12 +372,37 @@ def send_transactional_email_with_template(
if subject:
data["subject"] = EMAIL_SUBJECT_PREFIX + subject
+ log_args = {
+ "source": SOURCE_BREVO,
+ "args": {
+ "template_id": template_id,
+ "recipient_email": recipient_email,
+ "recipient_name": recipient_name,
+ "variables": variables,
+ "subject": subject,
+ "from_email": from_email,
+ "from_name": from_name,
+ },
+ }
+
+ # create log
+ template_transactional.send_logs.create(
+ recipient_content_object=recipient_content_object,
+ parent_content_object=parent_content_object,
+ extra_data={
+ **log_args,
+ "sent": settings.BITOUBI_ENV in ENV_NOT_ALLOWED, # considered successfully in tests and dev
+ },
+ )
+
if settings.BITOUBI_ENV not in ENV_NOT_ALLOWED:
try:
send_smtp_email = sib_api_v3_sdk.SendSmtpEmail(**data)
response = api_instance.send_transac_email(send_smtp_email)
logger.info("Brevo: send transactional email with template")
# {'message_id': '<202407151419.84958140835@smtp-relay.mailin.fr>', 'message_ids': None}
+ template_transactional.extra_data = {**log_args, "sent": True}
+ template_transactional.save(update_fields=["extra_data"])
return response.to_dict()
except ApiException as e:
print(f"Exception when calling SMTPApi->send_transac_email: {e}")
diff --git a/lemarche/utils/apis/api_mailjet.py b/lemarche/utils/apis/api_mailjet.py
index d1e95fd64..e9a5b8c1e 100644
--- a/lemarche/utils/apis/api_mailjet.py
+++ b/lemarche/utils/apis/api_mailjet.py
@@ -4,6 +4,7 @@
from django.conf import settings
from huey.contrib.djhuey import task
+from lemarche.conversations.constants import SOURCE_MAILJET
from lemarche.users import constants as user_constants
from lemarche.utils.constants import EMAIL_SUBJECT_PREFIX
@@ -95,6 +96,9 @@ def add_to_contact_list_async(email_address, properties, contact_list_id, client
@task()
def send_transactional_email_with_template(
+ template_transactional,
+ recipient_content_object,
+ parent_content_object,
template_id: int,
recipient_email: str,
recipient_name: str,
@@ -122,12 +126,37 @@ def send_transactional_email_with_template(
if not client:
client = get_default_client()
+ log_args = {
+ "source": SOURCE_MAILJET,
+ "args": {
+ "template_id": template_id,
+ "recipient_email": recipient_email,
+ "recipient_name": recipient_name,
+ "variables": variables,
+ "subject": subject,
+ "from_email": from_email,
+ "from_name": from_name,
+ },
+ }
+
+ # create log
+ template_transactional.send_logs.create(
+ recipient_content_object=recipient_content_object,
+ parent_content_object=parent_content_object,
+ extra_data={
+ **log_args,
+ "sent": settings.BITOUBI_ENV in ENV_NOT_ALLOWED, # considered successfully in tests and dev
+ },
+ )
+
if settings.BITOUBI_ENV not in ENV_NOT_ALLOWED:
try:
response = client.post(SEND_URL, json=data)
response.raise_for_status()
logger.info("Mailjet: send transactional email with template")
# {'Messages': [{'Status': 'success', 'CustomID': '', 'To': [{'Email': '', 'MessageUUID': '', 'MessageID': , 'MessageHref': 'https://api.mailjet.com/v3/REST/message/'}], 'Cc': [], 'Bcc': []}]} # noqa
+ template_transactional.extra_data = {**log_args, "sent": True}
+ template_transactional.save(update_fields=["extra_data"])
return response.json()
except requests.exceptions.HTTPError as e:
logger.error("Error while fetching `%s`: %s", e.request.url, e)
diff --git a/lemarche/utils/s3boto.py b/lemarche/utils/s3boto.py
deleted file mode 100644
index c42aadabc..000000000
--- a/lemarche/utils/s3boto.py
+++ /dev/null
@@ -1,520 +0,0 @@
-# copy from django-storage-1.8
-# replace force_text by force_str
-import io
-import mimetypes
-import os
-import warnings
-from datetime import datetime
-from gzip import GzipFile
-from tempfile import SpooledTemporaryFile
-
-from django.conf import settings as django_settings
-from django.core.exceptions import ImproperlyConfigured, SuspiciousOperation
-from django.core.files.base import File
-from django.core.files.storage import Storage
-from django.utils import timezone as tz
-from django.utils.deconstruct import deconstructible
-from django.utils.encoding import filepath_to_uri, force_bytes, force_str, smart_str
-from storages.utils import check_location, clean_name, get_available_overwrite_name, lookup_env, safe_join, setting
-
-
-try:
- from boto import __version__ as boto_version
- from boto.exception import S3ResponseError
- from boto.s3.connection import Location, S3Connection, SubdomainCallingFormat
- from boto.s3.key import Key as S3Key
- from boto.utils import ISO8601, parse_ts
-except ImportError:
- raise ImproperlyConfigured("Could not load Boto's S3 bindings.\n" "See https://github.com/boto/boto")
-
-
-boto_version_info = tuple([int(i) for i in boto_version.split("-")[0].split(".")])
-
-if boto_version_info[:2] < (2, 32):
- raise ImproperlyConfigured(
- "The installed Boto library must be 2.32 or " "higher.\nSee https://github.com/boto/boto"
- )
-
-warnings.warn(
- "The S3BotoStorage backend is deprecated in favor of the S3Boto3Storage backend "
- "and will be removed in django-storages 1.8. This backend is mostly in bugfix only "
- "mode and has been for quite a while (in much the same way as its underlying "
- "library 'boto'). For performance, security and new feature reasons it is _strongly_ "
- "recommended that you update to the S3Boto3Storage backend. Please see the migration docs "
- "https://django-storages.readthedocs.io/en/latest/backends/amazon-S3.html#migrating-boto-to-boto3.",
- DeprecationWarning,
-)
-
-
-@deconstructible
-class S3BotoStorageFile(File):
- """
- The default file object used by the S3BotoStorage backend.
- This file implements file streaming using boto's multipart
- uploading functionality. The file can be opened in read or
- write mode.
- This class extends Django's File class. However, the contained
- data is only the data contained in the current buffer. So you
- should not access the contained file object directly. You should
- access the data via this class.
- Warning: This file *must* be closed using the close() method in
- order to properly write the file to S3. Be sure to close the file
- in your application.
- """
-
- # TODO: Read/Write (rw) mode may be a bit undefined at the moment. Needs testing.
- # TODO: When Django drops support for Python 2.5, rewrite to use the
- # BufferedIO streams in the Python 2.6 io module.
- buffer_size = setting("AWS_S3_FILE_BUFFER_SIZE", 5242880)
-
- def __init__(self, name, mode, storage, buffer_size=None):
- self._storage = storage
- self.name = name[len(self._storage.location) :].lstrip("/") # noqa
- self._mode = mode
- self.key = storage.bucket.get_key(self._storage._encode_name(name))
- if not self.key and "w" in mode:
- self.key = storage.bucket.new_key(storage._encode_name(name))
- self._is_dirty = False
- self._file = None
- self._multipart = None
- # 5 MB is the minimum part size (if there is more than one part).
- # Amazon allows up to 10,000 parts. The default supports uploads
- # up to roughly 50 GB. Increase the part size to accommodate
- # for files larger than this.
- if buffer_size is not None:
- self.buffer_size = buffer_size
- self._write_counter = 0
-
- if not hasattr(django_settings, "AWS_DEFAULT_ACL"):
- warnings.warn(
- "The default behavior of S3BotoStorage is insecure. By default files "
- "and new buckets are saved with an ACL of 'public-read' (globally "
- "publicly readable). To change to using the bucket's default ACL "
- "set AWS_DEFAULT_ACL = None, otherwise to silence this warning "
- "explicitly set AWS_DEFAULT_ACL."
- )
-
- @property
- def size(self):
- return self.key.size
-
- def _get_file(self):
- if self._file is None:
- self._file = SpooledTemporaryFile(
- max_size=self._storage.max_memory_size,
- suffix=".S3BotoStorageFile",
- dir=setting("FILE_UPLOAD_TEMP_DIR"),
- )
- if "r" in self._mode:
- self._is_dirty = False
- self.key.get_contents_to_file(self._file)
- self._file.seek(0)
- if self._storage.gzip and self.key.content_encoding == "gzip":
- self._file = GzipFile(mode=self._mode, fileobj=self._file)
- return self._file
-
- def _set_file(self, value):
- self._file = value
-
- file = property(_get_file, _set_file)
-
- def read(self, *args, **kwargs):
- if "r" not in self._mode:
- raise AttributeError("File was not opened in read mode.")
- return super(S3BotoStorageFile, self).read(*args, **kwargs)
-
- def write(self, content, *args, **kwargs):
- if "w" not in self._mode:
- raise AttributeError("File was not opened in write mode.")
- self._is_dirty = True
- if self._multipart is None:
- provider = self.key.bucket.connection.provider
- upload_headers = {}
- if self._storage.default_acl:
- upload_headers[provider.acl_header] = self._storage.default_acl
- upload_headers.update(
- {"Content-Type": mimetypes.guess_type(self.key.name)[0] or self._storage.key_class.DefaultContentType}
- )
- upload_headers.update(self._storage.headers)
- self._multipart = self._storage.bucket.initiate_multipart_upload(
- self.key.name,
- headers=upload_headers,
- reduced_redundancy=self._storage.reduced_redundancy,
- encrypt_key=self._storage.encryption,
- )
- if self.buffer_size <= self._buffer_file_size:
- self._flush_write_buffer()
- return super(S3BotoStorageFile, self).write(force_bytes(content), *args, **kwargs)
-
- @property
- def _buffer_file_size(self):
- pos = self.file.tell()
- self.file.seek(0, os.SEEK_END)
- length = self.file.tell()
- self.file.seek(pos)
- return length
-
- def _flush_write_buffer(self):
- if self._buffer_file_size:
- self._write_counter += 1
- self.file.seek(0)
- headers = self._storage.headers.copy()
- self._multipart.upload_part_from_file(self.file, self._write_counter, headers=headers)
- self.file.seek(0)
- self.file.truncate()
-
- def close(self):
- if self._is_dirty:
- self._flush_write_buffer()
- self._multipart.complete_upload()
- else:
- if self._multipart is not None:
- self._multipart.cancel_upload()
- self.key.close()
- if self._file is not None:
- self._file.close()
- self._file = None
-
-
-@deconstructible
-class S3BotoStorage(Storage):
- """
- Amazon Simple Storage Service using Boto
- This storage backend supports opening files in read or write
- mode and supports streaming(buffering) data in chunks to S3
- when writing.
- """
-
- connection_class = S3Connection
- connection_response_error = S3ResponseError
- file_class = S3BotoStorageFile
- key_class = S3Key
- # used for looking up the access and secret key from env vars
- access_key_names = ["S3_STORAGE_ACCESS_KEY_ID", "AWS_ACCESS_KEY_ID"]
- secret_key_names = ["S3_STORAGE_SECRET_ACCESS_KEY", "AWS_SECRET_ACCESS_KEY"]
- security_token_names = ["AWS_SESSION_TOKEN", "AWS_SECURITY_TOKEN"]
- security_token = None
-
- access_key = setting("S3_STORAGE_ACCESS_KEY_ID", setting("AWS_ACCESS_KEY_ID"))
- secret_key = setting("S3_STORAGE_SECRET_ACCESS_KEY", setting("AWS_SECRET_ACCESS_KEY"))
- file_overwrite = setting("AWS_S3_FILE_OVERWRITE", True)
- headers = setting("AWS_HEADERS", {})
- bucket_name = setting("S3_STORAGE_BUCKET_NAME")
- auto_create_bucket = setting("AWS_AUTO_CREATE_BUCKET", False)
- default_acl = setting("AWS_DEFAULT_ACL", "public-read")
- bucket_acl = setting("AWS_BUCKET_ACL", default_acl)
- querystring_auth = setting("AWS_QUERYSTRING_AUTH", True)
- querystring_expire = setting("AWS_QUERYSTRING_EXPIRE", 3600)
- reduced_redundancy = setting("AWS_REDUCED_REDUNDANCY", False)
- location = setting("AWS_LOCATION", "")
- origin = setting("AWS_ORIGIN", Location.DEFAULT)
- encryption = setting("AWS_S3_ENCRYPTION", False)
- custom_domain = setting("AWS_S3_CUSTOM_DOMAIN")
- calling_format = setting("AWS_S3_CALLING_FORMAT", SubdomainCallingFormat())
- secure_urls = setting("AWS_S3_SECURE_URLS", True)
- file_name_charset = setting("AWS_S3_FILE_NAME_CHARSET", "utf-8")
- gzip = setting("AWS_IS_GZIPPED", False)
- preload_metadata = setting("AWS_PRELOAD_METADATA", False)
- gzip_content_types = setting(
- "GZIP_CONTENT_TYPES",
- (
- "text/css",
- "text/javascript",
- "application/javascript",
- "application/x-javascript",
- "image/svg+xml",
- ),
- )
- url_protocol = setting("AWS_S3_URL_PROTOCOL", "https:")
- host = setting("S3_STORAGE_ENDPOINT_DOMAIN", S3Connection.DefaultHost)
- use_ssl = setting("AWS_S3_USE_SSL", True)
- port = setting("AWS_S3_PORT")
- proxy = setting("AWS_S3_PROXY_HOST")
- proxy_port = setting("AWS_S3_PROXY_PORT")
- max_memory_size = setting("AWS_S3_MAX_MEMORY_SIZE", 0)
-
- def __init__(self, acl=None, bucket=None, **settings):
- # check if some of the settings we've provided as class attributes
- # need to be overwritten with values passed in here
- for name, value in settings.items():
- if hasattr(self, name):
- setattr(self, name, value)
-
- # For backward-compatibility of old differing parameter names
- if acl is not None:
- self.default_acl = acl
- if bucket is not None:
- self.bucket_name = bucket
-
- check_location(self)
-
- # Backward-compatibility: given the anteriority of the SECURE_URL setting
- # we fall back to https if specified in order to avoid the construction
- # of unsecure urls.
- if self.secure_urls:
- self.url_protocol = "https:"
-
- self._entries = {}
- self._bucket = None
- self._connection = None
- self._loaded_meta = False
-
- self.access_key, self.secret_key = self._get_access_keys()
- self.security_token = self._get_security_token()
-
- @property
- def connection(self):
- if self._connection is None:
- kwargs = self._get_connection_kwargs()
-
- self._connection = self.connection_class(self.access_key, self.secret_key, **kwargs)
- return self._connection
-
- def _get_connection_kwargs(self):
- return dict(
- security_token=self.security_token,
- is_secure=self.use_ssl,
- calling_format=self.calling_format,
- host=self.host,
- port=self.port,
- proxy=self.proxy,
- proxy_port=self.proxy_port,
- )
-
- @property
- def bucket(self):
- """
- Get the current bucket. If there is no current bucket object
- create it.
- """
- if self._bucket is None:
- self._bucket = self._get_or_create_bucket(self.bucket_name)
- return self._bucket
-
- @property
- def entries(self):
- """
- Get the locally cached files for the bucket.
- """
- if self.preload_metadata and not self._loaded_meta:
- self._entries.update(
- {self._decode_name(entry.key): entry for entry in self.bucket.list(prefix=self.location)}
- )
- self._loaded_meta = True
- return self._entries
-
- def _get_access_keys(self):
- """
- Gets the access keys to use when accessing S3. If none is
- provided in the settings then get them from the environment
- variables.
- """
- access_key = self.access_key or lookup_env(S3BotoStorage.access_key_names)
- secret_key = self.secret_key or lookup_env(S3BotoStorage.secret_key_names)
- return access_key, secret_key
-
- def _get_security_token(self):
- """
- Gets the security token to use when accessing S3. Get it from
- the environment variables.
- """
- security_token = self.security_token or lookup_env(S3BotoStorage.security_token_names)
- return security_token
-
- def _get_or_create_bucket(self, name):
- """
- Retrieves a bucket if it exists, otherwise creates it.
- """
- try:
- return self.connection.get_bucket(name, validate=self.auto_create_bucket)
- except self.connection_response_error:
- if self.auto_create_bucket:
- bucket = self.connection.create_bucket(name, location=self.origin)
- if not hasattr(django_settings, "AWS_BUCKET_ACL"):
- warnings.warn(
- "The default behavior of S3BotoStorage is insecure. By default new buckets "
- "are saved with an ACL of 'public-read' (globally publicly readable). To change "
- "to using Amazon's default of the bucket owner set AWS_DEFAULT_ACL = None, "
- "otherwise to silence this warning explicitly set AWS_DEFAULT_ACL."
- )
- if self.bucket_acl:
- bucket.set_acl(self.bucket_acl)
- return bucket
- raise ImproperlyConfigured(
- "Bucket %s does not exist. Buckets "
- "can be automatically created by "
- "setting AWS_AUTO_CREATE_BUCKET to "
- "``True``." % name
- )
-
- def _clean_name(self, name):
- """
- Cleans the name so that Windows style paths work
- """
- return clean_name(name)
-
- def _normalize_name(self, name):
- """
- Normalizes the name so that paths like /path/to/ignored/../something.txt
- work. We check to make sure that the path pointed to is not outside
- the directory specified by the LOCATION setting.
- """
- try:
- return safe_join(self.location, name)
- except ValueError:
- raise SuspiciousOperation("Attempted access to '%s' denied." % name)
-
- def _encode_name(self, name):
- return smart_str(name, encoding=self.file_name_charset)
-
- def _decode_name(self, name):
- return force_str(name, encoding=self.file_name_charset)
-
- def _compress_content(self, content):
- """Gzip a given string content."""
- zbuf = io.BytesIO()
- # The GZIP header has a modification time attribute (see http://www.zlib.org/rfc-gzip.html)
- # This means each time a file is compressed it changes even if the other contents don't change
- # For S3 this defeats detection of changes using MD5 sums on gzipped files
- # Fixing the mtime at 0.0 at compression time avoids this problem
- zfile = GzipFile(mode="wb", fileobj=zbuf, mtime=0.0)
- try:
- zfile.write(force_bytes(content.read()))
- finally:
- zfile.close()
- zbuf.seek(0)
- content.file = zbuf
- content.seek(0)
- return content
-
- def _open(self, name, mode="rb"):
- name = self._normalize_name(self._clean_name(name))
- f = self.file_class(name, mode, self)
- if not f.key:
- raise IOError("File does not exist: %s" % name)
- return f
-
- def _save(self, name, content):
- cleaned_name = self._clean_name(name)
- name = self._normalize_name(cleaned_name)
- headers = self.headers.copy()
- _type, encoding = mimetypes.guess_type(name)
- content_type = getattr(content, "content_type", None)
- content_type = content_type or _type or self.key_class.DefaultContentType
-
- # setting the content_type in the key object is not enough.
- headers.update({"Content-Type": content_type})
-
- if self.gzip and content_type in self.gzip_content_types:
- content = self._compress_content(content)
- headers.update({"Content-Encoding": "gzip"})
- elif encoding:
- # If the content already has a particular encoding, set it
- headers.update({"Content-Encoding": encoding})
-
- content.name = cleaned_name
- encoded_name = self._encode_name(name)
- key = self.bucket.get_key(encoded_name)
- if not key:
- key = self.bucket.new_key(encoded_name)
- if self.preload_metadata:
- self._entries[encoded_name] = key
- key.last_modified = datetime.utcnow().strftime(ISO8601)
-
- key.set_metadata("Content-Type", content_type)
- self._save_content(key, content, headers=headers)
- return cleaned_name
-
- def _save_content(self, key, content, headers):
- # only pass backwards incompatible arguments if they vary from the default
- kwargs = {}
- if self.encryption:
- kwargs["encrypt_key"] = self.encryption
- key.set_contents_from_file(
- content,
- headers=headers,
- policy=self.default_acl,
- reduced_redundancy=self.reduced_redundancy,
- rewind=True,
- **kwargs,
- )
-
- def _get_key(self, name):
- name = self._normalize_name(self._clean_name(name))
- if self.entries:
- return self.entries.get(name)
- return self.bucket.get_key(self._encode_name(name))
-
- def delete(self, name):
- name = self._normalize_name(self._clean_name(name))
- self.bucket.delete_key(self._encode_name(name))
-
- def exists(self, name):
- if not name: # root element aka the bucket
- try:
- self.bucket
- return True
- except ImproperlyConfigured:
- return False
-
- return self._get_key(name) is not None
-
- def listdir(self, name):
- name = self._normalize_name(self._clean_name(name))
- # for the bucket.list and logic below name needs to end in /
- # But for the root path "" we leave it as an empty string
- if name and not name.endswith("/"):
- name += "/"
-
- dirlist = self.bucket.list(self._encode_name(name))
- files = []
- dirs = set()
- base_parts = name.split("/")[:-1]
- for item in dirlist:
- parts = item.name.split("/")
- parts = parts[len(base_parts) :] # noqa
- if len(parts) == 1:
- # File
- files.append(parts[0])
- elif len(parts) > 1:
- # Directory
- dirs.add(parts[0])
- return list(dirs), files
-
- def size(self, name):
- return self._get_key(name).size
-
- def get_modified_time(self, name):
- dt = tz.make_aware(parse_ts(self._get_key(name).last_modified), tz.utc)
- return dt if setting("USE_TZ") else tz.make_naive(dt)
-
- def modified_time(self, name):
- dt = tz.make_aware(parse_ts(self._get_key(name).last_modified), tz.utc)
- return tz.make_naive(dt)
-
- def url(self, name, headers=None, response_headers=None, expire=None):
- # Preserve the trailing slash after normalizing the path.
- name = self._normalize_name(self._clean_name(name))
- if self.custom_domain:
- return "%s//%s/%s" % (self.url_protocol, self.custom_domain, filepath_to_uri(name))
-
- if expire is None:
- expire = self.querystring_expire
-
- return self.connection.generate_url(
- expire,
- method="GET",
- bucket=self.bucket.name,
- key=self._encode_name(name),
- headers=headers,
- query_auth=self.querystring_auth,
- force_http=not self.secure_urls,
- response_headers=response_headers,
- )
-
- def get_available_name(self, name, max_length=None):
- """Overwrite existing file with the same name."""
- name = self._clean_name(name)
- if self.file_overwrite:
- return get_available_overwrite_name(name, max_length)
- return super(S3BotoStorage, self).get_available_name(name, max_length)