Merge remote-tracking branch 'origin/master' into release

.github/workflows/release_drafter.yml

@@ -4,7 +4,7 @@ on:
   push:
     # branches to consider in the event; optional, defaults to all
     branches:
-      - master
+      - release
 
 jobs:
   update_release_draft:

.github/workflows/test.yml

@@ -1,8 +1,8 @@
 name: Test
 
 on:
-  - push
-  - pull_request
+  push:
+  pull_request:
 
 jobs:
   check_format:

.gitignore

@@ -1,5 +1,6 @@
 __pycache__/
 *.log
+*.orig
 .htmlcov/
 .coverage
 /conf/supervisord.conf
@@ -8,4 +9,5 @@ __pycache__/
 docker-compose.override.yml
 client/projects
 conf/nginx/certs/*
+conf/certbot/*
 Pipfile*

README.md

@@ -209,12 +209,14 @@ Run the django database migrations
     docker compose exec app python manage.py migrate
 
 
-## Create a certificate using Let's Encrypt
+## Create or renew a certificate using Let's Encrypt
 
 If you are running the server on a server with a public domain, you can install Let's Encrypt certificate by running the following command:
 
     ./scripts/init_letsencrypt.sh
 
+The same command can also be used to update an expired certificate.
+
 Note you may want to change the `LETSENCRYPT_EMAIL`, `LETSENCRYPT_RSA_KEY_SIZE` and `LETSENCRYPT_STAGING` variables.
 
 ### Infrastructure

docker-app/qfieldcloud/core/admin.py

@@ -16,6 +16,7 @@
 from django.contrib.admin.templatetags.admin_urls import admin_urlname
 from django.contrib.auth.models import Group
 from django.core.exceptions import PermissionDenied
+from django.db.models import Q
 from django.db.models.fields.json import JSONField
 from django.db.models.functions import Lower
 from django.forms import ModelForm, fields, widgets
@@ -48,6 +49,7 @@
     User,
     UserAccount,
 )
+from qfieldcloud.core.templatetags.filters import filesizeformat10
 from qfieldcloud.core.utils2 import jobs
 from rest_framework.authtoken.models import TokenProxy
 
@@ -368,6 +370,7 @@ class PersonAdmin(admin.ModelAdmin):
         "is_active",
         "date_joined",
         "last_login",
+        "storage_usage__field",
     )
     list_filter = (
         "type",
@@ -397,6 +400,7 @@ class PersonAdmin(admin.ModelAdmin):
     readonly_fields = (
         "date_joined",
         "last_login",
+        "storage_usage__field",
     )
 
     inlines = (
@@ -407,6 +411,13 @@ class PersonAdmin(admin.ModelAdmin):
     add_form_template = "admin/change_form.html"
     change_form_template = "admin/person_change_form.html"
 
+    @admin.display(description=_("Storage"))
+    def storage_usage__field(self, instance) -> str:
+        used_storage = filesizeformat10(instance.useraccount.storage_used_bytes)
+        free_storage = filesizeformat10(instance.useraccount.storage_free_bytes)
+        used_storage_perc = instance.useraccount.storage_used_ratio * 100
+        return f"{used_storage} {free_storage} ({used_storage_perc:.2f}%)"
+
     def save_model(self, request, obj, form, change):
         # Set the password to the value in the field if it's changed.
         if obj.pk:
@@ -612,6 +623,24 @@ def has_delete_permission(self, request, obj):
     #     return format_pre_json(instance.feedback)
 
 
+class IsFinalizedJobFilter(admin.SimpleListFilter):
+    title = _("finalized job")
+    parameter_name = "finalized"
+
+    def lookups(self, request, model_admin):
+        return (
+            ("finalized", _("finalized")),
+            ("not finalized", _("not finalized")),
+        )
+
+    def queryset(self, request, queryset):
+        q = Q(status="pending") | Q(status="started") | Q(status="queued")
+        if self.value() == "not finalized":
+            return queryset.filter(q)
+        else:
+            return queryset.filter(~q)
+
+
 class JobAdmin(admin.ModelAdmin):
     list_display = (
         "id",
@@ -623,7 +652,7 @@ class JobAdmin(admin.ModelAdmin):
         "created_at",
         "updated_at",
     )
-    list_filter = ("type", "status", "updated_at")
+    list_filter = ("type", "status", "updated_at", IsFinalizedJobFilter)
     list_select_related = ("project", "project__owner", "created_by")
     exclude = ("feedback", "output")
     ordering = ("-updated_at",)
@@ -727,6 +756,24 @@ def has_delete_permission(self, request, obj):
         return False
 
 
+class IsFinalizedDeltaJobFilter(admin.SimpleListFilter):
+    title = _("finalized delta job")
+    parameter_name = "finalized"
+
+    def lookups(self, request, model_admin):
+        return (
+            ("finalized", _("finalized")),
+            ("not finalized", _("not finalized")),
+        )
+
+    def queryset(self, request, queryset):
+        q = Q(last_status="pending") | Q(last_status="started")
+        if self.value() == "not finalized":
+            return queryset.filter(q)
+        else:
+            return queryset.filter(~q)
+
+
 class DeltaAdmin(admin.ModelAdmin):
     list_display = (
         "id",
@@ -738,7 +785,7 @@ class DeltaAdmin(admin.ModelAdmin):
         "created_at",
         "updated_at",
     )
-    list_filter = ("last_status", "updated_at")
+    list_filter = ("last_status", "updated_at", IsFinalizedDeltaJobFilter)
 
     actions = (
         "set_status_pending",
@@ -923,6 +970,7 @@ class OrganizationAdmin(admin.ModelAdmin):
         "email",
         "organization_owner__link",
         "date_joined",
+        "storage_usage__field",
     )
 
     search_fields = (
@@ -932,7 +980,10 @@ class OrganizationAdmin(admin.ModelAdmin):
         "organization_owner__email__iexact",
     )
 
-    readonly_fields = ("date_joined",)
+    readonly_fields = (
+        "date_joined",
+        "storage_usage__field",
+    )
 
     list_select_related = ("organization_owner",)
 
@@ -946,6 +997,13 @@ def organization_owner__link(self, instance):
             instance.organization_owner, instance.organization_owner.username
         )
 
+    @admin.display(description=_("Storage"))
+    def storage_usage__field(self, instance) -> str:
+        used_storage = filesizeformat10(instance.useraccount.storage_used_bytes)
+        free_storage = filesizeformat10(instance.useraccount.storage_free_bytes)
+        used_storage_perc = instance.useraccount.storage_used_ratio * 100
+        return f"{used_storage} {free_storage} ({used_storage_perc:.2f}%)"
+
     def get_search_results(self, request, queryset, search_term):
         filters = search_parser(
             request,

docker-app/qfieldcloud/core/templates/admin/project_files_widget.html

@@ -169,6 +169,16 @@
                     .then((files) => refreshTableContents({ files }))
                     .catch((error) => refreshTableContents({ error }));
             };
+            const filesize10 = n => {
+                switch(true) {
+                    case n < 10 ** 6:
+                        return (n / 10 ** 3).toFixed(3) + " KB";
+                    case n < 10 ** 9:
+                        return (n / 10 ** 6).toFixed(3) + " MB";
+                    default:
+                        return (n / 10 ** 9).toFixed(3) + " GB";
+                }
+            };
             const refreshTableContents = ({ error, isLoading = false, files = [] }) => {
                 $tbody.innerHTML = '';
                 $count.innerHTML = '';
@@ -196,7 +206,7 @@
 
                         $trow.querySelector('td:nth-child(1)').innerHTML = file.name;
                         $trow.querySelector('td:nth-child(2)').innerHTML = file.last_modified;
-                        $trow.querySelector('td:nth-child(3)').innerHTML = file.size;
+                        $trow.querySelector('td:nth-child(3)').innerHTML = `<span title="${file.size} bytes">${filesize10(file.size)} KB</span>`;
 
                         for (const version of file.versions) {
                             const $option = document.createElement('option');
@@ -220,9 +230,11 @@
                         });
 
                         $downloadBtn.addEventListener('click', () => {
-                            window.open(buildApiUrl(`files/${projectId}/${file.name}/`, {
+                            const pathToFile = `files/${projectId}/${file.name}/`
+                            const buildApiUrlWithPath = () => buildApiUrl(pathToFile, {
                                 version: $versionsSelect.value,
-                            }));
+                            });
+                            window.open($versionsSelect.value ? buildApiUrlWithPath() : pathToFile);
                         });
 
                         $deleteBtn.addEventListener('click', () => {

docker-app/qfieldcloud/core/templatetags/filters.py

@@ -0,0 +1,52 @@
+from django import template
+from django.utils import formats
+from django.utils.html import avoid_wrapping
+from django.utils.translation import gettext as _
+from django.utils.translation import ngettext
+
+register = template.Library()
+
+
+@register.filter(is_safe=True)
+def filesizeformat10(bytes_) -> str:
+    """
+    Format the value like a 'human-readable' file size (i.e. 13 KB, 4.1 MB,
+    102 bytes, etc.).
+
+    Unlike Django's `filesizeformat` which uses powers of 2 (e.g. 1024KB==1MB), `filesizeformat10` uses powers of 10 (e.g. 1000KB==1MB)
+    """
+    try:
+        bytes_ = int(bytes_)
+    except (TypeError, ValueError, UnicodeDecodeError):
+        value = ngettext("%(size)d byte", "%(size)d bytes", 0) % {"size": 0}
+        return avoid_wrapping(value)
+
+    def filesize_number_format(value):
+        return formats.number_format(round(value, 1), 1)
+
+    KB = 10**3
+    MB = 10**6
+    GB = 10**9
+    TB = 10**12
+    PB = 10**15
+
+    negative = bytes_ < 0
+    if negative:
+        bytes_ = -bytes_  # Allow formatting of negative numbers.
+
+    if bytes_ < KB:
+        value = ngettext("%(size)d byte", "%(size)d bytes", bytes_) % {"size": bytes_}
+    elif bytes_ < MB:
+        value = _("%s KB") % filesize_number_format(bytes_ / KB)
+    elif bytes_ < GB:
+        value = _("%s MB") % filesize_number_format(bytes_ / MB)
+    elif bytes_ < TB:
+        value = _("%s GB") % filesize_number_format(bytes_ / GB)
+    elif bytes_ < PB:
+        value = _("%s TB") % filesize_number_format(bytes_ / TB)
+    else:
+        value = _("%s PB") % filesize_number_format(bytes_ / PB)
+
+    if negative:
+        value = "-%s" % value
+    return avoid_wrapping(value)

docker-app/qfieldcloud/core/utils.py

@@ -121,11 +121,9 @@ def get_s3_bucket() -> mypy_boto3_s3.service_resource.Bucket:
 def get_s3_client() -> mypy_boto3_s3.Client:
     """Get a new S3 client instance using Django settings"""
 
-    s3_client = boto3.client(
+    s3_session = get_s3_session()
+    s3_client = s3_session.client(
         "s3",
-        region_name=settings.STORAGE_REGION_NAME,
-        aws_access_key_id=settings.STORAGE_ACCESS_KEY_ID,
-        aws_secret_access_key=settings.STORAGE_SECRET_ACCESS_KEY,
         endpoint_url=settings.STORAGE_ENDPOINT_URL,
     )
     return s3_client

docker-app/qfieldcloud/core/views/files_views.py

@@ -141,6 +141,12 @@ def post(self, request, projectid, filename, format=None):
             logger.info(
                 'The key "file" was not found in `request.data`.',
                 extra={
+                    "data_for_key_text_content": str(request.data.get("text", ""))[
+                        :1000
+                    ],
+                    "data_for_key_text_len": len(request.data.get("text", "")),
+                    "request_content_length": request.META.get("CONTENT_LENGTH"),
+                    "request_content_type": request.META.get("CONTENT_TYPE"),
                     "request_data": list(request.data.keys()),
                     "request_files": list(request.FILES.keys()),
                 },

docker-app/qfieldcloud/settings.py

@@ -144,7 +144,9 @@
         ],
         "APP_DIRS": True,
         "OPTIONS": {
-            "builtins": [],
+            "builtins": [
+                "qfieldcloud.core.templatetags.filters",
+            ],
             "context_processors": [
                 "django.template.context_processors.debug",
                 "django.template.context_processors.request",
@@ -263,12 +265,13 @@
     SENTRY_SAMPLE_RATE = float(os.environ.get("SENTRY_SAMPLE_RATE", 1))
 
     def before_send(event, hint):
-        from qfieldcloud.core.exceptions import ProjectAlreadyExistsError
+        from qfieldcloud.core.exceptions import ProjectAlreadyExistsError, QuotaError
         from rest_framework.exceptions import ValidationError
 
         ignored_exceptions = (
             ValidationError,
             ProjectAlreadyExistsError,
+            QuotaError,
         )
 
         if "exc_info" in hint:

docker-app/qfieldcloud/subscription/models.py

@@ -19,6 +19,8 @@
 
 from .exceptions import NotPremiumPlanException
 
+logger = logging.getLogger(__name__)
+
 
 def get_subscription_model() -> "Subscription":
     return apps.get_model(settings.QFIELDCLOUD_SUBSCRIPTION_MODEL)
@@ -655,7 +657,7 @@ def update_subscription(
                 update_fields.append(attr_name)
                 setattr(subscription, attr_name, attr_value)
 
-            logging.info(f"Updated subscription's fields: {', '.join(update_fields)}")
+            logger.info(f"Updated subscription's fields: {', '.join(update_fields)}")
 
             subscription.save(update_fields=update_fields)
 
@@ -726,6 +728,9 @@ def create_subscription(
             ), "Creating a trial plan requires `active_since` to be a valid datetime object"
 
             active_until = active_since + timedelta(days=config.TRIAL_PERIOD_DAYS)
+            logger.info(
+                f"Creating trial subscription from {active_since=} to {active_until=}"
+            )
             trial_subscription = cls.objects.create(
                 plan=plan,
                 account=account,
@@ -748,6 +753,7 @@ def create_subscription(
             regular_plan = plan
             regular_active_since = active_since
 
+        logger.info(f"Creating regular subscription from {regular_active_since}")
         regular_subscription = cls.objects.create(
             plan=regular_plan,
             account=account,

docker-app/requirements.txt

@@ -13,7 +13,7 @@ coreschema==0.0.4
 cryptography==36.0.1
 defusedxml==0.7.1
 Deprecated==1.2.13
-Django==3.2.17
+Django==3.2.18
 django-allauth==0.44.0
 django-appconf==1.0.5
 django-auditlog==2.2.2

docker-compose.override.dev.yml

@@ -3,8 +3,6 @@ version: '3.9'
 services:
 
   app:
-    depends_on:
-      - geodb
     build:
       args:
         DEBUG_BUILD: ${DEBUG}