Merge branch 'main' into FIND-137-Migration-of-DO-from-ds-wagtail-to-…

…ds-search

README.md

@@ -54,28 +54,33 @@ docker compose exec dev format
 
 In addition to the [base Docker image variables](https://github.com/nationalarchives/docker/blob/main/docker/tna-python-django/README.md#environment-variables), this application has support for:
 
-| Variable                          | Purpose                                                   | Default                      |
-| --------------------------------- | --------------------------------------------------------- | ---------------------------- |
-| `DJANGO_SETTINGS_MODULE`          | The configuration to use                                  | `config.settings.production` |
-| `DEBUG`                           | If true, allow debugging                                  | `False`                      |
-| `COOKIE_DOMAIN`                   | The domain to save cookie preferences against             | _none_                       |
-| `DATABASE_NAME`                   | The name of the Postgres database                         | _none_                       |
-| `DATABASE_USER`                   | The username needed to access the Postgres database       | _none_                       |
-| `DATABASE_PASSWORD`               | The password needed to access the Postgres database       | _none_                       |
-| `DATABASE_HOST`                   | The Postgres database host                                | _none_                       |
-| `DATABASE_PORT`                   | The Postgres database port                                | `5432`                       |
-| `CSP_IMG_SRC`                     | A comma separated list of CSP rules for `img-src`         | `'self'`                     |
-| `CSP_SCRIPT_SRC`                  | A comma separated list of CSP rules for `script-src`      | `'self'`                     |
-| `CSP_SCRIPT_SRC_ELEM`             | A comma separated list of CSP rules for `script-src-elem` | `'self'`                     |
-| `CSP_STYLE_SRC`                   | A comma separated list of CSP rules for `style-src`       | `'self'`                     |
-| `CSP_STYLE_SRC_ELEM`              | A comma separated list of CSP rules for `style-src-elem`  | `'self'`                     |
-| `CSP_FONT_SRC`                    | A comma separated list of CSP rules for `font-src`        | `'self'`                     |
-| `CSP_CONNECT_SRC`                 | A comma separated list of CSP rules for `connect-src`     | `'self'`                     |
-| `CSP_MEDIA_SRC`                   | A comma separated list of CSP rules for `media-src`       | `'self'`                     |
-| `CSP_WORKER_SRC`                  | A comma separated list of CSP rules for `worker-src`      | `'self'`                     |
-| `CSP_FRAME_SRC`                   | A comma separated list of CSP rules for `frame-src`       | `'self'`                     |
-| `GA4_ID`                          | The Google Analytics 4 ID                                 | _none_                       |
-| `ROSETTA_API_VERIFY_CERTIFICATES` | Verify certificate for API                                | `True`                       |
+| Variable                          | Purpose                                                   | Default                                                     |
+| --------------------------------- | --------------------------------------------------------- | ----------------------------------------------------------- |
+| `DJANGO_SETTINGS_MODULE`          | The configuration to use                                  | `config.settings.production`                                |
+| `DEBUG`                           | If true, allow debugging                                  | `False`                                                     |
+| `COOKIE_DOMAIN`                   | The domain to save cookie preferences against             | _none_                                                      |
+| `DATABASE_NAME`                   | The name of the Postgres database                         | _none_                                                      |
+| `DATABASE_USER`                   | The username needed to access the Postgres database       | _none_                                                      |
+| `DATABASE_PASSWORD`               | The password needed to access the Postgres database       | _none_                                                      |
+| `DATABASE_HOST`                   | The Postgres database host                                | _none_                                                      |
+| `DATABASE_PORT`                   | The Postgres database port                                | `5432`                                                      |
+| `CSP_IMG_SRC`                     | A comma separated list of CSP rules for `img-src`         | `'self'`                                                    |
+| `CSP_SCRIPT_SRC`                  | A comma separated list of CSP rules for `script-src`      | `'self'`                                                    |
+| `CSP_SCRIPT_SRC_ELEM`             | A comma separated list of CSP rules for `script-src-elem` | `'self'`                                                    |
+| `CSP_STYLE_SRC`                   | A comma separated list of CSP rules for `style-src`       | `'self'`                                                    |
+| `CSP_STYLE_SRC_ELEM`              | A comma separated list of CSP rules for `style-src-elem`  | `'self'`                                                    |
+| `CSP_FONT_SRC`                    | A comma separated list of CSP rules for `font-src`        | `'self'`                                                    |
+| `CSP_CONNECT_SRC`                 | A comma separated list of CSP rules for `connect-src`     | `'self'`                                                    |
+| `CSP_MEDIA_SRC`                   | A comma separated list of CSP rules for `media-src`       | `'self'`                                                    |
+| `CSP_WORKER_SRC`                  | A comma separated list of CSP rules for `worker-src`      | `'self'`                                                    |
+| `CSP_FRAME_SRC`                   | A comma separated list of CSP rules for `frame-src`       | `'self'`                                                    |
+| `GA4_ID`                          | The Google Analytics 4 ID                                 | _none_                                                      |
+| `ROSETTA_API_VERIFY_CERTIFICATES` | Verify certificate for API                                | `True`                                                      |
+| `ENVIRONMENT_NAME`                | The name of the environment (for reporting purposes)      | `production`                                                |
+| `SENTRY_DSN`                      | The ID of the Sentry client project to catch issues       | _none_                                                      |
+| `SENTRY_SAMPLE_RATE`              | How often to sample traces and profiles (0-1.0)           | production: `0.1`, staging: `0.25`, develop: `1`, test: `0` |
+
+See [Sentry's official guide](https://docs.sentry.io/platforms/python/guides/django/) for further information on configuring Sentry for Django projects.
 
 `.env` variables:
 

app/__init__.py

@@ -0,0 +1,14 @@
+import sentry_sdk
+from django.conf import settings
+from sentry_sdk.integrations.django import DjangoIntegration
+
+if settings.SENTRY_DSN:
+    sentry_sdk.init(
+        dsn=settings.SENTRY_DSN,
+        environment=settings.ENVIRONMENT_NAME,
+        release=settings.BUILD_VERSION,
+        integrations=[DjangoIntegration()],
+        sample_rate=settings.SENTRY_SAMPLE_RATE,
+        traces_sample_rate=settings.SENTRY_SAMPLE_RATE,
+        profiles_sample_rate=settings.SENTRY_SAMPLE_RATE,
+    )

app/ciim/utils.py

@@ -1,3 +1,4 @@
+import logging
 from typing import Any, Dict, Optional
 
 from django.urls import NoReverseMatch, reverse
@@ -8,6 +9,8 @@ class ValueExtractionError(Exception):
     pass
 
 
+logger = logging.getLogger(__name__)
+
 NOT_PROVIDED = "__np__"
 
 
@@ -60,15 +63,22 @@ def extract(
     return current
 
 
-def format_link(link_html: str) -> Dict[str, str]:
+def format_link(link_html: str, inc_msg: str = "") -> Dict[str, str]:
     """
     Extracts iaid and text from a link HTML string, e.g. "<a href="C5789">DEFE 31</a>"
     and returns as dict in the format: `{"id":"C5789", "href": "/catalogue/id/C5789/", "text":"DEFE 31"}
+
+    inc_msg includes message with logger if sepcified
+    Ex:inc_msg <method_name>:Record(<id):"
     """
     document = pq(link_html)
     id = document.attr("href")
     try:
         href = reverse("details-page-machine-readable", kwargs={"id": id})
     except NoReverseMatch:
         href = ""
+        # warning for partially valid data
+        logger.warning(
+            f"{inc_msg}format_link:No reverse match for details-page-machine-readable with id={id}"
+        )
     return {"id": id or "", "href": href, "text": document.text()}

app/deliveryoptions/utils.py

@@ -1,4 +1,5 @@
 import json
+import logging
 import os
 import re
 from enum import IntEnum
@@ -15,6 +16,8 @@
 from django.conf import settings
 from django.http import HttpRequest
 
+logger = logging.getLogger(__name__)
+
 
 class Reader(IntEnum):
     UNDEFINED = -1
@@ -616,7 +619,9 @@ def get_dev_reader_type() -> Reader:
                 )  # Return the corresponding Reader enum value
 
         except Exception as e:
-            pass
+            logger.warning(
+                f"Override reader type '{override_reader_type}' cannot be determined - returning UNDEFINED ({type(e)}: {e.args})"
+            )
 
     return Reader.UNDEFINED  # Default to UNDEFINED
 
@@ -627,6 +632,9 @@ def get_reader_type(request: HttpRequest) -> Reader:
     try:
         visitor_ip_address = get_client_ip(request)
     except Exception as e:
+        logger.warning(
+            f"Cannot determine the users ip address - returning OFFSITE ({type(e)}: {e.args})"
+        )
         return Reader.OFFSITE
 
     if is_dev(

app/records/field_labels.py

@@ -0,0 +1,43 @@
+# format <record_field_name> : <human readable label name>
+FIELD_LABELS = {
+    "reference_number": "Reference",
+    "title": "Title",
+    "summary_title": "Summary title",
+    "date_covering": "Date",
+    "creator": "Creator",
+    "dimensions": "Dimensions",
+    "former_department_reference": "Former reference in its original department",
+    "former_pro_reference": "Former reference in The National Archives",
+    "language": "Language",
+    "legal_status": "Legal status",
+    "map_designation": "Map designation",
+    "map_scale": "Map scale",
+    "note": "Note",
+    "physical_condition": "Physical condition",
+    "physical_description": "Physical description",
+    "held_by": "Held by",
+    "access_condition": "Access conditions",
+    "closure_status": "Closure status",
+    "record_opening": "Record opening date",
+    "accruals": "Accruals",
+    "accumulation_dates": "Accumulation dates",
+    "appraisal_information": "Selection and destruction information",
+    "copies_information": "Copies information",
+    "custodial_history": "Custodial history",
+    "immediate_source_of_acquisition": "Immediate source of acquisition",
+    "location_of_originals": "Originals held at",
+    "restrictions_on_use": "Restrictions on use",
+    "administrative_background": "Administrative / biographical background",
+    "arrangement": "Arrangement",
+    "publication_note": "Publication note",
+    "related_materials": "Related material",
+    "description": "Description",
+    "separated_materials": "Separated material",
+    "unpublished_finding_aids": "Unpublished finding aids",
+    "hierarchy": "Where am I in the catalogue?",
+    "next": "Next",
+    "previous": "Previous",
+    "parent": "This record is in",
+    "related_records": "Related records",
+    "related_articles": "Related content",
+}

app/records/models.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
+import logging
 import re
-from collections.abc import Sequence
 from typing import Any, Optional
 
 from app.ciim.models import APIModel
@@ -16,6 +16,8 @@
 
 from .converters import IDConverter
 
+logger = logging.getLogger(__name__)
+
 
 class Record(APIModel):
     """A 'lazy' data-interaction layer for record data retrieved from the Client API"""
@@ -226,7 +228,10 @@ def held_by_url(self) -> str:
                     kwargs={"id": self.held_by_id},
                 )
             except NoReverseMatch:
-                pass
+                # warning for partially valid record
+                logger.warning(
+                    f"held_by_url:Record({self.iaid}):No reverse match for details-page-machine-readable with held_by_id={self.held_by_id}"
+                )
         return ""
 
     @cached_property
@@ -302,10 +307,13 @@ def publication_note(self) -> list[str]:
     @cached_property
     def related_materials(self) -> tuple[dict[str, Any], ...]:
         """Returns transformed data which is a tuple of dict if found, empty tuple otherwise."""
+        inc_msg = f"related_materials:Record({self.iaid}):"
         return tuple(
             dict(
                 description=item.get("description", ""),
-                links=list(format_link(val) for val in item.get("links", ())),
+                links=list(
+                    format_link(val, inc_msg) for val in item.get("links", ())
+                ),
             )
             for item in self.template.get("relatedMaterials", ())
         )
@@ -318,10 +326,13 @@ def description(self) -> str:
     @cached_property
     def separated_materials(self) -> tuple[dict[str, Any], ...]:
         """Returns transformed data which is a tuple of dict if found, empty tuple otherwise."""
+        inc_msg = f"separated_materials:Record({self.iaid}):"
         return tuple(
             dict(
                 description=item.get("description", ""),
-                links=list(format_link(val) for val in item.get("links", ())),
+                links=list(
+                    format_link(val, inc_msg) for val in item.get("links", ())
+                ),
             )
             for item in self.template.get("separatedMaterials", ())
         )

app/records/template_tags/records_tags.py

@@ -0,0 +1,6 @@
+from app.records.field_labels import FIELD_LABELS
+
+
+def record_field_label(record_field_name: str) -> str:
+    """returns human readable label for pre configured record field name, otherwise Invalid name"""
+    return FIELD_LABELS.get(record_field_name, "UNRECOGNISED FIELD NAME")

app/records/views.py

@@ -1,14 +1,17 @@
+import logging
+
 from app.ciim.exceptions import DoesNotExist
 from app.deliveryoptions.delivery_options_api import get_delivery_option
 from app.deliveryoptions.utils import (
     AvailabilityCondition,
     construct_delivery_options,
-    get_reader_type,
 )
 from app.records.api import records_client
 from django.http import Http404
 from django.template.response import TemplateResponse
 
+logger = logging.getLogger(__name__)
+
 
 def record_detail_view(request, id):
     """
@@ -47,6 +50,10 @@ def record_detail_view(request, id):
 
     except Exception as e:
         # Built in order exception option
+        logger.warning(
+            f"DORIS Connection error - returning OrderException from Availability Conditions {e.args}"
+        )
+
         do_ctx = construct_delivery_options(
             [
                 {

config/jinja2.py

@@ -2,6 +2,7 @@
 import re
 from datetime import datetime
 
+from app.records.template_tags.records_tags import record_field_label
 from django.conf import settings
 from django.templatetags.static import static
 from django.urls import reverse
@@ -53,4 +54,5 @@ def environment(**options):
         }
     )
     env.filters.update({"slugify": slugify})
+    env.filters.update({"record_field_label": record_field_label})
     return env

config/settings/develop.py

@@ -11,6 +11,8 @@
 
 FORCE_HTTPS = strtobool(os.getenv("FORCE_HTTPS", "False"))
 
+SENTRY_SAMPLE_RATE = float(os.getenv("SENTRY_SAMPLE_RATE", "1.0"))
+
 DJANGO_SERVE_STATIC = strtobool(os.getenv("DJANGO_SERVE_STATIC", "True"))
 
 if not DEBUG and DJANGO_SERVE_STATIC:
@@ -20,9 +22,12 @@
         }
     }
 
-
 if DEBUG:
 
+    # logging
+    LOGGING["root"]["level"] = "DEBUG"  # noqa: F405
+
+    # debug toolbar
     INSTALLED_APPS += [  # noqa: F405
         "debug_toolbar",
     ]

config/settings/staging.py

@@ -5,3 +5,5 @@
 
 # TODO: This invalidates the CSP nonces
 CACHE_DEFAULT_TIMEOUT = int(os.environ.get("CACHE_DEFAULT_TIMEOUT", "60"))
+
+SENTRY_SAMPLE_RATE = float(os.getenv("SENTRY_SAMPLE_RATE", "0.25"))

config/settings/test.py

@@ -26,3 +26,6 @@
 STATICFILES_STORAGE = "django.contrib.staticfiles.storage.StaticFilesStorage"
 
 CLIENT_BASE_URL = "https://rosetta.test/data"
+
+ENVIRONMENT_NAME = "test"
+SENTRY_SAMPLE_RATE = 0

docker-compose.yml

@@ -9,6 +9,7 @@ services:
       - .env
     environment:
       - ENVIRONMENT=develop
+      - ENVIRONMENT_NAME=develop
       - DJANGO_SETTINGS_MODULE=config.settings.develop
       - SECRET_KEY=abc123
       - NPM_DEVELOP_COMMAND=dev