Merge pull request #332 from nationalarchives/chore/filter-on-sns-sub…

…scription

Filter SQS on trigger_enrichment presence

CHANGELOG.md

@@ -1,3 +1,9 @@
-1: Ensure "No Year" does not appear in <ref> tags to avoid validation errors.
+== v2 ==
+Update Terraform to filter only trigger_enrichment messages.
+Enrichment lambdas do not attempt to filter based on body JSON.
 
-0.1.0: Before December 2023, versions were not tracked and the version string was "0.1.0"
+== v1 ==
+Ensure "No Year" does not appear in <ref> tags to avoid validation errors.
+
+== v0.1.0 ==
+Before December 2023, versions were not tracked and the version string was "0.1.0"

src/lambdas/determine_legislation_provisions/index.py

@@ -43,7 +43,7 @@ def add_timestamp_and_engine_version(file_data):
         "uk:tna-enrichment-engine",
         attrs={"xmlns:uk": "https://caselaw.nationalarchives.gov.uk/akn"},
     )
-    enrichment_version.string = "1"
+    enrichment_version.string = "2"
     soup.proprietary.append(enrichment_version)
     soup.FRBRManifestation.FRBRdate.insert_after(enriched_date)
 

src/lambdas/fetch_xml/index.py

@@ -90,25 +90,21 @@ def process_event(sqs_rec, api_endpoint):
     and upload to destination S3 bucket
     """
     message = json.loads(sqs_rec["body"])
-    status, query = read_message(message)
+    status, query = read_message(
+        message
+    )  # query is the URL of the item requested to be enriched
     print("Judgment status:", status)
     print("Judgment query:", query)
 
-    if status == "published":
-        print("Judgment:", query)
-        source_key = query.replace("/", "-")
-        print("Source key:", source_key)
-
-        # fetch the xml content
-        xml_content = fetch_judgment_urllib(
-            api_endpoint, query, API_USERNAME, API_PASSWORD
-        )
-        # print(xml_content)
-        upload_contents(source_key, xml_content)
-        lock_judgment_urllib(api_endpoint, query, API_USERNAME, API_PASSWORD)
-        check_lock_judgment_urllib(api_endpoint, query, API_USERNAME, API_PASSWORD)
-    else:
-        print("Judgment not published.")
+    source_key = query.replace("/", "-")
+    print("Source key:", source_key)
+
+    # fetch the xml content
+    xml_content = fetch_judgment_urllib(api_endpoint, query, API_USERNAME, API_PASSWORD)
+    # print(xml_content)
+    upload_contents(source_key, xml_content)
+    lock_judgment_urllib(api_endpoint, query, API_USERNAME, API_PASSWORD)
+    check_lock_judgment_urllib(api_endpoint, query, API_USERNAME, API_PASSWORD)
 
 
 ############################################

terraform/modules/lambda_s3/queue.tf

@@ -463,15 +463,19 @@ resource "aws_lambda_event_source_mapping" "sqs_replacements_fetch_xml_event_sou
 }
 
 resource "aws_sns_topic_subscription" "fetch_xml_queue_subscription" {
-  count     = var.environment != "production" ? 1 : 0
-  topic_arn = "arn:aws:sns:eu-west-2:626206937213:caselaw-stg-judgment-updated"
-  protocol  = "sqs"
-  endpoint  = aws_sqs_queue.fetch_xml_queue.arn
+  count               = var.environment != "production" ? 1 : 0
+  topic_arn           = "arn:aws:sns:eu-west-2:626206937213:caselaw-stg-judgment-updated"
+  protocol            = "sqs"
+  endpoint            = aws_sqs_queue.fetch_xml_queue.arn
+  filter_policy_scope = "MessageAttributes"
+  filter_policy       = "{\"trigger_enrichment\": [{\"exists\": true}]}"
 }
 
 resource "aws_sns_topic_subscription" "fetch_xml_queue_subscription_prod" {
-  count     = var.environment == "production" ? 1 : 0
-  topic_arn = "arn:aws:sns:eu-west-2:276505630421:caselaw-judgment-updated"
-  protocol  = "sqs"
-  endpoint  = aws_sqs_queue.fetch_xml_queue.arn
+  count               = var.environment == "production" ? 1 : 0
+  topic_arn           = "arn:aws:sns:eu-west-2:276505630421:caselaw-judgment-updated"
+  protocol            = "sqs"
+  endpoint            = aws_sqs_queue.fetch_xml_queue.arn
+  filter_policy_scope = "MessageAttributes"
+  filter_policy       = "{\"trigger_enrichment\": [{\"exists\": true}]}"
 }