Allow ingest of metadata not in fields

apps/ingest/services.py

@@ -19,11 +19,6 @@ def clean_metadata(metadata):
     :rtype: dict
     """
     metadata = {key.casefold().replace(' ', '_'): value for key, value in metadata.items()}
-    fields = [
-        *(f.name for f in Manifest._meta.get_fields()),
-        "related",  # used for related external links
-    ]
-    invalid_keys = []
 
     for key in metadata.keys():
         if key != 'metadata' and isinstance(metadata[key], list):
@@ -33,12 +28,6 @@ def clean_metadata(metadata):
                         metadata[key] = metadata[key][0][meta_key]
             else:
                 metadata[key] = ', '.join(metadata[key])
-        if key not in fields:
-            invalid_keys.append(key)
-
-    # TODO: Update this method to allow all "invalid" keys to populate Manifest.metadata JSONField
-    for invalid_key in invalid_keys:
-        metadata.pop(invalid_key)
 
     return metadata
 
@@ -47,7 +36,7 @@ def create_related_links(manifest, related_str):
     Create RelatedLink objects from supplied related links string and associate each with supplied
     Manifest. String should consist of semicolon-separated URLs.
     :param manifest:
-    :type related_str: iiif.manifest.models.Manifest
+    :type manifest: iiif.manifest.models.Manifest
     :param related_str:
     :type related_str: str
     :rtype: None
@@ -61,6 +50,41 @@ def create_related_links(manifest, related_str):
             is_structured_data=False,  # assume this is not meant for seeAlso
         )
 
+def set_metadata(manifest, metadata):
+    """
+    Update Manifest.metadata using supplied metadata dict
+    :param manifest:
+    :type manifest: iiif.manifest.models.Manifest
+    :param metadata:
+    :type metadata: dict
+    :rtype: None
+    """
+    fields = [f.name for f in Manifest._meta.get_fields()]
+    for (key, value) in metadata.items():
+        if key == "related":
+            # add RelatedLinks from metadata spreadsheet key "related"
+            create_related_links(manifest, value)
+        elif key in fields:
+            setattr(manifest, key, value)
+        else:
+            # all other keys go into Manifest.metadata JSONField
+            if isinstance(manifest.metadata, list):
+                # add label and value to list
+                manifest.metadata.append({"label": key, "value": value})
+            elif isinstance(manifest.metadata, dict):
+                # convert to list of {label, value} as expected by iiif spec
+                manifest.metadata = [
+                    *[
+                        {"label": k, "value": v}
+                        for (k, v) in manifest.metadata.items()
+                    ],
+                    {"label": key, "value": value},
+                ]
+            else:
+                # instantiate as list
+                manifest.metadata = [{"label": key, "value": value}]
+    manifest.save()
+
 def create_manifest(ingest):
     """
     Create or update a Manifest from supplied metadata and images.
@@ -81,14 +105,7 @@ def create_manifest(ingest):
             manifest, created = Manifest.objects.get_or_create(pid=metadata['pid'].replace('_', '-'))
         else:
             manifest = Manifest.objects.create()
-        for (key, value) in metadata.items():
-            if key == "related":
-                # add RelatedLinks from metadata spreadsheet key "related"
-                create_related_links(manifest, value)
-            else:
-                # all other keys should exist as fields on Manifest (for now)
-                setattr(manifest, key, value)
-            # TODO: if the key doesn't exist on Manifest model, add it to Manifest.metadata
+        set_metadata(manifest, metadata)
     else:
         manifest = Manifest()
 

apps/ingest/tasks.py

@@ -17,7 +17,7 @@
 from apps.ingest.models import IngestTaskWatcher
 
 from .mail import send_email_on_failure, send_email_on_success
-from .services import create_manifest, create_related_links
+from .services import create_manifest, create_related_links, set_metadata
 
 # Use `apps.get_model` to avoid circular import error. Because the parameters used to
 # create a background task have to be serializable, we can't just pass in the model object.
@@ -128,13 +128,7 @@ def create_canvases_from_s3_ingest(metadata, ingest_id):
         manifest = Manifest.objects.get(pid=pid)
     except Manifest.DoesNotExist:
         manifest = Manifest.objects.create(pid=pid)
-    for (key, value) in metadata.items():
-        if key == "related":
-            # add RelatedLinks from metadata spreadsheet key "related"
-            create_related_links(manifest, value)
-        else:
-            # all other keys should exist as fields on Manifest (for now)
-            setattr(manifest, key, value)
+    set_metadata(manifest, metadata)
     # Image server: set from ingest
     ingest = S3Ingest.objects.get(pk=ingest_id)
     manifest.image_server = ingest.image_server