Merge pull request #1032 from ecds/feature/bulk-manifest-update

Allow bulk manifest metadata update via CSV

apps/iiif/manifests/admin.py

@@ -2,14 +2,15 @@
 from django.contrib import admin
 from django.http import HttpResponseRedirect
 from django.http.request import HttpRequest
+from django.urls import reverse
 from django.urls.conf import path
 from import_export import resources, fields
 from import_export.admin import ImportExportModelAdmin
 from import_export.widgets import ManyToManyWidget, ForeignKeyWidget
 from django_summernote.admin import SummernoteModelAdmin
 from .models import Manifest, Note, ImageServer, RelatedLink
 from .forms import ManifestAdminForm
-from .views import AddToCollectionsView
+from .views import AddToCollectionsView, MetadataImportView
 from ..kollections.models import Collection
 
 class ManifestResource(resources.ModelResource):
@@ -56,6 +57,7 @@ class ManifestAdmin(ImportExportModelAdmin, SummernoteModelAdmin, admin.ModelAdm
     form = ManifestAdminForm
     actions = ['add_to_collections_action']
     inlines = [RelatedLinksInline]
+    change_list_template = 'admin/change_list_override.html'
 
     def add_to_collections_action(self, request, queryset):
         """Action choose manifests to add to collections"""
@@ -72,6 +74,12 @@ def get_urls(self):
                 self.admin_site.admin_view(AddToCollectionsView.as_view()),
                 {'model_admin': self, },
                 name="AddManifestsToCollections",
+            ),
+            path(
+                'manifest_metadata_import/',
+                self.admin_site.admin_view(MetadataImportView.as_view()),
+                {'model_admin': self, },
+                name="MultiManifestMetadataImport",
             )
         ]
         return my_urls + urls

apps/iiif/manifests/forms.py

@@ -1,9 +1,14 @@
 """Django Forms for export."""
+import csv
+from io import StringIO
 import logging
 from django import forms
 from django.contrib.admin import site as admin_site, widgets
-from .models import Language, Manifest
-from ..canvases.models import Canvas
+from django.core.validators import FileExtensionValidator
+
+from apps.iiif.manifests.models import Manifest
+from apps.iiif.canvases.models import Canvas
+from apps.ingest.services import normalize_header
 
 LOGGER = logging.getLogger(__name__)
 
@@ -49,3 +54,42 @@ def __init__(self, *args, **kwargs):
                self.instance._meta.get_field('collections').remote_field,
                admin_site,
         )
+
+class ManifestCSVImportForm(forms.Form):
+    """Form to import a CSV and update the metadata for multiple manifests"""
+
+    csv_file = forms.FileField(
+        required=True,
+        validators=[FileExtensionValidator(allowed_extensions=['csv'])],
+        label="CSV File",
+        help_text="""
+            <p>Provide a CSV with a <strong>pid</strong> column, whose value in each row must match
+            the PID of an existing volume. Additional columns will be used to update the volume's
+            metadata.</p>
+            <p>Columns matching <strong>Manifest</strong> model field names will update those
+            fields directly, and any additional columns will be used to populate the volume's
+            <strong>metadata</strong> JSON field.</p>
+        """,
+    )
+
+    def clean(self):
+        # check csv has pid column
+        super().clean()
+        csv_file = self.cleaned_data.get('csv_file')
+        if csv_file:
+            reader = csv.DictReader(
+                normalize_header(
+                    StringIO(csv_file.read().decode('utf-8'))
+                ),
+            )
+            if 'pid' not in reader.fieldnames:
+                self.add_error(
+                    'metadata_spreadsheet',
+                    forms.ValidationError(
+                        """Spreadsheet must have pid column. Check to ensure there
+                        are no stray characters in the header row."""
+                    ),
+                )
+            # return back to start of file so we can read again
+            csv_file.seek(0, 0)
+        return self.cleaned_data

apps/iiif/manifests/templates/admin/change_list_override.html

@@ -0,0 +1,26 @@
+{% extends "admin/change_list.html" %}
+
+{% load i18n %}
+
+{# adapted from django-import-export #}
+{% block object-tools %}
+  <ul class="object-tools">
+    {% block object-tools-items %}
+      {% include "admin/import_export/change_list_import_item.html" %}
+      {% if has_import_permission %}
+        <li>
+          <a
+            href='{% url "admin:MultiManifestMetadataImport" %}'
+            class="import_link"
+          >
+            {% trans "Bulk update" %}
+          </a>
+        </li>
+      {% endif %}
+      {% include "admin/import_export/change_list_export_item.html" %}
+      {% if has_add_permission %}
+        {{ block.super }}
+      {% endif %}
+    {% endblock %}
+  </ul>
+{% endblock %}

apps/iiif/manifests/templates/manifest_metadata_import.html

@@ -0,0 +1,30 @@
+{% extends "admin/base_site.html" %}
+{% load i18n admin_urls %}
+{% load static %}
+
+{% block extrastyle %}
+  <link rel="stylesheet" type="text/css" href="/static/admin/css/forms.css">
+{% endblock %}
+
+{% block breadcrumbs %}
+<div class="breadcrumbs">
+  <a href="{% url 'admin:index' %}">{% trans 'Home' %}</a>
+  &rsaquo; <a href="{% url 'admin:manifests_manifest_changelist' %}">Manifests</a>  
+  &rsaquo; {{ title }}
+</div>
+{% endblock %}
+
+{% block content %}
+  <form enctype="multipart/form-data" method="post">{% csrf_token %}
+    <fieldset class="module aligned ">
+      <div class="form-row">
+        {{ form }}
+      </div>
+    </fieldset>
+    {% block submit_buttons_bottom %}
+      <div class="submit-row">
+        <input type="submit" value="{% trans 'Submit' %}" class="default" name="_save">
+      </div>
+    {% endblock %}
+  </form>
+{% endblock %}

apps/iiif/manifests/views.py

@@ -1,4 +1,6 @@
 """Django views for manifests"""
+import csv
+from io import StringIO
 import json
 import logging
 from datetime import datetime
@@ -10,8 +12,10 @@
 from django.core.serializers import serialize
 from django.contrib.sitemaps import Sitemap
 from django.urls import reverse
+
+from apps.ingest.services import normalize_header, set_metadata
 from .models import Manifest
-from .forms import ManifestsCollectionsForm
+from .forms import ManifestCSVImportForm, ManifestsCollectionsForm
 
 LOGGER = logging.getLogger(__name__)
 
@@ -154,3 +158,39 @@ def get_success_url(self):
             self.request, messages.SUCCESS, 'Successfully added manifests to collections'
         )
         return reverse('admin:manifests_manifest_changelist')
+
+class MetadataImportView(FormView):
+    """Admin page to import a CSV and update multiple Manifests' metadata"""
+
+    template_name = 'manifest_metadata_import.html'
+    form_class = ManifestCSVImportForm
+
+    def get_context_data(self, **kwargs):
+        """Set page title on context data"""
+        context = super().get_context_data(**kwargs)
+        context['title'] = 'Manifest metadata bulk update (CSV import)'
+        return context
+
+    def form_valid(self, form):
+        """Read the CSV file and, find associated manifests, and update metadata"""
+        csv_file = form.cleaned_data.get('csv_file')
+        csv_io = StringIO(csv_file.read().decode('utf-8'))
+        reader = csv.DictReader(normalize_header(csv_io))
+        for row in reader:
+            try:
+                # try to find manifest
+                manifest = Manifest.objects.get(pid=row['pid'])
+                # use ingest set_metadata function
+                set_metadata(manifest, metadata=row)
+            except Manifest.DoesNotExist:
+                messages.add_message(
+                    self.request, messages.ERROR, f'Manifest with pid {row["pid"]} not found'
+                )
+        return super().form_valid(form)
+
+    def get_success_url(self):
+        """Return to the manifest change list with a success message"""
+        messages.add_message(
+            self.request, messages.SUCCESS, 'Successfully updated manifests'
+        )
+        return reverse('admin:manifests_manifest_changelist')