diff --git a/apps/iiif/canvases/tests/test_model.py b/apps/iiif/canvases/tests/test_model.py index c84b887a2..bba0a8ce3 100644 --- a/apps/iiif/canvases/tests/test_model.py +++ b/apps/iiif/canvases/tests/test_model.py @@ -4,66 +4,126 @@ from django.test import TestCase, Client from boto3 import client, resource from botocore.exceptions import ClientError -from moto import mock_s3 +from moto import mock_aws from django.conf import settings from apps.iiif.manifests.tests.factories import ManifestFactory, ImageServerFactory from .factories import CanvasFactory, CanvasNoDimensionsFactory + class TestCanvasModels(TestCase): - fixtures = ['kollections.json', 'manifests.json', 'canvases.json', 'annotations.json'] + fixtures = [ + "kollections.json", + "manifests.json", + "canvases.json", + "annotations.json", + ] def setUp(self): self.client = Client() - self.canvas = Canvas.objects.get(pk='7261fae2-a24e-4a1c-9743-516f6c4ea0c9') + self.canvas = Canvas.objects.get(pk="7261fae2-a24e-4a1c-9743-516f6c4ea0c9") self.manifest = self.canvas.manifest - self.assumed_canvas_pid = 'fedora:emory:5622' - self.assumed_canvas_resource = '5622' - self.assumed_volume_pid = 'readux:st7r6' - self.assumed_iiif_base = 'https://loris.library.emory.edu' + self.assumed_canvas_pid = "fedora:emory:5622" + self.assumed_canvas_resource = "5622" + self.assumed_volume_pid = "readux:st7r6" + self.assumed_iiif_base = "https://loris.library.emory.edu" def test_properties(self): # httpretty.register_uri(httpretty.GET, 'https://loris.library.emory.edu') - assert self.canvas.identifier == "%s/iiif/%s/canvas/%s" % (settings.HOSTNAME, self.assumed_volume_pid, self.assumed_canvas_pid) - assert self.canvas.service_id == "%s/%s" % (self.assumed_iiif_base, quote(self.assumed_canvas_pid)) - assert self.canvas.anno_id == "%s/iiif/%s/annotation/%s" % (settings.HOSTNAME, self.assumed_volume_pid, self.assumed_canvas_pid) - assert self.canvas.thumbnail == "%s/%s/full/200,/0/default.jpg" % (self.assumed_iiif_base, self.assumed_canvas_resource) - assert self.canvas.social_media == "%s/%s/full/600,/0/default.jpg" % (self.assumed_iiif_base, self.assumed_canvas_resource) - assert self.canvas.twitter_media1 == "%s/%s/full/600,/0/default.jpg" % (self.assumed_iiif_base, self.assumed_canvas_resource) - assert self.canvas.twitter_media2 == "%s/%s/full/600,/0/default.jpg" % (self.assumed_iiif_base, self.assumed_canvas_resource) - assert self.canvas.uri == "%s/iiif/%s/" % (settings.HOSTNAME, self.assumed_volume_pid) - assert self.canvas.thumbnail_crop_landscape == "%s/%s/full/,250/0/default.jpg" % (self.assumed_iiif_base, self.assumed_canvas_resource) - assert self.canvas.thumbnail_crop_tallwide == "%s/%s/pct:5,5,90,90/,250/0/default.jpg" % (self.assumed_iiif_base, self.assumed_canvas_resource) - assert self.canvas.thumbnail_crop_volume == "%s/%s/pct:15,15,70,70/,600/0/default.jpg" % (self.assumed_iiif_base, self.assumed_canvas_resource) + assert self.canvas.identifier == "%s/iiif/%s/canvas/%s" % ( + settings.HOSTNAME, + self.assumed_volume_pid, + self.assumed_canvas_pid, + ) + assert self.canvas.service_id == "%s/%s" % ( + self.assumed_iiif_base, + quote(self.assumed_canvas_pid), + ) + assert self.canvas.anno_id == "%s/iiif/%s/annotation/%s" % ( + settings.HOSTNAME, + self.assumed_volume_pid, + self.assumed_canvas_pid, + ) + assert self.canvas.thumbnail == "%s/%s/full/200,/0/default.jpg" % ( + self.assumed_iiif_base, + self.assumed_canvas_resource, + ) + assert self.canvas.social_media == "%s/%s/full/600,/0/default.jpg" % ( + self.assumed_iiif_base, + self.assumed_canvas_resource, + ) + assert self.canvas.twitter_media1 == "%s/%s/full/600,/0/default.jpg" % ( + self.assumed_iiif_base, + self.assumed_canvas_resource, + ) + assert self.canvas.twitter_media2 == "%s/%s/full/600,/0/default.jpg" % ( + self.assumed_iiif_base, + self.assumed_canvas_resource, + ) + assert self.canvas.uri == "%s/iiif/%s/" % ( + settings.HOSTNAME, + self.assumed_volume_pid, + ) + assert ( + self.canvas.thumbnail_crop_landscape + == "%s/%s/full/,250/0/default.jpg" + % (self.assumed_iiif_base, self.assumed_canvas_resource) + ) + assert ( + self.canvas.thumbnail_crop_tallwide + == "%s/%s/pct:5,5,90,90/,250/0/default.jpg" + % (self.assumed_iiif_base, self.assumed_canvas_resource) + ) + assert ( + self.canvas.thumbnail_crop_volume + == "%s/%s/pct:15,15,70,70/,600/0/default.jpg" + % (self.assumed_iiif_base, self.assumed_canvas_resource) + ) - @mock_s3 + @mock_aws def test_delete_canvas_from_s3(self): """When deleted, it should delete the S3 objects""" - conn = resource('s3', region_name='us-east-1') - conn.create_bucket(Bucket='eagles') - image_server = ImageServerFactory.create(storage_service='s3', storage_path='eagles') + conn = resource("s3", region_name="us-east-1") + conn.create_bucket(Bucket="eagles") + image_server = ImageServerFactory.create( + storage_service="s3", storage_path="eagles" + ) manifest = ManifestFactory.create(image_server=image_server) canvas = CanvasFactory.create( manifest=manifest, - pid=f'{manifest.pid}_00000002.jpg', - ocr_file_path=f'https://{image_server.storage_path}.s3.amazonaws.com/{manifest.pid}/_*ocr_*/00000002.tsv' + pid=f"{manifest.pid}_00000002.jpg", + ocr_file_path=f"https://{image_server.storage_path}.s3.amazonaws.com/{manifest.pid}/_*ocr_*/00000002.tsv", ) - with open(join(settings.APPS_DIR, 'iiif/canvases/fixtures/00000002.jpg'), 'rb') as image_file: - client('s3').upload_fileobj( + with open( + join(settings.APPS_DIR, "iiif/canvases/fixtures/00000002.jpg"), "rb" + ) as image_file: + client("s3").upload_fileobj( image_file, Bucket=image_server.storage_path, - Key=f'{canvas.manifest.pid}/00000002.jpg' + Key=f"{canvas.manifest.pid}/00000002.jpg", ) - with open(join(settings.APPS_DIR, 'iiif/canvases/fixtures/00000002.tsv'), 'rb') as image_file: - client('s3').upload_fileobj( + with open( + join(settings.APPS_DIR, "iiif/canvases/fixtures/00000002.tsv"), "rb" + ) as image_file: + client("s3").upload_fileobj( image_file, Bucket=image_server.storage_path, - Key=f'{canvas.manifest.pid}/_*ocr*_/00000002.tsv' + Key=f"{canvas.manifest.pid}/_*ocr*_/00000002.tsv", ) - self.assertEqual(f'{canvas.manifest.pid}/00000002.jpg', canvas.file_name) - s3_image_obj = resource('s3').Object(image_server.storage_path, canvas.file_name) - s3_ocr_obj = resource('s3').Object(image_server.storage_path, f'{canvas.manifest.pid}/_*ocr*_/00000002.tsv') - self.assertEqual(s3_image_obj.get()['ResponseMetadata']['HTTPHeaders']['etag'], s3_image_obj.e_tag) - self.assertEqual(s3_ocr_obj.get()['ResponseMetadata']['HTTPHeaders']['etag'], s3_ocr_obj.e_tag) + self.assertEqual(f"{canvas.manifest.pid}/00000002.jpg", canvas.file_name) + s3_image_obj = resource("s3").Object( + image_server.storage_path, canvas.file_name + ) + s3_ocr_obj = resource("s3").Object( + image_server.storage_path, f"{canvas.manifest.pid}/_*ocr*_/00000002.tsv" + ) + self.assertEqual( + s3_image_obj.get()["ResponseMetadata"]["HTTPHeaders"]["etag"], + s3_image_obj.e_tag, + ) + self.assertEqual( + s3_ocr_obj.get()["ResponseMetadata"]["HTTPHeaders"]["etag"], + s3_ocr_obj.e_tag, + ) canvas.delete() @@ -73,14 +133,14 @@ def test_delete_canvas_from_s3(self): try: s3_image_obj.get() except ClientError as error: - get_image_error = error.response['Error']['Code'] + get_image_error = error.response["Error"]["Code"] try: s3_ocr_obj.get() except ClientError as error: - get_ocr_error = error.response['Error']['Code'] + get_ocr_error = error.response["Error"]["Code"] - self.assertEqual(get_image_error, 'NoSuchKey') - self.assertEqual(get_ocr_error, 'NoSuchKey') + self.assertEqual(get_image_error, "NoSuchKey") + self.assertEqual(get_ocr_error, "NoSuchKey") def test_no_manifest(self): canvas = Canvas() @@ -93,14 +153,14 @@ def test_string_representation(self): assert str(canvas) == canvas.pid def test_get_image_info(self): - image_server = ImageServerFactory.create(server_base='http://fake.info') + image_server = ImageServerFactory.create(server_base="http://fake.info") manifest = ManifestFactory.create(image_server=image_server) canvas = CanvasFactory.create(manifest=manifest) - assert canvas.image_info['height'] == 3000 - assert canvas.image_info['width'] == 3000 + assert canvas.image_info["height"] == 3000 + assert canvas.image_info["width"] == 3000 def test_setting_height_width_from_iiif(self): - image_server = ImageServerFactory.create(server_base='http://fake.info') + image_server = ImageServerFactory.create(server_base="http://fake.info") manifest = ManifestFactory.create(image_server=image_server) canvas = CanvasFactory.build() canvas.height = 0 @@ -125,4 +185,4 @@ def test_setting_resource(self): canvas = CanvasNoDimensionsFactory.build(manifest=ManifestFactory.create()) assert canvas.resource is None canvas.before_save() - assert canvas.resource == canvas.pid \ No newline at end of file + assert canvas.resource == canvas.pid diff --git a/apps/iiif/canvases/tests/test_services.py b/apps/iiif/canvases/tests/test_services.py index cf436d9ec..dd046cd3d 100644 --- a/apps/iiif/canvases/tests/test_services.py +++ b/apps/iiif/canvases/tests/test_services.py @@ -1,11 +1,12 @@ """ Test cases for :class:`apps.iiif.canvases` """ + import json from os.path import join import boto3 import httpretty -from moto import mock_s3 +from moto import mock_aws from django.test import TestCase, Client from django.urls import reverse from django.core.serializers import serialize @@ -22,114 +23,113 @@ class CanvasTests(TestCase): - fixtures = ['kollections.json', 'manifests.json', 'canvases.json', 'annotations.json'] + fixtures = [ + "kollections.json", + "manifests.json", + "canvases.json", + "annotations.json", + ] def setUp(self): self.client = Client() - self.canvas = Canvas.objects.get(pk='7261fae2-a24e-4a1c-9743-516f6c4ea0c9') + self.canvas = Canvas.objects.get(pk="7261fae2-a24e-4a1c-9743-516f6c4ea0c9") self.manifest = self.canvas.manifest - self.assumed_canvas_pid = 'fedora:emory:5622' - self.assumed_canvas_resource = '5622' - self.assumed_volume_pid = 'readux:st7r6' - self.assumed_iiif_base = 'https://loris.library.emory.edu' + self.assumed_canvas_pid = "fedora:emory:5622" + self.assumed_canvas_resource = "5622" + self.assumed_volume_pid = "readux:st7r6" + self.assumed_iiif_base = "https://loris.library.emory.edu" - def set_up_mock_s3(self, manifest): - conn = boto3.resource('s3', region_name='us-east-1') + def set_up_mock_aws(self, manifest): + conn = boto3.resource("s3", region_name="us-east-1") conn.create_bucket(Bucket=manifest.image_server.storage_path) def test_app_config(self): - assert CanvasesConfig.verbose_name == 'Canvases' - assert CanvasesConfig.name == 'apps.iiif.canvases' + assert CanvasesConfig.verbose_name == "Canvases" + assert CanvasesConfig.name == "apps.iiif.canvases" def test_ia_ocr_creation(self): valid_ia_ocr_response = { - 'ocr': [ - [ - ['III', [120, 1600, 180, 1494, 1597]] - ], - [ - ['chambray', [78, 1734, 116, 1674, 1734]] - ], - [ - ['tacos', [142, 1938, 188, 1854, 1938]] - ], - [ - ['freegan', [114, 2246, 196, 2156, 2245]] - ], - [ - ['Kombucha', [180, 2528, 220, 2444, 2528]] - ], + "ocr": [ + [["III", [120, 1600, 180, 1494, 1597]]], + [["chambray", [78, 1734, 116, 1674, 1734]]], + [["tacos", [142, 1938, 188, 1854, 1938]]], + [["freegan", [114, 2246, 196, 2156, 2245]]], + [["Kombucha", [180, 2528, 220, 2444, 2528]]], [ - ['succulents', [558, 535, 588, 501, 535]], - ['Thundercats', [928, 534, 1497, 478, 527]] + ["succulents", [558, 535, 588, 501, 535]], + ["Thundercats", [928, 534, 1497, 478, 527]], ], [ - ['poke', [557, 617, 646, 575, 614]], - ['VHS', [700, 612, 1147, 555, 610]], - ['chartreuse ', [1191, 616, 1209, 589, 609]], - ['pabst', [1266, 603, 1292, 569, 603]], - ['8-bit', [1354, 602, 1419, 549, 600]], - ['narwhal', [1471, 613, 1566, 553, 592]], - ['XOXO', [1609, 604, 1670, 538, 596]], - ['post-ironic', [1713, 603, 1826, 538, 590]], - ['synth', [1847, 588, 1859, 574, 588]] + ["poke", [557, 617, 646, 575, 614]], + ["VHS", [700, 612, 1147, 555, 610]], + ["chartreuse ", [1191, 616, 1209, 589, 609]], + ["pabst", [1266, 603, 1292, 569, 603]], + ["8-bit", [1354, 602, 1419, 549, 600]], + ["narwhal", [1471, 613, 1566, 553, 592]], + ["XOXO", [1609, 604, 1670, 538, 596]], + ["post-ironic", [1713, 603, 1826, 538, 590]], + ["synth", [1847, 588, 1859, 574, 588]], ], - [ - ['lumbersexual', [1741, 2928, 1904, 2881, 2922]] - ] + [["lumbersexual", [1741, 2928, 1904, 2881, 2922]]], ] } - canvas = Canvas.objects.get(pid='15210893.5622.emory.edu$95') - canvas.manifest.image_server.server_base = 'https://iiif.archivelab.org/iiif/' + canvas = Canvas.objects.get(pid="15210893.5622.emory.edu$95") + canvas.manifest.image_server.server_base = "https://iiif.archivelab.org/iiif/" ocr = services.add_positional_ocr(canvas, valid_ia_ocr_response) assert len(ocr) == 17 for word in ocr: - assert 'w' in word - assert 'h' in word - assert 'x' in word - assert 'y' in word - assert 'content' in word - assert isinstance(word['w'], int) - assert isinstance(word['h'], int) - assert isinstance(word['x'], int) - assert isinstance(word['y'], int) - assert isinstance(word['content'], str) + assert "w" in word + assert "h" in word + assert "x" in word + assert "y" in word + assert "content" in word + assert isinstance(word["w"], int) + assert isinstance(word["h"], int) + assert isinstance(word["x"], int) + assert isinstance(word["y"], int) + assert isinstance(word["content"], str) def test_fedora_ocr_creation(self): - valid_fedora_positional_response = """523\t 116\t 151\t 45\tDistillery\r\n 704\t 117\t 148\t 52\tplaid,"\r\n""".encode('UTF-8-sig') + valid_fedora_positional_response = """523\t 116\t 151\t 45\tDistillery\r\n 704\t 117\t 148\t 52\tplaid,"\r\n""".encode( + "UTF-8-sig" + ) ocr = services.add_positional_ocr(self.canvas, valid_fedora_positional_response) assert len(ocr) == 2 for word in ocr: - assert 'w' in word - assert 'h' in word - assert 'x' in word - assert 'y' in word - assert 'content' in word - assert isinstance(word['w'], int) - assert isinstance(word['h'], int) - assert isinstance(word['x'], int) - assert isinstance(word['y'], int) - assert isinstance(word['content'], str) + assert "w" in word + assert "h" in word + assert "x" in word + assert "y" in word + assert "content" in word + assert isinstance(word["w"], int) + assert isinstance(word["h"], int) + assert isinstance(word["x"], int) + assert isinstance(word["y"], int) + assert isinstance(word["content"], str) def test_ocr_from_tei(self): - tei = open('apps/iiif/canvases/fixtures/tei.xml', 'r').read() + tei = open("apps/iiif/canvases/fixtures/tei.xml", "r").read() ocr = services.parse_tei_ocr(tei) - assert ocr[1]['content'] == 'AEN DEN LESIIU' - assert ocr[1]['h'] == 28 - assert ocr[1]['w'] == 461 - assert ocr[1]['x'] == 814 - assert ocr[1]['y'] == 185 + assert ocr[1]["content"] == "AEN DEN LESIIU" + assert ocr[1]["h"] == 28 + assert ocr[1]["w"] == 461 + assert ocr[1]["x"] == 814 + assert ocr[1]["y"] == 185 def test_line_by_line_from_tei(self): - canvas = CanvasFactory.create(default_ocr='line', manifest=ManifestFactory.create()) - ocr_file = open(join(settings.APPS_DIR, 'iiif/canvases/fixtures/tei.xml'), 'r').read() + canvas = CanvasFactory.create( + default_ocr="line", manifest=ManifestFactory.create() + ) + ocr_file = open( + join(settings.APPS_DIR, "iiif/canvases/fixtures/tei.xml"), "r" + ).read() tei = services.parse_tei_ocr(ocr_file) services.add_ocr_annotations(canvas, tei) updated_canvas = Canvas.objects.get(pk=canvas.pk) ocr = updated_canvas.annotation_set.first() - assert 'mm' in ocr.content + assert "mm" in ocr.content assert ocr.h == 26 assert ocr.w == 90 assert ocr.x == 916 @@ -139,8 +139,12 @@ def test_line_by_line_from_tei(self): assert anno.order == num def test_ocr_from_tsv(self): - manifest = ManifestFactory.create(image_server=ImageServerFactory.create(server_base='https://images.readux.ecds.emory.fake/')) - canvas = CanvasFactory(manifest=manifest, pid='boo') + manifest = ManifestFactory.create( + image_server=ImageServerFactory.create( + server_base="https://images.readux.ecds.emory.fake/" + ) + ) + canvas = CanvasFactory(manifest=manifest, pid="boo") manifest.refresh_from_db() # AnnotationFactory.create(x=459, y=391, w=89, h=43, order=1, canvas=canvas) # AnnotationFactory.create(x=453, y=397, w=397, h=3, order=2, canvas=canvas) @@ -153,13 +157,13 @@ def test_ocr_from_tsv(self): assert ocr.y == 391 assert ocr.w == 89 assert ocr.h == 43 - assert 'Jordan' in ocr.content + assert "Jordan" in ocr.content ocr2 = canvas.annotation_set.all()[1] assert ocr2.x == 453 assert ocr2.y == 397 assert ocr2.w == 397 assert ocr2.h == 3 - assert '> ' in ocr2.content + assert "> " in ocr2.content assert canvas.annotation_set.all().count() == 5 def test_no_tei_from_empty_result(self): @@ -167,274 +171,318 @@ def test_no_tei_from_empty_result(self): assert ocr is None def test_from_bad_tei(self): - tei = open('apps/iiif/canvases/fixtures/bad_tei.xml', 'r').read() + tei = open("apps/iiif/canvases/fixtures/bad_tei.xml", "r").read() self.assertRaises(XMLSyntaxError, services.parse_tei_ocr, tei) def test_canvas_detail(self): - kwargs = {'manifest': self.manifest.pid, 'pid': self.canvas.pid} - url = reverse('RenderCanvasDetail', kwargs=kwargs) + kwargs = {"manifest": self.manifest.pid, "pid": self.canvas.pid} + url = reverse("RenderCanvasDetail", kwargs=kwargs) response = self.client.get(url) - serialized_canvas = json.loads(response.content.decode('UTF-8-sig')) + serialized_canvas = json.loads(response.content.decode("UTF-8-sig")) assert response.status_code == 200 - assert serialized_canvas['@id'] == self.canvas.identifier - assert serialized_canvas['label'] == str(self.canvas.position) - assert serialized_canvas['images'][0]['@id'] == self.canvas.anno_id - assert serialized_canvas['images'][0]['resource']['@id'] == "%s/full/full/0/default.jpg" % (self.canvas.resource_id) + assert serialized_canvas["@id"] == self.canvas.identifier + assert serialized_canvas["label"] == str(self.canvas.position) + assert serialized_canvas["images"][0]["@id"] == self.canvas.anno_id + assert serialized_canvas["images"][0]["resource"][ + "@id" + ] == "%s/full/full/0/default.jpg" % (self.canvas.resource_id) def test_canvas_list(self): - kwargs = { 'manifest': self.manifest.pid } - url = reverse('RenderCanvasList', kwargs=kwargs) + kwargs = {"manifest": self.manifest.pid} + url = reverse("RenderCanvasList", kwargs=kwargs) response = self.client.get(url) - canvas_list = json.loads(response.content.decode('UTF-8-sig')) + canvas_list = json.loads(response.content.decode("UTF-8-sig")) assert response.status_code == 200 assert len(canvas_list) == 2 def test_wide_image_crops(self): - pid = '15210893.5622.emory.edu$95' + pid = "15210893.5622.emory.edu$95" canvas = Canvas.objects.get(pid=pid) - assert canvas.thumbnail_crop_landscape == "%s/%s/pct:25,0,50,100/,250/0/default.jpg" % (canvas.manifest.image_server.server_base, canvas.resource) - assert canvas.thumbnail_crop_tallwide == "%s/%s/pct:5,5,90,90/250,/0/default.jpg" % (canvas.manifest.image_server.server_base, canvas.resource) - assert canvas.thumbnail_crop_volume == "%s/%s/pct:25,15,50,85/,600/0/default.jpg" % (canvas.manifest.image_server.server_base, canvas.resource) + assert ( + canvas.thumbnail_crop_landscape + == "%s/%s/pct:25,0,50,100/,250/0/default.jpg" + % (canvas.manifest.image_server.server_base, canvas.resource) + ) + assert ( + canvas.thumbnail_crop_tallwide + == "%s/%s/pct:5,5,90,90/250,/0/default.jpg" + % (canvas.manifest.image_server.server_base, canvas.resource) + ) + assert ( + canvas.thumbnail_crop_volume + == "%s/%s/pct:25,15,50,85/,600/0/default.jpg" + % (canvas.manifest.image_server.server_base, canvas.resource) + ) def test_result_property(self): canvas = CanvasFactory.create(manifest=ManifestFactory.create()) for order, word in enumerate(["a", "retto", ",", "dio", "Quefìa", "de'"]): - AnnotationFactory.create(content=word, canvas=canvas, order=order+1) + AnnotationFactory.create(content=word, canvas=canvas, order=order + 1) assert canvas.result == "a retto , dio Quef\u00eca de'" def test_no_tei_for_internet_archive(self): - self.canvas.manifest.image_server.server_base = 'https://iiif.archivelab.org/iiif/' + self.canvas.manifest.image_server.server_base = ( + "https://iiif.archivelab.org/iiif/" + ) assert services.fetch_tei_ocr(self.canvas) is None def test_fetch_positional_ocr(self): - self.canvas.manifest.image_server.server_base = 'https://iiif.archivelab.org/iiif/' - self.canvas.manifest.pid = 'atlantacitydirec1908foot' - self.canvas.pid = '1' - assert services.fetch_positional_ocr(self.canvas)['ocr'] is not None + self.canvas.manifest.image_server.server_base = ( + "https://iiif.archivelab.org/iiif/" + ) + self.canvas.manifest.pid = "atlantacitydirec1908foot" + self.canvas.pid = "1" + assert services.fetch_positional_ocr(self.canvas)["ocr"] is not None def test_fetch_positional_ocr_with_offset(self): - self.canvas.manifest.image_server.server_base = 'https://iiif.archivelab.org/iiif/' - self.canvas.manifest.pid = 'atlantacitydirec1908foot' - self.canvas.pid = '$1' - assert services.fetch_positional_ocr(self.canvas)['ocr'] is not None + self.canvas.manifest.image_server.server_base = ( + "https://iiif.archivelab.org/iiif/" + ) + self.canvas.manifest.pid = "atlantacitydirec1908foot" + self.canvas.pid = "$1" + assert services.fetch_positional_ocr(self.canvas)["ocr"] is not None # def test_fetch_positional_ocr_that_return_none(self): # self.canvas.manifest.image_server.server_base = 'oxford' # assert services.fetch_positional_ocr(self.canvas) is None - @mock_s3 + @mock_aws def test_ocr_in_s3(self): bucket_name = encode_noid() manifest = ManifestFactory.create( - image_server = ImageServerFactory.create(storage_service = 's3', storage_path=bucket_name, server_base='images.readux.ecds.emory') + image_server=ImageServerFactory.create( + storage_service="s3", + storage_path=bucket_name, + server_base="images.readux.ecds.emory", + ) ) - self.set_up_mock_s3(manifest) - tsv_file_path = 'apps/iiif/canvases/fixtures/00000002.tsv' + self.set_up_mock_aws(manifest) + tsv_file_path = "apps/iiif/canvases/fixtures/00000002.tsv" canvas = manifest.canvas_set.first() - canvas.ocr_file_path = f'{manifest.pid}/_*ocr*_/00000002.tsv' - manifest.image_server.bucket.upload_file(tsv_file_path, f'{manifest.pid}/_*ocr*_/00000002.tsv') + canvas.ocr_file_path = f"{manifest.pid}/_*ocr*_/00000002.tsv" + manifest.image_server.bucket.upload_file( + tsv_file_path, f"{manifest.pid}/_*ocr*_/00000002.tsv" + ) fetched_ocr = services.fetch_positional_ocr(canvas) - assert open(tsv_file_path, 'rb').read() == fetched_ocr + assert open(tsv_file_path, "rb").read() == fetched_ocr - @mock_s3 + @mock_aws def test_fetched_ocr_result_is_string(self): - """ Test when fetched OCR is a string. """ + """Test when fetched OCR is a string.""" bucket_name = encode_noid() manifest = ManifestFactory.create( - image_server = ImageServerFactory.create(storage_service = 's3', storage_path=bucket_name, server_base='images.readux.ecds.emory') + image_server=ImageServerFactory.create( + storage_service="s3", + storage_path=bucket_name, + server_base="images.readux.ecds.emory", + ) ) - self.set_up_mock_s3(manifest) - tsv_file_path = 'apps/iiif/canvases/fixtures/00000002.tsv' + self.set_up_mock_aws(manifest) + tsv_file_path = "apps/iiif/canvases/fixtures/00000002.tsv" canvas = manifest.canvas_set.first() - canvas.ocr_file_path = f'{manifest.pid}/_*ocr*_/00000002.tsv' - manifest.image_server.bucket.upload_file(tsv_file_path, f'{manifest.pid}/_*ocr*_/00000002.tsv') - ocr_result = open(tsv_file_path, 'r').read() + canvas.ocr_file_path = f"{manifest.pid}/_*ocr*_/00000002.tsv" + manifest.image_server.bucket.upload_file( + tsv_file_path, f"{manifest.pid}/_*ocr*_/00000002.tsv" + ) + ocr_result = open(tsv_file_path, "r").read() assert isinstance(ocr_result, str) ocr = services.add_positional_ocr(canvas, ocr_result) assert len(ocr) == 10 - assert ocr[0]['content'] == 'Manuscript' + assert ocr[0]["content"] == "Manuscript" - @mock_s3 + @mock_aws def test_fetched_ocr_result_is_bytes(self): - """ Test when fetched OCR is a bytes. """ + """Test when fetched OCR is a bytes.""" bucket_name = encode_noid() manifest = ManifestFactory.create( - image_server = ImageServerFactory.create(storage_service = 's3', storage_path=bucket_name, server_base='images.readux.ecds.emory') + image_server=ImageServerFactory.create( + storage_service="s3", + storage_path=bucket_name, + server_base="images.readux.ecds.emory", + ) ) - self.set_up_mock_s3(manifest) - tsv_file_path = 'apps/iiif/canvases/fixtures/00000002.tsv' + self.set_up_mock_aws(manifest) + tsv_file_path = "apps/iiif/canvases/fixtures/00000002.tsv" canvas = manifest.canvas_set.first() - canvas.ocr_file_path = f'{manifest.pid}/_*ocr*_/00000002.tsv' - manifest.image_server.bucket.upload_file(tsv_file_path, f'{manifest.pid}/_*ocr*_/00000002.tsv') - ocr_result = open(tsv_file_path, 'rb').read() + canvas.ocr_file_path = f"{manifest.pid}/_*ocr*_/00000002.tsv" + manifest.image_server.bucket.upload_file( + tsv_file_path, f"{manifest.pid}/_*ocr*_/00000002.tsv" + ) + ocr_result = open(tsv_file_path, "rb").read() assert isinstance(ocr_result, bytes) ocr = services.add_positional_ocr(canvas, ocr_result) assert len(ocr) == 10 - assert ocr[0]['content'] == 'Manuscript' + assert ocr[0]["content"] == "Manuscript" def test_from_alto_ocr(self): - """ Test parsing ALTO OCR """ - alto = open('apps/iiif/canvases/fixtures/alto.xml', 'rb').read() + """Test parsing ALTO OCR""" + alto = open("apps/iiif/canvases/fixtures/alto.xml", "rb").read() ocr = services.parse_alto_ocr(alto) - assert ocr[0]['content'] == 'MAGNA' - assert ocr[0]['h'] == 164 - assert ocr[0]['w'] == 758 - assert ocr[0]['x'] == 1894 - assert ocr[0]['y'] == 1787 + assert ocr[0]["content"] == "MAGNA" + assert ocr[0]["h"] == 164 + assert ocr[0]["w"] == 758 + assert ocr[0]["x"] == 1894 + assert ocr[0]["y"] == 1787 def test_from_hocr(self): - """ Test parsing hOCR """ - hocr = open('apps/iiif/canvases/fixtures/hocr.hocr', 'rb').read() + """Test parsing hOCR""" + hocr = open("apps/iiif/canvases/fixtures/hocr.hocr", "rb").read() ocr = services.parse_hocr_ocr(hocr) - assert ocr[0]['content'] == 'MAGNA' - assert ocr[0]['h'] == 164 - assert ocr[0]['w'] == 758 - assert ocr[0]['x'] == 1894 - assert ocr[0]['y'] == 1787 + assert ocr[0]["content"] == "MAGNA" + assert ocr[0]["h"] == 164 + assert ocr[0]["w"] == 758 + assert ocr[0]["x"] == 1894 + assert ocr[0]["y"] == 1787 def test_from_bad_hocr(self): - """ Test parsing bad hOCR """ - bad_hocr = open('apps/iiif/canvases/fixtures/bad_hocr.hocr', 'rb').read() - self.assertRaises(services.HocrValidationError, services.parse_hocr_ocr, bad_hocr) + """Test parsing bad hOCR""" + bad_hocr = open("apps/iiif/canvases/fixtures/bad_hocr.hocr", "rb").read() + self.assertRaises( + services.HocrValidationError, services.parse_hocr_ocr, bad_hocr + ) def test_identifying_alto_xml(self): - """ Test identifying XML file as ALTO OCR """ - alto = open('apps/iiif/canvases/fixtures/alto.xml', 'rb').read() + """Test identifying XML file as ALTO OCR""" + alto = open("apps/iiif/canvases/fixtures/alto.xml", "rb").read() ocr = services.parse_xml_ocr(alto) - assert ocr[0]['content'] == 'MAGNA' - assert ocr[0]['h'] == 164 - assert ocr[0]['w'] == 758 - assert ocr[0]['x'] == 1894 - assert ocr[0]['y'] == 1787 + assert ocr[0]["content"] == "MAGNA" + assert ocr[0]["h"] == 164 + assert ocr[0]["w"] == 758 + assert ocr[0]["x"] == 1894 + assert ocr[0]["y"] == 1787 def test_identifying_hocr_xml(self): - """ Test identifying XML file as hOCR """ - hocr = open('apps/iiif/canvases/fixtures/hocr.hocr', 'rb').read() + """Test identifying XML file as hOCR""" + hocr = open("apps/iiif/canvases/fixtures/hocr.hocr", "rb").read() ocr = services.parse_xml_ocr(hocr) - assert ocr[0]['content'] == 'MAGNA' - assert ocr[0]['h'] == 164 - assert ocr[0]['w'] == 758 - assert ocr[0]['x'] == 1894 - assert ocr[0]['y'] == 1787 + assert ocr[0]["content"] == "MAGNA" + assert ocr[0]["h"] == 164 + assert ocr[0]["w"] == 758 + assert ocr[0]["x"] == 1894 + assert ocr[0]["y"] == 1787 def test_identifying_tei_xml(self): - """ Test identifying XML file as hOCR """ - tei = open('apps/iiif/canvases/fixtures/tei.xml', 'r').read() + """Test identifying XML file as hOCR""" + tei = open("apps/iiif/canvases/fixtures/tei.xml", "r").read() ocr = services.parse_xml_ocr(tei) - assert ocr[1]['content'] == 'AEN DEN LESIIU' - assert ocr[1]['h'] == 28 - assert ocr[1]['w'] == 461 - assert ocr[1]['x'] == 814 - assert ocr[1]['y'] == 185 + assert ocr[1]["content"] == "AEN DEN LESIIU" + assert ocr[1]["h"] == 28 + assert ocr[1]["w"] == 461 + assert ocr[1]["x"] == 814 + assert ocr[1]["y"] == 185 def test_identification_failure(self): - """ Test identifying XML on non-XML fails """ - tsv = open('apps/iiif/canvases/fixtures/sample.tsv', 'r').read() + """Test identifying XML on non-XML fails""" + tsv = open("apps/iiif/canvases/fixtures/sample.tsv", "r").read() self.assertRaises(XMLSyntaxError, services.parse_xml_ocr, tsv) def test_unidentifiable_xml(self): - """ Test identifying XML that is not TEI, ALTO, or hOCR """ - hops = open('apps/iiif/canvases/fixtures/hops.xml', 'rb').read() + """Test identifying XML that is not TEI, ALTO, or hOCR""" + hops = open("apps/iiif/canvases/fixtures/hops.xml", "rb").read() ocr = services.parse_xml_ocr(hops) assert ocr is None - @mock_s3 + @mock_aws def test_add_alto_ocr_by_filename(self): - """ Test get_ocr when OCR is ALTO file (by filename). """ + """Test get_ocr when OCR is ALTO file (by filename).""" bucket_name = encode_noid() manifest = ManifestFactory.create( - image_server = ImageServerFactory.create( - storage_service = 's3', + image_server=ImageServerFactory.create( + storage_service="s3", storage_path=bucket_name, - server_base='images.readux.ecds.emory' + server_base="images.readux.ecds.emory", ) ) - self.set_up_mock_s3(manifest) - tsv_file_path = 'apps/iiif/canvases/fixtures/alto.xml' + self.set_up_mock_aws(manifest) + tsv_file_path = "apps/iiif/canvases/fixtures/alto.xml" canvas = manifest.canvas_set.first() - canvas.ocr_file_path = f'{manifest.pid}/_*ocr*_/alto.xml' - manifest.image_server.bucket.upload_file(tsv_file_path, f'{manifest.pid}/_*ocr*_/alto.xml') + canvas.ocr_file_path = f"{manifest.pid}/_*ocr*_/alto.xml" + manifest.image_server.bucket.upload_file( + tsv_file_path, f"{manifest.pid}/_*ocr*_/alto.xml" + ) ocr = services.get_ocr(canvas) - assert ocr[0]['content'] == 'MAGNA' - assert ocr[0]['h'] == 164 - assert ocr[0]['w'] == 758 - assert ocr[0]['x'] == 1894 - assert ocr[0]['y'] == 1787 + assert ocr[0]["content"] == "MAGNA" + assert ocr[0]["h"] == 164 + assert ocr[0]["w"] == 758 + assert ocr[0]["x"] == 1894 + assert ocr[0]["y"] == 1787 - @mock_s3 + @mock_aws def test_add_hocr_by_filename(self): - """ Test get_ocr when OCR is hOCR file (by filename). """ + """Test get_ocr when OCR is hOCR file (by filename).""" bucket_name = encode_noid() manifest = ManifestFactory.create( - image_server = ImageServerFactory.create( - storage_service = 's3', + image_server=ImageServerFactory.create( + storage_service="s3", storage_path=bucket_name, - server_base='images.readux.ecds.emory' + server_base="images.readux.ecds.emory", ) ) - self.set_up_mock_s3(manifest) - tsv_file_path = 'apps/iiif/canvases/fixtures/hocr.hocr' + self.set_up_mock_aws(manifest) + tsv_file_path = "apps/iiif/canvases/fixtures/hocr.hocr" canvas = manifest.canvas_set.first() - canvas.ocr_file_path = f'{manifest.pid}/_*ocr*_/hocr.hocr' - manifest.image_server.bucket.upload_file(tsv_file_path, f'{manifest.pid}/_*ocr*_/hocr.hocr') + canvas.ocr_file_path = f"{manifest.pid}/_*ocr*_/hocr.hocr" + manifest.image_server.bucket.upload_file( + tsv_file_path, f"{manifest.pid}/_*ocr*_/hocr.hocr" + ) ocr = services.get_ocr(canvas) - assert ocr[0]['content'] == 'MAGNA' - assert ocr[0]['h'] == 164 - assert ocr[0]['w'] == 758 - assert ocr[0]['x'] == 1894 - assert ocr[0]['y'] == 1787 + assert ocr[0]["content"] == "MAGNA" + assert ocr[0]["h"] == 164 + assert ocr[0]["w"] == 758 + assert ocr[0]["x"] == 1894 + assert ocr[0]["y"] == 1787 - @mock_s3 + @mock_aws def test_add_json_ocr_by_filename(self): - """ Test get_ocr when OCR is JSON file (by filename). """ + """Test get_ocr when OCR is JSON file (by filename).""" bucket_name = encode_noid() manifest = ManifestFactory.create( - image_server = ImageServerFactory.create( - storage_service = 's3', + image_server=ImageServerFactory.create( + storage_service="s3", storage_path=bucket_name, - server_base='images.readux.ecds.emory' + server_base="images.readux.ecds.emory", ) ) - self.set_up_mock_s3(manifest) - tsv_file_path = 'apps/iiif/canvases/fixtures/ocr_words.json' + self.set_up_mock_aws(manifest) + tsv_file_path = "apps/iiif/canvases/fixtures/ocr_words.json" canvas = manifest.canvas_set.first() - canvas.ocr_file_path = f'{manifest.pid}/_*ocr*_/ocr_words.json' + canvas.ocr_file_path = f"{manifest.pid}/_*ocr*_/ocr_words.json" manifest.image_server.bucket.upload_file( - tsv_file_path, f'{manifest.pid}/_*ocr*_/ocr_words.json' + tsv_file_path, f"{manifest.pid}/_*ocr*_/ocr_words.json" ) ocr = services.get_ocr(canvas) - assert ocr[0]['content'] == 'Dope' + assert ocr[0]["content"] == "Dope" def test_none_ocr(self): - """ Test add_positional_ocr when fetched OCR is None. """ + """Test add_positional_ocr when fetched OCR is None.""" ocr = services.add_positional_ocr(self.canvas, None) assert ocr is None def test_none_alto_ocr(self): - """ Test add_positional_ocr when fetched OCR is None. """ + """Test add_positional_ocr when fetched OCR is None.""" ocr = services.parse_alto_ocr(None) assert ocr is None def test_not_tsv(self): - """ Test is_tsv with something that is not TSV """ - not_tsv = 'test string' + """Test is_tsv with something that is not TSV""" + not_tsv = "test string" is_tsv = services.is_tsv(not_tsv) self.assertFalse(is_tsv) @httpretty.httprettified(allow_net_connect=False) def test_ocr_from_oa_annotation(self): - """ Test deserializing OA annotations """ + """Test deserializing OA annotations""" canvas = CanvasFactory.create(manifest=ManifestFactory.create()) for _ in range(0, 12): AnnotationFactory.create(canvas=canvas) anno_list = serialize( - 'annotation_list', + "annotation_list", [canvas], - version='v2', - owners=[UserFactory.create(username='ocr')] + version="v2", + owners=[UserFactory.create(username="ocr")], ) canvas.annotation_set.clear() @@ -444,11 +492,13 @@ def test_ocr_from_oa_annotation(self): httpretty.register_uri( httpretty.GET, - f'https://readux.io/iiif/v2/{canvas.manifest.pid}/list/{canvas.pid}', + f"https://readux.io/iiif/v2/{canvas.manifest.pid}/list/{canvas.pid}", body=anno_list, - content_type="text/json" + content_type="text/json", ) - services.add_oa_annotations(f'https://readux.io/iiif/v2/{canvas.manifest.pid}/list/{canvas.pid}') + services.add_oa_annotations( + f"https://readux.io/iiif/v2/{canvas.manifest.pid}/list/{canvas.pid}" + ) canvas.refresh_from_db() assert canvas.annotation_set.count() == 12 diff --git a/apps/iiif/manifests/forms.py b/apps/iiif/manifests/forms.py index 0755a12af..a6bd07532 100644 --- a/apps/iiif/manifests/forms.py +++ b/apps/iiif/manifests/forms.py @@ -1,4 +1,5 @@ """Django Forms for export.""" + import csv from io import StringIO import logging @@ -8,7 +9,7 @@ from apps.iiif.manifests.models import Manifest from apps.iiif.canvases.models import Canvas -from apps.ingest.services import normalize_header +from .services import normalize_header LOGGER = logging.getLogger(__name__) @@ -17,50 +18,76 @@ # http://stackoverflow.com/questions/3927018/django-how-to-check-if-field-widget-is-checkbox-in-the-template setattr( forms.Field, - 'is_checkbox', - lambda self: isinstance(self.widget, forms.CheckboxInput) + "is_checkbox", + lambda self: isinstance(self.widget, forms.CheckboxInput), ) + class ManifestAdminForm(forms.ModelForm): """Form for adding or changing a manifest""" + class Meta: model = Manifest fields = ( - 'id', 'pid', 'label', 'summary', 'author', - 'published_city', 'published_date_edtf', 'published_date', 'publisher', 'languages', - 'pdf', 'metadata', 'viewingdirection', 'collections', - 'image_server', 'start_canvas', 'attribution', 'logo', 'logo_url', 'license', 'scanned_by', 'identifier', 'identifier_uri' + "id", + "pid", + "label", + "summary", + "author", + "published_city", + "published_date_edtf", + "published_date", + "publisher", + "languages", + "pdf", + "metadata", + "viewingdirection", + "collections", + "image_server", + "start_canvas", + "attribution", + "logo", + "logo_url", + "license", + "scanned_by", + "identifier", + "identifier_uri", ) + def __init__(self, *args, **kwargs): super(ManifestAdminForm, self).__init__(*args, **kwargs) if ( - 'instance' in kwargs and - hasattr(kwargs['instance'], 'canvas_set') and kwargs['instance'].canvas_set.exists() + "instance" in kwargs + and hasattr(kwargs["instance"], "canvas_set") + and kwargs["instance"].canvas_set.exists() ): - self.fields['start_canvas'].queryset = kwargs['instance'].canvas_set.all() + self.fields["start_canvas"].queryset = kwargs["instance"].canvas_set.all() else: - self.fields['start_canvas'].queryset = Canvas.objects.none() + self.fields["start_canvas"].queryset = Canvas.objects.none() + class ManifestsCollectionsForm(forms.ModelForm): """Form to add manifests to collections.""" + class Meta: model = Manifest - fields=('collections',) + fields = ("collections",) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.fields['collections'].widget = widgets.RelatedFieldWidgetWrapper( - self.fields['collections'].widget, - self.instance._meta.get_field('collections').remote_field, - admin_site, + self.fields["collections"].widget = widgets.RelatedFieldWidgetWrapper( + self.fields["collections"].widget, + self.instance._meta.get_field("collections").remote_field, + admin_site, ) + class ManifestCSVImportForm(forms.Form): """Form to import a CSV and update the metadata for multiple manifests""" csv_file = forms.FileField( required=True, - validators=[FileExtensionValidator(allowed_extensions=['csv'])], + validators=[FileExtensionValidator(allowed_extensions=["csv"])], label="CSV File", help_text="""

Provide a CSV with a pid column, whose value in each row must match @@ -75,16 +102,14 @@ class ManifestCSVImportForm(forms.Form): def clean(self): # check csv has pid column super().clean() - csv_file = self.cleaned_data.get('csv_file') + csv_file = self.cleaned_data.get("csv_file") if csv_file: reader = csv.DictReader( - normalize_header( - StringIO(csv_file.read().decode('utf-8')) - ), + normalize_header(StringIO(csv_file.read().decode("utf-8"))), ) - if 'pid' not in reader.fieldnames: + if "pid" not in reader.fieldnames: self.add_error( - 'metadata_spreadsheet', + "metadata_spreadsheet", forms.ValidationError( """Spreadsheet must have pid column. Check to ensure there are no stray characters in the header row.""" diff --git a/apps/ingest/services.py b/apps/iiif/manifests/services.py similarity index 69% rename from apps/ingest/services.py rename to apps/iiif/manifests/services.py index a5660c65d..85e3a5021 100644 --- a/apps/ingest/services.py +++ b/apps/iiif/manifests/services.py @@ -1,4 +1,5 @@ """ Module of service classes and methods for ingest. """ + import itertools import re from mimetypes import guess_type @@ -7,7 +8,7 @@ from django.apps import apps from tablib.core import Dataset -from apps.iiif.manifests.models import Manifest, RelatedLink +from .models import Manifest, RelatedLink def clean_metadata(metadata): @@ -24,20 +25,22 @@ def clean_metadata(metadata): key.casefold().replace(" ", "_") if key.casefold().replace(" ", "_") in fields else key - ): value for key, value in metadata.items() + ): value + for key, value in metadata.items() } for key in metadata.keys(): - if key != 'metadata' and isinstance(metadata[key], list): + if key != "metadata" and isinstance(metadata[key], list): if isinstance(metadata[key][0], dict): for meta_key in metadata[key][0].keys(): - if 'value' in meta_key: + if "value" in meta_key: metadata[key] = metadata[key][0][meta_key] else: - metadata[key] = ', '.join(metadata[key]) + metadata[key] = ", ".join(metadata[key]) return metadata + def create_related_links(manifest, related_str): """ Create RelatedLink objects from supplied related links string and associate each with supplied @@ -53,10 +56,12 @@ def create_related_links(manifest, related_str): RelatedLink.objects.create( manifest=manifest, link=link, - format=format or "text/html", # assume web page if MIME type cannot be determined + format=format + or "text/html", # assume web page if MIME type cannot be determined is_structured_data=False, # assume this is not meant for seeAlso ) + def set_metadata(manifest, metadata): """ Update Manifest.metadata using supplied metadata dict @@ -67,7 +72,7 @@ def set_metadata(manifest, metadata): :rtype: None """ fields = [f.name for f in Manifest._meta.get_fields()] - for (key, value) in metadata.items(): + for key, value in metadata.items(): casefolded_key = key.casefold().replace(" ", "_") if casefolded_key == "related": # add RelatedLinks from metadata spreadsheet key "related" @@ -105,6 +110,7 @@ def set_metadata(manifest, metadata): manifest.metadata = [{"label": key, "value": value}] manifest.save() + def create_manifest(ingest): """ Create or update a Manifest from supplied metadata and images. @@ -121,8 +127,10 @@ def create_manifest(ingest): except TypeError: metadata = None if metadata: - if 'pid' in metadata: - manifest, created = Manifest.objects.get_or_create(pid=metadata['pid'].replace('_', '-')) + if "pid" in metadata: + manifest, created = Manifest.objects.get_or_create( + pid=metadata["pid"].replace("_", "-") + ) else: manifest = Manifest.objects.create() set_metadata(manifest, metadata) @@ -132,7 +140,7 @@ def create_manifest(ingest): manifest.image_server = ingest.image_server # This was giving me a 'django.core.exceptions.AppRegistryNotReady: Models aren't loaded yet' error. - Remote = apps.get_model('ingest.remote') + Remote = apps.get_model("ingest.remote") # Ensure that manifest has an ID before updating the M2M relationship manifest.save() @@ -145,12 +153,13 @@ def create_manifest(ingest): RelatedLink( manifest=manifest, link=ingest.remote_url, - format='application/ld+json', + format="application/ld+json", is_structured_data=True, ).save() return manifest + def extract_image_server(canvas): """Determines the IIIF image server URL for a given IIIF Canvas @@ -159,14 +168,15 @@ def extract_image_server(canvas): :return: IIIF image server URL :rtype: str """ - url = urlparse(canvas['images'][0]['resource']['service']['@id']) - parts = url.path.split('/') + url = urlparse(canvas["images"][0]["resource"]["service"]["@id"]) + parts = url.path.split("/") parts.pop() - base_path = '/'.join(parts) + base_path = "/".join(parts) host = url.hostname if url.port is not None: - host = '{h}:{p}'.format(h=url.hostname, p=url.port) - return '{s}://{h}{p}'.format(s=url.scheme, h=host, p=base_path) + host = "{h}:{p}".format(h=url.hostname, p=url.port) + return "{s}://{h}{p}".format(s=url.scheme, h=host, p=base_path) + def parse_iiif_v2_manifest(data): """Parse IIIF Manifest based on v2.1.1 or the presentation API. @@ -180,65 +190,77 @@ def parse_iiif_v2_manifest(data): properties = {} manifest_data = [] - if 'metadata' in data: - manifest_data.append({ 'metadata': data['metadata'] }) + if "metadata" in data: + manifest_data.append({"metadata": data["metadata"]}) - for iiif_metadata in [{prop['label']: prop['value']} for prop in data['metadata']]: + for iiif_metadata in [ + {prop["label"]: prop["value"]} for prop in data["metadata"] + ]: properties.update(iiif_metadata) # Sometimes, the label appears as a list. - if 'label' in data.keys() and isinstance(data['label'], list): - data['label'] = ' '.join(data['label']) + if "label" in data.keys() and isinstance(data["label"], list): + data["label"] = " ".join(data["label"]) - manifest_data.extend([{prop: data[prop]} for prop in data if isinstance(data[prop], str)]) + manifest_data.extend( + [{prop: data[prop]} for prop in data if isinstance(data[prop], str)] + ) for datum in manifest_data: properties.update(datum) - uri = urlparse(data['@id']) + uri = urlparse(data["@id"]) if not uri.query: - properties['pid'] = uri.path.split('/')[-2] + properties["pid"] = uri.path.split("/")[-2] else: - properties['pid'] = uri.query - - if 'description' in data.keys(): - if isinstance(data['description'], list): - if isinstance(data['description'][0], dict): - en = [lang['@value'] for lang in data['description'] if lang['@language'] == 'en'] - properties['summary'] = data['description'][0]['@value'] if not en else en[0] + properties["pid"] = uri.query + + if "description" in data.keys(): + if isinstance(data["description"], list): + if isinstance(data["description"][0], dict): + en = [ + lang["@value"] + for lang in data["description"] + if lang["@language"] == "en" + ] + properties["summary"] = ( + data["description"][0]["@value"] if not en else en[0] + ) else: - properties['summary'] = data['description'][0] + properties["summary"] = data["description"][0] else: - properties['summary'] = data['description'] + properties["summary"] = data["description"] - if 'logo' in properties: - properties['logo_url'] = properties['logo'] - properties.pop('logo') + if "logo" in properties: + properties["logo_url"] = properties["logo"] + properties.pop("logo") manifest_metadata = clean_metadata(properties) return manifest_metadata + def parse_iiif_v2_canvas(canvas): """ """ - canvas_id = canvas['@id'].split('/') - pid = canvas_id[-1] if canvas_id[-1] != 'canvas' else canvas_id[-2] + canvas_id = canvas["@id"].split("/") + pid = canvas_id[-1] if canvas_id[-1] != "canvas" else canvas_id[-2] - service = urlparse(canvas['images'][0]['resource']['service']['@id']) - resource = unquote(service.path.split('/').pop()) + service = urlparse(canvas["images"][0]["resource"]["service"]["@id"]) + resource = unquote(service.path.split("/").pop()) - summary = canvas['description'] if 'description' in canvas.keys() else '' - label = canvas['label'] if 'label' in canvas.keys() else '' + summary = canvas["description"] if "description" in canvas.keys() else "" + label = canvas["label"] if "label" in canvas.keys() else "" return { - 'pid': pid, - 'height': canvas['height'], - 'width': canvas['width'], - 'summary': summary, - 'label': label, - 'resource': resource + "pid": pid, + "height": canvas["height"], + "width": canvas["width"], + "summary": summary, + "label": label, + "resource": resource, } + def get_metadata_from(files): """ Find metadata file in uploaded files. @@ -249,16 +271,20 @@ def get_metadata_from(files): for file in files: if metadata is not None: continue - if 'zip' in guess_type(file.name)[0]: + if "zip" in guess_type(file.name)[0]: continue - if 'metadata' in file.name.casefold(): + if "metadata" in file.name.casefold(): stream = file.read() - if 'csv' in guess_type(file.name)[0] or 'tab-separated' in guess_type(file.name)[0]: - metadata = Dataset().load(stream.decode('utf-8-sig'), format='csv').dict + if ( + "csv" in guess_type(file.name)[0] + or "tab-separated" in guess_type(file.name)[0] + ): + metadata = Dataset().load(stream.decode("utf-8-sig"), format="csv").dict else: metadata = Dataset().load(stream).dict return metadata + def get_associated_meta(all_metadata, file): """ Associate metadata with filename. @@ -267,17 +293,21 @@ def get_associated_meta(all_metadata, file): :rtype: dict """ file_meta = {} - extless_filename = file.name[0:file.name.rindex('.')] + extless_filename = file.name[0 : file.name.rindex(".")] for meta_dict in all_metadata: metadata_found_filename = None for key, val in meta_dict.items(): - if key.casefold() == 'filename': + if key.casefold() == "filename": metadata_found_filename = val # Match filename column, case-sensitive, against filename - if metadata_found_filename and metadata_found_filename in (extless_filename, file.name): + if metadata_found_filename and metadata_found_filename in ( + extless_filename, + file.name, + ): file_meta = meta_dict return file_meta + def normalize_header(iterator): """Normalize the header row of a metadata CSV""" # ignore unicode characters and strip whitespace diff --git a/apps/iiif/manifests/tests/tests.py b/apps/iiif/manifests/tests/tests.py index 476493c9e..1370ec18e 100644 --- a/apps/iiif/manifests/tests/tests.py +++ b/apps/iiif/manifests/tests/tests.py @@ -1,10 +1,11 @@ """ Test class for IIIF Manifests """ + import json import random import boto3 -from moto import mock_s3 +from moto import mock_aws from datetime import datetime from time import sleep from django.test import TestCase, Client @@ -24,15 +25,17 @@ USER = get_user_model() + # FIXME: Extend `TestCase` to mock all HTTP requests. class ManifestTests(TestCase): """Tests for IIIF Manifests""" + fixtures = [ - 'users.json', - 'kollections.json', - 'manifests.json', - 'canvases.json', - 'annotations.json' + "users.json", + "kollections.json", + "manifests.json", + "canvases.json", + "annotations.json", ] def setUp(self): @@ -40,19 +43,18 @@ def setUp(self): self.user = get_user_model().objects.get(pk=111) self.factory = RequestFactory() self.client = Client() - self.volume = ManifestFactory.create( - publisher='ECDS', - published_city='Atlanta' - ) + self.volume = ManifestFactory.create(publisher="ECDS", published_city="Atlanta") for num in [1, 2, 3]: CanvasFactory.create( pid=str(random.randrange(2000, 5000)), manifest=self.volume, - position=num + position=num, ) self.volume.save() self.start_canvas = self.volume.start_canvas - self.default_start_canvas = self.volume.canvas_set.filter(is_starting_page=False).first() + self.default_start_canvas = self.volume.canvas_set.filter( + is_starting_page=False + ).first() self.assumed_label = self.volume.label self.assumed_pid = self.volume @@ -86,9 +88,11 @@ def setUp(self): def test_properties(self): assert self.volume.publisher_bib == "Atlanta : ECDS" - assert self.volume.logo.url.endswith('/media/logos/ecds.png') + assert self.volume.logo.url.endswith("/media/logos/ecds.png") assert self.volume.baseurl.endswith("/iiif/v2/%s" % (self.volume.pid)) - assert self.volume.start_canvas.identifier.endswith("/iiif/%s/canvas/%s" % (self.volume.pid, self.start_canvas.pid)) + assert self.volume.start_canvas.identifier.endswith( + "/iiif/%s/canvas/%s" % (self.volume.pid, self.start_canvas.pid) + ) def test_default_start_canvas(self): image_server = ImageServerFactory.create(server_base="https://fake.info") @@ -111,19 +115,27 @@ def test_sitemap(self): def test_ris_view(self): ris = ManifestRis() - assert ris.get_context_data(volume=self.assumed_pid.pid)['volume'] == self.volume + assert ( + ris.get_context_data(volume=self.assumed_pid.pid)["volume"] == self.volume + ) def test_plain_export_view(self): - kwargs = { 'pid': self.volume.pid, 'version': 'v2' } - url = reverse('PlainExport', kwargs=kwargs) + kwargs = {"pid": self.volume.pid, "version": "v2"} + url = reverse("PlainExport", kwargs=kwargs) response = self.client.get(url) assert response.status_code == 200 def test_autocomplete_label(self): - assert Manifest.objects.all().first().autocomplete_label() == Manifest.objects.all().first().pid + assert ( + Manifest.objects.all().first().autocomplete_label() + == Manifest.objects.all().first().pid + ) def test_absolute_url(self): - assert Manifest.objects.all().first().get_absolute_url() == "%s/volume/%s" % (settings.HOSTNAME, Manifest.objects.all().first().pid) + assert Manifest.objects.all().first().get_absolute_url() == "%s/volume/%s" % ( + settings.HOSTNAME, + Manifest.objects.all().first().pid, + ) # def test_manifest_search_vector_exists(self): # assert self.volume.search_vector is None @@ -134,80 +146,82 @@ def test_absolute_url(self): def test_multiple_starting_canvases(self): volume = ManifestFactory.create() for index, _ in enumerate(range(4)): - CanvasFactory.create(manifest=volume, is_starting_page=True, position=index+1) + CanvasFactory.create( + manifest=volume, is_starting_page=True, position=index + 1 + ) sleep(2) # volume.refresh_from_db() manifest = json.loads( serialize( - 'manifest', + "manifest", [volume], - version='v2', - annotators='Tom', - exportdate=datetime.utcnow() + version="v2", + annotators="Tom", + exportdate=datetime.utcnow(), ) ) - first_canvas = volume.canvas_set.all().order_by('position').first() - assert volume.start_canvas.pid in manifest['thumbnail']['@id'] + first_canvas = volume.canvas_set.all().order_by("position").first() + assert volume.start_canvas.pid in manifest["thumbnail"]["@id"] def test_no_starting_canvases(self): manifest = ManifestFactory.create() try: manifest.canvas_set.all().get(is_starting_page=True) except Canvas.DoesNotExist as error: - assert str(error) == 'Canvas matching query does not exist.' + assert str(error) == "Canvas matching query does not exist." manifest.refresh_from_db() - serialized_manifest = json.loads( - serialize( - 'manifest', - [manifest] - ) + serialized_manifest = json.loads(serialize("manifest", [manifest])) + assert ( + manifest.canvas_set.all().first().pid + in serialized_manifest["thumbnail"]["@id"] ) - assert manifest.canvas_set.all().first().pid in serialized_manifest['thumbnail']['@id'] def test_default_iiif_image_server_url(self): image_server = ImageServer() assert image_server.server_base == settings.IIIF_IMAGE_SERVER_BASE def test_serialized_related_links(self): - """ It should add a list of links for the "seeAlso" key. """ + """It should add a list of links for the "seeAlso" key.""" manifest = ManifestFactory.create() - no_links = json.loads( - serialize( - 'manifest', - [manifest] - ) - ) - assert not no_links['seeAlso'] + no_links = json.loads(serialize("manifest", [manifest])) + assert not no_links["seeAlso"] - link = RelatedLink(link='images.org', manifest=manifest, is_structured_data=True) + link = RelatedLink( + link="images.org", manifest=manifest, is_structured_data=True + ) link.save() manifest.refresh_from_db() - with_links = json.loads( - serialize( - 'manifest', - [manifest] - ) - ) + with_links = json.loads(serialize("manifest", [manifest])) - assert 'seeAlso' in with_links.keys() - assert isinstance(with_links['seeAlso'], list) - assert len(with_links['seeAlso']) == 1 - assert with_links['seeAlso'][0] == 'images.org' + assert "seeAlso" in with_links.keys() + assert isinstance(with_links["seeAlso"], list) + assert len(with_links["seeAlso"]) == 1 + assert with_links["seeAlso"][0] == "images.org" - @mock_s3 + @mock_aws def test_renameing_pid_when_images_are_in_s3(self): - """ It should copy the canvas files to a folder with new pid and delete old pid. """ - image_server = ImageServerFactory.create(storage_path='earthgang', storage_service='s3') + """It should copy the canvas files to a folder with new pid and delete old pid.""" + image_server = ImageServerFactory.create( + storage_path="earthgang", storage_service="s3" + ) manifest = ManifestFactory(image_server=image_server) - conn = boto3.resource('s3', region_name='us-east-1') + conn = boto3.resource("s3", region_name="us-east-1") conn.create_bucket(Bucket=image_server.storage_path) original_pid = manifest.pid - image_server.bucket.upload_file('apps/iiif/canvases/fixtures/00000002.jpg', f'{manifest.pid}/00000002.jpg') - assert f'{manifest.pid}/00000002.jpg' in [f.key for f in image_server.bucket.objects.all()] + image_server.bucket.upload_file( + "apps/iiif/canvases/fixtures/00000002.jpg", f"{manifest.pid}/00000002.jpg" + ) + assert f"{manifest.pid}/00000002.jpg" in [ + f.key for f in image_server.bucket.objects.all() + ] manifest.pid = encode_noid(0) manifest.save() manifest.refresh_from_db() - assert f'{original_pid}/00000002.jpg' not in [f.key for f in image_server.bucket.objects.all()] + assert f"{original_pid}/00000002.jpg" not in [ + f.key for f in image_server.bucket.objects.all() + ] assert original_pid not in [f.key for f in image_server.bucket.objects.all()] - assert f'{manifest.pid}/00000002.jpg' in [f.key for f in image_server.bucket.objects.all()] + assert f"{manifest.pid}/00000002.jpg" in [ + f.key for f in image_server.bucket.objects.all() + ] diff --git a/apps/iiif/manifests/views.py b/apps/iiif/manifests/views.py index 3642ae210..6bf05bb04 100644 --- a/apps/iiif/manifests/views.py +++ b/apps/iiif/manifests/views.py @@ -15,23 +15,25 @@ from django.contrib.sitemaps import Sitemap from django.urls import reverse -from apps.ingest.services import normalize_header, set_metadata +from .services import normalize_header, set_metadata from .models import Manifest from .forms import ManifestCSVImportForm, ManifestsCollectionsForm LOGGER = logging.getLogger(__name__) + class ManifestDetail(View): """Endpoint for a specific IIIF manifest.""" + def get_queryset(self): """Get requested manifest object. :return: Manifest object :rtype: django.db.models.QuerySet """ - return Manifest.objects.filter(pid=self.kwargs['pid']) + return Manifest.objects.filter(pid=self.kwargs["pid"]) - def get(self, request, *args, **kwargs): # pylint: disable = unused-argument + def get(self, request, *args, **kwargs): # pylint: disable = unused-argument """Responds to HTTP GET request for specific manifest. :return: IIIF representation of a manifest/volume @@ -42,35 +44,35 @@ def get(self, request, *args, **kwargs): # pylint: disable = unused-argument annotators = [] if request.user.is_authenticated: annotators.append(request.user) - annotators_string = ', '.join([str(i.name) for i in annotators]) - if '2' in kwargs['version']: + annotators_string = ", ".join([str(i.name) for i in annotators]) + if "2" in kwargs["version"]: return JsonResponse( json.loads( serialize( - 'manifest', + "manifest", self.get_queryset(), - version=kwargs['version'], + version=kwargs["version"], annotators=annotators_string, exportdate=datetime.utcnow(), - current_user=request.user + current_user=request.user, ) ), - safe=False + safe=False, ) - elif '3' in kwargs['version']: + elif "3" in kwargs["version"]: return JsonResponse( json.loads( serialize( - 'manifest_v3', - self.get_queryset(), - current_user=request.user + "manifest_v3", self.get_queryset(), current_user=request.user ) ), - safe=False + safe=False, ) + class AllVolumesCollection(View): """Endpoint for all volumes collection.""" + def get_queryset(self): """Get all manifest objects. @@ -79,7 +81,7 @@ def get_queryset(self): """ return Manifest.objects.all() - def get(self, request, *args, **kwargs): # pylint: disable = unused-argument + def get(self, request, *args, **kwargs): # pylint: disable = unused-argument """Responds to HTTP GET request for all manifests. :return: IIIF representation of all volumes collection @@ -91,28 +93,28 @@ def get(self, request, *args, **kwargs): # pylint: disable = unused-argument "@context": "http://iiif.io/api/presentation/2/context.json", "label": "All Readux volumes", "manifests": json.loads( - serialize( - 'all_volumes_manifest', - self.get_queryset(), - is_list=True - ) - ) + serialize("all_volumes_manifest", self.get_queryset(), is_list=True) + ), } - return JsonResponse(collection,safe=False) + return JsonResponse(collection, safe=False) + class ManifestSitemap(Sitemap): """Django site map for manifests""" + # priority unknown def items(self): return Manifest.objects.all() def location(self, item): - return reverse('ManifestRender', kwargs={'version': 'v2', 'pid': item.pid}) + return reverse("ManifestRender", kwargs={"version": "v2", "pid": item.pid}) + class ManifestRis(TemplateView): """Manifest Ris""" - content_type = 'application/x-research-info-systems; charset=UTF-8' + + content_type = "application/x-research-info-systems; charset=UTF-8" template_name = "citation.ris" def get_context_data(self, **kwargs): @@ -122,23 +124,24 @@ def get_context_data(self, **kwargs): :rtype: [type] """ context = super().get_context_data(**kwargs) - context['volume'] = Manifest.objects.filter(pid=kwargs['volume']).first() + context["volume"] = Manifest.objects.filter(pid=kwargs["volume"]).first() return context + class AddToCollectionsView(FormView): """Intermediate page to choose collections to which you are adding manifests""" - template_name = 'add_manifests_to_collections.html' + template_name = "add_manifests_to_collections.html" form_class = ManifestsCollectionsForm def get_context_data(self, **kwargs): - ids = self.request.GET.get('ids', '').split(',') + ids = self.request.GET.get("ids", "").split(",") manifests = Manifest.objects.filter(pk__in=ids) - model_admin = self.kwargs['model_admin'] + model_admin = self.kwargs["model_admin"] context = super().get_context_data(**kwargs) - context['model_admin'] = model_admin.admin_site.each_context(self.request) - context['manifests'] = manifests - context['title'] = 'Add selected manifests to collection(s)' + context["model_admin"] = model_admin.admin_site.each_context(self.request) + context["manifests"] = manifests + context["title"] = "Add selected manifests to collection(s)" return context def form_valid(self, form): @@ -148,54 +151,59 @@ def form_valid(self, form): def add_manifests_to_collections(self, form): """Adds selected manifests to selected collections from form""" context = self.get_context_data() - manifests = context['manifests'] + manifests = context["manifests"] if form.is_valid(): - collections = form.cleaned_data['collections'] + collections = form.cleaned_data["collections"] for manifest in manifests: manifest.collections.add(*collections) manifest.save() def get_success_url(self): messages.add_message( - self.request, messages.SUCCESS, 'Successfully added manifests to collections' + self.request, + messages.SUCCESS, + "Successfully added manifests to collections", ) - return reverse('admin:manifests_manifest_changelist') + return reverse("admin:manifests_manifest_changelist") + class MetadataImportView(FormView): """Admin page to import a CSV and update multiple Manifests' metadata""" - template_name = 'manifest_metadata_import.html' + template_name = "manifest_metadata_import.html" form_class = ManifestCSVImportForm def get_context_data(self, **kwargs): """Set page title on context data""" context = super().get_context_data(**kwargs) - context['title'] = 'Manifest metadata bulk update (CSV import)' + context["title"] = "Manifest metadata bulk update (CSV import)" return context def form_valid(self, form): """Read the CSV file and, find associated manifests, and update metadata""" form.is_valid() - csv_file = form.cleaned_data.get('csv_file') - csv_io = StringIO(csv_file.read().decode('utf-8')) + csv_file = form.cleaned_data.get("csv_file") + csv_io = StringIO(csv_file.read().decode("utf-8")) reader = csv.DictReader(normalize_header(csv_io)) for row in reader: try: # try to find manifest - manifest = Manifest.objects.get(pid=row['pid']) + manifest = Manifest.objects.get(pid=row["pid"]) # use ingest set_metadata function set_metadata(manifest, metadata=row) except Manifest.DoesNotExist: messages.add_message( - self.request, messages.ERROR, f'Manifest with pid {row["pid"]} not found' + self.request, + messages.ERROR, + f'Manifest with pid {row["pid"]} not found', ) return super().form_valid(form) def get_success_url(self): """Return to the manifest change list with a success message""" # https://stackoverflow.com/questions/11938164/why-dont-my-django-unittests-know-that-messagemiddleware-is-installed - if os.environ['DJANGO_ENV'] != 'test': + if os.environ["DJANGO_ENV"] != "test": messages.add_message( - self.request, messages.SUCCESS, 'Successfully updated manifests' + self.request, messages.SUCCESS, "Successfully updated manifests" ) - return reverse('admin:manifests_manifest_changelist') + return reverse("admin:manifests_manifest_changelist") diff --git a/apps/ingest/__init__.py b/apps/ingest/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/apps/ingest/admin.py b/apps/ingest/admin.py deleted file mode 100644 index 1536e6fcb..000000000 --- a/apps/ingest/admin.py +++ /dev/null @@ -1,322 +0,0 @@ -"""[summary]""" -import csv -from io import StringIO -import logging -from mimetypes import guess_type -from os import environ, listdir, path, remove, rmdir - -from django import forms -from django.contrib import admin -from django.core.files.base import ContentFile -from django.shortcuts import redirect -from django.urls import reverse -from django.utils.html import format_html -from django_celery_results.models import TaskResult - -from apps.ingest import tasks - -from .forms import BulkVolumeUploadForm -from .models import Bulk, IngestTaskWatcher, Local, Remote, S3Ingest -from .services import (clean_metadata, create_manifest, get_associated_meta, - get_metadata_from, normalize_header) - -LOGGER = logging.getLogger(__name__) -class LocalAdmin(admin.ModelAdmin): - """Django admin ingest.models.local resource.""" - fields = ('bundle', 'image_server', 'collections') - show_save_and_add_another = False - - def save_model(self, request, obj, form, change): - obj.save() - obj.manifest = create_manifest(obj) - obj.creator = request.user - obj.save() - obj.refresh_from_db() - super().save_model(request, obj, form, change) - if environ["DJANGO_ENV"] != 'test': # pragma: no cover - local_task_id = tasks.create_canvas_form_local_task.apply_async(args=[obj.id]) - local_task_result = TaskResult(task_id=local_task_id) - local_task_result.save() - file = request.FILES['bundle'] - IngestTaskWatcher.manager.create_watcher( - task_id=local_task_id, - task_result=local_task_result, - task_creator=request.user, - associated_manifest=obj.manifest, - filename=file.name - ) - else: - tasks.create_canvas_form_local_task(obj.id) - - def response_add(self, request, obj, post_url_continue=None): - obj.refresh_from_db() - manifest_id = obj.manifest.id - return redirect('/admin/manifests/manifest/{m}/change/'.format(m=manifest_id)) - - class Meta: # pylint: disable=too-few-public-methods, missing-class-docstring - model = Local - -class RemoteAdmin(admin.ModelAdmin): - """Django admin ingest.models.remote resource.""" - fields = ('remote_url',) - show_save_and_add_another = False - - def save_model(self, request, obj, form, change): - obj.manifest = create_manifest(obj) - obj.save() - obj.refresh_from_db() - super().save_model(request, obj, form, change) - if environ["DJANGO_ENV"] != 'test': # pragma: no cover - remote_task_id = tasks.create_remote_canvases.delay(obj.id) - remote_task_result = TaskResult(task_id=remote_task_id) - remote_task_result.save() - IngestTaskWatcher.manager.create_watcher( - task_id=remote_task_id, - task_result=remote_task_result, - task_creator=request.user, - filename=obj.remote_url, - associated_manifest=obj.manifest - ) - - def response_add(self, request, obj, post_url_continue=None): - obj.refresh_from_db() - manifest_id = obj.manifest.id - return redirect('/admin/manifests/manifest/{m}/change/'.format(m=manifest_id)) - -class BulkAdmin(admin.ModelAdmin): - """Django admin ingest.models.bulk resource.""" - - form = BulkVolumeUploadForm - - def save_model(self, request, obj, form, change): - # Save M2M relationships with collections so we can access them later - if form.is_valid(): - form.save(commit=False) - form.save_m2m() - obj.save() - # Get files from multi upload form - files = request.FILES.getlist("volume_files") - # Find the metadata file and load it into list of dicts - all_metadata = get_metadata_from(files) - for index, file in enumerate(files): - # Skip metadata file now - if 'metadata' in file.name.casefold() and 'zip' not in guess_type(file.name)[0]: - continue - - LOGGER.debug(f'Creating local ingest for {file.name}') - - # Associate metadata with zipfile - if all_metadata is not None: - file_meta = clean_metadata(get_associated_meta(all_metadata, file)) - else: - file_meta = {} - - # Create a Local object - new_local = Local.objects.create( - bulk=obj, - image_server=obj.image_server, - creator=request.user - ) - new_local.collections.set(obj.collections.all()) - if file_meta: - new_local.metadata=file_meta - - # Save tempfile in bundle_from_bulk - with ContentFile(file.read()) as file_content: - new_local.bundle_from_bulk.save(file.name, file_content) - new_local.save() - new_local.refresh_from_db() - - # Queue task to upload to S3 - if environ["DJANGO_ENV"] != 'test': # pragma: no cover - delay = index * 60 - upload_task = tasks.upload_to_s3_task.apply_async( - (new_local.id,), - countdown=delay - ) - upload_task_result = TaskResult(task_id=upload_task.id) - upload_task_result.save() - IngestTaskWatcher.manager.create_watcher( - task_id=upload_task.id, - task_result=upload_task_result, - task_creator=request.user, - filename=file.name - ) - obj.refresh_from_db() - super().save_model(request, obj, form, change) - - def response_add(self, request, obj, post_url_continue=None): - # Delete local file - file_path = obj.volume_files.path - if path.isfile(file_path): - remove(file_path) - else: - LOGGER.error(f"Could not cleanup {file_path}") - dir_path = file_path[0:file_path.rindex('/')] - if not path.isfile(file_path) and len(listdir(dir_path)) == 0: - rmdir(dir_path) - obj.delete() - url_to = reverse('admin:ingest_ingesttaskwatcher_changelist') - return redirect(url_to) - - class Meta: # pylint: disable=too-few-public-methods, missing-class-docstring - model = Bulk - -class TaskWatcherAdmin(admin.ModelAdmin): - """Django admin for ingest.models.IngestTaskWatcher resource.""" - - list_display = ( - "id", - "filename", - "task_name", - "task_status", - "task_creator", - "date_created", - "date_done", - ) - fields = ( - "id", - "filename", - "task_name", - "task_status", - "associated_manifest", - "task_creator", - "date_created", - "date_done", - ) - list_filter = ('task_result__task_name', 'task_result__status', 'task_creator', 'associated_manifest__pid') - search_fields = ('filename',) - date_hierarchy = 'task_result__date_created' - empty_value_display = '(none)' - - def task_status(self, obj): - """ Returns the task result with a link to view its details """ - if obj.task_result: - url = reverse('admin:%s_%s_change' % ( - obj.task_result._meta.app_label, - obj.task_result._meta.model_name - ), args=[obj.task_result.id] ) - return format_html( - "{label}", - url=url, - label=obj.task_result.status - ) - return None - task_status.admin_order_field = 'task_result__status' - - def task_name(self, obj): - """ Returns the task name for this task """ - if obj.task_result: - return obj.task_result.task_name - return None - task_name.admin_order_field = 'task_result__task_name' - - def date_created(self, obj): - """ Returns the creation date for this task """ - if obj.task_result: - return obj.task_result.date_created - return None - date_created.admin_order_field = 'task_result__date_created' - - def date_done(self, obj): - """ Returns the finished date for this task """ - if obj.task_result: - return obj.task_result.date_done - return None - date_done.admin_order_field = 'task_result__date_done' - - def has_add_permission(self, request): - return False - - def has_change_permission(self, request, obj=None): - return False - - -class S3IngestForm(forms.ModelForm): - """Override the admin form in order to add validation to clean method""" - - class Meta: # pylint: disable=too-few-public-methods, missing-class-docstring - model = S3Ingest - exclude = ("creator",) - - def clean(self): - # check s3 bucket is name not url, and csv has pid column - super().clean() - if any( - self.cleaned_data.get('s3_bucket').casefold().startswith(protocol) - for protocol in ('http:', 'https:', 's3:') - ): - self.add_error( - 's3_bucket', - forms.ValidationError('S3 Bucket should use name, not URL'), - ) - csv_file = self.cleaned_data.get('metadata_spreadsheet') - if csv_file: - reader = csv.DictReader( - normalize_header( - StringIO(csv_file.read().decode('utf-8')) - ), - ) - if 'pid' not in reader.fieldnames: - self.add_error( - 'metadata_spreadsheet', - forms.ValidationError( - """Spreadsheet must have pid column. Check to ensure there - are no stray characters in the header row.""" - ), - ) - - return self.cleaned_data - -class S3IngestAdmin(admin.ModelAdmin): - """Django admin for ingest.models.S3Ingest resource.""" - - form = S3IngestForm - - def save_model(self, request, obj, form, change): - # Save M2M relationships with collections so we can access them later - if form.is_valid(): - form.save(commit=False) - form.save_m2m() - obj.creator = request.user - obj.save() - obj.refresh_from_db() - # Get spreadsheet with metadata to match each volume - obj.metadata_spreadsheet.seek(0) - metadata = csv.DictReader( - normalize_header( - StringIO(obj.metadata_spreadsheet.read().decode('utf-8')) - ), - ) - for index, row in enumerate(metadata): - # each row needs to have a pid to match it with a volume directory in s3 - if "pid" not in row: - LOGGER.error(f'No pid found in row {index}') - continue - - LOGGER.debug(f'Creating manifest for {row["pid"]}') - file_meta = clean_metadata(row) - # Queue task to create canvases - if environ["DJANGO_ENV"] != 'test': # pragma: no cover - create_canvases_task = tasks.create_canvases_from_s3_ingest.apply_async( - (file_meta, obj.pk) - ) - create_canvases_task_result = TaskResult(task_id=create_canvases_task.id) - create_canvases_task_result.save() - IngestTaskWatcher.manager.create_watcher( - task_id=create_canvases_task.id, - task_result=create_canvases_task_result, - task_creator=request.user, - filename=f"s3://{obj.s3_bucket}/{obj.s3_prefix}/{row['pid']}" - ) - super().save_model(request, obj, form, change) - - def response_add(self, request, obj, post_url_continue=None): - # redirect to the task results list after saving - return redirect(reverse("admin:django_celery_results_taskresult_changelist")) - -admin.site.register(Local, LocalAdmin) -admin.site.register(Remote, RemoteAdmin) -admin.site.register(Bulk, BulkAdmin) -admin.site.register(IngestTaskWatcher, TaskWatcherAdmin) -admin.site.register(S3Ingest, S3IngestAdmin) diff --git a/apps/ingest/apps.py b/apps/ingest/apps.py deleted file mode 100644 index 1db540292..000000000 --- a/apps/ingest/apps.py +++ /dev/null @@ -1,8 +0,0 @@ -"""Configuration for :class:`apps.ingest`""" -from django.apps import AppConfig - - -class IngestConfig(AppConfig): - """Ingest config""" - name = 'apps.ingest' - verbose_name = 'Ingest' diff --git a/apps/ingest/celery.py b/apps/ingest/celery.py deleted file mode 100644 index 0b506b526..000000000 --- a/apps/ingest/celery.py +++ /dev/null @@ -1,16 +0,0 @@ -""" -Celery config for ingest tasks. -""" -import os -from celery import Celery -from django.conf import settings -# import config.settings.local as settings - -# set the default Django settings module for the 'celery' program. -os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'config.settings.local') -app = Celery('apps.ingest', result_extended=True) - -# Using a string here means the worker will not have to -# pickle the object when using Windows. -app.config_from_object('django.conf:settings') -app.autodiscover_tasks(lambda: settings.INSTALLED_APPS) diff --git a/apps/ingest/fixtures/bundle.zip b/apps/ingest/fixtures/bundle.zip deleted file mode 100644 index 65227898a..000000000 Binary files a/apps/ingest/fixtures/bundle.zip and /dev/null differ diff --git a/apps/ingest/fixtures/bundle_with_junk.zip b/apps/ingest/fixtures/bundle_with_junk.zip deleted file mode 100644 index 22f61f654..000000000 Binary files a/apps/ingest/fixtures/bundle_with_junk.zip and /dev/null differ diff --git a/apps/ingest/fixtures/bundle_with_underscores.zip b/apps/ingest/fixtures/bundle_with_underscores.zip deleted file mode 100644 index 6e266031b..000000000 Binary files a/apps/ingest/fixtures/bundle_with_underscores.zip and /dev/null differ diff --git a/apps/ingest/fixtures/csv_meta.zip b/apps/ingest/fixtures/csv_meta.zip deleted file mode 100644 index 44025ef88..000000000 Binary files a/apps/ingest/fixtures/csv_meta.zip and /dev/null differ diff --git a/apps/ingest/fixtures/heidelberg.json b/apps/ingest/fixtures/heidelberg.json deleted file mode 100644 index 1430de151..000000000 --- a/apps/ingest/fixtures/heidelberg.json +++ /dev/null @@ -1,1841 +0,0 @@ -{ - "@context" : "http://iiif.io/api/presentation/2/context.json", - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/manifest", - "@type" : "sc:Manifest", - "attribution" : [ - { - "@language" : "de", - "@value" : "Universit\u00e4tsbibliothek Heidelberg" - }, - { - "@language" : "en", - "@value" : "Heidelberg University Library" - } - ], - "description" : [ - { - "@language" : "de", - "@value" : "Nvovi Disegni Dell' Architettvre, E Piante De Palazzi Di Roma De Piv Celebri Architetti; Universit\u00e4tsbibliothek Heidelberg, C 6426-2-10 GROSS RES" - }, - { - "@language" : "en", - "@value" : "Nvovi Disegni Dell' Architettvre, E Piante De Palazzi Di Roma De Piv Celebri Architetti; Universit\u00e4tsbibliothek Heidelberg, C 6426-2-10 GROSS RES" - } - ], - "label" : "Nvovi Disegni Dell' Architettvre, E Piante De Palazzi Di Roma De Piv Celebri Architetti", - "license" : "http://creativecommons.org/publicdomain/mark/1.0/deed.de", - "logo" : "https://www.ub.uni-heidelberg.de/nav4/grafik/layout/ub_logo2.gif", - "metadata" : [ - { - "label" : "Author", - "value" : [ - "Ferrerio, Pietro; Rossi, Giovanni Giacomo de [Editor]" - ] - }, - { - "label" : "Location", - "value" : [ - "[Rom]" - ] - }, - { - "label" : "Date", - "value" : [ - "[1655]" - ] - }, - { - "label" : "Published", - "value" : [ - "[Rom], [1655]" - ] - }, - { - "label" : "Identifier (URN)", - "value" : [ - "http://nbn-resolving.de/urn:nbn:de:bsz:16-diglit-25939" - ] - }, - { - "label" : "Identifier (DOI)", - "value" : [ - "https://doi.org/10.11588/diglit.2593" - ] - }, - { - "label" : "Local Identifier", - "value" : [ - "Cicognara, 3719-2" - ] - }, - { - "label" : "Local Identifier", - "value" : [ - "dcl:d2v" - ] - }, - { - "label" : "Bibliographic Information", - "value" : [ - "http://katalog.ub.uni-heidelberg.de/cgi-bin/search.cgi?query=si:1355847117+-(teil:ezblf)&pos=1" - ] - }, - { - "label" : "Shelfmark", - "value" : [ - "Universit\u00e4tsbibliothek Heidelberg, C 6426-2-10 GROSS RES" - ] - } - ], - "seeAlso" : "https://digi.ub.uni-heidelberg.de/diglit/ferrerio1655bd2/mets", - "sequences" : [ - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/sequence/normal", - "@type" : "sc:Sequence", - "canvases" : [ - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0001", - "@type" : "sc:Canvas", - "height" : 3729, - "images" : [ - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/annotation/0001-image", - "@type" : "oa:Annotation", - "motivation" : "sc:painting", - "on" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0001", - "resource" : { - "@id" : "https://digi.ub.uni-heidelberg.de/diglitData/image/ferrerio1655bd2/4/Tafel_001.jpg", - "@type" : "dctypes:Image", - "format" : "image/jpeg", - "height" : 3729, - "service" : { - "@context" : "http://iiif.io/api/image/2/context.json", - "@id" : "https://digi.ub.uni-heidelberg.de/iiif/2/ferrerio1655bd2%3ATafel_001.jpg", - "profile" : "http://iiif.io/api/image/2/profiles/level2.json" - }, - "width" : 5053 - } - } - ], - "label" : "Tafel_001", - "width" : 5053 - }, - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0002", - "@type" : "sc:Canvas", - "height" : 3729, - "images" : [ - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/annotation/0002-image", - "@type" : "oa:Annotation", - "motivation" : "sc:painting", - "on" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0002", - "resource" : { - "@id" : "https://digi.ub.uni-heidelberg.de/diglitData/image/ferrerio1655bd2/4/Tafel_002.jpg", - "@type" : "dctypes:Image", - "format" : "image/jpeg", - "height" : 3729, - "service" : { - "@context" : "http://iiif.io/api/image/2/context.json", - "@id" : "https://digi.ub.uni-heidelberg.de/iiif/2/ferrerio1655bd2%3ATafel_002.jpg", - "profile" : "http://iiif.io/api/image/2/profiles/level2.json" - }, - "width" : 5053 - } - } - ], - "label" : "Tafel_002", - "width" : 5053 - }, - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0003", - "@type" : "sc:Canvas", - "height" : 3729, - "images" : [ - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/annotation/0003-image", - "@type" : "oa:Annotation", - "motivation" : "sc:painting", - "on" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0003", - "resource" : { - "@id" : "https://digi.ub.uni-heidelberg.de/diglitData/image/ferrerio1655bd2/4/Tafel_003.jpg", - "@type" : "dctypes:Image", - "format" : "image/jpeg", - "height" : 3729, - "service" : { - "@context" : "http://iiif.io/api/image/2/context.json", - "@id" : "https://digi.ub.uni-heidelberg.de/iiif/2/ferrerio1655bd2%3ATafel_003.jpg", - "profile" : "http://iiif.io/api/image/2/profiles/level2.json" - }, - "width" : 5053 - } - } - ], - "label" : "Tafel_003", - "width" : 5053 - }, - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0004", - "@type" : "sc:Canvas", - "height" : 3729, - "images" : [ - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/annotation/0004-image", - "@type" : "oa:Annotation", - "motivation" : "sc:painting", - "on" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0004", - "resource" : { - "@id" : "https://digi.ub.uni-heidelberg.de/diglitData/image/ferrerio1655bd2/4/Tafel_004.jpg", - "@type" : "dctypes:Image", - "format" : "image/jpeg", - "height" : 3729, - "service" : { - "@context" : "http://iiif.io/api/image/2/context.json", - "@id" : "https://digi.ub.uni-heidelberg.de/iiif/2/ferrerio1655bd2%3ATafel_004.jpg", - "profile" : "http://iiif.io/api/image/2/profiles/level2.json" - }, - "width" : 5053 - } - } - ], - "label" : "Tafel_004", - "width" : 5053 - }, - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0005", - "@type" : "sc:Canvas", - "height" : 3729, - "images" : [ - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/annotation/0005-image", - "@type" : "oa:Annotation", - "motivation" : "sc:painting", - "on" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0005", - "resource" : { - "@id" : "https://digi.ub.uni-heidelberg.de/diglitData/image/ferrerio1655bd2/4/Tafel_005.jpg", - "@type" : "dctypes:Image", - "format" : "image/jpeg", - "height" : 3729, - "service" : { - "@context" : "http://iiif.io/api/image/2/context.json", - "@id" : "https://digi.ub.uni-heidelberg.de/iiif/2/ferrerio1655bd2%3ATafel_005.jpg", - "profile" : "http://iiif.io/api/image/2/profiles/level2.json" - }, - "width" : 5053 - } - } - ], - "label" : "Tafel_005", - "width" : 5053 - }, - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0006", - "@type" : "sc:Canvas", - "height" : 3729, - "images" : [ - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/annotation/0006-image", - "@type" : "oa:Annotation", - "motivation" : "sc:painting", - "on" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0006", - "resource" : { - "@id" : "https://digi.ub.uni-heidelberg.de/diglitData/image/ferrerio1655bd2/4/Tafel_006.jpg", - "@type" : "dctypes:Image", - "format" : "image/jpeg", - "height" : 3729, - "service" : { - "@context" : "http://iiif.io/api/image/2/context.json", - "@id" : "https://digi.ub.uni-heidelberg.de/iiif/2/ferrerio1655bd2%3ATafel_006.jpg", - "profile" : "http://iiif.io/api/image/2/profiles/level2.json" - }, - "width" : 5053 - } - } - ], - "label" : "Tafel_006", - "width" : 5053 - }, - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0007", - "@type" : "sc:Canvas", - "height" : 3729, - "images" : [ - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/annotation/0007-image", - "@type" : "oa:Annotation", - "motivation" : "sc:painting", - "on" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0007", - "resource" : { - "@id" : "https://digi.ub.uni-heidelberg.de/diglitData/image/ferrerio1655bd2/4/Tafel_007.jpg", - "@type" : "dctypes:Image", - "format" : "image/jpeg", - "height" : 3729, - "service" : { - "@context" : "http://iiif.io/api/image/2/context.json", - "@id" : "https://digi.ub.uni-heidelberg.de/iiif/2/ferrerio1655bd2%3ATafel_007.jpg", - "profile" : "http://iiif.io/api/image/2/profiles/level2.json" - }, - "width" : 5053 - } - } - ], - "label" : "Tafel_007", - "width" : 5053 - }, - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0008", - "@type" : "sc:Canvas", - "height" : 3729, - "images" : [ - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/annotation/0008-image", - "@type" : "oa:Annotation", - "motivation" : "sc:painting", - "on" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0008", - "resource" : { - "@id" : "https://digi.ub.uni-heidelberg.de/diglitData/image/ferrerio1655bd2/4/Tafel_008.jpg", - "@type" : "dctypes:Image", - "format" : "image/jpeg", - "height" : 3729, - "service" : { - "@context" : "http://iiif.io/api/image/2/context.json", - "@id" : "https://digi.ub.uni-heidelberg.de/iiif/2/ferrerio1655bd2%3ATafel_008.jpg", - "profile" : "http://iiif.io/api/image/2/profiles/level2.json" - }, - "width" : 5053 - } - } - ], - "label" : "Tafel_008", - "width" : 5053 - }, - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0009", - "@type" : "sc:Canvas", - "height" : 3729, - "images" : [ - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/annotation/0009-image", - "@type" : "oa:Annotation", - "motivation" : "sc:painting", - "on" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0009", - "resource" : { - "@id" : "https://digi.ub.uni-heidelberg.de/diglitData/image/ferrerio1655bd2/4/Tafel_009.jpg", - "@type" : "dctypes:Image", - "format" : "image/jpeg", - "height" : 3729, - "service" : { - "@context" : "http://iiif.io/api/image/2/context.json", - "@id" : "https://digi.ub.uni-heidelberg.de/iiif/2/ferrerio1655bd2%3ATafel_009.jpg", - "profile" : "http://iiif.io/api/image/2/profiles/level2.json" - }, - "width" : 5053 - } - } - ], - "label" : "Tafel_009", - "width" : 5053 - }, - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0010", - "@type" : "sc:Canvas", - "height" : 3729, - "images" : [ - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/annotation/0010-image", - "@type" : "oa:Annotation", - "motivation" : "sc:painting", - "on" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0010", - "resource" : { - "@id" : "https://digi.ub.uni-heidelberg.de/diglitData/image/ferrerio1655bd2/4/Tafel_010.jpg", - "@type" : "dctypes:Image", - "format" : "image/jpeg", - "height" : 3729, - "service" : { - "@context" : "http://iiif.io/api/image/2/context.json", - "@id" : "https://digi.ub.uni-heidelberg.de/iiif/2/ferrerio1655bd2%3ATafel_010.jpg", - "profile" : "http://iiif.io/api/image/2/profiles/level2.json" - }, - "width" : 5053 - } - } - ], - "label" : "Tafel_010", - "width" : 5053 - }, - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0011", - "@type" : "sc:Canvas", - "height" : 3729, - "images" : [ - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/annotation/0011-image", - "@type" : "oa:Annotation", - "motivation" : "sc:painting", - "on" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0011", - "resource" : { - "@id" : "https://digi.ub.uni-heidelberg.de/diglitData/image/ferrerio1655bd2/4/Tafel_011.jpg", - "@type" : "dctypes:Image", - "format" : "image/jpeg", - "height" : 3729, - "service" : { - "@context" : "http://iiif.io/api/image/2/context.json", - "@id" : "https://digi.ub.uni-heidelberg.de/iiif/2/ferrerio1655bd2%3ATafel_011.jpg", - "profile" : "http://iiif.io/api/image/2/profiles/level2.json" - }, - "width" : 5053 - } - } - ], - "label" : "Tafel_011", - "width" : 5053 - }, - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0012", - "@type" : "sc:Canvas", - "height" : 3729, - "images" : [ - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/annotation/0012-image", - "@type" : "oa:Annotation", - "motivation" : "sc:painting", - "on" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0012", - "resource" : { - "@id" : "https://digi.ub.uni-heidelberg.de/diglitData/image/ferrerio1655bd2/4/Tafel_012.jpg", - "@type" : "dctypes:Image", - "format" : "image/jpeg", - "height" : 3729, - "service" : { - "@context" : "http://iiif.io/api/image/2/context.json", - "@id" : "https://digi.ub.uni-heidelberg.de/iiif/2/ferrerio1655bd2%3ATafel_012.jpg", - "profile" : "http://iiif.io/api/image/2/profiles/level2.json" - }, - "width" : 5053 - } - } - ], - "label" : "Tafel_012", - "width" : 5053 - }, - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0013", - "@type" : "sc:Canvas", - "height" : 3729, - "images" : [ - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/annotation/0013-image", - "@type" : "oa:Annotation", - "motivation" : "sc:painting", - "on" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0013", - "resource" : { - "@id" : "https://digi.ub.uni-heidelberg.de/diglitData/image/ferrerio1655bd2/4/Tafel_013.jpg", - "@type" : "dctypes:Image", - "format" : "image/jpeg", - "height" : 3729, - "service" : { - "@context" : "http://iiif.io/api/image/2/context.json", - "@id" : "https://digi.ub.uni-heidelberg.de/iiif/2/ferrerio1655bd2%3ATafel_013.jpg", - "profile" : "http://iiif.io/api/image/2/profiles/level2.json" - }, - "width" : 5053 - } - } - ], - "label" : "Tafel_013", - "width" : 5053 - }, - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0014", - "@type" : "sc:Canvas", - "height" : 3729, - "images" : [ - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/annotation/0014-image", - "@type" : "oa:Annotation", - "motivation" : "sc:painting", - "on" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0014", - "resource" : { - "@id" : "https://digi.ub.uni-heidelberg.de/diglitData/image/ferrerio1655bd2/4/Tafel_014.jpg", - "@type" : "dctypes:Image", - "format" : "image/jpeg", - "height" : 3729, - "service" : { - "@context" : "http://iiif.io/api/image/2/context.json", - "@id" : "https://digi.ub.uni-heidelberg.de/iiif/2/ferrerio1655bd2%3ATafel_014.jpg", - "profile" : "http://iiif.io/api/image/2/profiles/level2.json" - }, - "width" : 5053 - } - } - ], - "label" : "Tafel_014", - "width" : 5053 - }, - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0015", - "@type" : "sc:Canvas", - "height" : 3729, - "images" : [ - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/annotation/0015-image", - "@type" : "oa:Annotation", - "motivation" : "sc:painting", - "on" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0015", - "resource" : { - "@id" : "https://digi.ub.uni-heidelberg.de/diglitData/image/ferrerio1655bd2/4/Tafel_015.jpg", - "@type" : "dctypes:Image", - "format" : "image/jpeg", - "height" : 3729, - "service" : { - "@context" : "http://iiif.io/api/image/2/context.json", - "@id" : "https://digi.ub.uni-heidelberg.de/iiif/2/ferrerio1655bd2%3ATafel_015.jpg", - "profile" : "http://iiif.io/api/image/2/profiles/level2.json" - }, - "width" : 5053 - } - } - ], - "label" : "Tafel_015", - "width" : 5053 - }, - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0016", - "@type" : "sc:Canvas", - "height" : 3729, - "images" : [ - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/annotation/0016-image", - "@type" : "oa:Annotation", - "motivation" : "sc:painting", - "on" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0016", - "resource" : { - "@id" : "https://digi.ub.uni-heidelberg.de/diglitData/image/ferrerio1655bd2/4/Tafel_016.jpg", - "@type" : "dctypes:Image", - "format" : "image/jpeg", - "height" : 3729, - "service" : { - "@context" : "http://iiif.io/api/image/2/context.json", - "@id" : "https://digi.ub.uni-heidelberg.de/iiif/2/ferrerio1655bd2%3ATafel_016.jpg", - "profile" : "http://iiif.io/api/image/2/profiles/level2.json" - }, - "width" : 5053 - } - } - ], - "label" : "Tafel_016", - "width" : 5053 - }, - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0017", - "@type" : "sc:Canvas", - "height" : 3729, - "images" : [ - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/annotation/0017-image", - "@type" : "oa:Annotation", - "motivation" : "sc:painting", - "on" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0017", - "resource" : { - "@id" : "https://digi.ub.uni-heidelberg.de/diglitData/image/ferrerio1655bd2/4/Tafel_017.jpg", - "@type" : "dctypes:Image", - "format" : "image/jpeg", - "height" : 3729, - "service" : { - "@context" : "http://iiif.io/api/image/2/context.json", - "@id" : "https://digi.ub.uni-heidelberg.de/iiif/2/ferrerio1655bd2%3ATafel_017.jpg", - "profile" : "http://iiif.io/api/image/2/profiles/level2.json" - }, - "width" : 5053 - } - } - ], - "label" : "Tafel_017", - "width" : 5053 - }, - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0018", - "@type" : "sc:Canvas", - "height" : 3729, - "images" : [ - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/annotation/0018-image", - "@type" : "oa:Annotation", - "motivation" : "sc:painting", - "on" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0018", - "resource" : { - "@id" : "https://digi.ub.uni-heidelberg.de/diglitData/image/ferrerio1655bd2/4/Tafel_018.jpg", - "@type" : "dctypes:Image", - "format" : "image/jpeg", - "height" : 3729, - "service" : { - "@context" : "http://iiif.io/api/image/2/context.json", - "@id" : "https://digi.ub.uni-heidelberg.de/iiif/2/ferrerio1655bd2%3ATafel_018.jpg", - "profile" : "http://iiif.io/api/image/2/profiles/level2.json" - }, - "width" : 5053 - } - } - ], - "label" : "Tafel_018", - "width" : 5053 - }, - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0019", - "@type" : "sc:Canvas", - "height" : 3729, - "images" : [ - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/annotation/0019-image", - "@type" : "oa:Annotation", - "motivation" : "sc:painting", - "on" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0019", - "resource" : { - "@id" : "https://digi.ub.uni-heidelberg.de/diglitData/image/ferrerio1655bd2/4/Tafel_019.jpg", - "@type" : "dctypes:Image", - "format" : "image/jpeg", - "height" : 3729, - "service" : { - "@context" : "http://iiif.io/api/image/2/context.json", - "@id" : "https://digi.ub.uni-heidelberg.de/iiif/2/ferrerio1655bd2%3ATafel_019.jpg", - "profile" : "http://iiif.io/api/image/2/profiles/level2.json" - }, - "width" : 5053 - } - } - ], - "label" : "Tafel_019", - "width" : 5053 - }, - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0020", - "@type" : "sc:Canvas", - "height" : 3729, - "images" : [ - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/annotation/0020-image", - "@type" : "oa:Annotation", - "motivation" : "sc:painting", - "on" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0020", - "resource" : { - "@id" : "https://digi.ub.uni-heidelberg.de/diglitData/image/ferrerio1655bd2/4/Tafel_020.jpg", - "@type" : "dctypes:Image", - "format" : "image/jpeg", - "height" : 3729, - "service" : { - "@context" : "http://iiif.io/api/image/2/context.json", - "@id" : "https://digi.ub.uni-heidelberg.de/iiif/2/ferrerio1655bd2%3ATafel_020.jpg", - "profile" : "http://iiif.io/api/image/2/profiles/level2.json" - }, - "width" : 5053 - } - } - ], - "label" : "Tafel_020", - "width" : 5053 - }, - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0021", - "@type" : "sc:Canvas", - "height" : 3729, - "images" : [ - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/annotation/0021-image", - "@type" : "oa:Annotation", - "motivation" : "sc:painting", - "on" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0021", - "resource" : { - "@id" : "https://digi.ub.uni-heidelberg.de/diglitData/image/ferrerio1655bd2/4/Tafel_021.jpg", - "@type" : "dctypes:Image", - "format" : "image/jpeg", - "height" : 3729, - "service" : { - "@context" : "http://iiif.io/api/image/2/context.json", - "@id" : "https://digi.ub.uni-heidelberg.de/iiif/2/ferrerio1655bd2%3ATafel_021.jpg", - "profile" : "http://iiif.io/api/image/2/profiles/level2.json" - }, - "width" : 5053 - } - } - ], - "label" : "Tafel_021", - "width" : 5053 - }, - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0022", - "@type" : "sc:Canvas", - "height" : 3729, - "images" : [ - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/annotation/0022-image", - "@type" : "oa:Annotation", - "motivation" : "sc:painting", - "on" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0022", - "resource" : { - "@id" : "https://digi.ub.uni-heidelberg.de/diglitData/image/ferrerio1655bd2/4/Tafel_022.jpg", - "@type" : "dctypes:Image", - "format" : "image/jpeg", - "height" : 3729, - "service" : { - "@context" : "http://iiif.io/api/image/2/context.json", - "@id" : "https://digi.ub.uni-heidelberg.de/iiif/2/ferrerio1655bd2%3ATafel_022.jpg", - "profile" : "http://iiif.io/api/image/2/profiles/level2.json" - }, - "width" : 5053 - } - } - ], - "label" : "Tafel_022", - "width" : 5053 - }, - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0023", - "@type" : "sc:Canvas", - "height" : 3729, - "images" : [ - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/annotation/0023-image", - "@type" : "oa:Annotation", - "motivation" : "sc:painting", - "on" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0023", - "resource" : { - "@id" : "https://digi.ub.uni-heidelberg.de/diglitData/image/ferrerio1655bd2/4/Tafel_023.jpg", - "@type" : "dctypes:Image", - "format" : "image/jpeg", - "height" : 3729, - "service" : { - "@context" : "http://iiif.io/api/image/2/context.json", - "@id" : "https://digi.ub.uni-heidelberg.de/iiif/2/ferrerio1655bd2%3ATafel_023.jpg", - "profile" : "http://iiif.io/api/image/2/profiles/level2.json" - }, - "width" : 5053 - } - } - ], - "label" : "Tafel_023", - "width" : 5053 - }, - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0024", - "@type" : "sc:Canvas", - "height" : 3729, - "images" : [ - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/annotation/0024-image", - "@type" : "oa:Annotation", - "motivation" : "sc:painting", - "on" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0024", - "resource" : { - "@id" : "https://digi.ub.uni-heidelberg.de/diglitData/image/ferrerio1655bd2/4/Tafel_024.jpg", - "@type" : "dctypes:Image", - "format" : "image/jpeg", - "height" : 3729, - "service" : { - "@context" : "http://iiif.io/api/image/2/context.json", - "@id" : "https://digi.ub.uni-heidelberg.de/iiif/2/ferrerio1655bd2%3ATafel_024.jpg", - "profile" : "http://iiif.io/api/image/2/profiles/level2.json" - }, - "width" : 5053 - } - } - ], - "label" : "Tafel_024", - "width" : 5053 - }, - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0025", - "@type" : "sc:Canvas", - "height" : 3729, - "images" : [ - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/annotation/0025-image", - "@type" : "oa:Annotation", - "motivation" : "sc:painting", - "on" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0025", - "resource" : { - "@id" : "https://digi.ub.uni-heidelberg.de/diglitData/image/ferrerio1655bd2/4/Tafel_025.jpg", - "@type" : "dctypes:Image", - "format" : "image/jpeg", - "height" : 3729, - "service" : { - "@context" : "http://iiif.io/api/image/2/context.json", - "@id" : "https://digi.ub.uni-heidelberg.de/iiif/2/ferrerio1655bd2%3ATafel_025.jpg", - "profile" : "http://iiif.io/api/image/2/profiles/level2.json" - }, - "width" : 5053 - } - } - ], - "label" : "Tafel_025", - "width" : 5053 - }, - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0026", - "@type" : "sc:Canvas", - "height" : 3729, - "images" : [ - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/annotation/0026-image", - "@type" : "oa:Annotation", - "motivation" : "sc:painting", - "on" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0026", - "resource" : { - "@id" : "https://digi.ub.uni-heidelberg.de/diglitData/image/ferrerio1655bd2/4/Tafel_026.jpg", - "@type" : "dctypes:Image", - "format" : "image/jpeg", - "height" : 3729, - "service" : { - "@context" : "http://iiif.io/api/image/2/context.json", - "@id" : "https://digi.ub.uni-heidelberg.de/iiif/2/ferrerio1655bd2%3ATafel_026.jpg", - "profile" : "http://iiif.io/api/image/2/profiles/level2.json" - }, - "width" : 5053 - } - } - ], - "label" : "Tafel_026", - "width" : 5053 - }, - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0027", - "@type" : "sc:Canvas", - "height" : 3729, - "images" : [ - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/annotation/0027-image", - "@type" : "oa:Annotation", - "motivation" : "sc:painting", - "on" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0027", - "resource" : { - "@id" : "https://digi.ub.uni-heidelberg.de/diglitData/image/ferrerio1655bd2/4/Tafel_027.jpg", - "@type" : "dctypes:Image", - "format" : "image/jpeg", - "height" : 3729, - "service" : { - "@context" : "http://iiif.io/api/image/2/context.json", - "@id" : "https://digi.ub.uni-heidelberg.de/iiif/2/ferrerio1655bd2%3ATafel_027.jpg", - "profile" : "http://iiif.io/api/image/2/profiles/level2.json" - }, - "width" : 5053 - } - } - ], - "label" : "Tafel_027", - "width" : 5053 - }, - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0028", - "@type" : "sc:Canvas", - "height" : 3729, - "images" : [ - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/annotation/0028-image", - "@type" : "oa:Annotation", - "motivation" : "sc:painting", - "on" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0028", - "resource" : { - "@id" : "https://digi.ub.uni-heidelberg.de/diglitData/image/ferrerio1655bd2/4/Tafel_028.jpg", - "@type" : "dctypes:Image", - "format" : "image/jpeg", - "height" : 3729, - "service" : { - "@context" : "http://iiif.io/api/image/2/context.json", - "@id" : "https://digi.ub.uni-heidelberg.de/iiif/2/ferrerio1655bd2%3ATafel_028.jpg", - "profile" : "http://iiif.io/api/image/2/profiles/level2.json" - }, - "width" : 5053 - } - } - ], - "label" : "Tafel_028", - "width" : 5053 - }, - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0029", - "@type" : "sc:Canvas", - "height" : 3729, - "images" : [ - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/annotation/0029-image", - "@type" : "oa:Annotation", - "motivation" : "sc:painting", - "on" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0029", - "resource" : { - "@id" : "https://digi.ub.uni-heidelberg.de/diglitData/image/ferrerio1655bd2/4/Tafel_029.jpg", - "@type" : "dctypes:Image", - "format" : "image/jpeg", - "height" : 3729, - "service" : { - "@context" : "http://iiif.io/api/image/2/context.json", - "@id" : "https://digi.ub.uni-heidelberg.de/iiif/2/ferrerio1655bd2%3ATafel_029.jpg", - "profile" : "http://iiif.io/api/image/2/profiles/level2.json" - }, - "width" : 5053 - } - } - ], - "label" : "Tafel_029", - "width" : 5053 - }, - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0030", - "@type" : "sc:Canvas", - "height" : 3729, - "images" : [ - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/annotation/0030-image", - "@type" : "oa:Annotation", - "motivation" : "sc:painting", - "on" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0030", - "resource" : { - "@id" : "https://digi.ub.uni-heidelberg.de/diglitData/image/ferrerio1655bd2/4/Tafel_030.jpg", - "@type" : "dctypes:Image", - "format" : "image/jpeg", - "height" : 3729, - "service" : { - "@context" : "http://iiif.io/api/image/2/context.json", - "@id" : "https://digi.ub.uni-heidelberg.de/iiif/2/ferrerio1655bd2%3ATafel_030.jpg", - "profile" : "http://iiif.io/api/image/2/profiles/level2.json" - }, - "width" : 5053 - } - } - ], - "label" : "Tafel_030", - "width" : 5053 - }, - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0031", - "@type" : "sc:Canvas", - "height" : 3729, - "images" : [ - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/annotation/0031-image", - "@type" : "oa:Annotation", - "motivation" : "sc:painting", - "on" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0031", - "resource" : { - "@id" : "https://digi.ub.uni-heidelberg.de/diglitData/image/ferrerio1655bd2/4/Tafel_031.jpg", - "@type" : "dctypes:Image", - "format" : "image/jpeg", - "height" : 3729, - "service" : { - "@context" : "http://iiif.io/api/image/2/context.json", - "@id" : "https://digi.ub.uni-heidelberg.de/iiif/2/ferrerio1655bd2%3ATafel_031.jpg", - "profile" : "http://iiif.io/api/image/2/profiles/level2.json" - }, - "width" : 5053 - } - } - ], - "label" : "Tafel_031", - "width" : 5053 - }, - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0032", - "@type" : "sc:Canvas", - "height" : 3729, - "images" : [ - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/annotation/0032-image", - "@type" : "oa:Annotation", - "motivation" : "sc:painting", - "on" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0032", - "resource" : { - "@id" : "https://digi.ub.uni-heidelberg.de/diglitData/image/ferrerio1655bd2/4/Tafel_032.jpg", - "@type" : "dctypes:Image", - "format" : "image/jpeg", - "height" : 3729, - "service" : { - "@context" : "http://iiif.io/api/image/2/context.json", - "@id" : "https://digi.ub.uni-heidelberg.de/iiif/2/ferrerio1655bd2%3ATafel_032.jpg", - "profile" : "http://iiif.io/api/image/2/profiles/level2.json" - }, - "width" : 5053 - } - } - ], - "label" : "Tafel_032", - "width" : 5053 - }, - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0033", - "@type" : "sc:Canvas", - "height" : 3729, - "images" : [ - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/annotation/0033-image", - "@type" : "oa:Annotation", - "motivation" : "sc:painting", - "on" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0033", - "resource" : { - "@id" : "https://digi.ub.uni-heidelberg.de/diglitData/image/ferrerio1655bd2/4/Tafel_033.jpg", - "@type" : "dctypes:Image", - "format" : "image/jpeg", - "height" : 3729, - "service" : { - "@context" : "http://iiif.io/api/image/2/context.json", - "@id" : "https://digi.ub.uni-heidelberg.de/iiif/2/ferrerio1655bd2%3ATafel_033.jpg", - "profile" : "http://iiif.io/api/image/2/profiles/level2.json" - }, - "width" : 5053 - } - } - ], - "label" : "Tafel_033", - "width" : 5053 - }, - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0034", - "@type" : "sc:Canvas", - "height" : 3729, - "images" : [ - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/annotation/0034-image", - "@type" : "oa:Annotation", - "motivation" : "sc:painting", - "on" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0034", - "resource" : { - "@id" : "https://digi.ub.uni-heidelberg.de/diglitData/image/ferrerio1655bd2/4/Tafel_034.jpg", - "@type" : "dctypes:Image", - "format" : "image/jpeg", - "height" : 3729, - "service" : { - "@context" : "http://iiif.io/api/image/2/context.json", - "@id" : "https://digi.ub.uni-heidelberg.de/iiif/2/ferrerio1655bd2%3ATafel_034.jpg", - "profile" : "http://iiif.io/api/image/2/profiles/level2.json" - }, - "width" : 5053 - } - } - ], - "label" : "Tafel_034", - "width" : 5053 - }, - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0035", - "@type" : "sc:Canvas", - "height" : 3729, - "images" : [ - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/annotation/0035-image", - "@type" : "oa:Annotation", - "motivation" : "sc:painting", - "on" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0035", - "resource" : { - "@id" : "https://digi.ub.uni-heidelberg.de/diglitData/image/ferrerio1655bd2/4/Tafel_035.jpg", - "@type" : "dctypes:Image", - "format" : "image/jpeg", - "height" : 3729, - "service" : { - "@context" : "http://iiif.io/api/image/2/context.json", - "@id" : "https://digi.ub.uni-heidelberg.de/iiif/2/ferrerio1655bd2%3ATafel_035.jpg", - "profile" : "http://iiif.io/api/image/2/profiles/level2.json" - }, - "width" : 5053 - } - } - ], - "label" : "Tafel_035", - "width" : 5053 - }, - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0036", - "@type" : "sc:Canvas", - "height" : 3729, - "images" : [ - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/annotation/0036-image", - "@type" : "oa:Annotation", - "motivation" : "sc:painting", - "on" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0036", - "resource" : { - "@id" : "https://digi.ub.uni-heidelberg.de/diglitData/image/ferrerio1655bd2/4/Tafel_036.jpg", - "@type" : "dctypes:Image", - "format" : "image/jpeg", - "height" : 3729, - "service" : { - "@context" : "http://iiif.io/api/image/2/context.json", - "@id" : "https://digi.ub.uni-heidelberg.de/iiif/2/ferrerio1655bd2%3ATafel_036.jpg", - "profile" : "http://iiif.io/api/image/2/profiles/level2.json" - }, - "width" : 5053 - } - } - ], - "label" : "Tafel_036", - "width" : 5053 - }, - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0037", - "@type" : "sc:Canvas", - "height" : 3729, - "images" : [ - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/annotation/0037-image", - "@type" : "oa:Annotation", - "motivation" : "sc:painting", - "on" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0037", - "resource" : { - "@id" : "https://digi.ub.uni-heidelberg.de/diglitData/image/ferrerio1655bd2/4/Tafel_037.jpg", - "@type" : "dctypes:Image", - "format" : "image/jpeg", - "height" : 3729, - "service" : { - "@context" : "http://iiif.io/api/image/2/context.json", - "@id" : "https://digi.ub.uni-heidelberg.de/iiif/2/ferrerio1655bd2%3ATafel_037.jpg", - "profile" : "http://iiif.io/api/image/2/profiles/level2.json" - }, - "width" : 5053 - } - } - ], - "label" : "Tafel_037", - "width" : 5053 - }, - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0038", - "@type" : "sc:Canvas", - "height" : 3729, - "images" : [ - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/annotation/0038-image", - "@type" : "oa:Annotation", - "motivation" : "sc:painting", - "on" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0038", - "resource" : { - "@id" : "https://digi.ub.uni-heidelberg.de/diglitData/image/ferrerio1655bd2/4/Tafel_038.jpg", - "@type" : "dctypes:Image", - "format" : "image/jpeg", - "height" : 3729, - "service" : { - "@context" : "http://iiif.io/api/image/2/context.json", - "@id" : "https://digi.ub.uni-heidelberg.de/iiif/2/ferrerio1655bd2%3ATafel_038.jpg", - "profile" : "http://iiif.io/api/image/2/profiles/level2.json" - }, - "width" : 5053 - } - } - ], - "label" : "Tafel_038", - "width" : 5053 - }, - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0039", - "@type" : "sc:Canvas", - "height" : 3729, - "images" : [ - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/annotation/0039-image", - "@type" : "oa:Annotation", - "motivation" : "sc:painting", - "on" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0039", - "resource" : { - "@id" : "https://digi.ub.uni-heidelberg.de/diglitData/image/ferrerio1655bd2/4/Tafel_039.jpg", - "@type" : "dctypes:Image", - "format" : "image/jpeg", - "height" : 3729, - "service" : { - "@context" : "http://iiif.io/api/image/2/context.json", - "@id" : "https://digi.ub.uni-heidelberg.de/iiif/2/ferrerio1655bd2%3ATafel_039.jpg", - "profile" : "http://iiif.io/api/image/2/profiles/level2.json" - }, - "width" : 5053 - } - } - ], - "label" : "Tafel_039", - "width" : 5053 - }, - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0040", - "@type" : "sc:Canvas", - "height" : 3729, - "images" : [ - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/annotation/0040-image", - "@type" : "oa:Annotation", - "motivation" : "sc:painting", - "on" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0040", - "resource" : { - "@id" : "https://digi.ub.uni-heidelberg.de/diglitData/image/ferrerio1655bd2/4/Tafel_040.jpg", - "@type" : "dctypes:Image", - "format" : "image/jpeg", - "height" : 3729, - "service" : { - "@context" : "http://iiif.io/api/image/2/context.json", - "@id" : "https://digi.ub.uni-heidelberg.de/iiif/2/ferrerio1655bd2%3ATafel_040.jpg", - "profile" : "http://iiif.io/api/image/2/profiles/level2.json" - }, - "width" : 5053 - } - } - ], - "label" : "Tafel_040", - "width" : 5053 - }, - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0041", - "@type" : "sc:Canvas", - "height" : 3729, - "images" : [ - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/annotation/0041-image", - "@type" : "oa:Annotation", - "motivation" : "sc:painting", - "on" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0041", - "resource" : { - "@id" : "https://digi.ub.uni-heidelberg.de/diglitData/image/ferrerio1655bd2/4/Tafel_041.jpg", - "@type" : "dctypes:Image", - "format" : "image/jpeg", - "height" : 3729, - "service" : { - "@context" : "http://iiif.io/api/image/2/context.json", - "@id" : "https://digi.ub.uni-heidelberg.de/iiif/2/ferrerio1655bd2%3ATafel_041.jpg", - "profile" : "http://iiif.io/api/image/2/profiles/level2.json" - }, - "width" : 5053 - } - } - ], - "label" : "Tafel_041", - "width" : 5053 - }, - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0042", - "@type" : "sc:Canvas", - "height" : 3729, - "images" : [ - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/annotation/0042-image", - "@type" : "oa:Annotation", - "motivation" : "sc:painting", - "on" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0042", - "resource" : { - "@id" : "https://digi.ub.uni-heidelberg.de/diglitData/image/ferrerio1655bd2/4/Tafel_042.jpg", - "@type" : "dctypes:Image", - "format" : "image/jpeg", - "height" : 3729, - "service" : { - "@context" : "http://iiif.io/api/image/2/context.json", - "@id" : "https://digi.ub.uni-heidelberg.de/iiif/2/ferrerio1655bd2%3ATafel_042.jpg", - "profile" : "http://iiif.io/api/image/2/profiles/level2.json" - }, - "width" : 5053 - } - } - ], - "label" : "Tafel_042", - "width" : 5053 - }, - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0043", - "@type" : "sc:Canvas", - "height" : 3729, - "images" : [ - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/annotation/0043-image", - "@type" : "oa:Annotation", - "motivation" : "sc:painting", - "on" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0043", - "resource" : { - "@id" : "https://digi.ub.uni-heidelberg.de/diglitData/image/ferrerio1655bd2/4/Tafel_043.jpg", - "@type" : "dctypes:Image", - "format" : "image/jpeg", - "height" : 3729, - "service" : { - "@context" : "http://iiif.io/api/image/2/context.json", - "@id" : "https://digi.ub.uni-heidelberg.de/iiif/2/ferrerio1655bd2%3ATafel_043.jpg", - "profile" : "http://iiif.io/api/image/2/profiles/level2.json" - }, - "width" : 5053 - } - } - ], - "label" : "Tafel_043", - "width" : 5053 - }, - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0044", - "@type" : "sc:Canvas", - "height" : 3729, - "images" : [ - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/annotation/0044-image", - "@type" : "oa:Annotation", - "motivation" : "sc:painting", - "on" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0044", - "resource" : { - "@id" : "https://digi.ub.uni-heidelberg.de/diglitData/image/ferrerio1655bd2/4/Tafel_044.jpg", - "@type" : "dctypes:Image", - "format" : "image/jpeg", - "height" : 3729, - "service" : { - "@context" : "http://iiif.io/api/image/2/context.json", - "@id" : "https://digi.ub.uni-heidelberg.de/iiif/2/ferrerio1655bd2%3ATafel_044.jpg", - "profile" : "http://iiif.io/api/image/2/profiles/level2.json" - }, - "width" : 5053 - } - } - ], - "label" : "Tafel_044", - "width" : 5053 - }, - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0045", - "@type" : "sc:Canvas", - "height" : 3729, - "images" : [ - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/annotation/0045-image", - "@type" : "oa:Annotation", - "motivation" : "sc:painting", - "on" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0045", - "resource" : { - "@id" : "https://digi.ub.uni-heidelberg.de/diglitData/image/ferrerio1655bd2/4/Tafel_045.jpg", - "@type" : "dctypes:Image", - "format" : "image/jpeg", - "height" : 3729, - "service" : { - "@context" : "http://iiif.io/api/image/2/context.json", - "@id" : "https://digi.ub.uni-heidelberg.de/iiif/2/ferrerio1655bd2%3ATafel_045.jpg", - "profile" : "http://iiif.io/api/image/2/profiles/level2.json" - }, - "width" : 5053 - } - } - ], - "label" : "Tafel_045", - "width" : 5053 - }, - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0046", - "@type" : "sc:Canvas", - "height" : 3729, - "images" : [ - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/annotation/0046-image", - "@type" : "oa:Annotation", - "motivation" : "sc:painting", - "on" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0046", - "resource" : { - "@id" : "https://digi.ub.uni-heidelberg.de/diglitData/image/ferrerio1655bd2/4/Tafel_046.jpg", - "@type" : "dctypes:Image", - "format" : "image/jpeg", - "height" : 3729, - "service" : { - "@context" : "http://iiif.io/api/image/2/context.json", - "@id" : "https://digi.ub.uni-heidelberg.de/iiif/2/ferrerio1655bd2%3ATafel_046.jpg", - "profile" : "http://iiif.io/api/image/2/profiles/level2.json" - }, - "width" : 5053 - } - } - ], - "label" : "Tafel_046", - "width" : 5053 - }, - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0047", - "@type" : "sc:Canvas", - "height" : 3729, - "images" : [ - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/annotation/0047-image", - "@type" : "oa:Annotation", - "motivation" : "sc:painting", - "on" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0047", - "resource" : { - "@id" : "https://digi.ub.uni-heidelberg.de/diglitData/image/ferrerio1655bd2/4/Tafel_047.jpg", - "@type" : "dctypes:Image", - "format" : "image/jpeg", - "height" : 3729, - "service" : { - "@context" : "http://iiif.io/api/image/2/context.json", - "@id" : "https://digi.ub.uni-heidelberg.de/iiif/2/ferrerio1655bd2%3ATafel_047.jpg", - "profile" : "http://iiif.io/api/image/2/profiles/level2.json" - }, - "width" : 5053 - } - } - ], - "label" : "Tafel_047", - "width" : 5053 - }, - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0048", - "@type" : "sc:Canvas", - "height" : 3729, - "images" : [ - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/annotation/0048-image", - "@type" : "oa:Annotation", - "motivation" : "sc:painting", - "on" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0048", - "resource" : { - "@id" : "https://digi.ub.uni-heidelberg.de/diglitData/image/ferrerio1655bd2/4/Tafel_048.jpg", - "@type" : "dctypes:Image", - "format" : "image/jpeg", - "height" : 3729, - "service" : { - "@context" : "http://iiif.io/api/image/2/context.json", - "@id" : "https://digi.ub.uni-heidelberg.de/iiif/2/ferrerio1655bd2%3ATafel_048.jpg", - "profile" : "http://iiif.io/api/image/2/profiles/level2.json" - }, - "width" : 5053 - } - } - ], - "label" : "Tafel_048", - "width" : 5053 - }, - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0049", - "@type" : "sc:Canvas", - "height" : 3729, - "images" : [ - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/annotation/0049-image", - "@type" : "oa:Annotation", - "motivation" : "sc:painting", - "on" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0049", - "resource" : { - "@id" : "https://digi.ub.uni-heidelberg.de/diglitData/image/ferrerio1655bd2/4/Tafel_049.jpg", - "@type" : "dctypes:Image", - "format" : "image/jpeg", - "height" : 3729, - "service" : { - "@context" : "http://iiif.io/api/image/2/context.json", - "@id" : "https://digi.ub.uni-heidelberg.de/iiif/2/ferrerio1655bd2%3ATafel_049.jpg", - "profile" : "http://iiif.io/api/image/2/profiles/level2.json" - }, - "width" : 5053 - } - } - ], - "label" : "Tafel_049", - "width" : 5053 - }, - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0050", - "@type" : "sc:Canvas", - "height" : 3729, - "images" : [ - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/annotation/0050-image", - "@type" : "oa:Annotation", - "motivation" : "sc:painting", - "on" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0050", - "resource" : { - "@id" : "https://digi.ub.uni-heidelberg.de/diglitData/image/ferrerio1655bd2/4/Tafel_050.jpg", - "@type" : "dctypes:Image", - "format" : "image/jpeg", - "height" : 3729, - "service" : { - "@context" : "http://iiif.io/api/image/2/context.json", - "@id" : "https://digi.ub.uni-heidelberg.de/iiif/2/ferrerio1655bd2%3ATafel_050.jpg", - "profile" : "http://iiif.io/api/image/2/profiles/level2.json" - }, - "width" : 5053 - } - } - ], - "label" : "Tafel_050", - "width" : 5053 - }, - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0051", - "@type" : "sc:Canvas", - "height" : 3729, - "images" : [ - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/annotation/0051-image", - "@type" : "oa:Annotation", - "motivation" : "sc:painting", - "on" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0051", - "resource" : { - "@id" : "https://digi.ub.uni-heidelberg.de/diglitData/image/ferrerio1655bd2/4/Tafel_051.jpg", - "@type" : "dctypes:Image", - "format" : "image/jpeg", - "height" : 3729, - "service" : { - "@context" : "http://iiif.io/api/image/2/context.json", - "@id" : "https://digi.ub.uni-heidelberg.de/iiif/2/ferrerio1655bd2%3ATafel_051.jpg", - "profile" : "http://iiif.io/api/image/2/profiles/level2.json" - }, - "width" : 5053 - } - } - ], - "label" : "Tafel_051", - "width" : 5053 - }, - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0052", - "@type" : "sc:Canvas", - "height" : 3729, - "images" : [ - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/annotation/0052-image", - "@type" : "oa:Annotation", - "motivation" : "sc:painting", - "on" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0052", - "resource" : { - "@id" : "https://digi.ub.uni-heidelberg.de/diglitData/image/ferrerio1655bd2/4/Tafel_052.jpg", - "@type" : "dctypes:Image", - "format" : "image/jpeg", - "height" : 3729, - "service" : { - "@context" : "http://iiif.io/api/image/2/context.json", - "@id" : "https://digi.ub.uni-heidelberg.de/iiif/2/ferrerio1655bd2%3ATafel_052.jpg", - "profile" : "http://iiif.io/api/image/2/profiles/level2.json" - }, - "width" : 5053 - } - } - ], - "label" : "Tafel_052", - "width" : 5053 - }, - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0053", - "@type" : "sc:Canvas", - "height" : 3729, - "images" : [ - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/annotation/0053-image", - "@type" : "oa:Annotation", - "motivation" : "sc:painting", - "on" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0053", - "resource" : { - "@id" : "https://digi.ub.uni-heidelberg.de/diglitData/image/ferrerio1655bd2/4/Tafel_053.jpg", - "@type" : "dctypes:Image", - "format" : "image/jpeg", - "height" : 3729, - "service" : { - "@context" : "http://iiif.io/api/image/2/context.json", - "@id" : "https://digi.ub.uni-heidelberg.de/iiif/2/ferrerio1655bd2%3ATafel_053.jpg", - "profile" : "http://iiif.io/api/image/2/profiles/level2.json" - }, - "width" : 5053 - } - } - ], - "label" : "Tafel_053", - "width" : 5053 - }, - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0054", - "@type" : "sc:Canvas", - "height" : 3729, - "images" : [ - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/annotation/0054-image", - "@type" : "oa:Annotation", - "motivation" : "sc:painting", - "on" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0054", - "resource" : { - "@id" : "https://digi.ub.uni-heidelberg.de/diglitData/image/ferrerio1655bd2/4/Tafel_054.jpg", - "@type" : "dctypes:Image", - "format" : "image/jpeg", - "height" : 3729, - "service" : { - "@context" : "http://iiif.io/api/image/2/context.json", - "@id" : "https://digi.ub.uni-heidelberg.de/iiif/2/ferrerio1655bd2%3ATafel_054.jpg", - "profile" : "http://iiif.io/api/image/2/profiles/level2.json" - }, - "width" : 5053 - } - } - ], - "label" : "Tafel_054", - "width" : 5053 - }, - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0055", - "@type" : "sc:Canvas", - "height" : 3729, - "images" : [ - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/annotation/0055-image", - "@type" : "oa:Annotation", - "motivation" : "sc:painting", - "on" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0055", - "resource" : { - "@id" : "https://digi.ub.uni-heidelberg.de/diglitData/image/ferrerio1655bd2/4/Tafel_055.jpg", - "@type" : "dctypes:Image", - "format" : "image/jpeg", - "height" : 3729, - "service" : { - "@context" : "http://iiif.io/api/image/2/context.json", - "@id" : "https://digi.ub.uni-heidelberg.de/iiif/2/ferrerio1655bd2%3ATafel_055.jpg", - "profile" : "http://iiif.io/api/image/2/profiles/level2.json" - }, - "width" : 5053 - } - } - ], - "label" : "Tafel_055", - "width" : 5053 - }, - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0056", - "@type" : "sc:Canvas", - "height" : 3729, - "images" : [ - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/annotation/0056-image", - "@type" : "oa:Annotation", - "motivation" : "sc:painting", - "on" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0056", - "resource" : { - "@id" : "https://digi.ub.uni-heidelberg.de/diglitData/image/ferrerio1655bd2/4/Tafel_056.jpg", - "@type" : "dctypes:Image", - "format" : "image/jpeg", - "height" : 3729, - "service" : { - "@context" : "http://iiif.io/api/image/2/context.json", - "@id" : "https://digi.ub.uni-heidelberg.de/iiif/2/ferrerio1655bd2%3ATafel_056.jpg", - "profile" : "http://iiif.io/api/image/2/profiles/level2.json" - }, - "width" : 5053 - } - } - ], - "label" : "Tafel_056", - "width" : 5053 - }, - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0057", - "@type" : "sc:Canvas", - "height" : 3729, - "images" : [ - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/annotation/0057-image", - "@type" : "oa:Annotation", - "motivation" : "sc:painting", - "on" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0057", - "resource" : { - "@id" : "https://digi.ub.uni-heidelberg.de/diglitData/image/ferrerio1655bd2/4/Tafel_057.jpg", - "@type" : "dctypes:Image", - "format" : "image/jpeg", - "height" : 3729, - "service" : { - "@context" : "http://iiif.io/api/image/2/context.json", - "@id" : "https://digi.ub.uni-heidelberg.de/iiif/2/ferrerio1655bd2%3ATafel_057.jpg", - "profile" : "http://iiif.io/api/image/2/profiles/level2.json" - }, - "width" : 5053 - } - } - ], - "label" : "Tafel_057", - "width" : 5053 - }, - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0058", - "@type" : "sc:Canvas", - "height" : 3729, - "images" : [ - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/annotation/0058-image", - "@type" : "oa:Annotation", - "motivation" : "sc:painting", - "on" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0058", - "resource" : { - "@id" : "https://digi.ub.uni-heidelberg.de/diglitData/image/ferrerio1655bd2/4/Tafel_058.jpg", - "@type" : "dctypes:Image", - "format" : "image/jpeg", - "height" : 3729, - "service" : { - "@context" : "http://iiif.io/api/image/2/context.json", - "@id" : "https://digi.ub.uni-heidelberg.de/iiif/2/ferrerio1655bd2%3ATafel_058.jpg", - "profile" : "http://iiif.io/api/image/2/profiles/level2.json" - }, - "width" : 5053 - } - } - ], - "label" : "Tafel_058", - "width" : 5053 - }, - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0059", - "@type" : "sc:Canvas", - "height" : 3729, - "images" : [ - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/annotation/0059-image", - "@type" : "oa:Annotation", - "motivation" : "sc:painting", - "on" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0059", - "resource" : { - "@id" : "https://digi.ub.uni-heidelberg.de/diglitData/image/ferrerio1655bd2/4/Tafel_059.jpg", - "@type" : "dctypes:Image", - "format" : "image/jpeg", - "height" : 3729, - "service" : { - "@context" : "http://iiif.io/api/image/2/context.json", - "@id" : "https://digi.ub.uni-heidelberg.de/iiif/2/ferrerio1655bd2%3ATafel_059.jpg", - "profile" : "http://iiif.io/api/image/2/profiles/level2.json" - }, - "width" : 5053 - } - } - ], - "label" : "Tafel_059", - "width" : 5053 - }, - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0060", - "@type" : "sc:Canvas", - "height" : 3729, - "images" : [ - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/annotation/0060-image", - "@type" : "oa:Annotation", - "motivation" : "sc:painting", - "on" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0060", - "resource" : { - "@id" : "https://digi.ub.uni-heidelberg.de/diglitData/image/ferrerio1655bd2/4/Tafel_060.jpg", - "@type" : "dctypes:Image", - "format" : "image/jpeg", - "height" : 3729, - "service" : { - "@context" : "http://iiif.io/api/image/2/context.json", - "@id" : "https://digi.ub.uni-heidelberg.de/iiif/2/ferrerio1655bd2%3ATafel_060.jpg", - "profile" : "http://iiif.io/api/image/2/profiles/level2.json" - }, - "width" : 5053 - } - } - ], - "label" : "Tafel_060", - "width" : 5053 - }, - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0061", - "@type" : "sc:Canvas", - "height" : 3556, - "images" : [ - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/annotation/0061-image", - "@type" : "oa:Annotation", - "motivation" : "sc:painting", - "on" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0061", - "resource" : { - "@id" : "https://digi.ub.uni-heidelberg.de/diglitData/image/ferrerio1655bd2/4/Tafel_061.jpg", - "@type" : "dctypes:Image", - "format" : "image/jpeg", - "height" : 3556, - "service" : { - "@context" : "http://iiif.io/api/image/2/context.json", - "@id" : "https://digi.ub.uni-heidelberg.de/iiif/2/ferrerio1655bd2%3ATafel_061.jpg", - "profile" : "http://iiif.io/api/image/2/profiles/level2.json" - }, - "width" : 4455 - } - } - ], - "label" : "Tafel_061", - "width" : 4455 - } - ], - "label" : [ - { - "@language" : "de", - "@value" : "Standardabfolge" - }, - { - "@language" : "en", - "@value" : "Normal Sequence" - } - ], - "startCanvas" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0001", - "viewingDirection" : "left-to-right", - "viewingHint" : "paged" - } - ], - "service" : "https://digi.ub.uni-heidelberg.de/diglit/ferrerio1655bd2", - "structures" : [ - { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/range/r0", - "@type" : "sc:Range", - "canvases" : [ - "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0001", - "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0002", - "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0003", - "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0004", - "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0005", - "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0006", - "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0007", - "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0008", - "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0009", - "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0010", - "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0011", - "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0012", - "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0013", - "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0014", - "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0015", - "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0016", - "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0017", - "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0018", - "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0019", - "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0020", - "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0021", - "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0022", - "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0023", - "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0024", - "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0025", - "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0026", - "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0027", - "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0028", - "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0029", - "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0030", - "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0031", - "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0032", - "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0033", - "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0034", - "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0035", - "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0036", - "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0037", - "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0038", - "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0039", - "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0040", - "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0041", - "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0042", - "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0043", - "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0044", - "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0045", - "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0046", - "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0047", - "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0048", - "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0049", - "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0050", - "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0051", - "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0052", - "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0053", - "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0054", - "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0055", - "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0056", - "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0057", - "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0058", - "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0059", - "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0060", - "https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/canvas/0061" - ], - "label" : "Tafel 1 - 61" - } - ], - "thumbnail" : { - "@id" : "https://digi.ub.uni-heidelberg.de/diglit/ferrerio1655bd2/introimage", - "format" : "image/jpeg", - "height" : 283, - "width" : 399 - }, - "viewingDirection" : "left-to-right", - "viewingHint" : "paged", - "within" : [ - "https://digi.ub.uni-heidelberg.de/diglit/ferrerio1655ga" - ] -} diff --git a/apps/ingest/fixtures/manifest-label-as-array.json b/apps/ingest/fixtures/manifest-label-as-array.json deleted file mode 100644 index 01152bf1d..000000000 --- a/apps/ingest/fixtures/manifest-label-as-array.json +++ /dev/null @@ -1,232 +0,0 @@ -{ - "@context": "http://iiif.io/api/presentation/2/context.json", - "@id": "https://fake.archivelab.io/iiif/09359080.4757.emory.edu/manifest.json", - "@type": "sc:Manifest", - "attribution": "The Internet Archive", - "description": "Respect Frederick Douglass.", - "label": ["Address", "by", "American", "Hero", "Frederick", "Douglass"], - "logo": "https://encrypted-tbn2.gstatic.com/images?q=tbn:ANd9GcReMN4l9cgu_qb1OwflFeyfHcjp8aUfVNSJ9ynk2IfuHwW1I4mDSw", - "metadata": [ - { - "label": "title", - "value": "Address by Hon. Frederick Douglass, delivered in the Metropolitan A.M.E. Church, Washington, D.C., Tuesday, January 9th, 1894, on the lessons of the hour : in which he discusses the various aspects of the so-called, but mis-called, Negro problem" - }, - { - "label": "publisher", - "value": "Baltimore : Press of Thomas & Evans" - }, - { - "label": "subject", - "value": "African Americans" - }, - { - "label": "date", - "value": "1894" - }, - { - "label": "contributor", - "value": "Emory University, Robert W. Woodruff Library" - }, - { - "label": "creator", - "value": "Douglass, Frederick, 1818-1895. Metropolitan A.M.E. Church (Washington, D.C.)" - } - ], - "related": "http://archive.org/details/09359080.4757.emory.edu", - "seeAlso": "http://archive.org/metadata/09359080.4757.emory.edu", - "sequences": [ - { - "@context": "http://iiif.io/api/image/2/context.json", - "@id": "https://fake.archivelab.io/iiif/09359080.4757.emory.edu/canvas/default", - "@type": "sc:Sequence", - "canvases": [ - { - "@context": "http://iiif.io/api/presentation/2/context.json", - "@id": "https://fake.archivelab.io/iiif/09359080.4757.emory.edu$0/canvas", - "@type": "sc:Canvas", - "description": "", - "height": 2908, - "images": [ - { - "@context": "http://iiif.io/api/image/2/context.json", - "@id": "https://fake.archivelab.io/iiif/09359080.4757.emory.edu$0/annotation", - "@type": "oa:Annotation", - "motivation": "sc:painting", - "on": "https://fake.archivelab.io/iiif/09359080.4757.emory.edu$0/annotation", - "resource": { - "@id": "https://fake.archivelab.io/iiif/09359080.4757.emory.edu$0/full/full/0/default.jpg", - "@type": "dctypes:Image", - "format": "image/jpeg", - "height": 2908, - "service": { - "@context": "http://iiif.io/api/image/2/context.json", - "@id": "https://fake.archivelab.io/iiif/09359080.4757.emory.edu$0", - "profile": "https://iiif.io/api/image/2/profiles/level2.json" - }, - "width": 2039 - } - } - ], - "label": "p. ", - "width": 2039 - }, - { - "@context": "http://iiif.io/api/presentation/2/context.json", - "@id": "https://fake.archivelab.io/iiif/09359080.4757.emory.edu$1/canvas", - "@type": "sc:Canvas", - "description": "", - "height": 2908, - "images": [ - { - "@context": "http://iiif.io/api/image/2/context.json", - "@id": "https://fake.archivelab.io/iiif/09359080.4757.emory.edu$1/annotation", - "@type": "oa:Annotation", - "motivation": "sc:painting", - "on": "https://fake.archivelab.io/iiif/09359080.4757.emory.edu$1/annotation", - "resource": { - "@id": "https://fake.archivelab.io/iiif/09359080.4757.emory.edu$1/full/full/0/default.jpg", - "@type": "dctypes:Image", - "format": "image/jpeg", - "height": 2908, - "service": { - "@context": "http://iiif.io/api/image/2/context.json", - "@id": "https://fake.archivelab.io/iiif/09359080.4757.emory.edu$1", - "profile": "https://iiif.io/api/image/2/profiles/level2.json" - }, - "width": 2039 - } - } - ], - "label": "p. ", - "width": 2039 - }, - { - "@context": "http://iiif.io/api/presentation/2/context.json", - "@id": "https://fake.archivelab.io/iiif/09359080.4757.emory.edu$2/canvas", - "@type": "sc:Canvas", - "description": "", - "height": 2908, - "images": [ - { - "@context": "http://iiif.io/api/image/2/context.json", - "@id": "https://fake.archivelab.io/iiif/09359080.4757.emory.edu$2/annotation", - "@type": "oa:Annotation", - "motivation": "sc:painting", - "on": "https://fake.archivelab.io/iiif/09359080.4757.emory.edu$2/annotation", - "resource": { - "@id": "https://fake.archivelab.io/iiif/09359080.4757.emory.edu$2/full/full/0/default.jpg", - "@type": "dctypes:Image", - "format": "image/jpeg", - "height": 2908, - "service": { - "@context": "http://iiif.io/api/image/2/context.json", - "@id": "https://fake.archivelab.io/iiif/09359080.4757.emory.edu$2", - "profile": "https://iiif.io/api/image/2/profiles/level2.json" - }, - "width": 2039 - } - } - ], - "label": "p. ", - "width": 2039 - }, - { - "@context": "http://iiif.io/api/presentation/2/context.json", - "@id": "https://fake.archivelab.io/iiif/09359080.4757.emory.edu$3/canvas", - "@type": "sc:Canvas", - "description": "", - "height": 2909, - "images": [ - { - "@context": "http://iiif.io/api/image/2/context.json", - "@id": "https://fake.archivelab.io/iiif/09359080.4757.emory.edu$3/annotation", - "@type": "oa:Annotation", - "motivation": "sc:painting", - "on": "https://fake.archivelab.io/iiif/09359080.4757.emory.edu$3/annotation", - "resource": { - "@id": "https://fake.archivelab.io/iiif/09359080.4757.emory.edu$3/full/full/0/default.jpg", - "@type": "dctypes:Image", - "format": "image/jpeg", - "height": 2909, - "service": { - "@context": "http://iiif.io/api/image/2/context.json", - "@id": "https://fake.archivelab.io/iiif/09359080.4757.emory.edu$3", - "profile": "https://iiif.io/api/image/2/profiles/level2.json" - }, - "width": 2040 - } - } - ], - "label": "p. ", - "width": 2040 - }, - { - "@context": "http://iiif.io/api/presentation/2/context.json", - "@id": "https://fake.archivelab.io/iiif/09359080.4757.emory.edu$4/canvas", - "@type": "sc:Canvas", - "description": "", - "height": 2908, - "images": [ - { - "@context": "http://iiif.io/api/image/2/context.json", - "@id": "https://fake.archivelab.io/iiif/09359080.4757.emory.edu$4/annotation", - "@type": "oa:Annotation", - "motivation": "sc:painting", - "on": "https://fake.archivelab.io/iiif/09359080.4757.emory.edu$4/annotation", - "resource": { - "@id": "https://fake.archivelab.io/iiif/09359080.4757.emory.edu$4/full/full/0/default.jpg", - "@type": "dctypes:Image", - "format": "image/jpeg", - "height": 2908, - "service": { - "@context": "http://iiif.io/api/image/2/context.json", - "@id": "https://fake.archivelab.io/iiif/09359080.4757.emory.edu$4", - "profile": "https://iiif.io/api/image/2/profiles/level2.json" - }, - "width": 2039 - } - } - ], - "label": "p. ", - "width": 2039 - }, - { - "@context": "http://iiif.io/api/presentation/2/context.json", - "@id": "https://fake.archivelab.io/iiif/09359080.4757.emory.edu$5/canvas", - "@type": "sc:Canvas", - "description": "", - "height": 2909, - "images": [ - { - "@context": "http://iiif.io/api/image/2/context.json", - "@id": "https://fake.archivelab.io/iiif/09359080.4757.emory.edu$5/annotation", - "@type": "oa:Annotation", - "motivation": "sc:painting", - "on": "https://fake.archivelab.io/iiif/09359080.4757.emory.edu$5/annotation", - "resource": { - "@id": "https://fake.archivelab.io/iiif/09359080.4757.emory.edu$5/full/full/0/default.jpg", - "@type": "dctypes:Image", - "format": "image/jpeg", - "height": 2909, - "service": { - "@context": "http://iiif.io/api/image/2/context.json", - "@id": "https://fake.archivelab.io/iiif/09359080.4757.emory.edu$5", - "profile": "https://iiif.io/api/image/2/profiles/level2.json" - }, - "width": 2040 - } - } - ], - "label": "p. ", - "width": 2040 - } - ], - "label": "default" - } - ], - "thumbnail": { - "@id": "https://ia801601.us.archive.org/BookReader/BookReaderPreview.php?id=09359080.4757.emory.edu&subPrefix=09359080_4757&itemPath=/0/items/09359080.4757.emory.edu&server=ia801601.us.archive.org&page=preview&" - }, - "viewingHint": "paged", - "viewingDirection": "left-to-right" -} \ No newline at end of file diff --git a/apps/ingest/fixtures/manifest.json b/apps/ingest/fixtures/manifest.json deleted file mode 100644 index 78e69050b..000000000 --- a/apps/ingest/fixtures/manifest.json +++ /dev/null @@ -1,232 +0,0 @@ -{ - "@context": "http://iiif.io/api/presentation/2/context.json", - "@id": "https://fake.archivelab.io/iiif/09359080.4757.emory.edu/manifest.json", - "@type": "sc:Manifest", - "attribution": "The Internet Archive", - "description": "Respect Frederick Douglass.", - "label": "Address by Hon. Frederick Douglass", - "logo": "https://encrypted-tbn2.gstatic.com/images?q=tbn:ANd9GcReMN4l9cgu_qb1OwflFeyfHcjp8aUfVNSJ9ynk2IfuHwW1I4mDSw", - "metadata": [ - { - "label": "title", - "value": "Address by Hon. Frederick Douglass, delivered in the Metropolitan A.M.E. Church, Washington, D.C., Tuesday, January 9th, 1894, on the lessons of the hour : in which he discusses the various aspects of the so-called, but mis-called, Negro problem" - }, - { - "label": "publisher", - "value": "Baltimore : Press of Thomas & Evans" - }, - { - "label": "subject", - "value": "African Americans" - }, - { - "label": "date", - "value": "1894" - }, - { - "label": "contributor", - "value": "Emory University, Robert W. Woodruff Library" - }, - { - "label": "creator", - "value": "Douglass, Frederick, 1818-1895. Metropolitan A.M.E. Church (Washington, D.C.)" - } - ], - "related": "http://archive.org/details/09359080.4757.emory.edu", - "seeAlso": "http://archive.org/metadata/09359080.4757.emory.edu", - "sequences": [ - { - "@context": "http://iiif.io/api/image/2/context.json", - "@id": "https://fake.archivelab.io/iiif/09359080.4757.emory.edu/canvas/default", - "@type": "sc:Sequence", - "canvases": [ - { - "@context": "http://iiif.io/api/presentation/2/context.json", - "@id": "https://fake.archivelab.io/iiif/09359080.4757.emory.edu$0/canvas", - "@type": "sc:Canvas", - "description": "", - "height": 2908, - "images": [ - { - "@context": "http://iiif.io/api/image/2/context.json", - "@id": "https://fake.archivelab.io/iiif/09359080.4757.emory.edu$0/annotation", - "@type": "oa:Annotation", - "motivation": "sc:painting", - "on": "https://fake.archivelab.io/iiif/09359080.4757.emory.edu$0/annotation", - "resource": { - "@id": "https://fake.archivelab.io/iiif/09359080.4757.emory.edu$0/full/full/0/default.jpg", - "@type": "dctypes:Image", - "format": "image/jpeg", - "height": 2908, - "service": { - "@context": "http://iiif.io/api/image/2/context.json", - "@id": "https://fake.archivelab.io/iiif/09359080.4757.emory.edu$0", - "profile": "https://iiif.io/api/image/2/profiles/level2.json" - }, - "width": 2039 - } - } - ], - "label": "p. ", - "width": 2039 - }, - { - "@context": "http://iiif.io/api/presentation/2/context.json", - "@id": "https://fake.archivelab.io/iiif/09359080.4757.emory.edu$1/canvas", - "@type": "sc:Canvas", - "description": "", - "height": 2908, - "images": [ - { - "@context": "http://iiif.io/api/image/2/context.json", - "@id": "https://fake.archivelab.io/iiif/09359080.4757.emory.edu$1/annotation", - "@type": "oa:Annotation", - "motivation": "sc:painting", - "on": "https://fake.archivelab.io/iiif/09359080.4757.emory.edu$1/annotation", - "resource": { - "@id": "https://fake.archivelab.io/iiif/09359080.4757.emory.edu$1/full/full/0/default.jpg", - "@type": "dctypes:Image", - "format": "image/jpeg", - "height": 2908, - "service": { - "@context": "http://iiif.io/api/image/2/context.json", - "@id": "https://fake.archivelab.io/iiif/09359080.4757.emory.edu$1", - "profile": "https://iiif.io/api/image/2/profiles/level2.json" - }, - "width": 2039 - } - } - ], - "label": "p. ", - "width": 2039 - }, - { - "@context": "http://iiif.io/api/presentation/2/context.json", - "@id": "https://fake.archivelab.io/iiif/09359080.4757.emory.edu$2/canvas", - "@type": "sc:Canvas", - "description": "", - "height": 2908, - "images": [ - { - "@context": "http://iiif.io/api/image/2/context.json", - "@id": "https://fake.archivelab.io/iiif/09359080.4757.emory.edu$2/annotation", - "@type": "oa:Annotation", - "motivation": "sc:painting", - "on": "https://fake.archivelab.io/iiif/09359080.4757.emory.edu$2/annotation", - "resource": { - "@id": "https://fake.archivelab.io/iiif/09359080.4757.emory.edu$2/full/full/0/default.jpg", - "@type": "dctypes:Image", - "format": "image/jpeg", - "height": 2908, - "service": { - "@context": "http://iiif.io/api/image/2/context.json", - "@id": "https://fake.archivelab.io/iiif/09359080.4757.emory.edu$2", - "profile": "https://iiif.io/api/image/2/profiles/level2.json" - }, - "width": 2039 - } - } - ], - "label": "p. ", - "width": 2039 - }, - { - "@context": "http://iiif.io/api/presentation/2/context.json", - "@id": "https://fake.archivelab.io/iiif/09359080.4757.emory.edu$3/canvas", - "@type": "sc:Canvas", - "description": "", - "height": 2909, - "images": [ - { - "@context": "http://iiif.io/api/image/2/context.json", - "@id": "https://fake.archivelab.io/iiif/09359080.4757.emory.edu$3/annotation", - "@type": "oa:Annotation", - "motivation": "sc:painting", - "on": "https://fake.archivelab.io/iiif/09359080.4757.emory.edu$3/annotation", - "resource": { - "@id": "https://fake.archivelab.io/iiif/09359080.4757.emory.edu$3/full/full/0/default.jpg", - "@type": "dctypes:Image", - "format": "image/jpeg", - "height": 2909, - "service": { - "@context": "http://iiif.io/api/image/2/context.json", - "@id": "https://fake.archivelab.io/iiif/09359080.4757.emory.edu$3", - "profile": "https://iiif.io/api/image/2/profiles/level2.json" - }, - "width": 2040 - } - } - ], - "label": "p. ", - "width": 2040 - }, - { - "@context": "http://iiif.io/api/presentation/2/context.json", - "@id": "https://fake.archivelab.io/iiif/09359080.4757.emory.edu$4/canvas", - "@type": "sc:Canvas", - "description": "", - "height": 2908, - "images": [ - { - "@context": "http://iiif.io/api/image/2/context.json", - "@id": "https://fake.archivelab.io/iiif/09359080.4757.emory.edu$4/annotation", - "@type": "oa:Annotation", - "motivation": "sc:painting", - "on": "https://fake.archivelab.io/iiif/09359080.4757.emory.edu$4/annotation", - "resource": { - "@id": "https://fake.archivelab.io/iiif/09359080.4757.emory.edu$4/full/full/0/default.jpg", - "@type": "dctypes:Image", - "format": "image/jpeg", - "height": 2908, - "service": { - "@context": "http://iiif.io/api/image/2/context.json", - "@id": "https://fake.archivelab.io/iiif/09359080.4757.emory.edu$4", - "profile": "https://iiif.io/api/image/2/profiles/level2.json" - }, - "width": 2039 - } - } - ], - "label": "p. ", - "width": 2039 - }, - { - "@context": "http://iiif.io/api/presentation/2/context.json", - "@id": "https://fake.archivelab.io/iiif/09359080.4757.emory.edu$5/canvas", - "@type": "sc:Canvas", - "description": "", - "height": 2909, - "images": [ - { - "@context": "http://iiif.io/api/image/2/context.json", - "@id": "https://fake.archivelab.io/iiif/09359080.4757.emory.edu$5/annotation", - "@type": "oa:Annotation", - "motivation": "sc:painting", - "on": "https://fake.archivelab.io/iiif/09359080.4757.emory.edu$5/annotation", - "resource": { - "@id": "https://fake.archivelab.io/iiif/09359080.4757.emory.edu$5/full/full/0/default.jpg", - "@type": "dctypes:Image", - "format": "image/jpeg", - "height": 2909, - "service": { - "@context": "http://iiif.io/api/image/2/context.json", - "@id": "https://fake.archivelab.io/iiif/09359080.4757.emory.edu$5", - "profile": "https://iiif.io/api/image/2/profiles/level2.json" - }, - "width": 2040 - } - } - ], - "label": "p. ", - "width": 2040 - } - ], - "label": "default" - } - ], - "thumbnail": { - "@id": "https://ia801601.us.archive.org/BookReader/BookReaderPreview.php?id=09359080.4757.emory.edu&subPrefix=09359080_4757&itemPath=/0/items/09359080.4757.emory.edu&server=ia801601.us.archive.org&page=preview&" - }, - "viewingHint": "paged", - "viewingDirection": "left-to-right" -} \ No newline at end of file diff --git a/apps/ingest/fixtures/metadata.csv b/apps/ingest/fixtures/metadata.csv deleted file mode 100644 index 087862bc1..000000000 --- a/apps/ingest/fixtures/metadata.csv +++ /dev/null @@ -1,2 +0,0 @@ -pid,Filename,Label,Summary,Author,Published city,Published date,Publisher -1875-Gospel-UTL-1-jt1,no_meta_file,Test Bundle,Test file,Test author,Test City,2021,Pubilsher test \ No newline at end of file diff --git a/apps/ingest/fixtures/metadata.zip b/apps/ingest/fixtures/metadata.zip deleted file mode 100644 index bee3de718..000000000 Binary files a/apps/ingest/fixtures/metadata.zip and /dev/null differ diff --git a/apps/ingest/fixtures/nested_volume.zip b/apps/ingest/fixtures/nested_volume.zip deleted file mode 100644 index 002236f5b..000000000 Binary files a/apps/ingest/fixtures/nested_volume.zip and /dev/null differ diff --git a/apps/ingest/fixtures/nga.json b/apps/ingest/fixtures/nga.json deleted file mode 100644 index 9f5d45323..000000000 --- a/apps/ingest/fixtures/nga.json +++ /dev/null @@ -1,64 +0,0 @@ -{ - "@context": "http://iiif.io/api/presentation/2/context.json", - "@id": "https://www.nga.gov/content/ngaweb/api/v1/iiif/presentation/manifest.json?cultObj:id=164652", - "@type": "sc:Manifest", - "label": "Autre veue du Campo Vacine", - "description": "Autre veue du Campo Vacine", - "logo": "https://www.nga.gov/etc/designs/ngaweb/images/nga-seal.png", - "attribution": "Provided by National Gallery of Art", - "metadata": [ - { - "label": "Artist", - "value": "Silvestre, Israël " - }, - { - "label": "Accession Number", - "value": "1981.69.12.242" - }, - { - "label": "Creation Year", - "value": "1660" - }, - { - "label": "Title", - "value": "Autre veue du Campo Vacine" - } - ], - "viewingDirection": "left-to-right", - "viewingHint": "individuals", - "sequences": [{ - "@type": "sc:Sequence", - "label": "Autre veue du Campo Vacine", - "canvases": [{ - "@id": "https://cq.nga.gov/content/ngaweb/api/v1/iiif/presentation/manifest/sequence/canvas.json?cultObj:id=164652", - "@type": "sc:Canvas", - "width": 4000, - "height": 2074, - "label": "Autre veue du Campo Vacine", - "images": [{ - "@type": "oa:Annotation", - "motivation": "sc:painting", - "resource": { - "@id": "https://www.nga.gov/content/ngaweb/api/v1/iiif/presentation/manifest/sequence/canvas/resource.json?cultObj:id=164652", - "@type": "dctypes:Image", - "format": "image/jpeg", - "height": 2074, - "width": 4000, - "service": { - "@context": "http://iiif.io/api/image/2/context.json", - "@id": "https://media.nga.gov/iiif/cba52a13-d390-4ee4-8965-250593c600ba", - "profile": "http://iiif.io/api/image/2/level2.json" - } - }, - "on": "https://cq.nga.gov/content/ngaweb/api/v1/iiif/presentation/manifest/sequence/canvas.json?cultObj:id=164652" - }], - "thumbnail": { - "@id": "https://media.nga.gov/iiif/cba52a13-d390-4ee4-8965-250593c600ba/full/!100,100/0/default.jpg", - "@type": "dctypes:Image", - "format": "image/jpeg", - "width": 100, - "height": 100 - } - }] - }] - } \ No newline at end of file diff --git a/apps/ingest/fixtures/no_meta_file.zip b/apps/ingest/fixtures/no_meta_file.zip deleted file mode 100644 index 889649226..000000000 Binary files a/apps/ingest/fixtures/no_meta_file.zip and /dev/null differ diff --git a/apps/ingest/fixtures/princeton.json b/apps/ingest/fixtures/princeton.json deleted file mode 100644 index 55d43c992..000000000 --- a/apps/ingest/fixtures/princeton.json +++ /dev/null @@ -1,147 +0,0 @@ -{ - "@context": "http://iiif.io/api/presentation/2/context.json", - "@type": "sc:Manifest", - "@id": "https://figgy.princeton.edu/concern/scanned_resources/e38d7ebb-3ae6-4eb8-a3ef-d70b5184b005/manifest", - "label": ["Nvova pianta et alzata della citt"], - "description": ["Con l'aggivunta delle nove"], - "viewingHint": "paged", - "viewingDirection": "left-to-right", - "metadata": [{ - "label": "Author", - "value": ["Falda, Giovanni Battista, approximately 1640-1678"] - }, { - "label": "Alternative", - "value": ["Nuova pianta et alzata della città di Roma con tutte le strade, piazze et edificii de tempi, palazzi, giardini et altre fabbriche antiche e moderne come si trovano al presente nel pontificato di N.S. Papa Innocentio XI con le loro dichiarationi nomi et indice copiosissimo"] - }, { - "label": "Extent", - "value": ["1 map (12 sheets) mounted on cloth ; 138 x 151 cm. in case 54 cm.", "Graphic scales given in Italian miles."] - }, { - "label": "Identifier", - "value": ["\u003ca href='http://arks.princeton.edu/ark:/88435/5d86p0240' alt='Identifier'\u003ehttp://arks.princeton.edu/ark:/88435/5d86p0240\u003c/a\u003e"] - }, { - "label": "Replaces", - "value": ["pudl0023/4166894"] - }, { - "label": "Title", - "value": [{ - "@value": "Nvova pianta et alzata della città di Roma con tvtte le strade, piazze et edificii de tempi, palazzi, giardini et altre fabbriche antiche e moderne come si trovano al presente nel pontificato di N.S. Papa Innocentio XI con le loro dichiarationi nomi et indice copiosissimo / disegniata et intagliata da Gio", - "@language": "ita" - }] - }, { - "label": "Type", - "value": ["Maps", "Maps, Pictorial"] - }, { - "label": "Contributor", - "value": ["Rossi, Giovanni Giacomo de, 1627-1691", "Widman, Georgio"] - }, { - "label": "Creator", - "value": ["Falda, Giovanni Battista, approximately 1640-1678"] - }, { - "label": "Date", - "value": ["1730"] - }, { - "label": "Language", - "value": ["Italian"] - }, { - "label": "Local Identifier", - "value": ["p0p097960k"] - }, { - "label": "Publisher", - "value": ["[Roma] : G. G. de Rossi, 1730."] - }, { - "label": "Subject", - "value": ["Rome (Italy)—Maps—Early works to 1800"] - }, { - "label": "Call Number", - "value": ["Oversize G6714.R7 F342 1730e", "Electronic Resource"] - }, { - "label": "Location", - "value": ["SAX Oversize G6714.R7 F342 1730e", "SAX Electronic Resource", "ELF1 Oversize G6714.R7 F342 1730e", "ELF1 Electronic Resource"] - }, { - "label": "Electronic Locations", - "value": [{ - "id": null, - "internal_resource": "LabeledURI", - "created_at": null, - "updated_at": null, - "new_record": true, - "uri": { - "@id": "https://catalog.princeton.edu/catalog/4166894#view" - }, - "label": "Digital content" - }] - }, { - "label": "Member Of Collections", - "value": ["Marquand Library Selections"] - }], - "sequences": [{ - "@type": "sc:Sequence", - "@id": "https://figgy.princeton.edu/concern/scanned_resources/e38d7ebb-3ae6-4eb8-a3ef-d70b5184b005/manifest/sequence/normal", - "rendering": [{ - "@id": "https://figgy.princeton.edu/concern/scanned_resources/e38d7ebb-3ae6-4eb8-a3ef-d70b5184b005/pdf", - "label": "Download as PDF", - "format": "application/pdf" - }], - "canvases": [{ - "@type": "sc:Canvas", - "@id": "https://figgy.princeton.edu/concern/scanned_resources/e38d7ebb-3ae6-4eb8-a3ef-d70b5184b005/manifest/canvas/1ba0d507-a39d-4720-bb7d-2abe2cfa4b1e", - "label": "Composite view", - "local_identifier": "p1257bv430", - "rendering": [{ - "@id": "https://figgy.princeton.edu/downloads/1ba0d507-a39d-4720-bb7d-2abe2cfa4b1e/file/64074eff-c630-44a8-9733-485e5e0e0f34", - "label": "Download the original file", - "format": "image/tiff" - }], - "width": 6945, - "height": 7200, - "images": [{ - "@type": "oa:Annotation", - "motivation": "sc:painting", - "resource": { - "@type": "dctypes:Image", - "@id": "https://iiif-cloud.princeton.edu/iiif/2/0b%2F21%2F99%2F0b21998720154beaacc8218b5063373d%2Fintermediate_file/full/1000,/0/default.jpg", - "height": 7200, - "width": 6945, - "format": "image/jpeg", - "service": { - "@context": "http://iiif.io/api/image/2/context.json", - "@id": "https://iiif-cloud.princeton.edu/iiif/2/0b%2F21%2F99%2F0b21998720154beaacc8218b5063373d%2Fintermediate_file", - "profile": "http://iiif.io/api/image/2/level2.json" - } - }, - "@id": "https://figgy.princeton.edu/concern/scanned_resources/e38d7ebb-3ae6-4eb8-a3ef-d70b5184b005/manifest/image/1ba0d507-a39d-4720-bb7d-2abe2cfa4b1e", - "on": "https://figgy.princeton.edu/concern/scanned_resources/e38d7ebb-3ae6-4eb8-a3ef-d70b5184b005/manifest/canvas/1ba0d507-a39d-4720-bb7d-2abe2cfa4b1e" - }] - }], - "viewingHint": "paged" - }], - "structures": [{ - "@type": "sc:Range", - "@id": "https://figgy.princeton.edu/concern/scanned_resources/e38d7ebb-3ae6-4eb8-a3ef-d70b5184b005/manifest/range/rb5280354-6427-4407-b83e-364733b1a988", - "label": "", - "viewingHint": "top", - "ranges": [], - "canvases": ["https://figgy.princeton.edu/concern/scanned_resources/e38d7ebb-3ae6-4eb8-a3ef-d70b5184b005/manifest/canvas/1ba0d507-a39d-4720-bb7d-2abe2cfa4b1e"] - }], - "seeAlso": [{ - "@id": "https://figgy.princeton.edu/catalog/e38d7ebb-3ae6-4eb8-a3ef-d70b5184b005.jsonld", - "format": "application/ld+json" - }, { - "@id": "https://bibdata.princeton.edu/bibliographic/4166894", - "format": "text/xml" - }], - "license": "http://rightsstatements.org/vocab/NKC/1.0/", - "thumbnail": { - "@id": "https://iiif-cloud.princeton.edu/iiif/2/0b%2F21%2F99%2F0b21998720154beaacc8218b5063373d%2Fintermediate_file/full/!200,150/0/default.jpg", - "service": { - "@context": "http://iiif.io/api/image/2/context.json", - "@id": "https://iiif-cloud.princeton.edu/iiif/2/0b%2F21%2F99%2F0b21998720154beaacc8218b5063373d%2Fintermediate_file", - "profile": "http://iiif.io/api/image/2/level2.json" - } - }, - "rendering": { - "@id": "http://arks.princeton.edu/ark:/88435/5d86p0240", - "format": "text/html" - }, - "logo": "https://figgy.princeton.edu/assets/pul_logo_icon-7b5f9384dfa5ca04f4851c6ee9e44e2d6953e55f893472a3e205e1591d3b2ca6.png" -} \ No newline at end of file diff --git a/apps/ingest/fixtures/single-image.zip b/apps/ingest/fixtures/single-image.zip deleted file mode 100644 index f7ab31fb4..000000000 Binary files a/apps/ingest/fixtures/single-image.zip and /dev/null differ diff --git a/apps/ingest/fixtures/tsv.zip b/apps/ingest/fixtures/tsv.zip deleted file mode 100644 index d0a4c02f8..000000000 Binary files a/apps/ingest/fixtures/tsv.zip and /dev/null differ diff --git a/apps/ingest/forms.py b/apps/ingest/forms.py deleted file mode 100644 index a4cc66868..000000000 --- a/apps/ingest/forms.py +++ /dev/null @@ -1,16 +0,0 @@ -from django import forms -from django.forms import ClearableFileInput -from .models import Bulk - - -class MultipleFileInput(forms.ClearableFileInput): - allow_multiple_selected = True - - -class BulkVolumeUploadForm(forms.ModelForm): - class Meta: - model = Bulk - fields = ["image_server", "volume_files", "collections"] - widgets = { - "volume_files": MultipleFileInput, - } diff --git a/apps/ingest/mail.py b/apps/ingest/mail.py deleted file mode 100644 index 89c33104c..000000000 --- a/apps/ingest/mail.py +++ /dev/null @@ -1,68 +0,0 @@ -from traceback import format_tb -from django.urls.base import reverse -from django.template.loader import get_template -from django.conf import settings -from django.core.mail import send_mail - -def send_email_on_failure(task_watcher=None, exception=None, traceback=None): - """Function to send an email on task failure signal from Celery. - - :param task_watcher: The task watcher object - :type task_watcher: app.ingest.models.TaskWatcher - :param exception: Exception instance raised - :type exception: Exception - :param traceback: Stack trace object - :type traceback: traceback - """ - context = {} - if task_watcher is not None: - context['filename'] = task_watcher.filename - if exception is not None: - context['exception'] = exception.__repr__() - if traceback is not None: - context['traceback'] = '\n'.join(format_tb(traceback)) - context['result_url'] = settings.HOSTNAME + reverse( - "admin:%s_%s_change" - % ( - task_watcher.task_result._meta.app_label, - task_watcher.task_result._meta.model_name, - ), - args=[task_watcher.task_result.id], - ) - html_email = get_template('ingest_failure_email.html').render(context) - text_email = get_template('ingest_failure_email.txt').render(context) - if task_watcher is not None and task_watcher.task_creator is not None: - send_mail( - '[Readux] Failed: Ingest ' + task_watcher.filename, - text_email, - settings.READUX_EMAIL_SENDER, - [task_watcher.task_creator.email], - fail_silently=False, - html_message=html_email - ) - -def send_email_on_success(task_watcher=None): - context = {} - if task_watcher is not None: - context['filename'] = task_watcher.filename - if task_watcher is not None and task_watcher.associated_manifest is not None: - context['manifest_url'] = settings.HOSTNAME + reverse( - 'admin:manifests_manifest_change', args=(task_watcher.associated_manifest.id,) - ) - context['manifest_pid'] = task_watcher.associated_manifest.pid - context['volume_url'] = task_watcher.associated_manifest.get_volume_url() - else: - context['manifests_list_url'] = settings.HOSTNAME + reverse( - 'admin:manifests_manifest_changelist' - ) - html_email = get_template('ingest_success_email.html').render(context) - text_email = get_template('ingest_success_email.txt').render(context) - if task_watcher is not None and task_watcher.task_creator is not None: - send_mail( - '[Readux] Ingest complete: ' + task_watcher.filename, - text_email, - settings.READUX_EMAIL_SENDER, - [task_watcher.task_creator.email], - fail_silently=False, - html_message=html_email - ) \ No newline at end of file diff --git a/apps/ingest/migrations/0001_initial.py b/apps/ingest/migrations/0001_initial.py deleted file mode 100644 index 14d62394c..000000000 --- a/apps/ingest/migrations/0001_initial.py +++ /dev/null @@ -1,21 +0,0 @@ -# Generated by Django 2.2.10 on 2020-10-21 15:50 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - initial = True - - dependencies = [ - ] - - operations = [ - migrations.CreateModel( - name='Local', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('bundle', models.FileField(upload_to='tmp/')), - ], - ), - ] diff --git a/apps/ingest/migrations/0001_squashed_0005_auto_20201027_1653.py b/apps/ingest/migrations/0001_squashed_0005_auto_20201027_1653.py deleted file mode 100644 index 54fe41309..000000000 --- a/apps/ingest/migrations/0001_squashed_0005_auto_20201027_1653.py +++ /dev/null @@ -1,30 +0,0 @@ -# Generated by Django 2.2.10 on 2020-10-29 15:21 - -import apps.ingest.models -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - replaces = [('ingest', '0001_initial'), ('ingest', '0002_auto_20201021_1610'), ('ingest', '0003_auto_20201027_1452'), ('ingest', '0004_auto_20201027_1605'), ('ingest', '0005_auto_20201027_1653')] - - initial = True - - dependencies = [ - ('manifests', '0012_auto_20200819_1608'), - ('canvases', '0005_auto_20201027_1452'), - ] - - operations = [ - migrations.CreateModel( - name='Local', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('bundle', models.FileField(upload_to='')), - ('temp_file_path', models.FilePathField(path='/tmp/tmpz0h5pz94')), - ('image_server', models.ForeignKey(null=True, on_delete=django.db.models.deletion.DO_NOTHING, to='manifests.ImageServer')), - ('manifest', models.ForeignKey(null=True, on_delete=django.db.models.deletion.DO_NOTHING, to='manifests.Manifest')), - ], - ), - ] diff --git a/apps/ingest/migrations/0002_auto_20201021_1610.py b/apps/ingest/migrations/0002_auto_20201021_1610.py deleted file mode 100644 index 5aedf96ac..000000000 --- a/apps/ingest/migrations/0002_auto_20201021_1610.py +++ /dev/null @@ -1,19 +0,0 @@ -# Generated by Django 2.2.10 on 2020-10-21 16:10 - -import apps.ingest.models -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('ingest', '0001_initial'), - ] - - operations = [ - migrations.AlterField( - model_name='local', - name='bundle', - field=models.FileField(upload_to=''), - ), - ] diff --git a/apps/ingest/migrations/0002_auto_20210106_1658.py b/apps/ingest/migrations/0002_auto_20210106_1658.py deleted file mode 100644 index d7de0b6d6..000000000 --- a/apps/ingest/migrations/0002_auto_20210106_1658.py +++ /dev/null @@ -1,23 +0,0 @@ -# Generated by Django 2.2.10 on 2021-01-06 16:58 - -import apps.ingest.models -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - dependencies = [ - ('ingest', '0001_squashed_0005_auto_20201027_1653'), - ] - - operations = [ - migrations.CreateModel( - name='Remote', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('remote_url', models.CharField(max_length=255)), - ('manifest', models.ForeignKey(null=True, on_delete=django.db.models.deletion.DO_NOTHING, to='manifests.Manifest')), - ], - ), - ] diff --git a/apps/ingest/migrations/0002_auto_20210107_2159.py b/apps/ingest/migrations/0002_auto_20210107_2159.py deleted file mode 100644 index 7c91a5cb7..000000000 --- a/apps/ingest/migrations/0002_auto_20210107_2159.py +++ /dev/null @@ -1,15 +0,0 @@ -# Generated by Django 2.2.10 on 2021-01-07 21:59 - -import apps.ingest.models -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('ingest', '0001_squashed_0005_auto_20201027_1653'), - ] - - operations = [ - - ] diff --git a/apps/ingest/migrations/0003_auto_20201027_1452.py b/apps/ingest/migrations/0003_auto_20201027_1452.py deleted file mode 100644 index e6fa623ab..000000000 --- a/apps/ingest/migrations/0003_auto_20201027_1452.py +++ /dev/null @@ -1,21 +0,0 @@ -# Generated by Django 2.2.10 on 2020-10-27 14:52 - -import apps.ingest.models -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - dependencies = [ - ('canvases', '0005_auto_20201027_1452'), - ('ingest', '0002_auto_20201021_1610'), - ] - - operations = [ - migrations.AddField( - model_name='local', - name='image_server', - field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='canvases.IServer'), - ) - ] diff --git a/apps/ingest/migrations/0003_auto_20210108_1619.py b/apps/ingest/migrations/0003_auto_20210108_1619.py deleted file mode 100644 index 526bc6232..000000000 --- a/apps/ingest/migrations/0003_auto_20210108_1619.py +++ /dev/null @@ -1,15 +0,0 @@ -# Generated by Django 2.2.10 on 2021-01-08 16:19 - -import apps.ingest.models -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('ingest', '0002_auto_20210106_1658'), - ] - - operations = [ - - ] diff --git a/apps/ingest/migrations/0004_auto_20201027_1605.py b/apps/ingest/migrations/0004_auto_20201027_1605.py deleted file mode 100644 index 1291e996b..000000000 --- a/apps/ingest/migrations/0004_auto_20201027_1605.py +++ /dev/null @@ -1,15 +0,0 @@ -# Generated by Django 2.2.10 on 2020-10-27 16:05 - -import apps.ingest.models -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('ingest', '0003_auto_20201027_1452'), - ] - - operations = [ - - ] diff --git a/apps/ingest/migrations/0004_auto_20210108_1922.py b/apps/ingest/migrations/0004_auto_20210108_1922.py deleted file mode 100644 index 32fa05958..000000000 --- a/apps/ingest/migrations/0004_auto_20210108_1922.py +++ /dev/null @@ -1,15 +0,0 @@ -# Generated by Django 2.2.10 on 2021-01-08 19:22 - -import apps.ingest.models -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('ingest', '0003_auto_20210108_1619'), - ] - - operations = [ - - ] diff --git a/apps/ingest/migrations/0005_auto_20201027_1653.py b/apps/ingest/migrations/0005_auto_20201027_1653.py deleted file mode 100644 index dbeb94a81..000000000 --- a/apps/ingest/migrations/0005_auto_20201027_1653.py +++ /dev/null @@ -1,26 +0,0 @@ -# Generated by Django 2.2.10 on 2020-10-27 16:53 - -import apps.ingest.models -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - dependencies = [ - ('manifests', '0012_auto_20200819_1608'), - ('ingest', '0004_auto_20201027_1605'), - ] - - operations = [ - migrations.AddField( - model_name='local', - name='manifest', - field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.DO_NOTHING, to='manifests.Manifest'), - ), - migrations.AlterField( - model_name='local', - name='image_server', - field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.DO_NOTHING, to='manifests.ImageServer'), - ) - ] diff --git a/apps/ingest/migrations/0005_auto_20210119_1936.py b/apps/ingest/migrations/0005_auto_20210119_1936.py deleted file mode 100644 index da61d2c4c..000000000 --- a/apps/ingest/migrations/0005_auto_20210119_1936.py +++ /dev/null @@ -1,21 +0,0 @@ -# Generated by Django 2.2.10 on 2021-01-19 19:36 - -import apps.ingest.models -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - dependencies = [ - ('ingest', '0004_auto_20210108_1922'), - ('manifests', '0015_auto_20210108_1922') - ] - - operations = [ - migrations.AlterField( - model_name='local', - name='image_server', - field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.DO_NOTHING, to='manifests.ImageServer'), - ), - ] diff --git a/apps/ingest/migrations/0006_auto_20210122_1646.py b/apps/ingest/migrations/0006_auto_20210122_1646.py deleted file mode 100644 index 64d37a74e..000000000 --- a/apps/ingest/migrations/0006_auto_20210122_1646.py +++ /dev/null @@ -1,15 +0,0 @@ -# Generated by Django 2.2.10 on 2021-01-22 16:46 - -import apps.ingest.models -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('ingest', '0005_auto_20210119_1936'), - ] - - operations = [ - - ] diff --git a/apps/ingest/migrations/0007_merge_20210208_2009.py b/apps/ingest/migrations/0007_merge_20210208_2009.py deleted file mode 100644 index 3d2772de2..000000000 --- a/apps/ingest/migrations/0007_merge_20210208_2009.py +++ /dev/null @@ -1,14 +0,0 @@ -# Generated by Django 2.2.10 on 2021-02-08 20:09 - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('ingest', '0006_auto_20210122_1646'), - ('ingest', '0002_auto_20210107_2159'), - ] - - operations = [ - ] diff --git a/apps/ingest/migrations/0008_auto_20210309_1840.py b/apps/ingest/migrations/0008_auto_20210309_1840.py deleted file mode 100644 index 79c60a295..000000000 --- a/apps/ingest/migrations/0008_auto_20210309_1840.py +++ /dev/null @@ -1,15 +0,0 @@ -# Generated by Django 2.2.10 on 2021-03-09 18:40 - -import apps.ingest.models -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('ingest', '0007_merge_20210208_2009'), - ] - - operations = [ - - ] diff --git a/apps/ingest/migrations/0009_auto_20210805_1731.py b/apps/ingest/migrations/0009_auto_20210805_1731.py deleted file mode 100644 index bc58097ba..000000000 --- a/apps/ingest/migrations/0009_auto_20210805_1731.py +++ /dev/null @@ -1,37 +0,0 @@ -# Generated by Django 2.2.10 on 2021-08-05 17:31 - -import apps.ingest.models -from django.db import migrations, models -import storages.backends.s3boto3 -import uuid - - -class Migration(migrations.Migration): - - dependencies = [ - ('ingest', '0008_auto_20210309_1840'), - ] - - operations = [ - migrations.CreateModel( - name='Bulk', - fields=[ - ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), - ], - ), - migrations.CreateModel( - name='BulkUploads', - fields=[ - ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), - ('volume_file', models.FileField(storage=storages.backends.s3boto3.S3Boto3Storage, upload_to=apps.ingest.models.bulk_path)), - ], - ), - migrations.AlterModelOptions( - name='local', - options={'verbose_name_plural': 'Local'}, - ), - migrations.AlterModelOptions( - name='remote', - options={'verbose_name_plural': 'Remote'}, - ), - ] diff --git a/apps/ingest/migrations/0010_auto_20210805_1743.py b/apps/ingest/migrations/0010_auto_20210805_1743.py deleted file mode 100644 index f974954ac..000000000 --- a/apps/ingest/migrations/0010_auto_20210805_1743.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 2.2.10 on 2021-08-05 17:43 - -import apps.ingest.models -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('ingest', '0009_auto_20210805_1731'), - ] - - operations = [ - migrations.RenameModel( - old_name='BulkUploads', - new_name='Volume', - ) - ] diff --git a/apps/ingest/migrations/0011_auto_20210805_1748.py b/apps/ingest/migrations/0011_auto_20210805_1748.py deleted file mode 100644 index 38aee87a3..000000000 --- a/apps/ingest/migrations/0011_auto_20210805_1748.py +++ /dev/null @@ -1,21 +0,0 @@ -# Generated by Django 2.2.10 on 2021-08-05 17:48 - -import apps.ingest.models -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - dependencies = [ - ('ingest', '0010_auto_20210805_1743'), - ] - - operations = [ - migrations.AddField( - model_name='volume', - name='bulk', - field=models.ForeignKey(default='6b48744e-30a6-4de3-8ff5-5a527cf48e4e', on_delete=django.db.models.deletion.DO_NOTHING, to='ingest.Bulk'), - preserve_default=False, - ) - ] diff --git a/apps/ingest/migrations/0015_auto_20210811_1605.py b/apps/ingest/migrations/0015_auto_20210811_1605.py deleted file mode 100644 index 6edcf2461..000000000 --- a/apps/ingest/migrations/0015_auto_20210811_1605.py +++ /dev/null @@ -1,25 +0,0 @@ -# Generated by Django 2.2.10 on 2021-08-11 16:05 - -import apps.ingest.models -import apps.ingest.storages -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('ingest', '0011_auto_20210805_1748'), - ] - - operations = [ - migrations.AlterField( - model_name='local', - name='bundle', - field=models.FileField(storage=apps.ingest.storages.TmpStorage(), upload_to=''), - ), - migrations.AlterField( - model_name='volume', - name='volume_file', - field=models.FileField(storage=apps.ingest.storages.TmpStorage(), upload_to=apps.ingest.models.bulk_path), - ), - ] diff --git a/apps/ingest/migrations/0016_auto_20210812_1339.py b/apps/ingest/migrations/0016_auto_20210812_1339.py deleted file mode 100644 index 4dc09b46f..000000000 --- a/apps/ingest/migrations/0016_auto_20210812_1339.py +++ /dev/null @@ -1,19 +0,0 @@ -# Generated by Django 2.2.10 on 2021-08-12 13:39 - -import apps.ingest.models -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('ingest', '0015_auto_20210811_1605'), - ] - - operations = [ - migrations.AddField( - model_name='local', - name='bundle_local', - field=models.BooleanField(default=False), - ) - ] diff --git a/apps/ingest/migrations/0017_auto_20210812_1358.py b/apps/ingest/migrations/0017_auto_20210812_1358.py deleted file mode 100644 index 66437331a..000000000 --- a/apps/ingest/migrations/0017_auto_20210812_1358.py +++ /dev/null @@ -1,23 +0,0 @@ -# Generated by Django 2.2.10 on 2021-08-12 13:58 - -import apps.ingest.models -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('ingest', '0016_auto_20210812_1339'), - ] - - operations = [ - migrations.RemoveField( - model_name='local', - name='bundle_local', - ), - migrations.AddField( - model_name='local', - name='local_bundle_path', - field=models.CharField(blank=True, max_length=100, null=True), - ) - ] diff --git a/apps/ingest/migrations/0018_auto_20210813_1514.py b/apps/ingest/migrations/0018_auto_20210813_1514.py deleted file mode 100644 index 38279670a..000000000 --- a/apps/ingest/migrations/0018_auto_20210813_1514.py +++ /dev/null @@ -1,29 +0,0 @@ -# Generated by Django 2.2.10 on 2021-08-13 15:14 - -import apps.ingest.models -import apps.ingest.storages -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('ingest', '0017_auto_20210812_1358'), - ] - - operations = [ - migrations.RemoveField( - model_name='local', - name='temp_file_path', - ), - migrations.AlterField( - model_name='local', - name='bundle', - field=models.FileField(storage=apps.ingest.storages.IngestStorage(), upload_to=''), - ), - migrations.AlterField( - model_name='volume', - name='volume_file', - field=models.FileField(storage=apps.ingest.storages.IngestStorage(), upload_to=apps.ingest.models.bulk_path), - ), - ] diff --git a/apps/ingest/migrations/0019_auto_20210819_1310.py b/apps/ingest/migrations/0019_auto_20210819_1310.py deleted file mode 100644 index 4cbd56d10..000000000 --- a/apps/ingest/migrations/0019_auto_20210819_1310.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 2.2.23 on 2021-08-19 13:10 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('ingest', '0018_auto_20210813_1514'), - ] - - operations = [ - migrations.AlterField( - model_name='local', - name='local_bundle_path', - field=models.CharField(blank=True, max_length=500, null=True), - ), - ] diff --git a/apps/ingest/migrations/0020_auto_20210915_1327.py b/apps/ingest/migrations/0020_auto_20210915_1327.py deleted file mode 100644 index c22e494ba..000000000 --- a/apps/ingest/migrations/0020_auto_20210915_1327.py +++ /dev/null @@ -1,29 +0,0 @@ -# Generated by Django 2.2.23 on 2021-09-15 13:27 - -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - dependencies = [ - ('manifests', '0020_auto_20210915_1327'), - ('ingest', '0019_auto_20210819_1310'), - ] - - operations = [ - migrations.AlterModelOptions( - name='bulk', - options={'verbose_name_plural': 'Bulk'}, - ), - migrations.AddField( - model_name='bulk', - name='image_server', - field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.DO_NOTHING, to='manifests.ImageServer'), - ), - migrations.AlterField( - model_name='volume', - name='bulk', - field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='ingest.Bulk'), - ), - ] diff --git a/apps/ingest/migrations/0021_auto_20210915_1433.py b/apps/ingest/migrations/0021_auto_20210915_1433.py deleted file mode 100644 index 619066660..000000000 --- a/apps/ingest/migrations/0021_auto_20210915_1433.py +++ /dev/null @@ -1,24 +0,0 @@ -# Generated by Django 2.2.23 on 2021-09-15 14:33 - -import apps.ingest.models -import apps.ingest.storages -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('ingest', '0020_auto_20210915_1327'), - ] - - operations = [ - migrations.AddField( - model_name='bulk', - name='volume_files', - field=models.FileField(default='test', storage=apps.ingest.storages.IngestStorage(), upload_to=apps.ingest.models.bulk_path), - preserve_default=False, - ), - migrations.DeleteModel( - name='Volume', - ), - ] diff --git a/apps/ingest/migrations/0022_auto_20210915_1651.py b/apps/ingest/migrations/0022_auto_20210915_1651.py deleted file mode 100644 index 54b998785..000000000 --- a/apps/ingest/migrations/0022_auto_20210915_1651.py +++ /dev/null @@ -1,25 +0,0 @@ -# Generated by Django 2.2.23 on 2021-09-15 16:51 - -import apps.ingest.storages -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - dependencies = [ - ('ingest', '0021_auto_20210915_1433'), - ] - - operations = [ - migrations.AddField( - model_name='local', - name='bulk', - field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='local_uploads', to='ingest.Bulk'), - ), - migrations.AlterField( - model_name='bulk', - name='volume_files', - field=models.FileField(storage=apps.ingest.storages.IngestStorage(), upload_to=''), - ), - ] diff --git a/apps/ingest/migrations/0023_auto_20210916_1803.py b/apps/ingest/migrations/0023_auto_20210916_1803.py deleted file mode 100644 index 5d25b056a..000000000 --- a/apps/ingest/migrations/0023_auto_20210916_1803.py +++ /dev/null @@ -1,24 +0,0 @@ -# Generated by Django 2.2.23 on 2021-09-16 18:03 - -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - dependencies = [ - ('ingest', '0022_auto_20210915_1651'), - ] - - operations = [ - migrations.AlterField( - model_name='bulk', - name='volume_files', - field=models.FileField(upload_to=''), - ), - migrations.AlterField( - model_name='local', - name='bulk', - field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.DO_NOTHING, related_name='local_uploads', to='ingest.Bulk'), - ), - ] diff --git a/apps/ingest/migrations/0024_auto_20210916_1809.py b/apps/ingest/migrations/0024_auto_20210916_1809.py deleted file mode 100644 index c187f29ea..000000000 --- a/apps/ingest/migrations/0024_auto_20210916_1809.py +++ /dev/null @@ -1,19 +0,0 @@ -# Generated by Django 2.2.23 on 2021-09-16 18:09 - -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - dependencies = [ - ('ingest', '0023_auto_20210916_1803'), - ] - - operations = [ - migrations.AlterField( - model_name='local', - name='bulk', - field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='local_uploads', to='ingest.Bulk'), - ), - ] diff --git a/apps/ingest/migrations/0025_auto_20210920_2130.py b/apps/ingest/migrations/0025_auto_20210920_2130.py deleted file mode 100644 index 81d791c25..000000000 --- a/apps/ingest/migrations/0025_auto_20210920_2130.py +++ /dev/null @@ -1,25 +0,0 @@ -# Generated by Django 2.2.23 on 2021-09-20 21:30 - -import apps.ingest.models -import django.contrib.postgres.fields.jsonb -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('ingest', '0024_auto_20210916_1809'), - ] - - operations = [ - migrations.AddField( - model_name='local', - name='metadata', - field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=dict), - ), - migrations.AlterField( - model_name='bulk', - name='volume_files', - field=models.FileField(upload_to=apps.ingest.models.bulk_path), - ), - ] diff --git a/apps/ingest/migrations/0026_remote_metadata.py b/apps/ingest/migrations/0026_remote_metadata.py deleted file mode 100644 index 59225d180..000000000 --- a/apps/ingest/migrations/0026_remote_metadata.py +++ /dev/null @@ -1,19 +0,0 @@ -# Generated by Django 2.2.23 on 2021-09-20 22:08 - -import django.contrib.postgres.fields.jsonb -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('ingest', '0025_auto_20210920_2130'), - ] - - operations = [ - migrations.AddField( - model_name='remote', - name='metadata', - field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=dict), - ), - ] diff --git a/apps/ingest/migrations/0027_remove_local_local_bundle_path.py b/apps/ingest/migrations/0027_remove_local_local_bundle_path.py deleted file mode 100644 index 5e8ac7adf..000000000 --- a/apps/ingest/migrations/0027_remove_local_local_bundle_path.py +++ /dev/null @@ -1,17 +0,0 @@ -# Generated by Django 2.2.23 on 2021-09-22 19:57 - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('ingest', '0026_remote_metadata'), - ] - - operations = [ - migrations.RemoveField( - model_name='local', - name='local_bundle_path', - ), - ] diff --git a/apps/ingest/migrations/0028_ingesttaskwatcher.py b/apps/ingest/migrations/0028_ingesttaskwatcher.py deleted file mode 100644 index ff69456d5..000000000 --- a/apps/ingest/migrations/0028_ingesttaskwatcher.py +++ /dev/null @@ -1,29 +0,0 @@ -# Generated by Django 2.2.23 on 2021-09-28 18:39 - -from django.conf import settings -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - dependencies = [ - migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ('ingest', '0027_remove_local_local_bundle_path'), - ] - - operations = [ - migrations.CreateModel( - name='IngestTaskWatcher', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('filename', models.CharField(max_length=255, null=True)), - ('task_id', models.CharField(max_length=255, null=True)), - ('task_creator', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='created_tasks', to=settings.AUTH_USER_MODEL)), - ('task_result', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='django_celery_results.TaskResult')), - ], - options={ - 'verbose_name_plural': 'IngestTaskWatcher', - }, - ), - ] diff --git a/apps/ingest/migrations/0029_auto_20210928_1851.py b/apps/ingest/migrations/0029_auto_20210928_1851.py deleted file mode 100644 index 2044fb009..000000000 --- a/apps/ingest/migrations/0029_auto_20210928_1851.py +++ /dev/null @@ -1,20 +0,0 @@ -# Generated by Django 2.2.23 on 2021-09-28 18:51 - -from django.db import migrations -import django.db.models.manager - - -class Migration(migrations.Migration): - - dependencies = [ - ('ingest', '0028_ingesttaskwatcher'), - ] - - operations = [ - migrations.AlterModelManagers( - name='ingesttaskwatcher', - managers=[ - ('manager', django.db.models.manager.Manager()), - ], - ), - ] diff --git a/apps/ingest/migrations/0030_auto_20211007_2031.py b/apps/ingest/migrations/0030_auto_20211007_2031.py deleted file mode 100644 index ce9d6da7a..000000000 --- a/apps/ingest/migrations/0030_auto_20211007_2031.py +++ /dev/null @@ -1,25 +0,0 @@ -# Generated by Django 2.2.24 on 2021-10-07 20:31 - -from django.conf import settings -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - dependencies = [ - migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ('ingest', '0029_auto_20210928_1851'), - ] - - operations = [ - migrations.AlterModelOptions( - name='ingesttaskwatcher', - options={'verbose_name_plural': 'Ingest Statuses'}, - ), - migrations.AddField( - model_name='local', - name='creator', - field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='created_locals', to=settings.AUTH_USER_MODEL), - ), - ] diff --git a/apps/ingest/migrations/0031_ingesttaskwatcher_associated_manifest.py b/apps/ingest/migrations/0031_ingesttaskwatcher_associated_manifest.py deleted file mode 100644 index c6cbd4e36..000000000 --- a/apps/ingest/migrations/0031_ingesttaskwatcher_associated_manifest.py +++ /dev/null @@ -1,20 +0,0 @@ -# Generated by Django 2.2.24 on 2021-10-12 16:12 - -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - dependencies = [ - ('manifests', '0029_auto_20211012_1612'), - ('ingest', '0030_auto_20211007_2031'), - ] - - operations = [ - migrations.AddField( - model_name='ingesttaskwatcher', - name='associated_manifest', - field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='manifests.Manifest'), - ), - ] diff --git a/apps/ingest/migrations/0032_auto_20211026_1736.py b/apps/ingest/migrations/0032_auto_20211026_1736.py deleted file mode 100644 index 02a285ddc..000000000 --- a/apps/ingest/migrations/0032_auto_20211026_1736.py +++ /dev/null @@ -1,25 +0,0 @@ -# Generated by Django 2.2.24 on 2021-10-26 17:36 - -import apps.ingest.models -import apps.ingest.storages -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('ingest', '0031_ingesttaskwatcher_associated_manifest'), - ] - - operations = [ - migrations.AddField( - model_name='local', - name='bundle_from_bulk', - field=models.FileField(null=True, upload_to=apps.ingest.models.bulk_path), - ), - migrations.AlterField( - model_name='local', - name='bundle', - field=models.FileField(null=True, storage=apps.ingest.storages.IngestStorage(), upload_to=''), - ), - ] diff --git a/apps/ingest/migrations/0033_auto_20211028_1527.py b/apps/ingest/migrations/0033_auto_20211028_1527.py deleted file mode 100644 index a10e2faec..000000000 --- a/apps/ingest/migrations/0033_auto_20211028_1527.py +++ /dev/null @@ -1,36 +0,0 @@ -# Generated by Django 2.2.24 on 2021-10-28 15:27 - -import apps.ingest.models -import apps.ingest.storages -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('kollections', '0011_auto_20211028_1527'), - ('ingest', '0032_auto_20211026_1736'), - ] - - operations = [ - migrations.AddField( - model_name='bulk', - name='collections', - field=models.ManyToManyField(blank=True, help_text='Optional: Collections to attach to ALL volumes ingested in this form.', null=True, to='kollections.Collection'), - ), - migrations.AddField( - model_name='local', - name='collections', - field=models.ManyToManyField(blank=True, help_text='Optional: Collections to attach to the volume ingested in this form.', null=True, to='kollections.Collection'), - ), - migrations.AlterField( - model_name='local', - name='bundle', - field=models.FileField(blank=True, null=True, storage=apps.ingest.storages.IngestStorage(), upload_to=''), - ), - migrations.AlterField( - model_name='local', - name='bundle_from_bulk', - field=models.FileField(blank=True, null=True, upload_to=apps.ingest.models.bulk_path), - ), - ] diff --git a/apps/ingest/migrations/0034_auto_20220112_2229.py b/apps/ingest/migrations/0034_auto_20220112_2229.py deleted file mode 100644 index 45f825c64..000000000 --- a/apps/ingest/migrations/0034_auto_20220112_2229.py +++ /dev/null @@ -1,23 +0,0 @@ -# Generated by Django 2.2.24 on 2022-01-12 22:29 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('ingest', '0033_auto_20211028_1527'), - ] - - operations = [ - migrations.AlterField( - model_name='bulk', - name='collections', - field=models.ManyToManyField(blank=True, help_text='Optional: Collections to attach to ALL volumes ingested in this form.', to='kollections.Collection'), - ), - migrations.AlterField( - model_name='local', - name='collections', - field=models.ManyToManyField(blank=True, help_text='Optional: Collections to attach to the volume ingested in this form.', to='kollections.Collection'), - ), - ] diff --git a/apps/ingest/migrations/0035_s3ingest.py b/apps/ingest/migrations/0035_s3ingest.py deleted file mode 100644 index fc80f8c0d..000000000 --- a/apps/ingest/migrations/0035_s3ingest.py +++ /dev/null @@ -1,36 +0,0 @@ -# Generated by Django 3.2.12 on 2023-04-05 21:10 - -import apps.ingest.models -import apps.ingest.storages -from django.conf import settings -from django.db import migrations, models -import django.db.models.deletion -import uuid - - -class Migration(migrations.Migration): - - dependencies = [ - migrations.swappable_dependency(settings.AUTH_USER_MODEL), - ('manifests', '0055_alter_manifest_logo_url'), - ('kollections', '0023_auto_20220607_1453'), - ('ingest', '0034_auto_20220112_2229'), - ] - - operations = [ - migrations.CreateModel( - name='S3Ingest', - fields=[ - ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), - ('s3_bucket', models.CharField(help_text="The name of a publicly-accessible S3 bucket containing volumes to ingest,\n either at the bucket root or within subfolder(s). Each volume should have its own\n subfolder, with the volume's PID as its name.", max_length=255)), - ('s3_prefix', models.CharField(blank=True, help_text='Optional: Subfolder(s) within the S3 bucket containing your volumes to ingest.', max_length=255, null=True)), - ('metadata_spreadsheet', models.FileField(help_text='A spreadsheet file (CSV) with a row for each volume, including the\n volume PID, which must match the subfolder name in the S3 bucket.', storage=apps.ingest.storages.IngestStorage(), upload_to='')), - ('collections', models.ManyToManyField(blank=True, help_text='Optional: Collections to attach to ALL volumes ingested in this form.', to='kollections.Collection')), - ('creator', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='created_s3ingests', to=settings.AUTH_USER_MODEL)), - ('image_server', models.ForeignKey(null=True, on_delete=django.db.models.deletion.DO_NOTHING, to='manifests.imageserver')), - ], - options={ - 'verbose_name_plural': 'Amazon S3 Ingests', - }, - ), - ] diff --git a/apps/ingest/migrations/0036_alter_s3ingest_metadata_spreadsheet.py b/apps/ingest/migrations/0036_alter_s3ingest_metadata_spreadsheet.py deleted file mode 100644 index 1d6914f44..000000000 --- a/apps/ingest/migrations/0036_alter_s3ingest_metadata_spreadsheet.py +++ /dev/null @@ -1,44 +0,0 @@ -# Generated by Django 3.2.12 on 2023-04-20 20:24 - -import django.core.validators -from django.db import migrations, models - - -class Migration(migrations.Migration): - dependencies = [ - ("ingest", "0035_s3ingest"), - ] - - operations = [ - migrations.AlterField( - model_name="s3ingest", - name="metadata_spreadsheet", - field=models.FileField( - help_text="A spreadsheet file (CSV) with a row for each volume, including the\n volume PID (column name pid), which must match the subfolder name in the\n S3 bucket.", - upload_to="", - validators=[ - django.core.validators.FileExtensionValidator( - allowed_extensions=["csv"] - ) - ], - ), - ), - migrations.AlterField( - model_name="s3ingest", - name="s3_bucket", - field=models.CharField( - help_text="The name of a publicly-accessible S3 bucket containing volumes to\n ingest, either at the bucket root or within subfolder(s). Each volume should have its own\n subfolder, with the volume's PID as its name.\n
\n Example: if the bucket's URL is\n https://my-bucket.s3.us-east-1.amazonaws.com/, its name is my-bucket.", - max_length=255, - ), - ), - migrations.AlterField( - model_name="s3ingest", - name="s3_prefix", - field=models.CharField( - blank=True, - help_text="Optional: Subfolder(s) within the S3 bucket containing your volumes to ingest.\n
\n NOTE: If the volume PID subfolders are not at the bucket root, this field is\n required.", - max_length=255, - null=True, - ), - ), - ] diff --git a/apps/ingest/migrations/0037_auto_20230925_2040.py b/apps/ingest/migrations/0037_auto_20230925_2040.py deleted file mode 100644 index fb80acad3..000000000 --- a/apps/ingest/migrations/0037_auto_20230925_2040.py +++ /dev/null @@ -1,28 +0,0 @@ -# Generated by Django 3.2.15 on 2023-09-25 20:40 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('ingest', '0036_alter_s3ingest_metadata_spreadsheet'), - ] - - operations = [ - migrations.AlterField( - model_name='local', - name='metadata', - field=models.JSONField(blank=True, default=dict), - ), - migrations.AlterField( - model_name='remote', - name='metadata', - field=models.JSONField(blank=True, default=dict), - ), - migrations.AlterField( - model_name='s3ingest', - name='s3_bucket', - field=models.CharField(help_text="The name of a publicly-accessible S3 bucket containing volumes to\n ingest, either at the bucket root or within subfolder(s). Each volume should have its own\n subfolder, with the volume's PID as its name.\n
\n Example: if the bucket's URL is\n https://my-bucket.s3.us-east-1.amazonaws.com/, its name is my-bucket.", max_length=255), - ), - ] diff --git a/apps/ingest/migrations/__init__.py b/apps/ingest/migrations/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/apps/ingest/models.py b/apps/ingest/models.py deleted file mode 100644 index e85168bef..000000000 --- a/apps/ingest/models.py +++ /dev/null @@ -1,757 +0,0 @@ -""" Model classes for ingesting volumes. """ -import logging -import os -import uuid -from io import BytesIO -from mimetypes import guess_type -from tempfile import gettempdir - -import pysftp -import httpretty -from boto3 import client -from django.conf import settings -from django.core.files.base import ContentFile -from django.core.validators import FileExtensionValidator -from django.db import models -from django_celery_results.models import TaskResult -from requests import get -from stream_unzip import TruncatedDataError, stream_unzip -from tablib import Dataset - -from apps.iiif.canvases.models import Canvas -from apps.iiif.canvases.tasks import add_oa_ocr_task, add_ocr_task -from apps.iiif.kollections.models import Collection -from apps.iiif.manifests.models import ImageServer, Manifest -from apps.ingest import services -from apps.utils.fetch import fetch_url - -from .storages import IngestStorage - -LOGGER = logging.getLogger(__name__) -logging.getLogger('botocore').setLevel(logging.ERROR) -logging.getLogger('boto3').setLevel(logging.ERROR) -logging.getLogger('s3transfer').setLevel(logging.ERROR) -logging.getLogger('httpretty').setLevel(logging.CRITICAL) -logging.getLogger('httpretty.core').setLevel(logging.CRITICAL) -logging.getLogger('paramiko').setLevel(logging.CRITICAL) -logging.getLogger('elasticsearch.base').setLevel(logging.ERROR) -logging.getLogger('responses').setLevel(logging.ERROR) - -def bulk_path(instance, filename): - return os.path.join('bulk', str(instance.id), filename ) - -class IngestTaskWatcherManager(models.Manager): - """ Manager class for associating user and ingest data with a task result """ - def create_watcher(self, filename, task_id, task_result, task_creator, associated_manifest=None): - """ - Creates an instance of IngestTaskWatcher with provided params - """ - watcher = self.create( - filename=filename, - task_id=task_id, - task_result=task_result, - task_creator=task_creator, - associated_manifest=associated_manifest - ) - return watcher - - -class IngestTaskWatcher(models.Model): - """ Model class for associating user and ingest data with a task result """ - filename = models.CharField(max_length=255, null=True) - task_id = models.CharField(max_length=255, null=True) - task_result = models.ForeignKey(TaskResult, on_delete=models.CASCADE, null=True) - task_creator = models.ForeignKey( - settings.AUTH_USER_MODEL, - on_delete=models.CASCADE, - null=True, - related_name='created_tasks' - ) - associated_manifest = models.ForeignKey(Manifest, on_delete=models.SET_NULL, null=True) - manager = IngestTaskWatcherManager() - - class Meta: - verbose_name_plural = 'Ingest Statuses' - -class IngestAbstractModel(models.Model): - metadata = models.JSONField(default=dict, blank=True) - manifest = models.ForeignKey(Manifest, on_delete=models.DO_NOTHING, null=True) - - class Meta: # pylint: disable=too-few-public-methods, missing-class-docstring - abstract = True - - -class Bulk(models.Model): - """ Model class for bulk ingesting volumes from local files. """ - id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) - image_server = models.ForeignKey(ImageServer, on_delete=models.DO_NOTHING, null=True) - volume_files = models.FileField(blank=False, upload_to=bulk_path) - collections = models.ManyToManyField( - Collection, - blank=True, - help_text="Optional: Collections to attach to ALL volumes ingested in this form." - ) - - class Meta: - verbose_name_plural = 'Bulk' - -class Local(IngestAbstractModel): - """ Model class for ingesting a volume from local files. """ - bulk = models.ForeignKey(Bulk, related_name='local_uploads', on_delete=models.SET_NULL, null=True) - bundle_from_bulk = models.FileField(null=True, blank=True, upload_to=bulk_path) - bundle = models.FileField(null=True, blank=True, storage=IngestStorage()) - image_server = models.ForeignKey(ImageServer, on_delete=models.DO_NOTHING, null=True) - creator = models.ForeignKey( - settings.AUTH_USER_MODEL, - on_delete=models.SET_NULL, - null=True, - related_name='created_locals' - ) - collections = models.ManyToManyField( - Collection, - blank=True, - help_text="Optional: Collections to attach to the volume ingested in this form." - ) - - remote_dir = None - remote_files = [] - - class Meta: - verbose_name_plural = 'Local' - - @property - def s3_client(self): - if self.image_server.storage_service == 's3': - return client('s3') - - return None - - def create_sftp_connection(self): - - if self.image_server.storage_service == 'sftp': - connection_options = pysftp.CnOpts() - - # if os.environ['DJANGO_ENV'] == 'test': - connection_options.hostkeys = None - - return pysftp.Connection(**self.image_server.sftp_connection, cnopts=connection_options) - - return None - - def open_metadata(self): - """ - Set metadata property from extracted metadata from file. - """ - try: - for zipped_file, _, unzipped_chunks in stream_unzip(self.__zipped_chunks()): - _, file_name, file_type = self.__file_info(zipped_file) - tmp_file = bytes() - for chunk in unzipped_chunks: - if file_type and not self._is_junk(file_name) and 'metadata' in file_name: - tmp_file += chunk - if len(tmp_file) > 0 and file_type and not self._is_junk(file_name): - if 'csv' in file_type or 'tab-separated' in file_type: - format = 'csv' if 'csv' in file_type else 'tsv' - metadata = Dataset().load(tmp_file.decode('utf-8-sig'), format=format) - elif 'officedocument' in file_type: - metadata = Dataset().load(BytesIO(tmp_file)) - if metadata is not None: - self.metadata = services.clean_metadata(metadata.dict[0]) - return - except TruncatedDataError: - # TODO: Why does `apps.ingest.tests.test_admin.IngestAdminTest.test_local_admin_save` raise this? - pass - - def volume_to_s3(self): - """ Upload images and OCR files to an S3 bucket - - :return: Tuple of two lists. One list of image files and one of OCR files - :rtype: tuple - """ - self.__unzip_bundle() - - return ( - [file.key for file in self.image_server.bucket.objects.filter(Prefix=f'{self.manifest.pid}/') if '_*ocr*_' not in file.key and file.key.split('/')[0] == self.manifest.pid], - [file.key for file in self.image_server.bucket.objects.filter(Prefix=f'{self.manifest.pid}/') if '_*ocr*_' in file.key and file.key.split('/')[0] == self.manifest.pid] - ) - - def volume_to_sftp(self): - """ Upload images and OCR files via SFTP - - :return: Tuple of two lists. One list of image files and one of OCR files - :rtype: tuple - """ - - # sftp = self.create_sftp_connection() - # sftp.mkdir(self.manifest.pid) - # with sftp.cd(self.manifest.pid): - # sftp.mkdir('_*ocr*_') - - # sftp.close() - self.__unzip_bundle() - return self.__list_sftp_files() - - def bundle_to_s3(self): - """Uploads the zipfile stored in bundle_from_bulk to S3 - :param file_path: File path for uploaded file - :type file_path: str - """ - if bool(self.bundle_from_bulk): - # Save to bundle - if not bool(self.bundle): - if not os.path.isfile(self.bundle_from_bulk.path): - raise Exception(f"Could not find file: {self.bundle_from_bulk.path}") - bulk_name = self.bundle_from_bulk.name - with ContentFile(self.bundle_from_bulk.read()) as file_content: - self.bundle.save(bulk_name, file_content) - - # Delete tempfile - key_count = client('s3').list_objects_v2( - Bucket=self.bundle.field.storage.bucket_name, - Prefix=f'bulk/{self.id}' - )['KeyCount'] - - assert key_count == 1 - - if bool(self.bundle): - old_path = self.bundle_from_bulk.path - os.remove(old_path) - dir_path = old_path[0:old_path.rindex('/')] - if not os.path.isfile(old_path) and len(os.listdir(dir_path)) == 0: - os.rmdir(dir_path) - self.bundle_from_bulk.delete() - - @property - def file_list(self): - """Returns a list of files in the zip. Used for testing. - - :return: List of files in zip. - :rtype: list - """ - files = [] - for zipped_file, file_size, unzipped_chunks in stream_unzip(self.__zipped_chunks()): - file_path, file_name, file_type = self.__file_info(zipped_file) - files.append(file_path) - # Not looping through the chunks throws an UnexpectedSignatureError - for _ in unzipped_chunks: - pass - - return files - - def create_canvases(self): - """ - Create Canvas objects for each image file. - """ - image_files, ocr_files = ([], []) - - if self.image_server.storage_service == 's3': - image_files, ocr_files = self.volume_to_s3() - elif self.image_server.storage_service == 'sftp': - image_files, ocr_files = self.volume_to_sftp() - - if len(image_files) == 0: - # TODO: Throw an error here? - pass - - for index, key in enumerate(sorted(image_files)): - image_file = key.split('/')[-1]#.replace('_', '-') - - if not image_file: - continue - - ocr_file_path = None - if len(ocr_files) > 0: - image_name = '.'.join(image_file.split('.')[:-1]) - - try: - ocr_key = [key for key in ocr_files if image_name in key][0] - ocr_file_path = ocr_key - except IndexError: - # Every image may not have a matching OCR file - ocr_file_path = None - pass - - pid = image_file if self.manifest.pid in image_file else f'{self.manifest.pid}_{image_file}' - position = index + 1 - canvas, created = Canvas.objects.get_or_create( - manifest=self.manifest, - pid=pid, - ocr_file_path=ocr_file_path, - position=position - ) - - if created and canvas.ocr_file_path is not None: - if os.environ['DJANGO_ENV'] == 'test': - add_ocr_task(canvas.id) - else: - ocr_task_id = add_ocr_task.delay(canvas.id) - ocr_task_result = TaskResult(task_id=ocr_task_id) - ocr_task_result.save() - IngestTaskWatcher.manager.create_watcher( - task_id=ocr_task_id, - task_result=ocr_task_result, - task_creator=self.creator, - filename=canvas.ocr_file_path, - associated_manifest=self.manifest - ) - - # Request the thumbnail so a cached version is created on IIIF server. - if os.environ['DJANGO_ENV'] != 'test': - get(canvas.thumbnail) - - - if self.manifest.canvas_set.count() == len(image_files): - self.delete() - else: - # TODO: Log or though an error/waring? - pass - - def __unzip_bundle(self): - """ - Unzip and upload image and OCR files in the bundle, without loading the entire ZIP file - into memory or any of its uncompressed files. - """ - for zipped_file, _, unzipped_chunks in stream_unzip(self.__zipped_chunks()): - file_path, file_name, file_type = self.__file_info(zipped_file) - - has_type_or_is_hocr = file_name.endswith('.hocr') or file_type - tmp_file = bytes() - for chunk in unzipped_chunks: - if has_type_or_is_hocr and not self._is_junk(file_name): - tmp_file += chunk - if (file_type or file_name.endswith('.hocr')) and not self._is_junk(file_name): - file_name = file_name.replace('_', '-') - if file_type and 'image' in file_type and 'images' in file_path: - self.__upload_file(tmp_file, file_name, file_path) - if file_type: - is_ocr_file_type = ( - 'text' in file_type - or 'xml' in file_type - or 'json' in file_type - or 'html' in file_type - ) - if 'ocr' in file_path and ( - file_name.endswith('.hocr') or is_ocr_file_type - ): - self.__upload_file(tmp_file, file_name, file_path) - - def __upload_file(self, file, file_name, path): - remote_path = f'{self.manifest.pid}{self.image_server.path_delineator}{file_name}' - if self.image_server.storage_service == 's3': - if 'ocr' in path: - remote_path = f'{self.manifest.pid}/_*ocr*_/{file_name}' - self.image_server.bucket.upload_fileobj( - BytesIO(file), - remote_path - ) - elif self.image_server.storage_service == 'sftp': - # Check if the file will be stored in a sub directory. - self.remote_dir = os.path.dirname(remote_path) - # Make any needed local directories. - if self.remote_dir and not os.path.exists(os.path.join(gettempdir(), self.remote_dir)): - os.makedirs(os.path.join(gettempdir(), self.remote_dir)) - - # Define the full local path - local_path = os.path.join(gettempdir(), remote_path) - - - # Write the file to the local disk. - with open(local_path, 'wb') as f: - f.write(BytesIO(file).getbuffer()) - # for _ in range(0,5): - # print(f'local {local_path}') - - sftp = self.create_sftp_connection() - - if self.remote_dir: - # Make remote directories if needed. - if not sftp.isdir(self.remote_dir): - sftp.makedirs(self.remote_dir) - - # Change to the remote directory before upload. - sftp.chdir(self.remote_dir) - - sftp.put(local_path) - os.remove(local_path) - sftp.close() - - return remote_path - - def __list_sftp_files(self): - sftp = self.create_sftp_connection() - - remote_files = [] - - if self.remote_dir: - remote_files = [remote_file for remote_file in sftp.listdir(self.manifest.pid)] - else: - remote_files = [remote_file for remote_file in sftp.listdir() if f'{self.manifest.pid}{self.image_server.path_delineator}' in remote_file] - - files = ( - [os.path.join(self.remote_dir or '', image) for image in remote_files if 'image' in guess_type(image)[0]], - [os.path.join(self.remote_dir or '', ocr_file) for ocr_file in remote_files if 'image' not in guess_type(ocr_file)[0]], - ) - sftp.close() - - return files - - @staticmethod - def _is_junk(path): - file = path.split('/')[-1] - return file.startswith('.') or file.startswith('~') or file.startswith('__') - - @staticmethod - def __file_info(path): - path = path.decode('UTF-8') - return [ - path, - path.split('/')[-1], - guess_type(path)[0] - ] - - def __zipped_chunks(self): - try: - yield from self.bundle.file.obj.get()['Body'].iter_chunks(chunk_size=10240) - except Exception as error: - LOGGER.error(error) - - -class Remote(IngestAbstractModel): - """ Model class for ingesting a volume from remote manifest. """ - remote_url = models.CharField(max_length=255) - - class Meta: - verbose_name_plural = 'Remote' - - @property - def image_server(self): - """ Image server the Manifest """ - iiif_url = services.extract_image_server( - self.remote_manifest['sequences'][0]['canvases'][0] - ) - server, _created = ImageServer.objects.get_or_create(server_base=iiif_url) - - return server - - @property - def remote_manifest(self): - """Serialized remote IIIF Manifest data. - - :return: IIIF Manifest - :rtype: dict - """ - # if os.environ['DJANGO_ENV'] == 'test': - # return json.loads(open(os.path.join(settings.APPS_DIR, 'ingest/fixtures/manifest.json')).read()) - - return fetch_url(self.remote_url) - - def open_metadata(self): - """ Take a remote IIIF manifest and create a derivative version. """ - if self.remote_manifest['@context'] == 'http://iiif.io/api/presentation/2/context.json': - self.metadata = services.parse_iiif_v2_manifest(self.remote_manifest) - - def create_canvases(self): - # TODO: What if there are multiple sequences? Is that even allowed in IIIF? - for position, sc_canvas in enumerate(self.remote_manifest['sequences'][0]['canvases'], start=1): - canvas_metadata = None - # TODO: we will need some sort of check for IIIF API version, but not - # everyone includes a context for each canvas. - # if canvas['@context'] == 'http://iiif.io/api/presentation/2/context.json': - canvas_metadata = services.parse_iiif_v2_canvas(sc_canvas) - - if canvas_metadata is not None: - canvas, _ = Canvas.objects.get_or_create( - pid=canvas_metadata['pid'], - manifest=self.manifest, - position=position - ) - - for (key, value) in canvas_metadata.items(): - setattr(canvas, key, value) - canvas.save() - canvas.refresh_from_db() - - if os.environ['DJANGO_ENV'] != 'test': - add_ocr_task.delay(canvas.id) - - if 'otherContent' in sc_canvas and len(sc_canvas['otherContent']) > 0: - for content in sc_canvas['otherContent']: - if content['label'] == 'OCR Text': - if os.environ['DJANGO_ENV'] != 'test': - add_oa_ocr_task.delay(content['@id']) - else: - add_oa_ocr_task(content['@id']) - - -class S3Ingest(models.Model): - """ Model class for bulk ingesting volumes from an Amazon AWS S3 bucket. """ - id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) - s3_bucket = models.CharField( - null=False, - blank=False, - max_length=255, - help_text="""The name of a publicly-accessible S3 bucket containing volumes to - ingest, either at the bucket root or within subfolder(s). Each volume should have its own - subfolder, with the volume's PID as its name. -
- Example: if the bucket's URL is - https://my-bucket.s3.us-east-1.amazonaws.com/, its name is my-bucket.""", - ) - s3_prefix = models.CharField( - null=True, - blank=True, - max_length=255, - help_text="""Optional: Subfolder(s) within the S3 bucket containing your volumes to ingest. -
- NOTE: If the volume PID subfolders are not at the bucket root, this field is - required.""", - ) - metadata_spreadsheet = models.FileField( - null=False, - blank=False, - help_text="""A spreadsheet file (CSV) with a row for each volume, including the - volume PID (column name pid), which must match the subfolder name in the - S3 bucket.""", - validators=[FileExtensionValidator(allowed_extensions=['csv'])], - ) - image_server = models.ForeignKey(ImageServer, on_delete=models.DO_NOTHING, null=True) - collections = models.ManyToManyField( - Collection, - blank=True, - help_text="Optional: Collections to attach to ALL volumes ingested in this form." - ) - creator = models.ForeignKey( - settings.AUTH_USER_MODEL, - on_delete=models.SET_NULL, - null=True, - related_name="created_s3ingests" - ) - - class Meta: - verbose_name_plural = 'Amazon S3 Ingests' - - def create_canvases_for(self, pid): - """Create Canvas objects for each image file in a single volume, by pid.""" - image_files, ocr_files = ([], []) - - # get the list of all subfolders (aws parlance "Common Prefixes") at the root, specified - # by the s3_bucket and the s3_prefix - root_prefix = f"{self.s3_prefix}/" or "" - s3 = client("s3") - - # use paginator in case list is > 1000 items - paginator = s3.get_paginator("list_objects_v2") - root = paginator.paginate(Bucket=self.s3_bucket, Prefix=root_prefix, Delimiter="/") - for prefix_result in root.search("CommonPrefixes"): - # This really only seems to be an issue when testing. - # Maybe Moto doesn't properly mock this? - if prefix_result is None: - for s3_object in s3.list_objects_v2(Bucket=self.s3_bucket, Prefix=pid)['Contents']: - key = s3_object.get("Key") - # at this point, we just want to handle all files and ignore folder structure, - # so we can ignore keys ending in / - if key.endswith("/"): - continue - # determine if this is image or ocr and upload it - self.disambiguate_and_upload(pid, key) - else: - subfolder_name = prefix_result.get("Prefix") - subfolder_name = subfolder_name[:-1] if subfolder_name.endswith("/") else subfolder_name - # if this subfolder does not match the pid name, skip it - if subfolder_name.split("/")[-1] != pid: - continue - - LOGGER.debug(f"Subfolder found for pid: {pid}") - - subfolder_pages = paginator.paginate( - Bucket=self.s3_bucket, - Prefix=f"{root_prefix}{pid}/" if root_prefix else f"{pid}/", - ) - for page in subfolder_pages: - for s3_object in page.get("Contents"): - key = s3_object.get("Key") - # at this point, we just want to handle all files and ignore folder structure, - # so we can ignore keys ending in / - if key.endswith("/"): - continue - # determine if this is image or ocr and upload it - self.disambiguate_and_upload(pid, key) - - # collect all uploaded files - image_files, ocr_files = self.image_server_paths(pid) - - # the following is adapted closely from Local.create_canvases: - - if len(image_files) == 0: - raise Exception(f"No images found for pid {pid} in s3 bucket {self.s3_bucket}") - - # retrieve the manifest; will have been created in task function if not already present - manifest = Manifest.objects.get(pid=pid) - - # create canvases for each image - for index, img_key in enumerate(sorted(image_files)): - image_file = img_key.split("/")[-1] - if not image_file: - continue - ocr_file_path = None - if len(ocr_files) > 0: - image_name = ".".join(image_file.split(".")[:-1]) - try: - ocr_key = [k for k in ocr_files if image_name in k][0] - ocr_file_path = ocr_key - except IndexError: - # Every image may not have a matching OCR file - ocr_file_path = None - pass - canvas_pid = image_file if manifest.pid in image_file else f"{manifest.pid}_{image_file}" - position = index + 1 - canvas, created = Canvas.objects.get_or_create( - manifest=manifest, - pid=canvas_pid, - ocr_file_path=ocr_file_path, - position=position - ) - - if created and canvas.ocr_file_path is not None: - if os.environ["DJANGO_ENV"] == "test": - add_ocr_task(canvas.id) - else: - ocr_task_id = add_ocr_task.delay(canvas.id) - ocr_task_result = TaskResult(task_id=ocr_task_id) - ocr_task_result.save() - IngestTaskWatcher.manager.create_watcher( - task_id=ocr_task_id, - task_result=ocr_task_result, - task_creator=self.creator, - filename=canvas.ocr_file_path, - associated_manifest=manifest - ) - - # Request the thumbnail so a cached version is created on IIIF server. - if os.environ["DJANGO_ENV"] != "test": - get(canvas.thumbnail) - - # cleanup by deleting ingest if there is a canvas for each image - if manifest.canvas_set.count() == len(image_files): - self.delete() - else: - # TODO: Log or though an error/waring? - pass - - - def image_server_paths(self, pid): - """ Retrieve paths from image server (S3 bucket or SFTP) - - :return: Tuple of two lists. One list of image files and one of OCR files - :rtype: tuple - """ - if self.image_server.storage_service == "s3": - return ( - [ - file.key - for file in self.image_server.bucket.objects.filter( - Prefix=f"{pid}/" - ) - if "_*ocr*_" not in file.key - and file.key.split("/")[0] == pid - ], - [ - file.key - for file in self.image_server.bucket.objects.filter( - Prefix=f"{pid}/" - ) - if "_*ocr*_" in file.key - and file.key.split("/")[0] == pid - ], - ) - elif self.image_server.storage_service == "sftp": - if os.environ['DJANGO_ENV'] == 'test': - httpretty.disable() - return self.__list_sftp_files(pid) - return ([], []) - - def disambiguate_and_upload(self, pid, key): - """Determine if a file is an image or OCR and send to the upload function appropriately""" - file_path, file_name, file_type = self.__file_info(key) - if (file_type or file_name.endswith(".hocr")) and not Local._is_junk(file_name): - file_name = file_name.replace("_", "-") - if file_type and "image" in file_type and "images" in file_path: - self.__upload_file(pid, key, file_name, is_ocr=False) - if file_type: - is_ocr_file_type = ( - "text" in file_type - or "xml" in file_type - or "json" in file_type - or "html" in file_type - ) - if "ocr" in file_path and ( - file_name.endswith(".hocr") or is_ocr_file_type - ): - self.__upload_file(pid, key, file_name, is_ocr=True) - - def __upload_file(self, pid, key, file_name, is_ocr): - """Get a file from the s3 bucket and upload to the image server""" - # if we're uploading to s3 image server, just use s3 bucket copy function! - remote_path = f"{pid}{self.image_server.path_delineator}{file_name}" - if self.image_server.storage_service == "s3": - if is_ocr: - remote_path = f"{pid}/_*ocr*_/{file_name}" - LOGGER.debug(f"Copying {key} to s3 bucket") - self.image_server.bucket.copy({ "Bucket": self.s3_bucket, "Key": key }, remote_path) - # otherwise we need to actually download the file and reupload it to sftp - elif self.image_server.storage_service == "sftp": - # Check if the file will be stored in a sub directory - remote_dir = os.path.dirname(remote_path) - # Make any needed local directories - if remote_dir and not os.path.exists(os.path.join(gettempdir(), remote_dir)): - os.makedirs(os.path.join(gettempdir(), remote_dir)) - # Define the full local path - local_path = os.path.join(gettempdir(), remote_path) - # Download the file from S3 and write the file to the local disk - LOGGER.debug(f"Downloading {key} from s3 bucket") - with open(local_path, "wb") as f: - client("s3").download_fileobj(self.s3_bucket, key, f) - # Upload via sftp - sftp = self.create_sftp_connection() - if remote_dir: - # Make remote directories if needed. - if not sftp.isdir(remote_dir): - sftp.makedirs(remote_dir) - # Change to the remote directory before upload. - sftp.chdir(remote_dir) - LOGGER.debug(f"Uploading {file_name} to SFTP server") - sftp.put(local_path) - os.remove(local_path) - sftp.close() - - def create_sftp_connection(self): - """Pared-down adaptation of Local.create_sftp_connection""" - if os.environ['DJANGO_ENV'] == 'test': - httpretty.disable() - connection_options = pysftp.CnOpts() - connection_options.hostkeys = None - return pysftp.Connection(**self.image_server.sftp_connection, cnopts=connection_options) - - def __list_sftp_files(self, pid): - """Pared-down adaptation of Local.__list_sftp_files""" - if os.environ['DJANGO_ENV'] == 'test': - httpretty.disable() - sftp = self.create_sftp_connection() - remote_files = [remote_file for remote_file in sftp.listdir() if f"{pid}{self.image_server.path_delineator}" in remote_file] - - files = ( - [image for image in remote_files if "image" in guess_type(image)[0]], - [ocr_file for ocr_file in remote_files if "image" not in guess_type(ocr_file)[0]], - ) - - sftp.close() - return files - - @staticmethod - def __file_info(path): - """Pared-down adaptation of Local.__file_info""" - return [ - path, - path.split('/')[-1], - guess_type(path)[0] - ] \ No newline at end of file diff --git a/apps/ingest/storages.py b/apps/ingest/storages.py deleted file mode 100644 index b10f47b5d..000000000 --- a/apps/ingest/storages.py +++ /dev/null @@ -1,8 +0,0 @@ -from storages.backends.s3boto3 import S3Boto3Storage - -class TmpStorage(S3Boto3Storage): - bucket_name = 'readux' - location = 'tmp' - -class IngestStorage(S3Boto3Storage): - bucket_name = 'readux-ingest' \ No newline at end of file diff --git a/apps/ingest/tasks.py b/apps/ingest/tasks.py deleted file mode 100644 index 25500b216..000000000 --- a/apps/ingest/tasks.py +++ /dev/null @@ -1,174 +0,0 @@ -# pylint: disable = unused-argument - -""" Common tasks for ingest. """ -import logging -from os import environ - -from boto3 import client -from celery import Celery -from celery.signals import task_failure, task_success -from django.apps import apps -from django.conf import settings -from django.contrib.auth import get_user_model -from django.core.files.base import ContentFile -from django_celery_results.models import TaskResult - -from apps.iiif.canvases.models import Canvas -from apps.ingest.models import IngestTaskWatcher - -from .mail import send_email_on_failure, send_email_on_success -from .services import create_manifest, create_related_links, set_metadata - -# Use `apps.get_model` to avoid circular import error. Because the parameters used to -# create a background task have to be serializable, we can't just pass in the model object. -Local = apps.get_model('ingest.local') # pylint: disable = invalid-name -Remote = apps.get_model('ingest.remote') -S3Ingest = apps.get_model('ingest.S3Ingest') -Manifest = apps.get_model('manifests.Manifest') - -LOGGER = logging.getLogger(__name__) - -app = Celery('apps.ingest', result_extended=True) -app.config_from_object('django.conf:settings') -app.autodiscover_tasks(lambda: settings.INSTALLED_APPS) - -@app.task(name='creating_canvases_from_local', autoretry_for=(Exception,), retry_backoff=True, max_retries=20) -def create_canvas_form_local_task(ingest_id): - """Background task to create canvases and upload images. - - :param ingest_id: Primary key for .models.Local object - :type ingest_id: UUID - - """ - local_ingest = Local.objects.get(pk=ingest_id) - - if local_ingest.manifest is None: - local_ingest.manifest = create_manifest(local_ingest) - local_ingest.save() - local_ingest.refresh_from_db() - - local_ingest.create_canvases() - # Sometimes, the IIIF server is not ready to process the image by the time the canvas is saved to - # the database. As a double check loop through to make sure the height and width has been saved. - if environ['DJANGO_ENV'] != 'test': - for canvas in local_ingest.manifest.canvas_set.all(): - ensure_dimensions.delay(canvas.id) - -@app.task(name='ensure_dimensions', autoretry_for=(AssertionError,), retry_backoff=True, max_retries=20) -def ensure_dimensions(canvas_id): - """Task to ensure all the canvases have a height and width. Often, the IIF server has not processed - the image file by the time the Canvas object is saved. - - :param canvas_id: ID of Canvas to check. - :type canvas_id: int - """ - canvas = Canvas.objects.get(id=canvas_id) - canvas.save() - canvas.refresh_from_db() - assert canvas.width > 0 - -@app.task(name='creating_canvases_from_remote', autoretry_for=(Exception,), retry_backoff=True, max_retries=20) -def create_remote_canvases(ingest_id, *args, **kwargs): - """Task to create Canvas objects from remote IIIF manifest - - :param ingest_id: Primary key for .models.Remote object - :type ingest: UUID - """ - remote_ingest = Remote.objects.get(pk=ingest_id) - - if remote_ingest.manifest is None: - remote_ingest.manifest = create_manifest(remote_ingest) - remote_ingest.save() - remote_ingest.refresh_from_db() - - remote_ingest.create_canvases() - -@app.task(name='uploading_to_s3', autoretry_for=(Exception,), retry_backoff=True, max_retries=20) -def upload_to_s3_task(local_id): - """Task to create Canvas objects from remote IIIF manifest - - :param local_id: Primary key for .models.Local object - :type local_id: UUID - """ - local = Local.objects.get(pk=local_id) - - # Upload tempfile to S3 - local.bundle_to_s3() - - # Create manifest now that we have a file - if local.manifest is None: - local.manifest = create_manifest(local) - - # Queue task to create canvases etc. - local.save() - local.refresh_from_db() - local_task_id = create_canvas_form_local_task.delay(local_id) - local_task_result = TaskResult(task_id=local_task_id) - local_task_result.save() - IngestTaskWatcher.manager.create_watcher( - task_id=local_task_id, - task_result=local_task_result, - task_creator=get_user_model().objects.get(pk=local.creator.id), - associated_manifest=local.manifest, - filename=local.bundle.name - ) - -@app.task(name='creating_canvases_from_s3', autoretry_for=(Exception,), retry_backoff=True, max_retries=20) -def create_canvases_from_s3_ingest(metadata, ingest_id): - """Background task to create canvases and upload images. - - :param metadata: Dict containing key/value pairs for manifest metadata of a single volume - :type ingest_id: dict - :param ingest_id: Primary key for .models.S3Ingest object - :type ingest_id: UUID - """ - # Associate metadata with correct volume - pid = metadata["pid"].replace("_", "-") - try: - manifest = Manifest.objects.get(pid=pid) - except Manifest.DoesNotExist: - manifest = Manifest.objects.create(pid=pid) - set_metadata(manifest, metadata) - # Image server: set from ingest - ingest = S3Ingest.objects.get(pk=ingest_id) - manifest.image_server = ingest.image_server - # Collections: Ensure that manifest has an ID before updating the M2M relationship - manifest.save() - manifest.refresh_from_db() - # Collections: set from ingest - manifest.collections.set(ingest.collections.all()) - # Save again once relationship is set - manifest.save() - LOGGER.debug(f"creating canvases for {pid}") - ingest.create_canvases_for(pid) - # Sometimes, the IIIF server is not ready to process the image by the time the canvas is saved to - # the database. As a double check loop through to make sure the height and width has been saved. - if environ['DJANGO_ENV'] != 'test': - for canvas in manifest.canvas_set.all(): - ensure_dimensions.delay(canvas.id) - - -@task_failure.connect -def send_email_on_failure_task(sender=None, exception=None, task_id=None, traceback=None, *args, **kwargs): - """Function to send an email on task failure signal from Celery. - - :param sender: The task object - :type sender: celery.task - """ - if sender is not None and 'creating_canvases_from_local' in sender.name: - task_watcher = IngestTaskWatcher.manager.get(task_id=task_id) - if task_watcher is not None: - send_email_on_failure(task_watcher, exception, traceback, *args, **kwargs) - -@task_success.connect -def send_email_on_success_task(sender=None, **kwargs): - """Function to send an email on task success signal from Celery. - - :param sender: The task object - :type sender: celery.task - """ - if sender is not None and 'creating_canvases_from_local' in sender.name: - task_id = sender.request.id - task_watcher = IngestTaskWatcher.manager.get(task_id=task_id) - if task_watcher is not None: - send_email_on_success(task_watcher) diff --git a/apps/ingest/templates/.gitkeep b/apps/ingest/templates/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/apps/ingest/templates/admin/ingest/bulk/change_form.html b/apps/ingest/templates/admin/ingest/bulk/change_form.html deleted file mode 100644 index 051b8962d..000000000 --- a/apps/ingest/templates/admin/ingest/bulk/change_form.html +++ /dev/null @@ -1,133 +0,0 @@ -{% extends "admin/change_form.html" %} -{% load i18n admin_urls %} -{% load static %} - -{% block extrastyle %} - {{ block.super }} - -{% endblock %} -{% block submit_buttons_bottom %} -

- - -
-{% endblock %} -{% block admin_change_form_document_ready %} - {{ block.super }} - -{% endblock %} -{% block content %} - {{ block.super }} -
-
Uploading...
-
0%
-
- -
-
-

- You must leave this window open during upload. -

-

- Once upload completes, you will be sent to a new page. - You may then navigate away while the rest of the ingest completes; you - will receive an email to notify you when the ingest has completed. -

-
-
-{% endblock %} \ No newline at end of file diff --git a/apps/ingest/templates/admin/ingest/local/change_form.html b/apps/ingest/templates/admin/ingest/local/change_form.html deleted file mode 100644 index 1d87df60a..000000000 --- a/apps/ingest/templates/admin/ingest/local/change_form.html +++ /dev/null @@ -1,41 +0,0 @@ -{% extends "admin/change_form.html" %} -{% load i18n admin_urls %} -{% load static %} - -{% block extrastyle %} - {{ block.super }} - -{% endblock %} -{% block submit_buttons_bottom %} - -{% endblock %} -{% block admin_change_form_document_ready %} - {{ block.super }} - -{% endblock %} -{% block content %} - {{block.super}} -
loading
-{% endblock %} \ No newline at end of file diff --git a/apps/ingest/templates/admin/ingest/remote/change_form.html b/apps/ingest/templates/admin/ingest/remote/change_form.html deleted file mode 100644 index 1d87df60a..000000000 --- a/apps/ingest/templates/admin/ingest/remote/change_form.html +++ /dev/null @@ -1,41 +0,0 @@ -{% extends "admin/change_form.html" %} -{% load i18n admin_urls %} -{% load static %} - -{% block extrastyle %} - {{ block.super }} - -{% endblock %} -{% block submit_buttons_bottom %} - -{% endblock %} -{% block admin_change_form_document_ready %} - {{ block.super }} - -{% endblock %} -{% block content %} - {{block.super}} -
loading
-{% endblock %} \ No newline at end of file diff --git a/apps/ingest/tests/__init__.py b/apps/ingest/tests/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/apps/ingest/tests/factories.py b/apps/ingest/tests/factories.py deleted file mode 100644 index 2775bce24..000000000 --- a/apps/ingest/tests/factories.py +++ /dev/null @@ -1,46 +0,0 @@ -from os.path import join -from django_celery_results.models import TaskResult -from factory.django import DjangoModelFactory, FileField -from factory import Faker, SubFactory -from django.conf import settings -from apps.users.tests.factories import UserFactory -from apps.iiif.manifests.tests.factories import ImageServerFactory, ManifestFactory -from apps.ingest.models import Bulk, Local, Remote, IngestTaskWatcher - -class LocalFactory(DjangoModelFactory): - class Meta: - model = Local - - bundle = FileField(filename='bundle.zip', filepath=join(settings.APPS_DIR, 'ingest/fixtures/bundle.zip')) - image_server = SubFactory(ImageServerFactory) - manifest = None - -class RemoteFactory(DjangoModelFactory): - class Meta: - model = Remote - - remote_url = Faker('url') - -class BulkFactory(DjangoModelFactory): - class Meta: - model = Bulk - - volume_files = FileField(filename='bundle.zip', filepath=join(settings.APPS_DIR, 'ingest/fixtures/bundle.zip')) - image_server = SubFactory(ImageServerFactory) - -class TaskResultFactory(DjangoModelFactory): - class Meta: - model = TaskResult - - task_id = '1' - task_name = 'fake_task' - -class IngestTaskWatcherFactory(DjangoModelFactory): - class Meta: - model = IngestTaskWatcher - - task_id = '1' - filename = Faker('file_path') - task_result = SubFactory(TaskResultFactory) - task_creator = SubFactory(UserFactory) - associated_manifest = SubFactory(ManifestFactory) diff --git a/apps/ingest/tests/mock_sftp.py b/apps/ingest/tests/mock_sftp.py deleted file mode 100644 index f7ffb448c..000000000 --- a/apps/ingest/tests/mock_sftp.py +++ /dev/null @@ -1,57 +0,0 @@ -""" -Starts and stops a mock SFTP server using the Python `sftpserver` module. -https://github.com/rspivak/sftpserver -""" -import os -import signal -import socket -import subprocess -import tempfile -import logging - -LOGGER = logging.getLogger(__name__) - - -class MockSFTP: - """ - The `sftpserver` module does not provide an obvious way to start a server, do - some stuff, and then stop the server. There is a plugin for PyTest, but it - seems to create a read-only server. I don't love this, but it will work for now. - :( - """ - def __init__(self): - self.key_file = os.path.join(tempfile.gettempdir(), 'readux_test_sshkey') - self.port = 3373 - self.host = 'localhost' - self.server = None - - if not os.path.exists(self.key_file): - subprocess.run(f'ssh-keygen -b 2048 -t rsa -f {self.key_file} -q -N ""', shell=True, check=True) - - server_command = f'sftpserver -k {self.key_file} -p {self.port} --host {self.host}' - - if not self.server: - try: - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.bind((self.host, self.port)) - sock.close() - self.server = subprocess.Popen( - server_command, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - shell=True, - start_new_session=True - ) - except OSError: - # For CircleCI, the sftp server is started in the background. - pass - - def stop_server(self): - """ Kill the process and remove the key file. """ - if self.server: - try: - os.killpg(self.server.pid, signal.SIGTERM) - os.remove(self.key_file) - os.remove(f'{self.key_file}.pub') - except PermissionError: - LOGGER.error(f'Failed to kill mock SFTP server with pit {self.server.pid}') \ No newline at end of file diff --git a/apps/ingest/tests/test_admin.py b/apps/ingest/tests/test_admin.py deleted file mode 100644 index fdf56f9fa..000000000 --- a/apps/ingest/tests/test_admin.py +++ /dev/null @@ -1,408 +0,0 @@ -from os.path import join -from os import path, makedirs -from getpass import getuser -from shutil import rmtree -import boto3 -import httpretty -from django.conf import settings -from django.contrib.admin.sites import AdminSite -from django.core import files -from django.http import HttpResponseRedirect -from django.test import TestCase -from django.test.client import RequestFactory -from django.urls.base import reverse -from django_celery_results.models import TaskResult -from moto import mock_s3 -from apps.ingest.forms import BulkVolumeUploadForm -from apps.iiif.canvases.models import Canvas -from apps.iiif.kollections.models import Collection -from apps.iiif.kollections.tests.factories import CollectionFactory -from apps.iiif.manifests.models import Manifest -from apps.iiif.manifests.tests.factories import ManifestFactory, ImageServerFactory -from apps.ingest.models import Bulk, Local, Remote, IngestTaskWatcher -from apps.ingest.admin import BulkAdmin, LocalAdmin, RemoteAdmin, TaskWatcherAdmin -from apps.users.tests.factories import UserFactory -from .factories import BulkFactory, LocalFactory, RemoteFactory, TaskResultFactory -from .mock_sftp import MockSFTP - -@mock_s3 -class IngestAdminTest(TestCase): - @classmethod - def setUpClass(cls): - cls.sftp_server = MockSFTP() - - @classmethod - def tearDownClass(cls): - cls.sftp_server.stop_server() - - def setUp(self): - """ Set instance variables. """ - self.fixture_path = join(settings.APPS_DIR, 'ingest/fixtures/') - - self.s3_image_server = ImageServerFactory( - server_base='http://images.readux.ecds.emory', - storage_service='s3', - storage_path='readux' - ) - - self.sftp_image_server = ImageServerFactory( - server_base=f'http://{self.sftp_server.host}', - storage_service='sftp', - storage_path='admin_images', - sftp_port=self.sftp_server.port, - private_key_path=self.sftp_server.key_file, - sftp_user=getuser(), - path_delineator='_' - ) - - self.user = UserFactory.create(is_superuser=True) - - self.task_result = TaskResultFactory() - self.task_watcher = IngestTaskWatcher.manager.create_watcher( - task_id='1', - task_result=self.task_result, - task_creator=self.user, - filename='test_fake.zip' - ) - - # Create fake bucket for moto's mock S3 service. - conn = boto3.resource('s3', region_name='us-east-1') - conn.create_bucket(Bucket='readux') - conn.create_bucket(Bucket='readux-ingest') - - def test_local_admin_save_s3(self): - """It should add a create a manifest and canvases and delete the Local object""" - local = LocalFactory.build( - image_server=self.s3_image_server - ) - - original_manifest_count = Manifest.objects.count() - original_canvas_count = Canvas.objects.count() - - request_factory = RequestFactory() - - with open(join(self.fixture_path, 'no_meta_file.zip'), 'rb') as f: - content = files.base.ContentFile(f.read()) - - local.bundle = files.File(content.file, 'no_meta_file.zip') - - req = request_factory.post('/admin/ingest/local/add/', data={}) - req.user = self.user - - local_model_admin = LocalAdmin(model=Local, admin_site=AdminSite()) - local_model_admin.save_model(obj=local, request=req, form=None, change=None) - - # Saving should kick off the task to create the canvases and then delete - # the `Local` ingest object when done. - try: - local.refresh_from_db() - assert False - except Local.DoesNotExist: - assert True - - # A new `Manifest` should have been created along with the canvases - # in the ingest - assert Manifest.objects.count() == original_manifest_count + 1 - assert Canvas.objects.count() == original_canvas_count + 10 - - @httpretty.httprettified(allow_net_connect=True) - def test_local_admin_save_sftp(self): - """It should add a create a manifest and canvases and delete the Local object""" - httpretty.disable() - local = LocalFactory.build( - image_server=self.sftp_image_server - ) - - makedirs(local.image_server.storage_path, exist_ok=True) - - original_manifest_count = Manifest.objects.count() - original_canvas_count = Canvas.objects.count() - - request_factory = RequestFactory() - - with open(join(self.fixture_path, 'no_meta_file.zip'), 'rb') as file: - content = files.base.ContentFile(file.read()) - - local.bundle = files.File(content.file, 'no_meta_file.zip') - - req = request_factory.post('/admin/ingest/local/add/', data={}) - req.user = self.user - - local_model_admin = LocalAdmin(model=Local, admin_site=AdminSite()) - local_model_admin.save_model(obj=local, request=req, form=None, change=None) - - # Saving should kick off the task to create the canvases and then delete - # the `Local` ingest object when done. - try: - local.refresh_from_db() - assert False - except Local.DoesNotExist: - assert True - - rmtree(local.image_server.storage_path) - - # A new `Manifest` should have been created along with the canvases - # in the ingest - assert Manifest.objects.count() == original_manifest_count + 1 - assert Canvas.objects.count() == original_canvas_count + 10 - - def test_local_ingest_with_collections(self): - """It should add chosen collections to the Local's manifests""" - local = LocalFactory.build( - image_server=self.s3_image_server - ) - - # Force evaluation to get the true current list of manifests - manifests_before = list(Manifest.objects.all()) - - # Assign collections to Local - for _ in range(3): - CollectionFactory.create() - collections = Collection.objects.all() - local.save() - local.collections.set(collections) - assert len(local.collections.all()) == 3 - - # Make a local ingest - request_factory = RequestFactory() - with open(join(self.fixture_path, 'no_meta_file.zip'), 'rb') as f: - content = files.base.ContentFile(f.read()) - local.bundle = files.File(content.file, 'no_meta_file.zip') - req = request_factory.post('/admin/ingest/local/add/', data={}) - req.user = self.user - local_model_admin = LocalAdmin(model=Local, admin_site=AdminSite()) - local_model_admin.save_model(obj=local, request=req, form=None, change=None) - - # Get the newly created manifest by comparing current list to the list before - manifests_after = list(Manifest.objects.all()) - new_manifests = [x for x in manifests_after if x not in manifests_before] - assert len(new_manifests) == 1 - assert isinstance(new_manifests[0], Manifest) - - # The new manifest shouhld get the Local's collections - assert new_manifests[0].collections.count() == 3 - - def test_local_admin_response_add(self): - """It should redirect to new manifest""" - - local = LocalFactory.create(manifest=ManifestFactory.create()) - - local_model_admin = LocalAdmin(model=Local, admin_site=AdminSite()) - response = local_model_admin.response_add(obj=local, request=None) - - assert isinstance(response, HttpResponseRedirect) - assert response.url == f'/admin/manifests/manifest/{local.manifest.id}/change/' - - def test_remote_admin_save(self): - """It should add a manifest to the Local object""" - httpretty.enable(allow_net_connect=True) - - body = None - with open(path.join(self.fixture_path, 'manifest.json'), 'r') as file: - body = file.read() - - httpretty.register_uri( - httpretty.GET, - 'https://dooley.net/manifest.json', - body=body.replace('\n', ''), - content_type="text/json" - ) - - remote = RemoteFactory.create( - remote_url='https://dooley.net/manifest.json' - ) - - assert remote.manifest is None - - remote_model_admin = RemoteAdmin(model=Remote, admin_site=AdminSite()) - remote_model_admin.save_model(obj=remote, request=None, form=None, change=None) - - remote.refresh_from_db() - assert remote.manifest is not None - - httpretty.disable() - - def test_remote_admin_response_add(self): - """It should redirect to new manifest""" - - remote = RemoteFactory.create( - manifest=ManifestFactory.create(), - remote_url='https://poop.org/manifest.json' # pylint: disable=line-too-long - ) - - remote_model_admin = RemoteAdmin(model=Remote, admin_site=AdminSite()) - response = remote_model_admin.response_add(obj=remote, request=None) - - assert isinstance(response, HttpResponseRedirect) - assert response.url == f'/admin/manifests/manifest/{remote.manifest.id}/change/' - - def test_bulk_admin_save(self): - """It should add a Local object to this Bulk object""" - bulk = BulkFactory.create() - - assert len(bulk.local_uploads.all()) == 0 - - request_factory = RequestFactory() - req = request_factory.post('/admin/ingest/bulk/add/') - req.user = self.user - - bulk_model_admin = BulkAdmin(model=Bulk, admin_site=AdminSite()) - mock_form = BulkVolumeUploadForm() - req.FILES['volume_files'] = bulk.volume_files - bulk_model_admin.save_model(obj=bulk, request=req, form=mock_form, change=None) - - bulk.refresh_from_db() - assert len(bulk.local_uploads.all()) == 1 - - def test_bulk_admin_save_multiple(self): - """It should add three Local objects to this Bulk object""" - bulk = BulkFactory.create() - - assert len(bulk.local_uploads.all()) == 0 - - # Add 3 files to POST request - data = {} - file_list = [bulk.volume_files] - filepath2 = join(settings.APPS_DIR, 'ingest/fixtures/bundle_with_underscores.zip') - with open(filepath2, 'rb') as f: - content1 = files.base.ContentFile(f.read()) - file2 = files.File(content1.file, 'bundle_with_underscores.zip') - filepath3 = join(settings.APPS_DIR, 'ingest/fixtures/no_meta_file.zip') - with open(filepath3, 'rb') as f: - content2 = files.base.ContentFile(f.read()) - file3 = files.File(content2.file, 'no_meta_file.zip') - file_list.append(file2) - file_list.append(file3) - data['volume_files'] = file_list - - request_factory = RequestFactory() - req = request_factory.post('/admin/ingest/bulk/add/', data=data) - req.user = self.user - - bulk_model_admin = BulkAdmin(model=Bulk, admin_site=AdminSite()) - mock_form = BulkVolumeUploadForm() - bulk_model_admin.save_model(obj=bulk, request=req, form=mock_form, change=None) - - bulk.refresh_from_db() - assert len(bulk.local_uploads.all()) == 3 - - def test_bulk_admin_response_add(self): - """It should delete the Bulk object and redirect to manifests list""" - - bulk = BulkFactory.create() - bulk_model_admin = BulkAdmin(model=Bulk, admin_site=AdminSite()) - response = bulk_model_admin.response_add(obj=bulk, request=None) - - with self.assertRaises(Bulk.DoesNotExist): - bulk.refresh_from_db() - assert isinstance(response, HttpResponseRedirect) - assert response.url == reverse('admin:ingest_ingesttaskwatcher_changelist') - - def test_bulk_admin_with_external_metadata(self): - """It should add the metadata to the matching Local object""" - bulk = BulkFactory.create(image_server=self.s3_image_server) - - data = {} - data['volume_files'] = [] - - # Mock upload metadata csv with matching pid for zip - filepath1 = join(settings.APPS_DIR, 'ingest/fixtures/metadata.csv') - with open(filepath1, 'rb') as open_file: - content1 = files.base.ContentFile(open_file.read()) - file1 = files.File(content1.file, 'metadata.csv') - data['volume_files'].append(file1) - - # Mock upload a zip with no metadata - filepath2 = join(settings.APPS_DIR, 'ingest/fixtures/no_meta_file.zip') - with open(filepath2, 'rb') as open_file: - content2 = files.base.ContentFile(open_file.read()) - file2 = files.File(content2.file, 'no_meta_file.zip') - data['volume_files'].append(file2) - - request_factory = RequestFactory() - req = request_factory.post('/admin/ingest/bulk/add/', data=data) - req.user = self.user - - bulk_model_admin = BulkAdmin(model=Bulk, admin_site=AdminSite()) - mock_form = BulkVolumeUploadForm() - bulk_model_admin.save_model(obj=bulk, request=req, form=mock_form, change=None) - - bulk.refresh_from_db() - - local = bulk.local_uploads.first() - assert local.metadata is not None - assert isinstance(local.metadata, dict) - assert len(local.metadata) != 0 - assert local.metadata['label'] == 'Test Bundle' - - - def test_task_watcher_admin_functions(self): - """It should get the appropriate values from the watcher's associated TaskResult""" - watcher = self.task_watcher - assert isinstance(watcher.task_result, TaskResult) - assert watcher.task_id == watcher.task_result.task_id - assert watcher.task_result.task_name == 'fake_task' - assert watcher.task_result.status == 'PENDING' - - watcher_admin = TaskWatcherAdmin(model=IngestTaskWatcher, admin_site=AdminSite()) - assert watcher_admin.task_name(watcher) == 'fake_task' - assert 'PENDING' in watcher_admin.task_status(watcher) - - def test_bulk_ingest_with_collections(self): - """It should add the collections to the matching Local object""" - bulk = BulkFactory.create() - for _ in range(3): - CollectionFactory.create() - collections = Collection.objects.all() - bulk.save() - bulk.collections.set(collections) - data = {} - data['volume_files'] = [bulk.volume_files] - - request_factory = RequestFactory() - req = request_factory.post('/admin/ingest/bulk/add/', data=data) - req.user = self.user - - bulk_model_admin = BulkAdmin(model=Bulk, admin_site=AdminSite()) - mock_form = BulkVolumeUploadForm() - bulk_model_admin.save_model(obj=bulk, request=req, form=mock_form, change=None) - bulk.refresh_from_db() - created_local = bulk.local_uploads.first() - - assert created_local.collections.count() == 3 - - - # def test_local_admin_save_update_manifest(self): - # """It should add a manifest to the Local object""" - # local = LocalFactory.create() - - # assert local.manifest is None - - # request_factory = RequestFactory() - - # with open(join(self.fixture_path, 'no_meta_file.zip'), 'rb') as f: - # content = files.base.ContentFile(f.read()) - - # local.bundle = files.File(content.file, 'no_meta_file.zip') - - # req = request_factory.post('/admin/ingest/local/add/', data={}) - - # local_model_admin = LocalAdmin(model=Local, admin_site=AdminSite()) - # local_model_admin.save_model(obj=local, request=req, form=None, change=None) - - # local.refresh_from_db() - - # assert local.manifest is not None - # assert not local.manifest.label - - # manifest = local.manifest - # new_label = Faker._get_faker().name() - # manifest.label = new_label - # manifest.save() - - # assert manifest.label == new_label - - # create_canvas_form_local_task(local.id) - # manifest.refresh_from_db() - - # assert manifest.label == new_label diff --git a/apps/ingest/tests/test_local.py b/apps/ingest/tests/test_local.py deleted file mode 100644 index af4cd3bef..000000000 --- a/apps/ingest/tests/test_local.py +++ /dev/null @@ -1,387 +0,0 @@ -""" Tests for local ingest """ -from os import path, remove, mkdir -from getpass import getuser -from shutil import rmtree -import logging -import pytest -import boto3 -import httpretty -from moto import mock_s3 -from django.test import TestCase -from django.core.files.uploadedfile import SimpleUploadedFile -from django.conf import settings -from apps.iiif.canvases.models import Canvas -from apps.iiif.manifests.tests.factories import ManifestFactory, ImageServerFactory -from .mock_sftp import MockSFTP -from ..models import Local -from ..services import create_manifest -from ..storages import IngestStorage - -pytestmark = pytest.mark.django_db(transaction=True) # pylint: disable = invalid-name -LOGGER = logging.getLogger(__name__) - -@mock_s3 -class LocalTest(TestCase): - """ Tests for ingest.models.Local """ - @classmethod - def setUpClass(cls): - cls.sftp_server = MockSFTP() - - @classmethod - def tearDownClass(cls): - cls.sftp_server.stop_server() - - def setUp(self): - """ Set instance variables. """ - LOGGER.debug('SETTING UP') - self.fixture_path = path.join(settings.APPS_DIR, 'ingest/fixtures/') - self.image_server = ImageServerFactory( - server_base='http://readux.s3.amazonaws.com', - storage_service='s3', - storage_path='readux' - ) - # Create fake bucket for moto's mock S3 service. - conn = boto3.resource('s3', region_name='us-east-1') - conn.create_bucket(Bucket=self.image_server.storage_path) - conn.create_bucket(Bucket='readux-ingest') - - def mock_local(self, bundle, with_manifest=False, metadata={}, from_bulk=False): - # Note, I tried to use the factory here, but could not get it to override the file for bundle. - local = Local( - image_server = self.image_server, - metadata = metadata - ) - local.save() - file = SimpleUploadedFile( - name=bundle, - content=open(path.join(self.fixture_path, bundle), 'rb').read() - ) - if from_bulk: - local.bundle_from_bulk.save(bundle, file) - else: - local.bundle = file - - local.save() - - if with_manifest: - local.manifest = create_manifest(local) - local.save() - - return local - - def sftp_image_server(self): - if path.isdir('images'): - rmtree('images') - mkdir('images') - return ImageServerFactory( - server_base=f'http://{self.sftp_server.host}', - storage_service='sftp', - storage_path='images', - sftp_port=self.sftp_server.port, - private_key_path=self.sftp_server.key_file, - sftp_user=getuser() - ) - - - def test_bundle_upload(self): - """ It should upload the images using a fake S3 service from moto. """ - for bundle in ['bundle.zip', 'nested_volume.zip', 'csv_meta.zip']: - self.mock_local(bundle) - assert bundle in [f.key for f in IngestStorage().bucket.objects.all()] - - def test_image_upload_to_s3(self): - local = self.mock_local('bundle.zip', with_manifest=True) - - local.volume_to_s3() - - image_files = [f.key for f in local.image_server.bucket.objects.filter(Prefix=local.manifest.pid)] - - assert f'{local.manifest.pid}/00000008.jpg' in image_files - - def test_upload_to_sftp_with_default_delineator(self): - httpretty.disable() - local = self.mock_local('bundle.zip', with_manifest=True) - local.image_server = self.sftp_image_server() - - image_files, ocr_files = local.volume_to_sftp() - - assert path.exists( - path.join( - local.image_server.storage_path, - f'{local.manifest.pid}{local.image_server.path_delineator}00000005.jpg' - ) - ) - - assert path.exists( - path.join( - local.image_server.storage_path, - f'{local.manifest.pid}{local.image_server.path_delineator}00000005.tsv' - ) - ) - - rmtree(local.image_server.storage_path) - - assert path.join(local.manifest.pid, '00000008.tsv') in ocr_files - assert path.join(local.manifest.pid, '00000008.jpg') in image_files - - def test_ocr_upload_to_s3(self): - local = self.mock_local('nested_volume.zip', with_manifest=True) - - local.volume_to_s3() - - ocr_files = [f.key for f in local.image_server.bucket.objects.filter(Prefix=local.manifest.pid)] - - assert f'{local.manifest.pid}/_*ocr*_/00000008.tsv' in ocr_files - - def test_upload_to_sftp_with_underscore_delineator(self): - httpretty.disable() - local = self.mock_local('bundle.zip', with_manifest=True) - local.image_server = self.sftp_image_server() - local.image_server.path_delineator = '_' - - image_files, ocr_files = local.volume_to_sftp() - - assert path.exists( - path.join( - local.image_server.storage_path, - f'{local.manifest.pid}{local.image_server.path_delineator}00000005.jpg' - ) - ) - - assert path.exists( - path.join( - local.image_server.storage_path, - f'{local.manifest.pid}{local.image_server.path_delineator}00000003.tsv' - ) - ) - - rmtree(local.image_server.storage_path) - - assert f'{local.manifest.pid}_00000008.jpg' in image_files - assert f'{local.manifest.pid}_00000008.tsv' in ocr_files - - def test_metadata_from_excel(self): - """ It should create a manifest with metadata supplied in an Excel file. """ - local = self.mock_local('bundle.zip', with_manifest=True) - - assert 'pid' in local.metadata.keys() - - for key in local.metadata.keys(): - assert str(local.metadata[key]) == str(getattr(local.manifest, key)) - - def test_metadata_from_csv(self): - """ It should create a manifest with metadata supplied in a CSV file. """ - local = self.mock_local('csv_meta.zip', with_manifest=True) - - assert 'pid' in local.metadata.keys() - - for key in local.metadata.keys(): - assert local.metadata[key] == getattr(local.manifest, key) - - def test_metadata_from_tsv(self): - """ It should create a manifest with metadata supplied in a CSV file. """ - local = self.mock_local('tsv.zip', with_manifest=True) - - assert 'pid' in local.metadata.keys() - - for key in local.metadata.keys(): - assert local.metadata[key] == getattr(local.manifest, key) - - def test_no_metadata_file(self): - """ It should create a Manifest even when no metadata file is supplied. """ - local = self.mock_local('no_meta_file.zip', with_manifest=True) - - assert isinstance(local.manifest.pid, str) - assert len(local.manifest.pid) == 8 - - def test_single_image(self): - """ It should work when only one image is present. """ - local = self.mock_local('single-image.zip', with_manifest=True) - - local.volume_to_s3() - - image_files = [f.key for f in local.image_server.bucket.objects.filter(Prefix=local.manifest.pid)] - - assert f'{local.manifest.pid}/0011.jpg' in image_files - - def test_removing_junk(self): - """ - Any hidden files should not be uploaded. - """ - local = self.mock_local('bundle_with_junk.zip', with_manifest=True) - - local.volume_to_s3() - - ingest_files = [f.key for f in local.image_server.bucket.objects.filter(Prefix=local.manifest.pid)] - - assert 'ocr/.junk.tsv' in local.file_list - assert 'images/.00000010.jpg' in local.file_list - assert f'{local.manifest.pid}/00000009.jpg' in ingest_files - assert f'{local.manifest.pid}/.00000010.jpg' not in ingest_files - assert f'{local.manifest.pid}/_*ocr*_/00000003.tsv' in ingest_files - assert f'{local.manifest.pid}/_*ocr*_/.junk.tsv' not in ingest_files - - def test_removing_underscores(self): - """ - Any hidden files should be removed. - """ - local = self.mock_local('bundle_with_underscores.zip', with_manifest=True) - - local.volume_to_s3() - - ingest_files = [f.key for f in local.image_server.bucket.objects.filter(Prefix=local.manifest.pid)] - - underscore_files = [f for f in local.file_list if '_' in f] - assert len(underscore_files) == 10 - assert len([f for f in local.file_list if '-' in f]) == 0 - for underscore in [f for f in local.file_list if '_' in f]: - assert underscore not in ingest_files - - def test_when_metadata_in_filename(self): - """ - Make sure it doesn't get get confused when the word "metadata" is in - every path. - """ - local = self.mock_local('metadata.zip', with_manifest=True) - - image_files, ocr_files = local.volume_to_s3() - - files_in_zip = local.file_list - ingest_files = [f.key for f in local.image_server.bucket.objects.filter(Prefix=local.manifest.pid)] - - assert 'metadata/images/' in files_in_zip - assert all('metadata' in f for f in files_in_zip) - assert any('.DS_Store' in f for f in files_in_zip) - assert any('__MACOSX' in f for f in files_in_zip) - assert any('~$metadata.xlsx' in f for f in files_in_zip) - assert any('metadata/ocr/0001.tsv' in f for f in files_in_zip) - assert local.metadata['pid'] == 't9wtf-sample' - assert local.metadata['label'] == 't9wtf-sample' - assert '~$metadata.xlsx' not in ingest_files - assert f'{local.manifest.pid}/_*ocr*_/0001.tsv' in ocr_files - assert f'{local.manifest.pid}/0001.jpg' in image_files - assert len(ingest_files) == 2 - - def test_when_underscore_in_pid(self): - local = self.mock_local('no_meta_file.zip') - local.manifest = ManifestFactory.create( - image_server = self.image_server, - pid='p_i_d' - ) - - local.volume_to_s3() - local.volume_to_s3() - - ingest_files = [f.key for f in local.image_server.bucket.objects.filter(Prefix=local.manifest.pid)] - - assert all('p-i-d' in f for f in ingest_files) - - def test_creating_canvases(self): - """ - Make sure it doesn't get get confused when the word "metadata" is in - every path. - """ - local = self.mock_local('bundle.zip', with_manifest=True) - local.create_canvases() - - pid = local.manifest.pid - - assert local.manifest.canvas_set.all().count() == 10 - assert Canvas.objects.get(pid=f'{pid}_00000001.jpg').position == 1 - assert Canvas.objects.get(pid=f'{pid}_00000002.jpg').position == 2 - assert Canvas.objects.get(pid=f'{pid}_00000003.jpg').position == 3 - assert Canvas.objects.get(pid=f'{pid}_00000004.jpg').position == 4 - assert Canvas.objects.get(pid=f'{pid}_00000005.jpg').position == 5 - assert Canvas.objects.get(pid=f'{pid}_00000006.jpg').position == 6 - assert Canvas.objects.get(pid=f'{pid}_00000007.jpg').position == 7 - assert Canvas.objects.get(pid=f'{pid}_00000008.jpg').position == 8 - assert Canvas.objects.get(pid=f'{pid}_00000009.jpg').position == 9 - assert Canvas.objects.get(pid=f'{pid}_00000010.jpg').position == 10 - - - # def test_it_downloads_zip_when_local_bundle_path_is_not_none(self): - # local = self.mock_local('metadata.zip', with_manifest=True) - # local.local_bundle_path = 'swoop' - # files_in_zip = local.file_list - # assert 'metadata/images/' in files_in_zip - # assert exists(join(gettempdir(), 'metadata.zip')) - # assert local.local_bundle_path == join(gettempdir(), 'metadata.zip') - - # def test_it_cleans_up(self): - # local = self.mock_local('single-image.zip', with_manifest=True) - # local.local_bundle_path = 'swoop' - # local.zip_ref - # assert exists(join(gettempdir(), 'single-image.zip')) - # local.create_canvases() - # manifest = Manifest.objects.get(pk=local.manifest.id) - # assert manifest.canvas_set.count() == 1 - # assert exists(join(gettempdir(), 'single-image.zip')) is False - # try: - # Local.objects.get(pk=local.id) - # except Local.DoesNotExist: - # pass - - def test_it_creates_manifest_with_metadata_property(self): - metadata = { - 'pid': '808', - 'label': 'Goodie Mob' - } - local = self.mock_local('no_meta_file.zip', metadata=metadata) - local.manifest = create_manifest(local) - assert local.manifest.pid == '808' - assert local.manifest.label == 'Goodie Mob' - - def test_create_related_links(self): - metadata = { - 'pid': '808', - 'related': 'https://github.com/ecds/readux/tree/develop;https://archive.org/download/cherokeehymnbook00boud/cherokeehymnbook00boud.pdf' - } - local = self.mock_local('no_meta_file.zip', metadata=metadata) - local.manifest = create_manifest(local) - related_links = local.manifest.related_links - # should get 2 from metadata, 1 from volume url - assert len(related_links) == 3 - # should get github link format as text/html - assert any([link["@id"] == "https://github.com/ecds/readux/tree/develop" for link in related_links]) - assert any([link["format"] == "text/html" for link in related_links]) - # should get pdf format too - assert any([link["@id"] == "https://archive.org/download/cherokeehymnbook00boud/cherokeehymnbook00boud.pdf" for link in related_links]) - assert any([link["format"] == "application/pdf" for link in related_links]) - - def test_moving_bulk_bundle_to_s3(self): - """ - It should upload Local.bundle_from_bulk to mock S3 by saving it to - Local.bundle, then it should clean up tempfiles - """ - # Make sure local with from_bulk is mocked correctly - local = self.mock_local('bundle.zip', from_bulk=True) - assert bool(local.bundle_from_bulk) is True - bulk_name = local.bundle_from_bulk.name - bulk_path = local.bundle_from_bulk.path - dir_path = bulk_path[0:bulk_path.rindex('/')] - assert bulk_name[bulk_name.rindex('/')+1:] == 'bundle.zip' - assert path.isfile(bulk_path) is True - assert path.isdir(dir_path) is True - - # Call bundle_to_s3() and test that it uploaded the file - local.bundle_to_s3() - assert bool(local.bundle) is True - assert local.bundle.name[local.bundle.name.rindex('/')+1:] == 'bundle.zip' - assert local.bundle.storage.exists(f'bulk/{local.id}/bundle.zip') # pylint: disable=no-member - - # Test tempfile cleanup - assert bool(local.bundle_from_bulk) is False - assert path.isfile(bulk_path) is False - assert path.isdir(dir_path) is False - - def test_bundle_to_s3_fails_for_deleted_tempfile(self): - """ - It should raise an exception because we deleted the tempfile before - running bundle_to_s3 - """ - local = self.mock_local('bundle.zip', from_bulk=True) - bulk_path = local.bundle_from_bulk.path - assert path.isfile(bulk_path) is True - remove(bulk_path) - assert path.isfile(bulk_path) is False - self.assertRaises(Exception, local.bundle_to_s3) diff --git a/apps/ingest/tests/test_mail.py b/apps/ingest/tests/test_mail.py deleted file mode 100644 index dd37f7c8c..000000000 --- a/apps/ingest/tests/test_mail.py +++ /dev/null @@ -1,33 +0,0 @@ -from html import escape -from traceback import format_tb -from django.core import mail -from django.test import TestCase -from apps.utils.fake_traceback import FakeException, FakeTraceback -from ..mail import send_email_on_failure, send_email_on_success -from .factories import IngestTaskWatcherFactory - -class MailTest(TestCase): - def test_send_failure_email(self): - task_watcher = IngestTaskWatcherFactory.create() - fake_tb = FakeTraceback() - fake_exc = FakeException('error').with_traceback(fake_tb) - - send_email_on_failure(task_watcher, fake_exc, fake_tb) - - # Test that one message has been sent. - self.assertEqual(len(mail.outbox), 1) - - # Verify that the subject of the first message is correct. - self.assertEqual(mail.outbox[0].subject, f'[Readux] Failed: Ingest {task_watcher.filename}') - assert escape(format_tb(fake_tb)[0]) in mail.outbox[0].body - - def test_send_success_email(self): - task_watcher = IngestTaskWatcherFactory.create() - - send_email_on_success(task_watcher) - - # Test that one message has been sent. - self.assertEqual(len(mail.outbox), 1) - - # Verify that the subject of the first message is correct. - self.assertEqual(mail.outbox[0].subject, f'[Readux] Ingest complete: {task_watcher.filename}') diff --git a/apps/ingest/tests/test_remote.py b/apps/ingest/tests/test_remote.py deleted file mode 100644 index c23f77702..000000000 --- a/apps/ingest/tests/test_remote.py +++ /dev/null @@ -1,171 +0,0 @@ -""" Tests for Remote Ingests """ -import os -import httpretty -from django.test import TestCase -from django.conf import settings -from apps.iiif.manifests.tests.factories import ManifestFactory -from .factories import RemoteFactory -from ..services import (clean_metadata, create_manifest, get_associated_meta, - get_metadata_from, normalize_header) - - -class RemoteTest(TestCase): - """ Tests for ingest.models.Remote """ - def setUp(self): - """ Set instance variables. """ - self.fixture_path = os.path.join(settings.APPS_DIR, 'ingest/fixtures/') - - @httpretty.httprettified(allow_net_connect=False) - def test_metadata_from_remote_manifest(self): - """ It should extract properties from remote IIIF manifest. """ - body = None - with open(os.path.join(self.fixture_path, 'manifest.json'), 'r') as file: - body = file.read() - - httpretty.register_uri( - httpretty.GET, - 'https://iiif.archivelab.org/iiif/09359080.4757.emory.edu/manifest.json', - body=body.replace('\n', ''), - content_type="text/json" - ) - - remote = RemoteFactory.create( - remote_url='https://iiif.archivelab.org/iiif/09359080.4757.emory.edu/manifest.json' # pylint: disable=line-too-long - ) - - remote.open_metadata() - assert remote.metadata['pid'] == '09359080.4757.emory.edu' - assert remote.metadata['label'] == 'Address by Hon. Frederick Douglass' - assert remote.metadata['summary'] == 'Respect Frederick Douglass.' - assert remote.metadata['viewingdirection'] == 'left-to-right' - assert remote.metadata['publisher'] == 'Baltimore : Press of Thomas & Evans' - assert remote.metadata['attribution'] == 'The Internet Archive' - assert isinstance(remote.metadata['metadata'], list) - assert 'logo' not in remote.metadata - - # @httpretty.httprettified(allow_net_connect=True) - def test_create_canvases_from_remote_manifest(self): - """ It should extract properties from remote IIIF manifest. """ - - manifest = ManifestFactory.create(canvases=None) - httpretty.enable(allow_net_connect=True) - body = None - with open(os.path.join(self.fixture_path, 'manifest.json'), 'r') as file: - body = file.read() - - httpretty.register_uri( - httpretty.GET, - 'https://iiif.archivelab.org/iiif/09359080.4757.emory.edu/manifest.json', - body=body.replace('\n', ''), - content_type="text/json" - ) - - remote = RemoteFactory.create( - remote_url='https://iiif.archivelab.org/iiif/09359080.4757.emory.edu/manifest.json', # pylint: disable=line-too-long - manifest=manifest - ) - - remote.create_canvases() - self.assertEqual(remote.manifest.canvas_set.count(), 6) - - httpretty.disable() - - def test_heidelberg_manifest(self): - manifest = ManifestFactory.create(canvases=None) - httpretty.enable(allow_net_connect=True) - - body = None - with open(os.path.join(self.fixture_path, 'heidelberg.json'), 'r') as file: - body = file.read() - - httpretty.register_uri( - httpretty.GET, - 'https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/manifest', - body=body.replace('\n', ''), - content_type="text/json" - ) - - remote = RemoteFactory.create( - remote_url='https://digi.ub.uni-heidelberg.de/diglit/iiif/ferrerio1655bd2/manifest', # pylint: disable=line-too-long - manifest=manifest - ) - - remote.open_metadata() - assert remote.metadata['pid'] == 'ferrerio1655bd2' - assert remote.metadata['label'] == "Nvovi Disegni Dell' Architettvre, E Piante De Palazzi Di Roma De Piv Celebri Architetti" - assert remote.metadata['summary'] == "Nvovi Disegni Dell' Architettvre, E Piante De Palazzi Di Roma De Piv Celebri Architetti; Universitätsbibliothek Heidelberg, C 6426-2-10 GROSS RES" - assert remote.metadata['viewingdirection'] == 'left-to-right' - assert remote.metadata['logo_url'] == 'https://www.ub.uni-heidelberg.de/nav4/grafik/layout/ub_logo2.gif' - assert remote.metadata['author'] == 'Ferrerio, Pietro; Rossi, Giovanni Giacomo de [Editor]' - assert isinstance(remote.metadata['metadata'], list) - - remote.create_canvases() - self.assertEqual(remote.manifest.canvas_set.count(), 61) - - httpretty.disable() - - def test_princeton_manifest(self): - manifest = ManifestFactory.create(canvases=None) - httpretty.enable(allow_net_connect=True) - - body = None - with open(os.path.join(self.fixture_path, 'princeton.json'), 'r') as file: - body = file.read() - - httpretty.register_uri( - httpretty.GET, - 'https://figgy.princeton.edu/e38d7ebb-3ae6-4eb8-a3ef-d70b5184b005/manifest', - body=body.replace('\n', ''), - content_type="text/json" - ) - - remote = RemoteFactory.create( - remote_url='https://figgy.princeton.edu/e38d7ebb-3ae6-4eb8-a3ef-d70b5184b005/manifest', # pylint: disable=line-too-long - manifest=manifest - ) - - remote.open_metadata() - assert remote.metadata['pid'] == 'e38d7ebb-3ae6-4eb8-a3ef-d70b5184b005' - assert remote.metadata['label'] == "Nvova pianta et alzata della citt" - assert remote.metadata['summary'] == "Con l'aggivunta delle nove" - assert remote.metadata['viewingdirection'] == 'left-to-right' - assert remote.metadata['logo_url'] == 'https://figgy.princeton.edu/assets/pul_logo_icon-7b5f9384dfa5ca04f4851c6ee9e44e2d6953e55f893472a3e205e1591d3b2ca6.png' - assert isinstance(remote.metadata['metadata'], list) - - remote.create_canvases() - self.assertEqual(remote.manifest.canvas_set.count(), 1) - - httpretty.disable() - - def test_nga_manifest(self): - manifest = ManifestFactory.create(canvases=None) - httpretty.enable(allow_net_connect=True) - - body = None - with open(os.path.join(self.fixture_path, 'nga.json'), 'r') as file: - body = file.read() - - httpretty.register_uri( - httpretty.GET, - 'https://www.nga.gov/content/ngaweb/api/v1/iiif/presentation/manifest.json?cultObj:id=164652', - body=body.replace('\n', ''), - content_type="text/json" - ) - - remote = RemoteFactory.create( - remote_url='https://www.nga.gov/content/ngaweb/api/v1/iiif/presentation/manifest.json?cultObj:id=164652', # pylint: disable=line-too-long - manifest=manifest - ) - - remote.open_metadata() - assert remote.metadata['pid'] == 'cultObj:id=164652' - assert remote.metadata['label'] == "Autre veue du Campo Vacine" - assert remote.metadata['summary'] == "Autre veue du Campo Vacine" - assert remote.metadata['viewingdirection'] == 'left-to-right' - assert remote.metadata['logo_url'] == 'https://www.nga.gov/etc/designs/ngaweb/images/nga-seal.png' - assert isinstance(remote.metadata['metadata'], list) - - remote.create_canvases() - self.assertEqual(remote.manifest.canvas_set.count(), 1) - - httpretty.disable() \ No newline at end of file diff --git a/apps/ingest/tests/test_s3.py b/apps/ingest/tests/test_s3.py deleted file mode 100644 index f7e3895cb..000000000 --- a/apps/ingest/tests/test_s3.py +++ /dev/null @@ -1,130 +0,0 @@ -from os import path, remove, mkdir, listdir -from shutil import rmtree -from io import StringIO -import pytest -import boto3 -import httpretty -from csv import DictReader -from moto import mock_s3 -from zipfile import ZipFile -from getpass import getuser -from django.test import TestCase -from django.core.files.uploadedfile import SimpleUploadedFile -from django.conf import settings -from apps.iiif.canvases.models import Canvas -from apps.iiif.manifests.tests.factories import ManifestFactory, ImageServerFactory -from .mock_sftp import MockSFTP -from apps.iiif.manifests.tests.factories import ManifestFactory, ImageServerFactory -from ..models import S3Ingest -from ..services import (clean_metadata, create_manifest, get_associated_meta, - get_metadata_from, normalize_header) - -pytestmark = pytest.mark.django_db(transaction=True) # pylint: disable = invalid-name - -@mock_s3 -class S3Test(TestCase): - """ Tests for ingest.models.S3Ingest """ - @classmethod - def setUpClass(cls): - cls.sftp_server = MockSFTP() - cls.fixture_path = path.join(settings.APPS_DIR, 'ingest/fixtures/') - cls.test_bundle_path = path.join(cls.fixture_path, 'test_bundle') - cls.test_pid = 'sqn75' - super(S3Test, cls).setUpClass() - - @classmethod - def tearDownClass(cls): - cls.sftp_server.stop_server() - - if path.isdir(cls.test_bundle_path): - rmtree(cls.test_bundle_path) - - if path.isdir('images'): - rmtree('images') - - - def setUp(self): - conn = boto3.resource('s3', region_name='us-east-1') - conn.create_bucket(Bucket='s3-ingest') - - - if path.isdir(self.test_bundle_path): - rmtree(self.test_bundle_path) - - mkdir(self.test_bundle_path) - - with ZipFile(path.join(self.fixture_path, 'csv_meta.zip'), 'r') as bundle_ref: - bundle_ref.extractall(self.test_bundle_path) - - s3_client = boto3.client('s3') - - for filename in listdir(path.join(self.test_bundle_path, 'images')): - with open(path.join(self.test_bundle_path, 'images', filename), 'rb') as file: - s3_client.upload_fileobj( - file, - Bucket='s3-ingest', - Key=f'{self.test_pid}/images/{filename}' - ) - - for filename in listdir(path.join(self.test_bundle_path, 'ocr')): - with open(path.join(self.test_bundle_path, 'ocr', filename), 'rb') as file: - s3_client.upload_fileobj( - file, - Bucket='s3-ingest', - Key=f'{self.test_pid}/ocr/{filename}' - ) - - if path.isdir('images'): - rmtree('images') - - mkdir('images') - - self.image_server = ImageServerFactory( - server_base=f'http://{self.sftp_server.host}', - storage_service='sftp', - storage_path='images', - sftp_port=self.sftp_server.port, - private_key_path=self.sftp_server.key_file, - sftp_user=getuser(), - path_delineator='_' - ) - - def test_ingest_from_s3(self): - httpretty.disable() - ingest = S3Ingest( - image_server=self.image_server, - s3_bucket='s3-ingest', - metadata_spreadsheet=SimpleUploadedFile( - name='metadata.csv', - content=open(path.join(self.test_bundle_path, 'metadata.csv'), 'rb').read() - ) - ) - - manifest = ManifestFactory.create( - image_server=ingest.image_server, - pid=self.test_pid, - canvases=None - ) - - ingest.create_canvases_for(self.test_pid) - - assert path.exists( - path.join( - ingest.image_server.storage_path, - f'{self.test_pid}{ingest.image_server.path_delineator}00000005.jpg' - ) - ) - - assert path.exists( - path.join( - ingest.image_server.storage_path, - f'{self.test_pid}{ingest.image_server.path_delineator}00000003.tsv' - ) - ) - - manifest.refresh_from_db() - canvas = manifest.canvas_set.all()[4] - canvas.refresh_from_db() - assert canvas.annotation_set.all().count() == 72 - assert manifest.canvas_set.all().count() == 10 - diff --git a/apps/ingest/tests/test_services.py b/apps/ingest/tests/test_services.py deleted file mode 100644 index 7d7fd9d1f..000000000 --- a/apps/ingest/tests/test_services.py +++ /dev/null @@ -1,117 +0,0 @@ -""" Tests for ingest.services """ -import os -import json -import boto3 -import httpretty -from moto import mock_s3 -from django.test import TestCase -from django.core.files.uploadedfile import SimpleUploadedFile -from django.conf import settings -from factory.django import FileField -from apps.iiif.canvases.tests.factories import CanvasFactory -from apps.iiif.manifests.models import Manifest -from apps.iiif.manifests.tests.factories import ManifestFactory, ImageServerFactory -import apps.ingest.services as services -from .factories import LocalFactory, RemoteFactory - -class ServicesTest(TestCase): - """ Tests for ingest.services """ - def setUp(self): - """ Set instance variables. """ - self.fixture_path = os.path.join(settings.APPS_DIR, 'ingest/fixtures/') - - def test_cleaning_metadata(self): - """ It should normalize keys that match a Manifest field. """ - fake_metadata = { - 'pid': 'blm', - 'invalid': 'trump', - 'summary': 'idk', - 'PUBLISHER': 'ecds', - 'Published City': 'atlanta' - } - - cleaned_metadata = services.clean_metadata(fake_metadata) - - assert 'Published City' not in cleaned_metadata.keys() - assert 'PUBLISHER' not in cleaned_metadata.keys() - assert cleaned_metadata['published_city'] == fake_metadata['Published City'] - assert cleaned_metadata['publisher'] == fake_metadata['PUBLISHER'] - - def test_extracting_image_server_with_port(self): - """ If should return a URL with a specified port. """ - canvas = { - 'images': [{ - 'resource': { - 'service': { - '@id': "https://readux.org:8000/iiif/readux:thgg7_1.jp2" - } - } - }] - } - image_server_url = services.extract_image_server(canvas) - assert image_server_url == 'https://readux.org:8000/iiif' - - def test_extracting_image_server_without_port(self): - """ If should return a URL with no port. """ - canvas = { - 'images': [{ - 'resource': { - 'service': { - '@id': "https://readux.org/iiif/readux:thgg7_1.jp2" - } - } - }] - } - image_server_url = services.extract_image_server(canvas) - assert image_server_url == 'https://readux.org/iiif' - - def test_adding_related_link_to_remote_ingest_manifest(self): - httpretty.enable(allow_net_connect=True) - - body = None - with open(os.path.join(self.fixture_path, 'manifest.json'), 'r') as file: - body = file.read() - - httpretty.register_uri( - httpretty.GET, - 'https://swoop.net/manifest.json', - body=body.replace('\n', ''), - content_type="text/json" - ) - - remote = RemoteFactory.create( - remote_url='https://swoop.net/manifest.json' # pylint: disable=line-too-long - ) - manifest = services.create_manifest(remote) - related_links = manifest.relatedlink_set.all() - assert any([link.link == remote.remote_url for link in related_links]) - - httpretty.disable() - - def test_parse_v2_manifest_with_label_as_list(self): - data = json.loads(open(os.path.join(settings.APPS_DIR, 'ingest/fixtures/manifest-label-as-array.json')).read()) - metadata = services.parse_iiif_v2_manifest(data) - self.assertEqual(metadata['label'], 'Address by American Hero Frederick Douglass') - - @mock_s3 - def test_when_pid_not_in_metadata(self): - image_server = ImageServerFactory.create() - conn = boto3.resource('s3', region_name='us-east-1') - conn.create_bucket(Bucket=image_server.storage_path) - conn.create_bucket(Bucket='readux-ingest') - for _ in range(1, 5): - ManifestFactory.create() - local = LocalFactory.create( - image_server=image_server, - bundle=FileField( - filename='no_meta_file.zip', - filepath=os.path.join(settings.APPS_DIR, 'ingest/fixtures/no_meta_file.zip') - ) - ) - local.metadata['label'] = 'Southernplayalisticadillacmuzik' - local.manifest = None - assert 'pid' not in local.metadata - assert dict(local.metadata) is not None - local.manifest = services.create_manifest(local) - assert local.manifest.label == 'Southernplayalisticadillacmuzik' - assert local.manifest.pid is not None diff --git a/apps/ingest/tmp/.gitkeep b/apps/ingest/tmp/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/apps/ingest/views.py b/apps/ingest/views.py deleted file mode 100644 index 91ea44a21..000000000 --- a/apps/ingest/views.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.shortcuts import render - -# Create your views here. diff --git a/config/settings/base.py b/config/settings/base.py index 8c2aa1366..80283f29b 100644 --- a/config/settings/base.py +++ b/config/settings/base.py @@ -5,16 +5,16 @@ import environ ROOT_DIR = environ.Path(__file__) - 3 # (readux/config/settings/base.py - 3 = /) -APPS_DIR = ROOT_DIR.path('apps') +APPS_DIR = ROOT_DIR.path("apps") env = environ.Env() -READ_DOT_ENV_FILE = env.bool('DJANGO_READ_DOT_ENV_FILE', default=False) +READ_DOT_ENV_FILE = env.bool("DJANGO_READ_DOT_ENV_FILE", default=False) if READ_DOT_ENV_FILE: # OS environment variables take precedence over variables from .env - env.read_env(str(ROOT_DIR.path('.env'))) + env.read_env(str(ROOT_DIR.path(".env"))) -DJANGO_ENV = env('DJANGO_ENV', default='develop') +DJANGO_ENV = env("DJANGO_ENV", default="develop") # GENERAL # ------------------------------------------------------------------------------ @@ -24,9 +24,9 @@ # http://en.wikipedia.org/wiki/List_of_tz_zones_by_name # though not all of them may be available with every OS. # In Windows, this must be set to your system time zone. -TIME_ZONE = 'UTC' +TIME_ZONE = "UTC" # https://docs.djangoproject.com/en/dev/ref/settings/#language-code -LANGUAGE_CODE = 'en-us' +LANGUAGE_CODE = "en-us" # https://docs.djangoproject.com/en/dev/ref/settings/#site-id SITE_ID = 1 # https://docs.djangoproject.com/en/dev/ref/settings/#use-i18n @@ -41,83 +41,81 @@ # https://docs.djangoproject.com/en/dev/ref/settings/#databases DATABASES = { - 'default': env.db('DATABASE_URL', default='postgres:///readux'), + "default": env.db("DATABASE_URL", default="postgres:///readux"), } -DATABASES['default']['ATOMIC_REQUESTS'] = True -DEFAULT_AUTO_FIELD = 'django.db.models.AutoField' +DATABASES["default"]["ATOMIC_REQUESTS"] = True +DEFAULT_AUTO_FIELD = "django.db.models.AutoField" FILE_UPLOAD_HANDLERS = ["django.core.files.uploadhandler.TemporaryFileUploadHandler"] # URLS # ------------------------------------------------------------------------------ # https://docs.djangoproject.com/en/dev/ref/settings/#root-urlconf -ROOT_URLCONF = 'config.urls' +ROOT_URLCONF = "config.urls" # https://docs.djangoproject.com/en/dev/ref/settings/#wsgi-application -WSGI_APPLICATION = 'config.wsgi.application' +WSGI_APPLICATION = "config.wsgi.application" # APPS # ------------------------------------------------------------------------------ DJANGO_APPS = [ - 'django.contrib.auth', - 'django.contrib.contenttypes', - 'django.contrib.sessions', - 'django.contrib.sites', - 'django.contrib.messages', - 'django.contrib.staticfiles', - 'django.contrib.sitemaps', - 'django.contrib.postgres', - 'django.contrib.humanize', # Handy template tags + "django.contrib.auth", + "django.contrib.contenttypes", + "django.contrib.sessions", + "django.contrib.sites", + "django.contrib.messages", + "django.contrib.staticfiles", + "django.contrib.sitemaps", + "django.contrib.postgres", + "django.contrib.humanize", # Handy template tags # has to be included before django.contrib.admin - 'modeltranslation', - 'django.contrib.admin', - 'import_export', - 'django_summernote', - + "modeltranslation", + "django.contrib.admin", + "import_export", + "django_summernote", ] THIRD_PARTY_APPS = [ - 'allauth', - 'allauth.account', - 'allauth.socialaccount', - 'allauth.socialaccount.providers.facebook', - 'allauth.socialaccount.providers.github', - 'allauth.socialaccount.providers.google', - 'allauth.socialaccount.providers.twitter', - 'corsheaders', - 'crispy_forms', - 'django_celery_results', - 'django_elasticsearch_dsl', - 'modelcluster', - 'sass_processor', - 'storages', - 'taggit', - 'wagtail.contrib.forms', - 'wagtail.contrib.redirects', - 'wagtail.embeds', - 'wagtail.sites', - 'wagtail.users', - 'wagtail.snippets', - 'wagtail.documents', - 'wagtail.images', - 'wagtail.search', - 'wagtail.admin', - 'wagtail', - 'wagtailautocomplete', - 'wagtail.contrib.modeladmin', # Don't repeat if it's there already - 'wagtailmenus', + "allauth", + "allauth.account", + "allauth.socialaccount", + "allauth.socialaccount.providers.facebook", + "allauth.socialaccount.providers.github", + "allauth.socialaccount.providers.google", + "allauth.socialaccount.providers.twitter", + "corsheaders", + "crispy_forms", + "django_celery_results", + "django_elasticsearch_dsl", + "modelcluster", + "sass_processor", + "storages", + "taggit", + "wagtail.contrib.forms", + "wagtail.contrib.redirects", + "wagtail.embeds", + "wagtail.sites", + "wagtail.users", + "wagtail.snippets", + "wagtail.documents", + "wagtail.images", + "wagtail.search", + "wagtail.admin", + "wagtail", + "wagtailautocomplete", + "wagtail.contrib.modeladmin", # Don't repeat if it's there already + "wagtailmenus", ] LOCAL_APPS = [ - 'apps.users.apps.UsersAppConfig', - 'apps.iiif.annotations.apps.AnnotationsConfig', - 'apps.iiif.canvases.apps.CanvasesConfig', - 'apps.iiif.kollections.apps.KollectionsConfig', - 'apps.iiif.manifests.apps.ManifestsConfig', - 'apps.readux.apps.ReaduxConfig', - 'apps.cms.apps.CmsConfig', - 'apps.templates', - 'apps.custom_styles.apps.CustomStylesConfig', - 'apps.ingest.apps.IngestConfig', - 'apps.export.apps.ExportConfig', - 'apps.ocr.apps.OCRConfig' + "apps.users.apps.UsersAppConfig", + "apps.iiif.annotations.apps.AnnotationsConfig", + "apps.iiif.canvases.apps.CanvasesConfig", + "apps.iiif.kollections.apps.KollectionsConfig", + "apps.iiif.manifests.apps.ManifestsConfig", + "apps.readux.apps.ReaduxConfig", + "apps.cms.apps.CmsConfig", + "apps.templates", + "apps.custom_styles.apps.CustomStylesConfig", + "apps.export.apps.ExportConfig", + "apps.ocr.apps.OCRConfig", ] # https://docs.djangoproject.com/en/dev/ref/settings/#installed-apps INSTALLED_APPS = DJANGO_APPS + THIRD_PARTY_APPS + LOCAL_APPS @@ -125,48 +123,46 @@ # MIGRATIONS # ------------------------------------------------------------------------------ # https://docs.djangoproject.com/en/dev/ref/settings/#migration-modules -MIGRATION_MODULES = { - 'sites': 'apps.contrib.sites.migrations' -} +MIGRATION_MODULES = {"sites": "apps.contrib.sites.migrations"} # AUTHENTICATION # ------------------------------------------------------------------------------ # https://docs.djangoproject.com/en/dev/ref/settings/#authentication-backends AUTHENTICATION_BACKENDS = [ - 'django.contrib.auth.backends.ModelBackend', - 'allauth.account.auth_backends.AuthenticationBackend', + "django.contrib.auth.backends.ModelBackend", + "allauth.account.auth_backends.AuthenticationBackend", ] # https://docs.djangoproject.com/en/dev/ref/settings/#auth-user-model -AUTH_USER_MODEL = 'users.User' +AUTH_USER_MODEL = "users.User" # https://docs.djangoproject.com/en/dev/ref/settings/#login-redirect-url -LOGIN_REDIRECT_URL = 'users:redirect' +LOGIN_REDIRECT_URL = "users:redirect" # https://docs.djangoproject.com/en/dev/ref/settings/#login-url -LOGIN_URL = 'account_login' +LOGIN_URL = "account_login" # PASSWORDS # ------------------------------------------------------------------------------ # https://docs.djangoproject.com/en/dev/ref/settings/#password-hashers PASSWORD_HASHERS = [ # https://docs.djangoproject.com/en/dev/topics/auth/passwords/#using-argon2-with-django - 'django.contrib.auth.hashers.Argon2PasswordHasher', - 'django.contrib.auth.hashers.PBKDF2PasswordHasher', - 'django.contrib.auth.hashers.PBKDF2SHA1PasswordHasher', - 'django.contrib.auth.hashers.BCryptSHA256PasswordHasher', - 'django.contrib.auth.hashers.BCryptPasswordHasher', + "django.contrib.auth.hashers.Argon2PasswordHasher", + "django.contrib.auth.hashers.PBKDF2PasswordHasher", + "django.contrib.auth.hashers.PBKDF2SHA1PasswordHasher", + "django.contrib.auth.hashers.BCryptSHA256PasswordHasher", + "django.contrib.auth.hashers.BCryptPasswordHasher", ] # https://docs.djangoproject.com/en/dev/ref/settings/#auth-password-validators AUTH_PASSWORD_VALIDATORS = [ { - 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', + "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator", }, { - 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', + "NAME": "django.contrib.auth.password_validation.MinimumLengthValidator", }, { - 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', + "NAME": "django.contrib.auth.password_validation.CommonPasswordValidator", }, { - 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', + "NAME": "django.contrib.auth.password_validation.NumericPasswordValidator", }, ] @@ -174,45 +170,43 @@ # ------------------------------------------------------------------------------ # https://docs.djangoproject.com/en/dev/ref/settings/#middleware MIDDLEWARE = [ - 'corsheaders.middleware.CorsMiddleware', - 'django.middleware.security.SecurityMiddleware', - 'django.contrib.sessions.middleware.SessionMiddleware', - 'django.middleware.common.CommonMiddleware', - 'django.middleware.csrf.CsrfViewMiddleware', - 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.contrib.messages.middleware.MessageMiddleware', - 'django.middleware.clickjacking.XFrameOptionsMiddleware', - 'wagtail.contrib.redirects.middleware.RedirectMiddleware', + "corsheaders.middleware.CorsMiddleware", + "django.middleware.security.SecurityMiddleware", + "django.contrib.sessions.middleware.SessionMiddleware", + "django.middleware.common.CommonMiddleware", + "django.middleware.csrf.CsrfViewMiddleware", + "django.contrib.auth.middleware.AuthenticationMiddleware", + "django.contrib.messages.middleware.MessageMiddleware", + "django.middleware.clickjacking.XFrameOptionsMiddleware", + "wagtail.contrib.redirects.middleware.RedirectMiddleware", "allauth.account.middleware.AccountMiddleware", ] # STATIC # ------------------------------------------------------------------------------ # https://docs.djangoproject.com/en/dev/ref/settings/#static-root -STATIC_ROOT = str(ROOT_DIR('staticfiles')) +STATIC_ROOT = str(ROOT_DIR("staticfiles")) # https://docs.djangoproject.com/en/dev/ref/settings/#static-url -STATIC_URL = '/static/' +STATIC_URL = "/static/" # https://docs.djangoproject.com/en/dev/ref/contrib/staticfiles/#std:setting-STATICFILES_DIRS STATICFILES_DIRS = [ - str(APPS_DIR.path('static')), + str(APPS_DIR.path("static")), ] # https://docs.djangoproject.com/en/dev/ref/contrib/staticfiles/#staticfiles-finders STATICFILES_FINDERS = [ - 'django.contrib.staticfiles.finders.FileSystemFinder', - 'django.contrib.staticfiles.finders.AppDirectoriesFinder', - 'sass_processor.finders.CssFinder', + "django.contrib.staticfiles.finders.FileSystemFinder", + "django.contrib.staticfiles.finders.AppDirectoriesFinder", + "sass_processor.finders.CssFinder", ] -COMPRESS_PRECOMPILERS = ( - ('text/x-scss', 'django_libsass.SassCompiler'), -) +COMPRESS_PRECOMPILERS = (("text/x-scss", "django_libsass.SassCompiler"),) # MEDIA # ------------------------------------------------------------------------------ # https://docs.djangoproject.com/en/dev/ref/settings/#media-root -MEDIA_ROOT = str(APPS_DIR('media')) +MEDIA_ROOT = str(APPS_DIR("media")) # https://docs.djangoproject.com/en/dev/ref/settings/#media-url -MEDIA_URL = '/media/' +MEDIA_URL = "/media/" # TEMPLATES # ------------------------------------------------------------------------------ @@ -220,64 +214,61 @@ TEMPLATES = [ { # https://docs.djangoproject.com/en/dev/ref/settings/#std:setting-TEMPLATES-BACKEND - 'BACKEND': 'django.template.backends.django.DjangoTemplates', + "BACKEND": "django.template.backends.django.DjangoTemplates", # https://docs.djangoproject.com/en/dev/ref/settings/#template-dirs - 'DIRS': [ - str(APPS_DIR.path('templates')), - str(APPS_DIR.path('ingest/templates')), + "DIRS": [ + str(APPS_DIR.path("templates")), ], - 'OPTIONS': { + "OPTIONS": { # https://docs.djangoproject.com/en/dev/ref/settings/#template-debug - 'debug': DEBUG, + "debug": DEBUG, # https://docs.djangoproject.com/en/dev/ref/settings/#template-loaders # https://docs.djangoproject.com/en/dev/ref/templates/api/#loader-types - 'loaders': [ - 'django.template.loaders.filesystem.Loader', - 'django.template.loaders.app_directories.Loader', + "loaders": [ + "django.template.loaders.filesystem.Loader", + "django.template.loaders.app_directories.Loader", ], # https://docs.djangoproject.com/en/dev/ref/settings/#template-context-processors - 'context_processors': [ - 'django.template.context_processors.debug', - 'django.template.context_processors.request', - 'django.contrib.auth.context_processors.auth', - 'django.template.context_processors.i18n', - 'django.template.context_processors.media', - 'django.template.context_processors.static', - 'django.template.context_processors.tz', - 'django.contrib.messages.context_processors.messages', - 'wagtail.contrib.settings.context_processors.settings', - 'wagtailmenus.context_processors.wagtailmenus', - 'apps.custom_styles.context_processors.add_custom_style', - 'apps.templates.context_processors.has_ga_tracking_id', - 'apps.templates.context_processors.ga_tracking_id', - 'apps.readux.context_processors.current_version', + "context_processors": [ + "django.template.context_processors.debug", + "django.template.context_processors.request", + "django.contrib.auth.context_processors.auth", + "django.template.context_processors.i18n", + "django.template.context_processors.media", + "django.template.context_processors.static", + "django.template.context_processors.tz", + "django.contrib.messages.context_processors.messages", + "wagtail.contrib.settings.context_processors.settings", + "wagtailmenus.context_processors.wagtailmenus", + "apps.custom_styles.context_processors.add_custom_style", + "apps.templates.context_processors.has_ga_tracking_id", + "apps.templates.context_processors.ga_tracking_id", + "apps.readux.context_processors.current_version", ], }, }, ] # http://django-crispy-forms.readthedocs.io/en/latest/install.html#template-packs -CRISPY_TEMPLATE_PACK = 'bootstrap4' +CRISPY_TEMPLATE_PACK = "bootstrap4" # FIXTURES # ------------------------------------------------------------------------------ # https://docs.djangoproject.com/en/dev/ref/settings/#fixture-dirs -FIXTURE_DIRS = ( - str(APPS_DIR.path('fixtures')), -) +FIXTURE_DIRS = (str(APPS_DIR.path("fixtures")),) # EMAIL # ------------------------------------------------------------------------------ # https://docs.djangoproject.com/en/dev/ref/settings/#email-backend # EMAIL_BACKEND = env('DJANGO_EMAIL_BACKEND', default='django.core.mail.backends.smtp.EmailBackend') -EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend' +EMAIL_BACKEND = "django.core.mail.backends.console.EmailBackend" # ADMIN # ------------------------------------------------------------------------------ # Django Admin URL. -ADMIN_URL = 'admin/' +ADMIN_URL = "admin/" # https://docs.djangoproject.com/en/dev/ref/settings/#admins ADMINS = [ - ("""Emory Center for Digital Scholarship""", 'jay.varner@emory.edu'), + ("""Emory Center for Digital Scholarship""", "jay.varner@emory.edu"), ] # https://docs.djangoproject.com/en/dev/ref/settings/#managers MANAGERS = ADMINS @@ -285,41 +276,35 @@ # django-allauth # ------------------------------------------------------------------------------ -ACCOUNT_ALLOW_REGISTRATION = env.bool('DJANGO_ACCOUNT_ALLOW_REGISTRATION', True) +ACCOUNT_ALLOW_REGISTRATION = env.bool("DJANGO_ACCOUNT_ALLOW_REGISTRATION", True) # https://django-allauth.readthedocs.io/en/latest/configuration.html -ACCOUNT_AUTHENTICATION_METHOD = 'username' +ACCOUNT_AUTHENTICATION_METHOD = "username" # https://django-allauth.readthedocs.io/en/latest/configuration.html ACCOUNT_EMAIL_REQUIRED = True # https://django-allauth.readthedocs.io/en/latest/configuration.html -ACCOUNT_EMAIL_VERIFICATION = 'mandatory' +ACCOUNT_EMAIL_VERIFICATION = "mandatory" # https://django-allauth.readthedocs.io/en/latest/configuration.html -ACCOUNT_ADAPTER = 'apps.users.adapters.AccountAdapter' +ACCOUNT_ADAPTER = "apps.users.adapters.AccountAdapter" # https://django-allauth.readthedocs.io/en/latest/configuration.html -SOCIALACCOUNT_ADAPTER = 'apps.users.adapters.SocialAccountAdapter' +SOCIALACCOUNT_ADAPTER = "apps.users.adapters.SocialAccountAdapter" -SOCIALACCOUNT_FORMS = { - 'signup': 'apps.users.forms.ReaduxSocialSignupForm' -} +SOCIALACCOUNT_FORMS = {"signup": "apps.users.forms.ReaduxSocialSignupForm"} SOCIALACCOUNT_AUTO_SIGNUP = False ACCOUNT_AUTHENTICATED_LOGIN_REDIRECTS = False -SOCIALACCOUNT_PROVIDERS = { - 'facebook': { - 'SCOPE': ['email', 'public_profile'] - } -} +SOCIALACCOUNT_PROVIDERS = {"facebook": {"SCOPE": ["email", "public_profile"]}} # Translations # ------------------------------------------------------------------------------ gettext = lambda s: s LANGUAGES = ( - ('en', gettext('English')), - ('de', gettext('German')), + ("en", gettext("English")), + ("de", gettext("German")), ) -DEFAULT_LANGUAGE = 'en' +DEFAULT_LANGUAGE = "en" SERIALIZATION_MODULES = { "canvas": "apps.iiif.serializers.canvas", @@ -337,18 +322,16 @@ "manifest_v3": "apps.iiif.serializers.v3.manifest", } -WAGTAIL_SITE_NAME = 'Readux' -WAGTAILADMIN_BASE_URL = 'https://localhost' +WAGTAIL_SITE_NAME = "Readux" +WAGTAILADMIN_BASE_URL = "https://localhost" # CORS HEADERS CORS_ORIGIN_ALLOW_ALL = True # Elasticsearch -ELASTICSEARCH_DSL={ - 'default': { - 'hosts': env('ELASTICSEARCH_URL', default='http://localhost:9200') - }, +ELASTICSEARCH_DSL = { + "default": {"hosts": env("ELASTICSEARCH_URL", default="http://localhost:9200")}, } -SOCIALACCOUNT_STORE_TOKENS=True +SOCIALACCOUNT_STORE_TOKENS = True diff --git a/requirements/base.txt b/requirements/base.txt index 8b409e502..49d3b6b3a 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -14,7 +14,6 @@ git+https://github.com/ecds/python-edtf@214972b # Django # ------------------------------------------------------------------------------ -# TODO: To upgrade to Django 3, we have to redo the full text search on the manifest model. # TODO: When we get to update, check this re: summernote https://github.com/summernote/django-summernote/issues/449 Django>=3.2.0,<4.0 # https://www.djangoproject.com/ django-environ==0.11.2 # https://github.com/joke2k/django-environ @@ -96,7 +95,7 @@ iiif-prezi~=0.3.0 # https://github.com/iiif-prezi/iiif-prezi httpretty~=1.1.4 # https://pypi.org/project/httpretty/ #mock~=4.0.2 cssutils~=2.9.0 # https://pypi.org/project/cssutils/ -moto==4.2.13 # https://github.com/spulec/moto +moto==5.0.12 # https://github.com/spulec/moto factory-boy~=3.2.1 # https://github.com/FactoryBoy/factory_boy faker~=20.1.0