Merge branch '2.2.1' into release

.coveragerc

@@ -11,7 +11,7 @@ omit =
 exclude_lines =
     # Ignore imports
     from
-    import
+    import\s
     logger
     LOGGER
     pragma: no cover

.gitignore

@@ -67,11 +67,10 @@ pip-delete-this-directory.txt
 # Unit test / coverage reports
 htmlcov/
 .tox/
-.coverage
 .cache
 nosetests.xml
 coverage.xml
-.coveragerc
+.coverage*
 # Packages
 *.egg
 *.egg-info

CHANGELOG.rst

@@ -2,6 +2,10 @@
 
 CHANGELOG
 =========
+Release 2.2.1
+---------------------
+* Improvements to bulk ingest
+* Export tasks converted to Celery.
 
 Release 2.2.0
 ---------------------

apps/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "2.2.0"
+__version__ = "2.2.1"
 __version_info__ = tuple(
     [
         int(num) if num.isdigit() else num

apps/export/apps.py

@@ -0,0 +1,6 @@
+"""Django app configuration for Export"""
+from django.apps import AppConfig
+
+class ExportConfig(AppConfig):
+    """Configuration for Export Django app"""
+    name = 'apps.export'

apps/iiif/manifests/export.py → apps/export/export.py

@@ -673,7 +673,7 @@ def update_gitrepo(self):
 
         if os.environ['DJANGO_ENV'] != 'test':
             # run the script to import IIIF as jekyll site content
-            self.import_iiif_jekyll(self.manifest, self.jekyll_site_dir)
+            self.import_iiif_jekyll(self.manifest, self.jekyll_site_dir) # pragma: no cover
 
             # add any files that could be updated to the git index
             repo.index.add([ # pragma: no cover

apps/iiif/manifests/tasks.py → apps/export/tasks.py

@@ -1,15 +1,17 @@
 """Module to manage background export task."""
 import logging
 import os
-from background_task import background
+from celery import Celery
+from django.conf import settings
 from apps.users.models import User
+from apps.iiif.manifests.models import Manifest
 from .export import JekyllSiteExport
-from .models import Manifest
-
 
 LOGGER = logging.getLogger(__name__)
 
-@background(schedule=1)
+app = Celery('apps.readux', result_extended=True)
+
+@app.task(name='github_export', autoretry_for=(Exception,), retry_backoff=True, max_retries=20)
 def github_export_task(
         manifest_pid, version, github_repo=None,
         user_id=None, owner_ids=None, deep_zoom=False):
@@ -45,7 +47,7 @@ def github_export_task(
     jekyll_exporter.github_export(user.email)
     LOGGER.info('Background github export finished.')
 
-@background(schedule=1)
+@app.task(name='download_export', autoretry_for=(Exception,), retry_backoff=True, max_retries=20)
 def download_export_task(
         manifest_pid, version, github_repo=None,
         user_id=None, owner_ids=None, deep_zoom=False):
@@ -79,10 +81,10 @@ def download_export_task(
     )
 
     zipfile_name = jekyll_exporter.download_export(user.email, manifest)
-    delete_download_task(zipfile_name)
+    delete_download_task.apply_async((zipfile_name), countdown=86400)
     LOGGER.info('Background download export finished.')
 
-@background(schedule=86400)
+@app.task(name='delete_download', autoretry_for=(Exception,), retry_backoff=True, max_retries=20)
 def delete_download_task(download_path):
     """Background delete download task.
 

apps/readux/tests/test_export.py → apps/export/tests/test_export.py

@@ -14,8 +14,8 @@
 from apps.iiif.manifests.models import Manifest
 from apps.iiif.manifests.views import ManifestExport, JekyllExport
 from apps.iiif.canvases.models import Canvas
-from apps.iiif.manifests.export import IiifManifestExport, JekyllSiteExport, GithubExportException
-from apps.iiif.manifests.github import GithubApi
+from apps.export.export import IiifManifestExport, JekyllSiteExport, GithubExportException, ExportException
+from apps.export.github import GithubApi
 from apps.users.tests.factories import UserFactory, SocialAccountFactory, SocialAppFactory, SocialTokenFactory
 from iiif_prezi.loader import ManifestReader
 
@@ -121,6 +121,15 @@ def test_jekyll_site_export(self):
         # verify user annotation count is correct
         assert len(os.listdir(os.path.join(jekyll_path, '_annotations'))) == 1
 
+    def test_jekyll_export_error(self):
+        export = JekyllSiteExport(self.volume, 'v2', owners=[self.user.id], deep_zoom='exclude')
+        export.jekyll_site_dir = 'nope'
+        try:
+            export.import_iiif_jekyll(self.volume, 'non_existing_directory')
+            assert False
+        except ExportException:
+            assert True
+
     def test_get_zip_file(self):
         # Make an empty file
         dummy_file = os.path.join(tempfile.tempdir, 'file.txt')
@@ -137,8 +146,8 @@ def test_manifest_export(self):
         response = self.manifest_export_view(request, pid=self.volume.pid, version='v2')
         assert isinstance(response.getvalue(), bytes)
 
-    def test_setting_jekyll_site_dir(self):
-        self.jse
+    # def test_setting_jekyll_site_dir(self):
+    #     self.jse
 
     # Things I want to test:
     # * Unzip the IIIF zip file
@@ -174,10 +183,17 @@ def test_jekyll_export_include_download(self):
         )
         assert isinstance(response.getvalue(), bytes)
 
+    @httpretty.httprettified(allow_net_connect=False)
     def test_jekyll_export_to_github(self):
-        '''
-        Docstring
-        '''
+        httpretty.register_uri(
+            httpretty.GET,
+            'https://api.github.com/users/{u}/repos?per_page=3'.format(u=self.jse.github_username),
+            body='[{"name":"marx"}]',
+            content_type="text/json"
+        )
+        httpretty.register_uri(
+            httpretty.POST, 'https://api.github.com/user/repos'
+        )
         kwargs = {'pid': self.volume.pid, 'version': 'v2'}
         url = reverse('JekyllExport', kwargs=kwargs)
         kwargs['deep_zoom'] = 'exclude'

apps/readux/tests/test_github.py → apps/export/tests/test_github.py

@@ -1,7 +1,7 @@
 from django.test import TestCase
-from apps.iiif.manifests.github import GithubApi, GithubApiException, GithubAccountNotFound
-from apps.users.tests.factories import UserFactory, SocialAccountFactory, SocialAppFactory, SocialTokenFactory
 import httpretty
+from apps.users.tests.factories import UserFactory, SocialAccountFactory, SocialAppFactory, SocialTokenFactory
+from apps.export.github import GithubApi, GithubApiException, GithubAccountNotFound
 
 class TestGithubApi(TestCase):
     def setUp(self):

apps/iiif/canvases/migrations/0014_auto_20211026_1736.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.24 on 2021-10-26 17:36
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('canvases', '0013_auto_20211018_1913'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='canvas',
+            name='pid',
+            field=models.CharField(default='2qkw6szd', help_text="Unique ID. Do not use _'s or spaces in the pid.", max_length=255),
+        ),
+    ]

apps/iiif/canvases/migrations/0015_auto_20211028_1527.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.24 on 2021-10-28 15:27
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('canvases', '0014_auto_20211026_1736'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='canvas',
+            name='pid',
+            field=models.CharField(default='2qpxp706', help_text="Unique ID. Do not use _'s or spaces in the pid.", max_length=255),
+        ),
+    ]

apps/iiif/canvases/tasks.py

@@ -1,6 +1,5 @@
 """ Common tasks for canvases. """
 from celery import Celery
-from background_task import background
 from django.apps import apps
 from ..annotations.models import Annotation
 from .models import Canvas
@@ -11,9 +10,7 @@
 # create a background task have to be serializable, we can't just pass in the model object.
 # Canvas = apps.get_model('canvases.canvas')
 
-app = Celery('apps.ingest')
-app.config_from_object('django.conf:settings')
-app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)
+app = Celery('apps.readux')
 
 @app.task(name='adding_ocr_to_canvas', autoretry_for=(Canvas.DoesNotExist,), retry_backoff=5)
 def add_ocr_task(canvas_id, *args, **kwargs):

apps/iiif/kollections/migrations/0010_auto_20211026_1736.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.24 on 2021-10-26 17:36
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('kollections', '0009_auto_20211018_1913'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='collection',
+            name='pid',
+            field=models.CharField(default='2qkw6szd', help_text="Unique ID. Do not use _'s or spaces in the pid.", max_length=255),
+        ),
+    ]

apps/iiif/kollections/migrations/0011_auto_20211028_1527.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.24 on 2021-10-28 15:27
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('kollections', '0010_auto_20211026_1736'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='collection',
+            name='pid',
+            field=models.CharField(default='2qpxp706', help_text="Unique ID. Do not use _'s or spaces in the pid.", max_length=255),
+        ),
+    ]

apps/iiif/manifests/admin.py

@@ -1,11 +1,14 @@
 """Django admin module for maninfests"""
 from django.contrib import admin
+from django.http import HttpResponseRedirect
+from django.urls.conf import path
 from import_export import resources, fields
 from import_export.admin import ImportExportModelAdmin
 from import_export.widgets import ManyToManyWidget, ForeignKeyWidget
 from django_summernote.admin import SummernoteModelAdmin
 from .models import Manifest, Note, ImageServer
 from .forms import ManifestAdminForm
+from .views import AddToCollectionsView
 from ..kollections.models import Collection
 
 class ManifestResource(resources.ModelResource):
@@ -38,6 +41,26 @@ class ManifestAdmin(ImportExportModelAdmin, SummernoteModelAdmin, admin.ModelAdm
     search_fields = ('id', 'label', 'author', 'published_date')
     summernote_fields = ('summary',)
     form = ManifestAdminForm
+    actions = ['add_to_collections_action']
+
+    def add_to_collections_action(self, request, queryset):
+        """Action choose manifests to add to collections"""
+        selected = queryset.values_list('pk', flat=True)
+        selected_ids = ','.join(str(pk) for pk in selected)
+        return HttpResponseRedirect(f'add_to_collections/?ids={selected_ids}')
+    add_to_collections_action.short_description = 'Add selected manifests to collection(s)'
+
+    def get_urls(self):
+        urls = super().get_urls()
+        my_urls = [
+            path(
+                'add_to_collections/',
+                self.admin_site.admin_view(AddToCollectionsView.as_view()),
+                {'model_admin': self, },
+                name="AddManifestsToCollections",
+            )
+        ]
+        return my_urls + urls
 
 class NoteAdmin(admin.ModelAdmin):
     """Django admin configuration for a note."""

apps/iiif/manifests/forms.py

@@ -1,6 +1,9 @@
 """Django Forms for export."""
 import logging
 from django import forms
+from django.contrib.admin import site as admin_site, widgets
+
+from apps.iiif.kollections.models import Collection
 from .models import Manifest
 from ..canvases.models import Canvas
 
@@ -84,3 +87,17 @@ def __init__(self, *args, **kwargs):
             self.fields['start_canvas'].queryset = kwargs['instance'].canvas_set.all()
         else:
             self.fields['start_canvas'].queryset = Canvas.objects.none()
+
+class ManifestsCollectionsForm(forms.ModelForm):
+    """Form to add manifests to collections."""
+    class Meta:
+        model = Manifest
+        fields=('collections',)
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.fields['collections'].widget = widgets.RelatedFieldWidgetWrapper(
+               self.fields['collections'].widget,
+               self.instance._meta.get_field('collections').remote_field,           
+               admin_site,
+        )

apps/iiif/manifests/migrations/0031_auto_20211026_1736.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.24 on 2021-10-26 17:36
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('manifests', '0030_auto_20211018_1913'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='manifest',
+            name='pid',
+            field=models.CharField(default='2qkw6szd', help_text="Unique ID. Do not use _'s or spaces in the pid.", max_length=255),
+        ),
+    ]

apps/iiif/manifests/migrations/0032_auto_20211028_1527.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.24 on 2021-10-28 15:27
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('manifests', '0031_auto_20211026_1736'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='manifest',
+            name='pid',
+            field=models.CharField(default='2qpxp706', help_text="Unique ID. Do not use _'s or spaces in the pid.", max_length=255),
+        ),
+    ]