From c0c651d6ebb3893cf2be04626c2204843979a625 Mon Sep 17 00:00:00 2001 From: Jay Varner Date: Mon, 29 Nov 2021 13:06:13 -0500 Subject: [PATCH 1/7] Fix celery configs --- apps/export/celery.py | 16 ++++++++++++++++ apps/export/tasks.py | 4 +++- apps/iiif/canvases/celery.py | 16 ++++++++++++++++ apps/iiif/canvases/tasks.py | 4 +++- apps/ingest/celery.py | 16 ++++++++++++++++ apps/ingest/tasks.py | 4 +++- 6 files changed, 57 insertions(+), 3 deletions(-) create mode 100644 apps/export/celery.py create mode 100644 apps/iiif/canvases/celery.py create mode 100644 apps/ingest/celery.py diff --git a/apps/export/celery.py b/apps/export/celery.py new file mode 100644 index 000000000..7dfbe4c55 --- /dev/null +++ b/apps/export/celery.py @@ -0,0 +1,16 @@ +""" +Celery config for ingest tasks. +""" +import os +from celery import Celery +from django.conf import settings +# import config.settings.local as settings + +# set the default Django settings module for the 'celery' program. +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'config.settings.local') +app = Celery('apps.export', result_extended=True) + +# Using a string here means the worker will not have to +# pickle the object when using Windows. +app.config_from_object('django.conf:settings') +app.autodiscover_tasks(lambda: settings.INSTALLED_APPS) diff --git a/apps/export/tasks.py b/apps/export/tasks.py index 77df29244..4ef1cec97 100644 --- a/apps/export/tasks.py +++ b/apps/export/tasks.py @@ -9,7 +9,9 @@ LOGGER = logging.getLogger(__name__) -app = Celery('apps.readux', result_extended=True) +app = Celery('apps.export', result_extended=True) +app.config_from_object('django.conf:settings') +app.autodiscover_tasks(lambda: settings.INSTALLED_APPS) @app.task(name='github_export', autoretry_for=(Exception,), retry_backoff=True, max_retries=20) def github_export_task( diff --git a/apps/iiif/canvases/celery.py b/apps/iiif/canvases/celery.py new file mode 100644 index 000000000..a4b5ff29e --- /dev/null +++ b/apps/iiif/canvases/celery.py @@ -0,0 +1,16 @@ +""" +Celery config for ingest tasks. +""" +import os +from celery import Celery +from django.conf import settings +# import config.settings.local as settings + +# set the default Django settings module for the 'celery' program. +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'config.settings.local') +app = Celery('apps.iiif.canvases', result_extended=True) + +# Using a string here means the worker will not have to +# pickle the object when using Windows. +app.config_from_object('django.conf:settings') +app.autodiscover_tasks(lambda: settings.INSTALLED_APPS) diff --git a/apps/iiif/canvases/tasks.py b/apps/iiif/canvases/tasks.py index dcc8e1fe6..35ad6f59b 100644 --- a/apps/iiif/canvases/tasks.py +++ b/apps/iiif/canvases/tasks.py @@ -10,7 +10,9 @@ # create a background task have to be serializable, we can't just pass in the model object. # Canvas = apps.get_model('canvases.canvas') -app = Celery('apps.readux') +app = Celery('apps.iiif.canvases') +app.config_from_object('django.conf:settings') +app.autodiscover_tasks(lambda: settings.INSTALLED_APPS) @app.task(name='adding_ocr_to_canvas', autoretry_for=(Canvas.DoesNotExist,), retry_backoff=5) def add_ocr_task(canvas_id, *args, **kwargs): diff --git a/apps/ingest/celery.py b/apps/ingest/celery.py new file mode 100644 index 000000000..0b506b526 --- /dev/null +++ b/apps/ingest/celery.py @@ -0,0 +1,16 @@ +""" +Celery config for ingest tasks. +""" +import os +from celery import Celery +from django.conf import settings +# import config.settings.local as settings + +# set the default Django settings module for the 'celery' program. +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'config.settings.local') +app = Celery('apps.ingest', result_extended=True) + +# Using a string here means the worker will not have to +# pickle the object when using Windows. +app.config_from_object('django.conf:settings') +app.autodiscover_tasks(lambda: settings.INSTALLED_APPS) diff --git a/apps/ingest/tasks.py b/apps/ingest/tasks.py index 7bf3e6cc6..adfb66b88 100644 --- a/apps/ingest/tasks.py +++ b/apps/ingest/tasks.py @@ -20,7 +20,9 @@ LOGGER = logging.getLogger(__name__) -app = Celery('apps.readux', result_extended=True) +app = Celery('apps.ingest', result_extended=True) +app.config_from_object('django.conf:settings') +app.autodiscover_tasks(lambda: settings.INSTALLED_APPS) @app.task(name='creating_canvases_from_local', autoretry_for=(Exception,), retry_backoff=True, max_retries=20) def create_canvas_form_local_task(ingest_id): From e6c92d53837ea8c313c6a8bc1f7a2291c364e348 Mon Sep 17 00:00:00 2001 From: Jay Varner Date: Wed, 1 Dec 2021 10:09:18 -0500 Subject: [PATCH 2/7] Move all export code to export app --- .coveragerc | 2 + apps/export/export.py | 2 +- apps/export/forms.py | 68 +++++++++++ apps/export/tests/test_export.py | 39 ++++++- apps/export/tests/test_github.py | 25 +++- apps/export/tests/test_views.py | 24 ++++ apps/export/urls.py | 9 ++ apps/export/views.py | 149 ++++++++++++++++++++++++ apps/iiif/manifests/forms.py | 55 +-------- apps/iiif/manifests/tests/test_views.py | 14 +-- apps/iiif/manifests/tests/tests.py | 13 --- apps/iiif/manifests/urls.py | 3 - apps/iiif/manifests/views.py | 143 +---------------------- apps/ingest/admin.py | 6 +- apps/readux/views.py | 2 +- apps/templates/jekyll_export.html | 8 +- config/urls.py | 1 + 17 files changed, 323 insertions(+), 240 deletions(-) create mode 100644 apps/export/forms.py create mode 100644 apps/export/tests/test_views.py create mode 100644 apps/export/urls.py create mode 100644 apps/export/views.py diff --git a/.coveragerc b/.coveragerc index 48f76908b..cf193ed70 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,9 +1,11 @@ [report] omit = *tests.py + *celery.py *tests/* apps/cms/* apps/contrib/* + apps/utils/fake_traceback.py *migrations* # Exclude tasks *tasks.py diff --git a/apps/export/export.py b/apps/export/export.py index 4145f017d..5a98f2548 100644 --- a/apps/export/export.py +++ b/apps/export/export.py @@ -13,7 +13,7 @@ # pylint: disable = unused-import, ungrouped-imports try: from yaml import CLoader as Loader, CDumper as Dumper -except ImportError: +except ImportError: # pragma: no cover from yaml import Loader, Dumper # pylint: enable = unused-import, ungrouped-imports # TODO: Can we be more efficient in how we import git? diff --git a/apps/export/forms.py b/apps/export/forms.py new file mode 100644 index 000000000..60d57b5f9 --- /dev/null +++ b/apps/export/forms.py @@ -0,0 +1,68 @@ +"""Django Forms for export.""" +import logging +from django import forms +from django.contrib.admin import site as admin_site, widgets + +from apps.iiif.kollections.models import Collection +from apps.iiif.manifests.models import Manifest +from apps.iiif.canvases.models import Canvas + +LOGGER = logging.getLogger(__name__) + +# add is_checkbox method to all form fields, to enable template logic. +# thanks to: +# http://stackoverflow.com/questions/3927018/django-how-to-check-if-field-widget-is-checkbox-in-the-template +setattr( + forms.Field, + 'is_checkbox', + lambda self: isinstance(self.widget, forms.CheckboxInput) +) + +class JekyllExportForm(forms.Form): + """Form to provide export options.""" + #: export mode + mode = forms.ChoiceField( + label='Export mode', + choices=[ + ('download', 'Download Jekyll site export'), + ('github', 'Publish Jekyll site on GitHub'), + ], + initial='none', + widget=forms.RadioSelect(attrs={'class': 'uk-radio'}), + help_text='Choose how to export your annotated volume.' + ) + #: help text for export mode choices + mode_help = [ + 'Download a zip file with all Jekyll site contents', + '''Create or update a GitHub repository with the generated Jekyll + site content and publish it using Github Pages''' + ] + + #: github repository name to be created + github_repo = forms.SlugField( + label='GitHub repository name', required=False, + widget=forms.TextInput(attrs={'class': 'rdx-input uk-input'}), + help_text='Name of the repository to be created or updated, which will also ' + + 'determine the GitHub pages URL.') + + #: options that are relevant to jekyll export but not to TEI + jekyll_options = [ + # 'page_one', + # 'deep_zoom', + # 'image_hosting' + ] + + # flag to allow suppressing annotation choice display when + # user does not belong to any annotation groups + hide_annotation_choice = False + + def __init__(self, user, *args, **kwargs): + self.user = user + + # initialize normally + super(JekyllExportForm, self).__init__(*args, **kwargs) + # If the person has not authorized GitHub access, remove the GitHub + # options and select download by default. + if 'github' not in user.socialaccount_list: + self.fields['mode'].choices = self.fields['mode'].choices[:1] + self.fields['mode'].widget.attrs = {'class': 'uk-radio', 'checked': True} diff --git a/apps/export/tests/test_export.py b/apps/export/tests/test_export.py index 1dbaf6064..ac689a4d1 100644 --- a/apps/export/tests/test_export.py +++ b/apps/export/tests/test_export.py @@ -6,22 +6,20 @@ import tempfile import zipfile import httpretty +from slugify import slugify from django.test import TestCase, Client from django.test import RequestFactory -from django.conf import settings from django.contrib.auth import get_user_model from django.urls import reverse from apps.iiif.manifests.models import Manifest -from apps.iiif.manifests.views import ManifestExport, JekyllExport from apps.iiif.canvases.models import Canvas from apps.export.export import IiifManifestExport, JekyllSiteExport, GithubExportException, ExportException from apps.export.github import GithubApi -from apps.users.tests.factories import UserFactory, SocialAccountFactory, SocialAppFactory, SocialTokenFactory -from iiif_prezi.loader import ManifestReader +from apps.users.tests.factories import SocialAccountFactory, SocialAppFactory, SocialTokenFactory +from apps.export.views import JekyllExport, ManifestExport User = get_user_model() - class ManifestExportTests(TestCase): fixtures = ['users.json', 'kollections.json', 'manifests.json', 'canvases.json', 'annotations.json', 'userannotation.json'] @@ -184,7 +182,34 @@ def test_jekyll_export_include_download(self): assert isinstance(response.getvalue(), bytes) @httpretty.httprettified(allow_net_connect=False) - def test_jekyll_export_to_github(self): + def test_jekyll_export_to_github_repo_name_has_spaces(self): + httpretty.register_uri( + httpretty.GET, + 'https://api.github.com/users/{u}/repos?per_page=3'.format(u=self.jse.github_username), + body='[{"name":"marx"}]', + content_type="text/json" + ) + httpretty.register_uri( + httpretty.POST, 'https://api.github.com/user/repos' + ) + kwargs = {'pid': self.volume.pid, 'version': 'v2'} + url = reverse('JekyllExport', kwargs=kwargs) + kwargs['deep_zoom'] = 'exclude' + kwargs['mode'] = 'github' + kwargs['github_repo'] = 'has spaces' + request = self.factory.post(url, data=kwargs) + request.user = self.user + response = self.jekyll_export_view( + request, + pid=self.volume.pid, + version='v2', + content_type="application/x-www-form-urlencoded" + ) + assert response.status_code == 200 + assert 'https://github.com/zaphod/has-spaces' in response.content.decode('utf-8') + + @httpretty.httprettified(allow_net_connect=False) + def test_jekyll_export_to_github_repo_name_not_given(self): httpretty.register_uri( httpretty.GET, 'https://api.github.com/users/{u}/repos?per_page=3'.format(u=self.jse.github_username), @@ -207,6 +232,8 @@ def test_jekyll_export_to_github(self): content_type="application/x-www-form-urlencoded" ) assert response.status_code == 200 + assert f'https://github.com/zaphod/{slugify(self.volume.label, lowercase=False, max_length=50)}' in response.content.decode('utf-8') + assert f'https://zaphod.github.io/{slugify(self.volume.label, lowercase=False, max_length=50)}' in response.content.decode('utf-8') def test_use_github(self): assert isinstance(self.jse.github, GithubApi) diff --git a/apps/export/tests/test_github.py b/apps/export/tests/test_github.py index 162ab2633..5e526cc40 100644 --- a/apps/export/tests/test_github.py +++ b/apps/export/tests/test_github.py @@ -55,6 +55,16 @@ def test_get_oauth_scopes(self): scopes = self.gh.oauth_scopes(test=True) assert scopes == ['repo', 'user'] + @httpretty.activate + def test_get_oauth_scopes_is_none(self): + httpretty.register_uri( + httpretty.GET, + 'https://api.github.com/user', + status=404 + ) + scopes = self.gh.oauth_scopes(test=True) + assert scopes is None + @httpretty.activate def test_create_repo(self): httpretty.register_uri(httpretty.POST, 'https://api.github.com/user/repos', body='hello', status=201) @@ -80,6 +90,19 @@ def test_list_repos(self): repos = self.gh.list_repos('karl') assert len(repos) == 6 + @httpretty.activate + def test_list_user_repos_is_none(self): + httpretty.register_uri( + httpretty.GET, + 'https://api.github.com/user/repos', + status=404 + ) + + repos = self.gh.list_user_repos() + for _ in range(0, 20): + print(f'****{repos}') + assert repos is None + @httpretty.activate def test_list_user_repos(self): resp_body = '[{},{},{}]' @@ -108,7 +131,7 @@ def test_create_pr(self): body='{}', status=201 ) - pr = self.gh.create_pull_request('joseph', 'pitty', 'barco', 'larentowicz') + pr = self.gh.create_pull_request('joseph', 'pitty', 'barco', 'larentowicz', 'araujo') assert pr == {} @httpretty.activate diff --git a/apps/export/tests/test_views.py b/apps/export/tests/test_views.py new file mode 100644 index 000000000..ab7915115 --- /dev/null +++ b/apps/export/tests/test_views.py @@ -0,0 +1,24 @@ +''' +''' +from django.test import TestCase +from django.contrib.auth import get_user_model +from allauth.socialaccount.models import SocialAccount +from apps.export.forms import JekyllExportForm + +class ManifestTests(TestCase): + fixtures = ['users.json'] + + def setUp(self): + self.user = get_user_model().objects.get(pk=111) + + def test_form_mode_choices_no_github(self): + form = JekyllExportForm(user=self.user) + assert len(form.fields['mode'].choices) == 1 + assert form.fields['mode'].choices[0] != 'github' + + def test_form_mode_choices_with_github(self): + sa = SocialAccount(provider='github', user=self.user) + sa.save() + form = JekyllExportForm(user=self.user) + assert len(form.fields['mode'].choices) == 2 + assert form.fields['mode'].choices[1][0] == 'github' diff --git a/apps/export/urls.py b/apps/export/urls.py new file mode 100644 index 000000000..7e96ee517 --- /dev/null +++ b/apps/export/urls.py @@ -0,0 +1,9 @@ +"""URL patterns for manifests.""" +from django.urls import path +from . import views + +urlpatterns = [ + path('iiif///plain', views.PlainExport.as_view(), name="PlainExport"), + path('iiif///export', views.ManifestExport.as_view(), name="ManifestExport"), + path('iiif///jekyllexport', views.JekyllExport.as_view(), name="JekyllExport"), +] diff --git a/apps/export/views.py b/apps/export/views.py new file mode 100644 index 000000000..2c1b868f9 --- /dev/null +++ b/apps/export/views.py @@ -0,0 +1,149 @@ +"""Django views for manifests""" +import logging +from os import environ +from slugify import slugify +from django.http import HttpResponse +from django.shortcuts import render +from django.views import View +from django.views.generic.base import TemplateView +from apps.export.export import IiifManifestExport +from apps.export.github import GithubApi +from apps.export.tasks import github_export_task +from apps.export.tasks import download_export_task +from apps.iiif.canvases.models import Canvas +from apps.iiif.manifests.models import Manifest +from .forms import JekyllExportForm + +LOGGER = logging.getLogger(__name__) + +class ManifestExport(View): + """View for maifest export.""" + + def get_queryset(self): + """Get requested manifest + + :return: Manifest object to be exported. + :rtype: django.db.models.QuerySet + """ + return Manifest.objects.filter(pid=self.kwargs['pid']) + + def post(self, request, *args, **kwargs): # pylint: disable = unused-argument + """[summary] + + :param request: [description] + :type request: [type] + :return: [description] + :rtype: [type] + """ + # we should probably move this out of the view, into a library + manifest = self.get_queryset()[0] + owners = [request.user.id] + + zip = IiifManifestExport.get_zip(manifest, kwargs['version'], owners=owners) + resp = HttpResponse(zip, content_type = "application/x-zip-compressed") + resp['Content-Disposition'] = 'attachment; filename=iiif_export.zip' + + return resp + +class JekyllExport(TemplateView): + """Jekyll Export view""" + template_name = "jekyll_export.html" + + form_class = JekyllExportForm + + def get_queryset(self): + """Get requested manifest. + + :return: Manifest to be exported. + :rtype: apps.iiif.manifests.models.Manifest + """ + return Manifest.objects.filter(pid=self.kwargs['pid']) + + def post(self, request, *args, **kwargs): # pylint: disable = unused-argument + """HTTP POST for manifest export.""" + # we should probably move this out of the view, into a library + manifest = self.get_queryset()[0] + + export_form = JekyllExportForm(self.request.user, data=request.POST) + export_form.is_valid() + cleaned_data = export_form.clean() + + export_mode = export_form.cleaned_data['mode'] + if 'github_repo' not in cleaned_data or not cleaned_data['github_repo']: + fallback = request.POST['github_repo'] if 'github_repo' in request.POST else manifest.label + github_repo = slugify(fallback, lowercase=False, max_length=50) + else: + github_repo = cleaned_data['github_repo'] + + owners = [request.user.id] # TODO switch to form group vs. solo control + + # TODO Actually use the git repo and export mode + if export_mode == 'download': + context = self.get_context_data() + manifest_pid = manifest.pid + if environ['DJANGO_ENV'] != 'test': # pragma: no cover + download_export_task.delay( + manifest_pid, + kwargs['version'], + github_repo=github_repo, + deep_zoom=False, + owner_ids=owners, + user_id=self.request.user.id + ) + context['email'] = request.user.email + context['mode'] = "download" + return render(request, self.template_name, context) + + #github exports + context = self.get_context_data() + manifest_pid = manifest.pid + + if environ['DJANGO_ENV'] != 'test': # pragma: no cover + github_export_task.delay( + manifest_pid, + kwargs['version'], + github_repo=github_repo, + deep_zoom=False, + owner_ids=owners, + user_id=self.request.user.id + ) + else: + github_export_task( + manifest_pid, + kwargs['version'], + github_repo=github_repo, + deep_zoom=False, + owner_ids=owners, + user_id=self.request.user.id + ) + + context['email'] = request.user.email + context['mode'] = "github" + context['repo_link'] = f'https://github.com/{GithubApi.github_username(self.request.user)}/{github_repo}' + context['site_link'] = f'https://{GithubApi.github_username(self.request.user)}.github.io/{github_repo}' + return render(request, self.template_name, context) + + def get_context_data(self, **kwargs): + context_data = super(JekyllExport, self).get_context_data(**kwargs) + return context_data + +class PlainExport(View): + """Plain export""" + def get_queryset(self): + """Get requested manifest + + :return: Manifest to be exported. + :rtype: apps.iiif.manifests.models.Manifest + """ + manifest = Manifest.objects.get(pid=self.kwargs['pid']) + return Canvas.objects.filter(manifest=manifest.id).order_by('position') + + def get(self, request, *args, **kwargs): + """HTTP GET endpoint for plain export. + + :rtype: django.http.HttpResponse + """ + annotations = [] + for canvas in self.get_queryset() : + annotations.append(canvas.result) + return HttpResponse(' '.join(annotations)) diff --git a/apps/iiif/manifests/forms.py b/apps/iiif/manifests/forms.py index f8ef13bc6..a0a548b66 100644 --- a/apps/iiif/manifests/forms.py +++ b/apps/iiif/manifests/forms.py @@ -2,8 +2,6 @@ import logging from django import forms from django.contrib.admin import site as admin_site, widgets - -from apps.iiif.kollections.models import Collection from .models import Manifest from ..canvases.models import Canvas @@ -18,57 +16,6 @@ lambda self: isinstance(self.widget, forms.CheckboxInput) ) -class JekyllExportForm(forms.Form): - """Form to provide export options.""" - #: export mode - mode = forms.ChoiceField( - label='Export mode', - choices=[ - ('download', 'Download Jekyll site export'), - ('github', 'Publish Jekyll site on GitHub'), - ], - initial='none', - widget=forms.RadioSelect(attrs={'class': 'uk-radio'}), - help_text='Choose how to export your annotated volume.' - ) - #: help text for export mode choices - mode_help = [ - 'Download a zip file with all Jekyll site contents', - '''Create or update a GitHub repository with the generated Jekyll - site content and publish it using Github Pages''' - ] - - #: github repository name to be created - github_repo = forms.SlugField( - label='GitHub repository name', required=False, - widget=forms.TextInput(attrs={'class': 'rdx-input uk-input'}), - help_text='Name of the repository to be created or updated, which will also ' + - 'determine the GitHub pages URL.') - - #: options that are relevant to jekyll export but not to TEI - jekyll_options = [ - # 'page_one', - # 'deep_zoom', - # 'image_hosting' - ] - # used in the template to flag fields so javascript can hide them - # when TEI export is selected - - # flag to allow suppressing annotation choice display when - # user does not belong to any annotation groups - hide_annotation_choice = False - - def __init__(self, user, *args, **kwargs): - self.user = user - - # initialize normally - super(JekyllExportForm, self).__init__(*args, **kwargs) - # If the person has not authorized GitHub access, remove the GitHub - # options and select download by default. - if 'github' not in user.socialaccount_list: - self.fields['mode'].choices = self.fields['mode'].choices[:1] - self.fields['mode'].widget.attrs = {'class': 'uk-radio', 'checked': True} - class ManifestAdminForm(forms.ModelForm): class Meta: model = Manifest @@ -98,6 +45,6 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.fields['collections'].widget = widgets.RelatedFieldWidgetWrapper( self.fields['collections'].widget, - self.instance._meta.get_field('collections').remote_field, + self.instance._meta.get_field('collections').remote_field, admin_site, ) diff --git a/apps/iiif/manifests/tests/test_views.py b/apps/iiif/manifests/tests/test_views.py index 1a62c2250..ae3c71f4f 100644 --- a/apps/iiif/manifests/tests/test_views.py +++ b/apps/iiif/manifests/tests/test_views.py @@ -14,7 +14,7 @@ from ..admin import ManifestAdmin from ..views import AddToCollectionsView, ManifestSitemap, ManifestRis from ..models import Manifest -from ..forms import JekyllExportForm, ManifestsCollectionsForm +from ..forms import ManifestsCollectionsForm from .factories import ManifestFactory, EmptyManifestFactory from ...canvases.models import Canvas from ...canvases.tests.factories import CanvasFactory @@ -89,18 +89,6 @@ def test_autocomplete_label(self): def test_absolute_url(self): assert Manifest.objects.all().first().get_absolute_url() == "%s/volume/%s" % (settings.HOSTNAME, Manifest.objects.all().first().pid) - def test_form_mode_choices_no_github(self): - form = JekyllExportForm(user=self.user) - assert len(form.fields['mode'].choices) == 1 - assert form.fields['mode'].choices[0] != 'github' - - def test_form_mode_choices_with_github(self): - sa = SocialAccount(provider='github', user=self.user) - sa.save() - form = JekyllExportForm(user=self.user) - assert len(form.fields['mode'].choices) == 2 - assert form.fields['mode'].choices[1][0] == 'github' - def test_manifest_search_vector_exists(self): volume = ManifestFactory.create() assert not self.volume.search_vector diff --git a/apps/iiif/manifests/tests/tests.py b/apps/iiif/manifests/tests/tests.py index 2d072cf6a..3a3dcc515 100644 --- a/apps/iiif/manifests/tests/tests.py +++ b/apps/iiif/manifests/tests/tests.py @@ -18,7 +18,6 @@ from apps.utils.noid import encode_noid from ..views import ManifestSitemap, ManifestRis from ..models import Manifest, ImageServer, RelatedLink -from ..forms import JekyllExportForm from .factories import ManifestFactory, ImageServerFactory from ...canvases.models import Canvas from ...canvases.tests.factories import CanvasFactory @@ -123,18 +122,6 @@ def test_autocomplete_label(self): def test_absolute_url(self): assert Manifest.objects.all().first().get_absolute_url() == "%s/volume/%s" % (settings.HOSTNAME, Manifest.objects.all().first().pid) - def test_form_mode_choices_no_github(self): - form = JekyllExportForm(user=self.user) - assert len(form.fields['mode'].choices) == 1 - assert form.fields['mode'].choices[0] != 'github' - - def test_form_mode_choices_with_github(self): - sa = SocialAccount(provider='github', user=self.user) - sa.save() - form = JekyllExportForm(user=self.user) - assert len(form.fields['mode'].choices) == 2 - assert form.fields['mode'].choices[1][0] == 'github' - def test_manifest_search_vector_exists(self): assert self.volume.search_vector is None self.volume.save() diff --git a/apps/iiif/manifests/urls.py b/apps/iiif/manifests/urls.py index b70133c25..fe204f706 100644 --- a/apps/iiif/manifests/urls.py +++ b/apps/iiif/manifests/urls.py @@ -3,9 +3,6 @@ from . import views urlpatterns = [ - path('iiif///plain', views.PlainExport.as_view(), name="PlainExport"), path('iiif///manifest', views.ManifestDetail.as_view(), name="ManifestRender"), - path('iiif///export', views.ManifestExport.as_view(), name="ManifestExport"), - path('iiif///jekyllexport', views.JekyllExport.as_view(), name="JekyllExport"), path('volume//citation.ris', views.ManifestRis.as_view(), name='ris' ) ] diff --git a/apps/iiif/manifests/views.py b/apps/iiif/manifests/views.py index 6cb45d94a..f4dd28d15 100644 --- a/apps/iiif/manifests/views.py +++ b/apps/iiif/manifests/views.py @@ -1,25 +1,17 @@ """Django views for manifests""" import json import logging -from os import environ from datetime import datetime from django.contrib import messages from django.http import JsonResponse -from django.http import HttpResponse -from django.shortcuts import render -from django.template.response import TemplateResponse from django.views import View from django.views.generic.base import TemplateView from django.views.generic.edit import FormView from django.core.serializers import serialize from django.contrib.sitemaps import Sitemap from django.urls import reverse -from ..canvases.models import Canvas from .models import Manifest -from apps.export.export import IiifManifestExport -from .forms import JekyllExportForm, ManifestsCollectionsForm -from apps.export.tasks import github_export_task -from apps.export.tasks import download_export_task +from .forms import ManifestsCollectionsForm LOGGER = logging.getLogger(__name__) @@ -82,139 +74,6 @@ def get_context_data(self, **kwargs): context['volume'] = Manifest.objects.filter(pid=kwargs['volume']).first() return context - -class ManifestExport(View): - """View for maifest export.""" - - def get_queryset(self): - """Get requested manifest - - :return: Manifest object to be exported. - :rtype: django.db.models.QuerySet - """ - return Manifest.objects.filter(pid=self.kwargs['pid']) - - def post(self, request, *args, **kwargs): # pylint: disable = unused-argument - """[summary] - - :param request: [description] - :type request: [type] - :return: [description] - :rtype: [type] - """ - # we should probably move this out of the view, into a library - manifest = self.get_queryset()[0] - owners = [request.user.id] - - zip = IiifManifestExport.get_zip(manifest, kwargs['version'], owners=owners) - resp = HttpResponse(zip, content_type = "application/x-zip-compressed") - resp['Content-Disposition'] = 'attachment; filename=iiif_export.zip' - - return resp - -class JekyllExport(TemplateView): - """Jekyll Export view""" - template_name = "jekyll_export.html" - - form_class = JekyllExportForm - - def get_queryset(self): - """Get requested manifest. - - :return: Manifest to be exported. - :rtype: apps.iiif.manifests.models.Manifest - """ - return Manifest.objects.filter(pid=self.kwargs['pid']) - - def post(self, request, *args, **kwargs): # pylint: disable = unused-argument - """HTTP POST for manifest export.""" - # we should probably move this out of the view, into a library - manifest = self.get_queryset()[0] - LOGGER.debug(request.POST) - LOGGER.debug(dir(self.args)) - - export_form = JekyllExportForm(self.request.user, data=request.POST) - - # FIXME this needs to return an error. - if not export_form.is_valid(): - LOGGER.debug("Export form is not valid: %s", export_form.errors) - cleaned_data = export_form.clean() - LOGGER.debug("Cleaned Data: %s", dir(cleaned_data)) - - export_mode = export_form.cleaned_data['mode'] - github_repo = export_form.cleaned_data['github_repo'] - - owners = [request.user.id] # TODO switch to form group vs. solo control - - # TODO Actually use the git repo and export mode - if export_mode == 'download': - context = self.get_context_data() - manifest_pid = manifest.pid - if environ['DJANGO_ENV'] != 'test': # pragma: no cover - download_export_task.delay( - manifest_pid, - kwargs['version'], - github_repo=github_repo, - deep_zoom=False, - owner_ids=owners, - user_id=self.request.user.id - ) - context['email'] = request.user.email - context['mode'] = "download" - return render(request, self.template_name, context) - - #github exports - context = self.get_context_data() - manifest_pid = manifest.pid - - if environ['DJANGO_ENV'] != 'test': # pragma: no cover - github_export_task.delay( - manifest_pid, - kwargs['version'], - github_repo=github_repo, - deep_zoom=False, - owner_ids=owners, - user_id=self.request.user.id - ) - else: - github_export_task( - manifest_pid, - kwargs['version'], - github_repo=github_repo, - deep_zoom=False, - owner_ids=owners, - user_id=self.request.user.id - ) - - context['email'] = request.user.email - context['mode'] = "github" - return render(request, self.template_name, context) - - def get_context_data(self, **kwargs): - context_data = super(JekyllExport, self).get_context_data(**kwargs) - return context_data - -class PlainExport(View): - """Plain export""" - def get_queryset(self): - """Get requested manifest - - :return: Manifest to be exported. - :rtype: apps.iiif.manifests.models.Manifest - """ - manifest = Manifest.objects.get(pid=self.kwargs['pid']) - return Canvas.objects.filter(manifest=manifest.id).order_by('position') - - def get(self, request, *args, **kwargs): - """HTTP GET endpoint for plain export. - - :rtype: django.http.HttpResponse - """ - annotations = [] - for canvas in self.get_queryset() : - annotations.append(canvas.result) - return HttpResponse(' '.join(annotations)) - class AddToCollectionsView(FormView): """Intermediate page to choose collections to which you are adding manifests""" diff --git a/apps/ingest/admin.py b/apps/ingest/admin.py index b99b34954..2a9cf1ef9 100644 --- a/apps/ingest/admin.py +++ b/apps/ingest/admin.py @@ -26,7 +26,7 @@ def save_model(self, request, obj, form, change): obj.save() obj.refresh_from_db() super().save_model(request, obj, form, change) - if environ["DJANGO_ENV"] != 'test': + if environ["DJANGO_ENV"] != 'test': # pragma: no cover local_task_id = tasks.create_canvas_form_local_task.delay(obj.id) local_task_result = TaskResult(task_id=local_task_id) local_task_result.save() @@ -59,7 +59,7 @@ def save_model(self, request, obj, form, change): obj.save() obj.refresh_from_db() super().save_model(request, obj, form, change) - if environ["DJANGO_ENV"] != 'test': + if environ["DJANGO_ENV"] != 'test': # pragma: no cover remote_task_id = tasks.create_remote_canvases.delay(obj.id) remote_task_result = TaskResult(task_id=remote_task_id) remote_task_result.save() @@ -119,7 +119,7 @@ def save_model(self, request, obj, form, change): new_local.refresh_from_db() # Queue task to upload to S3 - if environ["DJANGO_ENV"] != 'test': + if environ["DJANGO_ENV"] != 'test': # pragma: no cover upload_task = tasks.upload_to_s3_task.delay( local_id=new_local.id, ) diff --git a/apps/readux/views.py b/apps/readux/views.py index e624b9ce0..12651c097 100644 --- a/apps/readux/views.py +++ b/apps/readux/views.py @@ -12,12 +12,12 @@ from django.utils.datastructures import MultiValueDictKeyError import config.settings.local as settings from apps.export.export import JekyllSiteExport +from apps.export.forms import JekyllExportForm from .models import UserAnnotation from ..cms.models import Page, CollectionsPage, VolumesPage from ..iiif.kollections.models import Collection from ..iiif.canvases.models import Canvas from ..iiif.manifests.models import Manifest -from ..iiif.manifests.forms import JekyllExportForm SORT_OPTIONS = ['title', 'author', 'date published', 'date added'] ORDER_OPTIONS = ['asc', 'desc'] diff --git a/apps/templates/jekyll_export.html b/apps/templates/jekyll_export.html index 4d1912283..0d179d3c3 100644 --- a/apps/templates/jekyll_export.html +++ b/apps/templates/jekyll_export.html @@ -4,7 +4,7 @@ {% block content %}

- {{volume.label}} + {{volume.label}}

{% endblock content %} @@ -15,11 +15,13 @@

Static Site Export

{% if mode == "github" %}
GitHub export started.
-

Your digital edition is being exported to GitHub. You will receive an email +

Your digital edition is being exported to GitHub. You will receive an email at {{ email }} when the export is complete.

+

Your repo will be available at {{ repo_link }}. The + site will be available at {{ site_link}}.

{% else %}
Download export started.
-

Your digital edition is being exported. You will receive an email +

Your digital edition is being exported. You will receive an email at {{ email }} when the export is complete with a link to download the file.

{% endif %} diff --git a/config/urls.py b/config/urls.py index 862d889cd..1a3ad7962 100644 --- a/config/urls.py +++ b/config/urls.py @@ -41,6 +41,7 @@ re_path(r'^', include('apps.iiif.manifests.urls')), re_path(r'^', include('apps.iiif.annotations.urls')), re_path(r'^', include('apps.iiif.kollections.urls')), + re_path(r'^', include('apps.export.urls')), path('accounts/', include('allauth.urls')),# re_path(r'^', include('readux.collection.urls')), # re_path(r'^', include('readux.volumes.urls')), # path("", TemplateView.as_view(template_name="pages/home.html"), name="home"), From 6e4b3eb4f5a2d62df366b58868859d7afd6a8963 Mon Sep 17 00:00:00 2001 From: Jay Varner Date: Thu, 2 Dec 2021 08:49:17 -0500 Subject: [PATCH 3/7] Fix scheduling clean up of export download file --- apps/export/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/export/tasks.py b/apps/export/tasks.py index 4ef1cec97..33dbe80cc 100644 --- a/apps/export/tasks.py +++ b/apps/export/tasks.py @@ -83,7 +83,7 @@ def download_export_task( ) zipfile_name = jekyll_exporter.download_export(user.email, manifest) - delete_download_task.apply_async((zipfile_name), countdown=86400) + delete_download_task.apply_async((zipfile_name,), countdown=86400) LOGGER.info('Background download export finished.') @app.task(name='delete_download', autoretry_for=(Exception,), retry_backoff=True, max_retries=20) From b7f11732a55380213976d0eb9b6db97fa49b8aec Mon Sep 17 00:00:00 2001 From: Jay Varner Date: Thu, 2 Dec 2021 11:51:47 -0500 Subject: [PATCH 4/7] Fix for when github_repo is left blank --- apps/export/tests/test_export.py | 5 +++-- apps/export/views.py | 10 +++++++--- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/apps/export/tests/test_export.py b/apps/export/tests/test_export.py index ac689a4d1..9f05ab6b6 100644 --- a/apps/export/tests/test_export.py +++ b/apps/export/tests/test_export.py @@ -223,6 +223,7 @@ def test_jekyll_export_to_github_repo_name_not_given(self): url = reverse('JekyllExport', kwargs=kwargs) kwargs['deep_zoom'] = 'exclude' kwargs['mode'] = 'github' + kwargs['github_repo'] = '' request = self.factory.post(url, data=kwargs) request.user = self.user response = self.jekyll_export_view( @@ -232,8 +233,8 @@ def test_jekyll_export_to_github_repo_name_not_given(self): content_type="application/x-www-form-urlencoded" ) assert response.status_code == 200 - assert f'https://github.com/zaphod/{slugify(self.volume.label, lowercase=False, max_length=50)}' in response.content.decode('utf-8') - assert f'https://zaphod.github.io/{slugify(self.volume.label, lowercase=False, max_length=50)}' in response.content.decode('utf-8') + assert f'https://github.com/zaphod/{slugify(self.volume.label, lowercase=False, max_length=40)}' in response.content.decode('utf-8') + assert f'https://zaphod.github.io/{slugify(self.volume.label, lowercase=False, max_length=40)}' in response.content.decode('utf-8') def test_use_github(self): assert isinstance(self.jse.github, GithubApi) diff --git a/apps/export/views.py b/apps/export/views.py index 2c1b868f9..d70512d03 100644 --- a/apps/export/views.py +++ b/apps/export/views.py @@ -69,9 +69,13 @@ def post(self, request, *args, **kwargs): # pylint: disable = unused-argument cleaned_data = export_form.clean() export_mode = export_form.cleaned_data['mode'] - if 'github_repo' not in cleaned_data or not cleaned_data['github_repo']: - fallback = request.POST['github_repo'] if 'github_repo' in request.POST else manifest.label - github_repo = slugify(fallback, lowercase=False, max_length=50) + + if 'github_repo' not in cleaned_data: + # If the person used spaces, `github_repo` will not be in `cleaned_data` + github_repo = slugify(request.POST['github_repo'], lowercase=False, max_length=40) + elif 'github_repo' in cleaned_data and not cleaned_data['github_repo']: + # if `github_repo` was left blank, it will be an empty `str` + github_repo = slugify(manifest.label, lowercase=False, max_length=40) else: github_repo = cleaned_data['github_repo'] From ca0bf6e1cf619a5a74c7e9d7783c8437e0fd945a Mon Sep 17 00:00:00 2001 From: Jay Varner Date: Thu, 2 Dec 2021 12:20:28 -0500 Subject: [PATCH 5/7] Revert char limit for repo names --- apps/export/tests/test_export.py | 4 ++-- apps/export/views.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/export/tests/test_export.py b/apps/export/tests/test_export.py index 9f05ab6b6..a6b9f985b 100644 --- a/apps/export/tests/test_export.py +++ b/apps/export/tests/test_export.py @@ -233,8 +233,8 @@ def test_jekyll_export_to_github_repo_name_not_given(self): content_type="application/x-www-form-urlencoded" ) assert response.status_code == 200 - assert f'https://github.com/zaphod/{slugify(self.volume.label, lowercase=False, max_length=40)}' in response.content.decode('utf-8') - assert f'https://zaphod.github.io/{slugify(self.volume.label, lowercase=False, max_length=40)}' in response.content.decode('utf-8') + assert f'https://github.com/zaphod/{slugify(self.volume.label, lowercase=False, max_length=50)}' in response.content.decode('utf-8') + assert f'https://zaphod.github.io/{slugify(self.volume.label, lowercase=False, max_length=50)}' in response.content.decode('utf-8') def test_use_github(self): assert isinstance(self.jse.github, GithubApi) diff --git a/apps/export/views.py b/apps/export/views.py index d70512d03..e1bbd952f 100644 --- a/apps/export/views.py +++ b/apps/export/views.py @@ -72,10 +72,10 @@ def post(self, request, *args, **kwargs): # pylint: disable = unused-argument if 'github_repo' not in cleaned_data: # If the person used spaces, `github_repo` will not be in `cleaned_data` - github_repo = slugify(request.POST['github_repo'], lowercase=False, max_length=40) + github_repo = slugify(request.POST['github_repo'], lowercase=False, max_length=50) elif 'github_repo' in cleaned_data and not cleaned_data['github_repo']: # if `github_repo` was left blank, it will be an empty `str` - github_repo = slugify(manifest.label, lowercase=False, max_length=40) + github_repo = slugify(manifest.label, lowercase=False, max_length=50) else: github_repo = cleaned_data['github_repo'] From a2a2f8e6bb8d4f6199cf7266a0781ff4381c1774 Mon Sep 17 00:00:00 2001 From: Jay Varner Date: Thu, 2 Dec 2021 12:20:56 -0500 Subject: [PATCH 6/7] Update version of iiif-to-jekyll gem --- Gemfile | 2 +- Gemfile.lock | 44 ++++++++++++++++++++++++++++++-------------- 2 files changed, 31 insertions(+), 15 deletions(-) diff --git a/Gemfile b/Gemfile index 0698de18b..23ac2d778 100644 --- a/Gemfile +++ b/Gemfile @@ -1,3 +1,3 @@ source 'https://rubygems.org' -gem 'iiif_to_jekyll', '0.9.4', git: 'https://github.com/ecds/iiif-to-jekyll.git' +gem 'iiif_to_jekyll', '0.9.5', git: 'https://github.com/ecds/iiif-to-jekyll.git' diff --git a/Gemfile.lock b/Gemfile.lock index db8c59ba5..a2fab2207 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,46 +1,62 @@ GIT remote: https://github.com/ecds/iiif-to-jekyll.git - revision: 6f734546fe58c8d24ab07d24d29eee43a564a701 + revision: de5365cd30e5b70756328f2d584e0400c70eaf47 specs: - iiif_to_jekyll (0.9.4) + iiif_to_jekyll (0.9.5) iiif-presentation (~> 0.2.0) openssl GEM remote: https://rubygems.org/ specs: - activesupport (6.1.0) + activesupport (6.1.4.1) concurrent-ruby (~> 1.0, >= 1.0.2) i18n (>= 1.6, < 2) minitest (>= 5.1) tzinfo (~> 2.0) zeitwerk (~> 2.3) - concurrent-ruby (1.1.7) - faraday (1.3.0) + concurrent-ruby (1.1.9) + faraday (1.8.0) + faraday-em_http (~> 1.0) + faraday-em_synchrony (~> 1.0) + faraday-excon (~> 1.1) + faraday-httpclient (~> 1.0.1) faraday-net_http (~> 1.0) + faraday-net_http_persistent (~> 1.1) + faraday-patron (~> 1.0) + faraday-rack (~> 1.0) multipart-post (>= 1.2, < 3) - ruby2_keywords - faraday-net_http (1.0.0) - i18n (1.8.7) + ruby2_keywords (>= 0.0.4) + faraday-em_http (1.0.0) + faraday-em_synchrony (1.0.0) + faraday-excon (1.1.0) + faraday-httpclient (1.0.1) + faraday-net_http (1.0.1) + faraday-net_http_persistent (1.2.0) + faraday-patron (1.0.0) + faraday-rack (1.0.0) + i18n (1.8.11) concurrent-ruby (~> 1.0) iiif-presentation (0.2.0) activesupport (>= 3.2.18) faraday (>= 0.9) json - json (2.5.1) - minitest (5.14.2) + ipaddr (1.2.3) + json (2.6.1) + minitest (5.14.4) multipart-post (2.1.1) - openssl (2.2.0) - ruby2_keywords (0.0.2) + openssl (2.2.1) + ipaddr + ruby2_keywords (0.0.5) tzinfo (2.0.4) concurrent-ruby (~> 1.0) - zeitwerk (2.4.2) + zeitwerk (2.5.1) PLATFORMS ruby DEPENDENCIES - iiif_to_jekyll (= 0.9.4)! + iiif_to_jekyll (= 0.9.5)! BUNDLED WITH 2.1.4 From a941bde0c5f5b6aed7071fca1912e04b7a54e5dd Mon Sep 17 00:00:00 2001 From: Jay Varner Date: Thu, 2 Dec 2021 12:44:53 -0500 Subject: [PATCH 7/7] Bump version number and update changelog --- CHANGELOG.rst | 4 ++++ apps/__init__.py | 2 +- apps/readux/__init__.py | 2 +- readux/__init__.py | 2 +- 4 files changed, 7 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index f4d606484..0f9209d33 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,6 +2,10 @@ CHANGELOG ========= +Release 2.2.2 +--------------------- +* Bug fixes for naming GitHub repository for export. + Release 2.2.1 --------------------- * Improvements to bulk ingest diff --git a/apps/__init__.py b/apps/__init__.py index b73cbe3cc..09fa92559 100644 --- a/apps/__init__.py +++ b/apps/__init__.py @@ -1,4 +1,4 @@ -__version__ = "2.2.1" +__version__ = "2.2.2" __version_info__ = tuple( [ int(num) if num.isdigit() else num diff --git a/apps/readux/__init__.py b/apps/readux/__init__.py index 36a511eca..f1edb192f 100644 --- a/apps/readux/__init__.py +++ b/apps/readux/__init__.py @@ -1 +1 @@ -__version__ = '2.2.1' +__version__ = '2.2.2' diff --git a/readux/__init__.py b/readux/__init__.py index 3739befa8..f185a5c62 100644 --- a/readux/__init__.py +++ b/readux/__init__.py @@ -1,4 +1,4 @@ -__version_info__ = (2, 2, 0, None) +__version_info__ = (2, 2, 2, None) # Dot-connect all but the last. Last is dash-connected if not None. __version__ = '.'.join([str(i) for i in __version_info__[:-1]])