Move export to async task and add scaffolding for polling the status …

…of the task

studygroups/tasks.py

@@ -30,6 +30,10 @@
 import os
 import re
 import dateutil.parser
+import tempfile
+import unicodecsv as csv
+import json
+
 
 logger = logging.getLogger(__name__)
 
@@ -615,3 +619,57 @@ def send_cofacilitator_removed_email(study_group_id, user_id, actor_user_id):
     )
     msg.attach_alternative(html_body, 'text/html')
     msg.send()
+
+
+@shared_task
+def export_signups(user_id):
+    """ user_id - the person requesting the export """
+    # get data for export
+    # write to stream/buffer
+    # upload to AWS S3
+    # Need a id/ref for creating a link that user can use
+    # add view for AWS presigned URL
+
+
+    object_list = Application.objects.all().prefetch_related('study_group', 'study_group__course')
+
+    temp_file = tempfile.TemporaryFile(mode='w+b')
+
+    ts = timezone.now().utcnow().isoformat()
+    signup_questions = ['support', 'goals', 'computer_access']
+    field_names = [
+        'id', 'uuid', 'study group id', 'study group uuid', 'study group name', 'course',
+        'location', 'name', 'email', 'mobile', 'signed up at'
+    ] + signup_questions + ['use_internet', 'survey completed', 'communications opt-in']
+    writer = csv.writer(temp_file)
+    writer.writerow(field_names)
+    for signup in object_list:
+        signup_data = json.loads(signup.signup_questions)
+        digital_literacy = 'n/a'
+        if signup_data.get('use_internet'):
+            digital_literacy = dict(Application.DIGITAL_LITERACY_CHOICES)[signup_data.get('use_internet')]
+        writer.writerow(
+            [
+                signup.id,
+                signup.uuid,
+                signup.study_group_id,
+                signup.study_group.uuid,
+                signup.study_group.name,
+                signup.study_group.course.title,
+                signup.study_group.venue_name,
+                signup.name,
+                signup.email,
+                signup.mobile,
+                signup.created_at,
+            ] +
+            [ signup_data.get(key, 'n/a') for key in signup_questions ] +
+            [
+                digital_literacy,
+                'yes' if signup.learnersurveyresponse_set.count() else 'no'
+            ] +
+            [ signup.communications_opt_in ]
+        )
+
+    # TODO - Upload to AWS S3 and generate presigned URL
+    return { "presign_url": "TODO" }
+

studygroups/urls.py

@@ -1,4 +1,4 @@
-from django.urls import include, re_path
+from django.urls import include, re_path, path
 from django.views.generic import TemplateView
 from django.views.generic.base import RedirectView
 
@@ -31,6 +31,7 @@
 from studygroups.views import InvitationConfirm
 from studygroups.views import OptOutView
 from studygroups.views import ExportSignupsView
+from studygroups.views import ExportStatusView
 from studygroups.views import ExportFacilitatorsView
 from studygroups.views import ExportStudyGroupsView
 from studygroups.views import ExportCoursesView
@@ -145,6 +146,8 @@
     re_path(r'^staff/dash/stats/$', StatsDashView.as_view(), name='studygroups_staff_dash_stats'),
 
     re_path(r'^export/signups/$', ExportSignupsView.as_view(), name='studygroups_export_signups'),
+
+    path('export/status/<uuid:task_id>/', ExportStatusView.as_view(), name='studygroups_export_status'),
     re_path(r'^export/facilitators/$', ExportFacilitatorsView.as_view(), name='studygroups_export_facilitators'),
     re_path(r'^export/studygroups/$', ExportStudyGroupsView.as_view(), name='studygroups_export_studygroups'),
     re_path(r'^export/courses/$', ExportCoursesView.as_view(), name='studygroups_export_courses'),

studygroups/views/staff.py

@@ -13,6 +13,7 @@
 from django.utils.decorators import method_decorator
 from django.utils.translation import gettext as _
 from django.views.generic.base import View
+from django.views.generic.detail import SingleObjectMixin
 from django.views.generic import ListView
 from django.views.generic import TemplateView
 from django.views.generic.edit import FormView
@@ -24,6 +25,7 @@
 from django.db.models import Subquery
 from django.db.models import F, Case, When, Value, Sum, IntegerField
 
+from django_celery_results.models import TaskResult
 
 from studygroups.models import Application
 from studygroups.models import StudyGroup
@@ -33,9 +35,11 @@
 from learnwithpeople import __version__ as VERSION
 from learnwithpeople import GIT_REVISION
 from ..tasks import send_community_digest
-
+from ..tasks import export_signups
 from studygroups.forms import DigestGenerateForm
 
+from uxhelpers.utils import json_response
+
 
 @method_decorator(user_is_staff, name='dispatch')
 class StaffDashView(TemplateView):
@@ -45,6 +49,8 @@ def get_context_data(self, **kwargs):
         context = super().get_context_data(**kwargs)
         context['version'] = VERSION
         context['git_revision'] = GIT_REVISION
+        # TODO - see if there are any pending exports
+
         return context
 
 
@@ -66,55 +72,23 @@ def form_valid(self, form):
 
 
 @method_decorator(user_is_staff, name='dispatch')
-class ExportSignupsView(ListView):
+class ExportSignupsView(View):
 
-    def get_queryset(self):
-        return Application.objects.all().prefetch_related('study_group', 'study_group__course')
+    def get(self, request, *args, **kwargs):
+        result = export_signups.delay(request.user.id)
+        return json_response(request, {'task_id': result.id})
 
 
-    def csv(self, **kwargs):
-        response = http.HttpResponse(content_type="text/csv")
-        ts = timezone.now().utcnow().isoformat()
-        response['Content-Disposition'] = 'attachment; filename="signups-{}.csv"'.format(ts)
-        signup_questions = ['support', 'goals', 'computer_access']
-        field_names = [
-            'id', 'uuid', 'study group id', 'study group uuid', 'study group name', 'course',
-            'location', 'name', 'email', 'mobile', 'signed up at'
-        ] + signup_questions + ['use_internet', 'survey completed', 'communications opt-in']
-        writer = csv.writer(response)
-        writer.writerow(field_names)
-        for signup in self.object_list:
-            signup_data = json.loads(signup.signup_questions)
-            digital_literacy = 'n/a'
-            if signup_data.get('use_internet'):
-                digital_literacy = dict(Application.DIGITAL_LITERACY_CHOICES)[signup_data.get('use_internet')]
-            writer.writerow(
-                [
-                    signup.id,
-                    signup.uuid,
-                    signup.study_group_id,
-                    signup.study_group.uuid,
-                    signup.study_group.name,
-                    signup.study_group.course.title,
-                    signup.study_group.venue_name,
-                    signup.name,
-                    signup.email,
-                    signup.mobile,
-                    signup.created_at,
-                ] +
-                [ signup_data.get(key, 'n/a') for key in signup_questions ] +
-                [
-                    digital_literacy,
-                    'yes' if signup.learnersurveyresponse_set.count() else 'no'
-                ] +
-                [ signup.communications_opt_in ]
-            )
-        return response
-
+@method_decorator(user_is_staff, name='dispatch')
+class ExportStatusView(SingleObjectMixin, View):
+    model = TaskResult
+    pk_url_kwarg = 'task_id'
 
     def get(self, request, *args, **kwargs):
-        self.object_list = self.get_queryset()
-        return self.csv(**kwargs)
+        result = get_object_or_404(TaskResult, task_id= kwargs.get('task_id'))
+        # TODO - check that the task was an export
+        # TODO - return usefull data for the task
+        return json_response(request, {'task_id': result.task_id, 'status': result.status})
 
 
 @method_decorator(user_is_staff, name='dispatch')