Merge pull request #193 from datosgobar/188-analytics-export

Export de analytics a CSV

.gitignore

@@ -13,6 +13,7 @@ coverage.xml
 
 #Media
 media/*
+protected/*
 
 #Autoenv files
 .autoenv.zsh

conf/settings/base.py

@@ -151,7 +151,7 @@
 
 VENDOR_APPS = (
     "django_rq",
-    'import_export',
+    'sendfile',
     'des'
 )
 
@@ -290,4 +290,7 @@
 ENV_TYPE = env('ENV_TYPE', default='')
 
 # Tarea a ser croneada para indexación
-READ_DATAJSON_SHELL_CMD = env('READ_DATAJSON_BIN_PATH', default='')
+READ_DATAJSON_SHELL_CMD = env('READ_DATAJSON_BIN_PATH', default='')
+
+PROTECTED_MEDIA_DIR = ROOT_DIR('protected')
+ANALYTICS_CSV_FILENAME = 'analytics.csv'

conf/settings/local_example.py

@@ -17,3 +17,5 @@
         'PASSWORD': env('DATABASE_PASSWORD'),
     }
 }
+
+SENDFILE_BACKEND = 'sendfile.backends.development'

conf/settings/production.py

@@ -40,3 +40,7 @@
 }
 
 INSTALLED_APPS += 'raven.contrib.django.raven_compat',
+
+SENDFILE_BACKEND = 'sendfile.backends.nginx'
+SENDFILE_ROOT = PROTECTED_MEDIA_DIR
+SENDFILE_URL = env('PROTECTED_INTERNAL_URL', default='/protected')

conf/settings/staging.py

@@ -37,3 +37,6 @@
 }
 
 INSTALLED_APPS = INSTALLED_APPS + ('raven.contrib.django.raven_compat',)
+SENDFILE_BACKEND = 'sendfile.backends.nginx'
+SENDFILE_ROOT = PROTECTED_MEDIA_DIR
+SENDFILE_URL = env('PROTECTED_INTERNAL_URL', default='/protected')

requirements/base.txt

@@ -12,6 +12,6 @@ numpy
 django-rq==1.0.0
 pyyaml
 django-ipware==1.1.6
-django-import-export==0.5.1
+django-sendfile==0.3.11
 python-crontab==2.2.8
 django-des==2.2.0

series_tiempo_ar_api/apps/analytics/admin.py

@@ -1,30 +1,16 @@
 # -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
-from import_export import resources
 from django.contrib import admin
-from import_export.admin import ImportExportModelAdmin
 
 from series_tiempo_ar_api.apps.analytics.models import Query
 
 
-class QueryResource(resources.ModelResource):
-    class Meta:
-        model = Query
-        fields = export_order = (
-            'timestamp',
-            'ip_address',
-            'ids',
-            'params',
-        )
-
-
-class QueryAdmin(ImportExportModelAdmin):
-    list_display = ('timestamp', 'params',)
+class QueryAdmin(admin.ModelAdmin):
+    list_display = ('timestamp', 'ip_address', 'params',)
     readonly_fields = ('timestamp', 'params', 'ip_address', 'args', 'ids')
 
     search_fields = ('timestamp', 'params', 'ip_address', 'args', 'ids')
-    resource_class = QueryResource
 
 
 admin.site.register(Query, QueryAdmin)

series_tiempo_ar_api/apps/analytics/tasks.py

@@ -0,0 +1,39 @@
+#! coding: utf-8
+import os
+import json
+
+import unicodecsv
+from django.conf import settings
+from django_rq import job
+
+from .models import Query
+from .utils import kong_milliseconds_to_tzdatetime
+
+
+@job("default")
+def analytics(ids, args_string, ip_address, params, timestamp_milliseconds):
+    params_json = json.dumps(params)
+    timestamp = kong_milliseconds_to_tzdatetime(timestamp_milliseconds)
+    query = Query(ids=ids, args=args_string, ip_address=ip_address, params=params_json, timestamp=timestamp)
+    query.save()
+
+
+@job("default")
+def export(path=None):
+    queryset = Query.objects.all()
+    filepath = path or os.path.join(settings.PROTECTED_MEDIA_DIR, settings.ANALYTICS_CSV_FILENAME)
+
+    fields = [
+        Query.timestamp,
+        Query.ip_address,
+        Query.ids,
+        Query.params
+    ]
+
+    with open(filepath, 'wb') as f:
+        writer = unicodecsv.writer(f)
+        # header
+        writer.writerow([field.field_name for field in fields])
+        for query in queryset.iterator():
+
+            writer.writerow([getattr(query, field.field_name) for field in fields])

series_tiempo_ar_api/apps/analytics/templates/admin/analytics/change_list.html

@@ -0,0 +1,7 @@
+{% extends "admin/change_list.html" %}
+
+{% block object-tools-items %}
+    <li><a href="{% url 'analytics:export_analytics' %}" class="link">EXPORT TO CSV</a></li>
+    {{ block.super }}
+
+{% endblock %}

series_tiempo_ar_api/apps/analytics/templates/export.html

@@ -0,0 +1,21 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <title>Export</title>
+</head>
+<body>
+
+<p>OK! En unos segundos el link de descarga ({% url 'analytics:read_analytics' %}) estará actualizado.</p>
+<p>Redirecting in 5 seconds...</p>
+<script type="application/javascript">
+    (function() {
+        var url = "{% url 'admin:analytics_query_changelist' %}";
+        window.setTimeout(function() {
+            location.href = url;
+        }, 5000);
+    })();
+
+</script>
+</body>
+</html>

series_tiempo_ar_api/apps/analytics/tests/__init__.py

@@ -0,0 +1 @@
+#!coding=utf8

series_tiempo_ar_api/apps/analytics/tests/export_tests.py

@@ -0,0 +1,32 @@
+#!coding=utf8
+import os
+
+from django.test import TestCase
+from django.utils import timezone
+
+from series_tiempo_ar_api.apps.analytics.models import Query
+from series_tiempo_ar_api.apps.analytics.tasks import export
+
+
+class ExportTests(TestCase):
+
+    filepath = 'test'
+
+    def test_export(self):
+        Query(args='test', params='test', ip_address='ip_addr', timestamp=timezone.now(), ids='').save()
+
+        export(path=self.filepath)
+
+        with open(self.filepath) as f:
+            self.assertEqual(len(f.readlines()), 2)
+
+    def test_export_empty(self):
+        export(path=self.filepath)
+
+        with open(self.filepath) as f:
+            # Esperado solo una línea de header
+            self.assertEqual(len(f.readlines()), 1)
+
+    def tearDown(self):
+        if os.path.exists(self.filepath):
+            os.remove(self.filepath)

series_tiempo_ar_api/apps/analytics/tests/samples/sample_analytics_dump.csv

@@ -0,0 +1,2 @@
+timestamp,ip_address,ids,params
+2018-02-16 16:59:04.275899+00:00,huh,test,params

series_tiempo_ar_api/apps/analytics/tests.py → series_tiempo_ar_api/apps/analytics/tests/tests.py

@@ -5,9 +5,9 @@
 from django.test import TestCase
 from django.urls import reverse
 
-from .analytics import analytics
-from .models import Query
-from .utils import kong_milliseconds_to_tzdatetime
+from series_tiempo_ar_api.apps.analytics.tasks import analytics
+from series_tiempo_ar_api.apps.analytics.models import Query
+from series_tiempo_ar_api.apps.analytics.utils import kong_milliseconds_to_tzdatetime
 
 
 class AnalyticsTests(TestCase):

series_tiempo_ar_api/apps/analytics/tests/view_tests.py

@@ -0,0 +1,52 @@
+#!coding=utf8
+import os
+import mock
+
+import sendfile
+from django.http import FileResponse
+from django.test import TestCase
+from django.urls import reverse
+from django.contrib.auth.models import User
+
+samples_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'samples')
+
+
+class ExportViewTests(TestCase):
+
+    def test_export_when_not_staff(self):
+        response = self.client.get(reverse('analytics:export_analytics'))
+
+        # Expected: admin login redirect
+        self.assertEqual(response.status_code, 302)
+
+    def test_export_as_staff(self):
+        user = User(username='user', password='pass', email='[email protected]', is_staff=True)
+        user.save()
+        self.client.force_login(user)
+        response = self.client.get(reverse('analytics:export_analytics'))
+
+        self.assertEqual(response.status_code, 200)
+
+
+class AnalyticsDownloadTests(TestCase):
+
+    def test_download_when_not_staff(self):
+        response = self.client.get(reverse('analytics:read_analytics'))
+
+        # Expected: admin login redirect
+        self.assertEqual(response.status_code, 302)
+
+    def test_download_as_staff(self):
+        user = User(username='user', password='pass', email='[email protected]', is_staff=True)
+        user.save()
+        self.client.force_login(user)
+
+        response_file = open(os.path.join(samples_dir, 'sample_analytics_dump.csv'))
+        sendfile.sendfile = mock.Mock(return_value=FileResponse(response_file))
+        response = self.client.get(reverse('analytics:read_analytics'))
+
+        self.assertEqual(response.status_code, 200)
+
+        with open(os.path.join(samples_dir, 'sample_analytics_dump.csv')) as response_file:
+            response_content = list(response.streaming_content)[0]  # Unwrap iterable
+            self.assertEqual(response_content, response_file.read())

series_tiempo_ar_api/apps/analytics/urls.py

@@ -1,7 +1,8 @@
 from django.conf.urls import url
-
-from .views import save
+from .views import save, read_analytics, export_analytics
 
 urlpatterns = [
-    url('^save/$', save, name='save')
+    url('^save/$', save, name='save'),
+    url('^analytics.csv', read_analytics, name='read_analytics'),
+    url('^export', export_analytics, name='export_analytics'),
 ]

series_tiempo_ar_api/apps/analytics/views.py

@@ -1,12 +1,17 @@
 # -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
+import os
 import json
 
+import sendfile
 from django.http import HttpResponse
+from django.conf import settings
 from django.views.decorators.csrf import csrf_exempt
+from django.contrib.admin.views.decorators import staff_member_required
+from django.shortcuts import render
 
-from .analytics import analytics
+from .tasks import analytics, export
 
 
 @csrf_exempt
@@ -34,3 +39,14 @@ def save(request):
         args = args[params_start:]
     analytics.delay(ids, args, ip_address, params, timestamp)
     return HttpResponse()
+
+
+@staff_member_required
+def read_analytics(request):
+    return sendfile.sendfile(request, os.path.join(settings.PROTECTED_MEDIA_DIR, 'analytics.csv'))
+
+
+@staff_member_required
+def export_analytics(request):
+    export.delay()
+    return HttpResponse(render(request, 'export.html'))