ADX-949 ADX-958 ckanext-validation upgrade

ckanext/unaids/actions.py

@@ -6,11 +6,11 @@
 import ckan.lib.dictization.model_save as model_save
 import ckan.lib.dictization.model_dictize as model_dictize
 import ckan.plugins.toolkit as t
-import ckanext.validation.helpers as validation_helpers
 import ckanext.unaids.custom_user_profile as custom_user_profile
 from ckan.common import _
 from ckanext.versions.logic.dataset_version_action import get_activity_id_from_dataset_version_name, activity_dataset_show
 from ckanext.unaids.logic import populate_data_dictionary_from_schema
+from ckanext.unaids.helpers import validation_load_json_schema
 
 NotFound = logic.NotFound
 NotAuthorized = logic.NotAuthorized
@@ -41,7 +41,7 @@ def get_table_schema(context, data_dict):
     schema_name = resource.get('schema')
     schema = False
     if schema_name:
-        schema = validation_helpers.validation_load_json_schema(schema_name)
+        schema = validation_load_json_schema(schema_name)
     if not schema:
         schema = {}
     return schema

ckanext/unaids/assets/css/validation.css

@@ -0,0 +1,23 @@
+.validation-badge{
+    display: inline-block;
+    margin: 0 0 20px 0;
+}
+
+
+.resource-list .validation-badge{
+    float: right;
+    margin: 0 150px 0 0;
+    text-align:  right;
+}
+
+@media (max-width: 991px) {
+    .resource-list .validation-badge{
+        margin: 0px;
+    }
+}
+
+.validation-badge p.badge-link{
+    text-align: center;
+    margin: 5px 0 0 0 ;
+    font-size: 14px;
+}

ckanext/unaids/assets/validation-badge.js

@@ -0,0 +1,55 @@
+this.ckan.module('validation-badge', function (jQuery) {
+    return{
+        options: {
+          resource: null,
+          status: 'created',
+          url: null
+        },
+        initialize: function () {
+          $.proxyAll(this, /_on/);
+          this._poll();
+          this.options.url = $(this.el).attr('href');
+          $(this.el).removeAttr('href');
+        },
+        _poll: function() {
+            var module = this;
+            $.ajax({
+                url: "/api/3/action/resource_validation_show?resource_id=" + this.options.resource,
+                type: "GET",
+                success: function(data){module._success(data);},
+                error: function(XMLHttpRequest, textStatus, errorThrown){module._error(XMLHttpRequest, textStatus, errorThrown);},
+                dataType: "json",
+                complete: setTimeout(function(){module._complete();}, 5000),
+                timeout: 2000
+            });
+        },
+        _success: function(data) {
+            this.options.status = data.result.status;
+            if(this.options.status != 'running' && this.options.status != 'created'){
+                this._update_badge();
+                $(this.el).attr('href', this.options.url);
+                if( this.options.status == 'failure' || this.options.status == 'error' ){
+                    $(this.el).find('.badge-link').removeClass('hidden');
+                }
+            }
+        },
+        _error: function(XMLHttpRequest, textStatus, errorThrown) {
+            this.options.status = 'error';
+            this._update_badge();
+            var message = 'No server connection. Please refresh your page.';
+            $(this.el).attr('title', message );
+            $(this.el).find('img').attr('title', message );
+        },
+        _complete: function() {
+            if(this.options.status == 'running' || this.options.status == 'created'){
+                this._poll();
+            }
+        },
+        _update_badge: function(){
+            var src = $('a.validation-badge').find('img').attr('src');
+            var src = src.split('-')[0] + "-" + this.options.status + '.gif';
+            $(this.el).find('img').attr('src', src);
+        }
+
+    }
+});

ckanext/unaids/assets/webassets.yml

@@ -7,6 +7,7 @@ main:
     - text_template.js
     - bootstrap_table.js
     - format_update.js
+    - validation-badge.js
   extra:
     preload:
       - base/main
@@ -64,3 +65,9 @@ MapSelectorCSS:
     filters: cssmin
     output: unaids/%(version)s_MapSelector.css
 
+ValidationBadgeCSS:
+    filters: cssmin
+    output: unaids/%(version)s_ValidationBadge.css
+    contents:
+        - css/validation.css
+

ckanext/unaids/helpers.py

@@ -2,14 +2,16 @@
 import logging
 import os
 import json
+import requests
 import six
 from ckan.lib.helpers import url_for_static_or_external, check_access, full_current_url
 from ckan.lib.i18n import get_lang
 from ckan.plugins.toolkit import get_action, request
 from ckan.plugins import toolkit
-from ckan.common import _, g, config
+from ckan.common import _, g, asbool, config
 from ckan.lib.helpers import build_nav_main as core_build_nav_main
 
+
 try:
     from html import escape as html_escape
 except ImportError:
@@ -20,6 +22,48 @@
 log = logging.getLogger()
 BULK_FILE_UPLOADER_DEFAULT_FIELDS = 'ckanext.bulk_file_uploader_default_fields'
 
+log = logging.getLogger(__name__)
+
+
+def _files_from_directory(path, extension=".json"):
+    listed_files = {}
+    for root, dirs, files in os.walk(path):
+        for file in files:
+            if extension in file:
+                name = file.split(".json")[0]
+                listed_files[name] = os.path.join(root, file)
+    return listed_files
+
+
+def get_schema_filepath(schema):
+    schema_directory = toolkit.config["ckanext.unaids.schema_directory"]
+    schemas = _files_from_directory(schema_directory)
+    return schemas.get(schema)
+
+
+def validation_load_json_schema(schema):
+    try:
+        # When updating a resource there's already an existing JSON schema
+        # attached to the resource
+        if isinstance(schema, dict):
+            return schema
+
+        if schema.startswith("http"):
+            r = requests.get(schema)
+            return r.json()
+
+        schema_filepath = get_schema_filepath(schema)
+        if schema_filepath:
+            with open(schema_filepath, "rb") as schema_file:
+                return json.load(schema_file)
+
+        return json.loads(schema)
+
+    except json.JSONDecodeError as e:
+        log.error("Error loading schema: " + schema)
+        log.exception(e)
+        return None
+
 
 def get_all_package_downloads(pkg_dict):
     """
@@ -185,3 +229,60 @@ def is_an_estimates_dataset(dataset_type_name):
 
 def url_encode(url):
     return quote(url, safe='/:?=&')
+
+
+def unaids_get_validation_badge(resource, in_listing=False):
+
+    if in_listing and not asbool(
+            toolkit.config.get('ckanext.validation.show_badges_in_listings', True)):
+        return ''
+
+    if not resource.get('validation_status'):
+        return ''
+
+    messages = {
+        'success': _('Valid data'),
+        'failure': _('Invalid data'),
+        'error': _('Error during validation'),
+        'unknown': _('Data validation unknown'),
+    }
+
+    if resource['validation_status'] in ['success', 'failure', 'error']:
+        status = resource['validation_status']
+    else:
+        status = 'unknown'
+
+    validation_url = toolkit.url_for(
+        'validation_read',
+        id=resource['package_id'],
+        resource_id=resource['id']
+    )
+
+    tags = ""
+    if status == 'unknown':
+        tags += "data-module='validation-badge' data-module-resource='{}'".format(
+            resource['id']
+        )
+
+    badge_url = url_for_static_or_external(
+        '/images/badges/{}-{}.gif'.format(toolkit.h.lang(), status))
+
+    link_visibility = ""
+    if status in ['success', 'unknown']:
+        link_visibility = 'hidden'
+
+    badge_html = '''
+<a href="{validation_url}" {tags} class="validation-badge">
+    <img src="{badge_url}" alt="{alt}" title="{title}"/>
+    <p class="small badge-link {link_visibility}">{badge_link}</p>
+</a>'''.format(
+        validation_url=validation_url,
+        tags=tags,
+        badge_url=badge_url,
+        alt=messages[status],
+        title=resource.get('validation_timestamp', ''),
+        link_visibility=link_visibility,
+        badge_link=_('View Error Report')
+    )
+
+    return badge_html