aligntment: convert badges to css, use webassets only, user iUploader…

… interface, move config settings from utils to settings class

ckanext/validation/helpers.py

@@ -1,61 +1,96 @@
 # encoding: utf-8
 import json
 
-from ckan.lib.helpers import url_for_static
+from six.moves.urllib.parse import urlparse
+from six import string_types
 from ckantoolkit import url_for, _, config, asbool, literal, h
 
+from ckanext.validation.utils import get_default_schema
+
 def get_helpers():
     validators = (
         get_validation_badge,
         validation_extract_report_from_errors,
         dump_json_value,
         bootstrap_version,
-        validation_dict,
-        use_webassets
+        validation_hide_source,
+        is_url_valid
     )
 
     return {"{}".format(func.__name__): func for func in validators}
 
 
 def get_validation_badge(resource, in_listing=False):
 
+    # afterDate = config.get('ckanext.validation.show_badges_after_last_modified_date', "")
+    # if afterDate and (not resource.get('last_modified')
+    #                   or h.date_str_to_datetime(afterDate)
+    #                   >= h.date_str_to_datetime(resource['last_modified'])):
+    #     return ''
+
     if in_listing and not asbool(
             config.get('ckanext.validation.show_badges_in_listings', True)):
         return ''
 
     if not resource.get('validation_status'):
         return ''
 
+    # if not _get_schema_or_default_schema(resource):
+    #     return ''
+
+    statuses = {
+        'success': _('valid'),
+        'failure': _('invalid'),
+        'invalid': _('invalid'),
+        'error': _('error'),
+        'unknown': _('unknown'),
+    }
+
     messages = {
         'success': _('Valid data'),
         'failure': _('Invalid data'),
+        'invalid': _('invalid data'),
         'error': _('Error during validation'),
         'unknown': _('Data validation unknown'),
     }
 
-    if resource['validation_status'] in ['success', 'failure', 'error']:
+    if resource['validation_status'] in ['success', 'failure', 'invalid', 'error']:
         status = resource['validation_status']
     else:
         status = 'unknown'
 
+    action = 'validation.read'
+
     validation_url = url_for(
-        'validation_read',
+        action,
         id=resource['package_id'],
         resource_id=resource['id'])
 
-    badge_url = url_for_static(
-        '/images/badges/data-{}-flat.svg'.format(status))
-
-    return '''
-<a href="{validation_url}" class="validation-badge">
-    <img src="{badge_url}" alt="{alt}" title="{title}"/>
+    return u'''
+<a href="{validation_url}" class="validation-badge" title="{alt} {title}">
+    <span class="prefix">{prefix}</span><span class="status {status}">{status_title}</span>
 </a>'''.format(
         validation_url=validation_url,
-        badge_url=badge_url,
+        prefix=_('data'),
+        status=status,
+        status_title=statuses[status],
         alt=messages[status],
         title=resource.get('validation_timestamp', ''))
 
 
+def _get_schema_or_default_schema(resource):
+
+    if asbool(resource.get('align_default_schema')):
+        schema = get_default_schema(resource['package_id'])
+    else:
+        schema = resource.get('schema')
+
+    if schema and isinstance(schema, string_types):
+        schema = schema if is_url_valid(schema) else json.loads(schema)
+
+    return schema
+
+
 def validation_extract_report_from_errors(errors):
 
     report = None
@@ -84,8 +119,6 @@ def validation_extract_report_from_errors(errors):
 
     return report, errors
 
-def validation_dict(validation_json):
-    return json.loads(validation_json)
 
 def dump_json_value(value, indent=None):
     """
@@ -109,5 +142,25 @@ def bootstrap_version():
         return '2'
 
 
-def use_webassets():
-    return int(h.ckan_version().split('.')[1]) >= 9
+def validation_hide_source(type):
+    """
+    Returns True if the given source type must be hidden on form.
+    Type is one of: upload, url or json.
+    For any unexpected type returns False
+    """
+    return asbool(config.get(
+        "ckanext.validation.form.hide_{}_source".format(type),
+    ))
+
+
+def is_url_valid(url):
+    """Basic checks for url validity"""
+    if not isinstance(url, string_types):
+        return False
+
+    try:
+        tokens = urlparse(url)
+    except ValueError:
+        return False
+
+    return all([getattr(tokens, attr) for attr in ('scheme', 'netloc')])

ckanext/validation/jobs.py

@@ -16,7 +16,7 @@
 import ckantoolkit as t
 
 from .model import Validation
-from . import utils
+from . import utils, settings
 
 
 log = logging.getLogger(__name__)
@@ -101,7 +101,7 @@ def run_validation_job(resource):
     report = _validate_table(source, _format=_format, schema=schema, **options)
 
     # Hide uploaded files
-    if type(report) == Report:
+    if isinstance(report, Report):
         report = report.to_dict()
 
     if 'tasks' in report:
@@ -135,7 +135,7 @@ def run_validation_job(resource):
         'validation_timestamp': validation.finished.isoformat(),
     }
 
-    if utils.get_update_mode_from_config() == 'sync':
+    if settings.get_update_mode_from_config() == 'sync':
         data_dict['_skip_next_validation'] = True,
 
     t.get_action('resource_patch')(

ckanext/validation/logic/action.py

@@ -14,13 +14,7 @@
 from ckanext.validation.model import Validation
 from ckanext.validation.interfaces import IDataValidation
 from ckanext.validation.jobs import run_validation_job
-from ckanext.validation import settings
-from ckanext.validation.utils import (
-    get_create_mode_from_config,
-    get_update_mode_from_config,
-    delete_local_uploaded_file,
-    validation_dictize,
-)
+from ckanext.validation import settings, utils
 
 
 log = logging.getLogger(__name__)
@@ -39,12 +33,6 @@ def get_actions():
     return {"{}".format(func.__name__): func for func in validators}
 
 
-def enqueue_job(*args, **kwargs):
-    try:
-        return t.enqueue_job(*args, **kwargs)
-    except AttributeError:
-        from ckanext.rq.jobs import enqueue as enqueue_job_legacy
-        return enqueue_job_legacy(*args, **kwargs)
 
 
 # Actions
@@ -77,11 +65,11 @@ def resource_validation_run(context, data_dict):
     async_job = data_dict.get(u'async', True)
 
     # Ensure format is supported
-    if not resource.get(u'format', u'').lower() in settings.SUPPORTED_FORMATS:
+    if not resource.get(u'format', u'').lower() in settings.get_supported_formats():
         raise t.ValidationError(
             {u'format': u'Unsupported resource format.' +
              u'Must be one of {}'.format(
-                 u','.join(settings.SUPPORTED_FORMATS))})
+                 u','.join(settings.get_supported_formats()))})
 
     # Ensure there is a URL or file upload
     if not resource.get(u'url') and not resource.get(u'url_type') == u'upload':
@@ -117,6 +105,9 @@ def resource_validation_run(context, data_dict):
         run_validation_job(resource)
 
 
+def enqueue_job(*args, **kwargs):
+    return t.enqueue_job(*args, **kwargs)
+
 @t.side_effect_free
 def resource_validation_show(context, data_dict):
     u'''
@@ -157,7 +148,7 @@ def resource_validation_show(context, data_dict):
         raise t.ObjectNotFound(
             'No validation report exists for this resource')
 
-    return validation_dictize(validation)
+    return utils.validation_dictize(validation)
 
 
 def resource_validation_delete(context, data_dict):
@@ -281,7 +272,7 @@ def resource_validation_run_batch(context, data_dict):
                 for resource in dataset['resources']:
 
                     if (not resource.get(u'format', u'').lower()
-                            in settings.SUPPORTED_FORMATS):
+                            in settings.get_supported_formats()):
                         continue
 
                     try:
@@ -381,7 +372,7 @@ def _add_default_formats(search_data_dict):
 
     filter_formats = []
 
-    for _format in settings.DEFAULT_SUPPORTED_FORMATS:
+    for _format in settings.get_supported_formats():
         filter_formats.extend([_format, _format.upper()])
 
     filter_formats_query = ['+res_format:"{0}"'.format(_format)
@@ -403,7 +394,7 @@ def resource_create(up_func, context, data_dict):
 
     '''
 
-    if get_create_mode_from_config() != 'sync':
+    if settings.get_create_mode_from_config() != 'sync':
         return up_func(context, data_dict)
 
     model = context['model']
@@ -509,7 +500,7 @@ def resource_update(up_func, context, data_dict):
 
     '''
 
-    if get_update_mode_from_config() != 'sync':
+    if settings.get_update_mode_from_config() != 'sync':
         return up_func(context, data_dict)
 
     model = context['model']
@@ -641,7 +632,7 @@ def _run_sync_validation(resource_id, local_upload=False, new_resource=True):
 
             # Delete uploaded file
             if local_upload:
-                delete_local_uploaded_file(resource_id)
+                utils.delete_local_uploaded_file(resource_id)
 
             if new_resource:
                 # Delete resource

ckanext/validation/plugin.py

@@ -8,15 +8,11 @@
 import ckan.plugins as p
 import ckantoolkit as tk
 
-from . import settings as s, utils, validators
+from . import settings as s, utils, validators, utils
 from .helpers import get_helpers
 from .logic import action, auth
 from .model import tables_exist
 
-from ckanext.validation.utils import (
-    get_create_mode_from_config,
-    get_update_mode_from_config,
-)
 from ckanext.validation.interfaces import IDataValidation
 from ckanext.validation import views, cli
 
@@ -59,7 +55,6 @@ def update_config(self, config_):
             log.debug(u'Validation tables exist')
 
         tk.add_template_directory(config_, u'templates')
-        tk.add_public_directory(config_, u'public')
         tk.add_resource(u'webassets', 'ckanext-validation')
 
     # IActions
@@ -112,7 +107,7 @@ def after_resource_create(self, context, data_dict):
 
         is_dataset = self._data_dict_is_dataset(data_dict)
 
-        if not get_create_mode_from_config() == u'async':
+        if not s.get_create_mode_from_config() == u'async':
             return
 
         if is_dataset:
@@ -141,7 +136,7 @@ def _handle_validation_for_resource(self, context, resource):
             ) and (
             # Make sure format is supported
             resource.get(u'format', u'').lower() in
-                s.SUPPORTED_FORMATS
+                s.get_supported_formats()
                 )):
             needs_validation = True
 
@@ -152,7 +147,7 @@ def _handle_validation_for_resource(self, context, resource):
                     log.debug('Skipping validation for resource %s', resource['id'])
                     return
 
-            utils._run_async_validation(resource[u'id'])
+            utils.run_async_validation(resource[u'id'])
 
     # CKAN < 2.10
     def before_update(self, context, current_resource, updated_resource):
@@ -172,7 +167,7 @@ def before_resource_update(self, context, current_resource, updated_resource):
                 package_id = existing_resource['package_id']
         self.packages_to_skip[package_id] = True
 
-        if not get_update_mode_from_config() == u'async':
+        if not s.get_update_mode_from_config() == u'async':
             return updated_resource
 
         needs_validation = False
@@ -190,7 +185,7 @@ def before_resource_update(self, context, current_resource, updated_resource):
             ) and (
             # Make sure format is supported
             updated_resource.get(u'format', u'').lower() in
-                s.SUPPORTED_FORMATS
+                s.get_supported_formats()
                 )):
             needs_validation = True
 
@@ -213,8 +208,8 @@ def after_resource_update(self, context, data_dict):
 
         # Need to allow create as well because resource_create calls
         # package_update
-        if (not get_update_mode_from_config() == u'async'
-                and not get_create_mode_from_config() == u'async'):
+        if (not s.get_update_mode_from_config() == u'async'
+                and not s.get_create_mode_from_config() == u'async'):
             return
 
         if context.get('_validation_performed'):
@@ -262,10 +257,10 @@ def after_resource_update(self, context, data_dict):
 
                 del self.resources_to_validate[resource_id]
 
-                utils._run_async_validation(resource_id)
+                utils.run_async_validation(resource_id)
 
-            if utils._should_remove_unsupported_resource_validation_reports(data_dict):
-                p.toolkit.enqueue_job(fn=utils._remove_unsupported_resource_validation_reports, args=[resource_id])
+            if utils.should_remove_unsupported_resource_validation_reports(data_dict):
+                p.toolkit.enqueue_job(fn=utils.remove_unsupported_resource_validation_reports, args=[resource_id])
 
     # IPackageController