chore: move from okfn container to ckan container which users 'Pytest 5'

.github/workflows/publish.yml

@@ -87,7 +87,7 @@ jobs:
     name: CKAN ${{ matrix.ckan-version }}
     runs-on: ubuntu-latest
     container:
-      image: openknowledge/ckan-dev:${{ matrix.ckan-version }}
+      image: ckan/ckan-dev:${{ matrix.ckan-image }}
       options: --user root
     services:
       solr:

.github/workflows/test.yml

@@ -38,7 +38,7 @@ jobs:
     name: CKAN ${{ matrix.ckan-version }}
     runs-on: ubuntu-latest
     container:
-      image: openknowledge/ckan-dev:${{ matrix.ckan-version }}
+      image: ckan/ckan-dev:${{ matrix.ckan-image }}
       options: --user root
     services:
       solr:
@@ -79,6 +79,11 @@ jobs:
         # Replace default path to CKAN core config file with the one on the container
         sed -i -e 's/use = config:.*/use = config:\/srv\/app\/src\/ckan\/test-core.ini/' test.ini
 
+    - name: Pin jinja2 for ckan 2.9 only (last jinja version that had escape class)
+      if: ${{ matrix.ckan-version == 2.9 }}
+      run: pip install "jinja2<3.1"
+      continue-on-error: ${{ matrix.experimental }}
+
     - name: Setup extension
       continue-on-error: ${{ matrix.experimental }}
       run: |

ckanext/validation/logic/action.py

@@ -241,15 +241,15 @@ def resource_validation_run_batch(context, data_dict):
     if isinstance(dataset_ids, str):
         try:
             dataset_ids = json.loads(dataset_ids)
-        except ValueError as e:
+        except ValueError:
             dataset_ids = [dataset_ids]
 
     search_params = data_dict.get('query')
     if isinstance(search_params, str):
         try:
             search_params = json.loads(search_params)
-        except ValueError as e:
-            msg = 'Error parsing search parameters'.format(search_params)
+        except ValueError:
+            msg = 'Error parsing search parameters {0}'.format(search_params)
             return {'output': msg}
 
     while True:
@@ -455,9 +455,9 @@ def resource_create(up_func, context, data_dict):
 
     if run_validation:
         is_local_upload = (
-            hasattr(upload, 'filename') and
-            upload.filename is not None and
-            isinstance(upload, uploader.ResourceUpload))
+            hasattr(upload, 'filename')
+            and upload.filename is not None
+            and isinstance(upload, uploader.ResourceUpload))
         _run_sync_validation(
             resource_id, local_upload=is_local_upload, new_resource=True)
 
@@ -532,8 +532,8 @@ def resource_update(up_func, context, data_dict):
         raise t.ObjectNotFound(t._('Resource was not found.'))
 
     # Persist the datastore_active extra if already present and not provided
-    if ('datastore_active' in resource.extras and
-            'datastore_active' not in data_dict):
+    if ('datastore_active' in resource.extras
+            and 'datastore_active' not in data_dict):
         data_dict['datastore_active'] = resource.extras['datastore_active']
 
     for plugin in plugins.PluginImplementations(plugins.IResourceController):
@@ -577,9 +577,9 @@ def resource_update(up_func, context, data_dict):
 
     if run_validation:
         is_local_upload = (
-            hasattr(upload, 'filename') and
-            upload.filename is not None and
-            isinstance(upload, uploader.ResourceUpload))
+            hasattr(upload, 'filename')
+            and upload.filename is not None
+            and isinstance(upload, uploader.ResourceUpload))
         _run_sync_validation(
             id, local_upload=is_local_upload, new_resource=False)
 
@@ -612,7 +612,7 @@ def _run_sync_validation(resource_id, local_upload=False, new_resource=True):
     except t.ValidationError as e:
         log.info(
             u'Could not run validation for resource %s: %s',
-                resource_id, e)
+            resource_id, e)
         return
 
     validation = t.get_action(u'resource_validation_show')(

ckanext/validation/tests/fixtures.py

@@ -3,8 +3,6 @@
 from ckan.lib import uploader
 from ckanext.validation.model import create_tables, tables_exist
 
-import ckantoolkit as t
-
 
 @pytest.fixture
 def validation_setup():

ckanext/validation/tests/helpers.py

@@ -1,5 +1,4 @@
 import builtins
-import cgi
 import functools
 from unittest import mock
 from io import BytesIO
@@ -8,7 +7,6 @@
 from pyfakefs import fake_filesystem
 
 import ckan.lib.uploader
-from ckan.plugins import toolkit
 from ckan.tests.helpers import change_config
 
 
@@ -146,6 +144,7 @@ def wrapper(*args, **kwargs):
 class MockFieldStorage(FileStorage):
     pass
 
+
 def get_mock_file(contents):
     mock_file = BytesIO()
     mock_file.write(contents.encode('utf8'))

ckanext/validation/tests/test_form.py

@@ -1,14 +1,13 @@
 import json
 import io
-from unittest import mock
 import datetime
 
 import pytest
 
 import ckantoolkit as t
 from ckantoolkit.tests.factories import Sysadmin, Dataset
 from ckantoolkit.tests.helpers import (
-    call_action, _get_test_app
+    call_action
 )
 
 from ckanext.validation.tests.helpers import VALID_CSV, INVALID_CSV
@@ -255,8 +254,6 @@ def test_resource_form_update_upload(self, app):
         value = {"fields": [{"name": "code"}, {"name": "department"}, {"name": "date"}]}
         json_value = bytes(json.dumps(value).encode('utf8'))
 
-        upload = ('schema_upload', 'schema.json', json_value)
-
         data = {
             "url": "https://example.com/data.csv",
             "id": "",
@@ -451,7 +448,7 @@ def test_resource_form_update_invalid(self, app):
         user = Sysadmin()
         env = {"REMOTE_USER": user["name"].encode("ascii")}
 
-        dataset2 = call_action("package_show", id=dataset["id"])
+        call_action("package_show", id=dataset["id"])
         response = app.post(
             url=_edit_resource_url(dataset['id'], dataset['resources'][0]['id']),
             extra_environ=env,

ckanext/validation/tests/test_interfaces.py

@@ -4,8 +4,6 @@
 from ckan import plugins as p
 from ckan.tests import helpers, factories
 
-import ckantoolkit as t
-
 from ckanext.validation.interfaces import IDataValidation
 from ckanext.validation.tests.helpers import VALID_REPORT
 
@@ -40,14 +38,13 @@ def setup(self):
         for plugin in p.PluginImplementations(IDataValidation):
             return plugin.reset_counter()
 
+
 @pytest.fixture
 def reset_counter():
     for plugin in p.PluginImplementations(IDataValidation):
         return plugin.reset_counter()
 
 
-
-
 @pytest.mark.usefixtures("clean_db", "validation_setup", "with_plugins", "reset_counter")
 @pytest.mark.ckan_config("ckan.plugins", "validation test_validation_plugin scheming_datasets")
 class TestInterfaceSync():
@@ -130,6 +127,7 @@ def test_can_validate_called_on_update_sync_no_validation(self, mock_validation)
 
         assert not mock_validation.called
 
+
 @pytest.mark.usefixtures("clean_db", "validation_setup", "with_plugins", "reset_counter")
 @pytest.mark.ckan_config("ckan.plugins", "validation test_validation_plugin scheming_datasets")
 @pytest.mark.ckan_config('ckanext.validation.run_on_create_sync', False)

ckanext/validation/tests/test_jobs.py

@@ -9,7 +9,7 @@
 from ckan.tests.helpers import call_action
 from ckan.tests import factories
 
-from ckanext.validation.model import create_tables, tables_exist, Validation
+from ckanext.validation.model import Validation
 from ckanext.validation.jobs import run_validation_job, uploader, Session
 from ckanext.validation.tests.helpers import (
     VALID_REPORT,

ckanext/validation/tests/test_logic.py

@@ -47,7 +47,7 @@ def test_resource_validation_wrong_format(self):
 
             call_action("resource_validation_run", resource_id=resource["id"])
 
-        assert "Unsupported resource format" in str(e)
+        assert "Unsupported resource format" in str(e.value)
 
     def test_resource_validation_no_url_or_upload(self):
 
@@ -57,7 +57,7 @@ def test_resource_validation_no_url_or_upload(self):
 
             call_action("resource_validation_run", resource_id=resource["id"])
 
-        assert "Resource must have a valid URL" in str(e)
+        assert "Resource must have a valid URL" in str(e.value)
 
     @mock.patch("ckanext.validation.logic.action.enqueue_job")
     def test_resource_validation_with_url(self, mock_enqueue_job):
@@ -199,7 +199,6 @@ def test_resource_validation_only_called_on_resource_updated(
         assert mock_enqueue_job.call_args[0][1][0]["id"] == resource_1_id
 
 
-
 @pytest.mark.usefixtures("clean_db", "validation_setup", "with_plugins")
 class TestResourceValidationShow(object):
     def test_resource_validation_show_param_missing(self):
@@ -530,8 +529,8 @@ def test_validation_fails_on_upload(self):
             )
 
         assert "validation" in e.value.error_dict
-        assert "missing-cell" in str(e)
-        assert 'Row at position "2" has a missing cell in field "d" at position "4"' in str(e)
+        assert "missing-cell" in str(e.value)
+        assert 'Row at position "2" has a missing cell in field "d" at position "4"' in str(e.value)
 
     @pytest.mark.usefixtures("mock_uploads")
     def test_validation_fails_no_validation_object_stored(self):
@@ -542,7 +541,7 @@ def test_validation_fails_no_validation_object_stored(self):
 
         dataset = factories.Dataset()
 
-        invalid_stream = io.BufferedReader(io.BytesIO(INVALID_CSV.encode('utf8')))
+        io.BufferedReader(io.BytesIO(INVALID_CSV.encode('utf8')))
 
         validation_count_before = model.Session.query(Validation).count()
 
@@ -626,8 +625,8 @@ def test_validation_fails_on_upload(self):
                 )
 
         assert "validation" in e.value.error_dict
-        assert "missing-cell" in str(e)
-        assert 'Row at position "2" has a missing cell in field "d" at position "4"' in str(e)
+        assert "missing-cell" in str(e.value)
+        assert 'Row at position "2" has a missing cell in field "d" at position "4"' in str(e.value)
 
     @pytest.mark.usefixtures("mock_uploads")
     def test_validation_fails_no_validation_object_stored(self):

ckanext/validation/tests/test_plugin.py

@@ -1,10 +1,9 @@
 import pytest
 from unittest import mock
 
-from ckan.tests.helpers import call_action, reset_db
+from ckan.tests.helpers import call_action
 from ckan.tests import factories
 
-from ckanext.validation.model import create_tables, tables_exist
 from ckanext.validation.jobs import run_validation_job