2.9.x Alignment - qld-gov-au to upstream + okfn to ckan containers

.github/workflows/publish.yml

@@ -87,7 +87,7 @@ jobs:
     name: CKAN ${{ matrix.ckan-version }}
     runs-on: ubuntu-latest
     container:
-      image: openknowledge/ckan-dev:${{ matrix.ckan-version }}
+      image: ckan/ckan-dev:${{ matrix.ckan-image }}
       options: --user root
     services:
       solr:

.github/workflows/test.yml

@@ -23,26 +23,30 @@ jobs:
         include: #ckan-image see https://github.com/ckan/ckan-docker-base, ckan-version controls other image tags
 #          - ckan-version: "2.11"
 #            ckan-image: "2.11-py3.10"
+#            ckan-solr: "2.11-solr9"
 #            experimental: false
 #          - ckan-version: "2.10"
 #            ckan-image: "2.10-py3.10"
+#            ckan-solr: "2.10-solr9"
 #            experimental: false
           - ckan-version: "2.9"
             ckan-image: "2.9-py3.9"
+            ckan-solr: "2.9-solr8"
             experimental: false
 #          - ckan-version: "master"
 #            ckan-image: "master"
+#            ckan-solr: "master-solr9"
 #            experimental: true  # master is unstable, good to know if we are compatible or not
       fail-fast: false
 
     name: CKAN ${{ matrix.ckan-version }}
     runs-on: ubuntu-latest
     container:
-      image: openknowledge/ckan-dev:${{ matrix.ckan-version }}
+      image: ckan/ckan-dev:${{ matrix.ckan-image }}
       options: --user root
     services:
       solr:
-        image: ckan/ckan-solr:${{ matrix.ckan-version }}-solr9
+        image: ckan/ckan-solr:${{ matrix.ckan-solr }}
       postgres:
         image: ckan/ckan-postgres-dev:${{ matrix.ckan-version }}
         env:
@@ -79,6 +83,11 @@ jobs:
         # Replace default path to CKAN core config file with the one on the container
         sed -i -e 's/use = config:.*/use = config:\/srv\/app\/src\/ckan\/test-core.ini/' test.ini
 
+    - name: Pin jinja2 for ckan 2.9 only (last jinja version that had escape class)
+      if: ${{ matrix.ckan-version == 2.9 }}
+      run: pip install "jinja2<3.1"
+      continue-on-error: ${{ matrix.experimental }}
+
     - name: Setup extension
       continue-on-error: ${{ matrix.experimental }}
       run: |
@@ -95,7 +104,7 @@ jobs:
         paths: "/tmp/artifacts/junit/*.xml"
       if: always()
 
-    - name: Upload coverage report to codecov
+    - name: "Upload coverage report to codecov flag: ${{ matrix.ckan-version }}"
       uses: codecov/codecov-action@v5
       continue-on-error: ${{ matrix.experimental }}
       with:

.gitignore

@@ -99,3 +99,4 @@ ENV/
 
 # mypy
 .mypy_cache/
+.idea

MANIFEST.in

@@ -1,4 +1,4 @@
-include README.rst
+include README.md
 include LICENSE
 include requirements.txt
-recursive-include ckanext/validation *.html *.json *.js *.less *.css *.mo
+recursive-include ckanext/validation *.html *.json *.js *.less *.css *.mo *.config *.yml

ckanext/validation/cli.py

@@ -1,22 +1,94 @@
-import sys
+# encoding: utf-8
 
 import click
 
-from ckanext.validation.model import create_tables, tables_exist
+from ckanext.validation import common
+
+
+def get_commands():
+    return [validation]
 
 
 @click.group()
 def validation():
-    """Harvests remotely mastered metadata."""
+    """Validation management commands.
+    """
     pass
 
 
-@validation.command()
+@validation.command(name='init-db')
 def init_db():
-    """Creates the necessary tables in the database."""
-    if tables_exist():
-        print(u"Validation tables already exist")
-        sys.exit(0)
+    """ Initialize database tables.
+    """
+    common.init_db()
+
+
+@validation.command(name='run')
+@click.option(u'-y', u'--yes',
+              help=u'Automatic yes to prompts. Assume "yes" as answer '
+                   u'to all prompts and run non-interactively',
+              default=False)
+@click.option('-r', '--resource',
+              multiple=True,
+              help=u'Run data validation on a particular resource (if the format is suitable).'
+                   u'It can be defined multiple times. Not to be used with -d or -s')
+@click.option('-d', '--dataset',
+              multiple=True,
+              help=u'Run data validation on all resources for a particular dataset (if the format is suitable).'
+                   u' You can use the dataset id or name, and it can be defined multiple times. '
+                   u'Not to be used with -r or -s')
+@click.option('-s', '--search',
+              default=False,
+              help=u'Extra search parameters that will be used for getting the datasets to run '
+                   u'validation on. It must be a JSON object like the one used by the `package_search` API call.'
+                   u' Supported fields are `q`, `fq` and `fq_list`. Check the documentation for examples. '
+                   u'Note that when using this you will have to specify the resource formats to target yourself.'
+                   u' Not to be used with -r or -d.')
+def run_validation(yes, resource, dataset, search):
+    '''Start asynchronous data validation on the site resources. If no
+    options are provided it will run validation on all resources of
+    the supported formats (`ckanext.validation.formats`). You can
+    specify particular datasets to run the validation on their
+    resources. You can also pass arbitrary search parameters to filter
+    the selected datasets.
+    '''
+    common.run_validation(yes, resource, dataset, search)
+
+
+@validation.command()
+@click.option(u'-o', u'--output',
+              help=u'Location of the CSV validation report file on the relevant commands.',
+              default=u'validation_errors_report.csv')
+def report(output):
+    '''Generate a report with all current data validation reports. This
+    will print an overview of the total number of tabular resources
+    and a breakdown of how many have a validation status of success,
+    failure or error. Additionally it will create a CSV report with all
+    failing resources, including the following fields:
+        * Dataset name
+        * Resource id
+        * Resource URL
+        * Status
+        * Validation report URL
+    '''
+    common.report(output)
+
+
+@validation.command(name='report-full')
+@click.option(u'-o', u'--output',
+              help=u'Location of the CSV validation report file on the relevant commands.',
+              default=u'validation_errors_report.csv')
+def report_full(output):
+    '''Generate a detailed report. This is similar to 'report'
+    but on the CSV report it will add a row for each error found on the
+    validation report (limited to ten occurrences of the same error
+    type per file). So the fields in the generated CSV report will be:
 
-    create_tables()
-    print(u"Validation tables created")
+        * Dataset name
+        * Resource id
+        * Resource URL
+        * Status
+        * Error code
+        * Error message
+    '''
+    common.report(output, full=True)