From f092290c60cbec77bc59ad386bdfdad2a9233570 Mon Sep 17 00:00:00 2001 From: Khalil <52969012+khail-k@users.noreply.github.com> Date: Tue, 20 Dec 2022 16:06:15 +0000 Subject: [PATCH 1/6] add custom error to schema validation --- schema_enforcer/schemas/jsonschema.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/schema_enforcer/schemas/jsonschema.py b/schema_enforcer/schemas/jsonschema.py index 7b3c7bd..728325e 100644 --- a/schema_enforcer/schemas/jsonschema.py +++ b/schema_enforcer/schemas/jsonschema.py @@ -58,7 +58,13 @@ def validate(self, data, strict=False): for err in validator.iter_errors(data): has_error = True - self.add_validation_error(err.message, absolute_path=list(err.absolute_path)) + + if 'errMessage' in err.schema: + message = f"{err.instance} {err.schema['errMessage']}" + else: + message = err.message + + self.add_validation_error(message, absolute_path=list(err.absolute_path)) if not has_error: self.add_validation_pass() From 04dacaac5aaa5acbebca68db0d5c02875504851b Mon Sep 17 00:00:00 2001 From: Khalil <52969012+khail-k@users.noreply.github.com> Date: Tue, 20 Dec 2022 18:32:05 +0000 Subject: [PATCH 2/6] make message generic --- schema_enforcer/schemas/jsonschema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schema_enforcer/schemas/jsonschema.py b/schema_enforcer/schemas/jsonschema.py index 728325e..45080f6 100644 --- a/schema_enforcer/schemas/jsonschema.py +++ b/schema_enforcer/schemas/jsonschema.py @@ -60,7 +60,7 @@ def validate(self, data, strict=False): has_error = True if 'errMessage' in err.schema: - message = f"{err.instance} {err.schema['errMessage']}" + message = err.schema['errMessage'] else: message = err.message From 886971d3ce9cf4463f1c57125ff81635ccb1040c Mon Sep 17 00:00:00 2001 From: Khalil <52969012+khail-k@users.noreply.github.com> Date: Tue, 20 Dec 2022 22:22:01 +0000 Subject: [PATCH 3/6] replace placeholder with instance data --- schema_enforcer/schemas/jsonschema.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/schema_enforcer/schemas/jsonschema.py b/schema_enforcer/schemas/jsonschema.py index 45080f6..98aa7fd 100644 --- a/schema_enforcer/schemas/jsonschema.py +++ b/schema_enforcer/schemas/jsonschema.py @@ -2,6 +2,7 @@ import copy import pkgutil import json +import re from jsonschema import Draft7Validator, draft7_format_checker # pylint: disable=import-self from schema_enforcer.schemas.validator import BaseValidation @@ -61,6 +62,7 @@ def validate(self, data, strict=False): if 'errMessage' in err.schema: message = err.schema['errMessage'] + message = re.sub(r'[s]\w+', str(err.instance), message) else: message = err.message From 0f55cfde39b8bbf41ebeec654f37d7266402120c Mon Sep 17 00:00:00 2001 From: Khalil <52969012+khail-k@users.noreply.github.com> Date: Wed, 21 Dec 2022 12:45:31 +0000 Subject: [PATCH 4/6] Simplify message tag simple replace on word in string. --- schema_enforcer/schemas/jsonschema.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/schema_enforcer/schemas/jsonschema.py b/schema_enforcer/schemas/jsonschema.py index 98aa7fd..69a81cb 100644 --- a/schema_enforcer/schemas/jsonschema.py +++ b/schema_enforcer/schemas/jsonschema.py @@ -2,7 +2,6 @@ import copy import pkgutil import json -import re from jsonschema import Draft7Validator, draft7_format_checker # pylint: disable=import-self from schema_enforcer.schemas.validator import BaseValidation @@ -62,7 +61,7 @@ def validate(self, data, strict=False): if 'errMessage' in err.schema: message = err.schema['errMessage'] - message = re.sub(r'[s]\w+', str(err.instance), message) + message = message.replace("$iData", str(err.instance)) else: message = err.message From 42e2d6e6d90a4091244146f3c88acbff3b924cd6 Mon Sep 17 00:00:00 2001 From: aggle-baggle <101641447+aggle-baggle@users.noreply.github.com> Date: Fri, 10 Feb 2023 17:09:20 +0000 Subject: [PATCH 5/6] Adding tests for custom error handling Added tests for custom error handling --- Dockerfile | 1 + schema_enforcer/schemas/jsonschema.py | 12 +-- .../hostvars/can-vancouver-hosts/hosts.yml | 10 +++ .../hostvars/chi-beijing-hosts/hosts.yml | 10 +++ .../hostvars/eng-london-hosts/hosts.yml | 12 +++ .../schema/definitions/arrays/hosts.yml | 5 ++ .../schema/definitions/objects/hosts.yml | 5 ++ .../schema/schemas/hosts.yml | 25 ++++++ tests/test_custom_error_messages.py | 81 +++++++++++++++++++ 9 files changed, 155 insertions(+), 6 deletions(-) create mode 100644 tests/fixtures/test_custom_errors/hostvars/can-vancouver-hosts/hosts.yml create mode 100644 tests/fixtures/test_custom_errors/hostvars/chi-beijing-hosts/hosts.yml create mode 100644 tests/fixtures/test_custom_errors/hostvars/eng-london-hosts/hosts.yml create mode 100644 tests/fixtures/test_custom_errors/schema/definitions/arrays/hosts.yml create mode 100644 tests/fixtures/test_custom_errors/schema/definitions/objects/hosts.yml create mode 100644 tests/fixtures/test_custom_errors/schema/schemas/hosts.yml create mode 100644 tests/test_custom_error_messages.py diff --git a/Dockerfile b/Dockerfile index c5801a4..4874218 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,6 +9,7 @@ WORKDIR /local # Poetry fails install without README.md being copied. COPY pyproject.toml poetry.lock README.md /local/ COPY schema_enforcer /local/schema_enforcer +COPY tests/ /local/tests/ RUN poetry config virtualenvs.create false \ && poetry install --no-interaction --no-ansi diff --git a/schema_enforcer/schemas/jsonschema.py b/schema_enforcer/schemas/jsonschema.py index 69a81cb..914bc78 100644 --- a/schema_enforcer/schemas/jsonschema.py +++ b/schema_enforcer/schemas/jsonschema.py @@ -58,14 +58,14 @@ def validate(self, data, strict=False): for err in validator.iter_errors(data): has_error = True + err_message = err.message - if 'errMessage' in err.schema: - message = err.schema['errMessage'] - message = message.replace("$iData", str(err.instance)) - else: - message = err.message + # Custom error message handling + if len(err.absolute_path) > 0 and 'err_message' in err.schema: + err_message = str(err.schema['err_message']) + err_message = err_message.replace('$instance', str(err.instance)) - self.add_validation_error(message, absolute_path=list(err.absolute_path)) + self.add_validation_error(err_message, absolute_path=list(err.absolute_path)) if not has_error: self.add_validation_pass() diff --git a/tests/fixtures/test_custom_errors/hostvars/can-vancouver-hosts/hosts.yml b/tests/fixtures/test_custom_errors/hostvars/can-vancouver-hosts/hosts.yml new file mode 100644 index 0000000..5bf213a --- /dev/null +++ b/tests/fixtures/test_custom_errors/hostvars/can-vancouver-hosts/hosts.yml @@ -0,0 +1,10 @@ +--- +hosts: + - name: host-1 + - name: host-2 + aliases: + - test-host-2 + - name: host-3 + aliases: + - 3 + - test-host-three diff --git a/tests/fixtures/test_custom_errors/hostvars/chi-beijing-hosts/hosts.yml b/tests/fixtures/test_custom_errors/hostvars/chi-beijing-hosts/hosts.yml new file mode 100644 index 0000000..b27acbd --- /dev/null +++ b/tests/fixtures/test_custom_errors/hostvars/chi-beijing-hosts/hosts.yml @@ -0,0 +1,10 @@ +--- +hosts: + - name: host-1 + - name: host-2 + aliases: + - test-host-2 + - name: host-3 + aliases: + - test-host-3 + - test-host-three diff --git a/tests/fixtures/test_custom_errors/hostvars/eng-london-hosts/hosts.yml b/tests/fixtures/test_custom_errors/hostvars/eng-london-hosts/hosts.yml new file mode 100644 index 0000000..2f66e6e --- /dev/null +++ b/tests/fixtures/test_custom_errors/hostvars/eng-london-hosts/hosts.yml @@ -0,0 +1,12 @@ +--- +hosts: + - name: host-1 + - name: host-2 + aliases: + - test-host-2 + - name: host-3 + aliases: + - test-host-3 + - test-host-three +hosts_2: + - name: host-1-2 diff --git a/tests/fixtures/test_custom_errors/schema/definitions/arrays/hosts.yml b/tests/fixtures/test_custom_errors/schema/definitions/arrays/hosts.yml new file mode 100644 index 0000000..561a65c --- /dev/null +++ b/tests/fixtures/test_custom_errors/schema/definitions/arrays/hosts.yml @@ -0,0 +1,5 @@ +--- +aliases: + type: "array" + items: + $ref: "file://../objects/hosts.yml#host_name" diff --git a/tests/fixtures/test_custom_errors/schema/definitions/objects/hosts.yml b/tests/fixtures/test_custom_errors/schema/definitions/objects/hosts.yml new file mode 100644 index 0000000..c419018 --- /dev/null +++ b/tests/fixtures/test_custom_errors/schema/definitions/objects/hosts.yml @@ -0,0 +1,5 @@ +--- +host_name: + type: "string" + pattern: ^(?!-)[a-zA-Z0-9-]{1,63}(?!-)$ + err_message: "'$instance' is not valid. Please see docs." diff --git a/tests/fixtures/test_custom_errors/schema/schemas/hosts.yml b/tests/fixtures/test_custom_errors/schema/schemas/hosts.yml new file mode 100644 index 0000000..f511fcb --- /dev/null +++ b/tests/fixtures/test_custom_errors/schema/schemas/hosts.yml @@ -0,0 +1,25 @@ +$schema: "http://json-schema.org/draft-07/schema#" +$id: "schemas/host_names" +type: "object" +additionalProperties: false +err_message: This should be ignored. +properties: + hosts: + type: "array" + items: + type: "object" + additionalProperties: false + required: [ "name" ] + properties: + name: + type: "string" + pattern: ^(?!-)[a-zA-Z0-9-]{1,63}(?!-)$ + err_message: "'$instance' is not valid. Please see docs." + aliases: + type: "array" + items: + type: "string" + pattern: ^(?!-)[a-zA-Z0-9-]{1,63}(?!-)$ + err_message: "'$instance' is not valid. Please see docs." +required: + - "hosts" diff --git a/tests/test_custom_error_messages.py b/tests/test_custom_error_messages.py new file mode 100644 index 0000000..6359f42 --- /dev/null +++ b/tests/test_custom_error_messages.py @@ -0,0 +1,81 @@ +# pylint: disable=redefined-outer-name +"""Tests to validate functions defined in jsonschema.py""" +import os +import pytest + +from schema_enforcer.schemas.jsonschema import JsonSchema +from schema_enforcer.validation import RESULT_PASS, RESULT_FAIL +from schema_enforcer.utils import load_file + +FIXTURES_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "fixtures", "test_custom_errors") +LOADED_SCHEMA_DATA = load_file(os.path.join(FIXTURES_DIR, "schema", "schemas", "hosts.yml")) + + +@pytest.fixture +def schema_instance(): + """JSONSchema schema instance.""" + schema_instance = JsonSchema( + schema=LOADED_SCHEMA_DATA, + filename="hosts.yml", + root=os.path.join(FIXTURES_DIR, "schema", "schemas"), + ) + return schema_instance + + +@pytest.fixture +def valid_instance_data(): + """Valid instance data loaded from YAML file.""" + return load_file(os.path.join(FIXTURES_DIR, "hostvars", "chi-beijing-hosts", "hosts.yml")) + + +@pytest.fixture +def invalid_instance_data(): + """Invalid instance data loaded from YAML file.""" + return load_file(os.path.join(FIXTURES_DIR, "hostvars", "can-vancouver-hosts", "hosts.yml")) + + +@pytest.fixture +def invalid_instance_data_root(): + """Invalid instance data with a root level error. Loaded from YAML file.""" + return load_file(os.path.join(FIXTURES_DIR, "hostvars", "eng-london-hosts", "hosts.yml")) + + +class TestCustomErrors: + """Tests custom error message handling in jsonschema.py""" + + @staticmethod + def test_validate(schema_instance, valid_instance_data, invalid_instance_data, invalid_instance_data_root): + """Tests validate method of JsonSchema class + + Args: + schema_instance (JsonSchema): Instance of JsonSchema class + """ + schema_instance.validate(data=valid_instance_data) + validation_results = schema_instance.get_results() + assert len(validation_results) == 1 + assert validation_results[0].schema_id == LOADED_SCHEMA_DATA.get("$id") + assert validation_results[0].result == RESULT_PASS + assert validation_results[0].message is None + schema_instance.clear_results() + + schema_instance.validate(data=invalid_instance_data) + validation_results = schema_instance.get_results() + assert len(validation_results) == 1 + assert validation_results[0].schema_id == LOADED_SCHEMA_DATA.get("$id") + assert validation_results[0].result == RESULT_FAIL + assert validation_results[0].message == "'3' is not valid. Please see docs." + assert validation_results[0].absolute_path == ["hosts", "2", "aliases", "0"] + schema_instance.clear_results() + + # Custom error message at root level should be ignored + schema_instance.validate(data=invalid_instance_data_root) + validation_results = schema_instance.get_results() + assert len(validation_results) == 1 + assert validation_results[0].schema_id == LOADED_SCHEMA_DATA.get("$id") + assert validation_results[0].result == RESULT_FAIL + assert ( + validation_results[0].message + == "Additional properties are not allowed ('hosts_2' was unexpected)" + ) + assert validation_results[0].absolute_path == [] + schema_instance.clear_results() From b113e8960929185cc8b0d99db9ee937ced9c189d Mon Sep 17 00:00:00 2001 From: aggle-baggle <101641447+aggle-baggle@users.noreply.github.com> Date: Fri, 10 Feb 2023 17:44:01 +0000 Subject: [PATCH 6/6] Update README.md Add snippet on using custom errors --- README.md | 57 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/README.md b/README.md index 1ada744..ed5f237 100755 --- a/README.md +++ b/README.md @@ -200,6 +200,63 @@ pip install 'jsonschema[rfc3987]' See the "Validating Formats" section in the [jsonschema documentation](https://github.com/python-jsonschema/jsonschema/blob/main/docs/validate.rst) for more information. +### Custom Error Message Support + +Schema enforcer is able to handle and return custom error messages that are defined in the JSON Schema itself using reserved keywords. These custom error messages override the default messages returned by JSON Schema. For example consider the following JSON Schema: + +``` +type: object +additionalProperties: false +properties: + data: + type: object + properties: + name: + $ref: "#/$defs/regex_sub_schema" + aliases: + type: array + items: + $ref: "#/$defs/regex_sub_schema" + +$defs: + regex_sub_schema: + type: string + pattern: regex + err_message: "'$instance' is not valid. Please see docs." +``` +When validating against test data with invalid entries for `name` and `aliases`, the custom error is returned: + +``` +FAIL | [ERROR] 'host' is not valid. Please see docs. [FILE] .//test.yml [PROPERTY] data:name +FAIL | [ERROR] 'alias-1' is not valid. Please see docs. [FILE] .//test.yml [PROPERTY] data:aliases:0 +FAIL | [ERROR] 'alias-2' is not valid. Please see docs. [FILE] .//test.yml [PROPERTY] data:aliases:1 +``` +The error messages returned contain the instance data that was being validated at the time of failure. This is achieved by using the `$instance` variable within the custom error message. All occurences of this variable will be replaced with the instance data being validated. + +Please bear in mind that custom errors defined at the `root` level of the schema are ignored. For example if we add a custom error at the root level: + +``` +type: object +additionalProperties: false +err_message: This error message will be ignored. +properties: + data: + [...] +``` +Then validate against data with an additional property at the root level, for example: +``` +data: + name: host + aliases: + - etc... +data2: + name: host2 +``` +The error returned will be the default JSON Schema error: +``` +FAIL | [ERROR] Additional properties are not allowed ('data2' was unexpected) [FILE] .//test.yml [PROPERTY] +``` + ### Where To Go Next Detailed documentation can be found in the README.md files inside of the `docs/` directory.