diff --git a/docs/test.markdown b/docs/test.markdown index b77bed87..a2323222 100644 --- a/docs/test.markdown +++ b/docs/test.markdown @@ -43,7 +43,12 @@ To create a test definition, you must write JSON documents that look like this: "data": { "type": 1 } - } + }, + { + "description": "Load from an external file, relative to the test", + "valid": true, + "dataPath": "../my-data.json" + }, ] } ``` diff --git a/src/command_test.cc b/src/command_test.cc index ec28a8c4..f4e049f7 100644 --- a/src/command_test.cc +++ b/src/command_test.cc @@ -2,8 +2,9 @@ #include #include -#include // EXIT_SUCCESS, EXIT_FAILURE -#include // std::cerr, std::cout +#include // EXIT_SUCCESS, EXIT_FAILURE +#include // std::filesystem +#include // std::cerr, std::cout #include "command.h" #include "utils.h" @@ -33,6 +34,28 @@ get_schema_object(const sourcemeta::jsontoolkit::URI &identifier, return std::nullopt; } +static auto +get_data(const sourcemeta::jsontoolkit::JSON &test_case, + const std::filesystem::path &base) -> sourcemeta::jsontoolkit::JSON { + assert(base.is_absolute()); + assert(test_case.is_object()); + assert(test_case.defines("data") || test_case.defines("dataPath")); + if (test_case.defines("data")) { + return test_case.at("data"); + } + + assert(test_case.defines("dataPath")); + assert(test_case.at("dataPath").is_string()); + const std::filesystem::path data_path{base / + test_case.at("dataPath").to_string()}; + try { + return sourcemeta::jsontoolkit::from_file(data_path); + } catch (...) { + std::cout << "\n"; + throw; + } +} + auto intelligence::jsonschema::cli::test( const std::span &arguments) -> int { const auto options{parse_options(arguments, {"h", "http"})}; @@ -146,9 +169,31 @@ auto intelligence::jsonschema::cli::test( return EXIT_FAILURE; } - if (!test_case.defines("data")) { - std::cout << "\nerror: Test case documents must contain a `data` " - "property\n at test case #" + if (!test_case.defines("data") && !test_case.defines("dataPath")) { + std::cout << "\nerror: Test case documents must contain a `data` or " + "`dataPath` property\n at test case #" + << index << "\n\n"; + std::cout << "Learn more here: " + "https://github.com/Intelligence-AI/jsonschema/blob/main/" + "docs/test.markdown\n"; + return EXIT_FAILURE; + } + + if (test_case.defines("data") && test_case.defines("dataPath")) { + std::cout + << "\nerror: Test case documents must contain either a `data` or " + "`dataPath` property, but not both\n at test case #" + << index << "\n\n"; + std::cout << "Learn more here: " + "https://github.com/Intelligence-AI/jsonschema/blob/main/" + "docs/test.markdown\n"; + return EXIT_FAILURE; + } + + if (test_case.defines("dataPath") && + !test_case.at("dataPath").is_string()) { + std::cout << "\nerror: Test case documents must set the `dataPath` " + "property to a string\n at test case #" << index << "\n\n"; std::cout << "Learn more here: " "https://github.com/Intelligence-AI/jsonschema/blob/main/" @@ -189,7 +234,7 @@ auto intelligence::jsonschema::cli::test( std::ostringstream error; const auto case_result{sourcemeta::jsontoolkit::evaluate( - schema_template, test_case.at("data"), + schema_template, get_data(test_case, entry.first.parent_path()), sourcemeta::jsontoolkit::SchemaCompilerEvaluationMode::Fast, pretty_evaluate_callback(error, {"$ref"}))}; diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index bc3ec1fc..5362e916 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -82,7 +82,9 @@ add_jsonschema_test_unix(test/fail_no_tests) add_jsonschema_test_unix(test/fail_tests_non_array) add_jsonschema_test_unix(test/fail_test_case_non_object) add_jsonschema_test_unix(test/fail_test_case_no_data) +add_jsonschema_test_unix(test/fail_test_case_data_and_data_path) add_jsonschema_test_unix(test/fail_test_case_non_string_description) +add_jsonschema_test_unix(test/fail_test_case_non_string_data_path) add_jsonschema_test_unix(test/fail_test_case_no_valid) add_jsonschema_test_unix(test/fail_test_case_non_boolean_valid) add_jsonschema_test_unix(test/fail_true_resolve_fragment) @@ -95,6 +97,8 @@ add_jsonschema_test_unix(test/pass_single_resolve_fragment_verbose) add_jsonschema_test_unix(test/pass_single_comment_verbose) add_jsonschema_test_unix(test/pass_single_no_description_verbose) add_jsonschema_test_unix(test/pass_single_no_test_description_verbose) +add_jsonschema_test_unix(test/pass_single_data_path) +add_jsonschema_test_unix(test/pass_single_data_path_verbose) add_jsonschema_test_unix(test/pass_multi_directory_resolve) add_jsonschema_test_unix(test/pass_multi_directory_resolve_verbose) diff --git a/test/test/fail_test_case_data_and_data_path.sh b/test/test/fail_test_case_data_and_data_path.sh new file mode 100755 index 00000000..7e7ad206 --- /dev/null +++ b/test/test/fail_test_case_data_and_data_path.sh @@ -0,0 +1,35 @@ +#!/bin/sh + +set -o errexit +set -o nounset + +TMP="$(mktemp -d)" +clean() { rm -rf "$TMP"; } +trap clean EXIT + +cat << 'EOF' > "$TMP/test.json" +{ + "target": "http://json-schema.org/draft-04/schema#", + "tests": [ + { + "valid": true, + "data": {}, + "dataPath": "./foo.json" + } + ] +} +EOF + +"$1" test "$TMP/test.json" 1> "$TMP/output.txt" 2>&1 \ + && CODE="$?" || CODE="$?" +test "$CODE" = "1" || exit 1 + +cat << EOF > "$TMP/expected.txt" +$(realpath "$TMP")/test.json: +error: Test case documents must contain either a \`data\` or \`dataPath\` property, but not both + at test case #1 + +Learn more here: https://github.com/Intelligence-AI/jsonschema/blob/main/docs/test.markdown +EOF + +diff "$TMP/output.txt" "$TMP/expected.txt" diff --git a/test/test/fail_test_case_no_data.sh b/test/test/fail_test_case_no_data.sh index 0d6e1aaa..a8e5420b 100755 --- a/test/test/fail_test_case_no_data.sh +++ b/test/test/fail_test_case_no_data.sh @@ -32,7 +32,7 @@ test "$CODE" = "1" || exit 1 cat << EOF > "$TMP/expected.txt" $(realpath "$TMP")/test.json: -error: Test case documents must contain a \`data\` property +error: Test case documents must contain a \`data\` or \`dataPath\` property at test case #3 Learn more here: https://github.com/Intelligence-AI/jsonschema/blob/main/docs/test.markdown diff --git a/test/test/fail_test_case_non_string_data_path.sh b/test/test/fail_test_case_non_string_data_path.sh new file mode 100755 index 00000000..a30edee2 --- /dev/null +++ b/test/test/fail_test_case_non_string_data_path.sh @@ -0,0 +1,34 @@ +#!/bin/sh + +set -o errexit +set -o nounset + +TMP="$(mktemp -d)" +clean() { rm -rf "$TMP"; } +trap clean EXIT + +cat << 'EOF' > "$TMP/test.json" +{ + "target": "http://json-schema.org/draft-04/schema#", + "tests": [ + { + "valid": true, + "dataPath": 1 + } + ] +} +EOF + +"$1" test "$TMP/test.json" 1> "$TMP/output.txt" 2>&1 \ + && CODE="$?" || CODE="$?" +test "$CODE" = "1" || exit 1 + +cat << EOF > "$TMP/expected.txt" +$(realpath "$TMP")/test.json: +error: Test case documents must set the \`dataPath\` property to a string + at test case #1 + +Learn more here: https://github.com/Intelligence-AI/jsonschema/blob/main/docs/test.markdown +EOF + +diff "$TMP/output.txt" "$TMP/expected.txt" diff --git a/test/test/pass_single_data_path.sh b/test/test/pass_single_data_path.sh new file mode 100755 index 00000000..d25cbb31 --- /dev/null +++ b/test/test/pass_single_data_path.sh @@ -0,0 +1,50 @@ +#!/bin/sh + +set -o errexit +set -o nounset + +TMP="$(mktemp -d)" +clean() { rm -rf "$TMP"; } +trap clean EXIT + +cat << 'EOF' > "$TMP/schema.json" +{ + "id": "https://example.com", + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "string" +} +EOF + +cat << 'EOF' > "$TMP/data-valid.json" +"Hello World" +EOF + +cat << 'EOF' > "$TMP/data-invalid.json" +{ "type": "Hello World" } +EOF + +cat << 'EOF' > "$TMP/test.json" +{ + "target": "https://example.com", + "tests": [ + { + "description": "First test", + "valid": true, + "dataPath": "./data-valid.json" + }, + { + "description": "Second test", + "valid": false, + "dataPath": "./data-invalid.json" + } + ] +} +EOF + +"$1" test "$TMP/test.json" --resolve "$TMP/schema.json" 1> "$TMP/output.txt" 2>&1 + +cat << EOF > "$TMP/expected.txt" +$(realpath "$TMP")/test.json: PASS 2/2 +EOF + +diff "$TMP/output.txt" "$TMP/expected.txt" diff --git a/test/test/pass_single_data_path_verbose.sh b/test/test/pass_single_data_path_verbose.sh new file mode 100755 index 00000000..6b295756 --- /dev/null +++ b/test/test/pass_single_data_path_verbose.sh @@ -0,0 +1,53 @@ +#!/bin/sh + +set -o errexit +set -o nounset + +TMP="$(mktemp -d)" +clean() { rm -rf "$TMP"; } +trap clean EXIT + +cat << 'EOF' > "$TMP/schema.json" +{ + "id": "https://example.com", + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "string" +} +EOF + +cat << 'EOF' > "$TMP/data-valid.json" +"Hello World" +EOF + +cat << 'EOF' > "$TMP/data-invalid.json" +{ "type": "Hello World" } +EOF + +cat << 'EOF' > "$TMP/test.json" +{ + "target": "https://example.com", + "tests": [ + { + "description": "First test", + "valid": true, + "dataPath": "./data-valid.json" + }, + { + "description": "Second test", + "valid": false, + "dataPath": "./data-invalid.json" + } + ] +} +EOF + +"$1" test "$TMP/test.json" --resolve "$TMP/schema.json" --verbose 1> "$TMP/output.txt" 2>&1 + +cat << EOF > "$TMP/expected.txt" +Importing schema into the resolution context: $(realpath "$TMP")/schema.json +$(realpath "$TMP")/test.json: + 1/2 PASS First test + 2/2 PASS Second test +EOF + +diff "$TMP/output.txt" "$TMP/expected.txt"