Skip to content

Commit

Permalink
[WIP] Revise and re-test the validate command
Browse files Browse the repository at this point in the history
Signed-off-by: Juan Cruz Viotti <[email protected]>
  • Loading branch information
jviotti committed Jun 19, 2024
1 parent 9fd98e8 commit 114dfa6
Show file tree
Hide file tree
Showing 46 changed files with 1,019 additions and 229 deletions.
41 changes: 24 additions & 17 deletions src/command_validate.cc
Original file line number Diff line number Diff line change
Expand Up @@ -32,33 +32,40 @@ auto intelligence::jsonschema::cli::validate(
return EXIT_FAILURE;
}

CLI_ENSURE(options.at("").size() >= 2, "You must pass an instance")
const auto &schema_path{options.at("").at(0)};
const auto custom_resolver{
resolver(options, options.contains("h") || options.contains("http"))};

const auto schema{sourcemeta::jsontoolkit::from_file(schema_path)};

if (!sourcemeta::jsontoolkit::is_schema(schema)) {
std::cerr << "error: The schema file you provided does not represent a "
"valid JSON Schema\n "
<< std::filesystem::canonical(schema_path).string() << "\n";
return EXIT_FAILURE;
}

bool result{true};
if (options.at("").size() >= 2) {
const auto &instance_path{options.at("").at(1)};
const auto schema_template{sourcemeta::jsontoolkit::compile(
schema, sourcemeta::jsontoolkit::default_schema_walker, custom_resolver,
sourcemeta::jsontoolkit::default_schema_compiler)};
const auto &instance_path{options.at("").at(1)};
const auto schema_template{sourcemeta::jsontoolkit::compile(
schema, sourcemeta::jsontoolkit::default_schema_walker, custom_resolver,
sourcemeta::jsontoolkit::default_schema_compiler)};

const auto instance{sourcemeta::jsontoolkit::from_file(instance_path)};
const auto instance{sourcemeta::jsontoolkit::from_file(instance_path)};

std::ostringstream error;
result = sourcemeta::jsontoolkit::evaluate(
schema_template, instance,
sourcemeta::jsontoolkit::SchemaCompilerEvaluationMode::Fast,
pretty_evaluate_callback(error));
std::ostringstream error;
result = sourcemeta::jsontoolkit::evaluate(
schema_template, instance,
sourcemeta::jsontoolkit::SchemaCompilerEvaluationMode::Fast,
pretty_evaluate_callback(error));

if (result) {
log_verbose(options) << "Valid\n";
} else {
std::cerr << error.str();
}
if (result) {
log_verbose(options)
<< "ok: " << std::filesystem::weakly_canonical(instance_path).string()
<< "\n matches "
<< std::filesystem::weakly_canonical(schema_path).string() << "\n";
} else {
std::cerr << error.str();
}

return result ? EXIT_SUCCESS : EXIT_FAILURE;
Expand Down
44 changes: 36 additions & 8 deletions src/main.cc
Original file line number Diff line number Diff line change
Expand Up @@ -95,24 +95,52 @@ auto main(int argc, char *argv[]) noexcept -> int {
argv + argc};
return jsonschema_main(program, command, arguments);
} catch (const sourcemeta::jsontoolkit::SchemaReferenceError &error) {
std::cerr << error.what() << ": " << error.id() << "\n";
std::cerr << " at schema location \"";
std::cerr << "error: " << error.what() << "\n " << error.id()
<< "\n at schema location \"";
sourcemeta::jsontoolkit::stringify(error.location(), std::cerr);
std::cerr << "\"\n";
return EXIT_FAILURE;
} catch (const sourcemeta::jsontoolkit::SchemaResolutionError &error) {
std::cerr << error.what() << ": " << error.id() << "\n";
std::cerr << "error: " << error.what() << "\n at " << error.id() << "\n";
return EXIT_FAILURE;
} catch (const sourcemeta::jsontoolkit::SchemaVocabularyError &error) {
std::cerr << "error: " << error.what() << "\n " << error.uri()
<< "\n\nTo request support for it, please open an issue "
"at\nhttps://github.com/intelligence-ai/jsonschema\n";
return EXIT_FAILURE;
} catch (const sourcemeta::jsontoolkit::FileParseError &error) {
std::cerr << error.path().string() << "\n " << error.what() << " at line "
<< error.line() << " and column " << error.column() << "\n";
std::cerr << "error: " << error.what() << " at line " << error.line()
<< " and column " << error.column() << "\n "
<< std::filesystem::weakly_canonical(error.path()).string()
<< "\n";
return EXIT_FAILURE;
} catch (const sourcemeta::jsontoolkit::ParseError &error) {
std::cerr << error.what() << " at line " << error.line() << " and column "
<< error.column() << "\n";
std::cerr << "error: " << error.what() << " at line " << error.line()
<< " and column " << error.column() << "\n";
return EXIT_FAILURE;
} catch (const std::filesystem::filesystem_error &error) {
// See https://en.cppreference.com/w/cpp/error/errc
if (error.code() == std::errc::no_such_file_or_directory) {
std::cerr << "error: " << error.code().message() << "\n "
<< std::filesystem::weakly_canonical(error.path1()).string()
<< "\n";
} else if (error.code() == std::errc::is_a_directory) {
std::cerr << "error: The input was supposed to be a file but it is a "
"directory\n "
<< std::filesystem::weakly_canonical(error.path1()).string()
<< "\n";
} else {
std::cerr << "error: " << error.what() << "\n";
}

return EXIT_FAILURE;
} catch (const std::runtime_error &error) {
std::cerr << "error: " << error.what() << "\n";
return EXIT_FAILURE;
} catch (const std::exception &error) {
std::cerr << "Error: " << error.what() << "\n";
std::cerr << "unexpected error: " << error.what()
<< "\nPlease report it at "
<< "https://github.com/intelligence-ai/jsonschema\n";
return EXIT_FAILURE;
}
}
18 changes: 9 additions & 9 deletions src/utils.cc
Original file line number Diff line number Diff line change
Expand Up @@ -187,11 +187,11 @@ auto pretty_evaluate_callback(std::ostringstream &output)
}

output << "error: " << sourcemeta::jsontoolkit::describe(step) << "\n";
output << " at instance location \"";
output << " at instance location \"";
sourcemeta::jsontoolkit::stringify(instance_location, output);
output << "\"\n";

output << " at evaluate path \"";
output << " at evaluate path \"";
sourcemeta::jsontoolkit::stringify(evaluate_path, output);
output << "\"\n";
};
Expand Down Expand Up @@ -219,15 +219,13 @@ static auto fallback_resolver(
return promise.get_future();
}

log_verbose(options) << "Attempting to fetch over HTTP: " << identifier
<< "\n";
log_verbose(options) << "Resolving over HTTP: " << identifier << "\n";
sourcemeta::hydra::http::ClientRequest request{std::string{identifier}};
request.method(sourcemeta::hydra::http::Method::GET);
sourcemeta::hydra::http::ClientResponse response{request.send().get()};
if (response.status() != sourcemeta::hydra::http::Status::OK) {
std::ostringstream error;
error << "Failed to fetch " << identifier
<< " over HTTP. Got status code: " << response.status();
error << response.status() << "\n at " << identifier;
throw std::runtime_error(error.str());
}

Expand All @@ -239,7 +237,7 @@ static auto fallback_resolver(
auto resolver(const std::map<std::string, std::vector<std::string>> &options,
const bool remote) -> sourcemeta::jsontoolkit::SchemaResolver {
sourcemeta::jsontoolkit::MapSchemaResolver dynamic_resolver{
[&remote, &options](std::string_view identifier) {
[remote, &options](std::string_view identifier) {
if (remote) {
return fallback_resolver(options, identifier);
} else {
Expand All @@ -251,7 +249,8 @@ auto resolver(const std::map<std::string, std::vector<std::string>> &options,
for (const auto &entry :
for_each_json(options.at("resolve"), parse_ignore(options),
parse_extensions(options))) {
log_verbose(options) << "Loading schema: " << entry.first << "\n";
log_verbose(options) << "Importing schema into the resolution context: "
<< entry.first.string() << "\n";
dynamic_resolver.add(entry.second);
}
}
Expand All @@ -260,7 +259,8 @@ auto resolver(const std::map<std::string, std::vector<std::string>> &options,
for (const auto &entry :
for_each_json(options.at("r"), parse_ignore(options),
parse_extensions(options))) {
log_verbose(options) << "Loading schema: " << entry.first << "\n";
log_verbose(options) << "Importing schema into the resolution context: "
<< entry.first.string() << "\n";
dynamic_resolver.add(entry.second);
}
}
Expand Down
45 changes: 34 additions & 11 deletions test/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -26,17 +26,6 @@ add_jsonschema_test_unix(format_directory_ignore_directory)
add_jsonschema_test_unix(format_directory_ignore_file)
add_jsonschema_test_unix(format_check_single_invalid)
add_jsonschema_test_unix(frame)
add_jsonschema_test_unix(validate_pass_draft4)
add_jsonschema_test_unix(validate_fail_draft4)
add_jsonschema_test_unix(validate_pass_draft6)
add_jsonschema_test_unix(validate_fail_draft6)
add_jsonschema_test_unix(validate_pass_draft7)
add_jsonschema_test_unix(validate_fail_draft7)
add_jsonschema_test_unix(validate_fail_remote_no_http)
add_jsonschema_test_unix(validate_fail_invalid_ref)
add_jsonschema_test_unix(validate_fail_no_schema)
add_jsonschema_test_unix(validate_fail_no_instance)
add_jsonschema_test_unix(validate_non_supported)
add_jsonschema_test_unix(bundle_non_remote)
add_jsonschema_test_unix(bundle_into_resolve_directory)
add_jsonschema_test_unix(bundle_remote_single_schema)
Expand All @@ -55,5 +44,39 @@ add_jsonschema_test_unix(metaschema_fail_non_schema)
add_jsonschema_test_unix(metaschema_pass_cwd)
add_jsonschema_test_unix(metaschema_pass_single)

# Validate
add_jsonschema_test_unix(validate/fail_instance_directory)
add_jsonschema_test_unix(validate/fail_instance_enoent)
add_jsonschema_test_unix(validate/fail_instance_invalid_json)
add_jsonschema_test_unix(validate/fail_invalid_ref)
add_jsonschema_test_unix(validate/fail_no_instance)
add_jsonschema_test_unix(validate/fail_no_schema)
add_jsonschema_test_unix(validate/fail_relative_external_ref_missing)
add_jsonschema_test_unix(validate/fail_resolve_enoent)
add_jsonschema_test_unix(validate/fail_resolve_directory_with_invalid_json)
add_jsonschema_test_unix(validate/fail_resolve_invalid_json)
add_jsonschema_test_unix(validate/fail_schema_directory)
add_jsonschema_test_unix(validate/fail_schema_enoent)
add_jsonschema_test_unix(validate/fail_schema_invalid_json)
add_jsonschema_test_unix(validate/fail_schema_non_schema)
add_jsonschema_test_unix(validate/fail_schema_unknown_dialect)
add_jsonschema_test_unix(validate/fail_schema_unsupported_dialect)
add_jsonschema_test_unix(validate/pass_resolve)
add_jsonschema_test_unix(validate/pass_resolve_custom_extension)
add_jsonschema_test_unix(validate/pass_resolve_verbose)
add_jsonschema_test_unix(validate/pass_verbose)
add_jsonschema_test_unix(validate/pass_draft4)
add_jsonschema_test_unix(validate/pass_draft6)
add_jsonschema_test_unix(validate/pass_draft7)
add_jsonschema_test_unix(validate/fail_draft4)
add_jsonschema_test_unix(validate/fail_draft6)
add_jsonschema_test_unix(validate/fail_draft7)

# CI specific tests
add_jsonschema_test_unix_ci(bundle_remote_http)
add_jsonschema_test_unix_ci(fail_validate_http_non_200)
add_jsonschema_test_unix_ci(fail_validate_http_non_200_verbose)
add_jsonschema_test_unix_ci(fail_validate_http_non_schema)
add_jsonschema_test_unix_ci(fail_validate_http_non_schema_verbose)
add_jsonschema_test_unix_ci(pass_validate_http)
add_jsonschema_test_unix_ci(pass_validate_http_verbose)
30 changes: 30 additions & 0 deletions test/ci/fail_validate_http_non_200.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
#!/bin/sh

set -o errexit
set -o nounset

TMP="$(mktemp -d)"
clean() { rm -rf "$TMP"; }
trap clean EXIT

cat << 'EOF' > "$TMP/schema.json"
{
"$schema": "http://json-schema.org/draft-07/schema#",
"allOf": [ { "$ref": "https://example.com" } ]
}
EOF

cat << 'EOF' > "$TMP/instance.json"
{ "type": "string" }
EOF

"$1" validate "$TMP/schema.json" "$TMP/instance.json" --http 2> "$TMP/stderr.txt" \
&& CODE="$?" || CODE="$?"
test "$CODE" = "1" || exit 1

cat << EOF > "$TMP/expected.txt"
error: 400 Bad Request
at https://example.com
EOF

diff "$TMP/stderr.txt" "$TMP/expected.txt"
31 changes: 31 additions & 0 deletions test/ci/fail_validate_http_non_200_verbose.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
#!/bin/sh

set -o errexit
set -o nounset

TMP="$(mktemp -d)"
clean() { rm -rf "$TMP"; }
trap clean EXIT

cat << 'EOF' > "$TMP/schema.json"
{
"$schema": "http://json-schema.org/draft-07/schema#",
"allOf": [ { "$ref": "https://example.com" } ]
}
EOF

cat << 'EOF' > "$TMP/instance.json"
{ "type": "string" }
EOF

"$1" validate "$TMP/schema.json" "$TMP/instance.json" --http --verbose 2> "$TMP/stderr.txt" \
&& CODE="$?" || CODE="$?"
test "$CODE" = "1" || exit 1

cat << EOF > "$TMP/expected.txt"
Resolving over HTTP: https://example.com
error: 400 Bad Request
at https://example.com
EOF

diff "$TMP/stderr.txt" "$TMP/expected.txt"
30 changes: 30 additions & 0 deletions test/ci/fail_validate_http_non_schema.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
#!/bin/sh

set -o errexit
set -o nounset

TMP="$(mktemp -d)"
clean() { rm -rf "$TMP"; }
trap clean EXIT

cat << 'EOF' > "$TMP/schema.json"
{
"$schema": "http://json-schema.org/draft-07/schema#",
"allOf": [ { "$ref": "https://jsonplaceholder.typicode.com/todos/1" } ]
}
EOF

cat << 'EOF' > "$TMP/instance.json"
{ "type": "string" }
EOF

"$1" validate "$TMP/schema.json" "$TMP/instance.json" --http 2> "$TMP/stderr.txt" \
&& CODE="$?" || CODE="$?"
test "$CODE" = "1" || exit 1

cat << EOF > "$TMP/expected.txt"
error: The JSON document is not a valid JSON Schema
at https://jsonplaceholder.typicode.com/todos/1
EOF

diff "$TMP/stderr.txt" "$TMP/expected.txt"
31 changes: 31 additions & 0 deletions test/ci/fail_validate_http_non_schema_verbose.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
#!/bin/sh

set -o errexit
set -o nounset

TMP="$(mktemp -d)"
clean() { rm -rf "$TMP"; }
trap clean EXIT

cat << 'EOF' > "$TMP/schema.json"
{
"$schema": "http://json-schema.org/draft-07/schema#",
"allOf": [ { "$ref": "https://jsonplaceholder.typicode.com/todos/1" } ]
}
EOF

cat << 'EOF' > "$TMP/instance.json"
{ "type": "string" }
EOF

"$1" validate "$TMP/schema.json" "$TMP/instance.json" --http --verbose 2> "$TMP/stderr.txt" \
&& CODE="$?" || CODE="$?"
test "$CODE" = "1" || exit 1

cat << EOF > "$TMP/expected.txt"
Resolving over HTTP: https://jsonplaceholder.typicode.com/todos/1
error: The JSON document is not a valid JSON Schema
at https://jsonplaceholder.typicode.com/todos/1
EOF

diff "$TMP/stderr.txt" "$TMP/expected.txt"
26 changes: 26 additions & 0 deletions test/ci/pass_validate_http.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
#!/bin/sh

set -o errexit
set -o nounset

TMP="$(mktemp -d)"
clean() { rm -rf "$TMP"; }
trap clean EXIT

cat << 'EOF' > "$TMP/schema.json"
{
"$schema": "http://json-schema.org/draft-07/schema#",
"allOf": [ { "$ref": "https://json.schemastore.org/mocharc.json" } ]
}
EOF

cat << 'EOF' > "$TMP/instance.json"
{ "exit": true }
EOF

"$1" validate "$TMP/schema.json" "$TMP/instance.json" --http 2> "$TMP/stderr.txt"

cat << EOF > "$TMP/expected.txt"
EOF

diff "$TMP/stderr.txt" "$TMP/expected.txt"
Loading

0 comments on commit 114dfa6

Please sign in to comment.