Improve output and error messages from the validate command

DEPENDENCIES

@@ -1,4 +1,4 @@
 vendorpull https://github.com/sourcemeta/vendorpull dea311b5bfb53b6926a4140267959ae334d3ecf4
 noa https://github.com/sourcemeta/noa 2bc3138b80e575786bec418c91fc2058c6836993
-jsontoolkit https://github.com/sourcemeta/jsontoolkit 0e2ac8987382685ad6dc50ca33d2e43a6701b023
+jsontoolkit https://github.com/sourcemeta/jsontoolkit 7c229e4243290ad255bba4f3775492147f3972ce
 hydra https://github.com/sourcemeta/hydra 3c53d3fdef79e9ba603d48470a508cc45472a0dc

src/command_validate.cc

@@ -32,33 +32,40 @@ auto intelligence::jsonschema::cli::validate(
     return EXIT_FAILURE;
   }
 
-  CLI_ENSURE(options.at("").size() >= 2, "You must pass an instance")
   const auto &schema_path{options.at("").at(0)};
   const auto custom_resolver{
       resolver(options, options.contains("h") || options.contains("http"))};
 
   const auto schema{sourcemeta::jsontoolkit::from_file(schema_path)};
 
+  if (!sourcemeta::jsontoolkit::is_schema(schema)) {
+    std::cerr << "error: The schema file you provided does not represent a "
+                 "valid JSON Schema\n  "
+              << std::filesystem::canonical(schema_path).string() << "\n";
+    return EXIT_FAILURE;
+  }
+
   bool result{true};
-  if (options.at("").size() >= 2) {
-    const auto &instance_path{options.at("").at(1)};
-    const auto schema_template{sourcemeta::jsontoolkit::compile(
-        schema, sourcemeta::jsontoolkit::default_schema_walker, custom_resolver,
-        sourcemeta::jsontoolkit::default_schema_compiler)};
+  const auto &instance_path{options.at("").at(1)};
+  const auto schema_template{sourcemeta::jsontoolkit::compile(
+      schema, sourcemeta::jsontoolkit::default_schema_walker, custom_resolver,
+      sourcemeta::jsontoolkit::default_schema_compiler)};
 
-    const auto instance{sourcemeta::jsontoolkit::from_file(instance_path)};
+  const auto instance{sourcemeta::jsontoolkit::from_file(instance_path)};
 
-    std::ostringstream error;
-    result = sourcemeta::jsontoolkit::evaluate(
-        schema_template, instance,
-        sourcemeta::jsontoolkit::SchemaCompilerEvaluationMode::Fast,
-        pretty_evaluate_callback(error));
+  std::ostringstream error;
+  result = sourcemeta::jsontoolkit::evaluate(
+      schema_template, instance,
+      sourcemeta::jsontoolkit::SchemaCompilerEvaluationMode::Fast,
+      pretty_evaluate_callback(error));
 
-    if (result) {
-      log_verbose(options) << "Valid\n";
-    } else {
-      std::cerr << error.str();
-    }
+  if (result) {
+    log_verbose(options)
+        << "ok: " << std::filesystem::weakly_canonical(instance_path).string()
+        << "\n  matches "
+        << std::filesystem::weakly_canonical(schema_path).string() << "\n";
+  } else {
+    std::cerr << error.str();
   }
 
   return result ? EXIT_SUCCESS : EXIT_FAILURE;

src/main.cc

@@ -95,24 +95,52 @@ auto main(int argc, char *argv[]) noexcept -> int {
                                              argv + argc};
     return jsonschema_main(program, command, arguments);
   } catch (const sourcemeta::jsontoolkit::SchemaReferenceError &error) {
-    std::cerr << error.what() << ": " << error.id() << "\n";
-    std::cerr << "    at schema location \"";
+    std::cerr << "error: " << error.what() << "\n  " << error.id()
+              << "\n    at schema location \"";
     sourcemeta::jsontoolkit::stringify(error.location(), std::cerr);
     std::cerr << "\"\n";
     return EXIT_FAILURE;
   } catch (const sourcemeta::jsontoolkit::SchemaResolutionError &error) {
-    std::cerr << error.what() << ": " << error.id() << "\n";
+    std::cerr << "error: " << error.what() << "\n  at " << error.id() << "\n";
+    return EXIT_FAILURE;
+  } catch (const sourcemeta::jsontoolkit::SchemaVocabularyError &error) {
+    std::cerr << "error: " << error.what() << "\n  " << error.uri()
+              << "\n\nTo request support for it, please open an issue "
+                 "at\nhttps://github.com/intelligence-ai/jsonschema\n";
     return EXIT_FAILURE;
   } catch (const sourcemeta::jsontoolkit::FileParseError &error) {
-    std::cerr << error.path().string() << "\n  " << error.what() << " at line "
-              << error.line() << " and column " << error.column() << "\n";
+    std::cerr << "error: " << error.what() << " at line " << error.line()
+              << " and column " << error.column() << "\n  "
+              << std::filesystem::weakly_canonical(error.path()).string()
+              << "\n";
     return EXIT_FAILURE;
   } catch (const sourcemeta::jsontoolkit::ParseError &error) {
-    std::cerr << error.what() << " at line " << error.line() << " and column "
-              << error.column() << "\n";
+    std::cerr << "error: " << error.what() << " at line " << error.line()
+              << " and column " << error.column() << "\n";
+    return EXIT_FAILURE;
+  } catch (const std::filesystem::filesystem_error &error) {
+    // See https://en.cppreference.com/w/cpp/error/errc
+    if (error.code() == std::errc::no_such_file_or_directory) {
+      std::cerr << "error: " << error.code().message() << "\n  "
+                << std::filesystem::weakly_canonical(error.path1()).string()
+                << "\n";
+    } else if (error.code() == std::errc::is_a_directory) {
+      std::cerr << "error: The input was supposed to be a file but it is a "
+                   "directory\n  "
+                << std::filesystem::weakly_canonical(error.path1()).string()
+                << "\n";
+    } else {
+      std::cerr << "error: " << error.what() << "\n";
+    }
+
+    return EXIT_FAILURE;
+  } catch (const std::runtime_error &error) {
+    std::cerr << "error: " << error.what() << "\n";
     return EXIT_FAILURE;
   } catch (const std::exception &error) {
-    std::cerr << "Error: " << error.what() << "\n";
+    std::cerr << "unexpected error: " << error.what()
+              << "\nPlease report it at "
+              << "https://github.com/intelligence-ai/jsonschema\n";
     return EXIT_FAILURE;
   }
 }

src/utils.cc

@@ -187,11 +187,11 @@ auto pretty_evaluate_callback(std::ostringstream &output)
     }
 
     output << "error: " << sourcemeta::jsontoolkit::describe(step) << "\n";
-    output << "    at instance location \"";
+    output << "  at instance location \"";
     sourcemeta::jsontoolkit::stringify(instance_location, output);
     output << "\"\n";
 
-    output << "    at evaluate path \"";
+    output << "  at evaluate path \"";
     sourcemeta::jsontoolkit::stringify(evaluate_path, output);
     output << "\"\n";
   };
@@ -219,15 +219,13 @@ static auto fallback_resolver(
     return promise.get_future();
   }
 
-  log_verbose(options) << "Attempting to fetch over HTTP: " << identifier
-                       << "\n";
+  log_verbose(options) << "Resolving over HTTP: " << identifier << "\n";
   sourcemeta::hydra::http::ClientRequest request{std::string{identifier}};
   request.method(sourcemeta::hydra::http::Method::GET);
   sourcemeta::hydra::http::ClientResponse response{request.send().get()};
   if (response.status() != sourcemeta::hydra::http::Status::OK) {
     std::ostringstream error;
-    error << "Failed to fetch " << identifier
-          << " over HTTP. Got status code: " << response.status();
+    error << response.status() << "\n  at " << identifier;
     throw std::runtime_error(error.str());
   }
 
@@ -251,7 +249,8 @@ auto resolver(const std::map<std::string, std::vector<std::string>> &options,
     for (const auto &entry :
          for_each_json(options.at("resolve"), parse_ignore(options),
                        parse_extensions(options))) {
-      log_verbose(options) << "Loading schema: " << entry.first << "\n";
+      log_verbose(options) << "Importing schema into the resolution context: "
+                           << entry.first.string() << "\n";
       dynamic_resolver.add(entry.second);
     }
   }
@@ -260,7 +259,8 @@ auto resolver(const std::map<std::string, std::vector<std::string>> &options,
     for (const auto &entry :
          for_each_json(options.at("r"), parse_ignore(options),
                        parse_extensions(options))) {
-      log_verbose(options) << "Loading schema: " << entry.first << "\n";
+      log_verbose(options) << "Importing schema into the resolution context: "
+                           << entry.first.string() << "\n";
       dynamic_resolver.add(entry.second);
     }
   }

test/CMakeLists.txt

@@ -26,17 +26,6 @@ add_jsonschema_test_unix(format_directory_ignore_directory)
 add_jsonschema_test_unix(format_directory_ignore_file)
 add_jsonschema_test_unix(format_check_single_invalid)
 add_jsonschema_test_unix(frame)
-add_jsonschema_test_unix(validate_pass_draft4)
-add_jsonschema_test_unix(validate_fail_draft4)
-add_jsonschema_test_unix(validate_pass_draft6)
-add_jsonschema_test_unix(validate_fail_draft6)
-add_jsonschema_test_unix(validate_pass_draft7)
-add_jsonschema_test_unix(validate_fail_draft7)
-add_jsonschema_test_unix(validate_fail_remote_no_http)
-add_jsonschema_test_unix(validate_fail_invalid_ref)
-add_jsonschema_test_unix(validate_fail_no_schema)
-add_jsonschema_test_unix(validate_fail_no_instance)
-add_jsonschema_test_unix(validate_non_supported)
 add_jsonschema_test_unix(bundle_non_remote)
 add_jsonschema_test_unix(bundle_into_resolve_directory)
 add_jsonschema_test_unix(bundle_remote_single_schema)
@@ -55,5 +44,39 @@ add_jsonschema_test_unix(metaschema_fail_non_schema)
 add_jsonschema_test_unix(metaschema_pass_cwd)
 add_jsonschema_test_unix(metaschema_pass_single)
 
+# Validate
+add_jsonschema_test_unix(validate/fail_instance_directory)
+add_jsonschema_test_unix(validate/fail_instance_enoent)
+add_jsonschema_test_unix(validate/fail_instance_invalid_json)
+add_jsonschema_test_unix(validate/fail_invalid_ref)
+add_jsonschema_test_unix(validate/fail_no_instance)
+add_jsonschema_test_unix(validate/fail_no_schema)
+add_jsonschema_test_unix(validate/fail_relative_external_ref_missing)
+add_jsonschema_test_unix(validate/fail_resolve_enoent)
+add_jsonschema_test_unix(validate/fail_resolve_directory_with_invalid_json)
+add_jsonschema_test_unix(validate/fail_resolve_invalid_json)
+add_jsonschema_test_unix(validate/fail_schema_directory)
+add_jsonschema_test_unix(validate/fail_schema_enoent)
+add_jsonschema_test_unix(validate/fail_schema_invalid_json)
+add_jsonschema_test_unix(validate/fail_schema_non_schema)
+add_jsonschema_test_unix(validate/fail_schema_unknown_dialect)
+add_jsonschema_test_unix(validate/fail_schema_unsupported_dialect)
+add_jsonschema_test_unix(validate/pass_resolve)
+add_jsonschema_test_unix(validate/pass_resolve_custom_extension)
+add_jsonschema_test_unix(validate/pass_resolve_verbose)
+add_jsonschema_test_unix(validate/pass_verbose)
+add_jsonschema_test_unix(validate/pass_draft4)
+add_jsonschema_test_unix(validate/pass_draft6)
+add_jsonschema_test_unix(validate/pass_draft7)
+add_jsonschema_test_unix(validate/fail_draft4)
+add_jsonschema_test_unix(validate/fail_draft6)
+add_jsonschema_test_unix(validate/fail_draft7)
+
 # CI specific tests
 add_jsonschema_test_unix_ci(bundle_remote_http)
+add_jsonschema_test_unix_ci(fail_validate_http_non_200)
+add_jsonschema_test_unix_ci(fail_validate_http_non_200_verbose)
+add_jsonschema_test_unix_ci(fail_validate_http_non_schema)
+add_jsonschema_test_unix_ci(fail_validate_http_non_schema_verbose)
+add_jsonschema_test_unix_ci(pass_validate_http)
+add_jsonschema_test_unix_ci(pass_validate_http_verbose)

test/ci/fail_validate_http_non_200.sh

@@ -0,0 +1,30 @@
+#!/bin/sh
+
+set -o errexit
+set -o nounset
+
+TMP="$(mktemp -d)"
+clean() { rm -rf "$TMP"; }
+trap clean EXIT
+
+cat << 'EOF' > "$TMP/schema.json"
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "allOf": [ { "$ref": "https://example.com" } ]
+}
+EOF
+
+cat << 'EOF' > "$TMP/instance.json"
+{ "type": "string" }
+EOF
+
+"$1" validate "$TMP/schema.json" "$TMP/instance.json" --http 2> "$TMP/stderr.txt" \
+  && CODE="$?" || CODE="$?"
+test "$CODE" = "1" || exit 1
+
+cat << EOF > "$TMP/expected.txt"
+error: 400 Bad Request
+  at https://example.com
+EOF
+
+diff "$TMP/stderr.txt" "$TMP/expected.txt"

test/ci/fail_validate_http_non_200_verbose.sh

@@ -0,0 +1,31 @@
+#!/bin/sh
+
+set -o errexit
+set -o nounset
+
+TMP="$(mktemp -d)"
+clean() { rm -rf "$TMP"; }
+trap clean EXIT
+
+cat << 'EOF' > "$TMP/schema.json"
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "allOf": [ { "$ref": "https://example.com" } ]
+}
+EOF
+
+cat << 'EOF' > "$TMP/instance.json"
+{ "type": "string" }
+EOF
+
+"$1" validate "$TMP/schema.json" "$TMP/instance.json" --http --verbose 2> "$TMP/stderr.txt" \
+  && CODE="$?" || CODE="$?"
+test "$CODE" = "1" || exit 1
+
+cat << EOF > "$TMP/expected.txt"
+Resolving over HTTP: https://example.com
+error: 400 Bad Request
+  at https://example.com
+EOF
+
+diff "$TMP/stderr.txt" "$TMP/expected.txt"

test/ci/fail_validate_http_non_schema.sh

@@ -0,0 +1,30 @@
+#!/bin/sh
+
+set -o errexit
+set -o nounset
+
+TMP="$(mktemp -d)"
+clean() { rm -rf "$TMP"; }
+trap clean EXIT
+
+cat << 'EOF' > "$TMP/schema.json"
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "allOf": [ { "$ref": "https://jsonplaceholder.typicode.com/todos/1" } ]
+}
+EOF
+
+cat << 'EOF' > "$TMP/instance.json"
+{ "type": "string" }
+EOF
+
+"$1" validate "$TMP/schema.json" "$TMP/instance.json" --http 2> "$TMP/stderr.txt" \
+  && CODE="$?" || CODE="$?"
+test "$CODE" = "1" || exit 1
+
+cat << EOF > "$TMP/expected.txt"
+error: The JSON document is not a valid JSON Schema
+  at https://jsonplaceholder.typicode.com/todos/1
+EOF
+
+diff "$TMP/stderr.txt" "$TMP/expected.txt"

test/ci/fail_validate_http_non_schema_verbose.sh

@@ -0,0 +1,31 @@
+#!/bin/sh
+
+set -o errexit
+set -o nounset
+
+TMP="$(mktemp -d)"
+clean() { rm -rf "$TMP"; }
+trap clean EXIT
+
+cat << 'EOF' > "$TMP/schema.json"
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "allOf": [ { "$ref": "https://jsonplaceholder.typicode.com/todos/1" } ]
+}
+EOF
+
+cat << 'EOF' > "$TMP/instance.json"
+{ "type": "string" }
+EOF
+
+"$1" validate "$TMP/schema.json" "$TMP/instance.json" --http --verbose 2> "$TMP/stderr.txt" \
+  && CODE="$?" || CODE="$?"
+test "$CODE" = "1" || exit 1
+
+cat << EOF > "$TMP/expected.txt"
+Resolving over HTTP: https://jsonplaceholder.typicode.com/todos/1
+error: The JSON document is not a valid JSON Schema
+  at https://jsonplaceholder.typicode.com/todos/1
+EOF
+
+diff "$TMP/stderr.txt" "$TMP/expected.txt"

test/ci/pass_validate_http.sh

@@ -0,0 +1,26 @@
+#!/bin/sh
+
+set -o errexit
+set -o nounset
+
+TMP="$(mktemp -d)"
+clean() { rm -rf "$TMP"; }
+trap clean EXIT
+
+cat << 'EOF' > "$TMP/schema.json"
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "allOf": [ { "$ref": "https://json.schemastore.org/mocharc.json" } ]
+}
+EOF
+
+cat << 'EOF' > "$TMP/instance.json"
+{ "exit": true }
+EOF
+
+"$1" validate "$TMP/schema.json" "$TMP/instance.json" --http 2> "$TMP/stderr.txt"
+
+cat << EOF > "$TMP/expected.txt"
+EOF
+
+diff "$TMP/stderr.txt" "$TMP/expected.txt"