Revisit metaschema command tests (#160)

Signed-off-by: Juan Cruz Viotti <[email protected]>

src/command_metaschema.cc

@@ -23,7 +23,9 @@ auto sourcemeta::jsonschema::cli::metaschema(
   for (const auto &entry : for_each_json(options.at(""), parse_ignore(options),
                                          parse_extensions(options))) {
     if (!sourcemeta::jsontoolkit::is_schema(entry.second)) {
-      std::cerr << "Not a schema: " << entry.first.string() << "\n";
+      std::cerr << "error: The schema file you provided does not represent a "
+                   "valid JSON Schema\n  "
+                << std::filesystem::canonical(entry.first).string() << "\n";
       return EXIT_FAILURE;
     }
 
@@ -45,12 +47,13 @@ auto sourcemeta::jsonschema::cli::metaschema(
             sourcemeta::jsontoolkit::SchemaCompilerEvaluationMode::Fast,
             std::ref(output))) {
       log_verbose(options)
-          << entry.first.string()
-          << ": The schema is valid with respect to its metaschema\n";
+          << "ok: " << std::filesystem::weakly_canonical(entry.first).string()
+          << "\n  matches " << dialect.value() << "\n";
     } else {
+      std::cerr << "fail: "
+                << std::filesystem::weakly_canonical(entry.first).string()
+                << "\n";
       print(output, std::cerr);
-      std::cerr << entry.first.string()
-                << ": The schema is NOT valid with respect to its metaschema\n";
       result = false;
     }
   }

test/CMakeLists.txt

@@ -25,12 +25,6 @@ add_jsonschema_test_unix(format_check_single_pass)
 add_jsonschema_test_unix(format_directory_ignore_directory)
 add_jsonschema_test_unix(format_directory_ignore_file)
 add_jsonschema_test_unix(format_check_single_invalid)
-add_jsonschema_test_unix(metaschema_fail_directory)
-add_jsonschema_test_unix(metaschema_fail_single)
-add_jsonschema_test_unix(metaschema_fail_non_schema)
-add_jsonschema_test_unix(metaschema_pass_cwd)
-add_jsonschema_test_unix(metaschema_pass_single)
-add_jsonschema_test_unix(metaschema_pass_2020_12)
 
 # Validate
 add_jsonschema_test_unix(validate/fail_instance_directory)
@@ -76,6 +70,14 @@ add_jsonschema_test_unix(validate/pass_many_verbose)
 add_jsonschema_test_unix(validate/fail_many)
 add_jsonschema_test_unix(validate/fail_many_verbose)
 
+# Metaschema
+add_jsonschema_test_unix(metaschema/fail_directory)
+add_jsonschema_test_unix(metaschema/fail_single)
+add_jsonschema_test_unix(metaschema/fail_non_schema)
+add_jsonschema_test_unix(metaschema/pass_cwd)
+add_jsonschema_test_unix(metaschema/pass_single)
+add_jsonschema_test_unix(metaschema/pass_2020_12)
+
 # Test
 add_jsonschema_test_unix(test/fail_true_single_resolve)
 add_jsonschema_test_unix(test/fail_true_single_resolve_verbose)

test/metaschema/fail_directory.sh

@@ -0,0 +1,50 @@
+#!/bin/sh
+
+set -o errexit
+set -o nounset
+
+TMP="$(mktemp -d)"
+clean() { rm -rf "$TMP"; }
+trap clean EXIT
+
+mkdir "$TMP/schemas"
+
+cat << 'EOF' > "$TMP/schemas/schema_1.json"
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "type": "string"
+}
+EOF
+
+cat << 'EOF' > "$TMP/schemas/schema_2.json"
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "type": 1
+}
+EOF
+
+"$1" metaschema "$TMP/schemas" 2>"$TMP/stderr.txt" && CODE="$?" || CODE="$?"
+test "$CODE" = "1" || exit 1
+
+cat << EOF > "$TMP/expected.txt"
+fail: $(realpath "$TMP")/schemas/schema_2.json
+error: Schema validation failure
+  The string value "object" was expected to equal one of the following values: "array", "boolean", "integer", "null", "number", "object", and "string"
+    at instance location "/type"
+    at evaluate path "/properties/type/anyOf/0/\$ref/enum"
+  The string value was expected to validate against the statically referenced schema
+    at instance location "/type"
+    at evaluate path "/properties/type/anyOf/0/\$ref"
+  The value was expected to be of type array but it was of type string
+    at instance location "/type"
+    at evaluate path "/properties/type/anyOf/1/type"
+  The string value was expected to validate against at least one of the 2 given subschemas
+    at instance location "/type"
+    at evaluate path "/properties/type/anyOf"
+  The object value was expected to validate against the 33 defined properties subschemas
+    at instance location ""
+    at evaluate path "/properties"
+EOF
+
+diff "$TMP/stderr.txt" "$TMP/expected.txt"
+

test/metaschema_fail_non_schema.sh → test/metaschema/fail_non_schema.sh

@@ -12,16 +12,11 @@ cat << 'EOF' > "$TMP/document.json"
 EOF
 
 "$1" metaschema "$TMP/document.json" 2>"$TMP/stderr.txt" && CODE="$?" || CODE="$?"
-
-if [ "$CODE" = "0" ]
-then
-  echo "FAIL" 1>&2
-  exit 1
-fi
+test "$CODE" = "1" || exit 1
 
 cat << EOF > "$TMP/expected.txt"
-Not a schema: $(realpath "$TMP/document.json")
+error: The schema file you provided does not represent a valid JSON Schema
+  $(realpath "$TMP/document.json")
 EOF
 
 diff "$TMP/stderr.txt" "$TMP/expected.txt"
-echo "PASS" 1>&2

test/metaschema/fail_single.sh

@@ -0,0 +1,40 @@
+#!/bin/sh
+
+set -o errexit
+set -o nounset
+
+TMP="$(mktemp -d)"
+clean() { rm -rf "$TMP"; }
+trap clean EXIT
+
+cat << 'EOF' > "$TMP/schema.json"
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "type": 1
+}
+EOF
+
+"$1" metaschema "$TMP/schema.json" 2>"$TMP/stderr.txt" && CODE="$?" || CODE="$?"
+test "$CODE" = "1" || exit 1
+
+cat << EOF > "$TMP/expected.txt"
+fail: $(realpath "$TMP")/schema.json
+error: Schema validation failure
+  The string value "object" was expected to equal one of the following values: "array", "boolean", "integer", "null", "number", "object", and "string"
+    at instance location "/type"
+    at evaluate path "/properties/type/anyOf/0/\$ref/enum"
+  The string value was expected to validate against the statically referenced schema
+    at instance location "/type"
+    at evaluate path "/properties/type/anyOf/0/\$ref"
+  The value was expected to be of type array but it was of type string
+    at instance location "/type"
+    at evaluate path "/properties/type/anyOf/1/type"
+  The string value was expected to validate against at least one of the 2 given subschemas
+    at instance location "/type"
+    at evaluate path "/properties/type/anyOf"
+  The object value was expected to validate against the 33 defined properties subschemas
+    at instance location ""
+    at evaluate path "/properties"
+EOF
+
+diff "$TMP/stderr.txt" "$TMP/expected.txt"

test/metaschema_pass_cwd.sh → test/metaschema/pass_cwd.sh

@@ -21,4 +21,14 @@ cat << 'EOF' > "$TMP/schema_2.json"
 }
 EOF
 
-"$1" metaschema --verbose
+cd "$TMP"
+"$1" metaschema --verbose > "$TMP/output.txt" 2>&1
+
+cat << EOF > "$TMP/expected.txt"
+ok: $(realpath "$TMP")/schema_1.json
+  matches http://json-schema.org/draft-04/schema#
+ok: $(realpath "$TMP")/schema_2.json
+  matches http://json-schema.org/draft-04/schema#
+EOF
+
+diff "$TMP/output.txt" "$TMP/expected.txt"

test/metaschema_pass_single.sh → test/metaschema/pass_single.sh

@@ -14,4 +14,9 @@ cat << 'EOF' > "$TMP/schema.json"
 }
 EOF
 
-"$1" metaschema "$TMP/schema.json"
+"$1" metaschema "$TMP/schema.json" > "$TMP/output.txt" 2>&1
+
+cat << EOF > "$TMP/expected.txt"
+EOF
+
+diff "$TMP/output.txt" "$TMP/expected.txt"