Automatically create JSON/YAML schema (#265)

.github/workflows/webviz-config.yml

@@ -76,6 +76,7 @@ jobs:
           webviz preferences --theme default
           pytest ./tests --headless --forked
           webviz docs --portable ./docs_build --skip-open
+          webviz schema
         
       - name: 🚢 Build and deploy Python package
         if: github.event_name == 'release' && matrix.python-version == '3.6' && matrix.pandas-version == '1.*'

INTRODUCTION.md

@@ -99,3 +99,17 @@ runs. E.g.
 ```bash
 webviz preferences --theme equinor --browser firefox
 ```
+
+#### YAML schema
+
+By running `webviz schema` you will get a YAML (or technically, a JSON) schema which you can use in text editors, which then will
+help you with auto-completion, detect mistakes immediately, and get hover description on different plugins.
+
+If you are using Visual Studio Code, we recommend [Red Hat's YAML extension](https://marketplace.visualstudio.com/items?itemName=redhat.vscode-yaml). After installing the extension, and adding something like
+```json
+{
+  ...
+  "yaml.schemas": { "file:///some/path/to/your/webviz_schema.json": ["*webviz*.yml", "*webviz*.yaml"]}
+}
+```
+to your `settings.json` file, you will get help from the editor on YAML files following the namepatterns to the right (might have to restart the editor after updating the settings).

examples/basic_example.yaml

@@ -33,10 +33,6 @@ pages:
     - SyntaxHighlighter:
         filename: ./basic_example.yaml
 
- - title: Tour example
-   content:
-    - ExampleTour:
-
  - title: Plot a table
    content:
     - TablePlotter:
@@ -82,8 +78,3 @@ pages:
           name: Kari Nordmann
           phone: 12345678
           email: [email protected]
-
- - title: Example portable
-   content:
-    - ExamplePortable:
-        some_number: 42

setup.py

@@ -4,13 +4,14 @@
     LONG_DESCRIPTION = fh.read()
 
 TESTS_REQUIRES = [
-    "pylint~=2.3",
-    "selenium~=3.141",
-    "mock",
-    "pytest-xdist",
-    "black",
     "bandit",
+    "black",
+    "jsonschema",
+    "mock",
     "mypy",
+    "pylint~=2.3",
+    "pytest-xdist",
+    "selenium~=3.141",
 ]
 
 setup(

tests/test_portable.py

@@ -29,7 +29,6 @@ def test_portable(dash_duo, tmp_path):
         "table_example",
         "pdf_example",
         "syntax_highlighting_example",
-        "tour_example",
         "plot_a_table",
         "last_page",
     ]:

tests/test_schema.py

@@ -0,0 +1,17 @@
+import pathlib
+
+import yaml
+import jsonschema
+
+from webviz_config._docs._create_schema import create_schema
+
+
+def test_schema():
+    """Tests both that the generated schema is valid,
+    and that the input configuration is valid according to the schema.
+    """
+
+    config = yaml.safe_load(
+        (pathlib.Path("examples") / "basic_example.yaml").read_text()
+    )
+    jsonschema.validate(instance=config, schema=create_schema())

webviz_config/_docs/_build_docs.py

@@ -26,8 +26,15 @@
 from webviz_config._config_parser import SPECIAL_ARGS
 
 
+class ArgInfo(TypedDict, total=False):
+    required: bool
+    default: Any
+    typehint: Any
+    typehint_string: str
+
+
 class PluginInfo(TypedDict):
-    arg_strings: Dict[str, str]
+    arg_info: Dict[str, ArgInfo]
     argument_description: Optional[str]
     data_input: Optional[str]
     description: Optional[str]
@@ -52,7 +59,7 @@ def _document_plugin(plugin: Tuple[str, Any]) -> PluginInfo:
     top_package_name = subpackage.split(".")[0]  # type: ignore
 
     plugin_info: PluginInfo = {
-        "arg_strings": {arg: "" for arg in argspec.args if arg not in SPECIAL_ARGS},
+        "arg_info": {arg: {} for arg in argspec.args if arg not in SPECIAL_ARGS},
         "argument_description": docstring_parts[1]
         if len(docstring_parts) > 1
         else None,
@@ -65,27 +72,21 @@ def _document_plugin(plugin: Tuple[str, Any]) -> PluginInfo:
         "package_version": pkg_resources.get_distribution(top_package_name).version,
     }
 
-    # Add default value and the string '# Optional' to plugin
-    # arguments with default values:
     if argspec.defaults is not None:
         for arg, default in dict(
             zip(reversed(argspec.args), reversed(argspec.defaults))
         ).items():
-            if default == "":
-                default = "''"
-            plugin_info["arg_strings"][arg] = f"{default}  # Optional."
+            plugin_info["arg_info"][arg]["default"] = default
 
-    # ...and for the other arguments add '# Required':
-    for arg, string in plugin_info["arg_strings"].items():
-        if string == "":
-            plugin_info["arg_strings"][arg] = "  # Required."
+    for arg, arg_info in plugin_info["arg_info"].items():
+        arg_info["required"] = "default" not in arg_info
 
-    # Add a human readable type hint (for arguments with type annotation):
     for arg, annotation in argspec.annotations.items():
-        if arg in plugin_info["arg_strings"]:
-            plugin_info["arg_strings"][
-                arg
-            ] += f" Type {_annotation_to_string(annotation)}."
+        if arg not in SPECIAL_ARGS:
+            plugin_info["arg_info"][arg]["typehint"] = annotation
+            plugin_info["arg_info"][arg]["typehint_string"] = _annotation_to_string(
+                annotation
+            )
 
     return plugin_info
 

webviz_config/_docs/_create_schema.py

@@ -0,0 +1,109 @@
+import copy
+import pathlib
+from typing import Any
+
+from ._build_docs import get_plugin_documentation
+
+
+JSON_SCHEMA = {
+    "$id": "https://github.com/equinor/webviz-config",
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "title": "Person",
+    "type": "object",
+    "properties": {
+        "title": {"description": "Title of your Webviz application.", "type": "string"},
+        "shared_settings": {"type": "object"},
+        "pages": {
+            "description": "Define the pages in your Webviz application.",
+            "type": "array",
+            "minLength": 1,
+            "items": {
+                "type": "object",
+                "properties": {
+                    "title": {"description": "Title of the page", "type": "string"},
+                    "content": {
+                        "description": "Content on the page",
+                        "type": "array",
+                        "items": {"oneOf": [{"type": "string"},]},
+                    },
+                },
+                "required": ["title", "content"],
+                "additionalProperties": False,
+            },
+        },
+    },
+    "required": ["title", "pages"],
+    "additionalProperties": False,
+}
+
+CONTACT_PERSON = {
+    "contact_person": {
+        "type": "object",
+        "properties": {
+            "name": {"type": "string"},
+            "email": {"type": "string"},
+            "phone": {"type": ["integer", "string"]},
+        },
+        "additionalProperties": False,
+    }
+}
+
+
+def create_schema() -> dict:
+    def json_type(typehint: Any) -> dict:
+        # pylint: disable=too-many-return-statements
+        if typehint == list:
+            return {"type": "array", "items": {}}
+        if typehint in (str, pathlib.Path):
+            return {"type": "string"}
+        if typehint == bool:
+            return {"type": "boolean"}
+        if typehint == int:
+            return {"type": "integer"}
+        if typehint == float:
+            return {"type": "number"}
+        if typehint == dict:
+            return {"type": "object"}
+        return {}
+
+    json_schema = copy.deepcopy(JSON_SCHEMA)
+
+    # fmt: off
+    content_schemas = json_schema["properties"]["pages"][  # type: ignore
+        "items"]["properties"]["content"]["items"]["oneOf"]
+    # fmt: on
+
+    for package_doc in get_plugin_documentation().values():
+        for plugin_doc in package_doc["plugins"]:
+            content_schemas.append(
+                {
+                    "type": "object",
+                    "properties": {
+                        plugin_doc["name"]: {
+                            "description": plugin_doc["description"],
+                            "type": "object",
+                            "properties": {
+                                **{
+                                    arg_name: json_type(arg_info.get("typehint"))
+                                    for arg_name, arg_info in plugin_doc[
+                                        "arg_info"
+                                    ].items()
+                                },
+                                **CONTACT_PERSON,
+                            },
+                            "required": [
+                                arg_name
+                                for arg_name, arg_info in plugin_doc["arg_info"].items()
+                                if arg_info["required"]
+                            ],
+                            "additionalProperties": False,
+                        }
+                    },
+                    "required": [plugin_doc["name"]],
+                    "additionalProperties": False,
+                    "minProperties": 1,
+                    "maxProperties": 1,
+                }
+            )
+
+    return json_schema

webviz_config/command_line.py

@@ -1,9 +1,12 @@
+import json
 import argparse
 import pathlib
 
 from ._build_webviz import build_webviz
 from .certificate._certificate_generator import create_ca
 from ._docs.open_docs import open_docs
+from ._docs._create_schema import create_schema
+from ._user_data_dir import user_data_dir
 from ._user_preferences import set_user_preferences, get_user_preference
 
 
@@ -143,6 +146,28 @@ def entrypoint_preferences(args: argparse.Namespace) -> None:
 
     parser_preferences.set_defaults(func=entrypoint_preferences)
 
+    # Add "schema" parser:
+
+    parser_schema = subparsers.add_parser(
+        "schema",
+        help="Create YAML (JSON) schema for webviz configuration "
+        "file (including all installed plugins)",
+    )
+
+    parser_schema.add_argument(
+        "--output",
+        type=pathlib.Path,
+        default=user_data_dir() / "webviz_schema.json",
+        help="Name of output JSON schema file. If not given, "
+        "it will be stored in your Webviz application settings folder.",
+    )
+
+    def entrypoint_schema(args: argparse.Namespace) -> None:
+        args.output.write_text(json.dumps(create_schema(), indent=4))
+        print(f"Schema written to {args.output}")
+
+    parser_schema.set_defaults(func=entrypoint_schema)
+
     # Do the argument parsing:
 
     args = parser.parse_args()

webviz_config/templates/README.md.jinja2

@@ -31,8 +31,8 @@
 
 ```yaml
     - {{ plugin["name"] }}:
-    {%- for arg, string in plugin["arg_strings"].items() %}
-        {{ arg }}: {{ string }}
+    {%- for arg, arg_info in plugin["arg_info"].items() %}
+        {{ arg }}: {{ arg_info["default"] | tojson if "default" in arg_info else "" }} # {{ "Required" if arg_info["required"] else "Optional" }}{{ ", type " + arg_info["typehint_string"] | string if "typehint_string" in arg_info }}.    
     {%- endfor %}
 ```