From b86a762814d1a3b32db66e6ffd88b8ebebb78a53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anders=20Fredrik=20Ki=C3=A6r?= <31612826+anders-kiaer@users.noreply.github.com> Date: Fri, 21 Aug 2020 21:30:33 +0200 Subject: [PATCH] Automatically create JSON/YAML schema (#265) --- .github/workflows/webviz-config.yml | 1 + INTRODUCTION.md | 14 +++ examples/basic_example.yaml | 9 -- setup.py | 11 +-- tests/test_portable.py | 1 - tests/test_schema.py | 17 ++++ webviz_config/_docs/_build_docs.py | 33 +++---- webviz_config/_docs/_create_schema.py | 109 +++++++++++++++++++++++ webviz_config/command_line.py | 25 ++++++ webviz_config/templates/README.md.jinja2 | 4 +- 10 files changed, 191 insertions(+), 33 deletions(-) create mode 100644 tests/test_schema.py create mode 100644 webviz_config/_docs/_create_schema.py diff --git a/.github/workflows/webviz-config.yml b/.github/workflows/webviz-config.yml index 83aa72eb..3ca77cf1 100644 --- a/.github/workflows/webviz-config.yml +++ b/.github/workflows/webviz-config.yml @@ -76,6 +76,7 @@ jobs: webviz preferences --theme default pytest ./tests --headless --forked webviz docs --portable ./docs_build --skip-open + webviz schema - name: 🚢 Build and deploy Python package if: github.event_name == 'release' && matrix.python-version == '3.6' && matrix.pandas-version == '1.*' diff --git a/INTRODUCTION.md b/INTRODUCTION.md index 65081e5c..014de787 100644 --- a/INTRODUCTION.md +++ b/INTRODUCTION.md @@ -99,3 +99,17 @@ runs. E.g. ```bash webviz preferences --theme equinor --browser firefox ``` + +#### YAML schema + +By running `webviz schema` you will get a YAML (or technically, a JSON) schema which you can use in text editors, which then will +help you with auto-completion, detect mistakes immediately, and get hover description on different plugins. + +If you are using Visual Studio Code, we recommend [Red Hat's YAML extension](https://marketplace.visualstudio.com/items?itemName=redhat.vscode-yaml). After installing the extension, and adding something like +```json +{ + ... + "yaml.schemas": { "file:///some/path/to/your/webviz_schema.json": ["*webviz*.yml", "*webviz*.yaml"]} +} +``` +to your `settings.json` file, you will get help from the editor on YAML files following the namepatterns to the right (might have to restart the editor after updating the settings). diff --git a/examples/basic_example.yaml b/examples/basic_example.yaml index c81af37f..5ba78bd9 100644 --- a/examples/basic_example.yaml +++ b/examples/basic_example.yaml @@ -33,10 +33,6 @@ pages: - SyntaxHighlighter: filename: ./basic_example.yaml - - title: Tour example - content: - - ExampleTour: - - title: Plot a table content: - TablePlotter: @@ -82,8 +78,3 @@ pages: name: Kari Nordmann phone: 12345678 email: someother@email.com - - - title: Example portable - content: - - ExamplePortable: - some_number: 42 diff --git a/setup.py b/setup.py index e5a5d102..59b18b1a 100644 --- a/setup.py +++ b/setup.py @@ -4,13 +4,14 @@ LONG_DESCRIPTION = fh.read() TESTS_REQUIRES = [ - "pylint~=2.3", - "selenium~=3.141", - "mock", - "pytest-xdist", - "black", "bandit", + "black", + "jsonschema", + "mock", "mypy", + "pylint~=2.3", + "pytest-xdist", + "selenium~=3.141", ] setup( diff --git a/tests/test_portable.py b/tests/test_portable.py index d2ff2522..349c048e 100644 --- a/tests/test_portable.py +++ b/tests/test_portable.py @@ -29,7 +29,6 @@ def test_portable(dash_duo, tmp_path): "table_example", "pdf_example", "syntax_highlighting_example", - "tour_example", "plot_a_table", "last_page", ]: diff --git a/tests/test_schema.py b/tests/test_schema.py new file mode 100644 index 00000000..5bf96fad --- /dev/null +++ b/tests/test_schema.py @@ -0,0 +1,17 @@ +import pathlib + +import yaml +import jsonschema + +from webviz_config._docs._create_schema import create_schema + + +def test_schema(): + """Tests both that the generated schema is valid, + and that the input configuration is valid according to the schema. + """ + + config = yaml.safe_load( + (pathlib.Path("examples") / "basic_example.yaml").read_text() + ) + jsonschema.validate(instance=config, schema=create_schema()) diff --git a/webviz_config/_docs/_build_docs.py b/webviz_config/_docs/_build_docs.py index 5bcb774d..ec1dff8e 100644 --- a/webviz_config/_docs/_build_docs.py +++ b/webviz_config/_docs/_build_docs.py @@ -26,8 +26,15 @@ from webviz_config._config_parser import SPECIAL_ARGS +class ArgInfo(TypedDict, total=False): + required: bool + default: Any + typehint: Any + typehint_string: str + + class PluginInfo(TypedDict): - arg_strings: Dict[str, str] + arg_info: Dict[str, ArgInfo] argument_description: Optional[str] data_input: Optional[str] description: Optional[str] @@ -52,7 +59,7 @@ def _document_plugin(plugin: Tuple[str, Any]) -> PluginInfo: top_package_name = subpackage.split(".")[0] # type: ignore plugin_info: PluginInfo = { - "arg_strings": {arg: "" for arg in argspec.args if arg not in SPECIAL_ARGS}, + "arg_info": {arg: {} for arg in argspec.args if arg not in SPECIAL_ARGS}, "argument_description": docstring_parts[1] if len(docstring_parts) > 1 else None, @@ -65,27 +72,21 @@ def _document_plugin(plugin: Tuple[str, Any]) -> PluginInfo: "package_version": pkg_resources.get_distribution(top_package_name).version, } - # Add default value and the string '# Optional' to plugin - # arguments with default values: if argspec.defaults is not None: for arg, default in dict( zip(reversed(argspec.args), reversed(argspec.defaults)) ).items(): - if default == "": - default = "''" - plugin_info["arg_strings"][arg] = f"{default} # Optional." + plugin_info["arg_info"][arg]["default"] = default - # ...and for the other arguments add '# Required': - for arg, string in plugin_info["arg_strings"].items(): - if string == "": - plugin_info["arg_strings"][arg] = " # Required." + for arg, arg_info in plugin_info["arg_info"].items(): + arg_info["required"] = "default" not in arg_info - # Add a human readable type hint (for arguments with type annotation): for arg, annotation in argspec.annotations.items(): - if arg in plugin_info["arg_strings"]: - plugin_info["arg_strings"][ - arg - ] += f" Type {_annotation_to_string(annotation)}." + if arg not in SPECIAL_ARGS: + plugin_info["arg_info"][arg]["typehint"] = annotation + plugin_info["arg_info"][arg]["typehint_string"] = _annotation_to_string( + annotation + ) return plugin_info diff --git a/webviz_config/_docs/_create_schema.py b/webviz_config/_docs/_create_schema.py new file mode 100644 index 00000000..9c24122f --- /dev/null +++ b/webviz_config/_docs/_create_schema.py @@ -0,0 +1,109 @@ +import copy +import pathlib +from typing import Any + +from ._build_docs import get_plugin_documentation + + +JSON_SCHEMA = { + "$id": "https://github.com/equinor/webviz-config", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Person", + "type": "object", + "properties": { + "title": {"description": "Title of your Webviz application.", "type": "string"}, + "shared_settings": {"type": "object"}, + "pages": { + "description": "Define the pages in your Webviz application.", + "type": "array", + "minLength": 1, + "items": { + "type": "object", + "properties": { + "title": {"description": "Title of the page", "type": "string"}, + "content": { + "description": "Content on the page", + "type": "array", + "items": {"oneOf": [{"type": "string"},]}, + }, + }, + "required": ["title", "content"], + "additionalProperties": False, + }, + }, + }, + "required": ["title", "pages"], + "additionalProperties": False, +} + +CONTACT_PERSON = { + "contact_person": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "email": {"type": "string"}, + "phone": {"type": ["integer", "string"]}, + }, + "additionalProperties": False, + } +} + + +def create_schema() -> dict: + def json_type(typehint: Any) -> dict: + # pylint: disable=too-many-return-statements + if typehint == list: + return {"type": "array", "items": {}} + if typehint in (str, pathlib.Path): + return {"type": "string"} + if typehint == bool: + return {"type": "boolean"} + if typehint == int: + return {"type": "integer"} + if typehint == float: + return {"type": "number"} + if typehint == dict: + return {"type": "object"} + return {} + + json_schema = copy.deepcopy(JSON_SCHEMA) + + # fmt: off + content_schemas = json_schema["properties"]["pages"][ # type: ignore + "items"]["properties"]["content"]["items"]["oneOf"] + # fmt: on + + for package_doc in get_plugin_documentation().values(): + for plugin_doc in package_doc["plugins"]: + content_schemas.append( + { + "type": "object", + "properties": { + plugin_doc["name"]: { + "description": plugin_doc["description"], + "type": "object", + "properties": { + **{ + arg_name: json_type(arg_info.get("typehint")) + for arg_name, arg_info in plugin_doc[ + "arg_info" + ].items() + }, + **CONTACT_PERSON, + }, + "required": [ + arg_name + for arg_name, arg_info in plugin_doc["arg_info"].items() + if arg_info["required"] + ], + "additionalProperties": False, + } + }, + "required": [plugin_doc["name"]], + "additionalProperties": False, + "minProperties": 1, + "maxProperties": 1, + } + ) + + return json_schema diff --git a/webviz_config/command_line.py b/webviz_config/command_line.py index 23b20cdf..17550132 100644 --- a/webviz_config/command_line.py +++ b/webviz_config/command_line.py @@ -1,9 +1,12 @@ +import json import argparse import pathlib from ._build_webviz import build_webviz from .certificate._certificate_generator import create_ca from ._docs.open_docs import open_docs +from ._docs._create_schema import create_schema +from ._user_data_dir import user_data_dir from ._user_preferences import set_user_preferences, get_user_preference @@ -143,6 +146,28 @@ def entrypoint_preferences(args: argparse.Namespace) -> None: parser_preferences.set_defaults(func=entrypoint_preferences) + # Add "schema" parser: + + parser_schema = subparsers.add_parser( + "schema", + help="Create YAML (JSON) schema for webviz configuration " + "file (including all installed plugins)", + ) + + parser_schema.add_argument( + "--output", + type=pathlib.Path, + default=user_data_dir() / "webviz_schema.json", + help="Name of output JSON schema file. If not given, " + "it will be stored in your Webviz application settings folder.", + ) + + def entrypoint_schema(args: argparse.Namespace) -> None: + args.output.write_text(json.dumps(create_schema(), indent=4)) + print(f"Schema written to {args.output}") + + parser_schema.set_defaults(func=entrypoint_schema) + # Do the argument parsing: args = parser.parse_args() diff --git a/webviz_config/templates/README.md.jinja2 b/webviz_config/templates/README.md.jinja2 index a1604e25..737aafa0 100644 --- a/webviz_config/templates/README.md.jinja2 +++ b/webviz_config/templates/README.md.jinja2 @@ -31,8 +31,8 @@ ```yaml - {{ plugin["name"] }}: - {%- for arg, string in plugin["arg_strings"].items() %} - {{ arg }}: {{ string }} + {%- for arg, arg_info in plugin["arg_info"].items() %} + {{ arg }}: {{ arg_info["default"] | tojson if "default" in arg_info else "" }} # {{ "Required" if arg_info["required"] else "Optional" }}{{ ", type " + arg_info["typehint_string"] | string if "typehint_string" in arg_info }}. {%- endfor %} ```