From 8b821d672cb2d279a69cbcdd52cb65381acc6012 Mon Sep 17 00:00:00 2001 From: Josh Karpel Date: Fri, 28 Jun 2024 21:26:12 -0500 Subject: [PATCH 1/8] generate mermaid diagrams --- {examples => docs/examples}/dag.yaml | 0 docs/hooks/mermaid.py | 35 +++++++++++++++++++++ docs/triggers.md | 9 ++++++ mkdocs.yml | 13 ++++++-- synth.yaml | 2 +- synthesize/cli.py | 8 +++++ synthesize/config.py | 46 +++++++++++++++++++++------- synthesize/execution.py | 3 +- tests/test_config.py | 4 ++- 9 files changed, 103 insertions(+), 17 deletions(-) rename {examples => docs/examples}/dag.yaml (100%) create mode 100644 docs/hooks/mermaid.py create mode 100644 docs/triggers.md diff --git a/examples/dag.yaml b/docs/examples/dag.yaml similarity index 100% rename from examples/dag.yaml rename to docs/examples/dag.yaml diff --git a/docs/hooks/mermaid.py b/docs/hooks/mermaid.py new file mode 100644 index 0000000..089a10c --- /dev/null +++ b/docs/hooks/mermaid.py @@ -0,0 +1,35 @@ +import re +import subprocess + +from mkdocs.config.defaults import MkDocsConfig +from mkdocs.structure.files import Files +from mkdocs.structure.pages import Page + + +def on_page_markdown( + markdown: str, + page: Page, + config: MkDocsConfig, + files: Files, +) -> str: + lines = [] + for line in markdown.splitlines(): + if match := re.match(r"@mermaid\(([\w\.\/]+)\)", line): + lines.append("```mermaid") + cmd = subprocess.run( + ("synth", "--mermaid", "--config", match.group(1)), + capture_output=True, + text=True, + check=False, + ) + + if cmd.returncode != 0: + raise Exception(f"Error: {cmd.stdout}\n{cmd.stderr}") + else: + lines.append(cmd.stdout) + + lines.append("```") + else: + lines.append(line) + + return "\n".join(lines) diff --git a/docs/triggers.md b/docs/triggers.md new file mode 100644 index 0000000..a0471c1 --- /dev/null +++ b/docs/triggers.md @@ -0,0 +1,9 @@ +# Triggers + +## After + +```yaml +--8<-- "docs/examples/dag.yaml" +``` + +@mermaid(docs/examples/dag.yaml) diff --git a/mkdocs.yml b/mkdocs.yml index 7c41b9b..ae464e0 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -9,6 +9,7 @@ extra_css: watch: - synthesize/ + - docs/ theme: name: material @@ -53,6 +54,9 @@ plugins: - https://docs.python.org/3/objects.inv - https://rich.readthedocs.io/en/stable/objects.inv +hooks: + - docs/hooks/mermaid.py + markdown_extensions: - admonition - pymdownx.details @@ -60,9 +64,13 @@ markdown_extensions: anchor_linenums: true - pymdownx.inlinehilite - pymdownx.snippets: - base_path: ['docs'] + base_path: [''] check_paths: true - - pymdownx.superfences + - pymdownx.superfences: + custom_fences: + - name: mermaid + class: mermaid + format: !!python/name:pymdownx.superfences.fence_code_format - pymdownx.tabbed: alternate_style: true - attr_list @@ -80,4 +88,5 @@ extra: nav: - Introduction: index.md + - triggers.md - changelog.md diff --git a/synth.yaml b/synth.yaml index 560406f..2b4ef6c 100644 --- a/synth.yaml +++ b/synth.yaml @@ -31,5 +31,5 @@ triggers: paths: - synthesize/ - tests/ - - examples/ + - docs/examples/ - pyproject.toml diff --git a/synthesize/cli.py b/synthesize/cli.py index 17a6918..82c2800 100644 --- a/synthesize/cli.py +++ b/synthesize/cli.py @@ -37,6 +37,10 @@ def run( envvar="SYNTHFILE", help="The path to the configuration file to execute.", ), + mermaid: bool = Option( + default=False, + help="If enabled, output the flow as a Mermaid diagram, and don't run the flow.", + ), dry: bool = Option( default=False, help="If enabled, do not run actually run the flow.", @@ -81,6 +85,10 @@ def run( ) raise Exit(code=1) + if mermaid: + print(selected_flow.mermaid()) + return + if dry: return diff --git a/synthesize/config.py b/synthesize/config.py index 54b86e9..9e478a1 100644 --- a/synthesize/config.py +++ b/synthesize/config.py @@ -4,6 +4,7 @@ import shutil from collections.abc import Mapping from colorsys import hsv_to_rgb +from functools import cached_property from pathlib import Path from random import random from textwrap import dedent @@ -15,6 +16,7 @@ from rich.color import Color from synthesize.model import Model +from synthesize.utils import md5 Args = dict[ Annotated[ @@ -36,6 +38,12 @@ ], str, ] +ID = Annotated[ + str, + Field( + pattern=r"\w+", + ), +] def random_color() -> str: @@ -88,7 +96,7 @@ class Once(Model): class After(Model): type: Literal["after"] = "after" - after: Annotated[frozenset[str], Field(min_length=1)] + after: Annotated[tuple[str, ...], Field(min_length=1)] class Restart(Model): @@ -128,13 +136,17 @@ class FlowNode(Model): color: Annotated[str, Field(default_factory=random_color)] + @cached_property + def uid(self) -> str: + return md5(self.model_dump_json(exclude={"color"}).encode()) + class UnresolvedFlowNode(Model): - target: Target | str + target: Target | ID args: Args = {} envs: Envs = {} - trigger: AnyTrigger | str = Once() + trigger: AnyTrigger | ID = Once() color: Annotated[str, Field(default_factory=random_color)] @@ -155,20 +167,32 @@ def resolve( class Flow(Model): - nodes: dict[str, FlowNode] + nodes: dict[ID, FlowNode] args: Args = {} envs: Envs = {} + def mermaid(self) -> str: + lines = ["flowchart TD"] + + for id, node in self.nodes.items(): + lines.append(f"{node.id}({id})") + + if node.trigger.type == "after": + for after in node.trigger.after: + lines.append(f" {self.nodes[after].id} --> {node.id}") + + return "\n ".join(lines).strip() + class UnresolvedFlow(Model): - nodes: dict[str, UnresolvedFlowNode] + nodes: dict[ID, UnresolvedFlowNode] args: Args = {} envs: Envs = {} def resolve( self, - targets: Mapping[str, Target], - triggers: Mapping[str, AnyTrigger], + targets: Mapping[ID, Target], + triggers: Mapping[ID, AnyTrigger], ) -> Flow: return Flow( nodes={id: node.resolve(id, targets, triggers) for id, node in self.nodes.items()}, @@ -178,9 +202,9 @@ def resolve( class Config(Model): - flows: dict[str, UnresolvedFlow] = {} - targets: dict[str, Target] = {} - triggers: dict[str, AnyTrigger] = {} + flows: dict[ID, UnresolvedFlow] = {} + targets: dict[ID, Target] = {} + triggers: dict[ID, AnyTrigger] = {} @classmethod def from_file(cls, file: Path) -> Config: @@ -191,5 +215,5 @@ def from_file(cls, file: Path) -> Config: else: raise NotImplementedError("Currently, only YAML files are supported.") - def resolve(self) -> Mapping[str, Flow]: + def resolve(self) -> Mapping[ID, Flow]: return {id: flow.resolve(self.targets, self.triggers) for id, flow in self.flows.items()} diff --git a/synthesize/execution.py b/synthesize/execution.py index 9fb85b6..a8ea05a 100644 --- a/synthesize/execution.py +++ b/synthesize/execution.py @@ -12,11 +12,10 @@ from synthesize.config import Args, Envs, FlowNode from synthesize.messages import ExecutionCompleted, ExecutionOutput, ExecutionStarted, Message -from synthesize.utils import md5 def write_script(node: FlowNode, args: Args, tmp_dir: Path) -> Path: - path = tmp_dir / f"{node.id}-{md5(node.model_dump_json().encode())}" + path = tmp_dir / f"{node.id}-{node.uid}" path.parent.mkdir(parents=True, exist_ok=True) path.write_text( diff --git a/tests/test_config.py b/tests/test_config.py index 7fc1bf1..b8d1245 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -19,7 +19,9 @@ ) -@pytest.mark.parametrize("example", list((Path(__file__).parent.parent / "examples").iterdir())) +@pytest.mark.parametrize( + "example", list((Path(__file__).parent.parent / "docs" / "examples").iterdir()) +) def test_config_examples_parse(example: Path) -> None: Config.from_file(example) From ffeebfb8d1596f78764f7c0cb781fc83597e5010 Mon Sep 17 00:00:00 2001 From: Josh Karpel Date: Fri, 28 Jun 2024 22:02:37 -0500 Subject: [PATCH 2/8] display triggers --- docs/assets/style.css | 6 ++++++ docs/triggers.md | 12 +++++++++++- synthesize/config.py | 22 +++++++++++++++++++--- synthesize/utils.py | 4 ++-- 4 files changed, 38 insertions(+), 6 deletions(-) diff --git a/docs/assets/style.css b/docs/assets/style.css index 3a22901..45fda70 100644 --- a/docs/assets/style.css +++ b/docs/assets/style.css @@ -64,3 +64,9 @@ a.autorefs-external::after { a.autorefs-external:hover::after { background-color: var(--md-accent-fg-color); } + +/* Styles for SVG screenshots */ + +.mermaid { + text-align: center; +} diff --git a/docs/triggers.md b/docs/triggers.md index a0471c1..55b31ed 100644 --- a/docs/triggers.md +++ b/docs/triggers.md @@ -1,9 +1,19 @@ # Triggers -## After +## Trigger Types + +### After ```yaml --8<-- "docs/examples/dag.yaml" ``` @mermaid(docs/examples/dag.yaml) + +## Combining Triggers + +```yaml +--8<-- "synth.yaml" +``` + +@mermaid(synth.yaml) diff --git a/synthesize/config.py b/synthesize/config.py index 9e478a1..0637989 100644 --- a/synthesize/config.py +++ b/synthesize/config.py @@ -14,6 +14,7 @@ from jinja2 import Environment from pydantic import Field, field_validator from rich.color import Color +from typing_extensions import assert_never from synthesize.model import Model from synthesize.utils import md5 @@ -174,12 +175,27 @@ class Flow(Model): def mermaid(self) -> str: lines = ["flowchart TD"] + seen_watches = set() for id, node in self.nodes.items(): lines.append(f"{node.id}({id})") - if node.trigger.type == "after": - for after in node.trigger.after: - lines.append(f" {self.nodes[after].id} --> {node.id}") + match node.trigger: + case Once(): + pass + case After(after=after): + for a in after: + lines.append(f"{self.nodes[a].id} --> {node.id}") + case Restart(delay=delay): + lines.append(f"{node.id} -->|∞ {delay:.3g}s| {node.id}") + case Watch(paths=paths): + text = "\n".join(paths) + h = md5("".join(paths)) + if h not in seen_watches: + seen_watches.add(h) + lines.append(f'w_{h}[("{text}")]') + lines.append(f"w_{h} -->|👁| {node.id}") + case _: + assert_never(node.trigger) return "\n ".join(lines).strip() diff --git a/synthesize/utils.py b/synthesize/utils.py index 40af9d9..9c8bdd3 100644 --- a/synthesize/utils.py +++ b/synthesize/utils.py @@ -15,5 +15,5 @@ async def delayed() -> T: return create_task(delayed(), name=name) -def md5(data: bytes) -> str: - return hashlib.md5(data).hexdigest() +def md5(data: bytes | str) -> str: + return hashlib.md5(data if isinstance(data, bytes) else data.encode()).hexdigest() From 34d4a54e9ddfe109dc14eb908f560adc034ae651 Mon Sep 17 00:00:00 2001 From: Josh Karpel Date: Fri, 28 Jun 2024 22:09:37 -0500 Subject: [PATCH 3/8] add example for each trigger type --- docs/examples/{dag.yaml => after.yaml} | 2 +- docs/examples/once.yaml | 15 ++++++++++++++ docs/examples/restart.yaml | 19 +++++++++++++++++ docs/examples/watch.yaml | 19 +++++++++++++++++ docs/triggers.md | 28 ++++++++++++++++++++++++-- 5 files changed, 80 insertions(+), 3 deletions(-) rename docs/examples/{dag.yaml => after.yaml} (97%) create mode 100644 docs/examples/once.yaml create mode 100644 docs/examples/restart.yaml create mode 100644 docs/examples/watch.yaml diff --git a/docs/examples/dag.yaml b/docs/examples/after.yaml similarity index 97% rename from docs/examples/dag.yaml rename to docs/examples/after.yaml index 3a98ff4..eadb6e1 100644 --- a/docs/examples/dag.yaml +++ b/docs/examples/after.yaml @@ -23,5 +23,5 @@ flows: targets: sleep-and-echo: commands: | - sleep 2 + sleep 1 echo "Hi from {{ id }}!" diff --git a/docs/examples/once.yaml b/docs/examples/once.yaml new file mode 100644 index 0000000..bd35e4e --- /dev/null +++ b/docs/examples/once.yaml @@ -0,0 +1,15 @@ +flows: + default: + nodes: + 1: + target: sleep-and-echo + 2: + target: sleep-and-echo + 3: + target: sleep-and-echo + +targets: + sleep-and-echo: + commands: | + sleep 1 + echo "Hi from {{ id }}!" diff --git a/docs/examples/restart.yaml b/docs/examples/restart.yaml new file mode 100644 index 0000000..0659037 --- /dev/null +++ b/docs/examples/restart.yaml @@ -0,0 +1,19 @@ +flows: + default: + nodes: + 1: + target: sleep-and-echo + trigger: + type: restart + delay: 3 + 2: + target: sleep-and-echo + trigger: + type: restart + delay: 1 + +targets: + sleep-and-echo: + commands: | + sleep 1 + echo "Hi from {{ id }}!" diff --git a/docs/examples/watch.yaml b/docs/examples/watch.yaml new file mode 100644 index 0000000..ec22430 --- /dev/null +++ b/docs/examples/watch.yaml @@ -0,0 +1,19 @@ +flows: + default: + nodes: + 1: + target: sleep-and-echo + trigger: + type: watch + paths: ["synthesize/", "tests/"] + 2: + target: sleep-and-echo + trigger: + type: watch + paths: [ "docs/" ] + +targets: + sleep-and-echo: + commands: | + sleep 1 + echo "Hi from {{ id }}!" diff --git a/docs/triggers.md b/docs/triggers.md index 55b31ed..a84782c 100644 --- a/docs/triggers.md +++ b/docs/triggers.md @@ -2,13 +2,37 @@ ## Trigger Types +### Once + +```yaml +--8<-- "docs/examples/once.yaml" +``` + +@mermaid(docs/examples/once.yaml) + ### After ```yaml ---8<-- "docs/examples/dag.yaml" +--8<-- "docs/examples/after.yaml" +``` + +@mermaid(docs/examples/after.yaml) + +### Restart + +```yaml +--8<-- "docs/examples/restart.yaml" +``` + +@mermaid(docs/examples/restart.yaml) + +### Watch + +```yaml +--8<-- "docs/examples/watch.yaml" ``` -@mermaid(docs/examples/dag.yaml) +@mermaid(docs/examples/watch.yaml) ## Combining Triggers From c90ef46ede966d59a595576ac33b5981d690908b Mon Sep 17 00:00:00 2001 From: Josh Karpel Date: Fri, 28 Jun 2024 22:12:38 -0500 Subject: [PATCH 4/8] changelog and docs --- docs/changelog.md | 2 ++ synthesize/cli.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/changelog.md b/docs/changelog.md index 3533eb1..3bcaafb 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -14,6 +14,8 @@ and environment variables into target commands. Arguments and environment variables can be specified at either the flow, node, or target level, with the most specific taking precedence. +- [#43](https://github.com/JoshKarpel/synthesize/pull/43) + Mermaid diagrams can be generated for a flow using the `--mermaid` option. ### Changed diff --git a/synthesize/cli.py b/synthesize/cli.py index 82c2800..0f55619 100644 --- a/synthesize/cli.py +++ b/synthesize/cli.py @@ -39,7 +39,7 @@ def run( ), mermaid: bool = Option( default=False, - help="If enabled, output the flow as a Mermaid diagram, and don't run the flow.", + help="If enabled, output a description of the flow as a Mermaid diagram, and don't run the flow.", ), dry: bool = Option( default=False, From f7ea3eb87245e76a6afaf523590c7a3e13e14d50 Mon Sep 17 00:00:00 2001 From: Josh Karpel Date: Fri, 28 Jun 2024 22:19:42 -0500 Subject: [PATCH 5/8] more details in docs --- docs/triggers.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/docs/triggers.md b/docs/triggers.md index a84782c..2b3594c 100644 --- a/docs/triggers.md +++ b/docs/triggers.md @@ -4,6 +4,11 @@ ### Once +"Once" triggers run the node just one time during the flow. +This is the default trigger. + +Use this trigger when a command needs to run only one time during a flow. + ```yaml --8<-- "docs/examples/once.yaml" ``` @@ -12,6 +17,10 @@ ### After +"After" triggers run the node after some other nodes have completed. + +Use this trigger when a node depends on the output of another node. + ```yaml --8<-- "docs/examples/after.yaml" ``` @@ -20,6 +29,10 @@ ### Restart +"Restart" triggers run the node every time the node is completed. + +Use this trigger when you want to keep the node's command running. + ```yaml --8<-- "docs/examples/restart.yaml" ``` @@ -28,6 +41,11 @@ ### Watch +"Watch" triggers run the node every time one of the watched files changes +(directories are watched recursively). + +Use this trigger to run a node in reaction to changes in the filesystem. + ```yaml --8<-- "docs/examples/watch.yaml" ``` From 507292aa620308994058e2d6b8e76df5b71acf7c Mon Sep 17 00:00:00 2001 From: Josh Karpel Date: Fri, 28 Jun 2024 22:24:08 -0500 Subject: [PATCH 6/8] test mermaid --- tests/test_config.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/test_config.py b/tests/test_config.py index b8d1245..d199ddf 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -18,12 +18,13 @@ random_color, ) +EXAMPLES = list((Path(__file__).parent.parent / "docs" / "examples").iterdir()) -@pytest.mark.parametrize( - "example", list((Path(__file__).parent.parent / "docs" / "examples").iterdir()) -) -def test_config_examples_parse(example: Path) -> None: - Config.from_file(example) + +@pytest.mark.parametrize("example", EXAMPLES) +def test_can_generate_mermaid_from_examples(example: Path) -> None: + for flow in Config.from_file(example).resolve().values(): + flow.mermaid() def test_can_make_style_from_random_color() -> None: From e54f6d974affb5b2c83c6a6af8756c9ace212451 Mon Sep 17 00:00:00 2001 From: Josh Karpel Date: Fri, 28 Jun 2024 22:29:22 -0500 Subject: [PATCH 7/8] more tests --- .coveragerc | 2 ++ synth.yaml | 1 + synthesize/config.py | 4 ++-- tests/test_config.py | 3 ++- 4 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.coveragerc b/.coveragerc index 41c432b..e60703c 100644 --- a/.coveragerc +++ b/.coveragerc @@ -16,6 +16,8 @@ exclude_lines = def __repr__ if self\.debug + case never: + assert_never raise AssertionError raise NotImplementedError diff --git a/synth.yaml b/synth.yaml index 2b4ef6c..31583df 100644 --- a/synth.yaml +++ b/synth.yaml @@ -33,3 +33,4 @@ triggers: - tests/ - docs/examples/ - pyproject.toml + - .coveragerc diff --git a/synthesize/config.py b/synthesize/config.py index 0637989..5f129e6 100644 --- a/synthesize/config.py +++ b/synthesize/config.py @@ -194,8 +194,8 @@ def mermaid(self) -> str: seen_watches.add(h) lines.append(f'w_{h}[("{text}")]') lines.append(f"w_{h} -->|👁| {node.id}") - case _: - assert_never(node.trigger) + case never: + assert_never(never) return "\n ".join(lines).strip() diff --git a/tests/test_config.py b/tests/test_config.py index d199ddf..196c049 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -18,7 +18,8 @@ random_color, ) -EXAMPLES = list((Path(__file__).parent.parent / "docs" / "examples").iterdir()) +ROOT = Path(__file__).parent.parent +EXAMPLES = [*(ROOT / "docs" / "examples").iterdir(), ROOT / "synth.yaml"] @pytest.mark.parametrize("example", EXAMPLES) From bfe56c70efa709cea530c8ced609ecfb7727e51d Mon Sep 17 00:00:00 2001 From: Josh Karpel Date: Fri, 28 Jun 2024 22:30:39 -0500 Subject: [PATCH 8/8] tweak --- tests/test_config.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_config.py b/tests/test_config.py index 196c049..ccbfb89 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -19,7 +19,10 @@ ) ROOT = Path(__file__).parent.parent -EXAMPLES = [*(ROOT / "docs" / "examples").iterdir(), ROOT / "synth.yaml"] +EXAMPLES = [ + *(ROOT / "docs" / "examples").iterdir(), + ROOT / "synth.yaml", +] @pytest.mark.parametrize("example", EXAMPLES)