diff --git a/.coveragerc b/.coveragerc index 41c432b..e60703c 100644 --- a/.coveragerc +++ b/.coveragerc @@ -16,6 +16,8 @@ exclude_lines = def __repr__ if self\.debug + case never: + assert_never raise AssertionError raise NotImplementedError diff --git a/docs/assets/style.css b/docs/assets/style.css index 3a22901..45fda70 100644 --- a/docs/assets/style.css +++ b/docs/assets/style.css @@ -64,3 +64,9 @@ a.autorefs-external::after { a.autorefs-external:hover::after { background-color: var(--md-accent-fg-color); } + +/* Styles for SVG screenshots */ + +.mermaid { + text-align: center; +} diff --git a/docs/changelog.md b/docs/changelog.md index 3533eb1..3bcaafb 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -14,6 +14,8 @@ and environment variables into target commands. Arguments and environment variables can be specified at either the flow, node, or target level, with the most specific taking precedence. +- [#43](https://github.com/JoshKarpel/synthesize/pull/43) + Mermaid diagrams can be generated for a flow using the `--mermaid` option. ### Changed diff --git a/examples/dag.yaml b/docs/examples/after.yaml similarity index 97% rename from examples/dag.yaml rename to docs/examples/after.yaml index 3a98ff4..eadb6e1 100644 --- a/examples/dag.yaml +++ b/docs/examples/after.yaml @@ -23,5 +23,5 @@ flows: targets: sleep-and-echo: commands: | - sleep 2 + sleep 1 echo "Hi from {{ id }}!" diff --git a/docs/examples/once.yaml b/docs/examples/once.yaml new file mode 100644 index 0000000..bd35e4e --- /dev/null +++ b/docs/examples/once.yaml @@ -0,0 +1,15 @@ +flows: + default: + nodes: + 1: + target: sleep-and-echo + 2: + target: sleep-and-echo + 3: + target: sleep-and-echo + +targets: + sleep-and-echo: + commands: | + sleep 1 + echo "Hi from {{ id }}!" diff --git a/docs/examples/restart.yaml b/docs/examples/restart.yaml new file mode 100644 index 0000000..0659037 --- /dev/null +++ b/docs/examples/restart.yaml @@ -0,0 +1,19 @@ +flows: + default: + nodes: + 1: + target: sleep-and-echo + trigger: + type: restart + delay: 3 + 2: + target: sleep-and-echo + trigger: + type: restart + delay: 1 + +targets: + sleep-and-echo: + commands: | + sleep 1 + echo "Hi from {{ id }}!" diff --git a/docs/examples/watch.yaml b/docs/examples/watch.yaml new file mode 100644 index 0000000..ec22430 --- /dev/null +++ b/docs/examples/watch.yaml @@ -0,0 +1,19 @@ +flows: + default: + nodes: + 1: + target: sleep-and-echo + trigger: + type: watch + paths: ["synthesize/", "tests/"] + 2: + target: sleep-and-echo + trigger: + type: watch + paths: [ "docs/" ] + +targets: + sleep-and-echo: + commands: | + sleep 1 + echo "Hi from {{ id }}!" diff --git a/docs/hooks/mermaid.py b/docs/hooks/mermaid.py new file mode 100644 index 0000000..089a10c --- /dev/null +++ b/docs/hooks/mermaid.py @@ -0,0 +1,35 @@ +import re +import subprocess + +from mkdocs.config.defaults import MkDocsConfig +from mkdocs.structure.files import Files +from mkdocs.structure.pages import Page + + +def on_page_markdown( + markdown: str, + page: Page, + config: MkDocsConfig, + files: Files, +) -> str: + lines = [] + for line in markdown.splitlines(): + if match := re.match(r"@mermaid\(([\w\.\/]+)\)", line): + lines.append("```mermaid") + cmd = subprocess.run( + ("synth", "--mermaid", "--config", match.group(1)), + capture_output=True, + text=True, + check=False, + ) + + if cmd.returncode != 0: + raise Exception(f"Error: {cmd.stdout}\n{cmd.stderr}") + else: + lines.append(cmd.stdout) + + lines.append("```") + else: + lines.append(line) + + return "\n".join(lines) diff --git a/docs/triggers.md b/docs/triggers.md new file mode 100644 index 0000000..2b3594c --- /dev/null +++ b/docs/triggers.md @@ -0,0 +1,61 @@ +# Triggers + +## Trigger Types + +### Once + +"Once" triggers run the node just one time during the flow. +This is the default trigger. + +Use this trigger when a command needs to run only one time during a flow. + +```yaml +--8<-- "docs/examples/once.yaml" +``` + +@mermaid(docs/examples/once.yaml) + +### After + +"After" triggers run the node after some other nodes have completed. + +Use this trigger when a node depends on the output of another node. + +```yaml +--8<-- "docs/examples/after.yaml" +``` + +@mermaid(docs/examples/after.yaml) + +### Restart + +"Restart" triggers run the node every time the node is completed. + +Use this trigger when you want to keep the node's command running. + +```yaml +--8<-- "docs/examples/restart.yaml" +``` + +@mermaid(docs/examples/restart.yaml) + +### Watch + +"Watch" triggers run the node every time one of the watched files changes +(directories are watched recursively). + +Use this trigger to run a node in reaction to changes in the filesystem. + +```yaml +--8<-- "docs/examples/watch.yaml" +``` + +@mermaid(docs/examples/watch.yaml) + +## Combining Triggers + +```yaml +--8<-- "synth.yaml" +``` + +@mermaid(synth.yaml) diff --git a/mkdocs.yml b/mkdocs.yml index 7c41b9b..ae464e0 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -9,6 +9,7 @@ extra_css: watch: - synthesize/ + - docs/ theme: name: material @@ -53,6 +54,9 @@ plugins: - https://docs.python.org/3/objects.inv - https://rich.readthedocs.io/en/stable/objects.inv +hooks: + - docs/hooks/mermaid.py + markdown_extensions: - admonition - pymdownx.details @@ -60,9 +64,13 @@ markdown_extensions: anchor_linenums: true - pymdownx.inlinehilite - pymdownx.snippets: - base_path: ['docs'] + base_path: [''] check_paths: true - - pymdownx.superfences + - pymdownx.superfences: + custom_fences: + - name: mermaid + class: mermaid + format: !!python/name:pymdownx.superfences.fence_code_format - pymdownx.tabbed: alternate_style: true - attr_list @@ -80,4 +88,5 @@ extra: nav: - Introduction: index.md + - triggers.md - changelog.md diff --git a/synth.yaml b/synth.yaml index 560406f..31583df 100644 --- a/synth.yaml +++ b/synth.yaml @@ -31,5 +31,6 @@ triggers: paths: - synthesize/ - tests/ - - examples/ + - docs/examples/ - pyproject.toml + - .coveragerc diff --git a/synthesize/cli.py b/synthesize/cli.py index 17a6918..0f55619 100644 --- a/synthesize/cli.py +++ b/synthesize/cli.py @@ -37,6 +37,10 @@ def run( envvar="SYNTHFILE", help="The path to the configuration file to execute.", ), + mermaid: bool = Option( + default=False, + help="If enabled, output a description of the flow as a Mermaid diagram, and don't run the flow.", + ), dry: bool = Option( default=False, help="If enabled, do not run actually run the flow.", @@ -81,6 +85,10 @@ def run( ) raise Exit(code=1) + if mermaid: + print(selected_flow.mermaid()) + return + if dry: return diff --git a/synthesize/config.py b/synthesize/config.py index 54b86e9..5f129e6 100644 --- a/synthesize/config.py +++ b/synthesize/config.py @@ -4,6 +4,7 @@ import shutil from collections.abc import Mapping from colorsys import hsv_to_rgb +from functools import cached_property from pathlib import Path from random import random from textwrap import dedent @@ -13,8 +14,10 @@ from jinja2 import Environment from pydantic import Field, field_validator from rich.color import Color +from typing_extensions import assert_never from synthesize.model import Model +from synthesize.utils import md5 Args = dict[ Annotated[ @@ -36,6 +39,12 @@ ], str, ] +ID = Annotated[ + str, + Field( + pattern=r"\w+", + ), +] def random_color() -> str: @@ -88,7 +97,7 @@ class Once(Model): class After(Model): type: Literal["after"] = "after" - after: Annotated[frozenset[str], Field(min_length=1)] + after: Annotated[tuple[str, ...], Field(min_length=1)] class Restart(Model): @@ -128,13 +137,17 @@ class FlowNode(Model): color: Annotated[str, Field(default_factory=random_color)] + @cached_property + def uid(self) -> str: + return md5(self.model_dump_json(exclude={"color"}).encode()) + class UnresolvedFlowNode(Model): - target: Target | str + target: Target | ID args: Args = {} envs: Envs = {} - trigger: AnyTrigger | str = Once() + trigger: AnyTrigger | ID = Once() color: Annotated[str, Field(default_factory=random_color)] @@ -155,20 +168,47 @@ def resolve( class Flow(Model): - nodes: dict[str, FlowNode] + nodes: dict[ID, FlowNode] args: Args = {} envs: Envs = {} + def mermaid(self) -> str: + lines = ["flowchart TD"] + + seen_watches = set() + for id, node in self.nodes.items(): + lines.append(f"{node.id}({id})") + + match node.trigger: + case Once(): + pass + case After(after=after): + for a in after: + lines.append(f"{self.nodes[a].id} --> {node.id}") + case Restart(delay=delay): + lines.append(f"{node.id} -->|∞ {delay:.3g}s| {node.id}") + case Watch(paths=paths): + text = "\n".join(paths) + h = md5("".join(paths)) + if h not in seen_watches: + seen_watches.add(h) + lines.append(f'w_{h}[("{text}")]') + lines.append(f"w_{h} -->|👁| {node.id}") + case never: + assert_never(never) + + return "\n ".join(lines).strip() + class UnresolvedFlow(Model): - nodes: dict[str, UnresolvedFlowNode] + nodes: dict[ID, UnresolvedFlowNode] args: Args = {} envs: Envs = {} def resolve( self, - targets: Mapping[str, Target], - triggers: Mapping[str, AnyTrigger], + targets: Mapping[ID, Target], + triggers: Mapping[ID, AnyTrigger], ) -> Flow: return Flow( nodes={id: node.resolve(id, targets, triggers) for id, node in self.nodes.items()}, @@ -178,9 +218,9 @@ def resolve( class Config(Model): - flows: dict[str, UnresolvedFlow] = {} - targets: dict[str, Target] = {} - triggers: dict[str, AnyTrigger] = {} + flows: dict[ID, UnresolvedFlow] = {} + targets: dict[ID, Target] = {} + triggers: dict[ID, AnyTrigger] = {} @classmethod def from_file(cls, file: Path) -> Config: @@ -191,5 +231,5 @@ def from_file(cls, file: Path) -> Config: else: raise NotImplementedError("Currently, only YAML files are supported.") - def resolve(self) -> Mapping[str, Flow]: + def resolve(self) -> Mapping[ID, Flow]: return {id: flow.resolve(self.targets, self.triggers) for id, flow in self.flows.items()} diff --git a/synthesize/execution.py b/synthesize/execution.py index 9fb85b6..a8ea05a 100644 --- a/synthesize/execution.py +++ b/synthesize/execution.py @@ -12,11 +12,10 @@ from synthesize.config import Args, Envs, FlowNode from synthesize.messages import ExecutionCompleted, ExecutionOutput, ExecutionStarted, Message -from synthesize.utils import md5 def write_script(node: FlowNode, args: Args, tmp_dir: Path) -> Path: - path = tmp_dir / f"{node.id}-{md5(node.model_dump_json().encode())}" + path = tmp_dir / f"{node.id}-{node.uid}" path.parent.mkdir(parents=True, exist_ok=True) path.write_text( diff --git a/synthesize/utils.py b/synthesize/utils.py index 40af9d9..9c8bdd3 100644 --- a/synthesize/utils.py +++ b/synthesize/utils.py @@ -15,5 +15,5 @@ async def delayed() -> T: return create_task(delayed(), name=name) -def md5(data: bytes) -> str: - return hashlib.md5(data).hexdigest() +def md5(data: bytes | str) -> str: + return hashlib.md5(data if isinstance(data, bytes) else data.encode()).hexdigest() diff --git a/tests/test_config.py b/tests/test_config.py index 7fc1bf1..ccbfb89 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -18,10 +18,17 @@ random_color, ) +ROOT = Path(__file__).parent.parent +EXAMPLES = [ + *(ROOT / "docs" / "examples").iterdir(), + ROOT / "synth.yaml", +] -@pytest.mark.parametrize("example", list((Path(__file__).parent.parent / "examples").iterdir())) -def test_config_examples_parse(example: Path) -> None: - Config.from_file(example) + +@pytest.mark.parametrize("example", EXAMPLES) +def test_can_generate_mermaid_from_examples(example: Path) -> None: + for flow in Config.from_file(example).resolve().values(): + flow.mermaid() def test_can_make_style_from_random_color() -> None: