diff --git a/CHANGELOG.md b/CHANGELOG.md index ba24cc5b..208f4d93 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,24 @@ Types of changes # Latch SDK Changelog +## 2.34.0 - 2023-10-04 + +### Added + +* `directory` modifier for input / outputs +* Support `temp` by removing from compiled rules. All files / directories are +temporary because they are deleted at the end of each job on Latch. +* `multiext` output modifier +* `report` output modifier +* `params` in rules + +### Fixed + +* Replace skipped rules with `Ellipsis`. Supports rules nested in conditionals where previously an empty block was produced. +* Patched parser to generate compiled code for `workflow.include` calls Compiled workflow.include should carry `print_compilation` keyword (snakemake/snakemake#2469) +* Detect use of `conda` keyword and install in image. This effectively supports wrapper/conda keywords. +* `Iterable, Generator` cause issues as type hints when imported from `collections.abc` rather than `typing` + ## 2.33.0 - 2023-09-29 ### Added diff --git a/latch_cli/snakemake/single_task_snakemake.py b/latch_cli/snakemake/single_task_snakemake.py index acef3270..e2785401 100644 --- a/latch_cli/snakemake/single_task_snakemake.py +++ b/latch_cli/snakemake/single_task_snakemake.py @@ -9,6 +9,7 @@ from snakemake.parser import ( INDENT, Benchmark, + Include, Input, Log, Output, @@ -28,6 +29,10 @@ def eprint(x: str) -> None: print(x, file=sys.stderr) +print_compilation = os.environ.get("LATCH_PRINT_COMPILATION", False) +if print_compilation == "1": + print_compilation = True + data = json.loads(os.environ["LATCH_SNAKEMAKE_DATA"]) rules = data["rules"] outputs = data["outputs"] @@ -154,10 +159,23 @@ def emit_overrides(self, token): else: raise ValueError(f"tried to emit overrides for unknown state: {type(self)}") - positional_data = (render_annotated_str_list(x) for x in xs["positional"]) - keyword_data = ( - f"{k}={render_annotated_str_list(v)}" for k, v in xs["keyword"].items() - ) + if ( + isinstance(self, Output) + and len(xs["positional"]) > 0 + and xs["positional"][0].get("flags") is not None + and "multiext" in xs["positional"][0].get("flags") + ): + filename = repr(xs["positional"][0]["flags"]["multiext"]) + exts = [repr("." + x["value"].split(".")[-1]) for x in xs["positional"]] + positional_data = (f"multiext({filename},{','.join(exts)})",) + else: + positional_data = (render_annotated_str_list(x) for x in xs["positional"]) + + modifier_fn = render_annotated_str_list + if isinstance(self, Params): + modifier_fn = repr + + keyword_data = (f"{k}={modifier_fn(v)}" for k, v in xs["keyword"].items()) data = chain(positional_data, keyword_data) for x in data: @@ -188,17 +206,33 @@ def skipping_block_content(self, token): emitted_overrides.add(self.rulename) +def block_content_with_print_compilation(self, token): + if print_compilation: + yield f"{token.string}, print_compilation=True", token + else: + yield token.string, token + + Input.block_content = skipping_block_content Output.block_content = skipping_block_content Params.block_content = skipping_block_content Benchmark.block_content = skipping_block_content Log.block_content = skipping_block_content Ruleorder.block_content = lambda self, token: None +Include.block_content = block_content_with_print_compilation class SkippingRule(Rule): def start(self, aux=""): if self.rulename not in rules: + # Rules can be nested in conditional statements: + # + # if (): + # rule A: + # + # + # We want correct python code if we remove them. + yield "..." return yield from super().start(aux) diff --git a/latch_cli/snakemake/workflow.py b/latch_cli/snakemake/workflow.py index 17bf0219..102878a1 100644 --- a/latch_cli/snakemake/workflow.py +++ b/latch_cli/snakemake/workflow.py @@ -1243,7 +1243,8 @@ def get_fn_code( check=True, env={{ **os.environ, - "LATCH_SNAKEMAKE_DATA": {repr(json.dumps(snakemake_data))} + "LATCH_SNAKEMAKE_DATA": {repr(json.dumps(snakemake_data))}, + "LATCH_PRINT_COMPILATION": "1" }}, stdout=f ) @@ -1325,21 +1326,6 @@ def get_fn_code( finally: ignored_paths = {{".cache", ".snakemake/conda"}} ignored_names = {{".git", ".latch", "__pycache__"}} - - print("Recursive directory listing:") - stack = [(Path("."), 0)] - while len(stack) > 0: - cur, indent = stack.pop() - print(" " * indent + cur.name) - - if cur.is_dir(): - if cur.name in ignored_names or str(cur) in ignored_paths: - print(" " * indent + " ...") - continue - - for x in cur.iterdir(): - stack.append((x, indent + 1)) - """, 1, ) diff --git a/setup.py b/setup.py index 3831f10e..8f13abc2 100644 --- a/setup.py +++ b/setup.py @@ -13,7 +13,7 @@ setup( name="latch", - version="v2.33.0", + version="v2.34.0", author_email="kenny@latch.bio", description="The Latch SDK", packages=find_packages(),