From 7b6d57f9403ccf6fdca4e94a831dfb0e79364db4 Mon Sep 17 00:00:00 2001 From: Kenny Workman Date: Tue, 3 Oct 2023 18:45:39 -0700 Subject: [PATCH 01/16] render params + replace rules with ellipse --- latch_cli/snakemake/single_task_snakemake.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/latch_cli/snakemake/single_task_snakemake.py b/latch_cli/snakemake/single_task_snakemake.py index e347888d..7e2abbd4 100644 --- a/latch_cli/snakemake/single_task_snakemake.py +++ b/latch_cli/snakemake/single_task_snakemake.py @@ -155,9 +155,12 @@ def emit_overrides(self, token): raise ValueError(f"tried to emit overrides for unknown state: {type(self)}") positional_data = (render_annotated_str_list(x) for x in xs["positional"]) - keyword_data = ( - f"{k}={render_annotated_str_list(v)}" for k, v in xs["keyword"].items() - ) + + modifier_fn = render_annotated_str_list + if isinstance(self, Params): + modifier_fn = repr + + keyword_data = (f"{k}={modifier_fn(v)}" for k, v in xs["keyword"].items()) data = chain(positional_data, keyword_data) for x in data: @@ -199,6 +202,14 @@ def skipping_block_content(self, token): class SkippingRule(Rule): def start(self, aux=""): if self.rulename not in rules: + # Rules can be nested in conditional statements: + # + # if (): + # rule A: + # + # + # We want correct python code if we remove them. + yield "..." return yield from super().start(aux) From 28ac3e2a1696b6fa5e12225d461f7498f8887e26 Mon Sep 17 00:00:00 2001 From: Kenny Workman Date: Tue, 3 Oct 2023 19:50:54 -0700 Subject: [PATCH 02/16] support multiext --- latch_cli/snakemake/single_task_snakemake.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/latch_cli/snakemake/single_task_snakemake.py b/latch_cli/snakemake/single_task_snakemake.py index 7e2abbd4..79a2b520 100644 --- a/latch_cli/snakemake/single_task_snakemake.py +++ b/latch_cli/snakemake/single_task_snakemake.py @@ -154,7 +154,17 @@ def emit_overrides(self, token): else: raise ValueError(f"tried to emit overrides for unknown state: {type(self)}") - positional_data = (render_annotated_str_list(x) for x in xs["positional"]) + if ( + isinstance(self, Params) + and xs[0].get("flags") + and "multiext" in xs[0].get("flags") + ): + quote = "'" + positional_data = ( + f"multiext('{xs[0]['value']}',{','.join([quote + x['value'].split('.')[-1] + quote for x in xs])})" + ) + else: + positional_data = (render_annotated_str_list(x) for x in xs["positional"]) modifier_fn = render_annotated_str_list if isinstance(self, Params): From c067ebfbd455a471bfae005939499520facbe960 Mon Sep 17 00:00:00 2001 From: Kenny Workman Date: Tue, 3 Oct 2023 20:02:26 -0700 Subject: [PATCH 03/16] strip directory print after job failure --- latch_cli/snakemake/workflow.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/latch_cli/snakemake/workflow.py b/latch_cli/snakemake/workflow.py index 9e0c2f61..2ac22efb 100644 --- a/latch_cli/snakemake/workflow.py +++ b/latch_cli/snakemake/workflow.py @@ -1325,21 +1325,6 @@ def get_fn_code( finally: ignored_paths = {{".cache", ".snakemake/conda"}} ignored_names = {{".git", ".latch", "__pycache__"}} - - print("Recursive directory listing:") - stack = [(Path("."), 0)] - while len(stack) > 0: - cur, indent = stack.pop() - print(" " * indent + cur.name) - - if cur.is_dir(): - if cur.name in ignored_names or str(cur) in ignored_paths: - print(" " * indent + " ...") - continue - - for x in cur.iterdir(): - stack.append((x, indent + 1)) - """, 1, ) From df456424382ff2883d55884bc46bfc7507d6537f Mon Sep 17 00:00:00 2001 From: Kenny Workman Date: Tue, 3 Oct 2023 20:02:32 -0700 Subject: [PATCH 04/16] bug --- latch_cli/snakemake/single_task_snakemake.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/latch_cli/snakemake/single_task_snakemake.py b/latch_cli/snakemake/single_task_snakemake.py index 79a2b520..dfa0c13c 100644 --- a/latch_cli/snakemake/single_task_snakemake.py +++ b/latch_cli/snakemake/single_task_snakemake.py @@ -155,7 +155,7 @@ def emit_overrides(self, token): raise ValueError(f"tried to emit overrides for unknown state: {type(self)}") if ( - isinstance(self, Params) + isinstance(self, Output) and xs[0].get("flags") and "multiext" in xs[0].get("flags") ): From 7aa3e123205aff070a1f96afd54f9dda816762ae Mon Sep 17 00:00:00 2001 From: Kenny Workman Date: Tue, 3 Oct 2023 20:49:21 -0700 Subject: [PATCH 05/16] bug --- latch_cli/snakemake/single_task_snakemake.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/latch_cli/snakemake/single_task_snakemake.py b/latch_cli/snakemake/single_task_snakemake.py index dfa0c13c..e9902028 100644 --- a/latch_cli/snakemake/single_task_snakemake.py +++ b/latch_cli/snakemake/single_task_snakemake.py @@ -161,7 +161,9 @@ def emit_overrides(self, token): ): quote = "'" positional_data = ( - f"multiext('{xs[0]['value']}',{','.join([quote + x['value'].split('.')[-1] + quote for x in xs])})" + ( + f"multiext('{xs[0]['value']}',{','.join([quote + x['value'].split('.')[-1] + quote for x in xs])})" + ), ) else: positional_data = (render_annotated_str_list(x) for x in xs["positional"]) From 1b1510267a7eee46b73412848d9827300108d742 Mon Sep 17 00:00:00 2001 From: Kenny Workman Date: Wed, 4 Oct 2023 13:14:29 -0700 Subject: [PATCH 06/16] trisomy --- latch_cli/snakemake/single_task_snakemake.py | 19 +++++++++++++++++-- latch_cli/snakemake/workflow.py | 3 ++- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/latch_cli/snakemake/single_task_snakemake.py b/latch_cli/snakemake/single_task_snakemake.py index e9902028..72480d2f 100644 --- a/latch_cli/snakemake/single_task_snakemake.py +++ b/latch_cli/snakemake/single_task_snakemake.py @@ -9,6 +9,7 @@ from snakemake.parser import ( INDENT, Benchmark, + Include, Input, Log, Output, @@ -28,6 +29,11 @@ def eprint(x: str) -> None: print(x, file=sys.stderr) +print_compilation_env = os.environ["LATCH_PRINT_COMPILATION"] +print_compilation = False +if print_compilation_env == "1": + print_compilation = True + data = json.loads(os.environ["LATCH_SNAKEMAKE_DATA"]) rules = data["rules"] outputs = data["outputs"] @@ -156,9 +162,10 @@ def emit_overrides(self, token): if ( isinstance(self, Output) - and xs[0].get("flags") - and "multiext" in xs[0].get("flags") + and xs["positional"][0].get("flags") + and "multiext" in xs["positional"][0].get("flags") ): + eprint("inside") quote = "'" positional_data = ( ( @@ -203,12 +210,20 @@ def skipping_block_content(self, token): emitted_overrides.add(self.rulename) +def block_content_with_print_compilation(self, token): + if print_compilation: + yield f"{token.string}, print_compilation=True", token + else: + yield token.string, token + + Input.block_content = skipping_block_content Output.block_content = skipping_block_content Params.block_content = skipping_block_content Benchmark.block_content = skipping_block_content Log.block_content = skipping_block_content Ruleorder.block_content = lambda self, token: None +Include.block_content = block_content_with_print_compilation class SkippingRule(Rule): diff --git a/latch_cli/snakemake/workflow.py b/latch_cli/snakemake/workflow.py index 2ac22efb..300419a3 100644 --- a/latch_cli/snakemake/workflow.py +++ b/latch_cli/snakemake/workflow.py @@ -1243,7 +1243,8 @@ def get_fn_code( check=True, env={{ **os.environ, - "LATCH_SNAKEMAKE_DATA": {repr(json.dumps(snakemake_data))} + "LATCH_SNAKEMAKE_DATA": {repr(json.dumps(snakemake_data))}, + "LATCH_PRINT_COMPILATION": "1" }}, stdout=f ) From 00c68195cb69815324f39a4667ce500923d7d79e Mon Sep 17 00:00:00 2001 From: Kenny Workman Date: Wed, 4 Oct 2023 13:29:23 -0700 Subject: [PATCH 07/16] better --- latch_cli/snakemake/single_task_snakemake.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/latch_cli/snakemake/single_task_snakemake.py b/latch_cli/snakemake/single_task_snakemake.py index 72480d2f..c380eaea 100644 --- a/latch_cli/snakemake/single_task_snakemake.py +++ b/latch_cli/snakemake/single_task_snakemake.py @@ -29,8 +29,7 @@ def eprint(x: str) -> None: print(x, file=sys.stderr) -print_compilation_env = os.environ["LATCH_PRINT_COMPILATION"] -print_compilation = False +print_compilation_env = os.environ.get("LATCH_PRINT_COMPILATION", False) if print_compilation_env == "1": print_compilation = True From 0c70e54a0922c1a5fd55113d990ed0be0834170e Mon Sep 17 00:00:00 2001 From: Kenny Workman Date: Wed, 4 Oct 2023 13:32:57 -0700 Subject: [PATCH 08/16] fix --- latch_cli/snakemake/single_task_snakemake.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/latch_cli/snakemake/single_task_snakemake.py b/latch_cli/snakemake/single_task_snakemake.py index c380eaea..1621b1a6 100644 --- a/latch_cli/snakemake/single_task_snakemake.py +++ b/latch_cli/snakemake/single_task_snakemake.py @@ -168,7 +168,7 @@ def emit_overrides(self, token): quote = "'" positional_data = ( ( - f"multiext('{xs[0]['value']}',{','.join([quote + x['value'].split('.')[-1] + quote for x in xs])})" + f"multiext('{xs['positional'][0]['value']}',{','.join([quote + x['value'].split('.')[-1] + quote for x in xs])})" ), ) else: From 6e9216f539fb4c4551f3421452a90847f139fd32 Mon Sep 17 00:00:00 2001 From: Kenny Workman Date: Wed, 4 Oct 2023 13:52:00 -0700 Subject: [PATCH 09/16] more readable --- latch_cli/snakemake/single_task_snakemake.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/latch_cli/snakemake/single_task_snakemake.py b/latch_cli/snakemake/single_task_snakemake.py index 1621b1a6..7861b24a 100644 --- a/latch_cli/snakemake/single_task_snakemake.py +++ b/latch_cli/snakemake/single_task_snakemake.py @@ -164,13 +164,9 @@ def emit_overrides(self, token): and xs["positional"][0].get("flags") and "multiext" in xs["positional"][0].get("flags") ): - eprint("inside") - quote = "'" - positional_data = ( - ( - f"multiext('{xs['positional'][0]['value']}',{','.join([quote + x['value'].split('.')[-1] + quote for x in xs])})" - ), - ) + filename = repr(xs["positional"][0]["value"]) + exts = [repr(x["value"].split(".")[-1]) for x in xs["positional"]] + positional_data = (f"multiext({filename},{','.join(exts)})",) else: positional_data = (render_annotated_str_list(x) for x in xs["positional"]) From 725e4512a5e915fe8fb5dd9bec319b6321982d2c Mon Sep 17 00:00:00 2001 From: Kenny Workman Date: Wed, 4 Oct 2023 14:00:12 -0700 Subject: [PATCH 10/16] slop --- latch_cli/snakemake/single_task_snakemake.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/latch_cli/snakemake/single_task_snakemake.py b/latch_cli/snakemake/single_task_snakemake.py index 7861b24a..76a8a9ba 100644 --- a/latch_cli/snakemake/single_task_snakemake.py +++ b/latch_cli/snakemake/single_task_snakemake.py @@ -29,8 +29,8 @@ def eprint(x: str) -> None: print(x, file=sys.stderr) -print_compilation_env = os.environ.get("LATCH_PRINT_COMPILATION", False) -if print_compilation_env == "1": +print_compilation = os.environ.get("LATCH_PRINT_COMPILATION", False) +if print_compilation == "1": print_compilation = True data = json.loads(os.environ["LATCH_SNAKEMAKE_DATA"]) @@ -161,7 +161,7 @@ def emit_overrides(self, token): if ( isinstance(self, Output) - and xs["positional"][0].get("flags") + and xs["positional"][0].get("flags") is not None and "multiext" in xs["positional"][0].get("flags") ): filename = repr(xs["positional"][0]["value"]) From b9d9caef78ce3465d9c9ebb12b6bec5f7c1b0aab Mon Sep 17 00:00:00 2001 From: Kenny Workman Date: Wed, 4 Oct 2023 14:21:38 -0700 Subject: [PATCH 11/16] positionals can be empty --- latch_cli/snakemake/single_task_snakemake.py | 1 + 1 file changed, 1 insertion(+) diff --git a/latch_cli/snakemake/single_task_snakemake.py b/latch_cli/snakemake/single_task_snakemake.py index 76a8a9ba..fc7f44a2 100644 --- a/latch_cli/snakemake/single_task_snakemake.py +++ b/latch_cli/snakemake/single_task_snakemake.py @@ -161,6 +161,7 @@ def emit_overrides(self, token): if ( isinstance(self, Output) + and len(xs["positional"]) > 0 and xs["positional"][0].get("flags") is not None and "multiext" in xs["positional"][0].get("flags") ): From 4d97a9a279ca9d1c65544ab309a86611753f10e2 Mon Sep 17 00:00:00 2001 From: Kenny Workman Date: Wed, 4 Oct 2023 15:14:29 -0700 Subject: [PATCH 12/16] bug --- latch_cli/snakemake/single_task_snakemake.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/latch_cli/snakemake/single_task_snakemake.py b/latch_cli/snakemake/single_task_snakemake.py index fc7f44a2..6331e059 100644 --- a/latch_cli/snakemake/single_task_snakemake.py +++ b/latch_cli/snakemake/single_task_snakemake.py @@ -165,8 +165,8 @@ def emit_overrides(self, token): and xs["positional"][0].get("flags") is not None and "multiext" in xs["positional"][0].get("flags") ): - filename = repr(xs["positional"][0]["value"]) - exts = [repr(x["value"].split(".")[-1]) for x in xs["positional"]] + filename = repr(xs["positional"][0]["flags"]["multiext"]) + exts = [repr("." + x["value"].split(".")[-1]) for x in xs["positional"]] positional_data = (f"multiext({filename},{','.join(exts)})",) else: positional_data = (render_annotated_str_list(x) for x in xs["positional"]) From 0e11a96987f6562f06f18baad2a94a8d1b58bf26 Mon Sep 17 00:00:00 2001 From: Kenny Workman Date: Wed, 4 Oct 2023 19:12:08 -0700 Subject: [PATCH 13/16] retry job on conda create exception --- latch_cli/snakemake/serialize.py | 3 ++- latch_cli/snakemake/workflow.py | 27 +++++++++++++++++++-------- 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/latch_cli/snakemake/serialize.py b/latch_cli/snakemake/serialize.py index 382bddcd..35a5261b 100644 --- a/latch_cli/snakemake/serialize.py +++ b/latch_cli/snakemake/serialize.py @@ -321,12 +321,13 @@ def generate_snakemake_entrypoint( import shutil import subprocess from subprocess import CalledProcessError + import traceback from typing import NamedTuple import stat import sys from flytekit.extras.persistence import LatchPersistence - import traceback + from snakemake.exceptions import CreateCondaEnvironmentException from latch.resources.tasks import custom_task from latch.types.directory import LatchDir diff --git a/latch_cli/snakemake/workflow.py b/latch_cli/snakemake/workflow.py index 300419a3..17014be0 100644 --- a/latch_cli/snakemake/workflow.py +++ b/latch_cli/snakemake/workflow.py @@ -1269,14 +1269,25 @@ def get_fn_code( print("\n\n\n") try: - subprocess.run( - [sys.executable,{','.join(repr(x) for x in snakemake_args)}], - check=True, - env={{ - **os.environ, - "LATCH_SNAKEMAKE_DATA": {repr(json.dumps(snakemake_data))} - }} - ) + conda_retries = 0 + while conda_retries < 3: + try: + subprocess.run( + [sys.executable,{','.join(repr(x) for x in snakemake_args)}], + check=True, + env={{ + **os.environ, + "LATCH_SNAKEMAKE_DATA": {repr(json.dumps(snakemake_data))} + }} + ) + break + except CreateCondaEnvironmentException as e: + if conda_retries < 3: + print("Retrying snakemake\n\n") + conda_retries +=1 + continue + else: + raise e finally: if tail is not None: import signal From 75edee35051e76fa28a80e99e0e5eb5ec30e98bb Mon Sep 17 00:00:00 2001 From: Kenny Workman Date: Wed, 4 Oct 2023 20:25:28 -0700 Subject: [PATCH 14/16] Revert "retry job on conda create exception" This reverts commit 0e11a96987f6562f06f18baad2a94a8d1b58bf26. --- latch_cli/snakemake/serialize.py | 3 +-- latch_cli/snakemake/workflow.py | 27 ++++++++------------------- 2 files changed, 9 insertions(+), 21 deletions(-) diff --git a/latch_cli/snakemake/serialize.py b/latch_cli/snakemake/serialize.py index 35a5261b..382bddcd 100644 --- a/latch_cli/snakemake/serialize.py +++ b/latch_cli/snakemake/serialize.py @@ -321,13 +321,12 @@ def generate_snakemake_entrypoint( import shutil import subprocess from subprocess import CalledProcessError - import traceback from typing import NamedTuple import stat import sys from flytekit.extras.persistence import LatchPersistence - from snakemake.exceptions import CreateCondaEnvironmentException + import traceback from latch.resources.tasks import custom_task from latch.types.directory import LatchDir diff --git a/latch_cli/snakemake/workflow.py b/latch_cli/snakemake/workflow.py index 17014be0..300419a3 100644 --- a/latch_cli/snakemake/workflow.py +++ b/latch_cli/snakemake/workflow.py @@ -1269,25 +1269,14 @@ def get_fn_code( print("\n\n\n") try: - conda_retries = 0 - while conda_retries < 3: - try: - subprocess.run( - [sys.executable,{','.join(repr(x) for x in snakemake_args)}], - check=True, - env={{ - **os.environ, - "LATCH_SNAKEMAKE_DATA": {repr(json.dumps(snakemake_data))} - }} - ) - break - except CreateCondaEnvironmentException as e: - if conda_retries < 3: - print("Retrying snakemake\n\n") - conda_retries +=1 - continue - else: - raise e + subprocess.run( + [sys.executable,{','.join(repr(x) for x in snakemake_args)}], + check=True, + env={{ + **os.environ, + "LATCH_SNAKEMAKE_DATA": {repr(json.dumps(snakemake_data))} + }} + ) finally: if tail is not None: import signal From f2069ac18e0654f24a1b15848742c14cbcce5e00 Mon Sep 17 00:00:00 2001 From: Kenny Workman Date: Wed, 4 Oct 2023 20:41:00 -0700 Subject: [PATCH 15/16] changelog + setup --- CHANGELOG.md | 18 ++++++++++++++++++ setup.py | 2 +- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9515c75f..92399912 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,24 @@ Types of changes # Latch SDK Changelog +## 2.33.0 - 2023-10-04 + +### Added + +* `directory` modifier for input / outputs +* Support `temp` by removing from compiled rules. All files / directories are +temporary because they are deleted at the end of each job on Latch. +* `multiext` output modifier +* `report` output modifier +* `params` in rules + +### Fixed + +* Replace skipped rules with `Ellipsis`. Supports rules nested in conditionals where previously an empty block was produced. +* Patched parser to generate compiled code for `workflow.include` calls Compiled workflow.include should carry `print_compilation` keyword (snakemake/snakemake#2469) +* Detect use of `conda` keyword and install in image. This effectively supports wrapper/conda keywords. +* `Iterable, Generator` cause issues as type hints when imported from `collections.abc` rather than `typing` + ## 2.32.8 - 2023-09-07 ### Fixed diff --git a/setup.py b/setup.py index d09f1f05..3831f10e 100644 --- a/setup.py +++ b/setup.py @@ -13,7 +13,7 @@ setup( name="latch", - version="v2.32.8", + version="v2.33.0", author_email="kenny@latch.bio", description="The Latch SDK", packages=find_packages(), From 53fe74ddda5d9cda503b350151e5617aa0362218 Mon Sep 17 00:00:00 2001 From: Kenny Workman Date: Wed, 4 Oct 2023 20:50:38 -0700 Subject: [PATCH 16/16] bump setup version --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 3831f10e..8f13abc2 100644 --- a/setup.py +++ b/setup.py @@ -13,7 +13,7 @@ setup( name="latch", - version="v2.33.0", + version="v2.34.0", author_email="kenny@latch.bio", description="The Latch SDK", packages=find_packages(),