Merge pull request #326 from latchbio/kenny/sm-gatk

Kenny/sm gatk

CHANGELOG.md

@@ -16,6 +16,24 @@ Types of changes
 
 # Latch SDK Changelog
 
+## 2.34.0 - 2023-10-04
+
+### Added
+
+* `directory` modifier for input / outputs
+* Support `temp` by removing from compiled rules. All files / directories are
+temporary because they are deleted at the end of each job on Latch.
+* `multiext` output modifier
+* `report` output modifier
+* `params` in rules
+
+### Fixed
+
+* Replace skipped rules with `Ellipsis`. Supports rules nested in conditionals where previously an empty block was produced.
+* Patched parser to generate compiled code for `workflow.include` calls Compiled workflow.include should carry `print_compilation` keyword (snakemake/snakemake#2469)
+* Detect use of `conda` keyword and install in image. This effectively supports wrapper/conda keywords.
+* `Iterable, Generator` cause issues as type hints when imported from `collections.abc` rather than `typing`
+
 ## 2.33.0 - 2023-09-29
 
 ### Added

latch_cli/snakemake/single_task_snakemake.py

@@ -9,6 +9,7 @@
 from snakemake.parser import (
     INDENT,
     Benchmark,
+    Include,
     Input,
     Log,
     Output,
@@ -28,6 +29,10 @@ def eprint(x: str) -> None:
     print(x, file=sys.stderr)
 
 
+print_compilation = os.environ.get("LATCH_PRINT_COMPILATION", False)
+if print_compilation == "1":
+    print_compilation = True
+
 data = json.loads(os.environ["LATCH_SNAKEMAKE_DATA"])
 rules = data["rules"]
 outputs = data["outputs"]
@@ -154,10 +159,23 @@ def emit_overrides(self, token):
     else:
         raise ValueError(f"tried to emit overrides for unknown state: {type(self)}")
 
-    positional_data = (render_annotated_str_list(x) for x in xs["positional"])
-    keyword_data = (
-        f"{k}={render_annotated_str_list(v)}" for k, v in xs["keyword"].items()
-    )
+    if (
+        isinstance(self, Output)
+        and len(xs["positional"]) > 0
+        and xs["positional"][0].get("flags") is not None
+        and "multiext" in xs["positional"][0].get("flags")
+    ):
+        filename = repr(xs["positional"][0]["flags"]["multiext"])
+        exts = [repr("." + x["value"].split(".")[-1]) for x in xs["positional"]]
+        positional_data = (f"multiext({filename},{','.join(exts)})",)
+    else:
+        positional_data = (render_annotated_str_list(x) for x in xs["positional"])
+
+    modifier_fn = render_annotated_str_list
+    if isinstance(self, Params):
+        modifier_fn = repr
+
+    keyword_data = (f"{k}={modifier_fn(v)}" for k, v in xs["keyword"].items())
     data = chain(positional_data, keyword_data)
 
     for x in data:
@@ -188,17 +206,33 @@ def skipping_block_content(self, token):
     emitted_overrides.add(self.rulename)
 
 
+def block_content_with_print_compilation(self, token):
+    if print_compilation:
+        yield f"{token.string}, print_compilation=True", token
+    else:
+        yield token.string, token
+
+
 Input.block_content = skipping_block_content
 Output.block_content = skipping_block_content
 Params.block_content = skipping_block_content
 Benchmark.block_content = skipping_block_content
 Log.block_content = skipping_block_content
 Ruleorder.block_content = lambda self, token: None
+Include.block_content = block_content_with_print_compilation
 
 
 class SkippingRule(Rule):
     def start(self, aux=""):
         if self.rulename not in rules:
+            # Rules can be nested in conditional statements:
+            #
+            # if (<condition>):
+            #   rule A:
+            #       <stuff>
+            #
+            # We want correct python code if we remove them.
+            yield "..."
             return
 
         yield from super().start(aux)

latch_cli/snakemake/workflow.py

@@ -1243,7 +1243,8 @@ def get_fn_code(
                         check=True,
                         env={{
                             **os.environ,
-                            "LATCH_SNAKEMAKE_DATA": {repr(json.dumps(snakemake_data))}
+                            "LATCH_SNAKEMAKE_DATA": {repr(json.dumps(snakemake_data))},
+                            "LATCH_PRINT_COMPILATION": "1"
                         }},
                         stdout=f
                     )
@@ -1325,21 +1326,6 @@ def get_fn_code(
             finally:
                 ignored_paths = {{".cache", ".snakemake/conda"}}
                 ignored_names = {{".git", ".latch", "__pycache__"}}
-
-                print("Recursive directory listing:")
-                stack = [(Path("."), 0)]
-                while len(stack) > 0:
-                    cur, indent = stack.pop()
-                    print("  " * indent + cur.name)
-
-                    if cur.is_dir():
-                        if cur.name in ignored_names or str(cur) in ignored_paths:
-                            print("  " * indent + "  ...")
-                            continue
-
-                        for x in cur.iterdir():
-                            stack.append((x, indent + 1))
-
             """,
             1,
         )

setup.py

@@ -13,7 +13,7 @@
 
 setup(
     name="latch",
-    version="v2.33.0",
+    version="v2.34.0",
     author_email="[email protected]",
     description="The Latch SDK",
     packages=find_packages(),