Merge pull request #79 from rabix/feature/sample_sheet_support

Feature/sample sheet support

.gitignore

@@ -2,3 +2,4 @@
 /dist/
 /sbpack.egg-info/
 /.idea/
+/venv/

MANIFEST.in

@@ -0,0 +1 @@
+include sbpack/noncwl/js_templates/*.js

requirements.txt

@@ -2,4 +2,4 @@ ruamel.yaml >= 0.16
 sevenbridges-python >= 2.0
 nf-core==2.1
 cwlformat
-packaging
+packaging

sbpack/lib.py

@@ -1,7 +1,8 @@
-from typing import Union
+from typing import Union, Optional
 from copy import deepcopy
-import urllib.parse
-import urllib.request
+from urllib.parse import ParseResult, urlparse, urljoin
+from urllib.request import urlopen
+from urllib.error import HTTPError
 import pathlib
 import sys
 
@@ -71,7 +72,8 @@ def normalize_to_map(obj: Union[list, dict], key_field: str):
         raise RuntimeError("Expecting a dictionary or a list here")
 
 
-def normalize_to_list(obj: Union[list, dict], key_field: str, value_field: str):
+def normalize_to_list(
+        obj: Union[list, dict], key_field: str, value_field: Optional[str]):
     if isinstance(obj, list):
         return deepcopy(obj)
     elif isinstance(obj, dict):
@@ -89,8 +91,8 @@ def normalize_to_list(obj: Union[list, dict], key_field: str, value_field: str):
 
 
 # To deprecate
-def normalized_path(link: str, base_url: urllib.parse.ParseResult):
-    link_url = urllib.parse.urlparse(link)
+def normalized_path(link: str, base_url: ParseResult):
+    link_url = urlparse(link)
     if link_url.scheme in ["file://", ""]:
         new_url = base_url._replace(
             path=str((pathlib.Path(base_url.path) / pathlib.Path(link)).resolve())
@@ -101,7 +103,7 @@ def normalized_path(link: str, base_url: urllib.parse.ParseResult):
     return new_url
 
 
-def resolved_path(base_url: urllib.parse.ParseResult, link: str):
+def resolved_path(base_url: ParseResult, link: str):
     """
     Given a base_url ("this document") and a link ("string in this document")
     return a new url (urllib.parse.ParseResult) that allows us to retrieve the
@@ -110,12 +112,12 @@ def resolved_path(base_url: urllib.parse.ParseResult, link: str):
     2. Use the OS appropriate path resolution for local paths, and network
        apropriate resolution for network paths
     """
-    link_url = urllib.parse.urlparse(link)
+    link_url = urlparse(link)
     # The link will always Posix
 
     if link_url.scheme == "file://":
         # Absolute local path
-        new_url = urllib.parse.ParseResult(link_url)
+        new_url = ParseResult(str(link_url))
 
     elif link_url.scheme == "":
         # Relative path, can be local or remote
@@ -130,7 +132,7 @@ def resolved_path(base_url: urllib.parse.ParseResult, link: str):
 
         else:
             # Remote relative path
-            new_url = urllib.parse.urlparse(urllib.parse.urljoin(base_url.geturl(), link_url.path))
+            new_url = urlparse(urljoin(base_url.geturl(), link_url.path))
             # We need urljoin because we need to resolve relative links in a
             # platform independent manner
 
@@ -141,16 +143,16 @@ def resolved_path(base_url: urllib.parse.ParseResult, link: str):
     return new_url
 
 
-def load_linked_file(base_url: urllib.parse.ParseResult, link: str, is_import=False):
+def load_linked_file(base_url: ParseResult, link: str, is_import=False):
 
     new_url = resolved_path(base_url, link)
 
     if new_url.scheme in ["file://", ""]:
         contents = pathlib.Path(new_url.path).open().read()
     else:
         try:
-            contents = urllib.request.urlopen(new_url.geturl()).read().decode("utf-8")
-        except urllib.error.HTTPError as e:
+            contents = urlopen(new_url.geturl()).read().decode("utf-8")
+        except HTTPError as e:
             e.msg += f"\n===\nCould not find linked file: {new_url.geturl()}\n===\n"
             raise SystemExit(e)
 
@@ -177,7 +179,7 @@ def load_linked_file(base_url: urllib.parse.ParseResult, link: str, is_import=Fa
     return _node, new_url
 
 
-def _is_github_symbolic_link(base_url: urllib.parse.ParseResult, contents: str):
+def _is_github_symbolic_link(base_url: ParseResult, contents: str):
     """Look for remote path with contents that is a single line with no new
     line with an extension."""
     if base_url.scheme in ["file://", ""]:

sbpack/noncwl/Readme.md

@@ -7,25 +7,21 @@ to the Platform.
 ```
 $ sbpack_wdl -h
 
-usage: sbpack_wdl [-h] [--profile PROFILE] --appid APPID --entrypoint ENTRYPOINT --workflow-path WORKFLOW_PATH [--sb-package-id SB_PACKAGE_ID] [--womtool-input WOMTOOL_INPUT]
-                  [--sb-inputs SB_INPUTS] [--sb-outputs SB_OUTPUTS] [--sb-doc SB_DOC] [--dump-sb-app] [--no-package] [--womtool-path WOMTOOL_PATH] [--json] [--sb-schema SB_SCHEMA]
+usage: sbpack_wdl [-h] [--profile PROFILE] --appid APPID --entrypoint ENTRYPOINT --workflow-path WORKFLOW_PATH [--sb-package-id SB_PACKAGE_ID] [--womtool-input WOMTOOL_INPUT] [--sb-doc SB_DOC] [--dump-sb-app] [--no-package]
+                  [--womtool-path WOMTOOL_PATH] [--json] [--sb-schema SB_SCHEMA]
 
 optional arguments:
   -h, --help            show this help message and exit
   --profile PROFILE     SB platform profile as set in the SB API credentials file.
   --appid APPID         Takes the form {user or division}/{project}/{app_id}.
   --entrypoint ENTRYPOINT
-                        Relative path to the workflow from the main workflow directory. If not provided, 'main.nf' will be used if available. If not available, but a single '*.nf' is located in the workflow-path will be used. If more than one '*.nf' script is detected, an error is raised.
+                        Relative path to the workflow from the main workflow directory
   --workflow-path WORKFLOW_PATH
                         Path to the main workflow directory
   --sb-package-id SB_PACKAGE_ID
                         Id of an already uploaded package
   --womtool-input WOMTOOL_INPUT
                         Path to inputs .JSON generated by womtool
-  --sb-inputs SB_INPUTS
-                        Path to pre built sb app inputs schema
-  --sb-outputs SB_OUTPUTS
-                        Path to pre build sb app outputs schema
   --sb-doc SB_DOC       Path to a doc file for sb app. If not provided, README.md will be used if available
   --dump-sb-app         Dump created sb app to file if true and exit
   --no-package          Only provide a sb app schema and a git URL for entrypoint
@@ -53,14 +49,20 @@ to the Platform.
 ```
 $ sbpack_nf -h
 
-usage: nextflow.py [-h] [--profile PROFILE] --appid APPID --workflow-path
-                   WORKFLOW_PATH [--entrypoint ENTRYPOINT]
-                   [--sb-package-id SB_PACKAGE_ID] [--sb-doc SB_DOC]
-                   [--dump-sb-app] [--no-package]
-                   [--executor-version EXECUTOR_VERSION]
-                   [--execution-mode {single-instance,multi-instance}]
-                   [--json] [--sb-schema SB_SCHEMA]
-                   [--revision-note REVISION_NOTE]
+usage: sbpack_nf [-h]
+                 [--profile PROFILE]
+                 --appid APPID 
+                 --workflow-path WORKFLOW_PATH
+                 [--entrypoint ENTRYPOINT]
+                 [--sb-package-id SB_PACKAGE_ID]
+                 [--sb-doc SB_DOC]
+                 [--dump-sb-app]
+                 [--no-package]
+                 [--executor-version EXECUTOR_VERSION]
+                 [--execution-mode {single-instance,multi-instance}]
+                 [--json]
+                 [--sb-schema SB_SCHEMA]
+                 [--revision-note REVISION_NOTE]
 
 optional arguments:
   -h, --help            show this help message and exit
@@ -113,8 +115,10 @@ Note: The link between the original, and the new app will not be available.
 
 ```
 sbcopy -h
-usage: copy_app.py [-h] [--profile PROFILE [PROFILE ...]] --appid APPID
-                   --projectid PROJECTID
+usage: sbcopy [-h]
+              [--profile PROFILE [PROFILE ...]]
+              --appid APPID 
+              --projectid PROJECTID
 
 optional arguments:
   -h, --help            show this help message and exit

sbpack/noncwl/__init__.py

@@ -0,0 +1,9 @@
+import os
+
+PACKAGE_PATH = os.path.abspath(os.path.dirname(__file__))
+JS_PATH = os.path.join(PACKAGE_PATH, 'js_templates')
+
+
+def read_js_template(file_name):
+    with open(os.path.join(JS_PATH, file_name), 'r') as f:
+        return f.read()

sbpack/noncwl/constants.py

@@ -1,7 +1,10 @@
 from enum import Enum
+from sbpack.noncwl import read_js_template
+
 
 # ############################## Generic Bits ############################### #
 PACKAGE_SIZE_LIMIT = 100 * 1024 * 1024  # 100 MB
+REMOVE_INPUT_KEY = "REMOVE_THIS_KEY"
 
 
 # keep track of what extensions are applicable for processing
@@ -19,6 +22,9 @@ class EXTENSIONS:
 # ############################ CWL Standard Bits ############################ #
 # A generic SB input array of files that should be available on the
 # instance but are not explicitly provided to the execution as wdl params.
+SAMPLE_SHEET_FUNCTION = read_js_template("sample_sheet_generator.js")
+SAMPLE_SHEET_SWITCH = read_js_template("sample_sheet_switch.js")
+
 GENERIC_FILE_ARRAY_INPUT = {
     "id": "auxiliary_files",
     "type": "File[]?",
@@ -27,6 +33,14 @@ class EXTENSIONS:
            "required for workflow execution."
 }
 
+SAMPLE_SHEET_FILE_ARRAY_INPUT = {
+    "id": "file_input",
+    "type": "File[]?",
+    "label": "Input files",
+    "doc": "List of files that will be used to autogenerate the sample sheet "
+           "that is required for workflow execution."
+}
+
 GENERIC_NF_OUTPUT_DIRECTORY = {
     "id": "nf_workdir",
     "type": "Directory?",
@@ -50,22 +64,52 @@ class EXTENSIONS:
     }
 }
 
-# Requirements to be added to sb wrapper
+# Requirements for sb wrapper
+INLINE_JS_REQUIREMENT = {
+    'class': "InlineJavascriptRequirement"
+}
+LOAD_LISTING_REQUIREMENT = {
+    'class': "LoadListingRequirement"
+}
+AUX_FILES_REQUIREMENT = {
+    "class": "InitialWorkDirRequirement",
+    "listing": [
+        "$(inputs.auxiliary_files)"
+    ]
+}
+
+# Legacy - Delete after updating wdl.py
 WRAPPER_REQUIREMENTS = [
-    {
-        "class": "InlineJavascriptRequirement"
-    },
-    {
+    INLINE_JS_REQUIREMENT,
+    AUX_FILES_REQUIREMENT
+]
+
+
+def sample_sheet(
+        file_name, sample_sheet_input, format_, input_source, header, rows,
+        defaults, group_by):
+    basename = ".".join(file_name.split(".")[:-1])
+    ext = file_name.split(".")[-1]
+    new_name = f"{basename}.new.{ext}"
+
+    return {
         "class": "InitialWorkDirRequirement",
         "listing": [
-            "$(inputs.auxiliary_files)"
+            {
+                "entryname": f"${{ return {sample_sheet_input} ? {sample_sheet_input}.nameroot + '.new' + {sample_sheet_input}.nameext : '{file_name}' }}",
+                "entry": SAMPLE_SHEET_FUNCTION.format_map(locals()),
+                "writable": False
+            }
         ]
     }
-]
+
 
 # ############################## Nextflow Bits ############################## #
 # Keys that should be skipped when parsing nextflow tower yaml file
 
+NF_SCHEMA_DEFAULT_NAME = 'nextflow_schema.json'
+SB_SCHEMA_DEFAULT_NAME = 'sb_nextflow_schema'
+
 # Mappings of nextflow input fields to SB input fields
 #  nextflow_key: cwl_key mapping
 NF_TO_CWL_PORT_MAP = {
@@ -96,3 +140,6 @@ class EXTENSIONS:
 class ExecMode(Enum):
     single = 'single-instance'
     multi = 'multi-instance'
+
+    def __str__(self):
+        return self.value