Merge pull request #206 from pepkit/dev

Release 0.12.4

docs/README.md

@@ -1,4 +1,4 @@
-# <img src="img/looper_logo.svg" class="img-header"> a pipeline submitting engine
+# <img src="img/looper_logo.svg" class="img-header"> pipeline submitting engine
 
 [![PEP compatible](http://pepkit.github.io/img/PEP-compatible-green.svg)](http://pepkit.github.io)
 

docs/changelog.md

@@ -2,6 +2,17 @@
 
 This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) and [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) format. 
 
+## [0.12.4] -- 2019-07-1X
+### Added
+- Ability to declare `required_executables` in a `PipelineInterface`, to trigger a naive "runnability" check for a sample submission
+- A possibility to opt out of status page inclusion in the navbar 
+
+### Changed
+- The status tables now use DataTables jQuery plugin to make them interactive 
+
+### Fixed
+- Navbar links creation
+
 ## [0.12.3] -- 2019-06-20
 ### Fixed
 - Bug in `Sample` YAML naming, whereby a base `Sample` was being suffixed as a subtype would be, leading to a pipeline argument based on `yaml_file` that did not exist on disk.

docs/cluster-computing.md

@@ -55,6 +55,6 @@ Now, this variable will be available for use in a template as `{SINGULARITY_IMAG
 
 The other pipeline interface section that is available to templates is `resources`. This section uses a list of *resource packages* that vary based on sample input size. We use these in existing templates to adjust the amount of resources we need to request from a resource manager like SLURM. For example: `{MEM}`, `{CORES}`, and `{TIME}` are all defined in this section, and they vary for different input file sizes.
 
-[Read more about pipeline_interface.yaml here](http://looper.readthedocs.io/en/latest/pipeline-interface.html).
+[Read more about pipeline_interface.yaml here](pipeline-interface.md).
 
 *project_config.yaml*. Finally, project-level variables can also be populated from the `compute` section of a project config file. We don't recommend using this and it is not yet well documented, but it would enable you to make project-specific compute changes (such as billing a particular project to a particular SLURM resource account).

looper/__init__.py

@@ -145,17 +145,10 @@ def add_subparser(cmd):
                      "By default, pipelines will not be submitted if a sample name"
                      " is duplicated, since samples names should be unique.  "
                      " Set this option to override this setting. Default=False")
-
-        comp_spec = subparser.add_mutually_exclusive_group()
-        comp_spec.add_argument(
-                "--compute", dest=COMPUTE_KEY,
-                default=DEFAULT_COMPUTE_RESOURCES_NAME,
-                help="YAML file with looper environment compute settings.")
-        comp_spec.add_argument(
+        subparser.add_argument(
                 "--compute-package", dest=COMPUTE_KEY,
                 default=DEFAULT_COMPUTE_RESOURCES_NAME,
-                help="YAML file with looper environment compute settings.")
-
+                help="Name of computing resource package to use")
         subparser.add_argument(
                 "--resources",
                 help="Specification of individual computing resource settings; "

looper/_version.py

@@ -1 +1 @@
-__version__ = "0.12.3"
+__version__ = "0.12.4"

looper/conductor.py

@@ -6,7 +6,9 @@
 import subprocess
 import time
 
+from .const import OUTKEY
 from .exceptions import JobSubmissionException
+from .pipeline_interface import PL_KEY
 from .utils import \
     create_looper_args_text, grab_project_data, fetch_sample_flags
 
@@ -243,9 +245,8 @@ def add_sample(self, sample, rerun=False):
 
         this_sample_size = float(sample.input_file_size)
 
-        if use_this_sample and not skip_reasons:
-            assert argstring is not None, \
-                "Failed to create argstring for sample: {}".format(sample.name)
+        if _use_sample(use_this_sample, skip_reasons):
+            _check_argstring(argstring, sample.name)
             self._pool.append((sample, argstring))
             self._curr_size += this_sample_size
             if self.automatic and self._is_full(self._pool, self._curr_size):
@@ -304,20 +305,23 @@ def submit(self, force=False):
             # specific to subtype as applicable (should just be a single
             # subtype for each submission conductor, but some may just be
             # the base Sample while others are the single valid subtype.)
+            pipe_data = self.pl_iface[PL_KEY][self.pl_key]
+            try:
+                outputs = pipe_data[OUTKEY]
+            except KeyError:
+                _LOGGER.debug("No outputs for pipeline '{}'".format(self.pl_key))
+                add_outputs = lambda _: None
+            else:
+                def add_outputs(s):
+                    s[OUTKEY] = outputs
             for s, _ in self._pool:
-                if _is_base_sample(s):
-                    exp_fname = "{}{}".format(s.name, SAMPLE_YAML_EXT)
-                    exp_fpath = os.path.join(
-                            self.prj.submission_folder, exp_fname)
-                    if not os.path.isfile(exp_fpath):
-                        _LOGGER.warning(
-                            "Missing %s file will be created: '%s'",
-                            Sample.__name__, exp_fpath)
-                else:
+                if not _is_base_sample(s):
                     subtype_name = s.__class__.__name__
                     _LOGGER.debug("Writing %s representation to disk: '%s'",
                                   subtype_name, s.name)
-                    s.to_yaml(subs_folder_path=self.prj.submission_folder)
+                add_outputs(s)
+                yaml_path = s.to_yaml(subs_folder_path=self.prj.submission_folder)
+                _LOGGER.debug("Wrote sample YAML: {}".format(yaml_path))
 
             script = self.write_script(self._pool, self._curr_size)
 
@@ -451,5 +455,14 @@ def _reset_curr_skips(self):
         self._curr_skip_size = 0
 
 
+def _check_argstring(argstring, sample_name):
+    assert argstring is not None, \
+        "Failed to create argstring for sample: {}".format(sample_name)
+
+
 def _is_base_sample(s):
     return type(s) is Sample
+
+
+def _use_sample(flag, skips):
+    return flag and not skips

looper/const.py

@@ -5,8 +5,8 @@
 
 
 __all__ = ["APPEARANCE_BY_FLAG", "NO_DATA_PLACEHOLDER", "OUTKEY",
-           "PIPELINE_INTERFACES_KEY", "RESULTS_SUBDIR_KEY",
-           "SUBMISSION_SUBDIR_KEY", "TEMPLATES_DIRNAME"]
+           "PIPELINE_INTERFACES_KEY", "PIPELINE_REQUIREMENTS_KEY",
+           "RESULTS_SUBDIR_KEY", "SUBMISSION_SUBDIR_KEY", "TEMPLATES_DIRNAME"]
 
 APPEARANCE_BY_FLAG = {
     "completed": {
@@ -32,6 +32,7 @@
 }
 NO_DATA_PLACEHOLDER = "NA"
 PIPELINE_INTERFACES_KEY = "pipeline_interfaces"
+PIPELINE_REQUIREMENTS_KEY = "required_executables"
 OUTKEY = "outputs"
 RESULTS_SUBDIR_KEY = "results_subdir"
 SUBMISSION_SUBDIR_KEY = "submission_subdir"

looper/exceptions.py

@@ -1,20 +1,18 @@
 """ Exceptions for specific looper issues. """
 
 from abc import ABCMeta
-import sys
-if sys.version_info < (3, 3):
-    from collections import Iterable
-else:
-    from collections.abc import Iterable
+from collections import Iterable
 
 __author__ = "Vince Reuter"
 __email__ = "[email protected]"
 
 _all__ = ["DuplicatePipelineKeyException",
           "InvalidResourceSpecificationException",
-          "JobSubmissionException", "LooperError",
+          "JobSubmissionException",
+          "LooperError",
           "MissingPipelineConfigurationException",
-          "PipelineInterfaceConfigError"]
+          "PipelineInterfaceConfigError",
+          "PipelineInterfaceRequirementsError"]
 
 
 class LooperError(Exception):
@@ -40,7 +38,7 @@ class JobSubmissionException(LooperError):
     def __init__(self, sub_cmd, script):
         self.script = script
         reason = "Error for command {} and script '{}'".\
-                format(sub_cmd, self.script)
+            format(sub_cmd, self.script)
         super(JobSubmissionException, self).__init__(reason)
 
 
@@ -61,3 +59,12 @@ def __init__(self, context):
         if not isinstance(context, str) and isinstance(context, Iterable):
             context = "Missing section(s): {}".format(", ".join(context))
         super(PipelineInterfaceConfigError, self).__init__(context)
+
+
+class PipelineInterfaceRequirementsError(LooperError):
+    """ Invalid specification of pipeline requirements in interface config. """
+    def __init__(self, typename_by_requirement):
+        super(PipelineInterfaceRequirementsError, self).__init__(
+            "{} invalid requirements: {}".format(
+                len(typename_by_requirement), typename_by_requirement))
+        self.error_specs = typename_by_requirement