Add support for Caper (#352)

* Add caper

* Version bump + docs

* Restore release script

* Fix tests

awsf3-docker/Dockerfile

@@ -85,6 +85,9 @@ RUN wget https://github.com/broadinstitute/cromwell/releases/download/53.1/cromw
 RUN wget https://github.com/broadinstitute/cromwell/releases/download/35/cromwell-35.jar
 RUN wget https://github.com/broadinstitute/cromwell/blob/develop/LICENSE.txt  # cromwell license
 
+# Caper - uses cromwell 59 under the hood
+RUN pip install caper==1.6.3
+
 # awsf scripts
 COPY run.sh .
 COPY cron.sh .

awsf3-docker/run.sh

@@ -122,6 +122,7 @@ exl awsf3 upload_postrun_json -i $POSTRUN_JSON_FILE_NAME
 
 
 # setting additional env variables including LANGUAGE and language-related envs.
+# This file is created in decode_run_json
 exl source $ENV_FILE
 
 
@@ -209,8 +210,19 @@ cwd0=$(pwd)
 cd $LOCAL_WFDIR
 if [[ $LANGUAGE == 'wdl_v1' || $LANGUAGE == 'wdl' ]]
 then
-  exl java -jar /usr/local/bin/cromwell.jar run $MAIN_WDL -i $cwd0/$INPUT_YML_FILE -m $LOGJSONFILE
-  handle_error $?
+  if [[ $WORKFLOW_ENGINE == 'caper' ]]
+  then
+    exl echo "## Using the Caper workflow engine"
+    exl echo "## Initializing Caper"
+    # Since we are running caper in LOCAL_WFDIR we don't need to set local-loc-dir in the caper config
+    exl caper init local
+    exl caper run $MAIN_WDL -i $cwd0/$INPUT_YML_FILE -m $LOGJSONFILE
+    handle_error $?
+  else # default to cromwell
+    exl echo "## Using the Cromwell workflow engine"
+    exl java -jar /usr/local/bin/cromwell.jar run $MAIN_WDL -i $cwd0/$INPUT_YML_FILE -m $LOGJSONFILE
+    handle_error $?
+  fi
 elif [[ $LANGUAGE == 'wdl_draft2' ]]  # 'wdl' defaults to 'wdl_v1'
 then
   exl echo "## using cromwell-35 for WDL draft2"

awsf3/aws_run_workflow_generic.sh

@@ -208,7 +208,7 @@ until [ $tries -ge 3 ]; do
     break 
   else
     ((tries++))
-    sleep 1
+    sleep 60
   fi
 done
 send_log

awsf3/utils.py

@@ -185,6 +185,7 @@ def create_env_def_file(env_filename, runjson, language):
         if language in ['wdl', 'wdl_v1', 'wdl_draft2']:
             f_env.write("export WDL_URL={}\n".format(app.wdl_url))
             f_env.write("export MAIN_WDL={}\n".format(app.main_wdl))
+            f_env.write("export WORKFLOW_ENGINE={}\n".format(app.workflow_engine))
             f_env.write("export WDL_FILES=\"{}\"\n".format(' '.join(app.other_wdl_files.split(','))))
         elif language == 'snakemake':
             f_env.write("export SNAKEMAKE_URL={}\n".format(app.snakemake_url))

docs/wdl.rst

@@ -4,3 +4,6 @@ Workflow Description Language (WDL)
 
 Tibanna version < ``1.0.0`` supports WDL draft-2, through Cromwell binary version 35. Tibanna version >= ``1.0.0`` supports both WDL draft-2 and v1.0, through Cromwell binary version 35 and 53, respectively. This is because some of our old WDL pipelines written in draft-2 version no longer works with the new Cromwell version and we wanted to ensure the backward compatibility. But if you want to use WDL draft-2, specify ``"language": "wdl_draft2"`` instead of ``"language": "wdl"`` which defaults to WDL v1.0.
 
+Tibanna version >= ``1.7.0`` supports (Caper_) in addition to Cromwell. If you would like to use Caper, add ``"workflow_engine": "caper"`` to the Tibanna job description. Cromwell is the default.
+
+.. _Caper: https://github.com/ENCODE-DCC/caper

tests/awsf3/test_utils.py

@@ -69,6 +69,7 @@ def test_create_env_def_file_wdl_v1():
     runjson_dict = {'Job': {'App': {'language': 'wdl_v1',
                                     'wdl_url': 'someurl',
                                     'main_wdl': 'somewdl',
+                                    'workflow_engine': 'cromwell',
                                     'other_wdl_files': 'otherwdl1,otherwdl2'},
                             'Input': {'Env': {}},
                             'Output': {'output_bucket_directory': 'somebucket'},
@@ -83,6 +84,7 @@ def test_create_env_def_file_wdl_v1():
     right_content = ('export LANGUAGE=wdl_v1\n'
                      'export WDL_URL=someurl\n'
                      'export MAIN_WDL=somewdl\n'
+                     'export WORKFLOW_ENGINE=cromwell\n'
                      'export WDL_FILES="otherwdl1 otherwdl2"\n'
                      'export PRESERVED_ENV_OPTION=""\n'
                      'export DOCKER_ENV_OPTION=""\n')
@@ -98,6 +100,7 @@ def test_create_env_def_file_wdl_draft2():
     runjson_dict = {'Job': {'App': {'language': 'wdl_draft2',
                                     'wdl_url': 'someurl',
                                     'main_wdl': 'somewdl',
+                                    'workflow_engine': 'cromwell',
                                     'other_wdl_files': 'otherwdl1,otherwdl2'},
                             'Input': {'Env': {}},
                             'Output': {'output_bucket_directory': 'somebucket'},
@@ -112,6 +115,7 @@ def test_create_env_def_file_wdl_draft2():
     right_content = ('export LANGUAGE=wdl_draft2\n'
                      'export WDL_URL=someurl\n'
                      'export MAIN_WDL=somewdl\n'
+                     'export WORKFLOW_ENGINE=cromwell\n'
                      'export WDL_FILES="otherwdl1 otherwdl2"\n'
                      'export PRESERVED_ENV_OPTION=""\n'
                      'export DOCKER_ENV_OPTION=""\n')
@@ -127,6 +131,7 @@ def test_create_env_def_file_wdl():
     runjson_dict = {'Job': {'App': {'language': 'wdl',
                                     'wdl_url': 'someurl',
                                     'main_wdl': 'somewdl',
+                                    'workflow_engine': 'cromwell',
                                     'other_wdl_files': 'otherwdl1,otherwdl2'},
                             'Input': {'Env': {}},
                             'Output': {'output_bucket_directory': 'somebucket'},
@@ -141,6 +146,7 @@ def test_create_env_def_file_wdl():
     right_content = ('export LANGUAGE=wdl\n'
                      'export WDL_URL=someurl\n'
                      'export MAIN_WDL=somewdl\n'
+                     'export WORKFLOW_ENGINE=cromwell\n'
                      'export WDL_FILES="otherwdl1 otherwdl2"\n'
                      'export PRESERVED_ENV_OPTION=""\n'
                      'export DOCKER_ENV_OPTION=""\n')

tibanna/_version.py

@@ -1,4 +1,4 @@
 """Version information."""
 
 # The following line *must* be the last in the module, exactly as formatted:
-__version__ = "1.6.0"
+__version__ = "1.7.0"

tibanna/awsem.py

@@ -70,7 +70,7 @@ def __init__(self, log_bucket_directory=None):
 class AwsemRunJsonApp(SerializableObject):
     def __init__(self, App_name=None, App_version=None, language=None,
                  cwl_url=None, main_cwl=None, other_cwl_files=None,
-                 wdl_url=None, main_wdl=None, other_wdl_files=None,
+                 wdl_url=None, main_wdl=None, other_wdl_files=None, workflow_engine=None,
                  container_image=None, command=None,
                  snakemake_url=None, main_snakemake=None, other_snakemake_files=None):
         self.App_name = App_name
@@ -82,6 +82,7 @@ def __init__(self, App_name=None, App_version=None, language=None,
         self.wdl_url = wdl_url
         self.main_wdl = main_wdl
         self.other_wdl_files = other_wdl_files
+        self.workflow_engine = workflow_engine
         self.container_image = container_image
         self.command = command
         self.snakemake_url = snakemake_url

tibanna/ec2_utils.py

@@ -169,6 +169,8 @@ def fill_default(self):
                 self.wdl_directory_local = ''
             if not hasattr(self, 'wdl_directory_url'):
                 self.wdl_directory_url = ''
+            if not hasattr(self, 'workflow_engine'):
+                self.workflow_engine = 'cromwell'
             if not self.wdl_directory_local and not self.wdl_directory_url:
                 errmsg = "either %s or %s must be provided in args" % ('wdl_directory_url', 'wdl_directory_local')
                 raise MissingFieldInInputJsonException(errmsg)
@@ -578,6 +580,7 @@ def create_run_json_dict(self):
                 'main_wdl': args.wdl_main_filename,
                 'other_wdl_files': ','.join(args.wdl_child_filenames),
                 'wdl_url': args.wdl_directory_url,
+                'workflow_engine': args.workflow_engine
             })
         elif args.language == 'snakemake':
             app.update({