diff --git a/README.adoc b/README.adoc index 13b79d8..146583b 100644 --- a/README.adoc +++ b/README.adoc @@ -203,6 +203,15 @@ Use this tipe to describe a HPC job. ** `bootstrap`: Relative path to blueprint to the script that will be executed in the HPC at the install workflow to bootstrap the job (like data movements, binary download, etc.) ** `revert`: Relative path to blueprint to the script that will be executed in the HPC at the uninstall workflow, reverting the bootstrap or other clean up operations. ** `inputs`: List of inputs that will be passed to the scripts when executed in the HPC. +* `publish`: A list of outputs to be published after job execution. Each list item is a dictionary containing: +** `type`: Type of the external repository to be published. Only `CKAN` is supported for now. The rest of the parameters depends on the type. +** `type: CKAN` +*** `entrypoint`: ckan entrypoint +*** `api_key`: Individual user ckan api key. +*** `dataset`: Id of the dataset in which the file will be published. +*** `file_path`: Local path of the output file in the computation node. +*** `name`: Name used to publish the file in the repository. +*** `description`: Text describing the data file. * `skip_cleanup`: Set to true to not clean up orchestrator auxiliar files. Default `False`. NOTE: The variable $CURRENT_WORKDIR is available in all operations and scripts. It points to the working directory of the execution in the HPC from the _HOME_ directory: `/home/user/$CURRENT_WORKDIR/`. diff --git a/hpc_plugin/external_repositories/external_repository.py b/hpc_plugin/external_repositories/external_repository.py index 9937683..4c6874a 100644 --- a/hpc_plugin/external_repositories/external_repository.py +++ b/hpc_plugin/external_repositories/external_repository.py @@ -53,7 +53,7 @@ def publish(self, return ssh_client.execute_shell_command( call, workdir=workdir, - wait_result=False) # TODO: poner a true + wait_result=False) def _build_publish_call(self, logger): """ diff --git a/hpc_plugin/ssh.py b/hpc_plugin/ssh.py index d88d750..8ba1adb 100644 --- a/hpc_plugin/ssh.py +++ b/hpc_plugin/ssh.py @@ -95,7 +95,18 @@ def close_connection(self): def execute_shell_command(self, cmd, workdir=None, - wait_result=False): + wait_result=False, + detach=False): + """ Execute the command remotely + - if workdir is set: in the specific workdir + - if wait_result is set to True: blocks until it gather + the results + - if detach is set tu True: let the command running in the background. + it is incompatible with wait_result=True""" + if detach: + wait_result = False + cmd = "nohup " + cmd + " &" + if not workdir: return self.send_command(cmd, wait_result=wait_result) diff --git a/hpc_plugin/workflows.py b/hpc_plugin/workflows.py index 597faba..fc00e7e 100644 --- a/hpc_plugin/workflows.py +++ b/hpc_plugin/workflows.py @@ -96,7 +96,6 @@ def queue(self): self.winstance.send_event('..HPC job queued') init_state = 'PENDING' self.set_status(init_state) - # print result.task.dump() return result.task def publish(self): @@ -104,14 +103,13 @@ def publish(self): if not self.parent_node.is_job: return - self.winstance.send_event('Publishing HPC job..') + self.winstance.send_event('Publishing job outputs..') result = self.winstance.execute_operation('hpc.interfaces.' 'lifecycle.publish', kwargs={"name": self.name}) - # TODO: How to do it in non-blocking?? result.task.wait_for_terminated() if result.task.get_state() != tasks.TASK_FAILED: - self.winstance.send_event('..HPC job published') + self.winstance.send_event('..outputs sent for publication') return result.task @@ -125,7 +123,7 @@ def set_status(self, status): self._status == 'COMPLETED' if self.completed: - self.publish() # TODO: do it in another thread? + self.publish() if not self.parent_node.is_job: self.failed = False