README & Publish operation non-blocking #45

README.adoc

@@ -203,6 +203,15 @@ Use this tipe to describe a HPC job.
 ** `bootstrap`: Relative path to blueprint to the script that will be executed in the HPC at the install workflow to bootstrap the job (like data movements, binary download, etc.)
 ** `revert`: Relative path to blueprint to the script that will be executed in the HPC at the uninstall workflow, reverting the bootstrap or other clean up operations.
 ** `inputs`: List of inputs that will be passed to the scripts when executed in the HPC.
+* `publish`: A list of outputs to be published after job execution. Each list item is a dictionary containing:
+** `type`: Type of the external repository to be published. Only `CKAN` is supported for now. The rest of the parameters depends on the type.
+** `type: CKAN`
+*** `entrypoint`: ckan entrypoint
+*** `api_key`: Individual user ckan api key.
+*** `dataset`: Id of the dataset in which the file will be published.
+*** `file_path`: Local path of the output file in the computation node.
+*** `name`: Name used to publish the file in the repository.
+*** `description`: Text describing the data file.
 * `skip_cleanup`: Set to true to not clean up orchestrator auxiliar files. Default `False`.
 
 NOTE: The variable $CURRENT_WORKDIR is available in all operations and scripts. It points to the working directory of the execution in the HPC from the _HOME_ directory: `/home/user/$CURRENT_WORKDIR/`.

hpc_plugin/external_repositories/external_repository.py

@@ -53,7 +53,7 @@ def publish(self,
         return ssh_client.execute_shell_command(
             call,
             workdir=workdir,
-            wait_result=False)  # TODO: poner a true
+            wait_result=False)
 
     def _build_publish_call(self, logger):
         """

hpc_plugin/ssh.py

@@ -95,7 +95,18 @@ def close_connection(self):
     def execute_shell_command(self,
                               cmd,
                               workdir=None,
-                              wait_result=False):
+                              wait_result=False,
+                              detach=False):
+        """ Execute the command remotely
+        - if workdir is set: in the specific workdir
+        - if wait_result is set to True: blocks until it gather
+          the results
+        - if detach is set tu True: let the command running in the background.
+          it is incompatible with wait_result=True"""
+        if detach:
+            wait_result = False
+            cmd = "nohup " + cmd + " &"
+
         if not workdir:
             return self.send_command(cmd,
                                      wait_result=wait_result)

hpc_plugin/workflows.py

@@ -96,22 +96,20 @@ def queue(self):
             self.winstance.send_event('..HPC job queued')
             init_state = 'PENDING'
         self.set_status(init_state)
-        # print result.task.dump()
         return result.task
 
     def publish(self):
         """ Send the instance to the HPC queue if it is a Job """
         if not self.parent_node.is_job:
             return
 
-        self.winstance.send_event('Publishing HPC job..')
+        self.winstance.send_event('Publishing job outputs..')
         result = self.winstance.execute_operation('hpc.interfaces.'
                                                   'lifecycle.publish',
                                                   kwargs={"name": self.name})
-        # TODO: How to do it in non-blocking??
         result.task.wait_for_terminated()
         if result.task.get_state() != tasks.TASK_FAILED:
-            self.winstance.send_event('..HPC job published')
+            self.winstance.send_event('..outputs sent for publication')
 
         return result.task
 
@@ -125,7 +123,7 @@ def set_status(self, status):
                 self._status == 'COMPLETED'
 
             if self.completed:
-                self.publish() # TODO: do it in another thread?
+                self.publish()
 
             if not self.parent_node.is_job:
                 self.failed = False