stagein and stageout update

EOEPCA · Mar 27, 2023 · 709ccf4 · 709ccf4
1 parent 5104e52
commit 709ccf4
Showing 1 changed file with 96 additions and 0 deletions.
diff --git a/system/clusters/creodias/processing-and-chaining/proc-ades.yaml b/system/clusters/creodias/processing-and-chaining/proc-ades.yaml
@@ -112,6 +112,102 @@ spec:
         yaml: "{}"
 
 
+      stageout:
+        cwl: |
+          cwlVersion: v1.0
+          doc: "Run Stars for staging results"
+          class: CommandLineTool
+          hints:
+            DockerRequirement:
+              dockerPull: terradue/stars:2.3.0
+            "cwltool:Secrets":
+              secrets:
+                - ADES_STAGEOUT_AWS_SERVICEURL
+                - ADES_STAGEOUT_AWS_REGION
+                - ADES_STAGEOUT_AWS_ACCESS_KEY_ID
+                - ADES_STAGEOUT_AWS_SECRET_ACCESS_KEY
+          id: stars
+          inputs:
+            ADES_STAGEOUT_AWS_SERVICEURL:
+              type: string?
+            ADES_STAGEOUT_AWS_ACCESS_KEY_ID:
+              type: string?
+            ADES_STAGEOUT_AWS_SECRET_ACCESS_KEY:
+              type: string?
+            ADES_STAGEOUT_OUTPUT:
+              type: string?
+            ADES_STAGEOUT_AWS_REGION:
+              type: string?
+            process:
+              type: string
+          outputs:
+            s3_catalog_output:
+              outputBinding:
+                outputEval: ${ return inputs.ADES_STAGEOUT_OUTPUT + "/" + inputs.process + "/catalog.json"; }
+              type: string
+          baseCommand: ['/bin/bash', 'stageout.sh']
+          requirements:
+            InlineJavascriptRequirement: {}
+            EnvVarRequirement:
+              envDef:
+                PATH: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+            ResourceRequirement: {}
+            InitialWorkDirRequirement:
+              listing:
+              - entryname: stageout.sh
+                entry: |-
+                  #!/bin/bash
+                  AWS__ServiceURL=$(inputs.ADES_STAGEOUT_AWS_SERVICEURL)
+                  AWS__Region=$(inputs.ADES_STAGEOUT_AWS_REGION)
+                  AWS__AuthenticationRegion=$(inputs.ADES_STAGEOUT_AWS_REGION)
+                  AWS_ACCESS_KEY_ID=$(inputs.ADES_STAGEOUT_AWS_ACCESS_KEY_ID)
+                  AWS_SECRET_ACCESS_KEY=$(inputs.ADES_STAGEOUT_AWS_SECRET_ACCESS_KEY)
+
+                  Stars copy -v -r 4 -o $( inputs.ADES_STAGEOUT_OUTPUT + "/" + inputs.process ) \
+                  $(inputs.wf_outputs.path) + "/catalog.json"
+
+
+      stagein:
+        cwl: |
+          cwlVersion: v1.0
+          doc: "Run Stars for staging input data"
+          class: CommandLineTool
+          hints:
+            DockerRequirement:
+              dockerPull: terradue/stars:1.0.0-beta.11
+            "cwltool:Secrets":
+              secrets:
+              - ADES_STAGEIN_AWS_SERVICEURL
+              - ADES_STAGEIN_AWS_ACCESS_KEY_ID
+              - ADES_STAGEIN_AWS_SECRET_ACCESS_KEY
+          id: stars
+          inputs:
+            ADES_STAGEIN_AWS_SERVICEURL:
+              type: string?
+            ADES_STAGEIN_AWS_ACCESS_KEY_ID:
+              type: string?
+            ADES_STAGEIN_AWS_SECRET_ACCESS_KEY:
+              type: string?
+          outputs: {}
+          baseCommand: ['/bin/bash', 'stagein.sh']
+          requirements:
+
+            InitialWorkDirRequirement:
+              listing:
+              - entryname: stagein.sh
+                entry: |-
+                  #!/bin/bash
+                  export AWS__ServiceURL=$(inputs.ADES_STAGEIN_AWS_SERVICEURL)
+                  export AWS_ACCESS_KEY_ID=$(inputs.ADES_STAGEIN_AWS_ACCESS_KEY_ID)
+                  export AWS_SECRET_ACCESS_KEY=$(inputs.ADES_STAGEIN_AWS_SECRET_ACCESS_KEY)
+                  Stars copy -v -rel -r 4 -o ./ --harvest $1
+            EnvVarRequirement:
+              envDef:
+                PATH: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+            ResourceRequirement: {}
+
+
+
       # kubernetes storage class to be used for provisioning volumes. Must be a persistent volume claim compliant (glusterfs-storage)
       processingStorageClass: managed-nfs-storage
       # Size of the Kubernetes Tmp Volumes