Merge branch 'develop' into gpu_support

.github/workflows/changelog.yml

@@ -13,4 +13,4 @@ on:
 
 jobs:
   call-changelog-check-workflow:
-    uses: ASFHyP3/actions/.github/workflows/[email protected].1
+    uses: ASFHyP3/actions/.github/workflows/[email protected].2

.github/workflows/create-jira-issue.yml

@@ -6,7 +6,7 @@ on:
 
 jobs:
   call-create-jira-issue-workflow:
-    uses: ASFHyP3/actions/.github/workflows/[email protected].1
+    uses: ASFHyP3/actions/.github/workflows/[email protected].2
     secrets:
       JIRA_BASE_URL: ${{ secrets.JIRA_BASE_URL }}
       JIRA_USER_EMAIL: ${{ secrets.JIRA_USER_EMAIL }}

.github/workflows/deploy-daac.yml

@@ -27,8 +27,8 @@ jobs:
             deploy_ref: refs/heads/main
             job_files: job_spec/AUTORIFT.yml job_spec/INSAR_GAMMA.yml job_spec/RTC_GAMMA.yml job_spec/INSAR_ISCE_BURST.yml
             instance_types: r6id.xlarge,r6id.2xlarge,r6id.4xlarge,r6id.8xlarge,r6idn.xlarge,r6idn.2xlarge,r6idn.4xlarge,r6idn.8xlarge
-            default_max_vcpus: 1200
-            expanded_max_vcpus: 2400
+            default_max_vcpus: 1500
+            expanded_max_vcpus: 3000
             required_surplus: 2000
             security_environment: EDC
             ami_id: image_id_ecs_amz2
@@ -50,8 +50,8 @@ jobs:
               job_spec/RTC_GAMMA.yml
               job_spec/INSAR_ISCE_BURST.yml
             instance_types: r6id.xlarge,r6id.2xlarge,r6id.4xlarge,r6id.8xlarge,r6idn.xlarge,r6idn.2xlarge,r6idn.4xlarge,r6idn.8xlarge
-            default_max_vcpus: 1200
-            expanded_max_vcpus: 2400
+            default_max_vcpus: 1500
+            expanded_max_vcpus: 3000
             required_surplus: 2000
             security_environment: EDC
             ami_id: image_id_ecs_amz2
@@ -107,6 +107,6 @@ jobs:
   call-bump-version-workflow:
     if: github.ref == 'refs/heads/main'
     needs: deploy
-    uses: ASFHyP3/actions/.github/workflows/[email protected].1
+    uses: ASFHyP3/actions/.github/workflows/[email protected].2
     secrets:
       USER_TOKEN: ${{ secrets.TOOLS_BOT_PAK }}

.github/workflows/deploy-enterprise-test.yml

@@ -47,8 +47,9 @@ jobs:
             default_application_status: APPROVED
             cost_profile: DEFAULT
             job_files: >-
+              job_spec/ARIA_AUTORIFT.yml
               job_spec/ARIA_RAIDER.yml
-              job_spec/INSAR_ISCE_TEST.yml
+              job_spec/INSAR_ISCE.yml
             instance_types: m6id.xlarge,m6id.2xlarge,m6id.4xlarge,m6id.8xlarge,m6idn.xlarge,m6idn.2xlarge,m6idn.4xlarge,m6idn.8xlarge
             default_max_vcpus: 640
             expanded_max_vcpus: 640
@@ -78,7 +79,7 @@ jobs:
 
           - environment: hyp3-lavas-test
             domain: hyp3-lavas-test.asf.alaska.edu
-            template_bucket: cf-templates-10a5pjrsv3cgo-us-west-2 
+            template_bucket: cf-templates-10a5pjrsv3cgo-us-west-2
             image_tag: test
             product_lifetime_in_days: 14
             default_credits_per_user: 0

.github/workflows/deploy-enterprise.yml

@@ -26,8 +26,8 @@ jobs:
               job_spec/AUTORIFT_ITS_LIVE.yml
               job_spec/S1_CORRECTION_ITS_LIVE.yml
             instance_types: r6id.xlarge,r6idn.xlarge,r5dn.xlarge,r5d.xlarge
-            default_max_vcpus: 10000
-            expanded_max_vcpus: 10000
+            default_max_vcpus: 2000  # Max: 10,406
+            expanded_max_vcpus: 2000  # Max: 10,406
             required_surplus: 0
             security_environment: JPL-public
             ami_id: /aws/service/ecs/optimized-ami/amazon-linux-2023/recommended/image_id
@@ -42,11 +42,12 @@ jobs:
             default_application_status: APPROVED
             cost_profile: DEFAULT
             job_files: >-
+              job_spec/ARIA_AUTORIFT.yml
               job_spec/ARIA_RAIDER.yml
               job_spec/INSAR_ISCE.yml
-            instance_types: c6id.xlarge,c6id.2xlarge,c6id.4xlarge,c6id.8xlarge
-            default_max_vcpus: 0
-            expanded_max_vcpus: 0
+            instance_types: m6id.xlarge,m6id.2xlarge,m6id.4xlarge,m6id.8xlarge,m6idn.xlarge,m6idn.2xlarge,m6idn.4xlarge,m6idn.8xlarge
+            default_max_vcpus: 4000  # Max: 13000
+            expanded_max_vcpus: 4000  # Max: 13000
             required_surplus: 0
             security_environment: JPL-public
             ami_id: /aws/service/ecs/optimized-ami/amazon-linux-2023/recommended/image_id
@@ -61,11 +62,12 @@ jobs:
             default_application_status: APPROVED
             cost_profile: DEFAULT
             job_files: >-
+              job_spec/ARIA_AUTORIFT.yml
               job_spec/ARIA_RAIDER.yml
-              job_spec/INSAR_ISCE_TEST.yml
+              job_spec/INSAR_ISCE.yml
             instance_types: m6id.xlarge,m6id.2xlarge,m6id.4xlarge,m6id.8xlarge,m6idn.xlarge,m6idn.2xlarge,m6idn.4xlarge,m6idn.8xlarge
-            default_max_vcpus: 1000
-            expanded_max_vcpus: 1000
+            default_max_vcpus: 1000  # Max: 10316
+            expanded_max_vcpus: 1000  # Max: 10316
             required_surplus: 0
             security_environment: JPL-public
             ami_id: /aws/service/ecs/optimized-ami/amazon-linux-2023/recommended/image_id
@@ -79,10 +81,13 @@ jobs:
             default_credits_per_user: 0
             default_application_status: APPROVED
             cost_profile: DEFAULT
-            job_files: job_spec/INSAR_ISCE.yml
-            instance_types: c6id.xlarge,c6id.2xlarge,c6id.4xlarge,c6id.8xlarge
-            default_max_vcpus: 1600
-            expanded_max_vcpus: 1600
+            job_files: >-
+              job_spec/ARIA_AUTORIFT.yml
+              job_spec/ARIA_RAIDER.yml
+              job_spec/INSAR_ISCE.yml
+            instance_types: m6id.xlarge,m6id.2xlarge,m6id.4xlarge,m6id.8xlarge,m6idn.xlarge,m6idn.2xlarge,m6idn.4xlarge,m6idn.8xlarge
+            default_max_vcpus: 1600  # Max 1652
+            expanded_max_vcpus: 1600  # Max 1652
             required_surplus: 0
             security_environment: JPL-public
             ami_id: /aws/service/ecs/optimized-ami/amazon-linux-2023/recommended/image_id
@@ -228,7 +233,7 @@ jobs:
 
           - environment: hyp3-lavas
             domain: hyp3-lavas.asf.alaska.edu
-            template_bucket: cf-templates-10a5pjrsv3cgo-us-west-2 
+            template_bucket: cf-templates-10a5pjrsv3cgo-us-west-2
             image_tag: latest
             product_lifetime_in_days: 14
             default_credits_per_user: 0

.github/workflows/labeled-pr.yml

@@ -12,4 +12,4 @@ on:
 
 jobs:
   call-labeled-pr-check-workflow:
-    uses: ASFHyP3/actions/.github/workflows/[email protected].1
+    uses: ASFHyP3/actions/.github/workflows/[email protected].2

.github/workflows/release-template-comment.yml

@@ -7,7 +7,7 @@ on:
 
 jobs:
   call-release-checklist-workflow:
-    uses: ASFHyP3/actions/.github/workflows/[email protected].1
+    uses: ASFHyP3/actions/.github/workflows/[email protected].2
     permissions:
       pull-requests: write
     with:

.github/workflows/release.yml

@@ -7,7 +7,7 @@ on:
 
 jobs:
   call-release-workflow:
-    uses: ASFHyP3/actions/.github/workflows/[email protected].1
+    uses: ASFHyP3/actions/.github/workflows/[email protected].2
     with:
       release_prefix: HyP3
     secrets:

.github/workflows/static-analysis.yml

@@ -87,4 +87,4 @@ jobs:
           snyk iac test --severity-threshold=high
 
   call-secrets-analysis-workflow:
-    uses: ASFHyP3/actions/.github/workflows/[email protected].1
+    uses: ASFHyP3/actions/.github/workflows/[email protected].2

CHANGELOG.md

@@ -4,6 +4,32 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+
+## [7.7.1]
+
+### Removed
+- The `ESA_USERNAME` and `ESA_PASSWORD` secrets have been removed from the job specs that no longer require them (those that use the `hyp3-gamma`, `hyp3-isce2`, `hyp3-autorift`, or `hyp3-back-projection` images).
+
+
+## [7.7.0]
+
+### Added
+- `ARIA_AUTORIFT.yml` job spec for Solid Earth offset tracking in the ARIA JPL deployments
+
+### Changed
+- Increased throughput for `hyp3-a19-jpl` (0 -> 4,000 vCPUs) to support continued processing of ARIA GUNW products.
+- The `hyp3-a19-jpl` and `hyp3-nisar-jpl` deployments now use the `m6id[n]` instance families to reduce the high number of spot interruptions seen with wth `c6id` instance family.
+- Increased available vCPUs for DAAC deployments.
+
+### Removed
+- The `INSAR_ISCE_TEST.yml` job spec, which only differed from `INSAR_ISCE.yml` in support of different instance families, has been removed now that all ARIA JPL deployments are using the same instance families again.
+
+
+## [7.6.0]
+
+### Changed
+- Reduced throughput for `hyp3-its-live` to prevent Sentinel-2 processing from being rate limited (10,000 -> 2,000 vCPUs).
+
 ## [7.5.0]
 
 ### Added

job_spec/ARIA_AUTORIFT.yml

@@ -0,0 +1,63 @@
+AUTORIFT:
+  required_parameters:
+    - granules
+  parameters:
+    granules:
+      default:  '""'
+      api_schema:
+        type: array
+        minItems: 2
+        maxItems: 2
+        example:
+          - S2B_MSIL1C_20200105T152259_N0208_R039_T13CES_20200105T181230
+          - S2B_MSIL1C_20200315T152259_N0209_R039_T13CES_20200315T181115
+        items:
+          anyOf:
+            - description: The name of the Sentinel-1 SLC granule to process
+              type: string
+              pattern: "^S1[AB]_IW_SLC__1S[SD][VH]"
+              minLength: 67
+              maxLength: 67
+              example: S1A_IW_SLC__1SSV_20150621T120220_20150621T120232_006471_008934_72D8
+            - description: The name of the Sentinel-2 granule to process (ESA naming convention)
+              type: string
+              pattern: "^S2[AB]_MSIL1C_"
+              minLength: 60
+              maxLength: 60
+              example: S2A_MSIL1C_20200627T150921_N0209_R025_T22WEB_20200627T170912
+            - description: The name of the Landsat 4, 5, 7, 8 or 9 Collection 2 granule to process
+              type: string
+              pattern: "^L([CO]0[89]|E07|T0[45])_L1"
+              minLength: 40
+              maxLength: 40
+              example: LC08_L1GT_118112_20210107_20210107_02_T2
+    bucket_prefix:
+      default:  '""'
+    parameter_file:
+      api_schema:
+        description: Shapefile for determining the correct search parameters by geographic location. Path to shapefile must be understood by GDAL.
+        type: string
+        default: '/vsicurl/http://its-live-data.s3.amazonaws.com/autorift_parameters/v001/autorift_solidearth_0120m.shp'
+  cost_profiles:
+    DEFAULT:
+      cost: 1.0
+  validators: []
+  tasks:
+    - name: ''
+      image: ghcr.io/asfhyp3/hyp3-autorift
+      command:
+        - --bucket
+        - '!Ref Bucket'
+        - --bucket-prefix
+        - Ref::bucket_prefix
+        - --parameter-file
+        - Ref::parameter_file
+        - --naming-scheme
+        - ITS_LIVE_OD
+        - Ref::granules
+      timeout: 10800
+      vcpu: 1
+      memory: 31500
+      secrets:
+        - EARTHDATA_USERNAME
+        - EARTHDATA_PASSWORD

job_spec/AUTORIFT.yml

@@ -60,5 +60,3 @@ AUTORIFT:
       secrets:
         - EARTHDATA_USERNAME
         - EARTHDATA_PASSWORD
-        - ESA_USERNAME
-        - ESA_PASSWORD

job_spec/AUTORIFT_ITS_LIVE.yml

@@ -72,7 +72,5 @@ AUTORIFT:
       secrets:
         - EARTHDATA_USERNAME
         - EARTHDATA_PASSWORD
-        - ESA_USERNAME
-        - ESA_PASSWORD
         - PUBLISH_ACCESS_KEY_ID
         - PUBLISH_SECRET_ACCESS_KEY

job_spec/INSAR_GAMMA.yml

@@ -119,5 +119,3 @@ INSAR_GAMMA:
       secrets:
         - EARTHDATA_USERNAME
         - EARTHDATA_PASSWORD
-        - ESA_USERNAME
-        - ESA_PASSWORD

job_spec/INSAR_ISCE.yml

@@ -97,7 +97,7 @@ INSAR_ISCE:
       image: ghcr.io/access-cloud-based-insar/dockerizedtopsapp
       command:
         - ++omp-num-threads
-        - '4'
+        - '2'  # 2 for the m instance family; 4 for the c
         - --bucket
         - '!Ref Bucket'
         - --bucket-prefix

job_spec/INSAR_ISCE_BURST.yml

@@ -67,5 +67,3 @@ INSAR_ISCE_BURST:
       secrets:
         - EARTHDATA_USERNAME
         - EARTHDATA_PASSWORD
-        - ESA_USERNAME
-        - ESA_PASSWORD