Merge pull request #436 from nasa/release/4.0.0-er.1.0

Release 4.0.0-ER.1.0
nasa · May 2, 2024 · 6bb3c3b · 6bb3c3b
2 parents c4de131 + be94f24
commit 6bb3c3b
Show file tree

Hide file tree

Showing 6 changed files with 241 additions and 7 deletions.
diff --git a/.ci/jenkins/build-int-test/Jenkinsfile b/.ci/jenkins/build-int-test/Jenkinsfile
@@ -11,7 +11,7 @@ pipeline {
     }
     parameters {
         // TODO: update default value as newer PGE's are added
-        string(name: 'DOCKER_IMAGE_SUFFIXES', defaultValue: 'dswx_hls,cslc_s1,rtc_s1,dswx_s1,disp_s1',
+        string(name: 'DOCKER_IMAGE_SUFFIXES', defaultValue: 'dswx_hls,cslc_s1,rtc_s1,dswx_s1,disp_s1,dswx_ni',
                description: 'Comma-delimited list of PGE names to run integration tests for')
         string(name: 'ART_DOCKER_REGISTRY', defaultValue: 'artifactory-fn.jpl.nasa.gov:16001',
                description: 'Address of Artifactory-FN Docker registry for uploading Docker images.')

diff --git a/.ci/jenkins/build-test-deploy/Jenkinsfile b/.ci/jenkins/build-test-deploy/Jenkinsfile
@@ -11,7 +11,7 @@ pipeline {
     }
     parameters {
         // TODO: update default value as newer PGE's are added
-        string(name: 'DOCKER_IMAGE_SUFFIXES', defaultValue: 'dswx_hls,cslc_s1,rtc_s1,dswx_s1,disp_s1',
+        string(name: 'DOCKER_IMAGE_SUFFIXES', defaultValue: 'dswx_hls,cslc_s1,rtc_s1,dswx_s1,disp_s1,dswx_ni',
                description: 'Comma-delimited list of PGE names to include for build/test/deploy')
         booleanParam(name: 'PUBLISH_DOCS', defaultValue: true,
                      description: 'Publish sphinx documentation to ghpages')

diff --git a/.ci/jenkins/build-test/Jenkinsfile b/.ci/jenkins/build-test/Jenkinsfile
@@ -5,7 +5,7 @@ pipeline {
     environment {
         DOCKER_IMAGE_PREFIX = 'opera_pge'
         // TODO: update as newer PGE's are added
-        DOCKER_IMAGE_SUFFIXES = 'dswx_hls,cslc_s1,rtc_s1,dswx_s1,disp_s1'
+        DOCKER_IMAGE_SUFFIXES = 'dswx_hls,cslc_s1,rtc_s1,dswx_s1,disp_s1,dswx_ni'
         DOCKER_TAG = """${sh(
                      returnStdout: true,
                      script: 'echo ${GIT_BRANCH##*/}-build-test-$$'

diff --git a/examples/dswx_ni_sample_runconfig-v4.0.0-er.1.0.yaml b/examples/dswx_ni_sample_runconfig-v4.0.0-er.1.0.yaml
@@ -0,0 +1,232 @@
+# Sample RunConfig for use with the DSWx-NI PGE v4.0.0-er.1.0
+# This RunConfig should require minimal changes in order to be used with the
+# OPERA PCM.
+
+RunConfig:
+  # Name for the RunConfig, may be any string
+  Name: OPERA-DSWX-NI-PGE-SAMPLE-CONFIG
+
+  Groups:
+    # PGE-specific RunConfig section
+    # This section is only used by the PGE, however, paths to inputs/outputs
+    # should align with the similar sections of the SAS RunConfig
+    PGE:
+      PGENameGroup:
+        # Name of the PGE for use with this RunConfig, should always be
+        # DSWX_NI_PGE when using with the DSWx-NI PGE
+        PGEName: DSWX_NI_PGE
+
+      InputFilesGroup:
+        # List of input files/directories
+        # Must be a list of the input HDF5 GCOV NISAR products
+        InputFilePaths: [
+          /home/dswx_user/input_dir/ALPSRP271200660_gcov.h5,
+          /home/dswx_user/input_dir/ALPSRP271200670_gcov.h5
+        ]
+
+      DynamicAncillaryFilesGroup:
+        # Map of ancillary file types to paths to the file
+        # Paths must correspond to the file system within the Docker container
+        AncillaryFileMap:
+          # Path to the Digital Elevation Model
+          # Must be either a single .tif, or a .vrt that points to one
+          # or more .tif files
+          dem_file: /home/dswx_user/input_dir/dem.tif
+
+          # Path to the Discrete Classification Map
+          # Must point to a single .tif, or a .vrt that points to one or
+          # more .tif files
+          hand_file: /home/dswx_user/input_dir/hand.tif
+
+          # Path to the reference water map
+          # Must point to a single .tif, or a .vrt that points to one or
+          # more .tif files
+          reference_water_file: /home/dswx_user/input_dir/reference_water.tif
+
+          # Path to the Worldcover map
+          # Must point to a single .tif, or a .vrt that points to one or
+          # more .tif files
+          worldcover_file: /home/dswx_user/input_dir/worldcover.tif
+
+      ProductPathGroup:
+        # Path to where output products should be stored
+        # Must correspond to the file system within the Docker container,
+        # and must have write permissions for the User/Group used with
+        # the "Docker run" command
+        OutputProductPath: /home/dswx_user/output_dir
+
+        # Path to a scratch directory for the PGE and SAS to store
+        # intermediate files that will not be needed after PGE execution
+        # completes
+        # Must correspond to the file system within the Docker container,
+        # and must have write permissions for the User/Group used with
+        # the "Docker run" command
+        ScratchPath: /home/dswx_user/scratch_dir
+
+      PrimaryExecutable:
+        # Identifier for the PGE executable, should always be DSWX_NI for
+        # this PGE
+        ProductIdentifier: DSWX_NI
+
+        # Product version specific to output products
+        ProductVersion: "0.1"
+
+        # Path to the executable to run, path must be reachable from
+        # within the Docker container (i.e. findable with a "which" command)
+        ProgramPath: python3
+
+        # List of command-line options to use with ProgramPath
+        ProgramOptions:
+          - /home/dswx_user/OPERA/DSWX-SAR/src/dswx_sar/dswx_ni.py
+
+        # The Error Code base value added to the offset values to make
+        # error codes unique per-PGE
+        ErrorCodeBase: 600000
+
+        # Path to the Yamale schema used to validate the SAS portion
+        # of the RunConfig
+        # Path should correspond to the file system within the Docker
+        # container, and typically should reference a schema file bundled
+        # with the opera_pge installation directory within the container
+        # Consult the Docker image build scripts for more info
+        SchemaPath: /home/dswx_user/opera/pge/dswx_ni/schema/dswx_ni_sas_schema.yaml
+
+        # Path to the Yamale schema used specifically to validate the
+        # algorithm parameters config, which is stored within a separate
+        # yaml file referenced within the SAS portion of this RunConfig
+        AlgorithmParametersSchemaPath: /home/dswx_user/opera/pge/dswx_ni/schema/algorithm_parameters_ni_schema.yaml
+
+        # Path to the Jinja2 template used to generate the ISO xml
+        # metadata file
+        # Path should correspond to the file system within the Docker
+        # container, and typically should reference a template file bundled
+        # with the opera_pge installation directory within the container
+        # Consult the Docker image build scripts for more info
+        # TODO: add once available
+        IsoTemplatePath:
+
+      QAExecutable:
+        # Set to True to enable execution of an additional "Quality Assurance"
+        # application after SAS execution has completed
+        Enabled: False
+
+        # Path to the executable to run, path must be reachable from
+        # within the Docker container (i.e. findable with a "which" command)
+        ProgramPath:
+
+        # List of command-line options to use with ProgramPath
+        ProgramOptions:
+
+      DebugLevelGroup:
+        # Set to True to enable Debug mode (TODO this is currently a no-op)
+        DebugSwitch: False
+
+        # Set to True to have the PGE invoke the SAS/QA executables via
+        # a shell, rather than a Python subprocess
+        # This allows shell-style syntax to be used in ProgramPath and
+        # ProgramOptions, which can be useful for testing
+        ExecuteViaShell: False
+
+    # SAS-specific RunConfig section
+    # Prior to SAS execution by the PGE, the section below starting at "runconfig"
+    # is isolated into its own YAML file for use with the SAS
+    SAS:
+      runconfig:
+        name: dswx_ni_workflow_default
+
+        groups:
+          pge_name_group:
+            pge_name: DSWX_NI_PGE
+
+          input_file_group:
+            input_file_path: [
+              /home/dswx_user/input_dir/ALPSRP271200660_gcov.h5,
+              /home/dswx_user/input_dir/ALPSRP271200670_gcov.h5
+            ]
+
+            # Placeholder for historical GCOV data
+            input_file_historical_path:
+
+            # Specify the MGRS tile collection ID
+            input_mgrs_collection_id: MS_131_19
+
+          dynamic_ancillary_file_group:
+            # Digital elevation model (Required)
+            dem_file: /home/dswx_user/input_dir/dem.tif
+
+            # Digital elevation model source description
+            dem_file_description: 'Copernicus DEM GLO-30 2021 WGS84'
+
+            # ESA WorldCover map file
+            worldcover_file: /home/dswx_user/input_dir/worldcover.tif
+
+            # ESA WorldCover map description
+            worldcover_file_description: 'ESA WorldCover 10m 2020 v1.0'
+
+            # Reference water body map (Required)
+            # e.g., Pekel's water occurrence or seasonality map
+            reference_water_file: /home/dswx_user/input_dir/reference_water.tif
+
+            # Reference water description
+            reference_water_file_description: 'JRC Global Surface Water - collection from 1984 to 2021'
+
+            # Height Above Nearest Drainage (Required)
+            hand_file: /home/dswx_user/input_dir/hand.tif
+
+            # HAND description
+            hand_file_description: 'ASF HAND GLO30'
+
+            # placeholders for inundated vegetation
+            mean_backscattering:
+            standard_deviation_backscattering:
+
+            # Separate runconfig file for algorithm parameters
+            algorithm_parameters: /home/dswx_user/input_dir/algorithm_parameter_ni.yaml
+
+          static_ancillary_file_group:
+            static_ancillary_inputs_flag: True
+
+            # MGRS database sqlite file
+            mgrs_database_file: /home/dswx_user/input_dir/MGRS_tile.sqlite
+
+            # MGRS collection database sqlite file
+            mgrs_collection_database_file: /home/dswx_user/input_dir/MGRS_collection_db_DSWx-NI_v0.1.sqlite
+
+          primary_executable:
+            product_type: dswx_ni
+
+          product_path_group:
+            # Directory where PGE will place results
+            product_path: /home/dswx_user/output_dir
+
+            # Directory where SAS can write temporary data
+            scratch_path: /home/dswx_user/scratch_dir
+
+            # SAS writes the output product(s) to the following file
+            sas_output_path: /home/dswx_user/output_dir
+
+            # TODO: this should become a string once SAS schema is fixed
+            product_version: 0.1
+
+            # DSWx-NI product format
+            output_imagery_format: 'COG'
+
+          browse_image_group:
+            # Save a full-res Cloud-Optimized GEOTIFF DSWx-NI browse image and
+            # a modified-resolution PNG of the browse image for DSWx-NI
+            save_browse: True
+
+            # Setting `browse_image_height` and `browse_image_width` equal
+            # will maintain this original DSWx-NI aspect ratio
+            # of 3660 pixels x 3660 pixels for the PNG browse image.
+            # If these fields are left empty, 1024 x 1024 will be used.
+            # Height in pixels for the PNG browse image
+            browse_image_height: 1024
+
+            # Width in pixels for the PNG browse image
+            browse_image_width: 1024
+
+            # Save the full-res GeoTIFF Browse image in addition to the PNG
+            save_tif_to_output: True
+
+          log_file: /home/dswx_user/output_dir/dswx-ni.log
diff --git a/src/opera/pge/dswx_ni/dswx_ni_pge.py b/src/opera/pge/dswx_ni/dswx_ni_pge.py
@@ -83,6 +83,9 @@ class DSWxNIExecutor(DSWxNIPreProcessorMixin, DSWxNIPostProcessorMixin, PgeExecu
     LEVEL = "L3"
     """Processing Level for DSWx-NI Products"""
 
+    PGE_VERSION = "4.0.0-er.1.0"
+    """Version of the PGE (overrides default from base_pge)"""
+
     SAS_VERSION = "0.1"
     """Version of the SAS wrapped by this PGE, should be updated as needed"""
 

diff --git a/src/opera/pge/dswx_ni/schema/dswx_ni_sas_schema.yaml b/src/opera/pge/dswx_ni/schema/dswx_ni_sas_schema.yaml
@@ -13,7 +13,9 @@ runconfig:
             # REQUIRED - list of RTC products (directory or files)
             input_file_path: list(str(), min=1)
             # Placeholder for historical GCOV data
-            input_file_path_historical: list(str(), min=1, required=False)
+            input_file_historical_path: list(str(), min=1, required=False)
+            # Specify the MGRS tile collection ID
+            input_mgrs_collection_id: str(required=False)
 
         dynamic_ancillary_file_group:
             # Digital elevation model
@@ -81,9 +83,6 @@ runconfig:
 
             product_version: num(required=False)
 
-            # Specify the MGRS tile collection ID
-            output_mgrs_collection_id: str(required=False)
-
             # DSWx-NI product format (default is 'COG')
             output_imagery_format: enum('GTiff', 'COG', required=False)