Merge pull request #434 from nasa/release/3.0.0-rc.2.1

Release 3.0.0-RC.2.1

.ci/scripts/disp_s1/build_disp_s1.sh

@@ -24,7 +24,7 @@ BUILD_DATE_TIME=$(date -u +'%Y-%m-%dT%H:%M:%SZ')
 # defaults, SAS image should be updated as necessary for new image releases from ADT
 [ -z "${WORKSPACE}" ] && WORKSPACE=$(realpath $(dirname $(realpath $0))/../../..)
 [ -z "${TAG}" ] && TAG="${USER}-dev"
-[ -z "${SAS_IMAGE}" ] && SAS_IMAGE="artifactory-fn.jpl.nasa.gov:16001/gov/nasa/jpl/opera/adt/opera/disp-s1:0.3"
+[ -z "${SAS_IMAGE}" ] && SAS_IMAGE="artifactory-fn.jpl.nasa.gov:16001/gov/nasa/jpl/opera/adt/opera/disp-s1:0.3.1"
 
 echo "WORKSPACE: $WORKSPACE"
 echo "IMAGE: $IMAGE"

.ci/scripts/dswx_s1/build_dswx_s1.sh

@@ -24,7 +24,7 @@ BUILD_DATE_TIME=$(date -u +'%Y-%m-%dT%H:%M:%SZ')
 # defaults, SAS image should be updated as necessary for new image releases from ADT
 [ -z "${WORKSPACE}" ] && WORKSPACE=$(realpath $(dirname $(realpath $0))/../../..)
 [ -z "${TAG}" ] && TAG="${USER}-dev"
-[ -z "${SAS_IMAGE}" ] && SAS_IMAGE="artifactory-fn.jpl.nasa.gov:16001/gov/nasa/jpl/opera/adt/opera/dswx-s1:calval_0.4"
+[ -z "${SAS_IMAGE}" ] && SAS_IMAGE="artifactory-fn.jpl.nasa.gov:16001/gov/nasa/jpl/opera/adt/opera/dswx-s1:calval_0.4.2"
 
 echo "WORKSPACE: $WORKSPACE"
 echo "IMAGE: $IMAGE"

.ci/scripts/dswx_s1/dswx_s1_calval_0.4_runconfig.yaml → .ci/scripts/dswx_s1/dswx_s1_calval_0.4.2_runconfig.yaml

@@ -63,7 +63,7 @@ RunConfig:
 
       PrimaryExecutable:
         ProductIdentifier: DSWX_S1
-        ProductVersion: "0.1"
+        ProductVersion: "0.4"
         ProgramPath: python3
         ProgramOptions:
           - /home/dswx_user/OPERA/DSWX-SAR/src/dswx_sar/dswx_s1.py
@@ -157,7 +157,7 @@ RunConfig:
             product_path: /home/dswx_user/output_dir
             scratch_path: /home/dswx_user/scratch_dir
             sas_output_path: /home/dswx_user/output_dir
-            product_version: 0.1
+            product_version: "0.4"
             output_imagery_format: 'COG'
 
           browse_image_group:
@@ -170,5 +170,6 @@ RunConfig:
             set_hand_mask_to_nodata: False
             set_layover_shadow_to_nodata: False
             set_ocean_masked_to_nodata: False
+            save_tif_to_output: True
 
           log_file: /home/dswx_user/output_dir/dswx-s1.log

.ci/scripts/dswx_s1/test_int_dswx_s1.sh

@@ -28,9 +28,9 @@ SAMPLE_TIME=1
 # RUNCONFIG should be the name of the runconfig in s3://operasds-dev-pge/dswx_s1/
 [ -z "${WORKSPACE}" ] && WORKSPACE=$(realpath "$(dirname "$(realpath "$0")")"/../../..)
 [ -z "${PGE_TAG}" ] && PGE_TAG="${USER}-dev"
-[ -z "${INPUT_DATA}" ] && INPUT_DATA="dswx_s1_calval_0.4_expected_input.zip"
-[ -z "${EXPECTED_DATA}" ] && EXPECTED_DATA="dswx_s1_calval_0.4_expected_output.zip"
-[ -z "${RUNCONFIG}" ] && RUNCONFIG="dswx_s1_calval_0.4_runconfig.yaml"
+[ -z "${INPUT_DATA}" ] && INPUT_DATA="dswx_s1_calval_0.4.2_expected_input.zip"
+[ -z "${EXPECTED_DATA}" ] && EXPECTED_DATA="dswx_s1_calval_0.4.2_expected_output.zip"
+[ -z "${RUNCONFIG}" ] && RUNCONFIG="dswx_s1_calval_0.4.2_runconfig.yaml"
 [ -z "${TMP_ROOT}" ] && TMP_ROOT="$DEFAULT_TMP_ROOT"
 
 # Create the test output directory in the work space
@@ -124,7 +124,7 @@ else
         echo "output_file $output_file"
         output_file=$(basename -- "$output_file")
 
-        if [[ "${output_file##*/}" == *.tif* ]]
+        if [[ "${output_file##*/}" == *B0*.tif* ]]
         then
             for potential_product in B01_WTR B02_BWTR B03_CONF B04_DIAG
             do
@@ -142,7 +142,7 @@ else
 
             echo "tile code is $tile_code"
 
-            for potential_file in "$expected_data_dir"/*.tif*
+            for potential_file in "$expected_data_dir"/*B0*.tif*
             do
                 if [[ "$potential_file" == *"$tile_code"*"$product"* ]]; then
                     echo "expected file is $potential_file"

examples/disp_s1_sample_runconfig-v3.0.0-rc.2.0.yaml → examples/disp_s1_sample_runconfig-v3.0.0-rc.2.1.yaml

@@ -1,4 +1,4 @@
-# Sample RunConfig for use with the DISP-S1 PGE v3.0.0-rc.2.0
+# Sample RunConfig for use with the DISP-S1 PGE v3.0.0-rc.2.1
 # This RunConfig should require minimal changes in order to be used with the
 # OPERA PCM.
 

examples/dswx_s1_sample_runconfig-v3.0.0-rc.2.0.yaml → examples/dswx_s1_sample_runconfig-v3.0.0-rc.2.1.yaml

@@ -1,4 +1,4 @@
-# Sample RunConfig for use with the DSWx-S1 PGE v3.0.0-rc.2.0
+# Sample RunConfig for use with the DSWx-S1 PGE v3.0.0-rc.2.1
 # This RunConfig should require minimal changes in order to be used with the
 # OPERA PCM.
 
@@ -209,7 +209,7 @@ RunConfig:
             # SAS writes the output product(s) to the following file
             sas_output_path: /home/dswx_user/output_dir
 
-            product_version: 0.1
+            product_version: "0.1"
 
             # DSWx-S1 product format
             output_imagery_format: 'COG'

src/opera/_package.py

@@ -7,7 +7,7 @@
 Package information for the OPERA PGE repository.
 """
 
-__version__ = "3.0.0-rc.2.0"
+__version__ = "3.0.0-rc.2.1"
 __title__ = "opera-sds-pge"
 __summary__ = "OPERA SDS Product Generation Executable (PGE) Repository"
 __uri__ = "https://github.com/nasa/opera-sds-pge"

src/opera/pge/base/runconfig.py

@@ -442,7 +442,7 @@ def get_ancillary_filenames(self):
         for ancillary_mapping in self.ancillary_file_map.values():
             if isinstance(ancillary_mapping, list):
                 result.extend(ancillary_mapping)
-            else:
+            elif ancillary_mapping is not None:
                 result.append(ancillary_mapping)
 
         return result

src/opera/pge/disp_s1/disp_s1_pge.py

@@ -337,7 +337,7 @@ def _browse_filename(self, inter_filename):
             The file name to assign to browse image created by this PGE.
 
         """
-        browse_image_filename = f"{self._core_filename(inter_filename)}.png"
+        browse_image_filename = f"{self._core_filename(inter_filename)}_BROWSE.png"
 
         return browse_image_filename
 
@@ -373,9 +373,11 @@ def _compressed_cslc_filename(self, inter_filename):
         Returns the file name to use for compressed CSLC files produced by the
         DISP-S1 PGE.
 
-        The current convention is to maintain the filename assigned by the DISP-S1
-        SAS, but this function is still required to ensure the compressed CSLC products
-        are moved to the output directory defined by the RunConfig.
+        The compressed CSLC filename for the DISP-S1 PGE consists of:
+
+             <Project>_<Level>_COMPRESSED-CSLC-S1_<BurstID>_<ReferenceDateTime>_\
+             <FirstDateTime>_<LastDateTime>_<ProductGenerationDateTime>_\
+             <Polarization>_<ProductVersion>.h5
 
         Parameters
         ----------

src/opera/pge/disp_s1/templates/OPERA_ISO_metadata_L3_DISP_S1_template.xml.jinja2

@@ -976,6 +976,7 @@
                             </gmd:description>
                         </gmd:LI_Source>
                     </gmd:source>
+                    {%- if run_config.Groups.PGE.DynamicAncillaryFilesGroup.AncillaryFileMap.amplitude_dispersion_files is not none %}
                     {%- for amplitude_dispersion_file in run_config.Groups.PGE.DynamicAncillaryFilesGroup.AncillaryFileMap.amplitude_dispersion_files %}
                     <gmd:source>
                         <gmd:LI_Source>
@@ -985,6 +986,8 @@
                         </gmd:LI_Source>
                     </gmd:source>
                     {%- endfor %}
+                    {%- endif %}
+                    {%- if run_config.Groups.PGE.DynamicAncillaryFilesGroup.AncillaryFileMap.amplitude_mean_files is not none %}
                     {%- for amplitude_mean_file in run_config.Groups.PGE.DynamicAncillaryFilesGroup.AncillaryFileMap.amplitude_mean_files %}
                     <gmd:source>
                         <gmd:LI_Source>
@@ -994,6 +997,8 @@
                         </gmd:LI_Source>
                     </gmd:source>
                     {%- endfor %}
+                    {%- endif %}
+                    {%- if run_config.Groups.PGE.DynamicAncillaryFilesGroup.AncillaryFileMap.static_layers_files is not none %}
                     {%- for static_layers_file in run_config.Groups.PGE.DynamicAncillaryFilesGroup.AncillaryFileMap.static_layers_files %}
                     <gmd:source>
                         <gmd:LI_Source>
@@ -1003,6 +1008,8 @@
                         </gmd:LI_Source>
                     </gmd:source>
                     {%- endfor %}
+                    {%- endif %}
+                    {%- if run_config.Groups.PGE.DynamicAncillaryFilesGroup.AncillaryFileMap.ionosphere_files is not none %}
                     {%- for ionosphere_file in run_config.Groups.PGE.DynamicAncillaryFilesGroup.AncillaryFileMap.ionosphere_files %}
                     <gmd:source>
                         <gmd:LI_Source>
@@ -1012,6 +1019,8 @@
                         </gmd:LI_Source>
                     </gmd:source>
                     {%- endfor %}
+                    {%- endif %}
+                    {%- if run_config.Groups.PGE.DynamicAncillaryFilesGroup.AncillaryFileMap.troposphere_files is not none %}
                     {%- for troposphere_file in run_config.Groups.PGE.DynamicAncillaryFilesGroup.AncillaryFileMap.troposphere_files %}
                     <gmd:source>
                         <gmd:LI_Source>
@@ -1021,6 +1030,7 @@
                         </gmd:LI_Source>
                     </gmd:source>
                     {%- endfor %}
+                    {%- endif %}
                 </gmd:LI_Lineage>
             </gmd:lineage>
         </gmd:DQ_DataQuality>

src/opera/pge/dswx_s1/dswx_s1_pge.py

@@ -10,6 +10,7 @@
 """
 
 import glob
+import os.path
 from datetime import datetime
 from os.path import abspath, basename, dirname, exists, getsize, join, splitext
 
@@ -150,12 +151,12 @@ def _validate_output(self):
         Evaluates the output file(s) generated from SAS execution to ensure:
             - That the file(s) contains some content (size is greater than 0).
             - That the .tif output files (band data) end with 'B01_WTR',
-              'B02_BWTR', 'B03_CONF' or 'B04_DIAG'
+              'B02_BWTR', 'B03_CONF', 'B04_DIAG' or 'BROWSE'
             - That the there are the same number of each type of file, implying
               3 output bands per tile
 
         """
-        EXPECTED_NUM_BANDS: int = 4
+        EXPECTED_NUM_BANDS: int = 5
         band_dict = {}
         num_bands = []
         output_extension = '.tif'
@@ -388,6 +389,9 @@ def _browse_filename(self, inter_filename):
             The file name to assign to browse image created by this PGE.
 
         """
+        # Save the extension of the intermediate filename
+        inter_ext = os.path.splitext(inter_filename)[-1]
+
         # Find one of the tif files corresponding to the provided browse image
         # based on the tile ID
         tile_id = basename(inter_filename).split('_')[3]
@@ -407,7 +411,7 @@ def _browse_filename(self, inter_filename):
         tile_filename = self._tile_filename(tif_files[0])
 
         # Add the portion specific to browse images
-        return f"{tile_filename}_BROWSE.png"
+        return f"{tile_filename}_BROWSE{inter_ext}"
 
     def _ancillary_filename(self):
         """
@@ -785,7 +789,7 @@ class DSWxS1Executor(DSWxS1PreProcessorMixin, DSWxS1PostProcessorMixin, PgeExecu
     LEVEL = "L3"
     """Processing Level for DSWx-S1 Products"""
 
-    SAS_VERSION = "0.4"  # CalVal release https://github.com/opera-adt/DSWX-SAR/releases/tag/v0.4
+    SAS_VERSION = "0.4.2"  # CalVal release https://github.com/opera-adt/DSWX-SAR/releases/tag/v0.4.2
     """Version of the SAS wrapped by this PGE, should be updated as needed"""
 
     def __init__(self, pge_name, runconfig_path, **kwargs):
@@ -794,6 +798,6 @@ def __init__(self, pge_name, runconfig_path, **kwargs):
         # Used in base_pge.py to rename and keep track of files
         # renamed by the PGE
         self.rename_by_pattern_map = {
-            '*.tif*': self._geotiff_filename,
-            '*.png': self._browse_filename
+            '*B0*.tif*': self._geotiff_filename,
+            '*BROWSE*': self._browse_filename
         }

src/opera/pge/dswx_s1/schema/dswx_s1_sas_schema.yaml

@@ -72,7 +72,7 @@ runconfig:
             # according to proper file naming conventions.
             sas_output_path: str()
 
-            product_version: num(required=False)
+            product_version: str(required=False)
 
             # DSWx-S1 product format (default is 'COG')
             output_imagery_format: enum('GTiff', 'COG', required=False)
@@ -99,21 +99,24 @@ runconfig:
             browse_image_width: int(min=1, required=False)
 
             # Flag to collapse water classes if set to True. Default is True.
-            flag_collapse_wtr_classes:  bool(required=False)
+            flag_collapse_wtr_classes: bool(required=False)
 
             # Flag to exclude inundated vegetation from processing if set to True.
-            exclude_inundated_vegetation:  bool(required=False)
+            exclude_inundated_vegetation: bool(required=False)
 
             # Flag to set non-water pixels to NoData value if set to True.
-            set_not_water_to_nodata:  bool(required=False)
+            set_not_water_to_nodata: bool(required=False)
 
             # Flag to set HAND mask pixels to NoData value if set to True.
-            set_hand_mask_to_nodata:  bool(required=False)
+            set_hand_mask_to_nodata: bool(required=False)
 
             # Flag to set layover and shadow pixels to NoData value if set to True.
-            set_layover_shadow_to_nodata:  bool(required=False)
+            set_layover_shadow_to_nodata: bool(required=False)
 
             # Flag to set ocean-masked pixels to NoData value if set to True.
-            set_ocean_masked_to_nodata:  bool(required=False)
+            set_ocean_masked_to_nodata: bool(required=False)
 
-        log_file: str(required=False)
+            # Flag to save Geotiff to output directory if set to True.
+            save_tif_to_output: bool(required=False)
+
+        log_file: str(required=False)

src/opera/test/data/invalid_runconfig.yaml

@@ -13,6 +13,7 @@ RunConfig:
       DynamicAncillaryFilesGroup:
         AncillaryFileMap:
           DEMFile: input/input_dem.vrt
+          IonosphereFiles:
 
       ProductPathGroup:
         OutputProductPath: outputs/

src/opera/test/data/test_dswx_s1_config.yaml

@@ -27,6 +27,7 @@ RunConfig:
           - /bin/echo hello world > dswx_s1_pge_test/output_dir/OPERA_L3_DSWx-S1_T18MVA_20200702T231843Z_20230317T190549Z_v0.1_B03_CONF.tif;
           - /bin/echo hello world > dswx_s1_pge_test/output_dir/OPERA_L3_DSWx-S1_T18MVA_20200702T231843Z_20230317T190549Z_v0.1_B04_DIAG.tif;
           - /bin/echo hello world > dswx_s1_pge_test/output_dir/OPERA_L3_DSWx-S1_T18MVA_20200702T231843Z_20230317T190549Z_v0.1_BROWSE.png;
+          - /bin/echo hello world > dswx_s1_pge_test/output_dir/OPERA_L3_DSWx-S1_T18MVA_20200702T231843Z_20230317T190549Z_v0.1_BROWSE.tif;
           - /bin/echo DSWx-S1 invoked with RunConfig
         ErrorCodeBase: 400000
         SchemaPath: pge/dswx_s1/schema/dswx_s1_sas_schema.yaml

src/opera/test/data/valid_runconfig_extra_fields.yaml

@@ -18,6 +18,7 @@ RunConfig:
       DynamicAncillaryFilesGroup:
         AncillaryFileMap:
           DEMFile: input/input_dem.vrt
+          IonosphereFiles:
 
       ProductPathGroup:
         OutputProductPath: outputs/

src/opera/test/data/valid_runconfig_full.yaml

@@ -15,6 +15,7 @@ RunConfig:
       DynamicAncillaryFilesGroup:
         AncillaryFileMap:
           DEMFile: input/input_dem.vrt
+          IonosphereFiles:
 
       ProductPathGroup:
         OutputProductPath: outputs/

src/opera/test/data/valid_runconfig_no_sas.yaml

@@ -15,6 +15,7 @@ RunConfig:
       DynamicAncillaryFilesGroup:
         AncillaryFileMap:
           DEMFile: input/input_dem.vrt
+          IonosphereFiles:
 
       ProductPathGroup:
         OutputProductPath: outputs/

src/opera/test/pge/base/test_runconfig.py

@@ -43,7 +43,7 @@ def _compare_runconfig_to_expected(self, runconfig):
         self.assertEqual(runconfig.name, "OPERA-SAMPLE-PGE-SAS-CONFIG")
         self.assertEqual(runconfig.pge_name, "EXAMPLE_PGE")
         self.assertListEqual(runconfig.input_files, ["input/input_file01.h5", "input/input_file02.h5"])
-        self.assertDictEqual(runconfig.ancillary_file_map, {"DEMFile": "input/input_dem.vrt"})
+        self.assertDictEqual(runconfig.ancillary_file_map, {"DEMFile": "input/input_dem.vrt", 'IonosphereFiles': None})
         self.assertEqual(runconfig.output_product_path, "outputs/")
         self.assertEqual(runconfig.scratch_path, "temp/")
         self.assertEqual(runconfig.product_identifier, "EXAMPLE")
@@ -82,6 +82,13 @@ def test_full_pge_config_parse_and_validate(self):
         # this test
         self.assertIsInstance(runconfig.sas_config, dict)
 
+        # Ensure empty ancillary fields are filtered out of list returned
+        # from get_ancillary_filenames()
+        ancillary_filenames = runconfig.get_ancillary_filenames()
+
+        self.assertIn("input/input_dem.vrt", ancillary_filenames)
+        self.assertNotIn(None, ancillary_filenames)
+
     def test_pge_only_config_parse_and_validate(self):
         """
         Test basic parsing and validation of an input RunConfig that only
@@ -104,6 +111,13 @@ def test_pge_only_config_parse_and_validate(self):
         # Check that None was assigned for SAS config section
         self.assertIsNone(runconfig.sas_config)
 
+        # Ensure empty ancillary fields are filtered out of list returned
+        # from get_ancillary_filenames()
+        ancillary_filenames = runconfig.get_ancillary_filenames()
+
+        self.assertIn("input/input_dem.vrt", ancillary_filenames)
+        self.assertNotIn(None, ancillary_filenames)
+
     def test_strict_mode_validation(self):
         """Test validation of a RunConfig with strict_mode both enabled and disabled"""
         # Parse a valid runconfig, but modify it with fields not in the base