From 68a8dc240a01be592089cf91eb0dfbd6842ebe75 Mon Sep 17 00:00:00 2001 From: Scott Collins Date: Tue, 9 Apr 2024 09:51:20 -0500 Subject: [PATCH 01/14] Rolled version numbers for DSWx-S1 to SAS 0.4.1 and PGE 3.0.0-rc.2.1 --- .ci/scripts/dswx_s1/build_dswx_s1.sh | 2 +- ....2.0.yaml => dswx_s1_sample_runconfig-v3.0.0-rc.2.1.yaml} | 2 +- src/opera/pge/dswx_s1/dswx_s1_pge.py | 5 ++++- 3 files changed, 6 insertions(+), 3 deletions(-) rename examples/{dswx_s1_sample_runconfig-v3.0.0-rc.2.0.yaml => dswx_s1_sample_runconfig-v3.0.0-rc.2.1.yaml} (99%) diff --git a/.ci/scripts/dswx_s1/build_dswx_s1.sh b/.ci/scripts/dswx_s1/build_dswx_s1.sh index ec9590cf..65feeebf 100755 --- a/.ci/scripts/dswx_s1/build_dswx_s1.sh +++ b/.ci/scripts/dswx_s1/build_dswx_s1.sh @@ -24,7 +24,7 @@ BUILD_DATE_TIME=$(date -u +'%Y-%m-%dT%H:%M:%SZ') # defaults, SAS image should be updated as necessary for new image releases from ADT [ -z "${WORKSPACE}" ] && WORKSPACE=$(realpath $(dirname $(realpath $0))/../../..) [ -z "${TAG}" ] && TAG="${USER}-dev" -[ -z "${SAS_IMAGE}" ] && SAS_IMAGE="artifactory-fn.jpl.nasa.gov:16001/gov/nasa/jpl/opera/adt/opera/dswx-s1:calval_0.4" +[ -z "${SAS_IMAGE}" ] && SAS_IMAGE="artifactory-fn.jpl.nasa.gov:16001/gov/nasa/jpl/opera/adt/opera/dswx-s1:calval_0.4.1" echo "WORKSPACE: $WORKSPACE" echo "IMAGE: $IMAGE" diff --git a/examples/dswx_s1_sample_runconfig-v3.0.0-rc.2.0.yaml b/examples/dswx_s1_sample_runconfig-v3.0.0-rc.2.1.yaml similarity index 99% rename from examples/dswx_s1_sample_runconfig-v3.0.0-rc.2.0.yaml rename to examples/dswx_s1_sample_runconfig-v3.0.0-rc.2.1.yaml index 6757b22d..a4bc7778 100644 --- a/examples/dswx_s1_sample_runconfig-v3.0.0-rc.2.0.yaml +++ b/examples/dswx_s1_sample_runconfig-v3.0.0-rc.2.1.yaml @@ -1,4 +1,4 @@ -# Sample RunConfig for use with the DSWx-S1 PGE v3.0.0-rc.2.0 +# Sample RunConfig for use with the DSWx-S1 PGE v3.0.0-rc.2.1 # This RunConfig should require minimal changes in order to be used with the # OPERA PCM. diff --git a/src/opera/pge/dswx_s1/dswx_s1_pge.py b/src/opera/pge/dswx_s1/dswx_s1_pge.py index aa9e1602..f84554eb 100644 --- a/src/opera/pge/dswx_s1/dswx_s1_pge.py +++ b/src/opera/pge/dswx_s1/dswx_s1_pge.py @@ -785,7 +785,10 @@ class DSWxS1Executor(DSWxS1PreProcessorMixin, DSWxS1PostProcessorMixin, PgeExecu LEVEL = "L3" """Processing Level for DSWx-S1 Products""" - SAS_VERSION = "0.4" # CalVal release https://github.com/opera-adt/DSWX-SAR/releases/tag/v0.4 + PGE_VERSION = "3.0.0-rc.2.1" + """Version of the PGE (overrides default from base_pge)""" + + SAS_VERSION = "0.4.1" # CalVal release https://github.com/opera-adt/DSWX-SAR/releases/tag/v0.4.1 """Version of the SAS wrapped by this PGE, should be updated as needed""" def __init__(self, pge_name, runconfig_path, **kwargs): From 4893d098cf6661837f6fb3c08024e178eb61be33 Mon Sep 17 00:00:00 2001 From: Scott Collins Date: Tue, 9 Apr 2024 12:28:26 -0500 Subject: [PATCH 02/14] Increased expected num bands for DSWx-S1 output product check to 5 This is to account for a new BROWSE tif product added in SAS v0.4.1 --- src/opera/pge/dswx_s1/dswx_s1_pge.py | 4 ++-- src/opera/test/data/test_dswx_s1_config.yaml | 1 + src/opera/test/pge/dswx_s1/test_dswx_s1_pge.py | 8 ++++---- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/opera/pge/dswx_s1/dswx_s1_pge.py b/src/opera/pge/dswx_s1/dswx_s1_pge.py index f84554eb..d74a0fd6 100644 --- a/src/opera/pge/dswx_s1/dswx_s1_pge.py +++ b/src/opera/pge/dswx_s1/dswx_s1_pge.py @@ -150,12 +150,12 @@ def _validate_output(self): Evaluates the output file(s) generated from SAS execution to ensure: - That the file(s) contains some content (size is greater than 0). - That the .tif output files (band data) end with 'B01_WTR', - 'B02_BWTR', 'B03_CONF' or 'B04_DIAG' + 'B02_BWTR', 'B03_CONF', 'B04_DIAG' or 'BROWSE' - That the there are the same number of each type of file, implying 3 output bands per tile """ - EXPECTED_NUM_BANDS: int = 4 + EXPECTED_NUM_BANDS: int = 5 band_dict = {} num_bands = [] output_extension = '.tif' diff --git a/src/opera/test/data/test_dswx_s1_config.yaml b/src/opera/test/data/test_dswx_s1_config.yaml index 1fab4160..f910a456 100644 --- a/src/opera/test/data/test_dswx_s1_config.yaml +++ b/src/opera/test/data/test_dswx_s1_config.yaml @@ -27,6 +27,7 @@ RunConfig: - /bin/echo hello world > dswx_s1_pge_test/output_dir/OPERA_L3_DSWx-S1_T18MVA_20200702T231843Z_20230317T190549Z_v0.1_B03_CONF.tif; - /bin/echo hello world > dswx_s1_pge_test/output_dir/OPERA_L3_DSWx-S1_T18MVA_20200702T231843Z_20230317T190549Z_v0.1_B04_DIAG.tif; - /bin/echo hello world > dswx_s1_pge_test/output_dir/OPERA_L3_DSWx-S1_T18MVA_20200702T231843Z_20230317T190549Z_v0.1_BROWSE.png; + - /bin/echo hello world > dswx_s1_pge_test/output_dir/OPERA_L3_DSWx-S1_T18MVA_20200702T231843Z_20230317T190549Z_v0.1_BROWSE.tif; - /bin/echo DSWx-S1 invoked with RunConfig ErrorCodeBase: 400000 SchemaPath: pge/dswx_s1/schema/dswx_s1_sas_schema.yaml diff --git a/src/opera/test/pge/dswx_s1/test_dswx_s1_pge.py b/src/opera/test/pge/dswx_s1/test_dswx_s1_pge.py index a3340c8c..995159c7 100644 --- a/src/opera/test/pge/dswx_s1/test_dswx_s1_pge.py +++ b/src/opera/test/pge/dswx_s1/test_dswx_s1_pge.py @@ -278,7 +278,7 @@ def test_dswx_s1_pge_execution(self): # Lastly, check that the dummy output products were created output_tif_files = glob.glob(join(pge.runconfig.output_product_path, "*.tif")) - self.assertEqual(len(output_tif_files), 4) + self.assertEqual(len(output_tif_files), 5) output_browse_files = glob.glob(join(pge.runconfig.output_product_path, "*.png")) self.assertEqual(len(output_browse_files), 1) @@ -298,7 +298,7 @@ def test_filename_application(self): pge.run() - image_files = glob.glob(join(pge.runconfig.output_product_path, "*.tif")) + image_files = glob.glob(join(pge.runconfig.output_product_path, "*_B0*.tif")) for image_file in image_files: file_name = pge._geotiff_filename(image_file) @@ -755,8 +755,8 @@ def test_dswx_s1_pge_output_validation(self): # Test a misnamed band file. Post-processor should detect this and flag an error band_data = ('OPERA_L3_DSWx-S1_b1_B01_WTR.tif', 'OPERA_L3_DSWx-S1_b1_B02_BWTR.tif', 'OPERA_L3_DSWx-S1_b1_B03_CONF.tif', 'OPERA_L3_DSWx-S1_b1_B04_DIAG.tif', - 'OPERA_L3_DSWx-S1_b2_B01_WTR.tif', 'OPERA_L3_DSWx-S1_b2_B02_BWTR.tif', - 'OPERA_L3_DSWx-S1_b2_B04_DIAG.tif') + 'OPERA_L3_DSWx-S1_b1_BROWSE.tif', 'OPERA_L3_DSWx-S1_b2_B01_WTR.tif', + 'OPERA_L3_DSWx-S1_b2_B02_BWTR.tif', 'OPERA_L3_DSWx-S1_b2_B04_DIAG.tif') self.generate_band_data_output(band_data, clear=True) with open(test_runconfig_path, 'w', encoding='utf-8') as outfile: From 9536c10def0a07ec5801156b6c01167def86c369 Mon Sep 17 00:00:00 2001 From: Scott Collins Date: Wed, 10 Apr 2024 11:56:58 -0500 Subject: [PATCH 03/14] Added support for rename of DSWx-S1 tif browse products --- src/opera/pge/dswx_s1/dswx_s1_pge.py | 10 +++++++--- src/opera/test/pge/dswx_s1/test_dswx_s1_pge.py | 16 ++++++++++++++++ 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/src/opera/pge/dswx_s1/dswx_s1_pge.py b/src/opera/pge/dswx_s1/dswx_s1_pge.py index d74a0fd6..3393123a 100644 --- a/src/opera/pge/dswx_s1/dswx_s1_pge.py +++ b/src/opera/pge/dswx_s1/dswx_s1_pge.py @@ -10,6 +10,7 @@ """ import glob +import os.path from datetime import datetime from os.path import abspath, basename, dirname, exists, getsize, join, splitext @@ -388,6 +389,9 @@ def _browse_filename(self, inter_filename): The file name to assign to browse image created by this PGE. """ + # Save the extension of the intermediate filename + inter_ext = os.path.splitext(inter_filename)[-1] + # Find one of the tif files corresponding to the provided browse image # based on the tile ID tile_id = basename(inter_filename).split('_')[3] @@ -407,7 +411,7 @@ def _browse_filename(self, inter_filename): tile_filename = self._tile_filename(tif_files[0]) # Add the portion specific to browse images - return f"{tile_filename}_BROWSE.png" + return f"{tile_filename}_BROWSE{inter_ext}" def _ancillary_filename(self): """ @@ -797,6 +801,6 @@ def __init__(self, pge_name, runconfig_path, **kwargs): # Used in base_pge.py to rename and keep track of files # renamed by the PGE self.rename_by_pattern_map = { - '*.tif*': self._geotiff_filename, - '*.png': self._browse_filename + '*B0*.tif*': self._geotiff_filename, + '*BROWSE*': self._browse_filename } diff --git a/src/opera/test/pge/dswx_s1/test_dswx_s1_pge.py b/src/opera/test/pge/dswx_s1/test_dswx_s1_pge.py index 995159c7..cd223ec7 100644 --- a/src/opera/test/pge/dswx_s1/test_dswx_s1_pge.py +++ b/src/opera/test/pge/dswx_s1/test_dswx_s1_pge.py @@ -314,6 +314,22 @@ def test_filename_application(self): rf"B\d{{2}}_\w+.tif" self.assertEqual(re.match(file_name_regex, file_name).group(), file_name) + browse_files = glob.glob(join(pge.runconfig.output_product_path, "*BROWSE*")) + + for browse_file in browse_files: + file_name = pge._browse_filename(browse_file) + md = MockGdal.MockDSWxS1GdalDataset().GetMetadata() + # TODO: kludge since SAS hardcodes SPACECRAFT_NAME to "Sentinel-1A/B" + md['SPACECRAFT_NAME'] = 'Sentinel-1B' + file_name_regex = rf"{pge.PROJECT}_{pge.LEVEL}_" \ + rf"{md['PRODUCT_TYPE']}_" \ + rf"T\w{{5}}_" \ + rf"\d{{8}}T\d{{6}}Z_\d{{8}}T\d{{6}}Z_" \ + rf"{get_sensor_from_spacecraft_name(md['SPACECRAFT_NAME'])}_" \ + rf"30_v{pge.runconfig.product_version}_" \ + rf"BROWSE.(png|tif)" + self.assertEqual(re.match(file_name_regex, file_name).group(), file_name) + @patch.object(opera.util.tiff_utils, "gdal", MockGdal) def test_iso_metadata_creation(self): """ From ed76dfd53e04ff1d946c44f63b9ace89e2162556 Mon Sep 17 00:00:00 2001 From: Scott Collins Date: Thu, 18 Apr 2024 09:05:00 -0500 Subject: [PATCH 04/14] Rolled example runconfig for DISP-S1 to v3.0.0-rc.2.1 --- ...-rc.2.0.yaml => disp_s1_sample_runconfig-v3.0.0-rc.2.1.yaml} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename examples/{disp_s1_sample_runconfig-v3.0.0-rc.2.0.yaml => disp_s1_sample_runconfig-v3.0.0-rc.2.1.yaml} (99%) diff --git a/examples/disp_s1_sample_runconfig-v3.0.0-rc.2.0.yaml b/examples/disp_s1_sample_runconfig-v3.0.0-rc.2.1.yaml similarity index 99% rename from examples/disp_s1_sample_runconfig-v3.0.0-rc.2.0.yaml rename to examples/disp_s1_sample_runconfig-v3.0.0-rc.2.1.yaml index d995a762..62c52cf1 100644 --- a/examples/disp_s1_sample_runconfig-v3.0.0-rc.2.0.yaml +++ b/examples/disp_s1_sample_runconfig-v3.0.0-rc.2.1.yaml @@ -1,4 +1,4 @@ -# Sample RunConfig for use with the DISP-S1 PGE v3.0.0-rc.2.0 +# Sample RunConfig for use with the DISP-S1 PGE v3.0.0-rc.2.1 # This RunConfig should require minimal changes in order to be used with the # OPERA PCM. From c5533337a8ff8a074f322b4b9e5107a48d14e876 Mon Sep 17 00:00:00 2001 From: Scott Collins Date: Thu, 18 Apr 2024 09:05:53 -0500 Subject: [PATCH 05/14] Updated SAS schema for DSWx-S1 to 0.4.2 spec --- ...swx_s1_sample_runconfig-v3.0.0-rc.2.1.yaml | 2 +- .../dswx_s1/schema/dswx_s1_sas_schema.yaml | 19 +++++++++++-------- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/examples/dswx_s1_sample_runconfig-v3.0.0-rc.2.1.yaml b/examples/dswx_s1_sample_runconfig-v3.0.0-rc.2.1.yaml index a4bc7778..b9a9d019 100644 --- a/examples/dswx_s1_sample_runconfig-v3.0.0-rc.2.1.yaml +++ b/examples/dswx_s1_sample_runconfig-v3.0.0-rc.2.1.yaml @@ -209,7 +209,7 @@ RunConfig: # SAS writes the output product(s) to the following file sas_output_path: /home/dswx_user/output_dir - product_version: 0.1 + product_version: "0.1" # DSWx-S1 product format output_imagery_format: 'COG' diff --git a/src/opera/pge/dswx_s1/schema/dswx_s1_sas_schema.yaml b/src/opera/pge/dswx_s1/schema/dswx_s1_sas_schema.yaml index 0548e5c7..fe4e9ea1 100644 --- a/src/opera/pge/dswx_s1/schema/dswx_s1_sas_schema.yaml +++ b/src/opera/pge/dswx_s1/schema/dswx_s1_sas_schema.yaml @@ -72,7 +72,7 @@ runconfig: # according to proper file naming conventions. sas_output_path: str() - product_version: num(required=False) + product_version: str(required=False) # DSWx-S1 product format (default is 'COG') output_imagery_format: enum('GTiff', 'COG', required=False) @@ -99,21 +99,24 @@ runconfig: browse_image_width: int(min=1, required=False) # Flag to collapse water classes if set to True. Default is True. - flag_collapse_wtr_classes: bool(required=False) + flag_collapse_wtr_classes: bool(required=False) # Flag to exclude inundated vegetation from processing if set to True. - exclude_inundated_vegetation: bool(required=False) + exclude_inundated_vegetation: bool(required=False) # Flag to set non-water pixels to NoData value if set to True. - set_not_water_to_nodata: bool(required=False) + set_not_water_to_nodata: bool(required=False) # Flag to set HAND mask pixels to NoData value if set to True. - set_hand_mask_to_nodata: bool(required=False) + set_hand_mask_to_nodata: bool(required=False) # Flag to set layover and shadow pixels to NoData value if set to True. - set_layover_shadow_to_nodata: bool(required=False) + set_layover_shadow_to_nodata: bool(required=False) # Flag to set ocean-masked pixels to NoData value if set to True. - set_ocean_masked_to_nodata: bool(required=False) + set_ocean_masked_to_nodata: bool(required=False) - log_file: str(required=False) \ No newline at end of file + # Flag to save Geotiff to output directory if set to True. + save_tif_to_output: bool(required=False) + + log_file: str(required=False) From 3e0e597ad94d9c5122b5c9671c62633da65836b0 Mon Sep 17 00:00:00 2001 From: Scott Collins Date: Thu, 18 Apr 2024 09:12:03 -0500 Subject: [PATCH 06/14] Rolled SAS image version to 0.4.2 in build_dswx_s1.sh --- .ci/scripts/dswx_s1/build_dswx_s1.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/scripts/dswx_s1/build_dswx_s1.sh b/.ci/scripts/dswx_s1/build_dswx_s1.sh index 65feeebf..ed3353d7 100755 --- a/.ci/scripts/dswx_s1/build_dswx_s1.sh +++ b/.ci/scripts/dswx_s1/build_dswx_s1.sh @@ -24,7 +24,7 @@ BUILD_DATE_TIME=$(date -u +'%Y-%m-%dT%H:%M:%SZ') # defaults, SAS image should be updated as necessary for new image releases from ADT [ -z "${WORKSPACE}" ] && WORKSPACE=$(realpath $(dirname $(realpath $0))/../../..) [ -z "${TAG}" ] && TAG="${USER}-dev" -[ -z "${SAS_IMAGE}" ] && SAS_IMAGE="artifactory-fn.jpl.nasa.gov:16001/gov/nasa/jpl/opera/adt/opera/dswx-s1:calval_0.4.1" +[ -z "${SAS_IMAGE}" ] && SAS_IMAGE="artifactory-fn.jpl.nasa.gov:16001/gov/nasa/jpl/opera/adt/opera/dswx-s1:calval_0.4.2" echo "WORKSPACE: $WORKSPACE" echo "IMAGE: $IMAGE" From 7c84f5e3f9dea2f92d3b7716d54720e934d2b3b0 Mon Sep 17 00:00:00 2001 From: Scott Collins Date: Thu, 18 Apr 2024 09:13:02 -0500 Subject: [PATCH 07/14] Rolled repository verison to 3.0.0-rc.2.1 --- src/opera/_package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/opera/_package.py b/src/opera/_package.py index 60b66ea8..aed59704 100644 --- a/src/opera/_package.py +++ b/src/opera/_package.py @@ -7,7 +7,7 @@ Package information for the OPERA PGE repository. """ -__version__ = "3.0.0-rc.2.0" +__version__ = "3.0.0-rc.2.1" __title__ = "opera-sds-pge" __summary__ = "OPERA SDS Product Generation Executable (PGE) Repository" __uri__ = "https://github.com/nasa/opera-sds-pge" From ab867e4406a9e0ea7d9651b5978d0b294fe4842a Mon Sep 17 00:00:00 2001 From: Scott Collins Date: Thu, 18 Apr 2024 09:13:42 -0500 Subject: [PATCH 08/14] Rolled SAS version to 0.4.2 in dswx_s1_pge.py --- src/opera/pge/dswx_s1/dswx_s1_pge.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/opera/pge/dswx_s1/dswx_s1_pge.py b/src/opera/pge/dswx_s1/dswx_s1_pge.py index 3393123a..b66eb0ce 100644 --- a/src/opera/pge/dswx_s1/dswx_s1_pge.py +++ b/src/opera/pge/dswx_s1/dswx_s1_pge.py @@ -789,10 +789,7 @@ class DSWxS1Executor(DSWxS1PreProcessorMixin, DSWxS1PostProcessorMixin, PgeExecu LEVEL = "L3" """Processing Level for DSWx-S1 Products""" - PGE_VERSION = "3.0.0-rc.2.1" - """Version of the PGE (overrides default from base_pge)""" - - SAS_VERSION = "0.4.1" # CalVal release https://github.com/opera-adt/DSWX-SAR/releases/tag/v0.4.1 + SAS_VERSION = "0.4.2" # CalVal release https://github.com/opera-adt/DSWX-SAR/releases/tag/v0.4.2 """Version of the SAS wrapped by this PGE, should be updated as needed""" def __init__(self, pge_name, runconfig_path, **kwargs): From 052563abf32ab2184871ff5baa66df93f4cc4817 Mon Sep 17 00:00:00 2001 From: Scott Collins Date: Thu, 18 Apr 2024 09:52:00 -0500 Subject: [PATCH 09/14] Updated integration test for DSWx-S1 to use latest expected assets for 0.4.2 --- ...config.yaml => dswx_s1_calval_0.4.2_runconfig.yaml} | 5 +++-- .ci/scripts/dswx_s1/test_int_dswx_s1.sh | 10 +++++----- 2 files changed, 8 insertions(+), 7 deletions(-) rename .ci/scripts/dswx_s1/{dswx_s1_calval_0.4_runconfig.yaml => dswx_s1_calval_0.4.2_runconfig.yaml} (98%) diff --git a/.ci/scripts/dswx_s1/dswx_s1_calval_0.4_runconfig.yaml b/.ci/scripts/dswx_s1/dswx_s1_calval_0.4.2_runconfig.yaml similarity index 98% rename from .ci/scripts/dswx_s1/dswx_s1_calval_0.4_runconfig.yaml rename to .ci/scripts/dswx_s1/dswx_s1_calval_0.4.2_runconfig.yaml index c611c629..f69ff7d5 100644 --- a/.ci/scripts/dswx_s1/dswx_s1_calval_0.4_runconfig.yaml +++ b/.ci/scripts/dswx_s1/dswx_s1_calval_0.4.2_runconfig.yaml @@ -63,7 +63,7 @@ RunConfig: PrimaryExecutable: ProductIdentifier: DSWX_S1 - ProductVersion: "0.1" + ProductVersion: "0.4" ProgramPath: python3 ProgramOptions: - /home/dswx_user/OPERA/DSWX-SAR/src/dswx_sar/dswx_s1.py @@ -157,7 +157,7 @@ RunConfig: product_path: /home/dswx_user/output_dir scratch_path: /home/dswx_user/scratch_dir sas_output_path: /home/dswx_user/output_dir - product_version: 0.1 + product_version: "0.4" output_imagery_format: 'COG' browse_image_group: @@ -170,5 +170,6 @@ RunConfig: set_hand_mask_to_nodata: False set_layover_shadow_to_nodata: False set_ocean_masked_to_nodata: False + save_tif_to_output: True log_file: /home/dswx_user/output_dir/dswx-s1.log diff --git a/.ci/scripts/dswx_s1/test_int_dswx_s1.sh b/.ci/scripts/dswx_s1/test_int_dswx_s1.sh index a114a2c8..0fdc9387 100755 --- a/.ci/scripts/dswx_s1/test_int_dswx_s1.sh +++ b/.ci/scripts/dswx_s1/test_int_dswx_s1.sh @@ -28,9 +28,9 @@ SAMPLE_TIME=1 # RUNCONFIG should be the name of the runconfig in s3://operasds-dev-pge/dswx_s1/ [ -z "${WORKSPACE}" ] && WORKSPACE=$(realpath "$(dirname "$(realpath "$0")")"/../../..) [ -z "${PGE_TAG}" ] && PGE_TAG="${USER}-dev" -[ -z "${INPUT_DATA}" ] && INPUT_DATA="dswx_s1_calval_0.4_expected_input.zip" -[ -z "${EXPECTED_DATA}" ] && EXPECTED_DATA="dswx_s1_calval_0.4_expected_output.zip" -[ -z "${RUNCONFIG}" ] && RUNCONFIG="dswx_s1_calval_0.4_runconfig.yaml" +[ -z "${INPUT_DATA}" ] && INPUT_DATA="dswx_s1_calval_0.4.2_expected_input.zip" +[ -z "${EXPECTED_DATA}" ] && EXPECTED_DATA="dswx_s1_calval_0.4.2_expected_output.zip" +[ -z "${RUNCONFIG}" ] && RUNCONFIG="dswx_s1_calval_0.4.2_runconfig.yaml" [ -z "${TMP_ROOT}" ] && TMP_ROOT="$DEFAULT_TMP_ROOT" # Create the test output directory in the work space @@ -124,7 +124,7 @@ else echo "output_file $output_file" output_file=$(basename -- "$output_file") - if [[ "${output_file##*/}" == *.tif* ]] + if [[ "${output_file##*/}" == *B0*.tif* ]] then for potential_product in B01_WTR B02_BWTR B03_CONF B04_DIAG do @@ -142,7 +142,7 @@ else echo "tile code is $tile_code" - for potential_file in "$expected_data_dir"/*.tif* + for potential_file in "$expected_data_dir"/*B0*.tif* do if [[ "$potential_file" == *"$tile_code"*"$product"* ]]; then echo "expected file is $potential_file" From afa353fe3a1e39332fcb79bf392b7131c81e8a5e Mon Sep 17 00:00:00 2001 From: Scott Collins Date: Mon, 22 Apr 2024 11:02:04 -0700 Subject: [PATCH 10/14] Updated docstring for _compressed_cslc_filename() --- src/opera/pge/disp_s1/disp_s1_pge.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/opera/pge/disp_s1/disp_s1_pge.py b/src/opera/pge/disp_s1/disp_s1_pge.py index 94ecbce7..0fe9a7c3 100644 --- a/src/opera/pge/disp_s1/disp_s1_pge.py +++ b/src/opera/pge/disp_s1/disp_s1_pge.py @@ -373,9 +373,11 @@ def _compressed_cslc_filename(self, inter_filename): Returns the file name to use for compressed CSLC files produced by the DISP-S1 PGE. - The current convention is to maintain the filename assigned by the DISP-S1 - SAS, but this function is still required to ensure the compressed CSLC products - are moved to the output directory defined by the RunConfig. + The compressed CSLC filename for the DISP-S1 PGE consists of: + + __COMPRESSED-CSLC-S1___\ + ___\ + _.h5 Parameters ---------- From fb1f76da340776ef7c57bc77b869db5e3cddd56b Mon Sep 17 00:00:00 2001 From: Scott Collins Date: Wed, 24 Apr 2024 12:22:41 -0700 Subject: [PATCH 11/14] Rolled DISP-S1 SAS container version to 0.3.1 in build_disp_s1.sh --- .ci/scripts/disp_s1/build_disp_s1.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/scripts/disp_s1/build_disp_s1.sh b/.ci/scripts/disp_s1/build_disp_s1.sh index 065e5618..5002c731 100755 --- a/.ci/scripts/disp_s1/build_disp_s1.sh +++ b/.ci/scripts/disp_s1/build_disp_s1.sh @@ -24,7 +24,7 @@ BUILD_DATE_TIME=$(date -u +'%Y-%m-%dT%H:%M:%SZ') # defaults, SAS image should be updated as necessary for new image releases from ADT [ -z "${WORKSPACE}" ] && WORKSPACE=$(realpath $(dirname $(realpath $0))/../../..) [ -z "${TAG}" ] && TAG="${USER}-dev" -[ -z "${SAS_IMAGE}" ] && SAS_IMAGE="artifactory-fn.jpl.nasa.gov:16001/gov/nasa/jpl/opera/adt/opera/disp-s1:0.3" +[ -z "${SAS_IMAGE}" ] && SAS_IMAGE="artifactory-fn.jpl.nasa.gov:16001/gov/nasa/jpl/opera/adt/opera/disp-s1:0.3.1" echo "WORKSPACE: $WORKSPACE" echo "IMAGE: $IMAGE" From 64c11cefe148dad4cc9a9b0718812b7df3a688ad Mon Sep 17 00:00:00 2001 From: Scott Collins Date: Thu, 25 Apr 2024 12:20:56 -0700 Subject: [PATCH 12/14] Fixed file naming conventions for DISP browse product to include _BROWSE suffix --- src/opera/pge/disp_s1/disp_s1_pge.py | 2 +- .../test/pge/disp_s1/test_disp_s1_pge.py | 25 +++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/src/opera/pge/disp_s1/disp_s1_pge.py b/src/opera/pge/disp_s1/disp_s1_pge.py index 0fe9a7c3..583e97bd 100644 --- a/src/opera/pge/disp_s1/disp_s1_pge.py +++ b/src/opera/pge/disp_s1/disp_s1_pge.py @@ -337,7 +337,7 @@ def _browse_filename(self, inter_filename): The file name to assign to browse image created by this PGE. """ - browse_image_filename = f"{self._core_filename(inter_filename)}.png" + browse_image_filename = f"{self._core_filename(inter_filename)}_BROWSE.png" return browse_image_filename diff --git a/src/opera/test/pge/disp_s1/test_disp_s1_pge.py b/src/opera/test/pge/disp_s1/test_disp_s1_pge.py index 830d05e5..cfc41468 100644 --- a/src/opera/test/pge/disp_s1/test_disp_s1_pge.py +++ b/src/opera/test/pge/disp_s1/test_disp_s1_pge.py @@ -207,6 +207,7 @@ def test_disp_s1_pge_execution(self): self.assertTrue(os.path.exists(expected_catalog_metadata_file)) expected_inter_filename = abspath("disp_s1_pge_test/output_dir/20180101_20180330.unw.nc") + expected_browse_filename = abspath("disp_s1_pge_test/output_dir/20180101_20180330.unw.unwrapped_phase.png") # Check that the ISO metadata file was created and all placeholders were # filled in @@ -227,6 +228,14 @@ def test_disp_s1_pge_execution(self): ) self.assertTrue(os.path.exists(expected_disp_product)) + expected_browse_product = join( + pge.runconfig.output_product_path, + pge._browse_filename( + inter_filename=expected_browse_filename + ) + ) + self.assertTrue(os.path.exists(expected_browse_product)) + for compressed_cslc in [ 'compressed_t042_088905_iw1_20221107_20221119_20221213.h5', 'compressed_t042_088906_iw1_20221107_20221119_20221213.h5']: @@ -273,6 +282,22 @@ def test_filename_application(self): rf'\d{{8}}T\d{{6}}Z.nc' ) + png_files = glob.glob(join(output_dir, '*.png')) + png_file = png_files[0] + + expected_browse_filename = pge._browse_filename( + inter_filename=abspath("disp_s1_pge_test/output_dir/20180101_20180330.unw.unwrapped_phase.png") + ) + + self.assertRegex( + expected_browse_filename, + rf'{pge.PROJECT}_{pge.LEVEL}_{pge.NAME}_' + rf'IW_F{disp_metadata["identification"]["frame_id"]:05d}_VV_' + rf'\d{{8}}T\d{{6}}Z_\d{{8}}T\d{{6}}Z_' + rf'v{disp_metadata["identification"]["product_version"]}_' + rf'\d{{8}}T\d{{6}}Z_BROWSE.png' + ) + # Repeat tests for the Compressed CSLC product h5_files = glob.glob(join(output_dir, '*.h5')) h5_file = sorted(h5_files)[0] From 7dc4dfe6ca4c0e06e626005b3b955ba5b9d40b80 Mon Sep 17 00:00:00 2001 From: Scott Collins Date: Thu, 25 Apr 2024 15:30:42 -0700 Subject: [PATCH 13/14] Fixed bug in runconfig.get_ancillary_filenames() where None could be included for empty ancillary mappings from the runconfig --- src/opera/pge/base/runconfig.py | 2 +- src/opera/test/data/invalid_runconfig.yaml | 1 + .../test/data/valid_runconfig_extra_fields.yaml | 1 + src/opera/test/data/valid_runconfig_full.yaml | 1 + src/opera/test/data/valid_runconfig_no_sas.yaml | 1 + src/opera/test/pge/base/test_runconfig.py | 16 +++++++++++++++- 6 files changed, 20 insertions(+), 2 deletions(-) diff --git a/src/opera/pge/base/runconfig.py b/src/opera/pge/base/runconfig.py index c208a931..417aa645 100644 --- a/src/opera/pge/base/runconfig.py +++ b/src/opera/pge/base/runconfig.py @@ -442,7 +442,7 @@ def get_ancillary_filenames(self): for ancillary_mapping in self.ancillary_file_map.values(): if isinstance(ancillary_mapping, list): result.extend(ancillary_mapping) - else: + elif ancillary_mapping is not None: result.append(ancillary_mapping) return result diff --git a/src/opera/test/data/invalid_runconfig.yaml b/src/opera/test/data/invalid_runconfig.yaml index fa5b8e41..eb6325af 100644 --- a/src/opera/test/data/invalid_runconfig.yaml +++ b/src/opera/test/data/invalid_runconfig.yaml @@ -13,6 +13,7 @@ RunConfig: DynamicAncillaryFilesGroup: AncillaryFileMap: DEMFile: input/input_dem.vrt + IonosphereFiles: ProductPathGroup: OutputProductPath: outputs/ diff --git a/src/opera/test/data/valid_runconfig_extra_fields.yaml b/src/opera/test/data/valid_runconfig_extra_fields.yaml index 08b2a476..b9c2173b 100644 --- a/src/opera/test/data/valid_runconfig_extra_fields.yaml +++ b/src/opera/test/data/valid_runconfig_extra_fields.yaml @@ -18,6 +18,7 @@ RunConfig: DynamicAncillaryFilesGroup: AncillaryFileMap: DEMFile: input/input_dem.vrt + IonosphereFiles: ProductPathGroup: OutputProductPath: outputs/ diff --git a/src/opera/test/data/valid_runconfig_full.yaml b/src/opera/test/data/valid_runconfig_full.yaml index 3c6dd1da..f15b0649 100644 --- a/src/opera/test/data/valid_runconfig_full.yaml +++ b/src/opera/test/data/valid_runconfig_full.yaml @@ -15,6 +15,7 @@ RunConfig: DynamicAncillaryFilesGroup: AncillaryFileMap: DEMFile: input/input_dem.vrt + IonosphereFiles: ProductPathGroup: OutputProductPath: outputs/ diff --git a/src/opera/test/data/valid_runconfig_no_sas.yaml b/src/opera/test/data/valid_runconfig_no_sas.yaml index 230db405..dacee2e1 100644 --- a/src/opera/test/data/valid_runconfig_no_sas.yaml +++ b/src/opera/test/data/valid_runconfig_no_sas.yaml @@ -15,6 +15,7 @@ RunConfig: DynamicAncillaryFilesGroup: AncillaryFileMap: DEMFile: input/input_dem.vrt + IonosphereFiles: ProductPathGroup: OutputProductPath: outputs/ diff --git a/src/opera/test/pge/base/test_runconfig.py b/src/opera/test/pge/base/test_runconfig.py index dadd2d8e..2e785c61 100755 --- a/src/opera/test/pge/base/test_runconfig.py +++ b/src/opera/test/pge/base/test_runconfig.py @@ -43,7 +43,7 @@ def _compare_runconfig_to_expected(self, runconfig): self.assertEqual(runconfig.name, "OPERA-SAMPLE-PGE-SAS-CONFIG") self.assertEqual(runconfig.pge_name, "EXAMPLE_PGE") self.assertListEqual(runconfig.input_files, ["input/input_file01.h5", "input/input_file02.h5"]) - self.assertDictEqual(runconfig.ancillary_file_map, {"DEMFile": "input/input_dem.vrt"}) + self.assertDictEqual(runconfig.ancillary_file_map, {"DEMFile": "input/input_dem.vrt", 'IonosphereFiles': None}) self.assertEqual(runconfig.output_product_path, "outputs/") self.assertEqual(runconfig.scratch_path, "temp/") self.assertEqual(runconfig.product_identifier, "EXAMPLE") @@ -82,6 +82,13 @@ def test_full_pge_config_parse_and_validate(self): # this test self.assertIsInstance(runconfig.sas_config, dict) + # Ensure empty ancillary fields are filtered out of list returned + # from get_ancillary_filenames() + ancillary_filenames = runconfig.get_ancillary_filenames() + + self.assertIn("input/input_dem.vrt", ancillary_filenames) + self.assertNotIn(None, ancillary_filenames) + def test_pge_only_config_parse_and_validate(self): """ Test basic parsing and validation of an input RunConfig that only @@ -104,6 +111,13 @@ def test_pge_only_config_parse_and_validate(self): # Check that None was assigned for SAS config section self.assertIsNone(runconfig.sas_config) + # Ensure empty ancillary fields are filtered out of list returned + # from get_ancillary_filenames() + ancillary_filenames = runconfig.get_ancillary_filenames() + + self.assertIn("input/input_dem.vrt", ancillary_filenames) + self.assertNotIn(None, ancillary_filenames) + def test_strict_mode_validation(self): """Test validation of a RunConfig with strict_mode both enabled and disabled""" # Parse a valid runconfig, but modify it with fields not in the base From 785b745710bf2d2dea55fcaf20601588e552f3bf Mon Sep 17 00:00:00 2001 From: Scott Collins Date: Fri, 26 Apr 2024 10:32:04 -0700 Subject: [PATCH 14/14] Added conditionals to check for missing ancillary files in ISO xml template for DISP-S1 --- .../OPERA_ISO_metadata_L3_DISP_S1_template.xml.jinja2 | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/opera/pge/disp_s1/templates/OPERA_ISO_metadata_L3_DISP_S1_template.xml.jinja2 b/src/opera/pge/disp_s1/templates/OPERA_ISO_metadata_L3_DISP_S1_template.xml.jinja2 index d5e1a9e8..ff28f02a 100644 --- a/src/opera/pge/disp_s1/templates/OPERA_ISO_metadata_L3_DISP_S1_template.xml.jinja2 +++ b/src/opera/pge/disp_s1/templates/OPERA_ISO_metadata_L3_DISP_S1_template.xml.jinja2 @@ -976,6 +976,7 @@ + {%- if run_config.Groups.PGE.DynamicAncillaryFilesGroup.AncillaryFileMap.amplitude_dispersion_files is not none %} {%- for amplitude_dispersion_file in run_config.Groups.PGE.DynamicAncillaryFilesGroup.AncillaryFileMap.amplitude_dispersion_files %} @@ -985,6 +986,8 @@ {%- endfor %} + {%- endif %} + {%- if run_config.Groups.PGE.DynamicAncillaryFilesGroup.AncillaryFileMap.amplitude_mean_files is not none %} {%- for amplitude_mean_file in run_config.Groups.PGE.DynamicAncillaryFilesGroup.AncillaryFileMap.amplitude_mean_files %} @@ -994,6 +997,8 @@ {%- endfor %} + {%- endif %} + {%- if run_config.Groups.PGE.DynamicAncillaryFilesGroup.AncillaryFileMap.static_layers_files is not none %} {%- for static_layers_file in run_config.Groups.PGE.DynamicAncillaryFilesGroup.AncillaryFileMap.static_layers_files %} @@ -1003,6 +1008,8 @@ {%- endfor %} + {%- endif %} + {%- if run_config.Groups.PGE.DynamicAncillaryFilesGroup.AncillaryFileMap.ionosphere_files is not none %} {%- for ionosphere_file in run_config.Groups.PGE.DynamicAncillaryFilesGroup.AncillaryFileMap.ionosphere_files %} @@ -1012,6 +1019,8 @@ {%- endfor %} + {%- endif %} + {%- if run_config.Groups.PGE.DynamicAncillaryFilesGroup.AncillaryFileMap.troposphere_files is not none %} {%- for troposphere_file in run_config.Groups.PGE.DynamicAncillaryFilesGroup.AncillaryFileMap.troposphere_files %} @@ -1021,6 +1030,7 @@ {%- endfor %} + {%- endif %}