From 3727bca8e294b6ea7e64b09351f45abd8a871335 Mon Sep 17 00:00:00 2001 From: Scott Collins Date: Tue, 4 Apr 2023 09:25:13 -0700 Subject: [PATCH 1/7] Updated Artifactory path in build_rtc_s1.sh to point at gamma image --- .ci/scripts/build_rtc_s1.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/scripts/build_rtc_s1.sh b/.ci/scripts/build_rtc_s1.sh index f6496b68..6c9a058a 100755 --- a/.ci/scripts/build_rtc_s1.sh +++ b/.ci/scripts/build_rtc_s1.sh @@ -24,7 +24,7 @@ BUILD_DATE_TIME=$(date -u +'%Y-%m-%dT%H:%M:%SZ') # defaults, SAS image should be updated as necessary for new image releases from ADT [ -z "${WORKSPACE}" ] && WORKSPACE=$(realpath $(dirname $(realpath $0))/../..) [ -z "${TAG}" ] && TAG="${USER}-dev" -[ -z "${SAS_IMAGE}" ] && SAS_IMAGE="artifactory-fn.jpl.nasa.gov:16001/gov/nasa/jpl/opera/adt/opera/rtc:beta_0.2.1" +[ -z "${SAS_IMAGE}" ] && SAS_IMAGE="artifactory-fn.jpl.nasa.gov:16001/gov/nasa/jpl/opera/adt/opera/rtc:gamma_0.3" echo "WORKSPACE: $WORKSPACE" echo "IMAGE: $IMAGE" From cd043cd3acff7494252d9157ac9a46983b4bab52 Mon Sep 17 00:00:00 2001 From: Scott Collins Date: Tue, 4 Apr 2023 09:31:15 -0700 Subject: [PATCH 2/7] Updated Yamale schema for RTC-S1 PGE to gamma spec --- .../pge/rtc_s1/schema/rtc_s1_sas_schema.yaml | 64 +++++++++++++++++-- 1 file changed, 60 insertions(+), 4 deletions(-) diff --git a/src/opera/pge/rtc_s1/schema/rtc_s1_sas_schema.yaml b/src/opera/pge/rtc_s1/schema/rtc_s1_sas_schema.yaml index 728f7f2f..4f588086 100644 --- a/src/opera/pge/rtc_s1/schema/rtc_s1_sas_schema.yaml +++ b/src/opera/pge/rtc_s1/schema/rtc_s1_sas_schema.yaml @@ -14,8 +14,8 @@ runconfig: safe_file_path: list(str(), min=1) # Required. List of orbit (EOF) files orbit_file_path: list(str(), min=1) - # Required. The unique burst ID to process, leave empty to process all bursts - burst_id : list(str(), min=1, required=False) + # Optional. Burst ID to process (empty for all bursts) + burst_id: list(str(), min=1, required=False) dynamic_ancillary_file_group: # Digital Elevation Model. @@ -47,8 +47,18 @@ runconfig: # {output_dir}/{burst_id}/{product_id}_v{product_version}{suffix}.{ext} # If option `save_mosaics` is set, output mosaics are saved to: # {output_dir}/{product_id}_v{product_version}{suffix}.{ext} - # If the field `product_id`` is left empty, the prefix "rtc_product" - # will be used instead. + # + # If the `product_id` contains the substring "_{burst_id}", the + # substring will be substituted by either: + # - "_" followed by the burst ID, if the product is a burst; or + # - An empty string, if the product is a mosaic. + # + # For example, the `product_id` = `RTC-S1_{burst_id}_S1B` will become + # `RTC-S1_069-147170-IW1_S1B` for the burst t069-147170-IW1; and it + # will become `RTC-S1_S1B` for the mosaic product. + # + # If the field `product_id`` is left empty, the prefix + # "OPERA_L2_RTC-S1_{burst_id}" will be used instead. # `suffix` is only used when there are multiple output files. # `ext` is determined by geocoding_options.output_imagery_format. output_dir: str() @@ -56,7 +66,13 @@ runconfig: # RTC-S1 imagery save_bursts: bool(required=False) + + # Save mosaic of RTC-S1 bursts save_mosaics: bool(required=False) + + # Save browse image(s) + save_browse: bool(required=False) + output_imagery_format: enum('HDF5', 'NETCDF', 'ENVI', 'GTiff', 'COG', required=False) output_imagery_compression: str(required=False) output_imagery_nbits: int(min=1, required=False) @@ -130,6 +146,13 @@ processing_options: # Geocoding options geocoding: include('geocoding_options', required=False) + # Browse image + browse_image_group: include('browse_image_options', required=False) + + # Number of parallel processes for burst processing + num_workers: int(required=False) + + rtc_options: # RTC output type: empty value to turn off the RTC # The output_type defaults to "gamma0" if the key is absent @@ -148,6 +171,12 @@ rtc_options: geocoding_options: + # OPTIONAL - Apply RSLC metadata valid-samples sub-swath masking + apply_valid_samples_sub_swath_masking: bool(required=False) + + # OPTIONAL - Apply shadow masking + apply_shadow_masking: bool(required=False) + # Algorithm type, area projection or interpolation: sinc, bilinear, bicubic, nearest, and biquintic algorithm_type: enum('area_projection', 'sinc', 'bilinear', 'bicubic', 'nearest', 'biquintic', required=False) @@ -222,3 +251,30 @@ geocoding_options: bottom_right: x: num(required=False) y: num(required=False) + + +browse_image_options: + + # If neither height or width parameters are provided, the browse + # image is generated with the same pixel spacing of the RTC-S1 + # imagery (burst or mosaic). + + # If the height parameter is provided but the width is not provided, + # a new width is assigned in order to keep the aspect ratio + # of the RTC-S1 geographic grid. + + # Conversely, if the width parameter is provided but the height is not, + # a new height is assigned in order to keep the aspect ratio + # of the RTC-S1 geographic grid. + + # Height in pixels for the PNG browse image of RTC-S1 bursts. + browse_image_burst_height: int(min=1, required=False) + + # Width in pixels for the PNG browse image of RTC-S1 bursts + browse_image_burst_width: int(min=1, required=False) + + # Height in pixels for the PNG browse image of RTC-S1 mosaics. + browse_image_mosaic_height: int(min=1, required=False) + + # Width in pixels for the PNG browse image of RTC-S1 mosaics + browse_image_mosaic_width: int(min=1, required=False) From 284e54e767970fd1052740c251759a2ebb71d3b6 Mon Sep 17 00:00:00 2001 From: Scott Collins Date: Tue, 4 Apr 2023 09:49:45 -0700 Subject: [PATCH 3/7] Updated example RunConfig for RTC-S1 to v2.0.0-rc.1.0 spec --- ...tc_s1_sample_runconfig-v2.0.0-rc.1.0.yaml} | 86 ++++++++++++++----- 1 file changed, 66 insertions(+), 20 deletions(-) rename examples/{rtc_s1_sample_runconfig-v2.0.0-er.5.0.yaml => rtc_s1_sample_runconfig-v2.0.0-rc.1.0.yaml} (77%) diff --git a/examples/rtc_s1_sample_runconfig-v2.0.0-er.5.0.yaml b/examples/rtc_s1_sample_runconfig-v2.0.0-rc.1.0.yaml similarity index 77% rename from examples/rtc_s1_sample_runconfig-v2.0.0-er.5.0.yaml rename to examples/rtc_s1_sample_runconfig-v2.0.0-rc.1.0.yaml index 5ca21e4f..abef6391 100644 --- a/examples/rtc_s1_sample_runconfig-v2.0.0-er.5.0.yaml +++ b/examples/rtc_s1_sample_runconfig-v2.0.0-rc.1.0.yaml @@ -1,4 +1,4 @@ -# Sample RunConfig for use with the RTC-S1 PGE v2.0.0-er.5.0 +# Sample RunConfig for use with the RTC-S1 PGE v2.0.0-rc.1.0 # This RunConfig should require minimal changes in order to be used with the # OPERA PCM. @@ -139,14 +139,13 @@ RunConfig: dynamic_ancillary_file_group: # Digital elevation model dem_file: /home/rtc_user/input_dir/dem.tif + dem_description: Digital Elevation Model (DEM) for the NASA OPERA project (v1.0) based on the Copernicus DEM 30-m and Copernicus 90-m referenced to the WGS84 ellipsoid static_ancillary_file_group: # burst database sqlite file burst_database_file: /home/rtc_user/input_dir/opera_burst_database_deploy_2022_1212.sqlite3 product_group: - processing_type: 'NOMINAL' - product_version: 1.0 # This should match the path used for OutputProductPath @@ -161,14 +160,31 @@ RunConfig: # These field determines the intermediate file name used # by the SAS for its output products. These products will # be renamed by the PGE to match the OPERA file name conventions. - product_id: rtc_product + product_id: OPERA_L2_RTC-S1_T{burst_id} + # RTC-S1 imagery save_bursts: True + + # Save mosaic of RTC-S1 bursts save_mosaics: False + + # Save browse image(s) + save_browse: True + output_imagery_format: COG output_imagery_compression: ZSTD output_imagery_nbits: 16 + # Optional. Save secondary layers (e.g., inc. angle) within + # the HDF5 file + save_secondary_layers_as_hdf5: False + + # Save RTC-S1 metadata in the HDF5 format + # Optional for `output_imagery_format` equal to 'ENVI', 'GTiff', or + # 'COG', and enabled by default for `output_imagery_format` equal + # to 'HDF5' or 'NETCDF' or `save_secondary_layers_as_hdf5` is True + save_metadata: True + primary_executable: # This should match the value used for ProductIdentifier product_type: RTC_S1 @@ -180,6 +196,7 @@ RunConfig: # Check if ancillary input covers entirely output products check_ancillary_inputs_coverage: True + # Polarization channels to process. polarization: dual-pol # Options to run geo2rdr @@ -192,6 +209,9 @@ RunConfig: threshold: 1.0e-7 numiter: 25 + # DEM interpolation method + dem_interpolation_method: biquintic + # Apply absolute radiometric correction apply_absolute_radiometric_correction: True @@ -208,7 +228,6 @@ RunConfig: apply_dry_tropospheric_delay_correction: True # OPTIONAL - to control behavior of RTC module - # (only applicable if geocode.apply_rtc is True) rtc: # OPTIONAL - Choices: # "gamma0" (default) @@ -226,27 +245,40 @@ RunConfig: input_terrain_radiometry: beta0 # OPTIONAL - Minimum RTC area factor in dB - rtc_min_value_db: + rtc_min_value_db: -30 # RTC DEM upsampling dem_upsampling: 2 + # OPTIONAL - to provide the number of processes when processing the bursts in parallel + # "0" means that the number will be automatically decided based on + # the number of cores, `OMP_NUM_THREADS` in environment setting, + # and the number of burst to process in runconfig + num_workers: 0 + # OPTIONAL - Mechanism to specify output posting and DEM geocoding: - # OPTIONAL - + # OPTIONAL - Apply RSLC metadata valid-samples sub-swath masking + apply_valid_samples_sub_swath_masking: True + + # OPTIONAL - Apply shadow masking + apply_shadow_masking: True + + # OPTIONAL - Algorithm type, area projection or + # interpolation: sinc, bilinear, bicubic, nearest, and biquintic algorithm_type: area_projection - # OPTIONAL - Choices: "single_block", "geogrid", "geogrid_radargrid", and "auto" (default) + # OPTIONAL - Choices: "single_block", "geogrid", "geogrid_and_radargrid", and "auto" (default) memory_mode: auto # OPTIONAL - Processing upsampling factor applied to input geogrid geogrid_upsampling: 1 # Save the incidence angle - save_incidence_angle: False + save_incidence_angle: True # Save the local-incidence angle - save_local_inc_angle: False + save_local_inc_angle: True # Save the projection angle save_projection_angle: False @@ -258,16 +290,16 @@ RunConfig: save_range_slope: False # Save the number of looks used to compute GCOV - save_nlooks: False + save_nlooks: True # Save the RTC area factor used to compute GCOV - save_rtc_anf: False + save_rtc_anf: True # Save interpolated DEM used to compute GCOV save_dem: False # Save layover shadow mask - save_layover_shadow_mask: False + save_layover_shadow_mask: True # OPTIONAL - Absolute radiometric correction abs_rad_cal: 1 @@ -278,18 +310,32 @@ RunConfig: # OPTIONAL - Clip values below threshold clip_min: - # OPTIONAL - Double sampling of the radar-grid - # input sampling in the range direction + # Double SLC sampling in the range direction upsample_radargrid: False - output_epsg: - x_posting: 30 - y_posting: 30 - x_snap: 30 - y_snap: 30 top_left: x: y: bottom_right: x: y: + + browse_image_group: + + # If neither height or width parameters are provided, the browse + # image is generated with the same pixel spacing of the RTC-S1 + # imagery (burst or mosaic). + + # If the height parameter is provided but the width is not provided, + # a new width is assigned in order to keep the aspect ratio + # of the RTC-S1 geographic grid. + + # Conversely, if the width parameter is provided but the height is not, + # a new height is assigned in order to keep the aspect ratio + # of the RTC-S1 geographic grid. + + # Height in pixels for the PNG browse image of RTC-S1 bursts. + browse_image_burst_height: 1024 + + # Height in pixels for the PNG browse image of RTC-S1 mosaics. + browse_image_mosaic_height: 1024 From f6558b5260a598ca2aebc46ae42d521ae09246b2 Mon Sep 17 00:00:00 2001 From: Scott Collins Date: Tue, 4 Apr 2023 10:33:41 -0700 Subject: [PATCH 4/7] Updated version numbers in rtc_s1_pge.py --- src/opera/pge/rtc_s1/rtc_s1_pge.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/opera/pge/rtc_s1/rtc_s1_pge.py b/src/opera/pge/rtc_s1/rtc_s1_pge.py index a8dee194..5a8cd5d6 100644 --- a/src/opera/pge/rtc_s1/rtc_s1_pge.py +++ b/src/opera/pge/rtc_s1/rtc_s1_pge.py @@ -680,10 +680,10 @@ class RtcS1Executor(RtcS1PreProcessorMixin, RtcS1PostProcessorMixin, PgeExecutor LEVEL = "L2" """Processing Level for RTC-S1 Products""" - PGE_VERSION = "2.0.0-er.5.1" + PGE_VERSION = "2.0.0-rc.1.0" """Version of the PGE (overrides default from base_pge)""" - SAS_VERSION = "0.2.1" # Beta release https://github.com/opera-adt/RTC/releases/tag/v0.2.1 + SAS_VERSION = "0.3" # Gamma release https://github.com/opera-adt/RTC/releases/tag/v0.3 """Version of the SAS wrapped by this PGE, should be updated as needed""" SOURCE = "S1" From cb56972d2278c5ba412fdc324561f695b75781ec Mon Sep 17 00:00:00 2001 From: Scott Collins Date: Tue, 4 Apr 2023 11:29:09 -0700 Subject: [PATCH 5/7] Added file name convention support for the RTC-S1 static layers and browse products --- src/opera/pge/rtc_s1/rtc_s1_pge.py | 106 +++++++++++++++++++++++++++-- src/opera/util/metadata_utils.py | 6 +- 2 files changed, 105 insertions(+), 7 deletions(-) diff --git a/src/opera/pge/rtc_s1/rtc_s1_pge.py b/src/opera/pge/rtc_s1/rtc_s1_pge.py index 5a8cd5d6..a2d3e8bc 100644 --- a/src/opera/pge/rtc_s1/rtc_s1_pge.py +++ b/src/opera/pge/rtc_s1/rtc_s1_pge.py @@ -21,6 +21,7 @@ from opera.util.error_codes import ErrorCode from opera.util.input_validation import validate_slc_s1_inputs from opera.util.metadata_utils import get_rtc_s1_product_metadata +from opera.util.metadata_utils import get_sensor_from_spacecraft_name from opera.util.render_jinja2 import render_jinja2 from opera.util.time import get_time_for_filename @@ -94,7 +95,8 @@ def _validate_output(self): if not dirs and scratch_dir not in path: # Ignore files in 'output_dir' and scratch directory out_dir_walk_dict[basename(path)] = files - output_format = self.runconfig.sas_config['runconfig']['groups']['product_group']['output_imagery_format'] + sas_product_group = self.runconfig.sas_config['runconfig']['groups']['product_group'] + output_format = sas_product_group['output_imagery_format'] if output_format == 'NETCDF': expected_ext = ['nc'] @@ -103,6 +105,11 @@ def _validate_output(self): elif output_format in ('GTiff', 'COG', 'ENVI'): expected_ext = ['tiff', 'tif', 'h5'] + save_browse = sas_product_group['save_browse'] + + if save_browse: + expected_ext.append('png') + # Verify: files in subdirectories, file length, and proper extension. for dir_name_key, file_names in out_dir_walk_dict.items(): if len(file_names) == 0: @@ -221,12 +228,16 @@ def _rtc_filename(self, inter_filename): # Use doppler start time as the acq time and convert it to our format # used for file naming acquisition_time = product_metadata['identification']['zeroDopplerStartTime'] + + if not acquisition_time.endswith('Z'): + acquisition_time += 'Z' + acquisition_time = get_time_for_filename( - datetime.strptime(acquisition_time, "%Y-%m-%dT%H:%M:%S.%f") + datetime.strptime(acquisition_time, "%Y-%m-%dT%H:%M:%S.%fZ") ) # Get the sensor (should be either S1A or S1B) - sensor = product_metadata['identification']['missionId'] + sensor = get_sensor_from_spacecraft_name(product_metadata['identification']['platform']) # Spacing is assumed to be identical in both X and Y direction spacing = int(product_metadata['frequencyA']['xCoordinateSpacing']) @@ -249,6 +260,54 @@ def _rtc_filename(self, inter_filename): return rtc_filename + def _static_layer_filename(self, inter_filename): + """ + Returns the final file name for the static layer RTC product which + may be optionally produced by this PGE. There are currently 5 static layer + products which may be produced by this PGE, each identified by a + static layer name appended to the end of the intermediate filename. + + The filename for static layer RTC products consists of: + + _.tif + + Where is returned by RtcS1PostProcessorMixin._rtc_filename(), + and is the identifier for the specific static layer + as parsed from the intermediate filename. + + Parameters + ---------- + inter_filename : str + The intermediate filename of the static layer output product to generate + a filename for. This parameter may is used to derive the core RTC + file name component, to which the particular static layer name (nlooks, + shadow_mask, etc...) is appended to denote the type. + + Returns + ------- + static_layer_filename : str + The file name to assign to static layer GeoTIFF product(s) created + by this PGE. + + """ + filename, ext = os.path.splitext(basename(inter_filename)) + + # The name of the static layer should always follow the product version + # within the intermediate filename + sas_product_group = self.runconfig.sas_config['runconfig']['groups']['product_group'] + product_version = str(sas_product_group['product_version']) + + if not product_version.startswith('v'): + product_version = f"v{product_version}" + + static_layer_name = filename.split(product_version)[-1] + + rtc_filename = self._rtc_filename(inter_filename) + + static_layer_filename = f"{rtc_filename}{static_layer_name}.tif" + + return static_layer_filename + def _rtc_geotiff_filename(self, inter_filename): """ Returns the file name to use for GeoTIFF format RTC products produced @@ -285,6 +344,36 @@ def _rtc_geotiff_filename(self, inter_filename): return f"{rtc_filename}_{polarization}.tif" + def _browse_filename(self, inter_filename): + """ + Returns the final file name of the PNG browse image product which may + be optionally produced by this PGE. + + The filename for RTC metadata products consists of: + + _BROWSE.png + + Where is returned by RtcS1PostProcessorMixin._rtc_filename(). + + Parameters + ---------- + inter_filename : str + The intermediate filename of the output product to generate + a filename for. This parameter may be used to inspect the file + in order to derive any necessary components of the returned filename. + + Returns + ------- + browse_filename : str + The file name to assign to browse product created by this PGE. + + """ + rtc_filename = self._rtc_filename(inter_filename) + + browse_filename = f"{rtc_filename}_BROWSE.png" + + return browse_filename + def _rtc_metadata_filename(self, inter_filename): """ Returns the file name to use for RTC metadata products produced by this PGE. @@ -342,7 +431,7 @@ def _ancillary_filename(self): product_metadata = list(self._burst_metadata_cache.values())[0] # Get the sensor (should be either S1A or S1B) - sensor = product_metadata['identification']['missionId'] + sensor = get_sensor_from_spacecraft_name(product_metadata['identification']['platform']) # Spacing is assumed to be identical in both X and Y direction spacing = int(product_metadata['frequencyA']['xCoordinateSpacing']) @@ -692,7 +781,14 @@ def __init__(self, pge_name, runconfig_path, **kwargs): super().__init__(pge_name, runconfig_path, **kwargs) self.rename_by_pattern_map = { - "*.tif": self._rtc_geotiff_filename, + "*_VV.tif": self._rtc_geotiff_filename, + "*_VH.tif": self._rtc_geotiff_filename, + "*_rtc_anf.tif": self._static_layer_filename, + "*_nlooks.tif": self._static_layer_filename, + "*_local_incidence_angle.tif": self._static_layer_filename, + "*_layover_shadow_mask.tif": self._static_layer_filename, + "*_incidence_angle.tif": self._static_layer_filename, + "*.png": self._browse_filename, "*.h5": self._rtc_metadata_filename, "*.nc": self._rtc_metadata_filename } diff --git a/src/opera/util/metadata_utils.py b/src/opera/util/metadata_utils.py index 73fea03c..23a990d1 100644 --- a/src/opera/util/metadata_utils.py +++ b/src/opera/util/metadata_utils.py @@ -84,8 +84,8 @@ def CoordinateTransformation(src, dest): def get_sensor_from_spacecraft_name(spacecraft_name): """ - Returns the HLS sensor short name from the full spacecraft name. - The short name is used with output file naming conventions for DSWx-HLS + Returns the sensor short name from the full spacecraft name. + The short name is used with output file naming conventions for PGE products Parameters @@ -108,6 +108,8 @@ def get_sensor_from_spacecraft_name(spacecraft_name): return { 'LANDSAT-8': 'L8', 'LANDSAT-9': 'L9', + 'SENTINEL-1A': 'S1A', + 'SENTINEL-1B': 'S1B', 'SENTINEL-2A': 'S2A', 'SENTINEL-2B': 'S2B' }[spacecraft_name.upper()] From 7b35f717c1472ac6a2e4e02c1c237d442f2c2130 Mon Sep 17 00:00:00 2001 From: Scott Collins Date: Tue, 4 Apr 2023 14:30:16 -0700 Subject: [PATCH 6/7] Updated ISO template and metadata collection for RTC-S1 gamma delivery --- src/opera/pge/rtc_s1/rtc_s1_pge.py | 15 -- ...ISO_metadata_L2_RTC_S1_template.xml.jinja2 | 223 ++++++------------ src/opera/util/metadata_utils.py | 23 +- 3 files changed, 79 insertions(+), 182 deletions(-) diff --git a/src/opera/pge/rtc_s1/rtc_s1_pge.py b/src/opera/pge/rtc_s1/rtc_s1_pge.py index a2d3e8bc..aa360114 100644 --- a/src/opera/pge/rtc_s1/rtc_s1_pge.py +++ b/src/opera/pge/rtc_s1/rtc_s1_pge.py @@ -559,21 +559,6 @@ def _collect_rtc_product_metadata(self, metadata_product): output_product_metadata['frequencyA']['frequencyALength'] = len(output_product_metadata['frequencyA'] ['yCoordinates']) - # TODO: the following fields seems to be missing in the interface delivery products, - # but are documented, remove these kludges once they are actually available - if 'azimuthBandwidth' not in output_product_metadata['frequencyA']: - output_product_metadata['frequencyA']['azimuthBandwidth'] = 12345678.9 - - if 'noiseCorrectionFlag' not in output_product_metadata['frequencyA']: - output_product_metadata['frequencyA']['noiseCorrectionFlag'] = False - - if 'plannedDatatakeId' not in output_product_metadata['identification']: - output_product_metadata['identification']['plannedDatatakeId'] = ['datatake1', 'datatake2'] - - if 'plannedObservationId' not in output_product_metadata['identification']: - output_product_metadata['identification']['plannedObservationId'] = ['obs1', 'obs2'] - # TODO: end kludges - return output_product_metadata def _create_custom_metadata(self): diff --git a/src/opera/pge/rtc_s1/templates/OPERA_ISO_metadata_L2_RTC_S1_template.xml.jinja2 b/src/opera/pge/rtc_s1/templates/OPERA_ISO_metadata_L2_RTC_S1_template.xml.jinja2 index a15cc352..c5d46405 100644 --- a/src/opera/pge/rtc_s1/templates/OPERA_ISO_metadata_L2_RTC_S1_template.xml.jinja2 +++ b/src/opera/pge/rtc_s1/templates/OPERA_ISO_metadata_L2_RTC_S1_template.xml.jinja2 @@ -569,10 +569,10 @@ instrumentInformation - BurstID + AcquisitionMode - Burst identification (burst ID) + Acquisition Mode Unitless @@ -583,7 +583,7 @@ - {{ product_output.identification.burstID }}{# ISO_OPERA_burstID #} + {{ product_output.identification.acquisitionMode }}{# ISO_OPERA_acquisitionMode #} @@ -593,21 +593,21 @@ instrumentInformation - TrackNumber + BeamID - Track number + Beam identifier Unitless - int + string - {{ product_output.identification.trackNumber }}{# ISO_OPERA_trackNumber #} + {{ product_output.identification.beamID }}{# ISO_OPERA_beamID #} @@ -617,18 +617,45 @@ instrumentInformation - MissionID + BurstID - Mission identifier + Burst identification (burst ID) + + Unitless + string - {{ product_output.identification.missionId }}{# ISO_OPERA_missionID #} + {{ product_output.identification.burstID }}{# ISO_OPERA_burstID #} + + + + + + + instrumentInformation + + + TrackNumber + + + Track number + + + Unitless + + + int + + + + + {{ product_output.identification.trackNumber }}{# ISO_OPERA_trackNumber #} @@ -722,18 +749,18 @@ instrumentInformation - ZeroDopplerStartTime + Platform - Azimuth start time of the product + Platform name - dateTimeString + string - {{ product_output.identification.zeroDopplerStartTime }}{# ISO_OPERA_zeroDopplerStartTime #} + {{ product_output.identification.platform }}{# ISO_OPERA_platform #} @@ -743,10 +770,10 @@ instrumentInformation - ZeroDopplerEndTime + ZeroDopplerStartTime - Azimuth stop time of the product + Azimuth start time of the product dateTimeString @@ -754,28 +781,28 @@ - {{ product_output.identification.zeroDopplerEndTime }}{# ISO_OPERA_zeroDopplerEndTime #} + {{ product_output.identification.zeroDopplerStartTime }}{# ISO_OPERA_zeroDopplerStartTime #} - contentInformation + instrumentInformation - PlannedDatatakeID + ZeroDopplerEndTime - List of planned datatakes included in the product + Azimuth stop time of the product - string + dateTimeString - {{ product_output.identification.plannedDatatakeId|string }}{# ISO_OPERA_plannedDatatakeId #} + {{ product_output.identification.zeroDopplerEndTime }}{# ISO_OPERA_zeroDopplerEndTime #} @@ -785,10 +812,10 @@ contentInformation - PlannedObservationID + IsUrgentObservation - List of planned observations included in the product + List of booleans indicating if datatakes are nominal or urgent string @@ -796,7 +823,7 @@ - {{ product_output.identification.plannedObservationId|string }}{# ISO_OPERA_plannedObservationId #} + {{ product_output.identification.isUrgentObservation|string }}{# ISO_OPERA_isUrgentObservation #} @@ -806,10 +833,10 @@ contentInformation - IsUrgentObservation + ListOfFrequencies - List of booleans indicating if datatakes are nominal or urgent + List of frequency layers available in the product string @@ -817,7 +844,7 @@ - {{ product_output.identification.isUrgentObservation|string }}{# ISO_OPERA_isUrgentObservation #} + {{ product_output.identification.listOfFrequencies|string }}{# ISO_OPERA_listOfFrequencies #} @@ -827,18 +854,18 @@ contentInformation - ListOfFrequencies + DiagnosticModeFlag - List of frequency layers available in the product + Indicates if the radar mode is a diagnostic mode or not: True or False - string + boolean - {{ product_output.identification.listOfFrequencies|string }}{# ISO_OPERA_listOfFrequencies #} + {{ product_output.identification.diagnosticModeFlag }}{# ISO_OPERA_diagnosticModeFlag #} @@ -848,10 +875,10 @@ contentInformation - DiagnosticModeFlag + IsGeocoded - Indicates if the radar mode is a diagnostic mode or not: True or False + Flag to indicate radar geometry or geocoded product boolean @@ -859,28 +886,28 @@ - {{ product_output.identification.diagnosticModeFlag }}{# ISO_OPERA_diagnosticModeFlag #} + {{ product_output.identification.isGeocoded }}{# ISO_OPERA_isGeocoded #} - contentInformation + processingInformation - IsGeocoded + ProcessingDatetime - Flag to indicate radar geometry or geocoded product + Processing date and time - boolean + string - {{ product_output.identification.isGeocoded }}{# ISO_OPERA_isGeocoded #} + {{ product_output.identification.processingDateTime }}{# ISO_OPERA_processingDatetime #} @@ -997,30 +1024,6 @@ {{ product_output.frequencyA.rangeBandwidth }}{# ISO_OPERA_rangeBandwidth #} - - - - - processingParameter - - - AzimuthBandwidth - - - Processed azimuth bandwidth in Hz - - - Hertz - - - float - - - - - {{ product_output.frequencyA.azimuthBandwidth }}{# ISO_OPERA_azimuthBandwidth #} - - @@ -1093,69 +1096,6 @@ {{ product_output.frequencyA.zeroDopplerTimeSpacing }}{# ISO_OPERA_zeroDopplerTimeSpacing #} - - - - - processingParameter - - - FaradayRotationFlag - - - Flag to indicate if Faraday Rotation correction was applied - - - boolean - - - - - {{ product_output.frequencyA.faradayRotationFlag }}{# ISO_OPERA_faradayRotationFlag #} - - - - - - - processingParameter - - - NoiseCorrectionFlag - - - Flag to indicate if Noise correction was applied - - - boolean - - - - - {{ product_output.frequencyA.noiseCorrectionFlag }}{# ISO_OPERA_noiseCorrectionFlag #} - - - - - - - processingParameter - - - PolarizationOrientationFlag - - - Flag to indicate if Polarization Orientation correction was applied - - - boolean - - - - - {{ product_output.frequencyA.polarizationOrientationFlag }}{# ISO_OPERA_polarizationOrientationFlag #} - - @@ -1261,27 +1201,6 @@ {{ product_output.processingInformation.algorithms.radiometricTerrainCorrection }}{# ISO_OPERA_radiometricTerrainCorrection #} - - - - - processingInformation - - - RTCVersion - - - RTC-S1 SAS version used for processing - - - string - - - - - {{ product_output.processingInformation.algorithms.RTCVersion }}{# ISO_OPERA_RTCVersion #} - - @@ -1342,7 +1261,7 @@ - {{ product_output.processingInformation.inputs.l1SlcGranules|string }}{# ISO_OPERA_l1SlcGranules #} + {{ product_output.processingInformation.inputs.l1SLCGranules|string }}{# ISO_OPERA_l1SLCGranules #} @@ -1373,10 +1292,10 @@ processingInformation - AuxcalFiles + AnnotationFiles - List of input calibration files used + List of input annotation files used string @@ -1384,7 +1303,7 @@ - {{ product_output.processingInformation.inputs.auxcalFiles|string }}{# ISO_OPERA_auxcalFiles #} + {{ product_output.processingInformation.inputs.annotationFiles|string }}{# ISO_OPERA_annotationFiles #} diff --git a/src/opera/util/metadata_utils.py b/src/opera/util/metadata_utils.py index 23a990d1..5ad6daf7 100644 --- a/src/opera/util/metadata_utils.py +++ b/src/opera/util/metadata_utils.py @@ -322,14 +322,9 @@ def create_test_rtc_metadata_product(file_path): projection_dset = frequencyA_grp.create_dataset("projection", data=b'1234') listOfPolarizations_dset = frequencyA_grp.create_dataset("listOfPolarizations", data=np.array([b'VV', b'VH'])) rangeBandwidth_dset = frequencyA_grp.create_dataset('rangeBandwidth', data=56500000.0, dtype='float64') - azimuthBandwidth_dset = frequencyA_grp.create_dataset('azimuthBandwidth', data=56500000.0, dtype='float64') slantRangeSpacing_dset = frequencyA_grp.create_dataset('slantRangeSpacing', data=2.32956, dtype='float64') zeroDopplerTimeSpacing_dset = frequencyA_grp.create_dataset('zeroDopplerTimeSpacing', data=0.002055, dtype='float64') - faradayRotationFlag_dset = frequencyA_grp.create_dataset('faradayRotationFlag', data=True, dtype='bool') - noiseCorrectionFlag_dset = frequencyA_grp.create_dataset('noiseCorrectionFlag', data=True, dtype='bool') - polarizationOrientationFlag_dset = frequencyA_grp.create_dataset('polarizationOrientationFlag', - data=True, dtype='bool') radiometricTerrainCorrectionFlag_dset = frequencyA_grp.create_dataset('radiometricTerrainCorrectionFlag', data=True, dtype='bool') @@ -341,12 +336,12 @@ def create_test_rtc_metadata_product(file_path): processingInformation_inputs_grp = outfile.create_group(f"{S1_SLC_HDF5_PREFIX}" f"/RTC/metadata/processingInformation/inputs") demSource_dset = processingInformation_inputs_grp.create_dataset("demSource", data=b'dem.tif') - auxcalFiles = np.array([b'calibration-s1b-iw1-slc-vv-20180504t104508-20180504t104533-010770-013aee-004.xml', - b'noise-s1b-iw1-slc-vv-20180504t104508-20180504t104533-010770-013aee-004.xml']) - auxcalFiles_dset = processingInformation_inputs_grp.create_dataset("auxcalFiles", data=auxcalFiles) + annotationFiles = np.array([b'calibration-s1b-iw1-slc-vv-20180504t104508-20180504t104533-010770-013aee-004.xml', + b'noise-s1b-iw1-slc-vv-20180504t104508-20180504t104533-010770-013aee-004.xml']) + annotationFiles_dset = processingInformation_inputs_grp.create_dataset("annotationFiles", data=annotationFiles) configFiles_dset = processingInformation_inputs_grp.create_dataset("configFiles", data=b'rtc_s1.yaml') l1SlcGranules = np.array([b'S1B_IW_SLC__1SDV_20180504T104507_20180504T104535_010770_013AEE_919F.zip']) - l1SlcGranules_dset = processingInformation_inputs_grp.create_dataset("l1SlcGranules", data=l1SlcGranules) + l1SlcGranules_dset = processingInformation_inputs_grp.create_dataset("l1SLCGranules", data=l1SlcGranules) orbitFiles = np.array([b'S1B_OPER_AUX_POEORB_OPOD_20180524T110543_V20180503T225942_20180505T005942.EOF']) orbitFiles_dset = processingInformation_inputs_grp.create_dataset("orbitFiles", data=orbitFiles) @@ -357,12 +352,13 @@ def create_test_rtc_metadata_product(file_path): geocoding_dset = processingInformation_algorithms_grp.create_dataset("geocoding", data=b'area_projection') radiometricTerrainCorrection_dset = processingInformation_algorithms_grp.create_dataset( "radiometricTerrainCorrection", data=b'area_projection') - rtcVersion_dset = processingInformation_algorithms_grp.create_dataset("RTCVersion", data=b'0.2') isceVersion_dset = processingInformation_algorithms_grp.create_dataset("ISCEVersion", data=b'0.8.0-dev') s1ReaderVersion_dset = processingInformation_algorithms_grp.create_dataset("S1ReaderVersion", data=b'1.2.3') identification_grp = outfile.create_group(f"{S1_SLC_HDF5_PREFIX}/identification") absoluteOrbitNumber_dset = identification_grp.create_dataset("absoluteOrbitNumber", data=10770, dtype='int64') + acquisitionMode_dset = identification_grp.create_dataset("acquisitionMode", data=np.string_('Interferometric Wide (IW)')) + beamID_dset = identification_grp.create_dataset("beamID", data=np.string_('iw1')) boundingPolygon_dset = identification_grp.create_dataset( "boundingPolygon", data=b'POLYGON ((399015 3859970, 398975 3860000, ..., 399015 3859970))') burstID_dset = identification_grp.create_dataset("burstID", data=b't069_147170_iw1') @@ -372,12 +368,9 @@ def create_test_rtc_metadata_product(file_path): data=np.array([False, True]), dtype='bool') lookDirection_dset = identification_grp.create_dataset("lookDirection", data=b'Right') listOfFrequencies_dset = identification_grp.create_dataset("listOfFrequencies", data=np.array([b'A'])) - missionId_dest = identification_grp.create_dataset("missionId", data=b'S1B') orbitPassDirection_dset = identification_grp.create_dataset("orbitPassDirection", data=b'Descending') - plannedDatatakeId_dset = identification_grp.create_dataset("plannedDatatakeId", - data=np.array([b'datatake1', b'datatake2'])) - plannedObservationId_dset = identification_grp.create_dataset("plannedObservationId", - data=np.array([b'obs1', b'obs2'])) + platform_dset = identification_grp.create_dataset("platform", data=b'Sentinel-1B') + processingDateTime_dset = identification_grp.create_dataset("processingDateTime", data=np.string_('2023-03-23T20:32:18.962836Z')) processingType_dset = identification_grp.create_dataset("processingType", data=b'UNDEFINED') productType_dset = identification_grp.create_dataset("productType", data=b'SLC') productVersion_dset = identification_grp.create_dataset("productVersion", data=b'1.0') From 12ed47a76ee54de1153478286f3448bdb0624f99 Mon Sep 17 00:00:00 2001 From: Scott Collins Date: Wed, 5 Apr 2023 09:21:41 -0700 Subject: [PATCH 7/7] Updated unit tests for RTC-S1 to account for gamma integration changes --- src/opera/test/data/test_rtc_s1_config.yaml | 7 +++++++ src/opera/test/pge/rtc_s1/test_rtc_s1_pge.py | 19 +++++++++++++++++-- src/opera/test/util/test_metadata_utils.py | 2 +- 3 files changed, 25 insertions(+), 3 deletions(-) diff --git a/src/opera/test/data/test_rtc_s1_config.yaml b/src/opera/test/data/test_rtc_s1_config.yaml index db74d84f..85c67cc3 100644 --- a/src/opera/test/data/test_rtc_s1_config.yaml +++ b/src/opera/test/data/test_rtc_s1_config.yaml @@ -28,8 +28,14 @@ RunConfig: - 'python3 -c "from opera.util.metadata_utils import create_test_rtc_metadata_product; create_test_rtc_metadata_product(\"rtc_s1_test/output_dir/t069_147170_iw1/rtc_product_v1.0.h5\")";' - 'touch rtc_s1_test/output_dir/t069_147170_iw1/rtc_product_v1.0_VH.tif;' - 'touch rtc_s1_test/output_dir/t069_147170_iw1/rtc_product_v1.0_VV.tif;' + - 'touch rtc_s1_test/output_dir/t069_147170_iw1/rtc_product_v1.0.png;' + - 'touch rtc_s1_test/output_dir/t069_147170_iw1/rtc_product_v1.0_nlooks.tif;' + - 'touch rtc_s1_test/output_dir/t069_147170_iw1/rtc_product_v1.0_layover_shadow_mask.tif;' - 'dd if=/dev/urandom of=rtc_s1_test/output_dir/t069_147170_iw1/rtc_product_v1.0_VH.tif bs=1M count=1;' - 'dd if=/dev/urandom of=rtc_s1_test/output_dir/t069_147170_iw1/rtc_product_v1.0_VV.tif bs=1M count=1;' + - 'dd if=/dev/urandom of=rtc_s1_test/output_dir/t069_147170_iw1/rtc_product_v1.0.png bs=1M count=1;' + - 'dd if=/dev/urandom of=rtc_s1_test/output_dir/t069_147170_iw1/rtc_product_v1.0_nlooks.tif bs=1M count=1;' + - 'dd if=/dev/urandom of=rtc_s1_test/output_dir/t069_147170_iw1/rtc_product_v1.0_layover_shadow_mask.tif bs=1M count=1;' - '/bin/echo RTC-S1 invoked with RunConfig' ErrorCodeBase: 300000 SchemaPath: pge/rtc_s1/schema/rtc_s1_sas_schema.yaml @@ -78,6 +84,7 @@ RunConfig: product_id: rtc_product save_bursts: True save_mosaics: False + save_browse: True output_imagery_format: COG processing: diff --git a/src/opera/test/pge/rtc_s1/test_rtc_s1_pge.py b/src/opera/test/pge/rtc_s1/test_rtc_s1_pge.py index d19bba69..867f3b18 100644 --- a/src/opera/test/pge/rtc_s1/test_rtc_s1_pge.py +++ b/src/opera/test/pge/rtc_s1/test_rtc_s1_pge.py @@ -26,6 +26,7 @@ from opera.util import PgeLogger from opera.util.metadata_utils import create_test_rtc_metadata_product from opera.util.metadata_utils import get_rtc_s1_product_metadata +from opera.util.metadata_utils import get_sensor_from_spacecraft_name class RtcS1PgeTestCase(unittest.TestCase): @@ -166,11 +167,12 @@ def test_filename_application(self): output_file = output_files[0] rtc_metadata = get_rtc_s1_product_metadata(output_file) + sensor = get_sensor_from_spacecraft_name(rtc_metadata['identification']['platform']) file_name_regex = rf"{pge.PROJECT}_{pge.LEVEL}_{pge.NAME}-{pge.SOURCE}_" \ rf"\w{{4}}-\w{{6}}-\w{{3}}_" \ rf"\d{{8}}T\d{{6}}Z_\d{{8}}T\d{{6}}Z_" \ - rf"{rtc_metadata['identification']['missionId']}_" \ + rf"{sensor}_" \ rf"{int(rtc_metadata['frequencyA']['xCoordinateSpacing'])}_" \ rf"v{pge.runconfig.product_version}.h5" @@ -184,12 +186,23 @@ def test_filename_application(self): output_files = glob.glob(join(pge.runconfig.output_product_path, f"{core_filename}*.tif")) - self.assertEqual(len(output_files), 2) + self.assertEqual(len(output_files), 4) output_files = list(map(os.path.basename, output_files)) self.assertIn(f"{core_filename}_VV.tif", output_files) self.assertIn(f"{core_filename}_VH.tif", output_files) + self.assertIn(f"{core_filename}_nlooks.tif", output_files) + self.assertIn(f"{core_filename}_layover_shadow_mask.tif", output_files) + + # Finally, ensure file name was applied to the png browse image + output_files = glob.glob(join(pge.runconfig.output_product_path, f"{core_filename}*.png")) + + self.assertEqual(len(output_files), 1) + + output_files = list(map(os.path.basename, output_files)) + + self.assertIn(f"{core_filename}_BROWSE.png", output_files) def test_iso_metadata_creation(self): """ @@ -428,6 +441,7 @@ def test_expected_extension(self): product_group = runconfig_dict['RunConfig']['Groups']['SAS']['runconfig']['groups']['product_group'] product_group['output_imagery_format'] = "NETCDF" + product_group['save_browse'] = False # Don't check for the .png browse image with open(test_runconfig_path, 'w', encoding='utf-8') as outfile: yaml.safe_dump(runconfig_dict, outfile, sort_keys=False) @@ -453,6 +467,7 @@ def test_expected_extension(self): product_group = runconfig_dict['RunConfig']['Groups']['SAS']['runconfig']['groups']['product_group'] product_group['output_imagery_format'] = "HDF5" + product_group['save_browse'] = False # Don't check for the .png browse image with open(test_runconfig_path, 'w', encoding='utf-8') as outfile: yaml.safe_dump(runconfig_dict, outfile, sort_keys=False) diff --git a/src/opera/test/util/test_metadata_utils.py b/src/opera/test/util/test_metadata_utils.py index f156ed3e..ea5c35a4 100644 --- a/src/opera/test/util/test_metadata_utils.py +++ b/src/opera/test/util/test_metadata_utils.py @@ -73,7 +73,7 @@ def test_get_rtc_s1_product_metadata(self): self.assertAlmostEqual(product_output['frequencyA']['centerFrequency'], 5405000454.33435) self.assertEqual(product_output['orbit']['orbitType'], "POE") self.assertEqual(product_output['processingInformation']['inputs']['demSource'], 'dem.tif') - for po, eo in zip(product_output['processingInformation']['inputs']['auxcalFiles'], + for po, eo in zip(product_output['processingInformation']['inputs']['annotationFiles'], ['calibration-s1b-iw1-slc-vv-20180504t104508-20180504t104533-010770-013aee-004.xml', 'noise-s1b-iw1-slc-vv-20180504t104508-20180504t104533-010770-013aee-004.xml']): self.assertEqual(po, eo)