From 89e612bdea0c01f5c382727c4bbe63d0e26c0be2 Mon Sep 17 00:00:00 2001 From: Scott Collins Date: Tue, 4 Apr 2023 09:06:51 -0700 Subject: [PATCH 1/8] Updated Artifactory path in build_cslc_s1.sh to point at gamma image --- .ci/scripts/build_cslc_s1.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/scripts/build_cslc_s1.sh b/.ci/scripts/build_cslc_s1.sh index 92a95c1d..9fc967cd 100755 --- a/.ci/scripts/build_cslc_s1.sh +++ b/.ci/scripts/build_cslc_s1.sh @@ -24,7 +24,7 @@ BUILD_DATE_TIME=$(date -u +'%Y-%m-%dT%H:%M:%SZ') # defaults, SAS image should be updated as necessary for new image releases from ADT [ -z "${WORKSPACE}" ] && WORKSPACE=$(realpath $(dirname $(realpath $0))/../..) [ -z "${TAG}" ] && TAG="${USER}-dev" -[ -z "${SAS_IMAGE}" ] && SAS_IMAGE="artifactory-fn.jpl.nasa.gov:16001/gov/nasa/jpl/opera/adt/opera/cslc_s1:beta_0.1" +[ -z "${SAS_IMAGE}" ] && SAS_IMAGE="artifactory-fn.jpl.nasa.gov:16001/gov/nasa/jpl/opera/adt/opera/cslc_s1:gamma_0.1" echo "WORKSPACE: $WORKSPACE" echo "IMAGE: $IMAGE" From 5613254ddc31f061bc2b8f66140c153c8119cd2b Mon Sep 17 00:00:00 2001 From: Scott Collins Date: Tue, 4 Apr 2023 09:07:35 -0700 Subject: [PATCH 2/8] Updated Yamale schema for CSLC-S1 PGE to gamma spec --- .../cslc_s1/schema/cslc_s1_sas_schema.yaml | 56 ++++++++++++++----- 1 file changed, 41 insertions(+), 15 deletions(-) diff --git a/src/opera/pge/cslc_s1/schema/cslc_s1_sas_schema.yaml b/src/opera/pge/cslc_s1/schema/cslc_s1_sas_schema.yaml index e4c8b6e1..f39196ee 100644 --- a/src/opera/pge/cslc_s1/schema/cslc_s1_sas_schema.yaml +++ b/src/opera/pge/cslc_s1/schema/cslc_s1_sas_schema.yaml @@ -21,9 +21,13 @@ runconfig: dynamic_ancillary_file_group: # Digital Elevation Model. dem_file: str(required=False) + # TEC file in IONEX format for ionosphere correction + tec_file: str(required=False) + # Troposphere weather model + weather_model_file: str(required=False) static_ancillary_file_group: - # burst database database file + # burst database sqlite file burst_database_file: str(required=False) product_path_group: @@ -34,6 +38,8 @@ runconfig: # Intermediate file name. SAS writes the output to this file. # PGE may rename the product according to file naming convention sas_output_file: str() + # Product version + product_version: num(required=False) primary_executable: product_type: enum('CSLC_S1') @@ -44,6 +50,8 @@ runconfig: # Worker options (e.g. enable/disable GPU processing, select GPU device ID) worker: include('worker_options', required=False) + # Quality assurance options + quality_assurance: include('quality_assurance_options', required=False) --- # Group of processing options processing_options: @@ -53,16 +61,12 @@ processing_options: geocoding: include('geocoding_options', required=False) # Options to run geo2rdr geo2rdr: include('geo2rdr_options', required=False) - # Range split-spectrum options - range_split_spectrum: include('range_split_spectrum_options', required=False) # Options to generate model-based slant range and azimuth correction LUTs correction_luts: include('lut_options', required=False) # Computation of topo layers rdr2geo: include('rdr2geo_options', required=False) geocoding_options: - # Format of output file - output_format: enum('ENVI', 'GTiff', 'COG', required=False) # Boolean flag to enable/disable flattening flatten: bool(required=False) # Number of lines to process in batch @@ -84,21 +88,21 @@ geo2rdr_options: # Lines per block to process in batch lines_per_block: int(min=1, required=False) -range_split_spectrum_options: - # Boolean flag to enable/disable split-spectrum - enabled: bool(required=False) - # Lines per block to process in batch - lines_per_block: int(min=1, required=False) - # Bandwidth of the low-band sub-image - low_band_bandwidth: num(min=0, required=False) - # Bandwidth of the high-band sub-image - high_band_bandwidth: num(min=0, required=False) - lut_options: + # Boolean flag to activate/deactivate model-based + # corrections while geocoding the burst + enabled: bool(required=False) # LUT spacing in range direction in meters range_spacing: num(min=0, required=False) # LUT spacing in azimuth direction in seconds azimuth_spacing: num(min=0, required=False) + # Troposphere delay using weather model + troposphere: include('troposphere_options', required=False) + +troposphere_options: + # Type of troposphere delay. Any of 'dry', 'wet' or 'wet_dry' for + # the sum of wet and dry delays + delay_type: enum('dry', 'wet', 'wet_dry', required=False) rdr2geo_options: # Enable/disable computation of topo layers @@ -136,3 +140,25 @@ worker_options: # Index of the GPU to use for processing, optional. Defaults to the # first available CUDA device. Ignored if *gpu_enabled* is False. gpu_id: int(min=0, required=False) + +quality_assurance_options: + # Options for generating browse images + browse_image: include('browse_image_options', required=False) + # Enable/disable computation of statistics of CSLC rasters and corrections + perform_qa: bool(required=False) + # Enable/disable writing of QA information to JSON + output_to_json: bool(required=False) + +browse_image_options: + # Enable/disable generation of CSLC browse image + enabled: bool(required=False) + # How to convert complex CSLC to real + complex_to_real: enum('amplitude', 'intensity', 'logamplitude', required=False) + # Lower percentile of non-nan pixels to clip + percent_low: num(min=0, required=False) + # Higher percentile of non-nan pixels to clip + percent_high: num(min=0, required=False) + # Gamma exponent applied to normalized image + gamma: num(min=0, required=False) + # Enable/disable histogram equalization + equalize: bool(required=False) From e636f70fbe8cf9806314b24fc4aee1041711ff06 Mon Sep 17 00:00:00 2001 From: Scott Collins Date: Tue, 4 Apr 2023 09:09:17 -0700 Subject: [PATCH 3/8] Updated example RunConfig for CSLC-S1 to v2.0.0-rc.1.0 spec --- ...lc_s1_sample_runconfig-v2.0.0-rc.1.0.yaml} | 24 +++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) rename examples/{cslc_s1_sample_runconfig-v2.0.0-er.5.0.yaml => cslc_s1_sample_runconfig-v2.0.0-rc.1.0.yaml} (88%) diff --git a/examples/cslc_s1_sample_runconfig-v2.0.0-er.5.0.yaml b/examples/cslc_s1_sample_runconfig-v2.0.0-rc.1.0.yaml similarity index 88% rename from examples/cslc_s1_sample_runconfig-v2.0.0-er.5.0.yaml rename to examples/cslc_s1_sample_runconfig-v2.0.0-rc.1.0.yaml index 25fb2f0a..097ad806 100644 --- a/examples/cslc_s1_sample_runconfig-v2.0.0-er.5.0.yaml +++ b/examples/cslc_s1_sample_runconfig-v2.0.0-rc.1.0.yaml @@ -1,4 +1,4 @@ -# Sample RunConfig for use with the CSLC-S1 PGE v2.0.0-er.5.0 +# Sample RunConfig for use with the CSLC-S1 PGE v2.0.0-rc.1.0 # This RunConfig should require minimal changes in order to be used with the # OPERA PCM. @@ -34,6 +34,9 @@ RunConfig: # or more .tif files dem_file: /home/compass_user/input_dir/dem_4326.tiff + # TEC file in IONEX format for ionosphere correction + tec_file: /home/compass_user/input_dir/jplg1210.22i + # Burst database, must be an .sqlite3 file burst_database_file: /home/compass_user/input_dir/opera_burst_database_deploy_2022_1212.sqlite3 @@ -137,6 +140,7 @@ RunConfig: # This section should match the DynamicAncillaryFilesGroup of the PGE RunConfig dynamic_ancillary_file_group: dem_file: /home/compass_user/input_dir/dem_4326.tiff + tec_file: /home/compass_user/input_dir/jplg1210.22i static_ancillary_file_group: # burst database sqlite file @@ -152,6 +156,9 @@ RunConfig: # Intermediate file name, this field is currently unused sas_output_file: /home/compass_user/output_dir + # Product version specific to output products + product_version: 1.0 + primary_executable: # This should match the value used for ProductIdentifier product_type: CSLC_S1 @@ -161,7 +168,6 @@ RunConfig: processing: polarization: co-pol geocoding: - output_format: GTiff flatten: True lines_per_block: 1000 x_posting: 5 @@ -170,3 +176,17 @@ RunConfig: lines_per_block: 1000 threshold: 1.0e-8 numiter: 25 + rdr2geo: + enabled: True + threshold: 1.0e-8 + numiter: 25 + lines_per_block: 1000 + extraiter: 10 + compute_latitude: True + compute_longitude: True + compute_height: True + compute_layover_shadow_mask: True + compute_incidence_angle: True + compute_local_incidence_angle: True + compute_azimuth_angle: True + geocode_metadata_layers: True From 8edd9447e31ffc38aac2b0cbe664205f5a34d271 Mon Sep 17 00:00:00 2001 From: Scott Collins Date: Tue, 4 Apr 2023 09:10:24 -0700 Subject: [PATCH 4/8] Added checks for the new ancillary file types to metadata_utils.validate_slc_s1_inputs() --- src/opera/util/input_validation.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/opera/util/input_validation.py b/src/opera/util/input_validation.py index f7428c52..c9252ef3 100644 --- a/src/opera/util/input_validation.py +++ b/src/opera/util/input_validation.py @@ -90,6 +90,12 @@ def validate_slc_s1_inputs(runconfig, logger, name): check_input(value[i], logger, name, valid_extensions=('.EOF',)) elif key == 'dem_file': check_input(value, logger, name, valid_extensions=('.tif', '.tiff', '.vrt')) + elif key == 'tec_file': + check_input(value, logger, name, valid_extensions=('.22i',)) + elif key == 'weather_model_file': + # TODO: this might be utilized as an ancillary with later deliveries, + # but no example available currently + continue elif key in ('burst_id', 'dem_description'): # these fields are included in the SAS input paths, but are not # actually file paths, so skip them From f7025b09e8dcb875d25be393d672b3e4d88b3b44 Mon Sep 17 00:00:00 2001 From: Scott Collins Date: Tue, 4 Apr 2023 09:11:45 -0700 Subject: [PATCH 5/8] Updated version numbers in cslc_s1_pge.py --- src/opera/pge/cslc_s1/cslc_s1_pge.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/opera/pge/cslc_s1/cslc_s1_pge.py b/src/opera/pge/cslc_s1/cslc_s1_pge.py index 32ccad3e..49e932de 100644 --- a/src/opera/pge/cslc_s1/cslc_s1_pge.py +++ b/src/opera/pge/cslc_s1/cslc_s1_pge.py @@ -706,7 +706,10 @@ class CslcS1Executor(CslcS1PreProcessorMixin, CslcS1PostProcessorMixin, PgeExecu LEVEL = "L2" """Processing Level for CSLC-S1 Products""" - SAS_VERSION = "0.1.3" # Beta release https://github.com/opera-adt/COMPASS/releases/tag/v0.1.3 + PGE_VERSION = "2.0.0-rc.1.0" + """Version of the PGE (overrides default from base_pge)""" + + SAS_VERSION = "0.1.4" # Gamma release https://github.com/opera-adt/COMPASS/releases/tag/v0.1.4 """Version of the SAS wrapped by this PGE, should be updated as needed""" def __init__(self, pge_name, runconfig_path, **kwargs): From 2f7d1cc2f07ba86100de1edcc06df05956ea9528 Mon Sep 17 00:00:00 2001 From: Scott Collins Date: Tue, 4 Apr 2023 09:13:07 -0700 Subject: [PATCH 6/8] Added file name convention support for the CSLC-S1 static layers and browse products --- src/opera/pge/cslc_s1/cslc_s1_pge.py | 90 ++++++++++++++++++++++++---- 1 file changed, 78 insertions(+), 12 deletions(-) diff --git a/src/opera/pge/cslc_s1/cslc_s1_pge.py b/src/opera/pge/cslc_s1/cslc_s1_pge.py index 49e932de..f3a14ae3 100644 --- a/src/opera/pge/cslc_s1/cslc_s1_pge.py +++ b/src/opera/pge/cslc_s1/cslc_s1_pge.py @@ -9,7 +9,7 @@ from Sentinel-1 A/B (S1) PGE. """ - +import glob import os.path import re from datetime import datetime @@ -84,7 +84,6 @@ def _validate_output(self): (size is greater than 0). """ out_dir_walk_dict = {} - expected_ext = [] output_dir = os.path.abspath(self.runconfig.output_product_path) scratch_dir = os.path.abspath(self.runconfig.scratch_path) @@ -102,10 +101,7 @@ def _validate_output(self): self.logger.critical(self.name, ErrorCode.OUTPUT_NOT_FOUND, error_msg) - output_format = self.runconfig.sas_config['runconfig']['groups']['processing']['geocoding']['output_format'] - - if output_format in ('GTiff', 'COG', 'ENVI'): - expected_ext = ['tiff', 'tif', 'h5'] + expected_ext = ['tiff', 'tif', 'h5', 'png'] # Verify: files in subdirectories, file length, and proper extension. for dir_name_key, file_names in out_dir_walk_dict.items(): @@ -203,14 +199,23 @@ def _cslc_filename(self, inter_filename): # The burst ID should be included within the file path for each # output directory, extract it and prepare it for use within the # final filename - burst_id = Path(os.path.dirname(inter_filename)).parts[-2] - burst_id = burst_id.upper().replace('_', '-') + burst_id_dir = Path(os.path.dirname(inter_filename)).parts[-2] + burst_id = burst_id_dir.upper().replace('_', '-') if burst_id in self._burst_metadata_cache: cslc_metadata = self._burst_metadata_cache[burst_id] else: # Collect the metadata from the HDF5 output product - cslc_metadata = self._collect_cslc_product_metadata(inter_filename) + cslc_h5_product_pattern = join(os.path.dirname(inter_filename), f"{burst_id_dir}*.h5") + + # Find the main .h5 product path based on location of the current file + # and burst ID + cslc_h5_product_paths = glob.glob(cslc_h5_product_pattern) + + if len(cslc_h5_product_paths) != 1: + raise RuntimeError(f'Got unexpected number of CSLC .h5 paths: {cslc_h5_product_paths}') + + cslc_metadata = self._collect_cslc_product_metadata(cslc_h5_product_paths[0]) self._burst_metadata_cache[burst_id] = cslc_metadata @@ -268,6 +273,65 @@ def _h5_filename(self, inter_filename): return f"{cslc_filename}.h5" + def _static_layers_filename(self, inter_filename): + """ + Returns the file name to use for the static layers product produced by + the CSLC-S1 PGE. + + The HDF5 filename for the CSLC-S1 PGE consists of: + + _static_layers.h5 + + Where is returned by CslcS1PostProcessorMixin._cslc_filename() + + Parameters + ---------- + inter_filename : str + The intermediate filename of the output HDF5 product containing all + the output static layers. This parameter is used to derive the + core CSLC file name component, to which "_static_layers" will be + appended to denote the file type. + + Returns + ------- + static_layers_filename : str + The file name to assign to static layers product(s) created by this PGE. + + """ + cslc_filename = self._cslc_filename(inter_filename) + + static_layers_filename = f"{cslc_filename}_static_layers.h5" + + return static_layers_filename + + def _browse_filename(self, inter_filename): + """ + Returns the file name to use for the PNG browse image produced by + the CSLC-S1 PGE. + + The browse image filename for the CSLC-S1 PGE consists of: + + _BROWSE.png + + Where is returned by CslcS1PostProcessorMixin._cslc_filename() + + Parameters + ---------- + inter_filename : str + The intermediate filename of the output browse image to generate a + filename for. This parameter may be used to inspect the file in order + to derive any necessary components of the returned filename. + + Returns + ------- + browse_image_filename : str + The file name to assign to browse image created by this PGE. + + """ + cslc_filename = self._cslc_filename(inter_filename) + + return f"{cslc_filename}_BROWSE.png" + def _geotiff_filename(self, inter_filename): """ Returns the file name to use for GeoTIFF's produced by the CSLC-S1 PGE. @@ -485,9 +549,9 @@ def _collect_cslc_product_metadata(self, metadata_product): output_product_metadata['grids']['width'] = len(output_product_metadata['grids']['x_coordinates']) output_product_metadata['grids']['length'] = len(output_product_metadata['grids']['y_coordinates']) - # Remove some of the larger arrays from the metadata so we don't use + # Remove some of the larger arrays from the metadata, so we don't use # too much memory when caching the metadata for each burst - for key in ['VV', 'VH', 'x_coordinates', 'y_coordinates']: + for key in ['VV', 'VH', 'HH', 'HV', 'x_coordinates', 'y_coordinates']: array = output_product_metadata['grids'].pop(key, None) if array is not None: @@ -716,7 +780,9 @@ def __init__(self, pge_name, runconfig_path, **kwargs): super().__init__(pge_name, runconfig_path, **kwargs) self.rename_by_pattern_map = { - '*.h5': self._h5_filename, + 't*.h5': self._h5_filename, + 'static_layers*.h5': self._static_layers_filename, + '*.png': self._browse_filename, '*.slc': self._geotiff_filename, '*.tif*': self._geotiff_filename, '*.json': self._json_metadata_filename From 343bacd092f6d7b030f1ea051055b344ea6ab593 Mon Sep 17 00:00:00 2001 From: Scott Collins Date: Tue, 4 Apr 2023 09:13:33 -0700 Subject: [PATCH 7/8] Updated ISO template and metadata collection for CSLC-S1 gamma delivery --- ...SO_metadata_L2_CSLC_S1_template.xml.jinja2 | 96 +++++++++---------- src/opera/util/metadata_utils.py | 16 +++- 2 files changed, 61 insertions(+), 51 deletions(-) diff --git a/src/opera/pge/cslc_s1/templates/OPERA_ISO_metadata_L2_CSLC_S1_template.xml.jinja2 b/src/opera/pge/cslc_s1/templates/OPERA_ISO_metadata_L2_CSLC_S1_template.xml.jinja2 index 607eb5bf..04d7f06f 100644 --- a/src/opera/pge/cslc_s1/templates/OPERA_ISO_metadata_L2_CSLC_S1_template.xml.jinja2 +++ b/src/opera/pge/cslc_s1/templates/OPERA_ISO_metadata_L2_CSLC_S1_template.xml.jinja2 @@ -604,10 +604,10 @@ instrumentInformation - DiagnosticModeFlag + IsGeocoded - Indicates if the radar mode is a diagnostic mode or not + Flag to indicate radar geometry or geocoded product Unitless @@ -618,7 +618,7 @@ - {{ product_output.identification.diagnostic_mode_flag }}{# ISO_OPERA_diagnosticModeFlag #} + {{ product_output.identification.is_geocoded }}{# ISO_OPERA_isGeocoded #} @@ -628,10 +628,10 @@ instrumentInformation - IsGeocoded + IsUrgentObservation - Flag to indicate radar geometry or geocoded product + Boolean indicating if datatakes are nominal or urgent Unitless @@ -642,7 +642,7 @@ - {{ product_output.identification.is_geocoded }}{# ISO_OPERA_isGeocoded #} + {{ product_output.identification.is_urgent_observation }}{# ISO_OPERA_isUrgentObservation #} @@ -652,21 +652,21 @@ instrumentInformation - IsUrgentObservation + LookDirection - Boolean indicating if datatakes are nominal or urgent + Look direction can be left or right Unitless - boolean + string - {{ product_output.identification.is_urgent_observation }}{# ISO_OPERA_isUrgentObservation #} + {{ product_output.identification.look_direction }}{# ISO_OPERA_lookDirection #} @@ -676,10 +676,10 @@ instrumentInformation - ListOfFrequencies + MissionID - List of frequency layers available in the product + Mission identifier Unitless @@ -690,7 +690,7 @@ - {{ product_output.identification.list_of_frequencies|string }}{# ISO_OPERA_listOfFrequencies #} + {{ product_output.identification.mission_id }}{# ISO_OPERA_missionID #} @@ -700,10 +700,10 @@ instrumentInformation - LookDirection + OrbitPassDirection - Look direction can be left or right + Orbit direction can be ascending or descending Unitless @@ -714,7 +714,7 @@ - {{ product_output.identification.look_direction }}{# ISO_OPERA_lookDirection #} + {{ product_output.identification.orbit_pass_direction }}{# ISO_OPERA_orbitPassDirection #} @@ -724,10 +724,10 @@ instrumentInformation - MissionID + ProductType - Mission identifier + Product type Unitless @@ -738,7 +738,7 @@ - {{ product_output.identification.mission_id }}{# ISO_OPERA_missionID #} + {{ product_output.identification.product_type }}{# ISO_OPERA_productType #} @@ -748,10 +748,10 @@ instrumentInformation - OrbitPassDirection + ProductVersion - Orbit direction can be ascending or descending + Product version Unitless @@ -762,7 +762,7 @@ - {{ product_output.identification.orbit_pass_direction }}{# ISO_OPERA_orbitPassDirection #} + {{ product_output.identification.product_version }}{# ISO_OPERA_productVersion #} @@ -772,21 +772,21 @@ instrumentInformation - ProductType + TrackNumber - Product type + Track number Unitless - string + int - {{ product_output.identification.product_type }}{# ISO_OPERA_productType #} + {{ product_output.identification.track_number }}{# ISO_OPERA_trackNumber #} @@ -796,21 +796,18 @@ instrumentInformation - ProductVersion + ZeroDopplerStartTime - Product version + Azimuth start time of product - - Unitless - string - {{ product_output.identification.product_version }}{# ISO_OPERA_productVersion #} + {{ product_output.identification.zero_doppler_start_time }}{# ISO_OPERA_zeroDopplerStartTime #} @@ -820,34 +817,34 @@ instrumentInformation - TrackNumber + ZeroDopplerEndTime - Track number + Azimuth stop time of product - - Unitless - - int + string - {{ product_output.identification.track_number }}{# ISO_OPERA_trackNumber #} + {{ product_output.identification.zero_doppler_end_time }}{# ISO_OPERA_zeroDopplerEndTime #} - instrumentInformation + processingInformation - ZeroDopplerStartTime + CalibrationBasename + + Unitless + - Azimuth start time of product + Calibration file name string @@ -855,20 +852,23 @@ - {{ product_output.identification.zero_doppler_start_time }}{# ISO_OPERA_zeroDopplerStartTime #} + {{ product_output.calibration_information.basename }}{# ISO_OPERA_calibrationBasename #} - instrumentInformation + processingInformation - ZeroDopplerEndTime + NoiseBasename + + Unitless + - Azimuth stop time of product + Noise calibration file name string @@ -876,7 +876,7 @@ - {{ product_output.identification.zero_doppler_end_time }}{# ISO_OPERA_zeroDopplerEndTime #} + {{ product_output.noise_information.basename }}{# ISO_OPERA_noiseBasename #} @@ -918,7 +918,7 @@ - {{ product_output.processing_information.algorithms.ISCE_version }}{# ISO_OPERA_isceVersion #} + {{ product_output.processing_information.algorithms.ISCE3_version }}{# ISO_OPERA_isceVersion #} diff --git a/src/opera/util/metadata_utils.py b/src/opera/util/metadata_utils.py index 3011b2e7..73fea03c 100644 --- a/src/opera/util/metadata_utils.py +++ b/src/opera/util/metadata_utils.py @@ -406,6 +406,10 @@ def get_cslc_s1_product_metadata(file_name): 'identification': get_hdf5_group_as_dict(file_name, f"{S1_SLC_HDF5_PREFIX}/identification"), 'grids': get_hdf5_group_as_dict(file_name, f"{S1_SLC_HDF5_PREFIX}/CSLC/grids"), 'corrections': get_hdf5_group_as_dict(file_name, f"{S1_SLC_HDF5_PREFIX}/CSLC/corrections"), + 'calibration_information': get_hdf5_group_as_dict(file_name, + f"{S1_SLC_HDF5_PREFIX}/CSLC/metadata/calibration_information"), + 'noise_information': get_hdf5_group_as_dict(file_name, + f"{S1_SLC_HDF5_PREFIX}/CSLC/metadata/noise_information"), 'processing_information': get_hdf5_group_as_dict(file_name, f"{S1_SLC_HDF5_PREFIX}/CSLC/metadata/processing_information"), 'orbit': get_hdf5_group_as_dict(file_name, f"{S1_SLC_HDF5_PREFIX}/CSLC/metadata/orbit") @@ -433,11 +437,9 @@ def create_test_cslc_metadata_product(file_path): bounding_polygon_dset = identification_grp.create_dataset( "bounding_polygon", data=np.string_("POLYGON ((-118.77 33.67, -118.72 33.68, ..., -118.77 33.67))")) burst_id_dset = identification_grp.create_dataset("burst_id", data=np.string_("t064_135518_iw1")) - diagnostic_mode_flag_dset = identification_grp.create_dataset('diagnostic_mode_flag', data=False, dtype='bool') is_geocoded_flag_dset = identification_grp.create_dataset("is_geocoded", data=True, dtype='bool') is_urgent_observation_dset = identification_grp.create_dataset("is_urgent_observation", data=False, dtype='bool') - list_of_frequencies_dset = identification_grp.create_dataset("list_of_frequencies", data=np.array([b'A'])) look_direction_dset = identification_grp.create_dataset("look_direction", data=np.string_("Right")) mission_id_dset = identification_grp.create_dataset("mission_id", data=np.string_("S1A")) orbit_pass_direction_dset = identification_grp.create_dataset("orbit_pass_direction", @@ -461,10 +463,18 @@ def create_test_cslc_metadata_product(file_path): zero_doppler_time_spacing_dset = corrections_grp.create_dataset("zero_doppler_time_spacing", data=0.027999999991152436, dtype='float64') + calibration_information_grp = outfile.create_group(f"{S1_SLC_HDF5_PREFIX}/CSLC/metadata/calibration_information") + cal_basename_dset = calibration_information_grp.create_dataset("basename", + data=np.string_('calibration-s1a-iw1-slc-vv-20220501t015035-20220501t015102-043011-0522a4-004.xml')) + + noise_information_grp = outfile.create_group(f"{S1_SLC_HDF5_PREFIX}/CSLC/metadata/noise_information") + noise_basename_dset = noise_information_grp.create_dataset("basename", + data=np.string_('noise-s1a-iw1-slc-vv-20220501t015035-20220501t015102-043011-0522a4-004.xml')) + processing_information_grp = outfile.create_group(f"{S1_SLC_HDF5_PREFIX}/CSLC/metadata/processing_information") algorithms_grp = outfile.create_group(f"{S1_SLC_HDF5_PREFIX}/CSLC/metadata/processing_information/algorithms") COMPASS_version_dset = algorithms_grp.create_dataset("COMPASS_version", data=np.string_("0.1.3")) - ISCE_version_dset = algorithms_grp.create_dataset("ISCE_version", data=np.string_("0.9.0")) + ISCE3_version_dset = algorithms_grp.create_dataset("ISCE3_version", data=np.string_("0.9.0")) dem_interpolation_dset = algorithms_grp.create_dataset("dem_interpolation", data=np.string_("biquintic")) geocoding_interpolator_dset = algorithms_grp.create_dataset("geocoding_interpolator", data=np.string_("sinc interpolation")) From c4c0a405ebaadc990bb05195483dfe7720f5a2ed Mon Sep 17 00:00:00 2001 From: Scott Collins Date: Tue, 4 Apr 2023 09:13:53 -0700 Subject: [PATCH 8/8] Updated unit tests for CSLC-S1 to account for gamma integration changes --- src/opera/test/data/test_cslc_s1_config.yaml | 30 ++++--------- .../test/pge/cslc_s1/test_cslc_s1_pge.py | 42 +++++++++++++++++-- 2 files changed, 48 insertions(+), 24 deletions(-) diff --git a/src/opera/test/data/test_cslc_s1_config.yaml b/src/opera/test/data/test_cslc_s1_config.yaml index e7b264a7..dc1efa01 100644 --- a/src/opera/test/data/test_cslc_s1_config.yaml +++ b/src/opera/test/data/test_cslc_s1_config.yaml @@ -15,6 +15,8 @@ RunConfig: DynamicAncillaryFilesGroup: AncillaryFileMap: dem_file: cslc_pge_test/input_dir/dem_4326.tiff + tec_file: cslc_pge_test/input_dir/jplg1210.22i + burst_database_file: cslc_pge_test/input_dir/opera_burst_database_deploy_2022_1212.sqlite3 ProductPathGroup: OutputProductPath: cslc_pge_test/output_dir @@ -26,7 +28,11 @@ RunConfig: ProgramPath: mkdir ProgramOptions: - '-p cslc_pge_test/output_dir/t064_135518_iw1/20220501/;' - - 'python3 -c "from opera.util.metadata_utils import create_test_cslc_metadata_product; create_test_cslc_metadata_product(\"cslc_pge_test/output_dir/t064_135518_iw1/20220501/t064_135518_iw1_20220501_VV.h5\")";' + - 'touch cslc_pge_test/output_dir/t064_135518_iw1/20220501/static_layers_t064_135518_iw1.h5;' + - 'touch cslc_pge_test/output_dir/t064_135518_iw1/20220501/t064_135518_iw1_20220501.png;' + - 'dd if=/dev/urandom of=cslc_pge_test/output_dir/t064_135518_iw1/20220501/t064_135518_iw1_20220501.png bs=1M count=1;' + - 'dd if=/dev/urandom of=cslc_pge_test/output_dir/t064_135518_iw1/20220501/static_layers_t064_135518_iw1.h5 bs=1M count=1;' + - 'python3 -c "from opera.util.metadata_utils import create_test_cslc_metadata_product; create_test_cslc_metadata_product(\"cslc_pge_test/output_dir/t064_135518_iw1/20220501/t064_135518_iw1_20220501.h5\")";' - '/bin/echo CSLC-S1 invoked with RunConfig' ErrorCodeBase: 200000 SchemaPath: pge/cslc_s1/schema/cslc_s1_sas_schema.yaml @@ -59,6 +65,7 @@ RunConfig: dynamic_ancillary_file_group: dem_file: cslc_pge_test/input_dir/dem_4326.tiff + tec_file: cslc_pge_test/input_dir/jplg1210.22i static_ancillary_file_group: burst_database_file: cslc_pge_test/input_dir/db.sqlite3 @@ -67,26 +74,7 @@ RunConfig: product_path: output_dir scratch_path: output_dir/scratch_dir sas_output_file: output_s1_cslc + product_version: 1.0 primary_executable: product_type: CSLC_S1 - - processing: - polarization: co-pol - geocoding: - output_format: GTiff - flatten: True - lines_per_block: 1000 - x_posting: 5 - y_posting: 10 - x_snap: - y_snap: - geo2rdr: - lines_per_block: 1000 - threshold: 1.0e-8 - numiter: 25 - - worker: - internet_access: False - gpu_enabled: False - gpu_id: 0 \ No newline at end of file diff --git a/src/opera/test/pge/cslc_s1/test_cslc_s1_pge.py b/src/opera/test/pge/cslc_s1/test_cslc_s1_pge.py index 24a4caf6..7e3d178e 100644 --- a/src/opera/test/pge/cslc_s1/test_cslc_s1_pge.py +++ b/src/opera/test/pge/cslc_s1/test_cslc_s1_pge.py @@ -75,6 +75,10 @@ def setUp(self) -> None: f"touch {join(input_dir, 'dem_4326.tiff')}" ) + os.system( + f"touch {join(input_dir, 'jplg1210.22i')}" + ) + # 'db.sqlite3' simulates the burst_id database file os.system( f"touch {join(input_dir, 'db.sqlite3')}" @@ -171,7 +175,7 @@ def test_filename_application(self): # Grab the metadata generated from the PGE run, as it is used to generate # the final filename for output products - metadata_files = glob.glob(join(pge.runconfig.output_product_path, "*.h5")) + metadata_files = glob.glob(join(pge.runconfig.output_product_path, "*Z.h5")) self.assertEqual(len(metadata_files), 1) @@ -183,7 +187,7 @@ def test_filename_application(self): # Compare the filename returned by the PGE for JSON metadata files # to a regex which should match each component of the final filename file_name = pge._h5_filename( - inter_filename='cslc_pge_test/output_dir/t064_135518_iw1/20220501/t064_135518_iw1_20220501_VV.h5' + inter_filename='cslc_pge_test/output_dir/t064_135518_iw1/20220501/t064_135518_iw1_20220501.h5' ) file_name_regex = rf"{pge.PROJECT}_{pge.LEVEL}_{pge.NAME}-" \ @@ -197,6 +201,36 @@ def test_filename_application(self): self.assertIsNotNone(result) self.assertEqual(result.group(), file_name) + file_name = pge._static_layers_filename( + inter_filename='cslc_pge_test/output_dir/t064_135518_iw1/20220501/static_layers_t064_135518_iw1.h5' + ) + + file_name_regex = rf"{pge.PROJECT}_{pge.LEVEL}_{pge.NAME}-" \ + rf"{burst_metadata['platform_id']}_IW_" \ + rf"{cslc_metadata['identification']['burst_id'].upper().replace('_', '-')}_" \ + rf"{burst_metadata['polarization']}_" \ + rf"\d{{8}}T\d{{6}}Z_v{pge.runconfig.product_version}_\d{{8}}T\d{{6}}Z_static_layers.h5" + + result = re.match(file_name_regex, file_name) + + self.assertIsNotNone(result) + self.assertEqual(result.group(), file_name) + + file_name = pge._browse_filename( + inter_filename='cslc_pge_test/output_dir/t064_135518_iw1/20220501/t064_135518_iw1_20220501.png' + ) + + file_name_regex = rf"{pge.PROJECT}_{pge.LEVEL}_{pge.NAME}-" \ + rf"{burst_metadata['platform_id']}_IW_" \ + rf"{cslc_metadata['identification']['burst_id'].upper().replace('_', '-')}_" \ + rf"{burst_metadata['polarization']}_" \ + rf"\d{{8}}T\d{{6}}Z_v{pge.runconfig.product_version}_\d{{8}}T\d{{6}}Z_BROWSE.png" + + result = re.match(file_name_regex, file_name) + + self.assertIsNotNone(result) + self.assertEqual(result.group(), file_name) + def test_iso_metadata_creation(self): """ Test that the ISO metadata template is fully filled out when realistic @@ -213,7 +247,7 @@ def test_iso_metadata_creation(self): # Create a sample metadata file within the output directory of the PGE output_dir = join(os.curdir, "cslc_pge_test/output_dir") - cslc_metadata_path = join(output_dir, 't064_135518_iw1_20220501_VV.h5') + cslc_metadata_path = join(output_dir, 't064_135518_iw1_20220501.h5') create_test_cslc_metadata_product(cslc_metadata_path) @@ -228,6 +262,8 @@ def test_iso_metadata_creation(self): # Rendered template should not have any missing placeholders self.assertNotIn('!Not found!', iso_metadata) + os.unlink(cslc_metadata_path) + # Test bad iso_template_path test_runconfig_path = join(self.data_dir, 'invalid_cslc_s1_runconfig.yaml')