Merge pull request #262 from nasa/258_cslc_s1_gamma_integration

CSLC-S1 Gamma Integration

.ci/scripts/build_cslc_s1.sh

@@ -24,7 +24,7 @@ BUILD_DATE_TIME=$(date -u +'%Y-%m-%dT%H:%M:%SZ')
 # defaults, SAS image should be updated as necessary for new image releases from ADT
 [ -z "${WORKSPACE}" ] && WORKSPACE=$(realpath $(dirname $(realpath $0))/../..)
 [ -z "${TAG}" ] && TAG="${USER}-dev"
-[ -z "${SAS_IMAGE}" ] && SAS_IMAGE="artifactory-fn.jpl.nasa.gov:16001/gov/nasa/jpl/opera/adt/opera/cslc_s1:beta_0.1"
+[ -z "${SAS_IMAGE}" ] && SAS_IMAGE="artifactory-fn.jpl.nasa.gov:16001/gov/nasa/jpl/opera/adt/opera/cslc_s1:gamma_0.1"
 
 echo "WORKSPACE: $WORKSPACE"
 echo "IMAGE: $IMAGE"

examples/cslc_s1_sample_runconfig-v2.0.0-er.5.0.yaml → examples/cslc_s1_sample_runconfig-v2.0.0-rc.1.0.yaml

@@ -1,4 +1,4 @@
-# Sample RunConfig for use with the CSLC-S1 PGE v2.0.0-er.5.0
+# Sample RunConfig for use with the CSLC-S1 PGE v2.0.0-rc.1.0
 # This RunConfig should require minimal changes in order to be used with the
 # OPERA PCM.
 
@@ -34,6 +34,9 @@ RunConfig:
                     # or more .tif files
                     dem_file: /home/compass_user/input_dir/dem_4326.tiff
 
+                    # TEC file in IONEX format for ionosphere correction
+                    tec_file: /home/compass_user/input_dir/jplg1210.22i
+
                     # Burst database, must be an .sqlite3 file
                     burst_database_file: /home/compass_user/input_dir/opera_burst_database_deploy_2022_1212.sqlite3
 
@@ -137,6 +140,7 @@ RunConfig:
                     # This section should match the DynamicAncillaryFilesGroup of the PGE RunConfig
                     dynamic_ancillary_file_group:
                         dem_file: /home/compass_user/input_dir/dem_4326.tiff
+                        tec_file: /home/compass_user/input_dir/jplg1210.22i
 
                     static_ancillary_file_group:
                         # burst database sqlite file
@@ -152,6 +156,9 @@ RunConfig:
                         # Intermediate file name, this field is currently unused
                         sas_output_file: /home/compass_user/output_dir
 
+                        # Product version specific to output products
+                        product_version: 1.0
+
                     primary_executable:
                         # This should match the value used for ProductIdentifier
                         product_type: CSLC_S1
@@ -161,7 +168,6 @@ RunConfig:
                     processing:
                         polarization: co-pol
                         geocoding:
-                            output_format: GTiff
                             flatten: True
                             lines_per_block: 1000
                             x_posting: 5
@@ -170,3 +176,17 @@ RunConfig:
                             lines_per_block: 1000
                             threshold: 1.0e-8
                             numiter: 25
+                        rdr2geo:
+                            enabled: True
+                            threshold: 1.0e-8
+                            numiter: 25
+                            lines_per_block: 1000
+                            extraiter: 10
+                            compute_latitude: True
+                            compute_longitude: True
+                            compute_height: True
+                            compute_layover_shadow_mask: True
+                            compute_incidence_angle: True
+                            compute_local_incidence_angle: True
+                            compute_azimuth_angle: True
+                            geocode_metadata_layers: True

src/opera/pge/cslc_s1/cslc_s1_pge.py

@@ -9,7 +9,7 @@
 from Sentinel-1 A/B (S1) PGE.
 
 """
-
+import glob
 import os.path
 import re
 from datetime import datetime
@@ -84,7 +84,6 @@ def _validate_output(self):
         (size is greater than 0).
         """
         out_dir_walk_dict = {}
-        expected_ext = []
 
         output_dir = os.path.abspath(self.runconfig.output_product_path)
         scratch_dir = os.path.abspath(self.runconfig.scratch_path)
@@ -102,10 +101,7 @@ def _validate_output(self):
 
             self.logger.critical(self.name, ErrorCode.OUTPUT_NOT_FOUND, error_msg)
 
-        output_format = self.runconfig.sas_config['runconfig']['groups']['processing']['geocoding']['output_format']
-
-        if output_format in ('GTiff', 'COG', 'ENVI'):
-            expected_ext = ['tiff', 'tif', 'h5']
+        expected_ext = ['tiff', 'tif', 'h5', 'png']
 
         # Verify: files in subdirectories, file length, and proper extension.
         for dir_name_key, file_names in out_dir_walk_dict.items():
@@ -203,14 +199,23 @@ def _cslc_filename(self, inter_filename):
         # The burst ID should be included within the file path for each
         # output directory, extract it and prepare it for use within the
         # final filename
-        burst_id = Path(os.path.dirname(inter_filename)).parts[-2]
-        burst_id = burst_id.upper().replace('_', '-')
+        burst_id_dir = Path(os.path.dirname(inter_filename)).parts[-2]
+        burst_id = burst_id_dir.upper().replace('_', '-')
 
         if burst_id in self._burst_metadata_cache:
             cslc_metadata = self._burst_metadata_cache[burst_id]
         else:
             # Collect the metadata from the HDF5 output product
-            cslc_metadata = self._collect_cslc_product_metadata(inter_filename)
+            cslc_h5_product_pattern = join(os.path.dirname(inter_filename), f"{burst_id_dir}*.h5")
+
+            # Find the main .h5 product path based on location of the current file
+            # and burst ID
+            cslc_h5_product_paths = glob.glob(cslc_h5_product_pattern)
+
+            if len(cslc_h5_product_paths) != 1:
+                raise RuntimeError(f'Got unexpected number of CSLC .h5 paths: {cslc_h5_product_paths}')
+
+            cslc_metadata = self._collect_cslc_product_metadata(cslc_h5_product_paths[0])
 
             self._burst_metadata_cache[burst_id] = cslc_metadata
 
@@ -268,6 +273,65 @@ def _h5_filename(self, inter_filename):
 
         return f"{cslc_filename}.h5"
 
+    def _static_layers_filename(self, inter_filename):
+        """
+        Returns the file name to use for the static layers product produced by
+        the CSLC-S1 PGE.
+
+        The HDF5 filename for the CSLC-S1 PGE consists of:
+
+            <CSLC filename>_static_layers.h5
+
+        Where <CSLC filename> is returned by CslcS1PostProcessorMixin._cslc_filename()
+
+        Parameters
+        ----------
+        inter_filename : str
+            The intermediate filename of the output HDF5 product containing all
+            the output static layers. This parameter is used to derive the
+            core CSLC file name component, to which "_static_layers" will be
+            appended to denote the file type.
+
+        Returns
+        -------
+        static_layers_filename : str
+            The file name to assign to static layers product(s) created by this PGE.
+
+        """
+        cslc_filename = self._cslc_filename(inter_filename)
+
+        static_layers_filename = f"{cslc_filename}_static_layers.h5"
+
+        return static_layers_filename
+
+    def _browse_filename(self, inter_filename):
+        """
+        Returns the file name to use for the PNG browse image produced by
+        the CSLC-S1 PGE.
+
+        The browse image filename for the CSLC-S1 PGE consists of:
+
+            <CSLC filename>_BROWSE.png
+
+        Where <CSLC filename> is returned by CslcS1PostProcessorMixin._cslc_filename()
+
+        Parameters
+        ----------
+        inter_filename : str
+            The intermediate filename of the output browse image to generate a
+            filename for. This parameter may be used to inspect the file in order
+            to derive any necessary components of the returned filename.
+
+        Returns
+        -------
+        browse_image_filename : str
+            The file name to assign to browse image created by this PGE.
+
+        """
+        cslc_filename = self._cslc_filename(inter_filename)
+
+        return f"{cslc_filename}_BROWSE.png"
+
     def _geotiff_filename(self, inter_filename):
         """
         Returns the file name to use for GeoTIFF's produced by the CSLC-S1 PGE.
@@ -485,9 +549,9 @@ def _collect_cslc_product_metadata(self, metadata_product):
         output_product_metadata['grids']['width'] = len(output_product_metadata['grids']['x_coordinates'])
         output_product_metadata['grids']['length'] = len(output_product_metadata['grids']['y_coordinates'])
 
-        # Remove some of the larger arrays from the metadata so we don't use
+        # Remove some of the larger arrays from the metadata, so we don't use
         # too much memory when caching the metadata for each burst
-        for key in ['VV', 'VH', 'x_coordinates', 'y_coordinates']:
+        for key in ['VV', 'VH', 'HH', 'HV', 'x_coordinates', 'y_coordinates']:
             array = output_product_metadata['grids'].pop(key, None)
 
             if array is not None:
@@ -706,14 +770,19 @@ class CslcS1Executor(CslcS1PreProcessorMixin, CslcS1PostProcessorMixin, PgeExecu
     LEVEL = "L2"
     """Processing Level for CSLC-S1 Products"""
 
-    SAS_VERSION = "0.1.3"  # Beta release https://github.com/opera-adt/COMPASS/releases/tag/v0.1.3
+    PGE_VERSION = "2.0.0-rc.1.0"
+    """Version of the PGE (overrides default from base_pge)"""
+
+    SAS_VERSION = "0.1.4"  # Gamma release https://github.com/opera-adt/COMPASS/releases/tag/v0.1.4
     """Version of the SAS wrapped by this PGE, should be updated as needed"""
 
     def __init__(self, pge_name, runconfig_path, **kwargs):
         super().__init__(pge_name, runconfig_path, **kwargs)
 
         self.rename_by_pattern_map = {
-            '*.h5': self._h5_filename,
+            't*.h5': self._h5_filename,
+            'static_layers*.h5': self._static_layers_filename,
+            '*.png': self._browse_filename,
             '*.slc': self._geotiff_filename,
             '*.tif*': self._geotiff_filename,
             '*.json': self._json_metadata_filename

src/opera/pge/cslc_s1/schema/cslc_s1_sas_schema.yaml

@@ -21,9 +21,13 @@ runconfig:
     dynamic_ancillary_file_group:
       # Digital Elevation Model.
       dem_file: str(required=False)
+      # TEC file in IONEX format for ionosphere correction
+      tec_file: str(required=False)
+      # Troposphere weather model
+      weather_model_file: str(required=False)
 
     static_ancillary_file_group:
-      # burst database database file
+      # burst database sqlite file
       burst_database_file: str(required=False)
 
     product_path_group:
@@ -34,6 +38,8 @@ runconfig:
       # Intermediate file name. SAS writes the output to this file.
       # PGE may rename the product according to file naming convention
       sas_output_file: str()
+      # Product version
+      product_version: num(required=False)
 
     primary_executable:
       product_type: enum('CSLC_S1')
@@ -44,6 +50,8 @@ runconfig:
     # Worker options (e.g. enable/disable GPU processing, select GPU device ID)
     worker: include('worker_options', required=False)
 
+    # Quality assurance options
+    quality_assurance: include('quality_assurance_options', required=False)
 ---
 # Group of processing options
 processing_options:
@@ -53,16 +61,12 @@ processing_options:
   geocoding: include('geocoding_options', required=False)
   # Options to run geo2rdr
   geo2rdr: include('geo2rdr_options', required=False)
-  # Range split-spectrum options
-  range_split_spectrum: include('range_split_spectrum_options', required=False)
   # Options to generate model-based slant range and azimuth correction LUTs
   correction_luts: include('lut_options', required=False)
   # Computation of topo layers
   rdr2geo: include('rdr2geo_options', required=False)
 
 geocoding_options:
-  # Format of output file
-  output_format: enum('ENVI', 'GTiff', 'COG', required=False)
   # Boolean flag to enable/disable flattening
   flatten: bool(required=False)
   # Number of lines to process in batch
@@ -84,21 +88,21 @@ geo2rdr_options:
   # Lines per block to process in batch
   lines_per_block: int(min=1, required=False)
 
-range_split_spectrum_options:
-  # Boolean flag to enable/disable split-spectrum
-  enabled: bool(required=False)
-  # Lines per block to process in batch
-  lines_per_block: int(min=1, required=False)
-  # Bandwidth of the low-band sub-image
-  low_band_bandwidth: num(min=0, required=False)
-  # Bandwidth of the high-band sub-image
-  high_band_bandwidth: num(min=0, required=False)
-
 lut_options:
+  # Boolean flag to activate/deactivate model-based
+  # corrections while geocoding the burst
+  enabled: bool(required=False)
   # LUT spacing in range direction in meters
   range_spacing: num(min=0, required=False)
   # LUT spacing in azimuth direction in seconds
   azimuth_spacing: num(min=0, required=False)
+  # Troposphere delay using weather model
+  troposphere: include('troposphere_options', required=False)
+
+troposphere_options:
+  # Type of troposphere delay. Any of 'dry', 'wet' or 'wet_dry' for
+  # the sum of wet and dry delays
+  delay_type: enum('dry', 'wet', 'wet_dry', required=False)
 
 rdr2geo_options:
   # Enable/disable computation of topo layers
@@ -136,3 +140,25 @@ worker_options:
   # Index of the GPU to use for processing, optional. Defaults to the
   # first available CUDA device. Ignored if *gpu_enabled* is False.
   gpu_id: int(min=0, required=False)
+
+quality_assurance_options:
+  # Options for generating browse images
+  browse_image: include('browse_image_options', required=False)
+  # Enable/disable computation of statistics of CSLC rasters and corrections
+  perform_qa: bool(required=False)
+  # Enable/disable writing of QA information to JSON
+  output_to_json: bool(required=False)
+
+browse_image_options:
+  # Enable/disable generation of CSLC browse image
+  enabled: bool(required=False)
+  # How to convert complex CSLC to real
+  complex_to_real: enum('amplitude', 'intensity', 'logamplitude', required=False)
+  # Lower percentile of non-nan pixels to clip
+  percent_low: num(min=0, required=False)
+  # Higher percentile of non-nan pixels to clip
+  percent_high: num(min=0, required=False)
+  # Gamma exponent applied to normalized image
+  gamma: num(min=0, required=False)
+  # Enable/disable histogram equalization
+  equalize: bool(required=False)