Merge pull request #454 from nasa/447_dswx_ni_iso_template

DSWx-NI ISO XML Generation

.ci/scripts/dswx_ni/opera_pge_dswx_ni_delivery_0.1_interface_runconfig.yaml

@@ -102,8 +102,7 @@ RunConfig:
         # container, and typically should reference a template file bundled
         # with the opera_pge installation directory within the container
         # Consult the Docker image build scripts for more info
-        # TODO: add once available
-        IsoTemplatePath:
+        IsoTemplatePath: /home/dswx_user/opera/pge/dswx_ni/templates/OPERA_ISO_metadata_L3_DSWx_NI_template.xml.jinja2
 
       QAExecutable:
         # Set to True to enable execution of an additional "Quality Assurance"

examples/dswx_ni_sample_runconfig-v4.0.0-er.1.0.yaml

@@ -102,8 +102,7 @@ RunConfig:
         # container, and typically should reference a template file bundled
         # with the opera_pge installation directory within the container
         # Consult the Docker image build scripts for more info
-        # TODO: add once available
-        IsoTemplatePath:
+        IsoTemplatePath: /home/dswx_user/opera/pge/dswx_ni/templates/OPERA_ISO_metadata_L3_DSWx_NI_template.xml.jinja2
 
       QAExecutable:
         # Set to True to enable execution of an additional "Quality Assurance"

src/opera/pge/dswx_ni/dswx_ni_pge.py

@@ -15,6 +15,8 @@
 from opera.pge.base.base_pge import PgeExecutor
 from opera.pge.dswx_s1.dswx_s1_pge import DSWxS1PostProcessorMixin, DSWxS1PreProcessorMixin
 from opera.util.error_codes import ErrorCode
+from opera.util.geo_utils import get_geographic_boundaries_from_mgrs_tile
+from opera.util.mock_utils import MockGdal
 from opera.util.time import get_time_for_filename
 
 
@@ -60,31 +62,48 @@ class DSWxNIPostProcessorMixin(DSWxS1PostProcessorMixin):
 
     _post_mixin_name = "DSWxNIPostProcessorMixin"
     _cached_core_filename = None
+    _tile_metadata_cache = {}
+    _tile_filename_cache = {}
 
     def _validate_output_product_filenames(self):
         """
         This method validates output product file names assigned by the SAS
-        via a regular expression.  The output product file names should follow
+        via a regular expression. The output product file names should follow
         this conventions:
 
-         <PROJECT>_<LEVEL>_<PRODUCT TYPE>_<SOURCE>_<TILE ID>_<ACQUISITION TIMESTAMP>_
-         <CREATION TIMESTAMP>_<SENSOR>_<SPACING>_<PRODUCT VERSION>_<BAND INDEX>_
-         <BAND NAME>.<FILE EXTENSION>
+            <PROJECT>_<LEVEL>_<PRODUCT TYPE>_<SOURCE>_<TILE ID>_<ACQUISITION TIMESTAMP>_
+            <CREATION TIMESTAMP>_<SENSOR>_<SPACING>_<PRODUCT VERSION>_<BAND INDEX>_
+            <BAND NAME>.<FILE EXTENSION>
 
-         If the pattern does not match a critical error will cause a RuntimeError.
+        If the pattern does not match a critical error will cause a RuntimeError.
+        If the pattern does match, this function will also read the product metadata
+        from the GeoTIFF product, and cache it for later use.
 
         """
         pattern = re.compile(
-            r'(?P<project>OPERA)_(?P<level>L3)_(?P<product_type>DSWx)-(?P<source>NI)_(?P<tile_id>T[^\W_]{5})_'
+            r'(?P<file_id>(?P<project>OPERA)_(?P<level>L3)_(?P<product_type>DSWx)-(?P<source>NI)_(?P<tile_id>T[^\W_]{5})_'
             r'(?P<acquisition_ts>\d{8}T\d{6}Z)_(?P<creation_ts>\d{8}T\d{6}Z)_(?P<sensor>LSAR)_(?P<spacing>30)_'
-            r'(?P<product_version>v\d+[.]\d+)(_(?P<band_index>B\d{2})_'
-            r'(?P<band_name>WTR|BWTR|CONF|DIAG)|_BROWSE)?[.](?P<ext>tif|tiff|png)$')
+            r'(?P<product_version>v\d+[.]\d+))(_(?P<band_index>B\d{2})_'
+            r'(?P<band_name>WTR|BWTR|CONF|DIAG)|_BROWSE)?[.](?P<ext>tif|tiff|png)$'
+        )
 
         for output_file in self.runconfig.get_output_product_filenames():
-            if not pattern.match(basename(output_file)):
+            match_result = pattern.match(basename(output_file))
+            if not match_result:
                 error_msg = (f"Output file {output_file} does not match the output "
                              f"naming convention.")
                 self.logger.critical(self.name, ErrorCode.INVALID_OUTPUT, error_msg)
+            else:
+                tile_id = match_result.groupdict()['tile_id']
+                file_id = match_result.groupdict()['file_id']
+
+                if tile_id not in self._tile_metadata_cache:
+                    # Cache the metadata for this product for use when generating the ISO XML
+                    self._tile_metadata_cache[tile_id] = self._collect_dswx_ni_product_metadata(output_file)
+
+                if tile_id not in self._tile_filename_cache:
+                    # Cache the core filename for use when naming the ISO XML file
+                    self._tile_filename_cache[tile_id] =  file_id
 
     def _ancillary_filename(self):
         """
@@ -136,23 +155,86 @@ def _ancillary_filename(self):
 
         return ancillary_filename
 
-    def _log_filename(self):
+    def _collect_dswx_ni_product_metadata(self, geotiff_product):
         """
-        Returns the file name to use for the PGE/SAS log file produced by the DSWx-NI PGE.
+        Gathers the available metadata from an output DSWx-NI product for
+        use in filling out the ISO metadata template for the DSWx-NI PGE.
 
-        The log file name for the DSWx-NI PGE consists of:
+        Parameters
+        ----------
+        geotiff_product : str
+            Path the GeoTIFF product to collect metadata from.
 
-            <Ancillary filename>.log
+        Returns
+        -------
+        output_product_metadata : dict
+            Dictionary containing DSWx-S1 output product metadata, formatted
+            for use with the ISO metadata Jinja2 template.
+
+        """
+        # Extract all metadata assigned by the SAS at product creation time
+        # TODO: current DSWx-NI GeoTIFF products do not contain any metadata
+        #       so just use the mock set for the time being
+        output_product_metadata = MockGdal.MockDSWxNIGdalDataset().GetMetadata()
+
+        # Get the Military Grid Reference System (MGRS) tile code and zone
+        # identifier from the intermediate file name
+        mgrs_tile_id = basename(geotiff_product).split('_')[3]
+
+        output_product_metadata['tileCode'] = mgrs_tile_id
+        output_product_metadata['zoneIdentifier'] = mgrs_tile_id[:2]
+
+        # Translate the MGRS tile ID to a lat/lon bounding box
+        (lat_min,
+         lat_max,
+         lon_min,
+         lon_max) = get_geographic_boundaries_from_mgrs_tile(mgrs_tile_id)
+
+        output_product_metadata['geospatial_lon_min'] = lon_min
+        output_product_metadata['geospatial_lon_max'] = lon_max
+        output_product_metadata['geospatial_lat_min'] = lat_min
+        output_product_metadata['geospatial_lat_max'] = lat_max
+
+        # Add some fields on the dimensions of the data. These values should
+        # be the same for all DSWx-NI products, and were derived from the
+        # ADT product spec
+        output_product_metadata['xCoordinates'] = {
+            'size': 3660,  # pixels
+            'spacing': 30  # meters/pixel
+        }
+        output_product_metadata['yCoordinates'] = {
+            'size': 3660,  # pixels
+            'spacing': 30  # meters/pixel
+        }
+
+        return output_product_metadata
+
+    def _create_custom_metadata(self):
+        """
+        Creates the "custom data" dictionary used with the ISO metadata rendering.
 
-        Where <Ancillary filename> is returned by DSWxNIPostProcessorMixin._ancillary_filename()
+        Custom data contains all metadata information needed for the ISO template
+        that is not found within any of the other metadata sources (such as the
+        RunConfig, output product(s), or catalog metadata).
 
         Returns
         -------
-        log_filename : str
-            The file name to assign to the PGE/SAS log created by this PGE.
+        custom_metadata : dict
+            Dictionary containing the custom metadata as expected by the ISO
+            metadata Jinja2 template.
 
         """
-        return self._ancillary_filename() + ".log"
+        custom_metadata = {
+            'ISO_OPERA_FilePackageName': self._core_filename(),
+            'ISO_OPERA_ProducerGranuleId': self._core_filename(),
+            'MetadataProviderAction': "creation",
+            'GranuleFilename': self._core_filename(),
+            'ISO_OPERA_ProjectKeywords': ['OPERA', 'JPL', 'DSWx', 'Dynamic', 'Surface', 'Water', 'Extent'],
+            'ISO_OPERA_PlatformKeywords': ['NI'],
+            'ISO_OPERA_InstrumentKeywords': ['NISAR']
+        }
+
+        return custom_metadata
 
     def _stage_output_files(self):
         """
@@ -177,6 +259,32 @@ def _stage_output_files(self):
             msg = f"Failed to create valid catalog metadata, reason(s):\n {catalog_metadata.get_error_msg()}"
             self.logger.critical(self.name, ErrorCode.INVALID_CATALOG_METADATA, msg)
 
+        cat_meta_filename = self._catalog_metadata_filename()
+        cat_meta_filepath = join(self.runconfig.output_product_path, cat_meta_filename)
+
+        self.logger.info(self.name, ErrorCode.CREATING_CATALOG_METADATA,
+                         f"Writing Catalog Metadata to {cat_meta_filepath}")
+
+        try:
+            catalog_metadata.write(cat_meta_filepath)
+        except OSError as err:
+            msg = f"Failed to write catalog metadata file {cat_meta_filepath}, reason: {str(err)}"
+            self.logger.critical(self.name, ErrorCode.CATALOG_METADATA_CREATION_FAILED, msg)
+
+        # Generate the ISO metadata for use with product submission to DAAC(s)
+        # For DSWX-S1, each tile-set is assigned an ISO xml file
+        for tile_id, tile_metadata in self._tile_metadata_cache.items():
+            iso_metadata = self._create_iso_metadata(tile_metadata)
+
+            iso_meta_filename = self._iso_metadata_filename(tile_id)
+            iso_meta_filepath = join(self.runconfig.output_product_path, iso_meta_filename)
+
+            if iso_metadata:
+                self.logger.info(self.name, ErrorCode.RENDERING_ISO_METADATA,
+                                 f"Writing ISO Metadata to {iso_meta_filepath}")
+                with open(iso_meta_filepath, 'w', encoding='utf-8') as outfile:
+                    outfile.write(iso_metadata)
+
         # Write the QA application log to disk with the appropriate filename,
         # if necessary
         if self.runconfig.qa_enabled:
@@ -221,7 +329,7 @@ def run_postprocessor(self, **kwargs):
         self._run_sas_qa_executable()
 
         self._validate_output()
-        # TODO - stage_output_files()  is only partially implemented
+        self._validate_output_product_filenames()
         self._stage_output_files()