Merge pull request #540 from nasa/521_dswx_hsl_measured_params

Measured parameters for DSWx-HLS

.ci/scripts/dswx_hls/opera_pge_dswx_hls_delivery_4.1_final_runconfig.yaml

@@ -31,7 +31,7 @@ RunConfig:
                 ErrorCodeBase: 100000
                 SchemaPath: pge/dswx_hls/schema/dswx_hls_sas_schema.yaml
                 IsoTemplatePath: pge/dswx_hls/templates/OPERA_ISO_metadata_L3_DSWx_HLS_template.xml.jinja2
-
+                IsoMeasuredParameterDescriptions: pge/dswx_hls/templates/dswx_hls_measured_parameters.yaml
             QAExecutable:
                 Enabled: True
                 ProgramPath: /home/conda/opera/.ci/scripts/dswx_hls/compare_dswx_hls_products.sh

examples/dswx_hls_sample_runconfig-v1.0.2.yaml

@@ -94,6 +94,15 @@ RunConfig:
                 # Consult the Docker image build scripts for more info
                 IsoTemplatePath: /home/conda/opera/pge/dswx_hls/templates/OPERA_ISO_metadata_L3_DSWx_HLS_template.xml.jinja2
 
+                # Path to a YAML file mapping Measured Parameter metadata names to descriptions used to supplement the ISO xml
+                # metadata file
+                # Path should correspond to the file system within the Docker
+                # container, and typically should reference a template file bundled
+                # with the opera_pge installation directory within the container
+                # Consult the Docker image build scripts for more info
+                IsoMeasuredParameterDescriptions: /home/conda/opera/pge/dswx_hls/templates/dswx_hls_measured_parameters.yaml
+
+
             QAExecutable:
                 # Set to True to enable execution of an additional "Quality Assurance"
                 # application after SAS execution has completed

src/opera/pge/base/base_pge.py

@@ -19,25 +19,31 @@
 from os.path import abspath, basename, exists, join, splitext
 
 import numpy as np
-import yamale
-import yaml
+
 from pkg_resources import resource_filename
+
+import yamale
 from yamale import YamaleError
 
+import yaml
+
 import opera
 from opera.util.error_codes import ErrorCode
 from opera.util.logger import PgeLogger
 from opera.util.logger import default_log_file_name
 from opera.util.metfile import MetFile
 from opera.util.render_jinja2 import (python_type_to_xml_type,
                                       guess_attribute_display_name,
-                                      NumpyEncoder)
+                                      NumpyEncoder,
+                                      UNDEFINED_ERROR,
+                                      UNDEFINED_WARNING)
 from opera.util.run_utils import create_qa_command_line
 from opera.util.run_utils import create_sas_command_line
 from opera.util.run_utils import get_checksum
 from opera.util.run_utils import time_and_execute
 from opera.util.time import get_catalog_metadata_datetime_str
 from opera.util.time import get_time_for_filename
+
 from .runconfig import RunConfig
 
 
@@ -639,8 +645,8 @@ def _stage_output_files(self):
 
     def augment_measured_parameters(self, measured_parameters):
         """
-        Augment the measured parameters dict into a dict of dicts containing the needed fields for the MeasuredParameters
-        section of the ISO XML file.
+        Augment the measured parameters dict into a dict of dicts containing the needed
+        fields for the MeasuredParameters section of the ISO XML file.
 
         Parameters
         ----------
@@ -660,10 +666,10 @@ def augment_measured_parameters(self, measured_parameters):
             with open(descriptions_file) as f:
                 descriptions = yaml.safe_load(f)
 
-            missing_description_value = '!Not Found!'
+            missing_description_value = UNDEFINED_ERROR
         else:
             descriptions = dict()
-            missing_description_value = 'Not Provided'
+            missing_description_value = UNDEFINED_WARNING
 
         for name, value in measured_parameters.items():
             if isinstance(value, np.generic):
@@ -682,7 +688,7 @@ def augment_measured_parameters(self, measured_parameters):
 
             attr_description = descriptions[name].get('description', missing_description_value)
             data_type = descriptions[name].get('attribute_data_type', guessed_data_type)
-            attr_type = descriptions[name].get('attribute_type', "!Not Found!")
+            attr_type = descriptions[name].get('attribute_type', UNDEFINED_ERROR)
             attr_name = descriptions[name].get('display_name', guessed_attr_name)
             escape_html = descriptions[name].get('escape_html', False)
 

src/opera/pge/base/runconfig.py

@@ -330,10 +330,10 @@ def iso_template_path(self) -> str:
             return None
 
         return (
-                iso_template_path
-                if isabs(iso_template_path)
-                else resource_filename('opera', iso_template_path)
-            )
+            iso_template_path
+            if isabs(iso_template_path)
+            else resource_filename('opera', iso_template_path)
+        )
 
     @property
     def iso_measured_parameter_descriptions(self) -> str:

src/opera/pge/dswx_hls/dswx_hls_pge.py

@@ -417,6 +417,8 @@ def _collect_dswx_hls_product_metadata(self):
         output_products = self.runconfig.get_output_product_filenames()
         representative_product = None
 
+        output_product_metadata = dict()
+
         for output_product in output_products:
             if basename(output_product) in self.renamed_files.values() and basename(output_product).endswith("tif"):
                 representative_product = output_product
@@ -428,7 +430,8 @@ def _collect_dswx_hls_product_metadata(self):
 
         # Extract all metadata assigned by the SAS at product creation time
         try:
-            output_product_metadata = get_geotiff_metadata(representative_product)
+            measured_parameters = get_geotiff_metadata(representative_product)
+            output_product_metadata['MeasuredParameters'] = self.augment_measured_parameters(measured_parameters)
         except Exception as err:
             msg = f'Failed to extract metadata from {representative_product}, reason: {err}'
             self.logger.critical(self.name, ErrorCode.ISO_METADATA_COULD_NOT_EXTRACT_METADATA, msg)
@@ -454,26 +457,31 @@ def _collect_dswx_hls_product_metadata(self):
         output_product_metadata['geospatial_lat_min'] = lat_min
         output_product_metadata['geospatial_lat_max'] = lat_max
 
+        # Note: In DSWx-HLS SENSING_TIME represents the Worldwide Reference System (WRS) center sensing time.
+        # The ISO metadata calls for a beginning time and an ending time, so the center time will be written to both
+        # the beginning time slot and the ending time slot.
         # Split the sensing time into the beginning/end portions
-        sensing_time = output_product_metadata.pop('SENSING_TIME')
+        sensing_time_entry = output_product_metadata['MeasuredParameters'].pop('SENSING_TIME')
+        sensing_time_value = sensing_time_entry.get('value')
+        first_sensing_time = sensing_time_value
 
         # Sensing time can contain multiple times delimited by semicolon,
         # just take the first one
-        if ';' in sensing_time:
-            sensing_time = sensing_time.split(';', maxsplit=1)[0]
+        if ';' in sensing_time_value:
+            first_sensing_time = sensing_time_value.split(';', maxsplit=1)[0]
 
         # Certain datasets have been observed with multiple sensing times
         # concatenated by a plus sign, for this case just take the first of the
         # times
-        if '+' in sensing_time:
-            sensing_time = sensing_time.split('+', maxsplit=1)[0]
+        if '+' in first_sensing_time:
+            first_sensing_time = first_sensing_time.split('+', maxsplit=1)[0]
 
-        # Set beginning and end time to single time parsed, since ISO metadata
+        # Set beginning and ending time to the center time, since ISO metadata
         # requires both
-        sensing_time_begin = sensing_time_end = sensing_time
+        sensing_time_begin = sensing_time_end = first_sensing_time.strip()
 
-        output_product_metadata['sensingTimeBegin'] = sensing_time_begin.strip()
-        output_product_metadata['sensingTimeEnd'] = sensing_time_end.strip()
+        output_product_metadata['sensingTimeBegin'] = sensing_time_begin
+        output_product_metadata['sensingTimeEnd'] = sensing_time_end
 
         # Add some fields on the dimensions of the data. These values should
         # be the same for all DSWx-HLS products, and were derived from the
@@ -533,7 +541,7 @@ def _create_iso_metadata(self):
             the sourced metadata dictionaries.
 
         """
-        # Use the base PGE implemenation to validate existence of the template
+        # Use the base PGE implementation to validate existence of the template
         super()._create_iso_metadata()
 
         runconfig_dict = self.runconfig.asdict()