Merge pull request #422 from nasa/release/2.1.1

Release 2.1.1
nasa · Mar 25, 2024 · c681bf0 · c681bf0
2 parents f105dc5 + 9ecb3bf
commit c681bf0
Show file tree

Hide file tree

Showing 16 changed files with 37 additions and 187 deletions.
diff --git a/.ci/scripts/cslc_s1/build_cslc_s1.sh b/.ci/scripts/cslc_s1/build_cslc_s1.sh
@@ -24,7 +24,7 @@ BUILD_DATE_TIME=$(date -u +'%Y-%m-%dT%H:%M:%SZ')
 # defaults, SAS image should be updated as necessary for new image releases from ADT
 [ -z "${WORKSPACE}" ] && WORKSPACE=$(realpath $(dirname $(realpath $0))/../../..)
 [ -z "${TAG}" ] && TAG="${USER}-dev"
-[ -z "${SAS_IMAGE}" ] && SAS_IMAGE="artifactory-fn.jpl.nasa.gov:16001/gov/nasa/jpl/opera/adt/opera/cslc_s1:final_0.5.4"
+[ -z "${SAS_IMAGE}" ] && SAS_IMAGE="artifactory-fn.jpl.nasa.gov:16001/gov/nasa/jpl/opera/adt/opera/cslc_s1:final_0.5.5"
 
 echo "WORKSPACE: $WORKSPACE"
 echo "IMAGE: $IMAGE"

diff --git a/...cslc_s1_delivery_6.1_final_runconfig.yaml → ...cslc_s1_delivery_6.4_final_runconfig.yaml b/...cslc_s1_delivery_6.1_final_runconfig.yaml → ...cslc_s1_delivery_6.4_final_runconfig.yaml
diff --git a/..._static_delivery_6.1_final_runconfig.yaml → ..._static_delivery_6.4_final_runconfig.yaml b/..._static_delivery_6.1_final_runconfig.yaml → ..._static_delivery_6.4_final_runconfig.yaml
diff --git a/.ci/scripts/cslc_s1/test_int_cslc_s1.sh b/.ci/scripts/cslc_s1/test_int_cslc_s1.sh
@@ -26,9 +26,9 @@ SAMPLE_TIME=15
 # Test data should be uploaded to  s3://operasds-dev-pge/${PGE_NAME}/
 [ -z "${WORKSPACE}" ] && WORKSPACE=$(realpath "$(dirname "$(realpath "$0")")"/../../..)
 [ -z "${PGE_TAG}" ] && PGE_TAG="${USER}-dev"
-[ -z "${INPUT_DATA}" ] && INPUT_DATA="delivery_cslc_s1_final_0.5.1_expected_input_data.zip"
-[ -z "${EXPECTED_DATA}" ] && EXPECTED_DATA="delivery_cslc_s1_final_0.5.1_expected_output.zip"
-[ -z "${RUNCONFIG}" ] && RUNCONFIG="opera_pge_cslc_s1_delivery_6.1_final_runconfig.yaml"
+[ -z "${INPUT_DATA}" ] && INPUT_DATA="cslc_s1_final_0.5.5_expected_input_data.zip"
+[ -z "${EXPECTED_DATA}" ] && EXPECTED_DATA="cslc_s1_final_0.5.5_expected_output.zip"
+[ -z "${RUNCONFIG}" ] && RUNCONFIG="opera_pge_cslc_s1_delivery_6.4_final_runconfig.yaml"
 [ -z "${TMP_ROOT}" ] && TMP_ROOT="$DEFAULT_TMP_ROOT"
 
 # Create the test output directory in the workspace
@@ -53,7 +53,7 @@ input_dir="${TMP_DIR}/${INPUT_DATA%.*}/input_data"
 runconfig_dir="${TMP_DIR}/runconfig"
 
 # Copy the RunConfig for the static layers workflow
-static_runconfig="opera_pge_cslc_s1_static_delivery_6.1_final_runconfig.yaml"
+static_runconfig="opera_pge_cslc_s1_static_delivery_6.4_final_runconfig.yaml"
 local_static_runconfig="${SCRIPT_DIR}/${static_runconfig}"
 echo "Copying runconfig file $local_static_runconfig to $runconfig_dir/"
 cp ${local_static_runconfig} ${runconfig_dir}

diff --git a/.ci/scripts/rtc_s1/build_rtc_s1.sh b/.ci/scripts/rtc_s1/build_rtc_s1.sh
@@ -24,7 +24,7 @@ BUILD_DATE_TIME=$(date -u +'%Y-%m-%dT%H:%M:%SZ')
 # defaults, SAS image should be updated as necessary for new image releases from ADT
 [ -z "${WORKSPACE}" ] && WORKSPACE=$(realpath $(dirname $(realpath $0))/../../..)
 [ -z "${TAG}" ] && TAG="${USER}-dev"
-[ -z "${SAS_IMAGE}" ] && SAS_IMAGE="artifactory-fn.jpl.nasa.gov:16001/gov/nasa/jpl/opera/adt/opera/rtc:final_1.0.1"
+[ -z "${SAS_IMAGE}" ] && SAS_IMAGE="artifactory-fn.jpl.nasa.gov:16001/gov/nasa/jpl/opera/adt/opera/rtc:final_1.0.2"
 
 echo "WORKSPACE: $WORKSPACE"
 echo "IMAGE: $IMAGE"

diff --git a/...ge_rtc_s1_delivery_5_final_runconfig.yaml → ..._rtc_s1_delivery_5.2_final_runconfig.yaml b/...ge_rtc_s1_delivery_5_final_runconfig.yaml → ..._rtc_s1_delivery_5.2_final_runconfig.yaml
@@ -108,6 +108,8 @@ RunConfig:
 
                         product_data_access: "https://search.asf.alaska.edu/#/?dataset=OPERA-S1&productTypes=RTC"
 
+                        static_layers_data_access: "https://search.asf.alaska.edu/#/?dataset=OPERA-S1&productTypes=RTC-STATIC&operaBurstID={burst_id}&end={end_date}"
+
                         save_bursts: True
 
                         save_mosaics: False
@@ -135,18 +137,10 @@ RunConfig:
                         geocoding:
                             memory_mode: auto
 
-                            bursts_geogrid:
-                                top_left:
-                                    x:
-                                    y:
-                                bottom_right:
-                                    x:
-                                    y:
-
-                            estimated_geometric_accuracy_bias_x: 0.0
-                            estimated_geometric_accuracy_bias_y: 0.0
-                            estimated_geometric_accuracy_stddev_x: 0.0
-                            estimated_geometric_accuracy_stddev_y: 0.0
+                            estimated_geometric_accuracy_bias_x: -0.72
+                            estimated_geometric_accuracy_bias_y: -0.67
+                            estimated_geometric_accuracy_stddev_x: 0.70
+                            estimated_geometric_accuracy_stddev_y: 0.62
 
                         mosaicking:
                             mosaic_mode: first
diff --git a/...s1_static_delivery_5_final_runconfig.yaml → ..._static_delivery_5.2_final_runconfig.yaml b/...s1_static_delivery_5_final_runconfig.yaml → ..._static_delivery_5.2_final_runconfig.yaml
@@ -108,6 +108,8 @@ RunConfig:
 
                         product_data_access: "https://search.asf.alaska.edu/#/?dataset=OPERA-S1&productTypes=RTC"
 
+                        static_layers_data_access: "https://search.asf.alaska.edu/#/?dataset=OPERA-S1&productTypes=RTC-STATIC&operaBurstID={burst_id}&end={end_date}"
+
                         save_bursts: True
 
                         save_mosaics: False
@@ -130,23 +132,15 @@ RunConfig:
                         rtc:
                             output_type: gamma0
 
-                        num_workers: 3
+                        num_workers: 2
 
                         geocoding:
                             memory_mode: auto
 
-                            bursts_geogrid:
-                                top_left:
-                                    x:
-                                    y:
-                                bottom_right:
-                                    x:
-                                    y:
-
-                            estimated_geometric_accuracy_bias_x: 0.0
-                            estimated_geometric_accuracy_bias_y: 0.0
-                            estimated_geometric_accuracy_stddev_x: 0.0
-                            estimated_geometric_accuracy_stddev_y: 0.0
+                            estimated_geometric_accuracy_bias_x: -0.72
+                            estimated_geometric_accuracy_bias_y: -0.67
+                            estimated_geometric_accuracy_stddev_x: 0.70
+                            estimated_geometric_accuracy_stddev_y: 0.62
 
                         mosaicking:
                             mosaic_mode: first
diff --git a/.ci/scripts/rtc_s1/test_int_rtc_s1.sh b/.ci/scripts/rtc_s1/test_int_rtc_s1.sh
@@ -27,9 +27,9 @@ SAMPLE_TIME=15
 # RUNCONFIG should be the name of the runconfig in s3://operasds-dev-pge/${PGE_NAME}/
 [ -z "${WORKSPACE}" ] && WORKSPACE="$(realpath "$(dirname "$(realpath "$0")")"/../../..)"
 [ -z "${PGE_TAG}" ] && PGE_TAG="${USER}-dev"
-[ -z "${INPUT_DATA}" ] && INPUT_DATA="rtc_s1_delivery_5.1_final_1.0.1_expected_input.zip"
-[ -z "${EXPECTED_DATA}" ] && EXPECTED_DATA="rtc_s1_delivery_5.1_final_1.0.1_expected_output.zip"
-[ -z "${RUNCONFIG}" ] && RUNCONFIG="opera_pge_rtc_s1_delivery_5_final_runconfig.yaml"
+[ -z "${INPUT_DATA}" ] && INPUT_DATA="rtc_s1_final_1.0.2_expected_input_data.zip"
+[ -z "${EXPECTED_DATA}" ] && EXPECTED_DATA="rtc_s1_final_1.0.2_expected_output.zip"
+[ -z "${RUNCONFIG}" ] && RUNCONFIG="opera_pge_rtc_s1_delivery_5.2_final_runconfig.yaml"
 [ -z "${TMP_ROOT}" ] && TMP_ROOT="$DEFAULT_TMP_ROOT"
 
 # Create the test output directory in the work space
@@ -54,7 +54,7 @@ input_dir="${TMP_DIR}/${INPUT_DATA%.*}/input_dir"
 runconfig_dir="${TMP_DIR}/runconfig"
 
 # Copy the RunConfig for the static layers workflow
-static_runconfig="opera_pge_rtc_s1_static_delivery_5_final_runconfig.yaml"
+static_runconfig="opera_pge_rtc_s1_static_delivery_5.2_final_runconfig.yaml"
 local_static_runconfig="${SCRIPT_DIR}/${static_runconfig}"
 echo "Copying runconfig file $local_static_runconfig to $runconfig_dir/"
 cp ${local_static_runconfig} ${runconfig_dir}

diff --git a/...ples/cslc_s1_sample_runconfig-v2.1.0.yaml → ...ples/cslc_s1_sample_runconfig-v2.1.1.yaml b/...ples/cslc_s1_sample_runconfig-v2.1.0.yaml → ...ples/cslc_s1_sample_runconfig-v2.1.1.yaml
@@ -1,4 +1,4 @@
-# Sample RunConfig for use with the CSLC-S1 PGE v2.1.0
+# Sample RunConfig for use with the CSLC-S1 PGE v2.1.1
 # This RunConfig should require minimal changes in order to be used with the
 # OPERA PCM.
 

diff --git a/examples/rtc_s1_sample_runconfig-v2.1.0.yaml → examples/rtc_s1_sample_runconfig-v2.1.1.yaml b/examples/rtc_s1_sample_runconfig-v2.1.0.yaml → examples/rtc_s1_sample_runconfig-v2.1.1.yaml
@@ -1,4 +1,4 @@
-# Sample RunConfig for use with the RTC-S1 PGE v2.1.0
+# Sample RunConfig for use with the RTC-S1 PGE v2.1.1
 # This RunConfig should require minimal changes in order to be used with the
 # OPERA PCM.
 
@@ -174,6 +174,11 @@ RunConfig:
                         # Location from where the output product can be retrieved (URL or DOI)
                         product_data_access: "https://search.asf.alaska.edu/#/?dataset=OPERA-S1&productTypes=RTC"
 
+                        # Location from where the static layers product can be retrieved
+                        # The {burst_id} and {end_date} are template placeholders that will be
+                        # filled in by the SAS if they are present in the string defined in this config
+                        static_layers_data_access: "https://search.asf.alaska.edu/#/?dataset=OPERA-S1&productTypes=RTC-STATIC&operaBurstID={burst_id}&end={end_date}"
+
                         # RTC-S1 imagery
                         save_bursts: True
 

diff --git a/src/opera/pge/cslc_s1/cslc_s1_pge.py b/src/opera/pge/cslc_s1/cslc_s1_pge.py
@@ -857,10 +857,10 @@ class CslcS1Executor(CslcS1PreProcessorMixin, CslcS1PostProcessorMixin, PgeExecu
     LEVEL = "L2"
     """Processing Level for CSLC-S1 Products"""
 
-    PGE_VERSION = "2.1.0"
+    PGE_VERSION = "2.1.1"
     """Version of the PGE (overrides default from base_pge)"""
 
-    SAS_VERSION = "0.5.4"  # Final release https://github.com/opera-adt/COMPASS/releases/tag/v0.5.4
+    SAS_VERSION = "0.5.5"  # Final release https://github.com/opera-adt/COMPASS/releases/tag/v0.5.5
     """Version of the SAS wrapped by this PGE, should be updated as needed"""
 
     def __init__(self, pge_name, runconfig_path, **kwargs):

diff --git a/src/opera/pge/rtc_s1/rtc_s1_pge.py b/src/opera/pge/rtc_s1/rtc_s1_pge.py
@@ -754,28 +754,6 @@ def _collect_rtc_product_metadata(self, metadata_product):
             except ValueError as err:
                 self.logger.critical(self.name, ErrorCode.ISO_METADATA_RENDER_FAILED, str(err))
 
-        # We also need to create the URL for static layer data access
-        # This URL will be injected into metadata products after products
-        # have been renamed to meet naming conventions
-        url_template = (
-            "https://search.asf.alaska.edu/#/?dataset=OPERA-S1&"
-            "productTypes=RTC-STATIC&operaBurstID={burst_id}&end={sensor_date}"
-        )
-
-        burst_id = output_product_metadata['identification']['burstID']
-        burst_id = burst_id.upper().replace('_', '-')
-
-        acquisition_time = output_product_metadata['identification']['zeroDopplerStartTime']
-
-        if acquisition_time.endswith('Z'):
-            acquisition_time = acquisition_time[:-1]
-
-        sensor_date = datetime.strptime(acquisition_time, "%Y-%m-%dT%H:%M:%S.%f").strftime('%Y-%m-%d')
-
-        url = url_template.format(burst_id=burst_id, sensor_date=sensor_date)
-
-        output_product_metadata['identification']['staticLayersDataAccess'] = url
-
         return output_product_metadata
 
     def _create_custom_metadata(self):
@@ -874,11 +852,6 @@ def _stage_output_files(self):
         for output_product in output_products:
             self._assign_filename(output_product, self.runconfig.output_product_path)
 
-        # Assign static layer data access URL to metadata product. We want to
-        # do this prior to the checksum generation for the catalog metadata
-        # since we are changing the contents of the files.
-        self._inject_static_data_access_url()
-
         # Write the catalog metadata to disk with the appropriate filename
         catalog_metadata = self._create_catalog_metadata()
 
@@ -938,65 +911,6 @@ def _stage_output_files(self):
             # Log stream might be closed by this point so raise an Exception instead
             raise RuntimeError(msg)
 
-    def _inject_static_data_access_url(self):
-        """
-        Modifies all HDF5 metadata output products to inject the URL for
-        Static Layer Data Access. This is only performed for the baseline
-        RTC workflow.
-        """
-        self.logger.info(
-            self.name, ErrorCode.UPDATING_PRODUCT_METADATA,
-            'Injecting static layer data access URL into product metadata'
-        )
-
-        product_type = self.runconfig.product_type
-
-        # For baseline RTC products, we are creating a linkage to where the
-        # corresponding static layer product can be obtained
-        if product_type == "RTC_S1":
-            hdf5_path = "/identification/staticLayersDataAccess"
-            geotiff_key = "STATIC_LAYERS_DATA_ACCESS"
-        # For static layer RTC products, we are injecting a reference to where
-        # the current product can be obtained from (product data access)
-        else:
-            hdf5_path = "/identification/dataAccess"
-            geotiff_key = "PRODUCT_DATA_ACCESS"
-
-        output_products = self.runconfig.get_output_product_filenames()
-
-        # Iterate over all the HDF5 output products, by now they should all
-        # be staged within the top-level of the designated output directory
-        hdf5_products = list(filter(lambda filename: filename.endswith(".h5"), output_products))
-
-        for hdf5_product in hdf5_products:
-            burst_id = get_burst_id_from_file_name(hdf5_product)
-
-            # Get the static layer access URL we already assigned to the
-            # product metadata
-            product_metadata = self._burst_metadata_cache[burst_id]
-            url = product_metadata['identification']['staticLayersDataAccess']
-
-            # Modify the HDF5 file with the instantiated URL for this burst product
-            with h5py.File(hdf5_product, 'r+') as hf:
-                del hf[hdf5_path]
-                hf.create_dataset(
-                    hdf5_path, data=np.string_(url)
-                )
-
-        tif_products = list(filter(lambda filename: filename.endswith(".tif"), output_products))
-
-        for tif_product in tif_products:
-            burst_id = get_burst_id_from_file_name(tif_product)
-
-            product_metadata = self._burst_metadata_cache[burst_id]
-            url = product_metadata['identification']['staticLayersDataAccess']
-
-            kwargs = {geotiff_key: url}
-
-            set_geotiff_metadata(
-                tif_product, scratch_dir=self.runconfig.scratch_path, **kwargs
-            )
-
     def run_postprocessor(self, **kwargs):
         """
         Executes the post-processing steps for the RTC-S1 PGE.
@@ -1035,10 +949,10 @@ class RtcS1Executor(RtcS1PreProcessorMixin, RtcS1PostProcessorMixin, PgeExecutor
     LEVEL = "L2"
     """Processing Level for RTC-S1 Products"""
 
-    PGE_VERSION = "2.1.0"
+    PGE_VERSION = "2.1.1"
     """Version of the PGE (overrides default from base_pge)"""
 
-    SAS_VERSION = "1.0.1"  # Final release https://github.com/opera-adt/RTC/releases/tag/v1.0.1
+    SAS_VERSION = "1.0.2"  # Final release https://github.com/opera-adt/RTC/releases/tag/v1.0.2
     """Version of the SAS wrapped by this PGE, should be updated as needed"""
 
     SOURCE = "S1"

diff --git a/src/opera/scripts/pge_docker_entrypoint.sh b/src/opera/scripts/pge_docker_entrypoint.sh
@@ -10,9 +10,7 @@ PGE_PROGRAM_DIR=${PGE_DEST_DIR}
 export PYTHONPATH=$PYTHONPATH:${PGE_PROGRAM_DIR}
 
 # Run the PGE wrapped by the following:
-# 1) docker-stats-on-exit-shim to collect computer usage info.  TODO
-#    https://github.com/delcypher/docker-stats-on-exit-shim
-# 2) The "PGE main" script, which is part of the opera_pge repo copied into the
-#    SAS container
+# The "PGE main" script, which is part of the opera_pge repo copied into the
+# SAS container
 
 ${PGE_PROGRAM_DIR}/opera/scripts/pge_main.py "$@"