From 3d38eb1681e30f40cdcb0eac1cadd43af18e7088 Mon Sep 17 00:00:00 2001 From: Scott Collins Date: Wed, 20 Mar 2024 09:24:00 -0700 Subject: [PATCH 1/6] Rolled version numbers for CSLC-S1 to PGE v2.1.1 and SAS image v0.5.5 --- .ci/scripts/cslc_s1/build_cslc_s1.sh | 2 +- ...onfig-v2.1.0.yaml => cslc_s1_sample_runconfig-v2.1.1.yaml} | 2 +- src/opera/pge/cslc_s1/cslc_s1_pge.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) rename examples/{cslc_s1_sample_runconfig-v2.1.0.yaml => cslc_s1_sample_runconfig-v2.1.1.yaml} (99%) diff --git a/.ci/scripts/cslc_s1/build_cslc_s1.sh b/.ci/scripts/cslc_s1/build_cslc_s1.sh index f6f1d244..d0e9fb2e 100755 --- a/.ci/scripts/cslc_s1/build_cslc_s1.sh +++ b/.ci/scripts/cslc_s1/build_cslc_s1.sh @@ -24,7 +24,7 @@ BUILD_DATE_TIME=$(date -u +'%Y-%m-%dT%H:%M:%SZ') # defaults, SAS image should be updated as necessary for new image releases from ADT [ -z "${WORKSPACE}" ] && WORKSPACE=$(realpath $(dirname $(realpath $0))/../../..) [ -z "${TAG}" ] && TAG="${USER}-dev" -[ -z "${SAS_IMAGE}" ] && SAS_IMAGE="artifactory-fn.jpl.nasa.gov:16001/gov/nasa/jpl/opera/adt/opera/cslc_s1:final_0.5.4" +[ -z "${SAS_IMAGE}" ] && SAS_IMAGE="artifactory-fn.jpl.nasa.gov:16001/gov/nasa/jpl/opera/adt/opera/cslc_s1:final_0.5.5" echo "WORKSPACE: $WORKSPACE" echo "IMAGE: $IMAGE" diff --git a/examples/cslc_s1_sample_runconfig-v2.1.0.yaml b/examples/cslc_s1_sample_runconfig-v2.1.1.yaml similarity index 99% rename from examples/cslc_s1_sample_runconfig-v2.1.0.yaml rename to examples/cslc_s1_sample_runconfig-v2.1.1.yaml index cf1b0954..6230a534 100644 --- a/examples/cslc_s1_sample_runconfig-v2.1.0.yaml +++ b/examples/cslc_s1_sample_runconfig-v2.1.1.yaml @@ -1,4 +1,4 @@ -# Sample RunConfig for use with the CSLC-S1 PGE v2.1.0 +# Sample RunConfig for use with the CSLC-S1 PGE v2.1.1 # This RunConfig should require minimal changes in order to be used with the # OPERA PCM. diff --git a/src/opera/pge/cslc_s1/cslc_s1_pge.py b/src/opera/pge/cslc_s1/cslc_s1_pge.py index d7e53896..522a8746 100644 --- a/src/opera/pge/cslc_s1/cslc_s1_pge.py +++ b/src/opera/pge/cslc_s1/cslc_s1_pge.py @@ -857,10 +857,10 @@ class CslcS1Executor(CslcS1PreProcessorMixin, CslcS1PostProcessorMixin, PgeExecu LEVEL = "L2" """Processing Level for CSLC-S1 Products""" - PGE_VERSION = "2.1.0" + PGE_VERSION = "2.1.1" """Version of the PGE (overrides default from base_pge)""" - SAS_VERSION = "0.5.4" # Final release https://github.com/opera-adt/COMPASS/releases/tag/v0.5.4 + SAS_VERSION = "0.5.5" # Final release https://github.com/opera-adt/COMPASS/releases/tag/v0.5.5 """Version of the SAS wrapped by this PGE, should be updated as needed""" def __init__(self, pge_name, runconfig_path, **kwargs): From ca6d97fba46e001f9f9ee26ac5d5ee1187452211 Mon Sep 17 00:00:00 2001 From: Scott Collins Date: Wed, 20 Mar 2024 09:30:18 -0700 Subject: [PATCH 2/6] Removed some outdated TODO statements --- src/opera/scripts/pge_docker_entrypoint.sh | 6 ++---- src/opera/util/run_utils.py | 1 - src/opera/util/time.py | 2 -- 3 files changed, 2 insertions(+), 7 deletions(-) diff --git a/src/opera/scripts/pge_docker_entrypoint.sh b/src/opera/scripts/pge_docker_entrypoint.sh index d4798a19..4380da09 100755 --- a/src/opera/scripts/pge_docker_entrypoint.sh +++ b/src/opera/scripts/pge_docker_entrypoint.sh @@ -10,9 +10,7 @@ PGE_PROGRAM_DIR=${PGE_DEST_DIR} export PYTHONPATH=$PYTHONPATH:${PGE_PROGRAM_DIR} # Run the PGE wrapped by the following: -# 1) docker-stats-on-exit-shim to collect computer usage info. TODO -# https://github.com/delcypher/docker-stats-on-exit-shim -# 2) The "PGE main" script, which is part of the opera_pge repo copied into the -# SAS container +# The "PGE main" script, which is part of the opera_pge repo copied into the +# SAS container ${PGE_PROGRAM_DIR}/opera/scripts/pge_main.py "$@" diff --git a/src/opera/util/run_utils.py b/src/opera/util/run_utils.py index aa87a29d..98a90855 100644 --- a/src/opera/util/run_utils.py +++ b/src/opera/util/run_utils.py @@ -199,7 +199,6 @@ def time_and_execute(command_line, logger, execute_via_shell=False): if execute_via_shell: command_line = " ".join(command_line) - # TODO: support for timeout argument? run_result = subprocess.run(command_line, env=os.environ.copy(), check=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=execute_via_shell) diff --git a/src/opera/util/time.py b/src/opera/util/time.py index e248db06..7aa885c1 100644 --- a/src/opera/util/time.py +++ b/src/opera/util/time.py @@ -90,8 +90,6 @@ def get_catalog_metadata_datetime_str(date_time): resolution. """ - # TODO: Rework to support 0.1 nanosecond resolution - # That probably means ditching Python's datetime class. datetime_str = date_time.isoformat(sep='T', timespec='microseconds') + "0000" + "Z" return datetime_str From 9167f8462459772f9f61b8022ee040ede48fdba0 Mon Sep 17 00:00:00 2001 From: Scott Collins Date: Wed, 20 Mar 2024 10:44:06 -0700 Subject: [PATCH 3/6] Updated CSLC-S1 integration test script to use latest assets for delivery v6.4 --- ...> opera_pge_cslc_s1_delivery_6.4_final_runconfig.yaml} | 0 ..._pge_cslc_s1_static_delivery_6.4_final_runconfig.yaml} | 0 .ci/scripts/cslc_s1/test_int_cslc_s1.sh | 8 ++++---- 3 files changed, 4 insertions(+), 4 deletions(-) rename .ci/scripts/cslc_s1/{opera_pge_cslc_s1_delivery_6.1_final_runconfig.yaml => opera_pge_cslc_s1_delivery_6.4_final_runconfig.yaml} (100%) rename .ci/scripts/cslc_s1/{opera_pge_cslc_s1_static_delivery_6.1_final_runconfig.yaml => opera_pge_cslc_s1_static_delivery_6.4_final_runconfig.yaml} (100%) diff --git a/.ci/scripts/cslc_s1/opera_pge_cslc_s1_delivery_6.1_final_runconfig.yaml b/.ci/scripts/cslc_s1/opera_pge_cslc_s1_delivery_6.4_final_runconfig.yaml similarity index 100% rename from .ci/scripts/cslc_s1/opera_pge_cslc_s1_delivery_6.1_final_runconfig.yaml rename to .ci/scripts/cslc_s1/opera_pge_cslc_s1_delivery_6.4_final_runconfig.yaml diff --git a/.ci/scripts/cslc_s1/opera_pge_cslc_s1_static_delivery_6.1_final_runconfig.yaml b/.ci/scripts/cslc_s1/opera_pge_cslc_s1_static_delivery_6.4_final_runconfig.yaml similarity index 100% rename from .ci/scripts/cslc_s1/opera_pge_cslc_s1_static_delivery_6.1_final_runconfig.yaml rename to .ci/scripts/cslc_s1/opera_pge_cslc_s1_static_delivery_6.4_final_runconfig.yaml diff --git a/.ci/scripts/cslc_s1/test_int_cslc_s1.sh b/.ci/scripts/cslc_s1/test_int_cslc_s1.sh index a61aab62..13f2aa96 100755 --- a/.ci/scripts/cslc_s1/test_int_cslc_s1.sh +++ b/.ci/scripts/cslc_s1/test_int_cslc_s1.sh @@ -26,9 +26,9 @@ SAMPLE_TIME=15 # Test data should be uploaded to s3://operasds-dev-pge/${PGE_NAME}/ [ -z "${WORKSPACE}" ] && WORKSPACE=$(realpath "$(dirname "$(realpath "$0")")"/../../..) [ -z "${PGE_TAG}" ] && PGE_TAG="${USER}-dev" -[ -z "${INPUT_DATA}" ] && INPUT_DATA="delivery_cslc_s1_final_0.5.1_expected_input_data.zip" -[ -z "${EXPECTED_DATA}" ] && EXPECTED_DATA="delivery_cslc_s1_final_0.5.1_expected_output.zip" -[ -z "${RUNCONFIG}" ] && RUNCONFIG="opera_pge_cslc_s1_delivery_6.1_final_runconfig.yaml" +[ -z "${INPUT_DATA}" ] && INPUT_DATA="cslc_s1_final_0.5.5_expected_input_data.zip" +[ -z "${EXPECTED_DATA}" ] && EXPECTED_DATA="cslc_s1_final_0.5.5_expected_output.zip" +[ -z "${RUNCONFIG}" ] && RUNCONFIG="opera_pge_cslc_s1_delivery_6.4_final_runconfig.yaml" [ -z "${TMP_ROOT}" ] && TMP_ROOT="$DEFAULT_TMP_ROOT" # Create the test output directory in the workspace @@ -53,7 +53,7 @@ input_dir="${TMP_DIR}/${INPUT_DATA%.*}/input_data" runconfig_dir="${TMP_DIR}/runconfig" # Copy the RunConfig for the static layers workflow -static_runconfig="opera_pge_cslc_s1_static_delivery_6.1_final_runconfig.yaml" +static_runconfig="opera_pge_cslc_s1_static_delivery_6.4_final_runconfig.yaml" local_static_runconfig="${SCRIPT_DIR}/${static_runconfig}" echo "Copying runconfig file $local_static_runconfig to $runconfig_dir/" cp ${local_static_runconfig} ${runconfig_dir} From 7c091cda382f7548177951398fed1d18a374e970 Mon Sep 17 00:00:00 2001 From: Scott Collins Date: Wed, 20 Mar 2024 13:34:07 -0700 Subject: [PATCH 4/6] Removed all code and tests related to injection of the static layers access URL This code was breaking visualization layers in COG format output products, so the SAS now writes the URL into RTC product metadata. --- src/opera/pge/rtc_s1/rtc_s1_pge.py | 86 -------------------- src/opera/test/pge/rtc_s1/test_rtc_s1_pge.py | 54 +----------- 2 files changed, 1 insertion(+), 139 deletions(-) diff --git a/src/opera/pge/rtc_s1/rtc_s1_pge.py b/src/opera/pge/rtc_s1/rtc_s1_pge.py index 31c79f54..354456ad 100644 --- a/src/opera/pge/rtc_s1/rtc_s1_pge.py +++ b/src/opera/pge/rtc_s1/rtc_s1_pge.py @@ -754,28 +754,6 @@ def _collect_rtc_product_metadata(self, metadata_product): except ValueError as err: self.logger.critical(self.name, ErrorCode.ISO_METADATA_RENDER_FAILED, str(err)) - # We also need to create the URL for static layer data access - # This URL will be injected into metadata products after products - # have been renamed to meet naming conventions - url_template = ( - "https://search.asf.alaska.edu/#/?dataset=OPERA-S1&" - "productTypes=RTC-STATIC&operaBurstID={burst_id}&end={sensor_date}" - ) - - burst_id = output_product_metadata['identification']['burstID'] - burst_id = burst_id.upper().replace('_', '-') - - acquisition_time = output_product_metadata['identification']['zeroDopplerStartTime'] - - if acquisition_time.endswith('Z'): - acquisition_time = acquisition_time[:-1] - - sensor_date = datetime.strptime(acquisition_time, "%Y-%m-%dT%H:%M:%S.%f").strftime('%Y-%m-%d') - - url = url_template.format(burst_id=burst_id, sensor_date=sensor_date) - - output_product_metadata['identification']['staticLayersDataAccess'] = url - return output_product_metadata def _create_custom_metadata(self): @@ -874,11 +852,6 @@ def _stage_output_files(self): for output_product in output_products: self._assign_filename(output_product, self.runconfig.output_product_path) - # Assign static layer data access URL to metadata product. We want to - # do this prior to the checksum generation for the catalog metadata - # since we are changing the contents of the files. - self._inject_static_data_access_url() - # Write the catalog metadata to disk with the appropriate filename catalog_metadata = self._create_catalog_metadata() @@ -938,65 +911,6 @@ def _stage_output_files(self): # Log stream might be closed by this point so raise an Exception instead raise RuntimeError(msg) - def _inject_static_data_access_url(self): - """ - Modifies all HDF5 metadata output products to inject the URL for - Static Layer Data Access. This is only performed for the baseline - RTC workflow. - """ - self.logger.info( - self.name, ErrorCode.UPDATING_PRODUCT_METADATA, - 'Injecting static layer data access URL into product metadata' - ) - - product_type = self.runconfig.product_type - - # For baseline RTC products, we are creating a linkage to where the - # corresponding static layer product can be obtained - if product_type == "RTC_S1": - hdf5_path = "/identification/staticLayersDataAccess" - geotiff_key = "STATIC_LAYERS_DATA_ACCESS" - # For static layer RTC products, we are injecting a reference to where - # the current product can be obtained from (product data access) - else: - hdf5_path = "/identification/dataAccess" - geotiff_key = "PRODUCT_DATA_ACCESS" - - output_products = self.runconfig.get_output_product_filenames() - - # Iterate over all the HDF5 output products, by now they should all - # be staged within the top-level of the designated output directory - hdf5_products = list(filter(lambda filename: filename.endswith(".h5"), output_products)) - - for hdf5_product in hdf5_products: - burst_id = get_burst_id_from_file_name(hdf5_product) - - # Get the static layer access URL we already assigned to the - # product metadata - product_metadata = self._burst_metadata_cache[burst_id] - url = product_metadata['identification']['staticLayersDataAccess'] - - # Modify the HDF5 file with the instantiated URL for this burst product - with h5py.File(hdf5_product, 'r+') as hf: - del hf[hdf5_path] - hf.create_dataset( - hdf5_path, data=np.string_(url) - ) - - tif_products = list(filter(lambda filename: filename.endswith(".tif"), output_products)) - - for tif_product in tif_products: - burst_id = get_burst_id_from_file_name(tif_product) - - product_metadata = self._burst_metadata_cache[burst_id] - url = product_metadata['identification']['staticLayersDataAccess'] - - kwargs = {geotiff_key: url} - - set_geotiff_metadata( - tif_product, scratch_dir=self.runconfig.scratch_path, **kwargs - ) - def run_postprocessor(self, **kwargs): """ Executes the post-processing steps for the RTC-S1 PGE. diff --git a/src/opera/test/pge/rtc_s1/test_rtc_s1_pge.py b/src/opera/test/pge/rtc_s1/test_rtc_s1_pge.py index ed558024..19cf924d 100644 --- a/src/opera/test/pge/rtc_s1/test_rtc_s1_pge.py +++ b/src/opera/test/pge/rtc_s1/test_rtc_s1_pge.py @@ -16,21 +16,16 @@ import unittest from io import StringIO from os.path import abspath, join -from unittest.mock import patch - -from pkg_resources import resource_filename import yaml +from pkg_resources import resource_filename -import opera.util.tiff_utils from opera.pge import RunConfig from opera.pge.rtc_s1.rtc_s1_pge import RtcS1Executor from opera.util import PgeLogger from opera.util.dataset_utils import get_sensor_from_spacecraft_name from opera.util.h5_utils import create_test_rtc_metadata_product from opera.util.h5_utils import get_rtc_s1_product_metadata -from opera.util.mock_utils import mock_gdal_edit -from opera.util.mock_utils import mock_save_as_cog class RtcS1PgeTestCase(unittest.TestCase): @@ -93,8 +88,6 @@ def tearDown(self) -> None: os.chdir(self.test_dir) self.working_dir.cleanup() - @patch.object(opera.util.tiff_utils, "gdal_edit", mock_gdal_edit) - @patch.object(opera.util.tiff_utils, "save_as_cog", mock_save_as_cog) def test_rtc_s1_pge_execution(self): """ Test execution of the RtcS1Executor class and its associated mixins @@ -156,43 +149,6 @@ def test_rtc_s1_pge_execution(self): self.assertIn(f"RTC-S1 invoked with RunConfig {expected_sas_config_file}", log_contents) - @patch.object(opera.util.tiff_utils, "save_as_cog", mock_save_as_cog) - def test_static_layer_data_access_url_injection(self): - """Test injection of static data access URL into the output HDF5 product(s)""" - expected_url = 'https://search.asf.alaska.edu/#/?dataset=OPERA-S1&productTypes=RTC-STATIC&operaBurstID=T069-147170-IW1&end=2018-05-04' # noqa E501 - - runconfig_path = join(self.data_dir, 'test_rtc_s1_config.yaml') - - pge = RtcS1Executor(pge_name="RtcPgeTest", runconfig_path=runconfig_path) - - # Evaluate the arguments to the gdal_edit call since we cannot actually - # update the dummy .tif files - def _patched_gdal_edit(args): - self.assertEqual(len(args), 4) - self.assertEqual(args[0], 'gdal_edit.py') - self.assertEqual(args[1], '-mo') - self.assertTrue(args[2].startswith('STATIC_LAYERS_DATA_ACCESS=')) - self.assertTrue(args[2].endswith(expected_url)) - self.assertTrue(args[3].endswith('.tif')) - return 0 - - with patch.object(opera.util.tiff_utils, "gdal_edit", _patched_gdal_edit): - pge.run() - - # Grab the metadata generated from the PGE run - output_files = glob.glob(join(pge.runconfig.output_product_path, "*.h5")) - - self.assertEqual(len(output_files), 1) - - output_file = output_files[0] - - rtc_metadata = get_rtc_s1_product_metadata(output_file) - - self.assertIn('staticLayersDataAccess', rtc_metadata['identification']) - self.assertEqual(rtc_metadata['identification']['staticLayersDataAccess'], expected_url) - - @patch.object(opera.util.tiff_utils, "gdal_edit", mock_gdal_edit) - @patch.object(opera.util.tiff_utils, "save_as_cog", mock_save_as_cog) def test_filename_application(self): """Test the filename convention applied to RTC output products""" runconfig_path = join(self.data_dir, 'test_rtc_s1_config.yaml') @@ -250,8 +206,6 @@ def test_filename_application(self): self.assertIn(f"{core_filename}_BROWSE.png", output_files) - @patch.object(opera.util.tiff_utils, "gdal_edit", mock_gdal_edit) - @patch.object(opera.util.tiff_utils, "save_as_cog", mock_save_as_cog) def test_static_layer_filename_application(self): """Test the filename convention applied to RTC static layer output products""" runconfig_path = join(self.data_dir, 'test_rtc_s1_static_config.yaml') @@ -601,8 +555,6 @@ def test_expected_extension(self): if os.path.exists(test_runconfig_path): os.unlink(test_runconfig_path) - @patch.object(opera.util.tiff_utils, "gdal_edit", mock_gdal_edit) - @patch.object(opera.util.tiff_utils, "save_as_cog", mock_save_as_cog) def test_no_burst_id(self): """Test _iso_metadata_filename with bad burst_id""" runconfig_path = join(self.data_dir, 'test_rtc_s1_config.yaml') @@ -612,8 +564,6 @@ def test_no_burst_id(self): with self.assertRaises(RuntimeError): pge._iso_metadata_filename("bad_burst") - @patch.object(opera.util.tiff_utils, "gdal_edit", mock_gdal_edit) - @patch.object(opera.util.tiff_utils, "save_as_cog", mock_save_as_cog) def test_bad_iso_metadata_template(self): """Verify code when the QA application is enabled""" runconfig_path = join(self.data_dir, 'test_rtc_s1_config.yaml') @@ -645,8 +595,6 @@ def test_bad_iso_metadata_template(self): if os.path.exists(test_runconfig_path): os.unlink(test_runconfig_path) - @patch.object(opera.util.tiff_utils, "gdal_edit", mock_gdal_edit) - @patch.object(opera.util.tiff_utils, "save_as_cog", mock_save_as_cog) def test_qa_enabled(self): """Test the staging of the qa.log files.""" # Verify code when the QA application is enabled From 3b4d482cabed01cd4f97dd12a36f30e914f9b706 Mon Sep 17 00:00:00 2001 From: Scott Collins Date: Wed, 20 Mar 2024 13:36:19 -0700 Subject: [PATCH 5/6] Rolled version numbers for RTC-S1 to PGE v2.1.1 and SAS image v1.0.2 --- .ci/scripts/rtc_s1/build_rtc_s1.sh | 2 +- ...fig-v2.1.0.yaml => rtc_s1_sample_runconfig-v2.1.1.yaml} | 7 ++++++- src/opera/pge/rtc_s1/rtc_s1_pge.py | 4 ++-- 3 files changed, 9 insertions(+), 4 deletions(-) rename examples/{rtc_s1_sample_runconfig-v2.1.0.yaml => rtc_s1_sample_runconfig-v2.1.1.yaml} (95%) diff --git a/.ci/scripts/rtc_s1/build_rtc_s1.sh b/.ci/scripts/rtc_s1/build_rtc_s1.sh index ef63c8cd..e8fddb2d 100755 --- a/.ci/scripts/rtc_s1/build_rtc_s1.sh +++ b/.ci/scripts/rtc_s1/build_rtc_s1.sh @@ -24,7 +24,7 @@ BUILD_DATE_TIME=$(date -u +'%Y-%m-%dT%H:%M:%SZ') # defaults, SAS image should be updated as necessary for new image releases from ADT [ -z "${WORKSPACE}" ] && WORKSPACE=$(realpath $(dirname $(realpath $0))/../../..) [ -z "${TAG}" ] && TAG="${USER}-dev" -[ -z "${SAS_IMAGE}" ] && SAS_IMAGE="artifactory-fn.jpl.nasa.gov:16001/gov/nasa/jpl/opera/adt/opera/rtc:final_1.0.1" +[ -z "${SAS_IMAGE}" ] && SAS_IMAGE="artifactory-fn.jpl.nasa.gov:16001/gov/nasa/jpl/opera/adt/opera/rtc:final_1.0.2" echo "WORKSPACE: $WORKSPACE" echo "IMAGE: $IMAGE" diff --git a/examples/rtc_s1_sample_runconfig-v2.1.0.yaml b/examples/rtc_s1_sample_runconfig-v2.1.1.yaml similarity index 95% rename from examples/rtc_s1_sample_runconfig-v2.1.0.yaml rename to examples/rtc_s1_sample_runconfig-v2.1.1.yaml index efb804a6..b5c21ed7 100644 --- a/examples/rtc_s1_sample_runconfig-v2.1.0.yaml +++ b/examples/rtc_s1_sample_runconfig-v2.1.1.yaml @@ -1,4 +1,4 @@ -# Sample RunConfig for use with the RTC-S1 PGE v2.1.0 +# Sample RunConfig for use with the RTC-S1 PGE v2.1.1 # This RunConfig should require minimal changes in order to be used with the # OPERA PCM. @@ -174,6 +174,11 @@ RunConfig: # Location from where the output product can be retrieved (URL or DOI) product_data_access: "https://search.asf.alaska.edu/#/?dataset=OPERA-S1&productTypes=RTC" + # Location from where the static layers product can be retrieved + # The {burst_id} and {end_date} are template placeholders that will be + # filled in by the SAS if they are present in the string defined in this config + static_layers_data_access: "https://search.asf.alaska.edu/#/?dataset=OPERA-S1&productTypes=RTC-STATIC&operaBurstID={burst_id}&end={end_date}" + # RTC-S1 imagery save_bursts: True diff --git a/src/opera/pge/rtc_s1/rtc_s1_pge.py b/src/opera/pge/rtc_s1/rtc_s1_pge.py index 354456ad..f0294578 100644 --- a/src/opera/pge/rtc_s1/rtc_s1_pge.py +++ b/src/opera/pge/rtc_s1/rtc_s1_pge.py @@ -949,10 +949,10 @@ class RtcS1Executor(RtcS1PreProcessorMixin, RtcS1PostProcessorMixin, PgeExecutor LEVEL = "L2" """Processing Level for RTC-S1 Products""" - PGE_VERSION = "2.1.0" + PGE_VERSION = "2.1.1" """Version of the PGE (overrides default from base_pge)""" - SAS_VERSION = "1.0.1" # Final release https://github.com/opera-adt/RTC/releases/tag/v1.0.1 + SAS_VERSION = "1.0.2" # Final release https://github.com/opera-adt/RTC/releases/tag/v1.0.2 """Version of the SAS wrapped by this PGE, should be updated as needed""" SOURCE = "S1" From 9ecb3bf51a56516b36f37284ba414e30a6b04efa Mon Sep 17 00:00:00 2001 From: Scott Collins Date: Wed, 20 Mar 2024 13:55:23 -0700 Subject: [PATCH 6/6] Updated RTC-S1 integration test script to use latest assets for delivery v5.2 --- ..._rtc_s1_delivery_5.2_final_runconfig.yaml} | 18 ++++++----------- ..._static_delivery_5.2_final_runconfig.yaml} | 20 +++++++------------ .ci/scripts/rtc_s1/test_int_rtc_s1.sh | 8 ++++---- 3 files changed, 17 insertions(+), 29 deletions(-) rename .ci/scripts/rtc_s1/{opera_pge_rtc_s1_delivery_5_final_runconfig.yaml => opera_pge_rtc_s1_delivery_5.2_final_runconfig.yaml} (93%) rename .ci/scripts/rtc_s1/{opera_pge_rtc_s1_static_delivery_5_final_runconfig.yaml => opera_pge_rtc_s1_static_delivery_5.2_final_runconfig.yaml} (93%) diff --git a/.ci/scripts/rtc_s1/opera_pge_rtc_s1_delivery_5_final_runconfig.yaml b/.ci/scripts/rtc_s1/opera_pge_rtc_s1_delivery_5.2_final_runconfig.yaml similarity index 93% rename from .ci/scripts/rtc_s1/opera_pge_rtc_s1_delivery_5_final_runconfig.yaml rename to .ci/scripts/rtc_s1/opera_pge_rtc_s1_delivery_5.2_final_runconfig.yaml index 6e8eeda8..45cdf62e 100644 --- a/.ci/scripts/rtc_s1/opera_pge_rtc_s1_delivery_5_final_runconfig.yaml +++ b/.ci/scripts/rtc_s1/opera_pge_rtc_s1_delivery_5.2_final_runconfig.yaml @@ -108,6 +108,8 @@ RunConfig: product_data_access: "https://search.asf.alaska.edu/#/?dataset=OPERA-S1&productTypes=RTC" + static_layers_data_access: "https://search.asf.alaska.edu/#/?dataset=OPERA-S1&productTypes=RTC-STATIC&operaBurstID={burst_id}&end={end_date}" + save_bursts: True save_mosaics: False @@ -135,18 +137,10 @@ RunConfig: geocoding: memory_mode: auto - bursts_geogrid: - top_left: - x: - y: - bottom_right: - x: - y: - - estimated_geometric_accuracy_bias_x: 0.0 - estimated_geometric_accuracy_bias_y: 0.0 - estimated_geometric_accuracy_stddev_x: 0.0 - estimated_geometric_accuracy_stddev_y: 0.0 + estimated_geometric_accuracy_bias_x: -0.72 + estimated_geometric_accuracy_bias_y: -0.67 + estimated_geometric_accuracy_stddev_x: 0.70 + estimated_geometric_accuracy_stddev_y: 0.62 mosaicking: mosaic_mode: first diff --git a/.ci/scripts/rtc_s1/opera_pge_rtc_s1_static_delivery_5_final_runconfig.yaml b/.ci/scripts/rtc_s1/opera_pge_rtc_s1_static_delivery_5.2_final_runconfig.yaml similarity index 93% rename from .ci/scripts/rtc_s1/opera_pge_rtc_s1_static_delivery_5_final_runconfig.yaml rename to .ci/scripts/rtc_s1/opera_pge_rtc_s1_static_delivery_5.2_final_runconfig.yaml index 2c353316..efbeb338 100644 --- a/.ci/scripts/rtc_s1/opera_pge_rtc_s1_static_delivery_5_final_runconfig.yaml +++ b/.ci/scripts/rtc_s1/opera_pge_rtc_s1_static_delivery_5.2_final_runconfig.yaml @@ -108,6 +108,8 @@ RunConfig: product_data_access: "https://search.asf.alaska.edu/#/?dataset=OPERA-S1&productTypes=RTC" + static_layers_data_access: "https://search.asf.alaska.edu/#/?dataset=OPERA-S1&productTypes=RTC-STATIC&operaBurstID={burst_id}&end={end_date}" + save_bursts: True save_mosaics: False @@ -130,23 +132,15 @@ RunConfig: rtc: output_type: gamma0 - num_workers: 3 + num_workers: 2 geocoding: memory_mode: auto - bursts_geogrid: - top_left: - x: - y: - bottom_right: - x: - y: - - estimated_geometric_accuracy_bias_x: 0.0 - estimated_geometric_accuracy_bias_y: 0.0 - estimated_geometric_accuracy_stddev_x: 0.0 - estimated_geometric_accuracy_stddev_y: 0.0 + estimated_geometric_accuracy_bias_x: -0.72 + estimated_geometric_accuracy_bias_y: -0.67 + estimated_geometric_accuracy_stddev_x: 0.70 + estimated_geometric_accuracy_stddev_y: 0.62 mosaicking: mosaic_mode: first diff --git a/.ci/scripts/rtc_s1/test_int_rtc_s1.sh b/.ci/scripts/rtc_s1/test_int_rtc_s1.sh index e6ef1d85..26508682 100755 --- a/.ci/scripts/rtc_s1/test_int_rtc_s1.sh +++ b/.ci/scripts/rtc_s1/test_int_rtc_s1.sh @@ -27,9 +27,9 @@ SAMPLE_TIME=15 # RUNCONFIG should be the name of the runconfig in s3://operasds-dev-pge/${PGE_NAME}/ [ -z "${WORKSPACE}" ] && WORKSPACE="$(realpath "$(dirname "$(realpath "$0")")"/../../..)" [ -z "${PGE_TAG}" ] && PGE_TAG="${USER}-dev" -[ -z "${INPUT_DATA}" ] && INPUT_DATA="rtc_s1_delivery_5.1_final_1.0.1_expected_input.zip" -[ -z "${EXPECTED_DATA}" ] && EXPECTED_DATA="rtc_s1_delivery_5.1_final_1.0.1_expected_output.zip" -[ -z "${RUNCONFIG}" ] && RUNCONFIG="opera_pge_rtc_s1_delivery_5_final_runconfig.yaml" +[ -z "${INPUT_DATA}" ] && INPUT_DATA="rtc_s1_final_1.0.2_expected_input_data.zip" +[ -z "${EXPECTED_DATA}" ] && EXPECTED_DATA="rtc_s1_final_1.0.2_expected_output.zip" +[ -z "${RUNCONFIG}" ] && RUNCONFIG="opera_pge_rtc_s1_delivery_5.2_final_runconfig.yaml" [ -z "${TMP_ROOT}" ] && TMP_ROOT="$DEFAULT_TMP_ROOT" # Create the test output directory in the work space @@ -54,7 +54,7 @@ input_dir="${TMP_DIR}/${INPUT_DATA%.*}/input_dir" runconfig_dir="${TMP_DIR}/runconfig" # Copy the RunConfig for the static layers workflow -static_runconfig="opera_pge_rtc_s1_static_delivery_5_final_runconfig.yaml" +static_runconfig="opera_pge_rtc_s1_static_delivery_5.2_final_runconfig.yaml" local_static_runconfig="${SCRIPT_DIR}/${static_runconfig}" echo "Copying runconfig file $local_static_runconfig to $runconfig_dir/" cp ${local_static_runconfig} ${runconfig_dir}