Initial commit for stub implementation for DIST-S1 PGE

.ci/docker/Dockerfile_dist_s1

@@ -0,0 +1,67 @@
+# Dockerfile to produce the production DSWx-NI PGE Docker image for OPERA
+# Authors: Scott Collins, Ray Bambery
+
+# Default SAS image path, must be provided by the docker build call via --build-arg
+ARG SAS_IMAGE
+FROM $SAS_IMAGE
+
+ARG PGE_SOURCE_DIR
+ARG PGE_DEST_DIR=/home/ops
+ARG CONDA_ROOT=/home/ops/miniconda3
+
+ENV PGE_DEST_DIR=$PGE_DEST_DIR
+ENV CONDA_ROOT=$CONDA_ROOT
+
+ARG BUILD_DATE_TIME
+ARG BUILD_VERSION
+
+# labels
+# the label-schema convention: http://label-schema.org/rc1/
+LABEL org.label-schema.build-date=${BUILD_DATE_TIME} \
+  org.label-schema.version=${BUILD_VERSION} \
+  org.label-schema.license="Copyright 2024,\
+ by the California Institute of Technology.\
+ ALL RIGHTS RESERVED.\
+ United States Government sponsorship acknowledged.\
+ Any commercial use must be negotiated with the Office of Technology Transfer\
+ at the California Institute of Technology.\
+ This software may be subject to U.S. export control laws and regulations.\
+ By accepting this document, the user agrees to comply with all applicable\
+ U.S. export laws and regulations. User has the responsibility to obtain\
+ export licenses, or other export authority as may be required, before\
+ exporting such information to foreign countries or providing access to\
+ foreign persons." \
+  org.label-schema.name="OPERA Product Generation Executable (PGE) Image" \
+  org.label-schema.schema-version="1.0" \
+  org.label-schema.vendor="California Institute of Technology" \
+  maintainer="California Institute of Technology"
+
+# Copy the OPERA PGE software into the container
+# the base container has a default user "mamba" with UID/GID 1000/1000
+COPY --chown=dist_user:dist_user ${PGE_SOURCE_DIR} ${PGE_DEST_DIR}
+
+# Switch to root for installing into Conda Env
+USER 0:0
+
+# Set the default shell to run commands within the Conda environment named "DSXW-SAR" using bash
+SHELL ["conda", "run", "-n", "dist-s1-env", "/bin/bash", "-c"]
+
+# Install dependencies into existing Conda Env
+# Modifications to the destination dir permissions are also required here to ensure
+# we don't run into issues when running as an outside user (via docker run -u)
+RUN set -ex \
+    && cd ${PGE_DEST_DIR} \
+    && mkdir -p ${CONDA_ROOT}/bin \
+    && cp ${PGE_DEST_DIR}/opera/scripts/*_entrypoint.sh ${CONDA_ROOT}/bin \
+    && chmod +x ${CONDA_ROOT}/bin/*_entrypoint.sh \
+    && python -m pip install -r ${PGE_DEST_DIR}/opera/requirements.txt \
+    && conda install --yes --channel conda-forge hdf5 \
+    && chmod 777 $(find ${PGE_DEST_DIR} -type d)
+
+# Set the Docker entrypoint and clear the default command
+ENTRYPOINT ["conda", "run", "--no-capture-output", "-n", "dist-s1-env", "sh", "-c", "exec ${CONDA_ROOT}/bin/pge_docker_entrypoint.sh  \"${@}\"", "--"]
+CMD []
+
+# Set the user/group back to the default
+USER dist_user:dist_user
+

.ci/scripts/dist_s1/build_dist_s1.sh

@@ -0,0 +1,83 @@
+#!/bin/bash
+set -e
+
+# Source the build script utility functions
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
+
+. "${SCRIPT_DIR}"/../util/util.sh
+
+# Parse args
+parse_build_args "$@"
+
+echo '
+=====================================
+
+Building DIST-S1 PGE docker image...
+
+=====================================
+'
+
+PGE_NAME="dist_s1"
+IMAGE="opera_pge/${PGE_NAME}"
+BUILD_DATE_TIME=$(date -u +'%Y-%m-%dT%H:%M:%SZ')
+
+# defaults, SAS image should be updated as necessary for new image releases from ADT
+[ -z "${WORKSPACE}" ] && WORKSPACE=$(realpath $(dirname $(realpath $0))/../../..)
+[ -z "${TAG}" ] && TAG="${USER}-dev"
+[ -z "${SAS_IMAGE}" ] && SAS_IMAGE="ghcr.io/opera-adt/dist-s1:0.0.3"
+
+echo "WORKSPACE: $WORKSPACE"
+echo "IMAGE: $IMAGE"
+echo "TAG: $TAG"
+echo "SAS_IMAGE: $SAS_IMAGE"
+
+# Check that the .ci scripts directory exists
+if [ ! -d "${WORKSPACE}/.ci" ]; then
+  echo "Error: the .ci directory doesn't exist at ${WORKSPACE}/.ci"
+  exit 1
+fi
+
+# Create a directory on the host to use as a staging area for files to be
+# copied into the resulting Docker image.
+STAGING_DIR=$(mktemp -d -p ${WORKSPACE} docker_image_staging_XXXXXXXXXX)
+
+# Configure a trap to clean up on exit regardless of whether the build succeeds
+trap build_script_cleanup EXIT
+
+# Copy files to the staging area and build the PGE docker image
+mkdir -p ${STAGING_DIR}/opera/pge
+
+copy_pge_files $WORKSPACE $STAGING_DIR $PGE_NAME
+
+# Create a VERSION file in the staging area to track version and build time
+printf "pge_version: ${TAG}\pge_build_datetime: ${BUILD_DATE_TIME}\n" \
+    > ${STAGING_DIR}/opera/VERSION \
+
+# Remove the old Docker image, if it exists
+EXISTING_IMAGE_ID=$(docker images -q ${IMAGE}:${TAG})
+if [[ ! -z ${EXISTING_IMAGE_ID} ]]; then
+  docker rmi ${EXISTING_IMAGE_ID}
+fi
+
+# Select the appropriate platform to target the container for.
+# This is to support Apple M1 builds, which will default to (very slow) emulation
+# if the incorrect platform is specified.
+# Note that currently, the container build process on Apple M1 is significantly
+# slower than for other architectures.
+if [[ `uname -m` == "arm64" ]]; then
+  PLATFORM='--platform linux/arm64'
+else
+  PLATFORM='--platform linux/amd64'
+fi
+
+# Build the PGE docker image
+docker build ${PLATFORM} --progress plain --rm --force-rm -t ${IMAGE}:${TAG} \
+    --build-arg SAS_IMAGE=${SAS_IMAGE} \
+    --build-arg BUILD_DATE_TIME=${BUILD_DATE_TIME} \
+    --build-arg BUILD_VERSION=${TAG} \
+    --build-arg PGE_SOURCE_DIR=$(basename ${STAGING_DIR}) \
+    --file ${WORKSPACE}/.ci/docker/Dockerfile_dist_s1 ${WORKSPACE}
+
+echo "DIST-S1 PGE Docker image build complete"
+
+exit 0

.ci/scripts/dist_s1/test_dist_s1.sh

@@ -0,0 +1,96 @@
+#!/bin/bash
+# Script to execute unit tests on the OPERA DISP-S1 PGE Docker image
+
+set -e
+
+# Source the build script utility functions
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
+
+. "${SCRIPT_DIR}"/../util/util.sh
+
+# Parse args
+parse_build_args "$@"
+
+echo '
+=====================================
+
+Testing DIST-S1 PGE Docker image...
+
+=====================================
+'
+
+PGE_NAME="dist_s1"
+IMAGE="opera_pge/${PGE_NAME}"
+TEST_RESULTS_REL_DIR="test_results"
+CONTAINER_HOME="/home/ops"
+CONDA_ROOT="/opt/conda"
+
+# defaults
+[ -z "${WORKSPACE}" ] && WORKSPACE=$(realpath $(dirname $(realpath $0))/../../..)
+[ -z "${TAG}" ] && TAG="${USER}-dev"
+
+TEST_RESULTS_DIR="${WORKSPACE}/${TEST_RESULTS_REL_DIR}/${PGE_NAME}"
+
+echo "Test results output directory: ${TEST_RESULTS_DIR}"
+mkdir --parents ${TEST_RESULTS_DIR}
+chmod -R 775 ${TEST_RESULTS_DIR}
+
+# Use the environment of the docker image to run linting, tests, etc...
+# Note the change of working directory (-w) to a directory without
+# Python code so that import statements favor Python code found in the
+# Docker image rather than code found on the host.
+DOCKER_RUN="docker run --rm \
+    -v ${WORKSPACE}:/workspace \
+    -v ${WORKSPACE}/src/opera/test/data:${CONTAINER_HOME}/opera/test/data \
+    -w /workspace/${TEST_RESULTS_REL_DIR} \
+    -u ${UID}:$(id -g) \
+    --entrypoint ${CONDA_ROOT}/bin/pge_tests_entrypoint.sh \
+    ${IMAGE}:${TAG}"
+
+# Configure a trap to set permissions on exit regardless of whether the testing succeeds
+function set_perms {
+    # Open up permissions on all test results so we can be sure the CI system can
+    # delete them after results are archived within Jenkins
+    ${DOCKER_RUN} bash -c "find \
+        /workspace/${TEST_RESULTS_REL_DIR} -type d -exec chmod 775 {} +"
+
+    ${DOCKER_RUN} bash -c "find \
+        /workspace/${TEST_RESULTS_REL_DIR} -type f -exec chmod 664 {} +"
+}
+
+trap set_perms EXIT
+
+# linting and pep8 style check (configured by .flake8 and .pylintrc)
+${DOCKER_RUN} bash -c "source /usr/local/bin/_activate_current_env.sh; flake8 \
+    --config ${CONTAINER_HOME}/opera/.flake8 \
+    --jobs auto \
+    --exit-zero \
+    --application-import-names opera \
+    --output-file /workspace/${TEST_RESULTS_REL_DIR}/${PGE_NAME}/flake8.log \
+    ${CONTAINER_HOME}/opera"
+
+${DOCKER_RUN} bash -c "export HOME=/home/mamba/opera; source /usr/local/bin/_activate_current_env.sh; pylint \
+    --rcfile=${CONTAINER_HOME}/opera/.pylintrc \
+    --jobs 0 \
+    --exit-zero \
+    --output=/workspace/${TEST_RESULTS_REL_DIR}/${PGE_NAME}/pylint.log \
+    --enable-all-extensions \
+    ${CONTAINER_HOME}/opera"
+
+# pytest (including code coverage)
+${DOCKER_RUN} bash -c "source /usr/local/bin/_activate_current_env.sh; pytest \
+    --junit-xml=/workspace/${TEST_RESULTS_REL_DIR}/${PGE_NAME}/pytest-junit.xml \
+    --cov=${CONTAINER_HOME}/opera/pge/base \
+    --cov=${CONTAINER_HOME}/opera/pge/${PGE_NAME} \
+    --cov=${CONTAINER_HOME}/opera/scripts \
+    --cov=${CONTAINER_HOME}/opera/util \
+    --cov-report=term \
+    --cov-report=html:/workspace/${TEST_RESULTS_REL_DIR}/${PGE_NAME}/coverage_html \
+    /workspace/src/opera/test/pge/base \
+    /workspace/src/opera/test/pge/${PGE_NAME} \
+    /workspace/src/opera/test/scripts \
+    /workspace/src/opera/test/util > /workspace/${TEST_RESULTS_REL_DIR}/${PGE_NAME}/pytest.log 2>&1"
+
+echo "DIST-S1 PGE Docker image test complete"
+
+exit 0

.ci/scripts/dswx_ni/build_dswx_ni.sh

@@ -53,7 +53,7 @@ copy_pge_files $WORKSPACE $STAGING_DIR $PGE_NAME
 mkdir -p ${STAGING_DIR}/opera/pge/dswx_s1; cp -r ${WORKSPACE}/src/opera/pge/dswx_s1/dswx_s1_pge.py ${STAGING_DIR}/opera/pge/dswx_s1/
 
 # Create a VERSION file in the staging area to track version and build time
-printf "pge_version: ${TAG}\npge_build_datetime: ${BUILD_DATE_TIME}\n" \
+printf "pge_version: ${TAG}\pge_build_datetime: ${BUILD_DATE_TIME}\n" \
     > ${STAGING_DIR}/opera/VERSION \
 
 # Remove the old Docker image, if it exists

.ci/scripts/util/build_all_images.sh

@@ -34,6 +34,7 @@ ${BUILD_SCRIPTS_DIR}/rtc_s1/build_rtc_s1.sh --tag ${TAG} --workspace ${WORKSPACE
 ${BUILD_SCRIPTS_DIR}/dswx_s1/build_dswx_s1.sh --tag ${TAG} --workspace ${WORKSPACE}
 ${BUILD_SCRIPTS_DIR}/disp_s1/build_disp_s1.sh --tag ${TAG} --workspace ${WORKSPACE}
 ${BUILD_SCRIPTS_DIR}/dswx_ni/build_dswx_ni.sh --tag ${TAG} --workspace ${WORKSPACE}
+${BUILD_SCRIPTS_DIR}/dist_s1/build_dist_s1.sh --tag ${TAG} --workspace ${WORKSPACE}
 
 echo 'Build Complete'
 

.ci/scripts/util/test_all_images.sh

@@ -34,6 +34,7 @@ ${BUILD_SCRIPTS_DIR}/rtc_s1/test_rtc_s1.sh --tag ${TAG} --workspace ${WORKSPACE}
 ${BUILD_SCRIPTS_DIR}/dswx_s1/test_dswx_s1.sh --tag ${TAG} --workspace ${WORKSPACE}
 ${BUILD_SCRIPTS_DIR}/disp_s1/test_disp_s1.sh --tag ${TAG} --workspace ${WORKSPACE}
 ${BUILD_SCRIPTS_DIR}/dswx_ni/test_dswx_ni.sh --tag ${TAG} --workspace ${WORKSPACE}
+${BUILD_SCRIPTS_DIR}/dist_s1/test_dist_s1.sh --tag ${TAG} --workspace ${WORKSPACE}
 echo 'Build Complete'
 
 exit 0

src/opera/pge/dist_s1/dist_s1_pge.py

@@ -0,0 +1,93 @@
+#!/usr/bin/env python3
+
+"""
+==============
+dist_s1_pge.py
+==============
+Module defining the implementation for the Surface Disturbance (DIST) from Sentinel-1 A/C (S1) PGE.
+
+"""
+
+from opera.pge.base.base_pge import PreProcessorMixin, PgeExecutor, PostProcessorMixin
+
+
+class DistS1PreProcessorMixin(PreProcessorMixin):
+    """
+    Mixin class responsible for handling all pre-processing steps for the DIST-S1
+    PGE. The pre-processing phase is defined as all steps necessary prior
+    to SAS execution.
+
+    This particular pre-processor is currently a stub implementation, inheriting from the base pre-processor mixin
+    and adding nothing at this time. New functionalities will be added as new versions of the DIST-S1 SAS are released.
+
+    """
+
+    _pre_mixin_name = "DistS1PreProcessorMixin"
+    _valid_input_extensions = (".tif",)
+
+    def run_preprocessor(self, **kwargs):
+        """
+        Executes the pre-processing steps for DIST-S1 PGE initialization.
+        The DistS1PreProcessorMixin version of this class performs all actions
+        of the PreProcessorMixin class. Parameterization of the validation
+        functions is handled via specialized class attributes (i.e. _valid_input_extensions)
+
+        Parameters
+        ----------
+        **kwargs: dict
+            Any keyword arguments needed by the pre-processor
+        """
+        super().run_preprocessor(**kwargs)
+
+
+class DistS1PostProcessorMixin(PostProcessorMixin):
+    """
+    Mixin class responsible for handling all post-processing steps for the DIST-S1
+    PGE. The post-processing phase is defined as all steps required after SAS
+    execution has completed, prior to handover of output products to PCM.
+
+    This particular pre-processor is currently a stub implementation, inheriting from the base pre-processor mixin
+    and adding nothing at this time. New functionalities will be added as new versions of the DIST-S1 SAS are released.
+
+    """
+
+    _post_mixin_name = "DistS1PostProcessorMixin"
+    _cached_core_filename = None
+    _tile_metadata_cache = {}
+    _tile_filename_cache = {}
+
+    def run_postprocessor(self, **kwargs):
+        """
+        Executes the post-processing steps for the DSWx-NI PGE.
+        The DistS1PostProcessorMixin version of this method performs the same
+        steps as the base PostProcessorMixin.
+
+        Parameters
+        ----------
+        **kwargs: dict
+            Any keyword arguments needed by the post-processor
+        """
+        super().run_postprocessor(**kwargs)
+
+
+class DistS1Executor(DistS1PreProcessorMixin, DistS1PostProcessorMixin, PgeExecutor):
+    """
+    Main class for execution of the DIST-S1 PGE, including the SAS layer.
+    This class essentially rolls up the DIST-specific pre- and post-processor
+    functionality, while inheriting all other functionality for setup and execution
+    of the SAS from the base PgeExecutor class.
+    """
+
+    NAME = "DIST-S1"
+    """Short name for the DIST-S1 PGE"""
+
+    LEVEL = "L3"
+    """Processing Level for DIST-S1 Products"""
+
+    SAS_VERSION = "0.0.3"  # Beta release https://github.com/opera-adt/dist-s1/releases/tag/v0.0.3
+    """Version of the SAS wrapped by this PGE, should be updated as needed"""
+
+    def __init__(self, pge_name, runconfig_path, **kwargs):
+        super().__init__(pge_name, runconfig_path, **kwargs)
+
+        self.rename_by_pattern_map = {}

src/opera/pge/dist_s1/schema/dist_s1_sas_schema.yaml

@@ -0,0 +1,28 @@
+#
+# Yamale schema for the DIST-S1 SAS Configuration
+#
+run_config:
+
+  # PLACEHOLDER FOR pre_rtc_copol PARAMETER DESCRIPTION
+  pre_rtc_copol: list(str(), required=True)
+
+  # PLACEHOLDER FOR pre_rtc_crosspol PARAMETER DESCRIPTION
+  pre_rtc_crosspol: list(str(), required=True)
+
+  # PLACEHOLDER FOR post_rtc_copol PARAMETER DESCRIPTION
+  post_rtc_copol: list(str(), required=True)
+
+  # PLACEHOLDER FOR post_rtc_crosspol PARAMETER DESCRIPTION
+  post_rtc_crosspol: list(str(), required=True)
+
+  # PLACEHOLDER FOR mgrs_tile_id PARAMETER DESCRIPTION
+  mgrs_tile_id: str(required=True)
+
+  # PLACEHOLDER FOR dist_s1_alert_db_dir PARAMETER DESCRIPTION
+  dist_s1_alert_db_dir: str(required=False)
+
+  # PLACEHOLDER FOR dst_dir PARAMETER DESCRIPTION
+  dst_dir: str(required=False)
+
+  # PLACEHOLDER FOR water_mask PARAMETER DESCRIPTION
+  water_mask: str(required=False)

src/opera/pge/dist_s1/templates/dist_s1_measured_parameters.yaml

@@ -0,0 +1 @@
+{}