From 5ed0a75fcdefdb0016c7998f33dff91dbc084095 Mon Sep 17 00:00:00 2001 From: Svyatoslav Nikolsky Date: Fri, 12 Jan 2024 16:49:16 +0300 Subject: [PATCH] Run bridges zombienet tests on CI (#2439) Brridges zombienet tests are non-standard - zombienet currently missing multiple relay chains support (see e.g. https://github.com/paritytech/zombienet/pull/796), so we need to go live with two relay networks, their parachains + custom test runner (which e.g. doesn't shutdown net when its tests are finished and instead waits for both networks tests to complete). So we are stuck with native zombienet provider => this PR is an attempt to gather everything in a single docker container and run tests there ~Draft, because it is far from finishing - what I want now is to see how it works on CI~ --- .gitlab/pipeline/build.yml | 18 ++++++ .gitlab/pipeline/publish.yml | 22 +++++++ .gitlab/pipeline/zombienet.yml | 2 + .gitlab/pipeline/zombienet/bridges.yml | 54 +++++++++++++++++ bridges/zombienet/run-tests.sh | 50 +++++++++++++--- cumulus/scripts/bridges_common.sh | 24 +++----- cumulus/scripts/bridges_rococo_westend.sh | 12 ++-- ...ridges_zombienet_tests_injected.Dockerfile | 60 +++++++++++++++++++ 8 files changed, 214 insertions(+), 28 deletions(-) create mode 100644 .gitlab/pipeline/zombienet/bridges.yml create mode 100644 docker/dockerfiles/bridges_zombienet_tests_injected.Dockerfile diff --git a/.gitlab/pipeline/build.yml b/.gitlab/pipeline/build.yml index 20aa4a5c2a28..d998b62c8993 100644 --- a/.gitlab/pipeline/build.yml +++ b/.gitlab/pipeline/build.yml @@ -382,3 +382,21 @@ build-subkey-linux: # after_script: [""] # tags: # - osx + +# bridges + +# we need some non-binary artifacts in our bridges+zombienet image +prepare-bridges-zombienet-artifacts: + stage: build + extends: + - .docker-env + - .common-refs + - .run-immediately + - .collect-artifacts + before_script: + - mkdir -p ./artifacts/bridges-polkadot-sdk/bridges + - mkdir -p ./artifacts/bridges-polkadot-sdk/cumulus/zombienet + script: + - cp -r bridges/zombienet ./artifacts/bridges-polkadot-sdk/bridges/zombienet + - cp -r cumulus/scripts ./artifacts/bridges-polkadot-sdk/cumulus/scripts + - cp -r cumulus/zombienet/bridge-hubs ./artifacts/bridges-polkadot-sdk/cumulus/zombienet/bridge-hubs diff --git a/.gitlab/pipeline/publish.yml b/.gitlab/pipeline/publish.yml index 3b77f5363f62..b73acb560f67 100644 --- a/.gitlab/pipeline/publish.yml +++ b/.gitlab/pipeline/publish.yml @@ -66,6 +66,8 @@ publish-rustdoc: # note: images are used not only in zombienet but also in rococo, wococo and versi .build-push-image: image: $BUILDAH_IMAGE + extends: + - .zombienet-refs variables: DOCKERFILE: "" # docker/path-to.Dockerfile IMAGE_NAME: "" # docker.io/paritypr/image_name @@ -77,6 +79,7 @@ publish-rustdoc: --build-arg VCS_REF="${CI_COMMIT_SHA}" --build-arg BUILD_DATE="$(date -u '+%Y-%m-%dT%H:%M:%SZ')" --build-arg IMAGE_NAME="${IMAGE_NAME}" + --build-arg ZOMBIENET_IMAGE="${ZOMBIENET_IMAGE}" --tag "$IMAGE_NAME:${DOCKER_IMAGES_VERSION}" --file ${DOCKERFILE} . - echo "$PARITYPR_PASS" | @@ -163,3 +166,22 @@ build-push-image-substrate-pr: variables: DOCKERFILE: "docker/dockerfiles/substrate_injected.Dockerfile" IMAGE_NAME: "docker.io/paritypr/substrate" + +# unlike other images, bridges+zombienet image is based on Zombienet image that pulls required binaries +# from other fresh images (polkadot and cumulus) +build-push-image-bridges-zombienet-tests: + stage: publish + extends: + - .kubernetes-env + - .common-refs + - .build-push-image + needs: + - job: build-linux-stable + artifacts: true + - job: build-linux-stable-cumulus + artifacts: true + - job: prepare-bridges-zombienet-artifacts + artifacts: true + variables: + DOCKERFILE: "docker/dockerfiles/bridges_zombienet_tests_injected.Dockerfile" + IMAGE_NAME: "docker.io/paritypr/bridges-zombienet-tests" diff --git a/.gitlab/pipeline/zombienet.yml b/.gitlab/pipeline/zombienet.yml index d5845611c60d..4b95c86c3c41 100644 --- a/.gitlab/pipeline/zombienet.yml +++ b/.gitlab/pipeline/zombienet.yml @@ -10,3 +10,5 @@ include: - .gitlab/pipeline/zombienet/cumulus.yml # polkadot tests - .gitlab/pipeline/zombienet/polkadot.yml + # bridges tests + - .gitlab/pipeline/zombienet/bridges.yml diff --git a/.gitlab/pipeline/zombienet/bridges.yml b/.gitlab/pipeline/zombienet/bridges.yml new file mode 100644 index 000000000000..9414207a3bbf --- /dev/null +++ b/.gitlab/pipeline/zombienet/bridges.yml @@ -0,0 +1,54 @@ +# This file is part of .gitlab-ci.yml +# Here are all jobs that are executed during "zombienet" stage for bridges + +# common settings for all zombienet jobs +.zombienet-bridges-common: + before_script: + # Exit if the job is not merge queue + # - if [[ $CI_COMMIT_REF_NAME != *"gh-readonly-queue"* ]]; then echo "I will run only in a merge queue"; exit 0; fi + - echo "Zombienet Tests Config" + - echo "${ZOMBIENET_IMAGE}" + - echo "${GH_DIR}" + - echo "${LOCAL_DIR}" + - ls "${LOCAL_DIR}" + - export DEBUG=zombie,zombie::network-node + - export ZOMBIENET_INTEGRATION_TEST_IMAGE="${BRIDGES_ZOMBIENET_TESTS_IMAGE}":${BRIDGES_ZOMBIENET_TESTS_IMAGE_TAG} + - echo "${ZOMBIENET_INTEGRATION_TEST_IMAGE}" + stage: zombienet + image: "${BRIDGES_ZOMBIENET_TESTS_IMAGE}:${BRIDGES_ZOMBIENET_TESTS_IMAGE_TAG}" + needs: + - job: build-push-image-bridges-zombienet-tests + artifacts: true + extends: + - .kubernetes-env + - .zombienet-refs + variables: + BRIDGES_ZOMBIENET_TESTS_IMAGE_TAG: ${DOCKER_IMAGES_VERSION} + BRIDGES_ZOMBIENET_TESTS_IMAGE: "docker.io/paritypr/bridges-zombienet-tests" + GH_DIR: "https://github.com/paritytech/polkadot-sdk/tree/${CI_COMMIT_SHA}/bridges/zombienet" + LOCAL_DIR: "/builds/parity/mirrors/polkadot-sdk/bridges/zombienet" + FF_DISABLE_UMASK_FOR_DOCKER_EXECUTOR: 1 + RUN_IN_CONTAINER: "1" + artifacts: + name: "${CI_JOB_NAME}_${CI_COMMIT_REF_NAME}_zombienet_bridge_tests" + when: always + expire_in: 2 days + paths: + - ./zombienet-logs + after_script: + - mkdir -p ./zombienet-logs + # copy logs of tests runner (run-tests.sh) + - cp -r /tmp/bridges-zombienet-tests.*/tmp.*/tmp.* ./zombienet-logs/ + # copy logs of all nodes + - cp /tmp/zombie*/logs/* ./zombienet-logs/ +# following lines are causing spurious test failures ("At least one of the nodes fails to start") +# retry: 2 +# tags: +# - zombienet-polkadot-integration-test + +zombienet-bridges-0001-asset-transfer-works: + extends: + - .zombienet-bridges-common + script: + - /home/nonroot/bridges-polkadot-sdk/bridges/zombienet/run-tests.sh --docker + - echo "Done" diff --git a/bridges/zombienet/run-tests.sh b/bridges/zombienet/run-tests.sh index 4f80e06650ee..34487e13261f 100755 --- a/bridges/zombienet/run-tests.sh +++ b/bridges/zombienet/run-tests.sh @@ -1,18 +1,49 @@ #!/bin/bash #set -eu +set -x shopt -s nullglob -trap "trap - SIGTERM && kill -- -$$" SIGINT SIGTERM EXIT +trap "trap - SIGINT SIGTERM EXIT && kill -- -$$" SIGINT SIGTERM EXIT + +# whether to use paths for zombienet+bridges tests container or for local testing +ZOMBIENET_DOCKER_PATHS=0 +while [ $# -ne 0 ] +do + arg="$1" + case "$arg" in + --docker) + ZOMBIENET_DOCKER_PATHS=1 + ;; + esac + shift +done # assuming that we'll be using native provide && all processes will be executing locally # (we need absolute paths here, because they're used when scripts are called by zombienet from tmp folders) export POLKADOT_SDK_FOLDER=`realpath $(dirname "$0")/../..` export BRIDGE_TESTS_FOLDER=$POLKADOT_SDK_FOLDER/bridges/zombienet/tests -export POLKADOT_BINARY_PATH=$POLKADOT_SDK_FOLDER/target/release/polkadot -export POLKADOT_PARACHAIN_BINARY_PATH=$POLKADOT_SDK_FOLDER/target/release/polkadot-parachain -export POLKADOT_PARACHAIN_BINARY_PATH_FOR_ASSET_HUB_ROCOCO=$POLKADOT_PARACHAIN_BINARY_PATH -export POLKADOT_PARACHAIN_BINARY_PATH_FOR_ASSET_HUB_WESTEND=$POLKADOT_PARACHAIN_BINARY_PATH -export ZOMBIENET_BINARY_PATH=~/local_bridge_testing/bin/zombienet-linux + +# set pathc to binaries +if [ "$ZOMBIENET_DOCKER_PATHS" -eq 1 ]; then + export POLKADOT_BINARY_PATH=/usr/local/bin/polkadot + export POLKADOT_PARACHAIN_BINARY_PATH=/usr/local/bin/polkadot-parachain + export POLKADOT_PARACHAIN_BINARY_PATH_FOR_ASSET_HUB_ROCOCO=/usr/local/bin/polkadot-parachain + export POLKADOT_PARACHAIN_BINARY_PATH_FOR_ASSET_HUB_WESTEND=/usr/local/bin/polkadot-parachain + + export SUBSTRATE_RELAY_PATH=/usr/local/bin/substrate-relay + export ZOMBIENET_BINARY_PATH=/usr/local/bin/zombie +else + export POLKADOT_BINARY_PATH=$POLKADOT_SDK_FOLDER/target/release/polkadot + export POLKADOT_PARACHAIN_BINARY_PATH=$POLKADOT_SDK_FOLDER/target/release/polkadot-parachain + export POLKADOT_PARACHAIN_BINARY_PATH_FOR_ASSET_HUB_ROCOCO=$POLKADOT_PARACHAIN_BINARY_PATH + export POLKADOT_PARACHAIN_BINARY_PATH_FOR_ASSET_HUB_WESTEND=$POLKADOT_PARACHAIN_BINARY_PATH + + export SUBSTRATE_RELAY_PATH=~/local_bridge_testing/bin/substrate-relay + export ZOMBIENET_BINARY_PATH=~/local_bridge_testing/bin/zombienet-linux +fi + +# check if `wait` supports -p flag +if [ `printf "$BASH_VERSION\n5.1" | sort -V | head -n 1` = "5.1" ]; then IS_BASH_5_1=1; else IS_BASH_5_1=0; fi # check if `wait` supports -p flag if [ `printf "$BASH_VERSION\n5.1" | sort -V | head -n 1` = "5.1" ]; then IS_BASH_5_1=1; else IS_BASH_5_1=0; fi @@ -21,13 +52,17 @@ if [ `printf "$BASH_VERSION\n5.1" | sort -V | head -n 1` = "5.1" ]; then IS_BASH export LANE_ID="00000002" # tests configuration -ALL_TESTS_FOLDER=`mktemp -d` +ALL_TESTS_FOLDER=`mktemp -d /tmp/bridges-zombienet-tests.XXXXX` function start_coproc() { local command=$1 local name=$2 local coproc_log=`mktemp -p $TEST_FOLDER` coproc COPROC { + # otherwise zombienet uses some hardcoded paths + unset RUN_IN_CONTAINER + unset ZOMBIENET_IMAGE + $command >$coproc_log 2>&1 } TEST_COPROCS[$COPROC_PID, 0]=$name @@ -90,6 +125,7 @@ do echo "=== Shutting down. Log of failed process below ===" echo "=====================================================================" echo $coproc_stdout + exit 1 fi diff --git a/cumulus/scripts/bridges_common.sh b/cumulus/scripts/bridges_common.sh index 97ef8aa12595..5d5b7b7a482a 100755 --- a/cumulus/scripts/bridges_common.sh +++ b/cumulus/scripts/bridges_common.sh @@ -1,27 +1,21 @@ #!/bin/bash -function ensure_binaries() { - if [[ ! -f ~/local_bridge_testing/bin/polkadot ]]; then - echo " Required polkadot binary '~/local_bridge_testing/bin/polkadot' does not exist!" - echo " You need to build it and copy to this location!" - echo " Please, check ./parachains/runtimes/bridge-hubs/README.md (Prepare/Build/Deploy)" - exit 1 - fi - if [[ ! -f ~/local_bridge_testing/bin/polkadot-parachain ]]; then - echo " Required polkadot-parachain binary '~/local_bridge_testing/bin/polkadot-parachain' does not exist!" - echo " You need to build it and copy to this location!" - echo " Please, check ./parachains/runtimes/bridge-hubs/README.md (Prepare/Build/Deploy)" - exit 1 - fi +function relayer_path() { + local default_path=~/local_bridge_testing/bin/substrate-relay + local path="${SUBSTRATE_RELAY_PATH:-$default_path}" + echo "$path" } function ensure_relayer() { - if [[ ! -f ~/local_bridge_testing/bin/substrate-relay ]]; then - echo " Required substrate-relay binary '~/local_bridge_testing/bin/substrate-relay' does not exist!" + local path=$(relayer_path) + if [[ ! -f "$path" ]]; then + echo " Required substrate-relay binary '$path' does not exist!" echo " You need to build it and copy to this location!" echo " Please, check ./parachains/runtimes/bridge-hubs/README.md (Prepare/Build/Deploy)" exit 1 fi + + echo $path } function ensure_polkadot_js_api() { diff --git a/cumulus/scripts/bridges_rococo_westend.sh b/cumulus/scripts/bridges_rococo_westend.sh index c52b72e51fc5..3b6f8e892858 100755 --- a/cumulus/scripts/bridges_rococo_westend.sh +++ b/cumulus/scripts/bridges_rococo_westend.sh @@ -132,10 +132,10 @@ LANE_ID="00000002" XCM_VERSION=3 function init_ro_wnd() { - ensure_relayer + local relayer_path=$(ensure_relayer) RUST_LOG=runtime=trace,rpc=trace,bridge=trace \ - ~/local_bridge_testing/bin/substrate-relay init-bridge rococo-to-bridge-hub-westend \ + $relayer_path init-bridge rococo-to-bridge-hub-westend \ --source-host localhost \ --source-port 9942 \ --source-version-mode Auto \ @@ -146,10 +146,10 @@ function init_ro_wnd() { } function init_wnd_ro() { - ensure_relayer + local relayer_path=$(ensure_relayer) RUST_LOG=runtime=trace,rpc=trace,bridge=trace \ - ~/local_bridge_testing/bin/substrate-relay init-bridge westend-to-bridge-hub-rococo \ + $relayer_path init-bridge westend-to-bridge-hub-rococo \ --source-host localhost \ --source-port 9945 \ --source-version-mode Auto \ @@ -160,10 +160,10 @@ function init_wnd_ro() { } function run_relay() { - ensure_relayer + local relayer_path=$(ensure_relayer) RUST_LOG=runtime=trace,rpc=trace,bridge=trace \ - ~/local_bridge_testing/bin/substrate-relay relay-headers-and-messages bridge-hub-rococo-bridge-hub-westend \ + $relayer_path relay-headers-and-messages bridge-hub-rococo-bridge-hub-westend \ --rococo-host localhost \ --rococo-port 9942 \ --rococo-version-mode Auto \ diff --git a/docker/dockerfiles/bridges_zombienet_tests_injected.Dockerfile b/docker/dockerfiles/bridges_zombienet_tests_injected.Dockerfile new file mode 100644 index 000000000000..ebad4046d268 --- /dev/null +++ b/docker/dockerfiles/bridges_zombienet_tests_injected.Dockerfile @@ -0,0 +1,60 @@ +# this image is built on top of existing Zombienet image +ARG ZOMBIENET_IMAGE +# this image uses substrate-relay image built elsewhere +ARG SUBSTRATE_RELAY_IMAGE=docker.io/paritytech/substrate-relay:v2023-11-07-rococo-westend-initial-relayer + +# metadata +ARG VCS_REF +ARG BUILD_DATE +ARG IMAGE_NAME + +# we need `substrate-relay` binary, built elsewhere +FROM ${SUBSTRATE_RELAY_IMAGE} as relay-builder + +# the base image is the zombienet image - we are planning to run zombienet tests using native +# provider here +FROM ${ZOMBIENET_IMAGE} + +LABEL io.parity.image.authors="devops-team@parity.io" \ + io.parity.image.vendor="Parity Technologies" \ + io.parity.image.title="${IMAGE_NAME}" \ + io.parity.image.description="Bridges Zombienet tests." \ + io.parity.image.source="https://github.com/paritytech/polkadot-sdk/blob/${VCS_REF}/docker/dockerfiles/bridges_zombienet_tests_injected.Dockerfile" \ + io.parity.image.revision="${VCS_REF}" \ + io.parity.image.created="${BUILD_DATE}" \ + io.parity.image.documentation="https://github.com/paritytech/polkadot-sdk/bridges/zombienet" + +# show backtraces +ENV RUST_BACKTRACE 1 +USER root + +# for native provider to work (TODO: fix in zn docker?) +RUN apt-get update && apt-get install -y procps sudo +RUN yarn global add @polkadot/api-cli + +# add polkadot binary to the docker image +COPY ./artifacts/polkadot /usr/local/bin/ +COPY ./artifacts/polkadot-execute-worker /usr/local/bin/ +COPY ./artifacts/polkadot-prepare-worker /usr/local/bin/ +# add polkadot-parachain binary to the docker image +COPY ./artifacts/polkadot-parachain /usr/local/bin +# copy substrate-relay to the docker image +COPY --from=relay-builder /home/user/substrate-relay /usr/local/bin/ +# we need bridges zombienet runner and tests +RUN mkdir -p /home/nonroot/bridges-polkadot-sdk +COPY ./artifacts/bridges-polkadot-sdk /home/nonroot/bridges-polkadot-sdk +# also prepare `generate_hex_encoded_call` for running +RUN set -eux; \ + cd /home/nonroot/bridges-polkadot-sdk/cumulus/scripts/generate_hex_encoded_call; \ + npm install + +# check if executable works in this container +USER nonroot +RUN /usr/local/bin/polkadot --version +RUN /usr/local/bin/polkadot-parachain --version +RUN /usr/local/bin/substrate-relay --version + +# https://polkadot.js.org/apps/?rpc=ws://127.0.0.1:{PORT}#/explorer +EXPOSE 9942 9910 8943 9945 9010 8945 + +ENTRYPOINT ["/bin/bash", "-c", "/home/nonroot/bridges-polkadot-sdk/bridges/zombienet/run-tests.sh"]