diff --git a/.github/workflows/canary-integration-test.yml b/.github/workflows/canary-integration-test.yml index 77705687ff96..898fdbd5d16d 100644 --- a/.github/workflows/canary-integration-test.yml +++ b/.github/workflows/canary-integration-test.yml @@ -1549,8 +1549,8 @@ jobs: run: | tests/scripts/github-action-helper.sh deploy_first_rook_cluster kubectl create -f deploy/examples/object-multisite-test.yaml - # wait for multisite store to be created - tests/scripts/github-action-helper.sh wait_for_rgw rook-ceph + # wait for multisite-store to be created + tests/scripts/github-action-helper.sh wait_for cephobjectstore multisite-store rook-ceph 480 - name: prep second cluster pull realm config shell: bash --noprofile --norc -eo pipefail -x {0} @@ -1569,13 +1569,13 @@ jobs: tests/scripts/github-action-helper.sh deploy_second_rook_cluster kubectl create -f deploy/examples/object-multisite-pull-realm-test.yaml # wait for realms to be pulled and zone-b-multisite-store to be created - tests/scripts/github-action-helper.sh wait_for_rgw rook-ceph-secondary + tests/scripts/github-action-helper.sh wait_for cephobjectstore zone-b-multisite-store rook-ceph-secondary 480 - - name: wait for ceph cluster 1 to be ready + - name: wait for both ceph clusters to be ready shell: bash --noprofile --norc -eo pipefail -x {0} run: | - tests/scripts/validate_cluster.sh osd 1 - kubectl -n rook-ceph get pods + tests/scripts/github-action-helper.sh wait_for cephcluster my-cluster rook-ceph + tests/scripts/github-action-helper.sh wait_for cephcluster my-cluster rook-ceph-secondary - name: write an object to one cluster, read from the other shell: bash --noprofile --norc -eo pipefail -x {0} diff --git a/tests/scripts/github-action-helper.sh b/tests/scripts/github-action-helper.sh index 26a2b59e0646..fac62a6fa78d 100755 --- a/tests/scripts/github-action-helper.sh +++ b/tests/scripts/github-action-helper.sh @@ -476,23 +476,28 @@ function deploy_second_rook_cluster() { deploy_toolbox } -function wait_for_rgw() { - for _ in {1..120}; do - if [ "$(kubectl -n "$1" get pod -l app=rook-ceph-rgw --no-headers --field-selector=status.phase=Running | wc -l)" -ge 1 ]; then - echo "rgw pod is found" - break +function wait_for() { + local kind=${1?kind is required} + local name=${2?resource name is required} + local ns=${3:-rook-ceph} + local timeout=${4:-120} + local status=${5:-Ready} + + local start_time="${SECONDS}" + local elapsed_time=0 + while [[ $elapsed_time -lt $timeout ]]; do + if [[ "$(kubectl -n "$ns" get "$kind" "$name" -o 'jsonpath={..status.phase}')" == "$status" ]]; then + echo "${kind}/${name} in ${ns} is ${status} - elapsed time ${elapsed_time}s" + return 0 fi - echo "waiting for rgw pods" - sleep 5 - done - for _ in {1..120}; do - if [ "$(kubectl -n "$1" get deployment -l app=rook-ceph-rgw -o yaml | yq read - 'items[0].status.readyReplicas')" -ge 1 ]; then - echo "rgw is ready" - break - fi - echo "waiting for rgw becomes ready" + + elapsed_time=$((SECONDS - start_time)) + echo "waiting for ${kind}/${name} in ${ns} to be ${status} - elapsed time ${elapsed_time}s" sleep 5 done + + echo "timed out waiting for ${kind}/${name} in ${ns} to be ${status} - elapsed time ${elapsed_time}s " >&2 + exit 1 } function verify_operator_log_message() { @@ -561,7 +566,7 @@ function write_object_read_from_replica_cluster() { # a direct sub-shell. S3CMD_ERROR=0 ( - sleep 60 + sleep 300 kill -s SIGUSR1 $$ ) 2>/dev/null & trap "{ S3CMD_ERROR=1; break; }" SIGUSR1