diff --git a/.github/workflows/build-container.yml b/.github/workflows/build-container.yml index d7d807ac6..92cb88334 100644 --- a/.github/workflows/build-container.yml +++ b/.github/workflows/build-container.yml @@ -535,10 +535,10 @@ jobs: strategy: fail-fast: false matrix: - test: ["sanity", "state_transitions", "state_transitions_both_gws", "state_transitions_loop", "state_transitions_rand_loop", "late_registration", "late_registration_loop"] + test: ["sanity", "state_transitions", "state_transitions_both_gws", "state_transitions_loop", "state_transitions_rand_loop", "late_registration", "late_registration_loop", "4gws"] runs-on: ubuntu-latest env: - HUGEPAGES: 768 # 3 spdk instances + HUGEPAGES: 1024 # 4 spdk instances steps: - name: Checkout code @@ -568,12 +568,22 @@ jobs: sudo apt install -y docker-compose fi docker-compose --version - ./tests/ha/start_up.sh + test_specific_start_up="./tests/ha/start_up_${{ matrix.test }}.sh" + if [ -x "$test_specific_start_up" ]; then + $test_specific_start_up + else + ./tests/ha/start_up.sh + fi - name: Wait for gateways to be listening timeout-minutes: 3 run: | - source tests/ha/wait_gateways.sh + test_specific_wait="./tests/ha/wait_gateways_${{ matrix.test }}.sh" + if [ -x "$test_specific_wait" ]; then + $test_specific_wait + else + ./tests/ha/wait_gateways.sh + fi - name: List containers if: success() || failure() @@ -587,7 +597,12 @@ jobs: - name: Set up target run: | - source tests/ha/setup.sh + test_specific_setup="./tests/ha/setup_${{ matrix.test }}.sh" + if [ -x "$test_specific_setup" ]; then + $test_specific_setup + else + ./tests/ha/setup.sh + fi - name: Run HA ${{ matrix.test }} test timeout-minutes: 30 diff --git a/tests/ha/4gws.sh b/tests/ha/4gws.sh new file mode 100755 index 000000000..e5be7e12c --- /dev/null +++ b/tests/ha/4gws.sh @@ -0,0 +1,172 @@ +set -xe +rpc=/usr/libexec/spdk/scripts/rpc.py +cmd=nvmf_subsystem_get_listeners + +expect_optimized() { + GW_NAME=$1 + EXPECTED_OPTIMIZED=$2 + NQN=$3 + + socket=$(docker exec "$GW_NAME" find /var/run/ceph -name spdk.sock) + # Verify expected number of "optimized" + while true; do + response=$(docker exec "$GW_NAME" "$rpc" "-s" "$socket" "$cmd" "$NQN") + ana_states=$(echo "$response" | jq -r '.[0].ana_states') + + # Count the number of "optimized" groups + optimized_count=$(jq -nr --argjson ana_states "$ana_states" '$ana_states | map(select(.ana_state == "optimized")) | length') + + # Check if there is expected number of "optimized" group + if [ "$optimized_count" -eq "$EXPECTED_OPTIMIZED" ]; then + # Iterate through JSON array + for item in $(echo "$ana_states" | jq -c '.[]'); do + ana_group=$(echo "$item" | jq -r '.ana_group') + ana_state=$(echo "$item" | jq -r '.ana_state') + + # Check if ana_state is "optimized" + if [ "$ana_state" = "optimized" ]; then + echo "$ana_group" + fi + done + break + else + sleep 1 + continue + fi + done +} + +# GW name by index +gw_name() { + i=$1 + docker ps --format '{{.ID}}\t{{.Names}}' | awk '$2 ~ /nvmeof/ && $2 ~ /'$i'/ {print $1}' +} + +# Function to access numbers by index +access_number_by_index() { + numbers=$1 + index=$(expr $2 + 1) + number=$(echo "$numbers" | awk -v idx="$index" 'NR == idx {print}') + echo "$number" +} + +# verify that given numbers must be either 1 and 2 or 2 and 1 +verify_ana_groups() { + nr1=$1 + nr2=$2 + + if [ "$nr1" -eq 1 ] && [ "$nr2" -eq 2 ]; then + echo "Verified: first is 1 and second is 2" + elif [ "$nr1" -eq 2 ] && [ "$nr2" -eq 1 ]; then + echo "Verified: first is 2 and second is 1" + else + echo "Invalid numbers: first and second must be either 1 and 2 or 2 and 1" + exit 1 + fi +} + +# Function to choose n random number at 1..m range +choose_n_m() { + n=$1 + m=$2 + count=0 + numbers="" + + # Ensure m is greater than 1 to avoid division by zero errors + if [ "$m" -le 1 ]; then + echo "Upper limit m must be greater than 1." + exit 1 + fi + + while [ "$count" -lt "$n" ]; do + # Generate a random number between 1 and m + random_number=$(expr $RANDOM % $m + 1) + + # Check if the number is unique + is_unique=$(echo "$numbers" | grep -c "\<$random_number\>") + if [ "$is_unique" -eq 0 ]; then + # Add the unique number to the list + numbers="$numbers $random_number" + echo $random_number + count=$(expr $count + 1) + fi + done +} + +validate_all_active() { + for s in $(seq $NUM_SUBSYSTEMS); do + NQN="nqn.2016-06.io.spdk:cnode$s" + all_ana_states=$(for g in $(seq $NUM_GATEWAYS); do + GW_OPTIMIZED=$(expect_optimized $(gw_name $g) 1 $NQN) + gw_ana=$(access_number_by_index "$GW_OPTIMIZED" 0) + echo $gw_ana + done) + + if [ "$(echo "$all_ana_states" | sort -n)" != "$(seq $NUM_GATEWAYS)" ]; then + echo "all active state failure" + exit 1 + fi + done +} + + +# +# MAIN +# + +NUM_SUBSYSTEMS=2 +NUM_GATEWAYS=4 +FAILING_GATEWAYS=2 +# +# Step 1 validate all gateways are optimized for one of ANA group +# and all groups are unique +# + +validate_all_active + +# +# Step 2 failover +# + +gws_to_stop=$(choose_n_m $FAILING_GATEWAYS $NUM_GATEWAYS) +for i in $(seq 0 $(expr $FAILING_GATEWAYS - 1)); do + gw=$(access_number_by_index "$gws_to_stop" $i) + gw_name=$(gw_name $gw) + echo "Stop gw $gw_name" + docker stop $gw_name +done + +docker ps + +# expect remaining gws to have two optimized groups each +for i in $(seq 4); do + found=0 + for j in $(seq 0 $(expr $FAILING_GATEWAYS - 1)); do + if [ "$i" -eq "$(access_number_by_index "$gws_to_stop" $j)"]; then + found=1 + break + fi + done + + # if gw is a healthy one + if [ "$found" -eq "0" ]; then + for s in $(seq $NUM_SUBSYSTEMS); do + NQN="nqn.2016-06.io.spdk:cnode$s" + GW_OPTIMIZED=$(expect_optimized $(gw_name $i) 2 $NQN) + done + fi +done + +# +# Step 3 failback +# +for i in $(seq 0 $(expr $FAILING_GATEWAYS - 1)); do + gw=$(access_number_by_index "$gws_to_stop" $i) + gw_name=$(gw_name $gw) + echo "Start gw $gw_name" + docker start $gw_name +done + +docker ps + +validate_all_active diff --git a/tests/ha/setup_4gws.sh b/tests/ha/setup_4gws.sh new file mode 100755 index 000000000..a8bf515b9 --- /dev/null +++ b/tests/ha/setup_4gws.sh @@ -0,0 +1,35 @@ +set -xe + +# GW name by index +gw_name() { + i=$1 + docker ps --format '{{.ID}}\t{{.Names}}' | awk '$2 ~ /nvmeof/ && $2 ~ /'$i'/ {print $1}' +} + +gw_ip() { + docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$(gw_name $1)" +} + +# +# MAIN +# +NUM_SUBSYSTEMS=1 +NUM_GATEWAYS=4 +NUM_NAMESPACES=32 + +for i in $(seq $NUM_SUBSYSTEMS); do + NQN="nqn.2016-06.io.spdk:cnode$i" + docker-compose run --rm nvmeof-cli --server-address $(gw_ip 1) --server-port 5500 subsystem add --subsystem $NQN + for n in $(seq $NUM_NAMESPACES); do + IMAGE="image$n" + L=$(expr $n % $NUM_GATEWAYS + 1) + docker-compose run --rm nvmeof-cli --server-address $(gw_ip 1) --server-port 5500 namespace add --subsystem $NQN --rbd-pool rbd --rbd-image $IMAGE --size 10M --rbd-create-image -l $L + done + for g in $(seq $NUM_GATEWAYS); do + GW_NAME=$(gw_name $g) + GW_IP=$(gw_ip $g) + PORT=4420 + docker-compose run --rm nvmeof-cli --server-address $GW_IP --server-port 5500 listener add --subsystem $NQN --host-name $GW_NAME --traddr $GW_IP --trsvcid $PORT + done +done + diff --git a/tests/ha/start_up_4gws.sh b/tests/ha/start_up_4gws.sh new file mode 100755 index 000000000..725e1ff34 --- /dev/null +++ b/tests/ha/start_up_4gws.sh @@ -0,0 +1,8 @@ +# Check if GITHUB_WORKSPACE is defined +if [ -n "$GITHUB_WORKSPACE" ]; then + test_dir="$GITHUB_WORKSPACE/tests/ha" +else + test_dir=$(dirname $0) +fi + +$test_dir/start_up.sh 4 diff --git a/tests/ha/wait_gateways.sh b/tests/ha/wait_gateways.sh index 060065f86..b2ff94c80 100755 --- a/tests/ha/wait_gateways.sh +++ b/tests/ha/wait_gateways.sh @@ -1,4 +1,16 @@ -for i in $(seq 2); do +SCALE=2 +# Check if argument is provided +if [ $# -ge 1 ]; then + # Check if argument is an integer larger or equal than 1 + if [ "$1" -eq "$1" ] 2>/dev/null && [ "$1" -ge 1 ]; then + # Set variable to the provided argument + SCALE="$1" + else + echo "Error: Argument must be an integer larger than 1." >&2 + exit 1 + fi +fi +for i in $(seq $SCALE); do while true; do sleep 1 # Adjust the sleep duration as needed GW_NAME=$(docker ps --format '{{.ID}}\t{{.Names}}' | awk '$2 ~ /nvmeof/ && $2 ~ /'$i'/ {print $1}') diff --git a/tests/ha/wait_gateways_4gws.sh b/tests/ha/wait_gateways_4gws.sh new file mode 100755 index 000000000..5442ec90a --- /dev/null +++ b/tests/ha/wait_gateways_4gws.sh @@ -0,0 +1,8 @@ +# Check if GITHUB_WORKSPACE is defined +if [ -n "$GITHUB_WORKSPACE" ]; then + test_dir="$GITHUB_WORKSPACE/tests/ha" +else + test_dir=$(dirname $0) +fi + +$test_dir/wait_gateways.sh 4