4gws test

Signed-off-by: Alexander Indenbaum <[email protected]>

.github/workflows/build-container.yml

@@ -535,10 +535,10 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        test: ["sanity", "state_transitions", "state_transitions_both_gws", "state_transitions_loop", "state_transitions_rand_loop", "late_registration", "late_registration_loop"]
+        test: ["sanity", "state_transitions", "state_transitions_both_gws", "state_transitions_loop", "state_transitions_rand_loop", "late_registration", "late_registration_loop", "4gws"]
     runs-on: ubuntu-latest
     env:
-      HUGEPAGES: 768  # 3 spdk instances
+      HUGEPAGES: 1024  # 4 spdk instances
 
     steps:
       - name: Checkout code
@@ -568,12 +568,22 @@ jobs:
             sudo apt install -y docker-compose
           fi
           docker-compose --version
-          ./tests/ha/start_up.sh
+          test_specific_start_up="./tests/ha/start_up_${{ matrix.test }}.sh"
+          if [ -x "$test_specific_start_up" ]; then
+            $test_specific_start_up
+          else
+            ./tests/ha/start_up.sh
+          fi
 
       - name: Wait for gateways to be listening
         timeout-minutes: 3
         run: |
-          source tests/ha/wait_gateways.sh
+          test_specific_wait="./tests/ha/wait_gateways_${{ matrix.test }}.sh"
+          if [ -x "$test_specific_wait" ]; then
+            $test_specific_wait
+          else
+            ./tests/ha/wait_gateways.sh
+          fi
 
       - name: List containers
         if: success() || failure()
@@ -587,7 +597,12 @@ jobs:
 
       - name: Set up target
         run: |
-          source tests/ha/setup.sh
+          test_specific_setup="./tests/ha/setup_${{ matrix.test }}.sh"
+          if [ -x "$test_specific_setup" ]; then
+            $test_specific_setup
+          else
+            ./tests/ha/setup.sh
+          fi
 
       - name: Run HA ${{ matrix.test }} test
         timeout-minutes: 30

tests/ha/4gws.sh

@@ -0,0 +1,178 @@
+set -xe
+rpc=/usr/libexec/spdk/scripts/rpc.py
+cmd=nvmf_subsystem_get_listeners
+
+expect_optimized() {
+  GW_NAME=$1
+  EXPECTED_OPTIMIZED=$2
+  NQN=$3
+
+  socket=$(docker exec "$GW_NAME" find /var/run/ceph -name spdk.sock)
+  # Verify expected number of "optimized"
+  for i in $(seq 10); do
+    response=$(docker exec "$GW_NAME" "$rpc" "-s" "$socket" "$cmd" "$NQN")
+    ana_states=$(echo "$response" | jq -r '.[0].ana_states')
+
+    # Count the number of "optimized" groups
+    optimized_count=$(jq -nr --argjson ana_states "$ana_states" '$ana_states | map(select(.ana_state == "optimized")) | length')
+
+    # Check if there is expected number of "optimized" group
+    if [ "$optimized_count" -eq "$EXPECTED_OPTIMIZED" ]; then
+      # Iterate through JSON array
+      for item in $(echo "$ana_states" | jq -c '.[]'); do
+        ana_group=$(echo "$item" | jq -r '.ana_group')
+        ana_state=$(echo "$item" | jq -r '.ana_state')
+
+        # Check if ana_state is "optimized"
+        if [ "$ana_state" = "optimized" ]; then
+          echo "$ana_group"
+        fi
+      done
+      return
+    else
+      sleep 1
+      continue
+    fi
+  done
+  echo "‼️  expect_optimized timeout GW_NAME=$1 EXPECTED_OPTIMIZED=$2 NQN=$3"
+  exit 1 # failed
+}
+
+# GW name by index
+gw_name() {
+  i=$1
+  docker ps --format '{{.ID}}\t{{.Names}}' | awk '$2 ~ /nvmeof/ && $2 ~ /'$i'/ {print $1}'
+}
+
+# Function to access numbers by index
+access_number_by_index() {
+    numbers=$1
+    index=$(expr $2 + 1)
+    number=$(echo "$numbers" | awk -v idx="$index" 'NR == idx {print}')
+    echo "$number"
+}
+
+# verify that given numbers must be either 1 and 2 or 2 and 1
+verify_ana_groups() {
+    nr1=$1
+    nr2=$2
+
+    if [ "$nr1" -eq 1 ] && [ "$nr2" -eq 2 ]; then
+        echo "Verified: first is 1 and second is 2"
+    elif [ "$nr1" -eq 2 ] && [ "$nr2" -eq 1 ]; then
+        echo "Verified: first is 2 and second is 1"
+    else
+        echo "Invalid numbers: first and second must be either 1 and 2 or 2 and 1"
+        exit 1
+    fi
+}
+
+# Function to choose n random number at 1..m range
+choose_n_m() {
+    n=$1
+    m=$2
+    count=0
+    numbers=""
+
+    # Ensure m is greater than 1 to avoid division by zero errors
+    if [ "$m" -le 1 ]; then
+        echo "Upper limit m must be greater than 1."
+        exit 1
+    fi
+
+    while [ "$count" -lt "$n" ]; do
+        # Generate a random number between 1 and m
+        random_number=$(expr $RANDOM % $m + 1)
+
+        # Check if the number is unique
+        is_unique=$(echo "$numbers" | grep -c "\<$random_number\>")
+        if [ "$is_unique" -eq 0 ]; then
+            # Add the unique number to the list
+            numbers="$numbers $random_number"
+            echo $random_number
+            count=$(expr $count + 1)
+        fi
+    done
+}
+
+validate_all_active() {
+  for s in $(seq $NUM_SUBSYSTEMS); do
+    all_ana_states=$(for g in $(seq $NUM_GATEWAYS); do
+                       NQN="nqn.2016-06.io.spdk:cnode$s"
+                       GW_OPTIMIZED=$(expect_optimized "$(gw_name $g)" 1 "$NQN")
+                       gw_ana=$(access_number_by_index "$GW_OPTIMIZED" 0)
+                       echo $gw_ana
+                     done)
+
+    if [ "$(echo "$all_ana_states" | sort -n)" != "$(seq $NUM_GATEWAYS)" ]; then
+      echo "all active state failure"
+      exit 1
+    fi
+  done
+}
+
+
+#
+# MAIN
+#
+
+NUM_SUBSYSTEMS=2
+NUM_GATEWAYS=4
+FAILING_GATEWAYS=2
+#
+# Step 1 validate all gateways are optimized for one of ANA group
+# and all groups are unique
+#
+
+echo "ℹ️ Step 1"
+validate_all_active
+
+#
+# Step 2 failover
+#
+
+echo "ℹ️ Step 2"
+gws_to_stop=$(choose_n_m $FAILING_GATEWAYS $NUM_GATEWAYS)
+for i in $(seq 0 $(expr $FAILING_GATEWAYS - 1)); do
+  gw=$(access_number_by_index "$gws_to_stop" $i)
+  gw_name=$(gw_name $gw)
+  echo "ℹ️ Stop gw $gw_name i=$i gw=$gw"
+  docker stop $gw_name
+done
+
+docker ps
+
+# expect remaining gws to have two optimized groups each
+for i in $(seq 4); do
+  found=0
+  for j in $(seq 0 $(expr $FAILING_GATEWAYS - 1)); do
+    if [ "$i" -eq "$(access_number_by_index "$gws_to_stop" $j)"]; then
+      found=1
+      break
+    fi
+  done
+
+  # if gw is a healthy one
+  if [ "$found" -eq "0" ]; then
+     echo "ℹ️ Check healthy gw gw=$i"
+     for s in $(seq $NUM_SUBSYSTEMS); do
+       NQN="nqn.2016-06.io.spdk:cnode$s"
+       GW_OPTIMIZED=$(expect_optimized "$(gw_name $i)" 2 "$NQN")
+     done
+  fi
+done
+
+#
+# Step 3 failback
+#
+echo "ℹ️ Step 3"
+for i in $(seq 0 $(expr $FAILING_GATEWAYS - 1)); do
+  gw=$(access_number_by_index "$gws_to_stop" $i)
+  gw_name=$(gw_name $gw)
+  echo "ℹ️ Start gw $gw_name i=$i gw=$gw"
+  docker start $gw_name
+done
+
+docker ps
+
+validate_all_active

tests/ha/setup_4gws.sh

@@ -0,0 +1,72 @@
+set -xe
+
+# GW name by index
+gw_name() {
+  i=$1
+  docker ps --format '{{.ID}}\t{{.Names}}' | awk '$2 ~ /nvmeof/ && $2 ~ /'$i'/ {print $1}'
+}
+
+gw_ip() {
+  docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$(gw_name $1)"
+}
+
+num_subs() {
+  # Count the number of elements in the outer list
+  echo "$1" | jq length
+}
+
+num_nss() {
+  echo "$1" | jq --argjson index "$2" '.[$2].namespaces | length'
+}
+
+num_listeners() {
+  echo "$subs" | jq --argjson index "$s" '.[$s].listen_addresses | length'
+}
+
+#
+# MAIN
+#
+NUM_SUBSYSTEMS=2
+NUM_GATEWAYS=4
+NUM_NAMESPACES=32
+
+# Setup
+for i in $(seq $NUM_SUBSYSTEMS); do
+    NQN="nqn.2016-06.io.spdk:cnode$i"
+    docker-compose  run --rm nvmeof-cli --server-address $(gw_ip 1) --server-port 5500 subsystem add --subsystem  $NQN
+    for n in $(seq $NUM_NAMESPACES); do
+       IMAGE="image_${i}_${n}"
+       L=$(expr $n % $NUM_GATEWAYS + 1)
+       docker-compose  run --rm nvmeof-cli --server-address $(gw_ip 1) --server-port 5500 namespace add --subsystem $NQN --rbd-pool rbd --rbd-image $IMAGE --size 10M --rbd-create-image -l $L
+    done
+    for g in $(seq $NUM_GATEWAYS); do
+        GW_NAME=$(gw_name $g)
+        GW_IP=$(gw_ip $g)
+        PORT=4420
+        docker-compose  run --rm nvmeof-cli --server-address $GW_IP --server-port 5500 listener add  --subsystem $NQN --host-name $GW_NAME --traddr $GW_IP --trsvcid $PORT
+    done
+done
+
+# Verify
+for g in $(seq $NUM_GATEWAYS); do
+  for i in $(seq 10); do
+    echo "Verify $i $GW_NAME $GW_IP"
+    GW_NAME=$(gw_name $g)
+    GW_IP=$(gw_ip $g)
+    PORT=4420
+    subs=$(docker-compose  run --rm nvmeof-cli --server-address $GW_IP --server-port 5500 get_subsystems)
+
+    # verify all resources found in get subsystems
+    if [ "$(num_subs "$subs")" -ne $NUM_SUBSYSTEMS -o \
+         "$(num_nss "$subs" 0)" -ne $NUM_NAMESPACES -o \
+         "$(num_nss "$subs" 1)" -ne $NUM_NAMESPACES -o \
+         "$(num_listeners "$subs" 1)" -ne 1 -o \
+         "$(num_listeners "$subs" 1)" -ne 1 ]; then
+
+        echo "Not ready $i $GW_NAME $GW_IP"
+	sleep 5
+        continue
+    fi
+    echo "Ready $i $GW_NAME $GW_IP"
+  done
+done

tests/ha/start_up_4gws.sh

@@ -0,0 +1,8 @@
+# Check if GITHUB_WORKSPACE is defined
+if [ -n "$GITHUB_WORKSPACE" ]; then
+    test_dir="$GITHUB_WORKSPACE/tests/ha"
+else
+    test_dir=$(dirname $0)
+fi
+
+$test_dir/start_up.sh 4

tests/ha/wait_gateways.sh

@@ -1,4 +1,16 @@
-for i in $(seq 2); do
+SCALE=2
+# Check if argument is provided
+if [ $# -ge 1 ]; then
+    # Check if argument is an integer larger or equal than 1
+    if [ "$1" -eq "$1" ] 2>/dev/null && [ "$1" -ge 1 ]; then
+        # Set variable to the provided argument
+        SCALE="$1"
+    else
+        echo "Error: Argument must be an integer larger than 1." >&2
+        exit 1
+    fi
+fi
+for i in $(seq $SCALE); do
   while true; do
     sleep 1  # Adjust the sleep duration as needed
     GW_NAME=$(docker ps --format '{{.ID}}\t{{.Names}}' | awk '$2 ~ /nvmeof/ && $2 ~ /'$i'/ {print $1}')

tests/ha/wait_gateways_4gws.sh

@@ -0,0 +1,8 @@
+# Check if GITHUB_WORKSPACE is defined
+if [ -n "$GITHUB_WORKSPACE" ]; then
+    test_dir="$GITHUB_WORKSPACE/tests/ha"
+else
+    test_dir=$(dirname $0)
+fi
+
+$test_dir/wait_gateways.sh 4