DAOS-4753 test: Initial tests for validating multiple ranks.

Test-tag: test_osa_online_drain_with_multiple_ranks test_osa_offline_drain_with_multiple_ranks
Skip-unit-tests: true

Summary:
- Add new OSA tests to validate multiple ranks support via dmg commands.
- Updates are done for drain and reintegration tests (offline/online)

Signed-off-by: rpadma2 <[email protected]>

src/tests/ftest/osa/offline_drain.yaml

@@ -1,107 +1,110 @@
-hosts:
-  test_servers: 3
-  test_clients: 1
-timeout: 2400
-setup:
-  start_servers_once: false
-skip_add_log_msg: true
-server_config:
-  name: daos_server
-  engines_per_host: 2
-  engines:
-    0:
-      pinned_numa_node: 0
-      nr_xs_helpers: 1
-      fabric_iface: ib0
-      fabric_iface_port: 31416
-      log_file: daos_server0.log
-      log_mask: INFO,MEM=ERR
-      env_vars:
-        - DD_MASK=mgmt,md
-      storage: auto
-    1:
-      pinned_numa_node: 1
-      nr_xs_helpers: 1
-      fabric_iface: ib1
-      fabric_iface_port: 31516
-      log_file: daos_server1.log
-      log_mask: INFO,MEM=ERR
-      env_vars:
-        - DD_MASK=mgmt,md
-      storage: auto
-pool:
-  scm_size: 12000000000
-  nvme_size: 108000000000
-  svcn: 4
-  rebuild_timeout: 240
-  properties: scrub:timed
-container:
-  type: POSIX
-  control_method: daos
-  oclass: RP_3G6
-  properties: cksum:crc64,cksum_size:16384,srv_cksum:on,rd_fac:2
-dkeys:
-  single:
-    no_of_dkeys:
-      - 50
-akeys:
-  single:
-    no_of_akeys:
-      - 10
-record:
-  1KB:
-    length:
-      - 1024
-ior:
-  clientslots:
-    slots: 48
-  test_file: /testFile
-  repetitions: 1
-  dfs_destroy: false
-  iorflags:
-    write_flags: "-w -F -k -G 1"
-    read_flags: "-F -r -R -k -G 1"
-    api: DFS
-    dfs_oclass: RP_3G6
-    dfs_dir_oclass: RP_3G6
-  ior_test_sequence:
-    # - [scmsize, nvmesize, transfersize, blocksize]
-    #   The values are set to be in the multiples of 10.
-    #   Values are appx GB.
-    - [12000000000, 108000000000, 500000, 500000000]
-mdtest:
-  api: DFS
-  client_processes:
-    np: 30
-  num_of_files_dirs: 4067         # creating total of 120K files
-  test_dir: "/"
-  iteration: 1
-  dfs_destroy: false
-  dfs_oclass: RP_3G6
-  dfs_dir_oclass: RP_3G6
-  manager: "MPICH"
-  flags: "-u"
-  wr_size:
-    32K:
-      write_bytes: 32768
-      read_bytes: 32768
-  verbosity_value: 1
-  depth: 0
-test_obj_class:
-  oclass:
-    - RP_2G8
-    - RP_4G1
-    - EC_2P1G1
-aggregation:
-  test_with_aggregation: true
-rebuild:
-  test_with_rebuild: true
-checksum:
-  test_with_checksum: false
-snapshot:
-  test_with_snapshot: true
-test_ranks:
-  rank_list: ["2", "5"]
-pool_capacity:
-  pool_fillup: 10
-  pool_test_oclass: RP_2GX
+hosts:
+  test_servers: 3
+  test_clients: 1
+timeout: 2400
+setup:
+  start_servers_once: false
+skip_add_log_msg: true
+server_config:
+  name: daos_server
+  engines_per_host: 2
+  engines:
+    0:
+      pinned_numa_node: 0
+      nr_xs_helpers: 1
+      fabric_iface: ib0
+      fabric_iface_port: 31416
+      log_file: daos_server0.log
+      log_mask: INFO,MEM=ERR
+      env_vars:
+        - DD_MASK=mgmt,md
+      storage: auto
+    1:
+      pinned_numa_node: 1
+      nr_xs_helpers: 1
+      fabric_iface: ib1
+      fabric_iface_port: 31516
+      log_file: daos_server1.log
+      log_mask: INFO,MEM=ERR
+      env_vars:
+        - DD_MASK=mgmt,md
+      storage: auto
+pool:
+  scm_size: 12000000000
+  nvme_size: 108000000000
+  svcn: 4
+  rebuild_timeout: 240
+  properties: scrub:timed
+container:
+  type: POSIX
+  control_method: daos
+  oclass: RP_3G6
+  properties: cksum:crc64,cksum_size:16384,srv_cksum:on,rd_fac:2
+dkeys:
+  single:
+    no_of_dkeys:
+      - 50
+akeys:
+  single:
+    no_of_akeys:
+      - 10
+record:
+  1KB:
+    length:
+      - 1024
+ior:
+  clientslots:
+    slots: 48
+  test_file: /testFile
+  repetitions: 1
+  dfs_destroy: false
+  iorflags:
+    write_flags: "-w -F -k -G 1"
+    read_flags: "-F -r -R -k -G 1"
+    api: DFS
+    dfs_oclass: RP_3G6
+    dfs_dir_oclass: RP_3G6
+  ior_test_sequence:
+    # - [scmsize, nvmesize, transfersize, blocksize]
+    #   The values are set to be in the multiples of 10.
+    #   Values are appx GB.
+    - [12000000000, 108000000000, 500000, 500000000]
+mdtest:
+  api: DFS
+  client_processes:
+    np: 30
+  num_of_files_dirs: 4067         # creating total of 120K files
+  test_dir: "/"
+  iteration: 1
+  dfs_destroy: false
+  dfs_oclass: RP_3G6
+  dfs_dir_oclass: RP_3G6
+  manager: "MPICH"
+  flags: "-u"
+  wr_size:
+    32K:
+      write_bytes: 32768
+      read_bytes: 32768
+  verbosity_value: 1
+  depth: 0
+test_obj_class:
+  oclass:
+    - RP_2G8
+    - RP_4G1
+    - EC_2P1G1
+aggregation:
+  test_with_aggregation: true
+rebuild:
+  test_with_rebuild: true
+checksum:
+  test_with_checksum: false
+snapshot:
+  test_with_snapshot: true
+mutliple_ranks:
+  test_with_multiple_ranks: true
+  rank_list: ["1, 2"]
+test_ranks:
+  rank_list: ["2", "5"]
+pool_capacity:
+  pool_fillup: 10
+  pool_test_oclass: RP_2GX

src/tests/ftest/osa/online_drain.py

@@ -1,5 +1,6 @@
 """
   (C) Copyright 2020-2024 Intel Corporation.
+  (C) Copyright 2025 Hewlett Packard Enterprise Development LP
 
   SPDX-License-Identifier: BSD-2-Clause-Patent
 """
@@ -186,3 +187,19 @@ def test_osa_online_drain_mdtest(self):
         """
         self.log.info("Online Drain : With Mdtest")
         self.run_online_drain_test(1, app_name="mdtest")
+
+    def test_osa_online_drain_with_multiple_ranks(self):
+        """Test ID: DAOS-4753.
+
+        Test Description: Drain multiple ranks at the same time.
+
+        :avocado: tags=all,daily_regression
+        :avocado: tags=hw,medium
+        :avocado: tags=osa,osa_drain,online_drain
+        :avocado: tags=OSAOnlineDrain,test_osa_online_drain_with_multiple_ranks
+        """
+        self.log.info("Online Drain : Test with mutiple ranks")
+        self.test_with_multiple_ranks = self.params.get("test_with_multiple_ranks",
+                                                        '/run/multiple_ranks/*')
+        self.multiple_ranks = self.params.get("rank_list", '/run/multiple_ranks/*')
+        self.run_online_drain_test(1, data=True)

src/tests/ftest/osa/online_drain.yaml

@@ -90,3 +90,6 @@ rebuild:
   test_with_rebuild: true
 checksum:
   test_with_checksum: false
+mutliple_ranks:
+  test_with_multiple_ranks: true
+  rank_list: ["1, 2"]