From 1742f773e06242a6e151af74ba33cebb2e6db736 Mon Sep 17 00:00:00 2001
From: Shi Jin <sjina@amazon.com>
Date: Wed, 12 Apr 2023 22:11:07 +0000
Subject: [PATCH] [v1.18.x] fabtests/common: split cq_read and cq_readerr in
 ft_spin_for_comp

Introduce a separate function ft_read_cq to do the cq read part
in ft_get_cq_comp,

Also introduce tag as an input parameter for ft_read_cq and its
child functions. It is aimed to avoid setting the global ft_tag
which could be messy.

Signed-off-by: Shi Jin <sjina@amazon.com>
(cherry picked from commit 24197313f864938645c195e40661a42fc54efd25)
---
 fabtests/common/shared.c  | 35 +++++++++++++++++++++++++----------
 fabtests/include/shared.h |  2 ++
 2 files changed, 27 insertions(+), 10 deletions(-)

diff --git a/fabtests/common/shared.c b/fabtests/common/shared.c
index c0e126e678d..5a5bebc4ba9 100644
--- a/fabtests/common/shared.c
+++ b/fabtests/common/shared.c
@@ -2317,7 +2317,8 @@ ft_tag_is_valid(struct fid_cq * cq, struct fi_cq_err_entry *comp, uint64_t tag)
  * fi_cq_err_entry can be cast to any CQ entry format.
  */
 static int ft_spin_for_comp(struct fid_cq *cq, uint64_t *cur,
-			    uint64_t total, int timeout)
+			    uint64_t total, int timeout,
+			    uint64_t tag)
 {
 	struct fi_cq_err_entry comp;
 	struct timespec a, b;
@@ -2331,7 +2332,7 @@ static int ft_spin_for_comp(struct fid_cq *cq, uint64_t *cur,
 		if (ret > 0) {
 			if (timeout >= 0)
 				clock_gettime(CLOCK_MONOTONIC, &a);
-			if (!ft_tag_is_valid(cq, &comp, ft_tag ? ft_tag : rx_cq_cntr))
+			if (!ft_tag_is_valid(cq, &comp, tag ? tag : rx_cq_cntr))
 				return -FI_EOTHER;
 			(*cur)++;
 		} else if (ret < 0 && ret != -FI_EAGAIN) {
@@ -2352,7 +2353,8 @@ static int ft_spin_for_comp(struct fid_cq *cq, uint64_t *cur,
  * fi_cq_err_entry can be cast to any CQ entry format.
  */
 static int ft_wait_for_comp(struct fid_cq *cq, uint64_t *cur,
-			    uint64_t total, int timeout)
+			    uint64_t total, int timeout,
+			    uint64_t tag)
 {
 	struct fi_cq_err_entry comp;
 	int ret;
@@ -2360,7 +2362,7 @@ static int ft_wait_for_comp(struct fid_cq *cq, uint64_t *cur,
 	while (total - *cur > 0) {
 		ret = fi_cq_sread(cq, &comp, 1, NULL, timeout);
 		if (ret > 0) {
-			if (!ft_tag_is_valid(cq, &comp, ft_tag ? ft_tag : rx_cq_cntr))
+			if (!ft_tag_is_valid(cq, &comp, tag ? tag : rx_cq_cntr))
 				return -FI_EOTHER;
 			(*cur)++;
 		} else if (ret < 0 && ret != -FI_EAGAIN) {
@@ -2375,7 +2377,8 @@ static int ft_wait_for_comp(struct fid_cq *cq, uint64_t *cur,
  * fi_cq_err_entry can be cast to any CQ entry format.
  */
 static int ft_fdwait_for_comp(struct fid_cq *cq, uint64_t *cur,
-			    uint64_t total, int timeout)
+			    uint64_t total, int timeout,
+			    uint64_t tag)
 {
 	struct fi_cq_err_entry comp;
 	struct fid *fids[1];
@@ -2394,7 +2397,7 @@ static int ft_fdwait_for_comp(struct fid_cq *cq, uint64_t *cur,
 
 		ret = fi_cq_read(cq, &comp, 1);
 		if (ret > 0) {
-			if (!ft_tag_is_valid(cq, &comp, ft_tag ? ft_tag : rx_cq_cntr))
+			if (!ft_tag_is_valid(cq, &comp, tag ? tag : rx_cq_cntr))
 				return -FI_EOTHER;
 			(*cur)++;
 		} else if (ret < 0 && ret != -FI_EAGAIN) {
@@ -2405,21 +2408,33 @@ static int ft_fdwait_for_comp(struct fid_cq *cq, uint64_t *cur,
 	return 0;
 }
 
-int ft_get_cq_comp(struct fid_cq *cq, uint64_t *cur, uint64_t total, int timeout)
+int ft_read_cq(struct fid_cq *cq, uint64_t *cur,
+		uint64_t total, int timeout,
+		uint64_t tag)
 {
 	int ret;
+
 	switch (opts.comp_method) {
 	case FT_COMP_SREAD:
 	case FT_COMP_YIELD:
-		ret = ft_wait_for_comp(cq, cur, total, timeout);
+		ret = ft_wait_for_comp(cq, cur, total, timeout, tag);
 		break;
 	case FT_COMP_WAIT_FD:
-		ret = ft_fdwait_for_comp(cq, cur, total, timeout);
+		ret = ft_fdwait_for_comp(cq, cur, total, timeout, tag);
 		break;
 	default:
-		ret = ft_spin_for_comp(cq, cur, total, timeout);
+		ret = ft_spin_for_comp(cq, cur, total, timeout, tag);
 		break;
 	}
+	return ret;
+}
+
+int ft_get_cq_comp(struct fid_cq *cq, uint64_t *cur,
+		    uint64_t total, int timeout)
+{
+	int ret;
+
+	ret = ft_read_cq(cq, cur, total, timeout, ft_tag);
 
 	if (ret) {
 		if (ret == -FI_EAVAIL) {
diff --git a/fabtests/include/shared.h b/fabtests/include/shared.h
index 7c5751165d7..dcf84a90195 100644
--- a/fabtests/include/shared.h
+++ b/fabtests/include/shared.h
@@ -503,6 +503,8 @@ static inline bool ft_check_prefix_forced(struct fi_info *info,
 	return true;
 }
 
+int ft_read_cq(struct fid_cq *cq, uint64_t *cur, uint64_t total,
+		int timeout, uint64_t tag);
 int ft_sync(void);
 int ft_sync_pair(int status);
 int ft_fork_and_pair(void);