From 87176c458e0f245dd6c00f044c8ae8bc8eade0fd Mon Sep 17 00:00:00 2001 From: Darryl Abbate Date: Thu, 7 Sep 2023 17:42:26 -0700 Subject: [PATCH 1/8] configure.ac Fix `--with-lttng` causing `yes/` to populate {CPP,LD}FLAGS When configured with LTTng-UST support without specifying an installation prefix (--with-lttng), the resulting CPPFLAGS and LDFLAGS contain the (likely) non-existant `yes/` path. This is a little hacky since it technically breaks builds for people who use `yes/` for some reason, but seems to be the convention used elsewhere. Signed-off-by: Darryl Abbate (cherry picked from commit bbcbaf24af9c232a50a41c87f89d196e641f8bda) NOTE: This commit is cherry-picked from main to make CI build check work properly. Without this one, the '--with-lttng' configure option used by CI would lead to failure with the following error message: checking if libibverbs is linkable by libtool... no ...... configure: verbs provider: disabled configure: WARNING: verbs provider was requested, but cannot be compiled configure: error: Cannot continue Commented-by: Jianxin Xiong --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index d57ac08eb14..f4951b00325 100644 --- a/configure.ac +++ b/configure.ac @@ -744,7 +744,7 @@ AS_IF([test -n "$with_lttng" && test x"$with_lttng" != x"no"], [AC_MSG_ERROR([LTTNG is requested but no found. Please check your prefix])]) ]) -AS_IF([test "$have_lttng" = "1"], +AS_IF([test "$have_lttng" = "1" && test x"$with_lttng" != x"yes"], [CPPFLAGS="$CPPFLAGS $lttng_CPPFLAGS" LDFLAGS="$LDFLAGS $lttng_LDFLAGS"]) From b6099cf59cda7bf79ace16c5c6f16727fc7f8423 Mon Sep 17 00:00:00 2001 From: Ken Raffenetti Date: Fri, 26 May 2023 16:14:42 -0500 Subject: [PATCH 2/8] [v1.18.x] fabtests: Fix some header includes These are necessary to compile on MacOS. Signed-off-by: Ken Raffenetti (cherry picked from commit c0191d23df65fdd1de94e3535927556906b424f0) --- fabtests/functional/rdm_stress.c | 2 +- fabtests/regression/sighandler_test.c | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/fabtests/functional/rdm_stress.c b/fabtests/functional/rdm_stress.c index e463c356db8..7a212e3b337 100644 --- a/fabtests/functional/rdm_stress.c +++ b/fabtests/functional/rdm_stress.c @@ -34,7 +34,7 @@ #include #include #include -#include +#include #include #include diff --git a/fabtests/regression/sighandler_test.c b/fabtests/regression/sighandler_test.c index 61a2ff0a8f0..84cf532fd4a 100644 --- a/fabtests/regression/sighandler_test.c +++ b/fabtests/regression/sighandler_test.c @@ -40,6 +40,7 @@ #include #include #include +#include #include "shared.h" int main(int argc, char **argv) From 882de314544578ee972fd01343f44ff2c48ab2ce Mon Sep 17 00:00:00 2001 From: Ken Raffenetti Date: Fri, 26 May 2023 16:15:21 -0500 Subject: [PATCH 3/8] [v1.18.x] fabtests/common: Avoid using memset function name This causes a compile error on MacOS with clang 14.0.0 because memset is substituted as a builtin function by the preprocessor. common/hmem.c:172:25: error: no member named '__builtin___memset_chk' in 'struct ft_hmem_ops' return hmem_ops[iface].memset(device, buf, value, size); ~~~~~~~~~~~~~~~ ^ Signed-off-by: Ken Raffenetti (cherry picked from commit dd2ce59b61a538a8b3b9e137ac05e83b062cd851) --- fabtests/common/hmem.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/fabtests/common/hmem.c b/fabtests/common/hmem.c index f83de70d7ec..9342b0d5530 100644 --- a/fabtests/common/hmem.c +++ b/fabtests/common/hmem.c @@ -43,7 +43,7 @@ struct ft_hmem_ops { int (*alloc_host)(void **buf, size_t size); int (*free)(void *buf); int (*free_host)(void *buf); - int (*memset)(uint64_t device, void *buf, int value, size_t size); + int (*mem_set)(uint64_t device, void *buf, int value, size_t size); int (*copy_to_hmem)(uint64_t device, void *dst, const void *src, size_t size); int (*copy_from_hmem)(uint64_t device, void *dst, const void *src, @@ -58,7 +58,7 @@ static struct ft_hmem_ops hmem_ops[] = { .alloc_host = ft_default_alloc_host, .free = ft_host_free, .free_host = ft_default_free_host, - .memset = ft_host_memset, + .mem_set = ft_host_memset, .copy_to_hmem = ft_host_memcpy, .copy_from_hmem = ft_host_memcpy, }, @@ -69,7 +69,7 @@ static struct ft_hmem_ops hmem_ops[] = { .alloc_host = ft_cuda_alloc_host, .free = ft_cuda_free, .free_host = ft_cuda_free_host, - .memset = ft_cuda_memset, + .mem_set = ft_cuda_memset, .copy_to_hmem = ft_cuda_copy_to_hmem, .copy_from_hmem = ft_cuda_copy_from_hmem, }, @@ -80,7 +80,7 @@ static struct ft_hmem_ops hmem_ops[] = { .alloc_host = ft_default_alloc_host, .free = ft_rocr_free, .free_host = ft_default_free_host, - .memset = ft_rocr_memset, + .mem_set = ft_rocr_memset, .copy_to_hmem = ft_rocr_memcpy, .copy_from_hmem = ft_rocr_memcpy, }, @@ -91,7 +91,7 @@ static struct ft_hmem_ops hmem_ops[] = { .alloc_host = ft_default_alloc_host, .free = ft_ze_free, .free_host = ft_default_free_host, - .memset = ft_ze_memset, + .mem_set = ft_ze_memset, .copy_to_hmem = ft_ze_copy, .copy_from_hmem = ft_ze_copy, }, @@ -102,7 +102,7 @@ static struct ft_hmem_ops hmem_ops[] = { .alloc_host = ft_default_alloc_host, .free = ft_neuron_free, .free_host = ft_default_free_host, - .memset = ft_neuron_memset, + .mem_set = ft_neuron_memset, .copy_to_hmem = ft_neuron_memcpy_to_hmem, .copy_from_hmem = ft_neuron_memcpy_from_hmem, }, @@ -169,7 +169,7 @@ int ft_hmem_free_host(enum fi_hmem_iface iface, void *buf) int ft_hmem_memset(enum fi_hmem_iface iface, uint64_t device, void *buf, int value, size_t size) { - return hmem_ops[iface].memset(device, buf, value, size); + return hmem_ops[iface].mem_set(device, buf, value, size); } int ft_hmem_copy_to(enum fi_hmem_iface iface, uint64_t device, void *dst, From af19247ac74913498c6d29c5ae87cdf31f6dbd65 Mon Sep 17 00:00:00 2001 From: Ken Raffenetti Date: Fri, 2 Jun 2023 11:43:29 -0500 Subject: [PATCH 4/8] [v1.18.x] fabtests/common: Use dummy ft_pin_core on macOS sched_setaffinity is only available on Linux. Signed-off-by: Ken Raffenetti (cherry picked from commit 3425fa724bb9e18b62a3a95d6a075c23eed63c03) --- fabtests/common/shared.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/fabtests/common/shared.c b/fabtests/common/shared.c index 28f25109853..c0e126e678d 100644 --- a/fabtests/common/shared.c +++ b/fabtests/common/shared.c @@ -3707,6 +3707,7 @@ void ft_free_string_array(char **s) free(s); } +#ifndef __APPLE__ static const char *nexttoken(const char *str, int chr) { if (str) @@ -3760,6 +3761,12 @@ static int ft_pin_core(const char *core_list) return sched_setaffinity(0, sizeof(mask), &mask); } +#else +static int ft_pin_core(const char *core_list) +{ + return EXIT_FAILURE; +} +#endif static int ft_parse_pin_core_opt(char *optarg) { From 2b8d169ceb3d2aa21b8dd4baae11fc267723bf5c Mon Sep 17 00:00:00 2001 From: Ken Raffenetti Date: Fri, 26 May 2023 15:55:50 -0500 Subject: [PATCH 5/8] [v1.18.x] component/sock_test: Do not use epoll if not available Signed-off-by: Ken Raffenetti (cherry picked from commit 9ad3a20ffe2e6392cb22efecea6edbe772332eca) --- fabtests/component/sock_test.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/fabtests/component/sock_test.c b/fabtests/component/sock_test.c index 3e2115b5f75..9d5365cbaf4 100755 --- a/fabtests/component/sock_test.c +++ b/fabtests/component/sock_test.c @@ -30,6 +30,10 @@ * SOFTWARE. */ +#if HAVE_CONFIG_H +# include +#endif /* HAVE_CONFIG_H */ + #include #include #include @@ -43,7 +47,9 @@ #include #include #include +#if HAVE_EPOLL == 1 #include +#endif #include #include @@ -54,8 +60,10 @@ static int connections = 1000; uint64_t starttime, endtime; static struct pollfd *poll_set; +#if HAVE_EPOLL == 1 static struct epoll_event *ep_events; static int epfd = -1; +#endif static void show_header(void) @@ -323,6 +331,7 @@ static int test_poll(void) return 0; } +#if HAVE_EPOLL == 1 static int init_epoll(uint32_t events, int stride) { struct epoll_event event; @@ -418,6 +427,12 @@ static int test_epoll(void) return 0; } +#else +static int test_epoll() +{ + return 0; +} +#endif static void close_conns(void) { From 1742f773e06242a6e151af74ba33cebb2e6db736 Mon Sep 17 00:00:00 2001 From: Shi Jin Date: Wed, 12 Apr 2023 22:11:07 +0000 Subject: [PATCH 6/8] [v1.18.x] fabtests/common: split cq_read and cq_readerr in ft_spin_for_comp Introduce a separate function ft_read_cq to do the cq read part in ft_get_cq_comp, Also introduce tag as an input parameter for ft_read_cq and its child functions. It is aimed to avoid setting the global ft_tag which could be messy. Signed-off-by: Shi Jin (cherry picked from commit 24197313f864938645c195e40661a42fc54efd25) --- fabtests/common/shared.c | 35 +++++++++++++++++++++++++---------- fabtests/include/shared.h | 2 ++ 2 files changed, 27 insertions(+), 10 deletions(-) diff --git a/fabtests/common/shared.c b/fabtests/common/shared.c index c0e126e678d..5a5bebc4ba9 100644 --- a/fabtests/common/shared.c +++ b/fabtests/common/shared.c @@ -2317,7 +2317,8 @@ ft_tag_is_valid(struct fid_cq * cq, struct fi_cq_err_entry *comp, uint64_t tag) * fi_cq_err_entry can be cast to any CQ entry format. */ static int ft_spin_for_comp(struct fid_cq *cq, uint64_t *cur, - uint64_t total, int timeout) + uint64_t total, int timeout, + uint64_t tag) { struct fi_cq_err_entry comp; struct timespec a, b; @@ -2331,7 +2332,7 @@ static int ft_spin_for_comp(struct fid_cq *cq, uint64_t *cur, if (ret > 0) { if (timeout >= 0) clock_gettime(CLOCK_MONOTONIC, &a); - if (!ft_tag_is_valid(cq, &comp, ft_tag ? ft_tag : rx_cq_cntr)) + if (!ft_tag_is_valid(cq, &comp, tag ? tag : rx_cq_cntr)) return -FI_EOTHER; (*cur)++; } else if (ret < 0 && ret != -FI_EAGAIN) { @@ -2352,7 +2353,8 @@ static int ft_spin_for_comp(struct fid_cq *cq, uint64_t *cur, * fi_cq_err_entry can be cast to any CQ entry format. */ static int ft_wait_for_comp(struct fid_cq *cq, uint64_t *cur, - uint64_t total, int timeout) + uint64_t total, int timeout, + uint64_t tag) { struct fi_cq_err_entry comp; int ret; @@ -2360,7 +2362,7 @@ static int ft_wait_for_comp(struct fid_cq *cq, uint64_t *cur, while (total - *cur > 0) { ret = fi_cq_sread(cq, &comp, 1, NULL, timeout); if (ret > 0) { - if (!ft_tag_is_valid(cq, &comp, ft_tag ? ft_tag : rx_cq_cntr)) + if (!ft_tag_is_valid(cq, &comp, tag ? tag : rx_cq_cntr)) return -FI_EOTHER; (*cur)++; } else if (ret < 0 && ret != -FI_EAGAIN) { @@ -2375,7 +2377,8 @@ static int ft_wait_for_comp(struct fid_cq *cq, uint64_t *cur, * fi_cq_err_entry can be cast to any CQ entry format. */ static int ft_fdwait_for_comp(struct fid_cq *cq, uint64_t *cur, - uint64_t total, int timeout) + uint64_t total, int timeout, + uint64_t tag) { struct fi_cq_err_entry comp; struct fid *fids[1]; @@ -2394,7 +2397,7 @@ static int ft_fdwait_for_comp(struct fid_cq *cq, uint64_t *cur, ret = fi_cq_read(cq, &comp, 1); if (ret > 0) { - if (!ft_tag_is_valid(cq, &comp, ft_tag ? ft_tag : rx_cq_cntr)) + if (!ft_tag_is_valid(cq, &comp, tag ? tag : rx_cq_cntr)) return -FI_EOTHER; (*cur)++; } else if (ret < 0 && ret != -FI_EAGAIN) { @@ -2405,21 +2408,33 @@ static int ft_fdwait_for_comp(struct fid_cq *cq, uint64_t *cur, return 0; } -int ft_get_cq_comp(struct fid_cq *cq, uint64_t *cur, uint64_t total, int timeout) +int ft_read_cq(struct fid_cq *cq, uint64_t *cur, + uint64_t total, int timeout, + uint64_t tag) { int ret; + switch (opts.comp_method) { case FT_COMP_SREAD: case FT_COMP_YIELD: - ret = ft_wait_for_comp(cq, cur, total, timeout); + ret = ft_wait_for_comp(cq, cur, total, timeout, tag); break; case FT_COMP_WAIT_FD: - ret = ft_fdwait_for_comp(cq, cur, total, timeout); + ret = ft_fdwait_for_comp(cq, cur, total, timeout, tag); break; default: - ret = ft_spin_for_comp(cq, cur, total, timeout); + ret = ft_spin_for_comp(cq, cur, total, timeout, tag); break; } + return ret; +} + +int ft_get_cq_comp(struct fid_cq *cq, uint64_t *cur, + uint64_t total, int timeout) +{ + int ret; + + ret = ft_read_cq(cq, cur, total, timeout, ft_tag); if (ret) { if (ret == -FI_EAVAIL) { diff --git a/fabtests/include/shared.h b/fabtests/include/shared.h index 7c5751165d7..dcf84a90195 100644 --- a/fabtests/include/shared.h +++ b/fabtests/include/shared.h @@ -503,6 +503,8 @@ static inline bool ft_check_prefix_forced(struct fi_info *info, return true; } +int ft_read_cq(struct fid_cq *cq, uint64_t *cur, uint64_t total, + int timeout, uint64_t tag); int ft_sync(void); int ft_sync_pair(int status); int ft_fork_and_pair(void); From 0368e2a6975b7eca1817ecbfa34d23ed45053a47 Mon Sep 17 00:00:00 2001 From: Shi Jin Date: Wed, 12 Apr 2023 22:17:48 +0000 Subject: [PATCH 7/8] [v1.18.x] fabtests/functional: make rdm_tagged_peek test more general The current rdm_tagged_peek test uses sread which is not available for some provider like efa. This patch makes this test more general to support all available comp_method, including sread and the default non-wait read. It also makes the do_send retry fi_tsend when it hits EAGAIN. Signed-off-by: Shi Jin (cherry picked from commit 61fa45d5ffd74dc49574500d68c4f511d07b916f) --- fabtests/functional/rdm_tagged_peek.c | 38 +++++++++++++++------------ 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/fabtests/functional/rdm_tagged_peek.c b/fabtests/functional/rdm_tagged_peek.c index e26bf283c06..b8d11ed447a 100644 --- a/fabtests/functional/rdm_tagged_peek.c +++ b/fabtests/functional/rdm_tagged_peek.c @@ -47,12 +47,13 @@ static struct fi_context fi_context; static int wait_for_send_comp(int count) { int ret, completions = 0; - struct fi_cq_tagged_entry comp; + uint64_t cq_cntr; do { - ret = fi_cq_sread(txcq, &comp, 1, NULL, -1); - if (ret != 1) { - FT_PRINTERR("fi_cq_sread", ret); + cq_cntr = 0; + ret = ft_read_cq(txcq, &cq_cntr, 1, timeout, BASE_TAG + completions + 1); + if (ret != FI_SUCCESS) { + FT_PRINTERR("fi_cq_sread/fi_cq_read", ret); return ret; } completions++; @@ -61,14 +62,15 @@ static int wait_for_send_comp(int count) return 0; } + static int trecv_op(uint64_t tag, uint64_t flags, bool ignore_nomsg) { - struct fi_cq_tagged_entry comp; struct fi_msg_tagged msg = {0}; struct fi_cq_err_entry cq_err; struct iovec iov; void *desc; int ret; + uint64_t cq_cntr; if (!(flags & (FI_PEEK | FI_DISCARD))) { iov.iov_base = buf; @@ -90,11 +92,10 @@ static int trecv_op(uint64_t tag, uint64_t flags, bool ignore_nomsg) return ret; } - ret = fi_cq_sread(rxcq, &comp, 1, NULL, -1); - if (ret == 1) { - ret = 0; + cq_cntr = 0; + ret = ft_read_cq(rxcq, &cq_cntr, 1, timeout, tag); + if (ret == FI_SUCCESS) break; - } if (ret == -FI_EAVAIL) { ret = fi_cq_readerr(rxcq, &cq_err, 0); @@ -103,7 +104,7 @@ static int trecv_op(uint64_t tag, uint64_t flags, bool ignore_nomsg) else ret = -cq_err.err; } else { - FT_PRINTERR("fi_cq_sread", ret); + FT_PRINTERR("fi_cq_read/fi_cq_sread", ret); } } while (ret == -FI_ENOMSG && ignore_nomsg); @@ -282,9 +283,11 @@ static int do_sends(void) (void) ft_fill_buf(tx_buf, tx_size); for (i = 1; i <= SEND_CNT; i++) { - ret = fi_tsend(ep, tx_buf, tx_size, mr_desc, - remote_fi_addr, BASE_TAG + i, - &tx_ctx_arr[i].context); + do { + ret = fi_tsend(ep, tx_buf, tx_size, mr_desc, + remote_fi_addr, BASE_TAG + i, + &tx_ctx_arr[i].context); + } while (ret == -FI_EAGAIN); if (ret) return ret; } @@ -310,9 +313,11 @@ static int run(void) /* sync with sender before ft_finalize, since we sent * and received messages outside of the sequence numbers * maintained by common code */ - ret = fi_tsend(ep, tx_buf, 1, mr_desc, - remote_fi_addr, 0xabc, - &tx_ctx_arr[0].context); + do { + ret = fi_tsend(ep, tx_buf, 1, mr_desc, + remote_fi_addr, 0xabc, + &tx_ctx_arr[0].context); + } while (ret == -FI_EAGAIN); if (ret) return ret; @@ -342,7 +347,6 @@ int main(int argc, char **argv) opts = INIT_OPTS; opts.options |= FT_OPT_SIZE; opts.transfer_size = 64; /* Don't expect receiver buffering */ - opts.comp_method = FT_COMP_SREAD; opts.window_size = SEND_CNT; hints = fi_allocinfo(); From fb0b201e8e02d63e070524c0bdb55d2c7f55f1e8 Mon Sep 17 00:00:00 2001 From: Alexia Ingerson Date: Tue, 25 Apr 2023 17:38:40 -0700 Subject: [PATCH 8/8] [v1.18.x] fabtests/rdm_tagged_peek: fix race condition synchronization It was possible for the client to complete the finalize send before the server gets the last 0xabc completion, causing a tag mismatch when processing the incorrect receive. This removes the extra synchronization sends and adds an OOB sync to make sure all of the test sends have finished before quitting the test. Signed-off-by: Alexia Ingerson (cherry picked from commit 09ca77959f6c658454e3932fc072954868ec821d) --- fabtests/functional/rdm_tagged_peek.c | 25 ++----------------------- 1 file changed, 2 insertions(+), 23 deletions(-) diff --git a/fabtests/functional/rdm_tagged_peek.c b/fabtests/functional/rdm_tagged_peek.c index b8d11ed447a..596c9402ec0 100644 --- a/fabtests/functional/rdm_tagged_peek.c +++ b/fabtests/functional/rdm_tagged_peek.c @@ -309,34 +309,13 @@ static int run(void) ret = do_recvs(); if (ret) return ret; - - /* sync with sender before ft_finalize, since we sent - * and received messages outside of the sequence numbers - * maintained by common code */ - do { - ret = fi_tsend(ep, tx_buf, 1, mr_desc, - remote_fi_addr, 0xabc, - &tx_ctx_arr[0].context); - } while (ret == -FI_EAGAIN); - if (ret) - return ret; - - ret = wait_for_send_comp(1); - if (ret) - return ret; } else { ret = do_sends(); if (ret) return ret; - - ret = trecv_op(0xabc, 0, false); - if (ret) { - FT_PRINTERR("Receive sync", ret); - return ret; - } } - ft_finalize(); + ft_sync(); return 0; } @@ -345,7 +324,7 @@ int main(int argc, char **argv) int ret, op; opts = INIT_OPTS; - opts.options |= FT_OPT_SIZE; + opts.options |= FT_OPT_SIZE | FT_OPT_OOB_SYNC; opts.transfer_size = 64; /* Don't expect receiver buffering */ opts.window_size = SEND_CNT;