From ee9f69243d65c863dbaf2b640a765d666b91c13d Mon Sep 17 00:00:00 2001 From: Jessie Yang Date: Thu, 18 Jul 2024 16:27:51 -0700 Subject: [PATCH] prov/efa: Add unit tests for reordering messages in receive window Signed-off-by: Jessie Yang --- prov/efa/Makefile.include | 3 +- prov/efa/test/efa_unit_test_rdm_peer.c | 135 +++++++++++++++++++++++++ prov/efa/test/efa_unit_tests.c | 4 + prov/efa/test/efa_unit_tests.h | 4 + 4 files changed, 145 insertions(+), 1 deletion(-) create mode 100644 prov/efa/test/efa_unit_test_rdm_peer.c diff --git a/prov/efa/Makefile.include b/prov/efa/Makefile.include index 3a32c4fa556..c014dd72eda 100644 --- a/prov/efa/Makefile.include +++ b/prov/efa/Makefile.include @@ -146,7 +146,8 @@ nodist_prov_efa_test_efa_unit_test_SOURCES = \ prov/efa/test/efa_unit_test_send.c \ prov/efa/test/efa_unit_test_fork_support.c \ prov/efa/test/efa_unit_test_runt.c \ - prov/efa/test/efa_unit_test_mr.c + prov/efa/test/efa_unit_test_mr.c \ + prov/efa/test/efa_unit_test_rdm_peer.c efa_CPPFLAGS += -I$(top_srcdir)/include -I$(top_srcdir)/prov/efa/test $(cmocka_CPPFLAGS) diff --git a/prov/efa/test/efa_unit_test_rdm_peer.c b/prov/efa/test/efa_unit_test_rdm_peer.c new file mode 100644 index 00000000000..63c5df2a799 --- /dev/null +++ b/prov/efa/test/efa_unit_test_rdm_peer.c @@ -0,0 +1,135 @@ +/* SPDX-License-Identifier: BSD-2-Clause OR GPL-2.0-only */ +/* SPDX-FileCopyrightText: Copyright Amazon.com, Inc. or its affiliates. All rights reserved. */ + +#include "efa_unit_tests.h" + +/** + * @brief Test efa_rdm_peer_reorder_msg + * + * @param resource efa_resource + * + */ +void test_efa_rdm_peer_reorder_msg_impl(struct efa_resource *resource, + u_int32_t exp_msg_id, u_int32_t msg_id, + int expected_ret) +{ + struct efa_rdm_ep *efa_rdm_ep; + struct efa_rdm_peer *peer; + struct efa_ep_addr raw_addr; + fi_addr_t addr; + struct efa_unit_test_eager_rtm_pkt_attr pkt_attr = {0}; + size_t raw_addr_len = sizeof(raw_addr); + struct efa_rdm_peer_overflow_pke_list_entry *overflow_pke_list_entry; + struct efa_rdm_pke *pkt_entry, *overflow_pkt_entry; + struct dlist_entry *tmp; + u_int32_t overflow_msg_id; + int ret; + + efa_rdm_ep = container_of(resource->ep, struct efa_rdm_ep, + base_ep.util_ep.ep_fid); + + /* Create and register a fake peer */ + ret = fi_getname(&resource->ep->fid, &raw_addr, &raw_addr_len); + assert_int_equal(ret, 0); + raw_addr.qpn = 1; + raw_addr.qkey = 0x1234; + ret = fi_av_insert(resource->av, &raw_addr, 1, &addr, 0 /* flags */, NULL /* context */); + assert_int_equal(ret, 1); + peer = efa_rdm_ep_get_peer(efa_rdm_ep, addr); + assert_non_null(peer); + ofi_recvwin_buf_alloc(&peer->robuf, efa_env.recvwin_size); + dlist_init(&peer->overflow_pke_list); + + pkt_entry = efa_rdm_pke_alloc(efa_rdm_ep, efa_rdm_ep->efa_rx_pkt_pool, + EFA_RDM_PKE_FROM_EFA_RX_POOL); + assert_non_null(pkt_entry); + efa_rdm_ep->efa_rx_pkts_posted = efa_rdm_ep_get_rx_pool_size(efa_rdm_ep); + + pkt_attr.msg_id = msg_id; + pkt_attr.connid = raw_addr.qkey; + /* Packet type must be >= EFA_RDM_REQ_PKT_BEGIN */ + efa_unit_test_eager_msgrtm_pkt_construct(pkt_entry, &pkt_attr); + + (&peer->robuf)->exp_msg_id = exp_msg_id; + ret = efa_rdm_peer_reorder_msg(peer, efa_rdm_ep, pkt_entry); + assert_int_equal(ret, expected_ret); + (&peer->robuf)->exp_msg_id = 0; + + if (msg_id >= efa_env.recvwin_size + exp_msg_id) { + /* Check the overflow_pke_list contains the overflow msg_id */ + assert_int_equal(efa_unit_test_get_dlist_length(&peer->overflow_pke_list), 1); + dlist_foreach_container_safe ( + &peer->overflow_pke_list, + struct efa_rdm_peer_overflow_pke_list_entry, + overflow_pke_list_entry, entry, tmp) { + overflow_pkt_entry = overflow_pke_list_entry->pkt_entry; + overflow_msg_id = ((struct efa_rdm_rtm_base_hdr *) overflow_pkt_entry->wiredata)->msg_id; + assert_int_equal(overflow_msg_id, msg_id); + /* Clean up */ + dlist_remove(&overflow_pke_list_entry->entry); + efa_rdm_pke_release_rx(overflow_pke_list_entry->pkt_entry); + ofi_buf_free(overflow_pke_list_entry); + } + } else { + efa_rdm_pke_release_rx(pkt_entry); + } +} + +void test_efa_rdm_peer_reorder_expected_msg_id(struct efa_resource **state) { + struct efa_resource *resource = *state; + u_int32_t msg_id, exp_msg_id; + int expected_ret; + + efa_unit_test_resource_construct(resource, FI_EP_RDM); + + msg_id = 0; + exp_msg_id = 0; + expected_ret = 0; + /* Receiving expected message id should return 0 */ + test_efa_rdm_peer_reorder_msg_impl(resource, exp_msg_id, msg_id, expected_ret); +} + + +void test_efa_rdm_peer_reorder_smaller_msg_id(struct efa_resource **state) { + struct efa_resource *resource = *state; + u_int32_t msg_id, exp_msg_id; + int expected_ret; + + efa_unit_test_resource_construct(resource, FI_EP_RDM); + + msg_id = 1; + exp_msg_id = 10; + expected_ret = -FI_EALREADY; + /* Receiving message id smaller than expected should return -FI_EALREADY */ + test_efa_rdm_peer_reorder_msg_impl(resource, exp_msg_id, msg_id, expected_ret); +} + +void test_efa_rdm_peer_reorder_larger_msg_id(struct efa_resource **state) { + struct efa_resource *resource = *state; + u_int32_t msg_id, exp_msg_id; + int expected_ret; + + efa_unit_test_resource_construct(resource, FI_EP_RDM); + + msg_id = 10; + exp_msg_id = 0; + expected_ret = 1; + efa_env.rx_copy_ooo = 0; /* Do not copy this pkt entry */ + /* Receiving message id larger than expected should return 1 */ + test_efa_rdm_peer_reorder_msg_impl(resource, exp_msg_id, msg_id, expected_ret); +} + +void test_efa_rdm_peer_reorder_overflow_msg_id(struct efa_resource **state) { + struct efa_resource *resource = *state; + u_int32_t msg_id, exp_msg_id; + int expected_ret; + + efa_unit_test_resource_construct(resource, FI_EP_RDM); + + msg_id = 16384; + exp_msg_id = 0; + expected_ret = 1; + /* Message id that overflows the receive window should be put in the + * overflow_pke_list and return 1 */ + test_efa_rdm_peer_reorder_msg_impl(resource, exp_msg_id, msg_id, expected_ret); +} diff --git a/prov/efa/test/efa_unit_tests.c b/prov/efa/test/efa_unit_tests.c index 97571abb93a..919c8c91c11 100644 --- a/prov/efa/test/efa_unit_tests.c +++ b/prov/efa/test/efa_unit_tests.c @@ -186,6 +186,10 @@ int main(void) cmocka_unit_test_setup_teardown(test_efa_neuron_dmabuf_support_mr_fail_no_fallback, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown), cmocka_unit_test_setup_teardown(test_efa_neuron_dmabuf_support_require_dmabuf_fail_no_fallback, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown), cmocka_unit_test_setup_teardown(test_efa_synapseai_dmabuf_support_fd_fail_no_fallback, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown), + cmocka_unit_test_setup_teardown(test_efa_rdm_peer_reorder_expected_msg_id, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown), + cmocka_unit_test_setup_teardown(test_efa_rdm_peer_reorder_smaller_msg_id, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown), + cmocka_unit_test_setup_teardown(test_efa_rdm_peer_reorder_larger_msg_id, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown), + cmocka_unit_test_setup_teardown(test_efa_rdm_peer_reorder_overflow_msg_id, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown), }; cmocka_set_message_output(CM_OUTPUT_XML); diff --git a/prov/efa/test/efa_unit_tests.h b/prov/efa/test/efa_unit_tests.h index 25139e358f2..9b388d7f415 100644 --- a/prov/efa/test/efa_unit_tests.h +++ b/prov/efa/test/efa_unit_tests.h @@ -196,6 +196,10 @@ void test_efa_cuda_dmabuf_support_ibv_reg_mr_fail_fallback_keygen(); void test_efa_synapseai_dmabuf_support_fd_fail_no_fallback(); void test_efa_cuda_dmabuf_support_require_dmabuf_fail_no_fallback(); void test_efa_neuron_dmabuf_support_require_dmabuf_fail_no_fallback(); +void test_efa_rdm_peer_reorder_expected_msg_id(); +void test_efa_rdm_peer_reorder_smaller_msg_id(); +void test_efa_rdm_peer_reorder_larger_msg_id(); +void test_efa_rdm_peer_reorder_overflow_msg_id(); static inline int efa_unit_test_get_dlist_length(struct dlist_entry *head)