prov/efa: Add unit tests for reordering messages in receive window

Signed-off-by: Jessie Yang <[email protected]>

prov/efa/Makefile.include

@@ -146,7 +146,8 @@ nodist_prov_efa_test_efa_unit_test_SOURCES = \
 	prov/efa/test/efa_unit_test_send.c \
 	prov/efa/test/efa_unit_test_fork_support.c \
 	prov/efa/test/efa_unit_test_runt.c \
-	prov/efa/test/efa_unit_test_mr.c
+	prov/efa/test/efa_unit_test_mr.c \
+	prov/efa/test/efa_unit_test_rdm_peer.c
 
 
 efa_CPPFLAGS += -I$(top_srcdir)/include -I$(top_srcdir)/prov/efa/test $(cmocka_CPPFLAGS)

prov/efa/test/efa_unit_test_rdm_peer.c

@@ -0,0 +1,135 @@
+/* SPDX-License-Identifier: BSD-2-Clause OR GPL-2.0-only */
+/* SPDX-FileCopyrightText: Copyright Amazon.com, Inc. or its affiliates. All rights reserved. */
+
+#include "efa_unit_tests.h"
+
+/**
+ * @brief Test efa_rdm_peer_reorder_msg
+ *
+ * @param resource efa_resource
+ * 
+ */
+void test_efa_rdm_peer_reorder_msg_impl(struct efa_resource *resource,
+					u_int32_t exp_msg_id, u_int32_t msg_id,
+					int expected_ret)
+{
+	struct efa_rdm_ep *efa_rdm_ep;
+	struct efa_rdm_peer *peer;
+	struct efa_ep_addr raw_addr;
+	fi_addr_t addr;
+	struct efa_unit_test_eager_rtm_pkt_attr pkt_attr = {0};
+	size_t raw_addr_len = sizeof(raw_addr);
+	struct efa_rdm_peer_overflow_pke_list_entry *overflow_pke_list_entry;
+	struct efa_rdm_pke *pkt_entry, *overflow_pkt_entry;
+	struct dlist_entry *tmp;
+	u_int32_t overflow_msg_id;
+	int ret;
+
+	efa_rdm_ep = container_of(resource->ep, struct efa_rdm_ep,
+				  base_ep.util_ep.ep_fid);
+
+	/* Create and register a fake peer */
+	ret = fi_getname(&resource->ep->fid, &raw_addr, &raw_addr_len);
+	assert_int_equal(ret, 0);
+	raw_addr.qpn = 1;
+	raw_addr.qkey = 0x1234;
+	ret = fi_av_insert(resource->av, &raw_addr, 1, &addr, 0 /* flags */, NULL /* context */);
+	assert_int_equal(ret, 1);
+	peer = efa_rdm_ep_get_peer(efa_rdm_ep, addr);
+	assert_non_null(peer);
+	ofi_recvwin_buf_alloc(&peer->robuf, efa_env.recvwin_size);
+	dlist_init(&peer->overflow_pke_list);
+
+	pkt_entry = efa_rdm_pke_alloc(efa_rdm_ep, efa_rdm_ep->efa_rx_pkt_pool,
+				      EFA_RDM_PKE_FROM_EFA_RX_POOL);
+	assert_non_null(pkt_entry);
+	efa_rdm_ep->efa_rx_pkts_posted = efa_rdm_ep_get_rx_pool_size(efa_rdm_ep);
+
+	pkt_attr.msg_id = msg_id;
+	pkt_attr.connid = raw_addr.qkey;
+	/* Packet type must be >= EFA_RDM_REQ_PKT_BEGIN */
+	efa_unit_test_eager_msgrtm_pkt_construct(pkt_entry, &pkt_attr);
+
+	(&peer->robuf)->exp_msg_id = exp_msg_id;
+	ret = efa_rdm_peer_reorder_msg(peer, efa_rdm_ep, pkt_entry);
+	assert_int_equal(ret, expected_ret);
+	(&peer->robuf)->exp_msg_id = 0;
+
+	if (msg_id >= efa_env.recvwin_size + exp_msg_id) {
+		/* Check the overflow_pke_list contains the overflow msg_id */
+		assert_int_equal(efa_unit_test_get_dlist_length(&peer->overflow_pke_list), 1);
+		dlist_foreach_container_safe (
+			&peer->overflow_pke_list,
+			struct efa_rdm_peer_overflow_pke_list_entry,
+			overflow_pke_list_entry, entry, tmp) {
+			overflow_pkt_entry = overflow_pke_list_entry->pkt_entry;
+			overflow_msg_id = ((struct efa_rdm_rtm_base_hdr *) overflow_pkt_entry->wiredata)->msg_id;
+			assert_int_equal(overflow_msg_id, msg_id);
+			/* Clean up */
+			dlist_remove(&overflow_pke_list_entry->entry);
+			efa_rdm_pke_release_rx(overflow_pke_list_entry->pkt_entry);
+			ofi_buf_free(overflow_pke_list_entry);
+		}
+	} else {
+		efa_rdm_pke_release_rx(pkt_entry);
+	}
+}
+
+void test_efa_rdm_peer_reorder_expected_msg_id(struct efa_resource **state) {
+	struct efa_resource *resource = *state;
+	u_int32_t msg_id, exp_msg_id;
+	int expected_ret;
+
+	efa_unit_test_resource_construct(resource, FI_EP_RDM);
+
+	msg_id = 0;
+	exp_msg_id = 0;
+	expected_ret = 0;
+	/* Receiving expected message id should return 0 */
+	test_efa_rdm_peer_reorder_msg_impl(resource, exp_msg_id, msg_id, expected_ret);
+}
+
+
+void test_efa_rdm_peer_reorder_smaller_msg_id(struct efa_resource **state) {
+	struct efa_resource *resource = *state;
+	u_int32_t msg_id, exp_msg_id;
+	int expected_ret;
+
+	efa_unit_test_resource_construct(resource, FI_EP_RDM);
+
+	msg_id = 1;
+	exp_msg_id = 10;
+	expected_ret = -FI_EALREADY;
+	/* Receiving message id smaller than expected should return -FI_EALREADY */
+	test_efa_rdm_peer_reorder_msg_impl(resource, exp_msg_id, msg_id, expected_ret);
+}
+
+void test_efa_rdm_peer_reorder_larger_msg_id(struct efa_resource **state) {
+	struct efa_resource *resource = *state;
+	u_int32_t msg_id, exp_msg_id;
+	int expected_ret;
+
+	efa_unit_test_resource_construct(resource, FI_EP_RDM);
+
+	msg_id = 10;
+	exp_msg_id = 0;
+	expected_ret = 1;
+	efa_env.rx_copy_ooo = 0; /* Do not copy this pkt entry */
+	/* Receiving message id larger than expected should return 1 */
+	test_efa_rdm_peer_reorder_msg_impl(resource, exp_msg_id, msg_id, expected_ret);
+}
+
+void test_efa_rdm_peer_reorder_overflow_msg_id(struct efa_resource **state) {
+	struct efa_resource *resource = *state;
+	u_int32_t msg_id, exp_msg_id;
+	int expected_ret;
+
+	efa_unit_test_resource_construct(resource, FI_EP_RDM);
+
+	msg_id = 16384;
+	exp_msg_id = 0;
+	expected_ret = 1;
+	/* Message id that overflows the receive window should be put in the
+	 * overflow_pke_list and return 1 */
+	test_efa_rdm_peer_reorder_msg_impl(resource, exp_msg_id, msg_id, expected_ret);
+}

prov/efa/test/efa_unit_tests.c

@@ -186,6 +186,10 @@ int main(void)
 		cmocka_unit_test_setup_teardown(test_efa_neuron_dmabuf_support_mr_fail_no_fallback, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
 		cmocka_unit_test_setup_teardown(test_efa_neuron_dmabuf_support_require_dmabuf_fail_no_fallback, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
 		cmocka_unit_test_setup_teardown(test_efa_synapseai_dmabuf_support_fd_fail_no_fallback, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
+		cmocka_unit_test_setup_teardown(test_efa_rdm_peer_reorder_expected_msg_id, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
+		cmocka_unit_test_setup_teardown(test_efa_rdm_peer_reorder_smaller_msg_id, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
+		cmocka_unit_test_setup_teardown(test_efa_rdm_peer_reorder_larger_msg_id, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
+		cmocka_unit_test_setup_teardown(test_efa_rdm_peer_reorder_overflow_msg_id, efa_unit_test_mocks_setup, efa_unit_test_mocks_teardown),
 	};
 
 	cmocka_set_message_output(CM_OUTPUT_XML);

prov/efa/test/efa_unit_tests.h

@@ -196,6 +196,10 @@ void test_efa_cuda_dmabuf_support_ibv_reg_mr_fail_fallback_keygen();
 void test_efa_synapseai_dmabuf_support_fd_fail_no_fallback();
 void test_efa_cuda_dmabuf_support_require_dmabuf_fail_no_fallback();
 void test_efa_neuron_dmabuf_support_require_dmabuf_fail_no_fallback();
+void test_efa_rdm_peer_reorder_expected_msg_id();
+void test_efa_rdm_peer_reorder_smaller_msg_id();
+void test_efa_rdm_peer_reorder_larger_msg_id();
+void test_efa_rdm_peer_reorder_overflow_msg_id();
 
 static inline
 int efa_unit_test_get_dlist_length(struct dlist_entry *head)