fabtests/efa: Get device MR limit from ibv_query_device

It's better to query the device than to hardcode the limit in case the limit changes in the future Signed-off-by: Sai Sunku <[email protected]>
ofiwg · Dec 29, 2023 · 66dfbea · 66dfbea
1 parent f4c0be1
commit 66dfbea
Show file tree

Hide file tree

Showing 3 changed files with 41 additions and 17 deletions.
diff --git a/fabtests/prov/efa/src/efa_exhaust_mr_reg_common.c b/fabtests/prov/efa/src/efa_exhaust_mr_reg_common.c
@@ -2,23 +2,35 @@
 
 #include "efa_exhaust_mr_reg_common.h"
 
-int ft_efa_setup_ibv_pd(struct ibv_pd **pd)
-{
+int ft_efa_open_ibv_device(struct ibv_context **ctx) {
 	int num_dev = 0;
 	struct ibv_device **dev_list;
-	struct ibv_context *ctx;
 
 	dev_list = ibv_get_device_list(&num_dev);
 	if (num_dev < 1) {
 		FT_ERR("No ibv devices found");
 		ibv_free_device_list(dev_list);
-		return EXIT_FAILURE;
+		return -1;
 	} else if (num_dev > 1) {
 		FT_WARN("More than 1 ibv devices found! This test will only "
 			"exhaust MRs on the first device");
 	}
-	ctx = ibv_open_device(dev_list[0]);
+	*ctx = ibv_open_device(dev_list[0]);
 	ibv_free_device_list(dev_list);
+	return 0;
+}
+
+int ft_efa_get_max_mr(struct ibv_context *ctx) {
+	int ret;
+	struct ibv_device_attr dev_attr = {0};
+	ret = ibv_query_device(ctx, &dev_attr);
+	if (ret)
+		FT_ERR("ibv_query_device failed with %d\n", ret);
+	return dev_attr.max_mr;
+}
+
+int ft_efa_setup_ibv_pd(struct ibv_context *ctx, struct ibv_pd **pd)
+{
 	*pd = ibv_alloc_pd(ctx);
 	if (!*pd) {
 		FT_ERR("alloc_pd failed with error %d\n", errno);

diff --git a/fabtests/prov/efa/src/efa_exhaust_mr_reg_common.h b/fabtests/prov/efa/src/efa_exhaust_mr_reg_common.h
@@ -6,11 +6,12 @@
 #ifndef _EFA_EXHAUST_MR_REG_COMMON_H
 #define _EFA_EXHAUST_MR_REG_COMMON_H
 
-/* The EFA NIC currently supports a maximum of 256 * 1024 = 262144 registrations */
-#define EFA_MR_REG_LIMIT 262144
 #define EFA_MR_REG_BUF_SIZE 128
 
-int ft_efa_setup_ibv_pd(struct ibv_pd **pd);
+int ft_efa_open_ibv_device(struct ibv_context **ctx);
+int ft_efa_close_ibv_device(struct ibv_context *ctx);
+int ft_efa_get_max_mr(struct ibv_context *ctx);
+int ft_efa_setup_ibv_pd(struct ibv_context *ctx, struct ibv_pd **pd);
 void ft_efa_destroy_ibv_pd(struct ibv_pd *pd);
 int ft_efa_register_mr_reg(struct ibv_pd *pd, void **buffers, size_t buf_size,
 			   struct ibv_mr **mr_reg_vec, size_t count, size_t *registered);

diff --git a/fabtests/prov/efa/src/efa_exhaust_mr_reg_rdm_pingpong.c b/fabtests/prov/efa/src/efa_exhaust_mr_reg_rdm_pingpong.c
@@ -36,10 +36,11 @@ static int run(int (*pingpong_func)(void))
 
 int main(int argc, char **argv)
 {
-	int op, ret, err;
+	int op, ret, err, mr_reg_limit;
 	size_t registered;
-	void *buffers[EFA_MR_REG_LIMIT];
-	struct ibv_mr *mr_reg_vec[EFA_MR_REG_LIMIT];
+	void **buffers = NULL;
+	struct ibv_mr **mr_reg_vec = NULL;
+	struct ibv_context *ibv_ctx;
 	struct ibv_pd *pd;
 
 	opts = INIT_OPTS;
@@ -91,18 +92,28 @@ int main(int argc, char **argv)
 
 	ft_sync();
 	if (opts.dst_addr) {
-		err = ft_efa_alloc_bufs(buffers, EFA_MR_REG_BUF_SIZE,
-					EFA_MR_REG_LIMIT);
-		if (err)
-			FT_PRINTERR("alloc bufs", -err);
+		ret = ft_efa_open_ibv_device(&ibv_ctx);
+		if (ret)
+			FT_PRINTERR("ibv_open_device", -1);
+
+		mr_reg_limit = ft_efa_get_max_mr(ibv_ctx);
+		printf("Memory registration limit on device %d\n", mr_reg_limit);
 
-		err = ft_efa_setup_ibv_pd(&pd);
+		buffers = malloc(sizeof(void *) * mr_reg_limit);
+		mr_reg_vec = malloc(sizeof(struct ibv_reg_mr *) * mr_reg_limit);
+
+		err = ft_efa_setup_ibv_pd(ibv_ctx, &pd);
 		if (err)
 			FT_PRINTERR("ibv protection domain", -err);
 
 		printf("Exhausting MRs on client\n");
+		err = ft_efa_alloc_bufs(buffers, EFA_MR_REG_BUF_SIZE,
+					mr_reg_limit);
+		if (err)
+			FT_PRINTERR("alloc bufs", -err);
+
 		err = ft_efa_register_mr_reg(pd, buffers, EFA_MR_REG_BUF_SIZE,
-					     mr_reg_vec, EFA_MR_REG_LIMIT,
+					     mr_reg_vec, mr_reg_limit,
 					     &registered);
 		if (err)
 			FT_PRINTERR("ibv mr reg", -err);