DAOS-13883 cart: control request_post_init/incr through env vars (#12565

) Add D_POST_INIT / D_POST_INCR env vars This enables tuning of multi-recv buffer sizes and number of server preposted recvs when multi-recv is not enabled. Signed-off-by: Jerome Soumagne <[email protected]>
daos-stack · Nov 30, 2023 · c11b258 · c11b258
1 parent 5daa960
commit c11b258
Show file tree

Hide file tree

Showing 4 changed files with 21 additions and 5 deletions.
diff --git a/src/cart/crt_hg.c b/src/cart/crt_hg.c
@@ -885,6 +885,9 @@ crt_hg_class_init(int provider, int idx, bool primary, hg_class_t **ret_hg_class
 	if (prov_data->cpg_max_unexp_size > 0)
 		init_info.na_init_info.max_unexpected_size = prov_data->cpg_max_unexp_size;
 
+	init_info.request_post_init = crt_gdata.cg_post_init;
+	init_info.request_post_incr = crt_gdata.cg_post_incr;
+
 	hg_class = HG_Init_opt(info_string, crt_is_service(), &init_info);
 	if (hg_class == NULL) {
 		D_ERROR("Could not initialize HG class.\n");

diff --git a/src/cart/crt_hg.h b/src/cart/crt_hg.h
@@ -27,6 +27,10 @@
 /** number of prepost HG handles when enable pool */
 #define CRT_HG_POOL_PREPOST_NUM	(16)
 
+/** default values for init / incr to prepost handles */
+#define CRT_HG_POST_INIT        (512)
+#define CRT_HG_POST_INCR        (512)
+
 struct crt_rpc_priv;
 struct crt_common_hdr;
 struct crt_corpc_hdr;

diff --git a/src/cart/crt_init.c b/src/cart/crt_init.c
@@ -96,6 +96,8 @@ dump_envariables(void)
 			     "D_PORT_AUTO_ADJUST",
 			     "D_POLL_TIMEOUT",
 			     "D_LOG_FILE_APPEND_RANK",
+			     "D_POST_INIT",
+			     "D_POST_INCR",
 			     "DAOS_SIGNAL_REGISTER"};
 
 	D_INFO("-- ENVARS: --\n");
@@ -258,12 +260,13 @@ prov_data_init(struct crt_prov_gdata *prov_data, crt_provider_t provider,
 /* first step init - for initializing crt_gdata */
 static int data_init(int server, crt_init_options_t *opt)
 {
-	uint32_t	timeout;
+	uint32_t        timeout = 0;
 	uint32_t	credits;
 	uint32_t	fi_univ_size = 0;
 	uint32_t	mem_pin_enable = 0;
 	uint32_t	is_secondary;
 	char		ucx_ib_fork_init = 0;
+	uint32_t        post_init = CRT_HG_POST_INIT, post_incr = CRT_HG_POST_INCR;
 	int		rc = 0;
 
 	D_DEBUG(DB_ALL, "initializing crt_gdata...\n");
@@ -273,6 +276,12 @@ static int data_init(int server, crt_init_options_t *opt)
 	D_DEBUG(DB_ALL, "Starting RPCID %#lx. Num cores: %ld\n",
 		crt_gdata.cg_rpcid, crt_gdata.cg_num_cores);
 
+	/* Set context post init / post incr to tune number of pre-posted recvs */
+	d_getenv_int("D_POST_INIT", &post_init);
+	crt_gdata.cg_post_init = post_init;
+	d_getenv_int("D_POST_INCR", &post_incr);
+	crt_gdata.cg_post_incr = post_incr;
+
 	is_secondary = 0;
 	/* Apply CART-890 workaround for server side only */
 	if (server) {
@@ -285,13 +294,9 @@ static int data_init(int server, crt_init_options_t *opt)
 		 * is running using a secondary provider
 		 */
 		d_getenv_int("CRT_SECONDARY_PROVIDER", &is_secondary);
-
 	}
-
 	crt_gdata.cg_provider_is_primary = (is_secondary) ? 0 : 1;
 
-	timeout = 0;
-
 	if (opt && opt->cio_crt_timeout != 0)
 		timeout = opt->cio_crt_timeout;
 	else

diff --git a/src/cart/crt_internal_types.h b/src/cart/crt_internal_types.h
@@ -91,6 +91,10 @@ struct crt_gdata {
 	/** */
 	struct crt_prov_gdata	*cg_prov_gdata_secondary;
 
+	/** Hints to mercury for request post init (ignored for clients) */
+	uint32_t                 cg_post_init;
+	uint32_t                 cg_post_incr;
+
 	/** global timeout value (second) for all RPCs */
 	uint32_t		cg_timeout;