diff --git a/man/fi_efa.7.md b/man/fi_efa.7.md index f2bb7a3e9ce..686c5e74a67 100644 --- a/man/fi_efa.7.md +++ b/man/fi_efa.7.md @@ -317,6 +317,10 @@ due to other requirements. See https://github.com/ofiwg/libfabric/blob/main/prov/efa/docs/efa_rdm_protocol_v4.md for details. +*FI_EFA_USE_UNSOLICITED_WRITE_RECV* +: Use device's unsolicited write recv functionality when it's available. (Default: 1). +Setting this environment variable to 0 can disable this feature. + # SEE ALSO [`fabric`(7)](fabric.7.html), diff --git a/prov/efa/src/efa_base_ep.c b/prov/efa/src/efa_base_ep.c index 5eb69ad5892..8c55fee2387 100644 --- a/prov/efa/src/efa_base_ep.c +++ b/prov/efa/src/efa_base_ep.c @@ -181,7 +181,7 @@ int efa_qp_create(struct efa_qp **qp, struct ibv_qp_init_attr_ex *init_attr_ex) } else { assert(init_attr_ex->qp_type == IBV_QPT_DRIVER); #if HAVE_CAPS_UNSOLICITED_WRITE_RECV - if (efa_device_support_unsolicited_write_recv()) + if (efa_rdm_use_unsolicited_write_recv()) efa_attr.flags |= EFADV_QP_FLAGS_UNSOLICITED_WRITE_RECV; #endif efa_attr.driver_qp_type = EFADV_QP_DRIVER_TYPE_SRD; diff --git a/prov/efa/src/efa_cq.h b/prov/efa/src/efa_cq.h index e3486f43f46..18cf435023b 100644 --- a/prov/efa/src/efa_cq.h +++ b/prov/efa/src/efa_cq.h @@ -131,7 +131,7 @@ static inline int efa_cq_ibv_cq_ex_open(struct fi_cq_attr *attr, }; #if HAVE_CAPS_UNSOLICITED_WRITE_RECV - if (efa_device_support_unsolicited_write_recv()) + if (efa_rdm_use_unsolicited_write_recv()) efadv_cq_init_attr.wc_flags |= EFADV_WC_EX_WITH_IS_UNSOLICITED; #endif diff --git a/prov/efa/src/efa_env.c b/prov/efa/src/efa_env.c index 1c54851acf5..84614a06426 100644 --- a/prov/efa/src/efa_env.c +++ b/prov/efa/src/efa_env.c @@ -37,6 +37,7 @@ struct efa_env efa_env = { .host_id_file = "/sys/devices/virtual/dmi/id/board_asset_tag", /* Available on EC2 instances and containers */ .use_sm2 = false, .huge_page_setting = EFA_ENV_HUGE_PAGE_UNSPEC, + .use_unsolicited_write_recv = 1, }; /** @@ -141,6 +142,7 @@ void efa_env_param_get(void) fi_param_get_size_t(&efa_prov, "inter_max_gdrcopy_message_size", &efa_env.efa_max_gdrcopy_msg_size); fi_param_get_bool(&efa_prov, "use_sm2", &efa_env.use_sm2); + fi_param_get_bool(&efa_prov, "use_unsolicited_write_recv", &efa_env.use_unsolicited_write_recv); int use_huge_page; if (fi_param_get_bool(&efa_prov, "use_huge_page", &use_huge_page) ==0) { @@ -216,6 +218,8 @@ void efa_env_define() "Using huge page memory has a small performance advantage, but can " "cause system to run out of huge page memory. By default, EFA provider " "will use huge page unless FI_EFA_FORK_SAFE is set to 1/on/true."); + fi_param_define(&efa_prov, "use_unsolicited_write_recv", FI_PARAM_BOOL, + "Use device's unsolicited write recv functionality when it's available. (Default: true)"); } diff --git a/prov/efa/src/efa_env.h b/prov/efa/src/efa_env.h index 2d315d7bedb..48742fe4097 100644 --- a/prov/efa/src/efa_env.h +++ b/prov/efa/src/efa_env.h @@ -77,6 +77,7 @@ struct efa_env { char *host_id_file; int use_sm2; enum efa_env_huge_page_setting huge_page_setting; + int use_unsolicited_write_recv; }; /** diff --git a/prov/efa/src/rdm/efa_rdm_cq.c b/prov/efa/src/rdm/efa_rdm_cq.c index ba1b218a31a..3d0578147cd 100644 --- a/prov/efa/src/rdm/efa_rdm_cq.c +++ b/prov/efa/src/rdm/efa_rdm_cq.c @@ -83,7 +83,7 @@ static struct fi_ops efa_rdm_cq_fi_ops = { static inline bool efa_rdm_cq_wc_is_unsolicited(struct ibv_cq_ex *ibv_cq_ex) { - return efa_device_support_unsolicited_write_recv() ? efadv_wc_is_unsolicited(efadv_cq_from_ibv_cq_ex(ibv_cq_ex)) : false; + return efa_rdm_use_unsolicited_write_recv() ? efadv_wc_is_unsolicited(efadv_cq_from_ibv_cq_ex(ibv_cq_ex)) : false; } #else diff --git a/prov/efa/src/rdm/efa_rdm_util.h b/prov/efa/src/rdm/efa_rdm_util.h index 02c91772c33..1b2fc1da0a2 100644 --- a/prov/efa/src/rdm/efa_rdm_util.h +++ b/prov/efa/src/rdm/efa_rdm_util.h @@ -30,4 +30,10 @@ static inline void efa_rdm_poison_mem_region(void *ptr, size_t size) } #endif +static inline +bool efa_rdm_use_unsolicited_write_recv() +{ + return efa_env.use_unsolicited_write_recv && efa_device_support_unsolicited_write_recv(); +} + #endif /* _EFA_RDM_UTIL_H */