From 11cc8254267f0f3b4565e293fcaf7ec9f5ebda7f Mon Sep 17 00:00:00 2001 From: Shi Jin Date: Wed, 6 Nov 2024 19:43:27 +0000 Subject: [PATCH] [v1.22.x] prov/efa: Fall back to zero sl when non-zero sl qp creation failed When driver doesn't support non-zero sl, the qp creation will fail. In this case, fallback to zero sl qp. Signed-off-by: Shi Jin (cherry picked from commit 230b840d833ab02fef23a6e4c201c0e3a3c2f518) --- prov/efa/src/efa_base_ep.c | 13 ++++++++++++- prov/efa/src/efa_base_ep.h | 3 ++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/prov/efa/src/efa_base_ep.c b/prov/efa/src/efa_base_ep.c index d022c0e3ef6..db75afa01e0 100644 --- a/prov/efa/src/efa_base_ep.c +++ b/prov/efa/src/efa_base_ep.c @@ -186,14 +186,25 @@ int efa_qp_create(struct efa_qp **qp, struct ibv_qp_init_attr_ex *init_attr_ex, #endif efa_attr.driver_qp_type = EFADV_QP_DRIVER_TYPE_SRD; #if HAVE_EFADV_SL + efa_attr.sl = EFA_QP_DEFAULT_SERVICE_LEVEL; if (tclass == FI_TC_LOW_LATENCY) - efa_attr.sl = EFA_QP_DEFAULT_SERVICE_LEVEL; + efa_attr.sl = EFA_QP_LOW_LATENCY_SERVICE_LEVEL; #endif (*qp)->ibv_qp = efadv_create_qp_ex( init_attr_ex->pd->context, init_attr_ex, &efa_attr, sizeof(struct efadv_qp_init_attr)); } +#if HAVE_EFADV_SL + if (!(*qp)->ibv_qp && tclass == FI_TC_LOW_LATENCY) { + EFA_INFO(FI_LOG_EP_CTRL, "ibv_create_qp failed with sl %u, errno: %d. Retrying with default sl.\n", efa_attr.sl, errno); + efa_attr.sl = EFA_QP_DEFAULT_SERVICE_LEVEL; + (*qp)->ibv_qp = efadv_create_qp_ex( + init_attr_ex->pd->context, init_attr_ex, &efa_attr, + sizeof(struct efadv_qp_init_attr)); + } +#endif + if (!(*qp)->ibv_qp) { EFA_WARN(FI_LOG_EP_CTRL, "ibv_create_qp failed. errno: %d\n", errno); free(*qp); diff --git a/prov/efa/src/efa_base_ep.h b/prov/efa/src/efa_base_ep.h index 6cde8f9f4bf..6a761ce4dc0 100644 --- a/prov/efa/src/efa_base_ep.h +++ b/prov/efa/src/efa_base_ep.h @@ -12,7 +12,8 @@ #include "ofi_util.h" #include "rdm/efa_rdm_protocol.h" -#define EFA_QP_DEFAULT_SERVICE_LEVEL 8 +#define EFA_QP_DEFAULT_SERVICE_LEVEL 0 +#define EFA_QP_LOW_LATENCY_SERVICE_LEVEL 8 struct efa_qp { struct ibv_qp *ibv_qp;