improve the setup of LCI_MAX_SINGLE_MESSAGE_SIZE for infiniband;

also change its default to -1.
uiuc-hpc · Nov 10, 2024 · 0616894 · 0616894
1 parent 32165e2
commit 0616894
Show file tree

Hide file tree

Showing 6 changed files with 25 additions and 14 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -207,7 +207,7 @@ if(NOT LCI_WITH_LCT_ONLY)
   set_property(CACHE LCI_RDV_PROTOCOL_DEFAULT PROPERTY STRINGS write writeimm)
 
   set(LCI_MAX_SINGLE_MESSAGE_SIZE_DEFAULT
-      2000000000
+      0x7FFFFFFF
       CACHE STRING "Default single low-level message max size")
 
   mark_as_advanced(

diff --git a/lci/api/lci_config.h.in b/lci/api/lci_config.h.in
@@ -40,7 +40,7 @@
 #cmakedefine01 LCI_IBV_ENABLE_TD_DEFAULT
 #cmakedefine01 LCI_ENABLE_PRG_NET_ENDPOINT_DEFAULT
 #define LCI_RDV_PROTOCOL_DEFAULT "@LCI_RDV_PROTOCOL_DEFAULT@"
-#define LCI_MAX_SINGLE_MESSAGE_SIZE_DEFAULT @LCI_MAX_SINGLE_MESSAGE_SIZE_DEFAULT@
+#define LCI_MAX_SINGLE_MESSAGE_SIZE_DEFAULT (@LCI_MAX_SINGLE_MESSAGE_SIZE_DEFAULT@)
 
 #define LCI_CQ_MAX_POLL 16
 #define LCI_SERVER_MAX_ENDPOINTS 8

diff --git a/lci/backend/ibv/server_ibv.c b/lci/backend/ibv/server_ibv.c
@@ -76,16 +76,6 @@ void LCISI_event_polling_thread_fina(LCISI_server_t* server)
 
 void LCISD_server_init(LCIS_server_t* s)
 {
-  // Check configurations
-  if (LCI_MAX_SINGLE_MESSAGE_SIZE >= 2 << 31) {
-    // ibverbs' max message is 2GiB (or 2GB?)
-    LCI_MAX_SINGLE_MESSAGE_SIZE = 2 << 31 - 1;
-    LCI_Warn(
-        "Reduce LCI_MAX_SINGLE_MESSAGE_SIZE to %lu"
-        "as required by libibverbs max message size\n",
-        LCI_MAX_SINGLE_MESSAGE_SIZE);
-  }
-
   LCISI_server_t* server = LCIU_malloc(sizeof(LCISI_server_t));
   *s = (LCIS_server_t)server;
 
@@ -205,6 +195,15 @@ void LCISD_server_init(LCIS_server_t* s)
           mtu_str(server->port_attr.max_mtu),
           mtu_str(server->port_attr.active_mtu));
 
+  // Check max_msg_sz
+  if (LCI_MAX_SINGLE_MESSAGE_SIZE > server->port_attr.max_msg_sz) {
+    LCI_MAX_SINGLE_MESSAGE_SIZE = server->port_attr.max_msg_sz;
+    LCI_Log(LCI_LOG_INFO, "ibv",
+            "Reduce LCI_MAX_SINGLE_MESSAGE_SIZE to %lu "
+            "as required by libibverbs max message size\n",
+            LCI_MAX_SINGLE_MESSAGE_SIZE);
+  }
+
   // query the gid
   server->gid_idx = LCI_IBV_GID_IDX;
   if (server->gid_idx < 0 &&

diff --git a/lci/backend/ofi/server_ofi.c b/lci/backend/ofi/server_ofi.c
@@ -102,7 +102,7 @@ void LCISD_server_init(LCIS_server_t* s)
   if (server->info->ep_attr->max_msg_size < LCI_MAX_SINGLE_MESSAGE_SIZE) {
     LCI_MAX_SINGLE_MESSAGE_SIZE = server->info->ep_attr->max_msg_size;
     LCI_Warn(
-        "Reduce LCI_MAX_SINGLE_MESSAGE_SIZE to %lu"
+        "Reduce LCI_MAX_SINGLE_MESSAGE_SIZE to %lu "
         "as required by the libfabric max_msg_size attribute\n",
         LCI_MAX_SINGLE_MESSAGE_SIZE);
   }

diff --git a/lci/runtime/env.c b/lci/runtime/env.c
@@ -101,7 +101,7 @@ void LCII_env_init(int num_proc, int rank)
                (LCI_PACKET_SIZE - sizeof(struct LCII_packet_context) -
                 sizeof(struct LCII_packet_rtr_t)) /
                    sizeof(struct LCII_packet_rtr_rbuffer_info_t));
-  LCI_MAX_SINGLE_MESSAGE_SIZE = LCIU_getenv_or(
+  LCI_MAX_SINGLE_MESSAGE_SIZE = LCIU_getenv_or_ul(
       "LCI_MAX_SINGLE_MESSAGE_SIZE", LCI_MAX_SINGLE_MESSAGE_SIZE_DEFAULT);
   LCI_OFI_CXI_TRY_NO_HACK = LCIU_getenv_or("LCI_OFI_CXI_TRY_NO_HACK", false);
   {

diff --git a/lci/sys/lciu_misc.h b/lci/sys/lciu_misc.h
@@ -104,6 +104,18 @@ static inline int LCIU_getenv_or(char* env, int def)
   LCI_Log(LCI_LOG_INFO, "env", "set %s to be %d\n", env, ret);
   return ret;
 }
+static inline size_t LCIU_getenv_or_ul(char* env, size_t def)
+{
+  size_t ret;
+  char* val = getenv(env);
+  if (val != NULL) {
+    LCI_Assert(sscanf(val, "%zu", &ret) == 1, "Unknown value: %lu\n", val);
+  } else {
+    ret = def;
+  }
+  LCI_Log(LCI_LOG_INFO, "env", "set %s to be %lu\n", env, ret);
+  return ret;
+}
 
 static inline void LCIU_spin_for_nsec(double t)
 {