libibumad, libibmad: add support for XDR infiniband tools

This patch adds support for SMI/GSI separation in XDR-supported HCAs by
introducing new APIs and structures in libibumad and libibmad.

In libibumad:
- Add umad_get_smi_gsi_pairs() and umad_get_smi_gsi_pairs_by_name().
- Introduce umad_ca_pair to represent SMI/GSI pairs.

In libibmad:
- Add mad_rpc_open_port2() and mad_rpc_close_port2().
- Return an ibmad_ports_pair structure from mad_rpc_open_port2().

In smpquery:
Demonstrate the new API to retrieve SMI/GSI details and properly use
  the SMI or GSI port based on the call type:
  - GMP calls (e.g., resolve_portid_str()) use the GSI port.
  - SMP calls (e.g., smp_query_via()) use the SMI port.

These changes improve SMI/GSI separation handling and enhance the
usability of the API.

Signed-off-by: Asaf Mazor <[email protected]>

debian/libibmad5.symbols

@@ -156,3 +156,6 @@ libibmad.so.5 libibmad5 #MINVER#
  smp_set_via@IBMAD_1.3 1.3.11
  str2drpath@IBMAD_1.3 1.3.11
  xdump@IBMAD_1.3 1.3.11
+ mad_rpc_open_port2@IBMAD_1.5 5.55.0
+ mad_rpc_close_port2@IBMAD_1.5 5.55.0
+

debian/libibumad3.symbols

@@ -44,3 +44,5 @@ libibumad.so.3 libibumad3 #MINVER#
  umad_sort_ca_device_list@IBUMAD_1.2 3.2.30
  umad_status@IBUMAD_1.0 1.3.9
  umad_unregister@IBUMAD_1.0 1.3.9
+ umad_get_smi_gsi_pairs@IBUMAD_1.4 3.4.55
+ umad_get_smi_gsi_pair_by_ca_name@IBUMAD_1.4 3.4.55

infiniband-diags/smpquery.c

@@ -48,6 +48,7 @@
 #include "ibdiag_common.h"
 
 static struct ibmad_port *srcport;
+static struct ibmad_ports_pair *srcports;
 
 static op_fn_t node_desc, node_info, port_info, switch_info, pkey_table,
     sl2vl_table, vlarb_table, guid_info, mlnx_ext_port_info, port_info_extended;
@@ -475,17 +476,19 @@ int main(int argc, char **argv)
 	if (!(fn = match_op(match_tbl, argv[0])))
 		IBEXIT("operation '%s' not supported", argv[0]);
 
-	srcport = mad_rpc_open_port(ibd_ca, ibd_ca_port, mgmt_classes, 3);
-	if (!srcport)
+	srcports = mad_rpc_open_port2(ibd_ca, ibd_ca_port, mgmt_classes, 3, 1);
+	if (!srcports)
 		IBEXIT("Failed to open '%s' port '%d'", ibd_ca, ibd_ca_port);
 
+	srcport = srcports->smi.port;
+
 	smp_mkey_set(srcport, ibd_mkey);
 
 	node_name_map = open_node_name_map(node_name_map_file);
 
 	if (ibd_dest_type != IB_DEST_DRSLID) {
-		if (resolve_portid_str(ibd_ca, ibd_ca_port, &portid, argv[1],
-				       ibd_dest_type, ibd_sm_id, srcport) < 0)
+		if (resolve_portid_str(srcports->gsi.ca_name, ibd_ca_port, &portid, argv[1],
+				       ibd_dest_type, ibd_sm_id, srcports->gsi.port) < 0)
 			IBEXIT("can't resolve destination port %s", argv[1]);
 		if ((err = fn(&portid, argv + 2, argc - 2)))
 			IBEXIT("operation %s: %s", argv[0], err);
@@ -494,13 +497,13 @@ int main(int argc, char **argv)
 
 		memset(concat, 0, 64);
 		snprintf(concat, sizeof(concat), "%s %s", argv[1], argv[2]);
-		if (resolve_portid_str(ibd_ca, ibd_ca_port, &portid, concat,
+		if (resolve_portid_str(srcports->smi.ca_name, ibd_ca_port, &portid, concat,
 				       ibd_dest_type, ibd_sm_id, srcport) < 0)
 			IBEXIT("can't resolve destination port %s", concat);
 		if ((err = fn(&portid, argv + 3, argc - 3)))
 			IBEXIT("operation %s: %s", argv[0], err);
 	}
 	close_node_name_map(node_name_map);
-	mad_rpc_close_port(srcport);
+	mad_rpc_close_port2(srcports);
 	exit(0);
 }

libibmad/CMakeLists.txt

@@ -9,7 +9,7 @@ publish_internal_headers(util
 
 rdma_library(ibmad libibmad.map
   # See Documentation/versioning.md
-  5 5.4.${PACKAGE_VERSION}
+  5 5.5.${PACKAGE_VERSION}
   bm.c
   cc.c
   dump.c

libibmad/libibmad.map

@@ -161,3 +161,10 @@ IBMAD_1.4 {
 		mad_dump_linkspeedexten2;
 		mad_dump_linkspeedextsup2;
 } IBMAD_1.3;
+
+IBMAD_1.5 {
+	global:
+		mad_rpc_open_port2;
+		mad_rpc_close_port2;
+} IBMAD_1.4;
+

libibmad/mad.h

@@ -1408,6 +1408,16 @@ typedef struct ib_bm_call {
 	uint64_t bkey;
 } ib_bm_call_t;
 
+typedef struct ibmad_ports_item {
+	struct ibmad_port *port;
+	char ca_name[20];
+} ibmad_ports_item_t;
+
+struct ibmad_ports_pair {
+	ibmad_ports_item_t smi;
+	ibmad_ports_item_t gsi;
+};
+
 #define IB_MIN_UCAST_LID	1
 #define IB_MAX_UCAST_LID	(0xc000-1)
 #define IB_MIN_MCAST_LID	0xc000
@@ -1483,7 +1493,10 @@ int madrpc_set_retries(int retries);
 int madrpc_set_timeout(int timeout);
 struct ibmad_port *mad_rpc_open_port(char *dev_name, int dev_port,
 				     int *mgmt_classes, int num_classes);
+struct ibmad_ports_pair *mad_rpc_open_port2(char *dev_name, int dev_port,
+				     int *mgmt_classes, int num_classes, unsigned enforce_smi);
 void mad_rpc_close_port(struct ibmad_port *srcport);
+void mad_rpc_close_port2(struct  ibmad_ports_pair *srcport);
 
 /*
  * On redirection, the dport argument is updated with the redirection target,

libibmad/rpc.c

@@ -436,3 +436,152 @@ void mad_rpc_close_port(struct ibmad_port *port)
 	umad_close_port(port->port_id);
 	free(port);
 }
+
+static int get_smi_gsi_pair(const char *ca_name, int portnum, struct ibmad_ports_pair *ports_pair,
+			    unsigned enforce_smi)
+{
+	struct umad_ca_pair ca_pair;
+	int smi_port_id = -1;
+	int gsi_port_id = -1;
+	int rc = -1;
+	rc = umad_get_smi_gsi_pair_by_ca_name(ca_name, portnum, &ca_pair, enforce_smi);
+
+	if (rc < 0) {
+		IBWARN("Can't open UMAD port (%s) (%s:%d)", strerror(-rc), ca_name, portnum);
+		return rc;
+	}
+	smi_port_id = umad_open_port(ca_pair.smi_name, ca_pair.smi_preferred_port);
+
+	if (smi_port_id < 0 && enforce_smi) {
+		IBWARN("Can't open SMI UMAD port (%s) (%s:%d)", strerror(-smi_port_id), ca_pair.smi_name, ca_pair.smi_preferred_port);
+		return smi_port_id;
+	}
+
+	gsi_port_id = umad_open_port(ca_pair.gsi_name, ca_pair.gsi_preferred_port);
+
+	if (gsi_port_id < 0) {
+		IBWARN("Can't open GSI UMAD port (%s) (%s:%d)", strerror(-gsi_port_id), ca_pair.gsi_name, ca_pair.gsi_preferred_port);
+		umad_close_port(smi_port_id);
+		return gsi_port_id;
+	}
+
+	ports_pair->smi.port->port_id = smi_port_id;
+	ports_pair->gsi.port->port_id = gsi_port_id;
+	strncpy(ports_pair->smi.ca_name, ca_pair.smi_name, UMAD_CA_NAME_LEN);
+	strncpy(ports_pair->gsi.ca_name, ca_pair.gsi_name, UMAD_CA_NAME_LEN);
+
+	return 0;
+}
+
+static struct ibmad_port *get_port_for_class(struct ibmad_ports_pair *ports_pair, int mgmt_class, unsigned enforce_smi)
+{
+	if (mgmt_class != IB_SMI_CLASS && mgmt_class != IB_SMI_DIRECT_CLASS) {
+		if (ports_pair->gsi.port->port_id < 0) {
+			IBWARN("required port for GSI is invalid");
+			return NULL;
+		}
+		return ports_pair->gsi.port;
+	}
+
+	if (ports_pair->smi.port->port_id < 0) {
+		if (enforce_smi) {
+			IBWARN("required port for SMI is invalid");
+			return NULL;
+		}
+		return ports_pair->gsi.port;
+	}
+
+	return ports_pair->smi.port;
+}
+
+struct ibmad_ports_pair *mad_rpc_open_port2(char *dev_name, int dev_port,
+				     int *mgmt_classes, int num_classes, unsigned enforce_smi)
+{
+	struct ibmad_port *smi;
+	struct ibmad_port *gsi;
+	struct ibmad_ports_pair *ports_pair;
+	char *debug_level_env;
+
+	if (num_classes >= MAX_CLASS) {
+		IBWARN("too many classes %d requested", num_classes);
+		errno = EINVAL;
+		return NULL;
+	}
+
+	if (umad_init() < 0) {
+		IBWARN("can't init UMAD library");
+		errno = ENODEV;
+		return NULL;
+	}
+
+	debug_level_env = getenv("LIBIBMAD_DEBUG_LEVEL");
+	if (debug_level_env) {
+		ibdebug = atoi(debug_level_env);
+	}
+
+	smi = malloc(sizeof(*smi));
+	if (!smi) {
+		errno = ENOMEM;
+		return NULL;
+	}
+	memset(smi, 0, sizeof(*smi));
+
+	gsi = malloc(sizeof(*gsi));
+	if (!gsi) {
+		errno = ENOMEM;
+		return NULL;
+	}
+	memset(gsi, 0, sizeof(*gsi));
+
+	ports_pair = malloc(sizeof(*ports_pair));
+	if (!ports_pair) {
+		errno = ENOMEM;
+		return NULL;
+	}
+	memset(ports_pair, 0, sizeof(*ports_pair));
+	ports_pair->smi.port = smi;
+	ports_pair->gsi.port = gsi;
+
+	if ((get_smi_gsi_pair(dev_name, dev_port, ports_pair, enforce_smi)) < 0) {
+		IBWARN("can't open UMAD port (%s:%d)", dev_name, dev_port);
+		if (!errno)
+			errno = EIO;
+		free(smi);
+		free(gsi);
+		free(ports_pair);
+		return NULL;
+	}
+
+	memset(ports_pair->smi.port->class_agents, 0xff, sizeof ports_pair->smi.port->class_agents);
+	memset(ports_pair->gsi.port->class_agents, 0xff, sizeof ports_pair->gsi.port->class_agents);
+	while (num_classes--) {
+		uint8_t rmpp_version = 0;
+		int mgmt = *mgmt_classes++;
+		struct ibmad_port *p = get_port_for_class(ports_pair, mgmt, enforce_smi);
+		if (mgmt == IB_SA_CLASS)
+			rmpp_version = 1;
+		if (mgmt < 0 || mgmt >= MAX_CLASS || !p ||
+		    mad_register_client_via(mgmt, rmpp_version, p) < 0) {
+			IBWARN("client_register for mgmt %d failed", mgmt);
+			if (!errno)
+				errno = EINVAL;
+			umad_close_port(ports_pair->smi.port->port_id);
+			umad_close_port(ports_pair->gsi.port->port_id);
+			free(ports_pair->smi.port);
+			free(ports_pair->gsi.port);
+			free(ports_pair);
+			return NULL;
+		}
+	}
+
+	return ports_pair;
+}
+
+void mad_rpc_close_port2(struct ibmad_ports_pair *srcport)
+{
+	umad_close_port(srcport->smi.port->port_id);
+	umad_close_port(srcport->gsi.port->port_id);
+	free(srcport->smi.port);
+	free(srcport->gsi.port);
+	free(srcport);
+}

libibumad/CMakeLists.txt

@@ -10,7 +10,7 @@ publish_headers(infiniband
 
 rdma_library(ibumad libibumad.map
   # See Documentation/versioning.md
-  3 3.3.${PACKAGE_VERSION}
+  3 3.4.${PACKAGE_VERSION}
   sysfs.c
   umad.c
   umad_str.c

libibumad/libibumad.map

@@ -55,3 +55,10 @@ IBUMAD_1.3 {
 	global:
 		umad_open_smi_port;
 } IBUMAD_1.2;
+
+IBUMAD_1.4 {
+	global:
+		umad_get_smi_gsi_pairs;
+		umad_get_smi_gsi_pair_by_ca_name;
+} IBUMAD_1.3;
+

libibumad/man/umad_get_smi_gsi_pair_by_ca_name.3

@@ -0,0 +1,57 @@
+.. -*- rst -*-
+.. Licensed under the OpenIB.org BSD license (FreeBSD Variant) - See COPYING.md
+
+umad_get_smi_gsi_pair_by_ca_name(3)
+===================================
+
+Retrieve SMI/GSI pair information based on device name and port number.
+
+Synopsis
+--------
+
+.. code-block:: c
+
+    #include <infiniband/umad.h>
+
+    int umad_get_smi_gsi_pair_by_ca_name(const char *devname, uint8_t portnum,
+                                        struct umad_ca_pair *ca, unsigned enforce_smi);
+
+Description
+-----------
+
+``umad_get_smi_gsi_pair_by_ca_name()`` fills the provided ``ca`` structure with the SMI and GSI pair information
+for the specified device name and port number.
+
+The ``devname`` parameter specifies the name of the device, and ``portnum`` is the associated port number.
+
+``enforce_smi`` parameter if enabled, will look only for pairs that has both SMI and GSI interfaces.
+
+The ``struct umad_ca_pair`` is defined in ``<infiniband/umad.h>`` and includes the following members:
+
+.. code-block:: c
+
+    struct umad_ca_pair {
+        char smi_name[UMAD_CA_NAME_LEN];
+        /* Name of the SMI */
+        uint32_t smi_preferred_port;
+        /* Preferred port for the SMI */
+        char gsi_name[UMAD_CA_NAME_LEN];
+        /* Name of the GSI */
+        uint32_t gsi_preferred_port;
+        /* Preferred port for the GSI */
+    };
+
+The function populates this structure with the relevant data for the given ``devname`` and ``portnum``.
+
+Return Value
+------------
+
+``umad_get_smi_gsi_pair_by_ca_name()`` returns:
+
+- **0**: If the specified device and port are found and the structure is successfully populated.
+- **1**: If no matching device or port is found.
+
+Authors
+-------
+
+- Asaf Mazor <[email protected]>

libibumad/man/umad_get_smi_gsi_pairs.3

@@ -0,0 +1,50 @@
+.. -*- rst -*-
+.. Licensed under the OpenIB.org BSD license (FreeBSD Variant) - See COPYING.md
+
+umad_get_smi_gsi_pairs(3)
+=========================
+
+Get CAs as SMI/GSI pairs.
+
+Synopsis
+--------
+
+.. code-block:: c
+
+    #include <infiniband/umad.h>
+
+    int umad_get_smi_gsi_pairs(struct umad_ca_pair cas[], size_t max);
+
+Description
+-----------
+
+``umad_get_smi_gsi_pairs()`` fills a user-allocated array of ``struct umad_ca_pair``.
+It fills up to ``max`` devices.
+
+The argument ``cas`` is an array of ``struct umad_ca_pair`` as specified in ``<infiniband/umad.h>``:
+
+.. code-block:: c
+
+    struct umad_ca_pair {
+        char smi_name[UMAD_CA_NAME_LEN];
+        /* Name of the SMI */
+        uint32_t smi_preferred_port;
+        /* Preferred port for the SMI */
+        char gsi_name[UMAD_CA_NAME_LEN];
+        /* Name of the GSI */
+        uint32_t gsi_preferred_port;
+        /* Preferred port for the GSI */
+    };
+
+The ``smi_preferred_port`` and ``gsi_preferred_port`` fields represent the first ports found active
+for the corresponding SMI/GSI device.
+
+Return Value
+------------
+
+``umad_get_smi_gsi_pairs()`` returns the number of devices filled, or **-1** on error.
+
+Authors
+-------
+
+- Asaf Mazor <[email protected]>