From 5576f08be108183a1a56044c161825fcf1a20689 Mon Sep 17 00:00:00 2001 From: Ian Ziemba Date: Tue, 19 Sep 2023 12:16:55 -0500 Subject: [PATCH] core: Support multiple auth keys per EP fi_domain_attr::max_ep_auth_key_cnt is used to reported the number of authorization keys supported by an endpoint. If this value is non-zero, connectionless endpoints must implement FI_AV_AUTH_KEY. FI_AV_AUTH_KEY is set by libfabric users via fi_domain_attr:::auth_key_size to denoted if MR and EP authorization keys from the AV instead of MR and EP attrs. When set, providers will ignore fi_ep_attr::auth_key during endpoint enable. From MRs, fi_mr_regattr() must be used with fi_mr_attr::auth_key pointing to a struct fi_mr_auth_key and fi_mr_attr:auth_key_size equal to sizeof(struct fi_mr_auth_key). fi_mr_auth_key::av should point to the AV the MR authorization keys should come from. If FI_DIRECTED_RECV is supported, fi_mr_auth_key::key is used to restrict the MR to a specific authorization key. All eligible authorization keys must be pre-inserted into the AV via fi_av_insert_auth_key(). Acceptable flags are the following: - FI_TRANSMIT: Restrict the authorization key to outbound data transfers. This includes send message, RMA, and atomic operations. - FI_RECV: Restrict the authorization key to inbound data transfers. This includes received messages and target MRs of RMA and atomic operations. fi_av_insert_auth_key() output is an fi_addr_t handle specific to this authorization key. If the EP is configured with FI_DIRECTED_RECV, this fi_addr_t can be used to match all EP addrs associated with this authorization key. Calling fi_av_remove() with this fi_addr_t will delete the authorization key. -FI_EBUSY will be returned from fi_av_remove() should this key still be used by en EP. In other words, all EPs using this authorization key need to be closed for fi_av_remove() to succeed. Once the AV is bound to an EP and the EP is successfully enabled, the EP will be configured to support all auth keys in the AV at that point in time. Users must provide an authorization key fi_addr_t with fi_av_insert_{addr, svc, sym}. This is done by using the fi_addr as input. For fi_av_insert_{addr, sym}, since fi_addr may be an array, only index 0 will be looked at for the authorization key fi_addr_t. That is only a single authorization key fi_addr_t will be supported for these functions. The output of fi_av_insert_{addr, svc, sym} is an fi_addr_t mapping to a specific tuple. For FI_EADDRNOTAVAIL CQ errors, fi_cq_err_entry::src_addr will return the authorization key handle associated with the incoming data transfer. This, combined with the existing behavior of fi_cq_err_entry::err_data enables users to generate a fi_addr_t mapping to the specific tuple which triggered the FI_EADDRNOTAVAIL event. Signed-off-by: Ian Ziemba --- include/rdma/fabric.h | 1 + include/rdma/fi_domain.h | 28 ++++++++++++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/include/rdma/fabric.h b/include/rdma/fabric.h index 5de2a6aebf2..042b9a2ec20 100644 --- a/include/rdma/fabric.h +++ b/include/rdma/fabric.h @@ -226,6 +226,7 @@ enum { #define FI_ADDR_NOTAVAIL ((uint64_t) -1) #define FI_KEY_NOTAVAIL ((uint64_t) -1) #define FI_SHARED_CONTEXT SIZE_MAX +#define FI_AV_AUTH_KEY SIZE_MAX typedef uint64_t fi_addr_t; enum fi_av_type { diff --git a/include/rdma/fi_domain.h b/include/rdma/fi_domain.h index 957fb1d4494..3ab95baef77 100644 --- a/include/rdma/fi_domain.h +++ b/include/rdma/fi_domain.h @@ -102,6 +102,11 @@ struct fi_ops_av { char *buf, size_t *len); int (*av_set)(struct fid_av *av, struct fi_av_set_attr *attr, struct fid_av_set **av_set, void *context); + int (*insert_auth_key)(struct fid_av *av, const void *auth_key, + size_t auth_key_size, fi_addr_t *fi_addr, + uint64_t flags); + int (*lookup_auth_key)(struct fid_av *av, fi_addr_t fi_addr, + void *auth_key, size_t *auth_key_size); }; struct fid_av { @@ -135,6 +140,11 @@ static inline int fi_hmem_ze_device(int driver_index, int device_index) return driver_index << 16 | device_index; } +struct fi_mr_auth_key { + struct fid_av *av; + fi_addr_t key; +}; + struct fi_mr_attr { const struct iovec *mr_iov; size_t iov_count; @@ -523,6 +533,24 @@ fi_av_straddr(struct fid_av *av, const void *addr, char *buf, size_t *len) return av->ops->straddr(av, addr, buf, len); } +static inline int +fi_av_insert_auth_key(struct fid_av *av, const void *auth_key, + size_t auth_key_size, fi_addr_t *fi_addr, uint64_t flags) +{ + return FI_CHECK_OP(av->ops, struct fi_ops_av, insert_auth_key) ? + av->ops->insert_auth_key(av, auth_key, auth_key_size, fi_addr, + flags) : -FI_ENOSYS; +} + +static inline int +fi_av_lookup_auth_key(struct fid_av *av, fi_addr_t addr, void *auth_key, + size_t *auth_key_size) +{ + return FI_CHECK_OP(av->ops, struct fi_ops_av, lookup_auth_key) ? + av->ops->lookup_auth_key(av, addr, auth_key, auth_key_size) : + -FI_ENOSYS; +} + static inline fi_addr_t fi_rx_addr(fi_addr_t fi_addr, int rx_index, int rx_ctx_bits) {