From 22aa7e5add1ee84a71a60a873d69a653b0323945 Mon Sep 17 00:00:00 2001 From: "Cabrera, Jorge" Date: Fri, 21 Jun 2024 16:32:19 -0700 Subject: [PATCH] [v1.22.x] prov/shm: Added support for write() method to submit DSA work DSA driver disabled mapping workqueues to user address space for some products in order to address errata. The DSA driver instead provides write system call as an alternative method to submit work. This commit adds the support to use the write system call. Signed-off-by: Jorge Cabrera (cherry picked from commit bbd62f22c10be9180350cb71dc24e7cab539edf9) --- prov/shm/src/smr_dsa.c | 152 ++++++++++++++++++++++++++++++++++------- 1 file changed, 126 insertions(+), 26 deletions(-) diff --git a/prov/shm/src/smr_dsa.c b/prov/shm/src/smr_dsa.c index 5b82b2aed2a..daa91cbeb4e 100644 --- a/prov/shm/src/smr_dsa.c +++ b/prov/shm/src/smr_dsa.c @@ -50,6 +50,7 @@ #include #include #include +#include // _mm_pause #include "smr_util.h" #include "smr_dsa.h" @@ -79,6 +80,13 @@ struct dsa_cmd_context { void *entry_ptr; }; +struct wq_handle { + union { + void *mmapped; + int fd; + }; +}; + struct smr_dsa_context { struct dsa_hw_desc dsa_work_desc[MAX_CMD_BATCH_SIZE * CMD_CONTEXT_COUNT]; @@ -89,9 +97,11 @@ struct smr_dsa_context { struct dsa_cmd_context dsa_cmd_context[CMD_CONTEXT_COUNT]; struct dsa_bitmap dsa_bitmap; - void *wq_portal[MAX_WQS_PER_EP]; + struct wq_handle wq_handle[MAX_WQS_PER_EP]; int wq_count; int next_wq; + int (*submit_cmd)(struct wq_handle *wq_handle, struct dsa_hw_desc *desc); + void (*close_wq)(struct wq_handle *wq_handle); unsigned long copy_type_stats[2]; unsigned long page_fault_stats[2]; @@ -132,32 +142,41 @@ struct dsa_ops { static void *libdsa_handle = NULL; static struct dsa_ops dsa_ops; -static inline unsigned char dsa_enqcmd(struct dsa_hw_desc *desc, - volatile void *reg) +static int dsa_enq_cmd(struct wq_handle *wq_handle, struct dsa_hw_desc *desc) { + volatile void *reg = (void *) wq_handle->mmapped; unsigned char retry; asm volatile(".byte 0xf2, 0x0f, 0x38, 0xf8, 0x02\t\n" "setz %0\t\n" : "=r"(retry) : "a"(reg), "d"(desc)); - return retry; + return (int) retry; +} + +static int dsa_write_cmd(struct wq_handle *wq_handle, struct dsa_hw_desc *desc) +{ + int ret; + int wq_fd = wq_handle->fd; + + ret = write(wq_fd, desc, sizeof(*desc)); + return ret != sizeof(*desc) ? 1 : 0; } static __always_inline void dsa_desc_submit(struct smr_dsa_context *dsa_context, struct dsa_hw_desc *hw) { - int enq_status; + int status; // make sure writes (e.g., comp.status = 0) are ordered wrt to enqcmd { asm volatile("sfence":::"memory"); } do { - enq_status = dsa_enqcmd(hw, - dsa_context->wq_portal[dsa_context->next_wq]); - dsa_context->next_wq = (dsa_context->next_wq + 1) % - (dsa_context->wq_count); - } while (enq_status); + status = dsa_context->submit_cmd( + &dsa_context->wq_handle[dsa_context->next_wq], hw); + dsa_context->next_wq = + (dsa_context->next_wq + 1) % (dsa_context->wq_count); + } while (status); } static int dsa_open_wq(struct accfg_wq *wq) @@ -184,6 +203,33 @@ static int dsa_open_wq(struct accfg_wq *wq) return fd; } +static bool dsa_write_is_supported(int fd) +{ + int ret; + int max_comp_checks = 65536; + struct dsa_hw_desc desc __attribute__((aligned(64))) = {0}; + struct dsa_completion_record comp __attribute__((aligned(32))) = {0}; + + desc.opcode = DSA_OPCODE_NOOP; + desc.flags = IDXD_OP_FLAG_CRAV | IDXD_OP_FLAG_RCR; + desc.completion_addr = (unsigned long) ∁ + + // make sure writes (e.g., comp.status = 0) are ordered wrt to write() + { asm volatile("sfence":::"memory"); } + + ret = write(fd, &desc, sizeof(desc)); + + if (ret != sizeof(desc)) + return false; + + while (comp.status == DSA_COMP_NONE && max_comp_checks > 0) { + max_comp_checks--; + _mm_pause(); + } + + return comp.status == DSA_COMP_SUCCESS; +} + static void *dsa_idxd_wq_mmap(struct accfg_wq *wq) { int fd; @@ -195,31 +241,55 @@ static void *dsa_idxd_wq_mmap(struct accfg_wq *wq) MAP_SHARED | MAP_POPULATE, fd, 0); close(fd); - if (wq_reg == MAP_FAILED) { - FI_WARN(&smr_prov, FI_LOG_EP_CTRL, "d_idxd_wq_mmap error\n"); + if (wq_reg == MAP_FAILED) return NULL; - } return wq_reg; } -static void dsa_idxd_wq_unmap(void *wq) +static int dsa_idxd_wq_fd(struct accfg_wq *wq, bool check_write_support) { - munmap(wq, LIMITED_MSIX_PORTAL_OFFSET); + int fd; + + fd = dsa_open_wq(wq); + + if (fd < 0) + return fd; + + if (check_write_support) { + if (!dsa_write_is_supported(fd)) { + close(fd); + return -1; + } + } + return fd; +} + +static void dsa_idxd_wq_unmap(struct wq_handle *wq_handle) +{ + munmap(wq_handle->mmapped, LIMITED_MSIX_PORTAL_OFFSET); +} + +static void dsa_idxd_wq_close(struct wq_handle *wq_handle) +{ + close(wq_handle->fd); } static int dsa_idxd_init_wq_array(int shared, int numa_node, - void **wq_array) + struct smr_dsa_context *dsa_context) { static struct accfg_ctx *ctx; struct accfg_wq *wq; void *wq_reg; + int fd; enum accfg_device_state dstate; enum accfg_wq_state wstate; enum accfg_wq_type type; int mode; int wq_count = 0; struct accfg_device *device; + bool wq_mmap_support = true; + bool wq_write_support = false; if ((*dsa_ops.accfg_new)(&ctx) < 0) return 0; @@ -266,18 +336,49 @@ static int dsa_idxd_init_wq_array(int shared, int numa_node, "DSA WQ: %s\n", (*dsa_ops.accfg_wq_get_devname)(wq)); - wq_reg = dsa_idxd_wq_mmap(wq); - if (wq_reg == NULL) - continue; - wq_array[wq_count] = wq_reg; - wq_count++; - break; + fd = -1; + wq_reg = NULL; + + if (wq_mmap_support) { + wq_reg = dsa_idxd_wq_mmap(wq); + if (!wq_reg && wq_count == 0) { + wq_mmap_support = false; + wq_write_support = true; + } else if (wq_reg != NULL) { + dsa_context->wq_handle[wq_count].mmapped = wq_reg; + } + } + + if (wq_write_support) { + fd = dsa_idxd_wq_fd(wq, wq_count == 0); + if (fd < 0 && wq_count == 0) + wq_write_support = false; + else if (fd >= 0) + dsa_context->wq_handle[wq_count].fd = fd; + } + + if (wq_reg || fd >= 0 ) { + wq_count++; + break; + } + } if (wq_count >= MAX_WQS_PER_EP) break; } + if (wq_mmap_support) { + dsa_context->submit_cmd = dsa_enq_cmd; + dsa_context->close_wq = dsa_idxd_wq_unmap; + } else if (wq_write_support) { + dsa_context->submit_cmd = dsa_write_cmd; + dsa_context->close_wq = dsa_idxd_wq_close; + } else { + assert(wq_count == 0); + } + + (*dsa_ops.accfg_unref)(ctx); return wq_count; } @@ -830,12 +931,11 @@ void smr_dsa_context_init(struct smr_ep *ep) dsa_context = ep->dsa_context; memset(dsa_context, 0, sizeof(*dsa_context)); - wq_count = dsa_idxd_init_wq_array(1, numa_node, - dsa_context->wq_portal); + wq_count = dsa_idxd_init_wq_array(1, numa_node, dsa_context); if (wq_count == 0) { FI_WARN(&smr_prov, FI_LOG_EP_CTRL, - "error calling dsa_idxd_init_wq_array()\n"); + "error: wq mmap and wq write not supported\n"); goto wq_get_error; } @@ -879,7 +979,7 @@ void smr_dsa_context_cleanup(struct smr_ep *ep) "doing cleanup\n"); for (i = 0; i < dsa_context->wq_count; i++) - dsa_idxd_wq_unmap(dsa_context->wq_portal[i]); + dsa_context->close_wq(&dsa_context->wq_handle[i]); free(ep->dsa_context); }