diff --git a/prov/shm/src/smr_dsa.c b/prov/shm/src/smr_dsa.c index 5b82b2aed2a..daa91cbeb4e 100644 --- a/prov/shm/src/smr_dsa.c +++ b/prov/shm/src/smr_dsa.c @@ -50,6 +50,7 @@ #include #include #include +#include // _mm_pause #include "smr_util.h" #include "smr_dsa.h" @@ -79,6 +80,13 @@ struct dsa_cmd_context { void *entry_ptr; }; +struct wq_handle { + union { + void *mmapped; + int fd; + }; +}; + struct smr_dsa_context { struct dsa_hw_desc dsa_work_desc[MAX_CMD_BATCH_SIZE * CMD_CONTEXT_COUNT]; @@ -89,9 +97,11 @@ struct smr_dsa_context { struct dsa_cmd_context dsa_cmd_context[CMD_CONTEXT_COUNT]; struct dsa_bitmap dsa_bitmap; - void *wq_portal[MAX_WQS_PER_EP]; + struct wq_handle wq_handle[MAX_WQS_PER_EP]; int wq_count; int next_wq; + int (*submit_cmd)(struct wq_handle *wq_handle, struct dsa_hw_desc *desc); + void (*close_wq)(struct wq_handle *wq_handle); unsigned long copy_type_stats[2]; unsigned long page_fault_stats[2]; @@ -132,32 +142,41 @@ struct dsa_ops { static void *libdsa_handle = NULL; static struct dsa_ops dsa_ops; -static inline unsigned char dsa_enqcmd(struct dsa_hw_desc *desc, - volatile void *reg) +static int dsa_enq_cmd(struct wq_handle *wq_handle, struct dsa_hw_desc *desc) { + volatile void *reg = (void *) wq_handle->mmapped; unsigned char retry; asm volatile(".byte 0xf2, 0x0f, 0x38, 0xf8, 0x02\t\n" "setz %0\t\n" : "=r"(retry) : "a"(reg), "d"(desc)); - return retry; + return (int) retry; +} + +static int dsa_write_cmd(struct wq_handle *wq_handle, struct dsa_hw_desc *desc) +{ + int ret; + int wq_fd = wq_handle->fd; + + ret = write(wq_fd, desc, sizeof(*desc)); + return ret != sizeof(*desc) ? 1 : 0; } static __always_inline void dsa_desc_submit(struct smr_dsa_context *dsa_context, struct dsa_hw_desc *hw) { - int enq_status; + int status; // make sure writes (e.g., comp.status = 0) are ordered wrt to enqcmd { asm volatile("sfence":::"memory"); } do { - enq_status = dsa_enqcmd(hw, - dsa_context->wq_portal[dsa_context->next_wq]); - dsa_context->next_wq = (dsa_context->next_wq + 1) % - (dsa_context->wq_count); - } while (enq_status); + status = dsa_context->submit_cmd( + &dsa_context->wq_handle[dsa_context->next_wq], hw); + dsa_context->next_wq = + (dsa_context->next_wq + 1) % (dsa_context->wq_count); + } while (status); } static int dsa_open_wq(struct accfg_wq *wq) @@ -184,6 +203,33 @@ static int dsa_open_wq(struct accfg_wq *wq) return fd; } +static bool dsa_write_is_supported(int fd) +{ + int ret; + int max_comp_checks = 65536; + struct dsa_hw_desc desc __attribute__((aligned(64))) = {0}; + struct dsa_completion_record comp __attribute__((aligned(32))) = {0}; + + desc.opcode = DSA_OPCODE_NOOP; + desc.flags = IDXD_OP_FLAG_CRAV | IDXD_OP_FLAG_RCR; + desc.completion_addr = (unsigned long) ∁ + + // make sure writes (e.g., comp.status = 0) are ordered wrt to write() + { asm volatile("sfence":::"memory"); } + + ret = write(fd, &desc, sizeof(desc)); + + if (ret != sizeof(desc)) + return false; + + while (comp.status == DSA_COMP_NONE && max_comp_checks > 0) { + max_comp_checks--; + _mm_pause(); + } + + return comp.status == DSA_COMP_SUCCESS; +} + static void *dsa_idxd_wq_mmap(struct accfg_wq *wq) { int fd; @@ -195,31 +241,55 @@ static void *dsa_idxd_wq_mmap(struct accfg_wq *wq) MAP_SHARED | MAP_POPULATE, fd, 0); close(fd); - if (wq_reg == MAP_FAILED) { - FI_WARN(&smr_prov, FI_LOG_EP_CTRL, "d_idxd_wq_mmap error\n"); + if (wq_reg == MAP_FAILED) return NULL; - } return wq_reg; } -static void dsa_idxd_wq_unmap(void *wq) +static int dsa_idxd_wq_fd(struct accfg_wq *wq, bool check_write_support) { - munmap(wq, LIMITED_MSIX_PORTAL_OFFSET); + int fd; + + fd = dsa_open_wq(wq); + + if (fd < 0) + return fd; + + if (check_write_support) { + if (!dsa_write_is_supported(fd)) { + close(fd); + return -1; + } + } + return fd; +} + +static void dsa_idxd_wq_unmap(struct wq_handle *wq_handle) +{ + munmap(wq_handle->mmapped, LIMITED_MSIX_PORTAL_OFFSET); +} + +static void dsa_idxd_wq_close(struct wq_handle *wq_handle) +{ + close(wq_handle->fd); } static int dsa_idxd_init_wq_array(int shared, int numa_node, - void **wq_array) + struct smr_dsa_context *dsa_context) { static struct accfg_ctx *ctx; struct accfg_wq *wq; void *wq_reg; + int fd; enum accfg_device_state dstate; enum accfg_wq_state wstate; enum accfg_wq_type type; int mode; int wq_count = 0; struct accfg_device *device; + bool wq_mmap_support = true; + bool wq_write_support = false; if ((*dsa_ops.accfg_new)(&ctx) < 0) return 0; @@ -266,18 +336,49 @@ static int dsa_idxd_init_wq_array(int shared, int numa_node, "DSA WQ: %s\n", (*dsa_ops.accfg_wq_get_devname)(wq)); - wq_reg = dsa_idxd_wq_mmap(wq); - if (wq_reg == NULL) - continue; - wq_array[wq_count] = wq_reg; - wq_count++; - break; + fd = -1; + wq_reg = NULL; + + if (wq_mmap_support) { + wq_reg = dsa_idxd_wq_mmap(wq); + if (!wq_reg && wq_count == 0) { + wq_mmap_support = false; + wq_write_support = true; + } else if (wq_reg != NULL) { + dsa_context->wq_handle[wq_count].mmapped = wq_reg; + } + } + + if (wq_write_support) { + fd = dsa_idxd_wq_fd(wq, wq_count == 0); + if (fd < 0 && wq_count == 0) + wq_write_support = false; + else if (fd >= 0) + dsa_context->wq_handle[wq_count].fd = fd; + } + + if (wq_reg || fd >= 0 ) { + wq_count++; + break; + } + } if (wq_count >= MAX_WQS_PER_EP) break; } + if (wq_mmap_support) { + dsa_context->submit_cmd = dsa_enq_cmd; + dsa_context->close_wq = dsa_idxd_wq_unmap; + } else if (wq_write_support) { + dsa_context->submit_cmd = dsa_write_cmd; + dsa_context->close_wq = dsa_idxd_wq_close; + } else { + assert(wq_count == 0); + } + + (*dsa_ops.accfg_unref)(ctx); return wq_count; } @@ -830,12 +931,11 @@ void smr_dsa_context_init(struct smr_ep *ep) dsa_context = ep->dsa_context; memset(dsa_context, 0, sizeof(*dsa_context)); - wq_count = dsa_idxd_init_wq_array(1, numa_node, - dsa_context->wq_portal); + wq_count = dsa_idxd_init_wq_array(1, numa_node, dsa_context); if (wq_count == 0) { FI_WARN(&smr_prov, FI_LOG_EP_CTRL, - "error calling dsa_idxd_init_wq_array()\n"); + "error: wq mmap and wq write not supported\n"); goto wq_get_error; } @@ -879,7 +979,7 @@ void smr_dsa_context_cleanup(struct smr_ep *ep) "doing cleanup\n"); for (i = 0; i < dsa_context->wq_count; i++) - dsa_idxd_wq_unmap(dsa_context->wq_portal[i]); + dsa_context->close_wq(&dsa_context->wq_handle[i]); free(ep->dsa_context); }