Skip to content

Commit

Permalink
[v1.21.x] prov/shm: Added support for write() method to submit DSA work
Browse files Browse the repository at this point in the history 
DSA driver disabled mapping workqueues to user address space for some
products in order to address errata. The DSA driver instead provides write
system call as an alternative method to submit work. This commit adds the
support to use the write system call.

Signed-off-by: Jorge Cabrera <[email protected]>
(cherry picked from commit bbd62f2)
  • Loading branch information
@jcabr020 @j-xiong
jcabr020 authored and j-xiong committed Jul 19, 2024
1 parent 8fe4a79 commit bbe9c9a
Showing 1 changed file with 126 additions and 26 deletions.
152 changes: 126 additions & 26 deletions prov/shm/src/smr_dsa.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@
#include <accel-config/libaccel_config.h>
#include <linux/idxd.h>
#include <numa.h>
#include <immintrin.h> // _mm_pause
#include "smr_util.h"
#include "smr_dsa.h"

Expand Down Expand Up @@ -79,6 +80,13 @@ struct dsa_cmd_context {
void *entry_ptr;
};

struct wq_handle {
union {
void *mmapped;
int fd;
};
};

struct smr_dsa_context {
struct dsa_hw_desc dsa_work_desc[MAX_CMD_BATCH_SIZE *
CMD_CONTEXT_COUNT];
Expand All @@ -89,9 +97,11 @@ struct smr_dsa_context {
struct dsa_cmd_context dsa_cmd_context[CMD_CONTEXT_COUNT];

struct dsa_bitmap dsa_bitmap;
void *wq_portal[MAX_WQS_PER_EP];
struct wq_handle wq_handle[MAX_WQS_PER_EP];
int wq_count;
int next_wq;
int (*submit_cmd)(struct wq_handle *wq_handle, struct dsa_hw_desc *desc);
void (*close_wq)(struct wq_handle *wq_handle);

unsigned long copy_type_stats[2];
unsigned long page_fault_stats[2];
Expand Down Expand Up @@ -132,32 +142,41 @@ struct dsa_ops {
static void *libdsa_handle = NULL;
static struct dsa_ops dsa_ops;

static inline unsigned char dsa_enqcmd(struct dsa_hw_desc *desc,
volatile void *reg)
static int dsa_enq_cmd(struct wq_handle *wq_handle, struct dsa_hw_desc *desc)
{
volatile void *reg = (void *) wq_handle->mmapped;
unsigned char retry;

asm volatile(".byte 0xf2, 0x0f, 0x38, 0xf8, 0x02\t\n"
"setz %0\t\n"
: "=r"(retry)
: "a"(reg), "d"(desc));
return retry;
return (int) retry;
}

static int dsa_write_cmd(struct wq_handle *wq_handle, struct dsa_hw_desc *desc)
{
int ret;
int wq_fd = wq_handle->fd;

ret = write(wq_fd, desc, sizeof(*desc));
return ret != sizeof(*desc) ? 1 : 0;
}

static __always_inline void dsa_desc_submit(struct smr_dsa_context *dsa_context,
struct dsa_hw_desc *hw)
{
int enq_status;
int status;

// make sure writes (e.g., comp.status = 0) are ordered wrt to enqcmd
{ asm volatile("sfence":::"memory"); }

do {
enq_status = dsa_enqcmd(hw,
dsa_context->wq_portal[dsa_context->next_wq]);
dsa_context->next_wq = (dsa_context->next_wq + 1) %
(dsa_context->wq_count);
} while (enq_status);
status = dsa_context->submit_cmd(
&dsa_context->wq_handle[dsa_context->next_wq], hw);
dsa_context->next_wq =
(dsa_context->next_wq + 1) % (dsa_context->wq_count);
} while (status);
}

static int dsa_open_wq(struct accfg_wq *wq)
Expand All @@ -184,6 +203,33 @@ static int dsa_open_wq(struct accfg_wq *wq)
return fd;
}

static bool dsa_write_is_supported(int fd)
{
int ret;
int max_comp_checks = 65536;
struct dsa_hw_desc desc __attribute__((aligned(64))) = {0};
struct dsa_completion_record comp __attribute__((aligned(32))) = {0};

desc.opcode = DSA_OPCODE_NOOP;
desc.flags = IDXD_OP_FLAG_CRAV | IDXD_OP_FLAG_RCR;
desc.completion_addr = (unsigned long) &comp;

// make sure writes (e.g., comp.status = 0) are ordered wrt to write()
{ asm volatile("sfence":::"memory"); }

ret = write(fd, &desc, sizeof(desc));

if (ret != sizeof(desc))
return false;

while (comp.status == DSA_COMP_NONE && max_comp_checks > 0) {
max_comp_checks--;
_mm_pause();
}

return comp.status == DSA_COMP_SUCCESS;
}

static void *dsa_idxd_wq_mmap(struct accfg_wq *wq)
{
int fd;
Expand All @@ -195,31 +241,55 @@ static void *dsa_idxd_wq_mmap(struct accfg_wq *wq)
MAP_SHARED | MAP_POPULATE, fd, 0);
close(fd);

if (wq_reg == MAP_FAILED) {
FI_WARN(&smr_prov, FI_LOG_EP_CTRL, "d_idxd_wq_mmap error\n");
if (wq_reg == MAP_FAILED)
return NULL;
}

return wq_reg;
}

static void dsa_idxd_wq_unmap(void *wq)
static int dsa_idxd_wq_fd(struct accfg_wq *wq, bool check_write_support)
{
munmap(wq, LIMITED_MSIX_PORTAL_OFFSET);
int fd;

fd = dsa_open_wq(wq);

if (fd < 0)
return fd;

if (check_write_support) {
if (!dsa_write_is_supported(fd)) {
close(fd);
return -1;
}
}
return fd;
}

static void dsa_idxd_wq_unmap(struct wq_handle *wq_handle)
{
munmap(wq_handle->mmapped, LIMITED_MSIX_PORTAL_OFFSET);
}

static void dsa_idxd_wq_close(struct wq_handle *wq_handle)
{
close(wq_handle->fd);
}

static int dsa_idxd_init_wq_array(int shared, int numa_node,
void **wq_array)
struct smr_dsa_context *dsa_context)
{
static struct accfg_ctx *ctx;
struct accfg_wq *wq;
void *wq_reg;
int fd;
enum accfg_device_state dstate;
enum accfg_wq_state wstate;
enum accfg_wq_type type;
int mode;
int wq_count = 0;
struct accfg_device *device;
bool wq_mmap_support = true;
bool wq_write_support = false;

if ((*dsa_ops.accfg_new)(&ctx) < 0)
return 0;
Expand Down Expand Up @@ -266,18 +336,49 @@ static int dsa_idxd_init_wq_array(int shared, int numa_node,
"DSA WQ: %s\n",
(*dsa_ops.accfg_wq_get_devname)(wq));

wq_reg = dsa_idxd_wq_mmap(wq);
if (wq_reg == NULL)
continue;
wq_array[wq_count] = wq_reg;
wq_count++;
break;
fd = -1;
wq_reg = NULL;

if (wq_mmap_support) {
wq_reg = dsa_idxd_wq_mmap(wq);
if (!wq_reg && wq_count == 0) {
wq_mmap_support = false;
wq_write_support = true;
} else if (wq_reg != NULL) {
dsa_context->wq_handle[wq_count].mmapped = wq_reg;
}
}

if (wq_write_support) {
fd = dsa_idxd_wq_fd(wq, wq_count == 0);
if (fd < 0 && wq_count == 0)
wq_write_support = false;
else if (fd >= 0)
dsa_context->wq_handle[wq_count].fd = fd;
}

if (wq_reg || fd >= 0 ) {
wq_count++;
break;
}

}

if (wq_count >= MAX_WQS_PER_EP)
break;
}

if (wq_mmap_support) {
dsa_context->submit_cmd = dsa_enq_cmd;
dsa_context->close_wq = dsa_idxd_wq_unmap;
} else if (wq_write_support) {
dsa_context->submit_cmd = dsa_write_cmd;
dsa_context->close_wq = dsa_idxd_wq_close;
} else {
assert(wq_count == 0);
}


(*dsa_ops.accfg_unref)(ctx);
return wq_count;
}
Expand Down Expand Up @@ -830,12 +931,11 @@ void smr_dsa_context_init(struct smr_ep *ep)
dsa_context = ep->dsa_context;
memset(dsa_context, 0, sizeof(*dsa_context));

wq_count = dsa_idxd_init_wq_array(1, numa_node,
dsa_context->wq_portal);
wq_count = dsa_idxd_init_wq_array(1, numa_node, dsa_context);

if (wq_count == 0) {
FI_WARN(&smr_prov, FI_LOG_EP_CTRL,
"error calling dsa_idxd_init_wq_array()\n");
"error: wq mmap and wq write not supported\n");
goto wq_get_error;
}

Expand Down Expand Up @@ -879,7 +979,7 @@ void smr_dsa_context_cleanup(struct smr_ep *ep)
"doing cleanup\n");

for (i = 0; i < dsa_context->wq_count; i++)
dsa_idxd_wq_unmap(dsa_context->wq_portal[i]);
dsa_context->close_wq(&dsa_context->wq_handle[i]);

free(ep->dsa_context);
}
Expand Down

0 comments on commit bbe9c9a

Please sign in to comment.