util: Update util_cq to support err_data

The issue with the current implementation of ofi_cq_write_error() is that it just reuses the provider provided err_data pointer. This creates the issue where a provider does not know when it is safe to free the err_data pointer. To fix this, if err_data is valid, create a err_data copy internal to the util_cq code. This internal err_data copy will automatically be freed in ofi_cq_readerr() once copy into the users buffer. After a provider calls ofi_cq_write_error(), they can free their err_data buffer. Signed-off-by: Ian Ziemba <[email protected]>
ofiwg · Oct 27, 2023 · 2942f2a · 2942f2a
1 parent 9446270
commit 2942f2a
Showing 1 changed file with 15 additions and 0 deletions.
diff --git a/prov/util/src/util_cq.c b/prov/util/src/util_cq.c
@@ -85,6 +85,7 @@ static int util_cq_insert_error(struct util_cq *cq,
 				const struct fi_cq_err_entry *err_entry)
 {
 	struct util_cq_aux_entry *entry;
+	void *err_data;
 
 	assert(ofi_genlock_held(&cq->cq_lock));
 	assert(err_entry->err);
@@ -93,6 +94,18 @@ static int util_cq_insert_error(struct util_cq *cq,
 		return -FI_ENOMEM;
 
 	entry->comp = *err_entry;
+
+	if (err_entry->err_data_size) {
+		err_data = mem_dup(err_entry->err_data,
+				   err_entry->err_data_size);
+		if (!err_data) {
+			free(entry);
+			return -FI_ENOMEM;
+		}
+
+		entry->comp.err_data = err_data;
+	}
+
 	util_cq_insert_aux(cq, entry);
 	return 0;
 }
@@ -295,6 +308,8 @@ ssize_t ofi_cq_readerr(struct fid_cq *cq_fid, struct fi_cq_err_entry *buf,
 	ofi_cq_err_memcpy(api_version, buf, &aux_entry->comp);
 
 	slist_remove_head(&cq->aux_queue);
+	if (aux_entry->comp.err_data_size)
+		free(aux_entry->comp.err_data);
 	free(aux_entry);
 	if (slist_empty(&cq->aux_queue)) {
 		ofi_cirque_discard(cq->cirq);