From bca8e4a5057807d8bd3bd3c25809b9570b0475ba Mon Sep 17 00:00:00 2001
From: Venkata Sagar Muppandam
 <96546223+venkatasagarm@users.noreply.github.com>
Date: Mon, 3 Apr 2023 14:19:57 -0700
Subject: [PATCH] Updated dpdk ionic driver which has CMB support (#13)

* Updated dpdk ionic driver which has CMB support

* Updating version to v2.87-pen.2
---
 VERSION                                       |  2 +-
 .../stateful_tests/trex_general_test.py       | 10 +++
 src/dpdk/drivers/net/ionic/ionic_dev.c        |  9 ++-
 src/dpdk/drivers/net/ionic/ionic_dev.h        | 11 +++-
 src/dpdk/drivers/net/ionic/ionic_ethdev.c     | 19 ++++--
 src/dpdk/drivers/net/ionic/ionic_lif.c        | 34 ++++++----
 src/dpdk/drivers/net/ionic/ionic_main.c       |  3 -
 src/dpdk/drivers/net/ionic/ionic_mem_bypass.c |  9 +++
 src/dpdk/drivers/net/ionic/ionic_rxtx.c       | 16 ++++-
 src/dpdk/drivers/net/ionic/ionic_rxtx.h       | 64 +++++++++++++++++++
 src/dpdk/drivers/net/ionic/ionic_rxtx_sg.c    | 36 ++++-------
 .../drivers/net/ionic/ionic_rxtx_simple.c     | 38 ++++-------
 src/dpdk/lib/librte_ethdev/rte_ethdev.h       |  9 ++-
 13 files changed, 179 insertions(+), 81 deletions(-)

diff --git a/VERSION b/VERSION
index d575f54691..f96a70e9da 100755
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-v2.87-pen.1
+v2.87-pen.2
diff --git a/scripts/automation/regression/stateful_tests/trex_general_test.py b/scripts/automation/regression/stateful_tests/trex_general_test.py
index b5d8dd45ff..cde81f78e3 100755
--- a/scripts/automation/regression/stateful_tests/trex_general_test.py
+++ b/scripts/automation/regression/stateful_tests/trex_general_test.py
@@ -634,5 +634,15 @@ def get_per_driver_params(cls):
                 'no_vlan': True,
                 'no_ipv6': True,
             },
+
+            'net_ionic_pci': {
+                'rate_percent': 40,
+                'rate_percent_soft': 0.01 if cls.is_vf_nics else 1,
+                'total_pkts': 1000,
+                'rate_latency': 0.01 if cls.is_vf_nics else 1,
+                'latency_9k_enable': False if cls.is_vf_nics else True,
+                'latency_9k_max_average': 200,
+                'latency_9k_max_latency': 200,
+            },
         }
 
diff --git a/src/dpdk/drivers/net/ionic/ionic_dev.c b/src/dpdk/drivers/net/ionic/ionic_dev.c
index 67fcdc32de..bce14e0a53 100644
--- a/src/dpdk/drivers/net/ionic/ionic_dev.c
+++ b/src/dpdk/drivers/net/ionic/ionic_dev.c
@@ -369,8 +369,7 @@ ionic_q_init(struct ionic_queue *q, uint32_t index, uint16_t num_descs)
 	q->index = index;
 	q->num_descs = num_descs;
 	q->size_mask = num_descs - 1;
-	q->head_idx = 0;
-	q->tail_idx = 0;
+	ionic_q_reset(q);
 
 #ifdef DPDK_SIM
 	struct ionic_qcq *qcq = IONIC_Q_TO_QCQ(q);
@@ -380,10 +379,13 @@ ionic_q_init(struct ionic_queue *q, uint32_t index, uint16_t num_descs)
 }
 
 void
-ionic_q_map(struct ionic_queue *q, void *base, rte_iova_t base_pa)
+ionic_q_map(struct ionic_queue *q, void *base, rte_iova_t base_pa,
+			void *cmb_base, rte_iova_t cmb_base_pa)
 {
 	q->base = base;
 	q->base_pa = base_pa;
+	q->cmb_base = cmb_base;
+	q->cmb_base_pa = cmb_base_pa;
 }
 
 void
@@ -397,5 +399,6 @@ void
 ionic_q_reset(struct ionic_queue *q)
 {
 	q->head_idx = 0;
+	q->cmb_head_idx = 0;
 	q->tail_idx = 0;
 }
diff --git a/src/dpdk/drivers/net/ionic/ionic_dev.h b/src/dpdk/drivers/net/ionic/ionic_dev.h
index fbbcd0998b..0e31d1619b 100644
--- a/src/dpdk/drivers/net/ionic/ionic_dev.h
+++ b/src/dpdk/drivers/net/ionic/ionic_dev.h
@@ -134,12 +134,14 @@ struct ionic_queue {
 	uint16_t num_descs;
 	uint16_t num_segs;
 	uint16_t head_idx;
+	uint16_t cmb_head_idx;
 	uint16_t tail_idx;
 	uint16_t size_mask;
 	uint8_t type;
 	uint8_t hw_type;
 	uint8_t cos;
 	void *base;
+	void *cmb_base;
 	void *sg_base;
 	struct ionic_doorbell __iomem *db;
 	void **info;
@@ -151,6 +153,7 @@ struct ionic_queue {
 	uint32_t hw_index;
 	rte_iova_t base_pa;
 	rte_iova_t sg_base_pa;
+	rte_iova_t cmb_base_pa;
 };
 
 struct ionic_cq {
@@ -227,7 +230,8 @@ uint32_t ionic_cq_service(struct ionic_cq *cq, uint32_t work_to_do,
 
 int ionic_q_init(struct ionic_queue *q, uint32_t index, uint16_t num_descs);
 void ionic_q_reset(struct ionic_queue *q);
-void ionic_q_map(struct ionic_queue *q, void *base, rte_iova_t base_pa);
+void ionic_q_map(struct ionic_queue *q, void *base, rte_iova_t base_pa,
+				 void *cmb_base, rte_iova_t cmb_base_pa);
 void ionic_q_sg_map(struct ionic_queue *q, void *base, rte_iova_t base_pa);
 
 static inline uint16_t
@@ -248,7 +252,10 @@ ionic_q_flush(struct ionic_queue *q)
 {
 	uint64_t val = IONIC_DBELL_QID(q->hw_index) | q->head_idx;
 
-#if defined(DPDK_SIM)
+#if defined(IONIC_EMBEDDED)
+	asm volatile("dsb st" : : : "memory");
+	rte_write64_relaxed(rte_cpu_to_le_64(val), q->db);
+#elif defined(DPDK_SIM)
 	dpdk_sim_write_doorbell(q->name, q->hw_type, val);
 #else
 	rte_write64(rte_cpu_to_le_64(val), q->db);
diff --git a/src/dpdk/drivers/net/ionic/ionic_ethdev.c b/src/dpdk/drivers/net/ionic/ionic_ethdev.c
index 680b8fdac2..3c62fe84f4 100644
--- a/src/dpdk/drivers/net/ionic/ionic_ethdev.c
+++ b/src/dpdk/drivers/net/ionic/ionic_ethdev.c
@@ -298,6 +298,9 @@ ionic_dev_link_update(struct rte_eth_dev *eth_dev,
 		link.link_status = ETH_LINK_UP;
 		link.link_duplex = ETH_LINK_FULL_DUPLEX;
 		switch (adapter->link_speed) {
+		case  1000:
+			link.link_speed = ETH_SPEED_NUM_1G;
+			break;
 		case  10000:
 			link.link_speed = ETH_SPEED_NUM_10G;
 			break;
@@ -313,6 +316,9 @@ ionic_dev_link_update(struct rte_eth_dev *eth_dev,
 		case 100000:
 			link.link_speed = ETH_SPEED_NUM_100G;
 			break;
+		case 200000:
+			link.link_speed = ETH_SPEED_NUM_200G;
+			break;
 		default:
 			link.link_speed = ETH_SPEED_NUM_NONE;
 			break;
@@ -608,12 +614,13 @@ ionic_dev_rss_reta_query(struct rte_eth_dev *eth_dev,
 
 	num = reta_size / RTE_RETA_GROUP_SIZE;
 
-    for (i = 0; i < num; i++) {
-        for(j=0; j < RTE_RETA_GROUP_SIZE; j++) {
-            reta_conf->reta[j] = lif->rss_ind_tbl[(i*RTE_RETA_GROUP_SIZE)+j];
-        }
-        reta_conf++;
-    }
+	for (i = 0; i < num; i++) {
+		for (j = 0; j < RTE_RETA_GROUP_SIZE; j++) {
+			reta_conf->reta[j] =
+				lif->rss_ind_tbl[(i * RTE_RETA_GROUP_SIZE) + j];
+		}
+		reta_conf++;
+	}
 
 	return 0;
 }
diff --git a/src/dpdk/drivers/net/ionic/ionic_lif.c b/src/dpdk/drivers/net/ionic/ionic_lif.c
index 3985802b7f..dc435247ca 100644
--- a/src/dpdk/drivers/net/ionic/ionic_lif.c
+++ b/src/dpdk/drivers/net/ionic/ionic_lif.c
@@ -124,9 +124,7 @@ ionic_lif_get_abs_stats(const struct ionic_lif *lif, struct rte_eth_stats *stats
 	stats->imissed +=
 		ls->rx_ucast_drop_packets +
 		ls->rx_mcast_drop_packets +
-		ls->rx_bcast_drop_packets +
-		ls->rx_queue_empty +
-		ls->rx_queue_disabled;
+		ls->rx_bcast_drop_packets;
 
 	stats->ierrors +=
 		ls->rx_dma_error +
@@ -578,10 +576,11 @@ ionic_qcq_alloc(struct ionic_lif *lif,
 {
 	struct ionic_qcq *new;
 	uint32_t q_size, cq_size, sg_size, total_size;
-	void *q_base, *cq_base, *sg_base;
+	void *q_base, *cmb_q_base, *cq_base, *sg_base;
 	rte_iova_t q_base_pa = 0;
 	rte_iova_t cq_base_pa = 0;
 	rte_iova_t sg_base_pa = 0;
+	rte_iova_t cmb_q_base_pa = 0;
 	size_t page_size = rte_mem_page_size();
 	int err;
 
@@ -616,7 +615,7 @@ ionic_qcq_alloc(struct ionic_lif *lif,
 
 	/* Most queue types will store 1 ptr per descriptor */
 	new->q.info = rte_calloc_socket("ionic",
-				num_descs * num_segs, sizeof(void *),
+				(uint64_t)num_descs * num_segs, sizeof(void *),
 				page_size, socket_id);
 	if (!new->q.info) {
 		IONIC_PRINT(ERR, "Cannot allocate queue info");
@@ -672,19 +671,22 @@ ionic_qcq_alloc(struct ionic_lif *lif,
 			IONIC_PRINT(ERR, "Cannot reserve queue from NIC mem");
 			return -ENOMEM;
 		}
-		q_base = (void *)
+		cmb_q_base = (void *)
 			((uint64_t)lif->adapter->bars.bar[2].vaddr +
 			 lif->adapter->cmb_offset);
 		/* CMB PA is a relative address */
-		q_base_pa = lif->adapter->cmb_offset;
+		cmb_q_base_pa = lif->adapter->cmb_offset;
 		lif->adapter->cmb_offset += q_size;
+	} else {
+		cmb_q_base = NULL;
+		cmb_q_base_pa = 0;
 	}
 
 	IONIC_PRINT(DEBUG, "Q-Base-PA = %#jx CQ-Base-PA = %#jx "
 		"SG-base-PA = %#jx",
 		q_base_pa, cq_base_pa, sg_base_pa);
 
-	ionic_q_map(&new->q, q_base, q_base_pa);
+	ionic_q_map(&new->q, q_base, q_base_pa, cmb_q_base, cmb_q_base_pa);
 	ionic_cq_map(&new->cq, cq_base, cq_base_pa);
 
 	*qcq = new;
@@ -1615,8 +1617,8 @@ ionic_lif_txq_init(struct ionic_tx_qcq *txq)
 			.index = rte_cpu_to_le_32(q->index),
 			.flags = rte_cpu_to_le_16(IONIC_QINIT_F_ENA),
 			.intr_index = rte_cpu_to_le_16(IONIC_INTR_NONE),
+			.cos = q->cos,
 			.ring_size = rte_log2_u32(q->num_descs),
-			.ring_base = rte_cpu_to_le_64(q->base_pa),
 			.cq_ring_base = rte_cpu_to_le_64(cq->base_pa),
 			.sg_ring_base = rte_cpu_to_le_64(q->sg_base_pa),
 		},
@@ -1625,8 +1627,12 @@ ionic_lif_txq_init(struct ionic_tx_qcq *txq)
 
 	if (txq->flags & IONIC_QCQ_F_SG)
 		ctx.cmd.q_init.flags |= rte_cpu_to_le_16(IONIC_QINIT_F_SG);
-	if (txq->flags & IONIC_QCQ_F_CMB)
+	if (txq->flags & IONIC_QCQ_F_CMB) {
 		ctx.cmd.q_init.flags |= rte_cpu_to_le_16(IONIC_QINIT_F_CMB);
+		ctx.cmd.q_init.ring_base = rte_cpu_to_le_64(q->cmb_base_pa);
+	} else {
+		ctx.cmd.q_init.ring_base = rte_cpu_to_le_64(q->base_pa);
+	}
 
 	IONIC_PRINT(DEBUG, "txq_init.index %d", q->index);
 	IONIC_PRINT(DEBUG, "txq_init.ring_base 0x%" PRIx64 "", q->base_pa);
@@ -1670,8 +1676,8 @@ ionic_lif_rxq_init(struct ionic_rx_qcq *rxq)
 			.index = rte_cpu_to_le_32(q->index),
 			.flags = rte_cpu_to_le_16(IONIC_QINIT_F_ENA),
 			.intr_index = rte_cpu_to_le_16(IONIC_INTR_NONE),
+			.cos = q->cos,
 			.ring_size = rte_log2_u32(q->num_descs),
-			.ring_base = rte_cpu_to_le_64(q->base_pa),
 			.cq_ring_base = rte_cpu_to_le_64(cq->base_pa),
 			.sg_ring_base = rte_cpu_to_le_64(q->sg_base_pa),
 		},
@@ -1680,8 +1686,12 @@ ionic_lif_rxq_init(struct ionic_rx_qcq *rxq)
 
 	if (rxq->flags & IONIC_QCQ_F_SG)
 		ctx.cmd.q_init.flags |= rte_cpu_to_le_16(IONIC_QINIT_F_SG);
-	if (rxq->flags & IONIC_QCQ_F_CMB)
+	if (rxq->flags & IONIC_QCQ_F_CMB) {
 		ctx.cmd.q_init.flags |= rte_cpu_to_le_16(IONIC_QINIT_F_CMB);
+		ctx.cmd.q_init.ring_base = rte_cpu_to_le_64(q->cmb_base_pa);
+	} else {
+		ctx.cmd.q_init.ring_base = rte_cpu_to_le_64(q->base_pa);
+	}
 
 	IONIC_PRINT(DEBUG, "rxq_init.index %d", q->index);
 	IONIC_PRINT(DEBUG, "rxq_init.ring_base 0x%" PRIx64 "", q->base_pa);
diff --git a/src/dpdk/drivers/net/ionic/ionic_main.c b/src/dpdk/drivers/net/ionic/ionic_main.c
index e26d1d9924..d3eaf396c8 100644
--- a/src/dpdk/drivers/net/ionic/ionic_main.c
+++ b/src/dpdk/drivers/net/ionic/ionic_main.c
@@ -223,9 +223,6 @@ ionic_adminq_post(struct ionic_lif *lif, struct ionic_admin_ctx *ctx)
 	q->head_idx = Q_NEXT_TO_POST(q, 1);
 
 	/* Ring doorbell */
-#if !defined(RTE_ARCH_ARM64)
-	rte_wmb();
-#endif
 	ionic_q_flush(q);
 
 err_out:
diff --git a/src/dpdk/drivers/net/ionic/ionic_mem_bypass.c b/src/dpdk/drivers/net/ionic/ionic_mem_bypass.c
index 144f50bdad..6364d53a60 100644
--- a/src/dpdk/drivers/net/ionic/ionic_mem_bypass.c
+++ b/src/dpdk/drivers/net/ionic/ionic_mem_bypass.c
@@ -71,6 +71,9 @@ struct bypass_range {
 	struct bypass_pool *pools;
 };
 
+/* Support at most this much bypass memory, taken from *end* of range */
+#define IONIC_MEM_BYPASS_MAX (1ULL << 30) /* 1GB */
+
 static struct capmem_range capmem_ranges[CAPMEM_RANGES_MAX];
 static struct bypass_range bypass_info;
 
@@ -328,6 +331,12 @@ ionic_mem_read_capmem(struct ionic_adapter *adapter, bool do_map)
 		goto out;
 	}
 
+	/* Hack - only use a portion of the bypass region */
+	if (range->len > IONIC_MEM_BYPASS_MAX) {
+		range->start += range->len - IONIC_MEM_BYPASS_MAX;
+		range->len = IONIC_MEM_BYPASS_MAX;
+	}
+
 	map_off = range->start & ~(getpagesize() - 1);
 	map_sz = range->start + range->len - map_off;
 
diff --git a/src/dpdk/drivers/net/ionic/ionic_rxtx.c b/src/dpdk/drivers/net/ionic/ionic_rxtx.c
index c758391f56..ec6242990a 100644
--- a/src/dpdk/drivers/net/ionic/ionic_rxtx.c
+++ b/src/dpdk/drivers/net/ionic/ionic_rxtx.c
@@ -233,6 +233,12 @@ ionic_dev_tx_queue_setup(struct rte_eth_dev *eth_dev, uint16_t tx_queue_id,
 		tx_conf->tx_free_thresh ? tx_conf->tx_free_thresh :
 		nb_desc - IONIC_DEF_TXRX_BURST;
 
+	if (tx_conf->cos != 0) {
+		IONIC_PRINT(DEBUG, "Setting TxQ %u to CoS %u",
+			tx_queue_id, tx_conf->cos);
+		txq->qcq.q.cos = tx_conf->cos;
+	}
+
 	eth_dev->data->tx_queues[tx_queue_id] = txq;
 
 	return 0;
@@ -616,6 +622,12 @@ ionic_dev_rx_queue_setup(struct rte_eth_dev *eth_dev,
 	if (rx_conf->rx_deferred_start)
 		rxq->flags |= IONIC_QCQ_F_DEFERRED;
 
+	if (rx_conf->cos != 0) {
+		IONIC_PRINT(DEBUG, "Setting RxQ %u to CoS %u",
+			rx_queue_id, rx_conf->cos);
+		rxq->qcq.q.cos = rx_conf->cos;
+	}
+
 	eth_dev->data->rx_queues[rx_queue_id] = rxq;
 
 #ifdef IONIC_MEM_BYPASS
@@ -862,7 +874,7 @@ ionic_dev_rx_descriptor_status(void *rx_queue, uint16_t offset)
 {
 	struct ionic_rx_qcq *rxq = rx_queue;
 	struct ionic_qcq *qcq = &rxq->qcq;
-	struct ionic_rxq_comp *cq_desc;
+	volatile struct ionic_rxq_comp *cq_desc;
 	uint16_t mask, head, tail, pos;
 	bool done_color;
 
@@ -901,7 +913,7 @@ ionic_dev_tx_descriptor_status(void *tx_queue, uint16_t offset)
 {
 	struct ionic_tx_qcq *txq = tx_queue;
 	struct ionic_qcq *qcq = &txq->qcq;
-	struct ionic_txq_comp *cq_desc;
+	volatile struct ionic_txq_comp *cq_desc;
 	uint16_t mask, head, tail, pos, cq_pos;
 	bool done_color;
 
diff --git a/src/dpdk/drivers/net/ionic/ionic_rxtx.h b/src/dpdk/drivers/net/ionic/ionic_rxtx.h
index aee8966855..945f2086c4 100644
--- a/src/dpdk/drivers/net/ionic/ionic_rxtx.h
+++ b/src/dpdk/drivers/net/ionic/ionic_rxtx.h
@@ -71,6 +71,70 @@ uint16_t ionic_xmit_pkts_sg(void *tx_queue, struct rte_mbuf **tx_pkts,
 
 int ionic_rx_fill_sg(struct ionic_rx_qcq *rxq);
 
+static inline void
+ionic_rxq_flush(struct ionic_queue *q)
+{
+#ifndef DPDK_SIM
+#ifndef IONIC_EMBEDDED
+	struct ionic_rxq_desc *desc_base = q->base;
+	struct ionic_rxq_desc *cmb_desc_base = q->cmb_base;
+
+	if (q->cmb_base) {
+		if (q->head_idx < q->cmb_head_idx) {
+			// copy [cmb_head, num_descs)
+			rte_memcpy((void *)&cmb_desc_base[q->cmb_head_idx],
+					   (void *)&desc_base[q->cmb_head_idx],
+					   (q->num_descs - q->cmb_head_idx) * sizeof(*desc_base));
+			// copy [0, head)
+			rte_memcpy((void *)&cmb_desc_base[0],
+					   (void *)&desc_base[0],
+					   q->head_idx * sizeof(*desc_base));
+		} else {
+			// copy [cmb_head, head)
+			rte_memcpy((void *)&cmb_desc_base[q->cmb_head_idx],
+					   (void *)&desc_base[q->cmb_head_idx],
+					   (q->head_idx - q->cmb_head_idx) * sizeof(*desc_base));
+		}
+		q->cmb_head_idx = q->head_idx;
+	}
+#endif /* IONIC_EMBEDDED */
+#endif /* DPDK_SIM */
+
+	ionic_q_flush(q);
+}
+
+static inline void
+ionic_txq_flush(struct ionic_queue *q)
+{
+#ifndef DPDK_SIM
+#ifndef IONIC_EMBEDDED
+	struct ionic_txq_desc *desc_base = q->base;
+	struct ionic_txq_desc *cmb_desc_base = q->cmb_base;
+
+	if (q->cmb_base) {
+		if (q->head_idx < q->cmb_head_idx) {
+			// copy [cmb_head, num_descs)
+			rte_memcpy((void *)&cmb_desc_base[q->cmb_head_idx],
+					   (void *)&desc_base[q->cmb_head_idx],
+					   (q->num_descs - q->cmb_head_idx) * sizeof(*desc_base));
+			// copy [0, head)
+			rte_memcpy((void *)&cmb_desc_base[0],
+					   (void *)&desc_base[0],
+					   q->head_idx * sizeof(*desc_base));
+		} else {
+			// copy [cmb_head, head)
+			rte_memcpy((void *)&cmb_desc_base[q->cmb_head_idx],
+					   (void *)&desc_base[q->cmb_head_idx],
+					   (q->head_idx - q->cmb_head_idx) * sizeof(*desc_base));
+		}
+		q->cmb_head_idx = q->head_idx;
+	}
+#endif /* IONIC_EMBEDDED */
+#endif /* DPDK_SIM */
+
+	ionic_q_flush(q);
+}
+
 #if defined(IONIC_CODE_PERF_RX) || defined(IONIC_CODE_PERF_TX)
 static inline uint64_t
 ionic_tsc(void)
diff --git a/src/dpdk/drivers/net/ionic/ionic_rxtx_sg.c b/src/dpdk/drivers/net/ionic/ionic_rxtx_sg.c
index 66668107a9..f0435b4798 100644
--- a/src/dpdk/drivers/net/ionic/ionic_rxtx_sg.c
+++ b/src/dpdk/drivers/net/ionic/ionic_rxtx_sg.c
@@ -56,7 +56,8 @@ ionic_tx_flush_sg(struct ionic_tx_qcq *txq)
 	uint32_t cnt = 0;
 #endif
 	struct rte_mbuf *txm;
-	struct ionic_txq_comp *cq_desc, *cq_desc_base = cq->base;
+	struct ionic_txq_comp *cq_desc_base = cq->base;
+	volatile struct ionic_txq_comp *cq_desc;
 	void **info;
 	uint32_t i;
 
@@ -225,6 +226,9 @@ ionic_xmit_pkts_sg(void *tx_queue, struct rte_mbuf **tx_pkts,
 #endif
 	struct ionic_tx_qcq *txq = tx_queue;
 	struct ionic_queue *q = &txq->qcq.q;
+#ifdef IONIC_PREFETCH
+	struct ionic_txq_desc *desc_base = q->base;
+#endif
 	struct ionic_tx_stats *stats = &txq->stats;
 	struct rte_mbuf *mbuf;
 	uint32_t bytes_tx = 0;
@@ -233,16 +237,10 @@ ionic_xmit_pkts_sg(void *tx_queue, struct rte_mbuf **tx_pkts,
 	int err;
 
 #ifdef IONIC_PREFETCH
-	struct ionic_txq_desc *desc_base = q->base;
-#ifdef IONIC_EMBEDDED
 	rte_prefetch0(&desc_base[q->head_idx]);
-#else
-	if (!(txq->flags & IONIC_QCQ_F_CMB))
-		rte_prefetch0(&desc_base[q->head_idx]);
-#endif
 	rte_prefetch0(IONIC_INFO_PTR(q, q->head_idx));
 
-	if (tx_pkts) {
+	if (nb_pkts) {
 		rte_mbuf_prefetch_part1(tx_pkts[0]);
 		rte_mbuf_prefetch_part2(tx_pkts[0]);
 	}
@@ -266,12 +264,7 @@ ionic_xmit_pkts_sg(void *tx_queue, struct rte_mbuf **tx_pkts,
 	while (nb_tx < nb_pkts) {
 #ifdef IONIC_PREFETCH
 		uint16_t next_idx = Q_NEXT_TO_POST(q, 1);
-#ifdef IONIC_EMBEDDED
 		rte_prefetch0(&desc_base[next_idx]);
-#else
-		if (!(txq->flags & IONIC_QCQ_F_CMB))
-			rte_prefetch0(&desc_base[next_idx]);
-#endif
 		rte_prefetch0(IONIC_INFO_PTR(q, next_idx));
 
 		if (nb_tx + 1 < nb_pkts) {
@@ -296,8 +289,7 @@ ionic_xmit_pkts_sg(void *tx_queue, struct rte_mbuf **tx_pkts,
 	}
 
 	if (nb_tx > 0) {
-		rte_wmb();
-		ionic_q_flush(q);
+		ionic_txq_flush(q);
 
 		txq->last_wdog_cycles = rte_get_timer_cycles();
 
@@ -340,7 +332,7 @@ ionic_xmit_pkts_sg(void *tx_queue, struct rte_mbuf **tx_pkts,
  */
 static __rte_always_inline void
 ionic_rx_clean_one_sg(struct ionic_rx_qcq *rxq,
-		struct ionic_rxq_comp *cq_desc,
+		volatile struct ionic_rxq_comp *cq_desc,
 		struct ionic_rx_service *rx_svc)
 {
 	struct ionic_queue *q = &rxq->qcq.q;
@@ -540,7 +532,8 @@ ionic_rxq_service_sg(struct ionic_rx_qcq *rxq, uint32_t work_to_do,
 #ifdef IONIC_PREFETCH
 	struct ionic_rxq_desc *q_desc_base = q->base;
 #endif
-	struct ionic_rxq_comp *cq_desc, *cq_desc_base = cq->base;
+	struct ionic_rxq_comp *cq_desc_base = cq->base;
+	volatile struct ionic_rxq_comp *cq_desc;
 	uint32_t work_done = 0;
 	uint64_t then, now, hz, delta;
 	int ret;
@@ -560,12 +553,7 @@ ionic_rxq_service_sg(struct ionic_rx_qcq *rxq, uint32_t work_to_do,
 		/* Prefetch 4 x 16B comp */
 		rte_prefetch0(&cq_desc_base[Q_NEXT_TO_SRVC(cq, 4)]);
 		/* Prefetch 4 x 16B descriptors */
-#ifdef IONIC_EMBEDDED
 		rte_prefetch0(&q_desc_base[Q_NEXT_TO_POST(q, 4)]);
-#else
-		if (!(rxq->flags & IONIC_QCQ_F_CMB))
-			rte_prefetch0(&q_desc_base[Q_NEXT_TO_POST(q, 4)]);
-#endif
 #endif
 
 		/* Clean one descriptor */
@@ -587,7 +575,7 @@ ionic_rxq_service_sg(struct ionic_rx_qcq *rxq, uint32_t work_to_do,
 
 	/* Update the queue indices and ring the doorbell */
 	if (work_done) {
-		ionic_q_flush(q);
+		ionic_rxq_flush(q);
 
 		rxq->last_wdog_cycles = rte_get_timer_cycles();
 		rxq->wdog_ms = IONIC_Q_WDOG_MS;
@@ -669,7 +657,7 @@ ionic_rx_fill_sg(struct ionic_rx_qcq *rxq)
 		q->head_idx = Q_NEXT_TO_POST(q, 1);
 	}
 
-	ionic_q_flush(q);
+	ionic_rxq_flush(q);
 
 	return err;
 }
diff --git a/src/dpdk/drivers/net/ionic/ionic_rxtx_simple.c b/src/dpdk/drivers/net/ionic/ionic_rxtx_simple.c
index ed8661bae8..9fa799a3a7 100644
--- a/src/dpdk/drivers/net/ionic/ionic_rxtx_simple.c
+++ b/src/dpdk/drivers/net/ionic/ionic_rxtx_simple.c
@@ -56,7 +56,8 @@ ionic_tx_flush(struct ionic_tx_qcq *txq)
 	uint32_t cnt = 0;
 #endif
 	struct rte_mbuf *txm;
-	struct ionic_txq_comp *cq_desc, *cq_desc_base = cq->base;
+	struct ionic_txq_comp *cq_desc_base = cq->base;
+	volatile struct ionic_txq_comp *cq_desc;
 	void **info;
 
 	cq_desc = &cq_desc_base[cq->tail_idx];
@@ -194,6 +195,9 @@ ionic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
 #endif
 	struct ionic_tx_qcq *txq = tx_queue;
 	struct ionic_queue *q = &txq->qcq.q;
+#ifdef IONIC_PREFETCH
+	struct ionic_txq_desc *desc_base = q->base;
+#endif
 	struct ionic_tx_stats *stats = &txq->stats;
 	struct rte_mbuf *mbuf;
 	uint32_t bytes_tx = 0;
@@ -202,16 +206,10 @@ ionic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
 	int err;
 
 #ifdef IONIC_PREFETCH
-	struct ionic_txq_desc *desc_base = q->base;
-#ifdef IONIC_EMBEDDED
 	rte_prefetch0(&desc_base[q->head_idx]);
-#else
-	if (!(txq->flags & IONIC_QCQ_F_CMB))
-		rte_prefetch0(&desc_base[q->head_idx]);
-#endif
 	rte_prefetch0(&q->info[q->head_idx]);
 
-	if (tx_pkts) {
+	if (nb_pkts) {
 		rte_mbuf_prefetch_part1(tx_pkts[0]);
 		rte_mbuf_prefetch_part2(tx_pkts[0]);
 	}
@@ -235,12 +233,7 @@ ionic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
 	while (nb_tx < nb_pkts) {
 #ifdef IONIC_PREFETCH
 		uint16_t next_idx = Q_NEXT_TO_POST(q, 1);
-#ifdef IONIC_EMBEDDED
 		rte_prefetch0(&desc_base[next_idx]);
-#else
-		if (!(txq->flags & IONIC_QCQ_F_CMB))
-			rte_prefetch0(&desc_base[next_idx]);
-#endif
 		rte_prefetch0(&q->info[next_idx]);
 
 		if (nb_tx + 1 < nb_pkts) {
@@ -265,8 +258,7 @@ ionic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
 	}
 
 	if (nb_tx > 0) {
-		rte_wmb();
-		ionic_q_flush(q);
+		ionic_txq_flush(q);
 
 		txq->last_wdog_cycles = rte_get_timer_cycles();
 
@@ -309,7 +301,7 @@ ionic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
  */
 static __rte_always_inline void
 ionic_rx_clean_one(struct ionic_rx_qcq *rxq,
-		struct ionic_rxq_comp *cq_desc,
+		volatile struct ionic_rxq_comp *cq_desc,
 		struct ionic_rx_service *rx_svc)
 {
 	struct ionic_queue *q = &rxq->qcq.q;
@@ -452,10 +444,9 @@ ionic_rxq_service(struct ionic_rx_qcq *rxq, uint32_t work_to_do,
 #endif
 	struct ionic_cq *cq = &rxq->qcq.cq;
 	struct ionic_queue *q = &rxq->qcq.q;
-#ifdef IONIC_PREFETCH
 	struct ionic_rxq_desc *q_desc_base = q->base;
-#endif
-	struct ionic_rxq_comp *cq_desc, *cq_desc_base = cq->base;
+	struct ionic_rxq_comp *cq_desc_base = cq->base;
+	volatile struct ionic_rxq_comp *cq_desc;
 	uint32_t work_done = 0;
 	uint64_t then, now, hz, delta;
 	int ret;
@@ -476,12 +467,7 @@ ionic_rxq_service(struct ionic_rx_qcq *rxq, uint32_t work_to_do,
 		/* Prefetch 4 x 16B comp */
 		rte_prefetch0(&cq_desc_base[Q_NEXT_TO_SRVC(cq, 4)]);
 		/* Prefetch 4 x 16B descriptors */
-#ifdef IONIC_EMBEDDED
 		rte_prefetch0(&q_desc_base[Q_NEXT_TO_POST(q, 4)]);
-#else
-		if (!(rxq->flags & IONIC_QCQ_F_CMB))
-			rte_prefetch0(&q_desc_base[Q_NEXT_TO_POST(q, 4)]);
-#endif
 #endif
 
 		/* Clean one descriptor */
@@ -504,7 +490,7 @@ ionic_rxq_service(struct ionic_rx_qcq *rxq, uint32_t work_to_do,
 
 	/* Update the queue indices and ring the doorbell */
 	if (work_done) {
-		ionic_q_flush(q);
+		ionic_rxq_flush(q);
 
 		rxq->last_wdog_cycles = rte_get_timer_cycles();
 		rxq->wdog_ms = IONIC_Q_WDOG_MS;
@@ -586,7 +572,7 @@ ionic_rx_fill(struct ionic_rx_qcq *rxq)
 		q->head_idx = Q_NEXT_TO_POST(q, 1);
 	}
 
-	ionic_q_flush(q);
+	ionic_rxq_flush(q);
 
 	return err;
 }
diff --git a/src/dpdk/lib/librte_ethdev/rte_ethdev.h b/src/dpdk/lib/librte_ethdev/rte_ethdev.h
index 20da7e5a0e..232730abb9 100644
--- a/src/dpdk/lib/librte_ethdev/rte_ethdev.h
+++ b/src/dpdk/lib/librte_ethdev/rte_ethdev.h
@@ -300,6 +300,7 @@ struct rte_eth_stats {
 #define ETH_SPEED_NUM_50G      50000 /**<  50 Gbps */
 #define ETH_SPEED_NUM_56G      56000 /**<  56 Gbps */
 #define ETH_SPEED_NUM_100G    100000 /**< 100 Gbps */
+#define ETH_SPEED_NUM_200G    200000 /**< 200 Gbps */
 
 /**
  * A structure used to retrieve link-level information of an Ethernet port.
@@ -852,7 +853,9 @@ struct rte_eth_rxconf {
 	 */
 	uint64_t offloads;
 
-	uint64_t reserved_64s[2]; /**< Reserved for future fields */
+    uint8_t cos;              /**< Hack for Pensando Class of Service */
+    uint8_t reserved_8s[7];
+    uint64_t reserved_64s[1]; /**< Reserved for future fields */
 	void *reserved_ptrs[2];   /**< Reserved for future fields */
 };
 
@@ -873,7 +876,9 @@ struct rte_eth_txconf {
 	 */
 	uint64_t offloads;
 
-	uint64_t reserved_64s[2]; /**< Reserved for future fields */
+    uint8_t cos;              /**< Hack for Pensando Class of Service */
+    uint8_t reserved_8s[7];
+    uint64_t reserved_64s[1]; /**< Reserved for future fields */
 	void *reserved_ptrs[2];   /**< Reserved for future fields */
 };