Updated dpdk ionic driver which has CMB support (#13)

* Updated dpdk ionic driver which has CMB support

* Updating version to v2.87-pen.2

VERSION

@@ -1 +1 @@
-v2.87-pen.1
+v2.87-pen.2

scripts/automation/regression/stateful_tests/trex_general_test.py

@@ -634,5 +634,15 @@ def get_per_driver_params(cls):
                 'no_vlan': True,
                 'no_ipv6': True,
             },
+
+            'net_ionic_pci': {
+                'rate_percent': 40,
+                'rate_percent_soft': 0.01 if cls.is_vf_nics else 1,
+                'total_pkts': 1000,
+                'rate_latency': 0.01 if cls.is_vf_nics else 1,
+                'latency_9k_enable': False if cls.is_vf_nics else True,
+                'latency_9k_max_average': 200,
+                'latency_9k_max_latency': 200,
+            },
         }
 

src/dpdk/drivers/net/ionic/ionic_dev.c

@@ -369,8 +369,7 @@ ionic_q_init(struct ionic_queue *q, uint32_t index, uint16_t num_descs)
 	q->index = index;
 	q->num_descs = num_descs;
 	q->size_mask = num_descs - 1;
-	q->head_idx = 0;
-	q->tail_idx = 0;
+	ionic_q_reset(q);
 
 #ifdef DPDK_SIM
 	struct ionic_qcq *qcq = IONIC_Q_TO_QCQ(q);
@@ -380,10 +379,13 @@ ionic_q_init(struct ionic_queue *q, uint32_t index, uint16_t num_descs)
 }
 
 void
-ionic_q_map(struct ionic_queue *q, void *base, rte_iova_t base_pa)
+ionic_q_map(struct ionic_queue *q, void *base, rte_iova_t base_pa,
+			void *cmb_base, rte_iova_t cmb_base_pa)
 {
 	q->base = base;
 	q->base_pa = base_pa;
+	q->cmb_base = cmb_base;
+	q->cmb_base_pa = cmb_base_pa;
 }
 
 void
@@ -397,5 +399,6 @@ void
 ionic_q_reset(struct ionic_queue *q)
 {
 	q->head_idx = 0;
+	q->cmb_head_idx = 0;
 	q->tail_idx = 0;
 }

src/dpdk/drivers/net/ionic/ionic_dev.h

@@ -134,12 +134,14 @@ struct ionic_queue {
 	uint16_t num_descs;
 	uint16_t num_segs;
 	uint16_t head_idx;
+	uint16_t cmb_head_idx;
 	uint16_t tail_idx;
 	uint16_t size_mask;
 	uint8_t type;
 	uint8_t hw_type;
 	uint8_t cos;
 	void *base;
+	void *cmb_base;
 	void *sg_base;
 	struct ionic_doorbell __iomem *db;
 	void **info;
@@ -151,6 +153,7 @@ struct ionic_queue {
 	uint32_t hw_index;
 	rte_iova_t base_pa;
 	rte_iova_t sg_base_pa;
+	rte_iova_t cmb_base_pa;
 };
 
 struct ionic_cq {
@@ -227,7 +230,8 @@ uint32_t ionic_cq_service(struct ionic_cq *cq, uint32_t work_to_do,
 
 int ionic_q_init(struct ionic_queue *q, uint32_t index, uint16_t num_descs);
 void ionic_q_reset(struct ionic_queue *q);
-void ionic_q_map(struct ionic_queue *q, void *base, rte_iova_t base_pa);
+void ionic_q_map(struct ionic_queue *q, void *base, rte_iova_t base_pa,
+				 void *cmb_base, rte_iova_t cmb_base_pa);
 void ionic_q_sg_map(struct ionic_queue *q, void *base, rte_iova_t base_pa);
 
 static inline uint16_t
@@ -248,7 +252,10 @@ ionic_q_flush(struct ionic_queue *q)
 {
 	uint64_t val = IONIC_DBELL_QID(q->hw_index) | q->head_idx;
 
-#if defined(DPDK_SIM)
+#if defined(IONIC_EMBEDDED)
+	asm volatile("dsb st" : : : "memory");
+	rte_write64_relaxed(rte_cpu_to_le_64(val), q->db);
+#elif defined(DPDK_SIM)
 	dpdk_sim_write_doorbell(q->name, q->hw_type, val);
 #else
 	rte_write64(rte_cpu_to_le_64(val), q->db);

src/dpdk/drivers/net/ionic/ionic_ethdev.c

@@ -298,6 +298,9 @@ ionic_dev_link_update(struct rte_eth_dev *eth_dev,
 		link.link_status = ETH_LINK_UP;
 		link.link_duplex = ETH_LINK_FULL_DUPLEX;
 		switch (adapter->link_speed) {
+		case  1000:
+			link.link_speed = ETH_SPEED_NUM_1G;
+			break;
 		case  10000:
 			link.link_speed = ETH_SPEED_NUM_10G;
 			break;
@@ -313,6 +316,9 @@ ionic_dev_link_update(struct rte_eth_dev *eth_dev,
 		case 100000:
 			link.link_speed = ETH_SPEED_NUM_100G;
 			break;
+		case 200000:
+			link.link_speed = ETH_SPEED_NUM_200G;
+			break;
 		default:
 			link.link_speed = ETH_SPEED_NUM_NONE;
 			break;
@@ -608,12 +614,13 @@ ionic_dev_rss_reta_query(struct rte_eth_dev *eth_dev,
 
 	num = reta_size / RTE_RETA_GROUP_SIZE;
 
-    for (i = 0; i < num; i++) {
-        for(j=0; j < RTE_RETA_GROUP_SIZE; j++) {
-            reta_conf->reta[j] = lif->rss_ind_tbl[(i*RTE_RETA_GROUP_SIZE)+j];
-        }
-        reta_conf++;
-    }
+	for (i = 0; i < num; i++) {
+		for (j = 0; j < RTE_RETA_GROUP_SIZE; j++) {
+			reta_conf->reta[j] =
+				lif->rss_ind_tbl[(i * RTE_RETA_GROUP_SIZE) + j];
+		}
+		reta_conf++;
+	}
 
 	return 0;
 }

src/dpdk/drivers/net/ionic/ionic_lif.c

@@ -124,9 +124,7 @@ ionic_lif_get_abs_stats(const struct ionic_lif *lif, struct rte_eth_stats *stats
 	stats->imissed +=
 		ls->rx_ucast_drop_packets +
 		ls->rx_mcast_drop_packets +
-		ls->rx_bcast_drop_packets +
-		ls->rx_queue_empty +
-		ls->rx_queue_disabled;
+		ls->rx_bcast_drop_packets;
 
 	stats->ierrors +=
 		ls->rx_dma_error +
@@ -578,10 +576,11 @@ ionic_qcq_alloc(struct ionic_lif *lif,
 {
 	struct ionic_qcq *new;
 	uint32_t q_size, cq_size, sg_size, total_size;
-	void *q_base, *cq_base, *sg_base;
+	void *q_base, *cmb_q_base, *cq_base, *sg_base;
 	rte_iova_t q_base_pa = 0;
 	rte_iova_t cq_base_pa = 0;
 	rte_iova_t sg_base_pa = 0;
+	rte_iova_t cmb_q_base_pa = 0;
 	size_t page_size = rte_mem_page_size();
 	int err;
 
@@ -616,7 +615,7 @@ ionic_qcq_alloc(struct ionic_lif *lif,
 
 	/* Most queue types will store 1 ptr per descriptor */
 	new->q.info = rte_calloc_socket("ionic",
-				num_descs * num_segs, sizeof(void *),
+				(uint64_t)num_descs * num_segs, sizeof(void *),
 				page_size, socket_id);
 	if (!new->q.info) {
 		IONIC_PRINT(ERR, "Cannot allocate queue info");
@@ -672,19 +671,22 @@ ionic_qcq_alloc(struct ionic_lif *lif,
 			IONIC_PRINT(ERR, "Cannot reserve queue from NIC mem");
 			return -ENOMEM;
 		}
-		q_base = (void *)
+		cmb_q_base = (void *)
 			((uint64_t)lif->adapter->bars.bar[2].vaddr +
 			 lif->adapter->cmb_offset);
 		/* CMB PA is a relative address */
-		q_base_pa = lif->adapter->cmb_offset;
+		cmb_q_base_pa = lif->adapter->cmb_offset;
 		lif->adapter->cmb_offset += q_size;
+	} else {
+		cmb_q_base = NULL;
+		cmb_q_base_pa = 0;
 	}
 
 	IONIC_PRINT(DEBUG, "Q-Base-PA = %#jx CQ-Base-PA = %#jx "
 		"SG-base-PA = %#jx",
 		q_base_pa, cq_base_pa, sg_base_pa);
 
-	ionic_q_map(&new->q, q_base, q_base_pa);
+	ionic_q_map(&new->q, q_base, q_base_pa, cmb_q_base, cmb_q_base_pa);
 	ionic_cq_map(&new->cq, cq_base, cq_base_pa);
 
 	*qcq = new;
@@ -1615,8 +1617,8 @@ ionic_lif_txq_init(struct ionic_tx_qcq *txq)
 			.index = rte_cpu_to_le_32(q->index),
 			.flags = rte_cpu_to_le_16(IONIC_QINIT_F_ENA),
 			.intr_index = rte_cpu_to_le_16(IONIC_INTR_NONE),
+			.cos = q->cos,
 			.ring_size = rte_log2_u32(q->num_descs),
-			.ring_base = rte_cpu_to_le_64(q->base_pa),
 			.cq_ring_base = rte_cpu_to_le_64(cq->base_pa),
 			.sg_ring_base = rte_cpu_to_le_64(q->sg_base_pa),
 		},
@@ -1625,8 +1627,12 @@ ionic_lif_txq_init(struct ionic_tx_qcq *txq)
 
 	if (txq->flags & IONIC_QCQ_F_SG)
 		ctx.cmd.q_init.flags |= rte_cpu_to_le_16(IONIC_QINIT_F_SG);
-	if (txq->flags & IONIC_QCQ_F_CMB)
+	if (txq->flags & IONIC_QCQ_F_CMB) {
 		ctx.cmd.q_init.flags |= rte_cpu_to_le_16(IONIC_QINIT_F_CMB);
+		ctx.cmd.q_init.ring_base = rte_cpu_to_le_64(q->cmb_base_pa);
+	} else {
+		ctx.cmd.q_init.ring_base = rte_cpu_to_le_64(q->base_pa);
+	}
 
 	IONIC_PRINT(DEBUG, "txq_init.index %d", q->index);
 	IONIC_PRINT(DEBUG, "txq_init.ring_base 0x%" PRIx64 "", q->base_pa);
@@ -1670,8 +1676,8 @@ ionic_lif_rxq_init(struct ionic_rx_qcq *rxq)
 			.index = rte_cpu_to_le_32(q->index),
 			.flags = rte_cpu_to_le_16(IONIC_QINIT_F_ENA),
 			.intr_index = rte_cpu_to_le_16(IONIC_INTR_NONE),
+			.cos = q->cos,
 			.ring_size = rte_log2_u32(q->num_descs),
-			.ring_base = rte_cpu_to_le_64(q->base_pa),
 			.cq_ring_base = rte_cpu_to_le_64(cq->base_pa),
 			.sg_ring_base = rte_cpu_to_le_64(q->sg_base_pa),
 		},
@@ -1680,8 +1686,12 @@ ionic_lif_rxq_init(struct ionic_rx_qcq *rxq)
 
 	if (rxq->flags & IONIC_QCQ_F_SG)
 		ctx.cmd.q_init.flags |= rte_cpu_to_le_16(IONIC_QINIT_F_SG);
-	if (rxq->flags & IONIC_QCQ_F_CMB)
+	if (rxq->flags & IONIC_QCQ_F_CMB) {
 		ctx.cmd.q_init.flags |= rte_cpu_to_le_16(IONIC_QINIT_F_CMB);
+		ctx.cmd.q_init.ring_base = rte_cpu_to_le_64(q->cmb_base_pa);
+	} else {
+		ctx.cmd.q_init.ring_base = rte_cpu_to_le_64(q->base_pa);
+	}
 
 	IONIC_PRINT(DEBUG, "rxq_init.index %d", q->index);
 	IONIC_PRINT(DEBUG, "rxq_init.ring_base 0x%" PRIx64 "", q->base_pa);

src/dpdk/drivers/net/ionic/ionic_main.c

@@ -223,9 +223,6 @@ ionic_adminq_post(struct ionic_lif *lif, struct ionic_admin_ctx *ctx)
 	q->head_idx = Q_NEXT_TO_POST(q, 1);
 
 	/* Ring doorbell */
-#if !defined(RTE_ARCH_ARM64)
-	rte_wmb();
-#endif
 	ionic_q_flush(q);
 
 err_out:

src/dpdk/drivers/net/ionic/ionic_mem_bypass.c

@@ -71,6 +71,9 @@ struct bypass_range {
 	struct bypass_pool *pools;
 };
 
+/* Support at most this much bypass memory, taken from *end* of range */
+#define IONIC_MEM_BYPASS_MAX (1ULL << 30) /* 1GB */
+
 static struct capmem_range capmem_ranges[CAPMEM_RANGES_MAX];
 static struct bypass_range bypass_info;
 
@@ -328,6 +331,12 @@ ionic_mem_read_capmem(struct ionic_adapter *adapter, bool do_map)
 		goto out;
 	}
 
+	/* Hack - only use a portion of the bypass region */
+	if (range->len > IONIC_MEM_BYPASS_MAX) {
+		range->start += range->len - IONIC_MEM_BYPASS_MAX;
+		range->len = IONIC_MEM_BYPASS_MAX;
+	}
+
 	map_off = range->start & ~(getpagesize() - 1);
 	map_sz = range->start + range->len - map_off;
 

src/dpdk/drivers/net/ionic/ionic_rxtx.c

@@ -233,6 +233,12 @@ ionic_dev_tx_queue_setup(struct rte_eth_dev *eth_dev, uint16_t tx_queue_id,
 		tx_conf->tx_free_thresh ? tx_conf->tx_free_thresh :
 		nb_desc - IONIC_DEF_TXRX_BURST;
 
+	if (tx_conf->cos != 0) {
+		IONIC_PRINT(DEBUG, "Setting TxQ %u to CoS %u",
+			tx_queue_id, tx_conf->cos);
+		txq->qcq.q.cos = tx_conf->cos;
+	}
+
 	eth_dev->data->tx_queues[tx_queue_id] = txq;
 
 	return 0;
@@ -616,6 +622,12 @@ ionic_dev_rx_queue_setup(struct rte_eth_dev *eth_dev,
 	if (rx_conf->rx_deferred_start)
 		rxq->flags |= IONIC_QCQ_F_DEFERRED;
 
+	if (rx_conf->cos != 0) {
+		IONIC_PRINT(DEBUG, "Setting RxQ %u to CoS %u",
+			rx_queue_id, rx_conf->cos);
+		rxq->qcq.q.cos = rx_conf->cos;
+	}
+
 	eth_dev->data->rx_queues[rx_queue_id] = rxq;
 
 #ifdef IONIC_MEM_BYPASS
@@ -862,7 +874,7 @@ ionic_dev_rx_descriptor_status(void *rx_queue, uint16_t offset)
 {
 	struct ionic_rx_qcq *rxq = rx_queue;
 	struct ionic_qcq *qcq = &rxq->qcq;
-	struct ionic_rxq_comp *cq_desc;
+	volatile struct ionic_rxq_comp *cq_desc;
 	uint16_t mask, head, tail, pos;
 	bool done_color;
 
@@ -901,7 +913,7 @@ ionic_dev_tx_descriptor_status(void *tx_queue, uint16_t offset)
 {
 	struct ionic_tx_qcq *txq = tx_queue;
 	struct ionic_qcq *qcq = &txq->qcq;
-	struct ionic_txq_comp *cq_desc;
+	volatile struct ionic_txq_comp *cq_desc;
 	uint16_t mask, head, tail, pos, cq_pos;
 	bool done_color;
 

src/dpdk/drivers/net/ionic/ionic_rxtx.h

@@ -71,6 +71,70 @@ uint16_t ionic_xmit_pkts_sg(void *tx_queue, struct rte_mbuf **tx_pkts,
 
 int ionic_rx_fill_sg(struct ionic_rx_qcq *rxq);
 
+static inline void
+ionic_rxq_flush(struct ionic_queue *q)
+{
+#ifndef DPDK_SIM
+#ifndef IONIC_EMBEDDED
+	struct ionic_rxq_desc *desc_base = q->base;
+	struct ionic_rxq_desc *cmb_desc_base = q->cmb_base;
+
+	if (q->cmb_base) {
+		if (q->head_idx < q->cmb_head_idx) {
+			// copy [cmb_head, num_descs)
+			rte_memcpy((void *)&cmb_desc_base[q->cmb_head_idx],
+					   (void *)&desc_base[q->cmb_head_idx],
+					   (q->num_descs - q->cmb_head_idx) * sizeof(*desc_base));
+			// copy [0, head)
+			rte_memcpy((void *)&cmb_desc_base[0],
+					   (void *)&desc_base[0],
+					   q->head_idx * sizeof(*desc_base));
+		} else {
+			// copy [cmb_head, head)
+			rte_memcpy((void *)&cmb_desc_base[q->cmb_head_idx],
+					   (void *)&desc_base[q->cmb_head_idx],
+					   (q->head_idx - q->cmb_head_idx) * sizeof(*desc_base));
+		}
+		q->cmb_head_idx = q->head_idx;
+	}
+#endif /* IONIC_EMBEDDED */
+#endif /* DPDK_SIM */
+
+	ionic_q_flush(q);
+}
+
+static inline void
+ionic_txq_flush(struct ionic_queue *q)
+{
+#ifndef DPDK_SIM
+#ifndef IONIC_EMBEDDED
+	struct ionic_txq_desc *desc_base = q->base;
+	struct ionic_txq_desc *cmb_desc_base = q->cmb_base;
+
+	if (q->cmb_base) {
+		if (q->head_idx < q->cmb_head_idx) {
+			// copy [cmb_head, num_descs)
+			rte_memcpy((void *)&cmb_desc_base[q->cmb_head_idx],
+					   (void *)&desc_base[q->cmb_head_idx],
+					   (q->num_descs - q->cmb_head_idx) * sizeof(*desc_base));
+			// copy [0, head)
+			rte_memcpy((void *)&cmb_desc_base[0],
+					   (void *)&desc_base[0],
+					   q->head_idx * sizeof(*desc_base));
+		} else {
+			// copy [cmb_head, head)
+			rte_memcpy((void *)&cmb_desc_base[q->cmb_head_idx],
+					   (void *)&desc_base[q->cmb_head_idx],
+					   (q->head_idx - q->cmb_head_idx) * sizeof(*desc_base));
+		}
+		q->cmb_head_idx = q->head_idx;
+	}
+#endif /* IONIC_EMBEDDED */
+#endif /* DPDK_SIM */
+
+	ionic_q_flush(q);
+}
+
 #if defined(IONIC_CODE_PERF_RX) || defined(IONIC_CODE_PERF_TX)
 static inline uint64_t
 ionic_tsc(void)