ENIC driver updates for DPDK 16.04 and 2.2.0
Change-Id: I896925ea2208d318d353d486707b97f295c78426
Signed-off-by: John Lo <loj@cisco.com>
diff --git a/dpdk/dpdk-2.2.0_patches/0026-enic-Optimization-of-Tx-path-to-reduce-Host-CPU-over.patch b/dpdk/dpdk-2.2.0_patches/0026-enic-Optimization-of-Tx-path-to-reduce-Host-CPU-over.patch
new file mode 100644
index 0000000..4858b8f
--- /dev/null
+++ b/dpdk/dpdk-2.2.0_patches/0026-enic-Optimization-of-Tx-path-to-reduce-Host-CPU-over.patch
@@ -0,0 +1,1844 @@
+From ca6bbb723880e91d006de6cc485259da988859aa Mon Sep 17 00:00:00 2001
+From: John Daley <johndale@cisco.com>
+Date: Tue, 5 Apr 2016 15:19:06 -0700
+Subject: [PATCH 3/3] enic: Optimization of Tx path to reduce Host CPU
+ overhead, cleanup
+
+Optimizations and cleanup:
+- flatten packet send path
+- flatten mbuf free path
+- disable CQ entry writing and use CQ messages instead
+- use rte_mempool_put_bulk() to bulk return freed mbufs
+- remove unnecessary fields vnic_bufs struct, use contiguous array of cache
+ aligned divisible elements. No next pointers.
+- use local variables inside per packet loop instead of fields in structs.
+- factor book keeping out of the per packet tx loop where possible
+ (removed several conditionals)
+- put Tx and Rx code in 1 file (enic_rxtx.c)
+
+Reviewed-by: Nelson Escobar <neescoba@cisco.com>
+Signed-off-by: John Daley <johndale@cisco.com>
+---
+ drivers/net/enic/Makefile | 2 +-
+ drivers/net/enic/base/enic_vnic_wq.h | 79 ------
+ drivers/net/enic/base/vnic_cq.h | 37 +--
+ drivers/net/enic/base/vnic_rq.h | 2 +-
+ drivers/net/enic/base/vnic_wq.c | 89 +++---
+ drivers/net/enic/base/vnic_wq.h | 113 +-------
+ drivers/net/enic/enic.h | 27 +-
+ drivers/net/enic/enic_ethdev.c | 67 +----
+ drivers/net/enic/enic_main.c | 132 +++------
+ drivers/net/enic/enic_res.h | 81 +-----
+ drivers/net/enic/enic_rx.c | 361 -------------------------
+ drivers/net/enic/enic_rxtx.c | 505 +++++++++++++++++++++++++++++++++++
+ 12 files changed, 635 insertions(+), 860 deletions(-)
+ delete mode 100644 drivers/net/enic/base/enic_vnic_wq.h
+ delete mode 100644 drivers/net/enic/enic_rx.c
+ create mode 100644 drivers/net/enic/enic_rxtx.c
+
+diff --git a/drivers/net/enic/Makefile b/drivers/net/enic/Makefile
+index f316274..3926b79 100644
+--- a/drivers/net/enic/Makefile
++++ b/drivers/net/enic/Makefile
+@@ -53,7 +53,7 @@ VPATH += $(SRCDIR)/src
+ #
+ SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_ethdev.c
+ SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_main.c
+-SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_rx.c
++SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_rxtx.c
+ SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_clsf.c
+ SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_res.c
+ SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += base/vnic_cq.c
+diff --git a/drivers/net/enic/base/enic_vnic_wq.h b/drivers/net/enic/base/enic_vnic_wq.h
+deleted file mode 100644
+index b019109..0000000
+--- a/drivers/net/enic/base/enic_vnic_wq.h
++++ /dev/null
+@@ -1,79 +0,0 @@
+-/*
+- * Copyright 2008-2015 Cisco Systems, Inc. All rights reserved.
+- * Copyright 2007 Nuova Systems, Inc. All rights reserved.
+- *
+- * Copyright (c) 2015, Cisco Systems, Inc.
+- * All rights reserved.
+- *
+- * Redistribution and use in source and binary forms, with or without
+- * modification, are permitted provided that the following conditions
+- * are met:
+- *
+- * 1. Redistributions of source code must retain the above copyright
+- * notice, this list of conditions and the following disclaimer.
+- *
+- * 2. Redistributions in binary form must reproduce the above copyright
+- * notice, this list of conditions and the following disclaimer in
+- * the documentation and/or other materials provided with the
+- * distribution.
+- *
+- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+- * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+- * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+- * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+- * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+- * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+- * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+- * POSSIBILITY OF SUCH DAMAGE.
+- *
+- */
+-
+-#ifndef _ENIC_VNIC_WQ_H_
+-#define _ENIC_VNIC_WQ_H_
+-
+-#include "vnic_dev.h"
+-#include "vnic_cq.h"
+-
+-static inline void enic_vnic_post_wq_index(struct vnic_wq *wq)
+-{
+- struct vnic_wq_buf *buf = wq->to_use;
+-
+- /* Adding write memory barrier prevents compiler and/or CPU
+- * reordering, thus avoiding descriptor posting before
+- * descriptor is initialized. Otherwise, hardware can read
+- * stale descriptor fields.
+- */
+- wmb();
+- iowrite32(buf->index, &wq->ctrl->posted_index);
+-}
+-
+-static inline void enic_vnic_post_wq(struct vnic_wq *wq,
+- void *os_buf, dma_addr_t dma_addr,
+- unsigned int len, int sop,
+- uint8_t desc_skip_cnt, uint8_t cq_entry,
+- uint8_t compressed_send, uint64_t wrid)
+-{
+- struct vnic_wq_buf *buf = wq->to_use;
+-
+- buf->sop = sop;
+- buf->cq_entry = cq_entry;
+- buf->compressed_send = compressed_send;
+- buf->desc_skip_cnt = desc_skip_cnt;
+- buf->os_buf = os_buf;
+- buf->dma_addr = dma_addr;
+- buf->len = len;
+- buf->wr_id = wrid;
+-
+- buf = buf->next;
+- wq->ring.desc_avail -= desc_skip_cnt;
+- wq->to_use = buf;
+-
+- if (cq_entry)
+- enic_vnic_post_wq_index(wq);
+-}
+-
+-#endif /* _ENIC_VNIC_WQ_H_ */
+diff --git a/drivers/net/enic/base/vnic_cq.h b/drivers/net/enic/base/vnic_cq.h
+index 922391b..ffc1aaa 100644
+--- a/drivers/net/enic/base/vnic_cq.h
++++ b/drivers/net/enic/base/vnic_cq.h
+@@ -96,41 +96,46 @@ static inline unsigned int vnic_cq_service(struct vnic_cq *cq,
+ u8 type, u16 q_number, u16 completed_index, void *opaque),
+ void *opaque)
+ {
+- struct cq_desc *cq_desc;
++ struct cq_desc *cq_desc, *cq_desc_last;
+ unsigned int work_done = 0;
+ u16 q_number, completed_index;
+- u8 type, color;
+- struct rte_mbuf **rx_pkts = opaque;
+- unsigned int ret;
++ u8 type, color, type_color;
+
+ cq_desc = (struct cq_desc *)((u8 *)cq->ring.descs +
+ cq->ring.desc_size * cq->to_clean);
+- cq_desc_dec(cq_desc, &type, &color,
+- &q_number, &completed_index);
++
++ type_color = cq_desc->type_color;
++ color = (type_color >> CQ_DESC_COLOR_SHIFT) & CQ_DESC_COLOR_MASK;
++ if (color == cq->last_color)
++ return 0;
+
+ while (color != cq->last_color) {
+- if (opaque)
+- opaque = (void *)&(rx_pkts[work_done]);
++ cq_desc_last = cq_desc;
+
+- ret = (*q_service)(cq->vdev, cq_desc, type,
+- q_number, completed_index, opaque);
+ cq->to_clean++;
+ if (cq->to_clean == cq->ring.desc_count) {
+ cq->to_clean = 0;
+ cq->last_color = cq->last_color ? 0 : 1;
+ }
+
++ work_done++;
++ if (work_done >= work_to_do)
++ break;
++
+ cq_desc = (struct cq_desc *)((u8 *)cq->ring.descs +
+ cq->ring.desc_size * cq->to_clean);
+- cq_desc_dec(cq_desc, &type, &color,
+- &q_number, &completed_index);
+
+- if (ret)
+- work_done++;
+- if (work_done >= work_to_do)
+- break;
++ type_color = cq_desc->type_color;
++ color = (type_color >> CQ_DESC_COLOR_SHIFT)
++ & CQ_DESC_COLOR_MASK;
++
+ }
+
++ cq_desc_dec(cq_desc_last, &type, &color,
++ &q_number, &completed_index);
++
++ (*q_service)(cq->vdev, cq_desc, type,
++ q_number, completed_index, opaque);
+ return work_done;
+ }
+
+diff --git a/drivers/net/enic/base/vnic_rq.h b/drivers/net/enic/base/vnic_rq.h
+index e083ccc..424415c 100644
+--- a/drivers/net/enic/base/vnic_rq.h
++++ b/drivers/net/enic/base/vnic_rq.h
+@@ -74,7 +74,7 @@ struct vnic_rq {
+ struct vnic_dev_ring ring;
+ struct rte_mbuf **mbuf_ring; /* array of allocated mbufs */
+ unsigned int mbuf_next_idx; /* next mb to consume */
+- void *os_buf_head;
++ void *mb_head;
+ unsigned int pkts_outstanding;
+ uint16_t rx_nb_hold;
+ uint16_t rx_free_thresh;
+diff --git a/drivers/net/enic/base/vnic_wq.c b/drivers/net/enic/base/vnic_wq.c
+index a3ef417..ccbbd61 100644
+--- a/drivers/net/enic/base/vnic_wq.c
++++ b/drivers/net/enic/base/vnic_wq.c
+@@ -59,71 +59,30 @@ int vnic_wq_alloc_ring(struct vnic_dev *vdev, struct vnic_wq *wq,
+
+ static int vnic_wq_alloc_bufs(struct vnic_wq *wq)
+ {
+- struct vnic_wq_buf *buf;
+- unsigned int i, j, count = wq->ring.desc_count;
+- unsigned int blks = VNIC_WQ_BUF_BLKS_NEEDED(count);
+-
+- for (i = 0; i < blks; i++) {
+- wq->bufs[i] = kzalloc(VNIC_WQ_BUF_BLK_SZ(count), GFP_ATOMIC);
+- if (!wq->bufs[i])
+- return -ENOMEM;
+- }
+-
+- for (i = 0; i < blks; i++) {
+- buf = wq->bufs[i];
+- for (j = 0; j < VNIC_WQ_BUF_BLK_ENTRIES(count); j++) {
+- buf->index = i * VNIC_WQ_BUF_BLK_ENTRIES(count) + j;
+- buf->desc = (u8 *)wq->ring.descs +
+- wq->ring.desc_size * buf->index;
+- if (buf->index + 1 == count) {
+- buf->next = wq->bufs[0];
+- break;
+- } else if (j + 1 == VNIC_WQ_BUF_BLK_ENTRIES(count)) {
+- buf->next = wq->bufs[i + 1];
+- } else {
+- buf->next = buf + 1;
+- buf++;
+- }
+- }
+- }
+-
+- wq->to_use = wq->to_clean = wq->bufs[0];
+-
++ unsigned int count = wq->ring.desc_count;
++ /* Allocate the mbuf ring */
++ wq->bufs = (struct vnic_wq_buf *)rte_zmalloc_socket("wq->bufs",
++ sizeof(struct vnic_wq_buf) * count,
++ RTE_CACHE_LINE_SIZE, wq->socket_id);
++ wq->head_idx = 0;
++ wq->tail_idx = 0;
++ if (wq->bufs == NULL)
++ return -ENOMEM;
+ return 0;
+ }
+
+ void vnic_wq_free(struct vnic_wq *wq)
+ {
+ struct vnic_dev *vdev;
+- unsigned int i;
+
+ vdev = wq->vdev;
+
+ vnic_dev_free_desc_ring(vdev, &wq->ring);
+
+- for (i = 0; i < VNIC_WQ_BUF_BLKS_MAX; i++) {
+- if (wq->bufs[i]) {
+- kfree(wq->bufs[i]);
+- wq->bufs[i] = NULL;
+- }
+- }
+-
++ rte_free(wq->bufs);
+ wq->ctrl = NULL;
+ }
+
+-int vnic_wq_mem_size(struct vnic_wq *wq, unsigned int desc_count,
+- unsigned int desc_size)
+-{
+- int mem_size = 0;
+-
+- mem_size += vnic_dev_desc_ring_size(&wq->ring, desc_count, desc_size);
+-
+- mem_size += VNIC_WQ_BUF_BLKS_NEEDED(wq->ring.desc_count) *
+- VNIC_WQ_BUF_BLK_SZ(wq->ring.desc_count);
+-
+- return mem_size;
+-}
+-
+
+ int vnic_wq_alloc(struct vnic_dev *vdev, struct vnic_wq *wq, unsigned int index,
+ unsigned int desc_count, unsigned int desc_size)
+@@ -172,9 +131,8 @@ void vnic_wq_init_start(struct vnic_wq *wq, unsigned int cq_index,
+ iowrite32(error_interrupt_offset, &wq->ctrl->error_interrupt_offset);
+ iowrite32(0, &wq->ctrl->error_status);
+
+- wq->to_use = wq->to_clean =
+- &wq->bufs[fetch_index / VNIC_WQ_BUF_BLK_ENTRIES(count)]
+- [fetch_index % VNIC_WQ_BUF_BLK_ENTRIES(count)];
++ wq->head_idx = fetch_index;
++ wq->tail_idx = wq->head_idx;
+ }
+
+ void vnic_wq_init(struct vnic_wq *wq, unsigned int cq_index,
+@@ -184,6 +142,7 @@ void vnic_wq_init(struct vnic_wq *wq, unsigned int cq_index,
+ vnic_wq_init_start(wq, cq_index, 0, 0,
+ error_interrupt_enable,
+ error_interrupt_offset);
++ wq->last_completed_index = 0;
+ }
+
+ void vnic_wq_error_out(struct vnic_wq *wq, unsigned int error)
+@@ -219,22 +178,34 @@ int vnic_wq_disable(struct vnic_wq *wq)
+ return -ETIMEDOUT;
+ }
+
++static inline uint32_t
++buf_idx_incr(uint32_t n_descriptors, uint32_t idx)
++{
++ idx++;
++ if (unlikely(idx == n_descriptors))
++ idx = 0;
++ return idx;
++}
++
+ void vnic_wq_clean(struct vnic_wq *wq,
+- void (*buf_clean)(struct vnic_wq *wq, struct vnic_wq_buf *buf))
++ void (*buf_clean)(struct vnic_wq_buf *buf))
+ {
+ struct vnic_wq_buf *buf;
++ unsigned int to_clean = wq->tail_idx;
+
+- buf = wq->to_clean;
++ buf = &wq->bufs[to_clean];
+
+ while (vnic_wq_desc_used(wq) > 0) {
+
+- (*buf_clean)(wq, buf);
++ (*buf_clean)(buf);
++ to_clean = buf_idx_incr(wq->ring.desc_count, to_clean);
+
+- buf = wq->to_clean = buf->next;
++ buf = &wq->bufs[to_clean];
+ wq->ring.desc_avail++;
+ }
+
+- wq->to_use = wq->to_clean = wq->bufs[0];
++ wq->head_idx = 0;
++ wq->tail_idx = 0;
+
+ iowrite32(0, &wq->ctrl->fetch_index);
+ iowrite32(0, &wq->ctrl->posted_index);
+diff --git a/drivers/net/enic/base/vnic_wq.h b/drivers/net/enic/base/vnic_wq.h
+index c23de62..37c3ff9 100644
+--- a/drivers/net/enic/base/vnic_wq.h
++++ b/drivers/net/enic/base/vnic_wq.h
+@@ -64,42 +64,23 @@ struct vnic_wq_ctrl {
+ u32 pad9;
+ };
+
++/* 16 bytes */
+ struct vnic_wq_buf {
+- struct vnic_wq_buf *next;
+- dma_addr_t dma_addr;
+- void *os_buf;
+- unsigned int len;
+- unsigned int index;
+- int sop;
+- void *desc;
+- uint64_t wr_id; /* Cookie */
+- uint8_t cq_entry; /* Gets completion event from hw */
+- uint8_t desc_skip_cnt; /* Num descs to occupy */
+- uint8_t compressed_send; /* Both hdr and payload in one desc */
++ struct rte_mempool *pool;
++ void *mb;
+ };
+
+-/* Break the vnic_wq_buf allocations into blocks of 32/64 entries */
+-#define VNIC_WQ_BUF_MIN_BLK_ENTRIES 32
+-#define VNIC_WQ_BUF_DFLT_BLK_ENTRIES 64
+-#define VNIC_WQ_BUF_BLK_ENTRIES(entries) \
+- ((unsigned int)((entries < VNIC_WQ_BUF_DFLT_BLK_ENTRIES) ? \
+- VNIC_WQ_BUF_MIN_BLK_ENTRIES : VNIC_WQ_BUF_DFLT_BLK_ENTRIES))
+-#define VNIC_WQ_BUF_BLK_SZ(entries) \
+- (VNIC_WQ_BUF_BLK_ENTRIES(entries) * sizeof(struct vnic_wq_buf))
+-#define VNIC_WQ_BUF_BLKS_NEEDED(entries) \
+- DIV_ROUND_UP(entries, VNIC_WQ_BUF_BLK_ENTRIES(entries))
+-#define VNIC_WQ_BUF_BLKS_MAX VNIC_WQ_BUF_BLKS_NEEDED(4096)
+-
+ struct vnic_wq {
+ unsigned int index;
+ struct vnic_dev *vdev;
+ struct vnic_wq_ctrl __iomem *ctrl; /* memory-mapped */
+ struct vnic_dev_ring ring;
+- struct vnic_wq_buf *bufs[VNIC_WQ_BUF_BLKS_MAX];
+- struct vnic_wq_buf *to_use;
+- struct vnic_wq_buf *to_clean;
+- unsigned int pkts_outstanding;
++ struct vnic_wq_buf *bufs;
++ unsigned int head_idx;
++ unsigned int tail_idx;
+ unsigned int socket_id;
++ const struct rte_memzone *cqmsg_rz;
++ uint16_t last_completed_index;
+ };
+
+ static inline unsigned int vnic_wq_desc_avail(struct vnic_wq *wq)
+@@ -114,11 +95,6 @@ static inline unsigned int vnic_wq_desc_used(struct vnic_wq *wq)
+ return wq->ring.desc_count - wq->ring.desc_avail - 1;
+ }
+
+-static inline void *vnic_wq_next_desc(struct vnic_wq *wq)
+-{
+- return wq->to_use->desc;
+-}
+-
+ #define PI_LOG2_CACHE_LINE_SIZE 5
+ #define PI_INDEX_BITS 12
+ #define PI_INDEX_MASK ((1U << PI_INDEX_BITS) - 1)
+@@ -191,75 +167,6 @@ static inline u64 vnic_cached_posted_index(dma_addr_t addr, unsigned int len,
+ PI_PREFETCH_ADDR_MASK) << PI_PREFETCH_ADDR_OFF);
+ }
+
+-static inline void vnic_wq_post(struct vnic_wq *wq,
+- void *os_buf, dma_addr_t dma_addr,
+- unsigned int len, int sop, int eop,
+- uint8_t desc_skip_cnt, uint8_t cq_entry,
+- uint8_t compressed_send, uint64_t wrid)
+-{
+- struct vnic_wq_buf *buf = wq->to_use;
+-
+- buf->sop = sop;
+- buf->cq_entry = cq_entry;
+- buf->compressed_send = compressed_send;
+- buf->desc_skip_cnt = desc_skip_cnt;
+- buf->os_buf = os_buf;
+- buf->dma_addr = dma_addr;
+- buf->len = len;
+- buf->wr_id = wrid;
+-
+- buf = buf->next;
+- if (eop) {
+-#ifdef DO_PREFETCH
+- uint64_t wr = vnic_cached_posted_index(dma_addr, len,
+- buf->index);
+-#endif
+- /* Adding write memory barrier prevents compiler and/or CPU
+- * reordering, thus avoiding descriptor posting before
+- * descriptor is initialized. Otherwise, hardware can read
+- * stale descriptor fields.
+- */
+- wmb();
+-#ifdef DO_PREFETCH
+- /* Intel chipsets seem to limit the rate of PIOs that we can
+- * push on the bus. Thus, it is very important to do a single
+- * 64 bit write here. With two 32-bit writes, my maximum
+- * pkt/sec rate was cut almost in half. -AJF
+- */
+- iowrite64((uint64_t)wr, &wq->ctrl->posted_index);
+-#else
+- iowrite32(buf->index, &wq->ctrl->posted_index);
+-#endif
+- }
+- wq->to_use = buf;
+-
+- wq->ring.desc_avail -= desc_skip_cnt;
+-}
+-
+-static inline void vnic_wq_service(struct vnic_wq *wq,
+- struct cq_desc *cq_desc, u16 completed_index,
+- void (*buf_service)(struct vnic_wq *wq,
+- struct cq_desc *cq_desc, struct vnic_wq_buf *buf, void *opaque),
+- void *opaque)
+-{
+- struct vnic_wq_buf *buf;
+-
+- buf = wq->to_clean;
+- while (1) {
+-
+- (*buf_service)(wq, cq_desc, buf, opaque);
+-
+- wq->ring.desc_avail++;
+-
+- wq->to_clean = buf->next;
+-
+- if (buf->index == completed_index)
+- break;
+-
+- buf = wq->to_clean;
+- }
+-}
+-
+ void vnic_wq_free(struct vnic_wq *wq);
+ int vnic_wq_alloc(struct vnic_dev *vdev, struct vnic_wq *wq, unsigned int index,
+ unsigned int desc_count, unsigned int desc_size);
+@@ -275,8 +182,6 @@ unsigned int vnic_wq_error_status(struct vnic_wq *wq);
+ void vnic_wq_enable(struct vnic_wq *wq);
+ int vnic_wq_disable(struct vnic_wq *wq);
+ void vnic_wq_clean(struct vnic_wq *wq,
+- void (*buf_clean)(struct vnic_wq *wq, struct vnic_wq_buf *buf));
+-int vnic_wq_mem_size(struct vnic_wq *wq, unsigned int desc_count,
+- unsigned int desc_size);
++ void (*buf_clean)(struct vnic_wq_buf *buf));
+
+ #endif /* _VNIC_WQ_H_ */
+diff --git a/drivers/net/enic/enic.h b/drivers/net/enic/enic.h
+index 8c914f5..43b82a6 100644
+--- a/drivers/net/enic/enic.h
++++ b/drivers/net/enic/enic.h
+@@ -155,6 +155,30 @@ static inline struct enic *pmd_priv(struct rte_eth_dev *eth_dev)
+ return (struct enic *)eth_dev->data->dev_private;
+ }
+
++static inline uint32_t
++enic_ring_add(uint32_t n_descriptors, uint32_t i0, uint32_t i1)
++{
++ uint32_t d = i0 + i1;
++ d -= (d >= n_descriptors) ? n_descriptors : 0;
++ return d;
++}
++
++static inline uint32_t
++enic_ring_sub(uint32_t n_descriptors, uint32_t i0, uint32_t i1)
++{
++ int32_t d = i1 - i0;
++ return (uint32_t)((d < 0) ? ((int32_t)n_descriptors + d) : d);
++}
++
++static inline uint32_t
++enic_ring_incr(uint32_t n_descriptors, uint32_t idx)
++{
++ idx++;
++ if (unlikely(idx == n_descriptors))
++ idx = 0;
++ return idx;
++}
++
+ #define RTE_LIBRTE_ENIC_ASSERT_ENABLE
+ #ifdef RTE_LIBRTE_ENIC_ASSERT_ENABLE
+ #define ASSERT(x) do { \
+@@ -209,5 +233,6 @@ extern int enic_clsf_init(struct enic *enic);
+ extern void enic_clsf_destroy(struct enic *enic);
+ uint16_t enic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
+ uint16_t nb_pkts);
+-
++uint16_t enic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
++ uint16_t nb_pkts);
+ #endif /* _ENIC_H_ */
+diff --git a/drivers/net/enic/enic_ethdev.c b/drivers/net/enic/enic_ethdev.c
+index 6bea940..697ff82 100644
+--- a/drivers/net/enic/enic_ethdev.c
++++ b/drivers/net/enic/enic_ethdev.c
+@@ -519,71 +519,6 @@ static void enicpmd_remove_mac_addr(struct rte_eth_dev *eth_dev, __rte_unused ui
+ enic_del_mac_address(enic);
+ }
+
+-
+-static uint16_t enicpmd_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
+- uint16_t nb_pkts)
+-{
+- uint16_t index;
+- unsigned int frags;
+- unsigned int pkt_len;
+- unsigned int seg_len;
+- unsigned int inc_len;
+- unsigned int nb_segs;
+- struct rte_mbuf *tx_pkt, *next_tx_pkt;
+- struct vnic_wq *wq = (struct vnic_wq *)tx_queue;
+- struct enic *enic = vnic_dev_priv(wq->vdev);
+- unsigned short vlan_id;
+- unsigned short ol_flags;
+- uint8_t last_seg, eop;
+- unsigned int host_tx_descs = 0;
+-
+- for (index = 0; index < nb_pkts; index++) {
+- tx_pkt = *tx_pkts++;
+- inc_len = 0;
+- nb_segs = tx_pkt->nb_segs;
+- if (nb_segs > vnic_wq_desc_avail(wq)) {
+- if (index > 0)
+- enic_post_wq_index(wq);
+-
+- /* wq cleanup and try again */
+- if (!enic_cleanup_wq(enic, wq) ||
+- (nb_segs > vnic_wq_desc_avail(wq))) {
+- return index;
+- }
+- }
+-
+- pkt_len = tx_pkt->pkt_len;
+- vlan_id = tx_pkt->vlan_tci;
+- ol_flags = tx_pkt->ol_flags;
+- for (frags = 0; inc_len < pkt_len; frags++) {
+- if (!tx_pkt)
+- break;
+- next_tx_pkt = tx_pkt->next;
+- seg_len = tx_pkt->data_len;
+- inc_len += seg_len;
+-
+- host_tx_descs++;
+- last_seg = 0;
+- eop = 0;
+- if ((pkt_len == inc_len) || !next_tx_pkt) {
+- eop = 1;
+- /* post if last packet in batch or > thresh */
+- if ((index == (nb_pkts - 1)) ||
+- (host_tx_descs > ENIC_TX_POST_THRESH)) {
+- last_seg = 1;
+- host_tx_descs = 0;
+- }
+- }
+- enic_send_pkt(enic, wq, tx_pkt, (unsigned short)seg_len,
+- !frags, eop, last_seg, ol_flags, vlan_id);
+- tx_pkt = next_tx_pkt;
+- }
+- }
+-
+- enic_cleanup_wq(enic, wq);
+- return index;
+-}
+-
+ static const struct eth_dev_ops enicpmd_eth_dev_ops = {
+ .dev_configure = enicpmd_dev_configure,
+ .dev_start = enicpmd_dev_start,
+@@ -642,7 +577,7 @@ static int eth_enicpmd_dev_init(struct rte_eth_dev *eth_dev)
+ enic->rte_dev = eth_dev;
+ eth_dev->dev_ops = &enicpmd_eth_dev_ops;
+ eth_dev->rx_pkt_burst = &enic_recv_pkts;
+- eth_dev->tx_pkt_burst = &enicpmd_xmit_pkts;
++ eth_dev->tx_pkt_burst = &enic_xmit_pkts;
+
+ pdev = eth_dev->pci_dev;
+ rte_eth_copy_pci_info(eth_dev, pdev);
+diff --git a/drivers/net/enic/enic_main.c b/drivers/net/enic/enic_main.c
+index 646d87f..ba73604 100644
+--- a/drivers/net/enic/enic_main.c
++++ b/drivers/net/enic/enic_main.c
+@@ -40,11 +40,11 @@
+ #include <libgen.h>
+
+ #include <rte_pci.h>
+-#include <rte_memzone.h>
+ #include <rte_malloc.h>
+ #include <rte_mbuf.h>
+ #include <rte_string_fns.h>
+ #include <rte_ethdev.h>
++#include <rte_memzone.h>
+
+ #include "enic_compat.h"
+ #include "enic.h"
+@@ -58,7 +58,6 @@
+ #include "vnic_cq.h"
+ #include "vnic_intr.h"
+ #include "vnic_nic.h"
+-#include "enic_vnic_wq.h"
+
+ static inline struct rte_mbuf *
+ rte_rxmbuf_alloc(struct rte_mempool *mp)
+@@ -109,38 +108,17 @@ enic_rxmbuf_queue_release(struct enic *enic, struct vnic_rq *rq)
+ }
+ }
+
+-
+ void enic_set_hdr_split_size(struct enic *enic, u16 split_hdr_size)
+ {
+ vnic_set_hdr_split_size(enic->vdev, split_hdr_size);
+ }
+
+-static void enic_free_wq_buf(__rte_unused struct vnic_wq *wq, struct vnic_wq_buf *buf)
++static void enic_free_wq_buf(struct vnic_wq_buf *buf)
+ {
+- struct rte_mbuf *mbuf = (struct rte_mbuf *)buf->os_buf;
++ struct rte_mbuf *mbuf = (struct rte_mbuf *)buf->mb;
+
+ rte_mempool_put(mbuf->pool, mbuf);
+- buf->os_buf = NULL;
+-}
+-
+-static void enic_wq_free_buf(struct vnic_wq *wq,
+- __rte_unused struct cq_desc *cq_desc,
+- struct vnic_wq_buf *buf,
+- __rte_unused void *opaque)
+-{
+- enic_free_wq_buf(wq, buf);
+-}
+-
+-static int enic_wq_service(struct vnic_dev *vdev, struct cq_desc *cq_desc,
+- __rte_unused u8 type, u16 q_number, u16 completed_index, void *opaque)
+-{
+- struct enic *enic = vnic_dev_priv(vdev);
+-
+- vnic_wq_service(&enic->wq[q_number], cq_desc,
+- completed_index, enic_wq_free_buf,
+- opaque);
+-
+- return 0;
++ buf->mb = NULL;
+ }
+
+ static void enic_log_q_error(struct enic *enic)
+@@ -163,64 +141,6 @@ static void enic_log_q_error(struct enic *enic)
+ }
+ }
+
+-unsigned int enic_cleanup_wq(struct enic *enic, struct vnic_wq *wq)
+-{
+- unsigned int cq = enic_cq_wq(enic, wq->index);
+-
+- /* Return the work done */
+- return vnic_cq_service(&enic->cq[cq],
+- -1 /*wq_work_to_do*/, enic_wq_service, NULL);
+-}
+-
+-void enic_post_wq_index(struct vnic_wq *wq)
+-{
+- enic_vnic_post_wq_index(wq);
+-}
+-
+-void enic_send_pkt(struct enic *enic, struct vnic_wq *wq,
+- struct rte_mbuf *tx_pkt, unsigned short len,
+- uint8_t sop, uint8_t eop, uint8_t cq_entry,
+- uint16_t ol_flags, uint16_t vlan_tag)
+-{
+- struct wq_enet_desc *desc = vnic_wq_next_desc(wq);
+- uint16_t mss = 0;
+- uint8_t vlan_tag_insert = 0;
+- uint64_t bus_addr = (dma_addr_t)
+- (tx_pkt->buf_physaddr + tx_pkt->data_off);
+-
+- if (sop) {
+- if (ol_flags & PKT_TX_VLAN_PKT)
+- vlan_tag_insert = 1;
+-
+- if (enic->hw_ip_checksum) {
+- if (ol_flags & PKT_TX_IP_CKSUM)
+- mss |= ENIC_CALC_IP_CKSUM;
+-
+- if (ol_flags & PKT_TX_TCP_UDP_CKSUM)
+- mss |= ENIC_CALC_TCP_UDP_CKSUM;
+- }
+- }
+-
+- wq_enet_desc_enc(desc,
+- bus_addr,
+- len,
+- mss,
+- 0 /* header_length */,
+- 0 /* offload_mode WQ_ENET_OFFLOAD_MODE_CSUM */,
+- eop,
+- cq_entry,
+- 0 /* fcoe_encap */,
+- vlan_tag_insert,
+- vlan_tag,
+- 0 /* loopback */);
+-
+- enic_vnic_post_wq(wq, (void *)tx_pkt, bus_addr, len,
+- sop,
+- 1 /*desc_skip_cnt*/,
+- cq_entry,
+- 0 /*compressed send*/,
+- 0 /*wrid*/);
+-}
+
+ void enic_dev_stats_clear(struct enic *enic)
+ {
+@@ -297,12 +217,28 @@ void enic_init_vnic_resources(struct enic *enic)
+ unsigned int error_interrupt_enable = 1;
+ unsigned int error_interrupt_offset = 0;
+ unsigned int index = 0;
++ unsigned int cq_idx;
++
++ vnic_dev_stats_clear(enic->vdev);
+
+ for (index = 0; index < enic->rq_count; index++) {
+ vnic_rq_init(&enic->rq[index],
+ enic_cq_rq(enic, index),
+ error_interrupt_enable,
+ error_interrupt_offset);
++
++ cq_idx = enic_cq_rq(enic, index);
++ vnic_cq_init(&enic->cq[cq_idx],
++ 0 /* flow_control_enable */,
++ 1 /* color_enable */,
++ 0 /* cq_head */,
++ 0 /* cq_tail */,
++ 1 /* cq_tail_color */,
++ 0 /* interrupt_enable */,
++ 1 /* cq_entry_enable */,
++ 0 /* cq_message_enable */,
++ 0 /* interrupt offset */,
++ 0 /* cq_message_addr */);
+ }
+
+ for (index = 0; index < enic->wq_count; index++) {
+@@ -310,22 +246,19 @@ void enic_init_vnic_resources(struct enic *enic)
+ enic_cq_wq(enic, index),
+ error_interrupt_enable,
+ error_interrupt_offset);
+- }
+-
+- vnic_dev_stats_clear(enic->vdev);
+
+- for (index = 0; index < enic->cq_count; index++) {
+- vnic_cq_init(&enic->cq[index],
++ cq_idx = enic_cq_wq(enic, index);
++ vnic_cq_init(&enic->cq[cq_idx],
+ 0 /* flow_control_enable */,
+ 1 /* color_enable */,
+ 0 /* cq_head */,
+ 0 /* cq_tail */,
+ 1 /* cq_tail_color */,
+ 0 /* interrupt_enable */,
+- 1 /* cq_entry_enable */,
+- 0 /* cq_message_enable */,
++ 0 /* cq_entry_enable */,
++ 1 /* cq_message_enable */,
+ 0 /* interrupt offset */,
+- 0 /* cq_message_addr */);
++ (u64)enic->wq[index].cqmsg_rz->phys_addr);
+ }
+
+ vnic_intr_init(&enic->intr,
+@@ -569,6 +502,7 @@ void enic_free_wq(void *txq)
+ struct vnic_wq *wq = (struct vnic_wq *)txq;
+ struct enic *enic = vnic_dev_priv(wq->vdev);
+
++ rte_memzone_free(wq->cqmsg_rz);
+ vnic_wq_free(wq);
+ vnic_cq_free(&enic->cq[enic->rq_count + wq->index]);
+ }
+@@ -579,6 +513,8 @@ int enic_alloc_wq(struct enic *enic, uint16_t queue_idx,
+ int err;
+ struct vnic_wq *wq = &enic->wq[queue_idx];
+ unsigned int cq_index = enic_cq_wq(enic, queue_idx);
++ char name[NAME_MAX];
++ static int instance;
+
+ wq->socket_id = socket_id;
+ if (nb_desc) {
+@@ -614,6 +550,18 @@ int enic_alloc_wq(struct enic *enic, uint16_t queue_idx,
+ dev_err(enic, "error in allocation of cq for wq\n");
+ }
+
++ /* setup up CQ message */
++ snprintf((char *)name, sizeof(name),
++ "vnic_cqmsg-%s-%d-%d", enic->bdf_name, queue_idx,
++ instance++);
++
++ wq->cqmsg_rz = rte_memzone_reserve_aligned((const char *)name,
++ sizeof(uint32_t),
++ SOCKET_ID_ANY, 0,
++ ENIC_ALIGN);
++ if (!wq->cqmsg_rz)
++ return -ENOMEM;
++
+ return err;
+ }
+
+diff --git a/drivers/net/enic/enic_res.h b/drivers/net/enic/enic_res.h
+index 00fa71d..3e1bdf5 100644
+--- a/drivers/net/enic/enic_res.h
++++ b/drivers/net/enic/enic_res.h
+@@ -53,89 +53,10 @@
+
+ #define ENIC_NON_TSO_MAX_DESC 16
+ #define ENIC_DEFAULT_RX_FREE_THRESH 32
+-#define ENIC_TX_POST_THRESH (ENIC_MIN_WQ_DESCS / 2)
++#define ENIC_TX_XMIT_MAX 64
+
+ #define ENIC_SETTING(enic, f) ((enic->config.flags & VENETF_##f) ? 1 : 0)
+
+-static inline void enic_queue_wq_desc_ex(struct vnic_wq *wq,
+- void *os_buf, dma_addr_t dma_addr, unsigned int len,
+- unsigned int mss_or_csum_offset, unsigned int hdr_len,
+- int vlan_tag_insert, unsigned int vlan_tag,
+- int offload_mode, int cq_entry, int sop, int eop, int loopback)
+-{
+- struct wq_enet_desc *desc = vnic_wq_next_desc(wq);
+- u8 desc_skip_cnt = 1;
+- u8 compressed_send = 0;
+- u64 wrid = 0;
+-
+- wq_enet_desc_enc(desc,
+- (u64)dma_addr | VNIC_PADDR_TARGET,
+- (u16)len,
+- (u16)mss_or_csum_offset,
+- (u16)hdr_len, (u8)offload_mode,
+- (u8)eop, (u8)cq_entry,
+- 0, /* fcoe_encap */
+- (u8)vlan_tag_insert,
+- (u16)vlan_tag,
+- (u8)loopback);
+-
+- vnic_wq_post(wq, os_buf, dma_addr, len, sop, eop, desc_skip_cnt,
+- (u8)cq_entry, compressed_send, wrid);
+-}
+-
+-static inline void enic_queue_wq_desc_cont(struct vnic_wq *wq,
+- void *os_buf, dma_addr_t dma_addr, unsigned int len,
+- int eop, int loopback)
+-{
+- enic_queue_wq_desc_ex(wq, os_buf, dma_addr, len,
+- 0, 0, 0, 0, 0,
+- eop, 0 /* !SOP */, eop, loopback);
+-}
+-
+-static inline void enic_queue_wq_desc(struct vnic_wq *wq, void *os_buf,
+- dma_addr_t dma_addr, unsigned int len, int vlan_tag_insert,
+- unsigned int vlan_tag, int eop, int loopback)
+-{
+- enic_queue_wq_desc_ex(wq, os_buf, dma_addr, len,
+- 0, 0, vlan_tag_insert, vlan_tag,
+- WQ_ENET_OFFLOAD_MODE_CSUM,
+- eop, 1 /* SOP */, eop, loopback);
+-}
+-
+-static inline void enic_queue_wq_desc_csum(struct vnic_wq *wq,
+- void *os_buf, dma_addr_t dma_addr, unsigned int len,
+- int ip_csum, int tcpudp_csum, int vlan_tag_insert,
+- unsigned int vlan_tag, int eop, int loopback)
+-{
+- enic_queue_wq_desc_ex(wq, os_buf, dma_addr, len,
+- (ip_csum ? 1 : 0) + (tcpudp_csum ? 2 : 0),
+- 0, vlan_tag_insert, vlan_tag,
+- WQ_ENET_OFFLOAD_MODE_CSUM,
+- eop, 1 /* SOP */, eop, loopback);
+-}
+-
+-static inline void enic_queue_wq_desc_csum_l4(struct vnic_wq *wq,
+- void *os_buf, dma_addr_t dma_addr, unsigned int len,
+- unsigned int csum_offset, unsigned int hdr_len,
+- int vlan_tag_insert, unsigned int vlan_tag, int eop, int loopback)
+-{
+- enic_queue_wq_desc_ex(wq, os_buf, dma_addr, len,
+- csum_offset, hdr_len, vlan_tag_insert, vlan_tag,
+- WQ_ENET_OFFLOAD_MODE_CSUM_L4,
+- eop, 1 /* SOP */, eop, loopback);
+-}
+-
+-static inline void enic_queue_wq_desc_tso(struct vnic_wq *wq,
+- void *os_buf, dma_addr_t dma_addr, unsigned int len,
+- unsigned int mss, unsigned int hdr_len, int vlan_tag_insert,
+- unsigned int vlan_tag, int eop, int loopback)
+-{
+- enic_queue_wq_desc_ex(wq, os_buf, dma_addr, len,
+- mss, hdr_len, vlan_tag_insert, vlan_tag,
+- WQ_ENET_OFFLOAD_MODE_TSO,
+- eop, 1 /* SOP */, eop, loopback);
+-}
+-
+ struct enic;
+
+ int enic_get_vnic_config(struct enic *);
+diff --git a/drivers/net/enic/enic_rx.c b/drivers/net/enic/enic_rx.c
+deleted file mode 100644
+index 39bb55c..0000000
+--- a/drivers/net/enic/enic_rx.c
++++ /dev/null
+@@ -1,361 +0,0 @@
+-/*
+- * Copyright 2008-2014 Cisco Systems, Inc. All rights reserved.
+- * Copyright 2007 Nuova Systems, Inc. All rights reserved.
+- *
+- * Copyright (c) 2014, Cisco Systems, Inc.
+- * All rights reserved.
+- *
+- * Redistribution and use in source and binary forms, with or without
+- * modification, are permitted provided that the following conditions
+- * are met:
+- *
+- * 1. Redistributions of source code must retain the above copyright
+- * notice, this list of conditions and the following disclaimer.
+- *
+- * 2. Redistributions in binary form must reproduce the above copyright
+- * notice, this list of conditions and the following disclaimer in
+- * the documentation and/or other materials provided with the
+- * distribution.
+- *
+- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+- * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+- * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+- * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+- * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+- * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+- * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+- * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+- * POSSIBILITY OF SUCH DAMAGE.
+- *
+- */
+-
+-#include <rte_mbuf.h>
+-#include <rte_ethdev.h>
+-#include <rte_prefetch.h>
+-
+-#include "enic_compat.h"
+-#include "rq_enet_desc.h"
+-#include "enic.h"
+-
+-#define RTE_PMD_USE_PREFETCH
+-
+-#ifdef RTE_PMD_USE_PREFETCH
+-/*
+- * Prefetch a cache line into all cache levels.
+- */
+-#define rte_enic_prefetch(p) rte_prefetch0(p)
+-#else
+-#define rte_enic_prefetch(p) do {} while (0)
+-#endif
+-
+-#ifdef RTE_PMD_PACKET_PREFETCH
+-#define rte_packet_prefetch(p) rte_prefetch1(p)
+-#else
+-#define rte_packet_prefetch(p) do {} while (0)
+-#endif
+-
+-static inline struct rte_mbuf *
+-rte_rxmbuf_alloc(struct rte_mempool *mp)
+-{
+- struct rte_mbuf *m;
+-
+- m = __rte_mbuf_raw_alloc(mp);
+- __rte_mbuf_sanity_check_raw(m, 0);
+- return m;
+-}
+-
+-static inline uint16_t
+-enic_cq_rx_desc_ciflags(struct cq_enet_rq_desc *crd)
+-{
+- return le16_to_cpu(crd->completed_index_flags) & ~CQ_DESC_COMP_NDX_MASK;
+-}
+-
+-static inline uint16_t
+-enic_cq_rx_desc_bwflags(struct cq_enet_rq_desc *crd)
+-{
+- return(le16_to_cpu(crd->bytes_written_flags) &
+- ~CQ_ENET_RQ_DESC_BYTES_WRITTEN_MASK);
+-}
+-
+-static inline uint8_t
+-enic_cq_rx_desc_packet_error(uint16_t bwflags)
+-{
+- return((bwflags & CQ_ENET_RQ_DESC_FLAGS_TRUNCATED) ==
+- CQ_ENET_RQ_DESC_FLAGS_TRUNCATED);
+-}
+-
+-static inline uint8_t
+-enic_cq_rx_desc_eop(uint16_t ciflags)
+-{
+- return (ciflags & CQ_ENET_RQ_DESC_FLAGS_EOP)
+- == CQ_ENET_RQ_DESC_FLAGS_EOP;
+-}
+-
+-static inline uint8_t
+-enic_cq_rx_desc_csum_not_calc(struct cq_enet_rq_desc *cqrd)
+-{
+- return ((le16_to_cpu(cqrd->q_number_rss_type_flags) &
+- CQ_ENET_RQ_DESC_FLAGS_CSUM_NOT_CALC) ==
+- CQ_ENET_RQ_DESC_FLAGS_CSUM_NOT_CALC);
+-}
+-
+-static inline uint8_t
+-enic_cq_rx_desc_ipv4_csum_ok(struct cq_enet_rq_desc *cqrd)
+-{
+- return ((cqrd->flags & CQ_ENET_RQ_DESC_FLAGS_IPV4_CSUM_OK) ==
+- CQ_ENET_RQ_DESC_FLAGS_IPV4_CSUM_OK);
+-}
+-
+-static inline uint8_t
+-enic_cq_rx_desc_tcp_udp_csum_ok(struct cq_enet_rq_desc *cqrd)
+-{
+- return((cqrd->flags & CQ_ENET_RQ_DESC_FLAGS_TCP_UDP_CSUM_OK) ==
+- CQ_ENET_RQ_DESC_FLAGS_TCP_UDP_CSUM_OK);
+-}
+-
+-static inline uint8_t
+-enic_cq_rx_desc_rss_type(struct cq_enet_rq_desc *cqrd)
+-{
+- return (uint8_t)((le16_to_cpu(cqrd->q_number_rss_type_flags) >>
+- CQ_DESC_Q_NUM_BITS) & CQ_ENET_RQ_DESC_RSS_TYPE_MASK);
+-}
+-
+-static inline uint32_t
+-enic_cq_rx_desc_rss_hash(struct cq_enet_rq_desc *cqrd)
+-{
+- return le32_to_cpu(cqrd->rss_hash);
+-}
+-
+-static inline uint16_t
+-enic_cq_rx_desc_vlan(struct cq_enet_rq_desc *cqrd)
+-{
+- return le16_to_cpu(cqrd->vlan);
+-}
+-
+-static inline uint16_t
+-enic_cq_rx_desc_n_bytes(struct cq_desc *cqd)
+-{
+- struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd;
+- return le16_to_cpu(cqrd->bytes_written_flags) &
+- CQ_ENET_RQ_DESC_BYTES_WRITTEN_MASK;
+-}
+-
+-static inline uint8_t
+-enic_cq_rx_to_pkt_err_flags(struct cq_desc *cqd, uint64_t *pkt_err_flags_out)
+-{
+- struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd;
+- uint16_t bwflags;
+- int ret = 0;
+- uint64_t pkt_err_flags = 0;
+-
+- bwflags = enic_cq_rx_desc_bwflags(cqrd);
+- if (unlikely(enic_cq_rx_desc_packet_error(bwflags))) {
+- pkt_err_flags = PKT_RX_MAC_ERR;
+- ret = 1;
+- }
+- *pkt_err_flags_out = pkt_err_flags;
+- return ret;
+-}
+-
+-/*
+- * Lookup table to translate RX CQ flags to mbuf flags.
+- */
+-static inline uint32_t
+-enic_cq_rx_flags_to_pkt_type(struct cq_desc *cqd)
+-{
+- struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd;
+- uint8_t cqrd_flags = cqrd->flags;
+- static const uint32_t cq_type_table[128] __rte_cache_aligned = {
+- [32] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4,
+- [34] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4
+- | RTE_PTYPE_L4_UDP,
+- [36] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4
+- | RTE_PTYPE_L4_TCP,
+- [96] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4
+- | RTE_PTYPE_L4_FRAG,
+- [16] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6,
+- [18] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6
+- | RTE_PTYPE_L4_UDP,
+- [20] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6
+- | RTE_PTYPE_L4_TCP,
+- [80] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6
+- | RTE_PTYPE_L4_FRAG,
+- /* All others reserved */
+- };
+- cqrd_flags &= CQ_ENET_RQ_DESC_FLAGS_IPV4_FRAGMENT
+- | CQ_ENET_RQ_DESC_FLAGS_IPV4 | CQ_ENET_RQ_DESC_FLAGS_IPV6
+- | CQ_ENET_RQ_DESC_FLAGS_TCP | CQ_ENET_RQ_DESC_FLAGS_UDP;
+- return cq_type_table[cqrd_flags];
+-}
+-
+-static inline void
+-enic_cq_rx_to_pkt_flags(struct cq_desc *cqd, struct rte_mbuf *mbuf)
+-{
+- struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd;
+- uint16_t ciflags, bwflags, pkt_flags = 0;
+- ciflags = enic_cq_rx_desc_ciflags(cqrd);
+- bwflags = enic_cq_rx_desc_bwflags(cqrd);
+-
+- mbuf->ol_flags = 0;
+-
+- /* flags are meaningless if !EOP */
+- if (unlikely(!enic_cq_rx_desc_eop(ciflags)))
+- goto mbuf_flags_done;
+-
+- /* VLAN stripping */
+- if (bwflags & CQ_ENET_RQ_DESC_FLAGS_VLAN_STRIPPED) {
+- pkt_flags |= PKT_RX_VLAN_PKT;
+- mbuf->vlan_tci = enic_cq_rx_desc_vlan(cqrd);
+- } else {
+- mbuf->vlan_tci = 0;
+- }
+-
+- /* RSS flag */
+- if (enic_cq_rx_desc_rss_type(cqrd)) {
+- pkt_flags |= PKT_RX_RSS_HASH;
+- mbuf->hash.rss = enic_cq_rx_desc_rss_hash(cqrd);
+- }
+-
+- /* checksum flags */
+- if (!enic_cq_rx_desc_csum_not_calc(cqrd) &&
+- (mbuf->packet_type & RTE_PTYPE_L3_IPV4)) {
+- if (unlikely(!enic_cq_rx_desc_ipv4_csum_ok(cqrd)))
+- pkt_flags |= PKT_RX_IP_CKSUM_BAD;
+- if (mbuf->packet_type & (RTE_PTYPE_L4_UDP | RTE_PTYPE_L4_TCP)) {
+- if (unlikely(!enic_cq_rx_desc_tcp_udp_csum_ok(cqrd)))
+- pkt_flags |= PKT_RX_L4_CKSUM_BAD;
+- }
+- }
+-
+- mbuf_flags_done:
+- mbuf->ol_flags = pkt_flags;
+-}
+-
+-static inline uint32_t
+-enic_ring_add(uint32_t n_descriptors, uint32_t i0, uint32_t i1)
+-{
+- uint32_t d = i0 + i1;
+- ASSERT(i0 < n_descriptors);
+- ASSERT(i1 < n_descriptors);
+- d -= (d >= n_descriptors) ? n_descriptors : 0;
+- return d;
+-}
+-
+-
+-uint16_t
+-enic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
+- uint16_t nb_pkts)
+-{
+- struct vnic_rq *rq = rx_queue;
+- struct enic *enic = vnic_dev_priv(rq->vdev);
+- unsigned int rx_id;
+- struct rte_mbuf *nmb, *rxmb;
+- uint16_t nb_rx = 0;
+- uint16_t nb_hold;
+- struct vnic_cq *cq;
+- volatile struct cq_desc *cqd_ptr;
+- uint8_t color;
+-
+- cq = &enic->cq[enic_cq_rq(enic, rq->index)];
+- rx_id = cq->to_clean; /* index of cqd, rqd, mbuf_table */
+- cqd_ptr = (struct cq_desc *)(cq->ring.descs) + rx_id;
+-
+- nb_hold = rq->rx_nb_hold; /* mbufs held by software */
+-
+- while (nb_rx < nb_pkts) {
+- volatile struct rq_enet_desc *rqd_ptr;
+- dma_addr_t dma_addr;
+- struct cq_desc cqd;
+- uint64_t ol_err_flags;
+- uint8_t packet_error;
+-
+- /* Check for pkts available */
+- color = (cqd_ptr->type_color >> CQ_DESC_COLOR_SHIFT)
+- & CQ_DESC_COLOR_MASK;
+- if (color == cq->last_color)
+- break;
+-
+- /* Get the cq descriptor and rq pointer */
+- cqd = *cqd_ptr;
+- rqd_ptr = (struct rq_enet_desc *)(rq->ring.descs) + rx_id;
+-
+- /* allocate a new mbuf */
+- nmb = rte_rxmbuf_alloc(rq->mp);
+- if (nmb == NULL) {
+- dev_err(enic, "RX mbuf alloc failed port=%u qid=%u",
+- enic->port_id, (unsigned)rq->index);
+- rte_eth_devices[enic->port_id].
+- data->rx_mbuf_alloc_failed++;
+- break;
+- }
+-
+- /* A packet error means descriptor and data are untrusted */
+- packet_error = enic_cq_rx_to_pkt_err_flags(&cqd, &ol_err_flags);
+-
+- /* Get the mbuf to return and replace with one just allocated */
+- rxmb = rq->mbuf_ring[rx_id];
+- rq->mbuf_ring[rx_id] = nmb;
+-
+- /* Increment cqd, rqd, mbuf_table index */
+- rx_id++;
+- if (unlikely(rx_id == rq->ring.desc_count)) {
+- rx_id = 0;
+- cq->last_color = cq->last_color ? 0 : 1;
+- }
+-
+- /* Prefetch next mbuf & desc while processing current one */
+- cqd_ptr = (struct cq_desc *)(cq->ring.descs) + rx_id;
+- rte_enic_prefetch(cqd_ptr);
+- rte_enic_prefetch(rq->mbuf_ring[rx_id]);
+- rte_enic_prefetch((struct rq_enet_desc *)(rq->ring.descs)
+- + rx_id);
+-
+- /* Push descriptor for newly allocated mbuf */
+- dma_addr = (dma_addr_t)(nmb->buf_physaddr
+- + RTE_PKTMBUF_HEADROOM);
+- rqd_ptr->address = rte_cpu_to_le_64(dma_addr);
+- rqd_ptr->length_type = cpu_to_le16(nmb->buf_len
+- - RTE_PKTMBUF_HEADROOM);
+-
+- /* Fill in the rest of the mbuf */
+- rxmb->data_off = RTE_PKTMBUF_HEADROOM;
+- rxmb->nb_segs = 1;
+- rxmb->next = NULL;
+- rxmb->port = enic->port_id;
+- if (!packet_error) {
+- rxmb->pkt_len = enic_cq_rx_desc_n_bytes(&cqd);
+- rxmb->packet_type = enic_cq_rx_flags_to_pkt_type(&cqd);
+- enic_cq_rx_to_pkt_flags(&cqd, rxmb);
+- } else {
+- rxmb->pkt_len = 0;
+- rxmb->packet_type = 0;
+- rxmb->ol_flags = 0;
+- }
+- rxmb->data_len = rxmb->pkt_len;
+-
+- /* prefetch mbuf data for caller */
+- rte_packet_prefetch(RTE_PTR_ADD(rxmb->buf_addr,
+- RTE_PKTMBUF_HEADROOM));
+-
+- /* store the mbuf address into the next entry of the array */
+- rx_pkts[nb_rx++] = rxmb;
+- }
+-
+- nb_hold += nb_rx;
+- cq->to_clean = rx_id;
+-
+- if (nb_hold > rq->rx_free_thresh) {
+- rq->posted_index = enic_ring_add(rq->ring.desc_count,
+- rq->posted_index, nb_hold);
+- nb_hold = 0;
+- rte_mb();
+- iowrite32(rq->posted_index, &rq->ctrl->posted_index);
+- }
+-
+- rq->rx_nb_hold = nb_hold;
+-
+- return nb_rx;
+-}
+diff --git a/drivers/net/enic/enic_rxtx.c b/drivers/net/enic/enic_rxtx.c
+new file mode 100644
+index 0000000..71ca34e
+--- /dev/null
++++ b/drivers/net/enic/enic_rxtx.c
+@@ -0,0 +1,505 @@
++/*
++ * Copyright 2008-2016 Cisco Systems, Inc. All rights reserved.
++ * Copyright 2007 Nuova Systems, Inc. All rights reserved.
++ *
++ * Copyright (c) 2016, Cisco Systems, Inc.
++ * All rights reserved.
++ *
++ * Redistribution and use in source and binary forms, with or without
++ * modification, are permitted provided that the following conditions
++ * are met:
++ *
++ * 1. Redistributions of source code must retain the above copyright
++ * notice, this list of conditions and the following disclaimer.
++ *
++ * 2. Redistributions in binary form must reproduce the above copyright
++ * notice, this list of conditions and the following disclaimer in
++ * the documentation and/or other materials provided with the
++ * distribution.
++ *
++ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
++ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
++ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
++ * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
++ * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
++ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
++ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
++ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
++ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
++ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
++ * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
++ * POSSIBILITY OF SUCH DAMAGE.
++ *
++ */
++
++#include <rte_mbuf.h>
++#include <rte_ethdev.h>
++#include <rte_prefetch.h>
++#include <rte_memzone.h>
++
++#include "enic_compat.h"
++#include "rq_enet_desc.h"
++#include "enic.h"
++
++#define RTE_PMD_USE_PREFETCH
++
++#ifdef RTE_PMD_USE_PREFETCH
++/*
++ * Prefetch a cache line into all cache levels.
++ */
++#define rte_enic_prefetch(p) rte_prefetch0(p)
++#else
++#define rte_enic_prefetch(p) do {} while (0)
++#endif
++
++#ifdef RTE_PMD_PACKET_PREFETCH
++#define rte_packet_prefetch(p) rte_prefetch1(p)
++#else
++#define rte_packet_prefetch(p) do {} while (0)
++#endif
++
++static inline struct rte_mbuf *
++rte_rxmbuf_alloc(struct rte_mempool *mp)
++{
++ struct rte_mbuf *m;
++
++ m = __rte_mbuf_raw_alloc(mp);
++ __rte_mbuf_sanity_check_raw(m, 0);
++ return m;
++}
++
++static inline uint16_t
++enic_cq_rx_desc_ciflags(struct cq_enet_rq_desc *crd)
++{
++ return le16_to_cpu(crd->completed_index_flags) & ~CQ_DESC_COMP_NDX_MASK;
++}
++
++static inline uint16_t
++enic_cq_rx_desc_bwflags(struct cq_enet_rq_desc *crd)
++{
++ return(le16_to_cpu(crd->bytes_written_flags) &
++ ~CQ_ENET_RQ_DESC_BYTES_WRITTEN_MASK);
++}
++
++static inline uint8_t
++enic_cq_rx_desc_packet_error(uint16_t bwflags)
++{
++ return((bwflags & CQ_ENET_RQ_DESC_FLAGS_TRUNCATED) ==
++ CQ_ENET_RQ_DESC_FLAGS_TRUNCATED);
++}
++
++static inline uint8_t
++enic_cq_rx_desc_eop(uint16_t ciflags)
++{
++ return (ciflags & CQ_ENET_RQ_DESC_FLAGS_EOP)
++ == CQ_ENET_RQ_DESC_FLAGS_EOP;
++}
++
++static inline uint8_t
++enic_cq_rx_desc_csum_not_calc(struct cq_enet_rq_desc *cqrd)
++{
++ return ((le16_to_cpu(cqrd->q_number_rss_type_flags) &
++ CQ_ENET_RQ_DESC_FLAGS_CSUM_NOT_CALC) ==
++ CQ_ENET_RQ_DESC_FLAGS_CSUM_NOT_CALC);
++}
++
++static inline uint8_t
++enic_cq_rx_desc_ipv4_csum_ok(struct cq_enet_rq_desc *cqrd)
++{
++ return ((cqrd->flags & CQ_ENET_RQ_DESC_FLAGS_IPV4_CSUM_OK) ==
++ CQ_ENET_RQ_DESC_FLAGS_IPV4_CSUM_OK);
++}
++
++static inline uint8_t
++enic_cq_rx_desc_tcp_udp_csum_ok(struct cq_enet_rq_desc *cqrd)
++{
++ return((cqrd->flags & CQ_ENET_RQ_DESC_FLAGS_TCP_UDP_CSUM_OK) ==
++ CQ_ENET_RQ_DESC_FLAGS_TCP_UDP_CSUM_OK);
++}
++
++static inline uint8_t
++enic_cq_rx_desc_rss_type(struct cq_enet_rq_desc *cqrd)
++{
++ return (uint8_t)((le16_to_cpu(cqrd->q_number_rss_type_flags) >>
++ CQ_DESC_Q_NUM_BITS) & CQ_ENET_RQ_DESC_RSS_TYPE_MASK);
++}
++
++static inline uint32_t
++enic_cq_rx_desc_rss_hash(struct cq_enet_rq_desc *cqrd)
++{
++ return le32_to_cpu(cqrd->rss_hash);
++}
++
++static inline uint16_t
++enic_cq_rx_desc_vlan(struct cq_enet_rq_desc *cqrd)
++{
++ return le16_to_cpu(cqrd->vlan);
++}
++
++static inline uint16_t
++enic_cq_rx_desc_n_bytes(struct cq_desc *cqd)
++{
++ struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd;
++ return le16_to_cpu(cqrd->bytes_written_flags) &
++ CQ_ENET_RQ_DESC_BYTES_WRITTEN_MASK;
++}
++
++static inline uint8_t
++enic_cq_rx_to_pkt_err_flags(struct cq_desc *cqd, uint64_t *pkt_err_flags_out)
++{
++ struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd;
++ uint16_t bwflags;
++ int ret = 0;
++ uint64_t pkt_err_flags = 0;
++
++ bwflags = enic_cq_rx_desc_bwflags(cqrd);
++ if (unlikely(enic_cq_rx_desc_packet_error(bwflags))) {
++ pkt_err_flags = PKT_RX_MAC_ERR;
++ ret = 1;
++ }
++ *pkt_err_flags_out = pkt_err_flags;
++ return ret;
++}
++
++/*
++ * Lookup table to translate RX CQ flags to mbuf flags.
++ */
++static inline uint32_t
++enic_cq_rx_flags_to_pkt_type(struct cq_desc *cqd)
++{
++ struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd;
++ uint8_t cqrd_flags = cqrd->flags;
++ static const uint32_t cq_type_table[128] __rte_cache_aligned = {
++ [32] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4,
++ [34] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4
++ | RTE_PTYPE_L4_UDP,
++ [36] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4
++ | RTE_PTYPE_L4_TCP,
++ [96] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4
++ | RTE_PTYPE_L4_FRAG,
++ [16] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6,
++ [18] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6
++ | RTE_PTYPE_L4_UDP,
++ [20] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6
++ | RTE_PTYPE_L4_TCP,
++ [80] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6
++ | RTE_PTYPE_L4_FRAG,
++ /* All others reserved */
++ };
++ cqrd_flags &= CQ_ENET_RQ_DESC_FLAGS_IPV4_FRAGMENT
++ | CQ_ENET_RQ_DESC_FLAGS_IPV4 | CQ_ENET_RQ_DESC_FLAGS_IPV6
++ | CQ_ENET_RQ_DESC_FLAGS_TCP | CQ_ENET_RQ_DESC_FLAGS_UDP;
++ return cq_type_table[cqrd_flags];
++}
++
++static inline void
++enic_cq_rx_to_pkt_flags(struct cq_desc *cqd, struct rte_mbuf *mbuf)
++{
++ struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd;
++ uint16_t ciflags, bwflags, pkt_flags = 0;
++ ciflags = enic_cq_rx_desc_ciflags(cqrd);
++ bwflags = enic_cq_rx_desc_bwflags(cqrd);
++
++ mbuf->ol_flags = 0;
++
++ /* flags are meaningless if !EOP */
++ if (unlikely(!enic_cq_rx_desc_eop(ciflags)))
++ goto mbuf_flags_done;
++
++ /* VLAN stripping */
++ if (bwflags & CQ_ENET_RQ_DESC_FLAGS_VLAN_STRIPPED) {
++ pkt_flags |= PKT_RX_VLAN_PKT;
++ mbuf->vlan_tci = enic_cq_rx_desc_vlan(cqrd);
++ } else {
++ mbuf->vlan_tci = 0;
++ }
++
++ /* RSS flag */
++ if (enic_cq_rx_desc_rss_type(cqrd)) {
++ pkt_flags |= PKT_RX_RSS_HASH;
++ mbuf->hash.rss = enic_cq_rx_desc_rss_hash(cqrd);
++ }
++
++ /* checksum flags */
++ if (!enic_cq_rx_desc_csum_not_calc(cqrd) &&
++ (mbuf->packet_type & RTE_PTYPE_L3_IPV4)) {
++ if (unlikely(!enic_cq_rx_desc_ipv4_csum_ok(cqrd)))
++ pkt_flags |= PKT_RX_IP_CKSUM_BAD;
++ if (mbuf->packet_type & (RTE_PTYPE_L4_UDP | RTE_PTYPE_L4_TCP)) {
++ if (unlikely(!enic_cq_rx_desc_tcp_udp_csum_ok(cqrd)))
++ pkt_flags |= PKT_RX_L4_CKSUM_BAD;
++ }
++ }
++
++ mbuf_flags_done:
++ mbuf->ol_flags = pkt_flags;
++}
++
++uint16_t
++enic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
++ uint16_t nb_pkts)
++{
++ struct vnic_rq *rq = rx_queue;
++ struct enic *enic = vnic_dev_priv(rq->vdev);
++ unsigned int rx_id;
++ struct rte_mbuf *nmb, *rxmb;
++ uint16_t nb_rx = 0;
++ uint16_t nb_hold;
++ struct vnic_cq *cq;
++ volatile struct cq_desc *cqd_ptr;
++ uint8_t color;
++
++ cq = &enic->cq[enic_cq_rq(enic, rq->index)];
++ rx_id = cq->to_clean; /* index of cqd, rqd, mbuf_table */
++ cqd_ptr = (struct cq_desc *)(cq->ring.descs) + rx_id;
++
++ nb_hold = rq->rx_nb_hold; /* mbufs held by software */
++
++ while (nb_rx < nb_pkts) {
++ volatile struct rq_enet_desc *rqd_ptr;
++ dma_addr_t dma_addr;
++ struct cq_desc cqd;
++ uint64_t ol_err_flags;
++ uint8_t packet_error;
++
++ /* Check for pkts available */
++ color = (cqd_ptr->type_color >> CQ_DESC_COLOR_SHIFT)
++ & CQ_DESC_COLOR_MASK;
++ if (color == cq->last_color)
++ break;
++
++ /* Get the cq descriptor and rq pointer */
++ cqd = *cqd_ptr;
++ rqd_ptr = (struct rq_enet_desc *)(rq->ring.descs) + rx_id;
++
++ /* allocate a new mbuf */
++ nmb = rte_rxmbuf_alloc(rq->mp);
++ if (nmb == NULL) {
++ dev_err(enic, "RX mbuf alloc failed port=%u qid=%u",
++ enic->port_id, (unsigned)rq->index);
++ rte_eth_devices[enic->port_id].
++ data->rx_mbuf_alloc_failed++;
++ break;
++ }
++
++ /* A packet error means descriptor and data are untrusted */
++ packet_error = enic_cq_rx_to_pkt_err_flags(&cqd, &ol_err_flags);
++
++ /* Get the mbuf to return and replace with one just allocated */
++ rxmb = rq->mbuf_ring[rx_id];
++ rq->mbuf_ring[rx_id] = nmb;
++
++ /* Increment cqd, rqd, mbuf_table index */
++ rx_id++;
++ if (unlikely(rx_id == rq->ring.desc_count)) {
++ rx_id = 0;
++ cq->last_color = cq->last_color ? 0 : 1;
++ }
++
++ /* Prefetch next mbuf & desc while processing current one */
++ cqd_ptr = (struct cq_desc *)(cq->ring.descs) + rx_id;
++ rte_enic_prefetch(cqd_ptr);
++ rte_enic_prefetch(rq->mbuf_ring[rx_id]);
++ rte_enic_prefetch((struct rq_enet_desc *)(rq->ring.descs)
++ + rx_id);
++
++ /* Push descriptor for newly allocated mbuf */
++ dma_addr = (dma_addr_t)(nmb->buf_physaddr
++ + RTE_PKTMBUF_HEADROOM);
++ rqd_ptr->address = rte_cpu_to_le_64(dma_addr);
++ rqd_ptr->length_type = cpu_to_le16(nmb->buf_len
++ - RTE_PKTMBUF_HEADROOM);
++
++ /* Fill in the rest of the mbuf */
++ rxmb->data_off = RTE_PKTMBUF_HEADROOM;
++ rxmb->nb_segs = 1;
++ rxmb->next = NULL;
++ rxmb->port = enic->port_id;
++ if (!packet_error) {
++ rxmb->pkt_len = enic_cq_rx_desc_n_bytes(&cqd);
++ rxmb->packet_type = enic_cq_rx_flags_to_pkt_type(&cqd);
++ enic_cq_rx_to_pkt_flags(&cqd, rxmb);
++ } else {
++ rxmb->pkt_len = 0;
++ rxmb->packet_type = 0;
++ rxmb->ol_flags = 0;
++ }
++ rxmb->data_len = rxmb->pkt_len;
++
++ /* prefetch mbuf data for caller */
++ rte_packet_prefetch(RTE_PTR_ADD(rxmb->buf_addr,
++ RTE_PKTMBUF_HEADROOM));
++
++ /* store the mbuf address into the next entry of the array */
++ rx_pkts[nb_rx++] = rxmb;
++ }
++
++ nb_hold += nb_rx;
++ cq->to_clean = rx_id;
++
++ if (nb_hold > rq->rx_free_thresh) {
++ rq->posted_index = enic_ring_add(rq->ring.desc_count,
++ rq->posted_index, nb_hold);
++ nb_hold = 0;
++ rte_mb();
++ iowrite32(rq->posted_index, &rq->ctrl->posted_index);
++ }
++
++ rq->rx_nb_hold = nb_hold;
++
++ return nb_rx;
++}
++
++static inline void enic_free_wq_bufs(struct vnic_wq *wq, u16 completed_index)
++{
++ struct vnic_wq_buf *buf;
++ struct rte_mbuf *m, *free[ENIC_MAX_WQ_DESCS];
++ unsigned int nb_to_free, nb_free = 0, i;
++ struct rte_mempool *pool;
++ unsigned int tail_idx;
++ unsigned int desc_count = wq->ring.desc_count;
++
++ nb_to_free = enic_ring_sub(desc_count, wq->tail_idx, completed_index)
++ + 1;
++ tail_idx = wq->tail_idx;
++ buf = &wq->bufs[tail_idx];
++ pool = ((struct rte_mbuf *)buf->mb)->pool;
++ for (i = 0; i < nb_to_free; i++) {
++ buf = &wq->bufs[tail_idx];
++ m = (struct rte_mbuf *)(buf->mb);
++ if (likely(m->pool == pool)) {
++ ASSERT(nb_free < ENIC_MAX_WQ_DESCS);
++ free[nb_free++] = m;
++ } else {
++ rte_mempool_put_bulk(pool, (void *)free, nb_free);
++ free[0] = m;
++ nb_free = 1;
++ pool = m->pool;
++ }
++ tail_idx = enic_ring_incr(desc_count, tail_idx);
++ buf->mb = NULL;
++ }
++
++ rte_mempool_put_bulk(pool, (void **)free, nb_free);
++
++ wq->tail_idx = tail_idx;
++ wq->ring.desc_avail += nb_to_free;
++}
++
++unsigned int enic_cleanup_wq(__rte_unused struct enic *enic, struct vnic_wq *wq)
++{
++ u16 completed_index;
++
++ completed_index = *((uint32_t *)wq->cqmsg_rz->addr) & 0xffff;
++
++ if (wq->last_completed_index != completed_index) {
++ enic_free_wq_bufs(wq, completed_index);
++ wq->last_completed_index = completed_index;
++ }
++ return 0;
++}
++
++uint16_t enic_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,
++ uint16_t nb_pkts)
++{
++ uint16_t index;
++ unsigned int pkt_len, data_len;
++ unsigned int nb_segs;
++ struct rte_mbuf *tx_pkt;
++ struct vnic_wq *wq = (struct vnic_wq *)tx_queue;
++ struct enic *enic = vnic_dev_priv(wq->vdev);
++ unsigned short vlan_id;
++ unsigned short ol_flags;
++ unsigned int wq_desc_avail;
++ int head_idx;
++ struct vnic_wq_buf *buf;
++ unsigned int hw_ip_cksum_enabled;
++ unsigned int desc_count;
++ struct wq_enet_desc *descs, *desc_p, desc_tmp;
++ uint16_t mss;
++ uint8_t vlan_tag_insert;
++ uint8_t eop;
++ uint64_t bus_addr;
++
++ enic_cleanup_wq(enic, wq);
++ wq_desc_avail = vnic_wq_desc_avail(wq);
++ head_idx = wq->head_idx;
++ desc_count = wq->ring.desc_count;
++
++ nb_pkts = RTE_MIN(nb_pkts, ENIC_TX_XMIT_MAX);
++
++ hw_ip_cksum_enabled = enic->hw_ip_checksum;
++ for (index = 0; index < nb_pkts; index++) {
++ tx_pkt = *tx_pkts++;
++ nb_segs = tx_pkt->nb_segs;
++ if (nb_segs > wq_desc_avail) {
++ if (index > 0)
++ goto post;
++ goto done;
++ }
++
++ pkt_len = tx_pkt->pkt_len;
++ data_len = tx_pkt->data_len;
++ vlan_id = tx_pkt->vlan_tci;
++ ol_flags = tx_pkt->ol_flags;
++
++ mss = 0;
++ vlan_tag_insert = 0;
++ bus_addr = (dma_addr_t)
++ (tx_pkt->buf_physaddr + tx_pkt->data_off);
++
++ descs = (struct wq_enet_desc *)wq->ring.descs;
++ desc_p = descs + head_idx;
++
++ eop = (data_len == pkt_len);
++
++ if (ol_flags & PKT_TX_VLAN_PKT)
++ vlan_tag_insert = 1;
++
++ if (hw_ip_cksum_enabled && (ol_flags & PKT_TX_IP_CKSUM))
++ mss |= ENIC_CALC_IP_CKSUM;
++
++ if (hw_ip_cksum_enabled && (ol_flags & PKT_TX_TCP_UDP_CKSUM))
++ mss |= ENIC_CALC_TCP_UDP_CKSUM;
++
++ wq_enet_desc_enc(&desc_tmp, bus_addr, data_len, mss, 0, 0, eop,
++ eop, 0, vlan_tag_insert, vlan_id, 0);
++
++ *desc_p = desc_tmp;
++ buf = &wq->bufs[head_idx];
++ buf->mb = (void *)tx_pkt;
++ head_idx = enic_ring_incr(desc_count, head_idx);
++ wq_desc_avail--;
++
++ if (!eop) {
++ for (tx_pkt = tx_pkt->next; tx_pkt; tx_pkt =
++ tx_pkt->next) {
++ data_len = tx_pkt->data_len;
++
++ if (tx_pkt->next == NULL)
++ eop = 1;
++ desc_p = descs + head_idx;
++ bus_addr = (dma_addr_t)(tx_pkt->buf_physaddr
++ + tx_pkt->data_off);
++ wq_enet_desc_enc((struct wq_enet_desc *)
++ &desc_tmp, bus_addr, data_len,
++ mss, 0, 0, eop, eop, 0,
++ vlan_tag_insert, vlan_id, 0);
++
++ *desc_p = desc_tmp;
++ buf = &wq->bufs[head_idx];
++ buf->mb = (void *)tx_pkt;
++ head_idx = enic_ring_incr(desc_count, head_idx);
++ wq_desc_avail--;
++ }
++ }
++ }
++ post:
++ rte_wmb();
++ iowrite32(head_idx, &wq->ctrl->posted_index);
++ done:
++ wq->ring.desc_avail = wq_desc_avail;
++ wq->head_idx = head_idx;
++
++ return index;
++}
+--
+2.7.0
+