| From 0963caf48db910ca6f93c5cbff8d5d24b2be64dd Mon Sep 17 00:00:00 2001 |
| From: John Lo <loj@cisco.com> |
| Date: Mon, 7 Mar 2016 14:12:29 -0500 |
| Subject: [PATCH 2/2] Replacement of ENIC PMD receive path to improve |
| performance and code clarifty. |
| |
| --- |
| drivers/net/enic/Makefile | 1 + |
| drivers/net/enic/base/vnic_rq.c | 99 ++--------- |
| drivers/net/enic/base/vnic_rq.h | 147 +--------------- |
| drivers/net/enic/enic.h | 16 +- |
| drivers/net/enic/enic_ethdev.c | 27 ++- |
| drivers/net/enic/enic_main.c | 321 ++++++++++------------------------ |
| drivers/net/enic/enic_res.h | 16 +- |
| drivers/net/enic/enic_rx.c | 378 ++++++++++++++++++++++++++++++++++++++++ |
| 8 files changed, 519 insertions(+), 486 deletions(-) |
| create mode 100644 drivers/net/enic/enic_rx.c |
| |
| diff --git a/drivers/net/enic/Makefile b/drivers/net/enic/Makefile |
| index f0ee093..f316274 100644 |
| --- a/drivers/net/enic/Makefile |
| +++ b/drivers/net/enic/Makefile |
| @@ -53,6 +53,7 @@ VPATH += $(SRCDIR)/src |
| # |
| SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_ethdev.c |
| SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_main.c |
| +SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_rx.c |
| SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_clsf.c |
| SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += enic_res.c |
| SRCS-$(CONFIG_RTE_LIBRTE_ENIC_PMD) += base/vnic_cq.c |
| diff --git a/drivers/net/enic/base/vnic_rq.c b/drivers/net/enic/base/vnic_rq.c |
| index 1441604..cb62c5e 100644 |
| --- a/drivers/net/enic/base/vnic_rq.c |
| +++ b/drivers/net/enic/base/vnic_rq.c |
| @@ -35,77 +35,21 @@ |
| #include "vnic_dev.h" |
| #include "vnic_rq.h" |
| |
| -static int vnic_rq_alloc_bufs(struct vnic_rq *rq) |
| -{ |
| - struct vnic_rq_buf *buf; |
| - unsigned int i, j, count = rq->ring.desc_count; |
| - unsigned int blks = VNIC_RQ_BUF_BLKS_NEEDED(count); |
| - |
| - for (i = 0; i < blks; i++) { |
| - rq->bufs[i] = kzalloc(VNIC_RQ_BUF_BLK_SZ(count), GFP_ATOMIC); |
| - if (!rq->bufs[i]) |
| - return -ENOMEM; |
| - } |
| - |
| - for (i = 0; i < blks; i++) { |
| - buf = rq->bufs[i]; |
| - for (j = 0; j < VNIC_RQ_BUF_BLK_ENTRIES(count); j++) { |
| - buf->index = i * VNIC_RQ_BUF_BLK_ENTRIES(count) + j; |
| - buf->desc = (u8 *)rq->ring.descs + |
| - rq->ring.desc_size * buf->index; |
| - if (buf->index + 1 == count) { |
| - buf->next = rq->bufs[0]; |
| - break; |
| - } else if (j + 1 == VNIC_RQ_BUF_BLK_ENTRIES(count)) { |
| - buf->next = rq->bufs[i + 1]; |
| - } else { |
| - buf->next = buf + 1; |
| - buf++; |
| - } |
| - } |
| - } |
| - |
| - rq->to_use = rq->to_clean = rq->bufs[0]; |
| - |
| - return 0; |
| -} |
| - |
| -int vnic_rq_mem_size(struct vnic_rq *rq, unsigned int desc_count, |
| - unsigned int desc_size) |
| -{ |
| - int mem_size = 0; |
| - |
| - mem_size += vnic_dev_desc_ring_size(&rq->ring, desc_count, desc_size); |
| - |
| - mem_size += VNIC_RQ_BUF_BLKS_NEEDED(rq->ring.desc_count) * |
| - VNIC_RQ_BUF_BLK_SZ(rq->ring.desc_count); |
| - |
| - return mem_size; |
| -} |
| - |
| void vnic_rq_free(struct vnic_rq *rq) |
| { |
| struct vnic_dev *vdev; |
| - unsigned int i; |
| |
| vdev = rq->vdev; |
| |
| vnic_dev_free_desc_ring(vdev, &rq->ring); |
| |
| - for (i = 0; i < VNIC_RQ_BUF_BLKS_MAX; i++) { |
| - if (rq->bufs[i]) { |
| - kfree(rq->bufs[i]); |
| - rq->bufs[i] = NULL; |
| - } |
| - } |
| - |
| rq->ctrl = NULL; |
| } |
| |
| int vnic_rq_alloc(struct vnic_dev *vdev, struct vnic_rq *rq, unsigned int index, |
| unsigned int desc_count, unsigned int desc_size) |
| { |
| - int err; |
| + int rc; |
| char res_name[NAME_MAX]; |
| static int instance; |
| |
| @@ -121,18 +65,9 @@ int vnic_rq_alloc(struct vnic_dev *vdev, struct vnic_rq *rq, unsigned int index, |
| vnic_rq_disable(rq); |
| |
| snprintf(res_name, sizeof(res_name), "%d-rq-%d", instance++, index); |
| - err = vnic_dev_alloc_desc_ring(vdev, &rq->ring, desc_count, desc_size, |
| + rc = vnic_dev_alloc_desc_ring(vdev, &rq->ring, desc_count, desc_size, |
| rq->socket_id, res_name); |
| - if (err) |
| - return err; |
| - |
| - err = vnic_rq_alloc_bufs(rq); |
| - if (err) { |
| - vnic_rq_free(rq); |
| - return err; |
| - } |
| - |
| - return 0; |
| + return rc; |
| } |
| |
| void vnic_rq_init_start(struct vnic_rq *rq, unsigned int cq_index, |
| @@ -154,9 +89,6 @@ void vnic_rq_init_start(struct vnic_rq *rq, unsigned int cq_index, |
| iowrite32(fetch_index, &rq->ctrl->fetch_index); |
| iowrite32(posted_index, &rq->ctrl->posted_index); |
| |
| - rq->to_use = rq->to_clean = |
| - &rq->bufs[fetch_index / VNIC_RQ_BUF_BLK_ENTRIES(count)] |
| - [fetch_index % VNIC_RQ_BUF_BLK_ENTRIES(count)]; |
| } |
| |
| void vnic_rq_init(struct vnic_rq *rq, unsigned int cq_index, |
| @@ -176,6 +108,8 @@ void vnic_rq_init(struct vnic_rq *rq, unsigned int cq_index, |
| fetch_index, fetch_index, |
| error_interrupt_enable, |
| error_interrupt_offset); |
| + rq->rxst_idx = 0; |
| + rq->tot_pkts = 0; |
| } |
| |
| void vnic_rq_error_out(struct vnic_rq *rq, unsigned int error) |
| @@ -212,21 +146,20 @@ int vnic_rq_disable(struct vnic_rq *rq) |
| } |
| |
| void vnic_rq_clean(struct vnic_rq *rq, |
| - void (*buf_clean)(struct vnic_rq *rq, struct vnic_rq_buf *buf)) |
| + void (*buf_clean)(struct rte_mbuf **buf)) |
| { |
| - struct vnic_rq_buf *buf; |
| - u32 fetch_index; |
| + struct rte_mbuf **buf; |
| + u32 fetch_index, i; |
| unsigned int count = rq->ring.desc_count; |
| |
| - buf = rq->to_clean; |
| - |
| - while (vnic_rq_desc_used(rq) > 0) { |
| + buf = &rq->mbuf_ring[0]; |
| |
| - (*buf_clean)(rq, buf); |
| - |
| - buf = rq->to_clean = buf->next; |
| - rq->ring.desc_avail++; |
| + for (i = 0; i < count; i++) { |
| + (*buf_clean)(buf); |
| + buf++; |
| } |
| + rq->ring.desc_avail = count - 1; |
| + rq->rx_nb_hold = 0; |
| |
| /* Use current fetch_index as the ring starting point */ |
| fetch_index = ioread32(&rq->ctrl->fetch_index); |
| @@ -235,9 +168,7 @@ void vnic_rq_clean(struct vnic_rq *rq, |
| /* Hardware surprise removal: reset fetch_index */ |
| fetch_index = 0; |
| } |
| - rq->to_use = rq->to_clean = |
| - &rq->bufs[fetch_index / VNIC_RQ_BUF_BLK_ENTRIES(count)] |
| - [fetch_index % VNIC_RQ_BUF_BLK_ENTRIES(count)]; |
| + |
| iowrite32(fetch_index, &rq->ctrl->posted_index); |
| |
| vnic_dev_clear_desc_ring(&rq->ring); |
| diff --git a/drivers/net/enic/base/vnic_rq.h b/drivers/net/enic/base/vnic_rq.h |
| index 0f5c3c1..e083ccc 100644 |
| --- a/drivers/net/enic/base/vnic_rq.h |
| +++ b/drivers/net/enic/base/vnic_rq.h |
| @@ -66,42 +66,22 @@ struct vnic_rq_ctrl { |
| u32 pad10; |
| }; |
| |
| -/* Break the vnic_rq_buf allocations into blocks of 32/64 entries */ |
| -#define VNIC_RQ_BUF_MIN_BLK_ENTRIES 32 |
| -#define VNIC_RQ_BUF_DFLT_BLK_ENTRIES 64 |
| -#define VNIC_RQ_BUF_BLK_ENTRIES(entries) \ |
| - ((unsigned int)((entries < VNIC_RQ_BUF_DFLT_BLK_ENTRIES) ? \ |
| - VNIC_RQ_BUF_MIN_BLK_ENTRIES : VNIC_RQ_BUF_DFLT_BLK_ENTRIES)) |
| -#define VNIC_RQ_BUF_BLK_SZ(entries) \ |
| - (VNIC_RQ_BUF_BLK_ENTRIES(entries) * sizeof(struct vnic_rq_buf)) |
| -#define VNIC_RQ_BUF_BLKS_NEEDED(entries) \ |
| - DIV_ROUND_UP(entries, VNIC_RQ_BUF_BLK_ENTRIES(entries)) |
| -#define VNIC_RQ_BUF_BLKS_MAX VNIC_RQ_BUF_BLKS_NEEDED(4096) |
| - |
| -struct vnic_rq_buf { |
| - struct vnic_rq_buf *next; |
| - dma_addr_t dma_addr; |
| - void *os_buf; |
| - unsigned int os_buf_index; |
| - unsigned int len; |
| - unsigned int index; |
| - void *desc; |
| - uint64_t wr_id; |
| -}; |
| - |
| struct vnic_rq { |
| unsigned int index; |
| + unsigned int posted_index; |
| struct vnic_dev *vdev; |
| - struct vnic_rq_ctrl __iomem *ctrl; /* memory-mapped */ |
| + struct vnic_rq_ctrl __iomem *ctrl; /* memory-mapped */ |
| struct vnic_dev_ring ring; |
| - struct vnic_rq_buf *bufs[VNIC_RQ_BUF_BLKS_MAX]; |
| - struct vnic_rq_buf *to_use; |
| - struct vnic_rq_buf *to_clean; |
| + struct rte_mbuf **mbuf_ring; /* array of allocated mbufs */ |
| + unsigned int mbuf_next_idx; /* next mb to consume */ |
| void *os_buf_head; |
| unsigned int pkts_outstanding; |
| - |
| + uint16_t rx_nb_hold; |
| + uint16_t rx_free_thresh; |
| unsigned int socket_id; |
| struct rte_mempool *mp; |
| + uint16_t rxst_idx; |
| + uint32_t tot_pkts; |
| }; |
| |
| static inline unsigned int vnic_rq_desc_avail(struct vnic_rq *rq) |
| @@ -116,119 +96,13 @@ static inline unsigned int vnic_rq_desc_used(struct vnic_rq *rq) |
| return rq->ring.desc_count - rq->ring.desc_avail - 1; |
| } |
| |
| -static inline void *vnic_rq_next_desc(struct vnic_rq *rq) |
| -{ |
| - return rq->to_use->desc; |
| -} |
| - |
| -static inline unsigned int vnic_rq_next_index(struct vnic_rq *rq) |
| -{ |
| - return rq->to_use->index; |
| -} |
| - |
| -static inline void vnic_rq_post(struct vnic_rq *rq, |
| - void *os_buf, unsigned int os_buf_index, |
| - dma_addr_t dma_addr, unsigned int len, |
| - uint64_t wrid) |
| -{ |
| - struct vnic_rq_buf *buf = rq->to_use; |
| - |
| - buf->os_buf = os_buf; |
| - buf->os_buf_index = os_buf_index; |
| - buf->dma_addr = dma_addr; |
| - buf->len = len; |
| - buf->wr_id = wrid; |
| - |
| - buf = buf->next; |
| - rq->to_use = buf; |
| - rq->ring.desc_avail--; |
| - |
| - /* Move the posted_index every nth descriptor |
| - */ |
| - |
| -#ifndef VNIC_RQ_RETURN_RATE |
| -#define VNIC_RQ_RETURN_RATE 0xf /* keep 2^n - 1 */ |
| -#endif |
| - |
| - if ((buf->index & VNIC_RQ_RETURN_RATE) == 0) { |
| - /* Adding write memory barrier prevents compiler and/or CPU |
| - * reordering, thus avoiding descriptor posting before |
| - * descriptor is initialized. Otherwise, hardware can read |
| - * stale descriptor fields. |
| - */ |
| - wmb(); |
| - iowrite32(buf->index, &rq->ctrl->posted_index); |
| - } |
| -} |
| - |
| -static inline void vnic_rq_post_commit(struct vnic_rq *rq, |
| - void *os_buf, unsigned int os_buf_index, |
| - dma_addr_t dma_addr, unsigned int len) |
| -{ |
| - struct vnic_rq_buf *buf = rq->to_use; |
| - |
| - buf->os_buf = os_buf; |
| - buf->os_buf_index = os_buf_index; |
| - buf->dma_addr = dma_addr; |
| - buf->len = len; |
| - |
| - buf = buf->next; |
| - rq->to_use = buf; |
| - rq->ring.desc_avail--; |
| - |
| - /* Move the posted_index every descriptor |
| - */ |
| - |
| - /* Adding write memory barrier prevents compiler and/or CPU |
| - * reordering, thus avoiding descriptor posting before |
| - * descriptor is initialized. Otherwise, hardware can read |
| - * stale descriptor fields. |
| - */ |
| - wmb(); |
| - iowrite32(buf->index, &rq->ctrl->posted_index); |
| -} |
| |
| -static inline void vnic_rq_return_descs(struct vnic_rq *rq, unsigned int count) |
| -{ |
| - rq->ring.desc_avail += count; |
| -} |
| |
| enum desc_return_options { |
| VNIC_RQ_RETURN_DESC, |
| VNIC_RQ_DEFER_RETURN_DESC, |
| }; |
| |
| -static inline int vnic_rq_service(struct vnic_rq *rq, |
| - struct cq_desc *cq_desc, u16 completed_index, |
| - int desc_return, int (*buf_service)(struct vnic_rq *rq, |
| - struct cq_desc *cq_desc, struct vnic_rq_buf *buf, |
| - int skipped, void *opaque), void *opaque) |
| -{ |
| - struct vnic_rq_buf *buf; |
| - int skipped; |
| - int eop = 0; |
| - |
| - buf = rq->to_clean; |
| - while (1) { |
| - |
| - skipped = (buf->index != completed_index); |
| - |
| - if ((*buf_service)(rq, cq_desc, buf, skipped, opaque)) |
| - eop++; |
| - |
| - if (desc_return == VNIC_RQ_RETURN_DESC) |
| - rq->ring.desc_avail++; |
| - |
| - rq->to_clean = buf->next; |
| - |
| - if (!skipped) |
| - break; |
| - |
| - buf = rq->to_clean; |
| - } |
| - return eop; |
| -} |
| - |
| static inline int vnic_rq_fill(struct vnic_rq *rq, |
| int (*buf_fill)(struct vnic_rq *rq)) |
| { |
| @@ -274,8 +148,5 @@ unsigned int vnic_rq_error_status(struct vnic_rq *rq); |
| void vnic_rq_enable(struct vnic_rq *rq); |
| int vnic_rq_disable(struct vnic_rq *rq); |
| void vnic_rq_clean(struct vnic_rq *rq, |
| - void (*buf_clean)(struct vnic_rq *rq, struct vnic_rq_buf *buf)); |
| -int vnic_rq_mem_size(struct vnic_rq *rq, unsigned int desc_count, |
| - unsigned int desc_size); |
| - |
| + void (*buf_clean)(struct rte_mbuf **buf)); |
| #endif /* _VNIC_RQ_H_ */ |
| diff --git a/drivers/net/enic/enic.h b/drivers/net/enic/enic.h |
| index 9e78305..8c914f5 100644 |
| --- a/drivers/net/enic/enic.h |
| +++ b/drivers/net/enic/enic.h |
| @@ -45,6 +45,7 @@ |
| #include "vnic_nic.h" |
| #include "vnic_rss.h" |
| #include "enic_res.h" |
| +#include "cq_enet_desc.h" |
| |
| #define DRV_NAME "enic_pmd" |
| #define DRV_DESCRIPTION "Cisco VIC Ethernet NIC Poll-mode Driver" |
| @@ -154,6 +155,16 @@ static inline struct enic *pmd_priv(struct rte_eth_dev *eth_dev) |
| return (struct enic *)eth_dev->data->dev_private; |
| } |
| |
| +#define RTE_LIBRTE_ENIC_ASSERT_ENABLE |
| +#ifdef RTE_LIBRTE_ENIC_ASSERT_ENABLE |
| +#define ASSERT(x) do { \ |
| + if (!(x)) \ |
| + rte_panic("ENIC: x"); \ |
| +} while (0) |
| +#else |
| +#define ASSERT(x) |
| +#endif |
| + |
| extern void enic_fdir_stats_get(struct enic *enic, |
| struct rte_eth_fdir_stats *stats); |
| extern int enic_fdir_add_fltr(struct enic *enic, |
| @@ -193,9 +204,10 @@ extern void enic_send_pkt(struct enic *enic, struct vnic_wq *wq, |
| uint16_t ol_flags, uint16_t vlan_tag); |
| |
| extern void enic_post_wq_index(struct vnic_wq *wq); |
| -extern int enic_poll(struct vnic_rq *rq, struct rte_mbuf **rx_pkts, |
| - unsigned int budget, unsigned int *work_done); |
| extern int enic_probe(struct enic *enic); |
| extern int enic_clsf_init(struct enic *enic); |
| extern void enic_clsf_destroy(struct enic *enic); |
| +uint16_t enic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, |
| + uint16_t nb_pkts); |
| + |
| #endif /* _ENIC_H_ */ |
| diff --git a/drivers/net/enic/enic_ethdev.c b/drivers/net/enic/enic_ethdev.c |
| index 2a88043..6f2ada5 100644 |
| --- a/drivers/net/enic/enic_ethdev.c |
| +++ b/drivers/net/enic/enic_ethdev.c |
| @@ -255,7 +255,7 @@ static int enicpmd_dev_rx_queue_setup(struct rte_eth_dev *eth_dev, |
| uint16_t queue_idx, |
| uint16_t nb_desc, |
| unsigned int socket_id, |
| - __rte_unused const struct rte_eth_rxconf *rx_conf, |
| + const struct rte_eth_rxconf *rx_conf, |
| struct rte_mempool *mp) |
| { |
| int ret; |
| @@ -270,6 +270,10 @@ static int enicpmd_dev_rx_queue_setup(struct rte_eth_dev *eth_dev, |
| return ret; |
| } |
| |
| + enic->rq[queue_idx].rx_free_thresh = rx_conf->rx_free_thresh; |
| + dev_debug(enic, "Set queue_id:%u free thresh:%u\n", queue_idx, |
| + enic->rq[queue_idx].rx_free_thresh); |
| + |
| return enicpmd_dev_setup_intr(enic); |
| } |
| |
| @@ -429,6 +433,9 @@ static void enicpmd_dev_info_get(struct rte_eth_dev *eth_dev, |
| DEV_TX_OFFLOAD_IPV4_CKSUM | |
| DEV_TX_OFFLOAD_UDP_CKSUM | |
| DEV_TX_OFFLOAD_TCP_CKSUM; |
| + device_info->default_rxconf = (struct rte_eth_rxconf) { |
| + .rx_free_thresh = ENIC_DEFAULT_RX_FREE_THRESH |
| + }; |
| } |
| |
| static void enicpmd_dev_promiscuous_enable(struct rte_eth_dev *eth_dev) |
| @@ -538,18 +545,6 @@ static uint16_t enicpmd_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, |
| return index; |
| } |
| |
| -static uint16_t enicpmd_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, |
| - uint16_t nb_pkts) |
| -{ |
| - struct vnic_rq *rq = (struct vnic_rq *)rx_queue; |
| - unsigned int work_done; |
| - |
| - if (enic_poll(rq, rx_pkts, (unsigned int)nb_pkts, &work_done)) |
| - dev_err(enic, "error in enicpmd poll\n"); |
| - |
| - return work_done; |
| -} |
| - |
| static const struct eth_dev_ops enicpmd_eth_dev_ops = { |
| .dev_configure = enicpmd_dev_configure, |
| .dev_start = enicpmd_dev_start, |
| @@ -606,7 +601,7 @@ static int eth_enicpmd_dev_init(struct rte_eth_dev *eth_dev) |
| enic->port_id = eth_dev->data->port_id; |
| enic->rte_dev = eth_dev; |
| eth_dev->dev_ops = &enicpmd_eth_dev_ops; |
| - eth_dev->rx_pkt_burst = &enicpmd_recv_pkts; |
| + eth_dev->rx_pkt_burst = &enic_recv_pkts; |
| eth_dev->tx_pkt_burst = &enicpmd_xmit_pkts; |
| |
| pdev = eth_dev->pci_dev; |
| @@ -635,8 +630,8 @@ static struct eth_driver rte_enic_pmd = { |
| * Register as the [Poll Mode] Driver of Cisco ENIC device. |
| */ |
| static int |
| -rte_enic_pmd_init(const char *name __rte_unused, |
| - const char *params __rte_unused) |
| +rte_enic_pmd_init(__rte_unused const char *name, |
| + __rte_unused const char *params) |
| { |
| ENICPMD_FUNC_TRACE(); |
| |
| diff --git a/drivers/net/enic/enic_main.c b/drivers/net/enic/enic_main.c |
| index 07a9810..d0c9bff 100644 |
| --- a/drivers/net/enic/enic_main.c |
| +++ b/drivers/net/enic/enic_main.c |
| @@ -60,6 +60,17 @@ |
| #include "vnic_nic.h" |
| #include "enic_vnic_wq.h" |
| |
| +static inline struct rte_mbuf * |
| +rte_rxmbuf_alloc(struct rte_mempool *mp) |
| +{ |
| + struct rte_mbuf *m; |
| + |
| + m = __rte_mbuf_raw_alloc(mp); |
| + __rte_mbuf_sanity_check_raw(m, 0); |
| + return m; |
| +} |
| + |
| + |
| static inline int enic_is_sriov_vf(struct enic *enic) |
| { |
| return enic->pdev->id.device_id == PCI_DEVICE_ID_CISCO_VIC_ENET_VF; |
| @@ -80,16 +91,25 @@ static int is_eth_addr_valid(uint8_t *addr) |
| return !is_mcast_addr(addr) && !is_zero_addr(addr); |
| } |
| |
| -static inline struct rte_mbuf * |
| -enic_rxmbuf_alloc(struct rte_mempool *mp) |
| +static void |
| +enic_rxmbuf_queue_release(struct enic *enic, struct vnic_rq *rq) |
| { |
| - struct rte_mbuf *m; |
| + uint16_t i; |
| |
| - m = __rte_mbuf_raw_alloc(mp); |
| - __rte_mbuf_sanity_check_raw(m, 0); |
| - return m; |
| + if (!rq || !rq->mbuf_ring) { |
| + dev_debug(enic, "Pointer to rq or mbuf_ring is NULL"); |
| + return; |
| + } |
| + |
| + for (i = 0; i < enic->config.rq_desc_count; i++) { |
| + if (rq->mbuf_ring[i]) { |
| + rte_pktmbuf_free_seg(rq->mbuf_ring[i]); |
| + rq->mbuf_ring[i] = NULL; |
| + } |
| + } |
| } |
| |
| + |
| void enic_set_hdr_split_size(struct enic *enic, u16 split_hdr_size) |
| { |
| vnic_set_hdr_split_size(enic->vdev, split_hdr_size); |
| @@ -262,13 +282,13 @@ void enic_set_mac_address(struct enic *enic, uint8_t *mac_addr) |
| } |
| |
| static void |
| -enic_free_rq_buf(__rte_unused struct vnic_rq *rq, struct vnic_rq_buf *buf) |
| +enic_free_rq_buf(struct rte_mbuf **mbuf) |
| { |
| - if (!buf->os_buf) |
| + if (*mbuf == NULL) |
| return; |
| |
| - rte_pktmbuf_free((struct rte_mbuf *)buf->os_buf); |
| - buf->os_buf = NULL; |
| + rte_pktmbuf_free(*mbuf); |
| + mbuf = NULL; |
| } |
| |
| void enic_init_vnic_resources(struct enic *enic) |
| @@ -314,221 +334,47 @@ void enic_init_vnic_resources(struct enic *enic) |
| } |
| |
| |
| -static int enic_rq_alloc_buf(struct vnic_rq *rq) |
| +static int |
| +enic_alloc_rx_queue_mbufs(struct enic *enic, struct vnic_rq *rq) |
| { |
| - struct enic *enic = vnic_dev_priv(rq->vdev); |
| + struct rte_mbuf *mb; |
| + struct rq_enet_desc *rqd = rq->ring.descs; |
| + unsigned i; |
| dma_addr_t dma_addr; |
| - struct rq_enet_desc *desc = vnic_rq_next_desc(rq); |
| - uint8_t type = RQ_ENET_TYPE_ONLY_SOP; |
| - u16 split_hdr_size = vnic_get_hdr_split_size(enic->vdev); |
| - struct rte_mbuf *mbuf = enic_rxmbuf_alloc(rq->mp); |
| - struct rte_mbuf *hdr_mbuf = NULL; |
| - |
| - if (!mbuf) { |
| - dev_err(enic, "mbuf alloc in enic_rq_alloc_buf failed\n"); |
| - return -1; |
| - } |
| - |
| - if (unlikely(split_hdr_size)) { |
| - if (vnic_rq_desc_avail(rq) < 2) { |
| - rte_mempool_put(mbuf->pool, mbuf); |
| - return -1; |
| - } |
| - hdr_mbuf = enic_rxmbuf_alloc(rq->mp); |
| - if (!hdr_mbuf) { |
| - rte_mempool_put(mbuf->pool, mbuf); |
| - dev_err(enic, |
| - "hdr_mbuf alloc in enic_rq_alloc_buf failed\n"); |
| - return -1; |
| - } |
| - |
| - hdr_mbuf->data_off = RTE_PKTMBUF_HEADROOM; |
| - |
| - hdr_mbuf->nb_segs = 2; |
| - hdr_mbuf->port = enic->port_id; |
| - hdr_mbuf->next = mbuf; |
| - |
| - dma_addr = (dma_addr_t) |
| - (hdr_mbuf->buf_physaddr + hdr_mbuf->data_off); |
| - |
| - rq_enet_desc_enc(desc, dma_addr, type, split_hdr_size); |
| |
| - vnic_rq_post(rq, (void *)hdr_mbuf, 0 /*os_buf_index*/, dma_addr, |
| - (unsigned int)split_hdr_size, 0 /*wrid*/); |
| + dev_debug(enic, "queue %u, allocating %u rx queue mbufs", rq->index, |
| + rq->ring.desc_count); |
| |
| - desc = vnic_rq_next_desc(rq); |
| - type = RQ_ENET_TYPE_NOT_SOP; |
| - } else { |
| - mbuf->nb_segs = 1; |
| - mbuf->port = enic->port_id; |
| - } |
| - |
| - mbuf->data_off = RTE_PKTMBUF_HEADROOM; |
| - mbuf->next = NULL; |
| - |
| - dma_addr = (dma_addr_t) |
| - (mbuf->buf_physaddr + mbuf->data_off); |
| - |
| - rq_enet_desc_enc(desc, dma_addr, type, mbuf->buf_len); |
| - |
| - vnic_rq_post(rq, (void *)mbuf, 0 /*os_buf_index*/, dma_addr, |
| - (unsigned int)mbuf->buf_len, 0 /*wrid*/); |
| - |
| - return 0; |
| -} |
| - |
| -static int enic_rq_indicate_buf(struct vnic_rq *rq, |
| - struct cq_desc *cq_desc, struct vnic_rq_buf *buf, |
| - int skipped, void *opaque) |
| -{ |
| - struct enic *enic = vnic_dev_priv(rq->vdev); |
| - struct rte_mbuf **rx_pkt_bucket = (struct rte_mbuf **)opaque; |
| - struct rte_mbuf *rx_pkt = NULL; |
| - struct rte_mbuf *hdr_rx_pkt = NULL; |
| - |
| - u8 type, color, eop, sop, ingress_port, vlan_stripped; |
| - u8 fcoe, fcoe_sof, fcoe_fc_crc_ok, fcoe_enc_error, fcoe_eof; |
| - u8 tcp_udp_csum_ok, udp, tcp, ipv4_csum_ok; |
| - u8 ipv6, ipv4, ipv4_fragment, fcs_ok, rss_type, csum_not_calc; |
| - u8 packet_error; |
| - u16 q_number, completed_index, bytes_written, vlan_tci, checksum; |
| - u32 rss_hash; |
| - |
| - cq_enet_rq_desc_dec((struct cq_enet_rq_desc *)cq_desc, |
| - &type, &color, &q_number, &completed_index, |
| - &ingress_port, &fcoe, &eop, &sop, &rss_type, |
| - &csum_not_calc, &rss_hash, &bytes_written, |
| - &packet_error, &vlan_stripped, &vlan_tci, &checksum, |
| - &fcoe_sof, &fcoe_fc_crc_ok, &fcoe_enc_error, |
| - &fcoe_eof, &tcp_udp_csum_ok, &udp, &tcp, |
| - &ipv4_csum_ok, &ipv6, &ipv4, &ipv4_fragment, |
| - &fcs_ok); |
| - |
| - rx_pkt = (struct rte_mbuf *)buf->os_buf; |
| - buf->os_buf = NULL; |
| - |
| - if (unlikely(packet_error)) { |
| - dev_err(enic, "packet error\n"); |
| - rx_pkt->data_len = 0; |
| - return 0; |
| - } |
| - |
| - if (unlikely(skipped)) { |
| - rx_pkt->data_len = 0; |
| - return 0; |
| - } |
| - |
| - if (likely(!vnic_get_hdr_split_size(enic->vdev))) { |
| - /* No header split configured */ |
| - *rx_pkt_bucket = rx_pkt; |
| - rx_pkt->pkt_len = bytes_written; |
| - |
| - if (ipv4) { |
| - rx_pkt->packet_type = RTE_PTYPE_L3_IPV4; |
| - if (!csum_not_calc) { |
| - if (unlikely(!ipv4_csum_ok)) |
| - rx_pkt->ol_flags |= PKT_RX_IP_CKSUM_BAD; |
| - |
| - if ((tcp || udp) && (!tcp_udp_csum_ok)) |
| - rx_pkt->ol_flags |= PKT_RX_L4_CKSUM_BAD; |
| - } |
| - } else if (ipv6) |
| - rx_pkt->packet_type = RTE_PTYPE_L3_IPV6; |
| - } else { |
| - /* Header split */ |
| - if (sop && !eop) { |
| - /* This piece is header */ |
| - *rx_pkt_bucket = rx_pkt; |
| - rx_pkt->pkt_len = bytes_written; |
| - } else { |
| - if (sop && eop) { |
| - /* The packet is smaller than split_hdr_size */ |
| - *rx_pkt_bucket = rx_pkt; |
| - rx_pkt->pkt_len = bytes_written; |
| - if (ipv4) { |
| - rx_pkt->packet_type = RTE_PTYPE_L3_IPV4; |
| - if (!csum_not_calc) { |
| - if (unlikely(!ipv4_csum_ok)) |
| - rx_pkt->ol_flags |= |
| - PKT_RX_IP_CKSUM_BAD; |
| - |
| - if ((tcp || udp) && |
| - (!tcp_udp_csum_ok)) |
| - rx_pkt->ol_flags |= |
| - PKT_RX_L4_CKSUM_BAD; |
| - } |
| - } else if (ipv6) |
| - rx_pkt->packet_type = RTE_PTYPE_L3_IPV6; |
| - } else { |
| - /* Payload */ |
| - hdr_rx_pkt = *rx_pkt_bucket; |
| - hdr_rx_pkt->pkt_len += bytes_written; |
| - if (ipv4) { |
| - hdr_rx_pkt->packet_type = |
| - RTE_PTYPE_L3_IPV4; |
| - if (!csum_not_calc) { |
| - if (unlikely(!ipv4_csum_ok)) |
| - hdr_rx_pkt->ol_flags |= |
| - PKT_RX_IP_CKSUM_BAD; |
| - |
| - if ((tcp || udp) && |
| - (!tcp_udp_csum_ok)) |
| - hdr_rx_pkt->ol_flags |= |
| - PKT_RX_L4_CKSUM_BAD; |
| - } |
| - } else if (ipv6) |
| - hdr_rx_pkt->packet_type = |
| - RTE_PTYPE_L3_IPV6; |
| - } |
| + for (i = 0; i < rq->ring.desc_count; i++, rqd++) { |
| + mb = rte_rxmbuf_alloc(rq->mp); |
| + if (mb == NULL) { |
| + dev_err(enic, "RX mbuf alloc failed queue_id=%u", |
| + (unsigned)rq->index); |
| + return -ENOMEM; |
| } |
| - } |
| |
| - rx_pkt->data_len = bytes_written; |
| + dma_addr = (dma_addr_t)(mb->buf_physaddr + mb->data_off); |
| |
| - if (rss_hash) { |
| - rx_pkt->ol_flags |= PKT_RX_RSS_HASH; |
| - rx_pkt->hash.rss = rss_hash; |
| + rq_enet_desc_enc(rqd, dma_addr, RQ_ENET_TYPE_ONLY_SOP, |
| + mb->buf_len); |
| + rq->mbuf_ring[i] = mb; |
| } |
| |
| - if (vlan_tci) { |
| - rx_pkt->ol_flags |= PKT_RX_VLAN_PKT; |
| - rx_pkt->vlan_tci = vlan_tci; |
| - } |
| + /* make sure all prior writes are complete before doing the PIO write */ |
| + rte_rmb(); |
| |
| - return eop; |
| -} |
| + /* Post all but the last 2 cache lines' worth of descriptors */ |
| + rq->posted_index = rq->ring.desc_count - (2 * RTE_CACHE_LINE_SIZE |
| + / sizeof(struct rq_enet_desc)); |
| + rq->rx_nb_hold = 0; |
| |
| -static int enic_rq_service(struct vnic_dev *vdev, struct cq_desc *cq_desc, |
| - __rte_unused u8 type, u16 q_number, u16 completed_index, void *opaque) |
| -{ |
| - struct enic *enic = vnic_dev_priv(vdev); |
| - |
| - return vnic_rq_service(&enic->rq[q_number], cq_desc, |
| - completed_index, VNIC_RQ_RETURN_DESC, |
| - enic_rq_indicate_buf, opaque); |
| - |
| -} |
| + dev_debug(enic, "port=%u, qidx=%u, Write %u posted idx, %u sw held\n", |
| + enic->port_id, rq->index, rq->posted_index, rq->rx_nb_hold); |
| + iowrite32(rq->posted_index, &rq->ctrl->posted_index); |
| + rte_rmb(); |
| |
| -int enic_poll(struct vnic_rq *rq, struct rte_mbuf **rx_pkts, |
| - unsigned int budget, unsigned int *work_done) |
| -{ |
| - struct enic *enic = vnic_dev_priv(rq->vdev); |
| - unsigned int cq = enic_cq_rq(enic, rq->index); |
| - int err = 0; |
| - |
| - *work_done = vnic_cq_service(&enic->cq[cq], |
| - budget, enic_rq_service, (void *)rx_pkts); |
| - |
| - if (*work_done) { |
| - vnic_rq_fill(rq, enic_rq_alloc_buf); |
| + return 0; |
| |
| - /* Need at least one buffer on ring to get going */ |
| - if (vnic_rq_desc_used(rq) == 0) { |
| - dev_err(enic, "Unable to alloc receive buffers\n"); |
| - err = -1; |
| - } |
| - } |
| - return err; |
| } |
| |
| static void * |
| @@ -576,6 +422,7 @@ enic_intr_handler(__rte_unused struct rte_intr_handle *handle, |
| int enic_enable(struct enic *enic) |
| { |
| unsigned int index; |
| + int err; |
| struct rte_eth_dev *eth_dev = enic->rte_dev; |
| |
| eth_dev->data->dev_link.link_speed = vnic_dev_port_speed(enic->vdev); |
| @@ -586,15 +433,11 @@ int enic_enable(struct enic *enic) |
| dev_warning(enic, "Init of hash table for clsf failed."\ |
| "Flow director feature will not work\n"); |
| |
| - /* Fill RQ bufs */ |
| for (index = 0; index < enic->rq_count; index++) { |
| - vnic_rq_fill(&enic->rq[index], enic_rq_alloc_buf); |
| - |
| - /* Need at least one buffer on ring to get going |
| - */ |
| - if (vnic_rq_desc_used(&enic->rq[index]) == 0) { |
| - dev_err(enic, "Unable to alloc receive buffers\n"); |
| - return -1; |
| + err = enic_alloc_rx_queue_mbufs(enic, &enic->rq[index]); |
| + if (err) { |
| + dev_err(enic, "Failed to alloc RX queue mbufs\n"); |
| + return err; |
| } |
| } |
| |
| @@ -636,6 +479,9 @@ void enic_free_rq(void *rxq) |
| struct vnic_rq *rq = (struct vnic_rq *)rxq; |
| struct enic *enic = vnic_dev_priv(rq->vdev); |
| |
| + enic_rxmbuf_queue_release(enic, rq); |
| + rte_free(rq->mbuf_ring); |
| + rq->mbuf_ring = NULL; |
| vnic_rq_free(rq); |
| vnic_cq_free(&enic->cq[rq->index]); |
| } |
| @@ -664,7 +510,7 @@ int enic_alloc_rq(struct enic *enic, uint16_t queue_idx, |
| unsigned int socket_id, struct rte_mempool *mp, |
| uint16_t nb_desc) |
| { |
| - int err; |
| + int rc; |
| struct vnic_rq *rq = &enic->rq[queue_idx]; |
| |
| rq->socket_id = socket_id; |
| @@ -687,23 +533,35 @@ int enic_alloc_rq(struct enic *enic, uint16_t queue_idx, |
| } |
| |
| /* Allocate queue resources */ |
| - err = vnic_rq_alloc(enic->vdev, &enic->rq[queue_idx], queue_idx, |
| - enic->config.rq_desc_count, |
| - sizeof(struct rq_enet_desc)); |
| - if (err) { |
| + rc = vnic_rq_alloc(enic->vdev, rq, queue_idx, |
| + enic->config.rq_desc_count, sizeof(struct rq_enet_desc)); |
| + if (rc) { |
| dev_err(enic, "error in allocation of rq\n"); |
| - return err; |
| + goto err_exit; |
| } |
| |
| - err = vnic_cq_alloc(enic->vdev, &enic->cq[queue_idx], queue_idx, |
| + rc = vnic_cq_alloc(enic->vdev, &enic->cq[queue_idx], queue_idx, |
| socket_id, enic->config.rq_desc_count, |
| sizeof(struct cq_enet_rq_desc)); |
| - if (err) { |
| - vnic_rq_free(rq); |
| + if (rc) { |
| dev_err(enic, "error in allocation of cq for rq\n"); |
| + goto err_free_rq_exit; |
| } |
| |
| - return err; |
| + /* Allocate the mbuf ring */ |
| + rq->mbuf_ring = (struct rte_mbuf **)rte_zmalloc_socket("rq->mbuf_ring", |
| + sizeof(struct rte_mbuf *) * enic->config.rq_desc_count, |
| + RTE_CACHE_LINE_SIZE, rq->socket_id); |
| + |
| + if (rq->mbuf_ring != NULL) |
| + return 0; |
| + |
| + /* cleanup on error */ |
| + vnic_cq_free(&enic->cq[queue_idx]); |
| +err_free_rq_exit: |
| + vnic_rq_free(rq); |
| +err_exit: |
| + return -ENOMEM; |
| } |
| |
| void enic_free_wq(void *txq) |
| @@ -790,6 +648,7 @@ int enic_disable(struct enic *enic) |
| |
| for (i = 0; i < enic->wq_count; i++) |
| vnic_wq_clean(&enic->wq[i], enic_free_wq_buf); |
| + |
| for (i = 0; i < enic->rq_count; i++) |
| vnic_rq_clean(&enic->rq[i], enic_free_rq_buf); |
| for (i = 0; i < enic->cq_count; i++) |
| @@ -1074,7 +933,7 @@ int enic_probe(struct enic *enic) |
| |
| /* Set ingress vlan rewrite mode before vnic initialization */ |
| err = vnic_dev_set_ig_vlan_rewrite_mode(enic->vdev, |
| - IG_VLAN_REWRITE_MODE_PRIORITY_TAG_DEFAULT_VLAN); |
| + IG_VLAN_REWRITE_MODE_PASS_THRU); |
| if (err) { |
| dev_err(enic, |
| "Failed to set ingress vlan rewrite mode, aborting.\n"); |
| diff --git a/drivers/net/enic/enic_res.h b/drivers/net/enic/enic_res.h |
| index 49f7e22..33f2e84 100644 |
| --- a/drivers/net/enic/enic_res.h |
| +++ b/drivers/net/enic/enic_res.h |
| @@ -52,6 +52,7 @@ |
| #define ENIC_UNICAST_PERFECT_FILTERS 32 |
| |
| #define ENIC_NON_TSO_MAX_DESC 16 |
| +#define ENIC_DEFAULT_RX_FREE_THRESH 32 |
| |
| #define ENIC_SETTING(enic, f) ((enic->config.flags & VENETF_##f) ? 1 : 0) |
| |
| @@ -133,21 +134,6 @@ static inline void enic_queue_wq_desc_tso(struct vnic_wq *wq, |
| WQ_ENET_OFFLOAD_MODE_TSO, |
| eop, 1 /* SOP */, eop, loopback); |
| } |
| -static inline void enic_queue_rq_desc(struct vnic_rq *rq, |
| - void *os_buf, unsigned int os_buf_index, |
| - dma_addr_t dma_addr, unsigned int len) |
| -{ |
| - struct rq_enet_desc *desc = vnic_rq_next_desc(rq); |
| - u64 wrid = 0; |
| - u8 type = os_buf_index ? |
| - RQ_ENET_TYPE_NOT_SOP : RQ_ENET_TYPE_ONLY_SOP; |
| - |
| - rq_enet_desc_enc(desc, |
| - (u64)dma_addr | VNIC_PADDR_TARGET, |
| - type, (u16)len); |
| - |
| - vnic_rq_post(rq, os_buf, os_buf_index, dma_addr, len, wrid); |
| -} |
| |
| struct enic; |
| |
| diff --git a/drivers/net/enic/enic_rx.c b/drivers/net/enic/enic_rx.c |
| new file mode 100644 |
| index 0000000..12b1f62 |
| --- /dev/null |
| +++ b/drivers/net/enic/enic_rx.c |
| @@ -0,0 +1,378 @@ |
| +/* |
| + * Copyright 2008-2014 Cisco Systems, Inc. All rights reserved. |
| + * Copyright 2007 Nuova Systems, Inc. All rights reserved. |
| + * |
| + * Copyright (c) 2014, Cisco Systems, Inc. |
| + * All rights reserved. |
| + * |
| + * Redistribution and use in source and binary forms, with or without |
| + * modification, are permitted provided that the following conditions |
| + * are met: |
| + * |
| + * 1. Redistributions of source code must retain the above copyright |
| + * notice, this list of conditions and the following disclaimer. |
| + * |
| + * 2. Redistributions in binary form must reproduce the above copyright |
| + * notice, this list of conditions and the following disclaimer in |
| + * the documentation and/or other materials provided with the |
| + * distribution. |
| + * |
| + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS |
| + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE |
| + * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, |
| + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, |
| + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; |
| + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER |
| + * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT |
| + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN |
| + * ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE |
| + * POSSIBILITY OF SUCH DAMAGE. |
| + * |
| + */ |
| + |
| +#include <rte_mbuf.h> |
| +#include <rte_ethdev.h> |
| +#include <rte_prefetch.h> |
| + |
| +#include "enic_compat.h" |
| +#include "rq_enet_desc.h" |
| +#include "enic.h" |
| + |
| +#define RTE_PMD_USE_PREFETCH |
| + |
| +#ifdef RTE_PMD_USE_PREFETCH |
| +/* |
| + * Prefetch a cache line into all cache levels. |
| + */ |
| +#define rte_enic_prefetch(p) rte_prefetch0(p) |
| +#else |
| +#define rte_enic_prefetch(p) do {} while (0) |
| +#endif |
| + |
| +#ifdef RTE_PMD_PACKET_PREFETCH |
| +#define rte_packet_prefetch(p) rte_prefetch1(p) |
| +#else |
| +#define rte_packet_prefetch(p) do {} while (0) |
| +#endif |
| + |
| +static inline struct rte_mbuf * |
| +rte_rxmbuf_alloc(struct rte_mempool *mp) |
| +{ |
| + struct rte_mbuf *m; |
| + |
| + m = __rte_mbuf_raw_alloc(mp); |
| + __rte_mbuf_sanity_check_raw(m, 0); |
| + return m; |
| +} |
| + |
| +static inline uint16_t |
| +enic_cq_rx_desc_ciflags(struct cq_enet_rq_desc *crd) |
| +{ |
| + return le16_to_cpu(crd->completed_index_flags) & ~CQ_DESC_COMP_NDX_MASK; |
| +} |
| + |
| +static inline uint16_t |
| +enic_cq_rx_desc_bwflags(struct cq_enet_rq_desc *crd) |
| +{ |
| + return(le16_to_cpu(crd->bytes_written_flags) & |
| + ~CQ_ENET_RQ_DESC_BYTES_WRITTEN_MASK); |
| +} |
| + |
| +static inline uint8_t |
| +enic_cq_rx_desc_packet_error(uint16_t bwflags) |
| +{ |
| + return((bwflags & CQ_ENET_RQ_DESC_FLAGS_TRUNCATED) == |
| + CQ_ENET_RQ_DESC_FLAGS_TRUNCATED); |
| +} |
| + |
| +static inline uint8_t |
| +enic_cq_rx_desc_eop(uint16_t ciflags) |
| +{ |
| + return (ciflags & CQ_ENET_RQ_DESC_FLAGS_EOP) |
| + == CQ_ENET_RQ_DESC_FLAGS_EOP; |
| +} |
| + |
| +static inline uint8_t |
| +enic_cq_rx_desc_csum_not_calc(struct cq_enet_rq_desc *cqrd) |
| +{ |
| + return ((le16_to_cpu(cqrd->q_number_rss_type_flags) & |
| + CQ_ENET_RQ_DESC_FLAGS_CSUM_NOT_CALC) == |
| + CQ_ENET_RQ_DESC_FLAGS_CSUM_NOT_CALC); |
| +} |
| + |
| +static inline uint8_t |
| +enic_cq_rx_desc_ipv4_csum_ok(struct cq_enet_rq_desc *cqrd) |
| +{ |
| + return ((cqrd->flags & CQ_ENET_RQ_DESC_FLAGS_IPV4_CSUM_OK) == |
| + CQ_ENET_RQ_DESC_FLAGS_IPV4_CSUM_OK); |
| +} |
| + |
| +static inline uint8_t |
| +enic_cq_rx_desc_tcp_udp_csum_ok(struct cq_enet_rq_desc *cqrd) |
| +{ |
| + return((cqrd->flags & CQ_ENET_RQ_DESC_FLAGS_TCP_UDP_CSUM_OK) == |
| + CQ_ENET_RQ_DESC_FLAGS_TCP_UDP_CSUM_OK); |
| +} |
| + |
| +static inline uint8_t |
| +enic_cq_rx_desc_rss_type(struct cq_enet_rq_desc *cqrd) |
| +{ |
| + return (uint8_t)((le16_to_cpu(cqrd->q_number_rss_type_flags) >> |
| + CQ_DESC_Q_NUM_BITS) & CQ_ENET_RQ_DESC_RSS_TYPE_MASK); |
| +} |
| + |
| +static inline uint32_t |
| +enic_cq_rx_desc_rss_hash(struct cq_enet_rq_desc *cqrd) |
| +{ |
| + return le32_to_cpu(cqrd->rss_hash); |
| +} |
| + |
| +static inline uint8_t |
| +enic_cq_rx_desc_fcs_ok(struct cq_enet_rq_desc *cqrd) |
| +{ |
| + return ((cqrd->flags & CQ_ENET_RQ_DESC_FLAGS_FCS_OK) == |
| + CQ_ENET_RQ_DESC_FLAGS_FCS_OK); |
| +} |
| + |
| +static inline uint16_t |
| +enic_cq_rx_desc_vlan(struct cq_enet_rq_desc *cqrd) |
| +{ |
| + return le16_to_cpu(cqrd->vlan); |
| +} |
| + |
| +static inline uint16_t |
| +enic_cq_rx_desc_n_bytes(struct cq_desc *cqd) |
| +{ |
| + struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd; |
| + return le16_to_cpu(cqrd->bytes_written_flags) & |
| + CQ_ENET_RQ_DESC_BYTES_WRITTEN_MASK; |
| +} |
| + |
| +static inline uint64_t |
| +enic_cq_rx_to_pkt_err_flags(struct cq_desc *cqd) |
| +{ |
| + struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd; |
| + uint16_t bwflags; |
| + uint64_t pkt_err_flags = 0; |
| + |
| + bwflags = enic_cq_rx_desc_bwflags(cqrd); |
| + |
| + /* Check for packet error. Can't be more specific than MAC error */ |
| + if (enic_cq_rx_desc_packet_error(bwflags)) { |
| +#ifdef RTE_NEXT_ABI |
| + pkt_err_flags |= PKT_RX_MAC_ERR; |
| +#else |
| + pkt_err_flags |= PKT_EXT_RX_PKT_ERROR; |
| +#endif |
| + } |
| + |
| + /* Check for bad FCS. MAC error isn't quite, but no other choice */ |
| + if (!enic_cq_rx_desc_fcs_ok(cqrd)) { |
| +#ifdef RTE_NEXT_ABI |
| + pkt_err_flags |= PKT_RX_MAC_ERR; |
| +#else |
| + pkt_err_flags |= PKT_EXT_RX_BAD_FCS; |
| +#endif |
| + } |
| + return pkt_err_flags; |
| +} |
| + |
| +/* |
| + * Lookup table to translate RX CQ flags to mbuf flags. |
| + */ |
| +static inline uint32_t |
| +enic_cq_rx_flags_to_pkt_type(struct cq_desc *cqd) |
| +{ |
| + struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd; |
| + uint8_t cqrd_flags = cqrd->flags; |
| + static const uint32_t cq_type_table[128] __rte_cache_aligned = { |
| + [32] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4, |
| + [34] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4 |
| + | RTE_PTYPE_L4_UDP, |
| + [36] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4 |
| + | RTE_PTYPE_L4_TCP, |
| + [96] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV4 |
| + | RTE_PTYPE_L4_FRAG, |
| + [16] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6, |
| + [18] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6 |
| + | RTE_PTYPE_L4_UDP, |
| + [20] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6 |
| + | RTE_PTYPE_L4_TCP, |
| + [80] = RTE_PTYPE_L2_ETHER | RTE_PTYPE_L3_IPV6 |
| + | RTE_PTYPE_L4_FRAG, |
| + /* All others reserved */ |
| + }; |
| + cqrd_flags &= CQ_ENET_RQ_DESC_FLAGS_IPV4_FRAGMENT |
| + | CQ_ENET_RQ_DESC_FLAGS_IPV4 | CQ_ENET_RQ_DESC_FLAGS_IPV6 |
| + | CQ_ENET_RQ_DESC_FLAGS_TCP | CQ_ENET_RQ_DESC_FLAGS_UDP; |
| + return cq_type_table[cqrd_flags]; |
| +} |
| + |
| +static inline void |
| +enic_cq_rx_to_pkt_flags(struct cq_desc *cqd, struct rte_mbuf *mbuf) |
| +{ |
| + struct cq_enet_rq_desc *cqrd = (struct cq_enet_rq_desc *)cqd; |
| + uint16_t ciflags, bwflags, pkt_flags = 0; |
| + ciflags = enic_cq_rx_desc_ciflags(cqrd); |
| + bwflags = enic_cq_rx_desc_bwflags(cqrd); |
| + |
| + ASSERT(mbuf->ol_flags == 0); |
| + |
| + /* flags are meaningless if !EOP */ |
| + if (unlikely(!enic_cq_rx_desc_eop(ciflags))) |
| + goto mbuf_flags_done; |
| + |
| + /* VLAN stripping */ |
| + if (bwflags & CQ_ENET_RQ_DESC_FLAGS_VLAN_STRIPPED) { |
| + pkt_flags |= PKT_RX_VLAN_PKT; |
| + mbuf->vlan_tci = enic_cq_rx_desc_vlan(cqrd); |
| + } else { |
| + mbuf->vlan_tci = 0; |
| + } |
| + |
| + /* RSS flag */ |
| + if (enic_cq_rx_desc_rss_type(cqrd)) { |
| + pkt_flags |= PKT_RX_RSS_HASH; |
| + mbuf->hash.rss = enic_cq_rx_desc_rss_hash(cqrd); |
| + } |
| + |
| + /* checksum flags */ |
| + if (!enic_cq_rx_desc_csum_not_calc(cqrd) && |
| + (mbuf->packet_type & RTE_PTYPE_L3_IPV4)) { |
| + if (unlikely(!enic_cq_rx_desc_ipv4_csum_ok(cqrd))) |
| + pkt_flags |= PKT_RX_IP_CKSUM_BAD; |
| + if (mbuf->packet_type & (RTE_PTYPE_L4_UDP | RTE_PTYPE_L4_TCP)) { |
| + if (unlikely(!enic_cq_rx_desc_tcp_udp_csum_ok(cqrd))) |
| + pkt_flags |= PKT_RX_L4_CKSUM_BAD; |
| + } |
| + } |
| + |
| + mbuf_flags_done: |
| + mbuf->ol_flags = pkt_flags; |
| +} |
| + |
| +static inline uint32_t |
| +enic_ring_add(uint32_t n_descriptors, uint32_t i0, uint32_t i1) |
| +{ |
| + uint32_t d = i0 + i1; |
| + ASSERT(i0 < n_descriptors); |
| + ASSERT(i1 < n_descriptors); |
| + d -= (d >= n_descriptors) ? n_descriptors : 0; |
| + return d; |
| +} |
| + |
| + |
| +uint16_t |
| +enic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, |
| + uint16_t nb_pkts) |
| +{ |
| + struct vnic_rq *rq = rx_queue; |
| + struct enic *enic = vnic_dev_priv(rq->vdev); |
| + unsigned int rx_id; |
| + struct rte_mbuf *nmb, *rxmb; |
| + uint16_t nb_rx = 0; |
| + uint16_t nb_hold; |
| + struct vnic_cq *cq; |
| + volatile struct cq_desc *cqd_ptr; |
| + uint8_t color; |
| + |
| + cq = &enic->cq[enic_cq_rq(enic, rq->index)]; |
| + rx_id = cq->to_clean; /* index of cqd, rqd, mbuf_table */ |
| + cqd_ptr = (struct cq_desc *)(cq->ring.descs) + rx_id; |
| + |
| + nb_hold = rq->rx_nb_hold; /* mbufs held by software */ |
| + |
| + while (nb_rx < nb_pkts) { |
| + uint16_t rx_pkt_len; |
| + volatile struct rq_enet_desc *rqd_ptr; |
| + dma_addr_t dma_addr; |
| + struct cq_desc cqd; |
| + uint64_t ol_err_flags; |
| + |
| + /* Check for pkts available */ |
| + color = (cqd_ptr->type_color >> CQ_DESC_COLOR_SHIFT) |
| + & CQ_DESC_COLOR_MASK; |
| + if (color == cq->last_color) |
| + break; |
| + |
| + /* Get the cq descriptor and rq pointer */ |
| + cqd = *cqd_ptr; |
| + rqd_ptr = (struct rq_enet_desc *)(rq->ring.descs) + rx_id; |
| + |
| + /* allocate a new mbuf */ |
| + nmb = rte_rxmbuf_alloc(rq->mp); |
| + if (nmb == NULL) { |
| + dev_err(enic, "RX mbuf alloc failed port=%u qid=%u", |
| + enic->port_id, (unsigned)rq->index); |
| + rte_eth_devices[enic->port_id]. |
| + data->rx_mbuf_alloc_failed++; |
| + break; |
| + } |
| + |
| + /* Check for FCS or packet errors */ |
| + ol_err_flags = enic_cq_rx_to_pkt_err_flags(&cqd); |
| + if (ol_err_flags == 0) |
| + rx_pkt_len = enic_cq_rx_desc_n_bytes(&cqd); |
| + else |
| + rx_pkt_len = 0; |
| + |
| + /* Get the mbuf to return and replace with one just allocated */ |
| + rxmb = rq->mbuf_ring[rx_id]; |
| + rq->mbuf_ring[rx_id] = nmb; |
| + |
| + /* Increment cqd, rqd, mbuf_table index */ |
| + rx_id++; |
| + if (unlikely(rx_id == rq->ring.desc_count)) { |
| + rx_id = 0; |
| + cq->last_color = cq->last_color ? 0 : 1; |
| + } |
| + |
| + /* Prefetch next mbuf & desc while processing current one */ |
| + cqd_ptr = (struct cq_desc *)(cq->ring.descs) + rx_id; |
| + rte_enic_prefetch(cqd_ptr); |
| + rte_enic_prefetch(rq->mbuf_ring[rx_id]); |
| + rte_enic_prefetch((struct rq_enet_desc *)(rq->ring.descs) |
| + + rx_id); |
| + |
| + /* Push descriptor for newly allocated mbuf */ |
| + dma_addr = (dma_addr_t)(nmb->buf_physaddr + nmb->data_off); |
| + rqd_ptr->address = rte_cpu_to_le_64(dma_addr); |
| + rqd_ptr->length_type = cpu_to_le16(nmb->buf_len); |
| + |
| + /* Fill in the rest of the mbuf */ |
| + rxmb->data_off = RTE_PKTMBUF_HEADROOM; |
| + rxmb->nb_segs = 1; |
| + rxmb->next = NULL; |
| + rxmb->pkt_len = rx_pkt_len; |
| + rxmb->data_len = rx_pkt_len; |
| + rxmb->port = enic->port_id; |
| + rxmb->ol_flags = ol_err_flags; |
| + if (!ol_err_flags) |
| + enic_cq_rx_to_pkt_flags(&cqd, rxmb); |
| + rxmb->packet_type = enic_cq_rx_flags_to_pkt_type(&cqd); |
| + |
| + /* prefetch mbuf data for caller */ |
| + rte_packet_prefetch(RTE_PTR_ADD(rxmb->buf_addr, |
| + RTE_PKTMBUF_HEADROOM)); |
| + |
| + /* store the mbuf address into the next entry of the array */ |
| + rx_pkts[nb_rx++] = rxmb; |
| + } |
| + |
| + nb_hold += nb_rx; |
| + cq->to_clean = rx_id; |
| + |
| + if (nb_hold > rq->rx_free_thresh) { |
| + rq->posted_index = enic_ring_add(rq->ring.desc_count, |
| + rq->posted_index, nb_hold); |
| + nb_hold = 0; |
| + rte_mb(); |
| + iowrite32(rq->posted_index, &rq->ctrl->posted_index); |
| + } |
| + |
| + rq->rx_nb_hold = nb_hold; |
| + |
| + return nb_rx; |
| +} |
| -- |
| 1.9.1 |
| |