blob: 393245434109e3977e0a197791acd254b394f2e6 [file] [log] [blame]
Rakesh Nair9bcf2602017-01-06 16:02:16 +05301/*
2 * Copyright (c) 2014 - 2017, The Linux Foundation. All rights reserved.
3 *
4 * Permission to use, copy, modify, and/or distribute this software for
5 * any purpose with or without fee is hereby granted, provided that the
6 * above copyright notice and this permission notice appear in all copies.
7 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
8 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
9 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
10 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
11 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
12 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT
13 * OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
14 */
15
16#include <linux/platform_device.h>
17#include <linux/if_vlan.h>
Rakesh Nair888af952017-06-30 18:41:58 +053018#include <linux/kernel.h>
Rakesh Nair9bcf2602017-01-06 16:02:16 +053019#include "ess_edma.h"
20#include "edma.h"
21
22extern struct net_device *edma_netdev[EDMA_MAX_PORTID_SUPPORTED];
23bool edma_stp_rstp;
24u16 edma_ath_eth_type;
Rakesh Nair888af952017-06-30 18:41:58 +053025extern u8 edma_dscp2ac_tbl[EDMA_PRECEDENCE_MAX];
26extern u8 edma_per_prec_stats_enable;
Rakesh Nair9bcf2602017-01-06 16:02:16 +053027
28/* edma_skb_priority_offset()
29 * get edma skb priority
30 */
31static unsigned int edma_skb_priority_offset(struct sk_buff *skb)
32{
33 return (skb->priority >> 2) & 1;
34}
35
36/* edma_alloc_tx_ring()
37 * Allocate Tx descriptors ring
38 */
39static int edma_alloc_tx_ring(struct edma_common_info *edma_cinfo,
40 struct edma_tx_desc_ring *etdr)
41{
42 struct platform_device *pdev = edma_cinfo->pdev;
43 u16 sw_size = sizeof(struct edma_sw_desc) * etdr->count;
44
45 /* Initialize ring */
46 etdr->size = sizeof(struct edma_tx_desc) * etdr->count;
47 etdr->sw_next_to_fill = 0;
48 etdr->sw_next_to_clean = 0;
49
50 /* Allocate SW descriptors */
51 etdr->sw_desc = vzalloc(sw_size);
52 if (!etdr->sw_desc) {
53 dev_err(&pdev->dev, "buffer alloc of tx ring failed=%p", etdr);
54 return -ENOMEM;
55 }
56
57 /* Allocate HW descriptors */
58 etdr->hw_desc = dma_alloc_coherent(&pdev->dev, etdr->size, &etdr->dma,
59 GFP_KERNEL);
60 if (!etdr->hw_desc) {
61 dev_err(&pdev->dev, "descriptor allocation for tx ring failed");
62 vfree(etdr->sw_desc);
63 etdr->sw_desc = NULL;
64 return -ENOMEM;
65 }
66
67 return 0;
68}
69
70/* edma_free_tx_ring()
71 * Free tx rings allocated by edma_alloc_tx_rings
72 */
73static void edma_free_tx_ring(struct edma_common_info *edma_cinfo,
74 struct edma_tx_desc_ring *etdr)
75{
76 struct platform_device *pdev = edma_cinfo->pdev;
77
78 if (likely(etdr->hw_desc)) {
79 dma_free_coherent(&pdev->dev, etdr->size, etdr->hw_desc,
80 etdr->dma);
81
82 vfree(etdr->sw_desc);
83 etdr->sw_desc = NULL;
84 }
85}
86
87/* edma_alloc_rx_ring()
88 * allocate rx descriptor ring
89 */
90static int edma_alloc_rx_ring(struct edma_common_info *edma_cinfo,
91 struct edma_rfd_desc_ring *erxd)
92{
93 struct platform_device *pdev = edma_cinfo->pdev;
94 u16 sw_size = sizeof(struct edma_sw_desc) * erxd->count;
95
96 erxd->size = sizeof(struct edma_sw_desc) * erxd->count;
97 erxd->sw_next_to_fill = 0;
98 erxd->sw_next_to_clean = 0;
99
100 /* Allocate SW descriptors */
101 erxd->sw_desc = vzalloc(sw_size);
102 if (!erxd->sw_desc)
103 return -ENOMEM;
104
105 /* Alloc HW descriptors */
106 erxd->hw_desc = dma_alloc_coherent(&pdev->dev, erxd->size, &erxd->dma,
107 GFP_KERNEL);
108 if (!erxd->hw_desc) {
109 vfree(erxd->sw_desc);
110 erxd->sw_desc = NULL;
111 return -ENOMEM;
112 }
113
Rakesh Nair03b586c2017-04-03 18:28:58 +0530114 /* Initialize pending fill */
115 erxd->pending_fill = 0;
116
Rakesh Nair9bcf2602017-01-06 16:02:16 +0530117 return 0;
118}
119
120/* edma_free_rx_ring()
121 * Free rx ring allocated by alloc_rx_ring
122 */
123static void edma_free_rx_ring(struct edma_common_info *edma_cinfo,
124 struct edma_rfd_desc_ring *erxd)
125{
126 struct platform_device *pdev = edma_cinfo->pdev;
127
128 if (likely(erxd->hw_desc)) {
129 dma_free_coherent(&pdev->dev, erxd->size, erxd->hw_desc,
130 erxd->dma);
131
132 vfree(erxd->sw_desc);
133 erxd->sw_desc = NULL;
134 }
135}
136
137/* edma_configure_tx()
138 * Configure transmission control data
139 */
140static void edma_configure_tx(struct edma_common_info *edma_cinfo)
141{
142 u32 txq_ctrl_data;
143
144 txq_ctrl_data = (EDMA_TPD_BURST << EDMA_TXQ_NUM_TPD_BURST_SHIFT);
145 txq_ctrl_data |= EDMA_TXQ_CTRL_TPD_BURST_EN;
146 txq_ctrl_data |= (EDMA_TXF_BURST << EDMA_TXQ_TXF_BURST_NUM_SHIFT);
147 edma_write_reg(EDMA_REG_TXQ_CTRL, txq_ctrl_data);
148}
149
150/* edma_configure_rx()
151 * configure reception control data
152 */
153static void edma_configure_rx(struct edma_common_info *edma_cinfo)
154{
155 struct edma_hw *hw = &edma_cinfo->hw;
156 u32 rss_type, rx_desc1, rxq_ctrl_data;
157
158 /* Set RSS type */
159 rss_type = hw->rss_type;
160 edma_write_reg(EDMA_REG_RSS_TYPE, rss_type);
161
162 /* Set RFD burst number */
163 rx_desc1 = (EDMA_RFD_BURST << EDMA_RXQ_RFD_BURST_NUM_SHIFT);
164
165 /* Set RFD prefetch threshold */
166 rx_desc1 |= (EDMA_RFD_THR << EDMA_RXQ_RFD_PF_THRESH_SHIFT);
167
168 /* Set RFD in host ring low threshold to generte interrupt */
169 rx_desc1 |= (EDMA_RFD_LTHR << EDMA_RXQ_RFD_LOW_THRESH_SHIFT);
170 edma_write_reg(EDMA_REG_RX_DESC1, rx_desc1);
171
172 /* Set Rx FIFO threshold to start to DMA data to host */
173 rxq_ctrl_data = EDMA_FIFO_THRESH_128_BYTE;
174
175 /* Set RX remove vlan bit */
176 rxq_ctrl_data |= EDMA_RXQ_CTRL_RMV_VLAN;
177
178 edma_write_reg(EDMA_REG_RXQ_CTRL, rxq_ctrl_data);
179}
180
181/* edma_alloc_rx_buf()
182 * does skb allocation for the received packets.
183 */
184static int edma_alloc_rx_buf(struct edma_common_info
185 *edma_cinfo,
186 struct edma_rfd_desc_ring *erdr,
187 int cleaned_count, int queue_id)
188{
189 struct platform_device *pdev = edma_cinfo->pdev;
190 struct edma_rx_free_desc *rx_desc;
191 struct edma_sw_desc *sw_desc;
192 struct sk_buff *skb;
193 unsigned int i;
194 u16 prod_idx, length;
195 u32 reg_data;
196
197 if (cleaned_count > erdr->count) {
198 dev_err(&pdev->dev, "Incorrect cleaned_count %d",
199 cleaned_count);
200 return -1;
201 }
202
203 i = erdr->sw_next_to_fill;
204
205 while (cleaned_count) {
206 sw_desc = &erdr->sw_desc[i];
207 length = edma_cinfo->rx_head_buffer_len;
208
209 if (sw_desc->flags & EDMA_SW_DESC_FLAG_SKB_REUSE) {
210 skb = sw_desc->skb;
211
212 /* Clear REUSE flag */
213 sw_desc->flags &= ~EDMA_SW_DESC_FLAG_SKB_REUSE;
214 } else {
215 /* alloc skb */
216 skb = netdev_alloc_skb(edma_netdev[0], length);
217 if (!skb) {
218 /* Better luck next round */
219 sw_desc->flags = 0;
220 break;
221 }
222 }
223
224 if (!edma_cinfo->page_mode) {
225 sw_desc->dma = dma_map_single(&pdev->dev, skb->data,
226 length, DMA_FROM_DEVICE);
227 if (dma_mapping_error(&pdev->dev, sw_desc->dma)) {
228 WARN_ONCE(0, "EDMA DMA mapping failed for linear address %x", sw_desc->dma);
229 sw_desc->flags = 0;
230 sw_desc->skb = NULL;
231 dev_kfree_skb_any(skb);
232 break;
233 }
234
235 /*
236 * We should not exit from here with REUSE flag set
237 * This is to avoid re-using same sk_buff for next
238 * time around
239 */
240 sw_desc->flags = EDMA_SW_DESC_FLAG_SKB_HEAD;
241 sw_desc->length = length;
242 } else {
243 struct page *pg = alloc_page(GFP_ATOMIC);
244
245 if (!pg) {
246 sw_desc->flags = 0;
247 sw_desc->skb = NULL;
248 dev_kfree_skb_any(skb);
249 break;
250 }
251
252 sw_desc->dma = dma_map_page(&pdev->dev, pg, 0,
253 edma_cinfo->rx_page_buffer_len,
254 DMA_FROM_DEVICE);
255 if (dma_mapping_error(&pdev->dev, sw_desc->dma)) {
256 WARN_ONCE(0, "EDMA DMA mapping failed for page address %x", sw_desc->dma);
257 sw_desc->flags = 0;
258 sw_desc->skb = NULL;
259 __free_page(pg);
260 dev_kfree_skb_any(skb);
261 break;
262 }
263
264 skb_fill_page_desc(skb, 0, pg, 0,
265 edma_cinfo->rx_page_buffer_len);
266 sw_desc->flags = EDMA_SW_DESC_FLAG_SKB_FRAG;
267 sw_desc->length = edma_cinfo->rx_page_buffer_len;
268 }
269
270 /* Update the buffer info */
271 sw_desc->skb = skb;
272 rx_desc = (&(erdr->hw_desc)[i]);
273 rx_desc->buffer_addr = cpu_to_le64(sw_desc->dma);
274 if (++i == erdr->count)
275 i = 0;
276 cleaned_count--;
277 }
278
279 erdr->sw_next_to_fill = i;
280
281 if (i == 0)
282 prod_idx = erdr->count - 1;
283 else
284 prod_idx = i - 1;
285
286 /* Update the producer index */
287 edma_read_reg(EDMA_REG_RFD_IDX_Q(queue_id), &reg_data);
288 reg_data &= ~EDMA_RFD_PROD_IDX_BITS;
289 reg_data |= prod_idx;
290 edma_write_reg(EDMA_REG_RFD_IDX_Q(queue_id), reg_data);
Rakesh Nair03b586c2017-04-03 18:28:58 +0530291
292 /* If we couldn't allocate all the buffers,
293 * we increment the alloc failure counters
294 */
295 if (cleaned_count)
296 edma_cinfo->edma_ethstats.rx_alloc_fail_ctr++;
297
Rakesh Nair9bcf2602017-01-06 16:02:16 +0530298 return cleaned_count;
299}
300
301/* edma_init_desc()
302 * update descriptor ring size, buffer and producer/consumer index
303 */
304static void edma_init_desc(struct edma_common_info *edma_cinfo)
305{
306 struct edma_rfd_desc_ring *rfd_ring;
307 struct edma_tx_desc_ring *etdr;
308 int i = 0, j = 0;
309 u32 data = 0;
310 u16 hw_cons_idx = 0;
311
312 /* Set the base address of every TPD ring. */
313 for (i = 0; i < edma_cinfo->num_tx_queues; i++) {
314 etdr = edma_cinfo->tpd_ring[i];
315
316 /* Update descriptor ring base address */
317 edma_write_reg(EDMA_REG_TPD_BASE_ADDR_Q(i), (u32)etdr->dma);
318 edma_read_reg(EDMA_REG_TPD_IDX_Q(i), &data);
319
320 /* Calculate hardware consumer index */
321 hw_cons_idx = (data >> EDMA_TPD_CONS_IDX_SHIFT) & 0xffff;
322 etdr->sw_next_to_fill = hw_cons_idx;
323 etdr->sw_next_to_clean = hw_cons_idx;
324 data &= ~(EDMA_TPD_PROD_IDX_MASK << EDMA_TPD_PROD_IDX_SHIFT);
325 data |= hw_cons_idx;
326
327 /* update producer index */
328 edma_write_reg(EDMA_REG_TPD_IDX_Q(i), data);
329
330 /* update SW consumer index register */
331 edma_write_reg(EDMA_REG_TX_SW_CONS_IDX_Q(i), hw_cons_idx);
332
333 /* Set TPD ring size */
334 edma_write_reg(EDMA_REG_TPD_RING_SIZE,
335 edma_cinfo->tx_ring_count &
336 EDMA_TPD_RING_SIZE_MASK);
337 }
338
339 for (i = 0, j = 0; i < edma_cinfo->num_rx_queues; i++) {
340 rfd_ring = edma_cinfo->rfd_ring[j];
341 /* Update Receive Free descriptor ring base address */
342 edma_write_reg(EDMA_REG_RFD_BASE_ADDR_Q(j),
343 (u32)(rfd_ring->dma));
344 j += ((edma_cinfo->num_rx_queues == 4) ? 2 : 1);
345 }
346
347 data = edma_cinfo->rx_head_buffer_len;
348 if (edma_cinfo->page_mode)
349 data = edma_cinfo->rx_page_buffer_len;
350
351 data &= EDMA_RX_BUF_SIZE_MASK;
352 data <<= EDMA_RX_BUF_SIZE_SHIFT;
353
354 /* Update RFD ring size and RX buffer size */
355 data |= (edma_cinfo->rx_ring_count & EDMA_RFD_RING_SIZE_MASK)
356 << EDMA_RFD_RING_SIZE_SHIFT;
357
358 edma_write_reg(EDMA_REG_RX_DESC0, data);
359
360 /* Disable TX FIFO low watermark and high watermark */
361 edma_write_reg(EDMA_REG_TXF_WATER_MARK, 0);
362
363 /* Load all of base address above */
364 edma_read_reg(EDMA_REG_TX_SRAM_PART, &data);
365 data |= 1 << EDMA_LOAD_PTR_SHIFT;
366 edma_write_reg(EDMA_REG_TX_SRAM_PART, data);
367}
368
369/* edma_receive_checksum
370 * Api to check checksum on receive packets
371 */
372static void edma_receive_checksum(struct edma_rx_return_desc *rd,
373 struct sk_buff *skb)
374{
375 skb_checksum_none_assert(skb);
376
377 /* check the RRD IP/L4 checksum bit to see if
378 * its set, which in turn indicates checksum
379 * failure.
380 */
381 if (rd->rrd6 & EDMA_RRD_CSUM_FAIL_MASK)
382 return;
383
Rakesh Nair72e1d282017-05-19 22:21:01 +0530384 /*
385 * We disable checksum verification only if
386 * we have a TCP/UDP packet
387 */
388 if (rd->rrd7 & (EDMA_RRD_L4OFFSET_MASK << EDMA_RRD_L4OFFSET_SHIFT))
389 skb->ip_summed = CHECKSUM_UNNECESSARY;
Rakesh Nair9bcf2602017-01-06 16:02:16 +0530390}
391
392/* edma_clean_rfd()
393 * clean up rx resourcers on error
394 */
395static void edma_clean_rfd(struct platform_device *pdev,
396 struct edma_rfd_desc_ring *erdr,
397 u16 index,
398 int pos)
399{
400 struct edma_rx_free_desc *rx_desc = &(erdr->hw_desc[index]);
401 struct edma_sw_desc *sw_desc = &erdr->sw_desc[index];
402
403 /* Unmap non-first RFD positions in packet */
404 if (pos) {
405 if (likely(sw_desc->flags & EDMA_SW_DESC_FLAG_SKB_HEAD))
406 dma_unmap_single(&pdev->dev, sw_desc->dma,
407 sw_desc->length, DMA_FROM_DEVICE);
408 else
409 dma_unmap_page(&pdev->dev, sw_desc->dma,
410 sw_desc->length, DMA_FROM_DEVICE);
411 }
412
413 if (sw_desc->skb) {
414 dev_kfree_skb_any(sw_desc->skb);
415 sw_desc->skb = NULL;
416 }
417
418 sw_desc->flags = 0;
419 memset(rx_desc, 0, sizeof(struct edma_rx_free_desc));
420}
421
422/* edma_rx_complete_stp_rstp()
423 * Complete Rx processing for STP RSTP packets
424 */
425static void edma_rx_complete_stp_rstp(struct sk_buff *skb, int port_id, struct edma_rx_return_desc *rd)
426{
427 int i;
428 u32 priority;
429 u16 port_type;
430 u8 mac_addr[EDMA_ETH_HDR_LEN];
431
432 port_type = (rd->rrd1 >> EDMA_RRD_PORT_TYPE_SHIFT)
433 & EDMA_RRD_PORT_TYPE_MASK;
434 /* if port type is 0x4, then only proceed with
435 * other stp/rstp calculation
436 */
437 if (port_type == EDMA_RX_ATH_HDR_RSTP_PORT_TYPE) {
438 u8 bpdu_mac[6] = {0x01, 0x80, 0xc2, 0x00, 0x00, 0x00};
439
440 /* calculate the frame priority */
441 priority = (rd->rrd1 >> EDMA_RRD_PRIORITY_SHIFT)
442 & EDMA_RRD_PRIORITY_MASK;
443
444 for (i = 0; i < EDMA_ETH_HDR_LEN; i++)
445 mac_addr[i] = skb->data[i];
446
447 /* Check if destination mac addr is bpdu addr */
448 if (!memcmp(mac_addr, bpdu_mac, 6)) {
449 /* destination mac address is BPDU
450 * destination mac address, then add
451 * atheros header to the packet.
452 */
453 u16 athr_hdr = (EDMA_RX_ATH_HDR_VERSION << EDMA_RX_ATH_HDR_VERSION_SHIFT) |
454 (priority << EDMA_RX_ATH_HDR_PRIORITY_SHIFT) |
455 (EDMA_RX_ATH_HDR_RSTP_PORT_TYPE << EDMA_RX_ATH_PORT_TYPE_SHIFT) | port_id;
456 skb_push(skb, 4);
457 memcpy(skb->data, mac_addr, EDMA_ETH_HDR_LEN);
458 *(uint16_t *)&skb->data[12] = htons(edma_ath_eth_type);
459 *(uint16_t *)&skb->data[14] = htons(athr_hdr);
460 }
461 }
462}
463
464/* edma_rx_complete_fraglist()
465 * Complete Rx processing for fraglist skbs
466 */
467static int edma_rx_complete_fraglist(struct sk_buff *skb, u16 num_rfds, u16 length, u32 sw_next_to_clean,
468 struct edma_rfd_desc_ring *erdr, struct edma_common_info *edma_cinfo)
469{
470 struct platform_device *pdev = edma_cinfo->pdev;
471 struct edma_hw *hw = &edma_cinfo->hw;
472 struct sk_buff *skb_temp;
473 struct edma_sw_desc *sw_desc;
474 int i;
475 u16 size_remaining;
476
477 skb->data_len = 0;
478 skb->tail += (hw->rx_head_buff_size - 16);
479 skb->len = skb->truesize = length;
480 size_remaining = length - (hw->rx_head_buff_size - 16);
481
482 /* clean-up all related sw_descs */
483 for (i = 1; i < num_rfds; i++) {
484 struct sk_buff *skb_prev;
485
486 sw_desc = &erdr->sw_desc[sw_next_to_clean];
487 skb_temp = sw_desc->skb;
488
489 dma_unmap_single(&pdev->dev, sw_desc->dma,
490 sw_desc->length, DMA_FROM_DEVICE);
491
492 if (size_remaining < hw->rx_head_buff_size)
493 skb_put(skb_temp, size_remaining);
494 else
495 skb_put(skb_temp, hw->rx_head_buff_size);
496
497 /* If we are processing the first rfd, we link
498 * skb->frag_list to the skb corresponding to the
499 * first RFD
500 */
501 if (i == 1)
502 skb_shinfo(skb)->frag_list = skb_temp;
503 else
504 skb_prev->next = skb_temp;
505 skb_prev = skb_temp;
506 skb_temp->next = NULL;
507
508 skb->data_len += skb_temp->len;
509 size_remaining -= skb_temp->len;
510
511 /* Increment SW index */
512 sw_next_to_clean = (sw_next_to_clean + 1) & (erdr->count - 1);
513 }
514
515 return sw_next_to_clean;
516}
517
518/* edma_rx_complete_paged()
519 * Complete Rx processing for paged skbs
520 */
521static int edma_rx_complete_paged(struct sk_buff *skb, u16 num_rfds,
522 u16 length, u32 sw_next_to_clean,
523 struct edma_rfd_desc_ring *erdr,
524 struct edma_common_info *edma_cinfo)
525{
526 struct platform_device *pdev = edma_cinfo->pdev;
527 struct sk_buff *skb_temp;
528 struct edma_sw_desc *sw_desc;
529 int i;
530 u16 size_remaining;
531
532 skb_frag_t *frag = &skb_shinfo(skb)->frags[0];
533
534 /* Setup skbuff fields */
535 skb->len = length;
536
537 if (likely(num_rfds <= 1)) {
538 skb->data_len = length;
539 skb->truesize += edma_cinfo->rx_page_buffer_len;
540 skb_fill_page_desc(skb, 0, skb_frag_page(frag),
541 16, length);
542 } else {
543 frag->size -= 16;
544 skb->data_len = frag->size;
545 skb->truesize += edma_cinfo->rx_page_buffer_len;
546 size_remaining = length - frag->size;
547
548 skb_fill_page_desc(skb, 0, skb_frag_page(frag),
549 16, frag->size);
550
551 /* clean-up all related sw_descs */
552 for (i = 1; i < num_rfds; i++) {
553 sw_desc = &erdr->sw_desc[sw_next_to_clean];
554 skb_temp = sw_desc->skb;
555 frag = &skb_shinfo(skb_temp)->frags[0];
556 dma_unmap_page(&pdev->dev, sw_desc->dma,
557 sw_desc->length, DMA_FROM_DEVICE);
558
559 if (size_remaining < edma_cinfo->rx_page_buffer_len)
560 frag->size = size_remaining;
561
562 skb_fill_page_desc(skb, i, skb_frag_page(frag),
563 0, frag->size);
564
565 /* We used frag pages from skb_temp in skb */
566 skb_shinfo(skb_temp)->nr_frags = 0;
567 dev_kfree_skb_any(skb_temp);
568
569 skb->data_len += frag->size;
570 skb->truesize += edma_cinfo->rx_page_buffer_len;
571 size_remaining -= frag->size;
572
573 /* Increment SW index */
574 sw_next_to_clean = (sw_next_to_clean + 1) & (erdr->count - 1);
575 }
576 }
577
578 return sw_next_to_clean;
579}
580
581/*
582 * edma_rx_complete()
583 * Main api called from the poll function to process rx packets.
584 */
Rakesh Nair03b586c2017-04-03 18:28:58 +0530585static u16 edma_rx_complete(struct edma_common_info *edma_cinfo,
Rakesh Nair9bcf2602017-01-06 16:02:16 +0530586 int *work_done, int work_to_do, int queue_id,
587 struct napi_struct *napi)
588{
589 struct platform_device *pdev = edma_cinfo->pdev;
590 struct edma_rfd_desc_ring *erdr = edma_cinfo->rfd_ring[queue_id];
591 u16 hash_type, rrd[8], cleaned_count = 0, length = 0, num_rfds = 1,
592 sw_next_to_clean, hw_next_to_clean = 0, vlan = 0, ret_count = 0;
593 u32 data = 0;
594 u16 count = erdr->count, rfd_avail;
595 u8 queue_to_rxid[8] = {0, 0, 1, 1, 2, 2, 3, 3};
596
Rakesh Nair03b586c2017-04-03 18:28:58 +0530597 cleaned_count = erdr->pending_fill;
Rakesh Nair9bcf2602017-01-06 16:02:16 +0530598 sw_next_to_clean = erdr->sw_next_to_clean;
599
600 edma_read_reg(EDMA_REG_RFD_IDX_Q(queue_id), &data);
601 hw_next_to_clean = (data >> EDMA_RFD_CONS_IDX_SHIFT) &
602 EDMA_RFD_CONS_IDX_MASK;
603
604 do {
605 while (sw_next_to_clean != hw_next_to_clean) {
606 struct net_device *netdev;
607 struct edma_adapter *adapter;
608 struct edma_sw_desc *sw_desc;
609 struct sk_buff *skb;
610 struct edma_rx_return_desc *rd;
611 u8 *vaddr;
612 int port_id, i, drop_count = 0;
613 u32 priority;
614
615 if (!work_to_do)
616 break;
617
618 sw_desc = &erdr->sw_desc[sw_next_to_clean];
619 skb = sw_desc->skb;
620
621 /* Get RRD */
622 if (!edma_cinfo->page_mode) {
623 dma_unmap_single(&pdev->dev, sw_desc->dma,
624 sw_desc->length, DMA_FROM_DEVICE);
625 rd = (struct edma_rx_return_desc *)skb->data;
626
627 } else {
628 dma_unmap_page(&pdev->dev, sw_desc->dma,
629 sw_desc->length, DMA_FROM_DEVICE);
630 vaddr = kmap_atomic(skb_frag_page(&skb_shinfo(skb)->frags[0]));
631 memcpy((uint8_t *)&rrd[0], vaddr, 16);
632 rd = (struct edma_rx_return_desc *)rrd;
633 kunmap_atomic(vaddr);
634 }
635
636 /* Check if RRD is valid */
637 if (!(rd->rrd7 & EDMA_RRD_DESC_VALID)) {
638 dev_err(&pdev->dev, "Incorrect RRD DESC valid bit set");
639 edma_clean_rfd(pdev, erdr, sw_next_to_clean, 0);
640 sw_next_to_clean = (sw_next_to_clean + 1) &
641 (erdr->count - 1);
642 cleaned_count++;
643 continue;
644 }
645
646 /* Get the number of RFDs from RRD */
647 num_rfds = rd->rrd1 & EDMA_RRD_NUM_RFD_MASK;
648
649 /* Get Rx port ID from switch */
650 port_id = (rd->rrd1 >> EDMA_PORT_ID_SHIFT) & EDMA_PORT_ID_MASK;
651 if ((!port_id) || (port_id > EDMA_MAX_PORTID_SUPPORTED)) {
652 if (net_ratelimit()) {
653 dev_err(&pdev->dev, "Incorrect RRD source port bit set");
654 dev_err(&pdev->dev,
655 "RRD Dump\n rrd0:%x rrd1: %x rrd2: %x rrd3: %x rrd4: %x rrd5: %x rrd6: %x rrd7: %x",
656 rd->rrd0, rd->rrd1, rd->rrd2, rd->rrd3, rd->rrd4, rd->rrd5, rd->rrd6, rd->rrd7);
657 dev_err(&pdev->dev, "Num_rfds: %d, src_port: %d, pkt_size: %d, cvlan_tag: %d\n",
658 num_rfds, rd->rrd1 & EDMA_RRD_SRC_PORT_NUM_MASK,
659 rd->rrd6 & EDMA_RRD_PKT_SIZE_MASK, rd->rrd7 & EDMA_RRD_CVLAN);
660 }
661 for (i = 0; i < num_rfds; i++) {
662 edma_clean_rfd(pdev, erdr, sw_next_to_clean, i);
663 sw_next_to_clean = (sw_next_to_clean + 1) & (erdr->count - 1);
664 }
665
666 cleaned_count += num_rfds;
667 continue;
668 }
669
670 netdev = edma_cinfo->portid_netdev_lookup_tbl[port_id];
671 if (!netdev) {
672 dev_err(&pdev->dev, "Invalid netdev");
673 for (i = 0; i < num_rfds; i++) {
674 edma_clean_rfd(pdev, erdr, sw_next_to_clean, i);
675 sw_next_to_clean = (sw_next_to_clean + 1) & (erdr->count - 1);
676 }
677
678 cleaned_count += num_rfds;
679 continue;
680 }
681 adapter = netdev_priv(netdev);
682
683 /* This code is added to handle a usecase where high
684 * priority stream and a low priority stream are
685 * received simultaneously on DUT. The problem occurs
686 * if one of the Rx rings is full and the corresponding
687 * core is busy with other stuff. This causes ESS CPU
688 * port to backpressure all incoming traffic including
689 * high priority one. We monitor free descriptor count
690 * on each CPU and whenever it reaches threshold (< 80),
691 * we drop all low priority traffic and let only high
692 * priotiy traffic pass through. We can hence avoid
693 * ESS CPU port to send backpressure on high priroity
694 * stream.
695 */
696 priority = (rd->rrd1 >> EDMA_RRD_PRIORITY_SHIFT)
697 & EDMA_RRD_PRIORITY_MASK;
698 if (likely(!priority && !edma_cinfo->page_mode && (num_rfds <= 1))) {
699 rfd_avail = (count + sw_next_to_clean - hw_next_to_clean - 1) & (count - 1);
700 if (rfd_avail < EDMA_RFD_AVAIL_THR) {
701 sw_desc->flags |= EDMA_SW_DESC_FLAG_SKB_REUSE;
702 sw_next_to_clean = (sw_next_to_clean + 1) & (erdr->count - 1);
703 adapter->stats.rx_dropped++;
704 cleaned_count++;
705 drop_count++;
706 if (drop_count == 3) {
707 work_to_do--;
708 (*work_done)++;
709 drop_count = 0;
710 }
711 if (cleaned_count == EDMA_RX_BUFFER_WRITE) {
712 /* If buffer clean count reaches 16, we replenish HW buffers. */
713 ret_count = edma_alloc_rx_buf(edma_cinfo, erdr, cleaned_count, queue_id);
714 edma_write_reg(EDMA_REG_RX_SW_CONS_IDX_Q(queue_id),
715 sw_next_to_clean);
716 cleaned_count = ret_count;
Rakesh Nair03b586c2017-04-03 18:28:58 +0530717 erdr->pending_fill = ret_count;
Rakesh Nair9bcf2602017-01-06 16:02:16 +0530718 }
719 continue;
720 }
721 }
722
723 work_to_do--;
724 (*work_done)++;
725
726 /* Increment SW index */
727 sw_next_to_clean = (sw_next_to_clean + 1) &
728 (erdr->count - 1);
729
730 /* Get the packet size and allocate buffer */
731 length = rd->rrd6 & EDMA_RRD_PKT_SIZE_MASK;
732
733 if (edma_cinfo->page_mode) {
734 /* paged skb */
735 sw_next_to_clean = edma_rx_complete_paged(skb, num_rfds, length,
736 sw_next_to_clean,
737 erdr, edma_cinfo);
738 if (!pskb_may_pull(skb, ETH_HLEN)) {
739 cleaned_count += num_rfds;
740 dev_kfree_skb_any(skb);
741 continue;
742 }
743 } else {
744 /* single or fraglist skb */
745
746 /* Addition of 16 bytes is required, as in the packet
747 * first 16 bytes are rrd descriptors, so actual data
748 * starts from an offset of 16.
749 */
750 skb_reserve(skb, 16);
751 if (likely((num_rfds <= 1) || !edma_cinfo->fraglist_mode))
752 skb_put(skb, length);
753 else
754 sw_next_to_clean = edma_rx_complete_fraglist(skb, num_rfds, length,
755 sw_next_to_clean,
756 erdr, edma_cinfo);
757 }
758
759 cleaned_count += num_rfds;
760
761 if (edma_stp_rstp)
762 edma_rx_complete_stp_rstp(skb, port_id, rd);
763
764 skb->protocol = eth_type_trans(skb, netdev);
765
766 /* Record Rx queue for RFS/RPS and fill flow hash from HW */
767 skb_record_rx_queue(skb, queue_to_rxid[queue_id]);
768 if (netdev->features & NETIF_F_RXHASH) {
769 hash_type = (rd->rrd5 >> EDMA_HASH_TYPE_SHIFT);
770 if ((hash_type > EDMA_HASH_TYPE_START) && (hash_type < EDMA_HASH_TYPE_END))
771 skb_set_hash(skb, rd->rrd2, PKT_HASH_TYPE_L4);
772 }
773
774#ifdef CONFIG_NF_FLOW_COOKIE
775 skb->flow_cookie = rd->rrd3 & EDMA_RRD_FLOW_COOKIE_MASK;
776#endif
777 edma_receive_checksum(rd, skb);
778
779 /* Process VLAN HW acceleration indication provided by HW */
780 if (adapter->default_vlan_tag != rd->rrd4) {
781 vlan = rd->rrd4;
782 if (likely(rd->rrd7 & EDMA_RRD_CVLAN))
783 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vlan);
784 else if (rd->rrd1 & EDMA_RRD_SVLAN)
785 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021AD), vlan);
786 }
787
788 /* Update rx statistics */
789 adapter->stats.rx_packets++;
790 adapter->stats.rx_bytes += length;
791
792 /* Check if we reached refill threshold */
793 if (cleaned_count == EDMA_RX_BUFFER_WRITE) {
794 ret_count = edma_alloc_rx_buf(edma_cinfo, erdr, cleaned_count, queue_id);
795 edma_write_reg(EDMA_REG_RX_SW_CONS_IDX_Q(queue_id),
796 sw_next_to_clean);
797 cleaned_count = ret_count;
Rakesh Nair03b586c2017-04-03 18:28:58 +0530798 erdr->pending_fill = ret_count;
Rakesh Nair9bcf2602017-01-06 16:02:16 +0530799 }
800
Rakesh Nair888af952017-06-30 18:41:58 +0530801 /*
802 * We increment per-precedence counters for the rx packets
803 */
804 if (edma_per_prec_stats_enable) {
805 edma_cinfo->edma_ethstats.rx_prec[priority]++;
806 edma_cinfo->edma_ethstats.rx_ac[edma_dscp2ac_tbl[priority]]++;
807 }
808
Rakesh Nair9bcf2602017-01-06 16:02:16 +0530809 /* At this point skb should go to stack */
810 napi_gro_receive(napi, skb);
811 }
812
813 /* Check if we still have NAPI budget */
814 if (!work_to_do)
815 break;
816
817 /* Read index once again since we still have NAPI budget */
818 edma_read_reg(EDMA_REG_RFD_IDX_Q(queue_id), &data);
819 hw_next_to_clean = (data >> EDMA_RFD_CONS_IDX_SHIFT) &
820 EDMA_RFD_CONS_IDX_MASK;
821 } while (hw_next_to_clean != sw_next_to_clean);
822
823 erdr->sw_next_to_clean = sw_next_to_clean;
824
825 /* Refill here in case refill threshold wasn't reached */
826 if (likely(cleaned_count)) {
827 ret_count = edma_alloc_rx_buf(edma_cinfo, erdr, cleaned_count, queue_id);
Rakesh Nair03b586c2017-04-03 18:28:58 +0530828 erdr->pending_fill = ret_count;
829 if (ret_count) {
830 if(net_ratelimit())
831 dev_dbg(&pdev->dev, "Edma not getting memory for descriptors.\n");
832 }
833
Rakesh Nair9bcf2602017-01-06 16:02:16 +0530834 edma_write_reg(EDMA_REG_RX_SW_CONS_IDX_Q(queue_id),
835 erdr->sw_next_to_clean);
836 }
Rakesh Nair03b586c2017-04-03 18:28:58 +0530837
838 return erdr->pending_fill;
Rakesh Nair9bcf2602017-01-06 16:02:16 +0530839}
840
841/* edma_delete_rfs_filter()
842 * Remove RFS filter from switch
843 */
844static int edma_delete_rfs_filter(struct edma_adapter *adapter,
845 struct edma_rfs_filter_node *filter_node)
846{
847 int res = -1;
848
849 if (likely(adapter->set_rfs_rule))
850 res = (*adapter->set_rfs_rule)(adapter->netdev,
851#if (LINUX_VERSION_CODE <= KERNEL_VERSION(3, 18, 21))
852 filter_node->keys.src,
853 filter_node->keys.dst, filter_node->keys.port16[0],
854 filter_node->keys.port16[1],
855 filter_node->keys.ip_proto,
856#else
857 filter_node->keys.addrs.v4addrs.src,
858 filter_node->keys.addrs.v4addrs.dst, filter_node->keys.ports.src,
859 filter_node->keys.ports.dst,
860 filter_node->keys.basic.ip_proto,
861#endif
862 filter_node->rq_id,
863 0);
864
865 return res;
866}
867
868/* edma_add_rfs_filter()
869 * Add RFS filter to switch
870 */
871static int edma_add_rfs_filter(struct edma_adapter *adapter,
872 struct flow_keys *keys, u16 rq,
873 struct edma_rfs_filter_node *filter_node)
874{
875 int res = -1;
876
877#if (LINUX_VERSION_CODE <= KERNEL_VERSION(3, 18, 21))
878 filter_node->keys.src = keys->src;
879 filter_node->keys.dst = keys->dst;
880 filter_node->keys.ports = keys->ports;
881 filter_node->keys.ip_proto = keys->ip_proto;
882#else
883 filter_node->keys.addrs.v4addrs.src = keys->addrs.v4addrs.src;
884 filter_node->keys.addrs.v4addrs.dst = keys->addrs.v4addrs.dst;
885 filter_node->keys.ports.ports = keys->ports.ports;
886 filter_node->keys.basic.ip_proto = keys->basic.ip_proto;
887#endif
888
889 /* Call callback registered by ESS driver */
890 if (likely(adapter->set_rfs_rule))
891#if (LINUX_VERSION_CODE <= KERNEL_VERSION(3, 18, 21))
892 res = (*adapter->set_rfs_rule)(adapter->netdev, keys->src,
893 keys->dst, keys->port16[0], keys->port16[1],
894 keys->ip_proto, rq, 1);
895#else
896 res = (*adapter->set_rfs_rule)(adapter->netdev, keys->addrs.v4addrs.src,
897 keys->addrs.v4addrs.dst, keys->ports.src, keys->ports.dst,
898 keys->basic.ip_proto, rq, 1);
899#endif
900
901 return res;
902}
903
904/* edma_rfs_key_search()
905 * Look for existing RFS entry
906 */
907static struct edma_rfs_filter_node *edma_rfs_key_search(struct hlist_head *h,
908 struct flow_keys *key)
909{
910 struct edma_rfs_filter_node *p;
911
912 hlist_for_each_entry(p, h, node)
913#if (LINUX_VERSION_CODE <= KERNEL_VERSION(3, 18, 21))
914 if (p->keys.src == key->src &&
915 p->keys.dst == key->dst &&
916 p->keys.ports == key->ports &&
917 p->keys.ip_proto == key->ip_proto)
918#else
919 if (p->keys.addrs.v4addrs.src == key->addrs.v4addrs.src &&
920 p->keys.addrs.v4addrs.dst == key->addrs.v4addrs.dst &&
921 p->keys.ports.ports == key->ports.ports &&
922 p->keys.basic.ip_proto == key->basic.ip_proto)
923#endif
924 return p;
925 return NULL;
926}
927
928/* edma_initialise_rfs_flow_table()
929 * Initialise EDMA RFS flow table
930 */
931static void edma_initialise_rfs_flow_table(struct edma_adapter *adapter)
932{
933 int i;
934
935 spin_lock_init(&adapter->rfs.rfs_ftab_lock);
936
937 /* Initialize EDMA flow hash table */
938 for (i = 0; i < EDMA_RFS_FLOW_ENTRIES; i++)
939 INIT_HLIST_HEAD(&adapter->rfs.hlist_head[i]);
940
941 adapter->rfs.max_num_filter = EDMA_RFS_FLOW_ENTRIES;
942 adapter->rfs.filter_available = adapter->rfs.max_num_filter;
943 adapter->rfs.hashtoclean = 0;
944
945 /* Add timer to get periodic RFS updates from OS */
946 init_timer(&adapter->rfs.expire_rfs);
947 adapter->rfs.expire_rfs.function = edma_flow_may_expire;
948 adapter->rfs.expire_rfs.data = (unsigned long)adapter;
949 mod_timer(&adapter->rfs.expire_rfs, jiffies + HZ/4);
950}
951
952/* edma_free_rfs_flow_table()
953 * Free EDMA RFS flow table
954 */
955static void edma_free_rfs_flow_table(struct edma_adapter *adapter)
956{
957 int i;
958
959 /* Remove sync timer */
960 del_timer_sync(&adapter->rfs.expire_rfs);
961 spin_lock_bh(&adapter->rfs.rfs_ftab_lock);
962
963 /* Free EDMA RFS table entries */
964 adapter->rfs.filter_available = 0;
965
966 /* Clean-up EDMA flow hash table */
967 for (i = 0; i < EDMA_RFS_FLOW_ENTRIES; i++) {
968 struct hlist_head *hhead;
969 struct hlist_node *tmp;
970 struct edma_rfs_filter_node *filter_node;
971 int res;
972
973 hhead = &adapter->rfs.hlist_head[i];
974 hlist_for_each_entry_safe(filter_node, tmp, hhead, node) {
975 res = edma_delete_rfs_filter(adapter, filter_node);
976 if (res < 0)
977 dev_warn(&adapter->netdev->dev,
978 "EDMA going down but RFS entry %d not allowed to be flushed by Switch",
979 filter_node->flow_id);
980 hlist_del(&filter_node->node);
981 kfree(filter_node);
982 }
983 }
984 spin_unlock_bh(&adapter->rfs.rfs_ftab_lock);
985}
986
987/* edma_tx_unmap_and_free()
988 * clean TX buffer
989 */
990static inline void edma_tx_unmap_and_free(struct platform_device *pdev,
991 struct edma_sw_desc *sw_desc)
992{
993 struct sk_buff *skb = sw_desc->skb;
994
995 if (likely((sw_desc->flags & EDMA_SW_DESC_FLAG_SKB_HEAD) ||
996 (sw_desc->flags & EDMA_SW_DESC_FLAG_SKB_FRAGLIST)))
997 /* unmap_single for skb head area */
998 dma_unmap_single(&pdev->dev, sw_desc->dma,
999 sw_desc->length, DMA_TO_DEVICE);
1000 else if (sw_desc->flags & EDMA_SW_DESC_FLAG_SKB_FRAG)
1001 /* unmap page for paged fragments */
1002 dma_unmap_page(&pdev->dev, sw_desc->dma,
1003 sw_desc->length, DMA_TO_DEVICE);
1004
1005 if (likely(sw_desc->flags & EDMA_SW_DESC_FLAG_LAST))
1006 dev_kfree_skb_any(skb);
1007
1008 sw_desc->flags = 0;
1009}
1010
1011/* edma_tx_complete()
1012 * Used to clean tx queues and update hardware and consumer index
1013 */
1014static void edma_tx_complete(struct edma_common_info *edma_cinfo, int queue_id)
1015{
1016 struct edma_tx_desc_ring *etdr = edma_cinfo->tpd_ring[queue_id];
1017 struct edma_sw_desc *sw_desc;
1018 struct platform_device *pdev = edma_cinfo->pdev;
1019 int i;
1020
1021 u16 sw_next_to_clean = etdr->sw_next_to_clean;
1022 u16 hw_next_to_clean;
1023 u32 data = 0;
1024
1025 edma_read_reg(EDMA_REG_TPD_IDX_Q(queue_id), &data);
1026 hw_next_to_clean = (data >> EDMA_TPD_CONS_IDX_SHIFT) & EDMA_TPD_CONS_IDX_MASK;
1027
1028 /* clean the buffer here */
1029 while (sw_next_to_clean != hw_next_to_clean) {
1030 sw_desc = &etdr->sw_desc[sw_next_to_clean];
1031 edma_tx_unmap_and_free(pdev, sw_desc);
1032 sw_next_to_clean = (sw_next_to_clean + 1) & (etdr->count - 1);
1033 }
1034
1035 etdr->sw_next_to_clean = sw_next_to_clean;
1036
1037 /* update the TPD consumer index register */
1038 edma_write_reg(EDMA_REG_TX_SW_CONS_IDX_Q(queue_id), sw_next_to_clean);
1039
1040 /* Wake the queue if queue is stopped and netdev link is up */
1041 for (i = 0; i < EDMA_MAX_NETDEV_PER_QUEUE && etdr->nq[i] ; i++) {
1042 if (netif_tx_queue_stopped(etdr->nq[i])) {
1043 if ((etdr->netdev[i]) && netif_carrier_ok(etdr->netdev[i]))
1044 netif_tx_wake_queue(etdr->nq[i]);
1045 }
1046 }
1047}
1048
1049/* edma_get_tx_buffer()
1050 * Get sw_desc corresponding to the TPD
1051 */
1052static struct edma_sw_desc *edma_get_tx_buffer(struct edma_common_info *edma_cinfo,
1053 struct edma_tx_desc *tpd, int queue_id)
1054{
1055 struct edma_tx_desc_ring *etdr = edma_cinfo->tpd_ring[queue_id];
1056
1057 return &etdr->sw_desc[tpd - (struct edma_tx_desc *)etdr->hw_desc];
1058}
1059
1060/* edma_get_next_tpd()
1061 * Return a TPD descriptor for transfer
1062 */
1063static struct edma_tx_desc *edma_get_next_tpd(struct edma_common_info *edma_cinfo,
1064 int queue_id)
1065{
1066 struct edma_tx_desc_ring *etdr = edma_cinfo->tpd_ring[queue_id];
1067 u16 sw_next_to_fill = etdr->sw_next_to_fill;
1068 struct edma_tx_desc *tpd_desc =
1069 (&((struct edma_tx_desc *)(etdr->hw_desc))[sw_next_to_fill]);
1070
1071 etdr->sw_next_to_fill = (etdr->sw_next_to_fill + 1) & (etdr->count - 1);
1072
1073 return tpd_desc;
1074}
1075
1076/* edma_tpd_available()
1077 * Check number of free TPDs
1078 */
1079static inline u16 edma_tpd_available(struct edma_common_info *edma_cinfo,
1080 int queue_id)
1081{
1082 struct edma_tx_desc_ring *etdr = edma_cinfo->tpd_ring[queue_id];
1083
1084 u16 sw_next_to_fill;
1085 u16 sw_next_to_clean;
1086 u16 count = 0;
1087
1088 sw_next_to_clean = etdr->sw_next_to_clean;
1089 sw_next_to_fill = etdr->sw_next_to_fill;
1090
1091 if (likely(sw_next_to_clean <= sw_next_to_fill))
1092 count = etdr->count;
1093
1094 return count + sw_next_to_clean - sw_next_to_fill - 1;
1095}
1096
1097/* edma_tx_queue_get()
1098 * Get the starting number of the queue
1099 */
1100static inline int edma_tx_queue_get(struct edma_adapter *adapter,
1101 struct sk_buff *skb, int txq_id)
1102{
1103 /* skb->priority is used as an index to skb priority table
1104 * and based on packet priority, correspong queue is assigned.
1105 */
1106 return adapter->tx_start_offset[txq_id] + edma_skb_priority_offset(skb);
1107}
1108
1109/* edma_tx_update_hw_idx()
1110 * update the producer index for the ring transmitted
1111 */
1112static void edma_tx_update_hw_idx(struct edma_common_info *edma_cinfo,
1113 struct sk_buff *skb, int queue_id)
1114{
1115 struct edma_tx_desc_ring *etdr = edma_cinfo->tpd_ring[queue_id];
1116 u32 tpd_idx_data;
1117
1118 /* Read and update the producer index */
1119 edma_read_reg(EDMA_REG_TPD_IDX_Q(queue_id), &tpd_idx_data);
1120 tpd_idx_data &= ~EDMA_TPD_PROD_IDX_BITS;
1121 tpd_idx_data |= (etdr->sw_next_to_fill & EDMA_TPD_PROD_IDX_MASK)
1122 << EDMA_TPD_PROD_IDX_SHIFT;
1123
1124 edma_write_reg(EDMA_REG_TPD_IDX_Q(queue_id), tpd_idx_data);
1125}
1126
1127/* edma_rollback_tx()
1128 * Function to retrieve tx resources in case of error
1129 */
1130static void edma_rollback_tx(struct edma_adapter *adapter,
1131 struct edma_tx_desc *start_tpd, int queue_id)
1132{
1133 struct edma_tx_desc_ring *etdr = adapter->edma_cinfo->tpd_ring[queue_id];
1134 struct edma_sw_desc *sw_desc;
1135 struct edma_tx_desc *tpd = NULL;
1136 u16 start_index, index;
1137
1138 start_index = start_tpd - (struct edma_tx_desc *)(etdr->hw_desc);
1139
1140 index = start_index;
1141 while (index != etdr->sw_next_to_fill) {
1142 tpd = (&((struct edma_tx_desc *)(etdr->hw_desc))[index]);
1143 sw_desc = &etdr->sw_desc[index];
1144 edma_tx_unmap_and_free(adapter->pdev, sw_desc);
1145 memset(tpd, 0, sizeof(struct edma_tx_desc));
1146 if (++index == etdr->count)
1147 index = 0;
1148 }
1149 etdr->sw_next_to_fill = start_index;
1150}
1151
1152/* edma_tx_map_and_fill()
1153 * gets called from edma_xmit_frame
1154 *
1155 * This is where the dma of the buffer to be transmitted
1156 * gets mapped
1157 */
1158static int edma_tx_map_and_fill(struct edma_common_info *edma_cinfo,
1159 struct edma_adapter *adapter,
1160 struct sk_buff *skb, int queue_id,
1161 unsigned int flags_transmit,
1162 u16 from_cpu, u16 dp_bitmap,
1163 bool packet_is_rstp, int nr_frags)
1164{
1165 struct edma_sw_desc *sw_desc = NULL;
1166 struct platform_device *pdev = edma_cinfo->pdev;
1167 struct edma_tx_desc *tpd = NULL;
1168 struct edma_tx_desc *start_tpd = NULL;
1169 struct sk_buff *iter_skb;
1170 int i;
1171 u32 word1 = 0, word3 = 0, lso_word1 = 0, svlan_tag = 0;
1172 u16 buf_len, lso_desc_len = 0;
1173
1174 if (skb_is_gso(skb)) {
1175 /* TODO: What additional checks need to be performed here */
1176 if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4) {
1177 lso_word1 |= EDMA_TPD_IPV4_EN;
1178 ip_hdr(skb)->check = 0;
1179 tcp_hdr(skb)->check = ~csum_tcpudp_magic(ip_hdr(skb)->saddr,
1180 ip_hdr(skb)->daddr, 0, IPPROTO_TCP, 0);
1181 } else if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6) {
1182 lso_word1 |= EDMA_TPD_LSO_V2_EN;
1183 ipv6_hdr(skb)->payload_len = 0;
1184 tcp_hdr(skb)->check = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
1185 &ipv6_hdr(skb)->daddr, 0, IPPROTO_TCP, 0);
1186 } else
1187 return -EINVAL;
1188
1189 lso_word1 |= EDMA_TPD_LSO_EN | ((skb_shinfo(skb)->gso_size & EDMA_TPD_MSS_MASK) << EDMA_TPD_MSS_SHIFT) |
1190 (skb_transport_offset(skb) << EDMA_TPD_HDR_SHIFT);
1191 } else if (flags_transmit & EDMA_HW_CHECKSUM) {
1192 u8 css, cso;
1193 cso = skb_checksum_start_offset(skb);
1194 css = cso + skb->csum_offset;
1195
1196 word1 |= (EDMA_TPD_CUSTOM_CSUM_EN);
1197 word1 |= (cso >> 1) << EDMA_TPD_HDR_SHIFT;
1198 word1 |= ((css >> 1) << EDMA_TPD_CUSTOM_CSUM_SHIFT);
1199 }
1200
1201 if (skb->protocol == htons(ETH_P_PPP_SES))
1202 word1 |= EDMA_TPD_PPPOE_EN;
1203
1204 if (flags_transmit & EDMA_VLAN_TX_TAG_INSERT_FLAG) {
1205 switch (skb->vlan_proto) {
1206 case htons(ETH_P_8021Q):
1207 word3 |= (1 << EDMA_TX_INS_CVLAN);
1208#if (LINUX_VERSION_CODE <= KERNEL_VERSION(3, 18, 21))
1209 word3 |= vlan_tx_tag_get(skb) << EDMA_TX_CVLAN_TAG_SHIFT;
1210#else
1211 word3 |= skb_vlan_tag_get(skb) << EDMA_TX_CVLAN_TAG_SHIFT;
1212#endif
1213 break;
1214 case htons(ETH_P_8021AD):
1215 word1 |= (1 << EDMA_TX_INS_SVLAN);
1216#if (LINUX_VERSION_CODE <= KERNEL_VERSION(3, 18, 21))
1217 svlan_tag = vlan_tx_tag_get(skb) << EDMA_TX_SVLAN_TAG_SHIFT;
1218#else
1219 svlan_tag = skb_vlan_tag_get(skb) << EDMA_TX_SVLAN_TAG_SHIFT;
1220#endif
1221 break;
1222 default:
1223 dev_err(&pdev->dev, "no ctag or stag present\n");
1224 goto vlan_tag_error;
1225 }
1226 } else if (flags_transmit & EDMA_VLAN_TX_TAG_INSERT_DEFAULT_FLAG) {
1227 word3 |= (1 << EDMA_TX_INS_CVLAN);
1228 word3 |= (adapter->default_vlan_tag) << EDMA_TX_CVLAN_TAG_SHIFT;
1229 }
1230
1231 if (packet_is_rstp) {
1232 word3 |= dp_bitmap << EDMA_TPD_PORT_BITMAP_SHIFT;
1233 word3 |= from_cpu << EDMA_TPD_FROM_CPU_SHIFT;
1234 } else {
1235 word3 |= adapter->dp_bitmap << EDMA_TPD_PORT_BITMAP_SHIFT;
1236 }
1237
1238 buf_len = skb_headlen(skb);
1239
1240 if (lso_word1) {
1241 if (lso_word1 & EDMA_TPD_LSO_V2_EN) {
1242
1243 /* IPv6 LSOv2 descriptor */
1244 start_tpd = tpd = edma_get_next_tpd(edma_cinfo, queue_id);
1245 sw_desc = edma_get_tx_buffer(edma_cinfo, tpd, queue_id);
1246 sw_desc->flags |= EDMA_SW_DESC_FLAG_SKB_NONE;
1247
1248 /* LSOv2 descriptor overrides addr field to pass length */
1249 tpd->addr = cpu_to_le16(skb->len);
1250 tpd->svlan_tag = svlan_tag;
1251 tpd->word1 = word1 | lso_word1;
1252 tpd->word3 = word3;
1253 }
1254
1255 tpd = edma_get_next_tpd(edma_cinfo, queue_id);
1256 if (!start_tpd)
1257 start_tpd = tpd;
1258 sw_desc = edma_get_tx_buffer(edma_cinfo, tpd, queue_id);
1259
1260 /* The last buffer info contain the skb address,
1261 * so skb will be freed after unmap
1262 */
1263 sw_desc->length = lso_desc_len;
1264 sw_desc->flags |= EDMA_SW_DESC_FLAG_SKB_HEAD;
1265
1266 sw_desc->dma = dma_map_single(&adapter->pdev->dev,
1267 skb->data, buf_len, DMA_TO_DEVICE);
1268 if (dma_mapping_error(&pdev->dev, sw_desc->dma))
1269 goto dma_error;
1270
1271 tpd->addr = cpu_to_le32(sw_desc->dma);
1272 tpd->len = cpu_to_le16(buf_len);
1273
1274 tpd->svlan_tag = svlan_tag;
1275 tpd->word1 = word1 | lso_word1;
1276 tpd->word3 = word3;
1277
1278 /* The last buffer info contain the skb address,
1279 * so it will be freed after unmap
1280 */
1281 sw_desc->length = lso_desc_len;
1282 sw_desc->flags |= EDMA_SW_DESC_FLAG_SKB_HEAD;
1283
1284 buf_len = 0;
1285 }
1286
1287 if (likely(buf_len)) {
1288
1289 /* TODO Do not dequeue descriptor if there is a potential error */
1290 tpd = edma_get_next_tpd(edma_cinfo, queue_id);
1291
1292 if (!start_tpd)
1293 start_tpd = tpd;
1294
1295 sw_desc = edma_get_tx_buffer(edma_cinfo, tpd, queue_id);
1296
1297 /* The last buffer info contain the skb address,
1298 * so it will be free after unmap
1299 */
1300 sw_desc->length = buf_len;
1301 sw_desc->flags |= EDMA_SW_DESC_FLAG_SKB_HEAD;
1302 sw_desc->dma = dma_map_single(&adapter->pdev->dev,
1303 skb->data, buf_len, DMA_TO_DEVICE);
1304 if (dma_mapping_error(&pdev->dev, sw_desc->dma))
1305 goto dma_error;
1306
1307 tpd->addr = cpu_to_le32(sw_desc->dma);
1308 tpd->len = cpu_to_le16(buf_len);
1309
1310 tpd->svlan_tag = svlan_tag;
1311 tpd->word1 = word1 | lso_word1;
1312 tpd->word3 = word3;
1313 }
1314
1315 i = 0;
1316
1317 /* Walk through paged frags for head skb */
1318 while (nr_frags--) {
1319 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
1320 buf_len = skb_frag_size(frag);
1321 tpd = edma_get_next_tpd(edma_cinfo, queue_id);
1322 sw_desc = edma_get_tx_buffer(edma_cinfo, tpd, queue_id);
1323 sw_desc->length = buf_len;
1324 sw_desc->flags |= EDMA_SW_DESC_FLAG_SKB_FRAG;
1325
1326 sw_desc->dma = skb_frag_dma_map(&pdev->dev, frag, 0, buf_len, DMA_TO_DEVICE);
1327
1328 if (dma_mapping_error(NULL, sw_desc->dma))
1329 goto dma_error;
1330
1331 tpd->addr = cpu_to_le32(sw_desc->dma);
1332 tpd->len = cpu_to_le16(buf_len);
1333
1334 tpd->svlan_tag = svlan_tag;
1335 tpd->word1 = word1 | lso_word1;
1336 tpd->word3 = word3;
1337 i++;
1338 }
1339
1340 /* Walk through all fraglist skbs */
1341 skb_walk_frags(skb, iter_skb) {
1342 buf_len = iter_skb->len;
1343 tpd = edma_get_next_tpd(edma_cinfo, queue_id);
1344 sw_desc = edma_get_tx_buffer(edma_cinfo, tpd, queue_id);
1345 sw_desc->length = buf_len;
1346 sw_desc->dma = dma_map_single(&adapter->pdev->dev,
1347 iter_skb->data, buf_len, DMA_TO_DEVICE);
1348
1349 if (dma_mapping_error(NULL, sw_desc->dma))
1350 goto dma_error;
1351
1352 tpd->addr = cpu_to_le32(sw_desc->dma);
1353 tpd->len = cpu_to_le16(buf_len);
1354 tpd->svlan_tag = svlan_tag;
1355 tpd->word1 = word1 | lso_word1;
1356 tpd->word3 = word3;
1357 sw_desc->flags |= EDMA_SW_DESC_FLAG_SKB_FRAGLIST;
1358
1359 i = 0;
1360
1361 nr_frags = skb_shinfo(iter_skb)->nr_frags;
1362
1363 /* Walk through paged frags for this fraglist skb */
1364 while (nr_frags--) {
1365 skb_frag_t *frag = &skb_shinfo(iter_skb)->frags[i];
1366 buf_len = skb_frag_size(frag);
1367 tpd = edma_get_next_tpd(edma_cinfo, queue_id);
1368 sw_desc = edma_get_tx_buffer(edma_cinfo, tpd, queue_id);
1369 sw_desc->length = buf_len;
1370 sw_desc->flags |= EDMA_SW_DESC_FLAG_SKB_FRAG;
1371
1372 sw_desc->dma = skb_frag_dma_map(&pdev->dev, frag,
1373 0, buf_len, DMA_TO_DEVICE);
1374 if (dma_mapping_error(NULL, sw_desc->dma))
1375 goto dma_error;
1376
1377 tpd->addr = cpu_to_le32(sw_desc->dma);
1378 tpd->len = cpu_to_le16(buf_len);
1379 tpd->svlan_tag = svlan_tag;
1380 tpd->word1 = word1 | lso_word1;
1381 tpd->word3 = word3;
1382 i++;
1383 }
1384 }
1385
Rakesh Nair888af952017-06-30 18:41:58 +05301386 /* If sysctl support for per-precedence stats are enabled */
1387 if (edma_per_prec_stats_enable) {
1388 struct iphdr *ip_hdr = NULL;
1389 struct ipv6hdr *ip6_hdr = NULL;
1390 uint8_t precedence = 0xff;
1391
1392 if (likely(htons(ETH_P_IP) == skb->protocol)) {
1393 ip_hdr = (struct iphdr *)skb_network_header(skb);
1394 if (ip_hdr && ((ip_hdr->protocol == IPPROTO_UDP) || (ip_hdr->protocol == IPPROTO_TCP)))
1395 precedence = ip_hdr->tos >> EDMA_DSCP_PREC_SHIFT;
1396
1397 /* Increment per-precedence counters for tx packets
1398 * and set the precedence in the TPD.
1399 */
1400 edma_cinfo->edma_ethstats.tx_prec[precedence]++;
1401 edma_cinfo->edma_ethstats.tx_ac[edma_dscp2ac_tbl[precedence]]++;
1402 tpd->word3 |= precedence << EDMA_TPD_PRIO_SHIFT;
1403 } else if (htons(ETH_P_IPV6) == skb->protocol) {
1404 ip6_hdr = (struct ipv6hdr *)skb_network_header(skb);
1405 if (ip6_hdr && ((ip6_hdr->nexthdr == IPPROTO_UDP) || (ip6_hdr->nexthdr == IPPROTO_TCP)))
1406 precedence = ip6_hdr->priority >> EDMA_DSCP6_PREC_SHIFT;
1407
1408 /* Increment per-precedence counters for tx packets
1409 * and set the precedence in the TPD for v6 packets.
1410 */
1411 edma_cinfo->edma_ethstats.tx_prec[precedence]++;
1412 edma_cinfo->edma_ethstats.tx_ac[edma_dscp2ac_tbl[precedence]]++;
1413 tpd->word3 |= precedence << EDMA_TPD_PRIO_SHIFT;
1414 }
1415 }
1416
Rakesh Nair9bcf2602017-01-06 16:02:16 +05301417 /* If tpd or sw_desc is still unitiialized then we need to return */
1418 if ((!tpd) || (!sw_desc))
1419 return -EINVAL;
1420
1421 tpd->word1 |= 1 << EDMA_TPD_EOP_SHIFT;
1422
1423 sw_desc->skb = skb;
1424 sw_desc->flags |= EDMA_SW_DESC_FLAG_LAST;
1425
1426 return 0;
1427
1428dma_error:
1429 edma_rollback_tx(adapter, start_tpd, queue_id);
1430 dev_err(&pdev->dev, "TX DMA map failed\n");
1431vlan_tag_error:
1432 return -ENOMEM;
1433}
1434
1435/* edma_check_link()
1436 * check Link status
1437 */
1438static int edma_check_link(struct edma_adapter *adapter)
1439{
1440 struct phy_device *phydev = adapter->phydev;
1441
1442 if (!(adapter->poll_required))
1443 return __EDMA_LINKUP;
1444
1445 if (phydev->link)
1446 return __EDMA_LINKUP;
1447
1448 return __EDMA_LINKDOWN;
1449}
1450
1451/* edma_adjust_link()
1452 * check for edma link status
1453 */
1454void edma_adjust_link(struct net_device *netdev)
1455{
1456 int status;
1457 struct edma_adapter *adapter = netdev_priv(netdev);
1458 struct phy_device *phydev = adapter->phydev;
1459
1460 if (!test_bit(__EDMA_UP, &adapter->state_flags))
1461 return;
1462
1463 status = edma_check_link(adapter);
1464
1465 if (status == __EDMA_LINKUP && adapter->link_state == __EDMA_LINKDOWN) {
1466 dev_info(&adapter->pdev->dev, "%s: GMAC Link is up with phy_speed=%d\n", netdev->name, phydev->speed);
1467 adapter->link_state = __EDMA_LINKUP;
1468 netif_carrier_on(netdev);
1469 if (netif_running(netdev))
1470 netif_tx_wake_all_queues(netdev);
1471 } else if (status == __EDMA_LINKDOWN && adapter->link_state == __EDMA_LINKUP) {
1472 dev_info(&adapter->pdev->dev, "%s: GMAC Link is down\n", netdev->name);
1473 adapter->link_state = __EDMA_LINKDOWN;
1474 netif_carrier_off(netdev);
1475 netif_tx_stop_all_queues(netdev);
1476 }
1477}
1478
Bhaskar Valabojue429bab2017-03-15 09:01:23 +05301479/* edma_get_stats64()
Rakesh Nair9bcf2602017-01-06 16:02:16 +05301480 * Statistics api used to retreive the tx/rx statistics
1481 */
Bhaskar Valabojue429bab2017-03-15 09:01:23 +05301482struct rtnl_link_stats64 *edma_get_stats64(struct net_device *netdev,
1483 struct rtnl_link_stats64 *stats)
Rakesh Nair9bcf2602017-01-06 16:02:16 +05301484{
1485 struct edma_adapter *adapter = netdev_priv(netdev);
1486
Bhaskar Valabojue429bab2017-03-15 09:01:23 +05301487 memcpy(stats, &adapter->stats, sizeof(*stats));
1488
1489 return stats;
Rakesh Nair9bcf2602017-01-06 16:02:16 +05301490}
1491
1492/* edma_xmit()
1493 * Main api to be called by the core for packet transmission
1494 */
1495netdev_tx_t edma_xmit(struct sk_buff *skb,
1496 struct net_device *net_dev)
1497{
1498 struct edma_adapter *adapter = netdev_priv(net_dev);
1499 struct edma_common_info *edma_cinfo = adapter->edma_cinfo;
1500 struct edma_tx_desc_ring *etdr;
1501 u16 from_cpu = 0, dp_bitmap = 0, txq_id;
1502 int ret, nr_frags_first = 0, num_tpds_needed = 1, queue_id = 0;
1503 unsigned int flags_transmit = 0;
1504 bool packet_is_rstp = false;
1505 struct netdev_queue *nq = NULL;
1506
1507 if (skb_shinfo(skb)->nr_frags) {
1508 nr_frags_first = skb_shinfo(skb)->nr_frags;
1509
1510 /* It is unlikely below check hits, BUG_ON */
1511 BUG_ON(nr_frags_first > MAX_SKB_FRAGS);
1512
1513 num_tpds_needed += nr_frags_first;
1514 }
1515
1516 if (skb_has_frag_list(skb)) {
1517 struct sk_buff *iter_skb;
1518
1519 /* Walk through fraglist skbs making a note of nr_frags */
1520 skb_walk_frags(skb, iter_skb) {
1521 unsigned char nr_frags = skb_shinfo(iter_skb)->nr_frags;
1522
1523 /* It is unlikely below check hits, BUG_ON */
1524 BUG_ON(nr_frags > MAX_SKB_FRAGS);
1525
1526 /* One TPD for skb->data and more for nr_frags */
1527 num_tpds_needed += (1 + nr_frags);
1528 }
1529 }
1530
1531 if (edma_stp_rstp) {
1532 u16 ath_hdr, ath_eth_type;
1533 u8 mac_addr[EDMA_ETH_HDR_LEN];
1534 ath_eth_type = ntohs(*(uint16_t *)&skb->data[12]);
1535 if (ath_eth_type == edma_ath_eth_type) {
1536 packet_is_rstp = true;
1537 ath_hdr = htons(*(uint16_t *)&skb->data[14]);
1538 dp_bitmap = ath_hdr & EDMA_TX_ATH_HDR_PORT_BITMAP_MASK;
1539 from_cpu = (ath_hdr & EDMA_TX_ATH_HDR_FROM_CPU_MASK) >> EDMA_TX_ATH_HDR_FROM_CPU_SHIFT;
1540 memcpy(mac_addr, skb->data, EDMA_ETH_HDR_LEN);
1541
1542 skb_pull(skb, 4);
1543
1544 memcpy(skb->data, mac_addr, EDMA_ETH_HDR_LEN);
1545 }
1546 }
1547
1548 /* this will be one of the 4 TX queues exposed to linux kernel */
1549 txq_id = skb_get_queue_mapping(skb);
1550 queue_id = edma_tx_queue_get(adapter, skb, txq_id);
1551 etdr = edma_cinfo->tpd_ring[queue_id];
1552 nq = netdev_get_tx_queue(net_dev, txq_id);
1553
1554 local_bh_disable();
1555 /* Tx is not handled in bottom half context. Hence, we need to protect
1556 * Tx from tasks and bottom half
1557 */
1558
1559 if (num_tpds_needed > edma_tpd_available(edma_cinfo, queue_id)) {
1560 /* not enough descriptor, just stop queue */
1561 netif_tx_stop_queue(nq);
1562 local_bh_enable();
1563 dev_dbg(&net_dev->dev, "Not enough descriptors available");
1564 edma_cinfo->edma_ethstats.tx_desc_error++;
1565 return NETDEV_TX_BUSY;
1566 }
1567
1568 /* Check and mark VLAN tag offload */
1569#if (LINUX_VERSION_CODE <= KERNEL_VERSION(3, 18, 21))
1570 if (vlan_tx_tag_present(skb))
1571#else
1572 if (skb_vlan_tag_present(skb))
1573#endif
1574 flags_transmit |= EDMA_VLAN_TX_TAG_INSERT_FLAG;
1575 else if (adapter->default_vlan_tag)
1576 flags_transmit |= EDMA_VLAN_TX_TAG_INSERT_DEFAULT_FLAG;
1577
1578 /* Check and mark checksum offload */
1579 if (likely(skb->ip_summed == CHECKSUM_PARTIAL))
1580 flags_transmit |= EDMA_HW_CHECKSUM;
1581
1582 /* Map and fill descriptor for Tx */
1583 ret = edma_tx_map_and_fill(edma_cinfo, adapter, skb, queue_id,
1584 flags_transmit, from_cpu, dp_bitmap,
1585 packet_is_rstp, nr_frags_first);
1586 if (ret) {
1587 dev_kfree_skb_any(skb);
1588 adapter->stats.tx_errors++;
1589 goto netdev_okay;
1590 }
1591
1592 /* Update SW producer index */
1593 edma_tx_update_hw_idx(edma_cinfo, skb, queue_id);
1594
1595 /* update tx statistics */
1596 adapter->stats.tx_packets++;
1597 adapter->stats.tx_bytes += skb->len;
1598
1599netdev_okay:
1600 local_bh_enable();
1601 return NETDEV_TX_OK;
1602}
1603
1604/*
1605 * edma_flow_may_expire()
1606 * Timer function called periodically to delete the node
1607 */
1608void edma_flow_may_expire(unsigned long data)
1609{
1610 struct edma_adapter *adapter = (struct edma_adapter *)data;
1611 int j;
1612
1613 spin_lock_bh(&adapter->rfs.rfs_ftab_lock);
1614 for (j = 0; j < EDMA_RFS_EXPIRE_COUNT_PER_CALL; j++) {
1615 struct hlist_head *hhead;
1616 struct hlist_node *tmp;
1617 struct edma_rfs_filter_node *n;
1618 bool res;
1619
1620 hhead = &adapter->rfs.hlist_head[adapter->rfs.hashtoclean++];
1621 hlist_for_each_entry_safe(n, tmp, hhead, node) {
1622 res = rps_may_expire_flow(adapter->netdev, n->rq_id,
1623 n->flow_id, n->filter_id);
1624 if (res) {
1625 res = edma_delete_rfs_filter(adapter, n);
1626 if (res < 0)
1627 dev_dbg(&adapter->netdev->dev,
1628 "RFS entry %d not allowed to be flushed by Switch",
1629 n->flow_id);
1630 else {
1631 hlist_del(&n->node);
1632 kfree(n);
1633 adapter->rfs.filter_available++;
1634 }
1635 }
1636 }
1637 }
1638
1639 adapter->rfs.hashtoclean = adapter->rfs.hashtoclean & (EDMA_RFS_FLOW_ENTRIES - 1);
1640 spin_unlock_bh(&adapter->rfs.rfs_ftab_lock);
1641 mod_timer(&adapter->rfs.expire_rfs, jiffies + HZ/4);
1642}
1643
1644/* edma_rx_flow_steer()
1645 * Called by core to to steer the flow to CPU
1646 */
1647int edma_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb,
1648 u16 rxq, u32 flow_id)
1649{
1650 struct flow_keys keys;
1651 struct edma_rfs_filter_node *filter_node;
1652 struct edma_adapter *adapter = netdev_priv(dev);
1653 u16 hash_tblid;
1654 int res;
1655
1656 if (skb->protocol == htons(ETH_P_IPV6)) {
1657 res = -EPROTONOSUPPORT;
1658 goto no_protocol_err;
1659 }
1660
1661 /* Dissect flow parameters
1662 * We only support IPv4 + TCP/UDP
1663 */
1664#if (LINUX_VERSION_CODE <= KERNEL_VERSION(3, 18, 21))
1665 res = skb_flow_dissect(skb, &keys);
1666 if (!((keys.ip_proto == IPPROTO_TCP) || (keys.ip_proto == IPPROTO_UDP))) {
1667#else
1668 res = skb_flow_dissect_flow_keys(skb, &keys, 0);
1669 if (!((keys.basic.ip_proto == IPPROTO_TCP) || (keys.basic.ip_proto == IPPROTO_UDP))) {
1670#endif
1671 res = -EPROTONOSUPPORT;
1672 goto no_protocol_err;
1673 }
1674
1675 /* Check if table entry exists */
1676 hash_tblid = skb_get_hash_raw(skb) & EDMA_RFS_FLOW_ENTRIES_MASK;
1677
1678 spin_lock_bh(&adapter->rfs.rfs_ftab_lock);
1679 filter_node = edma_rfs_key_search(&adapter->rfs.hlist_head[hash_tblid], &keys);
1680
1681 if (filter_node) {
1682 if (rxq == filter_node->rq_id) {
1683 res = -EEXIST;
1684 goto out;
1685 } else {
1686 res = edma_delete_rfs_filter(adapter, filter_node);
1687 if (res < 0)
1688 dev_warn(&adapter->netdev->dev,
1689 "Cannot steer flow %d to different queue",
1690 filter_node->flow_id);
1691 else {
1692 adapter->rfs.filter_available++;
1693 res = edma_add_rfs_filter(adapter, &keys, rxq, filter_node);
1694 if (res < 0) {
1695 dev_warn(&adapter->netdev->dev,
1696 "Cannot steer flow %d to different queue",
1697 filter_node->flow_id);
1698 } else {
1699 adapter->rfs.filter_available--;
1700 filter_node->rq_id = rxq;
1701 filter_node->filter_id = res;
1702 }
1703 }
1704 }
1705 } else {
1706 if (adapter->rfs.filter_available == 0) {
1707 res = -EBUSY;
1708 goto out;
1709 }
1710
1711 filter_node = kmalloc(sizeof(*filter_node), GFP_ATOMIC);
1712 if (!filter_node) {
1713 res = -ENOMEM;
1714 goto out;
1715 }
1716
1717 res = edma_add_rfs_filter(adapter, &keys, rxq, filter_node);
1718 if (res < 0) {
1719 kfree(filter_node);
1720 goto out;
1721 }
1722
1723 adapter->rfs.filter_available--;
1724 filter_node->rq_id = rxq;
1725 filter_node->filter_id = res;
1726 filter_node->flow_id = flow_id;
1727 filter_node->keys = keys;
1728 INIT_HLIST_NODE(&filter_node->node);
1729 hlist_add_head(&filter_node->node, &adapter->rfs.hlist_head[hash_tblid]);
1730 }
1731
1732out:
1733 spin_unlock_bh(&adapter->rfs.rfs_ftab_lock);
1734no_protocol_err:
1735 return res;
1736}
1737
1738#ifdef CONFIG_RFS_ACCEL
1739/* edma_register_rfs_filter()
1740 * Add RFS filter callback
1741 */
1742int edma_register_rfs_filter(struct net_device *netdev,
1743 set_rfs_filter_callback_t set_filter)
1744{
1745 struct edma_adapter *adapter = netdev_priv(netdev);
1746
1747 spin_lock_bh(&adapter->rfs.rfs_ftab_lock);
1748
1749 if (adapter->set_rfs_rule) {
1750 spin_unlock_bh(&adapter->rfs.rfs_ftab_lock);
1751 return -1;
1752 }
1753
1754 adapter->set_rfs_rule = set_filter;
1755 spin_unlock_bh(&adapter->rfs.rfs_ftab_lock);
1756
1757 return 0;
1758}
1759#endif
1760
1761/* edma_select_xps_queue()
1762 * Called by Linux TX stack to populate Linux TX queue
1763 */
1764u16 edma_select_xps_queue(struct net_device *dev, struct sk_buff *skb,
1765 void *accel_priv, select_queue_fallback_t fallback)
1766{
1767#if (LINUX_VERSION_CODE <= KERNEL_VERSION(3, 18, 21))
1768 return smp_processor_id();
1769#else
1770 int cpu = get_cpu();
1771 put_cpu();
1772
1773 return cpu;
1774#endif
1775}
1776
1777/* edma_alloc_tx_rings()
1778 * Allocate rx rings
1779 */
1780int edma_alloc_tx_rings(struct edma_common_info *edma_cinfo)
1781{
1782 struct platform_device *pdev = edma_cinfo->pdev;
1783 int i, err = 0;
1784
1785 for (i = 0; i < edma_cinfo->num_tx_queues; i++) {
1786 err = edma_alloc_tx_ring(edma_cinfo, edma_cinfo->tpd_ring[i]);
1787 if (err) {
1788 dev_err(&pdev->dev, "Tx Queue alloc %u failed\n", i);
1789 return err;
1790 }
1791 }
1792
1793 return 0;
1794}
1795
1796/* edma_free_tx_rings()
1797 * Free tx rings
1798 */
1799void edma_free_tx_rings(struct edma_common_info *edma_cinfo)
1800{
1801 int i;
1802
1803 for (i = 0; i < edma_cinfo->num_tx_queues; i++)
1804 edma_free_tx_ring(edma_cinfo, edma_cinfo->tpd_ring[i]);
1805}
1806
1807/* edma_free_tx_resources()
1808 * Free buffers associated with tx rings
1809 */
1810void edma_free_tx_resources(struct edma_common_info *edma_cinfo)
1811{
1812 struct edma_tx_desc_ring *etdr;
1813 struct edma_sw_desc *sw_desc;
1814 struct platform_device *pdev = edma_cinfo->pdev;
1815 int i, j;
1816
1817 for (i = 0; i < edma_cinfo->num_tx_queues; i++) {
1818 etdr = edma_cinfo->tpd_ring[i];
1819 for (j = 0; j < EDMA_TX_RING_SIZE; j++) {
1820 sw_desc = &etdr->sw_desc[j];
1821 if (sw_desc->flags & (EDMA_SW_DESC_FLAG_SKB_HEAD |
1822 EDMA_SW_DESC_FLAG_SKB_FRAG | EDMA_SW_DESC_FLAG_SKB_FRAGLIST))
1823 edma_tx_unmap_and_free(pdev, sw_desc);
1824 }
1825 }
1826}
1827
1828/* edma_alloc_rx_rings()
1829 * Allocate rx rings
1830 */
1831int edma_alloc_rx_rings(struct edma_common_info *edma_cinfo)
1832{
1833 struct platform_device *pdev = edma_cinfo->pdev;
1834 int i, j, err = 0;
1835
1836 for (i = 0, j = 0; i < edma_cinfo->num_rx_queues; i++) {
1837 err = edma_alloc_rx_ring(edma_cinfo, edma_cinfo->rfd_ring[j]);
1838 if (err) {
1839 dev_err(&pdev->dev, "Rx Queue alloc%u failed\n", i);
1840 return err;
1841 }
1842 j += ((edma_cinfo->num_rx_queues == 4) ? 2 : 1);
1843 }
1844
1845 return 0;
1846}
1847
1848/* edma_free_rx_rings()
1849 * free rx rings
1850 */
1851void edma_free_rx_rings(struct edma_common_info *edma_cinfo)
1852{
1853 int i, j;
1854
1855 for (i = 0, j = 0; i < edma_cinfo->num_rx_queues; i++) {
1856 edma_free_rx_ring(edma_cinfo, edma_cinfo->rfd_ring[j]);
1857 j += ((edma_cinfo->num_rx_queues == 4) ? 2 : 1);
1858 }
1859}
1860
1861/* edma_free_queues()
1862 * Free the queues allocaated
1863 */
1864void edma_free_queues(struct edma_common_info *edma_cinfo)
1865{
1866 int i , j;
1867
1868 for (i = 0; i < edma_cinfo->num_tx_queues; i++) {
1869 if (edma_cinfo->tpd_ring[i])
1870 kfree(edma_cinfo->tpd_ring[i]);
1871 edma_cinfo->tpd_ring[i] = NULL;
1872 }
1873
1874 for (i = 0, j = 0; i < edma_cinfo->num_rx_queues; i++) {
1875 if (edma_cinfo->rfd_ring[j])
1876 kfree(edma_cinfo->rfd_ring[j]);
1877 edma_cinfo->rfd_ring[j] = NULL;
1878 j += ((edma_cinfo->num_rx_queues == 4) ? 2 : 1);
1879 }
1880
1881 edma_cinfo->num_rx_queues = 0;
1882 edma_cinfo->num_tx_queues = 0;
1883
1884 return;
1885}
1886
1887/* edma_free_rx_resources()
1888 * Free buffers associated with tx rings
1889 */
1890void edma_free_rx_resources(struct edma_common_info *edma_cinfo)
1891{
1892 struct edma_rfd_desc_ring *erdr;
1893 struct platform_device *pdev = edma_cinfo->pdev;
1894 int i, j, k;
1895
1896 for (i = 0, k = 0; i < edma_cinfo->num_rx_queues; i++) {
1897 erdr = edma_cinfo->rfd_ring[k];
1898 for (j = 0; j < EDMA_RX_RING_SIZE; j++) {
1899 /* unmap all descriptors while cleaning */
1900 edma_clean_rfd(pdev, erdr, j, 1);
1901 }
1902 k += ((edma_cinfo->num_rx_queues == 4) ? 2 : 1);
1903
1904 }
1905}
1906
1907/* edma_alloc_queues_tx()
1908 * Allocate memory for all rings
1909 */
1910int edma_alloc_queues_tx(struct edma_common_info *edma_cinfo)
1911{
1912 int i;
1913
1914 for (i = 0; i < edma_cinfo->num_tx_queues; i++) {
1915 struct edma_tx_desc_ring *etdr;
1916 etdr = kzalloc(sizeof(struct edma_tx_desc_ring), GFP_KERNEL);
1917 if (!etdr)
1918 goto err;
1919 etdr->count = edma_cinfo->tx_ring_count;
1920 edma_cinfo->tpd_ring[i] = etdr;
1921 }
1922
1923 return 0;
1924err:
1925 edma_free_queues(edma_cinfo);
1926 return -1;
1927}
1928
1929/* edma_alloc_queues_rx()
1930 * Allocate memory for all rings
1931 */
1932int edma_alloc_queues_rx(struct edma_common_info *edma_cinfo)
1933{
1934 int i, j;
1935
1936 for (i = 0, j = 0; i < edma_cinfo->num_rx_queues; i++) {
1937 struct edma_rfd_desc_ring *rfd_ring;
1938 rfd_ring = kzalloc(sizeof(struct edma_rfd_desc_ring),
1939 GFP_KERNEL);
1940 if (!rfd_ring)
1941 goto err;
1942 rfd_ring->count = edma_cinfo->rx_ring_count;
1943 edma_cinfo->rfd_ring[j] = rfd_ring;
1944 j += ((edma_cinfo->num_rx_queues == 4) ? 2 : 1);
1945 }
1946 return 0;
1947err:
1948 edma_free_queues(edma_cinfo);
1949 return -1;
1950}
1951
1952/* edma_clear_irq_status()
1953 * Clear interrupt status
1954 */
1955void edma_clear_irq_status(void)
1956{
1957 edma_write_reg(EDMA_REG_RX_ISR, 0xff);
1958 edma_write_reg(EDMA_REG_TX_ISR, 0xffff);
1959 edma_write_reg(EDMA_REG_MISC_ISR, 0x1fff);
1960 edma_write_reg(EDMA_REG_WOL_ISR, 0x1);
1961};
1962
1963/* edma_configure()
1964 * Configure skb, edma interrupts and control register.
1965 */
1966int edma_configure(struct edma_common_info *edma_cinfo)
1967{
1968 struct edma_hw *hw = &edma_cinfo->hw;
1969 u32 intr_modrt_data;
1970 u32 intr_ctrl_data = 0;
1971 int i, j, ret_count;
1972
1973 edma_read_reg(EDMA_REG_INTR_CTRL, &intr_ctrl_data);
1974 intr_ctrl_data &= ~(1 << EDMA_INTR_SW_IDX_W_TYP_SHIFT);
1975 intr_ctrl_data |= hw->intr_sw_idx_w << EDMA_INTR_SW_IDX_W_TYP_SHIFT;
1976 edma_write_reg(EDMA_REG_INTR_CTRL, intr_ctrl_data);
1977
1978 edma_clear_irq_status();
1979
1980 /* Clear any WOL status */
1981 edma_write_reg(EDMA_REG_WOL_CTRL, 0);
1982 intr_modrt_data = (EDMA_TX_IMT << EDMA_IRQ_MODRT_TX_TIMER_SHIFT);
1983 intr_modrt_data |= (EDMA_RX_IMT << EDMA_IRQ_MODRT_RX_TIMER_SHIFT);
1984 edma_write_reg(EDMA_REG_IRQ_MODRT_TIMER_INIT, intr_modrt_data);
1985 edma_configure_tx(edma_cinfo);
1986 edma_configure_rx(edma_cinfo);
1987
1988 /* Allocate the RX buffer */
1989 for (i = 0, j = 0; i < edma_cinfo->num_rx_queues; i++) {
1990 struct edma_rfd_desc_ring *ring = edma_cinfo->rfd_ring[j];
1991 ret_count = edma_alloc_rx_buf(edma_cinfo, ring, ring->count, j);
1992 if (ret_count)
1993 dev_dbg(&edma_cinfo->pdev->dev, "not all rx buffers allocated\n");
Rakesh Nair03b586c2017-04-03 18:28:58 +05301994 ring->pending_fill = ret_count;
Rakesh Nair9bcf2602017-01-06 16:02:16 +05301995 j += ((edma_cinfo->num_rx_queues == 4) ? 2 : 1);
1996 }
1997
1998 /* Configure descriptor Ring */
1999 edma_init_desc(edma_cinfo);
2000 return 0;
2001}
2002
2003/* edma_irq_enable()
2004 * Enable default interrupt generation settings
2005 */
2006void edma_irq_enable(struct edma_common_info *edma_cinfo)
2007{
2008 struct edma_hw *hw = &edma_cinfo->hw;
2009 int i, j;
2010
2011 edma_write_reg(EDMA_REG_RX_ISR, 0xff);
2012 for (i = 0, j = 0; i < edma_cinfo->num_rx_queues; i++) {
2013 edma_write_reg(EDMA_REG_RX_INT_MASK_Q(j), hw->rx_intr_mask);
2014 j += ((edma_cinfo->num_rx_queues == 4) ? 2 : 1);
2015 }
2016 edma_write_reg(EDMA_REG_TX_ISR, 0xffff);
2017 for (i = 0; i < edma_cinfo->num_tx_queues; i++)
2018 edma_write_reg(EDMA_REG_TX_INT_MASK_Q(i), hw->tx_intr_mask);
2019}
2020
2021/* edma_irq_disable()
2022 * Disable Interrupt
2023 */
2024void edma_irq_disable(struct edma_common_info *edma_cinfo)
2025{
2026 int i;
2027
2028 for (i = 0; i < EDMA_MAX_RECEIVE_QUEUE; i++)
2029 edma_write_reg(EDMA_REG_RX_INT_MASK_Q(i), 0x0);
2030
2031 for (i = 0; i < EDMA_MAX_TRANSMIT_QUEUE; i++)
2032 edma_write_reg(EDMA_REG_TX_INT_MASK_Q(i), 0x0);
2033 edma_write_reg(EDMA_REG_MISC_IMR, 0);
2034 edma_write_reg(EDMA_REG_WOL_IMR, 0);
2035}
2036
2037/* edma_free_irqs()
2038 * Free All IRQs
2039 */
2040void edma_free_irqs(struct edma_adapter *adapter)
2041{
2042 struct edma_common_info *edma_cinfo = adapter->edma_cinfo;
2043 int i, j;
2044 int k = ((edma_cinfo->num_rx_queues == 4) ? 1 : 2);
2045
2046 for (i = 0; i < CONFIG_NR_CPUS; i++) {
2047 for (j = edma_cinfo->edma_percpu_info[i].tx_start; j < (edma_cinfo->edma_percpu_info[i].tx_start + 4); j++)
2048 free_irq(edma_cinfo->tx_irq[j], &edma_cinfo->edma_percpu_info[i]);
2049
2050 for (j = edma_cinfo->edma_percpu_info[i].rx_start; j < (edma_cinfo->edma_percpu_info[i].rx_start + k); j++)
2051 free_irq(edma_cinfo->rx_irq[j], &edma_cinfo->edma_percpu_info[i]);
2052 }
2053}
2054
2055/* edma_enable_rx_ctrl()
2056 * Enable RX queue control
2057 */
2058void edma_enable_rx_ctrl(struct edma_hw *hw)
2059{
2060 u32 data;
2061
2062 edma_read_reg(EDMA_REG_RXQ_CTRL, &data);
2063 data |= EDMA_RXQ_CTRL_EN;
2064 edma_write_reg(EDMA_REG_RXQ_CTRL, data);
2065}
2066
2067
2068/* edma_enable_tx_ctrl()
2069 * Enable TX queue control
2070 */
2071void edma_enable_tx_ctrl(struct edma_hw *hw)
2072{
2073 u32 data;
2074
2075 edma_read_reg(EDMA_REG_TXQ_CTRL, &data);
2076 data |= EDMA_TXQ_CTRL_TXQ_EN;
2077 edma_write_reg(EDMA_REG_TXQ_CTRL, data);
2078}
2079
2080/* edma_stop_rx_tx()
2081 * Disable RX/TQ Queue control
2082 */
2083void edma_stop_rx_tx(struct edma_hw *hw)
2084{
2085 u32 data;
2086
2087 edma_read_reg(EDMA_REG_RXQ_CTRL, &data);
2088 data &= ~EDMA_RXQ_CTRL_EN;
2089 edma_write_reg(EDMA_REG_RXQ_CTRL, data);
2090 edma_read_reg(EDMA_REG_TXQ_CTRL, &data);
2091 data &= ~EDMA_TXQ_CTRL_TXQ_EN;
2092 edma_write_reg(EDMA_REG_TXQ_CTRL, data);
2093}
2094
2095/* edma_reset()
2096 * Reset the EDMA
2097 */
2098int edma_reset(struct edma_common_info *edma_cinfo)
2099{
2100 struct edma_hw *hw = &edma_cinfo->hw;
2101
2102 edma_irq_disable(edma_cinfo);
2103
2104 edma_clear_irq_status();
2105
2106 edma_stop_rx_tx(hw);
2107
2108 return 0;
2109}
2110
2111/* edma_fill_netdev()
2112 * Fill netdev for each etdr
2113 */
2114int edma_fill_netdev(struct edma_common_info *edma_cinfo, int queue_id,
2115 int dev, int txq_id)
2116{
2117 struct edma_tx_desc_ring *etdr;
2118 int i = 0;
2119
2120 etdr = edma_cinfo->tpd_ring[queue_id];
2121
2122 while (etdr->netdev[i])
2123 i++;
2124
2125 if (i >= EDMA_MAX_NETDEV_PER_QUEUE)
2126 return -1;
2127
2128 /* Populate the netdev associated with the tpd ring */
2129 etdr->netdev[i] = edma_netdev[dev];
2130 etdr->nq[i] = netdev_get_tx_queue(edma_netdev[dev], txq_id);
2131
2132 return 0;
2133}
2134
2135/* edma_change_mtu()
2136 * change the MTU of the NIC.
2137 */
2138int edma_change_mtu(struct net_device *netdev, int new_mtu)
2139{
2140 struct edma_adapter *adapter = netdev_priv(netdev);
2141 struct edma_common_info *edma_cinfo = adapter->edma_cinfo;
2142 int old_mtu = netdev->mtu;
2143 int max_frame_size = new_mtu + ETH_HLEN + ETH_FCS_LEN + (2 * VLAN_HLEN);
2144
2145 if ((max_frame_size < ETH_ZLEN + ETH_FCS_LEN) ||
2146 (max_frame_size > EDMA_MAX_JUMBO_FRAME_SIZE)) {
2147 dev_err(&edma_cinfo->pdev->dev, "MTU setting not correct\n");
2148 return -EINVAL;
2149 }
2150
2151 /* set MTU */
2152 if (old_mtu != new_mtu) {
2153 netdev->mtu = new_mtu;
2154 netdev_update_features(netdev);
2155 }
2156
2157 return 0;
2158}
2159
2160/* edma_set_mac()
2161 * Change the Ethernet Address of the NIC
2162 */
2163int edma_set_mac_addr(struct net_device *netdev, void *p)
2164{
2165 struct sockaddr *addr = p;
2166
2167 if (!is_valid_ether_addr(addr->sa_data))
2168 return -EINVAL;
2169
2170 if (netif_running(netdev))
2171 return -EBUSY;
2172
2173 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
2174 return 0;
2175}
2176
2177/* edma_set_stp_rstp()
2178 * set stp/rstp
2179 */
2180void edma_set_stp_rstp(bool rstp)
2181{
2182 edma_stp_rstp = rstp;
2183}
2184
2185/* edma_assign_ath_hdr_type()
2186 * assign atheros header eth type
2187 */
2188void edma_assign_ath_hdr_type(int eth_type)
2189{
2190 edma_ath_eth_type = eth_type & EDMA_ETH_TYPE_MASK;
2191}
2192
2193/* edma_get_default_vlan_tag()
2194 * Used by other modules to get the default vlan tag
2195 */
2196int edma_get_default_vlan_tag(struct net_device *netdev)
2197{
2198 struct edma_adapter *adapter = netdev_priv(netdev);
2199
2200 if (adapter->default_vlan_tag)
2201 return adapter->default_vlan_tag;
2202
2203 return 0;
2204}
2205
2206/* edma_open()
2207 * gets called when netdevice is up, start the queue.
2208 */
2209int edma_open(struct net_device *netdev)
2210{
2211 struct edma_adapter *adapter = netdev_priv(netdev);
2212 struct platform_device *pdev = adapter->edma_cinfo->pdev;
2213
2214 netif_tx_start_all_queues(netdev);
2215 edma_initialise_rfs_flow_table(adapter);
2216 set_bit(__EDMA_UP, &adapter->state_flags);
2217
2218 /* if Link polling is enabled, in our case enabled for WAN, then
2219 * do a phy start, else always set link as UP
2220 */
Rakesh Naired29f6b2017-04-04 15:48:08 +05302221 mutex_lock(&adapter->poll_mutex);
Rakesh Nair9bcf2602017-01-06 16:02:16 +05302222 if (adapter->poll_required) {
2223 if (!IS_ERR(adapter->phydev)) {
2224 phy_start(adapter->phydev);
2225 phy_start_aneg(adapter->phydev);
2226 adapter->link_state = __EDMA_LINKDOWN;
2227 } else {
2228 dev_dbg(&pdev->dev, "Invalid PHY device for a link polled interface\n");
2229 }
2230 } else {
2231 adapter->link_state = __EDMA_LINKUP;
2232 netif_carrier_on(netdev);
2233 }
Rakesh Naired29f6b2017-04-04 15:48:08 +05302234 mutex_unlock(&adapter->poll_mutex);
Rakesh Nair9bcf2602017-01-06 16:02:16 +05302235
2236 return 0;
2237}
2238
2239
2240/* edma_close()
2241 * gets called when netdevice is down, stops the queue.
2242 */
2243int edma_close(struct net_device *netdev)
2244{
2245 struct edma_adapter *adapter = netdev_priv(netdev);
2246
2247 edma_free_rfs_flow_table(adapter);
2248 netif_carrier_off(netdev);
2249 netif_tx_stop_all_queues(netdev);
2250
Rakesh Naired29f6b2017-04-04 15:48:08 +05302251 mutex_lock(&adapter->poll_mutex);
Rakesh Nair9bcf2602017-01-06 16:02:16 +05302252 if (adapter->poll_required) {
2253 if (!IS_ERR(adapter->phydev))
2254 phy_stop(adapter->phydev);
2255 }
Rakesh Naired29f6b2017-04-04 15:48:08 +05302256 mutex_unlock(&adapter->poll_mutex);
Rakesh Nair9bcf2602017-01-06 16:02:16 +05302257
2258 adapter->link_state = __EDMA_LINKDOWN;
2259
2260 /* Set GMAC state to UP before link state is checked
2261 */
2262 clear_bit(__EDMA_UP, &adapter->state_flags);
2263
2264 return 0;
2265}
2266
2267/* edma_poll
2268 * polling function that gets called when the napi gets scheduled.
2269 *
2270 * Main sequence of task performed in this api
2271 * is clear irq status -> clear_tx_irq -> clean_rx_irq->
2272 * enable interrupts.
2273 */
2274int edma_poll(struct napi_struct *napi, int budget)
2275{
2276 struct edma_per_cpu_queues_info *edma_percpu_info = container_of(napi,
2277 struct edma_per_cpu_queues_info, napi);
2278 struct edma_common_info *edma_cinfo = edma_percpu_info->edma_cinfo;
2279 u32 reg_data;
2280 u32 shadow_rx_status, shadow_tx_status;
2281 int queue_id;
2282 int i, work_done = 0;
Rakesh Nair03b586c2017-04-03 18:28:58 +05302283 u16 rx_pending_fill;
Rakesh Nair9bcf2602017-01-06 16:02:16 +05302284
2285 /* Store the Rx/Tx status by ANDing it with
2286 * appropriate CPU RX?TX mask
2287 */
2288 edma_read_reg(EDMA_REG_RX_ISR, &reg_data);
2289 edma_percpu_info->rx_status |= reg_data & edma_percpu_info->rx_mask;
2290 shadow_rx_status = edma_percpu_info->rx_status;
2291 edma_read_reg(EDMA_REG_TX_ISR, &reg_data);
2292 edma_percpu_info->tx_status |= reg_data & edma_percpu_info->tx_mask;
2293 shadow_tx_status = edma_percpu_info->tx_status;
2294
2295 /* Every core will have a start, which will be computed
2296 * in probe and stored in edma_percpu_info->tx_start variable.
2297 * We will shift the status bit by tx_start to obtain
2298 * status bits for the core on which the current processing
2299 * is happening. Since, there are 4 tx queues per core,
2300 * we will run the loop till we get the correct queue to clear.
2301 */
2302 while (edma_percpu_info->tx_status) {
2303 queue_id = ffs(edma_percpu_info->tx_status) - 1;
2304 edma_tx_complete(edma_cinfo, queue_id);
2305 edma_percpu_info->tx_status &= ~(1 << queue_id);
2306 }
2307
2308 /* Every core will have a start, which will be computed
2309 * in probe and stored in edma_percpu_info->tx_start variable.
2310 * We will shift the status bit by tx_start to obtain
2311 * status bits for the core on which the current processing
2312 * is happening. Since, there are 4 tx queues per core, we
2313 * will run the loop till we get the correct queue to clear.
2314 */
2315 while (edma_percpu_info->rx_status) {
2316 queue_id = ffs(edma_percpu_info->rx_status) - 1;
Rakesh Nair03b586c2017-04-03 18:28:58 +05302317 rx_pending_fill = edma_rx_complete(edma_cinfo, &work_done,
Rakesh Nair9bcf2602017-01-06 16:02:16 +05302318 budget, queue_id, napi);
2319
Rakesh Nair03b586c2017-04-03 18:28:58 +05302320 if (likely(work_done < budget)) {
2321 if (rx_pending_fill) {
2322 work_done = budget;
2323 break;
2324 }
2325
Rakesh Nair9bcf2602017-01-06 16:02:16 +05302326 edma_percpu_info->rx_status &= ~(1 << queue_id);
Rakesh Nair03b586c2017-04-03 18:28:58 +05302327 }
Rakesh Nair9bcf2602017-01-06 16:02:16 +05302328 else
2329 break;
2330 }
2331
2332 /* Clear the status register, to avoid the interrupts to
2333 * reoccur.This clearing of interrupt status register is
2334 * done here as writing to status register only takes place
2335 * once the producer/consumer index has been updated to
2336 * reflect that the packet transmission/reception went fine.
2337 */
2338 edma_write_reg(EDMA_REG_RX_ISR, shadow_rx_status);
2339 edma_write_reg(EDMA_REG_TX_ISR, shadow_tx_status);
2340
2341 /* If budget not fully consumed, exit the polling mode */
2342 if (likely(work_done < budget)) {
2343 napi_complete(napi);
2344
2345 /* re-enable the interrupts */
2346 for (i = 0; i < edma_cinfo->num_rxq_per_core; i++)
2347 edma_write_reg(EDMA_REG_RX_INT_MASK_Q(edma_percpu_info->rx_start + i), 0x1);
2348 for (i = 0; i < edma_cinfo->num_txq_per_core; i++)
2349 edma_write_reg(EDMA_REG_TX_INT_MASK_Q(edma_percpu_info->tx_start + i), 0x1);
2350 }
2351
2352 return work_done;
2353}
2354
2355/* edma interrupt()
2356 * interrupt handler
2357 */
2358irqreturn_t edma_interrupt(int irq, void *dev)
2359{
2360 struct edma_per_cpu_queues_info *edma_percpu_info = (struct edma_per_cpu_queues_info *) dev;
2361 struct edma_common_info *edma_cinfo = edma_percpu_info->edma_cinfo;
2362 int i;
2363
2364 /* Unmask the TX/RX interrupt register */
2365 for (i = 0; i < edma_cinfo->num_rxq_per_core; i++)
2366 edma_write_reg(EDMA_REG_RX_INT_MASK_Q(edma_percpu_info->rx_start + i), 0x0);
2367
2368 for (i = 0; i < edma_cinfo->num_txq_per_core; i++)
2369 edma_write_reg(EDMA_REG_TX_INT_MASK_Q(edma_percpu_info->tx_start + i), 0x0);
2370
2371 napi_schedule(&edma_percpu_info->napi);
2372
2373 return IRQ_HANDLED;
2374}