blob: 8d505a52500c34ffab4f7e6315cb8763e036ebf7 [file] [log] [blame]
Rakesh Nair9bcf2602017-01-06 16:02:16 +05301/*
2 * Copyright (c) 2014 - 2017, The Linux Foundation. All rights reserved.
3 *
4 * Permission to use, copy, modify, and/or distribute this software for
5 * any purpose with or without fee is hereby granted, provided that the
6 * above copyright notice and this permission notice appear in all copies.
7 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
8 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
9 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
10 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
11 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
12 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT
13 * OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
14 */
15
16#include <linux/platform_device.h>
17#include <linux/if_vlan.h>
18#include "ess_edma.h"
19#include "edma.h"
20
21extern struct net_device *edma_netdev[EDMA_MAX_PORTID_SUPPORTED];
22bool edma_stp_rstp;
23u16 edma_ath_eth_type;
24
25/* edma_skb_priority_offset()
26 * get edma skb priority
27 */
28static unsigned int edma_skb_priority_offset(struct sk_buff *skb)
29{
30 return (skb->priority >> 2) & 1;
31}
32
33/* edma_alloc_tx_ring()
34 * Allocate Tx descriptors ring
35 */
36static int edma_alloc_tx_ring(struct edma_common_info *edma_cinfo,
37 struct edma_tx_desc_ring *etdr)
38{
39 struct platform_device *pdev = edma_cinfo->pdev;
40 u16 sw_size = sizeof(struct edma_sw_desc) * etdr->count;
41
42 /* Initialize ring */
43 etdr->size = sizeof(struct edma_tx_desc) * etdr->count;
44 etdr->sw_next_to_fill = 0;
45 etdr->sw_next_to_clean = 0;
46
47 /* Allocate SW descriptors */
48 etdr->sw_desc = vzalloc(sw_size);
49 if (!etdr->sw_desc) {
50 dev_err(&pdev->dev, "buffer alloc of tx ring failed=%p", etdr);
51 return -ENOMEM;
52 }
53
54 /* Allocate HW descriptors */
55 etdr->hw_desc = dma_alloc_coherent(&pdev->dev, etdr->size, &etdr->dma,
56 GFP_KERNEL);
57 if (!etdr->hw_desc) {
58 dev_err(&pdev->dev, "descriptor allocation for tx ring failed");
59 vfree(etdr->sw_desc);
60 etdr->sw_desc = NULL;
61 return -ENOMEM;
62 }
63
64 return 0;
65}
66
67/* edma_free_tx_ring()
68 * Free tx rings allocated by edma_alloc_tx_rings
69 */
70static void edma_free_tx_ring(struct edma_common_info *edma_cinfo,
71 struct edma_tx_desc_ring *etdr)
72{
73 struct platform_device *pdev = edma_cinfo->pdev;
74
75 if (likely(etdr->hw_desc)) {
76 dma_free_coherent(&pdev->dev, etdr->size, etdr->hw_desc,
77 etdr->dma);
78
79 vfree(etdr->sw_desc);
80 etdr->sw_desc = NULL;
81 }
82}
83
84/* edma_alloc_rx_ring()
85 * allocate rx descriptor ring
86 */
87static int edma_alloc_rx_ring(struct edma_common_info *edma_cinfo,
88 struct edma_rfd_desc_ring *erxd)
89{
90 struct platform_device *pdev = edma_cinfo->pdev;
91 u16 sw_size = sizeof(struct edma_sw_desc) * erxd->count;
92
93 erxd->size = sizeof(struct edma_sw_desc) * erxd->count;
94 erxd->sw_next_to_fill = 0;
95 erxd->sw_next_to_clean = 0;
96
97 /* Allocate SW descriptors */
98 erxd->sw_desc = vzalloc(sw_size);
99 if (!erxd->sw_desc)
100 return -ENOMEM;
101
102 /* Alloc HW descriptors */
103 erxd->hw_desc = dma_alloc_coherent(&pdev->dev, erxd->size, &erxd->dma,
104 GFP_KERNEL);
105 if (!erxd->hw_desc) {
106 vfree(erxd->sw_desc);
107 erxd->sw_desc = NULL;
108 return -ENOMEM;
109 }
110
111 return 0;
112}
113
114/* edma_free_rx_ring()
115 * Free rx ring allocated by alloc_rx_ring
116 */
117static void edma_free_rx_ring(struct edma_common_info *edma_cinfo,
118 struct edma_rfd_desc_ring *erxd)
119{
120 struct platform_device *pdev = edma_cinfo->pdev;
121
122 if (likely(erxd->hw_desc)) {
123 dma_free_coherent(&pdev->dev, erxd->size, erxd->hw_desc,
124 erxd->dma);
125
126 vfree(erxd->sw_desc);
127 erxd->sw_desc = NULL;
128 }
129}
130
131/* edma_configure_tx()
132 * Configure transmission control data
133 */
134static void edma_configure_tx(struct edma_common_info *edma_cinfo)
135{
136 u32 txq_ctrl_data;
137
138 txq_ctrl_data = (EDMA_TPD_BURST << EDMA_TXQ_NUM_TPD_BURST_SHIFT);
139 txq_ctrl_data |= EDMA_TXQ_CTRL_TPD_BURST_EN;
140 txq_ctrl_data |= (EDMA_TXF_BURST << EDMA_TXQ_TXF_BURST_NUM_SHIFT);
141 edma_write_reg(EDMA_REG_TXQ_CTRL, txq_ctrl_data);
142}
143
144/* edma_configure_rx()
145 * configure reception control data
146 */
147static void edma_configure_rx(struct edma_common_info *edma_cinfo)
148{
149 struct edma_hw *hw = &edma_cinfo->hw;
150 u32 rss_type, rx_desc1, rxq_ctrl_data;
151
152 /* Set RSS type */
153 rss_type = hw->rss_type;
154 edma_write_reg(EDMA_REG_RSS_TYPE, rss_type);
155
156 /* Set RFD burst number */
157 rx_desc1 = (EDMA_RFD_BURST << EDMA_RXQ_RFD_BURST_NUM_SHIFT);
158
159 /* Set RFD prefetch threshold */
160 rx_desc1 |= (EDMA_RFD_THR << EDMA_RXQ_RFD_PF_THRESH_SHIFT);
161
162 /* Set RFD in host ring low threshold to generte interrupt */
163 rx_desc1 |= (EDMA_RFD_LTHR << EDMA_RXQ_RFD_LOW_THRESH_SHIFT);
164 edma_write_reg(EDMA_REG_RX_DESC1, rx_desc1);
165
166 /* Set Rx FIFO threshold to start to DMA data to host */
167 rxq_ctrl_data = EDMA_FIFO_THRESH_128_BYTE;
168
169 /* Set RX remove vlan bit */
170 rxq_ctrl_data |= EDMA_RXQ_CTRL_RMV_VLAN;
171
172 edma_write_reg(EDMA_REG_RXQ_CTRL, rxq_ctrl_data);
173}
174
175/* edma_alloc_rx_buf()
176 * does skb allocation for the received packets.
177 */
178static int edma_alloc_rx_buf(struct edma_common_info
179 *edma_cinfo,
180 struct edma_rfd_desc_ring *erdr,
181 int cleaned_count, int queue_id)
182{
183 struct platform_device *pdev = edma_cinfo->pdev;
184 struct edma_rx_free_desc *rx_desc;
185 struct edma_sw_desc *sw_desc;
186 struct sk_buff *skb;
187 unsigned int i;
188 u16 prod_idx, length;
189 u32 reg_data;
190
191 if (cleaned_count > erdr->count) {
192 dev_err(&pdev->dev, "Incorrect cleaned_count %d",
193 cleaned_count);
194 return -1;
195 }
196
197 i = erdr->sw_next_to_fill;
198
199 while (cleaned_count) {
200 sw_desc = &erdr->sw_desc[i];
201 length = edma_cinfo->rx_head_buffer_len;
202
203 if (sw_desc->flags & EDMA_SW_DESC_FLAG_SKB_REUSE) {
204 skb = sw_desc->skb;
205
206 /* Clear REUSE flag */
207 sw_desc->flags &= ~EDMA_SW_DESC_FLAG_SKB_REUSE;
208 } else {
209 /* alloc skb */
210 skb = netdev_alloc_skb(edma_netdev[0], length);
211 if (!skb) {
212 /* Better luck next round */
213 sw_desc->flags = 0;
214 break;
215 }
216 }
217
218 if (!edma_cinfo->page_mode) {
219 sw_desc->dma = dma_map_single(&pdev->dev, skb->data,
220 length, DMA_FROM_DEVICE);
221 if (dma_mapping_error(&pdev->dev, sw_desc->dma)) {
222 WARN_ONCE(0, "EDMA DMA mapping failed for linear address %x", sw_desc->dma);
223 sw_desc->flags = 0;
224 sw_desc->skb = NULL;
225 dev_kfree_skb_any(skb);
226 break;
227 }
228
229 /*
230 * We should not exit from here with REUSE flag set
231 * This is to avoid re-using same sk_buff for next
232 * time around
233 */
234 sw_desc->flags = EDMA_SW_DESC_FLAG_SKB_HEAD;
235 sw_desc->length = length;
236 } else {
237 struct page *pg = alloc_page(GFP_ATOMIC);
238
239 if (!pg) {
240 sw_desc->flags = 0;
241 sw_desc->skb = NULL;
242 dev_kfree_skb_any(skb);
243 break;
244 }
245
246 sw_desc->dma = dma_map_page(&pdev->dev, pg, 0,
247 edma_cinfo->rx_page_buffer_len,
248 DMA_FROM_DEVICE);
249 if (dma_mapping_error(&pdev->dev, sw_desc->dma)) {
250 WARN_ONCE(0, "EDMA DMA mapping failed for page address %x", sw_desc->dma);
251 sw_desc->flags = 0;
252 sw_desc->skb = NULL;
253 __free_page(pg);
254 dev_kfree_skb_any(skb);
255 break;
256 }
257
258 skb_fill_page_desc(skb, 0, pg, 0,
259 edma_cinfo->rx_page_buffer_len);
260 sw_desc->flags = EDMA_SW_DESC_FLAG_SKB_FRAG;
261 sw_desc->length = edma_cinfo->rx_page_buffer_len;
262 }
263
264 /* Update the buffer info */
265 sw_desc->skb = skb;
266 rx_desc = (&(erdr->hw_desc)[i]);
267 rx_desc->buffer_addr = cpu_to_le64(sw_desc->dma);
268 if (++i == erdr->count)
269 i = 0;
270 cleaned_count--;
271 }
272
273 erdr->sw_next_to_fill = i;
274
275 if (i == 0)
276 prod_idx = erdr->count - 1;
277 else
278 prod_idx = i - 1;
279
280 /* Update the producer index */
281 edma_read_reg(EDMA_REG_RFD_IDX_Q(queue_id), &reg_data);
282 reg_data &= ~EDMA_RFD_PROD_IDX_BITS;
283 reg_data |= prod_idx;
284 edma_write_reg(EDMA_REG_RFD_IDX_Q(queue_id), reg_data);
285 return cleaned_count;
286}
287
288/* edma_init_desc()
289 * update descriptor ring size, buffer and producer/consumer index
290 */
291static void edma_init_desc(struct edma_common_info *edma_cinfo)
292{
293 struct edma_rfd_desc_ring *rfd_ring;
294 struct edma_tx_desc_ring *etdr;
295 int i = 0, j = 0;
296 u32 data = 0;
297 u16 hw_cons_idx = 0;
298
299 /* Set the base address of every TPD ring. */
300 for (i = 0; i < edma_cinfo->num_tx_queues; i++) {
301 etdr = edma_cinfo->tpd_ring[i];
302
303 /* Update descriptor ring base address */
304 edma_write_reg(EDMA_REG_TPD_BASE_ADDR_Q(i), (u32)etdr->dma);
305 edma_read_reg(EDMA_REG_TPD_IDX_Q(i), &data);
306
307 /* Calculate hardware consumer index */
308 hw_cons_idx = (data >> EDMA_TPD_CONS_IDX_SHIFT) & 0xffff;
309 etdr->sw_next_to_fill = hw_cons_idx;
310 etdr->sw_next_to_clean = hw_cons_idx;
311 data &= ~(EDMA_TPD_PROD_IDX_MASK << EDMA_TPD_PROD_IDX_SHIFT);
312 data |= hw_cons_idx;
313
314 /* update producer index */
315 edma_write_reg(EDMA_REG_TPD_IDX_Q(i), data);
316
317 /* update SW consumer index register */
318 edma_write_reg(EDMA_REG_TX_SW_CONS_IDX_Q(i), hw_cons_idx);
319
320 /* Set TPD ring size */
321 edma_write_reg(EDMA_REG_TPD_RING_SIZE,
322 edma_cinfo->tx_ring_count &
323 EDMA_TPD_RING_SIZE_MASK);
324 }
325
326 for (i = 0, j = 0; i < edma_cinfo->num_rx_queues; i++) {
327 rfd_ring = edma_cinfo->rfd_ring[j];
328 /* Update Receive Free descriptor ring base address */
329 edma_write_reg(EDMA_REG_RFD_BASE_ADDR_Q(j),
330 (u32)(rfd_ring->dma));
331 j += ((edma_cinfo->num_rx_queues == 4) ? 2 : 1);
332 }
333
334 data = edma_cinfo->rx_head_buffer_len;
335 if (edma_cinfo->page_mode)
336 data = edma_cinfo->rx_page_buffer_len;
337
338 data &= EDMA_RX_BUF_SIZE_MASK;
339 data <<= EDMA_RX_BUF_SIZE_SHIFT;
340
341 /* Update RFD ring size and RX buffer size */
342 data |= (edma_cinfo->rx_ring_count & EDMA_RFD_RING_SIZE_MASK)
343 << EDMA_RFD_RING_SIZE_SHIFT;
344
345 edma_write_reg(EDMA_REG_RX_DESC0, data);
346
347 /* Disable TX FIFO low watermark and high watermark */
348 edma_write_reg(EDMA_REG_TXF_WATER_MARK, 0);
349
350 /* Load all of base address above */
351 edma_read_reg(EDMA_REG_TX_SRAM_PART, &data);
352 data |= 1 << EDMA_LOAD_PTR_SHIFT;
353 edma_write_reg(EDMA_REG_TX_SRAM_PART, data);
354}
355
356/* edma_receive_checksum
357 * Api to check checksum on receive packets
358 */
359static void edma_receive_checksum(struct edma_rx_return_desc *rd,
360 struct sk_buff *skb)
361{
362 skb_checksum_none_assert(skb);
363
364 /* check the RRD IP/L4 checksum bit to see if
365 * its set, which in turn indicates checksum
366 * failure.
367 */
368 if (rd->rrd6 & EDMA_RRD_CSUM_FAIL_MASK)
369 return;
370
371 skb->ip_summed = CHECKSUM_UNNECESSARY;
372}
373
374/* edma_clean_rfd()
375 * clean up rx resourcers on error
376 */
377static void edma_clean_rfd(struct platform_device *pdev,
378 struct edma_rfd_desc_ring *erdr,
379 u16 index,
380 int pos)
381{
382 struct edma_rx_free_desc *rx_desc = &(erdr->hw_desc[index]);
383 struct edma_sw_desc *sw_desc = &erdr->sw_desc[index];
384
385 /* Unmap non-first RFD positions in packet */
386 if (pos) {
387 if (likely(sw_desc->flags & EDMA_SW_DESC_FLAG_SKB_HEAD))
388 dma_unmap_single(&pdev->dev, sw_desc->dma,
389 sw_desc->length, DMA_FROM_DEVICE);
390 else
391 dma_unmap_page(&pdev->dev, sw_desc->dma,
392 sw_desc->length, DMA_FROM_DEVICE);
393 }
394
395 if (sw_desc->skb) {
396 dev_kfree_skb_any(sw_desc->skb);
397 sw_desc->skb = NULL;
398 }
399
400 sw_desc->flags = 0;
401 memset(rx_desc, 0, sizeof(struct edma_rx_free_desc));
402}
403
404/* edma_rx_complete_stp_rstp()
405 * Complete Rx processing for STP RSTP packets
406 */
407static void edma_rx_complete_stp_rstp(struct sk_buff *skb, int port_id, struct edma_rx_return_desc *rd)
408{
409 int i;
410 u32 priority;
411 u16 port_type;
412 u8 mac_addr[EDMA_ETH_HDR_LEN];
413
414 port_type = (rd->rrd1 >> EDMA_RRD_PORT_TYPE_SHIFT)
415 & EDMA_RRD_PORT_TYPE_MASK;
416 /* if port type is 0x4, then only proceed with
417 * other stp/rstp calculation
418 */
419 if (port_type == EDMA_RX_ATH_HDR_RSTP_PORT_TYPE) {
420 u8 bpdu_mac[6] = {0x01, 0x80, 0xc2, 0x00, 0x00, 0x00};
421
422 /* calculate the frame priority */
423 priority = (rd->rrd1 >> EDMA_RRD_PRIORITY_SHIFT)
424 & EDMA_RRD_PRIORITY_MASK;
425
426 for (i = 0; i < EDMA_ETH_HDR_LEN; i++)
427 mac_addr[i] = skb->data[i];
428
429 /* Check if destination mac addr is bpdu addr */
430 if (!memcmp(mac_addr, bpdu_mac, 6)) {
431 /* destination mac address is BPDU
432 * destination mac address, then add
433 * atheros header to the packet.
434 */
435 u16 athr_hdr = (EDMA_RX_ATH_HDR_VERSION << EDMA_RX_ATH_HDR_VERSION_SHIFT) |
436 (priority << EDMA_RX_ATH_HDR_PRIORITY_SHIFT) |
437 (EDMA_RX_ATH_HDR_RSTP_PORT_TYPE << EDMA_RX_ATH_PORT_TYPE_SHIFT) | port_id;
438 skb_push(skb, 4);
439 memcpy(skb->data, mac_addr, EDMA_ETH_HDR_LEN);
440 *(uint16_t *)&skb->data[12] = htons(edma_ath_eth_type);
441 *(uint16_t *)&skb->data[14] = htons(athr_hdr);
442 }
443 }
444}
445
446/* edma_rx_complete_fraglist()
447 * Complete Rx processing for fraglist skbs
448 */
449static int edma_rx_complete_fraglist(struct sk_buff *skb, u16 num_rfds, u16 length, u32 sw_next_to_clean,
450 struct edma_rfd_desc_ring *erdr, struct edma_common_info *edma_cinfo)
451{
452 struct platform_device *pdev = edma_cinfo->pdev;
453 struct edma_hw *hw = &edma_cinfo->hw;
454 struct sk_buff *skb_temp;
455 struct edma_sw_desc *sw_desc;
456 int i;
457 u16 size_remaining;
458
459 skb->data_len = 0;
460 skb->tail += (hw->rx_head_buff_size - 16);
461 skb->len = skb->truesize = length;
462 size_remaining = length - (hw->rx_head_buff_size - 16);
463
464 /* clean-up all related sw_descs */
465 for (i = 1; i < num_rfds; i++) {
466 struct sk_buff *skb_prev;
467
468 sw_desc = &erdr->sw_desc[sw_next_to_clean];
469 skb_temp = sw_desc->skb;
470
471 dma_unmap_single(&pdev->dev, sw_desc->dma,
472 sw_desc->length, DMA_FROM_DEVICE);
473
474 if (size_remaining < hw->rx_head_buff_size)
475 skb_put(skb_temp, size_remaining);
476 else
477 skb_put(skb_temp, hw->rx_head_buff_size);
478
479 /* If we are processing the first rfd, we link
480 * skb->frag_list to the skb corresponding to the
481 * first RFD
482 */
483 if (i == 1)
484 skb_shinfo(skb)->frag_list = skb_temp;
485 else
486 skb_prev->next = skb_temp;
487 skb_prev = skb_temp;
488 skb_temp->next = NULL;
489
490 skb->data_len += skb_temp->len;
491 size_remaining -= skb_temp->len;
492
493 /* Increment SW index */
494 sw_next_to_clean = (sw_next_to_clean + 1) & (erdr->count - 1);
495 }
496
497 return sw_next_to_clean;
498}
499
500/* edma_rx_complete_paged()
501 * Complete Rx processing for paged skbs
502 */
503static int edma_rx_complete_paged(struct sk_buff *skb, u16 num_rfds,
504 u16 length, u32 sw_next_to_clean,
505 struct edma_rfd_desc_ring *erdr,
506 struct edma_common_info *edma_cinfo)
507{
508 struct platform_device *pdev = edma_cinfo->pdev;
509 struct sk_buff *skb_temp;
510 struct edma_sw_desc *sw_desc;
511 int i;
512 u16 size_remaining;
513
514 skb_frag_t *frag = &skb_shinfo(skb)->frags[0];
515
516 /* Setup skbuff fields */
517 skb->len = length;
518
519 if (likely(num_rfds <= 1)) {
520 skb->data_len = length;
521 skb->truesize += edma_cinfo->rx_page_buffer_len;
522 skb_fill_page_desc(skb, 0, skb_frag_page(frag),
523 16, length);
524 } else {
525 frag->size -= 16;
526 skb->data_len = frag->size;
527 skb->truesize += edma_cinfo->rx_page_buffer_len;
528 size_remaining = length - frag->size;
529
530 skb_fill_page_desc(skb, 0, skb_frag_page(frag),
531 16, frag->size);
532
533 /* clean-up all related sw_descs */
534 for (i = 1; i < num_rfds; i++) {
535 sw_desc = &erdr->sw_desc[sw_next_to_clean];
536 skb_temp = sw_desc->skb;
537 frag = &skb_shinfo(skb_temp)->frags[0];
538 dma_unmap_page(&pdev->dev, sw_desc->dma,
539 sw_desc->length, DMA_FROM_DEVICE);
540
541 if (size_remaining < edma_cinfo->rx_page_buffer_len)
542 frag->size = size_remaining;
543
544 skb_fill_page_desc(skb, i, skb_frag_page(frag),
545 0, frag->size);
546
547 /* We used frag pages from skb_temp in skb */
548 skb_shinfo(skb_temp)->nr_frags = 0;
549 dev_kfree_skb_any(skb_temp);
550
551 skb->data_len += frag->size;
552 skb->truesize += edma_cinfo->rx_page_buffer_len;
553 size_remaining -= frag->size;
554
555 /* Increment SW index */
556 sw_next_to_clean = (sw_next_to_clean + 1) & (erdr->count - 1);
557 }
558 }
559
560 return sw_next_to_clean;
561}
562
563/*
564 * edma_rx_complete()
565 * Main api called from the poll function to process rx packets.
566 */
567static void edma_rx_complete(struct edma_common_info *edma_cinfo,
568 int *work_done, int work_to_do, int queue_id,
569 struct napi_struct *napi)
570{
571 struct platform_device *pdev = edma_cinfo->pdev;
572 struct edma_rfd_desc_ring *erdr = edma_cinfo->rfd_ring[queue_id];
573 u16 hash_type, rrd[8], cleaned_count = 0, length = 0, num_rfds = 1,
574 sw_next_to_clean, hw_next_to_clean = 0, vlan = 0, ret_count = 0;
575 u32 data = 0;
576 u16 count = erdr->count, rfd_avail;
577 u8 queue_to_rxid[8] = {0, 0, 1, 1, 2, 2, 3, 3};
578
579 sw_next_to_clean = erdr->sw_next_to_clean;
580
581 edma_read_reg(EDMA_REG_RFD_IDX_Q(queue_id), &data);
582 hw_next_to_clean = (data >> EDMA_RFD_CONS_IDX_SHIFT) &
583 EDMA_RFD_CONS_IDX_MASK;
584
585 do {
586 while (sw_next_to_clean != hw_next_to_clean) {
587 struct net_device *netdev;
588 struct edma_adapter *adapter;
589 struct edma_sw_desc *sw_desc;
590 struct sk_buff *skb;
591 struct edma_rx_return_desc *rd;
592 u8 *vaddr;
593 int port_id, i, drop_count = 0;
594 u32 priority;
595
596 if (!work_to_do)
597 break;
598
599 sw_desc = &erdr->sw_desc[sw_next_to_clean];
600 skb = sw_desc->skb;
601
602 /* Get RRD */
603 if (!edma_cinfo->page_mode) {
604 dma_unmap_single(&pdev->dev, sw_desc->dma,
605 sw_desc->length, DMA_FROM_DEVICE);
606 rd = (struct edma_rx_return_desc *)skb->data;
607
608 } else {
609 dma_unmap_page(&pdev->dev, sw_desc->dma,
610 sw_desc->length, DMA_FROM_DEVICE);
611 vaddr = kmap_atomic(skb_frag_page(&skb_shinfo(skb)->frags[0]));
612 memcpy((uint8_t *)&rrd[0], vaddr, 16);
613 rd = (struct edma_rx_return_desc *)rrd;
614 kunmap_atomic(vaddr);
615 }
616
617 /* Check if RRD is valid */
618 if (!(rd->rrd7 & EDMA_RRD_DESC_VALID)) {
619 dev_err(&pdev->dev, "Incorrect RRD DESC valid bit set");
620 edma_clean_rfd(pdev, erdr, sw_next_to_clean, 0);
621 sw_next_to_clean = (sw_next_to_clean + 1) &
622 (erdr->count - 1);
623 cleaned_count++;
624 continue;
625 }
626
627 /* Get the number of RFDs from RRD */
628 num_rfds = rd->rrd1 & EDMA_RRD_NUM_RFD_MASK;
629
630 /* Get Rx port ID from switch */
631 port_id = (rd->rrd1 >> EDMA_PORT_ID_SHIFT) & EDMA_PORT_ID_MASK;
632 if ((!port_id) || (port_id > EDMA_MAX_PORTID_SUPPORTED)) {
633 if (net_ratelimit()) {
634 dev_err(&pdev->dev, "Incorrect RRD source port bit set");
635 dev_err(&pdev->dev,
636 "RRD Dump\n rrd0:%x rrd1: %x rrd2: %x rrd3: %x rrd4: %x rrd5: %x rrd6: %x rrd7: %x",
637 rd->rrd0, rd->rrd1, rd->rrd2, rd->rrd3, rd->rrd4, rd->rrd5, rd->rrd6, rd->rrd7);
638 dev_err(&pdev->dev, "Num_rfds: %d, src_port: %d, pkt_size: %d, cvlan_tag: %d\n",
639 num_rfds, rd->rrd1 & EDMA_RRD_SRC_PORT_NUM_MASK,
640 rd->rrd6 & EDMA_RRD_PKT_SIZE_MASK, rd->rrd7 & EDMA_RRD_CVLAN);
641 }
642 for (i = 0; i < num_rfds; i++) {
643 edma_clean_rfd(pdev, erdr, sw_next_to_clean, i);
644 sw_next_to_clean = (sw_next_to_clean + 1) & (erdr->count - 1);
645 }
646
647 cleaned_count += num_rfds;
648 continue;
649 }
650
651 netdev = edma_cinfo->portid_netdev_lookup_tbl[port_id];
652 if (!netdev) {
653 dev_err(&pdev->dev, "Invalid netdev");
654 for (i = 0; i < num_rfds; i++) {
655 edma_clean_rfd(pdev, erdr, sw_next_to_clean, i);
656 sw_next_to_clean = (sw_next_to_clean + 1) & (erdr->count - 1);
657 }
658
659 cleaned_count += num_rfds;
660 continue;
661 }
662 adapter = netdev_priv(netdev);
663
664 /* This code is added to handle a usecase where high
665 * priority stream and a low priority stream are
666 * received simultaneously on DUT. The problem occurs
667 * if one of the Rx rings is full and the corresponding
668 * core is busy with other stuff. This causes ESS CPU
669 * port to backpressure all incoming traffic including
670 * high priority one. We monitor free descriptor count
671 * on each CPU and whenever it reaches threshold (< 80),
672 * we drop all low priority traffic and let only high
673 * priotiy traffic pass through. We can hence avoid
674 * ESS CPU port to send backpressure on high priroity
675 * stream.
676 */
677 priority = (rd->rrd1 >> EDMA_RRD_PRIORITY_SHIFT)
678 & EDMA_RRD_PRIORITY_MASK;
679 if (likely(!priority && !edma_cinfo->page_mode && (num_rfds <= 1))) {
680 rfd_avail = (count + sw_next_to_clean - hw_next_to_clean - 1) & (count - 1);
681 if (rfd_avail < EDMA_RFD_AVAIL_THR) {
682 sw_desc->flags |= EDMA_SW_DESC_FLAG_SKB_REUSE;
683 sw_next_to_clean = (sw_next_to_clean + 1) & (erdr->count - 1);
684 adapter->stats.rx_dropped++;
685 cleaned_count++;
686 drop_count++;
687 if (drop_count == 3) {
688 work_to_do--;
689 (*work_done)++;
690 drop_count = 0;
691 }
692 if (cleaned_count == EDMA_RX_BUFFER_WRITE) {
693 /* If buffer clean count reaches 16, we replenish HW buffers. */
694 ret_count = edma_alloc_rx_buf(edma_cinfo, erdr, cleaned_count, queue_id);
695 edma_write_reg(EDMA_REG_RX_SW_CONS_IDX_Q(queue_id),
696 sw_next_to_clean);
697 cleaned_count = ret_count;
698 }
699 continue;
700 }
701 }
702
703 work_to_do--;
704 (*work_done)++;
705
706 /* Increment SW index */
707 sw_next_to_clean = (sw_next_to_clean + 1) &
708 (erdr->count - 1);
709
710 /* Get the packet size and allocate buffer */
711 length = rd->rrd6 & EDMA_RRD_PKT_SIZE_MASK;
712
713 if (edma_cinfo->page_mode) {
714 /* paged skb */
715 sw_next_to_clean = edma_rx_complete_paged(skb, num_rfds, length,
716 sw_next_to_clean,
717 erdr, edma_cinfo);
718 if (!pskb_may_pull(skb, ETH_HLEN)) {
719 cleaned_count += num_rfds;
720 dev_kfree_skb_any(skb);
721 continue;
722 }
723 } else {
724 /* single or fraglist skb */
725
726 /* Addition of 16 bytes is required, as in the packet
727 * first 16 bytes are rrd descriptors, so actual data
728 * starts from an offset of 16.
729 */
730 skb_reserve(skb, 16);
731 if (likely((num_rfds <= 1) || !edma_cinfo->fraglist_mode))
732 skb_put(skb, length);
733 else
734 sw_next_to_clean = edma_rx_complete_fraglist(skb, num_rfds, length,
735 sw_next_to_clean,
736 erdr, edma_cinfo);
737 }
738
739 cleaned_count += num_rfds;
740
741 if (edma_stp_rstp)
742 edma_rx_complete_stp_rstp(skb, port_id, rd);
743
744 skb->protocol = eth_type_trans(skb, netdev);
745
746 /* Record Rx queue for RFS/RPS and fill flow hash from HW */
747 skb_record_rx_queue(skb, queue_to_rxid[queue_id]);
748 if (netdev->features & NETIF_F_RXHASH) {
749 hash_type = (rd->rrd5 >> EDMA_HASH_TYPE_SHIFT);
750 if ((hash_type > EDMA_HASH_TYPE_START) && (hash_type < EDMA_HASH_TYPE_END))
751 skb_set_hash(skb, rd->rrd2, PKT_HASH_TYPE_L4);
752 }
753
754#ifdef CONFIG_NF_FLOW_COOKIE
755 skb->flow_cookie = rd->rrd3 & EDMA_RRD_FLOW_COOKIE_MASK;
756#endif
757 edma_receive_checksum(rd, skb);
758
759 /* Process VLAN HW acceleration indication provided by HW */
760 if (adapter->default_vlan_tag != rd->rrd4) {
761 vlan = rd->rrd4;
762 if (likely(rd->rrd7 & EDMA_RRD_CVLAN))
763 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vlan);
764 else if (rd->rrd1 & EDMA_RRD_SVLAN)
765 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021AD), vlan);
766 }
767
768 /* Update rx statistics */
769 adapter->stats.rx_packets++;
770 adapter->stats.rx_bytes += length;
771
772 /* Check if we reached refill threshold */
773 if (cleaned_count == EDMA_RX_BUFFER_WRITE) {
774 ret_count = edma_alloc_rx_buf(edma_cinfo, erdr, cleaned_count, queue_id);
775 edma_write_reg(EDMA_REG_RX_SW_CONS_IDX_Q(queue_id),
776 sw_next_to_clean);
777 cleaned_count = ret_count;
778 }
779
780 /* At this point skb should go to stack */
781 napi_gro_receive(napi, skb);
782 }
783
784 /* Check if we still have NAPI budget */
785 if (!work_to_do)
786 break;
787
788 /* Read index once again since we still have NAPI budget */
789 edma_read_reg(EDMA_REG_RFD_IDX_Q(queue_id), &data);
790 hw_next_to_clean = (data >> EDMA_RFD_CONS_IDX_SHIFT) &
791 EDMA_RFD_CONS_IDX_MASK;
792 } while (hw_next_to_clean != sw_next_to_clean);
793
794 erdr->sw_next_to_clean = sw_next_to_clean;
795
796 /* Refill here in case refill threshold wasn't reached */
797 if (likely(cleaned_count)) {
798 ret_count = edma_alloc_rx_buf(edma_cinfo, erdr, cleaned_count, queue_id);
799 if (ret_count)
800 dev_dbg(&pdev->dev, "Not all buffers was reallocated");
801 edma_write_reg(EDMA_REG_RX_SW_CONS_IDX_Q(queue_id),
802 erdr->sw_next_to_clean);
803 }
804}
805
806/* edma_delete_rfs_filter()
807 * Remove RFS filter from switch
808 */
809static int edma_delete_rfs_filter(struct edma_adapter *adapter,
810 struct edma_rfs_filter_node *filter_node)
811{
812 int res = -1;
813
814 if (likely(adapter->set_rfs_rule))
815 res = (*adapter->set_rfs_rule)(adapter->netdev,
816#if (LINUX_VERSION_CODE <= KERNEL_VERSION(3, 18, 21))
817 filter_node->keys.src,
818 filter_node->keys.dst, filter_node->keys.port16[0],
819 filter_node->keys.port16[1],
820 filter_node->keys.ip_proto,
821#else
822 filter_node->keys.addrs.v4addrs.src,
823 filter_node->keys.addrs.v4addrs.dst, filter_node->keys.ports.src,
824 filter_node->keys.ports.dst,
825 filter_node->keys.basic.ip_proto,
826#endif
827 filter_node->rq_id,
828 0);
829
830 return res;
831}
832
833/* edma_add_rfs_filter()
834 * Add RFS filter to switch
835 */
836static int edma_add_rfs_filter(struct edma_adapter *adapter,
837 struct flow_keys *keys, u16 rq,
838 struct edma_rfs_filter_node *filter_node)
839{
840 int res = -1;
841
842#if (LINUX_VERSION_CODE <= KERNEL_VERSION(3, 18, 21))
843 filter_node->keys.src = keys->src;
844 filter_node->keys.dst = keys->dst;
845 filter_node->keys.ports = keys->ports;
846 filter_node->keys.ip_proto = keys->ip_proto;
847#else
848 filter_node->keys.addrs.v4addrs.src = keys->addrs.v4addrs.src;
849 filter_node->keys.addrs.v4addrs.dst = keys->addrs.v4addrs.dst;
850 filter_node->keys.ports.ports = keys->ports.ports;
851 filter_node->keys.basic.ip_proto = keys->basic.ip_proto;
852#endif
853
854 /* Call callback registered by ESS driver */
855 if (likely(adapter->set_rfs_rule))
856#if (LINUX_VERSION_CODE <= KERNEL_VERSION(3, 18, 21))
857 res = (*adapter->set_rfs_rule)(adapter->netdev, keys->src,
858 keys->dst, keys->port16[0], keys->port16[1],
859 keys->ip_proto, rq, 1);
860#else
861 res = (*adapter->set_rfs_rule)(adapter->netdev, keys->addrs.v4addrs.src,
862 keys->addrs.v4addrs.dst, keys->ports.src, keys->ports.dst,
863 keys->basic.ip_proto, rq, 1);
864#endif
865
866 return res;
867}
868
869/* edma_rfs_key_search()
870 * Look for existing RFS entry
871 */
872static struct edma_rfs_filter_node *edma_rfs_key_search(struct hlist_head *h,
873 struct flow_keys *key)
874{
875 struct edma_rfs_filter_node *p;
876
877 hlist_for_each_entry(p, h, node)
878#if (LINUX_VERSION_CODE <= KERNEL_VERSION(3, 18, 21))
879 if (p->keys.src == key->src &&
880 p->keys.dst == key->dst &&
881 p->keys.ports == key->ports &&
882 p->keys.ip_proto == key->ip_proto)
883#else
884 if (p->keys.addrs.v4addrs.src == key->addrs.v4addrs.src &&
885 p->keys.addrs.v4addrs.dst == key->addrs.v4addrs.dst &&
886 p->keys.ports.ports == key->ports.ports &&
887 p->keys.basic.ip_proto == key->basic.ip_proto)
888#endif
889 return p;
890 return NULL;
891}
892
893/* edma_initialise_rfs_flow_table()
894 * Initialise EDMA RFS flow table
895 */
896static void edma_initialise_rfs_flow_table(struct edma_adapter *adapter)
897{
898 int i;
899
900 spin_lock_init(&adapter->rfs.rfs_ftab_lock);
901
902 /* Initialize EDMA flow hash table */
903 for (i = 0; i < EDMA_RFS_FLOW_ENTRIES; i++)
904 INIT_HLIST_HEAD(&adapter->rfs.hlist_head[i]);
905
906 adapter->rfs.max_num_filter = EDMA_RFS_FLOW_ENTRIES;
907 adapter->rfs.filter_available = adapter->rfs.max_num_filter;
908 adapter->rfs.hashtoclean = 0;
909
910 /* Add timer to get periodic RFS updates from OS */
911 init_timer(&adapter->rfs.expire_rfs);
912 adapter->rfs.expire_rfs.function = edma_flow_may_expire;
913 adapter->rfs.expire_rfs.data = (unsigned long)adapter;
914 mod_timer(&adapter->rfs.expire_rfs, jiffies + HZ/4);
915}
916
917/* edma_free_rfs_flow_table()
918 * Free EDMA RFS flow table
919 */
920static void edma_free_rfs_flow_table(struct edma_adapter *adapter)
921{
922 int i;
923
924 /* Remove sync timer */
925 del_timer_sync(&adapter->rfs.expire_rfs);
926 spin_lock_bh(&adapter->rfs.rfs_ftab_lock);
927
928 /* Free EDMA RFS table entries */
929 adapter->rfs.filter_available = 0;
930
931 /* Clean-up EDMA flow hash table */
932 for (i = 0; i < EDMA_RFS_FLOW_ENTRIES; i++) {
933 struct hlist_head *hhead;
934 struct hlist_node *tmp;
935 struct edma_rfs_filter_node *filter_node;
936 int res;
937
938 hhead = &adapter->rfs.hlist_head[i];
939 hlist_for_each_entry_safe(filter_node, tmp, hhead, node) {
940 res = edma_delete_rfs_filter(adapter, filter_node);
941 if (res < 0)
942 dev_warn(&adapter->netdev->dev,
943 "EDMA going down but RFS entry %d not allowed to be flushed by Switch",
944 filter_node->flow_id);
945 hlist_del(&filter_node->node);
946 kfree(filter_node);
947 }
948 }
949 spin_unlock_bh(&adapter->rfs.rfs_ftab_lock);
950}
951
952/* edma_tx_unmap_and_free()
953 * clean TX buffer
954 */
955static inline void edma_tx_unmap_and_free(struct platform_device *pdev,
956 struct edma_sw_desc *sw_desc)
957{
958 struct sk_buff *skb = sw_desc->skb;
959
960 if (likely((sw_desc->flags & EDMA_SW_DESC_FLAG_SKB_HEAD) ||
961 (sw_desc->flags & EDMA_SW_DESC_FLAG_SKB_FRAGLIST)))
962 /* unmap_single for skb head area */
963 dma_unmap_single(&pdev->dev, sw_desc->dma,
964 sw_desc->length, DMA_TO_DEVICE);
965 else if (sw_desc->flags & EDMA_SW_DESC_FLAG_SKB_FRAG)
966 /* unmap page for paged fragments */
967 dma_unmap_page(&pdev->dev, sw_desc->dma,
968 sw_desc->length, DMA_TO_DEVICE);
969
970 if (likely(sw_desc->flags & EDMA_SW_DESC_FLAG_LAST))
971 dev_kfree_skb_any(skb);
972
973 sw_desc->flags = 0;
974}
975
976/* edma_tx_complete()
977 * Used to clean tx queues and update hardware and consumer index
978 */
979static void edma_tx_complete(struct edma_common_info *edma_cinfo, int queue_id)
980{
981 struct edma_tx_desc_ring *etdr = edma_cinfo->tpd_ring[queue_id];
982 struct edma_sw_desc *sw_desc;
983 struct platform_device *pdev = edma_cinfo->pdev;
984 int i;
985
986 u16 sw_next_to_clean = etdr->sw_next_to_clean;
987 u16 hw_next_to_clean;
988 u32 data = 0;
989
990 edma_read_reg(EDMA_REG_TPD_IDX_Q(queue_id), &data);
991 hw_next_to_clean = (data >> EDMA_TPD_CONS_IDX_SHIFT) & EDMA_TPD_CONS_IDX_MASK;
992
993 /* clean the buffer here */
994 while (sw_next_to_clean != hw_next_to_clean) {
995 sw_desc = &etdr->sw_desc[sw_next_to_clean];
996 edma_tx_unmap_and_free(pdev, sw_desc);
997 sw_next_to_clean = (sw_next_to_clean + 1) & (etdr->count - 1);
998 }
999
1000 etdr->sw_next_to_clean = sw_next_to_clean;
1001
1002 /* update the TPD consumer index register */
1003 edma_write_reg(EDMA_REG_TX_SW_CONS_IDX_Q(queue_id), sw_next_to_clean);
1004
1005 /* Wake the queue if queue is stopped and netdev link is up */
1006 for (i = 0; i < EDMA_MAX_NETDEV_PER_QUEUE && etdr->nq[i] ; i++) {
1007 if (netif_tx_queue_stopped(etdr->nq[i])) {
1008 if ((etdr->netdev[i]) && netif_carrier_ok(etdr->netdev[i]))
1009 netif_tx_wake_queue(etdr->nq[i]);
1010 }
1011 }
1012}
1013
1014/* edma_get_tx_buffer()
1015 * Get sw_desc corresponding to the TPD
1016 */
1017static struct edma_sw_desc *edma_get_tx_buffer(struct edma_common_info *edma_cinfo,
1018 struct edma_tx_desc *tpd, int queue_id)
1019{
1020 struct edma_tx_desc_ring *etdr = edma_cinfo->tpd_ring[queue_id];
1021
1022 return &etdr->sw_desc[tpd - (struct edma_tx_desc *)etdr->hw_desc];
1023}
1024
1025/* edma_get_next_tpd()
1026 * Return a TPD descriptor for transfer
1027 */
1028static struct edma_tx_desc *edma_get_next_tpd(struct edma_common_info *edma_cinfo,
1029 int queue_id)
1030{
1031 struct edma_tx_desc_ring *etdr = edma_cinfo->tpd_ring[queue_id];
1032 u16 sw_next_to_fill = etdr->sw_next_to_fill;
1033 struct edma_tx_desc *tpd_desc =
1034 (&((struct edma_tx_desc *)(etdr->hw_desc))[sw_next_to_fill]);
1035
1036 etdr->sw_next_to_fill = (etdr->sw_next_to_fill + 1) & (etdr->count - 1);
1037
1038 return tpd_desc;
1039}
1040
1041/* edma_tpd_available()
1042 * Check number of free TPDs
1043 */
1044static inline u16 edma_tpd_available(struct edma_common_info *edma_cinfo,
1045 int queue_id)
1046{
1047 struct edma_tx_desc_ring *etdr = edma_cinfo->tpd_ring[queue_id];
1048
1049 u16 sw_next_to_fill;
1050 u16 sw_next_to_clean;
1051 u16 count = 0;
1052
1053 sw_next_to_clean = etdr->sw_next_to_clean;
1054 sw_next_to_fill = etdr->sw_next_to_fill;
1055
1056 if (likely(sw_next_to_clean <= sw_next_to_fill))
1057 count = etdr->count;
1058
1059 return count + sw_next_to_clean - sw_next_to_fill - 1;
1060}
1061
1062/* edma_tx_queue_get()
1063 * Get the starting number of the queue
1064 */
1065static inline int edma_tx_queue_get(struct edma_adapter *adapter,
1066 struct sk_buff *skb, int txq_id)
1067{
1068 /* skb->priority is used as an index to skb priority table
1069 * and based on packet priority, correspong queue is assigned.
1070 */
1071 return adapter->tx_start_offset[txq_id] + edma_skb_priority_offset(skb);
1072}
1073
1074/* edma_tx_update_hw_idx()
1075 * update the producer index for the ring transmitted
1076 */
1077static void edma_tx_update_hw_idx(struct edma_common_info *edma_cinfo,
1078 struct sk_buff *skb, int queue_id)
1079{
1080 struct edma_tx_desc_ring *etdr = edma_cinfo->tpd_ring[queue_id];
1081 u32 tpd_idx_data;
1082
1083 /* Read and update the producer index */
1084 edma_read_reg(EDMA_REG_TPD_IDX_Q(queue_id), &tpd_idx_data);
1085 tpd_idx_data &= ~EDMA_TPD_PROD_IDX_BITS;
1086 tpd_idx_data |= (etdr->sw_next_to_fill & EDMA_TPD_PROD_IDX_MASK)
1087 << EDMA_TPD_PROD_IDX_SHIFT;
1088
1089 edma_write_reg(EDMA_REG_TPD_IDX_Q(queue_id), tpd_idx_data);
1090}
1091
1092/* edma_rollback_tx()
1093 * Function to retrieve tx resources in case of error
1094 */
1095static void edma_rollback_tx(struct edma_adapter *adapter,
1096 struct edma_tx_desc *start_tpd, int queue_id)
1097{
1098 struct edma_tx_desc_ring *etdr = adapter->edma_cinfo->tpd_ring[queue_id];
1099 struct edma_sw_desc *sw_desc;
1100 struct edma_tx_desc *tpd = NULL;
1101 u16 start_index, index;
1102
1103 start_index = start_tpd - (struct edma_tx_desc *)(etdr->hw_desc);
1104
1105 index = start_index;
1106 while (index != etdr->sw_next_to_fill) {
1107 tpd = (&((struct edma_tx_desc *)(etdr->hw_desc))[index]);
1108 sw_desc = &etdr->sw_desc[index];
1109 edma_tx_unmap_and_free(adapter->pdev, sw_desc);
1110 memset(tpd, 0, sizeof(struct edma_tx_desc));
1111 if (++index == etdr->count)
1112 index = 0;
1113 }
1114 etdr->sw_next_to_fill = start_index;
1115}
1116
1117/* edma_tx_map_and_fill()
1118 * gets called from edma_xmit_frame
1119 *
1120 * This is where the dma of the buffer to be transmitted
1121 * gets mapped
1122 */
1123static int edma_tx_map_and_fill(struct edma_common_info *edma_cinfo,
1124 struct edma_adapter *adapter,
1125 struct sk_buff *skb, int queue_id,
1126 unsigned int flags_transmit,
1127 u16 from_cpu, u16 dp_bitmap,
1128 bool packet_is_rstp, int nr_frags)
1129{
1130 struct edma_sw_desc *sw_desc = NULL;
1131 struct platform_device *pdev = edma_cinfo->pdev;
1132 struct edma_tx_desc *tpd = NULL;
1133 struct edma_tx_desc *start_tpd = NULL;
1134 struct sk_buff *iter_skb;
1135 int i;
1136 u32 word1 = 0, word3 = 0, lso_word1 = 0, svlan_tag = 0;
1137 u16 buf_len, lso_desc_len = 0;
1138
1139 if (skb_is_gso(skb)) {
1140 /* TODO: What additional checks need to be performed here */
1141 if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4) {
1142 lso_word1 |= EDMA_TPD_IPV4_EN;
1143 ip_hdr(skb)->check = 0;
1144 tcp_hdr(skb)->check = ~csum_tcpudp_magic(ip_hdr(skb)->saddr,
1145 ip_hdr(skb)->daddr, 0, IPPROTO_TCP, 0);
1146 } else if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6) {
1147 lso_word1 |= EDMA_TPD_LSO_V2_EN;
1148 ipv6_hdr(skb)->payload_len = 0;
1149 tcp_hdr(skb)->check = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
1150 &ipv6_hdr(skb)->daddr, 0, IPPROTO_TCP, 0);
1151 } else
1152 return -EINVAL;
1153
1154 lso_word1 |= EDMA_TPD_LSO_EN | ((skb_shinfo(skb)->gso_size & EDMA_TPD_MSS_MASK) << EDMA_TPD_MSS_SHIFT) |
1155 (skb_transport_offset(skb) << EDMA_TPD_HDR_SHIFT);
1156 } else if (flags_transmit & EDMA_HW_CHECKSUM) {
1157 u8 css, cso;
1158 cso = skb_checksum_start_offset(skb);
1159 css = cso + skb->csum_offset;
1160
1161 word1 |= (EDMA_TPD_CUSTOM_CSUM_EN);
1162 word1 |= (cso >> 1) << EDMA_TPD_HDR_SHIFT;
1163 word1 |= ((css >> 1) << EDMA_TPD_CUSTOM_CSUM_SHIFT);
1164 }
1165
1166 if (skb->protocol == htons(ETH_P_PPP_SES))
1167 word1 |= EDMA_TPD_PPPOE_EN;
1168
1169 if (flags_transmit & EDMA_VLAN_TX_TAG_INSERT_FLAG) {
1170 switch (skb->vlan_proto) {
1171 case htons(ETH_P_8021Q):
1172 word3 |= (1 << EDMA_TX_INS_CVLAN);
1173#if (LINUX_VERSION_CODE <= KERNEL_VERSION(3, 18, 21))
1174 word3 |= vlan_tx_tag_get(skb) << EDMA_TX_CVLAN_TAG_SHIFT;
1175#else
1176 word3 |= skb_vlan_tag_get(skb) << EDMA_TX_CVLAN_TAG_SHIFT;
1177#endif
1178 break;
1179 case htons(ETH_P_8021AD):
1180 word1 |= (1 << EDMA_TX_INS_SVLAN);
1181#if (LINUX_VERSION_CODE <= KERNEL_VERSION(3, 18, 21))
1182 svlan_tag = vlan_tx_tag_get(skb) << EDMA_TX_SVLAN_TAG_SHIFT;
1183#else
1184 svlan_tag = skb_vlan_tag_get(skb) << EDMA_TX_SVLAN_TAG_SHIFT;
1185#endif
1186 break;
1187 default:
1188 dev_err(&pdev->dev, "no ctag or stag present\n");
1189 goto vlan_tag_error;
1190 }
1191 } else if (flags_transmit & EDMA_VLAN_TX_TAG_INSERT_DEFAULT_FLAG) {
1192 word3 |= (1 << EDMA_TX_INS_CVLAN);
1193 word3 |= (adapter->default_vlan_tag) << EDMA_TX_CVLAN_TAG_SHIFT;
1194 }
1195
1196 if (packet_is_rstp) {
1197 word3 |= dp_bitmap << EDMA_TPD_PORT_BITMAP_SHIFT;
1198 word3 |= from_cpu << EDMA_TPD_FROM_CPU_SHIFT;
1199 } else {
1200 word3 |= adapter->dp_bitmap << EDMA_TPD_PORT_BITMAP_SHIFT;
1201 }
1202
1203 buf_len = skb_headlen(skb);
1204
1205 if (lso_word1) {
1206 if (lso_word1 & EDMA_TPD_LSO_V2_EN) {
1207
1208 /* IPv6 LSOv2 descriptor */
1209 start_tpd = tpd = edma_get_next_tpd(edma_cinfo, queue_id);
1210 sw_desc = edma_get_tx_buffer(edma_cinfo, tpd, queue_id);
1211 sw_desc->flags |= EDMA_SW_DESC_FLAG_SKB_NONE;
1212
1213 /* LSOv2 descriptor overrides addr field to pass length */
1214 tpd->addr = cpu_to_le16(skb->len);
1215 tpd->svlan_tag = svlan_tag;
1216 tpd->word1 = word1 | lso_word1;
1217 tpd->word3 = word3;
1218 }
1219
1220 tpd = edma_get_next_tpd(edma_cinfo, queue_id);
1221 if (!start_tpd)
1222 start_tpd = tpd;
1223 sw_desc = edma_get_tx_buffer(edma_cinfo, tpd, queue_id);
1224
1225 /* The last buffer info contain the skb address,
1226 * so skb will be freed after unmap
1227 */
1228 sw_desc->length = lso_desc_len;
1229 sw_desc->flags |= EDMA_SW_DESC_FLAG_SKB_HEAD;
1230
1231 sw_desc->dma = dma_map_single(&adapter->pdev->dev,
1232 skb->data, buf_len, DMA_TO_DEVICE);
1233 if (dma_mapping_error(&pdev->dev, sw_desc->dma))
1234 goto dma_error;
1235
1236 tpd->addr = cpu_to_le32(sw_desc->dma);
1237 tpd->len = cpu_to_le16(buf_len);
1238
1239 tpd->svlan_tag = svlan_tag;
1240 tpd->word1 = word1 | lso_word1;
1241 tpd->word3 = word3;
1242
1243 /* The last buffer info contain the skb address,
1244 * so it will be freed after unmap
1245 */
1246 sw_desc->length = lso_desc_len;
1247 sw_desc->flags |= EDMA_SW_DESC_FLAG_SKB_HEAD;
1248
1249 buf_len = 0;
1250 }
1251
1252 if (likely(buf_len)) {
1253
1254 /* TODO Do not dequeue descriptor if there is a potential error */
1255 tpd = edma_get_next_tpd(edma_cinfo, queue_id);
1256
1257 if (!start_tpd)
1258 start_tpd = tpd;
1259
1260 sw_desc = edma_get_tx_buffer(edma_cinfo, tpd, queue_id);
1261
1262 /* The last buffer info contain the skb address,
1263 * so it will be free after unmap
1264 */
1265 sw_desc->length = buf_len;
1266 sw_desc->flags |= EDMA_SW_DESC_FLAG_SKB_HEAD;
1267 sw_desc->dma = dma_map_single(&adapter->pdev->dev,
1268 skb->data, buf_len, DMA_TO_DEVICE);
1269 if (dma_mapping_error(&pdev->dev, sw_desc->dma))
1270 goto dma_error;
1271
1272 tpd->addr = cpu_to_le32(sw_desc->dma);
1273 tpd->len = cpu_to_le16(buf_len);
1274
1275 tpd->svlan_tag = svlan_tag;
1276 tpd->word1 = word1 | lso_word1;
1277 tpd->word3 = word3;
1278 }
1279
1280 i = 0;
1281
1282 /* Walk through paged frags for head skb */
1283 while (nr_frags--) {
1284 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
1285 buf_len = skb_frag_size(frag);
1286 tpd = edma_get_next_tpd(edma_cinfo, queue_id);
1287 sw_desc = edma_get_tx_buffer(edma_cinfo, tpd, queue_id);
1288 sw_desc->length = buf_len;
1289 sw_desc->flags |= EDMA_SW_DESC_FLAG_SKB_FRAG;
1290
1291 sw_desc->dma = skb_frag_dma_map(&pdev->dev, frag, 0, buf_len, DMA_TO_DEVICE);
1292
1293 if (dma_mapping_error(NULL, sw_desc->dma))
1294 goto dma_error;
1295
1296 tpd->addr = cpu_to_le32(sw_desc->dma);
1297 tpd->len = cpu_to_le16(buf_len);
1298
1299 tpd->svlan_tag = svlan_tag;
1300 tpd->word1 = word1 | lso_word1;
1301 tpd->word3 = word3;
1302 i++;
1303 }
1304
1305 /* Walk through all fraglist skbs */
1306 skb_walk_frags(skb, iter_skb) {
1307 buf_len = iter_skb->len;
1308 tpd = edma_get_next_tpd(edma_cinfo, queue_id);
1309 sw_desc = edma_get_tx_buffer(edma_cinfo, tpd, queue_id);
1310 sw_desc->length = buf_len;
1311 sw_desc->dma = dma_map_single(&adapter->pdev->dev,
1312 iter_skb->data, buf_len, DMA_TO_DEVICE);
1313
1314 if (dma_mapping_error(NULL, sw_desc->dma))
1315 goto dma_error;
1316
1317 tpd->addr = cpu_to_le32(sw_desc->dma);
1318 tpd->len = cpu_to_le16(buf_len);
1319 tpd->svlan_tag = svlan_tag;
1320 tpd->word1 = word1 | lso_word1;
1321 tpd->word3 = word3;
1322 sw_desc->flags |= EDMA_SW_DESC_FLAG_SKB_FRAGLIST;
1323
1324 i = 0;
1325
1326 nr_frags = skb_shinfo(iter_skb)->nr_frags;
1327
1328 /* Walk through paged frags for this fraglist skb */
1329 while (nr_frags--) {
1330 skb_frag_t *frag = &skb_shinfo(iter_skb)->frags[i];
1331 buf_len = skb_frag_size(frag);
1332 tpd = edma_get_next_tpd(edma_cinfo, queue_id);
1333 sw_desc = edma_get_tx_buffer(edma_cinfo, tpd, queue_id);
1334 sw_desc->length = buf_len;
1335 sw_desc->flags |= EDMA_SW_DESC_FLAG_SKB_FRAG;
1336
1337 sw_desc->dma = skb_frag_dma_map(&pdev->dev, frag,
1338 0, buf_len, DMA_TO_DEVICE);
1339 if (dma_mapping_error(NULL, sw_desc->dma))
1340 goto dma_error;
1341
1342 tpd->addr = cpu_to_le32(sw_desc->dma);
1343 tpd->len = cpu_to_le16(buf_len);
1344 tpd->svlan_tag = svlan_tag;
1345 tpd->word1 = word1 | lso_word1;
1346 tpd->word3 = word3;
1347 i++;
1348 }
1349 }
1350
1351 /* If tpd or sw_desc is still unitiialized then we need to return */
1352 if ((!tpd) || (!sw_desc))
1353 return -EINVAL;
1354
1355 tpd->word1 |= 1 << EDMA_TPD_EOP_SHIFT;
1356
1357 sw_desc->skb = skb;
1358 sw_desc->flags |= EDMA_SW_DESC_FLAG_LAST;
1359
1360 return 0;
1361
1362dma_error:
1363 edma_rollback_tx(adapter, start_tpd, queue_id);
1364 dev_err(&pdev->dev, "TX DMA map failed\n");
1365vlan_tag_error:
1366 return -ENOMEM;
1367}
1368
1369/* edma_check_link()
1370 * check Link status
1371 */
1372static int edma_check_link(struct edma_adapter *adapter)
1373{
1374 struct phy_device *phydev = adapter->phydev;
1375
1376 if (!(adapter->poll_required))
1377 return __EDMA_LINKUP;
1378
1379 if (phydev->link)
1380 return __EDMA_LINKUP;
1381
1382 return __EDMA_LINKDOWN;
1383}
1384
1385/* edma_adjust_link()
1386 * check for edma link status
1387 */
1388void edma_adjust_link(struct net_device *netdev)
1389{
1390 int status;
1391 struct edma_adapter *adapter = netdev_priv(netdev);
1392 struct phy_device *phydev = adapter->phydev;
1393
1394 if (!test_bit(__EDMA_UP, &adapter->state_flags))
1395 return;
1396
1397 status = edma_check_link(adapter);
1398
1399 if (status == __EDMA_LINKUP && adapter->link_state == __EDMA_LINKDOWN) {
1400 dev_info(&adapter->pdev->dev, "%s: GMAC Link is up with phy_speed=%d\n", netdev->name, phydev->speed);
1401 adapter->link_state = __EDMA_LINKUP;
1402 netif_carrier_on(netdev);
1403 if (netif_running(netdev))
1404 netif_tx_wake_all_queues(netdev);
1405 } else if (status == __EDMA_LINKDOWN && adapter->link_state == __EDMA_LINKUP) {
1406 dev_info(&adapter->pdev->dev, "%s: GMAC Link is down\n", netdev->name);
1407 adapter->link_state = __EDMA_LINKDOWN;
1408 netif_carrier_off(netdev);
1409 netif_tx_stop_all_queues(netdev);
1410 }
1411}
1412
Bhaskar Valabojue429bab2017-03-15 09:01:23 +05301413/* edma_get_stats64()
Rakesh Nair9bcf2602017-01-06 16:02:16 +05301414 * Statistics api used to retreive the tx/rx statistics
1415 */
Bhaskar Valabojue429bab2017-03-15 09:01:23 +05301416struct rtnl_link_stats64 *edma_get_stats64(struct net_device *netdev,
1417 struct rtnl_link_stats64 *stats)
Rakesh Nair9bcf2602017-01-06 16:02:16 +05301418{
1419 struct edma_adapter *adapter = netdev_priv(netdev);
1420
Bhaskar Valabojue429bab2017-03-15 09:01:23 +05301421 memcpy(stats, &adapter->stats, sizeof(*stats));
1422
1423 return stats;
Rakesh Nair9bcf2602017-01-06 16:02:16 +05301424}
1425
1426/* edma_xmit()
1427 * Main api to be called by the core for packet transmission
1428 */
1429netdev_tx_t edma_xmit(struct sk_buff *skb,
1430 struct net_device *net_dev)
1431{
1432 struct edma_adapter *adapter = netdev_priv(net_dev);
1433 struct edma_common_info *edma_cinfo = adapter->edma_cinfo;
1434 struct edma_tx_desc_ring *etdr;
1435 u16 from_cpu = 0, dp_bitmap = 0, txq_id;
1436 int ret, nr_frags_first = 0, num_tpds_needed = 1, queue_id = 0;
1437 unsigned int flags_transmit = 0;
1438 bool packet_is_rstp = false;
1439 struct netdev_queue *nq = NULL;
1440
1441 if (skb_shinfo(skb)->nr_frags) {
1442 nr_frags_first = skb_shinfo(skb)->nr_frags;
1443
1444 /* It is unlikely below check hits, BUG_ON */
1445 BUG_ON(nr_frags_first > MAX_SKB_FRAGS);
1446
1447 num_tpds_needed += nr_frags_first;
1448 }
1449
1450 if (skb_has_frag_list(skb)) {
1451 struct sk_buff *iter_skb;
1452
1453 /* Walk through fraglist skbs making a note of nr_frags */
1454 skb_walk_frags(skb, iter_skb) {
1455 unsigned char nr_frags = skb_shinfo(iter_skb)->nr_frags;
1456
1457 /* It is unlikely below check hits, BUG_ON */
1458 BUG_ON(nr_frags > MAX_SKB_FRAGS);
1459
1460 /* One TPD for skb->data and more for nr_frags */
1461 num_tpds_needed += (1 + nr_frags);
1462 }
1463 }
1464
1465 if (edma_stp_rstp) {
1466 u16 ath_hdr, ath_eth_type;
1467 u8 mac_addr[EDMA_ETH_HDR_LEN];
1468 ath_eth_type = ntohs(*(uint16_t *)&skb->data[12]);
1469 if (ath_eth_type == edma_ath_eth_type) {
1470 packet_is_rstp = true;
1471 ath_hdr = htons(*(uint16_t *)&skb->data[14]);
1472 dp_bitmap = ath_hdr & EDMA_TX_ATH_HDR_PORT_BITMAP_MASK;
1473 from_cpu = (ath_hdr & EDMA_TX_ATH_HDR_FROM_CPU_MASK) >> EDMA_TX_ATH_HDR_FROM_CPU_SHIFT;
1474 memcpy(mac_addr, skb->data, EDMA_ETH_HDR_LEN);
1475
1476 skb_pull(skb, 4);
1477
1478 memcpy(skb->data, mac_addr, EDMA_ETH_HDR_LEN);
1479 }
1480 }
1481
1482 /* this will be one of the 4 TX queues exposed to linux kernel */
1483 txq_id = skb_get_queue_mapping(skb);
1484 queue_id = edma_tx_queue_get(adapter, skb, txq_id);
1485 etdr = edma_cinfo->tpd_ring[queue_id];
1486 nq = netdev_get_tx_queue(net_dev, txq_id);
1487
1488 local_bh_disable();
1489 /* Tx is not handled in bottom half context. Hence, we need to protect
1490 * Tx from tasks and bottom half
1491 */
1492
1493 if (num_tpds_needed > edma_tpd_available(edma_cinfo, queue_id)) {
1494 /* not enough descriptor, just stop queue */
1495 netif_tx_stop_queue(nq);
1496 local_bh_enable();
1497 dev_dbg(&net_dev->dev, "Not enough descriptors available");
1498 edma_cinfo->edma_ethstats.tx_desc_error++;
1499 return NETDEV_TX_BUSY;
1500 }
1501
1502 /* Check and mark VLAN tag offload */
1503#if (LINUX_VERSION_CODE <= KERNEL_VERSION(3, 18, 21))
1504 if (vlan_tx_tag_present(skb))
1505#else
1506 if (skb_vlan_tag_present(skb))
1507#endif
1508 flags_transmit |= EDMA_VLAN_TX_TAG_INSERT_FLAG;
1509 else if (adapter->default_vlan_tag)
1510 flags_transmit |= EDMA_VLAN_TX_TAG_INSERT_DEFAULT_FLAG;
1511
1512 /* Check and mark checksum offload */
1513 if (likely(skb->ip_summed == CHECKSUM_PARTIAL))
1514 flags_transmit |= EDMA_HW_CHECKSUM;
1515
1516 /* Map and fill descriptor for Tx */
1517 ret = edma_tx_map_and_fill(edma_cinfo, adapter, skb, queue_id,
1518 flags_transmit, from_cpu, dp_bitmap,
1519 packet_is_rstp, nr_frags_first);
1520 if (ret) {
1521 dev_kfree_skb_any(skb);
1522 adapter->stats.tx_errors++;
1523 goto netdev_okay;
1524 }
1525
1526 /* Update SW producer index */
1527 edma_tx_update_hw_idx(edma_cinfo, skb, queue_id);
1528
1529 /* update tx statistics */
1530 adapter->stats.tx_packets++;
1531 adapter->stats.tx_bytes += skb->len;
1532
1533netdev_okay:
1534 local_bh_enable();
1535 return NETDEV_TX_OK;
1536}
1537
1538/*
1539 * edma_flow_may_expire()
1540 * Timer function called periodically to delete the node
1541 */
1542void edma_flow_may_expire(unsigned long data)
1543{
1544 struct edma_adapter *adapter = (struct edma_adapter *)data;
1545 int j;
1546
1547 spin_lock_bh(&adapter->rfs.rfs_ftab_lock);
1548 for (j = 0; j < EDMA_RFS_EXPIRE_COUNT_PER_CALL; j++) {
1549 struct hlist_head *hhead;
1550 struct hlist_node *tmp;
1551 struct edma_rfs_filter_node *n;
1552 bool res;
1553
1554 hhead = &adapter->rfs.hlist_head[adapter->rfs.hashtoclean++];
1555 hlist_for_each_entry_safe(n, tmp, hhead, node) {
1556 res = rps_may_expire_flow(adapter->netdev, n->rq_id,
1557 n->flow_id, n->filter_id);
1558 if (res) {
1559 res = edma_delete_rfs_filter(adapter, n);
1560 if (res < 0)
1561 dev_dbg(&adapter->netdev->dev,
1562 "RFS entry %d not allowed to be flushed by Switch",
1563 n->flow_id);
1564 else {
1565 hlist_del(&n->node);
1566 kfree(n);
1567 adapter->rfs.filter_available++;
1568 }
1569 }
1570 }
1571 }
1572
1573 adapter->rfs.hashtoclean = adapter->rfs.hashtoclean & (EDMA_RFS_FLOW_ENTRIES - 1);
1574 spin_unlock_bh(&adapter->rfs.rfs_ftab_lock);
1575 mod_timer(&adapter->rfs.expire_rfs, jiffies + HZ/4);
1576}
1577
1578/* edma_rx_flow_steer()
1579 * Called by core to to steer the flow to CPU
1580 */
1581int edma_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb,
1582 u16 rxq, u32 flow_id)
1583{
1584 struct flow_keys keys;
1585 struct edma_rfs_filter_node *filter_node;
1586 struct edma_adapter *adapter = netdev_priv(dev);
1587 u16 hash_tblid;
1588 int res;
1589
1590 if (skb->protocol == htons(ETH_P_IPV6)) {
1591 res = -EPROTONOSUPPORT;
1592 goto no_protocol_err;
1593 }
1594
1595 /* Dissect flow parameters
1596 * We only support IPv4 + TCP/UDP
1597 */
1598#if (LINUX_VERSION_CODE <= KERNEL_VERSION(3, 18, 21))
1599 res = skb_flow_dissect(skb, &keys);
1600 if (!((keys.ip_proto == IPPROTO_TCP) || (keys.ip_proto == IPPROTO_UDP))) {
1601#else
1602 res = skb_flow_dissect_flow_keys(skb, &keys, 0);
1603 if (!((keys.basic.ip_proto == IPPROTO_TCP) || (keys.basic.ip_proto == IPPROTO_UDP))) {
1604#endif
1605 res = -EPROTONOSUPPORT;
1606 goto no_protocol_err;
1607 }
1608
1609 /* Check if table entry exists */
1610 hash_tblid = skb_get_hash_raw(skb) & EDMA_RFS_FLOW_ENTRIES_MASK;
1611
1612 spin_lock_bh(&adapter->rfs.rfs_ftab_lock);
1613 filter_node = edma_rfs_key_search(&adapter->rfs.hlist_head[hash_tblid], &keys);
1614
1615 if (filter_node) {
1616 if (rxq == filter_node->rq_id) {
1617 res = -EEXIST;
1618 goto out;
1619 } else {
1620 res = edma_delete_rfs_filter(adapter, filter_node);
1621 if (res < 0)
1622 dev_warn(&adapter->netdev->dev,
1623 "Cannot steer flow %d to different queue",
1624 filter_node->flow_id);
1625 else {
1626 adapter->rfs.filter_available++;
1627 res = edma_add_rfs_filter(adapter, &keys, rxq, filter_node);
1628 if (res < 0) {
1629 dev_warn(&adapter->netdev->dev,
1630 "Cannot steer flow %d to different queue",
1631 filter_node->flow_id);
1632 } else {
1633 adapter->rfs.filter_available--;
1634 filter_node->rq_id = rxq;
1635 filter_node->filter_id = res;
1636 }
1637 }
1638 }
1639 } else {
1640 if (adapter->rfs.filter_available == 0) {
1641 res = -EBUSY;
1642 goto out;
1643 }
1644
1645 filter_node = kmalloc(sizeof(*filter_node), GFP_ATOMIC);
1646 if (!filter_node) {
1647 res = -ENOMEM;
1648 goto out;
1649 }
1650
1651 res = edma_add_rfs_filter(adapter, &keys, rxq, filter_node);
1652 if (res < 0) {
1653 kfree(filter_node);
1654 goto out;
1655 }
1656
1657 adapter->rfs.filter_available--;
1658 filter_node->rq_id = rxq;
1659 filter_node->filter_id = res;
1660 filter_node->flow_id = flow_id;
1661 filter_node->keys = keys;
1662 INIT_HLIST_NODE(&filter_node->node);
1663 hlist_add_head(&filter_node->node, &adapter->rfs.hlist_head[hash_tblid]);
1664 }
1665
1666out:
1667 spin_unlock_bh(&adapter->rfs.rfs_ftab_lock);
1668no_protocol_err:
1669 return res;
1670}
1671
1672#ifdef CONFIG_RFS_ACCEL
1673/* edma_register_rfs_filter()
1674 * Add RFS filter callback
1675 */
1676int edma_register_rfs_filter(struct net_device *netdev,
1677 set_rfs_filter_callback_t set_filter)
1678{
1679 struct edma_adapter *adapter = netdev_priv(netdev);
1680
1681 spin_lock_bh(&adapter->rfs.rfs_ftab_lock);
1682
1683 if (adapter->set_rfs_rule) {
1684 spin_unlock_bh(&adapter->rfs.rfs_ftab_lock);
1685 return -1;
1686 }
1687
1688 adapter->set_rfs_rule = set_filter;
1689 spin_unlock_bh(&adapter->rfs.rfs_ftab_lock);
1690
1691 return 0;
1692}
1693#endif
1694
1695/* edma_select_xps_queue()
1696 * Called by Linux TX stack to populate Linux TX queue
1697 */
1698u16 edma_select_xps_queue(struct net_device *dev, struct sk_buff *skb,
1699 void *accel_priv, select_queue_fallback_t fallback)
1700{
1701#if (LINUX_VERSION_CODE <= KERNEL_VERSION(3, 18, 21))
1702 return smp_processor_id();
1703#else
1704 int cpu = get_cpu();
1705 put_cpu();
1706
1707 return cpu;
1708#endif
1709}
1710
1711/* edma_alloc_tx_rings()
1712 * Allocate rx rings
1713 */
1714int edma_alloc_tx_rings(struct edma_common_info *edma_cinfo)
1715{
1716 struct platform_device *pdev = edma_cinfo->pdev;
1717 int i, err = 0;
1718
1719 for (i = 0; i < edma_cinfo->num_tx_queues; i++) {
1720 err = edma_alloc_tx_ring(edma_cinfo, edma_cinfo->tpd_ring[i]);
1721 if (err) {
1722 dev_err(&pdev->dev, "Tx Queue alloc %u failed\n", i);
1723 return err;
1724 }
1725 }
1726
1727 return 0;
1728}
1729
1730/* edma_free_tx_rings()
1731 * Free tx rings
1732 */
1733void edma_free_tx_rings(struct edma_common_info *edma_cinfo)
1734{
1735 int i;
1736
1737 for (i = 0; i < edma_cinfo->num_tx_queues; i++)
1738 edma_free_tx_ring(edma_cinfo, edma_cinfo->tpd_ring[i]);
1739}
1740
1741/* edma_free_tx_resources()
1742 * Free buffers associated with tx rings
1743 */
1744void edma_free_tx_resources(struct edma_common_info *edma_cinfo)
1745{
1746 struct edma_tx_desc_ring *etdr;
1747 struct edma_sw_desc *sw_desc;
1748 struct platform_device *pdev = edma_cinfo->pdev;
1749 int i, j;
1750
1751 for (i = 0; i < edma_cinfo->num_tx_queues; i++) {
1752 etdr = edma_cinfo->tpd_ring[i];
1753 for (j = 0; j < EDMA_TX_RING_SIZE; j++) {
1754 sw_desc = &etdr->sw_desc[j];
1755 if (sw_desc->flags & (EDMA_SW_DESC_FLAG_SKB_HEAD |
1756 EDMA_SW_DESC_FLAG_SKB_FRAG | EDMA_SW_DESC_FLAG_SKB_FRAGLIST))
1757 edma_tx_unmap_and_free(pdev, sw_desc);
1758 }
1759 }
1760}
1761
1762/* edma_alloc_rx_rings()
1763 * Allocate rx rings
1764 */
1765int edma_alloc_rx_rings(struct edma_common_info *edma_cinfo)
1766{
1767 struct platform_device *pdev = edma_cinfo->pdev;
1768 int i, j, err = 0;
1769
1770 for (i = 0, j = 0; i < edma_cinfo->num_rx_queues; i++) {
1771 err = edma_alloc_rx_ring(edma_cinfo, edma_cinfo->rfd_ring[j]);
1772 if (err) {
1773 dev_err(&pdev->dev, "Rx Queue alloc%u failed\n", i);
1774 return err;
1775 }
1776 j += ((edma_cinfo->num_rx_queues == 4) ? 2 : 1);
1777 }
1778
1779 return 0;
1780}
1781
1782/* edma_free_rx_rings()
1783 * free rx rings
1784 */
1785void edma_free_rx_rings(struct edma_common_info *edma_cinfo)
1786{
1787 int i, j;
1788
1789 for (i = 0, j = 0; i < edma_cinfo->num_rx_queues; i++) {
1790 edma_free_rx_ring(edma_cinfo, edma_cinfo->rfd_ring[j]);
1791 j += ((edma_cinfo->num_rx_queues == 4) ? 2 : 1);
1792 }
1793}
1794
1795/* edma_free_queues()
1796 * Free the queues allocaated
1797 */
1798void edma_free_queues(struct edma_common_info *edma_cinfo)
1799{
1800 int i , j;
1801
1802 for (i = 0; i < edma_cinfo->num_tx_queues; i++) {
1803 if (edma_cinfo->tpd_ring[i])
1804 kfree(edma_cinfo->tpd_ring[i]);
1805 edma_cinfo->tpd_ring[i] = NULL;
1806 }
1807
1808 for (i = 0, j = 0; i < edma_cinfo->num_rx_queues; i++) {
1809 if (edma_cinfo->rfd_ring[j])
1810 kfree(edma_cinfo->rfd_ring[j]);
1811 edma_cinfo->rfd_ring[j] = NULL;
1812 j += ((edma_cinfo->num_rx_queues == 4) ? 2 : 1);
1813 }
1814
1815 edma_cinfo->num_rx_queues = 0;
1816 edma_cinfo->num_tx_queues = 0;
1817
1818 return;
1819}
1820
1821/* edma_free_rx_resources()
1822 * Free buffers associated with tx rings
1823 */
1824void edma_free_rx_resources(struct edma_common_info *edma_cinfo)
1825{
1826 struct edma_rfd_desc_ring *erdr;
1827 struct platform_device *pdev = edma_cinfo->pdev;
1828 int i, j, k;
1829
1830 for (i = 0, k = 0; i < edma_cinfo->num_rx_queues; i++) {
1831 erdr = edma_cinfo->rfd_ring[k];
1832 for (j = 0; j < EDMA_RX_RING_SIZE; j++) {
1833 /* unmap all descriptors while cleaning */
1834 edma_clean_rfd(pdev, erdr, j, 1);
1835 }
1836 k += ((edma_cinfo->num_rx_queues == 4) ? 2 : 1);
1837
1838 }
1839}
1840
1841/* edma_alloc_queues_tx()
1842 * Allocate memory for all rings
1843 */
1844int edma_alloc_queues_tx(struct edma_common_info *edma_cinfo)
1845{
1846 int i;
1847
1848 for (i = 0; i < edma_cinfo->num_tx_queues; i++) {
1849 struct edma_tx_desc_ring *etdr;
1850 etdr = kzalloc(sizeof(struct edma_tx_desc_ring), GFP_KERNEL);
1851 if (!etdr)
1852 goto err;
1853 etdr->count = edma_cinfo->tx_ring_count;
1854 edma_cinfo->tpd_ring[i] = etdr;
1855 }
1856
1857 return 0;
1858err:
1859 edma_free_queues(edma_cinfo);
1860 return -1;
1861}
1862
1863/* edma_alloc_queues_rx()
1864 * Allocate memory for all rings
1865 */
1866int edma_alloc_queues_rx(struct edma_common_info *edma_cinfo)
1867{
1868 int i, j;
1869
1870 for (i = 0, j = 0; i < edma_cinfo->num_rx_queues; i++) {
1871 struct edma_rfd_desc_ring *rfd_ring;
1872 rfd_ring = kzalloc(sizeof(struct edma_rfd_desc_ring),
1873 GFP_KERNEL);
1874 if (!rfd_ring)
1875 goto err;
1876 rfd_ring->count = edma_cinfo->rx_ring_count;
1877 edma_cinfo->rfd_ring[j] = rfd_ring;
1878 j += ((edma_cinfo->num_rx_queues == 4) ? 2 : 1);
1879 }
1880 return 0;
1881err:
1882 edma_free_queues(edma_cinfo);
1883 return -1;
1884}
1885
1886/* edma_clear_irq_status()
1887 * Clear interrupt status
1888 */
1889void edma_clear_irq_status(void)
1890{
1891 edma_write_reg(EDMA_REG_RX_ISR, 0xff);
1892 edma_write_reg(EDMA_REG_TX_ISR, 0xffff);
1893 edma_write_reg(EDMA_REG_MISC_ISR, 0x1fff);
1894 edma_write_reg(EDMA_REG_WOL_ISR, 0x1);
1895};
1896
1897/* edma_configure()
1898 * Configure skb, edma interrupts and control register.
1899 */
1900int edma_configure(struct edma_common_info *edma_cinfo)
1901{
1902 struct edma_hw *hw = &edma_cinfo->hw;
1903 u32 intr_modrt_data;
1904 u32 intr_ctrl_data = 0;
1905 int i, j, ret_count;
1906
1907 edma_read_reg(EDMA_REG_INTR_CTRL, &intr_ctrl_data);
1908 intr_ctrl_data &= ~(1 << EDMA_INTR_SW_IDX_W_TYP_SHIFT);
1909 intr_ctrl_data |= hw->intr_sw_idx_w << EDMA_INTR_SW_IDX_W_TYP_SHIFT;
1910 edma_write_reg(EDMA_REG_INTR_CTRL, intr_ctrl_data);
1911
1912 edma_clear_irq_status();
1913
1914 /* Clear any WOL status */
1915 edma_write_reg(EDMA_REG_WOL_CTRL, 0);
1916 intr_modrt_data = (EDMA_TX_IMT << EDMA_IRQ_MODRT_TX_TIMER_SHIFT);
1917 intr_modrt_data |= (EDMA_RX_IMT << EDMA_IRQ_MODRT_RX_TIMER_SHIFT);
1918 edma_write_reg(EDMA_REG_IRQ_MODRT_TIMER_INIT, intr_modrt_data);
1919 edma_configure_tx(edma_cinfo);
1920 edma_configure_rx(edma_cinfo);
1921
1922 /* Allocate the RX buffer */
1923 for (i = 0, j = 0; i < edma_cinfo->num_rx_queues; i++) {
1924 struct edma_rfd_desc_ring *ring = edma_cinfo->rfd_ring[j];
1925 ret_count = edma_alloc_rx_buf(edma_cinfo, ring, ring->count, j);
1926 if (ret_count)
1927 dev_dbg(&edma_cinfo->pdev->dev, "not all rx buffers allocated\n");
1928 j += ((edma_cinfo->num_rx_queues == 4) ? 2 : 1);
1929 }
1930
1931 /* Configure descriptor Ring */
1932 edma_init_desc(edma_cinfo);
1933 return 0;
1934}
1935
1936/* edma_irq_enable()
1937 * Enable default interrupt generation settings
1938 */
1939void edma_irq_enable(struct edma_common_info *edma_cinfo)
1940{
1941 struct edma_hw *hw = &edma_cinfo->hw;
1942 int i, j;
1943
1944 edma_write_reg(EDMA_REG_RX_ISR, 0xff);
1945 for (i = 0, j = 0; i < edma_cinfo->num_rx_queues; i++) {
1946 edma_write_reg(EDMA_REG_RX_INT_MASK_Q(j), hw->rx_intr_mask);
1947 j += ((edma_cinfo->num_rx_queues == 4) ? 2 : 1);
1948 }
1949 edma_write_reg(EDMA_REG_TX_ISR, 0xffff);
1950 for (i = 0; i < edma_cinfo->num_tx_queues; i++)
1951 edma_write_reg(EDMA_REG_TX_INT_MASK_Q(i), hw->tx_intr_mask);
1952}
1953
1954/* edma_irq_disable()
1955 * Disable Interrupt
1956 */
1957void edma_irq_disable(struct edma_common_info *edma_cinfo)
1958{
1959 int i;
1960
1961 for (i = 0; i < EDMA_MAX_RECEIVE_QUEUE; i++)
1962 edma_write_reg(EDMA_REG_RX_INT_MASK_Q(i), 0x0);
1963
1964 for (i = 0; i < EDMA_MAX_TRANSMIT_QUEUE; i++)
1965 edma_write_reg(EDMA_REG_TX_INT_MASK_Q(i), 0x0);
1966 edma_write_reg(EDMA_REG_MISC_IMR, 0);
1967 edma_write_reg(EDMA_REG_WOL_IMR, 0);
1968}
1969
1970/* edma_free_irqs()
1971 * Free All IRQs
1972 */
1973void edma_free_irqs(struct edma_adapter *adapter)
1974{
1975 struct edma_common_info *edma_cinfo = adapter->edma_cinfo;
1976 int i, j;
1977 int k = ((edma_cinfo->num_rx_queues == 4) ? 1 : 2);
1978
1979 for (i = 0; i < CONFIG_NR_CPUS; i++) {
1980 for (j = edma_cinfo->edma_percpu_info[i].tx_start; j < (edma_cinfo->edma_percpu_info[i].tx_start + 4); j++)
1981 free_irq(edma_cinfo->tx_irq[j], &edma_cinfo->edma_percpu_info[i]);
1982
1983 for (j = edma_cinfo->edma_percpu_info[i].rx_start; j < (edma_cinfo->edma_percpu_info[i].rx_start + k); j++)
1984 free_irq(edma_cinfo->rx_irq[j], &edma_cinfo->edma_percpu_info[i]);
1985 }
1986}
1987
1988/* edma_enable_rx_ctrl()
1989 * Enable RX queue control
1990 */
1991void edma_enable_rx_ctrl(struct edma_hw *hw)
1992{
1993 u32 data;
1994
1995 edma_read_reg(EDMA_REG_RXQ_CTRL, &data);
1996 data |= EDMA_RXQ_CTRL_EN;
1997 edma_write_reg(EDMA_REG_RXQ_CTRL, data);
1998}
1999
2000
2001/* edma_enable_tx_ctrl()
2002 * Enable TX queue control
2003 */
2004void edma_enable_tx_ctrl(struct edma_hw *hw)
2005{
2006 u32 data;
2007
2008 edma_read_reg(EDMA_REG_TXQ_CTRL, &data);
2009 data |= EDMA_TXQ_CTRL_TXQ_EN;
2010 edma_write_reg(EDMA_REG_TXQ_CTRL, data);
2011}
2012
2013/* edma_stop_rx_tx()
2014 * Disable RX/TQ Queue control
2015 */
2016void edma_stop_rx_tx(struct edma_hw *hw)
2017{
2018 u32 data;
2019
2020 edma_read_reg(EDMA_REG_RXQ_CTRL, &data);
2021 data &= ~EDMA_RXQ_CTRL_EN;
2022 edma_write_reg(EDMA_REG_RXQ_CTRL, data);
2023 edma_read_reg(EDMA_REG_TXQ_CTRL, &data);
2024 data &= ~EDMA_TXQ_CTRL_TXQ_EN;
2025 edma_write_reg(EDMA_REG_TXQ_CTRL, data);
2026}
2027
2028/* edma_reset()
2029 * Reset the EDMA
2030 */
2031int edma_reset(struct edma_common_info *edma_cinfo)
2032{
2033 struct edma_hw *hw = &edma_cinfo->hw;
2034
2035 edma_irq_disable(edma_cinfo);
2036
2037 edma_clear_irq_status();
2038
2039 edma_stop_rx_tx(hw);
2040
2041 return 0;
2042}
2043
2044/* edma_fill_netdev()
2045 * Fill netdev for each etdr
2046 */
2047int edma_fill_netdev(struct edma_common_info *edma_cinfo, int queue_id,
2048 int dev, int txq_id)
2049{
2050 struct edma_tx_desc_ring *etdr;
2051 int i = 0;
2052
2053 etdr = edma_cinfo->tpd_ring[queue_id];
2054
2055 while (etdr->netdev[i])
2056 i++;
2057
2058 if (i >= EDMA_MAX_NETDEV_PER_QUEUE)
2059 return -1;
2060
2061 /* Populate the netdev associated with the tpd ring */
2062 etdr->netdev[i] = edma_netdev[dev];
2063 etdr->nq[i] = netdev_get_tx_queue(edma_netdev[dev], txq_id);
2064
2065 return 0;
2066}
2067
2068/* edma_change_mtu()
2069 * change the MTU of the NIC.
2070 */
2071int edma_change_mtu(struct net_device *netdev, int new_mtu)
2072{
2073 struct edma_adapter *adapter = netdev_priv(netdev);
2074 struct edma_common_info *edma_cinfo = adapter->edma_cinfo;
2075 int old_mtu = netdev->mtu;
2076 int max_frame_size = new_mtu + ETH_HLEN + ETH_FCS_LEN + (2 * VLAN_HLEN);
2077
2078 if ((max_frame_size < ETH_ZLEN + ETH_FCS_LEN) ||
2079 (max_frame_size > EDMA_MAX_JUMBO_FRAME_SIZE)) {
2080 dev_err(&edma_cinfo->pdev->dev, "MTU setting not correct\n");
2081 return -EINVAL;
2082 }
2083
2084 /* set MTU */
2085 if (old_mtu != new_mtu) {
2086 netdev->mtu = new_mtu;
2087 netdev_update_features(netdev);
2088 }
2089
2090 return 0;
2091}
2092
2093/* edma_set_mac()
2094 * Change the Ethernet Address of the NIC
2095 */
2096int edma_set_mac_addr(struct net_device *netdev, void *p)
2097{
2098 struct sockaddr *addr = p;
2099
2100 if (!is_valid_ether_addr(addr->sa_data))
2101 return -EINVAL;
2102
2103 if (netif_running(netdev))
2104 return -EBUSY;
2105
2106 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
2107 return 0;
2108}
2109
2110/* edma_set_stp_rstp()
2111 * set stp/rstp
2112 */
2113void edma_set_stp_rstp(bool rstp)
2114{
2115 edma_stp_rstp = rstp;
2116}
2117
2118/* edma_assign_ath_hdr_type()
2119 * assign atheros header eth type
2120 */
2121void edma_assign_ath_hdr_type(int eth_type)
2122{
2123 edma_ath_eth_type = eth_type & EDMA_ETH_TYPE_MASK;
2124}
2125
2126/* edma_get_default_vlan_tag()
2127 * Used by other modules to get the default vlan tag
2128 */
2129int edma_get_default_vlan_tag(struct net_device *netdev)
2130{
2131 struct edma_adapter *adapter = netdev_priv(netdev);
2132
2133 if (adapter->default_vlan_tag)
2134 return adapter->default_vlan_tag;
2135
2136 return 0;
2137}
2138
2139/* edma_open()
2140 * gets called when netdevice is up, start the queue.
2141 */
2142int edma_open(struct net_device *netdev)
2143{
2144 struct edma_adapter *adapter = netdev_priv(netdev);
2145 struct platform_device *pdev = adapter->edma_cinfo->pdev;
2146
2147 netif_tx_start_all_queues(netdev);
2148 edma_initialise_rfs_flow_table(adapter);
2149 set_bit(__EDMA_UP, &adapter->state_flags);
2150
2151 /* if Link polling is enabled, in our case enabled for WAN, then
2152 * do a phy start, else always set link as UP
2153 */
2154 if (adapter->poll_required) {
2155 if (!IS_ERR(adapter->phydev)) {
2156 phy_start(adapter->phydev);
2157 phy_start_aneg(adapter->phydev);
2158 adapter->link_state = __EDMA_LINKDOWN;
2159 } else {
2160 dev_dbg(&pdev->dev, "Invalid PHY device for a link polled interface\n");
2161 }
2162 } else {
2163 adapter->link_state = __EDMA_LINKUP;
2164 netif_carrier_on(netdev);
2165 }
2166
2167 return 0;
2168}
2169
2170
2171/* edma_close()
2172 * gets called when netdevice is down, stops the queue.
2173 */
2174int edma_close(struct net_device *netdev)
2175{
2176 struct edma_adapter *adapter = netdev_priv(netdev);
2177
2178 edma_free_rfs_flow_table(adapter);
2179 netif_carrier_off(netdev);
2180 netif_tx_stop_all_queues(netdev);
2181
2182 if (adapter->poll_required) {
2183 if (!IS_ERR(adapter->phydev))
2184 phy_stop(adapter->phydev);
2185 }
2186
2187 adapter->link_state = __EDMA_LINKDOWN;
2188
2189 /* Set GMAC state to UP before link state is checked
2190 */
2191 clear_bit(__EDMA_UP, &adapter->state_flags);
2192
2193 return 0;
2194}
2195
2196/* edma_poll
2197 * polling function that gets called when the napi gets scheduled.
2198 *
2199 * Main sequence of task performed in this api
2200 * is clear irq status -> clear_tx_irq -> clean_rx_irq->
2201 * enable interrupts.
2202 */
2203int edma_poll(struct napi_struct *napi, int budget)
2204{
2205 struct edma_per_cpu_queues_info *edma_percpu_info = container_of(napi,
2206 struct edma_per_cpu_queues_info, napi);
2207 struct edma_common_info *edma_cinfo = edma_percpu_info->edma_cinfo;
2208 u32 reg_data;
2209 u32 shadow_rx_status, shadow_tx_status;
2210 int queue_id;
2211 int i, work_done = 0;
2212
2213 /* Store the Rx/Tx status by ANDing it with
2214 * appropriate CPU RX?TX mask
2215 */
2216 edma_read_reg(EDMA_REG_RX_ISR, &reg_data);
2217 edma_percpu_info->rx_status |= reg_data & edma_percpu_info->rx_mask;
2218 shadow_rx_status = edma_percpu_info->rx_status;
2219 edma_read_reg(EDMA_REG_TX_ISR, &reg_data);
2220 edma_percpu_info->tx_status |= reg_data & edma_percpu_info->tx_mask;
2221 shadow_tx_status = edma_percpu_info->tx_status;
2222
2223 /* Every core will have a start, which will be computed
2224 * in probe and stored in edma_percpu_info->tx_start variable.
2225 * We will shift the status bit by tx_start to obtain
2226 * status bits for the core on which the current processing
2227 * is happening. Since, there are 4 tx queues per core,
2228 * we will run the loop till we get the correct queue to clear.
2229 */
2230 while (edma_percpu_info->tx_status) {
2231 queue_id = ffs(edma_percpu_info->tx_status) - 1;
2232 edma_tx_complete(edma_cinfo, queue_id);
2233 edma_percpu_info->tx_status &= ~(1 << queue_id);
2234 }
2235
2236 /* Every core will have a start, which will be computed
2237 * in probe and stored in edma_percpu_info->tx_start variable.
2238 * We will shift the status bit by tx_start to obtain
2239 * status bits for the core on which the current processing
2240 * is happening. Since, there are 4 tx queues per core, we
2241 * will run the loop till we get the correct queue to clear.
2242 */
2243 while (edma_percpu_info->rx_status) {
2244 queue_id = ffs(edma_percpu_info->rx_status) - 1;
2245 edma_rx_complete(edma_cinfo, &work_done,
2246 budget, queue_id, napi);
2247
2248 if (likely(work_done < budget))
2249 edma_percpu_info->rx_status &= ~(1 << queue_id);
2250 else
2251 break;
2252 }
2253
2254 /* Clear the status register, to avoid the interrupts to
2255 * reoccur.This clearing of interrupt status register is
2256 * done here as writing to status register only takes place
2257 * once the producer/consumer index has been updated to
2258 * reflect that the packet transmission/reception went fine.
2259 */
2260 edma_write_reg(EDMA_REG_RX_ISR, shadow_rx_status);
2261 edma_write_reg(EDMA_REG_TX_ISR, shadow_tx_status);
2262
2263 /* If budget not fully consumed, exit the polling mode */
2264 if (likely(work_done < budget)) {
2265 napi_complete(napi);
2266
2267 /* re-enable the interrupts */
2268 for (i = 0; i < edma_cinfo->num_rxq_per_core; i++)
2269 edma_write_reg(EDMA_REG_RX_INT_MASK_Q(edma_percpu_info->rx_start + i), 0x1);
2270 for (i = 0; i < edma_cinfo->num_txq_per_core; i++)
2271 edma_write_reg(EDMA_REG_TX_INT_MASK_Q(edma_percpu_info->tx_start + i), 0x1);
2272 }
2273
2274 return work_done;
2275}
2276
2277/* edma interrupt()
2278 * interrupt handler
2279 */
2280irqreturn_t edma_interrupt(int irq, void *dev)
2281{
2282 struct edma_per_cpu_queues_info *edma_percpu_info = (struct edma_per_cpu_queues_info *) dev;
2283 struct edma_common_info *edma_cinfo = edma_percpu_info->edma_cinfo;
2284 int i;
2285
2286 /* Unmask the TX/RX interrupt register */
2287 for (i = 0; i < edma_cinfo->num_rxq_per_core; i++)
2288 edma_write_reg(EDMA_REG_RX_INT_MASK_Q(edma_percpu_info->rx_start + i), 0x0);
2289
2290 for (i = 0; i < edma_cinfo->num_txq_per_core; i++)
2291 edma_write_reg(EDMA_REG_TX_INT_MASK_Q(edma_percpu_info->tx_start + i), 0x0);
2292
2293 napi_schedule(&edma_percpu_info->napi);
2294
2295 return IRQ_HANDLED;
2296}