Damjan Marion | c517020 | 2017-10-11 14:15:47 +0200 | [diff] [blame] | 1 | /* |
| 2 | *------------------------------------------------------------------ |
Damjan Marion | 83243a0 | 2016-02-29 13:09:30 +0100 | [diff] [blame] | 3 | * af_packet.c - linux kernel packet interface |
| 4 | * |
| 5 | * Copyright (c) 2016 Cisco and/or its affiliates. |
| 6 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 7 | * you may not use this file except in compliance with the License. |
| 8 | * You may obtain a copy of the License at: |
| 9 | * |
| 10 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 11 | * |
| 12 | * Unless required by applicable law or agreed to in writing, software |
| 13 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 15 | * See the License for the specific language governing permissions and |
| 16 | * limitations under the License. |
| 17 | *------------------------------------------------------------------ |
| 18 | */ |
| 19 | |
| 20 | #include <linux/if_packet.h> |
| 21 | |
| 22 | #include <vlib/vlib.h> |
| 23 | #include <vlib/unix/unix.h> |
| 24 | #include <vnet/ip/ip.h> |
| 25 | #include <vnet/ethernet/ethernet.h> |
Mohammed Hawari | 85c1943 | 2020-12-18 16:29:45 +0100 | [diff] [blame] | 26 | #include <vnet/interface/rx_queue_funcs.h> |
Damjan Marion | 2231150 | 2016-10-28 20:30:15 +0200 | [diff] [blame] | 27 | #include <vnet/feature/feature.h> |
Jakub Grajciar | 92b0275 | 2017-10-20 13:37:28 +0200 | [diff] [blame] | 28 | #include <vnet/ethernet/packet.h> |
Damjan Marion | 83243a0 | 2016-02-29 13:09:30 +0100 | [diff] [blame] | 29 | |
| 30 | #include <vnet/devices/af_packet/af_packet.h> |
| 31 | |
Chaoyu Jin | a608b60 | 2018-02-28 10:15:53 -0800 | [diff] [blame] | 32 | #define foreach_af_packet_input_error \ |
| 33 | _(PARTIAL_PKT, "partial packet") |
Damjan Marion | 83243a0 | 2016-02-29 13:09:30 +0100 | [diff] [blame] | 34 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 35 | typedef enum |
| 36 | { |
Damjan Marion | 83243a0 | 2016-02-29 13:09:30 +0100 | [diff] [blame] | 37 | #define _(f,s) AF_PACKET_INPUT_ERROR_##f, |
| 38 | foreach_af_packet_input_error |
| 39 | #undef _ |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 40 | AF_PACKET_INPUT_N_ERROR, |
Damjan Marion | 83243a0 | 2016-02-29 13:09:30 +0100 | [diff] [blame] | 41 | } af_packet_input_error_t; |
| 42 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 43 | static char *af_packet_input_error_strings[] = { |
Damjan Marion | 83243a0 | 2016-02-29 13:09:30 +0100 | [diff] [blame] | 44 | #define _(n,s) s, |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 45 | foreach_af_packet_input_error |
Damjan Marion | 83243a0 | 2016-02-29 13:09:30 +0100 | [diff] [blame] | 46 | #undef _ |
| 47 | }; |
| 48 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 49 | typedef struct |
| 50 | { |
Damjan Marion | 83243a0 | 2016-02-29 13:09:30 +0100 | [diff] [blame] | 51 | u32 next_index; |
| 52 | u32 hw_if_index; |
| 53 | int block; |
| 54 | struct tpacket2_hdr tph; |
| 55 | } af_packet_input_trace_t; |
| 56 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 57 | static u8 * |
| 58 | format_af_packet_input_trace (u8 * s, va_list * args) |
Damjan Marion | 83243a0 | 2016-02-29 13:09:30 +0100 | [diff] [blame] | 59 | { |
| 60 | CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *); |
| 61 | CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *); |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 62 | af_packet_input_trace_t *t = va_arg (*args, af_packet_input_trace_t *); |
Christophe Fontaine | d3c008d | 2017-10-02 18:10:54 +0200 | [diff] [blame] | 63 | u32 indent = format_get_indent (s); |
Damjan Marion | 83243a0 | 2016-02-29 13:09:30 +0100 | [diff] [blame] | 64 | |
| 65 | s = format (s, "af_packet: hw_if_index %d next-index %d", |
| 66 | t->hw_if_index, t->next_index); |
| 67 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 68 | s = |
| 69 | format (s, |
| 70 | "\n%Utpacket2_hdr:\n%Ustatus 0x%x len %u snaplen %u mac %u net %u" |
| 71 | "\n%Usec 0x%x nsec 0x%x vlan %U" |
Damjan Marion | 83243a0 | 2016-02-29 13:09:30 +0100 | [diff] [blame] | 72 | #ifdef TP_STATUS_VLAN_TPID_VALID |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 73 | " vlan_tpid %u" |
Damjan Marion | 83243a0 | 2016-02-29 13:09:30 +0100 | [diff] [blame] | 74 | #endif |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 75 | , |
| 76 | format_white_space, indent + 2, |
| 77 | format_white_space, indent + 4, |
| 78 | t->tph.tp_status, |
| 79 | t->tph.tp_len, |
| 80 | t->tph.tp_snaplen, |
| 81 | t->tph.tp_mac, |
| 82 | t->tph.tp_net, |
| 83 | format_white_space, indent + 4, |
| 84 | t->tph.tp_sec, |
| 85 | t->tph.tp_nsec, format_ethernet_vlan_tci, t->tph.tp_vlan_tci |
Damjan Marion | 83243a0 | 2016-02-29 13:09:30 +0100 | [diff] [blame] | 86 | #ifdef TP_STATUS_VLAN_TPID_VALID |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 87 | , t->tph.tp_vlan_tpid |
Damjan Marion | 83243a0 | 2016-02-29 13:09:30 +0100 | [diff] [blame] | 88 | #endif |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 89 | ); |
Damjan Marion | 83243a0 | 2016-02-29 13:09:30 +0100 | [diff] [blame] | 90 | return s; |
| 91 | } |
| 92 | |
| 93 | always_inline void |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 94 | buffer_add_to_chain (vlib_main_t * vm, u32 bi, u32 first_bi, u32 prev_bi) |
Damjan Marion | 83243a0 | 2016-02-29 13:09:30 +0100 | [diff] [blame] | 95 | { |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 96 | vlib_buffer_t *b = vlib_get_buffer (vm, bi); |
| 97 | vlib_buffer_t *first_b = vlib_get_buffer (vm, first_bi); |
| 98 | vlib_buffer_t *prev_b = vlib_get_buffer (vm, prev_bi); |
Damjan Marion | 83243a0 | 2016-02-29 13:09:30 +0100 | [diff] [blame] | 99 | |
| 100 | /* update first buffer */ |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 101 | first_b->total_length_not_including_first_buffer += b->current_length; |
Damjan Marion | 83243a0 | 2016-02-29 13:09:30 +0100 | [diff] [blame] | 102 | |
| 103 | /* update previous buffer */ |
| 104 | prev_b->next_buffer = bi; |
| 105 | prev_b->flags |= VLIB_BUFFER_NEXT_PRESENT; |
| 106 | |
| 107 | /* update current buffer */ |
| 108 | b->next_buffer = 0; |
Damjan Marion | 83243a0 | 2016-02-29 13:09:30 +0100 | [diff] [blame] | 109 | } |
| 110 | |
Jakub Grajciar | 92b0275 | 2017-10-20 13:37:28 +0200 | [diff] [blame] | 111 | static_always_inline void |
Nathan Skrzypczak | ffc6bdc | 2021-02-01 17:13:59 +0100 | [diff] [blame] | 112 | fill_gso_buffer_flags (vlib_buffer_t *b, u32 gso_size, u8 l4_hdr_sz) |
| 113 | { |
| 114 | b->flags |= VNET_BUFFER_F_GSO; |
| 115 | vnet_buffer2 (b)->gso_size = gso_size; |
| 116 | vnet_buffer2 (b)->gso_l4_hdr_sz = l4_hdr_sz; |
| 117 | } |
| 118 | |
| 119 | static_always_inline void |
| 120 | mark_tcp_udp_cksum_calc (vlib_buffer_t *b, u8 *l4_hdr_sz) |
Jakub Grajciar | 92b0275 | 2017-10-20 13:37:28 +0200 | [diff] [blame] | 121 | { |
| 122 | ethernet_header_t *eth = vlib_buffer_get_current (b); |
Mohsin Kazmi | 36f7a6a | 2021-05-05 14:26:38 +0200 | [diff] [blame] | 123 | vnet_buffer_oflags_t oflags = 0; |
Jakub Grajciar | 92b0275 | 2017-10-20 13:37:28 +0200 | [diff] [blame] | 124 | if (clib_net_to_host_u16 (eth->type) == ETHERNET_TYPE_IP4) |
| 125 | { |
| 126 | ip4_header_t *ip4 = |
| 127 | (vlib_buffer_get_current (b) + sizeof (ethernet_header_t)); |
Jakub Grajciar | 5665a22 | 2017-11-15 17:25:50 +0100 | [diff] [blame] | 128 | b->flags |= VNET_BUFFER_F_IS_IP4; |
Jakub Grajciar | 92b0275 | 2017-10-20 13:37:28 +0200 | [diff] [blame] | 129 | if (ip4->protocol == IP_PROTOCOL_TCP) |
| 130 | { |
Mohsin Kazmi | 6809538 | 2021-02-10 11:26:24 +0100 | [diff] [blame] | 131 | oflags |= VNET_BUFFER_OFFLOAD_F_TCP_CKSUM; |
Nathan Skrzypczak | ffc6bdc | 2021-02-01 17:13:59 +0100 | [diff] [blame] | 132 | tcp_header_t *tcp = (tcp_header_t *) (vlib_buffer_get_current (b) + |
| 133 | sizeof (ethernet_header_t) + |
| 134 | ip4_header_bytes (ip4)); |
Nathan Skrzypczak | ffc6bdc | 2021-02-01 17:13:59 +0100 | [diff] [blame] | 135 | *l4_hdr_sz = tcp_header_bytes (tcp); |
Jakub Grajciar | 92b0275 | 2017-10-20 13:37:28 +0200 | [diff] [blame] | 136 | } |
| 137 | else if (ip4->protocol == IP_PROTOCOL_UDP) |
| 138 | { |
Mohsin Kazmi | 6809538 | 2021-02-10 11:26:24 +0100 | [diff] [blame] | 139 | oflags |= VNET_BUFFER_OFFLOAD_F_UDP_CKSUM; |
Nathan Skrzypczak | ffc6bdc | 2021-02-01 17:13:59 +0100 | [diff] [blame] | 140 | udp_header_t *udp = (udp_header_t *) (vlib_buffer_get_current (b) + |
| 141 | sizeof (ethernet_header_t) + |
| 142 | ip4_header_bytes (ip4)); |
Nathan Skrzypczak | ffc6bdc | 2021-02-01 17:13:59 +0100 | [diff] [blame] | 143 | *l4_hdr_sz = sizeof (*udp); |
Jakub Grajciar | 92b0275 | 2017-10-20 13:37:28 +0200 | [diff] [blame] | 144 | } |
| 145 | vnet_buffer (b)->l3_hdr_offset = sizeof (ethernet_header_t); |
| 146 | vnet_buffer (b)->l4_hdr_offset = |
| 147 | sizeof (ethernet_header_t) + ip4_header_bytes (ip4); |
Mohsin Kazmi | 6809538 | 2021-02-10 11:26:24 +0100 | [diff] [blame] | 148 | if (oflags) |
| 149 | vnet_buffer_offload_flags_set (b, oflags); |
Jakub Grajciar | 92b0275 | 2017-10-20 13:37:28 +0200 | [diff] [blame] | 150 | } |
| 151 | else if (clib_net_to_host_u16 (eth->type) == ETHERNET_TYPE_IP6) |
| 152 | { |
| 153 | ip6_header_t *ip6 = |
| 154 | (vlib_buffer_get_current (b) + sizeof (ethernet_header_t)); |
| 155 | b->flags |= VNET_BUFFER_F_IS_IP6; |
| 156 | u16 ip6_hdr_len = sizeof (ip6_header_t); |
| 157 | if (ip6_ext_hdr (ip6->protocol)) |
| 158 | { |
| 159 | ip6_ext_header_t *p = (void *) (ip6 + 1); |
| 160 | ip6_hdr_len += ip6_ext_header_len (p); |
| 161 | while (ip6_ext_hdr (p->next_hdr)) |
| 162 | { |
| 163 | ip6_hdr_len += ip6_ext_header_len (p); |
| 164 | p = ip6_ext_next_header (p); |
| 165 | } |
| 166 | } |
| 167 | if (ip6->protocol == IP_PROTOCOL_TCP) |
| 168 | { |
Mohsin Kazmi | 6809538 | 2021-02-10 11:26:24 +0100 | [diff] [blame] | 169 | oflags |= VNET_BUFFER_OFFLOAD_F_TCP_CKSUM; |
Nathan Skrzypczak | ffc6bdc | 2021-02-01 17:13:59 +0100 | [diff] [blame] | 170 | tcp_header_t *tcp = |
| 171 | (tcp_header_t *) (vlib_buffer_get_current (b) + |
| 172 | sizeof (ethernet_header_t) + ip6_hdr_len); |
Nathan Skrzypczak | ffc6bdc | 2021-02-01 17:13:59 +0100 | [diff] [blame] | 173 | *l4_hdr_sz = tcp_header_bytes (tcp); |
Jakub Grajciar | 92b0275 | 2017-10-20 13:37:28 +0200 | [diff] [blame] | 174 | } |
| 175 | else if (ip6->protocol == IP_PROTOCOL_UDP) |
| 176 | { |
Mohsin Kazmi | 6809538 | 2021-02-10 11:26:24 +0100 | [diff] [blame] | 177 | oflags |= VNET_BUFFER_OFFLOAD_F_UDP_CKSUM; |
Nathan Skrzypczak | ffc6bdc | 2021-02-01 17:13:59 +0100 | [diff] [blame] | 178 | udp_header_t *udp = |
| 179 | (udp_header_t *) (vlib_buffer_get_current (b) + |
| 180 | sizeof (ethernet_header_t) + ip6_hdr_len); |
Nathan Skrzypczak | ffc6bdc | 2021-02-01 17:13:59 +0100 | [diff] [blame] | 181 | *l4_hdr_sz = sizeof (*udp); |
Jakub Grajciar | 92b0275 | 2017-10-20 13:37:28 +0200 | [diff] [blame] | 182 | } |
| 183 | vnet_buffer (b)->l3_hdr_offset = sizeof (ethernet_header_t); |
| 184 | vnet_buffer (b)->l4_hdr_offset = |
| 185 | sizeof (ethernet_header_t) + ip6_hdr_len; |
Mohsin Kazmi | 6809538 | 2021-02-10 11:26:24 +0100 | [diff] [blame] | 186 | if (oflags) |
| 187 | vnet_buffer_offload_flags_set (b, oflags); |
Jakub Grajciar | 92b0275 | 2017-10-20 13:37:28 +0200 | [diff] [blame] | 188 | } |
| 189 | } |
| 190 | |
Damjan Marion | 83243a0 | 2016-02-29 13:09:30 +0100 | [diff] [blame] | 191 | always_inline uword |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 192 | af_packet_device_input_fn (vlib_main_t * vm, vlib_node_runtime_t * node, |
Mohsin KAZMI | cf751ec | 2017-01-18 11:59:45 +0100 | [diff] [blame] | 193 | vlib_frame_t * frame, af_packet_if_t * apif) |
Damjan Marion | 83243a0 | 2016-02-29 13:09:30 +0100 | [diff] [blame] | 194 | { |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 195 | af_packet_main_t *apm = &af_packet_main; |
Damjan Marion | 83243a0 | 2016-02-29 13:09:30 +0100 | [diff] [blame] | 196 | struct tpacket2_hdr *tph; |
Damjan Marion | 8bdc63b | 2016-11-02 14:48:21 +0100 | [diff] [blame] | 197 | u32 next_index = VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT; |
Damjan Marion | 83243a0 | 2016-02-29 13:09:30 +0100 | [diff] [blame] | 198 | u32 block = 0; |
| 199 | u32 rx_frame; |
| 200 | u32 n_free_bufs; |
| 201 | u32 n_rx_packets = 0; |
| 202 | u32 n_rx_bytes = 0; |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 203 | u32 *to_next = 0; |
Damjan Marion | 83243a0 | 2016-02-29 13:09:30 +0100 | [diff] [blame] | 204 | u32 block_size = apif->rx_req->tp_block_size; |
| 205 | u32 frame_size = apif->rx_req->tp_frame_size; |
| 206 | u32 frame_num = apif->rx_req->tp_frame_nr; |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 207 | u8 *block_start = apif->rx_ring + block * block_size; |
Damjan Marion | 83243a0 | 2016-02-29 13:09:30 +0100 | [diff] [blame] | 208 | uword n_trace = vlib_get_trace_count (vm, node); |
Damjan Marion | 067cd62 | 2018-07-11 12:47:43 +0200 | [diff] [blame] | 209 | u32 thread_index = vm->thread_index; |
Damjan Marion | 8934a04 | 2019-02-09 23:29:26 +0100 | [diff] [blame] | 210 | u32 n_buffer_bytes = vlib_buffer_get_default_data_size (vm); |
Damjan Marion | 83243a0 | 2016-02-29 13:09:30 +0100 | [diff] [blame] | 211 | u32 min_bufs = apif->rx_req->tp_frame_size / n_buffer_bytes; |
| 212 | |
Damjan Marion | 586afd7 | 2017-04-05 19:18:20 +0200 | [diff] [blame] | 213 | n_free_bufs = vec_len (apm->rx_buffers[thread_index]); |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 214 | if (PREDICT_FALSE (n_free_bufs < VLIB_FRAME_SIZE)) |
Damjan Marion | 83243a0 | 2016-02-29 13:09:30 +0100 | [diff] [blame] | 215 | { |
Damjan Marion | 586afd7 | 2017-04-05 19:18:20 +0200 | [diff] [blame] | 216 | vec_validate (apm->rx_buffers[thread_index], |
Damjan Marion | 553f6bd | 2016-09-07 11:54:22 +0200 | [diff] [blame] | 217 | VLIB_FRAME_SIZE + n_free_bufs - 1); |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 218 | n_free_bufs += |
Damjan Marion | 586afd7 | 2017-04-05 19:18:20 +0200 | [diff] [blame] | 219 | vlib_buffer_alloc (vm, &apm->rx_buffers[thread_index][n_free_bufs], |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 220 | VLIB_FRAME_SIZE); |
Damjan Marion | 586afd7 | 2017-04-05 19:18:20 +0200 | [diff] [blame] | 221 | _vec_len (apm->rx_buffers[thread_index]) = n_free_bufs; |
Damjan Marion | 83243a0 | 2016-02-29 13:09:30 +0100 | [diff] [blame] | 222 | } |
| 223 | |
| 224 | rx_frame = apif->next_rx_frame; |
| 225 | tph = (struct tpacket2_hdr *) (block_start + rx_frame * frame_size); |
| 226 | while ((tph->tp_status & TP_STATUS_USER) && (n_free_bufs > min_bufs)) |
| 227 | { |
Damjan Marion | 2231150 | 2016-10-28 20:30:15 +0200 | [diff] [blame] | 228 | vlib_buffer_t *b0 = 0, *first_b0 = 0; |
Dave Barach | 13f3c45 | 2016-03-29 11:56:41 -0400 | [diff] [blame] | 229 | u32 next0 = next_index; |
Damjan Marion | 83243a0 | 2016-02-29 13:09:30 +0100 | [diff] [blame] | 230 | |
| 231 | u32 n_left_to_next; |
| 232 | vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next); |
| 233 | while ((tph->tp_status & TP_STATUS_USER) && (n_free_bufs > min_bufs) && |
| 234 | n_left_to_next) |
| 235 | { |
| 236 | u32 data_len = tph->tp_snaplen; |
| 237 | u32 offset = 0; |
BenoƮt Ganne | d530445 | 2016-04-08 22:25:05 -0700 | [diff] [blame] | 238 | u32 bi0 = 0, first_bi0 = 0, prev_bi0; |
Nathan Skrzypczak | ffc6bdc | 2021-02-01 17:13:59 +0100 | [diff] [blame] | 239 | u8 l4_hdr_sz = 0; |
Damjan Marion | 83243a0 | 2016-02-29 13:09:30 +0100 | [diff] [blame] | 240 | |
| 241 | while (data_len) |
| 242 | { |
| 243 | /* grab free buffer */ |
Damjan Marion | 553f6bd | 2016-09-07 11:54:22 +0200 | [diff] [blame] | 244 | u32 last_empty_buffer = |
Damjan Marion | 586afd7 | 2017-04-05 19:18:20 +0200 | [diff] [blame] | 245 | vec_len (apm->rx_buffers[thread_index]) - 1; |
Damjan Marion | 83243a0 | 2016-02-29 13:09:30 +0100 | [diff] [blame] | 246 | prev_bi0 = bi0; |
Damjan Marion | 586afd7 | 2017-04-05 19:18:20 +0200 | [diff] [blame] | 247 | bi0 = apm->rx_buffers[thread_index][last_empty_buffer]; |
Damjan Marion | 83243a0 | 2016-02-29 13:09:30 +0100 | [diff] [blame] | 248 | b0 = vlib_get_buffer (vm, bi0); |
Damjan Marion | 586afd7 | 2017-04-05 19:18:20 +0200 | [diff] [blame] | 249 | _vec_len (apm->rx_buffers[thread_index]) = last_empty_buffer; |
Damjan Marion | 83243a0 | 2016-02-29 13:09:30 +0100 | [diff] [blame] | 250 | n_free_bufs--; |
| 251 | |
| 252 | /* copy data */ |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 253 | u32 bytes_to_copy = |
| 254 | data_len > n_buffer_bytes ? n_buffer_bytes : data_len; |
Damjan Marion | c517020 | 2017-10-11 14:15:47 +0200 | [diff] [blame] | 255 | u32 vlan_len = 0; |
Akshaya N | 535f0bf | 2017-09-15 17:37:53 +0530 | [diff] [blame] | 256 | u32 bytes_copied = 0; |
Damjan Marion | 83243a0 | 2016-02-29 13:09:30 +0100 | [diff] [blame] | 257 | b0->current_data = 0; |
Akshaya N | 535f0bf | 2017-09-15 17:37:53 +0530 | [diff] [blame] | 258 | /* Kernel removes VLAN headers, so reconstruct VLAN */ |
| 259 | if (PREDICT_FALSE (tph->tp_status & TP_STATUS_VLAN_VALID)) |
| 260 | { |
| 261 | if (PREDICT_TRUE (offset == 0)) |
| 262 | { |
Dave Barach | 178cf49 | 2018-11-13 16:34:13 -0500 | [diff] [blame] | 263 | clib_memcpy_fast (vlib_buffer_get_current (b0), |
| 264 | (u8 *) tph + tph->tp_mac, |
| 265 | sizeof (ethernet_header_t)); |
Akshaya N | 535f0bf | 2017-09-15 17:37:53 +0530 | [diff] [blame] | 266 | ethernet_header_t *eth = vlib_buffer_get_current (b0); |
| 267 | ethernet_vlan_header_t *vlan = |
| 268 | (ethernet_vlan_header_t *) (eth + 1); |
| 269 | vlan->priority_cfi_and_id = |
| 270 | clib_host_to_net_u16 (tph->tp_vlan_tci); |
| 271 | vlan->type = eth->type; |
| 272 | eth->type = clib_host_to_net_u16 (ETHERNET_TYPE_VLAN); |
| 273 | vlan_len = sizeof (ethernet_vlan_header_t); |
| 274 | bytes_copied = sizeof (ethernet_header_t); |
| 275 | } |
| 276 | } |
Dave Barach | 178cf49 | 2018-11-13 16:34:13 -0500 | [diff] [blame] | 277 | clib_memcpy_fast (((u8 *) vlib_buffer_get_current (b0)) + |
| 278 | bytes_copied + vlan_len, |
| 279 | (u8 *) tph + tph->tp_mac + offset + |
| 280 | bytes_copied, (bytes_to_copy - bytes_copied)); |
Damjan Marion | 83243a0 | 2016-02-29 13:09:30 +0100 | [diff] [blame] | 281 | |
| 282 | /* fill buffer header */ |
Akshaya N | 535f0bf | 2017-09-15 17:37:53 +0530 | [diff] [blame] | 283 | b0->current_length = bytes_to_copy + vlan_len; |
Damjan Marion | 83243a0 | 2016-02-29 13:09:30 +0100 | [diff] [blame] | 284 | |
| 285 | if (offset == 0) |
| 286 | { |
| 287 | b0->total_length_not_including_first_buffer = 0; |
| 288 | b0->flags = VLIB_BUFFER_TOTAL_LENGTH_VALID; |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 289 | vnet_buffer (b0)->sw_if_index[VLIB_RX] = apif->sw_if_index; |
| 290 | vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0; |
Damjan Marion | 83243a0 | 2016-02-29 13:09:30 +0100 | [diff] [blame] | 291 | first_bi0 = bi0; |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 292 | first_b0 = vlib_get_buffer (vm, first_bi0); |
Jakub Grajciar | 92b0275 | 2017-10-20 13:37:28 +0200 | [diff] [blame] | 293 | if (tph->tp_status & TP_STATUS_CSUMNOTREADY) |
Nathan Skrzypczak | ffc6bdc | 2021-02-01 17:13:59 +0100 | [diff] [blame] | 294 | mark_tcp_udp_cksum_calc (first_b0, &l4_hdr_sz); |
| 295 | if (tph->tp_snaplen > apif->host_mtu) |
| 296 | fill_gso_buffer_flags (first_b0, apif->host_mtu, |
| 297 | l4_hdr_sz); |
Damjan Marion | 83243a0 | 2016-02-29 13:09:30 +0100 | [diff] [blame] | 298 | } |
| 299 | else |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 300 | buffer_add_to_chain (vm, bi0, first_bi0, prev_bi0); |
Damjan Marion | 83243a0 | 2016-02-29 13:09:30 +0100 | [diff] [blame] | 301 | |
| 302 | offset += bytes_to_copy; |
| 303 | data_len -= bytes_to_copy; |
| 304 | } |
| 305 | n_rx_packets++; |
| 306 | n_rx_bytes += tph->tp_snaplen; |
| 307 | to_next[0] = first_bi0; |
| 308 | to_next += 1; |
| 309 | n_left_to_next--; |
| 310 | |
Chaoyu Jin | a608b60 | 2018-02-28 10:15:53 -0800 | [diff] [blame] | 311 | /* drop partial packets */ |
| 312 | if (PREDICT_FALSE (tph->tp_len != tph->tp_snaplen)) |
| 313 | { |
| 314 | next0 = VNET_DEVICE_INPUT_NEXT_DROP; |
| 315 | first_b0->error = |
| 316 | node->errors[AF_PACKET_INPUT_ERROR_PARTIAL_PKT]; |
| 317 | } |
| 318 | else |
| 319 | { |
| 320 | next0 = VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT; |
Michael Yu | 90b34ed | 2019-08-30 00:50:16 +0800 | [diff] [blame] | 321 | |
| 322 | if (PREDICT_FALSE (apif->per_interface_next_index != ~0)) |
| 323 | next0 = apif->per_interface_next_index; |
| 324 | |
Chaoyu Jin | a608b60 | 2018-02-28 10:15:53 -0800 | [diff] [blame] | 325 | /* redirect if feature path enabled */ |
| 326 | vnet_feature_start_device_input_x1 (apif->sw_if_index, &next0, |
| 327 | first_b0); |
| 328 | } |
| 329 | |
Damjan Marion | 83243a0 | 2016-02-29 13:09:30 +0100 | [diff] [blame] | 330 | /* trace */ |
BenoƮt Ganne | 9a3973e | 2020-10-02 19:36:57 +0200 | [diff] [blame] | 331 | if (PREDICT_FALSE |
| 332 | (n_trace > 0 && vlib_trace_buffer (vm, node, next0, first_b0, |
| 333 | /* follow_chain */ 0))) |
Damjan Marion | 83243a0 | 2016-02-29 13:09:30 +0100 | [diff] [blame] | 334 | { |
| 335 | af_packet_input_trace_t *tr; |
Damjan Marion | 83243a0 | 2016-02-29 13:09:30 +0100 | [diff] [blame] | 336 | vlib_set_trace_count (vm, node, --n_trace); |
| 337 | tr = vlib_add_trace (vm, node, first_b0, sizeof (*tr)); |
| 338 | tr->next_index = next0; |
| 339 | tr->hw_if_index = apif->hw_if_index; |
Dave Barach | 178cf49 | 2018-11-13 16:34:13 -0500 | [diff] [blame] | 340 | clib_memcpy_fast (&tr->tph, tph, sizeof (struct tpacket2_hdr)); |
Damjan Marion | 83243a0 | 2016-02-29 13:09:30 +0100 | [diff] [blame] | 341 | } |
Damjan Marion | 2231150 | 2016-10-28 20:30:15 +0200 | [diff] [blame] | 342 | |
Damjan Marion | 83243a0 | 2016-02-29 13:09:30 +0100 | [diff] [blame] | 343 | /* enque and take next packet */ |
| 344 | vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next, |
| 345 | n_left_to_next, first_bi0, next0); |
| 346 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 347 | /* next packet */ |
Damjan Marion | 83243a0 | 2016-02-29 13:09:30 +0100 | [diff] [blame] | 348 | tph->tp_status = TP_STATUS_KERNEL; |
| 349 | rx_frame = (rx_frame + 1) % frame_num; |
| 350 | tph = (struct tpacket2_hdr *) (block_start + rx_frame * frame_size); |
| 351 | } |
| 352 | |
| 353 | vlib_put_next_frame (vm, node, next_index, n_left_to_next); |
| 354 | } |
| 355 | |
| 356 | apif->next_rx_frame = rx_frame; |
| 357 | |
| 358 | vlib_increment_combined_counter |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 359 | (vnet_get_main ()->interface_main.combined_sw_if_counters |
Damjan Marion | 83243a0 | 2016-02-29 13:09:30 +0100 | [diff] [blame] | 360 | + VNET_INTERFACE_COUNTER_RX, |
Damjan Marion | 586afd7 | 2017-04-05 19:18:20 +0200 | [diff] [blame] | 361 | vlib_get_thread_index (), apif->hw_if_index, n_rx_packets, n_rx_bytes); |
Damjan Marion | 83243a0 | 2016-02-29 13:09:30 +0100 | [diff] [blame] | 362 | |
Damjan Marion | 586afd7 | 2017-04-05 19:18:20 +0200 | [diff] [blame] | 363 | vnet_device_increment_rx_packets (thread_index, n_rx_packets); |
Damjan Marion | 83243a0 | 2016-02-29 13:09:30 +0100 | [diff] [blame] | 364 | return n_rx_packets; |
| 365 | } |
| 366 | |
Filip Tehlar | 608996d | 2019-03-04 03:03:13 -0800 | [diff] [blame] | 367 | VLIB_NODE_FN (af_packet_input_node) (vlib_main_t * vm, |
| 368 | vlib_node_runtime_t * node, |
| 369 | vlib_frame_t * frame) |
Damjan Marion | 83243a0 | 2016-02-29 13:09:30 +0100 | [diff] [blame] | 370 | { |
Damjan Marion | 83243a0 | 2016-02-29 13:09:30 +0100 | [diff] [blame] | 371 | u32 n_rx_packets = 0; |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 372 | af_packet_main_t *apm = &af_packet_main; |
Mohammed Hawari | 85c1943 | 2020-12-18 16:29:45 +0100 | [diff] [blame] | 373 | vnet_hw_if_rxq_poll_vector_t *pv; |
| 374 | pv = vnet_hw_if_get_rxq_poll_vector (vm, node); |
| 375 | for (int i = 0; i < vec_len (pv); i++) |
| 376 | { |
| 377 | af_packet_if_t *apif; |
| 378 | apif = vec_elt_at_index (apm->interfaces, pv[i].dev_instance); |
| 379 | if (apif->is_admin_up) |
| 380 | n_rx_packets += af_packet_device_input_fn (vm, node, frame, apif); |
| 381 | } |
Damjan Marion | 83243a0 | 2016-02-29 13:09:30 +0100 | [diff] [blame] | 382 | |
| 383 | return n_rx_packets; |
| 384 | } |
| 385 | |
Damjan Marion | 83243a0 | 2016-02-29 13:09:30 +0100 | [diff] [blame] | 386 | VLIB_REGISTER_NODE (af_packet_input_node) = { |
Damjan Marion | 83243a0 | 2016-02-29 13:09:30 +0100 | [diff] [blame] | 387 | .name = "af-packet-input", |
Damjan Marion | 7ca5aaa | 2019-09-24 18:10:49 +0200 | [diff] [blame] | 388 | .flags = VLIB_NODE_FLAG_TRACE_SUPPORTED, |
Damjan Marion | 51327ac | 2016-11-09 11:59:42 +0100 | [diff] [blame] | 389 | .sibling_of = "device-input", |
Damjan Marion | 83243a0 | 2016-02-29 13:09:30 +0100 | [diff] [blame] | 390 | .format_trace = format_af_packet_input_trace, |
| 391 | .type = VLIB_NODE_TYPE_INPUT, |
| 392 | .state = VLIB_NODE_STATE_INTERRUPT, |
| 393 | .n_errors = AF_PACKET_INPUT_N_ERROR, |
| 394 | .error_strings = af_packet_input_error_strings, |
Damjan Marion | 83243a0 | 2016-02-29 13:09:30 +0100 | [diff] [blame] | 395 | }; |
Damjan Marion | 1c80e83 | 2016-05-11 23:07:18 +0200 | [diff] [blame] | 396 | |
Damjan Marion | 00a9dca | 2016-08-17 17:05:46 +0200 | [diff] [blame] | 397 | |
| 398 | /* |
| 399 | * fd.io coding-style-patch-verification: ON |
| 400 | * |
| 401 | * Local Variables: |
| 402 | * eval: (c-set-style "gnu") |
| 403 | * End: |
| 404 | */ |