Damjan Marion | 8389fb9 | 2017-10-13 18:29:53 +0200 | [diff] [blame] | 1 | /* |
| 2 | *------------------------------------------------------------------ |
| 3 | * Copyright (c) 2016 Cisco and/or its affiliates. |
| 4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | * you may not use this file except in compliance with the License. |
| 6 | * You may obtain a copy of the License at: |
| 7 | * |
| 8 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | * |
| 10 | * Unless required by applicable law or agreed to in writing, software |
| 11 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | * See the License for the specific language governing permissions and |
| 14 | * limitations under the License. |
| 15 | *------------------------------------------------------------------ |
| 16 | */ |
| 17 | |
| 18 | #include <sys/types.h> |
| 19 | #include <sys/stat.h> |
| 20 | #include <fcntl.h> |
| 21 | #include <net/if.h> |
| 22 | #include <linux/if_tun.h> |
| 23 | #include <sys/ioctl.h> |
Damjan Marion | 8389fb9 | 2017-10-13 18:29:53 +0200 | [diff] [blame] | 24 | #include <sys/eventfd.h> |
| 25 | |
| 26 | #include <vlib/vlib.h> |
| 27 | #include <vlib/unix/unix.h> |
| 28 | #include <vnet/ethernet/ethernet.h> |
Damjan Marion | 8389fb9 | 2017-10-13 18:29:53 +0200 | [diff] [blame] | 29 | #include <vnet/feature/feature.h> |
Damjan Marion | 9410053 | 2020-11-06 23:25:57 +0100 | [diff] [blame] | 30 | #include <vnet/interface/rx_queue_funcs.h> |
Milan Lenco | 73e7f42 | 2017-12-14 10:04:25 +0100 | [diff] [blame] | 31 | #include <vnet/ip/ip4_packet.h> |
| 32 | #include <vnet/ip/ip6_packet.h> |
Andrew Yourtchenko | 6a7cff7 | 2018-10-12 16:09:22 +0200 | [diff] [blame] | 33 | #include <vnet/udp/udp_packet.h> |
Damjan Marion | 8389fb9 | 2017-10-13 18:29:53 +0200 | [diff] [blame] | 34 | #include <vnet/devices/virtio/virtio.h> |
Mohsin Kazmi | 6799f9b | 2021-04-28 18:55:45 +0200 | [diff] [blame] | 35 | #include <vnet/devices/virtio/virtio_inline.h> |
Damjan Marion | 8389fb9 | 2017-10-13 18:29:53 +0200 | [diff] [blame] | 36 | |
| 37 | static char *virtio_input_error_strings[] = { |
Mohsin Kazmi | 6799f9b | 2021-04-28 18:55:45 +0200 | [diff] [blame] | 38 | #define _(n, s) s, |
Damjan Marion | 8389fb9 | 2017-10-13 18:29:53 +0200 | [diff] [blame] | 39 | foreach_virtio_input_error |
| 40 | #undef _ |
| 41 | }; |
| 42 | |
| 43 | typedef struct |
| 44 | { |
| 45 | u32 next_index; |
| 46 | u32 hw_if_index; |
| 47 | u16 ring; |
| 48 | u16 len; |
Mohsin Kazmi | a7a2281 | 2020-08-31 17:17:16 +0200 | [diff] [blame] | 49 | virtio_net_hdr_v1_t hdr; |
Damjan Marion | 8389fb9 | 2017-10-13 18:29:53 +0200 | [diff] [blame] | 50 | } virtio_input_trace_t; |
| 51 | |
| 52 | static u8 * |
| 53 | format_virtio_input_trace (u8 * s, va_list * args) |
| 54 | { |
| 55 | CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *); |
| 56 | CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *); |
| 57 | virtio_input_trace_t *t = va_arg (*args, virtio_input_trace_t *); |
| 58 | u32 indent = format_get_indent (s); |
| 59 | |
| 60 | s = format (s, "virtio: hw_if_index %d next-index %d vring %u len %u", |
| 61 | t->hw_if_index, t->next_index, t->ring, t->len); |
| 62 | s = format (s, "\n%Uhdr: flags 0x%02x gso_type 0x%02x hdr_len %u " |
| 63 | "gso_size %u csum_start %u csum_offset %u num_buffers %u", |
| 64 | format_white_space, indent + 2, |
| 65 | t->hdr.flags, t->hdr.gso_type, t->hdr.hdr_len, t->hdr.gso_size, |
| 66 | t->hdr.csum_start, t->hdr.csum_offset, t->hdr.num_buffers); |
| 67 | return s; |
| 68 | } |
| 69 | |
| 70 | static_always_inline void |
Mohsin Kazmi | a7a2281 | 2020-08-31 17:17:16 +0200 | [diff] [blame] | 71 | virtio_needs_csum (vlib_buffer_t * b0, virtio_net_hdr_v1_t * hdr, |
Mohsin Kazmi | 206acf8 | 2020-04-06 14:19:54 +0200 | [diff] [blame] | 72 | u8 * l4_proto, u8 * l4_hdr_sz, virtio_if_type_t type) |
Andrew Yourtchenko | 6a7cff7 | 2018-10-12 16:09:22 +0200 | [diff] [blame] | 73 | { |
Andrew Yourtchenko | 6a7cff7 | 2018-10-12 16:09:22 +0200 | [diff] [blame] | 74 | if (hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM) |
Andrew Yourtchenko | 6a7cff7 | 2018-10-12 16:09:22 +0200 | [diff] [blame] | 75 | { |
Mohsin Kazmi | 206acf8 | 2020-04-06 14:19:54 +0200 | [diff] [blame] | 76 | u16 ethertype = 0, l2hdr_sz = 0; |
Mohsin Kazmi | 36f7a6a | 2021-05-05 14:26:38 +0200 | [diff] [blame] | 77 | vnet_buffer_oflags_t oflags = 0; |
Mohsin Kazmi | 38b0968 | 2020-06-03 18:20:17 +0200 | [diff] [blame] | 78 | |
| 79 | if (type == VIRTIO_IF_TYPE_TUN) |
| 80 | { |
| 81 | switch (b0->data[0] & 0xf0) |
| 82 | { |
| 83 | case 0x40: |
| 84 | ethertype = ETHERNET_TYPE_IP4; |
| 85 | break; |
| 86 | case 0x60: |
| 87 | ethertype = ETHERNET_TYPE_IP6; |
| 88 | break; |
| 89 | } |
| 90 | } |
| 91 | else |
Mohsin Kazmi | 14bea1b | 2019-07-29 11:39:26 +0200 | [diff] [blame] | 92 | { |
Mohsin Kazmi | 206acf8 | 2020-04-06 14:19:54 +0200 | [diff] [blame] | 93 | ethernet_header_t *eh = |
| 94 | (ethernet_header_t *) vlib_buffer_get_current (b0); |
| 95 | ethertype = clib_net_to_host_u16 (eh->type); |
| 96 | l2hdr_sz = sizeof (ethernet_header_t); |
Mohsin Kazmi | 14bea1b | 2019-07-29 11:39:26 +0200 | [diff] [blame] | 97 | |
Mohsin Kazmi | 206acf8 | 2020-04-06 14:19:54 +0200 | [diff] [blame] | 98 | if (ethernet_frame_is_tagged (ethertype)) |
Mohsin Kazmi | 14bea1b | 2019-07-29 11:39:26 +0200 | [diff] [blame] | 99 | { |
Mohsin Kazmi | 206acf8 | 2020-04-06 14:19:54 +0200 | [diff] [blame] | 100 | ethernet_vlan_header_t *vlan = |
| 101 | (ethernet_vlan_header_t *) (eh + 1); |
| 102 | |
Mohsin Kazmi | 14bea1b | 2019-07-29 11:39:26 +0200 | [diff] [blame] | 103 | ethertype = clib_net_to_host_u16 (vlan->type); |
| 104 | l2hdr_sz += sizeof (*vlan); |
Mohsin Kazmi | 206acf8 | 2020-04-06 14:19:54 +0200 | [diff] [blame] | 105 | if (ethertype == ETHERNET_TYPE_VLAN) |
| 106 | { |
| 107 | vlan++; |
| 108 | ethertype = clib_net_to_host_u16 (vlan->type); |
| 109 | l2hdr_sz += sizeof (*vlan); |
| 110 | } |
| 111 | } |
| 112 | } |
Mohsin Kazmi | 14bea1b | 2019-07-29 11:39:26 +0200 | [diff] [blame] | 113 | |
Mohsin Kazmi | 157a4ab | 2019-12-06 15:47:48 +0100 | [diff] [blame] | 114 | vnet_buffer (b0)->l2_hdr_offset = 0; |
| 115 | vnet_buffer (b0)->l3_hdr_offset = l2hdr_sz; |
Mohsin Kazmi | 38b0968 | 2020-06-03 18:20:17 +0200 | [diff] [blame] | 116 | |
Andrew Yourtchenko | 6a7cff7 | 2018-10-12 16:09:22 +0200 | [diff] [blame] | 117 | if (PREDICT_TRUE (ethertype == ETHERNET_TYPE_IP4)) |
| 118 | { |
Mohsin Kazmi | 72e7312 | 2019-10-22 13:33:13 +0200 | [diff] [blame] | 119 | ip4_header_t *ip4 = |
| 120 | (ip4_header_t *) (vlib_buffer_get_current (b0) + l2hdr_sz); |
Mohsin Kazmi | 157a4ab | 2019-12-06 15:47:48 +0100 | [diff] [blame] | 121 | vnet_buffer (b0)->l4_hdr_offset = l2hdr_sz + ip4_header_bytes (ip4); |
BenoƮt Ganne | 15d7fd0 | 2019-11-26 17:59:41 +0100 | [diff] [blame] | 122 | *l4_proto = ip4->protocol; |
Mohsin Kazmi | 6809538 | 2021-02-10 11:26:24 +0100 | [diff] [blame] | 123 | oflags |= VNET_BUFFER_OFFLOAD_F_IP_CKSUM; |
Andrew Yourtchenko | 6a7cff7 | 2018-10-12 16:09:22 +0200 | [diff] [blame] | 124 | b0->flags |= |
Mohsin Kazmi | 6809538 | 2021-02-10 11:26:24 +0100 | [diff] [blame] | 125 | (VNET_BUFFER_F_IS_IP4 | VNET_BUFFER_F_L2_HDR_OFFSET_VALID | |
| 126 | VNET_BUFFER_F_L3_HDR_OFFSET_VALID | |
Mohsin Kazmi | 157a4ab | 2019-12-06 15:47:48 +0100 | [diff] [blame] | 127 | VNET_BUFFER_F_L4_HDR_OFFSET_VALID); |
Andrew Yourtchenko | 6a7cff7 | 2018-10-12 16:09:22 +0200 | [diff] [blame] | 128 | } |
| 129 | else if (PREDICT_TRUE (ethertype == ETHERNET_TYPE_IP6)) |
| 130 | { |
Mohsin Kazmi | 72e7312 | 2019-10-22 13:33:13 +0200 | [diff] [blame] | 131 | ip6_header_t *ip6 = |
| 132 | (ip6_header_t *) (vlib_buffer_get_current (b0) + l2hdr_sz); |
Mohsin Kazmi | 157a4ab | 2019-12-06 15:47:48 +0100 | [diff] [blame] | 133 | vnet_buffer (b0)->l4_hdr_offset = l2hdr_sz + sizeof (ip6_header_t); |
Andrew Yourtchenko | 6a7cff7 | 2018-10-12 16:09:22 +0200 | [diff] [blame] | 134 | /* FIXME IPv6 EH traversal */ |
BenoƮt Ganne | 15d7fd0 | 2019-11-26 17:59:41 +0100 | [diff] [blame] | 135 | *l4_proto = ip6->protocol; |
Mohsin Kazmi | 157a4ab | 2019-12-06 15:47:48 +0100 | [diff] [blame] | 136 | b0->flags |= (VNET_BUFFER_F_IS_IP6 | |
| 137 | VNET_BUFFER_F_L2_HDR_OFFSET_VALID |
| 138 | | VNET_BUFFER_F_L3_HDR_OFFSET_VALID | |
| 139 | VNET_BUFFER_F_L4_HDR_OFFSET_VALID); |
Andrew Yourtchenko | 6a7cff7 | 2018-10-12 16:09:22 +0200 | [diff] [blame] | 140 | } |
BenoƮt Ganne | 15d7fd0 | 2019-11-26 17:59:41 +0100 | [diff] [blame] | 141 | if (*l4_proto == IP_PROTOCOL_TCP) |
Andrew Yourtchenko | 6a7cff7 | 2018-10-12 16:09:22 +0200 | [diff] [blame] | 142 | { |
Mohsin Kazmi | 6809538 | 2021-02-10 11:26:24 +0100 | [diff] [blame] | 143 | oflags |= VNET_BUFFER_OFFLOAD_F_TCP_CKSUM; |
Mohsin Kazmi | 157a4ab | 2019-12-06 15:47:48 +0100 | [diff] [blame] | 144 | tcp_header_t *tcp = (tcp_header_t *) (vlib_buffer_get_current (b0) + |
| 145 | vnet_buffer |
| 146 | (b0)->l4_hdr_offset); |
BenoƮt Ganne | 15d7fd0 | 2019-11-26 17:59:41 +0100 | [diff] [blame] | 147 | *l4_hdr_sz = tcp_header_bytes (tcp); |
Andrew Yourtchenko | 6a7cff7 | 2018-10-12 16:09:22 +0200 | [diff] [blame] | 148 | } |
BenoƮt Ganne | 15d7fd0 | 2019-11-26 17:59:41 +0100 | [diff] [blame] | 149 | else if (*l4_proto == IP_PROTOCOL_UDP) |
Andrew Yourtchenko | 6a7cff7 | 2018-10-12 16:09:22 +0200 | [diff] [blame] | 150 | { |
Mohsin Kazmi | 6809538 | 2021-02-10 11:26:24 +0100 | [diff] [blame] | 151 | oflags |= VNET_BUFFER_OFFLOAD_F_UDP_CKSUM; |
Mohsin Kazmi | 157a4ab | 2019-12-06 15:47:48 +0100 | [diff] [blame] | 152 | udp_header_t *udp = (udp_header_t *) (vlib_buffer_get_current (b0) + |
| 153 | vnet_buffer |
| 154 | (b0)->l4_hdr_offset); |
BenoƮt Ganne | 15d7fd0 | 2019-11-26 17:59:41 +0100 | [diff] [blame] | 155 | *l4_hdr_sz = sizeof (*udp); |
Andrew Yourtchenko | 6a7cff7 | 2018-10-12 16:09:22 +0200 | [diff] [blame] | 156 | } |
Mohsin Kazmi | 6809538 | 2021-02-10 11:26:24 +0100 | [diff] [blame] | 157 | if (oflags) |
| 158 | vnet_buffer_offload_flags_set (b0, oflags); |
Andrew Yourtchenko | 6a7cff7 | 2018-10-12 16:09:22 +0200 | [diff] [blame] | 159 | } |
BenoƮt Ganne | 15d7fd0 | 2019-11-26 17:59:41 +0100 | [diff] [blame] | 160 | } |
| 161 | |
| 162 | static_always_inline void |
Mohsin Kazmi | a7a2281 | 2020-08-31 17:17:16 +0200 | [diff] [blame] | 163 | fill_gso_buffer_flags (vlib_buffer_t * b0, virtio_net_hdr_v1_t * hdr, |
BenoƮt Ganne | 15d7fd0 | 2019-11-26 17:59:41 +0100 | [diff] [blame] | 164 | u8 l4_proto, u8 l4_hdr_sz) |
| 165 | { |
Andrew Yourtchenko | 6a7cff7 | 2018-10-12 16:09:22 +0200 | [diff] [blame] | 166 | if (hdr->gso_type == VIRTIO_NET_HDR_GSO_TCPV4) |
| 167 | { |
| 168 | ASSERT (hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM); |
| 169 | vnet_buffer2 (b0)->gso_size = hdr->gso_size; |
Mohsin Kazmi | 157a4ab | 2019-12-06 15:47:48 +0100 | [diff] [blame] | 170 | vnet_buffer2 (b0)->gso_l4_hdr_sz = l4_hdr_sz; |
Mohsin Kazmi | 72e7312 | 2019-10-22 13:33:13 +0200 | [diff] [blame] | 171 | b0->flags |= VNET_BUFFER_F_GSO | VNET_BUFFER_F_IS_IP4; |
Andrew Yourtchenko | 6a7cff7 | 2018-10-12 16:09:22 +0200 | [diff] [blame] | 172 | } |
| 173 | if (hdr->gso_type == VIRTIO_NET_HDR_GSO_TCPV6) |
| 174 | { |
| 175 | ASSERT (hdr->flags & VIRTIO_NET_HDR_F_NEEDS_CSUM); |
| 176 | vnet_buffer2 (b0)->gso_size = hdr->gso_size; |
Mohsin Kazmi | 157a4ab | 2019-12-06 15:47:48 +0100 | [diff] [blame] | 177 | vnet_buffer2 (b0)->gso_l4_hdr_sz = l4_hdr_sz; |
Mohsin Kazmi | 72e7312 | 2019-10-22 13:33:13 +0200 | [diff] [blame] | 178 | b0->flags |= VNET_BUFFER_F_GSO | VNET_BUFFER_F_IS_IP6; |
Andrew Yourtchenko | 6a7cff7 | 2018-10-12 16:09:22 +0200 | [diff] [blame] | 179 | } |
| 180 | } |
| 181 | |
Mohsin Kazmi | b977d3f | 2020-11-16 16:49:30 +0100 | [diff] [blame] | 182 | static_always_inline u16 |
| 183 | virtio_n_left_to_process (virtio_vring_t * vring, const int packed) |
| 184 | { |
| 185 | if (packed) |
| 186 | return vring->desc_in_use; |
| 187 | else |
| 188 | return vring->used->idx - vring->last_used_idx; |
| 189 | } |
| 190 | |
| 191 | static_always_inline u16 |
| 192 | virtio_get_slot_id (virtio_vring_t * vring, const int packed, u16 last, |
| 193 | u16 mask) |
| 194 | { |
| 195 | if (packed) |
| 196 | return vring->packed_desc[last].id; |
| 197 | else |
| 198 | return vring->used->ring[last & mask].id; |
| 199 | } |
| 200 | |
| 201 | static_always_inline u16 |
| 202 | virtio_get_len (virtio_vring_t * vring, const int packed, const int hdr_sz, |
| 203 | u16 last, u16 mask) |
| 204 | { |
| 205 | if (packed) |
| 206 | return vring->packed_desc[last].len - hdr_sz; |
| 207 | else |
| 208 | return vring->used->ring[last & mask].len - hdr_sz; |
| 209 | } |
| 210 | |
| 211 | #define increment_last(last, packed, vring) \ |
| 212 | do { \ |
| 213 | last++; \ |
| 214 | if (packed && last >= vring->size) \ |
| 215 | { \ |
| 216 | last = 0; \ |
| 217 | vring->used_wrap_counter ^= 1; \ |
| 218 | } \ |
| 219 | } while (0) |
| 220 | |
Damjan Marion | 8389fb9 | 2017-10-13 18:29:53 +0200 | [diff] [blame] | 221 | static_always_inline uword |
Mohsin Kazmi | 38b0968 | 2020-06-03 18:20:17 +0200 | [diff] [blame] | 222 | virtio_device_input_gso_inline (vlib_main_t * vm, vlib_node_runtime_t * node, |
| 223 | vlib_frame_t * frame, virtio_if_t * vif, |
Damjan Marion | 4769ed9 | 2020-10-18 14:00:43 +0200 | [diff] [blame] | 224 | virtio_vring_t * vring, virtio_if_type_t type, |
Mohsin Kazmi | b977d3f | 2020-11-16 16:49:30 +0100 | [diff] [blame] | 225 | int gso_enabled, int checksum_offload_enabled, |
| 226 | int packed) |
Damjan Marion | 8389fb9 | 2017-10-13 18:29:53 +0200 | [diff] [blame] | 227 | { |
| 228 | vnet_main_t *vnm = vnet_get_main (); |
Damjan Marion | 067cd62 | 2018-07-11 12:47:43 +0200 | [diff] [blame] | 229 | u32 thread_index = vm->thread_index; |
Damjan Marion | 8389fb9 | 2017-10-13 18:29:53 +0200 | [diff] [blame] | 230 | uword n_trace = vlib_get_trace_count (vm, node); |
Damjan Marion | d96b28a | 2020-10-19 17:15:58 +0200 | [diff] [blame] | 231 | u32 next_index; |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 232 | const int hdr_sz = vif->virtio_net_hdr_sz; |
Damjan Marion | 8389fb9 | 2017-10-13 18:29:53 +0200 | [diff] [blame] | 233 | u32 *to_next = 0; |
| 234 | u32 n_rx_packets = 0; |
| 235 | u32 n_rx_bytes = 0; |
| 236 | u16 mask = vring->size - 1; |
| 237 | u16 last = vring->last_used_idx; |
Mohsin Kazmi | b977d3f | 2020-11-16 16:49:30 +0100 | [diff] [blame] | 238 | u16 n_left = virtio_n_left_to_process (vring, packed); |
Mohsin Kazmi | 6259406 | 2021-11-05 16:13:57 +0000 | [diff] [blame] | 239 | vlib_buffer_t bt = {}; |
Damjan Marion | 8389fb9 | 2017-10-13 18:29:53 +0200 | [diff] [blame] | 240 | |
| 241 | if (n_left == 0) |
Damjan Marion | 4769ed9 | 2020-10-18 14:00:43 +0200 | [diff] [blame] | 242 | return 0; |
Damjan Marion | 8389fb9 | 2017-10-13 18:29:53 +0200 | [diff] [blame] | 243 | |
Mohsin Kazmi | 38b0968 | 2020-06-03 18:20:17 +0200 | [diff] [blame] | 244 | if (type == VIRTIO_IF_TYPE_TUN) |
Damjan Marion | d96b28a | 2020-10-19 17:15:58 +0200 | [diff] [blame] | 245 | { |
| 246 | next_index = VNET_DEVICE_INPUT_NEXT_IP4_INPUT; |
| 247 | } |
| 248 | else |
| 249 | { |
| 250 | next_index = VNET_DEVICE_INPUT_NEXT_ETHERNET_INPUT; |
| 251 | if (PREDICT_FALSE (vif->per_interface_next_index != ~0)) |
| 252 | next_index = vif->per_interface_next_index; |
| 253 | |
| 254 | /* only for l2, redirect if feature path enabled */ |
| 255 | vnet_feature_start_device_input_x1 (vif->sw_if_index, &next_index, &bt); |
| 256 | } |
Mohsin Kazmi | 38b0968 | 2020-06-03 18:20:17 +0200 | [diff] [blame] | 257 | |
Damjan Marion | 8389fb9 | 2017-10-13 18:29:53 +0200 | [diff] [blame] | 258 | while (n_left) |
| 259 | { |
| 260 | u32 n_left_to_next; |
| 261 | u32 next0 = next_index; |
Mohsin Kazmi | 38b0968 | 2020-06-03 18:20:17 +0200 | [diff] [blame] | 262 | |
Damjan Marion | 8389fb9 | 2017-10-13 18:29:53 +0200 | [diff] [blame] | 263 | vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next); |
| 264 | |
| 265 | while (n_left && n_left_to_next) |
| 266 | { |
Mohsin Kazmi | b977d3f | 2020-11-16 16:49:30 +0100 | [diff] [blame] | 267 | if (packed) |
| 268 | { |
| 269 | vring_packed_desc_t *d = &vring->packed_desc[last]; |
| 270 | u16 flags = d->flags; |
| 271 | if ((flags & VRING_DESC_F_AVAIL) != |
| 272 | (vring->used_wrap_counter << 7) |
| 273 | || (flags & VRING_DESC_F_USED) != |
| 274 | (vring->used_wrap_counter << 15)) |
| 275 | { |
| 276 | n_left = 0; |
| 277 | break; |
| 278 | } |
| 279 | } |
Mohsin Kazmi | 516e4ed | 2020-02-24 15:54:24 +0100 | [diff] [blame] | 280 | u8 l4_proto = 0, l4_hdr_sz = 0; |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 281 | u16 num_buffers = 1; |
Mohsin Kazmi | a7a2281 | 2020-08-31 17:17:16 +0200 | [diff] [blame] | 282 | virtio_net_hdr_v1_t *hdr; |
Mohsin Kazmi | b977d3f | 2020-11-16 16:49:30 +0100 | [diff] [blame] | 283 | u16 slot = virtio_get_slot_id (vring, packed, last, mask); |
| 284 | u16 len = virtio_get_len (vring, packed, hdr_sz, last, mask); |
Damjan Marion | 8389fb9 | 2017-10-13 18:29:53 +0200 | [diff] [blame] | 285 | u32 bi0 = vring->buffers[slot]; |
| 286 | vlib_buffer_t *b0 = vlib_get_buffer (vm, bi0); |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 287 | hdr = vlib_buffer_get_current (b0); |
Mohsin Kazmi | a7a2281 | 2020-08-31 17:17:16 +0200 | [diff] [blame] | 288 | if (hdr_sz == sizeof (virtio_net_hdr_v1_t)) |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 289 | num_buffers = hdr->num_buffers; |
Damjan Marion | 8389fb9 | 2017-10-13 18:29:53 +0200 | [diff] [blame] | 290 | |
Mohsin Kazmi | 8975dbd | 2020-06-24 16:19:19 +0200 | [diff] [blame] | 291 | b0->flags = VLIB_BUFFER_TOTAL_LENGTH_VALID; |
Mohsin Kazmi | 38b0968 | 2020-06-03 18:20:17 +0200 | [diff] [blame] | 292 | b0->current_data = 0; |
Damjan Marion | 8389fb9 | 2017-10-13 18:29:53 +0200 | [diff] [blame] | 293 | b0->current_length = len; |
Andrew Yourtchenko | 6a7cff7 | 2018-10-12 16:09:22 +0200 | [diff] [blame] | 294 | |
Mohsin Kazmi | 6d4af89 | 2020-01-03 15:11:53 +0000 | [diff] [blame] | 295 | if (checksum_offload_enabled) |
Mohsin Kazmi | 38b0968 | 2020-06-03 18:20:17 +0200 | [diff] [blame] | 296 | virtio_needs_csum (b0, hdr, &l4_proto, &l4_hdr_sz, type); |
BenoƮt Ganne | 15d7fd0 | 2019-11-26 17:59:41 +0100 | [diff] [blame] | 297 | |
Andrew Yourtchenko | 6a7cff7 | 2018-10-12 16:09:22 +0200 | [diff] [blame] | 298 | if (gso_enabled) |
BenoƮt Ganne | 15d7fd0 | 2019-11-26 17:59:41 +0100 | [diff] [blame] | 299 | fill_gso_buffer_flags (b0, hdr, l4_proto, l4_hdr_sz); |
Andrew Yourtchenko | 6a7cff7 | 2018-10-12 16:09:22 +0200 | [diff] [blame] | 300 | |
Damjan Marion | 8389fb9 | 2017-10-13 18:29:53 +0200 | [diff] [blame] | 301 | vnet_buffer (b0)->sw_if_index[VLIB_RX] = vif->sw_if_index; |
| 302 | vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0; |
| 303 | |
| 304 | /* if multisegment packet */ |
| 305 | if (PREDICT_FALSE (num_buffers > 1)) |
| 306 | { |
| 307 | vlib_buffer_t *pb, *cb; |
| 308 | pb = b0; |
Mohsin Kazmi | 38b0968 | 2020-06-03 18:20:17 +0200 | [diff] [blame] | 309 | b0->total_length_not_including_first_buffer = 0; |
Damjan Marion | 8389fb9 | 2017-10-13 18:29:53 +0200 | [diff] [blame] | 310 | while (num_buffers > 1) |
| 311 | { |
Mohsin Kazmi | b977d3f | 2020-11-16 16:49:30 +0100 | [diff] [blame] | 312 | increment_last (last, packed, vring); |
| 313 | u16 cslot = virtio_get_slot_id (vring, packed, last, mask); |
Mohsin Kazmi | a17381a | 2020-12-03 11:35:23 +0100 | [diff] [blame] | 314 | /* hdr size is 0 after 1st packet in chain buffers */ |
| 315 | u16 clen = virtio_get_len (vring, packed, 0, last, mask); |
Mohsin Kazmi | b977d3f | 2020-11-16 16:49:30 +0100 | [diff] [blame] | 316 | u32 cbi = vring->buffers[cslot]; |
Damjan Marion | 8389fb9 | 2017-10-13 18:29:53 +0200 | [diff] [blame] | 317 | cb = vlib_get_buffer (vm, cbi); |
| 318 | |
| 319 | /* current buffer */ |
Mohsin Kazmi | b977d3f | 2020-11-16 16:49:30 +0100 | [diff] [blame] | 320 | cb->current_length = clen; |
Damjan Marion | 8389fb9 | 2017-10-13 18:29:53 +0200 | [diff] [blame] | 321 | |
| 322 | /* previous buffer */ |
| 323 | pb->next_buffer = cbi; |
| 324 | pb->flags |= VLIB_BUFFER_NEXT_PRESENT; |
| 325 | |
| 326 | /* first buffer */ |
Mohsin Kazmi | b977d3f | 2020-11-16 16:49:30 +0100 | [diff] [blame] | 327 | b0->total_length_not_including_first_buffer += clen; |
Damjan Marion | 8389fb9 | 2017-10-13 18:29:53 +0200 | [diff] [blame] | 328 | |
| 329 | pb = cb; |
| 330 | vring->desc_in_use--; |
| 331 | num_buffers--; |
| 332 | n_left--; |
| 333 | } |
Mohsin Kazmi | 38b0968 | 2020-06-03 18:20:17 +0200 | [diff] [blame] | 334 | len += b0->total_length_not_including_first_buffer; |
Damjan Marion | 8389fb9 | 2017-10-13 18:29:53 +0200 | [diff] [blame] | 335 | } |
Mohsin Kazmi | 38b0968 | 2020-06-03 18:20:17 +0200 | [diff] [blame] | 336 | |
| 337 | if (type == VIRTIO_IF_TYPE_TUN) |
Mohsin Kazmi | 206acf8 | 2020-04-06 14:19:54 +0200 | [diff] [blame] | 338 | { |
| 339 | switch (b0->data[0] & 0xf0) |
| 340 | { |
| 341 | case 0x40: |
| 342 | next0 = VNET_DEVICE_INPUT_NEXT_IP4_INPUT; |
| 343 | break; |
| 344 | case 0x60: |
| 345 | next0 = VNET_DEVICE_INPUT_NEXT_IP6_INPUT; |
| 346 | break; |
| 347 | default: |
| 348 | next0 = VNET_DEVICE_INPUT_NEXT_DROP; |
| 349 | break; |
| 350 | } |
Damjan Marion | d96b28a | 2020-10-19 17:15:58 +0200 | [diff] [blame] | 351 | |
| 352 | if (PREDICT_FALSE (vif->per_interface_next_index != ~0)) |
| 353 | next0 = vif->per_interface_next_index; |
Mohsin Kazmi | 206acf8 | 2020-04-06 14:19:54 +0200 | [diff] [blame] | 354 | } |
Damjan Marion | d96b28a | 2020-10-19 17:15:58 +0200 | [diff] [blame] | 355 | else |
Mohsin Kazmi | 38b0968 | 2020-06-03 18:20:17 +0200 | [diff] [blame] | 356 | { |
Damjan Marion | d96b28a | 2020-10-19 17:15:58 +0200 | [diff] [blame] | 357 | /* copy feature arc data from template */ |
| 358 | b0->current_config_index = bt.current_config_index; |
| 359 | vnet_buffer (b0)->feature_arc_index = |
| 360 | vnet_buffer (&bt)->feature_arc_index; |
Mohsin Kazmi | 38b0968 | 2020-06-03 18:20:17 +0200 | [diff] [blame] | 361 | } |
Damjan Marion | 06c194d | 2019-11-13 10:12:53 +0100 | [diff] [blame] | 362 | |
Damjan Marion | 8389fb9 | 2017-10-13 18:29:53 +0200 | [diff] [blame] | 363 | /* trace */ |
BenoƮt Ganne | 9a3973e | 2020-10-02 19:36:57 +0200 | [diff] [blame] | 364 | if (PREDICT_FALSE (n_trace > 0 && vlib_trace_buffer (vm, node, next0, b0, /* follow_chain */ |
| 365 | 1))) |
Damjan Marion | 8389fb9 | 2017-10-13 18:29:53 +0200 | [diff] [blame] | 366 | { |
| 367 | virtio_input_trace_t *tr; |
Damjan Marion | 8389fb9 | 2017-10-13 18:29:53 +0200 | [diff] [blame] | 368 | vlib_set_trace_count (vm, node, --n_trace); |
| 369 | tr = vlib_add_trace (vm, node, b0, sizeof (*tr)); |
| 370 | tr->next_index = next0; |
| 371 | tr->hw_if_index = vif->hw_if_index; |
Mohsin Kazmi | 38b0968 | 2020-06-03 18:20:17 +0200 | [diff] [blame] | 372 | tr->len = len; |
Mohsin Kazmi | 049a2ea | 2021-11-11 19:17:27 +0100 | [diff] [blame] | 373 | clib_memcpy_fast (&tr->hdr, hdr, (hdr_sz == 12) ? 12 : 10); |
Damjan Marion | 8389fb9 | 2017-10-13 18:29:53 +0200 | [diff] [blame] | 374 | } |
| 375 | |
| 376 | /* enqueue buffer */ |
| 377 | to_next[0] = bi0; |
| 378 | vring->desc_in_use--; |
| 379 | to_next += 1; |
| 380 | n_left_to_next--; |
| 381 | n_left--; |
Mohsin Kazmi | b977d3f | 2020-11-16 16:49:30 +0100 | [diff] [blame] | 382 | increment_last (last, packed, vring); |
Damjan Marion | 8389fb9 | 2017-10-13 18:29:53 +0200 | [diff] [blame] | 383 | |
Damjan Marion | d96b28a | 2020-10-19 17:15:58 +0200 | [diff] [blame] | 384 | /* only tun interfaces may have different next index */ |
| 385 | if (type == VIRTIO_IF_TYPE_TUN) |
| 386 | vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next, |
| 387 | n_left_to_next, bi0, next0); |
Damjan Marion | 8389fb9 | 2017-10-13 18:29:53 +0200 | [diff] [blame] | 388 | |
| 389 | /* next packet */ |
| 390 | n_rx_packets++; |
Mohsin Kazmi | 38b0968 | 2020-06-03 18:20:17 +0200 | [diff] [blame] | 391 | n_rx_bytes += len; |
Damjan Marion | 8389fb9 | 2017-10-13 18:29:53 +0200 | [diff] [blame] | 392 | } |
| 393 | vlib_put_next_frame (vm, node, next_index, n_left_to_next); |
| 394 | } |
| 395 | vring->last_used_idx = last; |
| 396 | |
| 397 | vlib_increment_combined_counter (vnm->interface_main.combined_sw_if_counters |
| 398 | + VNET_INTERFACE_COUNTER_RX, thread_index, |
Steven Luong | efa119d | 2019-08-30 10:49:44 -0700 | [diff] [blame] | 399 | vif->sw_if_index, n_rx_packets, |
Damjan Marion | 8389fb9 | 2017-10-13 18:29:53 +0200 | [diff] [blame] | 400 | n_rx_bytes); |
| 401 | |
Damjan Marion | 8389fb9 | 2017-10-13 18:29:53 +0200 | [diff] [blame] | 402 | return n_rx_packets; |
| 403 | } |
| 404 | |
Mohsin Kazmi | 38b0968 | 2020-06-03 18:20:17 +0200 | [diff] [blame] | 405 | static_always_inline uword |
| 406 | virtio_device_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node, |
| 407 | vlib_frame_t * frame, virtio_if_t * vif, u16 qid, |
| 408 | virtio_if_type_t type) |
| 409 | { |
Damjan Marion | 4769ed9 | 2020-10-18 14:00:43 +0200 | [diff] [blame] | 410 | virtio_vring_t *vring = vec_elt_at_index (vif->rxq_vrings, qid); |
| 411 | const int hdr_sz = vif->virtio_net_hdr_sz; |
Damjan Marion | 4769ed9 | 2020-10-18 14:00:43 +0200 | [diff] [blame] | 412 | uword rv; |
| 413 | |
Mohsin Kazmi | b977d3f | 2020-11-16 16:49:30 +0100 | [diff] [blame] | 414 | if (vif->is_packed) |
| 415 | { |
Mohsin Kazmi | b977d3f | 2020-11-16 16:49:30 +0100 | [diff] [blame] | 416 | if (vif->gso_enabled) |
| 417 | rv = |
| 418 | virtio_device_input_gso_inline (vm, node, frame, vif, vring, type, |
| 419 | 1, 1, 1); |
| 420 | else if (vif->csum_offload_enabled) |
| 421 | rv = |
| 422 | virtio_device_input_gso_inline (vm, node, frame, vif, vring, type, |
| 423 | 0, 1, 1); |
| 424 | else |
| 425 | rv = |
| 426 | virtio_device_input_gso_inline (vm, node, frame, vif, vring, type, |
| 427 | 0, 0, 1); |
| 428 | |
| 429 | virtio_refill_vring_packed (vm, vif, type, vring, hdr_sz, |
| 430 | node->node_index); |
| 431 | } |
Mohsin Kazmi | 38b0968 | 2020-06-03 18:20:17 +0200 | [diff] [blame] | 432 | else |
Mohsin Kazmi | b977d3f | 2020-11-16 16:49:30 +0100 | [diff] [blame] | 433 | { |
Mohsin Kazmi | b977d3f | 2020-11-16 16:49:30 +0100 | [diff] [blame] | 434 | if (vif->gso_enabled) |
| 435 | rv = |
| 436 | virtio_device_input_gso_inline (vm, node, frame, vif, vring, type, |
| 437 | 1, 1, 0); |
| 438 | else if (vif->csum_offload_enabled) |
| 439 | rv = |
| 440 | virtio_device_input_gso_inline (vm, node, frame, vif, vring, type, |
| 441 | 0, 1, 0); |
| 442 | else |
| 443 | rv = |
| 444 | virtio_device_input_gso_inline (vm, node, frame, vif, vring, type, |
| 445 | 0, 0, 0); |
| 446 | |
| 447 | virtio_refill_vring_split (vm, vif, type, vring, hdr_sz, |
| 448 | node->node_index); |
| 449 | } |
Damjan Marion | 4769ed9 | 2020-10-18 14:00:43 +0200 | [diff] [blame] | 450 | return rv; |
Mohsin Kazmi | 38b0968 | 2020-06-03 18:20:17 +0200 | [diff] [blame] | 451 | } |
| 452 | |
Filip Tehlar | 608996d | 2019-03-04 03:03:13 -0800 | [diff] [blame] | 453 | VLIB_NODE_FN (virtio_input_node) (vlib_main_t * vm, |
| 454 | vlib_node_runtime_t * node, |
| 455 | vlib_frame_t * frame) |
Damjan Marion | 8389fb9 | 2017-10-13 18:29:53 +0200 | [diff] [blame] | 456 | { |
| 457 | u32 n_rx = 0; |
Damjan Marion | 9410053 | 2020-11-06 23:25:57 +0100 | [diff] [blame] | 458 | virtio_main_t *vim = &virtio_main; |
| 459 | vnet_hw_if_rxq_poll_vector_t *p, |
| 460 | *pv = vnet_hw_if_get_rxq_poll_vector (vm, node); |
Damjan Marion | 8389fb9 | 2017-10-13 18:29:53 +0200 | [diff] [blame] | 461 | |
Damjan Marion | 9410053 | 2020-11-06 23:25:57 +0100 | [diff] [blame] | 462 | vec_foreach (p, pv) |
| 463 | { |
| 464 | virtio_if_t *vif; |
| 465 | vif = vec_elt_at_index (vim->interfaces, p->dev_instance); |
| 466 | if (vif->flags & VIRTIO_IF_FLAG_ADMIN_UP) |
| 467 | { |
| 468 | if (vif->type == VIRTIO_IF_TYPE_TAP) |
| 469 | n_rx += virtio_device_input_inline ( |
| 470 | vm, node, frame, vif, p->queue_id, VIRTIO_IF_TYPE_TAP); |
| 471 | else if (vif->type == VIRTIO_IF_TYPE_PCI) |
| 472 | n_rx += virtio_device_input_inline ( |
| 473 | vm, node, frame, vif, p->queue_id, VIRTIO_IF_TYPE_PCI); |
| 474 | else if (vif->type == VIRTIO_IF_TYPE_TUN) |
| 475 | n_rx += virtio_device_input_inline ( |
| 476 | vm, node, frame, vif, p->queue_id, VIRTIO_IF_TYPE_TUN); |
| 477 | } |
| 478 | } |
Damjan Marion | 8389fb9 | 2017-10-13 18:29:53 +0200 | [diff] [blame] | 479 | |
| 480 | return n_rx; |
| 481 | } |
| 482 | |
| 483 | /* *INDENT-OFF* */ |
| 484 | VLIB_REGISTER_NODE (virtio_input_node) = { |
Damjan Marion | 8389fb9 | 2017-10-13 18:29:53 +0200 | [diff] [blame] | 485 | .name = "virtio-input", |
| 486 | .sibling_of = "device-input", |
| 487 | .format_trace = format_virtio_input_trace, |
Damjan Marion | 7ca5aaa | 2019-09-24 18:10:49 +0200 | [diff] [blame] | 488 | .flags = VLIB_NODE_FLAG_TRACE_SUPPORTED, |
Damjan Marion | 8389fb9 | 2017-10-13 18:29:53 +0200 | [diff] [blame] | 489 | .type = VLIB_NODE_TYPE_INPUT, |
| 490 | .state = VLIB_NODE_STATE_INTERRUPT, |
Mohsin Kazmi | ddd2183 | 2019-01-22 13:05:00 +0000 | [diff] [blame] | 491 | .n_errors = VIRTIO_INPUT_N_ERROR, |
Damjan Marion | 8389fb9 | 2017-10-13 18:29:53 +0200 | [diff] [blame] | 492 | .error_strings = virtio_input_error_strings, |
| 493 | }; |
Damjan Marion | 8389fb9 | 2017-10-13 18:29:53 +0200 | [diff] [blame] | 494 | /* *INDENT-ON* */ |
| 495 | |
| 496 | /* |
| 497 | * fd.io coding-style-patch-verification: ON |
| 498 | * |
| 499 | * Local Variables: |
| 500 | * eval: (c-set-style "gnu") |
| 501 | * End: |
| 502 | */ |