blob: 6c2fe34abf0fa08b2ac2f979e3d546a72a011650 [file] [log] [blame]
Damjan Marion8389fb92017-10-13 18:29:53 +02001/*
2 *------------------------------------------------------------------
3 * Copyright (c) 2016 Cisco and/or its affiliates.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at:
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 *------------------------------------------------------------------
16 */
17
18#include <sys/types.h>
19#include <sys/stat.h>
20#include <fcntl.h>
Damjan Marion8389fb92017-10-13 18:29:53 +020021
22#include <vlib/vlib.h>
23#include <vlib/unix/unix.h>
Mohsin Kazmi206acf82020-04-06 14:19:54 +020024#include <vnet/vnet.h>
Damjan Marion8389fb92017-10-13 18:29:53 +020025#include <vnet/ethernet/ethernet.h>
Mohsin Kazmi9e2a7852020-08-13 18:57:26 +020026#include <vnet/gso/gro_func.h>
Mohsin Kazmi0b042092020-04-17 16:50:56 +000027#include <vnet/gso/hdr_offset_parser.h>
Milan Lenco73e7f422017-12-14 10:04:25 +010028#include <vnet/ip/ip4_packet.h>
29#include <vnet/ip/ip6_packet.h>
Mohsin Kazmi6d4af892020-01-03 15:11:53 +000030#include <vnet/tcp/tcp_packet.h>
31#include <vnet/udp/udp_packet.h>
Damjan Marion8389fb92017-10-13 18:29:53 +020032#include <vnet/devices/virtio/virtio.h>
33
Mohsin Kazmi637c97c2021-05-12 14:51:29 +020034#define VIRTIO_TX_MAX_CHAIN_LEN 127
35
Damjan Marion8389fb92017-10-13 18:29:53 +020036#define foreach_virtio_tx_func_error \
37_(NO_FREE_SLOTS, "no free tx slots") \
38_(TRUNC_PACKET, "packet > buffer size -- truncated in tx ring") \
39_(PENDING_MSGS, "pending msgs in tx ring") \
Mohsin Kazmif50ef402020-04-09 22:11:29 +020040_(INDIRECT_DESC_ALLOC_FAILED, "indirect descriptor allocation failed - packet drop") \
41_(OUT_OF_ORDER, "out-of-order buffers in used ring") \
42_(GSO_PACKET_DROP, "gso disabled on itf -- gso packet drop") \
43_(CSUM_OFFLOAD_PACKET_DROP, "checksum offload disabled on itf -- csum offload packet drop")
Damjan Marion8389fb92017-10-13 18:29:53 +020044
45typedef enum
46{
Mohsin Kazmid6c15af2018-10-23 18:00:47 +020047#define _(f,s) VIRTIO_TX_ERROR_##f,
Damjan Marion8389fb92017-10-13 18:29:53 +020048 foreach_virtio_tx_func_error
49#undef _
Mohsin Kazmid6c15af2018-10-23 18:00:47 +020050 VIRTIO_TX_N_ERROR,
Damjan Marion8389fb92017-10-13 18:29:53 +020051} virtio_tx_func_error_t;
52
53static char *virtio_tx_func_error_strings[] = {
54#define _(n,s) s,
55 foreach_virtio_tx_func_error
56#undef _
57};
58
Damjan Marion8389fb92017-10-13 18:29:53 +020059static u8 *
60format_virtio_device (u8 * s, va_list * args)
61{
62 u32 dev_instance = va_arg (*args, u32);
63 int verbose = va_arg (*args, int);
64 u32 indent = format_get_indent (s);
65
66 s = format (s, "VIRTIO interface");
67 if (verbose)
68 {
69 s = format (s, "\n%U instance %u", format_white_space, indent + 2,
70 dev_instance);
71 }
72 return s;
73}
74
Mohsin Kazmib5ca5592020-06-26 13:48:26 +000075typedef struct
Damjan Marion8389fb92017-10-13 18:29:53 +020076{
Mohsin Kazmib5ca5592020-06-26 13:48:26 +000077 u32 buffer_index;
78 u32 sw_if_index;
Mohsin Kazmib5ca5592020-06-26 13:48:26 +000079 generic_header_offset_t gho;
Damjan Marion8c698872020-10-10 19:17:58 +020080 vlib_buffer_t buffer;
Mohsin Kazmib5ca5592020-06-26 13:48:26 +000081} virtio_tx_trace_t;
82
83static u8 *
84format_virtio_tx_trace (u8 * s, va_list * va)
85{
86 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*va, vlib_main_t *);
87 CLIB_UNUSED (vlib_node_t * node) = va_arg (*va, vlib_node_t *);
88 virtio_tx_trace_t *t = va_arg (*va, virtio_tx_trace_t *);
89 u32 indent = format_get_indent (s);
90
BenoƮt Ganne03f2a012021-07-20 16:49:13 +020091 s = format (s, "%Ubuffer 0x%x: %U\n", format_white_space, indent,
92 t->buffer_index, format_vnet_buffer_no_chain, &t->buffer);
Mohsin Kazmib3a8f4e2020-09-15 16:41:44 +000093 s =
94 format (s, "%U%U\n", format_white_space, indent,
95 format_generic_header_offset, &t->gho);
96 s =
97 format (s, "%U%U", format_white_space, indent,
98 format_ethernet_header_with_length, t->buffer.pre_data,
99 sizeof (t->buffer.pre_data));
Damjan Marion8389fb92017-10-13 18:29:53 +0200100 return s;
101}
102
Damjan Marion809d5c92021-03-24 12:24:58 +0100103static void
104virtio_tx_trace (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_buffer_t *b0,
105 u32 bi, int is_tun)
Mohsin Kazmi03b26952020-09-16 16:53:16 +0000106{
107 virtio_tx_trace_t *t;
108 t = vlib_add_trace (vm, node, b0, sizeof (t[0]));
109 t->sw_if_index = vnet_buffer (b0)->sw_if_index[VLIB_TX];
110 t->buffer_index = bi;
Damjan Mariond4a54712020-10-31 23:40:51 +0100111 if (is_tun)
Mohsin Kazmi03b26952020-09-16 16:53:16 +0000112 {
113 int is_ip4 = 0, is_ip6 = 0;
114
115 switch (((u8 *) vlib_buffer_get_current (b0))[0] & 0xf0)
116 {
117 case 0x40:
118 is_ip4 = 1;
119 break;
120 case 0x60:
121 is_ip6 = 1;
122 break;
123 default:
124 break;
125 }
126 vnet_generic_header_offset_parser (b0, &t->gho, 0, is_ip4, is_ip6);
127 }
128 else
129 vnet_generic_header_offset_parser (b0, &t->gho, 1,
130 b0->flags &
131 VNET_BUFFER_F_IS_IP4,
132 b0->flags & VNET_BUFFER_F_IS_IP6);
133
134 clib_memcpy_fast (&t->buffer, b0, sizeof (*b0) - sizeof (b0->pre_data));
135 clib_memcpy_fast (t->buffer.pre_data, vlib_buffer_get_current (b0),
136 sizeof (t->buffer.pre_data));
137}
138
Damjan Marion809d5c92021-03-24 12:24:58 +0100139static void
Mohsin Kazmia02ad342021-05-04 16:36:09 +0200140virtio_interface_drop_inline (vlib_main_t *vm, virtio_if_t *vif,
141 uword node_index, u32 *buffers, u16 n,
142 virtio_tx_func_error_t error)
Mohsin Kazmif50ef402020-04-09 22:11:29 +0200143{
144 vlib_error_count (vm, node_index, error, n);
Mohsin Kazmia02ad342021-05-04 16:36:09 +0200145 vlib_increment_simple_counter (vnet_main.interface_main.sw_if_counters +
146 VNET_INTERFACE_COUNTER_DROP,
147 vm->thread_index, vif->sw_if_index, n);
Mohsin Kazmif50ef402020-04-09 22:11:29 +0200148 vlib_buffer_free (vm, buffers, n);
149}
150
Damjan Marion809d5c92021-03-24 12:24:58 +0100151static void
152virtio_memset_ring_u32 (u32 *ring, u32 start, u32 ring_size, u32 n_buffers)
Mohsin Kazmi7f6d1452020-02-27 11:49:21 +0100153{
154 ASSERT (n_buffers <= ring_size);
155
156 if (PREDICT_TRUE (start + n_buffers <= ring_size))
157 {
158 clib_memset_u32 (ring + start, ~0, n_buffers);
159 }
160 else
161 {
162 clib_memset_u32 (ring + start, ~0, ring_size - start);
163 clib_memset_u32 (ring, ~0, n_buffers - (ring_size - start));
164 }
165}
166
Damjan Marion809d5c92021-03-24 12:24:58 +0100167static void
168virtio_free_used_device_desc_split (vlib_main_t *vm, virtio_vring_t *vring,
Mohsin Kazmib977d3f2020-11-16 16:49:30 +0100169 uword node_index)
Damjan Marion8389fb92017-10-13 18:29:53 +0200170{
171 u16 used = vring->desc_in_use;
172 u16 sz = vring->size;
173 u16 mask = sz - 1;
174 u16 last = vring->last_used_idx;
175 u16 n_left = vring->used->idx - last;
Mohsin Kazmi7f6d1452020-02-27 11:49:21 +0100176 u16 out_of_order_count = 0;
Damjan Marion8389fb92017-10-13 18:29:53 +0200177
178 if (n_left == 0)
179 return;
180
181 while (n_left)
182 {
Mohsin Kazmia7a22812020-08-31 17:17:16 +0200183 vring_used_elem_t *e = &vring->used->ring[last & mask];
Mohsin Kazmiaea0df32019-05-23 14:32:58 +0200184 u16 slot, n_buffers;
185 slot = n_buffers = e->id;
Damjan Marion8389fb92017-10-13 18:29:53 +0200186
Mohsin Kazmi7f6d1452020-02-27 11:49:21 +0100187 while (e->id == (n_buffers & mask))
Mohsin Kazmiaea0df32019-05-23 14:32:58 +0200188 {
189 n_left--;
190 last++;
191 n_buffers++;
Mohsin Kazmia7a22812020-08-31 17:17:16 +0200192 vring_desc_t *d = &vring->desc[e->id];
Mohsin Kazmice215992020-04-10 14:57:06 +0000193 u16 next;
194 while (d->flags & VRING_DESC_F_NEXT)
195 {
196 n_buffers++;
197 next = d->next;
198 d = &vring->desc[next];
199 }
Mohsin Kazmiaea0df32019-05-23 14:32:58 +0200200 if (n_left == 0)
201 break;
202 e = &vring->used->ring[last & mask];
203 }
204 vlib_buffer_free_from_ring (vm, vring->buffers, slot,
205 sz, (n_buffers - slot));
Mohsin Kazmi7f6d1452020-02-27 11:49:21 +0100206 virtio_memset_ring_u32 (vring->buffers, slot, sz, (n_buffers - slot));
Mohsin Kazmiaea0df32019-05-23 14:32:58 +0200207 used -= (n_buffers - slot);
208
209 if (n_left > 0)
210 {
Mohsin Kazmi7f6d1452020-02-27 11:49:21 +0100211 vlib_buffer_free (vm, &vring->buffers[e->id], 1);
212 vring->buffers[e->id] = ~0;
Mohsin Kazmiaea0df32019-05-23 14:32:58 +0200213 used--;
214 last++;
215 n_left--;
Mohsin Kazmi7f6d1452020-02-27 11:49:21 +0100216 out_of_order_count++;
217 vring->flags |= VRING_TX_OUT_OF_ORDER;
Mohsin Kazmiaea0df32019-05-23 14:32:58 +0200218 }
Damjan Marion8389fb92017-10-13 18:29:53 +0200219 }
Mohsin Kazmi7f6d1452020-02-27 11:49:21 +0100220
221 /*
222 * Some vhost-backends give buffers back in out-of-order fashion in used ring.
223 * It impacts the overall virtio-performance.
224 */
225 if (out_of_order_count)
226 vlib_error_count (vm, node_index, VIRTIO_TX_ERROR_OUT_OF_ORDER,
227 out_of_order_count);
228
Damjan Marion8389fb92017-10-13 18:29:53 +0200229 vring->desc_in_use = used;
230 vring->last_used_idx = last;
231}
232
Damjan Marion809d5c92021-03-24 12:24:58 +0100233static void
234virtio_free_used_device_desc_packed (vlib_main_t *vm, virtio_vring_t *vring,
Mohsin Kazmib977d3f2020-11-16 16:49:30 +0100235 uword node_index)
236{
237 vring_packed_desc_t *d;
238 u16 sz = vring->size;
239 u16 last = vring->last_used_idx;
240 u16 n_buffers = 0, start;
241 u16 flags;
242
243 if (vring->desc_in_use == 0)
244 return;
245
246 d = &vring->packed_desc[last];
247 flags = d->flags;
248 start = d->id;
249
250 while ((flags & VRING_DESC_F_AVAIL) == (vring->used_wrap_counter << 7) &&
251 (flags & VRING_DESC_F_USED) == (vring->used_wrap_counter << 15))
252 {
253 last++;
254 n_buffers++;
255
256 if (last >= sz)
257 {
258 last = 0;
259 vring->used_wrap_counter ^= 1;
260 }
261 d = &vring->packed_desc[last];
262 flags = d->flags;
263 }
264
265 if (n_buffers)
266 {
267 vlib_buffer_free_from_ring (vm, vring->buffers, start, sz, n_buffers);
268 virtio_memset_ring_u32 (vring->buffers, start, sz, n_buffers);
269 vring->desc_in_use -= n_buffers;
270 vring->last_used_idx = last;
271 }
272}
273
Damjan Marion809d5c92021-03-24 12:24:58 +0100274static void
275virtio_free_used_device_desc (vlib_main_t *vm, virtio_vring_t *vring,
Mohsin Kazmib977d3f2020-11-16 16:49:30 +0100276 uword node_index, int packed)
277{
278 if (packed)
279 virtio_free_used_device_desc_packed (vm, vring, node_index);
280 else
281 virtio_free_used_device_desc_split (vm, vring, node_index);
282
283}
284
Damjan Marion809d5c92021-03-24 12:24:58 +0100285static void
286set_checksum_offsets (vlib_buffer_t *b, virtio_net_hdr_v1_t *hdr,
Mohsin Kazmib977d3f2020-11-16 16:49:30 +0100287 const int is_l2)
Mohsin Kazmi6d4af892020-01-03 15:11:53 +0000288{
Mohsin Kazmi36f7a6a2021-05-05 14:26:38 +0200289 vnet_buffer_oflags_t oflags = vnet_buffer (b)->oflags;
Mohsin Kazmi68095382021-02-10 11:26:24 +0100290
Mohsin Kazmi6d4af892020-01-03 15:11:53 +0000291 if (b->flags & VNET_BUFFER_F_IS_IP4)
292 {
293 ip4_header_t *ip4;
Mohsin Kazmi0b042092020-04-17 16:50:56 +0000294 generic_header_offset_t gho = { 0 };
Mohsin Kazmi84f91fa2020-04-23 17:59:49 +0200295 vnet_generic_header_offset_parser (b, &gho, is_l2, 1 /* ip4 */ ,
296 0 /* ip6 */ );
Mohsin Kazmi6d4af892020-01-03 15:11:53 +0000297 hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
298 hdr->csum_start = gho.l4_hdr_offset; // 0x22;
Mohsin Kazmi68095382021-02-10 11:26:24 +0100299 if (oflags & VNET_BUFFER_OFFLOAD_F_TCP_CKSUM)
Mohsin Kazmi0937fdf2020-03-25 20:37:16 +0000300 {
Mohsin Kazmi0937fdf2020-03-25 20:37:16 +0000301 hdr->csum_offset = STRUCT_OFFSET_OF (tcp_header_t, checksum);
302 }
Mohsin Kazmi68095382021-02-10 11:26:24 +0100303 else if (oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM)
Mohsin Kazmi0937fdf2020-03-25 20:37:16 +0000304 {
Mohsin Kazmi0937fdf2020-03-25 20:37:16 +0000305 hdr->csum_offset = STRUCT_OFFSET_OF (udp_header_t, checksum);
306 }
Mohsin Kazmi6d4af892020-01-03 15:11:53 +0000307
308 /*
309 * virtio devices do not support IP4 checksum offload. So driver takes care
310 * of it while doing tx.
311 */
312 ip4 =
313 (ip4_header_t *) (vlib_buffer_get_current (b) + gho.l3_hdr_offset);
Mohsin Kazmi68095382021-02-10 11:26:24 +0100314 if (oflags & VNET_BUFFER_OFFLOAD_F_IP_CKSUM)
Mohsin Kazmi6d4af892020-01-03 15:11:53 +0000315 ip4->checksum = ip4_header_checksum (ip4);
316 }
317 else if (b->flags & VNET_BUFFER_F_IS_IP6)
318 {
Mohsin Kazmi0b042092020-04-17 16:50:56 +0000319 generic_header_offset_t gho = { 0 };
Mohsin Kazmi84f91fa2020-04-23 17:59:49 +0200320 vnet_generic_header_offset_parser (b, &gho, is_l2, 0 /* ip4 */ ,
321 1 /* ip6 */ );
Mohsin Kazmi6d4af892020-01-03 15:11:53 +0000322 hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
323 hdr->csum_start = gho.l4_hdr_offset; // 0x36;
Mohsin Kazmi68095382021-02-10 11:26:24 +0100324 if (oflags & VNET_BUFFER_OFFLOAD_F_TCP_CKSUM)
Mohsin Kazmi0937fdf2020-03-25 20:37:16 +0000325 {
Mohsin Kazmi0937fdf2020-03-25 20:37:16 +0000326 hdr->csum_offset = STRUCT_OFFSET_OF (tcp_header_t, checksum);
327 }
Mohsin Kazmi68095382021-02-10 11:26:24 +0100328 else if (oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM)
Mohsin Kazmi0937fdf2020-03-25 20:37:16 +0000329 {
Mohsin Kazmi0937fdf2020-03-25 20:37:16 +0000330 hdr->csum_offset = STRUCT_OFFSET_OF (udp_header_t, checksum);
331 }
Mohsin Kazmi6d4af892020-01-03 15:11:53 +0000332 }
333}
334
Damjan Marion809d5c92021-03-24 12:24:58 +0100335static void
336set_gso_offsets (vlib_buffer_t *b, virtio_net_hdr_v1_t *hdr, const int is_l2)
Mohsin Kazmif50ef402020-04-09 22:11:29 +0200337{
Mohsin Kazmi36f7a6a2021-05-05 14:26:38 +0200338 vnet_buffer_oflags_t oflags = vnet_buffer (b)->oflags;
Mohsin Kazmi68095382021-02-10 11:26:24 +0100339
Mohsin Kazmif50ef402020-04-09 22:11:29 +0200340 if (b->flags & VNET_BUFFER_F_IS_IP4)
341 {
342 ip4_header_t *ip4;
Mohsin Kazmi0b042092020-04-17 16:50:56 +0000343 generic_header_offset_t gho = { 0 };
Mohsin Kazmi84f91fa2020-04-23 17:59:49 +0200344 vnet_generic_header_offset_parser (b, &gho, is_l2, 1 /* ip4 */ ,
345 0 /* ip6 */ );
Mohsin Kazmif50ef402020-04-09 22:11:29 +0200346 hdr->gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
347 hdr->gso_size = vnet_buffer2 (b)->gso_size;
Mohsin Kazmi0b042092020-04-17 16:50:56 +0000348 hdr->hdr_len = gho.hdr_sz;
Mohsin Kazmif50ef402020-04-09 22:11:29 +0200349 hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
350 hdr->csum_start = gho.l4_hdr_offset; // 0x22;
351 hdr->csum_offset = STRUCT_OFFSET_OF (tcp_header_t, checksum);
352 ip4 =
353 (ip4_header_t *) (vlib_buffer_get_current (b) + gho.l3_hdr_offset);
354 /*
355 * virtio devices do not support IP4 checksum offload. So driver takes care
356 * of it while doing tx.
357 */
Mohsin Kazmi68095382021-02-10 11:26:24 +0100358 if (oflags & VNET_BUFFER_OFFLOAD_F_IP_CKSUM)
Mohsin Kazmif50ef402020-04-09 22:11:29 +0200359 ip4->checksum = ip4_header_checksum (ip4);
360 }
361 else if (b->flags & VNET_BUFFER_F_IS_IP6)
362 {
Mohsin Kazmi0b042092020-04-17 16:50:56 +0000363 generic_header_offset_t gho = { 0 };
Mohsin Kazmi84f91fa2020-04-23 17:59:49 +0200364 vnet_generic_header_offset_parser (b, &gho, is_l2, 0 /* ip4 */ ,
365 1 /* ip6 */ );
Mohsin Kazmif50ef402020-04-09 22:11:29 +0200366 hdr->gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
367 hdr->gso_size = vnet_buffer2 (b)->gso_size;
Mohsin Kazmi0b042092020-04-17 16:50:56 +0000368 hdr->hdr_len = gho.hdr_sz;
Mohsin Kazmif50ef402020-04-09 22:11:29 +0200369 hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
370 hdr->csum_start = gho.l4_hdr_offset; // 0x36;
371 hdr->csum_offset = STRUCT_OFFSET_OF (tcp_header_t, checksum);
372 }
373}
374
Damjan Marion809d5c92021-03-24 12:24:58 +0100375static u16
376add_buffer_to_slot (vlib_main_t *vm, vlib_node_runtime_t *node,
Mohsin Kazmia02ad342021-05-04 16:36:09 +0200377 virtio_if_t *vif, virtio_vring_t *vring, u32 bi,
378 u16 free_desc_count, u16 avail, u16 next, u16 mask,
379 int hdr_sz, int do_gso, int csum_offload, int is_pci,
380 int is_tun, int is_indirect, int is_any_layout)
Damjan Marion8389fb92017-10-13 18:29:53 +0200381{
382 u16 n_added = 0;
Mohsin Kazmia7a22812020-08-31 17:17:16 +0200383 vring_desc_t *d;
Damjan Mariond4a54712020-10-31 23:40:51 +0100384 int is_l2 = !is_tun;
Damjan Marion8389fb92017-10-13 18:29:53 +0200385 d = &vring->desc[next];
386 vlib_buffer_t *b = vlib_get_buffer (vm, bi);
Mohsin Kazmia7a22812020-08-31 17:17:16 +0200387 virtio_net_hdr_v1_t *hdr = vlib_buffer_get_current (b) - hdr_sz;
Damjan Mariond4a54712020-10-31 23:40:51 +0100388 u32 drop_inline = ~0;
Damjan Marion508cabc2018-02-08 19:49:22 +0100389
Damjan Mariond4a54712020-10-31 23:40:51 +0100390 clib_memset_u8 (hdr, 0, hdr_sz);
Mohsin Kazmi6d4af892020-01-03 15:11:53 +0000391
Mohsin Kazmif50ef402020-04-09 22:11:29 +0200392 if (b->flags & VNET_BUFFER_F_GSO)
Andrew Yourtchenko6a7cff72018-10-12 16:09:22 +0200393 {
Mohsin Kazmif50ef402020-04-09 22:11:29 +0200394 if (do_gso)
Mohsin Kazmi84f91fa2020-04-23 17:59:49 +0200395 set_gso_offsets (b, hdr, is_l2);
Mohsin Kazmif50ef402020-04-09 22:11:29 +0200396 else
Andrew Yourtchenko6a7cff72018-10-12 16:09:22 +0200397 {
Damjan Mariond4a54712020-10-31 23:40:51 +0100398 drop_inline = VIRTIO_TX_ERROR_GSO_PACKET_DROP;
399 goto done;
Andrew Yourtchenko6a7cff72018-10-12 16:09:22 +0200400 }
401 }
Mohsin Kazmi68095382021-02-10 11:26:24 +0100402 else if (b->flags & VNET_BUFFER_F_OFFLOAD)
Mohsin Kazmi6d4af892020-01-03 15:11:53 +0000403 {
Mohsin Kazmif50ef402020-04-09 22:11:29 +0200404 if (csum_offload)
Mohsin Kazmi84f91fa2020-04-23 17:59:49 +0200405 set_checksum_offsets (b, hdr, is_l2);
Mohsin Kazmif50ef402020-04-09 22:11:29 +0200406 else
407 {
Damjan Mariond4a54712020-10-31 23:40:51 +0100408 drop_inline = VIRTIO_TX_ERROR_CSUM_OFFLOAD_PACKET_DROP;
409 goto done;
Mohsin Kazmif50ef402020-04-09 22:11:29 +0200410 }
Mohsin Kazmi6d4af892020-01-03 15:11:53 +0000411 }
Damjan Marion8389fb92017-10-13 18:29:53 +0200412
Mohsin Kazmi83143712020-10-07 13:23:55 +0200413 if (PREDICT_FALSE (b->flags & VLIB_BUFFER_IS_TRACED))
414 {
Damjan Mariond4a54712020-10-31 23:40:51 +0100415 virtio_tx_trace (vm, node, b, bi, is_tun);
Mohsin Kazmi83143712020-10-07 13:23:55 +0200416 }
417
Damjan Marion8389fb92017-10-13 18:29:53 +0200418 if (PREDICT_TRUE ((b->flags & VLIB_BUFFER_NEXT_PRESENT) == 0))
419 {
Damjan Mariond4a54712020-10-31 23:40:51 +0100420 d->addr = ((is_pci) ? vlib_buffer_get_current_pa (vm, b) :
421 pointer_to_uword (vlib_buffer_get_current (b))) - hdr_sz;
Damjan Marion8389fb92017-10-13 18:29:53 +0200422 d->len = b->current_length + hdr_sz;
423 d->flags = 0;
424 }
Damjan Mariond4a54712020-10-31 23:40:51 +0100425 else if (is_indirect)
Damjan Marion8389fb92017-10-13 18:29:53 +0200426 {
Mohsin Kazmid6c15af2018-10-23 18:00:47 +0200427 /*
428 * We are using single vlib_buffer_t for indirect descriptor(s)
429 * chain. Single descriptor is 16 bytes and vlib_buffer_t
430 * has 2048 bytes space. So maximum long chain can have 128
431 * (=2048/16) indirect descriptors.
432 * It can easily support 65535 bytes of Jumbo frames with
433 * each data buffer size of 512 bytes minimum.
434 */
Mohsin Kazmiaea0df32019-05-23 14:32:58 +0200435 u32 indirect_buffer = 0;
436 if (PREDICT_FALSE (vlib_buffer_alloc (vm, &indirect_buffer, 1) == 0))
Mohsin Kazmif50ef402020-04-09 22:11:29 +0200437 {
Damjan Mariond4a54712020-10-31 23:40:51 +0100438 drop_inline = VIRTIO_TX_ERROR_INDIRECT_DESC_ALLOC_FAILED;
439 goto done;
Mohsin Kazmif50ef402020-04-09 22:11:29 +0200440 }
Mohsin Kazmiaea0df32019-05-23 14:32:58 +0200441
442 vlib_buffer_t *indirect_desc = vlib_get_buffer (vm, indirect_buffer);
Mohsin Kazmid6c15af2018-10-23 18:00:47 +0200443 indirect_desc->current_data = 0;
Mohsin Kazmiaea0df32019-05-23 14:32:58 +0200444 indirect_desc->flags |= VLIB_BUFFER_NEXT_PRESENT;
445 indirect_desc->next_buffer = bi;
446 bi = indirect_buffer;
Damjan Marion8389fb92017-10-13 18:29:53 +0200447
Mohsin Kazmia7a22812020-08-31 17:17:16 +0200448 vring_desc_t *id =
449 (vring_desc_t *) vlib_buffer_get_current (indirect_desc);
Mohsin Kazmid6c15af2018-10-23 18:00:47 +0200450 u32 count = 1;
Damjan Mariond4a54712020-10-31 23:40:51 +0100451 if (is_pci)
Damjan Marion8389fb92017-10-13 18:29:53 +0200452 {
Mohsin Kazmid6c15af2018-10-23 18:00:47 +0200453 d->addr = vlib_physmem_get_pa (vm, id);
454 id->addr = vlib_buffer_get_current_pa (vm, b) - hdr_sz;
Damjan Marion8389fb92017-10-13 18:29:53 +0200455
Mohsin Kazmid6c15af2018-10-23 18:00:47 +0200456 /*
457 * If VIRTIO_F_ANY_LAYOUT is not negotiated, then virtio_net_hdr
458 * should be presented in separate descriptor and data will start
459 * from next descriptor.
460 */
Damjan Mariond4a54712020-10-31 23:40:51 +0100461 if (is_any_layout)
Mohsin Kazmid6c15af2018-10-23 18:00:47 +0200462 id->len = b->current_length + hdr_sz;
463 else
464 {
465 id->len = hdr_sz;
466 id->flags = VRING_DESC_F_NEXT;
467 id->next = count;
468 count++;
469 id++;
470 id->addr = vlib_buffer_get_current_pa (vm, b);
471 id->len = b->current_length;
472 }
473 while (b->flags & VLIB_BUFFER_NEXT_PRESENT)
474 {
475 id->flags = VRING_DESC_F_NEXT;
476 id->next = count;
477 count++;
478 id++;
479 b = vlib_get_buffer (vm, b->next_buffer);
480 id->addr = vlib_buffer_get_current_pa (vm, b);
481 id->len = b->current_length;
Mohsin Kazmi637c97c2021-05-12 14:51:29 +0200482 if (PREDICT_FALSE (count == VIRTIO_TX_MAX_CHAIN_LEN))
483 {
484 if (b->flags & VLIB_BUFFER_NEXT_PRESENT)
485 vlib_error_count (vm, node->node_index,
486 VIRTIO_TX_ERROR_TRUNC_PACKET, 1);
487 break;
488 }
Mohsin Kazmid6c15af2018-10-23 18:00:47 +0200489 }
490 }
Mohsin Kazmi206acf82020-04-06 14:19:54 +0200491 else /* VIRTIO_IF_TYPE_[TAP | TUN] */
Mohsin Kazmid6c15af2018-10-23 18:00:47 +0200492 {
493 d->addr = pointer_to_uword (id);
494 /* first buffer in chain */
495 id->addr = pointer_to_uword (vlib_buffer_get_current (b)) - hdr_sz;
496 id->len = b->current_length + hdr_sz;
497
498 while (b->flags & VLIB_BUFFER_NEXT_PRESENT)
499 {
500 id->flags = VRING_DESC_F_NEXT;
501 id->next = count;
502 count++;
503 id++;
504 b = vlib_get_buffer (vm, b->next_buffer);
505 id->addr = pointer_to_uword (vlib_buffer_get_current (b));
506 id->len = b->current_length;
Mohsin Kazmi637c97c2021-05-12 14:51:29 +0200507 if (PREDICT_FALSE (count == VIRTIO_TX_MAX_CHAIN_LEN))
508 {
509 if (b->flags & VLIB_BUFFER_NEXT_PRESENT)
510 vlib_error_count (vm, node->node_index,
511 VIRTIO_TX_ERROR_TRUNC_PACKET, 1);
512 break;
513 }
Mohsin Kazmid6c15af2018-10-23 18:00:47 +0200514 }
515 }
516 id->flags = 0;
517 id->next = 0;
Mohsin Kazmia7a22812020-08-31 17:17:16 +0200518 d->len = count * sizeof (vring_desc_t);
Damjan Marion8389fb92017-10-13 18:29:53 +0200519 d->flags = VRING_DESC_F_INDIRECT;
520 }
Damjan Mariond4a54712020-10-31 23:40:51 +0100521 else if (is_pci)
Mohsin Kazmice215992020-04-10 14:57:06 +0000522 {
523 u16 count = next;
524 vlib_buffer_t *b_temp = b;
525 u16 n_buffers_in_chain = 1;
526
527 /*
528 * Check the length of the chain for the required number of
529 * descriptors. Return from here, retry to get more descriptors,
530 * if chain length is greater than available descriptors.
531 */
532 while (b_temp->flags & VLIB_BUFFER_NEXT_PRESENT)
533 {
534 n_buffers_in_chain++;
535 b_temp = vlib_get_buffer (vm, b_temp->next_buffer);
536 }
537
538 if (n_buffers_in_chain > free_desc_count)
539 return n_buffers_in_chain;
540
541 d->addr = vlib_buffer_get_current_pa (vm, b) - hdr_sz;
542 d->len = b->current_length + hdr_sz;
543
544 while (b->flags & VLIB_BUFFER_NEXT_PRESENT)
545 {
546 d->flags = VRING_DESC_F_NEXT;
547 vring->buffers[count] = bi;
548 b->flags &=
549 ~(VLIB_BUFFER_NEXT_PRESENT | VLIB_BUFFER_TOTAL_LENGTH_VALID);
550 bi = b->next_buffer;
551 b->next_buffer = 0;
552 n_added++;
553 count = (count + 1) & mask;
554 d->next = count;
555 d = &vring->desc[count];
556 b = vlib_get_buffer (vm, bi);
557 d->addr = vlib_buffer_get_current_pa (vm, b);
558 d->len = b->current_length;
559 }
560 d->flags = 0;
561 vring->buffers[count] = bi;
562 vring->avail->ring[avail & mask] = next;
563 n_added++;
564 return n_added;
565 }
566 else
567 {
568 ASSERT (0);
569 }
Damjan Marion8389fb92017-10-13 18:29:53 +0200570 vring->buffers[next] = bi;
571 vring->avail->ring[avail & mask] = next;
572 n_added++;
Damjan Mariond4a54712020-10-31 23:40:51 +0100573
574done:
575 if (drop_inline != ~0)
Mohsin Kazmia02ad342021-05-04 16:36:09 +0200576 virtio_interface_drop_inline (vm, vif, node->node_index, &bi, 1,
577 drop_inline);
Damjan Mariond4a54712020-10-31 23:40:51 +0100578
Damjan Marion8389fb92017-10-13 18:29:53 +0200579 return n_added;
580}
581
Damjan Marion809d5c92021-03-24 12:24:58 +0100582static u16
583add_buffer_to_slot_packed (vlib_main_t *vm, vlib_node_runtime_t *node,
Mohsin Kazmia02ad342021-05-04 16:36:09 +0200584 virtio_if_t *vif, virtio_vring_t *vring, u32 bi,
585 u16 next, int hdr_sz, int do_gso, int csum_offload,
586 int is_pci, int is_tun, int is_indirect,
587 int is_any_layout)
Mohsin Kazmib977d3f2020-11-16 16:49:30 +0100588{
589 u16 n_added = 0, flags = 0;
590 int is_l2 = !is_tun;
591 vring_packed_desc_t *d = &vring->packed_desc[next];
592 vlib_buffer_t *b = vlib_get_buffer (vm, bi);
593 virtio_net_hdr_v1_t *hdr = vlib_buffer_get_current (b) - hdr_sz;
594 u32 drop_inline = ~0;
595
596 clib_memset (hdr, 0, hdr_sz);
597
598 if (b->flags & VNET_BUFFER_F_GSO)
599 {
600 if (do_gso)
601 set_gso_offsets (b, hdr, is_l2);
602 else
603 {
604 drop_inline = VIRTIO_TX_ERROR_GSO_PACKET_DROP;
605 goto done;
606 }
607 }
Mohsin Kazmi68095382021-02-10 11:26:24 +0100608 else if (b->flags & VNET_BUFFER_F_OFFLOAD)
Mohsin Kazmib977d3f2020-11-16 16:49:30 +0100609 {
610 if (csum_offload)
611 set_checksum_offsets (b, hdr, is_l2);
612 else
613 {
614 drop_inline = VIRTIO_TX_ERROR_CSUM_OFFLOAD_PACKET_DROP;
615 goto done;
616 }
617 }
618 if (PREDICT_FALSE (b->flags & VLIB_BUFFER_IS_TRACED))
619 {
620 virtio_tx_trace (vm, node, b, bi, is_tun);
621 }
622
623 if (PREDICT_TRUE ((b->flags & VLIB_BUFFER_NEXT_PRESENT) == 0))
624 {
625 d->addr =
626 ((is_pci) ? vlib_buffer_get_current_pa (vm,
627 b) :
628 pointer_to_uword (vlib_buffer_get_current (b))) - hdr_sz;
629 d->len = b->current_length + hdr_sz;
630 }
631 else if (is_indirect)
632 {
633 /*
634 * We are using single vlib_buffer_t for indirect descriptor(s)
635 * chain. Single descriptor is 16 bytes and vlib_buffer_t
636 * has 2048 bytes space. So maximum long chain can have 128
637 * (=2048/16) indirect descriptors.
638 * It can easily support 65535 bytes of Jumbo frames with
639 * each data buffer size of 512 bytes minimum.
640 */
641 u32 indirect_buffer = 0;
642 if (PREDICT_FALSE (vlib_buffer_alloc (vm, &indirect_buffer, 1) == 0))
643 {
644 drop_inline = VIRTIO_TX_ERROR_INDIRECT_DESC_ALLOC_FAILED;
645 goto done;
646 }
647
648 vlib_buffer_t *indirect_desc = vlib_get_buffer (vm, indirect_buffer);
649 indirect_desc->current_data = 0;
650 indirect_desc->flags |= VLIB_BUFFER_NEXT_PRESENT;
651 indirect_desc->next_buffer = bi;
652 bi = indirect_buffer;
653
654 vring_packed_desc_t *id =
655 (vring_packed_desc_t *) vlib_buffer_get_current (indirect_desc);
656 u32 count = 1;
657 if (is_pci)
658 {
659 d->addr = vlib_physmem_get_pa (vm, id);
660 id->addr = vlib_buffer_get_current_pa (vm, b) - hdr_sz;
661
662 /*
663 * If VIRTIO_F_ANY_LAYOUT is not negotiated, then virtio_net_hdr
664 * should be presented in separate descriptor and data will start
665 * from next descriptor.
666 */
667 if (is_any_layout)
668 id->len = b->current_length + hdr_sz;
669 else
670 {
671 id->len = hdr_sz;
672 id->flags = 0;
673 id->id = 0;
674 count++;
675 id++;
676 id->addr = vlib_buffer_get_current_pa (vm, b);
677 id->len = b->current_length;
678 }
679 while (b->flags & VLIB_BUFFER_NEXT_PRESENT)
680 {
681 id->flags = 0;
682 id->id = 0;
683 count++;
684 id++;
685 b = vlib_get_buffer (vm, b->next_buffer);
686 id->addr = vlib_buffer_get_current_pa (vm, b);
687 id->len = b->current_length;
Mohsin Kazmi637c97c2021-05-12 14:51:29 +0200688 if (PREDICT_FALSE (count == VIRTIO_TX_MAX_CHAIN_LEN))
689 {
690 if (b->flags & VLIB_BUFFER_NEXT_PRESENT)
691 vlib_error_count (vm, node->node_index,
692 VIRTIO_TX_ERROR_TRUNC_PACKET, 1);
693 break;
694 }
Mohsin Kazmib977d3f2020-11-16 16:49:30 +0100695 }
696 }
697 id->flags = 0;
698 id->id = 0;
699 d->len = count * sizeof (vring_packed_desc_t);
700 flags = VRING_DESC_F_INDIRECT;
701 }
702 else
703 {
704 ASSERT (0);
705 }
706 if (vring->avail_wrap_counter)
707 {
708 flags |= VRING_DESC_F_AVAIL;
709 flags &= ~VRING_DESC_F_USED;
710 }
711 else
712 {
713 flags &= ~VRING_DESC_F_AVAIL;
714 flags |= VRING_DESC_F_USED;
715 }
716
717 d->id = next;
718 d->flags = flags;
719 vring->buffers[next] = bi;
720 n_added++;
721
722done:
723 if (drop_inline != ~0)
Mohsin Kazmia02ad342021-05-04 16:36:09 +0200724 virtio_interface_drop_inline (vm, vif, node->node_index, &bi, 1,
725 drop_inline);
Mohsin Kazmib977d3f2020-11-16 16:49:30 +0100726
727 return n_added;
728}
729
Damjan Marion809d5c92021-03-24 12:24:58 +0100730static uword
731virtio_interface_tx_packed_gso_inline (vlib_main_t *vm,
732 vlib_node_runtime_t *node,
733 virtio_if_t *vif, virtio_if_type_t type,
734 virtio_vring_t *vring, u32 *buffers,
735 u16 n_left, const int do_gso,
Mohsin Kazmib977d3f2020-11-16 16:49:30 +0100736 const int csum_offload)
737{
738 int is_pci = (type == VIRTIO_IF_TYPE_PCI);
739 int is_tun = (type == VIRTIO_IF_TYPE_TUN);
740 int is_indirect =
741 ((vif->features & VIRTIO_FEATURE (VIRTIO_RING_F_INDIRECT_DESC)) != 0);
742 int is_any_layout =
743 ((vif->features & VIRTIO_FEATURE (VIRTIO_F_ANY_LAYOUT)) != 0);
744 const int hdr_sz = vif->virtio_net_hdr_sz;
745 u16 sz = vring->size;
746 u16 used, next, n_buffers = 0, n_buffers_left = 0;
747 u16 n_vectors = n_left;
748
749
750 used = vring->desc_in_use;
751 next = vring->desc_next;
752
753 if (vif->packet_buffering)
754 {
755 n_buffers = n_buffers_left = virtio_vring_n_buffers (vring->buffering);
756
757 while (n_buffers_left && used < sz)
758 {
759 u16 n_added = 0;
760
761 u32 bi = virtio_vring_buffering_read_from_front (vring->buffering);
762 if (bi == ~0)
763 break;
Mohsin Kazmia02ad342021-05-04 16:36:09 +0200764 n_added = add_buffer_to_slot_packed (
765 vm, node, vif, vring, bi, next, hdr_sz, do_gso, csum_offload,
766 is_pci, is_tun, is_indirect, is_any_layout);
Mohsin Kazmib977d3f2020-11-16 16:49:30 +0100767 n_buffers_left--;
768 if (PREDICT_FALSE (n_added == 0))
769 continue;
770
771 used++;
772 next++;
773 if (next >= sz)
774 {
775 next = 0;
776 vring->avail_wrap_counter ^= 1;
777 }
778 }
779 }
780
781 while (n_left && used < sz)
782 {
783 u16 n_added = 0;
784
Mohsin Kazmia02ad342021-05-04 16:36:09 +0200785 n_added = add_buffer_to_slot_packed (
786 vm, node, vif, vring, buffers[0], next, hdr_sz, do_gso, csum_offload,
787 is_pci, is_tun, is_indirect, is_any_layout);
Mohsin Kazmib977d3f2020-11-16 16:49:30 +0100788 buffers++;
789 n_left--;
790 if (PREDICT_FALSE (n_added == 0))
791 continue;
792
793 used++;
794 next++;
795 if (next >= sz)
796 {
797 next = 0;
798 vring->avail_wrap_counter ^= 1;
799 }
800 }
801
802 if (n_left != n_vectors || n_buffers != n_buffers_left)
803 {
804 CLIB_MEMORY_STORE_BARRIER ();
805 vring->desc_next = next;
806 vring->desc_in_use = used;
807 CLIB_MEMORY_BARRIER ();
808 if (vring->device_event->flags != VRING_EVENT_F_DISABLE)
809 virtio_kick (vm, vring, vif);
810 }
811
812 return n_left;
813}
814
Damjan Marion809d5c92021-03-24 12:24:58 +0100815static void
816virtio_find_free_desc (virtio_vring_t *vring, u16 size, u16 mask, u16 req,
817 u16 next, u32 *first_free_desc_index,
818 u16 *free_desc_count)
Mohsin Kazmi7f6d1452020-02-27 11:49:21 +0100819{
820 u16 start = 0;
821 /* next is used as hint: from where to start looking */
822 for (u16 i = 0; i < size; i++, next++)
823 {
824 if (vring->buffers[next & mask] == ~0)
825 {
826 if (*first_free_desc_index == ~0)
827 {
828 *first_free_desc_index = (next & mask);
829 start = i;
830 (*free_desc_count)++;
831 req--;
832 if (req == 0)
833 break;
834 }
835 else
836 {
837 if (start + *free_desc_count == i)
838 {
839 (*free_desc_count)++;
840 req--;
841 if (req == 0)
842 break;
843 }
844 else
845 break;
846 }
847 }
848 }
849}
850
Damjan Marion809d5c92021-03-24 12:24:58 +0100851static u16
852virtio_interface_tx_split_gso_inline (vlib_main_t *vm,
853 vlib_node_runtime_t *node,
854 virtio_if_t *vif, virtio_if_type_t type,
855 virtio_vring_t *vring, u32 *buffers,
856 u16 n_left, int do_gso, int csum_offload)
Damjan Marion8389fb92017-10-13 18:29:53 +0200857{
Mohsin Kazmie347acb2020-09-28 10:26:33 +0000858 u16 used, next, avail, n_buffers = 0, n_buffers_left = 0;
Damjan Mariond4a54712020-10-31 23:40:51 +0100859 int is_pci = (type == VIRTIO_IF_TYPE_PCI);
860 int is_tun = (type == VIRTIO_IF_TYPE_TUN);
861 int is_indirect =
862 ((vif->features & VIRTIO_FEATURE (VIRTIO_RING_F_INDIRECT_DESC)) != 0);
863 int is_any_layout =
864 ((vif->features & VIRTIO_FEATURE (VIRTIO_F_ANY_LAYOUT)) != 0);
Damjan Marion8389fb92017-10-13 18:29:53 +0200865 u16 sz = vring->size;
Damjan Mariond4a54712020-10-31 23:40:51 +0100866 int hdr_sz = vif->virtio_net_hdr_sz;
Damjan Marion8389fb92017-10-13 18:29:53 +0200867 u16 mask = sz - 1;
Mohsin Kazmi2e6f6242020-10-26 12:41:41 +0100868 u16 n_vectors = n_left;
Damjan Marion8389fb92017-10-13 18:29:53 +0200869
870 used = vring->desc_in_use;
871 next = vring->desc_next;
872 avail = vring->avail->idx;
873
Mohsin Kazmi7f6d1452020-02-27 11:49:21 +0100874 u16 free_desc_count = 0;
875
876 if (PREDICT_FALSE (vring->flags & VRING_TX_OUT_OF_ORDER))
877 {
878 u32 first_free_desc_index = ~0;
879
880 virtio_find_free_desc (vring, sz, mask, n_left, next,
881 &first_free_desc_index, &free_desc_count);
882
883 if (free_desc_count)
884 next = first_free_desc_index;
885 }
886 else
887 free_desc_count = sz - used;
888
Mohsin Kazmie347acb2020-09-28 10:26:33 +0000889 if (vif->packet_buffering)
890 {
891 n_buffers = n_buffers_left = virtio_vring_n_buffers (vring->buffering);
892
893 while (n_buffers_left && free_desc_count)
894 {
895 u16 n_added = 0;
896
897 u32 bi = virtio_vring_buffering_read_from_front (vring->buffering);
898 if (bi == ~0)
899 break;
Mohsin Kazmi83143712020-10-07 13:23:55 +0200900
Mohsin Kazmia02ad342021-05-04 16:36:09 +0200901 n_added = add_buffer_to_slot (vm, node, vif, vring, bi,
902 free_desc_count, avail, next, mask,
903 hdr_sz, do_gso, csum_offload, is_pci,
904 is_tun, is_indirect, is_any_layout);
Mohsin Kazmie347acb2020-09-28 10:26:33 +0000905 if (PREDICT_FALSE (n_added == 0))
906 {
907 n_buffers_left--;
908 continue;
909 }
910 else if (PREDICT_FALSE (n_added > free_desc_count))
911 break;
912
913 avail++;
914 next = (next + n_added) & mask;
915 used += n_added;
916 n_buffers_left--;
917 free_desc_count -= n_added;
918 }
919 }
920
Mohsin Kazmi7f6d1452020-02-27 11:49:21 +0100921 while (n_left && free_desc_count)
Damjan Marion8389fb92017-10-13 18:29:53 +0200922 {
Mohsin Kazmid6c15af2018-10-23 18:00:47 +0200923 u16 n_added = 0;
Mohsin Kazmib5ca5592020-06-26 13:48:26 +0000924
Mohsin Kazmia02ad342021-05-04 16:36:09 +0200925 n_added =
926 add_buffer_to_slot (vm, node, vif, vring, buffers[0], free_desc_count,
927 avail, next, mask, hdr_sz, do_gso, csum_offload,
928 is_pci, is_tun, is_indirect, is_any_layout);
Mohsin Kazmif50ef402020-04-09 22:11:29 +0200929
930 if (PREDICT_FALSE (n_added == 0))
931 {
932 buffers++;
933 n_left--;
934 continue;
935 }
Mohsin Kazmice215992020-04-10 14:57:06 +0000936 else if (PREDICT_FALSE (n_added > free_desc_count))
937 break;
Mohsin Kazmif50ef402020-04-09 22:11:29 +0200938
Mohsin Kazmice215992020-04-10 14:57:06 +0000939 avail++;
Damjan Marion8389fb92017-10-13 18:29:53 +0200940 next = (next + n_added) & mask;
941 used += n_added;
942 buffers++;
943 n_left--;
Mohsin Kazmice215992020-04-10 14:57:06 +0000944 free_desc_count -= n_added;
Damjan Marion8389fb92017-10-13 18:29:53 +0200945 }
946
Mohsin Kazmi2e6f6242020-10-26 12:41:41 +0100947 if (n_left != n_vectors || n_buffers != n_buffers_left)
Damjan Marion8389fb92017-10-13 18:29:53 +0200948 {
Mohsin Kazmid8b7dec2020-12-10 09:49:21 +0000949 clib_atomic_store_seq_cst (&vring->avail->idx, avail);
Damjan Marion8389fb92017-10-13 18:29:53 +0200950 vring->desc_next = next;
951 vring->desc_in_use = used;
Mohsin Kazmid8b7dec2020-12-10 09:49:21 +0000952 if ((clib_atomic_load_seq_cst (&vring->used->flags) &
953 VRING_USED_F_NO_NOTIFY) == 0)
Mohsin Kazmid6c15af2018-10-23 18:00:47 +0200954 virtio_kick (vm, vring, vif);
Damjan Marion8389fb92017-10-13 18:29:53 +0200955 }
956
Mohsin Kazmi2e6f6242020-10-26 12:41:41 +0100957 return n_left;
Damjan Marion8389fb92017-10-13 18:29:53 +0200958}
959
Damjan Marion809d5c92021-03-24 12:24:58 +0100960static u16
961virtio_interface_tx_gso_inline (vlib_main_t *vm, vlib_node_runtime_t *node,
962 virtio_if_t *vif, virtio_if_type_t type,
963 virtio_vring_t *vring, u32 *buffers,
964 u16 n_left, int packed, int do_gso,
965 int csum_offload)
Mohsin Kazmib977d3f2020-11-16 16:49:30 +0100966{
967 if (packed)
968 return virtio_interface_tx_packed_gso_inline (vm, node, vif, type, vring,
969 buffers, n_left,
Mohsin Kazmib18b1902020-12-03 16:35:20 +0100970 do_gso, csum_offload);
Mohsin Kazmib977d3f2020-11-16 16:49:30 +0100971 else
972 return virtio_interface_tx_split_gso_inline (vm, node, vif, type, vring,
973 buffers, n_left,
Mohsin Kazmib18b1902020-12-03 16:35:20 +0100974 do_gso, csum_offload);
Mohsin Kazmib977d3f2020-11-16 16:49:30 +0100975}
976
Damjan Marion809d5c92021-03-24 12:24:58 +0100977static u16
978virtio_interface_tx_inline (vlib_main_t *vm, vlib_node_runtime_t *node,
979 virtio_if_t *vif, virtio_vring_t *vring,
980 virtio_if_type_t type, u32 *buffers, u16 n_left,
981 int packed)
Mohsin Kazmi38b09682020-06-03 18:20:17 +0200982{
983 vnet_main_t *vnm = vnet_get_main ();
984 vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, vif->hw_if_index);
985
Mohsin Kazmi5b3f5232021-02-10 12:03:53 +0100986 if (hw->caps & VNET_HW_INTERFACE_CAP_SUPPORTS_TCP_GSO)
Mohsin Kazmi2e6f6242020-10-26 12:41:41 +0100987 return virtio_interface_tx_gso_inline (vm, node, vif, type, vring,
Mohsin Kazmib977d3f2020-11-16 16:49:30 +0100988 buffers, n_left, packed,
989 1 /* do_gso */ ,
Damjan Mariondc181242020-10-20 12:11:06 +0200990 1 /* checksum offload */ );
Mohsin Kazmi5b3f5232021-02-10 12:03:53 +0100991 else if (hw->caps & VNET_HW_INTERFACE_CAP_SUPPORTS_L4_TX_CKSUM)
Mohsin Kazmi2e6f6242020-10-26 12:41:41 +0100992 return virtio_interface_tx_gso_inline (vm, node, vif, type, vring,
Mohsin Kazmib977d3f2020-11-16 16:49:30 +0100993 buffers, n_left, packed,
Mohsin Kazmi38b09682020-06-03 18:20:17 +0200994 0 /* no do_gso */ ,
Damjan Mariondc181242020-10-20 12:11:06 +0200995 1 /* checksum offload */ );
Mohsin Kazmi38b09682020-06-03 18:20:17 +0200996 else
Mohsin Kazmi2e6f6242020-10-26 12:41:41 +0100997 return virtio_interface_tx_gso_inline (vm, node, vif, type, vring,
Mohsin Kazmib977d3f2020-11-16 16:49:30 +0100998 buffers, n_left, packed,
Mohsin Kazmi38b09682020-06-03 18:20:17 +0200999 0 /* no do_gso */ ,
Damjan Mariondc181242020-10-20 12:11:06 +02001000 0 /* no checksum offload */ );
Mohsin Kazmi38b09682020-06-03 18:20:17 +02001001}
1002
Filip Tehlaraee73642019-03-13 05:50:44 -07001003VNET_DEVICE_CLASS_TX_FN (virtio_device_class) (vlib_main_t * vm,
1004 vlib_node_runtime_t * node,
1005 vlib_frame_t * frame)
Damjan Marion8389fb92017-10-13 18:29:53 +02001006{
1007 virtio_main_t *nm = &virtio_main;
1008 vnet_interface_output_runtime_t *rund = (void *) node->runtime_data;
1009 virtio_if_t *vif = pool_elt_at_index (nm->interfaces, rund->dev_instance);
Damjan Mariondc181242020-10-20 12:11:06 +02001010 u16 qid = vm->thread_index % vif->num_txqs;
1011 virtio_vring_t *vring = vec_elt_at_index (vif->txq_vrings, qid);
Mohsin Kazmiee1e30f2020-10-23 11:23:52 +02001012 u16 n_left = frame->n_vectors;
1013 u32 *buffers = vlib_frame_vector_args (frame);
1014 u32 to[GRO_TO_VECTOR_SIZE (n_left)];
Mohsin Kazmib977d3f2020-11-16 16:49:30 +01001015 int packed = vif->is_packed;
Mohsin Kazmie2e9fe52020-12-17 15:54:03 +01001016 u16 n_vectors = frame->n_vectors;
Damjan Mariondc181242020-10-20 12:11:06 +02001017
1018 clib_spinlock_lock_if_init (&vring->lockp);
1019
Mohsin Kazmiee1e30f2020-10-23 11:23:52 +02001020 if (vif->packet_coalesce)
1021 {
Mohsin Kazmie2e9fe52020-12-17 15:54:03 +01001022 n_vectors = n_left =
1023 vnet_gro_inline (vm, vring->flow_table, buffers, n_left, to);
Mohsin Kazmiee1e30f2020-10-23 11:23:52 +02001024 buffers = to;
1025 }
1026
Damjan Marion587f9132020-10-31 22:47:01 +01001027 u16 retry_count = 2;
1028
1029retry:
1030 /* free consumed buffers */
Mohsin Kazmib977d3f2020-11-16 16:49:30 +01001031 virtio_free_used_device_desc (vm, vring, node->node_index, packed);
Damjan Marion587f9132020-10-31 22:47:01 +01001032
Mohsin Kazmi38b09682020-06-03 18:20:17 +02001033 if (vif->type == VIRTIO_IF_TYPE_TAP)
Mohsin Kazmi2e6f6242020-10-26 12:41:41 +01001034 n_left = virtio_interface_tx_inline (vm, node, vif, vring,
Mohsin Kazmi81dac062020-11-11 20:28:35 +01001035 VIRTIO_IF_TYPE_TAP,
Mohsin Kazmie2e9fe52020-12-17 15:54:03 +01001036 &buffers[n_vectors - n_left],
Mohsin Kazmib977d3f2020-11-16 16:49:30 +01001037 n_left, packed);
Mohsin Kazmi38b09682020-06-03 18:20:17 +02001038 else if (vif->type == VIRTIO_IF_TYPE_PCI)
Mohsin Kazmi2e6f6242020-10-26 12:41:41 +01001039 n_left = virtio_interface_tx_inline (vm, node, vif, vring,
Mohsin Kazmi81dac062020-11-11 20:28:35 +01001040 VIRTIO_IF_TYPE_PCI,
Mohsin Kazmie2e9fe52020-12-17 15:54:03 +01001041 &buffers[n_vectors - n_left],
Mohsin Kazmib977d3f2020-11-16 16:49:30 +01001042 n_left, packed);
Mohsin Kazmi38b09682020-06-03 18:20:17 +02001043 else if (vif->type == VIRTIO_IF_TYPE_TUN)
Mohsin Kazmi2e6f6242020-10-26 12:41:41 +01001044 n_left = virtio_interface_tx_inline (vm, node, vif, vring,
Mohsin Kazmi81dac062020-11-11 20:28:35 +01001045 VIRTIO_IF_TYPE_TUN,
Mohsin Kazmie2e9fe52020-12-17 15:54:03 +01001046 &buffers[n_vectors - n_left],
Mohsin Kazmib977d3f2020-11-16 16:49:30 +01001047 n_left, packed);
Andrew Yourtchenko6a7cff72018-10-12 16:09:22 +02001048 else
Mohsin Kazmi38b09682020-06-03 18:20:17 +02001049 ASSERT (0);
1050
Damjan Marion587f9132020-10-31 22:47:01 +01001051 if (n_left && retry_count--)
1052 goto retry;
1053
Mohsin Kazmi2e6f6242020-10-26 12:41:41 +01001054 if (vif->packet_buffering && n_left)
1055 {
Mohsin Kazmi81dac062020-11-11 20:28:35 +01001056 u16 n_buffered = virtio_vring_buffering_store_packets (vring->buffering,
1057 &buffers
Mohsin Kazmie2e9fe52020-12-17 15:54:03 +01001058 [n_vectors
Mohsin Kazmi81dac062020-11-11 20:28:35 +01001059 - n_left],
1060 n_left);
Mohsin Kazmi2e6f6242020-10-26 12:41:41 +01001061 n_left -= n_buffered;
1062 }
1063 if (n_left)
Mohsin Kazmia02ad342021-05-04 16:36:09 +02001064 virtio_interface_drop_inline (vm, vif, node->node_index,
Mohsin Kazmie2e9fe52020-12-17 15:54:03 +01001065 &buffers[n_vectors - n_left], n_left,
Mohsin Kazmi2e6f6242020-10-26 12:41:41 +01001066 VIRTIO_TX_ERROR_NO_FREE_SLOTS);
1067
Damjan Mariondc181242020-10-20 12:11:06 +02001068 clib_spinlock_unlock_if_init (&vring->lockp);
1069
Mohsin Kazmi2e6f6242020-10-26 12:41:41 +01001070 return frame->n_vectors - n_left;
Damjan Marion8389fb92017-10-13 18:29:53 +02001071}
1072
1073static void
1074virtio_set_interface_next_node (vnet_main_t * vnm, u32 hw_if_index,
1075 u32 node_index)
1076{
1077 virtio_main_t *apm = &virtio_main;
1078 vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index);
1079 virtio_if_t *vif = pool_elt_at_index (apm->interfaces, hw->dev_instance);
1080
1081 /* Shut off redirection */
1082 if (node_index == ~0)
1083 {
1084 vif->per_interface_next_index = node_index;
1085 return;
1086 }
1087
1088 vif->per_interface_next_index =
1089 vlib_node_add_next (vlib_get_main (), virtio_input_node.index,
1090 node_index);
1091}
1092
1093static void
1094virtio_clear_hw_interface_counters (u32 instance)
1095{
1096 /* Nothing for now */
1097}
1098
Damjan Marion809d5c92021-03-24 12:24:58 +01001099static void
1100virtio_set_rx_interrupt (virtio_if_t *vif, virtio_vring_t *vring)
Mohsin Kazmi24841862020-12-02 16:39:42 +01001101{
1102 if (vif->is_packed)
1103 vring->driver_event->flags &= ~VRING_EVENT_F_DISABLE;
1104 else
1105 vring->avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT;
1106}
1107
Damjan Marion809d5c92021-03-24 12:24:58 +01001108static void
1109virtio_set_rx_polling (virtio_if_t *vif, virtio_vring_t *vring)
Mohsin Kazmi24841862020-12-02 16:39:42 +01001110{
1111 if (vif->is_packed)
1112 vring->driver_event->flags |= VRING_EVENT_F_DISABLE;
1113 else
1114 vring->avail->flags |= VRING_AVAIL_F_NO_INTERRUPT;
1115}
1116
Damjan Marion8389fb92017-10-13 18:29:53 +02001117static clib_error_t *
1118virtio_interface_rx_mode_change (vnet_main_t * vnm, u32 hw_if_index, u32 qid,
Damjan Marioneabd4242020-10-07 20:59:07 +02001119 vnet_hw_if_rx_mode mode)
Damjan Marion8389fb92017-10-13 18:29:53 +02001120{
Mohsin Kazmia5203b52020-10-12 13:01:24 +02001121 vlib_main_t *vm = vnm->vlib_main;
Damjan Marion8389fb92017-10-13 18:29:53 +02001122 virtio_main_t *mm = &virtio_main;
1123 vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index);
1124 virtio_if_t *vif = pool_elt_at_index (mm->interfaces, hw->dev_instance);
Mohsin Kazmi1017a1d2020-09-25 15:36:19 +02001125 virtio_vring_t *rx_vring = vec_elt_at_index (vif->rxq_vrings, qid);
Damjan Marion8389fb92017-10-13 18:29:53 +02001126
Mohsin Kazmib74fe322019-01-31 13:50:56 +00001127 if (vif->type == VIRTIO_IF_TYPE_PCI && !(vif->support_int_mode))
1128 {
Mohsin Kazmi24841862020-12-02 16:39:42 +01001129 virtio_set_rx_polling (vif, rx_vring);
Mohsin Kazmib74fe322019-01-31 13:50:56 +00001130 return clib_error_return (0, "interrupt mode is not supported");
1131 }
1132
Damjan Marioneabd4242020-10-07 20:59:07 +02001133 if (mode == VNET_HW_IF_RX_MODE_POLLING)
Mohsin Kazmi9e2a7852020-08-13 18:57:26 +02001134 {
Mohsin Kazmia5203b52020-10-12 13:01:24 +02001135 if (vif->packet_coalesce || vif->packet_buffering)
1136 {
1137 if (mm->interrupt_queues_count > 0)
1138 mm->interrupt_queues_count--;
1139 if (mm->interrupt_queues_count == 0)
1140 vlib_process_signal_event (vm,
1141 virtio_send_interrupt_node.index,
1142 VIRTIO_EVENT_STOP_TIMER, 0);
1143 }
Mohsin Kazmi24841862020-12-02 16:39:42 +01001144 virtio_set_rx_polling (vif, rx_vring);
Mohsin Kazmi9e2a7852020-08-13 18:57:26 +02001145 }
Damjan Marion8389fb92017-10-13 18:29:53 +02001146 else
Mohsin Kazmi9e2a7852020-08-13 18:57:26 +02001147 {
Mohsin Kazmie347acb2020-09-28 10:26:33 +00001148 if (vif->packet_coalesce || vif->packet_buffering)
Mohsin Kazmi1017a1d2020-09-25 15:36:19 +02001149 {
Mohsin Kazmia5203b52020-10-12 13:01:24 +02001150 mm->interrupt_queues_count++;
1151 if (mm->interrupt_queues_count == 1)
1152 vlib_process_signal_event (vm,
1153 virtio_send_interrupt_node.index,
1154 VIRTIO_EVENT_START_TIMER, 0);
Mohsin Kazmi1017a1d2020-09-25 15:36:19 +02001155 }
Mohsin Kazmi24841862020-12-02 16:39:42 +01001156 virtio_set_rx_interrupt (vif, rx_vring);
Mohsin Kazmi9e2a7852020-08-13 18:57:26 +02001157 }
Damjan Marion8389fb92017-10-13 18:29:53 +02001158
Mohsin Kazmia5203b52020-10-12 13:01:24 +02001159 rx_vring->mode = mode;
1160
Damjan Marion8389fb92017-10-13 18:29:53 +02001161 return 0;
1162}
1163
1164static clib_error_t *
1165virtio_interface_admin_up_down (vnet_main_t * vnm, u32 hw_if_index, u32 flags)
1166{
1167 virtio_main_t *mm = &virtio_main;
1168 vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index);
1169 virtio_if_t *vif = pool_elt_at_index (mm->interfaces, hw->dev_instance);
Damjan Marion8389fb92017-10-13 18:29:53 +02001170
1171 if (flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP)
Mohsin Kazmi529f0e92020-03-19 14:03:31 +01001172 {
1173 vif->flags |= VIRTIO_IF_FLAG_ADMIN_UP;
1174 vnet_hw_interface_set_flags (vnm, vif->hw_if_index,
1175 VNET_HW_INTERFACE_FLAG_LINK_UP);
1176 }
Damjan Marion8389fb92017-10-13 18:29:53 +02001177 else
Mohsin Kazmi529f0e92020-03-19 14:03:31 +01001178 {
1179 vif->flags &= ~VIRTIO_IF_FLAG_ADMIN_UP;
1180 vnet_hw_interface_set_flags (vnm, vif->hw_if_index, 0);
1181 }
Damjan Marion8389fb92017-10-13 18:29:53 +02001182 return 0;
1183}
1184
1185static clib_error_t *
1186virtio_subif_add_del_function (vnet_main_t * vnm,
1187 u32 hw_if_index,
1188 struct vnet_sw_interface_t *st, int is_add)
1189{
1190 /* Nothing for now */
1191 return 0;
1192}
1193
1194/* *INDENT-OFF* */
1195VNET_DEVICE_CLASS (virtio_device_class) = {
1196 .name = "virtio",
Damjan Marion8389fb92017-10-13 18:29:53 +02001197 .format_device_name = format_virtio_device_name,
1198 .format_device = format_virtio_device,
1199 .format_tx_trace = format_virtio_tx_trace,
Mohsin Kazmid6c15af2018-10-23 18:00:47 +02001200 .tx_function_n_errors = VIRTIO_TX_N_ERROR,
Damjan Marion8389fb92017-10-13 18:29:53 +02001201 .tx_function_error_strings = virtio_tx_func_error_strings,
1202 .rx_redirect_to_node = virtio_set_interface_next_node,
1203 .clear_counters = virtio_clear_hw_interface_counters,
1204 .admin_up_down_function = virtio_interface_admin_up_down,
1205 .subif_add_del_function = virtio_subif_add_del_function,
1206 .rx_mode_change_function = virtio_interface_rx_mode_change,
1207};
Mohsin Kazmi206acf82020-04-06 14:19:54 +02001208
Damjan Marion8389fb92017-10-13 18:29:53 +02001209/* *INDENT-ON* */
1210
1211/*
1212 * fd.io coding-style-patch-verification: ON
1213 *
1214 * Local Variables:
1215 * eval: (c-set-style "gnu")
1216 * End:
1217 */