blob: 8eeeb0d2c74d157ea415197f9e2331790650b0ed [file] [log] [blame]
Damjan Marion8389fb92017-10-13 18:29:53 +02001/*
2 *------------------------------------------------------------------
3 * Copyright (c) 2016 Cisco and/or its affiliates.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at:
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 *------------------------------------------------------------------
16 */
17
18#include <sys/types.h>
19#include <sys/stat.h>
20#include <fcntl.h>
Damjan Marion8389fb92017-10-13 18:29:53 +020021
22#include <vlib/vlib.h>
23#include <vlib/unix/unix.h>
Mohsin Kazmi206acf82020-04-06 14:19:54 +020024#include <vnet/vnet.h>
Damjan Marion8389fb92017-10-13 18:29:53 +020025#include <vnet/ethernet/ethernet.h>
Mohsin Kazmi9e2a7852020-08-13 18:57:26 +020026#include <vnet/gso/gro_func.h>
Mohsin Kazmi0b042092020-04-17 16:50:56 +000027#include <vnet/gso/hdr_offset_parser.h>
Milan Lenco73e7f422017-12-14 10:04:25 +010028#include <vnet/ip/ip4_packet.h>
29#include <vnet/ip/ip6_packet.h>
Mohsin Kazmif5462362021-02-23 15:55:04 +010030#include <vnet/ip/ip_psh_cksum.h>
Mohsin Kazmi6d4af892020-01-03 15:11:53 +000031#include <vnet/tcp/tcp_packet.h>
32#include <vnet/udp/udp_packet.h>
Damjan Marion8389fb92017-10-13 18:29:53 +020033#include <vnet/devices/virtio/virtio.h>
34
Mohsin Kazmi637c97c2021-05-12 14:51:29 +020035#define VIRTIO_TX_MAX_CHAIN_LEN 127
36
Damjan Marion8389fb92017-10-13 18:29:53 +020037#define foreach_virtio_tx_func_error \
38_(NO_FREE_SLOTS, "no free tx slots") \
39_(TRUNC_PACKET, "packet > buffer size -- truncated in tx ring") \
40_(PENDING_MSGS, "pending msgs in tx ring") \
Mohsin Kazmif50ef402020-04-09 22:11:29 +020041_(INDIRECT_DESC_ALLOC_FAILED, "indirect descriptor allocation failed - packet drop") \
42_(OUT_OF_ORDER, "out-of-order buffers in used ring") \
43_(GSO_PACKET_DROP, "gso disabled on itf -- gso packet drop") \
44_(CSUM_OFFLOAD_PACKET_DROP, "checksum offload disabled on itf -- csum offload packet drop")
Damjan Marion8389fb92017-10-13 18:29:53 +020045
46typedef enum
47{
Mohsin Kazmid6c15af2018-10-23 18:00:47 +020048#define _(f,s) VIRTIO_TX_ERROR_##f,
Damjan Marion8389fb92017-10-13 18:29:53 +020049 foreach_virtio_tx_func_error
50#undef _
Mohsin Kazmid6c15af2018-10-23 18:00:47 +020051 VIRTIO_TX_N_ERROR,
Damjan Marion8389fb92017-10-13 18:29:53 +020052} virtio_tx_func_error_t;
53
54static char *virtio_tx_func_error_strings[] = {
55#define _(n,s) s,
56 foreach_virtio_tx_func_error
57#undef _
58};
59
Damjan Marion8389fb92017-10-13 18:29:53 +020060static u8 *
61format_virtio_device (u8 * s, va_list * args)
62{
63 u32 dev_instance = va_arg (*args, u32);
64 int verbose = va_arg (*args, int);
65 u32 indent = format_get_indent (s);
66
67 s = format (s, "VIRTIO interface");
68 if (verbose)
69 {
70 s = format (s, "\n%U instance %u", format_white_space, indent + 2,
71 dev_instance);
72 }
73 return s;
74}
75
Mohsin Kazmib5ca5592020-06-26 13:48:26 +000076typedef struct
Damjan Marion8389fb92017-10-13 18:29:53 +020077{
Mohsin Kazmib5ca5592020-06-26 13:48:26 +000078 u32 buffer_index;
79 u32 sw_if_index;
Mohsin Kazmib5ca5592020-06-26 13:48:26 +000080 generic_header_offset_t gho;
Damjan Marion8c698872020-10-10 19:17:58 +020081 vlib_buffer_t buffer;
Mohsin Kazmib5ca5592020-06-26 13:48:26 +000082} virtio_tx_trace_t;
83
84static u8 *
85format_virtio_tx_trace (u8 * s, va_list * va)
86{
87 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*va, vlib_main_t *);
88 CLIB_UNUSED (vlib_node_t * node) = va_arg (*va, vlib_node_t *);
89 virtio_tx_trace_t *t = va_arg (*va, virtio_tx_trace_t *);
90 u32 indent = format_get_indent (s);
91
BenoƮt Ganne03f2a012021-07-20 16:49:13 +020092 s = format (s, "%Ubuffer 0x%x: %U\n", format_white_space, indent,
93 t->buffer_index, format_vnet_buffer_no_chain, &t->buffer);
Mohsin Kazmib3a8f4e2020-09-15 16:41:44 +000094 s =
95 format (s, "%U%U\n", format_white_space, indent,
96 format_generic_header_offset, &t->gho);
97 s =
98 format (s, "%U%U", format_white_space, indent,
99 format_ethernet_header_with_length, t->buffer.pre_data,
100 sizeof (t->buffer.pre_data));
Damjan Marion8389fb92017-10-13 18:29:53 +0200101 return s;
102}
103
Damjan Marion809d5c92021-03-24 12:24:58 +0100104static void
105virtio_tx_trace (vlib_main_t *vm, vlib_node_runtime_t *node, vlib_buffer_t *b0,
106 u32 bi, int is_tun)
Mohsin Kazmi03b26952020-09-16 16:53:16 +0000107{
108 virtio_tx_trace_t *t;
109 t = vlib_add_trace (vm, node, b0, sizeof (t[0]));
110 t->sw_if_index = vnet_buffer (b0)->sw_if_index[VLIB_TX];
111 t->buffer_index = bi;
Damjan Mariond4a54712020-10-31 23:40:51 +0100112 if (is_tun)
Mohsin Kazmi03b26952020-09-16 16:53:16 +0000113 {
114 int is_ip4 = 0, is_ip6 = 0;
115
116 switch (((u8 *) vlib_buffer_get_current (b0))[0] & 0xf0)
117 {
118 case 0x40:
119 is_ip4 = 1;
120 break;
121 case 0x60:
122 is_ip6 = 1;
123 break;
124 default:
125 break;
126 }
127 vnet_generic_header_offset_parser (b0, &t->gho, 0, is_ip4, is_ip6);
128 }
129 else
130 vnet_generic_header_offset_parser (b0, &t->gho, 1,
131 b0->flags &
132 VNET_BUFFER_F_IS_IP4,
133 b0->flags & VNET_BUFFER_F_IS_IP6);
134
135 clib_memcpy_fast (&t->buffer, b0, sizeof (*b0) - sizeof (b0->pre_data));
136 clib_memcpy_fast (t->buffer.pre_data, vlib_buffer_get_current (b0),
137 sizeof (t->buffer.pre_data));
138}
139
Damjan Marion809d5c92021-03-24 12:24:58 +0100140static void
Mohsin Kazmia02ad342021-05-04 16:36:09 +0200141virtio_interface_drop_inline (vlib_main_t *vm, virtio_if_t *vif,
142 uword node_index, u32 *buffers, u16 n,
143 virtio_tx_func_error_t error)
Mohsin Kazmif50ef402020-04-09 22:11:29 +0200144{
145 vlib_error_count (vm, node_index, error, n);
Mohsin Kazmia02ad342021-05-04 16:36:09 +0200146 vlib_increment_simple_counter (vnet_main.interface_main.sw_if_counters +
147 VNET_INTERFACE_COUNTER_DROP,
148 vm->thread_index, vif->sw_if_index, n);
Mohsin Kazmif50ef402020-04-09 22:11:29 +0200149 vlib_buffer_free (vm, buffers, n);
150}
151
Damjan Marion809d5c92021-03-24 12:24:58 +0100152static void
153virtio_memset_ring_u32 (u32 *ring, u32 start, u32 ring_size, u32 n_buffers)
Mohsin Kazmi7f6d1452020-02-27 11:49:21 +0100154{
155 ASSERT (n_buffers <= ring_size);
156
157 if (PREDICT_TRUE (start + n_buffers <= ring_size))
158 {
159 clib_memset_u32 (ring + start, ~0, n_buffers);
160 }
161 else
162 {
163 clib_memset_u32 (ring + start, ~0, ring_size - start);
164 clib_memset_u32 (ring, ~0, n_buffers - (ring_size - start));
165 }
166}
167
Damjan Marion809d5c92021-03-24 12:24:58 +0100168static void
169virtio_free_used_device_desc_split (vlib_main_t *vm, virtio_vring_t *vring,
Mohsin Kazmib977d3f2020-11-16 16:49:30 +0100170 uword node_index)
Damjan Marion8389fb92017-10-13 18:29:53 +0200171{
172 u16 used = vring->desc_in_use;
173 u16 sz = vring->size;
174 u16 mask = sz - 1;
175 u16 last = vring->last_used_idx;
176 u16 n_left = vring->used->idx - last;
Mohsin Kazmi7f6d1452020-02-27 11:49:21 +0100177 u16 out_of_order_count = 0;
Damjan Marion8389fb92017-10-13 18:29:53 +0200178
179 if (n_left == 0)
180 return;
181
182 while (n_left)
183 {
Mohsin Kazmia7a22812020-08-31 17:17:16 +0200184 vring_used_elem_t *e = &vring->used->ring[last & mask];
Mohsin Kazmiaea0df32019-05-23 14:32:58 +0200185 u16 slot, n_buffers;
186 slot = n_buffers = e->id;
Damjan Marion8389fb92017-10-13 18:29:53 +0200187
Mohsin Kazmi7f6d1452020-02-27 11:49:21 +0100188 while (e->id == (n_buffers & mask))
Mohsin Kazmiaea0df32019-05-23 14:32:58 +0200189 {
190 n_left--;
191 last++;
192 n_buffers++;
Mohsin Kazmia7a22812020-08-31 17:17:16 +0200193 vring_desc_t *d = &vring->desc[e->id];
Mohsin Kazmice215992020-04-10 14:57:06 +0000194 u16 next;
195 while (d->flags & VRING_DESC_F_NEXT)
196 {
197 n_buffers++;
198 next = d->next;
199 d = &vring->desc[next];
200 }
Mohsin Kazmiaea0df32019-05-23 14:32:58 +0200201 if (n_left == 0)
202 break;
203 e = &vring->used->ring[last & mask];
204 }
205 vlib_buffer_free_from_ring (vm, vring->buffers, slot,
206 sz, (n_buffers - slot));
Mohsin Kazmi7f6d1452020-02-27 11:49:21 +0100207 virtio_memset_ring_u32 (vring->buffers, slot, sz, (n_buffers - slot));
Mohsin Kazmiaea0df32019-05-23 14:32:58 +0200208 used -= (n_buffers - slot);
209
210 if (n_left > 0)
211 {
Mohsin Kazmi7f6d1452020-02-27 11:49:21 +0100212 vlib_buffer_free (vm, &vring->buffers[e->id], 1);
213 vring->buffers[e->id] = ~0;
Mohsin Kazmiaea0df32019-05-23 14:32:58 +0200214 used--;
215 last++;
216 n_left--;
Mohsin Kazmi7f6d1452020-02-27 11:49:21 +0100217 out_of_order_count++;
218 vring->flags |= VRING_TX_OUT_OF_ORDER;
Mohsin Kazmiaea0df32019-05-23 14:32:58 +0200219 }
Damjan Marion8389fb92017-10-13 18:29:53 +0200220 }
Mohsin Kazmi7f6d1452020-02-27 11:49:21 +0100221
222 /*
223 * Some vhost-backends give buffers back in out-of-order fashion in used ring.
224 * It impacts the overall virtio-performance.
225 */
226 if (out_of_order_count)
227 vlib_error_count (vm, node_index, VIRTIO_TX_ERROR_OUT_OF_ORDER,
228 out_of_order_count);
229
Damjan Marion8389fb92017-10-13 18:29:53 +0200230 vring->desc_in_use = used;
231 vring->last_used_idx = last;
232}
233
Damjan Marion809d5c92021-03-24 12:24:58 +0100234static void
235virtio_free_used_device_desc_packed (vlib_main_t *vm, virtio_vring_t *vring,
Mohsin Kazmib977d3f2020-11-16 16:49:30 +0100236 uword node_index)
237{
238 vring_packed_desc_t *d;
239 u16 sz = vring->size;
240 u16 last = vring->last_used_idx;
241 u16 n_buffers = 0, start;
242 u16 flags;
243
244 if (vring->desc_in_use == 0)
245 return;
246
247 d = &vring->packed_desc[last];
248 flags = d->flags;
249 start = d->id;
250
251 while ((flags & VRING_DESC_F_AVAIL) == (vring->used_wrap_counter << 7) &&
252 (flags & VRING_DESC_F_USED) == (vring->used_wrap_counter << 15))
253 {
254 last++;
255 n_buffers++;
256
257 if (last >= sz)
258 {
259 last = 0;
260 vring->used_wrap_counter ^= 1;
261 }
262 d = &vring->packed_desc[last];
263 flags = d->flags;
264 }
265
266 if (n_buffers)
267 {
268 vlib_buffer_free_from_ring (vm, vring->buffers, start, sz, n_buffers);
269 virtio_memset_ring_u32 (vring->buffers, start, sz, n_buffers);
270 vring->desc_in_use -= n_buffers;
271 vring->last_used_idx = last;
272 }
273}
274
Damjan Marion809d5c92021-03-24 12:24:58 +0100275static void
276virtio_free_used_device_desc (vlib_main_t *vm, virtio_vring_t *vring,
Mohsin Kazmib977d3f2020-11-16 16:49:30 +0100277 uword node_index, int packed)
278{
279 if (packed)
280 virtio_free_used_device_desc_packed (vm, vring, node_index);
281 else
282 virtio_free_used_device_desc_split (vm, vring, node_index);
283
284}
285
Damjan Marion809d5c92021-03-24 12:24:58 +0100286static void
287set_checksum_offsets (vlib_buffer_t *b, virtio_net_hdr_v1_t *hdr,
Mohsin Kazmib977d3f2020-11-16 16:49:30 +0100288 const int is_l2)
Mohsin Kazmi6d4af892020-01-03 15:11:53 +0000289{
Mohsin Kazmi36f7a6a2021-05-05 14:26:38 +0200290 vnet_buffer_oflags_t oflags = vnet_buffer (b)->oflags;
Mohsin Kazmi68095382021-02-10 11:26:24 +0100291
Mohsin Kazmi6d4af892020-01-03 15:11:53 +0000292 if (b->flags & VNET_BUFFER_F_IS_IP4)
293 {
294 ip4_header_t *ip4;
Mohsin Kazmi0b042092020-04-17 16:50:56 +0000295 generic_header_offset_t gho = { 0 };
Mohsin Kazmi84f91fa2020-04-23 17:59:49 +0200296 vnet_generic_header_offset_parser (b, &gho, is_l2, 1 /* ip4 */ ,
297 0 /* ip6 */ );
Mohsin Kazmi6d4af892020-01-03 15:11:53 +0000298 hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
299 hdr->csum_start = gho.l4_hdr_offset; // 0x22;
Mohsin Kazmif5462362021-02-23 15:55:04 +0100300
301 /*
302 * virtio devices do not support IP4 checksum offload. So driver takes
303 * care of it while doing tx.
304 */
305 ip4 = (ip4_header_t *) (vlib_buffer_get_current (b) + gho.l3_hdr_offset);
306 if (oflags & VNET_BUFFER_OFFLOAD_F_IP_CKSUM)
307 ip4->checksum = ip4_header_checksum (ip4);
308
309 /*
310 * virtio devices assume the l4 header is set to the checksum of the
311 * l3 pseudo-header, so we compute it before tx-ing
312 */
Mohsin Kazmi68095382021-02-10 11:26:24 +0100313 if (oflags & VNET_BUFFER_OFFLOAD_F_TCP_CKSUM)
Mohsin Kazmi0937fdf2020-03-25 20:37:16 +0000314 {
Mohsin Kazmif5462362021-02-23 15:55:04 +0100315 tcp_header_t *tcp =
316 (tcp_header_t *) (vlib_buffer_get_current (b) + gho.l4_hdr_offset);
317 tcp->checksum = ip4_pseudo_header_cksum (ip4);
Mohsin Kazmi0937fdf2020-03-25 20:37:16 +0000318 hdr->csum_offset = STRUCT_OFFSET_OF (tcp_header_t, checksum);
319 }
Mohsin Kazmi68095382021-02-10 11:26:24 +0100320 else if (oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM)
Mohsin Kazmi0937fdf2020-03-25 20:37:16 +0000321 {
Mohsin Kazmif5462362021-02-23 15:55:04 +0100322 udp_header_t *udp =
323 (udp_header_t *) (vlib_buffer_get_current (b) + gho.l4_hdr_offset);
324 udp->checksum = ip4_pseudo_header_cksum (ip4);
Mohsin Kazmi0937fdf2020-03-25 20:37:16 +0000325 hdr->csum_offset = STRUCT_OFFSET_OF (udp_header_t, checksum);
326 }
Mohsin Kazmi6d4af892020-01-03 15:11:53 +0000327 }
328 else if (b->flags & VNET_BUFFER_F_IS_IP6)
329 {
Mohsin Kazmif5462362021-02-23 15:55:04 +0100330 ip6_header_t *ip6;
Mohsin Kazmi0b042092020-04-17 16:50:56 +0000331 generic_header_offset_t gho = { 0 };
Mohsin Kazmi84f91fa2020-04-23 17:59:49 +0200332 vnet_generic_header_offset_parser (b, &gho, is_l2, 0 /* ip4 */ ,
333 1 /* ip6 */ );
Mohsin Kazmi6d4af892020-01-03 15:11:53 +0000334 hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
335 hdr->csum_start = gho.l4_hdr_offset; // 0x36;
Mohsin Kazmif5462362021-02-23 15:55:04 +0100336 ip6 = (ip6_header_t *) (vlib_buffer_get_current (b) + gho.l3_hdr_offset);
337
338 /*
339 * virtio devices assume the l4 header is set to the checksum of the
340 * l3 pseudo-header, so we compute it before tx-ing
341 */
Mohsin Kazmi68095382021-02-10 11:26:24 +0100342 if (oflags & VNET_BUFFER_OFFLOAD_F_TCP_CKSUM)
Mohsin Kazmi0937fdf2020-03-25 20:37:16 +0000343 {
Mohsin Kazmif5462362021-02-23 15:55:04 +0100344 tcp_header_t *tcp =
345 (tcp_header_t *) (vlib_buffer_get_current (b) + gho.l4_hdr_offset);
346 tcp->checksum = ip6_pseudo_header_cksum (ip6);
Mohsin Kazmi0937fdf2020-03-25 20:37:16 +0000347 hdr->csum_offset = STRUCT_OFFSET_OF (tcp_header_t, checksum);
348 }
Mohsin Kazmi68095382021-02-10 11:26:24 +0100349 else if (oflags & VNET_BUFFER_OFFLOAD_F_UDP_CKSUM)
Mohsin Kazmi0937fdf2020-03-25 20:37:16 +0000350 {
Mohsin Kazmif5462362021-02-23 15:55:04 +0100351 udp_header_t *udp =
352 (udp_header_t *) (vlib_buffer_get_current (b) + gho.l4_hdr_offset);
353 udp->checksum = ip6_pseudo_header_cksum (ip6);
Mohsin Kazmi0937fdf2020-03-25 20:37:16 +0000354 hdr->csum_offset = STRUCT_OFFSET_OF (udp_header_t, checksum);
355 }
Mohsin Kazmi6d4af892020-01-03 15:11:53 +0000356 }
357}
358
Damjan Marion809d5c92021-03-24 12:24:58 +0100359static void
360set_gso_offsets (vlib_buffer_t *b, virtio_net_hdr_v1_t *hdr, const int is_l2)
Mohsin Kazmif50ef402020-04-09 22:11:29 +0200361{
Mohsin Kazmi36f7a6a2021-05-05 14:26:38 +0200362 vnet_buffer_oflags_t oflags = vnet_buffer (b)->oflags;
Mohsin Kazmi68095382021-02-10 11:26:24 +0100363
Mohsin Kazmif50ef402020-04-09 22:11:29 +0200364 if (b->flags & VNET_BUFFER_F_IS_IP4)
365 {
366 ip4_header_t *ip4;
Mohsin Kazmi0b042092020-04-17 16:50:56 +0000367 generic_header_offset_t gho = { 0 };
Mohsin Kazmi84f91fa2020-04-23 17:59:49 +0200368 vnet_generic_header_offset_parser (b, &gho, is_l2, 1 /* ip4 */ ,
369 0 /* ip6 */ );
Mohsin Kazmif50ef402020-04-09 22:11:29 +0200370 hdr->gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
371 hdr->gso_size = vnet_buffer2 (b)->gso_size;
Mohsin Kazmi0b042092020-04-17 16:50:56 +0000372 hdr->hdr_len = gho.hdr_sz;
Mohsin Kazmif50ef402020-04-09 22:11:29 +0200373 hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
374 hdr->csum_start = gho.l4_hdr_offset; // 0x22;
375 hdr->csum_offset = STRUCT_OFFSET_OF (tcp_header_t, checksum);
376 ip4 =
377 (ip4_header_t *) (vlib_buffer_get_current (b) + gho.l3_hdr_offset);
378 /*
379 * virtio devices do not support IP4 checksum offload. So driver takes care
380 * of it while doing tx.
381 */
Mohsin Kazmi68095382021-02-10 11:26:24 +0100382 if (oflags & VNET_BUFFER_OFFLOAD_F_IP_CKSUM)
Mohsin Kazmif50ef402020-04-09 22:11:29 +0200383 ip4->checksum = ip4_header_checksum (ip4);
384 }
385 else if (b->flags & VNET_BUFFER_F_IS_IP6)
386 {
Mohsin Kazmi0b042092020-04-17 16:50:56 +0000387 generic_header_offset_t gho = { 0 };
Mohsin Kazmi84f91fa2020-04-23 17:59:49 +0200388 vnet_generic_header_offset_parser (b, &gho, is_l2, 0 /* ip4 */ ,
389 1 /* ip6 */ );
Mohsin Kazmif50ef402020-04-09 22:11:29 +0200390 hdr->gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
391 hdr->gso_size = vnet_buffer2 (b)->gso_size;
Mohsin Kazmi0b042092020-04-17 16:50:56 +0000392 hdr->hdr_len = gho.hdr_sz;
Mohsin Kazmif50ef402020-04-09 22:11:29 +0200393 hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
394 hdr->csum_start = gho.l4_hdr_offset; // 0x36;
395 hdr->csum_offset = STRUCT_OFFSET_OF (tcp_header_t, checksum);
396 }
397}
398
Damjan Marion809d5c92021-03-24 12:24:58 +0100399static u16
400add_buffer_to_slot (vlib_main_t *vm, vlib_node_runtime_t *node,
Mohsin Kazmia02ad342021-05-04 16:36:09 +0200401 virtio_if_t *vif, virtio_vring_t *vring, u32 bi,
402 u16 free_desc_count, u16 avail, u16 next, u16 mask,
403 int hdr_sz, int do_gso, int csum_offload, int is_pci,
404 int is_tun, int is_indirect, int is_any_layout)
Damjan Marion8389fb92017-10-13 18:29:53 +0200405{
406 u16 n_added = 0;
Mohsin Kazmia7a22812020-08-31 17:17:16 +0200407 vring_desc_t *d;
Damjan Mariond4a54712020-10-31 23:40:51 +0100408 int is_l2 = !is_tun;
Damjan Marion8389fb92017-10-13 18:29:53 +0200409 d = &vring->desc[next];
410 vlib_buffer_t *b = vlib_get_buffer (vm, bi);
Mohsin Kazmia7a22812020-08-31 17:17:16 +0200411 virtio_net_hdr_v1_t *hdr = vlib_buffer_get_current (b) - hdr_sz;
Damjan Mariond4a54712020-10-31 23:40:51 +0100412 u32 drop_inline = ~0;
Damjan Marion508cabc2018-02-08 19:49:22 +0100413
Damjan Mariond4a54712020-10-31 23:40:51 +0100414 clib_memset_u8 (hdr, 0, hdr_sz);
Mohsin Kazmi6d4af892020-01-03 15:11:53 +0000415
Mohsin Kazmif50ef402020-04-09 22:11:29 +0200416 if (b->flags & VNET_BUFFER_F_GSO)
Andrew Yourtchenko6a7cff72018-10-12 16:09:22 +0200417 {
Mohsin Kazmif50ef402020-04-09 22:11:29 +0200418 if (do_gso)
Mohsin Kazmi84f91fa2020-04-23 17:59:49 +0200419 set_gso_offsets (b, hdr, is_l2);
Mohsin Kazmif50ef402020-04-09 22:11:29 +0200420 else
Andrew Yourtchenko6a7cff72018-10-12 16:09:22 +0200421 {
Damjan Mariond4a54712020-10-31 23:40:51 +0100422 drop_inline = VIRTIO_TX_ERROR_GSO_PACKET_DROP;
423 goto done;
Andrew Yourtchenko6a7cff72018-10-12 16:09:22 +0200424 }
425 }
Mohsin Kazmi68095382021-02-10 11:26:24 +0100426 else if (b->flags & VNET_BUFFER_F_OFFLOAD)
Mohsin Kazmi6d4af892020-01-03 15:11:53 +0000427 {
Mohsin Kazmif50ef402020-04-09 22:11:29 +0200428 if (csum_offload)
Mohsin Kazmi84f91fa2020-04-23 17:59:49 +0200429 set_checksum_offsets (b, hdr, is_l2);
Mohsin Kazmif50ef402020-04-09 22:11:29 +0200430 else
431 {
Damjan Mariond4a54712020-10-31 23:40:51 +0100432 drop_inline = VIRTIO_TX_ERROR_CSUM_OFFLOAD_PACKET_DROP;
433 goto done;
Mohsin Kazmif50ef402020-04-09 22:11:29 +0200434 }
Mohsin Kazmi6d4af892020-01-03 15:11:53 +0000435 }
Damjan Marion8389fb92017-10-13 18:29:53 +0200436
Mohsin Kazmi83143712020-10-07 13:23:55 +0200437 if (PREDICT_FALSE (b->flags & VLIB_BUFFER_IS_TRACED))
438 {
Damjan Mariond4a54712020-10-31 23:40:51 +0100439 virtio_tx_trace (vm, node, b, bi, is_tun);
Mohsin Kazmi83143712020-10-07 13:23:55 +0200440 }
441
Damjan Marion8389fb92017-10-13 18:29:53 +0200442 if (PREDICT_TRUE ((b->flags & VLIB_BUFFER_NEXT_PRESENT) == 0))
443 {
Damjan Mariond4a54712020-10-31 23:40:51 +0100444 d->addr = ((is_pci) ? vlib_buffer_get_current_pa (vm, b) :
445 pointer_to_uword (vlib_buffer_get_current (b))) - hdr_sz;
Damjan Marion8389fb92017-10-13 18:29:53 +0200446 d->len = b->current_length + hdr_sz;
447 d->flags = 0;
448 }
Damjan Mariond4a54712020-10-31 23:40:51 +0100449 else if (is_indirect)
Damjan Marion8389fb92017-10-13 18:29:53 +0200450 {
Mohsin Kazmid6c15af2018-10-23 18:00:47 +0200451 /*
452 * We are using single vlib_buffer_t for indirect descriptor(s)
453 * chain. Single descriptor is 16 bytes and vlib_buffer_t
454 * has 2048 bytes space. So maximum long chain can have 128
455 * (=2048/16) indirect descriptors.
456 * It can easily support 65535 bytes of Jumbo frames with
457 * each data buffer size of 512 bytes minimum.
458 */
Mohsin Kazmiaea0df32019-05-23 14:32:58 +0200459 u32 indirect_buffer = 0;
460 if (PREDICT_FALSE (vlib_buffer_alloc (vm, &indirect_buffer, 1) == 0))
Mohsin Kazmif50ef402020-04-09 22:11:29 +0200461 {
Damjan Mariond4a54712020-10-31 23:40:51 +0100462 drop_inline = VIRTIO_TX_ERROR_INDIRECT_DESC_ALLOC_FAILED;
463 goto done;
Mohsin Kazmif50ef402020-04-09 22:11:29 +0200464 }
Mohsin Kazmiaea0df32019-05-23 14:32:58 +0200465
466 vlib_buffer_t *indirect_desc = vlib_get_buffer (vm, indirect_buffer);
Mohsin Kazmid6c15af2018-10-23 18:00:47 +0200467 indirect_desc->current_data = 0;
Mohsin Kazmiaea0df32019-05-23 14:32:58 +0200468 indirect_desc->flags |= VLIB_BUFFER_NEXT_PRESENT;
469 indirect_desc->next_buffer = bi;
470 bi = indirect_buffer;
Damjan Marion8389fb92017-10-13 18:29:53 +0200471
Mohsin Kazmia7a22812020-08-31 17:17:16 +0200472 vring_desc_t *id =
473 (vring_desc_t *) vlib_buffer_get_current (indirect_desc);
Mohsin Kazmid6c15af2018-10-23 18:00:47 +0200474 u32 count = 1;
Damjan Mariond4a54712020-10-31 23:40:51 +0100475 if (is_pci)
Damjan Marion8389fb92017-10-13 18:29:53 +0200476 {
Mohsin Kazmid6c15af2018-10-23 18:00:47 +0200477 d->addr = vlib_physmem_get_pa (vm, id);
478 id->addr = vlib_buffer_get_current_pa (vm, b) - hdr_sz;
Damjan Marion8389fb92017-10-13 18:29:53 +0200479
Mohsin Kazmid6c15af2018-10-23 18:00:47 +0200480 /*
481 * If VIRTIO_F_ANY_LAYOUT is not negotiated, then virtio_net_hdr
482 * should be presented in separate descriptor and data will start
483 * from next descriptor.
484 */
Damjan Mariond4a54712020-10-31 23:40:51 +0100485 if (is_any_layout)
Mohsin Kazmid6c15af2018-10-23 18:00:47 +0200486 id->len = b->current_length + hdr_sz;
487 else
488 {
489 id->len = hdr_sz;
490 id->flags = VRING_DESC_F_NEXT;
491 id->next = count;
492 count++;
493 id++;
494 id->addr = vlib_buffer_get_current_pa (vm, b);
495 id->len = b->current_length;
496 }
497 while (b->flags & VLIB_BUFFER_NEXT_PRESENT)
498 {
499 id->flags = VRING_DESC_F_NEXT;
500 id->next = count;
501 count++;
502 id++;
503 b = vlib_get_buffer (vm, b->next_buffer);
504 id->addr = vlib_buffer_get_current_pa (vm, b);
505 id->len = b->current_length;
Mohsin Kazmi637c97c2021-05-12 14:51:29 +0200506 if (PREDICT_FALSE (count == VIRTIO_TX_MAX_CHAIN_LEN))
507 {
508 if (b->flags & VLIB_BUFFER_NEXT_PRESENT)
509 vlib_error_count (vm, node->node_index,
510 VIRTIO_TX_ERROR_TRUNC_PACKET, 1);
511 break;
512 }
Mohsin Kazmid6c15af2018-10-23 18:00:47 +0200513 }
514 }
Mohsin Kazmi206acf82020-04-06 14:19:54 +0200515 else /* VIRTIO_IF_TYPE_[TAP | TUN] */
Mohsin Kazmid6c15af2018-10-23 18:00:47 +0200516 {
517 d->addr = pointer_to_uword (id);
518 /* first buffer in chain */
519 id->addr = pointer_to_uword (vlib_buffer_get_current (b)) - hdr_sz;
520 id->len = b->current_length + hdr_sz;
521
522 while (b->flags & VLIB_BUFFER_NEXT_PRESENT)
523 {
524 id->flags = VRING_DESC_F_NEXT;
525 id->next = count;
526 count++;
527 id++;
528 b = vlib_get_buffer (vm, b->next_buffer);
529 id->addr = pointer_to_uword (vlib_buffer_get_current (b));
530 id->len = b->current_length;
Mohsin Kazmi637c97c2021-05-12 14:51:29 +0200531 if (PREDICT_FALSE (count == VIRTIO_TX_MAX_CHAIN_LEN))
532 {
533 if (b->flags & VLIB_BUFFER_NEXT_PRESENT)
534 vlib_error_count (vm, node->node_index,
535 VIRTIO_TX_ERROR_TRUNC_PACKET, 1);
536 break;
537 }
Mohsin Kazmid6c15af2018-10-23 18:00:47 +0200538 }
539 }
540 id->flags = 0;
541 id->next = 0;
Mohsin Kazmia7a22812020-08-31 17:17:16 +0200542 d->len = count * sizeof (vring_desc_t);
Damjan Marion8389fb92017-10-13 18:29:53 +0200543 d->flags = VRING_DESC_F_INDIRECT;
544 }
Damjan Mariond4a54712020-10-31 23:40:51 +0100545 else if (is_pci)
Mohsin Kazmice215992020-04-10 14:57:06 +0000546 {
547 u16 count = next;
548 vlib_buffer_t *b_temp = b;
549 u16 n_buffers_in_chain = 1;
550
551 /*
552 * Check the length of the chain for the required number of
553 * descriptors. Return from here, retry to get more descriptors,
554 * if chain length is greater than available descriptors.
555 */
556 while (b_temp->flags & VLIB_BUFFER_NEXT_PRESENT)
557 {
558 n_buffers_in_chain++;
559 b_temp = vlib_get_buffer (vm, b_temp->next_buffer);
560 }
561
562 if (n_buffers_in_chain > free_desc_count)
563 return n_buffers_in_chain;
564
565 d->addr = vlib_buffer_get_current_pa (vm, b) - hdr_sz;
566 d->len = b->current_length + hdr_sz;
567
568 while (b->flags & VLIB_BUFFER_NEXT_PRESENT)
569 {
570 d->flags = VRING_DESC_F_NEXT;
571 vring->buffers[count] = bi;
572 b->flags &=
573 ~(VLIB_BUFFER_NEXT_PRESENT | VLIB_BUFFER_TOTAL_LENGTH_VALID);
574 bi = b->next_buffer;
575 b->next_buffer = 0;
576 n_added++;
577 count = (count + 1) & mask;
578 d->next = count;
579 d = &vring->desc[count];
580 b = vlib_get_buffer (vm, bi);
581 d->addr = vlib_buffer_get_current_pa (vm, b);
582 d->len = b->current_length;
583 }
584 d->flags = 0;
585 vring->buffers[count] = bi;
586 vring->avail->ring[avail & mask] = next;
587 n_added++;
588 return n_added;
589 }
590 else
591 {
592 ASSERT (0);
593 }
Damjan Marion8389fb92017-10-13 18:29:53 +0200594 vring->buffers[next] = bi;
595 vring->avail->ring[avail & mask] = next;
596 n_added++;
Damjan Mariond4a54712020-10-31 23:40:51 +0100597
598done:
599 if (drop_inline != ~0)
Mohsin Kazmia02ad342021-05-04 16:36:09 +0200600 virtio_interface_drop_inline (vm, vif, node->node_index, &bi, 1,
601 drop_inline);
Damjan Mariond4a54712020-10-31 23:40:51 +0100602
Damjan Marion8389fb92017-10-13 18:29:53 +0200603 return n_added;
604}
605
Damjan Marion809d5c92021-03-24 12:24:58 +0100606static u16
607add_buffer_to_slot_packed (vlib_main_t *vm, vlib_node_runtime_t *node,
Mohsin Kazmia02ad342021-05-04 16:36:09 +0200608 virtio_if_t *vif, virtio_vring_t *vring, u32 bi,
609 u16 next, int hdr_sz, int do_gso, int csum_offload,
610 int is_pci, int is_tun, int is_indirect,
611 int is_any_layout)
Mohsin Kazmib977d3f2020-11-16 16:49:30 +0100612{
613 u16 n_added = 0, flags = 0;
614 int is_l2 = !is_tun;
615 vring_packed_desc_t *d = &vring->packed_desc[next];
616 vlib_buffer_t *b = vlib_get_buffer (vm, bi);
617 virtio_net_hdr_v1_t *hdr = vlib_buffer_get_current (b) - hdr_sz;
618 u32 drop_inline = ~0;
619
620 clib_memset (hdr, 0, hdr_sz);
621
622 if (b->flags & VNET_BUFFER_F_GSO)
623 {
624 if (do_gso)
625 set_gso_offsets (b, hdr, is_l2);
626 else
627 {
628 drop_inline = VIRTIO_TX_ERROR_GSO_PACKET_DROP;
629 goto done;
630 }
631 }
Mohsin Kazmi68095382021-02-10 11:26:24 +0100632 else if (b->flags & VNET_BUFFER_F_OFFLOAD)
Mohsin Kazmib977d3f2020-11-16 16:49:30 +0100633 {
634 if (csum_offload)
635 set_checksum_offsets (b, hdr, is_l2);
636 else
637 {
638 drop_inline = VIRTIO_TX_ERROR_CSUM_OFFLOAD_PACKET_DROP;
639 goto done;
640 }
641 }
642 if (PREDICT_FALSE (b->flags & VLIB_BUFFER_IS_TRACED))
643 {
644 virtio_tx_trace (vm, node, b, bi, is_tun);
645 }
646
647 if (PREDICT_TRUE ((b->flags & VLIB_BUFFER_NEXT_PRESENT) == 0))
648 {
649 d->addr =
650 ((is_pci) ? vlib_buffer_get_current_pa (vm,
651 b) :
652 pointer_to_uword (vlib_buffer_get_current (b))) - hdr_sz;
653 d->len = b->current_length + hdr_sz;
654 }
655 else if (is_indirect)
656 {
657 /*
658 * We are using single vlib_buffer_t for indirect descriptor(s)
659 * chain. Single descriptor is 16 bytes and vlib_buffer_t
660 * has 2048 bytes space. So maximum long chain can have 128
661 * (=2048/16) indirect descriptors.
662 * It can easily support 65535 bytes of Jumbo frames with
663 * each data buffer size of 512 bytes minimum.
664 */
665 u32 indirect_buffer = 0;
666 if (PREDICT_FALSE (vlib_buffer_alloc (vm, &indirect_buffer, 1) == 0))
667 {
668 drop_inline = VIRTIO_TX_ERROR_INDIRECT_DESC_ALLOC_FAILED;
669 goto done;
670 }
671
672 vlib_buffer_t *indirect_desc = vlib_get_buffer (vm, indirect_buffer);
673 indirect_desc->current_data = 0;
674 indirect_desc->flags |= VLIB_BUFFER_NEXT_PRESENT;
675 indirect_desc->next_buffer = bi;
676 bi = indirect_buffer;
677
678 vring_packed_desc_t *id =
679 (vring_packed_desc_t *) vlib_buffer_get_current (indirect_desc);
680 u32 count = 1;
681 if (is_pci)
682 {
683 d->addr = vlib_physmem_get_pa (vm, id);
684 id->addr = vlib_buffer_get_current_pa (vm, b) - hdr_sz;
685
686 /*
687 * If VIRTIO_F_ANY_LAYOUT is not negotiated, then virtio_net_hdr
688 * should be presented in separate descriptor and data will start
689 * from next descriptor.
690 */
691 if (is_any_layout)
692 id->len = b->current_length + hdr_sz;
693 else
694 {
695 id->len = hdr_sz;
696 id->flags = 0;
697 id->id = 0;
698 count++;
699 id++;
700 id->addr = vlib_buffer_get_current_pa (vm, b);
701 id->len = b->current_length;
702 }
703 while (b->flags & VLIB_BUFFER_NEXT_PRESENT)
704 {
705 id->flags = 0;
706 id->id = 0;
707 count++;
708 id++;
709 b = vlib_get_buffer (vm, b->next_buffer);
710 id->addr = vlib_buffer_get_current_pa (vm, b);
711 id->len = b->current_length;
Mohsin Kazmi637c97c2021-05-12 14:51:29 +0200712 if (PREDICT_FALSE (count == VIRTIO_TX_MAX_CHAIN_LEN))
713 {
714 if (b->flags & VLIB_BUFFER_NEXT_PRESENT)
715 vlib_error_count (vm, node->node_index,
716 VIRTIO_TX_ERROR_TRUNC_PACKET, 1);
717 break;
718 }
Mohsin Kazmib977d3f2020-11-16 16:49:30 +0100719 }
720 }
721 id->flags = 0;
722 id->id = 0;
723 d->len = count * sizeof (vring_packed_desc_t);
724 flags = VRING_DESC_F_INDIRECT;
725 }
726 else
727 {
728 ASSERT (0);
729 }
730 if (vring->avail_wrap_counter)
731 {
732 flags |= VRING_DESC_F_AVAIL;
733 flags &= ~VRING_DESC_F_USED;
734 }
735 else
736 {
737 flags &= ~VRING_DESC_F_AVAIL;
738 flags |= VRING_DESC_F_USED;
739 }
740
741 d->id = next;
742 d->flags = flags;
743 vring->buffers[next] = bi;
744 n_added++;
745
746done:
747 if (drop_inline != ~0)
Mohsin Kazmia02ad342021-05-04 16:36:09 +0200748 virtio_interface_drop_inline (vm, vif, node->node_index, &bi, 1,
749 drop_inline);
Mohsin Kazmib977d3f2020-11-16 16:49:30 +0100750
751 return n_added;
752}
753
Damjan Marion809d5c92021-03-24 12:24:58 +0100754static uword
755virtio_interface_tx_packed_gso_inline (vlib_main_t *vm,
756 vlib_node_runtime_t *node,
757 virtio_if_t *vif, virtio_if_type_t type,
758 virtio_vring_t *vring, u32 *buffers,
759 u16 n_left, const int do_gso,
Mohsin Kazmib977d3f2020-11-16 16:49:30 +0100760 const int csum_offload)
761{
762 int is_pci = (type == VIRTIO_IF_TYPE_PCI);
763 int is_tun = (type == VIRTIO_IF_TYPE_TUN);
764 int is_indirect =
765 ((vif->features & VIRTIO_FEATURE (VIRTIO_RING_F_INDIRECT_DESC)) != 0);
766 int is_any_layout =
767 ((vif->features & VIRTIO_FEATURE (VIRTIO_F_ANY_LAYOUT)) != 0);
768 const int hdr_sz = vif->virtio_net_hdr_sz;
769 u16 sz = vring->size;
770 u16 used, next, n_buffers = 0, n_buffers_left = 0;
771 u16 n_vectors = n_left;
772
773
774 used = vring->desc_in_use;
775 next = vring->desc_next;
776
777 if (vif->packet_buffering)
778 {
779 n_buffers = n_buffers_left = virtio_vring_n_buffers (vring->buffering);
780
781 while (n_buffers_left && used < sz)
782 {
783 u16 n_added = 0;
784
785 u32 bi = virtio_vring_buffering_read_from_front (vring->buffering);
786 if (bi == ~0)
787 break;
Mohsin Kazmia02ad342021-05-04 16:36:09 +0200788 n_added = add_buffer_to_slot_packed (
789 vm, node, vif, vring, bi, next, hdr_sz, do_gso, csum_offload,
790 is_pci, is_tun, is_indirect, is_any_layout);
Mohsin Kazmib977d3f2020-11-16 16:49:30 +0100791 n_buffers_left--;
792 if (PREDICT_FALSE (n_added == 0))
793 continue;
794
795 used++;
796 next++;
797 if (next >= sz)
798 {
799 next = 0;
800 vring->avail_wrap_counter ^= 1;
801 }
802 }
Mohsin Kazmib7e4e6d2021-12-13 18:32:42 +0000803 virtio_txq_clear_scheduled (vring);
Mohsin Kazmib977d3f2020-11-16 16:49:30 +0100804 }
805
806 while (n_left && used < sz)
807 {
808 u16 n_added = 0;
809
Mohsin Kazmia02ad342021-05-04 16:36:09 +0200810 n_added = add_buffer_to_slot_packed (
811 vm, node, vif, vring, buffers[0], next, hdr_sz, do_gso, csum_offload,
812 is_pci, is_tun, is_indirect, is_any_layout);
Mohsin Kazmib977d3f2020-11-16 16:49:30 +0100813 buffers++;
814 n_left--;
815 if (PREDICT_FALSE (n_added == 0))
816 continue;
817
818 used++;
819 next++;
820 if (next >= sz)
821 {
822 next = 0;
823 vring->avail_wrap_counter ^= 1;
824 }
825 }
826
827 if (n_left != n_vectors || n_buffers != n_buffers_left)
828 {
829 CLIB_MEMORY_STORE_BARRIER ();
830 vring->desc_next = next;
831 vring->desc_in_use = used;
832 CLIB_MEMORY_BARRIER ();
833 if (vring->device_event->flags != VRING_EVENT_F_DISABLE)
834 virtio_kick (vm, vring, vif);
835 }
836
837 return n_left;
838}
839
Damjan Marion809d5c92021-03-24 12:24:58 +0100840static void
841virtio_find_free_desc (virtio_vring_t *vring, u16 size, u16 mask, u16 req,
842 u16 next, u32 *first_free_desc_index,
843 u16 *free_desc_count)
Mohsin Kazmi7f6d1452020-02-27 11:49:21 +0100844{
845 u16 start = 0;
846 /* next is used as hint: from where to start looking */
847 for (u16 i = 0; i < size; i++, next++)
848 {
849 if (vring->buffers[next & mask] == ~0)
850 {
851 if (*first_free_desc_index == ~0)
852 {
853 *first_free_desc_index = (next & mask);
854 start = i;
855 (*free_desc_count)++;
856 req--;
857 if (req == 0)
858 break;
859 }
860 else
861 {
862 if (start + *free_desc_count == i)
863 {
864 (*free_desc_count)++;
865 req--;
866 if (req == 0)
867 break;
868 }
869 else
870 break;
871 }
872 }
873 }
874}
875
Damjan Marion809d5c92021-03-24 12:24:58 +0100876static u16
877virtio_interface_tx_split_gso_inline (vlib_main_t *vm,
878 vlib_node_runtime_t *node,
879 virtio_if_t *vif, virtio_if_type_t type,
880 virtio_vring_t *vring, u32 *buffers,
881 u16 n_left, int do_gso, int csum_offload)
Damjan Marion8389fb92017-10-13 18:29:53 +0200882{
Mohsin Kazmie347acb2020-09-28 10:26:33 +0000883 u16 used, next, avail, n_buffers = 0, n_buffers_left = 0;
Damjan Mariond4a54712020-10-31 23:40:51 +0100884 int is_pci = (type == VIRTIO_IF_TYPE_PCI);
885 int is_tun = (type == VIRTIO_IF_TYPE_TUN);
886 int is_indirect =
887 ((vif->features & VIRTIO_FEATURE (VIRTIO_RING_F_INDIRECT_DESC)) != 0);
888 int is_any_layout =
889 ((vif->features & VIRTIO_FEATURE (VIRTIO_F_ANY_LAYOUT)) != 0);
Damjan Marion8389fb92017-10-13 18:29:53 +0200890 u16 sz = vring->size;
Damjan Mariond4a54712020-10-31 23:40:51 +0100891 int hdr_sz = vif->virtio_net_hdr_sz;
Damjan Marion8389fb92017-10-13 18:29:53 +0200892 u16 mask = sz - 1;
Mohsin Kazmi2e6f6242020-10-26 12:41:41 +0100893 u16 n_vectors = n_left;
Damjan Marion8389fb92017-10-13 18:29:53 +0200894
895 used = vring->desc_in_use;
896 next = vring->desc_next;
897 avail = vring->avail->idx;
898
Mohsin Kazmi7f6d1452020-02-27 11:49:21 +0100899 u16 free_desc_count = 0;
900
901 if (PREDICT_FALSE (vring->flags & VRING_TX_OUT_OF_ORDER))
902 {
903 u32 first_free_desc_index = ~0;
904
905 virtio_find_free_desc (vring, sz, mask, n_left, next,
906 &first_free_desc_index, &free_desc_count);
907
908 if (free_desc_count)
909 next = first_free_desc_index;
910 }
911 else
912 free_desc_count = sz - used;
913
Mohsin Kazmie347acb2020-09-28 10:26:33 +0000914 if (vif->packet_buffering)
915 {
916 n_buffers = n_buffers_left = virtio_vring_n_buffers (vring->buffering);
917
918 while (n_buffers_left && free_desc_count)
919 {
920 u16 n_added = 0;
921
922 u32 bi = virtio_vring_buffering_read_from_front (vring->buffering);
923 if (bi == ~0)
924 break;
Mohsin Kazmi83143712020-10-07 13:23:55 +0200925
Mohsin Kazmia02ad342021-05-04 16:36:09 +0200926 n_added = add_buffer_to_slot (vm, node, vif, vring, bi,
927 free_desc_count, avail, next, mask,
928 hdr_sz, do_gso, csum_offload, is_pci,
929 is_tun, is_indirect, is_any_layout);
Mohsin Kazmie347acb2020-09-28 10:26:33 +0000930 if (PREDICT_FALSE (n_added == 0))
931 {
932 n_buffers_left--;
933 continue;
934 }
935 else if (PREDICT_FALSE (n_added > free_desc_count))
936 break;
937
938 avail++;
939 next = (next + n_added) & mask;
940 used += n_added;
941 n_buffers_left--;
942 free_desc_count -= n_added;
943 }
Mohsin Kazmib7e4e6d2021-12-13 18:32:42 +0000944 virtio_txq_clear_scheduled (vring);
Mohsin Kazmie347acb2020-09-28 10:26:33 +0000945 }
946
Mohsin Kazmi7f6d1452020-02-27 11:49:21 +0100947 while (n_left && free_desc_count)
Damjan Marion8389fb92017-10-13 18:29:53 +0200948 {
Mohsin Kazmid6c15af2018-10-23 18:00:47 +0200949 u16 n_added = 0;
Mohsin Kazmib5ca5592020-06-26 13:48:26 +0000950
Mohsin Kazmia02ad342021-05-04 16:36:09 +0200951 n_added =
952 add_buffer_to_slot (vm, node, vif, vring, buffers[0], free_desc_count,
953 avail, next, mask, hdr_sz, do_gso, csum_offload,
954 is_pci, is_tun, is_indirect, is_any_layout);
Mohsin Kazmif50ef402020-04-09 22:11:29 +0200955
956 if (PREDICT_FALSE (n_added == 0))
957 {
958 buffers++;
959 n_left--;
960 continue;
961 }
Mohsin Kazmice215992020-04-10 14:57:06 +0000962 else if (PREDICT_FALSE (n_added > free_desc_count))
963 break;
Mohsin Kazmif50ef402020-04-09 22:11:29 +0200964
Mohsin Kazmice215992020-04-10 14:57:06 +0000965 avail++;
Damjan Marion8389fb92017-10-13 18:29:53 +0200966 next = (next + n_added) & mask;
967 used += n_added;
968 buffers++;
969 n_left--;
Mohsin Kazmice215992020-04-10 14:57:06 +0000970 free_desc_count -= n_added;
Damjan Marion8389fb92017-10-13 18:29:53 +0200971 }
972
Mohsin Kazmi2e6f6242020-10-26 12:41:41 +0100973 if (n_left != n_vectors || n_buffers != n_buffers_left)
Damjan Marion8389fb92017-10-13 18:29:53 +0200974 {
Mohsin Kazmid8b7dec2020-12-10 09:49:21 +0000975 clib_atomic_store_seq_cst (&vring->avail->idx, avail);
Damjan Marion8389fb92017-10-13 18:29:53 +0200976 vring->desc_next = next;
977 vring->desc_in_use = used;
Mohsin Kazmid8b7dec2020-12-10 09:49:21 +0000978 if ((clib_atomic_load_seq_cst (&vring->used->flags) &
979 VRING_USED_F_NO_NOTIFY) == 0)
Mohsin Kazmid6c15af2018-10-23 18:00:47 +0200980 virtio_kick (vm, vring, vif);
Damjan Marion8389fb92017-10-13 18:29:53 +0200981 }
982
Mohsin Kazmi2e6f6242020-10-26 12:41:41 +0100983 return n_left;
Damjan Marion8389fb92017-10-13 18:29:53 +0200984}
985
Damjan Marion809d5c92021-03-24 12:24:58 +0100986static u16
987virtio_interface_tx_gso_inline (vlib_main_t *vm, vlib_node_runtime_t *node,
988 virtio_if_t *vif, virtio_if_type_t type,
989 virtio_vring_t *vring, u32 *buffers,
990 u16 n_left, int packed, int do_gso,
991 int csum_offload)
Mohsin Kazmib977d3f2020-11-16 16:49:30 +0100992{
993 if (packed)
994 return virtio_interface_tx_packed_gso_inline (vm, node, vif, type, vring,
995 buffers, n_left,
Mohsin Kazmib18b1902020-12-03 16:35:20 +0100996 do_gso, csum_offload);
Mohsin Kazmib977d3f2020-11-16 16:49:30 +0100997 else
998 return virtio_interface_tx_split_gso_inline (vm, node, vif, type, vring,
999 buffers, n_left,
Mohsin Kazmib18b1902020-12-03 16:35:20 +01001000 do_gso, csum_offload);
Mohsin Kazmib977d3f2020-11-16 16:49:30 +01001001}
1002
Damjan Marion809d5c92021-03-24 12:24:58 +01001003static u16
1004virtio_interface_tx_inline (vlib_main_t *vm, vlib_node_runtime_t *node,
1005 virtio_if_t *vif, virtio_vring_t *vring,
1006 virtio_if_type_t type, u32 *buffers, u16 n_left,
1007 int packed)
Mohsin Kazmi38b09682020-06-03 18:20:17 +02001008{
1009 vnet_main_t *vnm = vnet_get_main ();
1010 vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, vif->hw_if_index);
1011
Mohsin Kazmi5b3f5232021-02-10 12:03:53 +01001012 if (hw->caps & VNET_HW_INTERFACE_CAP_SUPPORTS_TCP_GSO)
Mohsin Kazmi2e6f6242020-10-26 12:41:41 +01001013 return virtio_interface_tx_gso_inline (vm, node, vif, type, vring,
Mohsin Kazmib977d3f2020-11-16 16:49:30 +01001014 buffers, n_left, packed,
1015 1 /* do_gso */ ,
Damjan Mariondc181242020-10-20 12:11:06 +02001016 1 /* checksum offload */ );
Mohsin Kazmi5b3f5232021-02-10 12:03:53 +01001017 else if (hw->caps & VNET_HW_INTERFACE_CAP_SUPPORTS_L4_TX_CKSUM)
Mohsin Kazmi2e6f6242020-10-26 12:41:41 +01001018 return virtio_interface_tx_gso_inline (vm, node, vif, type, vring,
Mohsin Kazmib977d3f2020-11-16 16:49:30 +01001019 buffers, n_left, packed,
Mohsin Kazmi38b09682020-06-03 18:20:17 +02001020 0 /* no do_gso */ ,
Damjan Mariondc181242020-10-20 12:11:06 +02001021 1 /* checksum offload */ );
Mohsin Kazmi38b09682020-06-03 18:20:17 +02001022 else
Mohsin Kazmi2e6f6242020-10-26 12:41:41 +01001023 return virtio_interface_tx_gso_inline (vm, node, vif, type, vring,
Mohsin Kazmib977d3f2020-11-16 16:49:30 +01001024 buffers, n_left, packed,
Mohsin Kazmi38b09682020-06-03 18:20:17 +02001025 0 /* no do_gso */ ,
Damjan Mariondc181242020-10-20 12:11:06 +02001026 0 /* no checksum offload */ );
Mohsin Kazmi38b09682020-06-03 18:20:17 +02001027}
1028
Filip Tehlaraee73642019-03-13 05:50:44 -07001029VNET_DEVICE_CLASS_TX_FN (virtio_device_class) (vlib_main_t * vm,
1030 vlib_node_runtime_t * node,
1031 vlib_frame_t * frame)
Damjan Marion8389fb92017-10-13 18:29:53 +02001032{
1033 virtio_main_t *nm = &virtio_main;
1034 vnet_interface_output_runtime_t *rund = (void *) node->runtime_data;
1035 virtio_if_t *vif = pool_elt_at_index (nm->interfaces, rund->dev_instance);
Mohsin Kazmib7e4e6d2021-12-13 18:32:42 +00001036 vnet_hw_if_tx_frame_t *tf = vlib_frame_scalar_args (frame);
1037 u16 qid = tf->queue_id;
Damjan Mariondc181242020-10-20 12:11:06 +02001038 virtio_vring_t *vring = vec_elt_at_index (vif->txq_vrings, qid);
Mohsin Kazmiee1e30f2020-10-23 11:23:52 +02001039 u16 n_left = frame->n_vectors;
1040 u32 *buffers = vlib_frame_vector_args (frame);
1041 u32 to[GRO_TO_VECTOR_SIZE (n_left)];
Mohsin Kazmib977d3f2020-11-16 16:49:30 +01001042 int packed = vif->is_packed;
Mohsin Kazmie2e9fe52020-12-17 15:54:03 +01001043 u16 n_vectors = frame->n_vectors;
Damjan Mariondc181242020-10-20 12:11:06 +02001044
Mohsin Kazmib7e4e6d2021-12-13 18:32:42 +00001045 if (tf->shared_queue)
1046 clib_spinlock_lock (&vring->lockp);
Damjan Mariondc181242020-10-20 12:11:06 +02001047
Mohsin Kazmiee1e30f2020-10-23 11:23:52 +02001048 if (vif->packet_coalesce)
1049 {
Mohsin Kazmie2e9fe52020-12-17 15:54:03 +01001050 n_vectors = n_left =
1051 vnet_gro_inline (vm, vring->flow_table, buffers, n_left, to);
Mohsin Kazmiee1e30f2020-10-23 11:23:52 +02001052 buffers = to;
Mohsin Kazmib7e4e6d2021-12-13 18:32:42 +00001053 virtio_txq_clear_scheduled (vring);
Mohsin Kazmiee1e30f2020-10-23 11:23:52 +02001054 }
1055
Damjan Marion587f9132020-10-31 22:47:01 +01001056 u16 retry_count = 2;
1057
1058retry:
1059 /* free consumed buffers */
Mohsin Kazmib977d3f2020-11-16 16:49:30 +01001060 virtio_free_used_device_desc (vm, vring, node->node_index, packed);
Damjan Marion587f9132020-10-31 22:47:01 +01001061
Mohsin Kazmi38b09682020-06-03 18:20:17 +02001062 if (vif->type == VIRTIO_IF_TYPE_TAP)
Mohsin Kazmi2e6f6242020-10-26 12:41:41 +01001063 n_left = virtio_interface_tx_inline (vm, node, vif, vring,
Mohsin Kazmi81dac062020-11-11 20:28:35 +01001064 VIRTIO_IF_TYPE_TAP,
Mohsin Kazmie2e9fe52020-12-17 15:54:03 +01001065 &buffers[n_vectors - n_left],
Mohsin Kazmib977d3f2020-11-16 16:49:30 +01001066 n_left, packed);
Mohsin Kazmi38b09682020-06-03 18:20:17 +02001067 else if (vif->type == VIRTIO_IF_TYPE_PCI)
Mohsin Kazmi2e6f6242020-10-26 12:41:41 +01001068 n_left = virtio_interface_tx_inline (vm, node, vif, vring,
Mohsin Kazmi81dac062020-11-11 20:28:35 +01001069 VIRTIO_IF_TYPE_PCI,
Mohsin Kazmie2e9fe52020-12-17 15:54:03 +01001070 &buffers[n_vectors - n_left],
Mohsin Kazmib977d3f2020-11-16 16:49:30 +01001071 n_left, packed);
Mohsin Kazmi38b09682020-06-03 18:20:17 +02001072 else if (vif->type == VIRTIO_IF_TYPE_TUN)
Mohsin Kazmi2e6f6242020-10-26 12:41:41 +01001073 n_left = virtio_interface_tx_inline (vm, node, vif, vring,
Mohsin Kazmi81dac062020-11-11 20:28:35 +01001074 VIRTIO_IF_TYPE_TUN,
Mohsin Kazmie2e9fe52020-12-17 15:54:03 +01001075 &buffers[n_vectors - n_left],
Mohsin Kazmib977d3f2020-11-16 16:49:30 +01001076 n_left, packed);
Andrew Yourtchenko6a7cff72018-10-12 16:09:22 +02001077 else
Mohsin Kazmi38b09682020-06-03 18:20:17 +02001078 ASSERT (0);
1079
Damjan Marion587f9132020-10-31 22:47:01 +01001080 if (n_left && retry_count--)
1081 goto retry;
1082
Mohsin Kazmi2e6f6242020-10-26 12:41:41 +01001083 if (vif->packet_buffering && n_left)
1084 {
Mohsin Kazmi81dac062020-11-11 20:28:35 +01001085 u16 n_buffered = virtio_vring_buffering_store_packets (vring->buffering,
1086 &buffers
Mohsin Kazmie2e9fe52020-12-17 15:54:03 +01001087 [n_vectors
Mohsin Kazmi81dac062020-11-11 20:28:35 +01001088 - n_left],
1089 n_left);
Mohsin Kazmi2e6f6242020-10-26 12:41:41 +01001090 n_left -= n_buffered;
1091 }
1092 if (n_left)
Mohsin Kazmia02ad342021-05-04 16:36:09 +02001093 virtio_interface_drop_inline (vm, vif, node->node_index,
Mohsin Kazmie2e9fe52020-12-17 15:54:03 +01001094 &buffers[n_vectors - n_left], n_left,
Mohsin Kazmi2e6f6242020-10-26 12:41:41 +01001095 VIRTIO_TX_ERROR_NO_FREE_SLOTS);
1096
Mohsin Kazmib7e4e6d2021-12-13 18:32:42 +00001097 if (tf->shared_queue)
1098 clib_spinlock_unlock (&vring->lockp);
Damjan Mariondc181242020-10-20 12:11:06 +02001099
Mohsin Kazmi2e6f6242020-10-26 12:41:41 +01001100 return frame->n_vectors - n_left;
Damjan Marion8389fb92017-10-13 18:29:53 +02001101}
1102
1103static void
1104virtio_set_interface_next_node (vnet_main_t * vnm, u32 hw_if_index,
1105 u32 node_index)
1106{
1107 virtio_main_t *apm = &virtio_main;
1108 vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index);
1109 virtio_if_t *vif = pool_elt_at_index (apm->interfaces, hw->dev_instance);
1110
1111 /* Shut off redirection */
1112 if (node_index == ~0)
1113 {
1114 vif->per_interface_next_index = node_index;
1115 return;
1116 }
1117
1118 vif->per_interface_next_index =
1119 vlib_node_add_next (vlib_get_main (), virtio_input_node.index,
1120 node_index);
1121}
1122
1123static void
1124virtio_clear_hw_interface_counters (u32 instance)
1125{
1126 /* Nothing for now */
1127}
1128
Damjan Marion809d5c92021-03-24 12:24:58 +01001129static void
1130virtio_set_rx_interrupt (virtio_if_t *vif, virtio_vring_t *vring)
Mohsin Kazmi24841862020-12-02 16:39:42 +01001131{
1132 if (vif->is_packed)
1133 vring->driver_event->flags &= ~VRING_EVENT_F_DISABLE;
1134 else
1135 vring->avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT;
1136}
1137
Damjan Marion809d5c92021-03-24 12:24:58 +01001138static void
1139virtio_set_rx_polling (virtio_if_t *vif, virtio_vring_t *vring)
Mohsin Kazmi24841862020-12-02 16:39:42 +01001140{
1141 if (vif->is_packed)
1142 vring->driver_event->flags |= VRING_EVENT_F_DISABLE;
1143 else
1144 vring->avail->flags |= VRING_AVAIL_F_NO_INTERRUPT;
1145}
1146
Damjan Marion8389fb92017-10-13 18:29:53 +02001147static clib_error_t *
1148virtio_interface_rx_mode_change (vnet_main_t * vnm, u32 hw_if_index, u32 qid,
Damjan Marioneabd4242020-10-07 20:59:07 +02001149 vnet_hw_if_rx_mode mode)
Damjan Marion8389fb92017-10-13 18:29:53 +02001150{
1151 virtio_main_t *mm = &virtio_main;
1152 vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index);
1153 virtio_if_t *vif = pool_elt_at_index (mm->interfaces, hw->dev_instance);
Mohsin Kazmi1017a1d2020-09-25 15:36:19 +02001154 virtio_vring_t *rx_vring = vec_elt_at_index (vif->rxq_vrings, qid);
Damjan Marion8389fb92017-10-13 18:29:53 +02001155
Mohsin Kazmib74fe322019-01-31 13:50:56 +00001156 if (vif->type == VIRTIO_IF_TYPE_PCI && !(vif->support_int_mode))
1157 {
Mohsin Kazmi24841862020-12-02 16:39:42 +01001158 virtio_set_rx_polling (vif, rx_vring);
Mohsin Kazmib74fe322019-01-31 13:50:56 +00001159 return clib_error_return (0, "interrupt mode is not supported");
1160 }
1161
Damjan Marioneabd4242020-10-07 20:59:07 +02001162 if (mode == VNET_HW_IF_RX_MODE_POLLING)
Mohsin Kazmi24841862020-12-02 16:39:42 +01001163 virtio_set_rx_polling (vif, rx_vring);
Damjan Marion8389fb92017-10-13 18:29:53 +02001164 else
Mohsin Kazmi24841862020-12-02 16:39:42 +01001165 virtio_set_rx_interrupt (vif, rx_vring);
Damjan Marion8389fb92017-10-13 18:29:53 +02001166
Mohsin Kazmia5203b52020-10-12 13:01:24 +02001167 rx_vring->mode = mode;
1168
Damjan Marion8389fb92017-10-13 18:29:53 +02001169 return 0;
1170}
1171
1172static clib_error_t *
1173virtio_interface_admin_up_down (vnet_main_t * vnm, u32 hw_if_index, u32 flags)
1174{
1175 virtio_main_t *mm = &virtio_main;
1176 vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index);
1177 virtio_if_t *vif = pool_elt_at_index (mm->interfaces, hw->dev_instance);
Damjan Marion8389fb92017-10-13 18:29:53 +02001178
1179 if (flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP)
Mohsin Kazmi529f0e92020-03-19 14:03:31 +01001180 {
1181 vif->flags |= VIRTIO_IF_FLAG_ADMIN_UP;
1182 vnet_hw_interface_set_flags (vnm, vif->hw_if_index,
1183 VNET_HW_INTERFACE_FLAG_LINK_UP);
1184 }
Damjan Marion8389fb92017-10-13 18:29:53 +02001185 else
Mohsin Kazmi529f0e92020-03-19 14:03:31 +01001186 {
1187 vif->flags &= ~VIRTIO_IF_FLAG_ADMIN_UP;
1188 vnet_hw_interface_set_flags (vnm, vif->hw_if_index, 0);
1189 }
Damjan Marion8389fb92017-10-13 18:29:53 +02001190 return 0;
1191}
1192
1193static clib_error_t *
1194virtio_subif_add_del_function (vnet_main_t * vnm,
1195 u32 hw_if_index,
1196 struct vnet_sw_interface_t *st, int is_add)
1197{
1198 /* Nothing for now */
1199 return 0;
1200}
1201
1202/* *INDENT-OFF* */
1203VNET_DEVICE_CLASS (virtio_device_class) = {
1204 .name = "virtio",
Damjan Marion8389fb92017-10-13 18:29:53 +02001205 .format_device_name = format_virtio_device_name,
1206 .format_device = format_virtio_device,
1207 .format_tx_trace = format_virtio_tx_trace,
Mohsin Kazmid6c15af2018-10-23 18:00:47 +02001208 .tx_function_n_errors = VIRTIO_TX_N_ERROR,
Damjan Marion8389fb92017-10-13 18:29:53 +02001209 .tx_function_error_strings = virtio_tx_func_error_strings,
1210 .rx_redirect_to_node = virtio_set_interface_next_node,
1211 .clear_counters = virtio_clear_hw_interface_counters,
1212 .admin_up_down_function = virtio_interface_admin_up_down,
1213 .subif_add_del_function = virtio_subif_add_del_function,
1214 .rx_mode_change_function = virtio_interface_rx_mode_change,
1215};
Mohsin Kazmi206acf82020-04-06 14:19:54 +02001216
Damjan Marion8389fb92017-10-13 18:29:53 +02001217/* *INDENT-ON* */
1218
1219/*
1220 * fd.io coding-style-patch-verification: ON
1221 *
1222 * Local Variables:
1223 * eval: (c-set-style "gnu")
1224 * End:
1225 */