blob: 32c1eab6b34d114cff3bba7bd514d08acaf2e818 [file] [log] [blame]
Damjan Marion8389fb92017-10-13 18:29:53 +02001/*
2 *------------------------------------------------------------------
3 * Copyright (c) 2016 Cisco and/or its affiliates.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at:
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 *------------------------------------------------------------------
16 */
17
18#include <sys/types.h>
19#include <sys/stat.h>
20#include <fcntl.h>
Damjan Marion8389fb92017-10-13 18:29:53 +020021
22#include <vlib/vlib.h>
23#include <vlib/unix/unix.h>
Mohsin Kazmi206acf82020-04-06 14:19:54 +020024#include <vnet/vnet.h>
Damjan Marion8389fb92017-10-13 18:29:53 +020025#include <vnet/ethernet/ethernet.h>
Mohsin Kazmi9e2a7852020-08-13 18:57:26 +020026#include <vnet/gso/gro_func.h>
Mohsin Kazmi0b042092020-04-17 16:50:56 +000027#include <vnet/gso/hdr_offset_parser.h>
Milan Lenco73e7f422017-12-14 10:04:25 +010028#include <vnet/ip/ip4_packet.h>
29#include <vnet/ip/ip6_packet.h>
Mohsin Kazmi6d4af892020-01-03 15:11:53 +000030#include <vnet/tcp/tcp_packet.h>
31#include <vnet/udp/udp_packet.h>
Damjan Marion8389fb92017-10-13 18:29:53 +020032#include <vnet/devices/virtio/virtio.h>
33
34#define foreach_virtio_tx_func_error \
35_(NO_FREE_SLOTS, "no free tx slots") \
36_(TRUNC_PACKET, "packet > buffer size -- truncated in tx ring") \
37_(PENDING_MSGS, "pending msgs in tx ring") \
Mohsin Kazmif50ef402020-04-09 22:11:29 +020038_(INDIRECT_DESC_ALLOC_FAILED, "indirect descriptor allocation failed - packet drop") \
39_(OUT_OF_ORDER, "out-of-order buffers in used ring") \
40_(GSO_PACKET_DROP, "gso disabled on itf -- gso packet drop") \
41_(CSUM_OFFLOAD_PACKET_DROP, "checksum offload disabled on itf -- csum offload packet drop")
Damjan Marion8389fb92017-10-13 18:29:53 +020042
43typedef enum
44{
Mohsin Kazmid6c15af2018-10-23 18:00:47 +020045#define _(f,s) VIRTIO_TX_ERROR_##f,
Damjan Marion8389fb92017-10-13 18:29:53 +020046 foreach_virtio_tx_func_error
47#undef _
Mohsin Kazmid6c15af2018-10-23 18:00:47 +020048 VIRTIO_TX_N_ERROR,
Damjan Marion8389fb92017-10-13 18:29:53 +020049} virtio_tx_func_error_t;
50
51static char *virtio_tx_func_error_strings[] = {
52#define _(n,s) s,
53 foreach_virtio_tx_func_error
54#undef _
55};
56
Damjan Marion8389fb92017-10-13 18:29:53 +020057static u8 *
58format_virtio_device (u8 * s, va_list * args)
59{
60 u32 dev_instance = va_arg (*args, u32);
61 int verbose = va_arg (*args, int);
62 u32 indent = format_get_indent (s);
63
64 s = format (s, "VIRTIO interface");
65 if (verbose)
66 {
67 s = format (s, "\n%U instance %u", format_white_space, indent + 2,
68 dev_instance);
69 }
70 return s;
71}
72
Mohsin Kazmib5ca5592020-06-26 13:48:26 +000073typedef struct
Damjan Marion8389fb92017-10-13 18:29:53 +020074{
Mohsin Kazmib5ca5592020-06-26 13:48:26 +000075 u32 buffer_index;
76 u32 sw_if_index;
77 vlib_buffer_t buffer;
78 generic_header_offset_t gho;
79} virtio_tx_trace_t;
80
81static u8 *
82format_virtio_tx_trace (u8 * s, va_list * va)
83{
84 CLIB_UNUSED (vlib_main_t * vm) = va_arg (*va, vlib_main_t *);
85 CLIB_UNUSED (vlib_node_t * node) = va_arg (*va, vlib_node_t *);
86 virtio_tx_trace_t *t = va_arg (*va, virtio_tx_trace_t *);
87 u32 indent = format_get_indent (s);
88
Mohsin Kazmib3a8f4e2020-09-15 16:41:44 +000089 s = format (s, "%Ubuffer 0x%x: %U\n",
Mohsin Kazmib5ca5592020-06-26 13:48:26 +000090 format_white_space, indent,
91 t->buffer_index, format_vnet_buffer, &t->buffer);
Mohsin Kazmib3a8f4e2020-09-15 16:41:44 +000092 s =
93 format (s, "%U%U\n", format_white_space, indent,
94 format_generic_header_offset, &t->gho);
95 s =
96 format (s, "%U%U", format_white_space, indent,
97 format_ethernet_header_with_length, t->buffer.pre_data,
98 sizeof (t->buffer.pre_data));
Damjan Marion8389fb92017-10-13 18:29:53 +020099 return s;
100}
101
Mohsin Kazmiaea0df32019-05-23 14:32:58 +0200102static_always_inline void
Mohsin Kazmi03b26952020-09-16 16:53:16 +0000103virtio_tx_trace (vlib_main_t * vm, vlib_node_runtime_t * node,
104 virtio_if_type_t type, vlib_buffer_t * b0, u32 bi)
105{
106 virtio_tx_trace_t *t;
107 t = vlib_add_trace (vm, node, b0, sizeof (t[0]));
108 t->sw_if_index = vnet_buffer (b0)->sw_if_index[VLIB_TX];
109 t->buffer_index = bi;
110 if (type == VIRTIO_IF_TYPE_TUN)
111 {
112 int is_ip4 = 0, is_ip6 = 0;
113
114 switch (((u8 *) vlib_buffer_get_current (b0))[0] & 0xf0)
115 {
116 case 0x40:
117 is_ip4 = 1;
118 break;
119 case 0x60:
120 is_ip6 = 1;
121 break;
122 default:
123 break;
124 }
125 vnet_generic_header_offset_parser (b0, &t->gho, 0, is_ip4, is_ip6);
126 }
127 else
128 vnet_generic_header_offset_parser (b0, &t->gho, 1,
129 b0->flags &
130 VNET_BUFFER_F_IS_IP4,
131 b0->flags & VNET_BUFFER_F_IS_IP6);
132
133 clib_memcpy_fast (&t->buffer, b0, sizeof (*b0) - sizeof (b0->pre_data));
134 clib_memcpy_fast (t->buffer.pre_data, vlib_buffer_get_current (b0),
135 sizeof (t->buffer.pre_data));
136}
137
138static_always_inline void
Mohsin Kazmif50ef402020-04-09 22:11:29 +0200139virtio_interface_drop_inline (vlib_main_t * vm, uword node_index,
140 u32 * buffers, u16 n,
141 virtio_tx_func_error_t error)
142{
143 vlib_error_count (vm, node_index, error, n);
144 vlib_buffer_free (vm, buffers, n);
145}
146
147static_always_inline void
Mohsin Kazmi7f6d1452020-02-27 11:49:21 +0100148virtio_memset_ring_u32 (u32 * ring, u32 start, u32 ring_size, u32 n_buffers)
149{
150 ASSERT (n_buffers <= ring_size);
151
152 if (PREDICT_TRUE (start + n_buffers <= ring_size))
153 {
154 clib_memset_u32 (ring + start, ~0, n_buffers);
155 }
156 else
157 {
158 clib_memset_u32 (ring + start, ~0, ring_size - start);
159 clib_memset_u32 (ring, ~0, n_buffers - (ring_size - start));
160 }
161}
162
163static_always_inline void
164virtio_free_used_device_desc (vlib_main_t * vm, virtio_vring_t * vring,
165 uword node_index)
Damjan Marion8389fb92017-10-13 18:29:53 +0200166{
167 u16 used = vring->desc_in_use;
168 u16 sz = vring->size;
169 u16 mask = sz - 1;
170 u16 last = vring->last_used_idx;
171 u16 n_left = vring->used->idx - last;
Mohsin Kazmi7f6d1452020-02-27 11:49:21 +0100172 u16 out_of_order_count = 0;
Damjan Marion8389fb92017-10-13 18:29:53 +0200173
174 if (n_left == 0)
175 return;
176
177 while (n_left)
178 {
Mohsin Kazmia7a22812020-08-31 17:17:16 +0200179 vring_used_elem_t *e = &vring->used->ring[last & mask];
Mohsin Kazmiaea0df32019-05-23 14:32:58 +0200180 u16 slot, n_buffers;
181 slot = n_buffers = e->id;
Damjan Marion8389fb92017-10-13 18:29:53 +0200182
Mohsin Kazmi7f6d1452020-02-27 11:49:21 +0100183 while (e->id == (n_buffers & mask))
Mohsin Kazmiaea0df32019-05-23 14:32:58 +0200184 {
185 n_left--;
186 last++;
187 n_buffers++;
Mohsin Kazmia7a22812020-08-31 17:17:16 +0200188 vring_desc_t *d = &vring->desc[e->id];
Mohsin Kazmice215992020-04-10 14:57:06 +0000189 u16 next;
190 while (d->flags & VRING_DESC_F_NEXT)
191 {
192 n_buffers++;
193 next = d->next;
194 d = &vring->desc[next];
195 }
Mohsin Kazmiaea0df32019-05-23 14:32:58 +0200196 if (n_left == 0)
197 break;
198 e = &vring->used->ring[last & mask];
199 }
200 vlib_buffer_free_from_ring (vm, vring->buffers, slot,
201 sz, (n_buffers - slot));
Mohsin Kazmi7f6d1452020-02-27 11:49:21 +0100202 virtio_memset_ring_u32 (vring->buffers, slot, sz, (n_buffers - slot));
Mohsin Kazmiaea0df32019-05-23 14:32:58 +0200203 used -= (n_buffers - slot);
204
205 if (n_left > 0)
206 {
Mohsin Kazmi7f6d1452020-02-27 11:49:21 +0100207 vlib_buffer_free (vm, &vring->buffers[e->id], 1);
208 vring->buffers[e->id] = ~0;
Mohsin Kazmiaea0df32019-05-23 14:32:58 +0200209 used--;
210 last++;
211 n_left--;
Mohsin Kazmi7f6d1452020-02-27 11:49:21 +0100212 out_of_order_count++;
213 vring->flags |= VRING_TX_OUT_OF_ORDER;
Mohsin Kazmiaea0df32019-05-23 14:32:58 +0200214 }
Damjan Marion8389fb92017-10-13 18:29:53 +0200215 }
Mohsin Kazmi7f6d1452020-02-27 11:49:21 +0100216
217 /*
218 * Some vhost-backends give buffers back in out-of-order fashion in used ring.
219 * It impacts the overall virtio-performance.
220 */
221 if (out_of_order_count)
222 vlib_error_count (vm, node_index, VIRTIO_TX_ERROR_OUT_OF_ORDER,
223 out_of_order_count);
224
Damjan Marion8389fb92017-10-13 18:29:53 +0200225 vring->desc_in_use = used;
226 vring->last_used_idx = last;
227}
228
Mohsin Kazmi6d4af892020-01-03 15:11:53 +0000229static_always_inline void
Mohsin Kazmia7a22812020-08-31 17:17:16 +0200230set_checksum_offsets (vlib_buffer_t * b, virtio_net_hdr_v1_t * hdr, int is_l2)
Mohsin Kazmi6d4af892020-01-03 15:11:53 +0000231{
232 if (b->flags & VNET_BUFFER_F_IS_IP4)
233 {
234 ip4_header_t *ip4;
Mohsin Kazmi0b042092020-04-17 16:50:56 +0000235 generic_header_offset_t gho = { 0 };
Mohsin Kazmi84f91fa2020-04-23 17:59:49 +0200236 vnet_generic_header_offset_parser (b, &gho, is_l2, 1 /* ip4 */ ,
237 0 /* ip6 */ );
Mohsin Kazmi6d4af892020-01-03 15:11:53 +0000238 hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
239 hdr->csum_start = gho.l4_hdr_offset; // 0x22;
240 if (b->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM)
Mohsin Kazmi0937fdf2020-03-25 20:37:16 +0000241 {
Mohsin Kazmi0937fdf2020-03-25 20:37:16 +0000242 hdr->csum_offset = STRUCT_OFFSET_OF (tcp_header_t, checksum);
243 }
Mohsin Kazmi6d4af892020-01-03 15:11:53 +0000244 else if (b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM)
Mohsin Kazmi0937fdf2020-03-25 20:37:16 +0000245 {
Mohsin Kazmi0937fdf2020-03-25 20:37:16 +0000246 hdr->csum_offset = STRUCT_OFFSET_OF (udp_header_t, checksum);
247 }
Mohsin Kazmi6d4af892020-01-03 15:11:53 +0000248
249 /*
250 * virtio devices do not support IP4 checksum offload. So driver takes care
251 * of it while doing tx.
252 */
253 ip4 =
254 (ip4_header_t *) (vlib_buffer_get_current (b) + gho.l3_hdr_offset);
255 if (b->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM)
256 ip4->checksum = ip4_header_checksum (ip4);
257 }
258 else if (b->flags & VNET_BUFFER_F_IS_IP6)
259 {
Mohsin Kazmi0b042092020-04-17 16:50:56 +0000260 generic_header_offset_t gho = { 0 };
Mohsin Kazmi84f91fa2020-04-23 17:59:49 +0200261 vnet_generic_header_offset_parser (b, &gho, is_l2, 0 /* ip4 */ ,
262 1 /* ip6 */ );
Mohsin Kazmi6d4af892020-01-03 15:11:53 +0000263 hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
264 hdr->csum_start = gho.l4_hdr_offset; // 0x36;
265 if (b->flags & VNET_BUFFER_F_OFFLOAD_TCP_CKSUM)
Mohsin Kazmi0937fdf2020-03-25 20:37:16 +0000266 {
Mohsin Kazmi0937fdf2020-03-25 20:37:16 +0000267 hdr->csum_offset = STRUCT_OFFSET_OF (tcp_header_t, checksum);
268 }
Mohsin Kazmi6d4af892020-01-03 15:11:53 +0000269 else if (b->flags & VNET_BUFFER_F_OFFLOAD_UDP_CKSUM)
Mohsin Kazmi0937fdf2020-03-25 20:37:16 +0000270 {
Mohsin Kazmi0937fdf2020-03-25 20:37:16 +0000271 hdr->csum_offset = STRUCT_OFFSET_OF (udp_header_t, checksum);
272 }
Mohsin Kazmi6d4af892020-01-03 15:11:53 +0000273 }
274}
275
Mohsin Kazmif50ef402020-04-09 22:11:29 +0200276static_always_inline void
Mohsin Kazmia7a22812020-08-31 17:17:16 +0200277set_gso_offsets (vlib_buffer_t * b, virtio_net_hdr_v1_t * hdr, int is_l2)
Mohsin Kazmif50ef402020-04-09 22:11:29 +0200278{
279 if (b->flags & VNET_BUFFER_F_IS_IP4)
280 {
281 ip4_header_t *ip4;
Mohsin Kazmi0b042092020-04-17 16:50:56 +0000282 generic_header_offset_t gho = { 0 };
Mohsin Kazmi84f91fa2020-04-23 17:59:49 +0200283 vnet_generic_header_offset_parser (b, &gho, is_l2, 1 /* ip4 */ ,
284 0 /* ip6 */ );
Mohsin Kazmif50ef402020-04-09 22:11:29 +0200285 hdr->gso_type = VIRTIO_NET_HDR_GSO_TCPV4;
286 hdr->gso_size = vnet_buffer2 (b)->gso_size;
Mohsin Kazmi0b042092020-04-17 16:50:56 +0000287 hdr->hdr_len = gho.hdr_sz;
Mohsin Kazmif50ef402020-04-09 22:11:29 +0200288 hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
289 hdr->csum_start = gho.l4_hdr_offset; // 0x22;
290 hdr->csum_offset = STRUCT_OFFSET_OF (tcp_header_t, checksum);
291 ip4 =
292 (ip4_header_t *) (vlib_buffer_get_current (b) + gho.l3_hdr_offset);
293 /*
294 * virtio devices do not support IP4 checksum offload. So driver takes care
295 * of it while doing tx.
296 */
297 if (b->flags & VNET_BUFFER_F_OFFLOAD_IP_CKSUM)
298 ip4->checksum = ip4_header_checksum (ip4);
299 }
300 else if (b->flags & VNET_BUFFER_F_IS_IP6)
301 {
Mohsin Kazmi0b042092020-04-17 16:50:56 +0000302 generic_header_offset_t gho = { 0 };
Mohsin Kazmi84f91fa2020-04-23 17:59:49 +0200303 vnet_generic_header_offset_parser (b, &gho, is_l2, 0 /* ip4 */ ,
304 1 /* ip6 */ );
Mohsin Kazmif50ef402020-04-09 22:11:29 +0200305 hdr->gso_type = VIRTIO_NET_HDR_GSO_TCPV6;
306 hdr->gso_size = vnet_buffer2 (b)->gso_size;
Mohsin Kazmi0b042092020-04-17 16:50:56 +0000307 hdr->hdr_len = gho.hdr_sz;
Mohsin Kazmif50ef402020-04-09 22:11:29 +0200308 hdr->flags = VIRTIO_NET_HDR_F_NEEDS_CSUM;
309 hdr->csum_start = gho.l4_hdr_offset; // 0x36;
310 hdr->csum_offset = STRUCT_OFFSET_OF (tcp_header_t, checksum);
311 }
312}
313
Damjan Marion8389fb92017-10-13 18:29:53 +0200314static_always_inline u16
Mohsin Kazmi83143712020-10-07 13:23:55 +0200315add_buffer_to_slot (vlib_main_t * vm, vlib_node_runtime_t * node,
316 virtio_if_t * vif, virtio_if_type_t type,
317 virtio_vring_t * vring, u32 bi, u16 free_desc_count,
Mohsin Kazmice215992020-04-10 14:57:06 +0000318 u16 avail, u16 next, u16 mask, int do_gso,
Mohsin Kazmi83143712020-10-07 13:23:55 +0200319 int csum_offload)
Damjan Marion8389fb92017-10-13 18:29:53 +0200320{
321 u16 n_added = 0;
Mohsin Kazmid6c15af2018-10-23 18:00:47 +0200322 int hdr_sz = vif->virtio_net_hdr_sz;
Mohsin Kazmia7a22812020-08-31 17:17:16 +0200323 vring_desc_t *d;
Damjan Marion8389fb92017-10-13 18:29:53 +0200324 d = &vring->desc[next];
325 vlib_buffer_t *b = vlib_get_buffer (vm, bi);
Mohsin Kazmia7a22812020-08-31 17:17:16 +0200326 virtio_net_hdr_v1_t *hdr = vlib_buffer_get_current (b) - hdr_sz;
Mohsin Kazmi38b09682020-06-03 18:20:17 +0200327 int is_l2 = (type & (VIRTIO_IF_TYPE_TAP | VIRTIO_IF_TYPE_PCI));
Damjan Marion508cabc2018-02-08 19:49:22 +0100328
Dave Barachb7b92992018-10-17 10:38:51 -0400329 clib_memset (hdr, 0, hdr_sz);
Mohsin Kazmi6d4af892020-01-03 15:11:53 +0000330
Mohsin Kazmif50ef402020-04-09 22:11:29 +0200331 if (b->flags & VNET_BUFFER_F_GSO)
Andrew Yourtchenko6a7cff72018-10-12 16:09:22 +0200332 {
Mohsin Kazmif50ef402020-04-09 22:11:29 +0200333 if (do_gso)
Mohsin Kazmi84f91fa2020-04-23 17:59:49 +0200334 set_gso_offsets (b, hdr, is_l2);
Mohsin Kazmif50ef402020-04-09 22:11:29 +0200335 else
Andrew Yourtchenko6a7cff72018-10-12 16:09:22 +0200336 {
Mohsin Kazmi83143712020-10-07 13:23:55 +0200337 virtio_interface_drop_inline (vm, node->node_index, &bi, 1,
Mohsin Kazmif50ef402020-04-09 22:11:29 +0200338 VIRTIO_TX_ERROR_GSO_PACKET_DROP);
339 return n_added;
Andrew Yourtchenko6a7cff72018-10-12 16:09:22 +0200340 }
341 }
Mohsin Kazmif50ef402020-04-09 22:11:29 +0200342 else if (b->flags & (VNET_BUFFER_F_OFFLOAD_TCP_CKSUM |
343 VNET_BUFFER_F_OFFLOAD_UDP_CKSUM))
Mohsin Kazmi6d4af892020-01-03 15:11:53 +0000344 {
Mohsin Kazmif50ef402020-04-09 22:11:29 +0200345 if (csum_offload)
Mohsin Kazmi84f91fa2020-04-23 17:59:49 +0200346 set_checksum_offsets (b, hdr, is_l2);
Mohsin Kazmif50ef402020-04-09 22:11:29 +0200347 else
348 {
Mohsin Kazmi83143712020-10-07 13:23:55 +0200349 virtio_interface_drop_inline (vm, node->node_index, &bi, 1,
Mohsin Kazmif50ef402020-04-09 22:11:29 +0200350 VIRTIO_TX_ERROR_CSUM_OFFLOAD_PACKET_DROP);
351 return n_added;
352 }
Mohsin Kazmi6d4af892020-01-03 15:11:53 +0000353 }
Damjan Marion8389fb92017-10-13 18:29:53 +0200354
Mohsin Kazmi83143712020-10-07 13:23:55 +0200355 if (PREDICT_FALSE (b->flags & VLIB_BUFFER_IS_TRACED))
356 {
357 virtio_tx_trace (vm, node, type, b, bi);
358 }
359
Damjan Marion8389fb92017-10-13 18:29:53 +0200360 if (PREDICT_TRUE ((b->flags & VLIB_BUFFER_NEXT_PRESENT) == 0))
361 {
Mohsin Kazmid6c15af2018-10-23 18:00:47 +0200362 d->addr =
Mohsin Kazmi38b09682020-06-03 18:20:17 +0200363 ((type == VIRTIO_IF_TYPE_PCI) ? vlib_buffer_get_current_pa (vm,
364 b) :
Mohsin Kazmid6c15af2018-10-23 18:00:47 +0200365 pointer_to_uword (vlib_buffer_get_current (b))) - hdr_sz;
Damjan Marion8389fb92017-10-13 18:29:53 +0200366 d->len = b->current_length + hdr_sz;
367 d->flags = 0;
368 }
Mohsin Kazmice215992020-04-10 14:57:06 +0000369 else if (vif->features & VIRTIO_FEATURE (VIRTIO_RING_F_INDIRECT_DESC))
Damjan Marion8389fb92017-10-13 18:29:53 +0200370 {
Mohsin Kazmid6c15af2018-10-23 18:00:47 +0200371 /*
372 * We are using single vlib_buffer_t for indirect descriptor(s)
373 * chain. Single descriptor is 16 bytes and vlib_buffer_t
374 * has 2048 bytes space. So maximum long chain can have 128
375 * (=2048/16) indirect descriptors.
376 * It can easily support 65535 bytes of Jumbo frames with
377 * each data buffer size of 512 bytes minimum.
378 */
Mohsin Kazmiaea0df32019-05-23 14:32:58 +0200379 u32 indirect_buffer = 0;
380 if (PREDICT_FALSE (vlib_buffer_alloc (vm, &indirect_buffer, 1) == 0))
Mohsin Kazmif50ef402020-04-09 22:11:29 +0200381 {
Mohsin Kazmi83143712020-10-07 13:23:55 +0200382 virtio_interface_drop_inline (vm, node->node_index, &bi, 1,
Mohsin Kazmif50ef402020-04-09 22:11:29 +0200383 VIRTIO_TX_ERROR_INDIRECT_DESC_ALLOC_FAILED);
384 return n_added;
385 }
Mohsin Kazmiaea0df32019-05-23 14:32:58 +0200386
387 vlib_buffer_t *indirect_desc = vlib_get_buffer (vm, indirect_buffer);
Mohsin Kazmid6c15af2018-10-23 18:00:47 +0200388 indirect_desc->current_data = 0;
Mohsin Kazmiaea0df32019-05-23 14:32:58 +0200389 indirect_desc->flags |= VLIB_BUFFER_NEXT_PRESENT;
390 indirect_desc->next_buffer = bi;
391 bi = indirect_buffer;
Damjan Marion8389fb92017-10-13 18:29:53 +0200392
Mohsin Kazmia7a22812020-08-31 17:17:16 +0200393 vring_desc_t *id =
394 (vring_desc_t *) vlib_buffer_get_current (indirect_desc);
Mohsin Kazmid6c15af2018-10-23 18:00:47 +0200395 u32 count = 1;
Mohsin Kazmi38b09682020-06-03 18:20:17 +0200396 if (type == VIRTIO_IF_TYPE_PCI)
Damjan Marion8389fb92017-10-13 18:29:53 +0200397 {
Mohsin Kazmid6c15af2018-10-23 18:00:47 +0200398 d->addr = vlib_physmem_get_pa (vm, id);
399 id->addr = vlib_buffer_get_current_pa (vm, b) - hdr_sz;
Damjan Marion8389fb92017-10-13 18:29:53 +0200400
Mohsin Kazmid6c15af2018-10-23 18:00:47 +0200401 /*
402 * If VIRTIO_F_ANY_LAYOUT is not negotiated, then virtio_net_hdr
403 * should be presented in separate descriptor and data will start
404 * from next descriptor.
405 */
406 if (PREDICT_TRUE
407 (vif->features & VIRTIO_FEATURE (VIRTIO_F_ANY_LAYOUT)))
408 id->len = b->current_length + hdr_sz;
409 else
410 {
411 id->len = hdr_sz;
412 id->flags = VRING_DESC_F_NEXT;
413 id->next = count;
414 count++;
415 id++;
416 id->addr = vlib_buffer_get_current_pa (vm, b);
417 id->len = b->current_length;
418 }
419 while (b->flags & VLIB_BUFFER_NEXT_PRESENT)
420 {
421 id->flags = VRING_DESC_F_NEXT;
422 id->next = count;
423 count++;
424 id++;
425 b = vlib_get_buffer (vm, b->next_buffer);
426 id->addr = vlib_buffer_get_current_pa (vm, b);
427 id->len = b->current_length;
428 }
429 }
Mohsin Kazmi206acf82020-04-06 14:19:54 +0200430 else /* VIRTIO_IF_TYPE_[TAP | TUN] */
Mohsin Kazmid6c15af2018-10-23 18:00:47 +0200431 {
432 d->addr = pointer_to_uword (id);
433 /* first buffer in chain */
434 id->addr = pointer_to_uword (vlib_buffer_get_current (b)) - hdr_sz;
435 id->len = b->current_length + hdr_sz;
436
437 while (b->flags & VLIB_BUFFER_NEXT_PRESENT)
438 {
439 id->flags = VRING_DESC_F_NEXT;
440 id->next = count;
441 count++;
442 id++;
443 b = vlib_get_buffer (vm, b->next_buffer);
444 id->addr = pointer_to_uword (vlib_buffer_get_current (b));
445 id->len = b->current_length;
446 }
447 }
448 id->flags = 0;
449 id->next = 0;
Mohsin Kazmia7a22812020-08-31 17:17:16 +0200450 d->len = count * sizeof (vring_desc_t);
Damjan Marion8389fb92017-10-13 18:29:53 +0200451 d->flags = VRING_DESC_F_INDIRECT;
452 }
Mohsin Kazmi38b09682020-06-03 18:20:17 +0200453 else if (type == VIRTIO_IF_TYPE_PCI)
Mohsin Kazmice215992020-04-10 14:57:06 +0000454 {
455 u16 count = next;
456 vlib_buffer_t *b_temp = b;
457 u16 n_buffers_in_chain = 1;
458
459 /*
460 * Check the length of the chain for the required number of
461 * descriptors. Return from here, retry to get more descriptors,
462 * if chain length is greater than available descriptors.
463 */
464 while (b_temp->flags & VLIB_BUFFER_NEXT_PRESENT)
465 {
466 n_buffers_in_chain++;
467 b_temp = vlib_get_buffer (vm, b_temp->next_buffer);
468 }
469
470 if (n_buffers_in_chain > free_desc_count)
471 return n_buffers_in_chain;
472
473 d->addr = vlib_buffer_get_current_pa (vm, b) - hdr_sz;
474 d->len = b->current_length + hdr_sz;
475
476 while (b->flags & VLIB_BUFFER_NEXT_PRESENT)
477 {
478 d->flags = VRING_DESC_F_NEXT;
479 vring->buffers[count] = bi;
480 b->flags &=
481 ~(VLIB_BUFFER_NEXT_PRESENT | VLIB_BUFFER_TOTAL_LENGTH_VALID);
482 bi = b->next_buffer;
483 b->next_buffer = 0;
484 n_added++;
485 count = (count + 1) & mask;
486 d->next = count;
487 d = &vring->desc[count];
488 b = vlib_get_buffer (vm, bi);
489 d->addr = vlib_buffer_get_current_pa (vm, b);
490 d->len = b->current_length;
491 }
492 d->flags = 0;
493 vring->buffers[count] = bi;
494 vring->avail->ring[avail & mask] = next;
495 n_added++;
496 return n_added;
497 }
498 else
499 {
500 ASSERT (0);
501 }
Damjan Marion8389fb92017-10-13 18:29:53 +0200502 vring->buffers[next] = bi;
503 vring->avail->ring[avail & mask] = next;
504 n_added++;
505 return n_added;
506}
507
Mohsin Kazmi7f6d1452020-02-27 11:49:21 +0100508static_always_inline void
509virtio_find_free_desc (virtio_vring_t * vring, u16 size, u16 mask,
510 u16 req, u16 next, u32 * first_free_desc_index,
511 u16 * free_desc_count)
512{
513 u16 start = 0;
514 /* next is used as hint: from where to start looking */
515 for (u16 i = 0; i < size; i++, next++)
516 {
517 if (vring->buffers[next & mask] == ~0)
518 {
519 if (*first_free_desc_index == ~0)
520 {
521 *first_free_desc_index = (next & mask);
522 start = i;
523 (*free_desc_count)++;
524 req--;
525 if (req == 0)
526 break;
527 }
528 else
529 {
530 if (start + *free_desc_count == i)
531 {
532 (*free_desc_count)++;
533 req--;
534 if (req == 0)
535 break;
536 }
537 else
538 break;
539 }
540 }
541 }
542}
543
Damjan Marion8389fb92017-10-13 18:29:53 +0200544static_always_inline uword
Mohsin Kazmi38b09682020-06-03 18:20:17 +0200545virtio_interface_tx_gso_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
546 vlib_frame_t * frame, virtio_if_t * vif,
547 virtio_if_type_t type, int do_gso,
Mohsin Kazmi9e2a7852020-08-13 18:57:26 +0200548 int csum_offload, int do_gro)
Damjan Marion8389fb92017-10-13 18:29:53 +0200549{
Damjan Marion8389fb92017-10-13 18:29:53 +0200550 u16 n_left = frame->n_vectors;
Mohsin Kazmi09a3bc52019-04-02 11:45:08 +0000551 virtio_vring_t *vring;
552 u16 qid = vm->thread_index % vif->num_txqs;
553 vring = vec_elt_at_index (vif->txq_vrings, qid);
Mohsin Kazmie347acb2020-09-28 10:26:33 +0000554 u16 used, next, avail, n_buffers = 0, n_buffers_left = 0;
Damjan Marion8389fb92017-10-13 18:29:53 +0200555 u16 sz = vring->size;
556 u16 mask = sz - 1;
Mohsin Kazmi7f6d1452020-02-27 11:49:21 +0100557 u16 retry_count = 2;
Damjan Mariona3d59862018-11-10 10:23:00 +0100558 u32 *buffers = vlib_frame_vector_args (frame);
Mohsin Kazmi9e2a7852020-08-13 18:57:26 +0200559 u32 to[GRO_TO_VECTOR_SIZE (n_left)];
Damjan Marion8389fb92017-10-13 18:29:53 +0200560
Mohsin Kazmi09a3bc52019-04-02 11:45:08 +0000561 clib_spinlock_lock_if_init (&vring->lockp);
Damjan Marion829ee532018-02-16 16:13:32 +0100562
Mohsin Kazmia7a22812020-08-31 17:17:16 +0200563 if ((vring->used->flags & VRING_USED_F_NO_NOTIFY) == 0 &&
Mohsin Kazmid6c15af2018-10-23 18:00:47 +0200564 (vring->last_kick_avail_idx != vring->avail->idx))
565 virtio_kick (vm, vring, vif);
Damjan Marione40231b2018-12-20 10:44:47 +0100566
Mohsin Kazmi9e2a7852020-08-13 18:57:26 +0200567 if (do_gro)
568 {
569 n_left = vnet_gro_inline (vm, vring->flow_table, buffers, n_left, to);
570 buffers = to;
571 }
572
Mohsin Kazmi7f6d1452020-02-27 11:49:21 +0100573retry:
Damjan Marion8389fb92017-10-13 18:29:53 +0200574 /* free consumed buffers */
Mohsin Kazmi7f6d1452020-02-27 11:49:21 +0100575 virtio_free_used_device_desc (vm, vring, node->node_index);
Damjan Marion8389fb92017-10-13 18:29:53 +0200576
577 used = vring->desc_in_use;
578 next = vring->desc_next;
579 avail = vring->avail->idx;
580
Mohsin Kazmi7f6d1452020-02-27 11:49:21 +0100581 u16 free_desc_count = 0;
582
583 if (PREDICT_FALSE (vring->flags & VRING_TX_OUT_OF_ORDER))
584 {
585 u32 first_free_desc_index = ~0;
586
587 virtio_find_free_desc (vring, sz, mask, n_left, next,
588 &first_free_desc_index, &free_desc_count);
589
590 if (free_desc_count)
591 next = first_free_desc_index;
592 }
593 else
594 free_desc_count = sz - used;
595
Mohsin Kazmie347acb2020-09-28 10:26:33 +0000596 if (vif->packet_buffering)
597 {
598 n_buffers = n_buffers_left = virtio_vring_n_buffers (vring->buffering);
599
600 while (n_buffers_left && free_desc_count)
601 {
602 u16 n_added = 0;
603
604 u32 bi = virtio_vring_buffering_read_from_front (vring->buffering);
605 if (bi == ~0)
606 break;
Mohsin Kazmi83143712020-10-07 13:23:55 +0200607
Mohsin Kazmie347acb2020-09-28 10:26:33 +0000608 n_added =
Mohsin Kazmi83143712020-10-07 13:23:55 +0200609 add_buffer_to_slot (vm, node, vif, type, vring, bi,
610 free_desc_count, avail, next, mask, do_gso,
611 csum_offload);
Mohsin Kazmie347acb2020-09-28 10:26:33 +0000612 if (PREDICT_FALSE (n_added == 0))
613 {
614 n_buffers_left--;
615 continue;
616 }
617 else if (PREDICT_FALSE (n_added > free_desc_count))
618 break;
619
620 avail++;
621 next = (next + n_added) & mask;
622 used += n_added;
623 n_buffers_left--;
624 free_desc_count -= n_added;
625 }
626 }
627
Mohsin Kazmi7f6d1452020-02-27 11:49:21 +0100628 while (n_left && free_desc_count)
Damjan Marion8389fb92017-10-13 18:29:53 +0200629 {
Mohsin Kazmid6c15af2018-10-23 18:00:47 +0200630 u16 n_added = 0;
Mohsin Kazmib5ca5592020-06-26 13:48:26 +0000631
Mohsin Kazmid6c15af2018-10-23 18:00:47 +0200632 n_added =
Mohsin Kazmi83143712020-10-07 13:23:55 +0200633 add_buffer_to_slot (vm, node, vif, type, vring, buffers[0],
634 free_desc_count, avail, next, mask, do_gso,
635 csum_offload);
Mohsin Kazmif50ef402020-04-09 22:11:29 +0200636
637 if (PREDICT_FALSE (n_added == 0))
638 {
639 buffers++;
640 n_left--;
641 continue;
642 }
Mohsin Kazmice215992020-04-10 14:57:06 +0000643 else if (PREDICT_FALSE (n_added > free_desc_count))
644 break;
Mohsin Kazmif50ef402020-04-09 22:11:29 +0200645
Mohsin Kazmice215992020-04-10 14:57:06 +0000646 avail++;
Damjan Marion8389fb92017-10-13 18:29:53 +0200647 next = (next + n_added) & mask;
648 used += n_added;
649 buffers++;
650 n_left--;
Mohsin Kazmice215992020-04-10 14:57:06 +0000651 free_desc_count -= n_added;
Damjan Marion8389fb92017-10-13 18:29:53 +0200652 }
653
Mohsin Kazmie347acb2020-09-28 10:26:33 +0000654 if (n_left != frame->n_vectors || n_buffers != n_buffers_left)
Damjan Marion8389fb92017-10-13 18:29:53 +0200655 {
656 CLIB_MEMORY_STORE_BARRIER ();
657 vring->avail->idx = avail;
658 vring->desc_next = next;
659 vring->desc_in_use = used;
Mohsin Kazmia7a22812020-08-31 17:17:16 +0200660 if ((vring->used->flags & VRING_USED_F_NO_NOTIFY) == 0)
Mohsin Kazmid6c15af2018-10-23 18:00:47 +0200661 virtio_kick (vm, vring, vif);
Damjan Marion8389fb92017-10-13 18:29:53 +0200662 }
663
Damjan Marion8389fb92017-10-13 18:29:53 +0200664 if (n_left)
665 {
Mohsin Kazmi7f6d1452020-02-27 11:49:21 +0100666 if (retry_count--)
667 goto retry;
668
Mohsin Kazmie347acb2020-09-28 10:26:33 +0000669 if (vif->packet_buffering)
670 {
671
672 u16 n_buffered =
673 virtio_vring_buffering_store_packets (vring->buffering, buffers,
674 n_left);
675 buffers += n_buffered;
676 n_left -= n_buffered;
677 }
678 if (n_left)
679 virtio_interface_drop_inline (vm, node->node_index,
680 buffers, n_left,
681 VIRTIO_TX_ERROR_NO_FREE_SLOTS);
Damjan Marion8389fb92017-10-13 18:29:53 +0200682 }
683
Mohsin Kazmi09a3bc52019-04-02 11:45:08 +0000684 clib_spinlock_unlock_if_init (&vring->lockp);
Damjan Marion829ee532018-02-16 16:13:32 +0100685
Damjan Marion8389fb92017-10-13 18:29:53 +0200686 return frame->n_vectors - n_left;
687}
688
Mohsin Kazmi38b09682020-06-03 18:20:17 +0200689static_always_inline uword
690virtio_interface_tx_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
691 vlib_frame_t * frame, virtio_if_t * vif,
692 virtio_if_type_t type)
693{
694 vnet_main_t *vnm = vnet_get_main ();
695 vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, vif->hw_if_index);
696
697 if (hw->flags & VNET_HW_INTERFACE_FLAG_SUPPORTS_GSO)
698 return virtio_interface_tx_gso_inline (vm, node, frame, vif, type,
699 1 /* do_gso */ ,
Mohsin Kazmi9e2a7852020-08-13 18:57:26 +0200700 1 /* checksum offload */ ,
701 vif->packet_coalesce);
Mohsin Kazmi38b09682020-06-03 18:20:17 +0200702 else if (hw->flags & VNET_HW_INTERFACE_FLAG_SUPPORTS_TX_L4_CKSUM_OFFLOAD)
703 return virtio_interface_tx_gso_inline (vm, node, frame, vif, type,
704 0 /* no do_gso */ ,
Mohsin Kazmi9e2a7852020-08-13 18:57:26 +0200705 1 /* checksum offload */ ,
706 0 /* do_gro */ );
Mohsin Kazmi38b09682020-06-03 18:20:17 +0200707 else
708 return virtio_interface_tx_gso_inline (vm, node, frame, vif, type,
709 0 /* no do_gso */ ,
Mohsin Kazmi9e2a7852020-08-13 18:57:26 +0200710 0 /* no checksum offload */ ,
711 0 /* do_gro */ );
Mohsin Kazmi38b09682020-06-03 18:20:17 +0200712}
713
Filip Tehlaraee73642019-03-13 05:50:44 -0700714VNET_DEVICE_CLASS_TX_FN (virtio_device_class) (vlib_main_t * vm,
715 vlib_node_runtime_t * node,
716 vlib_frame_t * frame)
Damjan Marion8389fb92017-10-13 18:29:53 +0200717{
718 virtio_main_t *nm = &virtio_main;
719 vnet_interface_output_runtime_t *rund = (void *) node->runtime_data;
720 virtio_if_t *vif = pool_elt_at_index (nm->interfaces, rund->dev_instance);
Mohsin Kazmi09a3bc52019-04-02 11:45:08 +0000721
Mohsin Kazmi38b09682020-06-03 18:20:17 +0200722 if (vif->type == VIRTIO_IF_TYPE_TAP)
Mohsin Kazmi6d4af892020-01-03 15:11:53 +0000723 return virtio_interface_tx_inline (vm, node, frame, vif,
Mohsin Kazmi38b09682020-06-03 18:20:17 +0200724 VIRTIO_IF_TYPE_TAP);
725 else if (vif->type == VIRTIO_IF_TYPE_PCI)
726 return virtio_interface_tx_inline (vm, node, frame, vif,
727 VIRTIO_IF_TYPE_PCI);
728 else if (vif->type == VIRTIO_IF_TYPE_TUN)
729 return virtio_interface_tx_inline (vm, node, frame, vif,
730 VIRTIO_IF_TYPE_TUN);
Andrew Yourtchenko6a7cff72018-10-12 16:09:22 +0200731 else
Mohsin Kazmi38b09682020-06-03 18:20:17 +0200732 ASSERT (0);
733
734 return 0;
Damjan Marion8389fb92017-10-13 18:29:53 +0200735}
736
737static void
738virtio_set_interface_next_node (vnet_main_t * vnm, u32 hw_if_index,
739 u32 node_index)
740{
741 virtio_main_t *apm = &virtio_main;
742 vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index);
743 virtio_if_t *vif = pool_elt_at_index (apm->interfaces, hw->dev_instance);
744
745 /* Shut off redirection */
746 if (node_index == ~0)
747 {
748 vif->per_interface_next_index = node_index;
749 return;
750 }
751
752 vif->per_interface_next_index =
753 vlib_node_add_next (vlib_get_main (), virtio_input_node.index,
754 node_index);
755}
756
757static void
758virtio_clear_hw_interface_counters (u32 instance)
759{
760 /* Nothing for now */
761}
762
763static clib_error_t *
764virtio_interface_rx_mode_change (vnet_main_t * vnm, u32 hw_if_index, u32 qid,
Damjan Marioneabd4242020-10-07 20:59:07 +0200765 vnet_hw_if_rx_mode mode)
Damjan Marion8389fb92017-10-13 18:29:53 +0200766{
767 virtio_main_t *mm = &virtio_main;
768 vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index);
769 virtio_if_t *vif = pool_elt_at_index (mm->interfaces, hw->dev_instance);
Mohsin Kazmi1017a1d2020-09-25 15:36:19 +0200770 virtio_vring_t *rx_vring = vec_elt_at_index (vif->rxq_vrings, qid);
771 virtio_vring_t *tx_vring = 0;
Damjan Marion8389fb92017-10-13 18:29:53 +0200772
Mohsin Kazmib74fe322019-01-31 13:50:56 +0000773 if (vif->type == VIRTIO_IF_TYPE_PCI && !(vif->support_int_mode))
774 {
Mohsin Kazmi1017a1d2020-09-25 15:36:19 +0200775 rx_vring->avail->flags |= VRING_AVAIL_F_NO_INTERRUPT;
Mohsin Kazmib74fe322019-01-31 13:50:56 +0000776 return clib_error_return (0, "interrupt mode is not supported");
777 }
778
Damjan Marioneabd4242020-10-07 20:59:07 +0200779 if (mode == VNET_HW_IF_RX_MODE_POLLING)
Mohsin Kazmi9e2a7852020-08-13 18:57:26 +0200780 {
Mohsin Kazmi1017a1d2020-09-25 15:36:19 +0200781 vec_foreach (tx_vring, vif->txq_vrings)
782 {
783 /* only enable packet coalesce in poll mode */
784 gro_flow_table_set_is_enable (tx_vring->flow_table, 1);
Mohsin Kazmie347acb2020-09-28 10:26:33 +0000785 /* only enable packet buffering in poll mode */
786 virtio_vring_buffering_set_is_enable (tx_vring->buffering, 1);
Mohsin Kazmi1017a1d2020-09-25 15:36:19 +0200787 }
788 rx_vring->avail->flags |= VRING_AVAIL_F_NO_INTERRUPT;
Mohsin Kazmi9e2a7852020-08-13 18:57:26 +0200789 }
Damjan Marion8389fb92017-10-13 18:29:53 +0200790 else
Mohsin Kazmi9e2a7852020-08-13 18:57:26 +0200791 {
Mohsin Kazmie347acb2020-09-28 10:26:33 +0000792 if (vif->packet_coalesce || vif->packet_buffering)
Mohsin Kazmi1017a1d2020-09-25 15:36:19 +0200793 {
794 virtio_log_warning (vif,
Mohsin Kazmie347acb2020-09-28 10:26:33 +0000795 "interface %U is in interrupt mode, disabling packet coalescing or buffering",
Mohsin Kazmi1017a1d2020-09-25 15:36:19 +0200796 format_vnet_sw_if_index_name, vnet_get_main (),
797 vif->sw_if_index);
798 vec_foreach (tx_vring, vif->txq_vrings)
799 {
800 gro_flow_table_set_is_enable (tx_vring->flow_table, 0);
Mohsin Kazmie347acb2020-09-28 10:26:33 +0000801 virtio_vring_buffering_set_is_enable (tx_vring->buffering, 0);
Mohsin Kazmi1017a1d2020-09-25 15:36:19 +0200802 }
803 }
804 rx_vring->avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT;
Mohsin Kazmi9e2a7852020-08-13 18:57:26 +0200805 }
Damjan Marion8389fb92017-10-13 18:29:53 +0200806
807 return 0;
808}
809
810static clib_error_t *
811virtio_interface_admin_up_down (vnet_main_t * vnm, u32 hw_if_index, u32 flags)
812{
813 virtio_main_t *mm = &virtio_main;
814 vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index);
815 virtio_if_t *vif = pool_elt_at_index (mm->interfaces, hw->dev_instance);
Damjan Marion8389fb92017-10-13 18:29:53 +0200816
817 if (flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP)
Mohsin Kazmi529f0e92020-03-19 14:03:31 +0100818 {
819 vif->flags |= VIRTIO_IF_FLAG_ADMIN_UP;
820 vnet_hw_interface_set_flags (vnm, vif->hw_if_index,
821 VNET_HW_INTERFACE_FLAG_LINK_UP);
822 }
Damjan Marion8389fb92017-10-13 18:29:53 +0200823 else
Mohsin Kazmi529f0e92020-03-19 14:03:31 +0100824 {
825 vif->flags &= ~VIRTIO_IF_FLAG_ADMIN_UP;
826 vnet_hw_interface_set_flags (vnm, vif->hw_if_index, 0);
827 }
Damjan Marion8389fb92017-10-13 18:29:53 +0200828 return 0;
829}
830
831static clib_error_t *
832virtio_subif_add_del_function (vnet_main_t * vnm,
833 u32 hw_if_index,
834 struct vnet_sw_interface_t *st, int is_add)
835{
836 /* Nothing for now */
837 return 0;
838}
839
840/* *INDENT-OFF* */
841VNET_DEVICE_CLASS (virtio_device_class) = {
842 .name = "virtio",
Damjan Marion8389fb92017-10-13 18:29:53 +0200843 .format_device_name = format_virtio_device_name,
844 .format_device = format_virtio_device,
845 .format_tx_trace = format_virtio_tx_trace,
Mohsin Kazmid6c15af2018-10-23 18:00:47 +0200846 .tx_function_n_errors = VIRTIO_TX_N_ERROR,
Damjan Marion8389fb92017-10-13 18:29:53 +0200847 .tx_function_error_strings = virtio_tx_func_error_strings,
848 .rx_redirect_to_node = virtio_set_interface_next_node,
849 .clear_counters = virtio_clear_hw_interface_counters,
850 .admin_up_down_function = virtio_interface_admin_up_down,
851 .subif_add_del_function = virtio_subif_add_del_function,
852 .rx_mode_change_function = virtio_interface_rx_mode_change,
853};
Mohsin Kazmi206acf82020-04-06 14:19:54 +0200854
Damjan Marion8389fb92017-10-13 18:29:53 +0200855/* *INDENT-ON* */
856
857/*
858 * fd.io coding-style-patch-verification: ON
859 *
860 * Local Variables:
861 * eval: (c-set-style "gnu")
862 * End:
863 */