Damjan Marion | 8389fb9 | 2017-10-13 18:29:53 +0200 | [diff] [blame] | 1 | /* |
| 2 | *------------------------------------------------------------------ |
| 3 | * Copyright (c) 2016 Cisco and/or its affiliates. |
| 4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | * you may not use this file except in compliance with the License. |
| 6 | * You may obtain a copy of the License at: |
| 7 | * |
| 8 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | * |
| 10 | * Unless required by applicable law or agreed to in writing, software |
| 11 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | * See the License for the specific language governing permissions and |
| 14 | * limitations under the License. |
| 15 | *------------------------------------------------------------------ |
| 16 | */ |
| 17 | |
| 18 | #include <sys/types.h> |
| 19 | #include <sys/stat.h> |
| 20 | #include <fcntl.h> |
| 21 | #include <linux/virtio_net.h> |
| 22 | #include <linux/vhost.h> |
| 23 | |
| 24 | #include <vlib/vlib.h> |
| 25 | #include <vlib/unix/unix.h> |
| 26 | #include <vnet/ethernet/ethernet.h> |
Milan Lenco | 73e7f42 | 2017-12-14 10:04:25 +0100 | [diff] [blame] | 27 | #include <vnet/ip/ip4_packet.h> |
| 28 | #include <vnet/ip/ip6_packet.h> |
Damjan Marion | 8389fb9 | 2017-10-13 18:29:53 +0200 | [diff] [blame] | 29 | #include <vnet/devices/virtio/virtio.h> |
| 30 | |
| 31 | #define foreach_virtio_tx_func_error \ |
| 32 | _(NO_FREE_SLOTS, "no free tx slots") \ |
| 33 | _(TRUNC_PACKET, "packet > buffer size -- truncated in tx ring") \ |
| 34 | _(PENDING_MSGS, "pending msgs in tx ring") \ |
| 35 | _(NO_TX_QUEUES, "no tx queues") |
| 36 | |
| 37 | typedef enum |
| 38 | { |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 39 | #define _(f,s) VIRTIO_TX_ERROR_##f, |
Damjan Marion | 8389fb9 | 2017-10-13 18:29:53 +0200 | [diff] [blame] | 40 | foreach_virtio_tx_func_error |
| 41 | #undef _ |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 42 | VIRTIO_TX_N_ERROR, |
Damjan Marion | 8389fb9 | 2017-10-13 18:29:53 +0200 | [diff] [blame] | 43 | } virtio_tx_func_error_t; |
| 44 | |
| 45 | static char *virtio_tx_func_error_strings[] = { |
| 46 | #define _(n,s) s, |
| 47 | foreach_virtio_tx_func_error |
| 48 | #undef _ |
| 49 | }; |
| 50 | |
| 51 | u8 * |
| 52 | format_virtio_device_name (u8 * s, va_list * args) |
| 53 | { |
| 54 | u32 dev_instance = va_arg (*args, u32); |
| 55 | virtio_main_t *mm = &virtio_main; |
| 56 | virtio_if_t *vif = pool_elt_at_index (mm->interfaces, dev_instance); |
| 57 | |
| 58 | if (vif->type == VIRTIO_IF_TYPE_TAP) |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 59 | s = format (s, "tap%u", vif->id); |
| 60 | else if (vif->type == VIRTIO_IF_TYPE_PCI) |
| 61 | s = format (s, "virtio-%x/%x/%x/%x", vif->pci_addr.domain, |
| 62 | vif->pci_addr.bus, vif->pci_addr.slot, |
| 63 | vif->pci_addr.function); |
Damjan Marion | 8389fb9 | 2017-10-13 18:29:53 +0200 | [diff] [blame] | 64 | else |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 65 | s = format (s, "virtio-%lu", vif->dev_instance); |
Damjan Marion | 8389fb9 | 2017-10-13 18:29:53 +0200 | [diff] [blame] | 66 | |
| 67 | return s; |
| 68 | } |
| 69 | |
| 70 | static u8 * |
| 71 | format_virtio_device (u8 * s, va_list * args) |
| 72 | { |
| 73 | u32 dev_instance = va_arg (*args, u32); |
| 74 | int verbose = va_arg (*args, int); |
| 75 | u32 indent = format_get_indent (s); |
| 76 | |
| 77 | s = format (s, "VIRTIO interface"); |
| 78 | if (verbose) |
| 79 | { |
| 80 | s = format (s, "\n%U instance %u", format_white_space, indent + 2, |
| 81 | dev_instance); |
| 82 | } |
| 83 | return s; |
| 84 | } |
| 85 | |
| 86 | static u8 * |
| 87 | format_virtio_tx_trace (u8 * s, va_list * args) |
| 88 | { |
| 89 | s = format (s, "Unimplemented..."); |
| 90 | return s; |
| 91 | } |
| 92 | |
Steven | a624dbe | 2018-01-09 11:13:29 -0800 | [diff] [blame] | 93 | inline void |
Damjan Marion | 8389fb9 | 2017-10-13 18:29:53 +0200 | [diff] [blame] | 94 | virtio_free_used_desc (vlib_main_t * vm, virtio_vring_t * vring) |
| 95 | { |
| 96 | u16 used = vring->desc_in_use; |
| 97 | u16 sz = vring->size; |
| 98 | u16 mask = sz - 1; |
| 99 | u16 last = vring->last_used_idx; |
| 100 | u16 n_left = vring->used->idx - last; |
| 101 | |
| 102 | if (n_left == 0) |
| 103 | return; |
| 104 | |
| 105 | while (n_left) |
| 106 | { |
| 107 | struct vring_used_elem *e = &vring->used->ring[last & mask]; |
| 108 | u16 slot = e->id; |
Damjan Marion | 8389fb9 | 2017-10-13 18:29:53 +0200 | [diff] [blame] | 109 | |
| 110 | vlib_buffer_free (vm, &vring->buffers[slot], 1); |
| 111 | used--; |
| 112 | last++; |
| 113 | n_left--; |
| 114 | } |
| 115 | vring->desc_in_use = used; |
| 116 | vring->last_used_idx = last; |
| 117 | } |
| 118 | |
| 119 | static_always_inline u16 |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 120 | add_buffer_to_slot (vlib_main_t * vm, virtio_if_t * vif, |
| 121 | virtio_vring_t * vring, u32 bi, u16 avail, u16 next, |
| 122 | u16 mask) |
Damjan Marion | 8389fb9 | 2017-10-13 18:29:53 +0200 | [diff] [blame] | 123 | { |
| 124 | u16 n_added = 0; |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 125 | int hdr_sz = vif->virtio_net_hdr_sz; |
Damjan Marion | 8389fb9 | 2017-10-13 18:29:53 +0200 | [diff] [blame] | 126 | struct vring_desc *d; |
| 127 | d = &vring->desc[next]; |
| 128 | vlib_buffer_t *b = vlib_get_buffer (vm, bi); |
Damjan Marion | 508cabc | 2018-02-08 19:49:22 +0100 | [diff] [blame] | 129 | struct virtio_net_hdr_v1 *hdr = vlib_buffer_get_current (b) - hdr_sz; |
| 130 | |
Dave Barach | b7b9299 | 2018-10-17 10:38:51 -0400 | [diff] [blame] | 131 | clib_memset (hdr, 0, hdr_sz); |
Damjan Marion | 8389fb9 | 2017-10-13 18:29:53 +0200 | [diff] [blame] | 132 | |
| 133 | if (PREDICT_TRUE ((b->flags & VLIB_BUFFER_NEXT_PRESENT) == 0)) |
| 134 | { |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 135 | d->addr = |
| 136 | ((vif->type == VIRTIO_IF_TYPE_PCI) ? vlib_buffer_get_current_pa (vm, |
| 137 | b) : |
| 138 | pointer_to_uword (vlib_buffer_get_current (b))) - hdr_sz; |
Damjan Marion | 8389fb9 | 2017-10-13 18:29:53 +0200 | [diff] [blame] | 139 | d->len = b->current_length + hdr_sz; |
| 140 | d->flags = 0; |
| 141 | } |
| 142 | else |
| 143 | { |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 144 | /* |
| 145 | * We are using single vlib_buffer_t for indirect descriptor(s) |
| 146 | * chain. Single descriptor is 16 bytes and vlib_buffer_t |
| 147 | * has 2048 bytes space. So maximum long chain can have 128 |
| 148 | * (=2048/16) indirect descriptors. |
| 149 | * It can easily support 65535 bytes of Jumbo frames with |
| 150 | * each data buffer size of 512 bytes minimum. |
| 151 | */ |
| 152 | vlib_buffer_t *indirect_desc = |
| 153 | vlib_get_buffer (vm, vring->indirect_buffers[next]); |
| 154 | indirect_desc->current_data = 0; |
Damjan Marion | 8389fb9 | 2017-10-13 18:29:53 +0200 | [diff] [blame] | 155 | |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 156 | struct vring_desc *id = |
| 157 | (struct vring_desc *) vlib_buffer_get_current (indirect_desc); |
| 158 | u32 count = 1; |
| 159 | if (vif->type == VIRTIO_IF_TYPE_PCI) |
Damjan Marion | 8389fb9 | 2017-10-13 18:29:53 +0200 | [diff] [blame] | 160 | { |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 161 | d->addr = vlib_physmem_get_pa (vm, id); |
| 162 | id->addr = vlib_buffer_get_current_pa (vm, b) - hdr_sz; |
Damjan Marion | 8389fb9 | 2017-10-13 18:29:53 +0200 | [diff] [blame] | 163 | |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 164 | /* |
| 165 | * If VIRTIO_F_ANY_LAYOUT is not negotiated, then virtio_net_hdr |
| 166 | * should be presented in separate descriptor and data will start |
| 167 | * from next descriptor. |
| 168 | */ |
| 169 | if (PREDICT_TRUE |
| 170 | (vif->features & VIRTIO_FEATURE (VIRTIO_F_ANY_LAYOUT))) |
| 171 | id->len = b->current_length + hdr_sz; |
| 172 | else |
| 173 | { |
| 174 | id->len = hdr_sz; |
| 175 | id->flags = VRING_DESC_F_NEXT; |
| 176 | id->next = count; |
| 177 | count++; |
| 178 | id++; |
| 179 | id->addr = vlib_buffer_get_current_pa (vm, b); |
| 180 | id->len = b->current_length; |
| 181 | } |
| 182 | while (b->flags & VLIB_BUFFER_NEXT_PRESENT) |
| 183 | { |
| 184 | id->flags = VRING_DESC_F_NEXT; |
| 185 | id->next = count; |
| 186 | count++; |
| 187 | id++; |
| 188 | b = vlib_get_buffer (vm, b->next_buffer); |
| 189 | id->addr = vlib_buffer_get_current_pa (vm, b); |
| 190 | id->len = b->current_length; |
| 191 | } |
| 192 | } |
| 193 | else /* VIRTIO_IF_TYPE_TAP */ |
| 194 | { |
| 195 | d->addr = pointer_to_uword (id); |
| 196 | /* first buffer in chain */ |
| 197 | id->addr = pointer_to_uword (vlib_buffer_get_current (b)) - hdr_sz; |
| 198 | id->len = b->current_length + hdr_sz; |
| 199 | |
| 200 | while (b->flags & VLIB_BUFFER_NEXT_PRESENT) |
| 201 | { |
| 202 | id->flags = VRING_DESC_F_NEXT; |
| 203 | id->next = count; |
| 204 | count++; |
| 205 | id++; |
| 206 | b = vlib_get_buffer (vm, b->next_buffer); |
| 207 | id->addr = pointer_to_uword (vlib_buffer_get_current (b)); |
| 208 | id->len = b->current_length; |
| 209 | } |
| 210 | } |
| 211 | id->flags = 0; |
| 212 | id->next = 0; |
| 213 | d->len = count * sizeof (struct vring_desc); |
Damjan Marion | 8389fb9 | 2017-10-13 18:29:53 +0200 | [diff] [blame] | 214 | d->flags = VRING_DESC_F_INDIRECT; |
| 215 | } |
| 216 | vring->buffers[next] = bi; |
| 217 | vring->avail->ring[avail & mask] = next; |
| 218 | n_added++; |
| 219 | return n_added; |
| 220 | } |
| 221 | |
Damjan Marion | 8389fb9 | 2017-10-13 18:29:53 +0200 | [diff] [blame] | 222 | static_always_inline uword |
| 223 | virtio_interface_tx_inline (vlib_main_t * vm, vlib_node_runtime_t * node, |
| 224 | vlib_frame_t * frame, virtio_if_t * vif) |
| 225 | { |
| 226 | u8 qid = 0; |
| 227 | u16 n_left = frame->n_vectors; |
| 228 | virtio_vring_t *vring = vec_elt_at_index (vif->vrings, (qid << 1) + 1); |
| 229 | u16 used, next, avail; |
| 230 | u16 sz = vring->size; |
| 231 | u16 mask = sz - 1; |
Damjan Marion | a3d5986 | 2018-11-10 10:23:00 +0100 | [diff] [blame] | 232 | u32 *buffers = vlib_frame_vector_args (frame); |
Damjan Marion | 8389fb9 | 2017-10-13 18:29:53 +0200 | [diff] [blame] | 233 | |
Damjan Marion | 829ee53 | 2018-02-16 16:13:32 +0100 | [diff] [blame] | 234 | clib_spinlock_lock_if_init (&vif->lockp); |
| 235 | |
Damjan Marion | e40231b | 2018-12-20 10:44:47 +0100 | [diff] [blame] | 236 | if ((vring->used->flags & VIRTIO_RING_FLAG_MASK_INT) == 0 && |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 237 | (vring->last_kick_avail_idx != vring->avail->idx)) |
| 238 | virtio_kick (vm, vring, vif); |
Damjan Marion | e40231b | 2018-12-20 10:44:47 +0100 | [diff] [blame] | 239 | |
Damjan Marion | 8389fb9 | 2017-10-13 18:29:53 +0200 | [diff] [blame] | 240 | /* free consumed buffers */ |
| 241 | virtio_free_used_desc (vm, vring); |
| 242 | |
| 243 | used = vring->desc_in_use; |
| 244 | next = vring->desc_next; |
| 245 | avail = vring->avail->idx; |
| 246 | |
| 247 | while (n_left && used < sz) |
| 248 | { |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 249 | u16 n_added = 0; |
| 250 | n_added = |
| 251 | add_buffer_to_slot (vm, vif, vring, buffers[0], avail, next, mask); |
| 252 | if (!n_added) |
| 253 | break; |
Damjan Marion | 8389fb9 | 2017-10-13 18:29:53 +0200 | [diff] [blame] | 254 | avail += n_added; |
| 255 | next = (next + n_added) & mask; |
| 256 | used += n_added; |
| 257 | buffers++; |
| 258 | n_left--; |
| 259 | } |
| 260 | |
| 261 | if (n_left != frame->n_vectors) |
| 262 | { |
| 263 | CLIB_MEMORY_STORE_BARRIER (); |
| 264 | vring->avail->idx = avail; |
| 265 | vring->desc_next = next; |
| 266 | vring->desc_in_use = used; |
| 267 | if ((vring->used->flags & VIRTIO_RING_FLAG_MASK_INT) == 0) |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 268 | virtio_kick (vm, vring, vif); |
Damjan Marion | 8389fb9 | 2017-10-13 18:29:53 +0200 | [diff] [blame] | 269 | } |
| 270 | |
Damjan Marion | 8389fb9 | 2017-10-13 18:29:53 +0200 | [diff] [blame] | 271 | if (n_left) |
| 272 | { |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 273 | vlib_error_count (vm, node->node_index, VIRTIO_TX_ERROR_NO_FREE_SLOTS, |
Damjan Marion | 8389fb9 | 2017-10-13 18:29:53 +0200 | [diff] [blame] | 274 | n_left); |
| 275 | vlib_buffer_free (vm, buffers, n_left); |
| 276 | } |
| 277 | |
Damjan Marion | 829ee53 | 2018-02-16 16:13:32 +0100 | [diff] [blame] | 278 | clib_spinlock_unlock_if_init (&vif->lockp); |
| 279 | |
Damjan Marion | 8389fb9 | 2017-10-13 18:29:53 +0200 | [diff] [blame] | 280 | return frame->n_vectors - n_left; |
| 281 | } |
| 282 | |
| 283 | static uword |
| 284 | virtio_interface_tx (vlib_main_t * vm, |
| 285 | vlib_node_runtime_t * node, vlib_frame_t * frame) |
| 286 | { |
| 287 | virtio_main_t *nm = &virtio_main; |
| 288 | vnet_interface_output_runtime_t *rund = (void *) node->runtime_data; |
| 289 | virtio_if_t *vif = pool_elt_at_index (nm->interfaces, rund->dev_instance); |
| 290 | |
| 291 | return virtio_interface_tx_inline (vm, node, frame, vif); |
| 292 | } |
| 293 | |
| 294 | static void |
| 295 | virtio_set_interface_next_node (vnet_main_t * vnm, u32 hw_if_index, |
| 296 | u32 node_index) |
| 297 | { |
| 298 | virtio_main_t *apm = &virtio_main; |
| 299 | vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index); |
| 300 | virtio_if_t *vif = pool_elt_at_index (apm->interfaces, hw->dev_instance); |
| 301 | |
| 302 | /* Shut off redirection */ |
| 303 | if (node_index == ~0) |
| 304 | { |
| 305 | vif->per_interface_next_index = node_index; |
| 306 | return; |
| 307 | } |
| 308 | |
| 309 | vif->per_interface_next_index = |
| 310 | vlib_node_add_next (vlib_get_main (), virtio_input_node.index, |
| 311 | node_index); |
| 312 | } |
| 313 | |
| 314 | static void |
| 315 | virtio_clear_hw_interface_counters (u32 instance) |
| 316 | { |
| 317 | /* Nothing for now */ |
| 318 | } |
| 319 | |
| 320 | static clib_error_t * |
| 321 | virtio_interface_rx_mode_change (vnet_main_t * vnm, u32 hw_if_index, u32 qid, |
| 322 | vnet_hw_interface_rx_mode mode) |
| 323 | { |
| 324 | virtio_main_t *mm = &virtio_main; |
| 325 | vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index); |
| 326 | virtio_if_t *vif = pool_elt_at_index (mm->interfaces, hw->dev_instance); |
| 327 | virtio_vring_t *vring = vec_elt_at_index (vif->vrings, qid); |
| 328 | |
| 329 | if (mode == VNET_HW_INTERFACE_RX_MODE_POLLING) |
| 330 | vring->avail->flags |= VIRTIO_RING_FLAG_MASK_INT; |
| 331 | else |
| 332 | vring->avail->flags &= ~VIRTIO_RING_FLAG_MASK_INT; |
| 333 | |
| 334 | return 0; |
| 335 | } |
| 336 | |
| 337 | static clib_error_t * |
| 338 | virtio_interface_admin_up_down (vnet_main_t * vnm, u32 hw_if_index, u32 flags) |
| 339 | { |
| 340 | virtio_main_t *mm = &virtio_main; |
| 341 | vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, hw_if_index); |
| 342 | virtio_if_t *vif = pool_elt_at_index (mm->interfaces, hw->dev_instance); |
Damjan Marion | 8389fb9 | 2017-10-13 18:29:53 +0200 | [diff] [blame] | 343 | |
| 344 | if (flags & VNET_SW_INTERFACE_FLAG_ADMIN_UP) |
| 345 | vif->flags |= VIRTIO_IF_FLAG_ADMIN_UP; |
| 346 | else |
| 347 | vif->flags &= ~VIRTIO_IF_FLAG_ADMIN_UP; |
| 348 | |
Damjan Marion | 8389fb9 | 2017-10-13 18:29:53 +0200 | [diff] [blame] | 349 | return 0; |
| 350 | } |
| 351 | |
| 352 | static clib_error_t * |
| 353 | virtio_subif_add_del_function (vnet_main_t * vnm, |
| 354 | u32 hw_if_index, |
| 355 | struct vnet_sw_interface_t *st, int is_add) |
| 356 | { |
| 357 | /* Nothing for now */ |
| 358 | return 0; |
| 359 | } |
| 360 | |
| 361 | /* *INDENT-OFF* */ |
| 362 | VNET_DEVICE_CLASS (virtio_device_class) = { |
| 363 | .name = "virtio", |
| 364 | .tx_function = virtio_interface_tx, |
| 365 | .format_device_name = format_virtio_device_name, |
| 366 | .format_device = format_virtio_device, |
| 367 | .format_tx_trace = format_virtio_tx_trace, |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 368 | .tx_function_n_errors = VIRTIO_TX_N_ERROR, |
Damjan Marion | 8389fb9 | 2017-10-13 18:29:53 +0200 | [diff] [blame] | 369 | .tx_function_error_strings = virtio_tx_func_error_strings, |
| 370 | .rx_redirect_to_node = virtio_set_interface_next_node, |
| 371 | .clear_counters = virtio_clear_hw_interface_counters, |
| 372 | .admin_up_down_function = virtio_interface_admin_up_down, |
| 373 | .subif_add_del_function = virtio_subif_add_del_function, |
| 374 | .rx_mode_change_function = virtio_interface_rx_mode_change, |
| 375 | }; |
| 376 | |
| 377 | VLIB_DEVICE_TX_FUNCTION_MULTIARCH(virtio_device_class, |
| 378 | virtio_interface_tx) |
| 379 | /* *INDENT-ON* */ |
| 380 | |
| 381 | /* |
| 382 | * fd.io coding-style-patch-verification: ON |
| 383 | * |
| 384 | * Local Variables: |
| 385 | * eval: (c-set-style "gnu") |
| 386 | * End: |
| 387 | */ |