Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (c) 2018 Cisco and/or its affiliates. |
| 3 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | * you may not use this file except in compliance with the License. |
| 5 | * You may obtain a copy of the License at: |
| 6 | * |
| 7 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | * |
| 9 | * Unless required by applicable law or agreed to in writing, software |
| 10 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | * See the License for the specific language governing permissions and |
| 13 | * limitations under the License. |
| 14 | */ |
| 15 | |
| 16 | #include <fcntl.h> |
| 17 | #include <sys/ioctl.h> |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 18 | |
| 19 | #include <vppinfra/types.h> |
| 20 | #include <vlib/vlib.h> |
| 21 | #include <vlib/pci/pci.h> |
| 22 | #include <vnet/ethernet/ethernet.h> |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 23 | #include <vnet/ip/ip4_packet.h> |
| 24 | #include <vnet/ip/ip6_packet.h> |
| 25 | #include <vnet/devices/virtio/virtio.h> |
| 26 | #include <vnet/devices/virtio/pci.h> |
| 27 | |
| 28 | #define PCI_VENDOR_ID_VIRTIO 0x1af4 |
| 29 | #define PCI_DEVICE_ID_VIRTIO_NIC 0x1000 |
| 30 | /* Doesn't support modern device */ |
| 31 | #define PCI_DEVICE_ID_VIRTIO_NIC_MODERN 0x1041 |
| 32 | |
| 33 | #define PCI_CAPABILITY_LIST 0x34 |
| 34 | #define PCI_CAP_ID_VNDR 0x09 |
| 35 | #define PCI_CAP_ID_MSIX 0x11 |
| 36 | |
| 37 | #define PCI_MSIX_ENABLE 0x8000 |
| 38 | |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 39 | static pci_device_id_t virtio_pci_device_ids[] = { |
| 40 | { |
| 41 | .vendor_id = PCI_VENDOR_ID_VIRTIO, |
| 42 | .device_id = PCI_DEVICE_ID_VIRTIO_NIC}, |
| 43 | { |
| 44 | .vendor_id = PCI_VENDOR_ID_VIRTIO, |
| 45 | .device_id = PCI_DEVICE_ID_VIRTIO_NIC_MODERN}, |
| 46 | {0}, |
| 47 | }; |
| 48 | |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 49 | static u32 |
| 50 | virtio_pci_flag_change (vnet_main_t * vnm, vnet_hw_interface_t * hw, |
| 51 | u32 flags) |
| 52 | { |
| 53 | return 0; |
| 54 | } |
| 55 | |
| 56 | static clib_error_t * |
| 57 | virtio_pci_get_max_virtqueue_pairs (vlib_main_t * vm, virtio_if_t * vif) |
| 58 | { |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 59 | clib_error_t *error = 0; |
| 60 | u16 max_queue_pairs = 1; |
| 61 | |
| 62 | if (vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_MQ)) |
| 63 | { |
Mohsin Kazmi | a0a6833 | 2020-07-16 12:55:42 +0000 | [diff] [blame] | 64 | max_queue_pairs = vif->virtio_pci_func->get_max_queue_pairs (vm, vif); |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 65 | } |
| 66 | |
Damjan Marion | f41244f | 2019-11-08 17:41:06 +0100 | [diff] [blame] | 67 | virtio_log_debug (vif, "max queue pair is %x", max_queue_pairs); |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 68 | if (max_queue_pairs < 1 || max_queue_pairs > 0x8000) |
Mohsin Kazmi | 57d7321 | 2020-03-05 12:22:41 +0000 | [diff] [blame] | 69 | return clib_error_return (error, "max queue pair is %x," |
| 70 | " should be in range [1, 0x8000]", |
| 71 | max_queue_pairs); |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 72 | |
| 73 | vif->max_queue_pairs = max_queue_pairs; |
| 74 | return error; |
| 75 | } |
| 76 | |
| 77 | static void |
| 78 | virtio_pci_set_mac (vlib_main_t * vm, virtio_if_t * vif) |
| 79 | { |
Mohsin Kazmi | a0a6833 | 2020-07-16 12:55:42 +0000 | [diff] [blame] | 80 | if (vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_MAC)) |
| 81 | vif->virtio_pci_func->set_mac (vm, vif); |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 82 | } |
| 83 | |
| 84 | static u32 |
| 85 | virtio_pci_get_mac (vlib_main_t * vm, virtio_if_t * vif) |
| 86 | { |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 87 | if (vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_MAC)) |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 88 | { |
Mohsin Kazmi | a0a6833 | 2020-07-16 12:55:42 +0000 | [diff] [blame] | 89 | vif->virtio_pci_func->get_mac (vm, vif); |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 90 | return 0; |
| 91 | } |
| 92 | return 1; |
| 93 | } |
| 94 | |
| 95 | static u16 |
| 96 | virtio_pci_is_link_up (vlib_main_t * vm, virtio_if_t * vif) |
| 97 | { |
| 98 | /* |
| 99 | * Minimal driver: assumes link is up |
| 100 | */ |
| 101 | u16 status = 1; |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 102 | if (vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_STATUS)) |
Mohsin Kazmi | a0a6833 | 2020-07-16 12:55:42 +0000 | [diff] [blame] | 103 | status = vif->virtio_pci_func->get_device_status (vm, vif); |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 104 | return status; |
| 105 | } |
| 106 | |
| 107 | static void |
Mohsin Kazmi | 8046fdc | 2020-07-15 11:54:10 +0000 | [diff] [blame] | 108 | virtio_pci_irq_queue_handler (vlib_main_t * vm, vlib_pci_dev_handle_t h, |
| 109 | u16 line) |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 110 | { |
| 111 | vnet_main_t *vnm = vnet_get_main (); |
Mohsin Kazmi | 33cc5cf | 2019-01-21 15:19:39 +0000 | [diff] [blame] | 112 | virtio_main_t *vim = &virtio_main; |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 113 | uword pd = vlib_pci_get_private_data (vm, h); |
Mohsin Kazmi | 33cc5cf | 2019-01-21 15:19:39 +0000 | [diff] [blame] | 114 | virtio_if_t *vif = pool_elt_at_index (vim->interfaces, pd); |
Mohsin Kazmi | 8046fdc | 2020-07-15 11:54:10 +0000 | [diff] [blame] | 115 | line--; |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 116 | u16 qid = line; |
| 117 | |
| 118 | vnet_device_input_set_interrupt_pending (vnm, vif->hw_if_index, qid); |
| 119 | } |
| 120 | |
| 121 | static void |
Mohsin Kazmi | 8046fdc | 2020-07-15 11:54:10 +0000 | [diff] [blame] | 122 | virtio_pci_irq_config_handler (vlib_main_t * vm, vlib_pci_dev_handle_t h, |
| 123 | u16 line) |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 124 | { |
| 125 | vnet_main_t *vnm = vnet_get_main (); |
Mohsin Kazmi | 33cc5cf | 2019-01-21 15:19:39 +0000 | [diff] [blame] | 126 | virtio_main_t *vim = &virtio_main; |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 127 | uword pd = vlib_pci_get_private_data (vm, h); |
Mohsin Kazmi | 33cc5cf | 2019-01-21 15:19:39 +0000 | [diff] [blame] | 128 | virtio_if_t *vif = pool_elt_at_index (vim->interfaces, pd); |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 129 | |
| 130 | if (virtio_pci_is_link_up (vm, vif) & VIRTIO_NET_S_LINK_UP) |
| 131 | { |
| 132 | vif->flags |= VIRTIO_IF_FLAG_ADMIN_UP; |
| 133 | vnet_hw_interface_set_flags (vnm, vif->hw_if_index, |
| 134 | VNET_HW_INTERFACE_FLAG_LINK_UP); |
| 135 | } |
| 136 | else |
| 137 | { |
| 138 | vif->flags &= ~VIRTIO_IF_FLAG_ADMIN_UP; |
| 139 | vnet_hw_interface_set_flags (vnm, vif->hw_if_index, 0); |
| 140 | } |
| 141 | } |
| 142 | |
| 143 | static void |
| 144 | virtio_pci_irq_handler (vlib_main_t * vm, vlib_pci_dev_handle_t h) |
| 145 | { |
Mohsin Kazmi | 33cc5cf | 2019-01-21 15:19:39 +0000 | [diff] [blame] | 146 | virtio_main_t *vim = &virtio_main; |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 147 | uword pd = vlib_pci_get_private_data (vm, h); |
Mohsin Kazmi | 33cc5cf | 2019-01-21 15:19:39 +0000 | [diff] [blame] | 148 | virtio_if_t *vif = pool_elt_at_index (vim->interfaces, pd); |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 149 | u8 isr = 0; |
| 150 | u16 line = 0; |
| 151 | |
Mohsin Kazmi | a0a6833 | 2020-07-16 12:55:42 +0000 | [diff] [blame] | 152 | isr = vif->virtio_pci_func->get_isr (vm, vif); |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 153 | |
| 154 | /* |
| 155 | * If the lower bit is set: look through the used rings of |
| 156 | * all virtqueues for the device, to see if any progress has |
| 157 | * been made by the device which requires servicing. |
| 158 | */ |
| 159 | if (isr & VIRTIO_PCI_ISR_INTR) |
Mohsin Kazmi | 8046fdc | 2020-07-15 11:54:10 +0000 | [diff] [blame] | 160 | { |
| 161 | for (; line < vif->num_rxqs; line++) |
| 162 | virtio_pci_irq_queue_handler (vm, h, (line + 1)); |
| 163 | } |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 164 | |
| 165 | if (isr & VIRTIO_PCI_ISR_CONFIG) |
Mohsin Kazmi | 8046fdc | 2020-07-15 11:54:10 +0000 | [diff] [blame] | 166 | virtio_pci_irq_config_handler (vm, h, line); |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 167 | } |
| 168 | |
| 169 | inline void |
| 170 | device_status (vlib_main_t * vm, virtio_if_t * vif) |
| 171 | { |
| 172 | struct status_struct |
| 173 | { |
| 174 | u8 bit; |
| 175 | char *str; |
| 176 | }; |
| 177 | struct status_struct *status_entry; |
| 178 | static struct status_struct status_array[] = { |
| 179 | #define _(s,b) { .str = #s, .bit = b, }, |
| 180 | foreach_virtio_config_status_flags |
| 181 | #undef _ |
| 182 | {.str = NULL} |
| 183 | }; |
| 184 | |
| 185 | vlib_cli_output (vm, " status 0x%x", vif->status); |
| 186 | |
| 187 | status_entry = (struct status_struct *) &status_array; |
| 188 | while (status_entry->str) |
| 189 | { |
| 190 | if (vif->status & status_entry->bit) |
| 191 | vlib_cli_output (vm, " %s (%x)", status_entry->str, |
| 192 | status_entry->bit); |
| 193 | status_entry++; |
| 194 | } |
| 195 | } |
| 196 | |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 197 | static int |
Mohsin Kazmi | b977d3f | 2020-11-16 16:49:30 +0100 | [diff] [blame] | 198 | virtio_pci_send_ctrl_msg_packed (vlib_main_t * vm, virtio_if_t * vif, |
| 199 | virtio_ctrl_msg_t * data, u32 len) |
| 200 | { |
| 201 | virtio_vring_t *vring = vif->cxq_vring; |
| 202 | virtio_net_ctrl_ack_t status = VIRTIO_NET_ERR; |
| 203 | virtio_ctrl_msg_t result; |
| 204 | u32 buffer_index; |
| 205 | vlib_buffer_t *b; |
| 206 | u16 used, next; |
| 207 | u16 sz = vring->size; |
| 208 | u16 flags = 0, first_desc_flags = 0; |
| 209 | |
| 210 | used = vring->desc_in_use; |
| 211 | next = vring->desc_next; |
| 212 | vring_packed_desc_t *d = &vring->packed_desc[next]; |
| 213 | |
| 214 | if (vlib_buffer_alloc (vm, &buffer_index, 1)) |
| 215 | b = vlib_get_buffer (vm, buffer_index); |
| 216 | else |
| 217 | return VIRTIO_NET_ERR; |
| 218 | /* |
| 219 | * current_data may not be initialized with 0 and may contain |
| 220 | * previous offset. |
| 221 | */ |
| 222 | b->current_data = 0; |
| 223 | clib_memcpy (vlib_buffer_get_current (b), data, sizeof (virtio_ctrl_msg_t)); |
| 224 | |
| 225 | first_desc_flags = VRING_DESC_F_NEXT; |
| 226 | if (vring->avail_wrap_counter) |
| 227 | { |
| 228 | first_desc_flags |= VRING_DESC_F_AVAIL; |
| 229 | first_desc_flags &= ~VRING_DESC_F_USED; |
| 230 | } |
| 231 | else |
| 232 | { |
| 233 | first_desc_flags &= ~VRING_DESC_F_AVAIL; |
| 234 | first_desc_flags |= VRING_DESC_F_USED; |
| 235 | } |
| 236 | d->addr = vlib_buffer_get_current_pa (vm, b); |
| 237 | d->len = sizeof (virtio_net_ctrl_hdr_t); |
| 238 | d->id = next; |
| 239 | |
| 240 | next++; |
| 241 | if (next >= sz) |
| 242 | { |
| 243 | next = 0; |
| 244 | vring->avail_wrap_counter ^= 1; |
| 245 | } |
| 246 | used++; |
| 247 | |
| 248 | d = &vring->packed_desc[next]; |
| 249 | flags = VRING_DESC_F_NEXT; |
| 250 | if (vring->avail_wrap_counter) |
| 251 | { |
| 252 | flags |= VRING_DESC_F_AVAIL; |
| 253 | flags &= ~VRING_DESC_F_USED; |
| 254 | } |
| 255 | else |
| 256 | { |
| 257 | flags &= ~VRING_DESC_F_AVAIL; |
| 258 | flags |= VRING_DESC_F_USED; |
| 259 | } |
| 260 | d->addr = vlib_buffer_get_current_pa (vm, b) + |
| 261 | STRUCT_OFFSET_OF (virtio_ctrl_msg_t, data); |
| 262 | d->len = len; |
| 263 | d->id = next; |
| 264 | d->flags = flags; |
| 265 | |
| 266 | next++; |
| 267 | if (next >= sz) |
| 268 | { |
| 269 | next = 0; |
| 270 | vring->avail_wrap_counter ^= 1; |
| 271 | } |
| 272 | used++; |
| 273 | |
| 274 | d = &vring->packed_desc[next]; |
| 275 | flags = VRING_DESC_F_WRITE; |
| 276 | if (vring->avail_wrap_counter) |
| 277 | { |
| 278 | flags |= VRING_DESC_F_AVAIL; |
| 279 | flags &= ~VRING_DESC_F_USED; |
| 280 | } |
| 281 | else |
| 282 | { |
| 283 | flags &= ~VRING_DESC_F_AVAIL; |
| 284 | flags |= VRING_DESC_F_USED; |
| 285 | } |
| 286 | d->addr = vlib_buffer_get_current_pa (vm, b) + |
| 287 | STRUCT_OFFSET_OF (virtio_ctrl_msg_t, status); |
| 288 | d->len = sizeof (data->status); |
| 289 | d->id = next; |
| 290 | d->flags = flags; |
| 291 | |
| 292 | next++; |
| 293 | if (next >= sz) |
| 294 | { |
| 295 | next = 0; |
| 296 | vring->avail_wrap_counter ^= 1; |
| 297 | } |
| 298 | used++; |
| 299 | |
| 300 | CLIB_MEMORY_STORE_BARRIER (); |
| 301 | vring->packed_desc[vring->desc_next].flags = first_desc_flags; |
| 302 | vring->desc_next = next; |
| 303 | vring->desc_in_use = used; |
| 304 | CLIB_MEMORY_BARRIER (); |
| 305 | if (vring->device_event->flags != VRING_EVENT_F_DISABLE) |
| 306 | { |
| 307 | virtio_kick (vm, vring, vif); |
| 308 | } |
| 309 | |
| 310 | u16 last = vring->last_used_idx; |
| 311 | d = &vring->packed_desc[last]; |
| 312 | do |
| 313 | { |
| 314 | flags = d->flags; |
| 315 | } |
| 316 | while ((flags & VRING_DESC_F_AVAIL) != (vring->used_wrap_counter << 7) |
| 317 | || (flags & VRING_DESC_F_USED) != (vring->used_wrap_counter << 15)); |
| 318 | |
| 319 | last += 3; |
| 320 | if (last >= vring->size) |
| 321 | { |
| 322 | last = last - vring->size; |
| 323 | vring->used_wrap_counter ^= 1; |
| 324 | } |
| 325 | vring->desc_in_use -= 3; |
| 326 | vring->last_used_idx = last; |
| 327 | |
| 328 | CLIB_MEMORY_BARRIER (); |
| 329 | clib_memcpy (&result, vlib_buffer_get_current (b), |
| 330 | sizeof (virtio_ctrl_msg_t)); |
| 331 | virtio_log_debug (vif, "ctrl-queue: status %u", result.status); |
| 332 | status = result.status; |
| 333 | vlib_buffer_free (vm, &buffer_index, 1); |
| 334 | return status; |
| 335 | } |
| 336 | |
| 337 | static int |
| 338 | virtio_pci_send_ctrl_msg_split (vlib_main_t * vm, virtio_if_t * vif, |
| 339 | virtio_ctrl_msg_t * data, u32 len) |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 340 | { |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 341 | virtio_vring_t *vring = vif->cxq_vring; |
Mohsin Kazmi | a7a2281 | 2020-08-31 17:17:16 +0200 | [diff] [blame] | 342 | virtio_net_ctrl_ack_t status = VIRTIO_NET_ERR; |
| 343 | virtio_ctrl_msg_t result; |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 344 | u32 buffer_index; |
| 345 | vlib_buffer_t *b; |
| 346 | u16 used, next, avail; |
| 347 | u16 sz = vring->size; |
| 348 | u16 mask = sz - 1; |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 349 | |
| 350 | used = vring->desc_in_use; |
| 351 | next = vring->desc_next; |
| 352 | avail = vring->avail->idx; |
Mohsin Kazmi | a7a2281 | 2020-08-31 17:17:16 +0200 | [diff] [blame] | 353 | vring_desc_t *d = &vring->desc[next]; |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 354 | |
| 355 | if (vlib_buffer_alloc (vm, &buffer_index, 1)) |
| 356 | b = vlib_get_buffer (vm, buffer_index); |
| 357 | else |
| 358 | return VIRTIO_NET_ERR; |
| 359 | /* |
| 360 | * current_data may not be initialized with 0 and may contain |
| 361 | * previous offset. |
| 362 | */ |
| 363 | b->current_data = 0; |
Mohsin Kazmi | a7a2281 | 2020-08-31 17:17:16 +0200 | [diff] [blame] | 364 | clib_memcpy (vlib_buffer_get_current (b), data, sizeof (virtio_ctrl_msg_t)); |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 365 | d->flags = VRING_DESC_F_NEXT; |
| 366 | d->addr = vlib_buffer_get_current_pa (vm, b); |
Mohsin Kazmi | a7a2281 | 2020-08-31 17:17:16 +0200 | [diff] [blame] | 367 | d->len = sizeof (virtio_net_ctrl_hdr_t); |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 368 | vring->avail->ring[avail & mask] = next; |
| 369 | avail++; |
| 370 | next = (next + 1) & mask; |
| 371 | d->next = next; |
| 372 | used++; |
| 373 | |
| 374 | d = &vring->desc[next]; |
| 375 | d->flags = VRING_DESC_F_NEXT; |
| 376 | d->addr = vlib_buffer_get_current_pa (vm, b) + |
Mohsin Kazmi | a7a2281 | 2020-08-31 17:17:16 +0200 | [diff] [blame] | 377 | STRUCT_OFFSET_OF (virtio_ctrl_msg_t, data); |
Mohsin Kazmi | 0549378 | 2019-05-01 14:26:17 +0200 | [diff] [blame] | 378 | d->len = len; |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 379 | next = (next + 1) & mask; |
| 380 | d->next = next; |
| 381 | used++; |
| 382 | |
| 383 | d = &vring->desc[next]; |
| 384 | d->flags = VRING_DESC_F_WRITE; |
| 385 | d->addr = vlib_buffer_get_current_pa (vm, b) + |
Mohsin Kazmi | a7a2281 | 2020-08-31 17:17:16 +0200 | [diff] [blame] | 386 | STRUCT_OFFSET_OF (virtio_ctrl_msg_t, status); |
Mohsin Kazmi | 0549378 | 2019-05-01 14:26:17 +0200 | [diff] [blame] | 387 | d->len = sizeof (data->status); |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 388 | next = (next + 1) & mask; |
| 389 | used++; |
| 390 | |
| 391 | CLIB_MEMORY_STORE_BARRIER (); |
| 392 | vring->avail->idx = avail; |
| 393 | vring->desc_next = next; |
| 394 | vring->desc_in_use = used; |
| 395 | |
| 396 | if ((vring->used->flags & VIRTIO_RING_FLAG_MASK_INT) == 0) |
| 397 | { |
| 398 | virtio_kick (vm, vring, vif); |
| 399 | } |
| 400 | |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 401 | u16 last = vring->last_used_idx, n_left = 0; |
| 402 | n_left = vring->used->idx - last; |
| 403 | |
| 404 | while (n_left) |
| 405 | { |
Mohsin Kazmi | a7a2281 | 2020-08-31 17:17:16 +0200 | [diff] [blame] | 406 | vring_used_elem_t *e = &vring->used->ring[last & mask]; |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 407 | u16 slot = e->id; |
| 408 | |
| 409 | d = &vring->desc[slot]; |
| 410 | while (d->flags & VRING_DESC_F_NEXT) |
| 411 | { |
| 412 | used--; |
| 413 | slot = d->next; |
| 414 | d = &vring->desc[slot]; |
| 415 | } |
| 416 | used--; |
| 417 | last++; |
| 418 | n_left--; |
| 419 | } |
| 420 | vring->desc_in_use = used; |
| 421 | vring->last_used_idx = last; |
| 422 | |
| 423 | CLIB_MEMORY_BARRIER (); |
| 424 | clib_memcpy (&result, vlib_buffer_get_current (b), |
Mohsin Kazmi | a7a2281 | 2020-08-31 17:17:16 +0200 | [diff] [blame] | 425 | sizeof (virtio_ctrl_msg_t)); |
Damjan Marion | f41244f | 2019-11-08 17:41:06 +0100 | [diff] [blame] | 426 | virtio_log_debug (vif, "ctrl-queue: status %u", result.status); |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 427 | status = result.status; |
| 428 | vlib_buffer_free (vm, &buffer_index, 1); |
| 429 | return status; |
| 430 | } |
| 431 | |
Mohsin Kazmi | 0549378 | 2019-05-01 14:26:17 +0200 | [diff] [blame] | 432 | static int |
Mohsin Kazmi | b977d3f | 2020-11-16 16:49:30 +0100 | [diff] [blame] | 433 | virtio_pci_send_ctrl_msg (vlib_main_t * vm, virtio_if_t * vif, |
| 434 | virtio_ctrl_msg_t * data, u32 len) |
| 435 | { |
| 436 | if (vif->is_packed) |
| 437 | return virtio_pci_send_ctrl_msg_packed (vm, vif, data, len); |
| 438 | else |
| 439 | return virtio_pci_send_ctrl_msg_split (vm, vif, data, len); |
| 440 | } |
| 441 | |
| 442 | static int |
Mohsin Kazmi | 6d4af89 | 2020-01-03 15:11:53 +0000 | [diff] [blame] | 443 | virtio_pci_disable_offload (vlib_main_t * vm, virtio_if_t * vif) |
| 444 | { |
Mohsin Kazmi | a7a2281 | 2020-08-31 17:17:16 +0200 | [diff] [blame] | 445 | virtio_ctrl_msg_t offload_hdr; |
| 446 | virtio_net_ctrl_ack_t status = VIRTIO_NET_ERR; |
Mohsin Kazmi | 6d4af89 | 2020-01-03 15:11:53 +0000 | [diff] [blame] | 447 | |
| 448 | offload_hdr.ctrl.class = VIRTIO_NET_CTRL_GUEST_OFFLOADS; |
| 449 | offload_hdr.ctrl.cmd = VIRTIO_NET_CTRL_GUEST_OFFLOADS_SET; |
| 450 | offload_hdr.status = VIRTIO_NET_ERR; |
| 451 | u64 offloads = 0ULL; |
| 452 | clib_memcpy (offload_hdr.data, &offloads, sizeof (offloads)); |
| 453 | |
| 454 | status = |
| 455 | virtio_pci_send_ctrl_msg (vm, vif, &offload_hdr, sizeof (offloads)); |
| 456 | virtio_log_debug (vif, "disable offloads"); |
Mohsin Kazmi | a0a6833 | 2020-07-16 12:55:42 +0000 | [diff] [blame] | 457 | vif->remote_features = vif->virtio_pci_func->get_device_features (vm, vif); |
| 458 | vif->virtio_pci_func->get_driver_features (vm, vif); |
Mohsin Kazmi | 6d4af89 | 2020-01-03 15:11:53 +0000 | [diff] [blame] | 459 | return status; |
| 460 | } |
| 461 | |
| 462 | static int |
| 463 | virtio_pci_enable_checksum_offload (vlib_main_t * vm, virtio_if_t * vif) |
| 464 | { |
Mohsin Kazmi | a7a2281 | 2020-08-31 17:17:16 +0200 | [diff] [blame] | 465 | virtio_ctrl_msg_t csum_offload_hdr; |
| 466 | virtio_net_ctrl_ack_t status = VIRTIO_NET_ERR; |
Mohsin Kazmi | 6d4af89 | 2020-01-03 15:11:53 +0000 | [diff] [blame] | 467 | |
| 468 | csum_offload_hdr.ctrl.class = VIRTIO_NET_CTRL_GUEST_OFFLOADS; |
| 469 | csum_offload_hdr.ctrl.cmd = VIRTIO_NET_CTRL_GUEST_OFFLOADS_SET; |
| 470 | csum_offload_hdr.status = VIRTIO_NET_ERR; |
| 471 | u64 offloads = 0ULL; |
| 472 | offloads |= VIRTIO_FEATURE (VIRTIO_NET_F_GUEST_CSUM); |
| 473 | clib_memcpy (csum_offload_hdr.data, &offloads, sizeof (offloads)); |
| 474 | |
| 475 | status = |
| 476 | virtio_pci_send_ctrl_msg (vm, vif, &csum_offload_hdr, sizeof (offloads)); |
| 477 | virtio_log_debug (vif, "enable checksum offload"); |
Mohsin Kazmi | a0a6833 | 2020-07-16 12:55:42 +0000 | [diff] [blame] | 478 | vif->remote_features = vif->virtio_pci_func->get_device_features (vm, vif); |
| 479 | vif->features = vif->virtio_pci_func->get_driver_features (vm, vif); |
Mohsin Kazmi | 6d4af89 | 2020-01-03 15:11:53 +0000 | [diff] [blame] | 480 | return status; |
| 481 | } |
| 482 | |
| 483 | static int |
Mohsin Kazmi | bbd6b74 | 2019-05-02 13:54:59 +0200 | [diff] [blame] | 484 | virtio_pci_enable_gso (vlib_main_t * vm, virtio_if_t * vif) |
| 485 | { |
Mohsin Kazmi | a7a2281 | 2020-08-31 17:17:16 +0200 | [diff] [blame] | 486 | virtio_ctrl_msg_t gso_hdr; |
| 487 | virtio_net_ctrl_ack_t status = VIRTIO_NET_ERR; |
Mohsin Kazmi | bbd6b74 | 2019-05-02 13:54:59 +0200 | [diff] [blame] | 488 | |
| 489 | gso_hdr.ctrl.class = VIRTIO_NET_CTRL_GUEST_OFFLOADS; |
| 490 | gso_hdr.ctrl.cmd = VIRTIO_NET_CTRL_GUEST_OFFLOADS_SET; |
| 491 | gso_hdr.status = VIRTIO_NET_ERR; |
| 492 | u64 offloads = VIRTIO_FEATURE (VIRTIO_NET_F_GUEST_CSUM) |
| 493 | | VIRTIO_FEATURE (VIRTIO_NET_F_GUEST_TSO4) |
Mohsin Kazmi | 6d4af89 | 2020-01-03 15:11:53 +0000 | [diff] [blame] | 494 | | VIRTIO_FEATURE (VIRTIO_NET_F_GUEST_TSO6); |
Mohsin Kazmi | bbd6b74 | 2019-05-02 13:54:59 +0200 | [diff] [blame] | 495 | clib_memcpy (gso_hdr.data, &offloads, sizeof (offloads)); |
| 496 | |
| 497 | status = virtio_pci_send_ctrl_msg (vm, vif, &gso_hdr, sizeof (offloads)); |
Damjan Marion | f41244f | 2019-11-08 17:41:06 +0100 | [diff] [blame] | 498 | virtio_log_debug (vif, "enable gso"); |
Mohsin Kazmi | a0a6833 | 2020-07-16 12:55:42 +0000 | [diff] [blame] | 499 | vif->remote_features = vif->virtio_pci_func->get_device_features (vm, vif); |
| 500 | vif->virtio_pci_func->get_driver_features (vm, vif); |
Mohsin Kazmi | bbd6b74 | 2019-05-02 13:54:59 +0200 | [diff] [blame] | 501 | return status; |
| 502 | } |
| 503 | |
| 504 | static int |
Mohsin Kazmi | 6d4af89 | 2020-01-03 15:11:53 +0000 | [diff] [blame] | 505 | virtio_pci_offloads (vlib_main_t * vm, virtio_if_t * vif, int gso_enabled, |
| 506 | int csum_offload_enabled) |
| 507 | { |
| 508 | vnet_main_t *vnm = vnet_get_main (); |
| 509 | vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, vif->hw_if_index); |
| 510 | |
| 511 | if ((vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_CTRL_VQ)) && |
| 512 | (vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_CTRL_GUEST_OFFLOADS))) |
| 513 | { |
| 514 | if (gso_enabled |
| 515 | && (vif->features & (VIRTIO_FEATURE (VIRTIO_NET_F_HOST_TSO4) | |
| 516 | VIRTIO_FEATURE (VIRTIO_NET_F_HOST_TSO6)))) |
| 517 | { |
| 518 | if (virtio_pci_enable_gso (vm, vif)) |
| 519 | { |
| 520 | virtio_log_warning (vif, "gso is not enabled"); |
| 521 | } |
| 522 | else |
| 523 | { |
| 524 | vif->gso_enabled = 1; |
| 525 | vif->csum_offload_enabled = 0; |
| 526 | hw->flags |= VNET_HW_INTERFACE_FLAG_SUPPORTS_GSO | |
| 527 | VNET_HW_INTERFACE_FLAG_SUPPORTS_TX_L4_CKSUM_OFFLOAD; |
| 528 | } |
| 529 | } |
| 530 | else if (csum_offload_enabled |
| 531 | && (vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_CSUM))) |
| 532 | { |
| 533 | if (virtio_pci_enable_checksum_offload (vm, vif)) |
| 534 | { |
| 535 | virtio_log_warning (vif, "checksum offload is not enabled"); |
| 536 | } |
| 537 | else |
| 538 | { |
| 539 | vif->csum_offload_enabled = 1; |
| 540 | vif->gso_enabled = 0; |
| 541 | hw->flags &= ~VNET_HW_INTERFACE_FLAG_SUPPORTS_GSO; |
| 542 | hw->flags |= |
| 543 | VNET_HW_INTERFACE_FLAG_SUPPORTS_TX_L4_CKSUM_OFFLOAD; |
| 544 | } |
| 545 | } |
| 546 | else |
| 547 | { |
| 548 | if (virtio_pci_disable_offload (vm, vif)) |
| 549 | { |
| 550 | virtio_log_warning (vif, "offloads are not disabled"); |
| 551 | } |
| 552 | else |
| 553 | { |
| 554 | vif->csum_offload_enabled = 0; |
| 555 | vif->gso_enabled = 0; |
| 556 | hw->flags &= ~(VNET_HW_INTERFACE_FLAG_SUPPORTS_GSO | |
| 557 | VNET_HW_INTERFACE_FLAG_SUPPORTS_TX_L4_CKSUM_OFFLOAD); |
| 558 | } |
| 559 | } |
| 560 | } |
| 561 | |
| 562 | return 0; |
| 563 | } |
| 564 | |
| 565 | static int |
Mohsin Kazmi | 0549378 | 2019-05-01 14:26:17 +0200 | [diff] [blame] | 566 | virtio_pci_enable_multiqueue (vlib_main_t * vm, virtio_if_t * vif, |
| 567 | u16 num_queues) |
| 568 | { |
Mohsin Kazmi | a7a2281 | 2020-08-31 17:17:16 +0200 | [diff] [blame] | 569 | virtio_ctrl_msg_t mq_hdr; |
| 570 | virtio_net_ctrl_ack_t status = VIRTIO_NET_ERR; |
Mohsin Kazmi | 0549378 | 2019-05-01 14:26:17 +0200 | [diff] [blame] | 571 | |
| 572 | mq_hdr.ctrl.class = VIRTIO_NET_CTRL_MQ; |
| 573 | mq_hdr.ctrl.cmd = VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET; |
| 574 | mq_hdr.status = VIRTIO_NET_ERR; |
| 575 | clib_memcpy (mq_hdr.data, &num_queues, sizeof (num_queues)); |
| 576 | |
| 577 | status = virtio_pci_send_ctrl_msg (vm, vif, &mq_hdr, sizeof (num_queues)); |
Damjan Marion | f41244f | 2019-11-08 17:41:06 +0100 | [diff] [blame] | 578 | virtio_log_debug (vif, "multi-queue enable %u queues", num_queues); |
Mohsin Kazmi | 0549378 | 2019-05-01 14:26:17 +0200 | [diff] [blame] | 579 | return status; |
| 580 | } |
| 581 | |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 582 | static u8 |
| 583 | virtio_pci_queue_size_valid (u16 qsz) |
| 584 | { |
| 585 | if (qsz < 64 || qsz > 4096) |
| 586 | return 0; |
| 587 | if ((qsz % 64) != 0) |
| 588 | return 0; |
| 589 | return 1; |
| 590 | } |
| 591 | |
| 592 | clib_error_t * |
Mohsin Kazmi | b977d3f | 2020-11-16 16:49:30 +0100 | [diff] [blame] | 593 | virtio_pci_control_vring_packed_init (vlib_main_t * vm, virtio_if_t * vif, |
| 594 | u16 queue_num) |
| 595 | { |
| 596 | clib_error_t *error = 0; |
| 597 | u16 queue_size = 0; |
| 598 | virtio_vring_t *vring; |
| 599 | u32 i = 0; |
| 600 | void *ptr = NULL; |
| 601 | |
| 602 | queue_size = vif->virtio_pci_func->get_queue_size (vm, vif, queue_num); |
| 603 | |
| 604 | if (queue_size > 32768) |
| 605 | return clib_error_return (0, "ring size must be 32768 or lower"); |
| 606 | |
| 607 | if (queue_size == 0) |
| 608 | queue_size = 256; |
| 609 | |
| 610 | vec_validate_aligned (vif->cxq_vring, 0, CLIB_CACHE_LINE_BYTES); |
| 611 | vring = vec_elt_at_index (vif->cxq_vring, 0); |
| 612 | |
| 613 | i = |
| 614 | (((queue_size * sizeof (vring_packed_desc_t)) + |
| 615 | sizeof (vring_desc_event_t) + VIRTIO_PCI_VRING_ALIGN - |
| 616 | 1) & ~(VIRTIO_PCI_VRING_ALIGN - 1)) + sizeof (vring_desc_event_t); |
| 617 | |
| 618 | ptr = |
| 619 | vlib_physmem_alloc_aligned_on_numa (vm, i, VIRTIO_PCI_VRING_ALIGN, |
| 620 | vif->numa_node); |
| 621 | if (!ptr) |
| 622 | return vlib_physmem_last_error (vm); |
| 623 | clib_memset (ptr, 0, i); |
| 624 | |
| 625 | vring->packed_desc = ptr; |
| 626 | |
| 627 | vring->driver_event = ptr + (queue_size * sizeof (vring_packed_desc_t)); |
| 628 | vring->driver_event->off_wrap = 0; |
| 629 | vring->driver_event->flags = VRING_EVENT_F_DISABLE; |
| 630 | |
| 631 | vring->device_event = |
| 632 | ptr + |
| 633 | (((queue_size * sizeof (vring_packed_desc_t)) + |
| 634 | sizeof (vring_desc_event_t) + VIRTIO_PCI_VRING_ALIGN - |
| 635 | 1) & ~(VIRTIO_PCI_VRING_ALIGN - 1)); |
| 636 | vring->device_event->off_wrap = 0; |
| 637 | vring->device_event->flags = 0; |
| 638 | |
| 639 | vring->queue_id = queue_num; |
| 640 | vring->size = queue_size; |
| 641 | vring->avail_wrap_counter = 1; |
| 642 | vring->used_wrap_counter = 1; |
| 643 | |
| 644 | ASSERT (vring->buffers == 0); |
| 645 | |
| 646 | virtio_log_debug (vif, "control-queue: number %u, size %u", queue_num, |
| 647 | queue_size); |
| 648 | vif->virtio_pci_func->setup_queue (vm, vif, queue_num, (void *) vring); |
| 649 | vring->queue_notify_offset = |
| 650 | vif->notify_off_multiplier * |
| 651 | vif->virtio_pci_func->get_queue_notify_off (vm, vif, queue_num); |
| 652 | virtio_log_debug (vif, "queue-notify-offset: number %u, offset %u", |
| 653 | queue_num, vring->queue_notify_offset); |
| 654 | return error; |
| 655 | } |
| 656 | |
| 657 | clib_error_t * |
| 658 | virtio_pci_control_vring_split_init (vlib_main_t * vm, virtio_if_t * vif, |
| 659 | u16 queue_num) |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 660 | { |
| 661 | clib_error_t *error = 0; |
| 662 | u16 queue_size = 0; |
| 663 | virtio_vring_t *vring; |
Mohsin Kazmi | a7a2281 | 2020-08-31 17:17:16 +0200 | [diff] [blame] | 664 | vring_t vr; |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 665 | u32 i = 0; |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 666 | void *ptr = NULL; |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 667 | |
Mohsin Kazmi | a0a6833 | 2020-07-16 12:55:42 +0000 | [diff] [blame] | 668 | queue_size = vif->virtio_pci_func->get_queue_size (vm, vif, queue_num); |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 669 | if (!virtio_pci_queue_size_valid (queue_size)) |
| 670 | clib_warning ("queue size is not valid"); |
| 671 | |
| 672 | if (!is_pow2 (queue_size)) |
| 673 | return clib_error_return (0, "ring size must be power of 2"); |
| 674 | |
| 675 | if (queue_size > 32768) |
| 676 | return clib_error_return (0, "ring size must be 32768 or lower"); |
| 677 | |
| 678 | if (queue_size == 0) |
| 679 | queue_size = 256; |
| 680 | |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 681 | vec_validate_aligned (vif->cxq_vring, 0, CLIB_CACHE_LINE_BYTES); |
| 682 | vring = vec_elt_at_index (vif->cxq_vring, 0); |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 683 | i = vring_size (queue_size, VIRTIO_PCI_VRING_ALIGN); |
| 684 | i = round_pow2 (i, VIRTIO_PCI_VRING_ALIGN); |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 685 | ptr = |
| 686 | vlib_physmem_alloc_aligned_on_numa (vm, i, VIRTIO_PCI_VRING_ALIGN, |
| 687 | vif->numa_node); |
| 688 | if (!ptr) |
| 689 | return vlib_physmem_last_error (vm); |
| 690 | clib_memset (ptr, 0, i); |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 691 | vring_init (&vr, queue_size, ptr, VIRTIO_PCI_VRING_ALIGN); |
| 692 | vring->desc = vr.desc; |
| 693 | vring->avail = vr.avail; |
| 694 | vring->used = vr.used; |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 695 | vring->queue_id = queue_num; |
| 696 | vring->avail->flags = VIRTIO_RING_FLAG_MASK_INT; |
| 697 | |
| 698 | ASSERT (vring->buffers == 0); |
| 699 | |
| 700 | vring->size = queue_size; |
Damjan Marion | f41244f | 2019-11-08 17:41:06 +0100 | [diff] [blame] | 701 | virtio_log_debug (vif, "control-queue: number %u, size %u", queue_num, |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 702 | queue_size); |
Mohsin Kazmi | a0a6833 | 2020-07-16 12:55:42 +0000 | [diff] [blame] | 703 | vif->virtio_pci_func->setup_queue (vm, vif, queue_num, ptr); |
Mohsin Kazmi | 162a296 | 2020-09-29 10:01:25 +0000 | [diff] [blame] | 704 | vring->queue_notify_offset = |
| 705 | vif->notify_off_multiplier * |
| 706 | vif->virtio_pci_func->get_queue_notify_off (vm, vif, queue_num); |
| 707 | virtio_log_debug (vif, "queue-notify-offset: number %u, offset %u", |
| 708 | queue_num, vring->queue_notify_offset); |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 709 | |
| 710 | return error; |
| 711 | } |
| 712 | |
| 713 | clib_error_t * |
Mohsin Kazmi | b977d3f | 2020-11-16 16:49:30 +0100 | [diff] [blame] | 714 | virtio_pci_control_vring_init (vlib_main_t * vm, virtio_if_t * vif, |
| 715 | u16 queue_num) |
| 716 | { |
| 717 | if (vif->is_packed) |
| 718 | return virtio_pci_control_vring_packed_init (vm, vif, queue_num); |
| 719 | else |
| 720 | return virtio_pci_control_vring_split_init (vm, vif, queue_num); |
| 721 | } |
| 722 | |
| 723 | clib_error_t * |
| 724 | virtio_pci_vring_split_init (vlib_main_t * vm, virtio_if_t * vif, |
| 725 | u16 queue_num) |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 726 | { |
Benoît Ganne | b6b484d | 2020-09-15 10:58:07 +0200 | [diff] [blame] | 727 | vlib_thread_main_t *vtm = vlib_get_thread_main (); |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 728 | clib_error_t *error = 0; |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 729 | u16 queue_size = 0; |
| 730 | virtio_vring_t *vring; |
Mohsin Kazmi | a7a2281 | 2020-08-31 17:17:16 +0200 | [diff] [blame] | 731 | vring_t vr; |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 732 | u32 i = 0; |
| 733 | void *ptr = NULL; |
| 734 | |
Mohsin Kazmi | a0a6833 | 2020-07-16 12:55:42 +0000 | [diff] [blame] | 735 | queue_size = vif->virtio_pci_func->get_queue_size (vm, vif, queue_num); |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 736 | if (!virtio_pci_queue_size_valid (queue_size)) |
| 737 | clib_warning ("queue size is not valid"); |
| 738 | |
| 739 | if (!is_pow2 (queue_size)) |
| 740 | return clib_error_return (0, "ring size must be power of 2"); |
| 741 | |
| 742 | if (queue_size > 32768) |
| 743 | return clib_error_return (0, "ring size must be 32768 or lower"); |
| 744 | |
| 745 | if (queue_size == 0) |
| 746 | queue_size = 256; |
| 747 | |
| 748 | if (queue_num % 2) |
| 749 | { |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 750 | vec_validate_aligned (vif->txq_vrings, TX_QUEUE_ACCESS (queue_num), |
| 751 | CLIB_CACHE_LINE_BYTES); |
| 752 | vring = vec_elt_at_index (vif->txq_vrings, TX_QUEUE_ACCESS (queue_num)); |
Benoît Ganne | b6b484d | 2020-09-15 10:58:07 +0200 | [diff] [blame] | 753 | if (vif->max_queue_pairs < vtm->n_vlib_mains) |
| 754 | clib_spinlock_init (&vring->lockp); |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 755 | } |
| 756 | else |
| 757 | { |
| 758 | vec_validate_aligned (vif->rxq_vrings, RX_QUEUE_ACCESS (queue_num), |
| 759 | CLIB_CACHE_LINE_BYTES); |
| 760 | vring = vec_elt_at_index (vif->rxq_vrings, RX_QUEUE_ACCESS (queue_num)); |
| 761 | } |
| 762 | i = vring_size (queue_size, VIRTIO_PCI_VRING_ALIGN); |
| 763 | i = round_pow2 (i, VIRTIO_PCI_VRING_ALIGN); |
| 764 | ptr = |
| 765 | vlib_physmem_alloc_aligned_on_numa (vm, i, VIRTIO_PCI_VRING_ALIGN, |
| 766 | vif->numa_node); |
| 767 | if (!ptr) |
| 768 | return vlib_physmem_last_error (vm); |
| 769 | clib_memset (ptr, 0, i); |
| 770 | vring_init (&vr, queue_size, ptr, VIRTIO_PCI_VRING_ALIGN); |
| 771 | vring->desc = vr.desc; |
| 772 | vring->avail = vr.avail; |
| 773 | vring->used = vr.used; |
| 774 | vring->queue_id = queue_num; |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 775 | vring->avail->flags = VIRTIO_RING_FLAG_MASK_INT; |
Mohsin Kazmi | 9e2a785 | 2020-08-13 18:57:26 +0200 | [diff] [blame] | 776 | vring->flow_table = 0; |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 777 | |
| 778 | ASSERT (vring->buffers == 0); |
| 779 | vec_validate_aligned (vring->buffers, queue_size, CLIB_CACHE_LINE_BYTES); |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 780 | if (queue_num % 2) |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 781 | { |
Damjan Marion | f41244f | 2019-11-08 17:41:06 +0100 | [diff] [blame] | 782 | virtio_log_debug (vif, "tx-queue: number %u, size %u", queue_num, |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 783 | queue_size); |
Mohsin Kazmi | 7f6d145 | 2020-02-27 11:49:21 +0100 | [diff] [blame] | 784 | clib_memset_u32 (vring->buffers, ~0, queue_size); |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 785 | } |
| 786 | else |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 787 | { |
Damjan Marion | f41244f | 2019-11-08 17:41:06 +0100 | [diff] [blame] | 788 | virtio_log_debug (vif, "rx-queue: number %u, size %u", queue_num, |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 789 | queue_size); |
| 790 | } |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 791 | vring->size = queue_size; |
Mohsin Kazmi | a0a6833 | 2020-07-16 12:55:42 +0000 | [diff] [blame] | 792 | if (vif->virtio_pci_func->setup_queue (vm, vif, queue_num, ptr)) |
Mohsin Kazmi | 43b512c | 2019-04-30 17:25:26 +0200 | [diff] [blame] | 793 | return clib_error_return (0, "error in queue address setup"); |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 794 | |
Mohsin Kazmi | 162a296 | 2020-09-29 10:01:25 +0000 | [diff] [blame] | 795 | vring->queue_notify_offset = |
| 796 | vif->notify_off_multiplier * |
| 797 | vif->virtio_pci_func->get_queue_notify_off (vm, vif, queue_num); |
| 798 | virtio_log_debug (vif, "queue-notify-offset: number %u, offset %u", |
| 799 | queue_num, vring->queue_notify_offset); |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 800 | return error; |
| 801 | } |
| 802 | |
Mohsin Kazmi | b977d3f | 2020-11-16 16:49:30 +0100 | [diff] [blame] | 803 | clib_error_t * |
| 804 | virtio_pci_vring_packed_init (vlib_main_t * vm, virtio_if_t * vif, |
| 805 | u16 queue_num) |
| 806 | { |
| 807 | vlib_thread_main_t *vtm = vlib_get_thread_main (); |
| 808 | clib_error_t *error = 0; |
| 809 | u16 queue_size = 0; |
| 810 | virtio_vring_t *vring; |
| 811 | u32 i = 0; |
| 812 | void *ptr = NULL; |
| 813 | |
| 814 | queue_size = vif->virtio_pci_func->get_queue_size (vm, vif, queue_num); |
| 815 | |
| 816 | if (queue_size > 32768) |
| 817 | return clib_error_return (0, "ring size must be 32768 or lower"); |
| 818 | |
| 819 | if (queue_size == 0) |
| 820 | queue_size = 256; |
| 821 | |
| 822 | if (queue_num % 2) |
| 823 | { |
| 824 | vec_validate_aligned (vif->txq_vrings, TX_QUEUE_ACCESS (queue_num), |
| 825 | CLIB_CACHE_LINE_BYTES); |
| 826 | vring = vec_elt_at_index (vif->txq_vrings, TX_QUEUE_ACCESS (queue_num)); |
| 827 | if (vif->max_queue_pairs < vtm->n_vlib_mains) |
| 828 | clib_spinlock_init (&vring->lockp); |
| 829 | } |
| 830 | else |
| 831 | { |
| 832 | vec_validate_aligned (vif->rxq_vrings, RX_QUEUE_ACCESS (queue_num), |
| 833 | CLIB_CACHE_LINE_BYTES); |
| 834 | vring = vec_elt_at_index (vif->rxq_vrings, RX_QUEUE_ACCESS (queue_num)); |
| 835 | } |
| 836 | |
| 837 | i = |
| 838 | (((queue_size * sizeof (vring_packed_desc_t)) + |
| 839 | sizeof (vring_desc_event_t) + VIRTIO_PCI_VRING_ALIGN - |
| 840 | 1) & ~(VIRTIO_PCI_VRING_ALIGN - 1)) + sizeof (vring_desc_event_t); |
| 841 | |
| 842 | ptr = |
| 843 | vlib_physmem_alloc_aligned_on_numa (vm, i, VIRTIO_PCI_VRING_ALIGN, |
| 844 | vif->numa_node); |
| 845 | if (!ptr) |
| 846 | return vlib_physmem_last_error (vm); |
| 847 | |
| 848 | clib_memset (ptr, 0, i); |
| 849 | vring->packed_desc = ptr; |
| 850 | |
| 851 | vring->driver_event = ptr + (queue_size * sizeof (vring_packed_desc_t)); |
| 852 | vring->driver_event->off_wrap = 0; |
| 853 | vring->driver_event->flags = VRING_EVENT_F_DISABLE; |
| 854 | |
| 855 | vring->device_event = |
| 856 | ptr + |
| 857 | (((queue_size * sizeof (vring_packed_desc_t)) + |
| 858 | sizeof (vring_desc_event_t) + VIRTIO_PCI_VRING_ALIGN - |
| 859 | 1) & ~(VIRTIO_PCI_VRING_ALIGN - 1)); |
| 860 | vring->device_event->off_wrap = 0; |
| 861 | vring->device_event->flags = 0; |
| 862 | |
| 863 | vring->queue_id = queue_num; |
| 864 | |
| 865 | vring->avail_wrap_counter = 1; |
| 866 | vring->used_wrap_counter = 1; |
| 867 | |
| 868 | ASSERT (vring->buffers == 0); |
| 869 | vec_validate_aligned (vring->buffers, queue_size, CLIB_CACHE_LINE_BYTES); |
| 870 | if (queue_num % 2) |
| 871 | { |
| 872 | virtio_log_debug (vif, "tx-queue: number %u, size %u", queue_num, |
| 873 | queue_size); |
| 874 | clib_memset_u32 (vring->buffers, ~0, queue_size); |
| 875 | } |
| 876 | else |
| 877 | { |
| 878 | virtio_log_debug (vif, "rx-queue: number %u, size %u", queue_num, |
| 879 | queue_size); |
| 880 | } |
| 881 | vring->size = queue_size; |
| 882 | if (vif->virtio_pci_func->setup_queue (vm, vif, queue_num, (void *) vring)) |
| 883 | return clib_error_return (0, "error in queue address setup"); |
| 884 | |
| 885 | vring->queue_notify_offset = |
| 886 | vif->notify_off_multiplier * |
| 887 | vif->virtio_pci_func->get_queue_notify_off (vm, vif, queue_num); |
| 888 | virtio_log_debug (vif, "queue-notify-offset: number %u, offset %u", |
| 889 | queue_num, vring->queue_notify_offset); |
| 890 | |
| 891 | return error; |
| 892 | } |
| 893 | |
| 894 | clib_error_t * |
| 895 | virtio_pci_vring_init (vlib_main_t * vm, virtio_if_t * vif, u16 queue_num) |
| 896 | { |
| 897 | if (vif->is_packed) |
| 898 | return virtio_pci_vring_packed_init (vm, vif, queue_num); |
| 899 | else |
| 900 | return virtio_pci_vring_split_init (vm, vif, queue_num); |
| 901 | } |
| 902 | |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 903 | static void |
| 904 | virtio_negotiate_features (vlib_main_t * vm, virtio_if_t * vif, |
| 905 | u64 req_features) |
| 906 | { |
| 907 | /* |
| 908 | * if features are not requested |
| 909 | * default: all supported features |
| 910 | */ |
Mohsin Kazmi | bbd6b74 | 2019-05-02 13:54:59 +0200 | [diff] [blame] | 911 | u64 supported_features = VIRTIO_FEATURE (VIRTIO_NET_F_CSUM) |
| 912 | | VIRTIO_FEATURE (VIRTIO_NET_F_GUEST_CSUM) |
| 913 | | VIRTIO_FEATURE (VIRTIO_NET_F_CTRL_GUEST_OFFLOADS) |
| 914 | | VIRTIO_FEATURE (VIRTIO_NET_F_MTU) |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 915 | | VIRTIO_FEATURE (VIRTIO_NET_F_MAC) |
Mohsin Kazmi | bbd6b74 | 2019-05-02 13:54:59 +0200 | [diff] [blame] | 916 | | VIRTIO_FEATURE (VIRTIO_NET_F_GSO) |
| 917 | | VIRTIO_FEATURE (VIRTIO_NET_F_GUEST_TSO4) |
| 918 | | VIRTIO_FEATURE (VIRTIO_NET_F_GUEST_TSO6) |
| 919 | | VIRTIO_FEATURE (VIRTIO_NET_F_GUEST_UFO) |
| 920 | | VIRTIO_FEATURE (VIRTIO_NET_F_HOST_TSO4) |
| 921 | | VIRTIO_FEATURE (VIRTIO_NET_F_HOST_TSO6) |
| 922 | | VIRTIO_FEATURE (VIRTIO_NET_F_HOST_UFO) |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 923 | | VIRTIO_FEATURE (VIRTIO_NET_F_MRG_RXBUF) |
| 924 | | VIRTIO_FEATURE (VIRTIO_NET_F_STATUS) |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 925 | | VIRTIO_FEATURE (VIRTIO_NET_F_CTRL_VQ) |
| 926 | | VIRTIO_FEATURE (VIRTIO_NET_F_MQ) |
Mohsin Kazmi | b74fe32 | 2019-01-31 13:50:56 +0000 | [diff] [blame] | 927 | | VIRTIO_FEATURE (VIRTIO_F_NOTIFY_ON_EMPTY) |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 928 | | VIRTIO_FEATURE (VIRTIO_F_ANY_LAYOUT) |
| 929 | | VIRTIO_FEATURE (VIRTIO_RING_F_INDIRECT_DESC); |
| 930 | |
Mohsin Kazmi | 379aac3 | 2020-08-20 10:25:12 +0200 | [diff] [blame] | 931 | if (vif->is_modern) |
| 932 | supported_features |= VIRTIO_FEATURE (VIRTIO_F_VERSION_1); |
| 933 | |
Mohsin Kazmi | b977d3f | 2020-11-16 16:49:30 +0100 | [diff] [blame] | 934 | if (vif->is_packed) |
| 935 | { |
| 936 | supported_features |= |
| 937 | (VIRTIO_FEATURE (VIRTIO_F_RING_PACKED) | |
| 938 | VIRTIO_FEATURE (VIRTIO_F_IN_ORDER)); |
| 939 | } |
| 940 | |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 941 | if (req_features == 0) |
| 942 | { |
| 943 | req_features = supported_features; |
| 944 | } |
| 945 | |
| 946 | vif->features = req_features & vif->remote_features & supported_features; |
| 947 | |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 948 | if (vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_MTU)) |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 949 | { |
Mohsin Kazmi | a0a6833 | 2020-07-16 12:55:42 +0000 | [diff] [blame] | 950 | u16 mtu = 0; |
| 951 | mtu = vif->virtio_pci_func->get_mtu (vm, vif); |
| 952 | |
| 953 | if (mtu < 64) |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 954 | vif->features &= ~VIRTIO_FEATURE (VIRTIO_NET_F_MTU); |
| 955 | } |
| 956 | |
Mohsin Kazmi | b977d3f | 2020-11-16 16:49:30 +0100 | [diff] [blame] | 957 | if ((vif->features & (VIRTIO_FEATURE (VIRTIO_F_RING_PACKED))) == 0) |
| 958 | vif->is_packed = 0; |
| 959 | |
Mohsin Kazmi | a0a6833 | 2020-07-16 12:55:42 +0000 | [diff] [blame] | 960 | vif->virtio_pci_func->set_driver_features (vm, vif, vif->features); |
| 961 | vif->features = vif->virtio_pci_func->get_driver_features (vm, vif); |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 962 | } |
| 963 | |
| 964 | void |
| 965 | virtio_pci_read_device_feature (vlib_main_t * vm, virtio_if_t * vif) |
| 966 | { |
Mohsin Kazmi | a0a6833 | 2020-07-16 12:55:42 +0000 | [diff] [blame] | 967 | vif->remote_features = vif->virtio_pci_func->get_device_features (vm, vif); |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 968 | } |
| 969 | |
| 970 | int |
| 971 | virtio_pci_reset_device (vlib_main_t * vm, virtio_if_t * vif) |
| 972 | { |
| 973 | u8 status = 0; |
| 974 | |
| 975 | /* |
| 976 | * Reset the device |
| 977 | */ |
Mohsin Kazmi | a0a6833 | 2020-07-16 12:55:42 +0000 | [diff] [blame] | 978 | status = vif->virtio_pci_func->device_reset (vm, vif); |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 979 | |
| 980 | /* |
| 981 | * Set the Acknowledge status bit |
| 982 | */ |
Mohsin Kazmi | a0a6833 | 2020-07-16 12:55:42 +0000 | [diff] [blame] | 983 | vif->virtio_pci_func->set_status (vm, vif, VIRTIO_CONFIG_STATUS_ACK); |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 984 | |
| 985 | /* |
| 986 | * Set the Driver status bit |
| 987 | */ |
Mohsin Kazmi | a0a6833 | 2020-07-16 12:55:42 +0000 | [diff] [blame] | 988 | vif->virtio_pci_func->set_status (vm, vif, VIRTIO_CONFIG_STATUS_DRIVER); |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 989 | |
| 990 | /* |
| 991 | * Read the status and verify it |
| 992 | */ |
Mohsin Kazmi | a0a6833 | 2020-07-16 12:55:42 +0000 | [diff] [blame] | 993 | status = vif->virtio_pci_func->get_status (vm, vif); |
Mohsin Kazmi | 379aac3 | 2020-08-20 10:25:12 +0200 | [diff] [blame] | 994 | if ((status & VIRTIO_CONFIG_STATUS_ACK) |
| 995 | && (status & VIRTIO_CONFIG_STATUS_DRIVER)) |
| 996 | vif->status = status; |
| 997 | else |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 998 | return -1; |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 999 | |
| 1000 | return 0; |
| 1001 | } |
| 1002 | |
| 1003 | clib_error_t * |
Mohsin Kazmi | 379aac3 | 2020-08-20 10:25:12 +0200 | [diff] [blame] | 1004 | virtio_pci_read_caps (vlib_main_t * vm, virtio_if_t * vif, void **bar) |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1005 | { |
| 1006 | clib_error_t *error = 0; |
Mohsin Kazmi | a7a2281 | 2020-08-31 17:17:16 +0200 | [diff] [blame] | 1007 | virtio_pci_cap_t cap; |
Mohsin Kazmi | 379aac3 | 2020-08-20 10:25:12 +0200 | [diff] [blame] | 1008 | u8 pos, common_cfg = 0, notify = 0, dev_cfg = 0, isr = 0, pci_cfg = 0; |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1009 | vlib_pci_dev_handle_t h = vif->pci_dev_handle; |
| 1010 | |
| 1011 | if ((error = vlib_pci_read_config_u8 (vm, h, PCI_CAPABILITY_LIST, &pos))) |
Mohsin Kazmi | b74fe32 | 2019-01-31 13:50:56 +0000 | [diff] [blame] | 1012 | { |
Damjan Marion | f41244f | 2019-11-08 17:41:06 +0100 | [diff] [blame] | 1013 | virtio_log_error (vif, "error in reading capabilty list position"); |
Mohsin Kazmi | c96e64b | 2020-09-01 17:12:50 +0200 | [diff] [blame] | 1014 | return clib_error_return (error, |
| 1015 | "error in reading capabilty list position"); |
Mohsin Kazmi | b74fe32 | 2019-01-31 13:50:56 +0000 | [diff] [blame] | 1016 | } |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1017 | while (pos) |
| 1018 | { |
| 1019 | if ((error = |
| 1020 | vlib_pci_read_write_config (vm, h, VLIB_READ, pos, &cap, |
| 1021 | sizeof (cap)))) |
Mohsin Kazmi | b74fe32 | 2019-01-31 13:50:56 +0000 | [diff] [blame] | 1022 | { |
Damjan Marion | f41244f | 2019-11-08 17:41:06 +0100 | [diff] [blame] | 1023 | virtio_log_error (vif, "%s [%2x]", |
Mohsin Kazmi | b74fe32 | 2019-01-31 13:50:56 +0000 | [diff] [blame] | 1024 | "error in reading the capability at", pos); |
Mohsin Kazmi | c96e64b | 2020-09-01 17:12:50 +0200 | [diff] [blame] | 1025 | return clib_error_return (error, |
| 1026 | "error in reading the capability at [%2x]", |
| 1027 | pos); |
Mohsin Kazmi | b74fe32 | 2019-01-31 13:50:56 +0000 | [diff] [blame] | 1028 | } |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1029 | |
| 1030 | if (cap.cap_vndr == PCI_CAP_ID_MSIX) |
| 1031 | { |
Mohsin Kazmi | b74fe32 | 2019-01-31 13:50:56 +0000 | [diff] [blame] | 1032 | u16 flags, table_size, table_size_mask = 0x07FF; |
| 1033 | |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1034 | if ((error = |
| 1035 | vlib_pci_read_write_config (vm, h, VLIB_READ, pos + 2, &flags, |
| 1036 | sizeof (flags)))) |
Mohsin Kazmi | c96e64b | 2020-09-01 17:12:50 +0200 | [diff] [blame] | 1037 | return clib_error_return (error, |
| 1038 | "error in reading the capability at [%2x]", |
| 1039 | pos + 2); |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1040 | |
Mohsin Kazmi | b74fe32 | 2019-01-31 13:50:56 +0000 | [diff] [blame] | 1041 | table_size = flags & table_size_mask; |
Damjan Marion | f41244f | 2019-11-08 17:41:06 +0100 | [diff] [blame] | 1042 | virtio_log_debug (vif, "flags:0x%x %s 0x%x", flags, |
Mohsin Kazmi | b74fe32 | 2019-01-31 13:50:56 +0000 | [diff] [blame] | 1043 | "msix interrupt vector table-size", table_size); |
| 1044 | |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1045 | if (flags & PCI_MSIX_ENABLE) |
Mohsin Kazmi | b74fe32 | 2019-01-31 13:50:56 +0000 | [diff] [blame] | 1046 | { |
Damjan Marion | f41244f | 2019-11-08 17:41:06 +0100 | [diff] [blame] | 1047 | virtio_log_debug (vif, "msix interrupt enabled"); |
Mohsin Kazmi | b74fe32 | 2019-01-31 13:50:56 +0000 | [diff] [blame] | 1048 | vif->msix_enabled = VIRTIO_MSIX_ENABLED; |
Mohsin Kazmi | 8046fdc | 2020-07-15 11:54:10 +0000 | [diff] [blame] | 1049 | vif->msix_table_size = table_size; |
Mohsin Kazmi | b74fe32 | 2019-01-31 13:50:56 +0000 | [diff] [blame] | 1050 | } |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1051 | else |
Mohsin Kazmi | b74fe32 | 2019-01-31 13:50:56 +0000 | [diff] [blame] | 1052 | { |
Damjan Marion | f41244f | 2019-11-08 17:41:06 +0100 | [diff] [blame] | 1053 | virtio_log_debug (vif, "msix interrupt disabled"); |
Mohsin Kazmi | b74fe32 | 2019-01-31 13:50:56 +0000 | [diff] [blame] | 1054 | vif->msix_enabled = VIRTIO_MSIX_DISABLED; |
Mohsin Kazmi | 8046fdc | 2020-07-15 11:54:10 +0000 | [diff] [blame] | 1055 | vif->msix_table_size = 0; |
Mohsin Kazmi | b74fe32 | 2019-01-31 13:50:56 +0000 | [diff] [blame] | 1056 | } |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1057 | } |
| 1058 | |
| 1059 | if (cap.cap_vndr != PCI_CAP_ID_VNDR) |
| 1060 | { |
Damjan Marion | f41244f | 2019-11-08 17:41:06 +0100 | [diff] [blame] | 1061 | virtio_log_debug (vif, "[%2x] %s %2x ", pos, |
Mohsin Kazmi | 33cc5cf | 2019-01-21 15:19:39 +0000 | [diff] [blame] | 1062 | "skipping non VNDR cap id:", cap.cap_vndr); |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1063 | goto next; |
| 1064 | } |
| 1065 | |
Damjan Marion | f41244f | 2019-11-08 17:41:06 +0100 | [diff] [blame] | 1066 | virtio_log_debug (vif, |
Mohsin Kazmi | b74fe32 | 2019-01-31 13:50:56 +0000 | [diff] [blame] | 1067 | "[%4x] cfg type: %u, bar: %u, offset: %04x, len: %u", |
| 1068 | pos, cap.cfg_type, cap.bar, cap.offset, cap.length); |
Mohsin Kazmi | 379aac3 | 2020-08-20 10:25:12 +0200 | [diff] [blame] | 1069 | |
Mohsin Kazmi | 0a507d7 | 2020-09-01 18:18:48 +0200 | [diff] [blame] | 1070 | if (cap.bar >= 0 && cap.bar <= 5) |
Mohsin Kazmi | 133c91c1 | 2020-08-20 17:18:56 +0200 | [diff] [blame] | 1071 | { |
| 1072 | vif->bar = bar[cap.bar]; |
| 1073 | vif->bar_id = cap.bar; |
| 1074 | } |
| 1075 | else |
| 1076 | return clib_error_return (error, "invalid bar %u", cap.bar); |
Mohsin Kazmi | 379aac3 | 2020-08-20 10:25:12 +0200 | [diff] [blame] | 1077 | |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1078 | switch (cap.cfg_type) |
| 1079 | { |
| 1080 | case VIRTIO_PCI_CAP_COMMON_CFG: |
Mohsin Kazmi | 379aac3 | 2020-08-20 10:25:12 +0200 | [diff] [blame] | 1081 | vif->common_offset = cap.offset; |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1082 | common_cfg = 1; |
| 1083 | break; |
| 1084 | case VIRTIO_PCI_CAP_NOTIFY_CFG: |
Mohsin Kazmi | 379aac3 | 2020-08-20 10:25:12 +0200 | [diff] [blame] | 1085 | if ((error = |
| 1086 | vlib_pci_read_write_config (vm, h, VLIB_READ, |
| 1087 | pos + sizeof (cap), |
| 1088 | &vif->notify_off_multiplier, |
| 1089 | sizeof |
| 1090 | (vif->notify_off_multiplier)))) |
| 1091 | { |
| 1092 | virtio_log_error (vif, "notify off multiplier is not given"); |
| 1093 | } |
| 1094 | else |
| 1095 | { |
| 1096 | virtio_log_debug (vif, "notify off multiplier is %u", |
| 1097 | vif->notify_off_multiplier); |
| 1098 | vif->notify_offset = cap.offset; |
| 1099 | notify = 1; |
| 1100 | } |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1101 | break; |
| 1102 | case VIRTIO_PCI_CAP_DEVICE_CFG: |
Mohsin Kazmi | 379aac3 | 2020-08-20 10:25:12 +0200 | [diff] [blame] | 1103 | vif->device_offset = cap.offset; |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1104 | dev_cfg = 1; |
| 1105 | break; |
| 1106 | case VIRTIO_PCI_CAP_ISR_CFG: |
Mohsin Kazmi | 379aac3 | 2020-08-20 10:25:12 +0200 | [diff] [blame] | 1107 | vif->isr_offset = cap.offset; |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1108 | isr = 1; |
| 1109 | break; |
Mohsin Kazmi | 22c0ece | 2019-01-28 19:30:21 +0000 | [diff] [blame] | 1110 | case VIRTIO_PCI_CAP_PCI_CFG: |
| 1111 | if (cap.bar == 0) |
| 1112 | pci_cfg = 1; |
| 1113 | break; |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1114 | } |
| 1115 | next: |
| 1116 | pos = cap.cap_next; |
| 1117 | } |
| 1118 | |
Mohsin Kazmi | 379aac3 | 2020-08-20 10:25:12 +0200 | [diff] [blame] | 1119 | if (common_cfg == 0 || notify == 0 || dev_cfg == 0 || isr == 0) |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1120 | { |
Mohsin Kazmi | 379aac3 | 2020-08-20 10:25:12 +0200 | [diff] [blame] | 1121 | vif->virtio_pci_func = &virtio_pci_legacy_func; |
Mohsin Kazmi | 162a296 | 2020-09-29 10:01:25 +0000 | [diff] [blame] | 1122 | vif->notify_off_multiplier = 0; |
Damjan Marion | f41244f | 2019-11-08 17:41:06 +0100 | [diff] [blame] | 1123 | virtio_log_debug (vif, "legacy virtio pci device found"); |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1124 | return error; |
| 1125 | } |
| 1126 | |
Mohsin Kazmi | 379aac3 | 2020-08-20 10:25:12 +0200 | [diff] [blame] | 1127 | vif->is_modern = 1; |
| 1128 | vif->virtio_pci_func = &virtio_pci_modern_func; |
| 1129 | |
Mohsin Kazmi | 0e2bc63 | 2019-01-30 13:36:57 +0000 | [diff] [blame] | 1130 | if (!pci_cfg) |
Mohsin Kazmi | 162a296 | 2020-09-29 10:01:25 +0000 | [diff] [blame] | 1131 | { |
| 1132 | virtio_log_debug (vif, "modern virtio pci device found"); |
| 1133 | } |
| 1134 | else |
| 1135 | { |
| 1136 | virtio_log_debug (vif, "transitional virtio pci device found"); |
| 1137 | } |
Mohsin Kazmi | 0e2bc63 | 2019-01-30 13:36:57 +0000 | [diff] [blame] | 1138 | |
Mohsin Kazmi | 33cc5cf | 2019-01-21 15:19:39 +0000 | [diff] [blame] | 1139 | return error; |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1140 | } |
| 1141 | |
| 1142 | static clib_error_t * |
| 1143 | virtio_pci_device_init (vlib_main_t * vm, virtio_if_t * vif, |
Mohsin Kazmi | 379aac3 | 2020-08-20 10:25:12 +0200 | [diff] [blame] | 1144 | virtio_pci_create_if_args_t * args, void **bar) |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1145 | { |
| 1146 | clib_error_t *error = 0; |
Mohsin Kazmi | edf3b4b | 2019-10-30 15:00:04 +0100 | [diff] [blame] | 1147 | vlib_thread_main_t *vtm = vlib_get_thread_main (); |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1148 | u8 status = 0; |
| 1149 | |
Mohsin Kazmi | 379aac3 | 2020-08-20 10:25:12 +0200 | [diff] [blame] | 1150 | if ((error = virtio_pci_read_caps (vm, vif, bar))) |
Mohsin Kazmi | 57d7321 | 2020-03-05 12:22:41 +0000 | [diff] [blame] | 1151 | { |
| 1152 | args->rv = VNET_API_ERROR_UNSUPPORTED; |
| 1153 | virtio_log_error (vif, "Device is not supported"); |
Mohsin Kazmi | c96e64b | 2020-09-01 17:12:50 +0200 | [diff] [blame] | 1154 | return clib_error_return (error, "Device is not supported"); |
Mohsin Kazmi | 57d7321 | 2020-03-05 12:22:41 +0000 | [diff] [blame] | 1155 | } |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1156 | |
| 1157 | if (virtio_pci_reset_device (vm, vif) < 0) |
Mohsin Kazmi | b74fe32 | 2019-01-31 13:50:56 +0000 | [diff] [blame] | 1158 | { |
Mohsin Kazmi | 57d7321 | 2020-03-05 12:22:41 +0000 | [diff] [blame] | 1159 | args->rv = VNET_API_ERROR_INIT_FAILED; |
Damjan Marion | f41244f | 2019-11-08 17:41:06 +0100 | [diff] [blame] | 1160 | virtio_log_error (vif, "Failed to reset the device"); |
Mohsin Kazmi | c96e64b | 2020-09-01 17:12:50 +0200 | [diff] [blame] | 1161 | return clib_error_return (error, "Failed to reset the device"); |
Mohsin Kazmi | b74fe32 | 2019-01-31 13:50:56 +0000 | [diff] [blame] | 1162 | } |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1163 | /* |
| 1164 | * read device features and negotiate (user) requested features |
| 1165 | */ |
| 1166 | virtio_pci_read_device_feature (vm, vif); |
Mohsin Kazmi | ce21599 | 2020-04-10 14:57:06 +0000 | [diff] [blame] | 1167 | if ((vif->remote_features & VIRTIO_FEATURE (VIRTIO_RING_F_INDIRECT_DESC)) == |
| 1168 | 0) |
| 1169 | { |
| 1170 | virtio_log_warning (vif, "error encountered: vhost-net backend doesn't " |
| 1171 | "support VIRTIO_RING_F_INDIRECT_DESC features"); |
| 1172 | } |
| 1173 | if ((vif->remote_features & VIRTIO_FEATURE (VIRTIO_NET_F_MRG_RXBUF)) == 0) |
| 1174 | { |
| 1175 | virtio_log_warning (vif, "error encountered: vhost-net backend doesn't " |
| 1176 | "support VIRTIO_NET_F_MRG_RXBUF features"); |
| 1177 | } |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1178 | virtio_negotiate_features (vm, vif, args->features); |
| 1179 | |
| 1180 | /* |
| 1181 | * After FEATURE_OK, driver should not accept new feature bits |
| 1182 | */ |
Mohsin Kazmi | a0a6833 | 2020-07-16 12:55:42 +0000 | [diff] [blame] | 1183 | vif->virtio_pci_func->set_status (vm, vif, |
| 1184 | VIRTIO_CONFIG_STATUS_FEATURES_OK); |
| 1185 | status = vif->virtio_pci_func->get_status (vm, vif); |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1186 | if (!(status & VIRTIO_CONFIG_STATUS_FEATURES_OK)) |
Mohsin Kazmi | b74fe32 | 2019-01-31 13:50:56 +0000 | [diff] [blame] | 1187 | { |
Mohsin Kazmi | 57d7321 | 2020-03-05 12:22:41 +0000 | [diff] [blame] | 1188 | args->rv = VNET_API_ERROR_UNSUPPORTED; |
Damjan Marion | f41244f | 2019-11-08 17:41:06 +0100 | [diff] [blame] | 1189 | virtio_log_error (vif, |
Mohsin Kazmi | b74fe32 | 2019-01-31 13:50:56 +0000 | [diff] [blame] | 1190 | "error encountered: Device doesn't support requested features"); |
Mohsin Kazmi | c96e64b | 2020-09-01 17:12:50 +0200 | [diff] [blame] | 1191 | return clib_error_return (error, |
| 1192 | "Device doesn't support requested features"); |
Mohsin Kazmi | b74fe32 | 2019-01-31 13:50:56 +0000 | [diff] [blame] | 1193 | } |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1194 | vif->status = status; |
| 1195 | |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 1196 | /* |
| 1197 | * get or set the mac address |
| 1198 | */ |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1199 | if (virtio_pci_get_mac (vm, vif)) |
| 1200 | { |
| 1201 | f64 now = vlib_time_now (vm); |
| 1202 | u32 rnd; |
| 1203 | rnd = (u32) (now * 1e6); |
| 1204 | rnd = random_u32 (&rnd); |
| 1205 | |
| 1206 | memcpy (vif->mac_addr + 2, &rnd, sizeof (rnd)); |
| 1207 | vif->mac_addr[0] = 2; |
| 1208 | vif->mac_addr[1] = 0xfe; |
| 1209 | virtio_pci_set_mac (vm, vif); |
| 1210 | } |
| 1211 | |
| 1212 | virtio_set_net_hdr_size (vif); |
| 1213 | |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 1214 | /* |
| 1215 | * Initialize the virtqueues |
| 1216 | */ |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1217 | if ((error = virtio_pci_get_max_virtqueue_pairs (vm, vif))) |
Mohsin Kazmi | 57d7321 | 2020-03-05 12:22:41 +0000 | [diff] [blame] | 1218 | { |
| 1219 | args->rv = VNET_API_ERROR_EXCEEDED_NUMBER_OF_RANGES_CAPACITY; |
| 1220 | goto err; |
| 1221 | } |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1222 | |
Mohsin Kazmi | 8046fdc | 2020-07-15 11:54:10 +0000 | [diff] [blame] | 1223 | if (vif->msix_enabled == VIRTIO_MSIX_ENABLED) |
| 1224 | { |
| 1225 | if (vif->msix_table_size <= vif->max_queue_pairs) |
| 1226 | { |
| 1227 | virtio_log_error (vif, |
| 1228 | "error MSIX lines (%u) <= Number of RXQs (%u)", |
| 1229 | vif->msix_table_size, vif->max_queue_pairs); |
| 1230 | return clib_error_return (error, |
| 1231 | "error MSIX lines (%u) <= Number of RXQs (%u)", |
| 1232 | vif->msix_table_size, |
| 1233 | vif->max_queue_pairs); |
| 1234 | } |
| 1235 | } |
| 1236 | |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 1237 | for (int i = 0; i < vif->max_queue_pairs; i++) |
| 1238 | { |
| 1239 | if ((error = virtio_pci_vring_init (vm, vif, RX_QUEUE (i)))) |
Mohsin Kazmi | 43b512c | 2019-04-30 17:25:26 +0200 | [diff] [blame] | 1240 | { |
Mohsin Kazmi | 57d7321 | 2020-03-05 12:22:41 +0000 | [diff] [blame] | 1241 | args->rv = VNET_API_ERROR_INIT_FAILED; |
| 1242 | virtio_log_error (vif, "%s (%u) %s", "error in rxq-queue", |
| 1243 | RX_QUEUE (i), "initialization"); |
Mohsin Kazmi | a0a6833 | 2020-07-16 12:55:42 +0000 | [diff] [blame] | 1244 | error = |
| 1245 | clib_error_return (error, "%s (%u) %s", "error in rxq-queue", |
| 1246 | RX_QUEUE (i), "initialization"); |
| 1247 | goto err; |
Mohsin Kazmi | 43b512c | 2019-04-30 17:25:26 +0200 | [diff] [blame] | 1248 | } |
| 1249 | else |
| 1250 | { |
| 1251 | vif->num_rxqs++; |
| 1252 | } |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1253 | |
Mohsin Kazmi | edf3b4b | 2019-10-30 15:00:04 +0100 | [diff] [blame] | 1254 | if (i >= vtm->n_vlib_mains) |
| 1255 | { |
| 1256 | /* |
| 1257 | * There is 1:1 mapping between tx queue and vpp worker thread. |
| 1258 | * tx queue 0 is bind with thread index 0, tx queue 1 on thread |
| 1259 | * index 1 and so on. |
| 1260 | * Multiple worker threads can poll same tx queue when number of |
| 1261 | * workers are more than tx queues. In this case, 1:N mapping |
| 1262 | * between tx queue and vpp worker thread. |
| 1263 | */ |
| 1264 | virtio_log_debug (vif, "%s %u, %s", "tx-queue: number", |
| 1265 | TX_QUEUE (i), |
| 1266 | "no VPP worker thread is available"); |
| 1267 | continue; |
| 1268 | } |
| 1269 | |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 1270 | if ((error = virtio_pci_vring_init (vm, vif, TX_QUEUE (i)))) |
Mohsin Kazmi | 43b512c | 2019-04-30 17:25:26 +0200 | [diff] [blame] | 1271 | { |
Mohsin Kazmi | 57d7321 | 2020-03-05 12:22:41 +0000 | [diff] [blame] | 1272 | args->rv = VNET_API_ERROR_INIT_FAILED; |
| 1273 | virtio_log_error (vif, "%s (%u) %s", "error in txq-queue", |
| 1274 | TX_QUEUE (i), "initialization"); |
Mohsin Kazmi | a0a6833 | 2020-07-16 12:55:42 +0000 | [diff] [blame] | 1275 | error = |
| 1276 | clib_error_return (error, "%s (%u) %s", "error in txq-queue", |
| 1277 | TX_QUEUE (i), "initialization"); |
| 1278 | goto err; |
Mohsin Kazmi | 43b512c | 2019-04-30 17:25:26 +0200 | [diff] [blame] | 1279 | } |
| 1280 | else |
| 1281 | { |
| 1282 | vif->num_txqs++; |
| 1283 | } |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 1284 | } |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1285 | |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 1286 | if (vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_CTRL_VQ)) |
| 1287 | { |
| 1288 | if ((error = |
| 1289 | virtio_pci_control_vring_init (vm, vif, vif->max_queue_pairs * 2))) |
| 1290 | { |
Damjan Marion | f41244f | 2019-11-08 17:41:06 +0100 | [diff] [blame] | 1291 | virtio_log_warning (vif, "%s (%u) %s", "error in control-queue", |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 1292 | vif->max_queue_pairs * 2, "initialization"); |
| 1293 | if (vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_MQ)) |
| 1294 | vif->features &= ~VIRTIO_FEATURE (VIRTIO_NET_F_MQ); |
| 1295 | } |
| 1296 | } |
| 1297 | else |
| 1298 | { |
Damjan Marion | f41244f | 2019-11-08 17:41:06 +0100 | [diff] [blame] | 1299 | virtio_log_debug (vif, "control queue is not available"); |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 1300 | vif->cxq_vring = NULL; |
| 1301 | } |
| 1302 | |
| 1303 | /* |
| 1304 | * set the msix interrupts |
| 1305 | */ |
Mohsin Kazmi | b74fe32 | 2019-01-31 13:50:56 +0000 | [diff] [blame] | 1306 | if (vif->msix_enabled == VIRTIO_MSIX_ENABLED) |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1307 | { |
Mohsin Kazmi | 8046fdc | 2020-07-15 11:54:10 +0000 | [diff] [blame] | 1308 | int i, j; |
Mohsin Kazmi | a0a6833 | 2020-07-16 12:55:42 +0000 | [diff] [blame] | 1309 | if (vif->virtio_pci_func->set_config_irq (vm, vif, 0) == |
Mohsin Kazmi | b74fe32 | 2019-01-31 13:50:56 +0000 | [diff] [blame] | 1310 | VIRTIO_MSI_NO_VECTOR) |
Mohsin Kazmi | 8046fdc | 2020-07-15 11:54:10 +0000 | [diff] [blame] | 1311 | { |
| 1312 | virtio_log_warning (vif, "config vector 0 is not set"); |
| 1313 | } |
| 1314 | else |
| 1315 | { |
| 1316 | virtio_log_debug (vif, "config msix vector is set at 0"); |
| 1317 | } |
| 1318 | for (i = 0, j = 1; i < vif->max_queue_pairs; i++, j++) |
| 1319 | { |
Mohsin Kazmi | a0a6833 | 2020-07-16 12:55:42 +0000 | [diff] [blame] | 1320 | if (vif->virtio_pci_func->set_queue_irq (vm, vif, j, |
| 1321 | RX_QUEUE (i)) == |
Mohsin Kazmi | 8046fdc | 2020-07-15 11:54:10 +0000 | [diff] [blame] | 1322 | VIRTIO_MSI_NO_VECTOR) |
| 1323 | { |
| 1324 | virtio_log_warning (vif, "queue (%u) vector is not set at %u", |
| 1325 | RX_QUEUE (i), j); |
| 1326 | } |
| 1327 | else |
| 1328 | { |
| 1329 | virtio_log_debug (vif, "%s (%u) %s %u", "queue", |
| 1330 | RX_QUEUE (i), "msix vector is set at", j); |
| 1331 | } |
| 1332 | } |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1333 | } |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 1334 | |
| 1335 | /* |
| 1336 | * set the driver status OK |
| 1337 | */ |
Mohsin Kazmi | a0a6833 | 2020-07-16 12:55:42 +0000 | [diff] [blame] | 1338 | vif->virtio_pci_func->set_status (vm, vif, VIRTIO_CONFIG_STATUS_DRIVER_OK); |
| 1339 | vif->status = vif->virtio_pci_func->get_status (vm, vif); |
Mohsin Kazmi | 46f877e | 2019-04-08 10:36:18 +0200 | [diff] [blame] | 1340 | err: |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1341 | return error; |
| 1342 | } |
| 1343 | |
| 1344 | void |
| 1345 | virtio_pci_create_if (vlib_main_t * vm, virtio_pci_create_if_args_t * args) |
| 1346 | { |
| 1347 | vnet_main_t *vnm = vnet_get_main (); |
Mohsin Kazmi | 33cc5cf | 2019-01-21 15:19:39 +0000 | [diff] [blame] | 1348 | virtio_main_t *vim = &virtio_main; |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1349 | virtio_if_t *vif; |
| 1350 | vlib_pci_dev_handle_t h; |
| 1351 | clib_error_t *error = 0; |
Mohsin Kazmi | 8046fdc | 2020-07-15 11:54:10 +0000 | [diff] [blame] | 1352 | u32 interrupt_count = 0; |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1353 | |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1354 | /* *INDENT-OFF* */ |
Damjan Marion | b2c31b6 | 2020-12-13 21:47:40 +0100 | [diff] [blame] | 1355 | pool_foreach (vif, vim->interfaces) { |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1356 | if (vif->pci_addr.as_u32 == args->addr) |
| 1357 | { |
Mohsin Kazmi | 57d7321 | 2020-03-05 12:22:41 +0000 | [diff] [blame] | 1358 | args->rv = VNET_API_ERROR_ADDRESS_IN_USE; |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1359 | args->error = |
| 1360 | clib_error_return (error, "PCI address in use"); |
Mohsin Kazmi | 33cc5cf | 2019-01-21 15:19:39 +0000 | [diff] [blame] | 1361 | vlib_log (VLIB_LOG_LEVEL_ERR, vim->log_default, "%U: %s", |
| 1362 | format_vlib_pci_addr, &args->addr, |
| 1363 | " PCI address in use"); |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1364 | return; |
| 1365 | } |
Damjan Marion | b2c31b6 | 2020-12-13 21:47:40 +0100 | [diff] [blame] | 1366 | } |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1367 | /* *INDENT-ON* */ |
| 1368 | |
Mohsin Kazmi | 33cc5cf | 2019-01-21 15:19:39 +0000 | [diff] [blame] | 1369 | pool_get (vim->interfaces, vif); |
| 1370 | vif->dev_instance = vif - vim->interfaces; |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1371 | vif->per_interface_next_index = ~0; |
| 1372 | vif->pci_addr.as_u32 = args->addr; |
Mohsin Kazmi | b977d3f | 2020-11-16 16:49:30 +0100 | [diff] [blame] | 1373 | if (args->virtio_flags & VIRTIO_FLAG_PACKED) |
| 1374 | vif->is_packed = 1; |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1375 | |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1376 | if ((error = |
| 1377 | vlib_pci_device_open (vm, (vlib_pci_addr_t *) & vif->pci_addr, |
| 1378 | virtio_pci_device_ids, &h))) |
| 1379 | { |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1380 | args->rv = VNET_API_ERROR_INVALID_INTERFACE; |
| 1381 | args->error = |
| 1382 | clib_error_return (error, "pci-addr %U", format_vlib_pci_addr, |
| 1383 | &vif->pci_addr); |
Mohsin Kazmi | 33cc5cf | 2019-01-21 15:19:39 +0000 | [diff] [blame] | 1384 | vlib_log (VLIB_LOG_LEVEL_ERR, vim->log_default, "%U: %s", |
| 1385 | format_vlib_pci_addr, &vif->pci_addr, |
| 1386 | "error encountered on pci device open"); |
Benoît Ganne | 8f01183 | 2019-11-20 14:32:38 +0100 | [diff] [blame] | 1387 | pool_put (vim->interfaces, vif); |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1388 | return; |
| 1389 | } |
| 1390 | vif->pci_dev_handle = h; |
| 1391 | vlib_pci_set_private_data (vm, h, vif->dev_instance); |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 1392 | vif->numa_node = vlib_pci_get_numa_node (vm, h); |
Damjan Marion | f41244f | 2019-11-08 17:41:06 +0100 | [diff] [blame] | 1393 | vif->type = VIRTIO_IF_TYPE_PCI; |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1394 | |
| 1395 | if ((error = vlib_pci_bus_master_enable (vm, h))) |
Mohsin Kazmi | 33cc5cf | 2019-01-21 15:19:39 +0000 | [diff] [blame] | 1396 | { |
Damjan Marion | f41244f | 2019-11-08 17:41:06 +0100 | [diff] [blame] | 1397 | virtio_log_error (vif, "error encountered on pci bus master enable"); |
Mohsin Kazmi | 33cc5cf | 2019-01-21 15:19:39 +0000 | [diff] [blame] | 1398 | goto error; |
| 1399 | } |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1400 | |
Mohsin Kazmi | 379aac3 | 2020-08-20 10:25:12 +0200 | [diff] [blame] | 1401 | void *bar[6]; |
| 1402 | for (u32 i = 0; i <= 5; i++) |
| 1403 | { |
| 1404 | |
| 1405 | if ((error = vlib_pci_map_region (vm, h, i, &bar[i]))) |
| 1406 | { |
| 1407 | virtio_log_debug (vif, "no pci map region for bar %u", i); |
| 1408 | } |
| 1409 | else |
| 1410 | { |
| 1411 | virtio_log_debug (vif, "pci map region for bar %u at %p", i, |
| 1412 | bar[i]); |
| 1413 | } |
| 1414 | } |
| 1415 | |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1416 | if ((error = vlib_pci_io_region (vm, h, 0))) |
Mohsin Kazmi | 33cc5cf | 2019-01-21 15:19:39 +0000 | [diff] [blame] | 1417 | { |
Damjan Marion | f41244f | 2019-11-08 17:41:06 +0100 | [diff] [blame] | 1418 | virtio_log_error (vif, "error encountered on pci io region"); |
Mohsin Kazmi | 33cc5cf | 2019-01-21 15:19:39 +0000 | [diff] [blame] | 1419 | goto error; |
| 1420 | } |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1421 | |
Mohsin Kazmi | 8046fdc | 2020-07-15 11:54:10 +0000 | [diff] [blame] | 1422 | interrupt_count = vlib_pci_get_num_msix_interrupts (vm, h); |
| 1423 | if (interrupt_count > 1) |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1424 | { |
| 1425 | if ((error = vlib_pci_register_msix_handler (vm, h, 0, 1, |
Mohsin Kazmi | 8046fdc | 2020-07-15 11:54:10 +0000 | [diff] [blame] | 1426 | &virtio_pci_irq_config_handler))) |
Mohsin Kazmi | 33cc5cf | 2019-01-21 15:19:39 +0000 | [diff] [blame] | 1427 | { |
Mohsin Kazmi | 57d7321 | 2020-03-05 12:22:41 +0000 | [diff] [blame] | 1428 | args->rv = VNET_API_ERROR_INVALID_REGISTRATION; |
Damjan Marion | f41244f | 2019-11-08 17:41:06 +0100 | [diff] [blame] | 1429 | virtio_log_error (vif, |
Mohsin Kazmi | 33cc5cf | 2019-01-21 15:19:39 +0000 | [diff] [blame] | 1430 | "error encountered on pci register msix handler 0"); |
| 1431 | goto error; |
| 1432 | } |
Mohsin Kazmi | 8046fdc | 2020-07-15 11:54:10 +0000 | [diff] [blame] | 1433 | |
| 1434 | if ((error = |
| 1435 | vlib_pci_register_msix_handler (vm, h, 1, (interrupt_count - 1), |
| 1436 | &virtio_pci_irq_queue_handler))) |
Mohsin Kazmi | 33cc5cf | 2019-01-21 15:19:39 +0000 | [diff] [blame] | 1437 | { |
Mohsin Kazmi | 57d7321 | 2020-03-05 12:22:41 +0000 | [diff] [blame] | 1438 | args->rv = VNET_API_ERROR_INVALID_REGISTRATION; |
Damjan Marion | f41244f | 2019-11-08 17:41:06 +0100 | [diff] [blame] | 1439 | virtio_log_error (vif, |
Mohsin Kazmi | 33cc5cf | 2019-01-21 15:19:39 +0000 | [diff] [blame] | 1440 | "error encountered on pci register msix handler 1"); |
| 1441 | goto error; |
| 1442 | } |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1443 | |
Mohsin Kazmi | 8046fdc | 2020-07-15 11:54:10 +0000 | [diff] [blame] | 1444 | if ((error = vlib_pci_enable_msix_irq (vm, h, 0, interrupt_count))) |
Mohsin Kazmi | 33cc5cf | 2019-01-21 15:19:39 +0000 | [diff] [blame] | 1445 | { |
Damjan Marion | f41244f | 2019-11-08 17:41:06 +0100 | [diff] [blame] | 1446 | virtio_log_error (vif, "error encountered on pci enable msix irq"); |
Mohsin Kazmi | 33cc5cf | 2019-01-21 15:19:39 +0000 | [diff] [blame] | 1447 | goto error; |
| 1448 | } |
Mohsin Kazmi | b74fe32 | 2019-01-31 13:50:56 +0000 | [diff] [blame] | 1449 | vif->support_int_mode = 1; |
Damjan Marion | f41244f | 2019-11-08 17:41:06 +0100 | [diff] [blame] | 1450 | virtio_log_debug (vif, "device supports msix interrupts"); |
Mohsin Kazmi | b74fe32 | 2019-01-31 13:50:56 +0000 | [diff] [blame] | 1451 | } |
Mohsin Kazmi | 8046fdc | 2020-07-15 11:54:10 +0000 | [diff] [blame] | 1452 | else if (interrupt_count == 1) |
Mohsin Kazmi | b74fe32 | 2019-01-31 13:50:56 +0000 | [diff] [blame] | 1453 | { |
| 1454 | /* |
| 1455 | * if msix table-size is 1, fall back to intX. |
| 1456 | */ |
| 1457 | if ((error = |
| 1458 | vlib_pci_register_intx_handler (vm, h, &virtio_pci_irq_handler))) |
| 1459 | { |
Damjan Marion | f41244f | 2019-11-08 17:41:06 +0100 | [diff] [blame] | 1460 | virtio_log_error (vif, |
Mohsin Kazmi | b74fe32 | 2019-01-31 13:50:56 +0000 | [diff] [blame] | 1461 | "error encountered on pci register interrupt handler"); |
| 1462 | goto error; |
| 1463 | } |
| 1464 | vif->support_int_mode = 1; |
Damjan Marion | f41244f | 2019-11-08 17:41:06 +0100 | [diff] [blame] | 1465 | virtio_log_debug (vif, "pci register interrupt handler"); |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1466 | } |
| 1467 | else |
| 1468 | { |
Mohsin Kazmi | b74fe32 | 2019-01-31 13:50:56 +0000 | [diff] [blame] | 1469 | /* |
| 1470 | * WARN: intX is showing some weird behaviour. |
| 1471 | * Please don't use interrupt mode with UIO driver. |
| 1472 | */ |
| 1473 | vif->support_int_mode = 0; |
Damjan Marion | f41244f | 2019-11-08 17:41:06 +0100 | [diff] [blame] | 1474 | virtio_log_debug (vif, "driver is configured in poll mode only"); |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1475 | } |
| 1476 | |
| 1477 | if ((error = vlib_pci_intr_enable (vm, h))) |
Mohsin Kazmi | 33cc5cf | 2019-01-21 15:19:39 +0000 | [diff] [blame] | 1478 | { |
Damjan Marion | f41244f | 2019-11-08 17:41:06 +0100 | [diff] [blame] | 1479 | virtio_log_error (vif, "error encountered on pci interrupt enable"); |
Mohsin Kazmi | 33cc5cf | 2019-01-21 15:19:39 +0000 | [diff] [blame] | 1480 | goto error; |
| 1481 | } |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1482 | |
Mohsin Kazmi | 379aac3 | 2020-08-20 10:25:12 +0200 | [diff] [blame] | 1483 | if ((error = virtio_pci_device_init (vm, vif, args, bar))) |
Mohsin Kazmi | b74fe32 | 2019-01-31 13:50:56 +0000 | [diff] [blame] | 1484 | { |
Damjan Marion | f41244f | 2019-11-08 17:41:06 +0100 | [diff] [blame] | 1485 | virtio_log_error (vif, "error encountered on device init"); |
Mohsin Kazmi | b74fe32 | 2019-01-31 13:50:56 +0000 | [diff] [blame] | 1486 | goto error; |
| 1487 | } |
| 1488 | |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1489 | /* create interface */ |
| 1490 | error = ethernet_register_interface (vnm, virtio_device_class.index, |
| 1491 | vif->dev_instance, vif->mac_addr, |
| 1492 | &vif->hw_if_index, |
| 1493 | virtio_pci_flag_change); |
| 1494 | |
| 1495 | if (error) |
Mohsin Kazmi | 33cc5cf | 2019-01-21 15:19:39 +0000 | [diff] [blame] | 1496 | { |
Mohsin Kazmi | 57d7321 | 2020-03-05 12:22:41 +0000 | [diff] [blame] | 1497 | args->rv = VNET_API_ERROR_INVALID_REGISTRATION; |
Damjan Marion | f41244f | 2019-11-08 17:41:06 +0100 | [diff] [blame] | 1498 | virtio_log_error (vif, |
Mohsin Kazmi | 33cc5cf | 2019-01-21 15:19:39 +0000 | [diff] [blame] | 1499 | "error encountered on ethernet register interface"); |
| 1500 | goto error; |
| 1501 | } |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1502 | |
| 1503 | vnet_sw_interface_t *sw = vnet_get_hw_sw_interface (vnm, vif->hw_if_index); |
| 1504 | vif->sw_if_index = sw->sw_if_index; |
| 1505 | args->sw_if_index = sw->sw_if_index; |
| 1506 | |
| 1507 | vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, vif->hw_if_index); |
| 1508 | hw->flags |= VNET_HW_INTERFACE_FLAG_SUPPORTS_INT_MODE; |
Mohsin Kazmi | e347acb | 2020-09-28 10:26:33 +0000 | [diff] [blame] | 1509 | |
| 1510 | if (args->virtio_flags & VIRTIO_FLAG_BUFFERING) |
| 1511 | { |
| 1512 | error = virtio_set_packet_buffering (vif, args->buffering_size); |
| 1513 | if (error) |
| 1514 | { |
| 1515 | args->rv = VNET_API_ERROR_INIT_FAILED; |
| 1516 | virtio_log_error (vif, |
| 1517 | "error encountered during packet buffering init"); |
| 1518 | goto error; |
| 1519 | } |
| 1520 | } |
| 1521 | |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1522 | vnet_hw_interface_set_input_node (vnm, vif->hw_if_index, |
| 1523 | virtio_input_node.index); |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 1524 | u32 i = 0; |
| 1525 | vec_foreach_index (i, vif->rxq_vrings) |
| 1526 | { |
| 1527 | vnet_hw_interface_assign_rx_thread (vnm, vif->hw_if_index, i, ~0); |
| 1528 | virtio_vring_set_numa_node (vm, vif, RX_QUEUE (i)); |
| 1529 | /* Set default rx mode to POLLING */ |
| 1530 | vnet_hw_interface_set_rx_mode (vnm, vif->hw_if_index, i, |
Damjan Marion | eabd424 | 2020-10-07 20:59:07 +0200 | [diff] [blame] | 1531 | VNET_HW_IF_RX_MODE_POLLING); |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 1532 | } |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1533 | if (virtio_pci_is_link_up (vm, vif) & VIRTIO_NET_S_LINK_UP) |
| 1534 | { |
| 1535 | vif->flags |= VIRTIO_IF_FLAG_ADMIN_UP; |
| 1536 | vnet_hw_interface_set_flags (vnm, vif->hw_if_index, |
| 1537 | VNET_HW_INTERFACE_FLAG_LINK_UP); |
| 1538 | } |
| 1539 | else |
| 1540 | vnet_hw_interface_set_flags (vnm, vif->hw_if_index, 0); |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 1541 | |
Mohsin Kazmi | 6d4af89 | 2020-01-03 15:11:53 +0000 | [diff] [blame] | 1542 | virtio_pci_offloads (vm, vif, args->gso_enabled, |
| 1543 | args->checksum_offload_enabled); |
| 1544 | |
| 1545 | if ((vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_CTRL_VQ)) && |
| 1546 | (vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_MQ))) |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 1547 | { |
Mohsin Kazmi | 6d4af89 | 2020-01-03 15:11:53 +0000 | [diff] [blame] | 1548 | if (virtio_pci_enable_multiqueue (vm, vif, vif->max_queue_pairs)) |
| 1549 | virtio_log_warning (vif, "multiqueue is not set"); |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 1550 | } |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1551 | return; |
| 1552 | |
| 1553 | error: |
| 1554 | virtio_pci_delete_if (vm, vif); |
Mohsin Kazmi | 57d7321 | 2020-03-05 12:22:41 +0000 | [diff] [blame] | 1555 | if (args->rv == 0) |
| 1556 | args->rv = VNET_API_ERROR_INVALID_INTERFACE; |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1557 | args->error = error; |
| 1558 | } |
| 1559 | |
| 1560 | int |
| 1561 | virtio_pci_delete_if (vlib_main_t * vm, virtio_if_t * vif) |
| 1562 | { |
| 1563 | vnet_main_t *vnm = vnet_get_main (); |
Mohsin Kazmi | 33cc5cf | 2019-01-21 15:19:39 +0000 | [diff] [blame] | 1564 | virtio_main_t *vim = &virtio_main; |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1565 | u32 i = 0; |
| 1566 | |
| 1567 | if (vif->type != VIRTIO_IF_TYPE_PCI) |
| 1568 | return VNET_API_ERROR_INVALID_INTERFACE; |
| 1569 | |
| 1570 | vlib_pci_intr_disable (vm, vif->pci_dev_handle); |
| 1571 | |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 1572 | for (i = 0; i < vif->max_queue_pairs; i++) |
| 1573 | { |
Mohsin Kazmi | a0a6833 | 2020-07-16 12:55:42 +0000 | [diff] [blame] | 1574 | vif->virtio_pci_func->del_queue (vm, vif, RX_QUEUE (i)); |
| 1575 | vif->virtio_pci_func->del_queue (vm, vif, TX_QUEUE (i)); |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 1576 | } |
| 1577 | |
| 1578 | if (vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_CTRL_VQ)) |
Mohsin Kazmi | a0a6833 | 2020-07-16 12:55:42 +0000 | [diff] [blame] | 1579 | vif->virtio_pci_func->del_queue (vm, vif, vif->max_queue_pairs * 2); |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1580 | |
Mohsin Kazmi | c79d735 | 2020-09-01 17:45:56 +0200 | [diff] [blame] | 1581 | if (vif->virtio_pci_func) |
| 1582 | vif->virtio_pci_func->device_reset (vm, vif); |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1583 | |
| 1584 | if (vif->hw_if_index) |
| 1585 | { |
| 1586 | vnet_hw_interface_set_flags (vnm, vif->hw_if_index, 0); |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 1587 | vec_foreach_index (i, vif->rxq_vrings) |
| 1588 | { |
| 1589 | vnet_hw_interface_unassign_rx_thread (vnm, vif->hw_if_index, i); |
| 1590 | } |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1591 | ethernet_delete_interface (vnm, vif->hw_if_index); |
| 1592 | } |
| 1593 | |
| 1594 | vlib_pci_device_close (vm, vif->pci_dev_handle); |
| 1595 | |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 1596 | vec_foreach_index (i, vif->rxq_vrings) |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1597 | { |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 1598 | virtio_vring_t *vring = vec_elt_at_index (vif->rxq_vrings, i); |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1599 | if (vring->used) |
| 1600 | { |
Mohsin Kazmi | b977d3f | 2020-11-16 16:49:30 +0100 | [diff] [blame] | 1601 | virtio_free_buffers (vm, vring); |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 1602 | } |
| 1603 | vec_free (vring->buffers); |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 1604 | vlib_physmem_free (vm, vring->desc); |
| 1605 | } |
| 1606 | |
| 1607 | vec_foreach_index (i, vif->txq_vrings) |
| 1608 | { |
| 1609 | virtio_vring_t *vring = vec_elt_at_index (vif->txq_vrings, i); |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 1610 | if (vring->used) |
| 1611 | { |
Mohsin Kazmi | b977d3f | 2020-11-16 16:49:30 +0100 | [diff] [blame] | 1612 | virtio_free_buffers (vm, vring); |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1613 | } |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1614 | vec_free (vring->buffers); |
Mohsin Kazmi | b977d3f | 2020-11-16 16:49:30 +0100 | [diff] [blame] | 1615 | gro_flow_table_free (vring->flow_table); |
| 1616 | virtio_vring_buffering_free (vm, vring->buffering); |
Steven Luong | e558496 | 2019-06-25 13:47:06 -0700 | [diff] [blame] | 1617 | clib_spinlock_free (&vring->lockp); |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1618 | vlib_physmem_free (vm, vring->desc); |
| 1619 | } |
| 1620 | |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 1621 | if (vif->cxq_vring != NULL) |
| 1622 | { |
| 1623 | u16 last = vif->cxq_vring->last_used_idx; |
| 1624 | u16 n_left = vif->cxq_vring->used->idx - last; |
| 1625 | while (n_left) |
| 1626 | { |
| 1627 | last++; |
| 1628 | n_left--; |
| 1629 | } |
| 1630 | |
| 1631 | vif->cxq_vring->last_used_idx = last; |
| 1632 | vlib_physmem_free (vm, vif->cxq_vring->desc); |
| 1633 | } |
| 1634 | |
| 1635 | vec_free (vif->rxq_vrings); |
| 1636 | vec_free (vif->txq_vrings); |
| 1637 | vec_free (vif->cxq_vring); |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1638 | |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1639 | clib_error_free (vif->error); |
| 1640 | memset (vif, 0, sizeof (*vif)); |
Mohsin Kazmi | 33cc5cf | 2019-01-21 15:19:39 +0000 | [diff] [blame] | 1641 | pool_put (vim->interfaces, vif); |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1642 | |
| 1643 | return 0; |
| 1644 | } |
| 1645 | |
Mohsin Kazmi | 6d4af89 | 2020-01-03 15:11:53 +0000 | [diff] [blame] | 1646 | int |
| 1647 | virtio_pci_enable_disable_offloads (vlib_main_t * vm, virtio_if_t * vif, |
| 1648 | int gso_enabled, |
| 1649 | int checksum_offload_enabled, |
| 1650 | int offloads_disabled) |
| 1651 | { |
| 1652 | if (vif->type != VIRTIO_IF_TYPE_PCI) |
| 1653 | return VNET_API_ERROR_INVALID_INTERFACE; |
| 1654 | |
| 1655 | if (gso_enabled) |
| 1656 | virtio_pci_offloads (vm, vif, 1, 0); |
| 1657 | else if (checksum_offload_enabled) |
| 1658 | virtio_pci_offloads (vm, vif, 0, 1); |
| 1659 | else if (offloads_disabled) |
| 1660 | virtio_pci_offloads (vm, vif, 0, 0); |
| 1661 | |
| 1662 | return 0; |
| 1663 | } |
| 1664 | |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1665 | /* |
| 1666 | * fd.io coding-style-patch-verification: ON |
| 1667 | * |
| 1668 | * Local Variables: |
| 1669 | * eval: (c-set-style "gnu") |
| 1670 | * End: |
| 1671 | */ |