Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (c) 2018 Cisco and/or its affiliates. |
| 3 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | * you may not use this file except in compliance with the License. |
| 5 | * You may obtain a copy of the License at: |
| 6 | * |
| 7 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | * |
| 9 | * Unless required by applicable law or agreed to in writing, software |
| 10 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | * See the License for the specific language governing permissions and |
| 13 | * limitations under the License. |
| 14 | */ |
| 15 | |
| 16 | #include <fcntl.h> |
| 17 | #include <sys/ioctl.h> |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 18 | |
| 19 | #include <vppinfra/types.h> |
| 20 | #include <vlib/vlib.h> |
| 21 | #include <vlib/pci/pci.h> |
| 22 | #include <vnet/ethernet/ethernet.h> |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 23 | #include <vnet/ip/ip4_packet.h> |
| 24 | #include <vnet/ip/ip6_packet.h> |
| 25 | #include <vnet/devices/virtio/virtio.h> |
| 26 | #include <vnet/devices/virtio/pci.h> |
Damjan Marion | 9410053 | 2020-11-06 23:25:57 +0100 | [diff] [blame] | 27 | #include <vnet/interface/rx_queue_funcs.h> |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 28 | |
| 29 | #define PCI_VENDOR_ID_VIRTIO 0x1af4 |
| 30 | #define PCI_DEVICE_ID_VIRTIO_NIC 0x1000 |
| 31 | /* Doesn't support modern device */ |
| 32 | #define PCI_DEVICE_ID_VIRTIO_NIC_MODERN 0x1041 |
| 33 | |
| 34 | #define PCI_CAPABILITY_LIST 0x34 |
| 35 | #define PCI_CAP_ID_VNDR 0x09 |
| 36 | #define PCI_CAP_ID_MSIX 0x11 |
| 37 | |
| 38 | #define PCI_MSIX_ENABLE 0x8000 |
| 39 | |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 40 | static pci_device_id_t virtio_pci_device_ids[] = { |
| 41 | { |
| 42 | .vendor_id = PCI_VENDOR_ID_VIRTIO, |
| 43 | .device_id = PCI_DEVICE_ID_VIRTIO_NIC}, |
| 44 | { |
| 45 | .vendor_id = PCI_VENDOR_ID_VIRTIO, |
| 46 | .device_id = PCI_DEVICE_ID_VIRTIO_NIC_MODERN}, |
| 47 | {0}, |
| 48 | }; |
| 49 | |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 50 | static u32 |
| 51 | virtio_pci_flag_change (vnet_main_t * vnm, vnet_hw_interface_t * hw, |
| 52 | u32 flags) |
| 53 | { |
| 54 | return 0; |
| 55 | } |
| 56 | |
| 57 | static clib_error_t * |
| 58 | virtio_pci_get_max_virtqueue_pairs (vlib_main_t * vm, virtio_if_t * vif) |
| 59 | { |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 60 | clib_error_t *error = 0; |
| 61 | u16 max_queue_pairs = 1; |
| 62 | |
| 63 | if (vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_MQ)) |
| 64 | { |
Mohsin Kazmi | a0a6833 | 2020-07-16 12:55:42 +0000 | [diff] [blame] | 65 | max_queue_pairs = vif->virtio_pci_func->get_max_queue_pairs (vm, vif); |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 66 | } |
| 67 | |
Damjan Marion | f41244f | 2019-11-08 17:41:06 +0100 | [diff] [blame] | 68 | virtio_log_debug (vif, "max queue pair is %x", max_queue_pairs); |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 69 | if (max_queue_pairs < 1 || max_queue_pairs > 0x8000) |
Mohsin Kazmi | 57d7321 | 2020-03-05 12:22:41 +0000 | [diff] [blame] | 70 | return clib_error_return (error, "max queue pair is %x," |
| 71 | " should be in range [1, 0x8000]", |
| 72 | max_queue_pairs); |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 73 | |
| 74 | vif->max_queue_pairs = max_queue_pairs; |
| 75 | return error; |
| 76 | } |
| 77 | |
| 78 | static void |
| 79 | virtio_pci_set_mac (vlib_main_t * vm, virtio_if_t * vif) |
| 80 | { |
Mohsin Kazmi | a0a6833 | 2020-07-16 12:55:42 +0000 | [diff] [blame] | 81 | if (vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_MAC)) |
| 82 | vif->virtio_pci_func->set_mac (vm, vif); |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 83 | } |
| 84 | |
| 85 | static u32 |
| 86 | virtio_pci_get_mac (vlib_main_t * vm, virtio_if_t * vif) |
| 87 | { |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 88 | if (vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_MAC)) |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 89 | { |
Mohsin Kazmi | a0a6833 | 2020-07-16 12:55:42 +0000 | [diff] [blame] | 90 | vif->virtio_pci_func->get_mac (vm, vif); |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 91 | return 0; |
| 92 | } |
| 93 | return 1; |
| 94 | } |
| 95 | |
| 96 | static u16 |
| 97 | virtio_pci_is_link_up (vlib_main_t * vm, virtio_if_t * vif) |
| 98 | { |
| 99 | /* |
| 100 | * Minimal driver: assumes link is up |
| 101 | */ |
| 102 | u16 status = 1; |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 103 | if (vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_STATUS)) |
Mohsin Kazmi | a0a6833 | 2020-07-16 12:55:42 +0000 | [diff] [blame] | 104 | status = vif->virtio_pci_func->get_device_status (vm, vif); |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 105 | return status; |
| 106 | } |
| 107 | |
| 108 | static void |
Mohsin Kazmi | 8046fdc | 2020-07-15 11:54:10 +0000 | [diff] [blame] | 109 | virtio_pci_irq_queue_handler (vlib_main_t * vm, vlib_pci_dev_handle_t h, |
| 110 | u16 line) |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 111 | { |
| 112 | vnet_main_t *vnm = vnet_get_main (); |
Mohsin Kazmi | 33cc5cf | 2019-01-21 15:19:39 +0000 | [diff] [blame] | 113 | virtio_main_t *vim = &virtio_main; |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 114 | uword pd = vlib_pci_get_private_data (vm, h); |
Mohsin Kazmi | 33cc5cf | 2019-01-21 15:19:39 +0000 | [diff] [blame] | 115 | virtio_if_t *vif = pool_elt_at_index (vim->interfaces, pd); |
Mohsin Kazmi | 8046fdc | 2020-07-15 11:54:10 +0000 | [diff] [blame] | 116 | line--; |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 117 | u16 qid = line; |
| 118 | |
Damjan Marion | 9410053 | 2020-11-06 23:25:57 +0100 | [diff] [blame] | 119 | virtio_vring_t *vring = vec_elt_at_index (vif->rxq_vrings, qid); |
| 120 | vnet_hw_if_rx_queue_set_int_pending (vnm, vring->queue_index); |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 121 | } |
| 122 | |
| 123 | static void |
Mohsin Kazmi | 8046fdc | 2020-07-15 11:54:10 +0000 | [diff] [blame] | 124 | virtio_pci_irq_config_handler (vlib_main_t * vm, vlib_pci_dev_handle_t h, |
| 125 | u16 line) |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 126 | { |
| 127 | vnet_main_t *vnm = vnet_get_main (); |
Mohsin Kazmi | 33cc5cf | 2019-01-21 15:19:39 +0000 | [diff] [blame] | 128 | virtio_main_t *vim = &virtio_main; |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 129 | uword pd = vlib_pci_get_private_data (vm, h); |
Mohsin Kazmi | 33cc5cf | 2019-01-21 15:19:39 +0000 | [diff] [blame] | 130 | virtio_if_t *vif = pool_elt_at_index (vim->interfaces, pd); |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 131 | |
| 132 | if (virtio_pci_is_link_up (vm, vif) & VIRTIO_NET_S_LINK_UP) |
| 133 | { |
| 134 | vif->flags |= VIRTIO_IF_FLAG_ADMIN_UP; |
| 135 | vnet_hw_interface_set_flags (vnm, vif->hw_if_index, |
| 136 | VNET_HW_INTERFACE_FLAG_LINK_UP); |
| 137 | } |
| 138 | else |
| 139 | { |
| 140 | vif->flags &= ~VIRTIO_IF_FLAG_ADMIN_UP; |
| 141 | vnet_hw_interface_set_flags (vnm, vif->hw_if_index, 0); |
| 142 | } |
| 143 | } |
| 144 | |
| 145 | static void |
| 146 | virtio_pci_irq_handler (vlib_main_t * vm, vlib_pci_dev_handle_t h) |
| 147 | { |
Mohsin Kazmi | 33cc5cf | 2019-01-21 15:19:39 +0000 | [diff] [blame] | 148 | virtio_main_t *vim = &virtio_main; |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 149 | uword pd = vlib_pci_get_private_data (vm, h); |
Mohsin Kazmi | 33cc5cf | 2019-01-21 15:19:39 +0000 | [diff] [blame] | 150 | virtio_if_t *vif = pool_elt_at_index (vim->interfaces, pd); |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 151 | u8 isr = 0; |
| 152 | u16 line = 0; |
| 153 | |
Mohsin Kazmi | a0a6833 | 2020-07-16 12:55:42 +0000 | [diff] [blame] | 154 | isr = vif->virtio_pci_func->get_isr (vm, vif); |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 155 | |
| 156 | /* |
| 157 | * If the lower bit is set: look through the used rings of |
| 158 | * all virtqueues for the device, to see if any progress has |
| 159 | * been made by the device which requires servicing. |
| 160 | */ |
| 161 | if (isr & VIRTIO_PCI_ISR_INTR) |
Mohsin Kazmi | 8046fdc | 2020-07-15 11:54:10 +0000 | [diff] [blame] | 162 | { |
| 163 | for (; line < vif->num_rxqs; line++) |
| 164 | virtio_pci_irq_queue_handler (vm, h, (line + 1)); |
| 165 | } |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 166 | |
| 167 | if (isr & VIRTIO_PCI_ISR_CONFIG) |
Mohsin Kazmi | 8046fdc | 2020-07-15 11:54:10 +0000 | [diff] [blame] | 168 | virtio_pci_irq_config_handler (vm, h, line); |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 169 | } |
| 170 | |
| 171 | inline void |
| 172 | device_status (vlib_main_t * vm, virtio_if_t * vif) |
| 173 | { |
| 174 | struct status_struct |
| 175 | { |
| 176 | u8 bit; |
| 177 | char *str; |
| 178 | }; |
| 179 | struct status_struct *status_entry; |
| 180 | static struct status_struct status_array[] = { |
| 181 | #define _(s,b) { .str = #s, .bit = b, }, |
| 182 | foreach_virtio_config_status_flags |
| 183 | #undef _ |
| 184 | {.str = NULL} |
| 185 | }; |
| 186 | |
| 187 | vlib_cli_output (vm, " status 0x%x", vif->status); |
| 188 | |
| 189 | status_entry = (struct status_struct *) &status_array; |
| 190 | while (status_entry->str) |
| 191 | { |
| 192 | if (vif->status & status_entry->bit) |
| 193 | vlib_cli_output (vm, " %s (%x)", status_entry->str, |
| 194 | status_entry->bit); |
| 195 | status_entry++; |
| 196 | } |
| 197 | } |
| 198 | |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 199 | static int |
Mohsin Kazmi | b977d3f | 2020-11-16 16:49:30 +0100 | [diff] [blame] | 200 | virtio_pci_send_ctrl_msg_packed (vlib_main_t * vm, virtio_if_t * vif, |
| 201 | virtio_ctrl_msg_t * data, u32 len) |
| 202 | { |
| 203 | virtio_vring_t *vring = vif->cxq_vring; |
| 204 | virtio_net_ctrl_ack_t status = VIRTIO_NET_ERR; |
| 205 | virtio_ctrl_msg_t result; |
| 206 | u32 buffer_index; |
| 207 | vlib_buffer_t *b; |
| 208 | u16 used, next; |
| 209 | u16 sz = vring->size; |
| 210 | u16 flags = 0, first_desc_flags = 0; |
| 211 | |
| 212 | used = vring->desc_in_use; |
| 213 | next = vring->desc_next; |
| 214 | vring_packed_desc_t *d = &vring->packed_desc[next]; |
| 215 | |
| 216 | if (vlib_buffer_alloc (vm, &buffer_index, 1)) |
| 217 | b = vlib_get_buffer (vm, buffer_index); |
| 218 | else |
| 219 | return VIRTIO_NET_ERR; |
| 220 | /* |
| 221 | * current_data may not be initialized with 0 and may contain |
| 222 | * previous offset. |
| 223 | */ |
| 224 | b->current_data = 0; |
| 225 | clib_memcpy (vlib_buffer_get_current (b), data, sizeof (virtio_ctrl_msg_t)); |
| 226 | |
| 227 | first_desc_flags = VRING_DESC_F_NEXT; |
| 228 | if (vring->avail_wrap_counter) |
| 229 | { |
| 230 | first_desc_flags |= VRING_DESC_F_AVAIL; |
| 231 | first_desc_flags &= ~VRING_DESC_F_USED; |
| 232 | } |
| 233 | else |
| 234 | { |
| 235 | first_desc_flags &= ~VRING_DESC_F_AVAIL; |
| 236 | first_desc_flags |= VRING_DESC_F_USED; |
| 237 | } |
| 238 | d->addr = vlib_buffer_get_current_pa (vm, b); |
| 239 | d->len = sizeof (virtio_net_ctrl_hdr_t); |
| 240 | d->id = next; |
| 241 | |
| 242 | next++; |
| 243 | if (next >= sz) |
| 244 | { |
| 245 | next = 0; |
| 246 | vring->avail_wrap_counter ^= 1; |
| 247 | } |
| 248 | used++; |
| 249 | |
| 250 | d = &vring->packed_desc[next]; |
| 251 | flags = VRING_DESC_F_NEXT; |
| 252 | if (vring->avail_wrap_counter) |
| 253 | { |
| 254 | flags |= VRING_DESC_F_AVAIL; |
| 255 | flags &= ~VRING_DESC_F_USED; |
| 256 | } |
| 257 | else |
| 258 | { |
| 259 | flags &= ~VRING_DESC_F_AVAIL; |
| 260 | flags |= VRING_DESC_F_USED; |
| 261 | } |
| 262 | d->addr = vlib_buffer_get_current_pa (vm, b) + |
| 263 | STRUCT_OFFSET_OF (virtio_ctrl_msg_t, data); |
| 264 | d->len = len; |
| 265 | d->id = next; |
| 266 | d->flags = flags; |
| 267 | |
| 268 | next++; |
| 269 | if (next >= sz) |
| 270 | { |
| 271 | next = 0; |
| 272 | vring->avail_wrap_counter ^= 1; |
| 273 | } |
| 274 | used++; |
| 275 | |
| 276 | d = &vring->packed_desc[next]; |
| 277 | flags = VRING_DESC_F_WRITE; |
| 278 | if (vring->avail_wrap_counter) |
| 279 | { |
| 280 | flags |= VRING_DESC_F_AVAIL; |
| 281 | flags &= ~VRING_DESC_F_USED; |
| 282 | } |
| 283 | else |
| 284 | { |
| 285 | flags &= ~VRING_DESC_F_AVAIL; |
| 286 | flags |= VRING_DESC_F_USED; |
| 287 | } |
| 288 | d->addr = vlib_buffer_get_current_pa (vm, b) + |
| 289 | STRUCT_OFFSET_OF (virtio_ctrl_msg_t, status); |
| 290 | d->len = sizeof (data->status); |
| 291 | d->id = next; |
| 292 | d->flags = flags; |
| 293 | |
| 294 | next++; |
| 295 | if (next >= sz) |
| 296 | { |
| 297 | next = 0; |
| 298 | vring->avail_wrap_counter ^= 1; |
| 299 | } |
| 300 | used++; |
| 301 | |
| 302 | CLIB_MEMORY_STORE_BARRIER (); |
| 303 | vring->packed_desc[vring->desc_next].flags = first_desc_flags; |
| 304 | vring->desc_next = next; |
| 305 | vring->desc_in_use = used; |
| 306 | CLIB_MEMORY_BARRIER (); |
| 307 | if (vring->device_event->flags != VRING_EVENT_F_DISABLE) |
| 308 | { |
| 309 | virtio_kick (vm, vring, vif); |
| 310 | } |
| 311 | |
| 312 | u16 last = vring->last_used_idx; |
| 313 | d = &vring->packed_desc[last]; |
| 314 | do |
| 315 | { |
| 316 | flags = d->flags; |
| 317 | } |
| 318 | while ((flags & VRING_DESC_F_AVAIL) != (vring->used_wrap_counter << 7) |
| 319 | || (flags & VRING_DESC_F_USED) != (vring->used_wrap_counter << 15)); |
| 320 | |
| 321 | last += 3; |
| 322 | if (last >= vring->size) |
| 323 | { |
| 324 | last = last - vring->size; |
| 325 | vring->used_wrap_counter ^= 1; |
| 326 | } |
| 327 | vring->desc_in_use -= 3; |
| 328 | vring->last_used_idx = last; |
| 329 | |
| 330 | CLIB_MEMORY_BARRIER (); |
| 331 | clib_memcpy (&result, vlib_buffer_get_current (b), |
| 332 | sizeof (virtio_ctrl_msg_t)); |
| 333 | virtio_log_debug (vif, "ctrl-queue: status %u", result.status); |
| 334 | status = result.status; |
| 335 | vlib_buffer_free (vm, &buffer_index, 1); |
| 336 | return status; |
| 337 | } |
| 338 | |
| 339 | static int |
| 340 | virtio_pci_send_ctrl_msg_split (vlib_main_t * vm, virtio_if_t * vif, |
| 341 | virtio_ctrl_msg_t * data, u32 len) |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 342 | { |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 343 | virtio_vring_t *vring = vif->cxq_vring; |
Mohsin Kazmi | a7a2281 | 2020-08-31 17:17:16 +0200 | [diff] [blame] | 344 | virtio_net_ctrl_ack_t status = VIRTIO_NET_ERR; |
| 345 | virtio_ctrl_msg_t result; |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 346 | u32 buffer_index; |
| 347 | vlib_buffer_t *b; |
| 348 | u16 used, next, avail; |
| 349 | u16 sz = vring->size; |
| 350 | u16 mask = sz - 1; |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 351 | |
| 352 | used = vring->desc_in_use; |
| 353 | next = vring->desc_next; |
| 354 | avail = vring->avail->idx; |
Mohsin Kazmi | a7a2281 | 2020-08-31 17:17:16 +0200 | [diff] [blame] | 355 | vring_desc_t *d = &vring->desc[next]; |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 356 | |
| 357 | if (vlib_buffer_alloc (vm, &buffer_index, 1)) |
| 358 | b = vlib_get_buffer (vm, buffer_index); |
| 359 | else |
| 360 | return VIRTIO_NET_ERR; |
| 361 | /* |
| 362 | * current_data may not be initialized with 0 and may contain |
| 363 | * previous offset. |
| 364 | */ |
| 365 | b->current_data = 0; |
Mohsin Kazmi | a7a2281 | 2020-08-31 17:17:16 +0200 | [diff] [blame] | 366 | clib_memcpy (vlib_buffer_get_current (b), data, sizeof (virtio_ctrl_msg_t)); |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 367 | d->flags = VRING_DESC_F_NEXT; |
| 368 | d->addr = vlib_buffer_get_current_pa (vm, b); |
Mohsin Kazmi | a7a2281 | 2020-08-31 17:17:16 +0200 | [diff] [blame] | 369 | d->len = sizeof (virtio_net_ctrl_hdr_t); |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 370 | vring->avail->ring[avail & mask] = next; |
| 371 | avail++; |
| 372 | next = (next + 1) & mask; |
| 373 | d->next = next; |
| 374 | used++; |
| 375 | |
| 376 | d = &vring->desc[next]; |
| 377 | d->flags = VRING_DESC_F_NEXT; |
| 378 | d->addr = vlib_buffer_get_current_pa (vm, b) + |
Mohsin Kazmi | a7a2281 | 2020-08-31 17:17:16 +0200 | [diff] [blame] | 379 | STRUCT_OFFSET_OF (virtio_ctrl_msg_t, data); |
Mohsin Kazmi | 0549378 | 2019-05-01 14:26:17 +0200 | [diff] [blame] | 380 | d->len = len; |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 381 | next = (next + 1) & mask; |
| 382 | d->next = next; |
| 383 | used++; |
| 384 | |
| 385 | d = &vring->desc[next]; |
| 386 | d->flags = VRING_DESC_F_WRITE; |
| 387 | d->addr = vlib_buffer_get_current_pa (vm, b) + |
Mohsin Kazmi | a7a2281 | 2020-08-31 17:17:16 +0200 | [diff] [blame] | 388 | STRUCT_OFFSET_OF (virtio_ctrl_msg_t, status); |
Mohsin Kazmi | 0549378 | 2019-05-01 14:26:17 +0200 | [diff] [blame] | 389 | d->len = sizeof (data->status); |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 390 | next = (next + 1) & mask; |
| 391 | used++; |
| 392 | |
| 393 | CLIB_MEMORY_STORE_BARRIER (); |
| 394 | vring->avail->idx = avail; |
| 395 | vring->desc_next = next; |
| 396 | vring->desc_in_use = used; |
| 397 | |
| 398 | if ((vring->used->flags & VIRTIO_RING_FLAG_MASK_INT) == 0) |
| 399 | { |
| 400 | virtio_kick (vm, vring, vif); |
| 401 | } |
| 402 | |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 403 | u16 last = vring->last_used_idx, n_left = 0; |
| 404 | n_left = vring->used->idx - last; |
| 405 | |
| 406 | while (n_left) |
| 407 | { |
Mohsin Kazmi | a7a2281 | 2020-08-31 17:17:16 +0200 | [diff] [blame] | 408 | vring_used_elem_t *e = &vring->used->ring[last & mask]; |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 409 | u16 slot = e->id; |
| 410 | |
| 411 | d = &vring->desc[slot]; |
| 412 | while (d->flags & VRING_DESC_F_NEXT) |
| 413 | { |
| 414 | used--; |
| 415 | slot = d->next; |
| 416 | d = &vring->desc[slot]; |
| 417 | } |
| 418 | used--; |
| 419 | last++; |
| 420 | n_left--; |
| 421 | } |
| 422 | vring->desc_in_use = used; |
| 423 | vring->last_used_idx = last; |
| 424 | |
| 425 | CLIB_MEMORY_BARRIER (); |
| 426 | clib_memcpy (&result, vlib_buffer_get_current (b), |
Mohsin Kazmi | a7a2281 | 2020-08-31 17:17:16 +0200 | [diff] [blame] | 427 | sizeof (virtio_ctrl_msg_t)); |
Damjan Marion | f41244f | 2019-11-08 17:41:06 +0100 | [diff] [blame] | 428 | virtio_log_debug (vif, "ctrl-queue: status %u", result.status); |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 429 | status = result.status; |
| 430 | vlib_buffer_free (vm, &buffer_index, 1); |
| 431 | return status; |
| 432 | } |
| 433 | |
Mohsin Kazmi | 0549378 | 2019-05-01 14:26:17 +0200 | [diff] [blame] | 434 | static int |
Mohsin Kazmi | b977d3f | 2020-11-16 16:49:30 +0100 | [diff] [blame] | 435 | virtio_pci_send_ctrl_msg (vlib_main_t * vm, virtio_if_t * vif, |
| 436 | virtio_ctrl_msg_t * data, u32 len) |
| 437 | { |
| 438 | if (vif->is_packed) |
| 439 | return virtio_pci_send_ctrl_msg_packed (vm, vif, data, len); |
| 440 | else |
| 441 | return virtio_pci_send_ctrl_msg_split (vm, vif, data, len); |
| 442 | } |
| 443 | |
| 444 | static int |
Mohsin Kazmi | 6d4af89 | 2020-01-03 15:11:53 +0000 | [diff] [blame] | 445 | virtio_pci_disable_offload (vlib_main_t * vm, virtio_if_t * vif) |
| 446 | { |
Mohsin Kazmi | a7a2281 | 2020-08-31 17:17:16 +0200 | [diff] [blame] | 447 | virtio_ctrl_msg_t offload_hdr; |
| 448 | virtio_net_ctrl_ack_t status = VIRTIO_NET_ERR; |
Mohsin Kazmi | 6d4af89 | 2020-01-03 15:11:53 +0000 | [diff] [blame] | 449 | |
| 450 | offload_hdr.ctrl.class = VIRTIO_NET_CTRL_GUEST_OFFLOADS; |
| 451 | offload_hdr.ctrl.cmd = VIRTIO_NET_CTRL_GUEST_OFFLOADS_SET; |
| 452 | offload_hdr.status = VIRTIO_NET_ERR; |
| 453 | u64 offloads = 0ULL; |
| 454 | clib_memcpy (offload_hdr.data, &offloads, sizeof (offloads)); |
| 455 | |
| 456 | status = |
| 457 | virtio_pci_send_ctrl_msg (vm, vif, &offload_hdr, sizeof (offloads)); |
| 458 | virtio_log_debug (vif, "disable offloads"); |
Mohsin Kazmi | a0a6833 | 2020-07-16 12:55:42 +0000 | [diff] [blame] | 459 | vif->remote_features = vif->virtio_pci_func->get_device_features (vm, vif); |
| 460 | vif->virtio_pci_func->get_driver_features (vm, vif); |
Mohsin Kazmi | 6d4af89 | 2020-01-03 15:11:53 +0000 | [diff] [blame] | 461 | return status; |
| 462 | } |
| 463 | |
| 464 | static int |
| 465 | virtio_pci_enable_checksum_offload (vlib_main_t * vm, virtio_if_t * vif) |
| 466 | { |
Mohsin Kazmi | a7a2281 | 2020-08-31 17:17:16 +0200 | [diff] [blame] | 467 | virtio_ctrl_msg_t csum_offload_hdr; |
| 468 | virtio_net_ctrl_ack_t status = VIRTIO_NET_ERR; |
Mohsin Kazmi | 6d4af89 | 2020-01-03 15:11:53 +0000 | [diff] [blame] | 469 | |
| 470 | csum_offload_hdr.ctrl.class = VIRTIO_NET_CTRL_GUEST_OFFLOADS; |
| 471 | csum_offload_hdr.ctrl.cmd = VIRTIO_NET_CTRL_GUEST_OFFLOADS_SET; |
| 472 | csum_offload_hdr.status = VIRTIO_NET_ERR; |
| 473 | u64 offloads = 0ULL; |
| 474 | offloads |= VIRTIO_FEATURE (VIRTIO_NET_F_GUEST_CSUM); |
| 475 | clib_memcpy (csum_offload_hdr.data, &offloads, sizeof (offloads)); |
| 476 | |
| 477 | status = |
| 478 | virtio_pci_send_ctrl_msg (vm, vif, &csum_offload_hdr, sizeof (offloads)); |
| 479 | virtio_log_debug (vif, "enable checksum offload"); |
Mohsin Kazmi | a0a6833 | 2020-07-16 12:55:42 +0000 | [diff] [blame] | 480 | vif->remote_features = vif->virtio_pci_func->get_device_features (vm, vif); |
| 481 | vif->features = vif->virtio_pci_func->get_driver_features (vm, vif); |
Mohsin Kazmi | 6d4af89 | 2020-01-03 15:11:53 +0000 | [diff] [blame] | 482 | return status; |
| 483 | } |
| 484 | |
| 485 | static int |
Mohsin Kazmi | bbd6b74 | 2019-05-02 13:54:59 +0200 | [diff] [blame] | 486 | virtio_pci_enable_gso (vlib_main_t * vm, virtio_if_t * vif) |
| 487 | { |
Mohsin Kazmi | a7a2281 | 2020-08-31 17:17:16 +0200 | [diff] [blame] | 488 | virtio_ctrl_msg_t gso_hdr; |
| 489 | virtio_net_ctrl_ack_t status = VIRTIO_NET_ERR; |
Mohsin Kazmi | bbd6b74 | 2019-05-02 13:54:59 +0200 | [diff] [blame] | 490 | |
| 491 | gso_hdr.ctrl.class = VIRTIO_NET_CTRL_GUEST_OFFLOADS; |
| 492 | gso_hdr.ctrl.cmd = VIRTIO_NET_CTRL_GUEST_OFFLOADS_SET; |
| 493 | gso_hdr.status = VIRTIO_NET_ERR; |
| 494 | u64 offloads = VIRTIO_FEATURE (VIRTIO_NET_F_GUEST_CSUM) |
| 495 | | VIRTIO_FEATURE (VIRTIO_NET_F_GUEST_TSO4) |
Mohsin Kazmi | 6d4af89 | 2020-01-03 15:11:53 +0000 | [diff] [blame] | 496 | | VIRTIO_FEATURE (VIRTIO_NET_F_GUEST_TSO6); |
Mohsin Kazmi | bbd6b74 | 2019-05-02 13:54:59 +0200 | [diff] [blame] | 497 | clib_memcpy (gso_hdr.data, &offloads, sizeof (offloads)); |
| 498 | |
| 499 | status = virtio_pci_send_ctrl_msg (vm, vif, &gso_hdr, sizeof (offloads)); |
Damjan Marion | f41244f | 2019-11-08 17:41:06 +0100 | [diff] [blame] | 500 | virtio_log_debug (vif, "enable gso"); |
Mohsin Kazmi | a0a6833 | 2020-07-16 12:55:42 +0000 | [diff] [blame] | 501 | vif->remote_features = vif->virtio_pci_func->get_device_features (vm, vif); |
| 502 | vif->virtio_pci_func->get_driver_features (vm, vif); |
Mohsin Kazmi | bbd6b74 | 2019-05-02 13:54:59 +0200 | [diff] [blame] | 503 | return status; |
| 504 | } |
| 505 | |
| 506 | static int |
Mohsin Kazmi | 6d4af89 | 2020-01-03 15:11:53 +0000 | [diff] [blame] | 507 | virtio_pci_offloads (vlib_main_t * vm, virtio_if_t * vif, int gso_enabled, |
| 508 | int csum_offload_enabled) |
| 509 | { |
| 510 | vnet_main_t *vnm = vnet_get_main (); |
| 511 | vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, vif->hw_if_index); |
| 512 | |
| 513 | if ((vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_CTRL_VQ)) && |
| 514 | (vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_CTRL_GUEST_OFFLOADS))) |
| 515 | { |
| 516 | if (gso_enabled |
| 517 | && (vif->features & (VIRTIO_FEATURE (VIRTIO_NET_F_HOST_TSO4) | |
| 518 | VIRTIO_FEATURE (VIRTIO_NET_F_HOST_TSO6)))) |
| 519 | { |
| 520 | if (virtio_pci_enable_gso (vm, vif)) |
| 521 | { |
| 522 | virtio_log_warning (vif, "gso is not enabled"); |
| 523 | } |
| 524 | else |
| 525 | { |
| 526 | vif->gso_enabled = 1; |
| 527 | vif->csum_offload_enabled = 0; |
Mohsin Kazmi | 5b3f523 | 2021-02-10 12:03:53 +0100 | [diff] [blame] | 528 | hw->caps |= VNET_HW_INTERFACE_CAP_SUPPORTS_TCP_GSO | |
| 529 | VNET_HW_INTERFACE_CAP_SUPPORTS_TX_TCP_CKSUM | |
| 530 | VNET_HW_INTERFACE_CAP_SUPPORTS_TX_UDP_CKSUM; |
Mohsin Kazmi | 6d4af89 | 2020-01-03 15:11:53 +0000 | [diff] [blame] | 531 | } |
| 532 | } |
| 533 | else if (csum_offload_enabled |
| 534 | && (vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_CSUM))) |
| 535 | { |
| 536 | if (virtio_pci_enable_checksum_offload (vm, vif)) |
| 537 | { |
| 538 | virtio_log_warning (vif, "checksum offload is not enabled"); |
| 539 | } |
| 540 | else |
| 541 | { |
| 542 | vif->csum_offload_enabled = 1; |
| 543 | vif->gso_enabled = 0; |
Mohsin Kazmi | 5b3f523 | 2021-02-10 12:03:53 +0100 | [diff] [blame] | 544 | hw->caps &= ~VNET_HW_INTERFACE_CAP_SUPPORTS_TCP_GSO; |
| 545 | hw->caps |= VNET_HW_INTERFACE_CAP_SUPPORTS_TX_TCP_CKSUM | |
| 546 | VNET_HW_INTERFACE_CAP_SUPPORTS_TX_UDP_CKSUM; |
Mohsin Kazmi | 6d4af89 | 2020-01-03 15:11:53 +0000 | [diff] [blame] | 547 | } |
| 548 | } |
| 549 | else |
| 550 | { |
| 551 | if (virtio_pci_disable_offload (vm, vif)) |
| 552 | { |
| 553 | virtio_log_warning (vif, "offloads are not disabled"); |
| 554 | } |
| 555 | else |
| 556 | { |
| 557 | vif->csum_offload_enabled = 0; |
| 558 | vif->gso_enabled = 0; |
Mohsin Kazmi | 5b3f523 | 2021-02-10 12:03:53 +0100 | [diff] [blame] | 559 | hw->caps &= ~(VNET_HW_INTERFACE_CAP_SUPPORTS_L4_TX_CKSUM | |
| 560 | VNET_HW_INTERFACE_CAP_SUPPORTS_TCP_GSO); |
Mohsin Kazmi | 6d4af89 | 2020-01-03 15:11:53 +0000 | [diff] [blame] | 561 | } |
| 562 | } |
| 563 | } |
| 564 | |
| 565 | return 0; |
| 566 | } |
| 567 | |
| 568 | static int |
Mohsin Kazmi | 0549378 | 2019-05-01 14:26:17 +0200 | [diff] [blame] | 569 | virtio_pci_enable_multiqueue (vlib_main_t * vm, virtio_if_t * vif, |
| 570 | u16 num_queues) |
| 571 | { |
Mohsin Kazmi | a7a2281 | 2020-08-31 17:17:16 +0200 | [diff] [blame] | 572 | virtio_ctrl_msg_t mq_hdr; |
| 573 | virtio_net_ctrl_ack_t status = VIRTIO_NET_ERR; |
Mohsin Kazmi | 0549378 | 2019-05-01 14:26:17 +0200 | [diff] [blame] | 574 | |
| 575 | mq_hdr.ctrl.class = VIRTIO_NET_CTRL_MQ; |
| 576 | mq_hdr.ctrl.cmd = VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET; |
| 577 | mq_hdr.status = VIRTIO_NET_ERR; |
| 578 | clib_memcpy (mq_hdr.data, &num_queues, sizeof (num_queues)); |
| 579 | |
| 580 | status = virtio_pci_send_ctrl_msg (vm, vif, &mq_hdr, sizeof (num_queues)); |
Damjan Marion | f41244f | 2019-11-08 17:41:06 +0100 | [diff] [blame] | 581 | virtio_log_debug (vif, "multi-queue enable %u queues", num_queues); |
Mohsin Kazmi | 0549378 | 2019-05-01 14:26:17 +0200 | [diff] [blame] | 582 | return status; |
| 583 | } |
| 584 | |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 585 | static u8 |
| 586 | virtio_pci_queue_size_valid (u16 qsz) |
| 587 | { |
| 588 | if (qsz < 64 || qsz > 4096) |
| 589 | return 0; |
| 590 | if ((qsz % 64) != 0) |
| 591 | return 0; |
| 592 | return 1; |
| 593 | } |
| 594 | |
| 595 | clib_error_t * |
Mohsin Kazmi | b977d3f | 2020-11-16 16:49:30 +0100 | [diff] [blame] | 596 | virtio_pci_control_vring_packed_init (vlib_main_t * vm, virtio_if_t * vif, |
| 597 | u16 queue_num) |
| 598 | { |
| 599 | clib_error_t *error = 0; |
| 600 | u16 queue_size = 0; |
| 601 | virtio_vring_t *vring; |
| 602 | u32 i = 0; |
| 603 | void *ptr = NULL; |
| 604 | |
| 605 | queue_size = vif->virtio_pci_func->get_queue_size (vm, vif, queue_num); |
| 606 | |
| 607 | if (queue_size > 32768) |
| 608 | return clib_error_return (0, "ring size must be 32768 or lower"); |
| 609 | |
| 610 | if (queue_size == 0) |
| 611 | queue_size = 256; |
| 612 | |
| 613 | vec_validate_aligned (vif->cxq_vring, 0, CLIB_CACHE_LINE_BYTES); |
| 614 | vring = vec_elt_at_index (vif->cxq_vring, 0); |
| 615 | |
| 616 | i = |
| 617 | (((queue_size * sizeof (vring_packed_desc_t)) + |
| 618 | sizeof (vring_desc_event_t) + VIRTIO_PCI_VRING_ALIGN - |
| 619 | 1) & ~(VIRTIO_PCI_VRING_ALIGN - 1)) + sizeof (vring_desc_event_t); |
| 620 | |
| 621 | ptr = |
| 622 | vlib_physmem_alloc_aligned_on_numa (vm, i, VIRTIO_PCI_VRING_ALIGN, |
| 623 | vif->numa_node); |
| 624 | if (!ptr) |
| 625 | return vlib_physmem_last_error (vm); |
| 626 | clib_memset (ptr, 0, i); |
| 627 | |
| 628 | vring->packed_desc = ptr; |
| 629 | |
| 630 | vring->driver_event = ptr + (queue_size * sizeof (vring_packed_desc_t)); |
| 631 | vring->driver_event->off_wrap = 0; |
| 632 | vring->driver_event->flags = VRING_EVENT_F_DISABLE; |
| 633 | |
| 634 | vring->device_event = |
| 635 | ptr + |
| 636 | (((queue_size * sizeof (vring_packed_desc_t)) + |
| 637 | sizeof (vring_desc_event_t) + VIRTIO_PCI_VRING_ALIGN - |
| 638 | 1) & ~(VIRTIO_PCI_VRING_ALIGN - 1)); |
| 639 | vring->device_event->off_wrap = 0; |
| 640 | vring->device_event->flags = 0; |
| 641 | |
| 642 | vring->queue_id = queue_num; |
| 643 | vring->size = queue_size; |
| 644 | vring->avail_wrap_counter = 1; |
| 645 | vring->used_wrap_counter = 1; |
| 646 | |
| 647 | ASSERT (vring->buffers == 0); |
| 648 | |
| 649 | virtio_log_debug (vif, "control-queue: number %u, size %u", queue_num, |
| 650 | queue_size); |
| 651 | vif->virtio_pci_func->setup_queue (vm, vif, queue_num, (void *) vring); |
| 652 | vring->queue_notify_offset = |
| 653 | vif->notify_off_multiplier * |
| 654 | vif->virtio_pci_func->get_queue_notify_off (vm, vif, queue_num); |
| 655 | virtio_log_debug (vif, "queue-notify-offset: number %u, offset %u", |
| 656 | queue_num, vring->queue_notify_offset); |
| 657 | return error; |
| 658 | } |
| 659 | |
| 660 | clib_error_t * |
| 661 | virtio_pci_control_vring_split_init (vlib_main_t * vm, virtio_if_t * vif, |
| 662 | u16 queue_num) |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 663 | { |
| 664 | clib_error_t *error = 0; |
| 665 | u16 queue_size = 0; |
| 666 | virtio_vring_t *vring; |
Mohsin Kazmi | a7a2281 | 2020-08-31 17:17:16 +0200 | [diff] [blame] | 667 | vring_t vr; |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 668 | u32 i = 0; |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 669 | void *ptr = NULL; |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 670 | |
Mohsin Kazmi | a0a6833 | 2020-07-16 12:55:42 +0000 | [diff] [blame] | 671 | queue_size = vif->virtio_pci_func->get_queue_size (vm, vif, queue_num); |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 672 | if (!virtio_pci_queue_size_valid (queue_size)) |
| 673 | clib_warning ("queue size is not valid"); |
| 674 | |
| 675 | if (!is_pow2 (queue_size)) |
| 676 | return clib_error_return (0, "ring size must be power of 2"); |
| 677 | |
| 678 | if (queue_size > 32768) |
| 679 | return clib_error_return (0, "ring size must be 32768 or lower"); |
| 680 | |
| 681 | if (queue_size == 0) |
| 682 | queue_size = 256; |
| 683 | |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 684 | vec_validate_aligned (vif->cxq_vring, 0, CLIB_CACHE_LINE_BYTES); |
| 685 | vring = vec_elt_at_index (vif->cxq_vring, 0); |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 686 | i = vring_size (queue_size, VIRTIO_PCI_VRING_ALIGN); |
| 687 | i = round_pow2 (i, VIRTIO_PCI_VRING_ALIGN); |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 688 | ptr = |
| 689 | vlib_physmem_alloc_aligned_on_numa (vm, i, VIRTIO_PCI_VRING_ALIGN, |
| 690 | vif->numa_node); |
| 691 | if (!ptr) |
| 692 | return vlib_physmem_last_error (vm); |
| 693 | clib_memset (ptr, 0, i); |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 694 | vring_init (&vr, queue_size, ptr, VIRTIO_PCI_VRING_ALIGN); |
| 695 | vring->desc = vr.desc; |
| 696 | vring->avail = vr.avail; |
| 697 | vring->used = vr.used; |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 698 | vring->queue_id = queue_num; |
| 699 | vring->avail->flags = VIRTIO_RING_FLAG_MASK_INT; |
| 700 | |
| 701 | ASSERT (vring->buffers == 0); |
| 702 | |
| 703 | vring->size = queue_size; |
Damjan Marion | f41244f | 2019-11-08 17:41:06 +0100 | [diff] [blame] | 704 | virtio_log_debug (vif, "control-queue: number %u, size %u", queue_num, |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 705 | queue_size); |
Mohsin Kazmi | a0a6833 | 2020-07-16 12:55:42 +0000 | [diff] [blame] | 706 | vif->virtio_pci_func->setup_queue (vm, vif, queue_num, ptr); |
Mohsin Kazmi | 162a296 | 2020-09-29 10:01:25 +0000 | [diff] [blame] | 707 | vring->queue_notify_offset = |
| 708 | vif->notify_off_multiplier * |
| 709 | vif->virtio_pci_func->get_queue_notify_off (vm, vif, queue_num); |
| 710 | virtio_log_debug (vif, "queue-notify-offset: number %u, offset %u", |
| 711 | queue_num, vring->queue_notify_offset); |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 712 | |
| 713 | return error; |
| 714 | } |
| 715 | |
| 716 | clib_error_t * |
Mohsin Kazmi | b977d3f | 2020-11-16 16:49:30 +0100 | [diff] [blame] | 717 | virtio_pci_control_vring_init (vlib_main_t * vm, virtio_if_t * vif, |
| 718 | u16 queue_num) |
| 719 | { |
| 720 | if (vif->is_packed) |
| 721 | return virtio_pci_control_vring_packed_init (vm, vif, queue_num); |
| 722 | else |
| 723 | return virtio_pci_control_vring_split_init (vm, vif, queue_num); |
| 724 | } |
| 725 | |
| 726 | clib_error_t * |
| 727 | virtio_pci_vring_split_init (vlib_main_t * vm, virtio_if_t * vif, |
| 728 | u16 queue_num) |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 729 | { |
Benoît Ganne | b6b484d | 2020-09-15 10:58:07 +0200 | [diff] [blame] | 730 | vlib_thread_main_t *vtm = vlib_get_thread_main (); |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 731 | clib_error_t *error = 0; |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 732 | u16 queue_size = 0; |
| 733 | virtio_vring_t *vring; |
Mohsin Kazmi | a7a2281 | 2020-08-31 17:17:16 +0200 | [diff] [blame] | 734 | vring_t vr; |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 735 | u32 i = 0; |
| 736 | void *ptr = NULL; |
| 737 | |
Mohsin Kazmi | a0a6833 | 2020-07-16 12:55:42 +0000 | [diff] [blame] | 738 | queue_size = vif->virtio_pci_func->get_queue_size (vm, vif, queue_num); |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 739 | if (!virtio_pci_queue_size_valid (queue_size)) |
| 740 | clib_warning ("queue size is not valid"); |
| 741 | |
| 742 | if (!is_pow2 (queue_size)) |
| 743 | return clib_error_return (0, "ring size must be power of 2"); |
| 744 | |
| 745 | if (queue_size > 32768) |
| 746 | return clib_error_return (0, "ring size must be 32768 or lower"); |
| 747 | |
| 748 | if (queue_size == 0) |
| 749 | queue_size = 256; |
| 750 | |
| 751 | if (queue_num % 2) |
| 752 | { |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 753 | vec_validate_aligned (vif->txq_vrings, TX_QUEUE_ACCESS (queue_num), |
| 754 | CLIB_CACHE_LINE_BYTES); |
| 755 | vring = vec_elt_at_index (vif->txq_vrings, TX_QUEUE_ACCESS (queue_num)); |
Benoît Ganne | b6b484d | 2020-09-15 10:58:07 +0200 | [diff] [blame] | 756 | if (vif->max_queue_pairs < vtm->n_vlib_mains) |
| 757 | clib_spinlock_init (&vring->lockp); |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 758 | } |
| 759 | else |
| 760 | { |
| 761 | vec_validate_aligned (vif->rxq_vrings, RX_QUEUE_ACCESS (queue_num), |
| 762 | CLIB_CACHE_LINE_BYTES); |
| 763 | vring = vec_elt_at_index (vif->rxq_vrings, RX_QUEUE_ACCESS (queue_num)); |
| 764 | } |
| 765 | i = vring_size (queue_size, VIRTIO_PCI_VRING_ALIGN); |
| 766 | i = round_pow2 (i, VIRTIO_PCI_VRING_ALIGN); |
| 767 | ptr = |
| 768 | vlib_physmem_alloc_aligned_on_numa (vm, i, VIRTIO_PCI_VRING_ALIGN, |
| 769 | vif->numa_node); |
| 770 | if (!ptr) |
| 771 | return vlib_physmem_last_error (vm); |
| 772 | clib_memset (ptr, 0, i); |
| 773 | vring_init (&vr, queue_size, ptr, VIRTIO_PCI_VRING_ALIGN); |
| 774 | vring->desc = vr.desc; |
| 775 | vring->avail = vr.avail; |
| 776 | vring->used = vr.used; |
| 777 | vring->queue_id = queue_num; |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 778 | vring->avail->flags = VIRTIO_RING_FLAG_MASK_INT; |
Mohsin Kazmi | 9e2a785 | 2020-08-13 18:57:26 +0200 | [diff] [blame] | 779 | vring->flow_table = 0; |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 780 | |
| 781 | ASSERT (vring->buffers == 0); |
| 782 | vec_validate_aligned (vring->buffers, queue_size, CLIB_CACHE_LINE_BYTES); |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 783 | if (queue_num % 2) |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 784 | { |
Damjan Marion | f41244f | 2019-11-08 17:41:06 +0100 | [diff] [blame] | 785 | virtio_log_debug (vif, "tx-queue: number %u, size %u", queue_num, |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 786 | queue_size); |
Mohsin Kazmi | 7f6d145 | 2020-02-27 11:49:21 +0100 | [diff] [blame] | 787 | clib_memset_u32 (vring->buffers, ~0, queue_size); |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 788 | } |
| 789 | else |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 790 | { |
Damjan Marion | f41244f | 2019-11-08 17:41:06 +0100 | [diff] [blame] | 791 | virtio_log_debug (vif, "rx-queue: number %u, size %u", queue_num, |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 792 | queue_size); |
| 793 | } |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 794 | vring->size = queue_size; |
Mohsin Kazmi | a0a6833 | 2020-07-16 12:55:42 +0000 | [diff] [blame] | 795 | if (vif->virtio_pci_func->setup_queue (vm, vif, queue_num, ptr)) |
Mohsin Kazmi | 43b512c | 2019-04-30 17:25:26 +0200 | [diff] [blame] | 796 | return clib_error_return (0, "error in queue address setup"); |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 797 | |
Mohsin Kazmi | 162a296 | 2020-09-29 10:01:25 +0000 | [diff] [blame] | 798 | vring->queue_notify_offset = |
| 799 | vif->notify_off_multiplier * |
| 800 | vif->virtio_pci_func->get_queue_notify_off (vm, vif, queue_num); |
| 801 | virtio_log_debug (vif, "queue-notify-offset: number %u, offset %u", |
| 802 | queue_num, vring->queue_notify_offset); |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 803 | return error; |
| 804 | } |
| 805 | |
Mohsin Kazmi | b977d3f | 2020-11-16 16:49:30 +0100 | [diff] [blame] | 806 | clib_error_t * |
| 807 | virtio_pci_vring_packed_init (vlib_main_t * vm, virtio_if_t * vif, |
| 808 | u16 queue_num) |
| 809 | { |
| 810 | vlib_thread_main_t *vtm = vlib_get_thread_main (); |
| 811 | clib_error_t *error = 0; |
| 812 | u16 queue_size = 0; |
| 813 | virtio_vring_t *vring; |
| 814 | u32 i = 0; |
| 815 | void *ptr = NULL; |
| 816 | |
| 817 | queue_size = vif->virtio_pci_func->get_queue_size (vm, vif, queue_num); |
| 818 | |
| 819 | if (queue_size > 32768) |
| 820 | return clib_error_return (0, "ring size must be 32768 or lower"); |
| 821 | |
| 822 | if (queue_size == 0) |
| 823 | queue_size = 256; |
| 824 | |
| 825 | if (queue_num % 2) |
| 826 | { |
| 827 | vec_validate_aligned (vif->txq_vrings, TX_QUEUE_ACCESS (queue_num), |
| 828 | CLIB_CACHE_LINE_BYTES); |
| 829 | vring = vec_elt_at_index (vif->txq_vrings, TX_QUEUE_ACCESS (queue_num)); |
| 830 | if (vif->max_queue_pairs < vtm->n_vlib_mains) |
| 831 | clib_spinlock_init (&vring->lockp); |
| 832 | } |
| 833 | else |
| 834 | { |
| 835 | vec_validate_aligned (vif->rxq_vrings, RX_QUEUE_ACCESS (queue_num), |
| 836 | CLIB_CACHE_LINE_BYTES); |
| 837 | vring = vec_elt_at_index (vif->rxq_vrings, RX_QUEUE_ACCESS (queue_num)); |
| 838 | } |
| 839 | |
| 840 | i = |
| 841 | (((queue_size * sizeof (vring_packed_desc_t)) + |
| 842 | sizeof (vring_desc_event_t) + VIRTIO_PCI_VRING_ALIGN - |
| 843 | 1) & ~(VIRTIO_PCI_VRING_ALIGN - 1)) + sizeof (vring_desc_event_t); |
| 844 | |
| 845 | ptr = |
| 846 | vlib_physmem_alloc_aligned_on_numa (vm, i, VIRTIO_PCI_VRING_ALIGN, |
| 847 | vif->numa_node); |
| 848 | if (!ptr) |
| 849 | return vlib_physmem_last_error (vm); |
| 850 | |
| 851 | clib_memset (ptr, 0, i); |
| 852 | vring->packed_desc = ptr; |
| 853 | |
| 854 | vring->driver_event = ptr + (queue_size * sizeof (vring_packed_desc_t)); |
| 855 | vring->driver_event->off_wrap = 0; |
| 856 | vring->driver_event->flags = VRING_EVENT_F_DISABLE; |
| 857 | |
| 858 | vring->device_event = |
| 859 | ptr + |
| 860 | (((queue_size * sizeof (vring_packed_desc_t)) + |
| 861 | sizeof (vring_desc_event_t) + VIRTIO_PCI_VRING_ALIGN - |
| 862 | 1) & ~(VIRTIO_PCI_VRING_ALIGN - 1)); |
| 863 | vring->device_event->off_wrap = 0; |
| 864 | vring->device_event->flags = 0; |
| 865 | |
| 866 | vring->queue_id = queue_num; |
| 867 | |
| 868 | vring->avail_wrap_counter = 1; |
| 869 | vring->used_wrap_counter = 1; |
| 870 | |
| 871 | ASSERT (vring->buffers == 0); |
| 872 | vec_validate_aligned (vring->buffers, queue_size, CLIB_CACHE_LINE_BYTES); |
| 873 | if (queue_num % 2) |
| 874 | { |
| 875 | virtio_log_debug (vif, "tx-queue: number %u, size %u", queue_num, |
| 876 | queue_size); |
| 877 | clib_memset_u32 (vring->buffers, ~0, queue_size); |
| 878 | } |
| 879 | else |
| 880 | { |
| 881 | virtio_log_debug (vif, "rx-queue: number %u, size %u", queue_num, |
| 882 | queue_size); |
| 883 | } |
| 884 | vring->size = queue_size; |
| 885 | if (vif->virtio_pci_func->setup_queue (vm, vif, queue_num, (void *) vring)) |
| 886 | return clib_error_return (0, "error in queue address setup"); |
| 887 | |
| 888 | vring->queue_notify_offset = |
| 889 | vif->notify_off_multiplier * |
| 890 | vif->virtio_pci_func->get_queue_notify_off (vm, vif, queue_num); |
| 891 | virtio_log_debug (vif, "queue-notify-offset: number %u, offset %u", |
| 892 | queue_num, vring->queue_notify_offset); |
| 893 | |
| 894 | return error; |
| 895 | } |
| 896 | |
| 897 | clib_error_t * |
| 898 | virtio_pci_vring_init (vlib_main_t * vm, virtio_if_t * vif, u16 queue_num) |
| 899 | { |
| 900 | if (vif->is_packed) |
| 901 | return virtio_pci_vring_packed_init (vm, vif, queue_num); |
| 902 | else |
| 903 | return virtio_pci_vring_split_init (vm, vif, queue_num); |
| 904 | } |
| 905 | |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 906 | static void |
| 907 | virtio_negotiate_features (vlib_main_t * vm, virtio_if_t * vif, |
| 908 | u64 req_features) |
| 909 | { |
| 910 | /* |
| 911 | * if features are not requested |
| 912 | * default: all supported features |
| 913 | */ |
Mohsin Kazmi | bbd6b74 | 2019-05-02 13:54:59 +0200 | [diff] [blame] | 914 | u64 supported_features = VIRTIO_FEATURE (VIRTIO_NET_F_CSUM) |
| 915 | | VIRTIO_FEATURE (VIRTIO_NET_F_GUEST_CSUM) |
| 916 | | VIRTIO_FEATURE (VIRTIO_NET_F_CTRL_GUEST_OFFLOADS) |
| 917 | | VIRTIO_FEATURE (VIRTIO_NET_F_MTU) |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 918 | | VIRTIO_FEATURE (VIRTIO_NET_F_MAC) |
Mohsin Kazmi | bbd6b74 | 2019-05-02 13:54:59 +0200 | [diff] [blame] | 919 | | VIRTIO_FEATURE (VIRTIO_NET_F_GSO) |
| 920 | | VIRTIO_FEATURE (VIRTIO_NET_F_GUEST_TSO4) |
| 921 | | VIRTIO_FEATURE (VIRTIO_NET_F_GUEST_TSO6) |
| 922 | | VIRTIO_FEATURE (VIRTIO_NET_F_GUEST_UFO) |
| 923 | | VIRTIO_FEATURE (VIRTIO_NET_F_HOST_TSO4) |
| 924 | | VIRTIO_FEATURE (VIRTIO_NET_F_HOST_TSO6) |
| 925 | | VIRTIO_FEATURE (VIRTIO_NET_F_HOST_UFO) |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 926 | | VIRTIO_FEATURE (VIRTIO_NET_F_MRG_RXBUF) |
| 927 | | VIRTIO_FEATURE (VIRTIO_NET_F_STATUS) |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 928 | | VIRTIO_FEATURE (VIRTIO_NET_F_CTRL_VQ) |
| 929 | | VIRTIO_FEATURE (VIRTIO_NET_F_MQ) |
Mohsin Kazmi | b74fe32 | 2019-01-31 13:50:56 +0000 | [diff] [blame] | 930 | | VIRTIO_FEATURE (VIRTIO_F_NOTIFY_ON_EMPTY) |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 931 | | VIRTIO_FEATURE (VIRTIO_F_ANY_LAYOUT) |
| 932 | | VIRTIO_FEATURE (VIRTIO_RING_F_INDIRECT_DESC); |
| 933 | |
Mohsin Kazmi | 379aac3 | 2020-08-20 10:25:12 +0200 | [diff] [blame] | 934 | if (vif->is_modern) |
| 935 | supported_features |= VIRTIO_FEATURE (VIRTIO_F_VERSION_1); |
| 936 | |
Mohsin Kazmi | b977d3f | 2020-11-16 16:49:30 +0100 | [diff] [blame] | 937 | if (vif->is_packed) |
| 938 | { |
| 939 | supported_features |= |
| 940 | (VIRTIO_FEATURE (VIRTIO_F_RING_PACKED) | |
| 941 | VIRTIO_FEATURE (VIRTIO_F_IN_ORDER)); |
| 942 | } |
| 943 | |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 944 | if (req_features == 0) |
| 945 | { |
| 946 | req_features = supported_features; |
| 947 | } |
| 948 | |
| 949 | vif->features = req_features & vif->remote_features & supported_features; |
| 950 | |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 951 | if (vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_MTU)) |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 952 | { |
Mohsin Kazmi | a0a6833 | 2020-07-16 12:55:42 +0000 | [diff] [blame] | 953 | u16 mtu = 0; |
| 954 | mtu = vif->virtio_pci_func->get_mtu (vm, vif); |
| 955 | |
| 956 | if (mtu < 64) |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 957 | vif->features &= ~VIRTIO_FEATURE (VIRTIO_NET_F_MTU); |
| 958 | } |
| 959 | |
Mohsin Kazmi | b977d3f | 2020-11-16 16:49:30 +0100 | [diff] [blame] | 960 | if ((vif->features & (VIRTIO_FEATURE (VIRTIO_F_RING_PACKED))) == 0) |
| 961 | vif->is_packed = 0; |
| 962 | |
Mohsin Kazmi | a0a6833 | 2020-07-16 12:55:42 +0000 | [diff] [blame] | 963 | vif->virtio_pci_func->set_driver_features (vm, vif, vif->features); |
| 964 | vif->features = vif->virtio_pci_func->get_driver_features (vm, vif); |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 965 | } |
| 966 | |
| 967 | void |
| 968 | virtio_pci_read_device_feature (vlib_main_t * vm, virtio_if_t * vif) |
| 969 | { |
Mohsin Kazmi | a0a6833 | 2020-07-16 12:55:42 +0000 | [diff] [blame] | 970 | vif->remote_features = vif->virtio_pci_func->get_device_features (vm, vif); |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 971 | } |
| 972 | |
| 973 | int |
| 974 | virtio_pci_reset_device (vlib_main_t * vm, virtio_if_t * vif) |
| 975 | { |
| 976 | u8 status = 0; |
| 977 | |
| 978 | /* |
| 979 | * Reset the device |
| 980 | */ |
Mohsin Kazmi | a0a6833 | 2020-07-16 12:55:42 +0000 | [diff] [blame] | 981 | status = vif->virtio_pci_func->device_reset (vm, vif); |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 982 | |
| 983 | /* |
| 984 | * Set the Acknowledge status bit |
| 985 | */ |
Mohsin Kazmi | a0a6833 | 2020-07-16 12:55:42 +0000 | [diff] [blame] | 986 | vif->virtio_pci_func->set_status (vm, vif, VIRTIO_CONFIG_STATUS_ACK); |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 987 | |
| 988 | /* |
| 989 | * Set the Driver status bit |
| 990 | */ |
Mohsin Kazmi | a0a6833 | 2020-07-16 12:55:42 +0000 | [diff] [blame] | 991 | vif->virtio_pci_func->set_status (vm, vif, VIRTIO_CONFIG_STATUS_DRIVER); |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 992 | |
| 993 | /* |
| 994 | * Read the status and verify it |
| 995 | */ |
Mohsin Kazmi | a0a6833 | 2020-07-16 12:55:42 +0000 | [diff] [blame] | 996 | status = vif->virtio_pci_func->get_status (vm, vif); |
Mohsin Kazmi | 379aac3 | 2020-08-20 10:25:12 +0200 | [diff] [blame] | 997 | if ((status & VIRTIO_CONFIG_STATUS_ACK) |
| 998 | && (status & VIRTIO_CONFIG_STATUS_DRIVER)) |
| 999 | vif->status = status; |
| 1000 | else |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1001 | return -1; |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1002 | |
| 1003 | return 0; |
| 1004 | } |
| 1005 | |
| 1006 | clib_error_t * |
Mohsin Kazmi | 379aac3 | 2020-08-20 10:25:12 +0200 | [diff] [blame] | 1007 | virtio_pci_read_caps (vlib_main_t * vm, virtio_if_t * vif, void **bar) |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1008 | { |
| 1009 | clib_error_t *error = 0; |
Mohsin Kazmi | a7a2281 | 2020-08-31 17:17:16 +0200 | [diff] [blame] | 1010 | virtio_pci_cap_t cap; |
Mohsin Kazmi | 379aac3 | 2020-08-20 10:25:12 +0200 | [diff] [blame] | 1011 | u8 pos, common_cfg = 0, notify = 0, dev_cfg = 0, isr = 0, pci_cfg = 0; |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1012 | vlib_pci_dev_handle_t h = vif->pci_dev_handle; |
| 1013 | |
| 1014 | if ((error = vlib_pci_read_config_u8 (vm, h, PCI_CAPABILITY_LIST, &pos))) |
Mohsin Kazmi | b74fe32 | 2019-01-31 13:50:56 +0000 | [diff] [blame] | 1015 | { |
Damjan Marion | f41244f | 2019-11-08 17:41:06 +0100 | [diff] [blame] | 1016 | virtio_log_error (vif, "error in reading capabilty list position"); |
Mohsin Kazmi | c96e64b | 2020-09-01 17:12:50 +0200 | [diff] [blame] | 1017 | return clib_error_return (error, |
| 1018 | "error in reading capabilty list position"); |
Mohsin Kazmi | b74fe32 | 2019-01-31 13:50:56 +0000 | [diff] [blame] | 1019 | } |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1020 | while (pos) |
| 1021 | { |
| 1022 | if ((error = |
| 1023 | vlib_pci_read_write_config (vm, h, VLIB_READ, pos, &cap, |
| 1024 | sizeof (cap)))) |
Mohsin Kazmi | b74fe32 | 2019-01-31 13:50:56 +0000 | [diff] [blame] | 1025 | { |
Damjan Marion | f41244f | 2019-11-08 17:41:06 +0100 | [diff] [blame] | 1026 | virtio_log_error (vif, "%s [%2x]", |
Mohsin Kazmi | b74fe32 | 2019-01-31 13:50:56 +0000 | [diff] [blame] | 1027 | "error in reading the capability at", pos); |
Mohsin Kazmi | c96e64b | 2020-09-01 17:12:50 +0200 | [diff] [blame] | 1028 | return clib_error_return (error, |
| 1029 | "error in reading the capability at [%2x]", |
| 1030 | pos); |
Mohsin Kazmi | b74fe32 | 2019-01-31 13:50:56 +0000 | [diff] [blame] | 1031 | } |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1032 | |
| 1033 | if (cap.cap_vndr == PCI_CAP_ID_MSIX) |
| 1034 | { |
Mohsin Kazmi | b74fe32 | 2019-01-31 13:50:56 +0000 | [diff] [blame] | 1035 | u16 flags, table_size, table_size_mask = 0x07FF; |
| 1036 | |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1037 | if ((error = |
| 1038 | vlib_pci_read_write_config (vm, h, VLIB_READ, pos + 2, &flags, |
| 1039 | sizeof (flags)))) |
Mohsin Kazmi | c96e64b | 2020-09-01 17:12:50 +0200 | [diff] [blame] | 1040 | return clib_error_return (error, |
| 1041 | "error in reading the capability at [%2x]", |
| 1042 | pos + 2); |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1043 | |
Mohsin Kazmi | b74fe32 | 2019-01-31 13:50:56 +0000 | [diff] [blame] | 1044 | table_size = flags & table_size_mask; |
Damjan Marion | f41244f | 2019-11-08 17:41:06 +0100 | [diff] [blame] | 1045 | virtio_log_debug (vif, "flags:0x%x %s 0x%x", flags, |
Mohsin Kazmi | b74fe32 | 2019-01-31 13:50:56 +0000 | [diff] [blame] | 1046 | "msix interrupt vector table-size", table_size); |
| 1047 | |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1048 | if (flags & PCI_MSIX_ENABLE) |
Mohsin Kazmi | b74fe32 | 2019-01-31 13:50:56 +0000 | [diff] [blame] | 1049 | { |
Damjan Marion | f41244f | 2019-11-08 17:41:06 +0100 | [diff] [blame] | 1050 | virtio_log_debug (vif, "msix interrupt enabled"); |
Mohsin Kazmi | b74fe32 | 2019-01-31 13:50:56 +0000 | [diff] [blame] | 1051 | vif->msix_enabled = VIRTIO_MSIX_ENABLED; |
Mohsin Kazmi | 8046fdc | 2020-07-15 11:54:10 +0000 | [diff] [blame] | 1052 | vif->msix_table_size = table_size; |
Mohsin Kazmi | b74fe32 | 2019-01-31 13:50:56 +0000 | [diff] [blame] | 1053 | } |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1054 | else |
Mohsin Kazmi | b74fe32 | 2019-01-31 13:50:56 +0000 | [diff] [blame] | 1055 | { |
Damjan Marion | f41244f | 2019-11-08 17:41:06 +0100 | [diff] [blame] | 1056 | virtio_log_debug (vif, "msix interrupt disabled"); |
Mohsin Kazmi | b74fe32 | 2019-01-31 13:50:56 +0000 | [diff] [blame] | 1057 | vif->msix_enabled = VIRTIO_MSIX_DISABLED; |
Mohsin Kazmi | 8046fdc | 2020-07-15 11:54:10 +0000 | [diff] [blame] | 1058 | vif->msix_table_size = 0; |
Mohsin Kazmi | b74fe32 | 2019-01-31 13:50:56 +0000 | [diff] [blame] | 1059 | } |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1060 | } |
| 1061 | |
| 1062 | if (cap.cap_vndr != PCI_CAP_ID_VNDR) |
| 1063 | { |
Damjan Marion | f41244f | 2019-11-08 17:41:06 +0100 | [diff] [blame] | 1064 | virtio_log_debug (vif, "[%2x] %s %2x ", pos, |
Mohsin Kazmi | 33cc5cf | 2019-01-21 15:19:39 +0000 | [diff] [blame] | 1065 | "skipping non VNDR cap id:", cap.cap_vndr); |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1066 | goto next; |
| 1067 | } |
| 1068 | |
Damjan Marion | f41244f | 2019-11-08 17:41:06 +0100 | [diff] [blame] | 1069 | virtio_log_debug (vif, |
Mohsin Kazmi | b74fe32 | 2019-01-31 13:50:56 +0000 | [diff] [blame] | 1070 | "[%4x] cfg type: %u, bar: %u, offset: %04x, len: %u", |
| 1071 | pos, cap.cfg_type, cap.bar, cap.offset, cap.length); |
Mohsin Kazmi | 379aac3 | 2020-08-20 10:25:12 +0200 | [diff] [blame] | 1072 | |
Mohsin Kazmi | 0a507d7 | 2020-09-01 18:18:48 +0200 | [diff] [blame] | 1073 | if (cap.bar >= 0 && cap.bar <= 5) |
Mohsin Kazmi | 133c91c1 | 2020-08-20 17:18:56 +0200 | [diff] [blame] | 1074 | { |
| 1075 | vif->bar = bar[cap.bar]; |
| 1076 | vif->bar_id = cap.bar; |
| 1077 | } |
| 1078 | else |
| 1079 | return clib_error_return (error, "invalid bar %u", cap.bar); |
Mohsin Kazmi | 379aac3 | 2020-08-20 10:25:12 +0200 | [diff] [blame] | 1080 | |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1081 | switch (cap.cfg_type) |
| 1082 | { |
| 1083 | case VIRTIO_PCI_CAP_COMMON_CFG: |
Mohsin Kazmi | 379aac3 | 2020-08-20 10:25:12 +0200 | [diff] [blame] | 1084 | vif->common_offset = cap.offset; |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1085 | common_cfg = 1; |
| 1086 | break; |
| 1087 | case VIRTIO_PCI_CAP_NOTIFY_CFG: |
Mohsin Kazmi | 379aac3 | 2020-08-20 10:25:12 +0200 | [diff] [blame] | 1088 | if ((error = |
| 1089 | vlib_pci_read_write_config (vm, h, VLIB_READ, |
| 1090 | pos + sizeof (cap), |
| 1091 | &vif->notify_off_multiplier, |
| 1092 | sizeof |
| 1093 | (vif->notify_off_multiplier)))) |
| 1094 | { |
| 1095 | virtio_log_error (vif, "notify off multiplier is not given"); |
| 1096 | } |
| 1097 | else |
| 1098 | { |
| 1099 | virtio_log_debug (vif, "notify off multiplier is %u", |
| 1100 | vif->notify_off_multiplier); |
| 1101 | vif->notify_offset = cap.offset; |
| 1102 | notify = 1; |
| 1103 | } |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1104 | break; |
| 1105 | case VIRTIO_PCI_CAP_DEVICE_CFG: |
Mohsin Kazmi | 379aac3 | 2020-08-20 10:25:12 +0200 | [diff] [blame] | 1106 | vif->device_offset = cap.offset; |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1107 | dev_cfg = 1; |
| 1108 | break; |
| 1109 | case VIRTIO_PCI_CAP_ISR_CFG: |
Mohsin Kazmi | 379aac3 | 2020-08-20 10:25:12 +0200 | [diff] [blame] | 1110 | vif->isr_offset = cap.offset; |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1111 | isr = 1; |
| 1112 | break; |
Mohsin Kazmi | 22c0ece | 2019-01-28 19:30:21 +0000 | [diff] [blame] | 1113 | case VIRTIO_PCI_CAP_PCI_CFG: |
| 1114 | if (cap.bar == 0) |
| 1115 | pci_cfg = 1; |
| 1116 | break; |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1117 | } |
| 1118 | next: |
| 1119 | pos = cap.cap_next; |
| 1120 | } |
| 1121 | |
Mohsin Kazmi | 379aac3 | 2020-08-20 10:25:12 +0200 | [diff] [blame] | 1122 | if (common_cfg == 0 || notify == 0 || dev_cfg == 0 || isr == 0) |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1123 | { |
Mohsin Kazmi | 379aac3 | 2020-08-20 10:25:12 +0200 | [diff] [blame] | 1124 | vif->virtio_pci_func = &virtio_pci_legacy_func; |
Mohsin Kazmi | 162a296 | 2020-09-29 10:01:25 +0000 | [diff] [blame] | 1125 | vif->notify_off_multiplier = 0; |
Damjan Marion | f41244f | 2019-11-08 17:41:06 +0100 | [diff] [blame] | 1126 | virtio_log_debug (vif, "legacy virtio pci device found"); |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1127 | return error; |
| 1128 | } |
| 1129 | |
Mohsin Kazmi | 379aac3 | 2020-08-20 10:25:12 +0200 | [diff] [blame] | 1130 | vif->is_modern = 1; |
| 1131 | vif->virtio_pci_func = &virtio_pci_modern_func; |
| 1132 | |
Mohsin Kazmi | 0e2bc63 | 2019-01-30 13:36:57 +0000 | [diff] [blame] | 1133 | if (!pci_cfg) |
Mohsin Kazmi | 162a296 | 2020-09-29 10:01:25 +0000 | [diff] [blame] | 1134 | { |
| 1135 | virtio_log_debug (vif, "modern virtio pci device found"); |
| 1136 | } |
| 1137 | else |
| 1138 | { |
| 1139 | virtio_log_debug (vif, "transitional virtio pci device found"); |
| 1140 | } |
Mohsin Kazmi | 0e2bc63 | 2019-01-30 13:36:57 +0000 | [diff] [blame] | 1141 | |
Mohsin Kazmi | 33cc5cf | 2019-01-21 15:19:39 +0000 | [diff] [blame] | 1142 | return error; |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1143 | } |
| 1144 | |
| 1145 | static clib_error_t * |
| 1146 | virtio_pci_device_init (vlib_main_t * vm, virtio_if_t * vif, |
Mohsin Kazmi | 379aac3 | 2020-08-20 10:25:12 +0200 | [diff] [blame] | 1147 | virtio_pci_create_if_args_t * args, void **bar) |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1148 | { |
| 1149 | clib_error_t *error = 0; |
| 1150 | u8 status = 0; |
| 1151 | |
Mohsin Kazmi | 379aac3 | 2020-08-20 10:25:12 +0200 | [diff] [blame] | 1152 | if ((error = virtio_pci_read_caps (vm, vif, bar))) |
Mohsin Kazmi | 57d7321 | 2020-03-05 12:22:41 +0000 | [diff] [blame] | 1153 | { |
| 1154 | args->rv = VNET_API_ERROR_UNSUPPORTED; |
| 1155 | virtio_log_error (vif, "Device is not supported"); |
Mohsin Kazmi | c96e64b | 2020-09-01 17:12:50 +0200 | [diff] [blame] | 1156 | return clib_error_return (error, "Device is not supported"); |
Mohsin Kazmi | 57d7321 | 2020-03-05 12:22:41 +0000 | [diff] [blame] | 1157 | } |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1158 | |
| 1159 | if (virtio_pci_reset_device (vm, vif) < 0) |
Mohsin Kazmi | b74fe32 | 2019-01-31 13:50:56 +0000 | [diff] [blame] | 1160 | { |
Mohsin Kazmi | 57d7321 | 2020-03-05 12:22:41 +0000 | [diff] [blame] | 1161 | args->rv = VNET_API_ERROR_INIT_FAILED; |
Damjan Marion | f41244f | 2019-11-08 17:41:06 +0100 | [diff] [blame] | 1162 | virtio_log_error (vif, "Failed to reset the device"); |
Mohsin Kazmi | c96e64b | 2020-09-01 17:12:50 +0200 | [diff] [blame] | 1163 | return clib_error_return (error, "Failed to reset the device"); |
Mohsin Kazmi | b74fe32 | 2019-01-31 13:50:56 +0000 | [diff] [blame] | 1164 | } |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1165 | /* |
| 1166 | * read device features and negotiate (user) requested features |
| 1167 | */ |
| 1168 | virtio_pci_read_device_feature (vm, vif); |
Mohsin Kazmi | ce21599 | 2020-04-10 14:57:06 +0000 | [diff] [blame] | 1169 | if ((vif->remote_features & VIRTIO_FEATURE (VIRTIO_RING_F_INDIRECT_DESC)) == |
| 1170 | 0) |
| 1171 | { |
| 1172 | virtio_log_warning (vif, "error encountered: vhost-net backend doesn't " |
| 1173 | "support VIRTIO_RING_F_INDIRECT_DESC features"); |
| 1174 | } |
| 1175 | if ((vif->remote_features & VIRTIO_FEATURE (VIRTIO_NET_F_MRG_RXBUF)) == 0) |
| 1176 | { |
| 1177 | virtio_log_warning (vif, "error encountered: vhost-net backend doesn't " |
| 1178 | "support VIRTIO_NET_F_MRG_RXBUF features"); |
| 1179 | } |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1180 | virtio_negotiate_features (vm, vif, args->features); |
| 1181 | |
| 1182 | /* |
| 1183 | * After FEATURE_OK, driver should not accept new feature bits |
| 1184 | */ |
Mohsin Kazmi | a0a6833 | 2020-07-16 12:55:42 +0000 | [diff] [blame] | 1185 | vif->virtio_pci_func->set_status (vm, vif, |
| 1186 | VIRTIO_CONFIG_STATUS_FEATURES_OK); |
| 1187 | status = vif->virtio_pci_func->get_status (vm, vif); |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1188 | if (!(status & VIRTIO_CONFIG_STATUS_FEATURES_OK)) |
Mohsin Kazmi | b74fe32 | 2019-01-31 13:50:56 +0000 | [diff] [blame] | 1189 | { |
Mohsin Kazmi | 57d7321 | 2020-03-05 12:22:41 +0000 | [diff] [blame] | 1190 | args->rv = VNET_API_ERROR_UNSUPPORTED; |
Damjan Marion | f41244f | 2019-11-08 17:41:06 +0100 | [diff] [blame] | 1191 | virtio_log_error (vif, |
Mohsin Kazmi | b74fe32 | 2019-01-31 13:50:56 +0000 | [diff] [blame] | 1192 | "error encountered: Device doesn't support requested features"); |
Mohsin Kazmi | c96e64b | 2020-09-01 17:12:50 +0200 | [diff] [blame] | 1193 | return clib_error_return (error, |
| 1194 | "Device doesn't support requested features"); |
Mohsin Kazmi | b74fe32 | 2019-01-31 13:50:56 +0000 | [diff] [blame] | 1195 | } |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1196 | vif->status = status; |
| 1197 | |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 1198 | /* |
| 1199 | * get or set the mac address |
| 1200 | */ |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1201 | if (virtio_pci_get_mac (vm, vif)) |
| 1202 | { |
| 1203 | f64 now = vlib_time_now (vm); |
| 1204 | u32 rnd; |
| 1205 | rnd = (u32) (now * 1e6); |
| 1206 | rnd = random_u32 (&rnd); |
| 1207 | |
| 1208 | memcpy (vif->mac_addr + 2, &rnd, sizeof (rnd)); |
| 1209 | vif->mac_addr[0] = 2; |
| 1210 | vif->mac_addr[1] = 0xfe; |
| 1211 | virtio_pci_set_mac (vm, vif); |
| 1212 | } |
| 1213 | |
| 1214 | virtio_set_net_hdr_size (vif); |
| 1215 | |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 1216 | /* |
| 1217 | * Initialize the virtqueues |
| 1218 | */ |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1219 | if ((error = virtio_pci_get_max_virtqueue_pairs (vm, vif))) |
Mohsin Kazmi | 57d7321 | 2020-03-05 12:22:41 +0000 | [diff] [blame] | 1220 | { |
| 1221 | args->rv = VNET_API_ERROR_EXCEEDED_NUMBER_OF_RANGES_CAPACITY; |
| 1222 | goto err; |
| 1223 | } |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1224 | |
Mohsin Kazmi | 8046fdc | 2020-07-15 11:54:10 +0000 | [diff] [blame] | 1225 | if (vif->msix_enabled == VIRTIO_MSIX_ENABLED) |
| 1226 | { |
| 1227 | if (vif->msix_table_size <= vif->max_queue_pairs) |
| 1228 | { |
| 1229 | virtio_log_error (vif, |
| 1230 | "error MSIX lines (%u) <= Number of RXQs (%u)", |
| 1231 | vif->msix_table_size, vif->max_queue_pairs); |
| 1232 | return clib_error_return (error, |
| 1233 | "error MSIX lines (%u) <= Number of RXQs (%u)", |
| 1234 | vif->msix_table_size, |
| 1235 | vif->max_queue_pairs); |
| 1236 | } |
| 1237 | } |
| 1238 | |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 1239 | for (int i = 0; i < vif->max_queue_pairs; i++) |
| 1240 | { |
| 1241 | if ((error = virtio_pci_vring_init (vm, vif, RX_QUEUE (i)))) |
Mohsin Kazmi | 43b512c | 2019-04-30 17:25:26 +0200 | [diff] [blame] | 1242 | { |
Mohsin Kazmi | 57d7321 | 2020-03-05 12:22:41 +0000 | [diff] [blame] | 1243 | args->rv = VNET_API_ERROR_INIT_FAILED; |
| 1244 | virtio_log_error (vif, "%s (%u) %s", "error in rxq-queue", |
| 1245 | RX_QUEUE (i), "initialization"); |
Mohsin Kazmi | a0a6833 | 2020-07-16 12:55:42 +0000 | [diff] [blame] | 1246 | error = |
| 1247 | clib_error_return (error, "%s (%u) %s", "error in rxq-queue", |
| 1248 | RX_QUEUE (i), "initialization"); |
| 1249 | goto err; |
Mohsin Kazmi | 43b512c | 2019-04-30 17:25:26 +0200 | [diff] [blame] | 1250 | } |
| 1251 | else |
| 1252 | { |
| 1253 | vif->num_rxqs++; |
| 1254 | } |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1255 | |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 1256 | if ((error = virtio_pci_vring_init (vm, vif, TX_QUEUE (i)))) |
Mohsin Kazmi | 43b512c | 2019-04-30 17:25:26 +0200 | [diff] [blame] | 1257 | { |
Mohsin Kazmi | 57d7321 | 2020-03-05 12:22:41 +0000 | [diff] [blame] | 1258 | args->rv = VNET_API_ERROR_INIT_FAILED; |
| 1259 | virtio_log_error (vif, "%s (%u) %s", "error in txq-queue", |
| 1260 | TX_QUEUE (i), "initialization"); |
Mohsin Kazmi | a0a6833 | 2020-07-16 12:55:42 +0000 | [diff] [blame] | 1261 | error = |
| 1262 | clib_error_return (error, "%s (%u) %s", "error in txq-queue", |
| 1263 | TX_QUEUE (i), "initialization"); |
| 1264 | goto err; |
Mohsin Kazmi | 43b512c | 2019-04-30 17:25:26 +0200 | [diff] [blame] | 1265 | } |
| 1266 | else |
| 1267 | { |
| 1268 | vif->num_txqs++; |
| 1269 | } |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 1270 | } |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1271 | |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 1272 | if (vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_CTRL_VQ)) |
| 1273 | { |
| 1274 | if ((error = |
| 1275 | virtio_pci_control_vring_init (vm, vif, vif->max_queue_pairs * 2))) |
| 1276 | { |
Damjan Marion | f41244f | 2019-11-08 17:41:06 +0100 | [diff] [blame] | 1277 | virtio_log_warning (vif, "%s (%u) %s", "error in control-queue", |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 1278 | vif->max_queue_pairs * 2, "initialization"); |
| 1279 | if (vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_MQ)) |
| 1280 | vif->features &= ~VIRTIO_FEATURE (VIRTIO_NET_F_MQ); |
| 1281 | } |
| 1282 | } |
| 1283 | else |
| 1284 | { |
Damjan Marion | f41244f | 2019-11-08 17:41:06 +0100 | [diff] [blame] | 1285 | virtio_log_debug (vif, "control queue is not available"); |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 1286 | vif->cxq_vring = NULL; |
| 1287 | } |
| 1288 | |
| 1289 | /* |
| 1290 | * set the msix interrupts |
| 1291 | */ |
Mohsin Kazmi | b74fe32 | 2019-01-31 13:50:56 +0000 | [diff] [blame] | 1292 | if (vif->msix_enabled == VIRTIO_MSIX_ENABLED) |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1293 | { |
Mohsin Kazmi | 8046fdc | 2020-07-15 11:54:10 +0000 | [diff] [blame] | 1294 | int i, j; |
Mohsin Kazmi | a0a6833 | 2020-07-16 12:55:42 +0000 | [diff] [blame] | 1295 | if (vif->virtio_pci_func->set_config_irq (vm, vif, 0) == |
Mohsin Kazmi | b74fe32 | 2019-01-31 13:50:56 +0000 | [diff] [blame] | 1296 | VIRTIO_MSI_NO_VECTOR) |
Mohsin Kazmi | 8046fdc | 2020-07-15 11:54:10 +0000 | [diff] [blame] | 1297 | { |
| 1298 | virtio_log_warning (vif, "config vector 0 is not set"); |
| 1299 | } |
| 1300 | else |
| 1301 | { |
| 1302 | virtio_log_debug (vif, "config msix vector is set at 0"); |
| 1303 | } |
| 1304 | for (i = 0, j = 1; i < vif->max_queue_pairs; i++, j++) |
| 1305 | { |
Mohsin Kazmi | a0a6833 | 2020-07-16 12:55:42 +0000 | [diff] [blame] | 1306 | if (vif->virtio_pci_func->set_queue_irq (vm, vif, j, |
| 1307 | RX_QUEUE (i)) == |
Mohsin Kazmi | 8046fdc | 2020-07-15 11:54:10 +0000 | [diff] [blame] | 1308 | VIRTIO_MSI_NO_VECTOR) |
| 1309 | { |
| 1310 | virtio_log_warning (vif, "queue (%u) vector is not set at %u", |
| 1311 | RX_QUEUE (i), j); |
| 1312 | } |
| 1313 | else |
| 1314 | { |
| 1315 | virtio_log_debug (vif, "%s (%u) %s %u", "queue", |
| 1316 | RX_QUEUE (i), "msix vector is set at", j); |
| 1317 | } |
| 1318 | } |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1319 | } |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 1320 | |
| 1321 | /* |
| 1322 | * set the driver status OK |
| 1323 | */ |
Mohsin Kazmi | a0a6833 | 2020-07-16 12:55:42 +0000 | [diff] [blame] | 1324 | vif->virtio_pci_func->set_status (vm, vif, VIRTIO_CONFIG_STATUS_DRIVER_OK); |
| 1325 | vif->status = vif->virtio_pci_func->get_status (vm, vif); |
Mohsin Kazmi | 46f877e | 2019-04-08 10:36:18 +0200 | [diff] [blame] | 1326 | err: |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1327 | return error; |
| 1328 | } |
| 1329 | |
| 1330 | void |
| 1331 | virtio_pci_create_if (vlib_main_t * vm, virtio_pci_create_if_args_t * args) |
| 1332 | { |
| 1333 | vnet_main_t *vnm = vnet_get_main (); |
Mohsin Kazmi | 33cc5cf | 2019-01-21 15:19:39 +0000 | [diff] [blame] | 1334 | virtio_main_t *vim = &virtio_main; |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1335 | virtio_if_t *vif; |
| 1336 | vlib_pci_dev_handle_t h; |
| 1337 | clib_error_t *error = 0; |
Mohsin Kazmi | 8046fdc | 2020-07-15 11:54:10 +0000 | [diff] [blame] | 1338 | u32 interrupt_count = 0; |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1339 | |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1340 | /* *INDENT-OFF* */ |
Damjan Marion | b2c31b6 | 2020-12-13 21:47:40 +0100 | [diff] [blame] | 1341 | pool_foreach (vif, vim->interfaces) { |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1342 | if (vif->pci_addr.as_u32 == args->addr) |
| 1343 | { |
Mohsin Kazmi | 57d7321 | 2020-03-05 12:22:41 +0000 | [diff] [blame] | 1344 | args->rv = VNET_API_ERROR_ADDRESS_IN_USE; |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1345 | args->error = |
| 1346 | clib_error_return (error, "PCI address in use"); |
Mohsin Kazmi | 33cc5cf | 2019-01-21 15:19:39 +0000 | [diff] [blame] | 1347 | vlib_log (VLIB_LOG_LEVEL_ERR, vim->log_default, "%U: %s", |
| 1348 | format_vlib_pci_addr, &args->addr, |
| 1349 | " PCI address in use"); |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1350 | return; |
| 1351 | } |
Damjan Marion | b2c31b6 | 2020-12-13 21:47:40 +0100 | [diff] [blame] | 1352 | } |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1353 | /* *INDENT-ON* */ |
| 1354 | |
Mohsin Kazmi | 33cc5cf | 2019-01-21 15:19:39 +0000 | [diff] [blame] | 1355 | pool_get (vim->interfaces, vif); |
| 1356 | vif->dev_instance = vif - vim->interfaces; |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1357 | vif->per_interface_next_index = ~0; |
| 1358 | vif->pci_addr.as_u32 = args->addr; |
Mohsin Kazmi | b977d3f | 2020-11-16 16:49:30 +0100 | [diff] [blame] | 1359 | if (args->virtio_flags & VIRTIO_FLAG_PACKED) |
| 1360 | vif->is_packed = 1; |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1361 | |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1362 | if ((error = |
| 1363 | vlib_pci_device_open (vm, (vlib_pci_addr_t *) & vif->pci_addr, |
| 1364 | virtio_pci_device_ids, &h))) |
| 1365 | { |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1366 | args->rv = VNET_API_ERROR_INVALID_INTERFACE; |
| 1367 | args->error = |
| 1368 | clib_error_return (error, "pci-addr %U", format_vlib_pci_addr, |
| 1369 | &vif->pci_addr); |
Mohsin Kazmi | 33cc5cf | 2019-01-21 15:19:39 +0000 | [diff] [blame] | 1370 | vlib_log (VLIB_LOG_LEVEL_ERR, vim->log_default, "%U: %s", |
| 1371 | format_vlib_pci_addr, &vif->pci_addr, |
| 1372 | "error encountered on pci device open"); |
Benoît Ganne | 8f01183 | 2019-11-20 14:32:38 +0100 | [diff] [blame] | 1373 | pool_put (vim->interfaces, vif); |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1374 | return; |
| 1375 | } |
| 1376 | vif->pci_dev_handle = h; |
| 1377 | vlib_pci_set_private_data (vm, h, vif->dev_instance); |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 1378 | vif->numa_node = vlib_pci_get_numa_node (vm, h); |
Damjan Marion | f41244f | 2019-11-08 17:41:06 +0100 | [diff] [blame] | 1379 | vif->type = VIRTIO_IF_TYPE_PCI; |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1380 | |
| 1381 | if ((error = vlib_pci_bus_master_enable (vm, h))) |
Mohsin Kazmi | 33cc5cf | 2019-01-21 15:19:39 +0000 | [diff] [blame] | 1382 | { |
Damjan Marion | f41244f | 2019-11-08 17:41:06 +0100 | [diff] [blame] | 1383 | virtio_log_error (vif, "error encountered on pci bus master enable"); |
Mohsin Kazmi | 33cc5cf | 2019-01-21 15:19:39 +0000 | [diff] [blame] | 1384 | goto error; |
| 1385 | } |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1386 | |
Mohsin Kazmi | 379aac3 | 2020-08-20 10:25:12 +0200 | [diff] [blame] | 1387 | void *bar[6]; |
| 1388 | for (u32 i = 0; i <= 5; i++) |
| 1389 | { |
| 1390 | |
| 1391 | if ((error = vlib_pci_map_region (vm, h, i, &bar[i]))) |
| 1392 | { |
| 1393 | virtio_log_debug (vif, "no pci map region for bar %u", i); |
| 1394 | } |
| 1395 | else |
| 1396 | { |
| 1397 | virtio_log_debug (vif, "pci map region for bar %u at %p", i, |
| 1398 | bar[i]); |
| 1399 | } |
| 1400 | } |
| 1401 | |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1402 | if ((error = vlib_pci_io_region (vm, h, 0))) |
Mohsin Kazmi | 33cc5cf | 2019-01-21 15:19:39 +0000 | [diff] [blame] | 1403 | { |
Damjan Marion | f41244f | 2019-11-08 17:41:06 +0100 | [diff] [blame] | 1404 | virtio_log_error (vif, "error encountered on pci io region"); |
Mohsin Kazmi | 33cc5cf | 2019-01-21 15:19:39 +0000 | [diff] [blame] | 1405 | goto error; |
| 1406 | } |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1407 | |
Mohsin Kazmi | 8046fdc | 2020-07-15 11:54:10 +0000 | [diff] [blame] | 1408 | interrupt_count = vlib_pci_get_num_msix_interrupts (vm, h); |
| 1409 | if (interrupt_count > 1) |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1410 | { |
| 1411 | if ((error = vlib_pci_register_msix_handler (vm, h, 0, 1, |
Mohsin Kazmi | 8046fdc | 2020-07-15 11:54:10 +0000 | [diff] [blame] | 1412 | &virtio_pci_irq_config_handler))) |
Mohsin Kazmi | 33cc5cf | 2019-01-21 15:19:39 +0000 | [diff] [blame] | 1413 | { |
Mohsin Kazmi | 57d7321 | 2020-03-05 12:22:41 +0000 | [diff] [blame] | 1414 | args->rv = VNET_API_ERROR_INVALID_REGISTRATION; |
Damjan Marion | f41244f | 2019-11-08 17:41:06 +0100 | [diff] [blame] | 1415 | virtio_log_error (vif, |
Mohsin Kazmi | 33cc5cf | 2019-01-21 15:19:39 +0000 | [diff] [blame] | 1416 | "error encountered on pci register msix handler 0"); |
| 1417 | goto error; |
| 1418 | } |
Mohsin Kazmi | 8046fdc | 2020-07-15 11:54:10 +0000 | [diff] [blame] | 1419 | |
| 1420 | if ((error = |
| 1421 | vlib_pci_register_msix_handler (vm, h, 1, (interrupt_count - 1), |
| 1422 | &virtio_pci_irq_queue_handler))) |
Mohsin Kazmi | 33cc5cf | 2019-01-21 15:19:39 +0000 | [diff] [blame] | 1423 | { |
Mohsin Kazmi | 57d7321 | 2020-03-05 12:22:41 +0000 | [diff] [blame] | 1424 | args->rv = VNET_API_ERROR_INVALID_REGISTRATION; |
Damjan Marion | f41244f | 2019-11-08 17:41:06 +0100 | [diff] [blame] | 1425 | virtio_log_error (vif, |
Mohsin Kazmi | 33cc5cf | 2019-01-21 15:19:39 +0000 | [diff] [blame] | 1426 | "error encountered on pci register msix handler 1"); |
| 1427 | goto error; |
| 1428 | } |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1429 | |
Mohsin Kazmi | 8046fdc | 2020-07-15 11:54:10 +0000 | [diff] [blame] | 1430 | if ((error = vlib_pci_enable_msix_irq (vm, h, 0, interrupt_count))) |
Mohsin Kazmi | 33cc5cf | 2019-01-21 15:19:39 +0000 | [diff] [blame] | 1431 | { |
Damjan Marion | f41244f | 2019-11-08 17:41:06 +0100 | [diff] [blame] | 1432 | virtio_log_error (vif, "error encountered on pci enable msix irq"); |
Mohsin Kazmi | 33cc5cf | 2019-01-21 15:19:39 +0000 | [diff] [blame] | 1433 | goto error; |
| 1434 | } |
Mohsin Kazmi | b74fe32 | 2019-01-31 13:50:56 +0000 | [diff] [blame] | 1435 | vif->support_int_mode = 1; |
Damjan Marion | f41244f | 2019-11-08 17:41:06 +0100 | [diff] [blame] | 1436 | virtio_log_debug (vif, "device supports msix interrupts"); |
Mohsin Kazmi | b74fe32 | 2019-01-31 13:50:56 +0000 | [diff] [blame] | 1437 | } |
Mohsin Kazmi | 5db2f4a | 2021-02-22 18:27:57 +0000 | [diff] [blame] | 1438 | else |
Mohsin Kazmi | b74fe32 | 2019-01-31 13:50:56 +0000 | [diff] [blame] | 1439 | { |
| 1440 | /* |
Mohsin Kazmi | 5db2f4a | 2021-02-22 18:27:57 +0000 | [diff] [blame] | 1441 | * WARN: performance will be sub-optimal. |
| 1442 | * Fall back to intX, if msix table-size is 1 or |
| 1443 | * if UIO driver is being used. |
Mohsin Kazmi | b74fe32 | 2019-01-31 13:50:56 +0000 | [diff] [blame] | 1444 | */ |
| 1445 | if ((error = |
| 1446 | vlib_pci_register_intx_handler (vm, h, &virtio_pci_irq_handler))) |
| 1447 | { |
Damjan Marion | f41244f | 2019-11-08 17:41:06 +0100 | [diff] [blame] | 1448 | virtio_log_error (vif, |
Mohsin Kazmi | b74fe32 | 2019-01-31 13:50:56 +0000 | [diff] [blame] | 1449 | "error encountered on pci register interrupt handler"); |
| 1450 | goto error; |
| 1451 | } |
| 1452 | vif->support_int_mode = 1; |
Damjan Marion | f41244f | 2019-11-08 17:41:06 +0100 | [diff] [blame] | 1453 | virtio_log_debug (vif, "pci register interrupt handler"); |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1454 | } |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1455 | |
| 1456 | if ((error = vlib_pci_intr_enable (vm, h))) |
Mohsin Kazmi | 33cc5cf | 2019-01-21 15:19:39 +0000 | [diff] [blame] | 1457 | { |
Damjan Marion | f41244f | 2019-11-08 17:41:06 +0100 | [diff] [blame] | 1458 | virtio_log_error (vif, "error encountered on pci interrupt enable"); |
Mohsin Kazmi | 33cc5cf | 2019-01-21 15:19:39 +0000 | [diff] [blame] | 1459 | goto error; |
| 1460 | } |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1461 | |
Mohsin Kazmi | 379aac3 | 2020-08-20 10:25:12 +0200 | [diff] [blame] | 1462 | if ((error = virtio_pci_device_init (vm, vif, args, bar))) |
Mohsin Kazmi | b74fe32 | 2019-01-31 13:50:56 +0000 | [diff] [blame] | 1463 | { |
Damjan Marion | f41244f | 2019-11-08 17:41:06 +0100 | [diff] [blame] | 1464 | virtio_log_error (vif, "error encountered on device init"); |
Mohsin Kazmi | b74fe32 | 2019-01-31 13:50:56 +0000 | [diff] [blame] | 1465 | goto error; |
| 1466 | } |
| 1467 | |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1468 | /* create interface */ |
| 1469 | error = ethernet_register_interface (vnm, virtio_device_class.index, |
| 1470 | vif->dev_instance, vif->mac_addr, |
| 1471 | &vif->hw_if_index, |
| 1472 | virtio_pci_flag_change); |
| 1473 | |
| 1474 | if (error) |
Mohsin Kazmi | 33cc5cf | 2019-01-21 15:19:39 +0000 | [diff] [blame] | 1475 | { |
Mohsin Kazmi | 57d7321 | 2020-03-05 12:22:41 +0000 | [diff] [blame] | 1476 | args->rv = VNET_API_ERROR_INVALID_REGISTRATION; |
Damjan Marion | f41244f | 2019-11-08 17:41:06 +0100 | [diff] [blame] | 1477 | virtio_log_error (vif, |
Mohsin Kazmi | 33cc5cf | 2019-01-21 15:19:39 +0000 | [diff] [blame] | 1478 | "error encountered on ethernet register interface"); |
| 1479 | goto error; |
| 1480 | } |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1481 | |
| 1482 | vnet_sw_interface_t *sw = vnet_get_hw_sw_interface (vnm, vif->hw_if_index); |
| 1483 | vif->sw_if_index = sw->sw_if_index; |
| 1484 | args->sw_if_index = sw->sw_if_index; |
| 1485 | |
| 1486 | vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, vif->hw_if_index); |
Mohsin Kazmi | 5b3f523 | 2021-02-10 12:03:53 +0100 | [diff] [blame] | 1487 | hw->caps |= VNET_HW_INTERFACE_CAP_SUPPORTS_INT_MODE; |
Mohsin Kazmi | e347acb | 2020-09-28 10:26:33 +0000 | [diff] [blame] | 1488 | |
| 1489 | if (args->virtio_flags & VIRTIO_FLAG_BUFFERING) |
| 1490 | { |
| 1491 | error = virtio_set_packet_buffering (vif, args->buffering_size); |
| 1492 | if (error) |
| 1493 | { |
| 1494 | args->rv = VNET_API_ERROR_INIT_FAILED; |
| 1495 | virtio_log_error (vif, |
| 1496 | "error encountered during packet buffering init"); |
| 1497 | goto error; |
| 1498 | } |
| 1499 | } |
| 1500 | |
Damjan Marion | 9410053 | 2020-11-06 23:25:57 +0100 | [diff] [blame] | 1501 | virtio_vring_set_rx_queues (vm, vif); |
| 1502 | |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1503 | if (virtio_pci_is_link_up (vm, vif) & VIRTIO_NET_S_LINK_UP) |
| 1504 | { |
| 1505 | vif->flags |= VIRTIO_IF_FLAG_ADMIN_UP; |
| 1506 | vnet_hw_interface_set_flags (vnm, vif->hw_if_index, |
| 1507 | VNET_HW_INTERFACE_FLAG_LINK_UP); |
| 1508 | } |
| 1509 | else |
| 1510 | vnet_hw_interface_set_flags (vnm, vif->hw_if_index, 0); |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 1511 | |
Mohsin Kazmi | 6d4af89 | 2020-01-03 15:11:53 +0000 | [diff] [blame] | 1512 | virtio_pci_offloads (vm, vif, args->gso_enabled, |
| 1513 | args->checksum_offload_enabled); |
| 1514 | |
| 1515 | if ((vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_CTRL_VQ)) && |
| 1516 | (vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_MQ))) |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 1517 | { |
Mohsin Kazmi | 6d4af89 | 2020-01-03 15:11:53 +0000 | [diff] [blame] | 1518 | if (virtio_pci_enable_multiqueue (vm, vif, vif->max_queue_pairs)) |
| 1519 | virtio_log_warning (vif, "multiqueue is not set"); |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 1520 | } |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1521 | return; |
| 1522 | |
| 1523 | error: |
| 1524 | virtio_pci_delete_if (vm, vif); |
Mohsin Kazmi | 57d7321 | 2020-03-05 12:22:41 +0000 | [diff] [blame] | 1525 | if (args->rv == 0) |
| 1526 | args->rv = VNET_API_ERROR_INVALID_INTERFACE; |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1527 | args->error = error; |
| 1528 | } |
| 1529 | |
| 1530 | int |
| 1531 | virtio_pci_delete_if (vlib_main_t * vm, virtio_if_t * vif) |
| 1532 | { |
| 1533 | vnet_main_t *vnm = vnet_get_main (); |
Mohsin Kazmi | 33cc5cf | 2019-01-21 15:19:39 +0000 | [diff] [blame] | 1534 | virtio_main_t *vim = &virtio_main; |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1535 | u32 i = 0; |
| 1536 | |
| 1537 | if (vif->type != VIRTIO_IF_TYPE_PCI) |
| 1538 | return VNET_API_ERROR_INVALID_INTERFACE; |
| 1539 | |
| 1540 | vlib_pci_intr_disable (vm, vif->pci_dev_handle); |
| 1541 | |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 1542 | for (i = 0; i < vif->max_queue_pairs; i++) |
| 1543 | { |
Mohsin Kazmi | a0a6833 | 2020-07-16 12:55:42 +0000 | [diff] [blame] | 1544 | vif->virtio_pci_func->del_queue (vm, vif, RX_QUEUE (i)); |
| 1545 | vif->virtio_pci_func->del_queue (vm, vif, TX_QUEUE (i)); |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 1546 | } |
| 1547 | |
| 1548 | if (vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_CTRL_VQ)) |
Mohsin Kazmi | a0a6833 | 2020-07-16 12:55:42 +0000 | [diff] [blame] | 1549 | vif->virtio_pci_func->del_queue (vm, vif, vif->max_queue_pairs * 2); |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1550 | |
Mohsin Kazmi | c79d735 | 2020-09-01 17:45:56 +0200 | [diff] [blame] | 1551 | if (vif->virtio_pci_func) |
| 1552 | vif->virtio_pci_func->device_reset (vm, vif); |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1553 | |
| 1554 | if (vif->hw_if_index) |
| 1555 | { |
| 1556 | vnet_hw_interface_set_flags (vnm, vif->hw_if_index, 0); |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1557 | ethernet_delete_interface (vnm, vif->hw_if_index); |
| 1558 | } |
| 1559 | |
| 1560 | vlib_pci_device_close (vm, vif->pci_dev_handle); |
| 1561 | |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 1562 | vec_foreach_index (i, vif->rxq_vrings) |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1563 | { |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 1564 | virtio_vring_t *vring = vec_elt_at_index (vif->rxq_vrings, i); |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1565 | if (vring->used) |
| 1566 | { |
Mohsin Kazmi | b977d3f | 2020-11-16 16:49:30 +0100 | [diff] [blame] | 1567 | virtio_free_buffers (vm, vring); |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 1568 | } |
| 1569 | vec_free (vring->buffers); |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 1570 | vlib_physmem_free (vm, vring->desc); |
| 1571 | } |
| 1572 | |
| 1573 | vec_foreach_index (i, vif->txq_vrings) |
| 1574 | { |
| 1575 | virtio_vring_t *vring = vec_elt_at_index (vif->txq_vrings, i); |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 1576 | if (vring->used) |
| 1577 | { |
Mohsin Kazmi | b977d3f | 2020-11-16 16:49:30 +0100 | [diff] [blame] | 1578 | virtio_free_buffers (vm, vring); |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1579 | } |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1580 | vec_free (vring->buffers); |
Mohsin Kazmi | b977d3f | 2020-11-16 16:49:30 +0100 | [diff] [blame] | 1581 | gro_flow_table_free (vring->flow_table); |
| 1582 | virtio_vring_buffering_free (vm, vring->buffering); |
Steven Luong | e558496 | 2019-06-25 13:47:06 -0700 | [diff] [blame] | 1583 | clib_spinlock_free (&vring->lockp); |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1584 | vlib_physmem_free (vm, vring->desc); |
| 1585 | } |
| 1586 | |
Mohsin Kazmi | 09a3bc5 | 2019-04-02 11:45:08 +0000 | [diff] [blame] | 1587 | if (vif->cxq_vring != NULL) |
| 1588 | { |
| 1589 | u16 last = vif->cxq_vring->last_used_idx; |
| 1590 | u16 n_left = vif->cxq_vring->used->idx - last; |
| 1591 | while (n_left) |
| 1592 | { |
| 1593 | last++; |
| 1594 | n_left--; |
| 1595 | } |
| 1596 | |
| 1597 | vif->cxq_vring->last_used_idx = last; |
| 1598 | vlib_physmem_free (vm, vif->cxq_vring->desc); |
| 1599 | } |
| 1600 | |
| 1601 | vec_free (vif->rxq_vrings); |
| 1602 | vec_free (vif->txq_vrings); |
| 1603 | vec_free (vif->cxq_vring); |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1604 | |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1605 | clib_error_free (vif->error); |
| 1606 | memset (vif, 0, sizeof (*vif)); |
Mohsin Kazmi | 33cc5cf | 2019-01-21 15:19:39 +0000 | [diff] [blame] | 1607 | pool_put (vim->interfaces, vif); |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1608 | |
| 1609 | return 0; |
| 1610 | } |
| 1611 | |
Mohsin Kazmi | 6d4af89 | 2020-01-03 15:11:53 +0000 | [diff] [blame] | 1612 | int |
| 1613 | virtio_pci_enable_disable_offloads (vlib_main_t * vm, virtio_if_t * vif, |
| 1614 | int gso_enabled, |
| 1615 | int checksum_offload_enabled, |
| 1616 | int offloads_disabled) |
| 1617 | { |
| 1618 | if (vif->type != VIRTIO_IF_TYPE_PCI) |
| 1619 | return VNET_API_ERROR_INVALID_INTERFACE; |
| 1620 | |
| 1621 | if (gso_enabled) |
| 1622 | virtio_pci_offloads (vm, vif, 1, 0); |
| 1623 | else if (checksum_offload_enabled) |
| 1624 | virtio_pci_offloads (vm, vif, 0, 1); |
| 1625 | else if (offloads_disabled) |
| 1626 | virtio_pci_offloads (vm, vif, 0, 0); |
| 1627 | |
| 1628 | return 0; |
| 1629 | } |
| 1630 | |
Mohsin Kazmi | d6c15af | 2018-10-23 18:00:47 +0200 | [diff] [blame] | 1631 | /* |
| 1632 | * fd.io coding-style-patch-verification: ON |
| 1633 | * |
| 1634 | * Local Variables: |
| 1635 | * eval: (c-set-style "gnu") |
| 1636 | * End: |
| 1637 | */ |