blob: 0ff453eb204984390d54413d166b3d877ab282e9 [file] [log] [blame]
Mohsin Kazmid6c15af2018-10-23 18:00:47 +02001/*
2 * Copyright (c) 2018 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16#include <fcntl.h>
17#include <sys/ioctl.h>
Mohsin Kazmid6c15af2018-10-23 18:00:47 +020018
19#include <vppinfra/types.h>
20#include <vlib/vlib.h>
21#include <vlib/pci/pci.h>
22#include <vnet/ethernet/ethernet.h>
Mohsin Kazmid6c15af2018-10-23 18:00:47 +020023#include <vnet/ip/ip4_packet.h>
24#include <vnet/ip/ip6_packet.h>
25#include <vnet/devices/virtio/virtio.h>
26#include <vnet/devices/virtio/pci.h>
27
28#define PCI_VENDOR_ID_VIRTIO 0x1af4
29#define PCI_DEVICE_ID_VIRTIO_NIC 0x1000
30/* Doesn't support modern device */
31#define PCI_DEVICE_ID_VIRTIO_NIC_MODERN 0x1041
32
33#define PCI_CAPABILITY_LIST 0x34
34#define PCI_CAP_ID_VNDR 0x09
35#define PCI_CAP_ID_MSIX 0x11
36
37#define PCI_MSIX_ENABLE 0x8000
38
39static u32 msix_enabled = 0;
40
41#define PCI_CONFIG_SIZE ((msix_enabled == VIRTIO_MSIX_ENABLED) ? \
42 24 : 20)
43
44static pci_device_id_t virtio_pci_device_ids[] = {
45 {
46 .vendor_id = PCI_VENDOR_ID_VIRTIO,
47 .device_id = PCI_DEVICE_ID_VIRTIO_NIC},
48 {
49 .vendor_id = PCI_VENDOR_ID_VIRTIO,
50 .device_id = PCI_DEVICE_ID_VIRTIO_NIC_MODERN},
51 {0},
52};
53
54static void
55virtio_pci_legacy_read_config (vlib_main_t * vm, virtio_if_t * vif, void *dst,
56 int len, u32 addr)
57{
58 u32 size = 0;
59 vlib_pci_dev_handle_t h = vif->pci_dev_handle;
60
61 while (len > 0)
62 {
63 if (len >= 4)
64 {
65 size = 4;
66 vlib_pci_read_io_u32 (vm, h, PCI_CONFIG_SIZE + addr, dst);
67 }
68 else if (len >= 2)
69 {
70 size = 2;
71 vlib_pci_read_io_u16 (vm, h, PCI_CONFIG_SIZE + addr, dst);
72 }
73 else
74 {
75 size = 1;
76 vlib_pci_read_io_u8 (vm, h, PCI_CONFIG_SIZE + addr, dst);
77 }
78 dst = (u8 *) dst + size;
79 addr += size;
80 len -= size;
81 }
82}
83
84static void
85virtio_pci_legacy_write_config (vlib_main_t * vm, virtio_if_t * vif,
86 void *src, int len, u32 addr)
87{
88 u32 size = 0;
89 vlib_pci_dev_handle_t h = vif->pci_dev_handle;
90
91 while (len > 0)
92 {
93 if (len >= 4)
94 {
95 size = 4;
96 vlib_pci_write_io_u32 (vm, h, PCI_CONFIG_SIZE + addr, src);
97 }
98 else if (len >= 2)
99 {
100 size = 2;
101 vlib_pci_write_io_u16 (vm, h, PCI_CONFIG_SIZE + addr, src);
102 }
103 else
104 {
105 size = 1;
106 vlib_pci_write_io_u8 (vm, h, PCI_CONFIG_SIZE + addr, src);
107 }
108 src = (u8 *) src + size;
109 addr += size;
110 len -= size;
111 }
112}
113
114static u64
115virtio_pci_legacy_get_features (vlib_main_t * vm, virtio_if_t * vif)
116{
117 u32 features;
118 vlib_pci_read_io_u32 (vm, vif->pci_dev_handle, VIRTIO_PCI_HOST_FEATURES,
119 &features);
120 return features;
121}
122
123static u32
124virtio_pci_legacy_set_features (vlib_main_t * vm, virtio_if_t * vif,
125 u64 features)
126{
127 if ((features >> 32) != 0)
128 {
129 clib_warning ("only 32 bit features are allowed for legacy virtio!");
130 }
131 u32 feature = 0, guest_features = (u32) features;
132 vlib_pci_write_io_u32 (vm, vif->pci_dev_handle, VIRTIO_PCI_GUEST_FEATURES,
133 &guest_features);
134 vlib_pci_read_io_u32 (vm, vif->pci_dev_handle, VIRTIO_PCI_GUEST_FEATURES,
135 &feature);
136 return feature;
137}
138
139static u8
140virtio_pci_legacy_get_status (vlib_main_t * vm, virtio_if_t * vif)
141{
142 u8 status = 0;
143 vlib_pci_read_io_u8 (vm, vif->pci_dev_handle, VIRTIO_PCI_STATUS, &status);
144 return status;
145}
146
147static void
148virtio_pci_legacy_set_status (vlib_main_t * vm, virtio_if_t * vif, u8 status)
149{
150 if (status != VIRTIO_CONFIG_STATUS_RESET)
151 status |= virtio_pci_legacy_get_status (vm, vif);
152 vlib_pci_write_io_u8 (vm, vif->pci_dev_handle, VIRTIO_PCI_STATUS, &status);
153}
154
155static u8
156virtio_pci_legacy_reset (vlib_main_t * vm, virtio_if_t * vif)
157{
158 virtio_pci_legacy_set_status (vm, vif, VIRTIO_CONFIG_STATUS_RESET);
159 return virtio_pci_legacy_get_status (vm, vif);
160}
161
162static u8
163virtio_pci_legacy_get_isr (vlib_main_t * vm, virtio_if_t * vif)
164{
165 u8 isr = 0;
166 vlib_pci_read_io_u8 (vm, vif->pci_dev_handle, VIRTIO_PCI_ISR, &isr);
167 return isr;
168}
169
170static u16
171virtio_pci_legacy_get_queue_num (vlib_main_t * vm, virtio_if_t * vif,
172 u16 queue_id)
173{
174 u16 queue_num = 0;
175 vlib_pci_write_io_u16 (vm, vif->pci_dev_handle, VIRTIO_PCI_QUEUE_SEL,
176 &queue_id);
177 vlib_pci_read_io_u16 (vm, vif->pci_dev_handle, VIRTIO_PCI_QUEUE_NUM,
178 &queue_num);
179 return queue_num;
180}
181
182
183static void
184virtio_pci_legacy_setup_queue (vlib_main_t * vm, virtio_if_t * vif,
185 u16 queue_id, void *p)
186{
187 u64 addr = vlib_physmem_get_pa (vm, p) >> VIRTIO_PCI_QUEUE_ADDR_SHIFT;
188 vlib_pci_write_io_u16 (vm, vif->pci_dev_handle, VIRTIO_PCI_QUEUE_SEL,
189 &queue_id);
190 vlib_pci_write_io_u32 (vm, vif->pci_dev_handle, VIRTIO_PCI_QUEUE_PFN,
191 (u32 *) & addr);
192}
193
194static void
195virtio_pci_legacy_del_queue (vlib_main_t * vm, virtio_if_t * vif,
196 u16 queue_id)
197{
198 u32 src = 0;
199 vlib_pci_write_io_u16 (vm, vif->pci_dev_handle, VIRTIO_PCI_QUEUE_SEL,
200 &queue_id);
201 vlib_pci_write_io_u32 (vm, vif->pci_dev_handle, VIRTIO_PCI_QUEUE_PFN, &src);
202}
203
204inline void
205virtio_pci_legacy_notify_queue (vlib_main_t * vm, virtio_if_t * vif,
206 u16 queue_id)
207{
208 vlib_pci_write_io_u16 (vm, vif->pci_dev_handle, VIRTIO_PCI_QUEUE_NOTIFY,
209 &queue_id);
210}
211
212/* Enable one vector (0) for Link State Intrerrupt */
213static u16
214virtio_pci_legacy_set_config_irq (vlib_main_t * vm, virtio_if_t * vif,
215 u16 vec)
216{
217 vlib_pci_write_io_u16 (vm, vif->pci_dev_handle, VIRTIO_MSI_CONFIG_VECTOR,
218 &vec);
219 vlib_pci_read_io_u16 (vm, vif->pci_dev_handle, VIRTIO_MSI_CONFIG_VECTOR,
220 &vec);
221 return vec;
222}
223
224static u16
225virtio_pci_legacy_set_queue_irq (vlib_main_t * vm, virtio_if_t * vif, u16 vec,
226 u16 queue_id)
227{
228 vlib_pci_write_io_u16 (vm, vif->pci_dev_handle, VIRTIO_PCI_QUEUE_SEL,
229 &queue_id);
230 vlib_pci_write_io_u16 (vm, vif->pci_dev_handle, VIRTIO_MSI_QUEUE_VECTOR,
231 &vec);
232 vlib_pci_read_io_u16 (vm, vif->pci_dev_handle, VIRTIO_MSI_QUEUE_VECTOR,
233 &vec);
234 return vec;
235}
236
237static u32
238virtio_pci_flag_change (vnet_main_t * vnm, vnet_hw_interface_t * hw,
239 u32 flags)
240{
241 return 0;
242}
243
244static clib_error_t *
245virtio_pci_get_max_virtqueue_pairs (vlib_main_t * vm, virtio_if_t * vif)
246{
247 virtio_net_config_t config;
248 clib_error_t *error = 0;
249 u16 max_queue_pairs = 1;
250
251 if (vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_MQ))
252 {
253 virtio_pci_legacy_read_config (vm, vif, &config.max_virtqueue_pairs,
254 sizeof (config.max_virtqueue_pairs), 8);
255 max_queue_pairs = config.max_virtqueue_pairs;
256 }
257
258 if (max_queue_pairs < 1 || max_queue_pairs > 0x8000)
259 clib_error_return (error, "max queue pair is %x", max_queue_pairs);
260
261 vif->max_queue_pairs = max_queue_pairs;
262 return error;
263}
264
265static void
266virtio_pci_set_mac (vlib_main_t * vm, virtio_if_t * vif)
267{
268 virtio_pci_legacy_write_config (vm, vif, vif->mac_addr,
269 sizeof (vif->mac_addr), 0);
270}
271
272static u32
273virtio_pci_get_mac (vlib_main_t * vm, virtio_if_t * vif)
274{
275 if (vif->remote_features & VIRTIO_FEATURE (VIRTIO_NET_F_MAC))
276 {
277 virtio_pci_legacy_read_config (vm, vif, vif->mac_addr,
278 sizeof (vif->mac_addr), 0);
279 return 0;
280 }
281 return 1;
282}
283
284static u16
285virtio_pci_is_link_up (vlib_main_t * vm, virtio_if_t * vif)
286{
287 /*
288 * Minimal driver: assumes link is up
289 */
290 u16 status = 1;
291 if (vif->remote_features & VIRTIO_FEATURE (VIRTIO_NET_F_STATUS))
292 virtio_pci_legacy_read_config (vm, vif, &status, sizeof (status), /* mac */
293 6);
294 return status;
295}
296
297static void
298virtio_pci_irq_0_handler (vlib_main_t * vm, vlib_pci_dev_handle_t h, u16 line)
299{
300 vnet_main_t *vnm = vnet_get_main ();
Mohsin Kazmi33cc5cf2019-01-21 15:19:39 +0000301 virtio_main_t *vim = &virtio_main;
Mohsin Kazmid6c15af2018-10-23 18:00:47 +0200302 uword pd = vlib_pci_get_private_data (vm, h);
Mohsin Kazmi33cc5cf2019-01-21 15:19:39 +0000303 virtio_if_t *vif = pool_elt_at_index (vim->interfaces, pd);
Mohsin Kazmid6c15af2018-10-23 18:00:47 +0200304 u16 qid = line;
305
306 vnet_device_input_set_interrupt_pending (vnm, vif->hw_if_index, qid);
307}
308
309static void
310virtio_pci_irq_1_handler (vlib_main_t * vm, vlib_pci_dev_handle_t h, u16 line)
311{
312 vnet_main_t *vnm = vnet_get_main ();
Mohsin Kazmi33cc5cf2019-01-21 15:19:39 +0000313 virtio_main_t *vim = &virtio_main;
Mohsin Kazmid6c15af2018-10-23 18:00:47 +0200314 uword pd = vlib_pci_get_private_data (vm, h);
Mohsin Kazmi33cc5cf2019-01-21 15:19:39 +0000315 virtio_if_t *vif = pool_elt_at_index (vim->interfaces, pd);
Mohsin Kazmid6c15af2018-10-23 18:00:47 +0200316
317 if (virtio_pci_is_link_up (vm, vif) & VIRTIO_NET_S_LINK_UP)
318 {
319 vif->flags |= VIRTIO_IF_FLAG_ADMIN_UP;
320 vnet_hw_interface_set_flags (vnm, vif->hw_if_index,
321 VNET_HW_INTERFACE_FLAG_LINK_UP);
322 }
323 else
324 {
325 vif->flags &= ~VIRTIO_IF_FLAG_ADMIN_UP;
326 vnet_hw_interface_set_flags (vnm, vif->hw_if_index, 0);
327 }
328}
329
330static void
331virtio_pci_irq_handler (vlib_main_t * vm, vlib_pci_dev_handle_t h)
332{
Mohsin Kazmi33cc5cf2019-01-21 15:19:39 +0000333 virtio_main_t *vim = &virtio_main;
Mohsin Kazmid6c15af2018-10-23 18:00:47 +0200334 uword pd = vlib_pci_get_private_data (vm, h);
Mohsin Kazmi33cc5cf2019-01-21 15:19:39 +0000335 virtio_if_t *vif = pool_elt_at_index (vim->interfaces, pd);
Mohsin Kazmid6c15af2018-10-23 18:00:47 +0200336 u8 isr = 0;
337 u16 line = 0;
338
339 isr = virtio_pci_legacy_get_isr (vm, vif);
340
341 /*
342 * If the lower bit is set: look through the used rings of
343 * all virtqueues for the device, to see if any progress has
344 * been made by the device which requires servicing.
345 */
346 if (isr & VIRTIO_PCI_ISR_INTR)
347 virtio_pci_irq_0_handler (vm, h, line);
348
349 if (isr & VIRTIO_PCI_ISR_CONFIG)
350 virtio_pci_irq_1_handler (vm, h, line);
351}
352
353inline void
354device_status (vlib_main_t * vm, virtio_if_t * vif)
355{
356 struct status_struct
357 {
358 u8 bit;
359 char *str;
360 };
361 struct status_struct *status_entry;
362 static struct status_struct status_array[] = {
363#define _(s,b) { .str = #s, .bit = b, },
364 foreach_virtio_config_status_flags
365#undef _
366 {.str = NULL}
367 };
368
369 vlib_cli_output (vm, " status 0x%x", vif->status);
370
371 status_entry = (struct status_struct *) &status_array;
372 while (status_entry->str)
373 {
374 if (vif->status & status_entry->bit)
375 vlib_cli_output (vm, " %s (%x)", status_entry->str,
376 status_entry->bit);
377 status_entry++;
378 }
379}
380
381inline void
382debug_device_config_space (vlib_main_t * vm, virtio_if_t * vif)
383{
384 u32 data_u32;
385 u16 data_u16;
386 u8 data_u8;
387 vlib_pci_read_io_u32 (vm, vif->pci_dev_handle, VIRTIO_PCI_HOST_FEATURES,
388 &data_u32);
389 vlib_cli_output (vm, "remote features 0x%lx", data_u32);
390 vlib_pci_read_io_u32 (vm, vif->pci_dev_handle, VIRTIO_PCI_GUEST_FEATURES,
391 &data_u32);
392 vlib_cli_output (vm, "guest features 0x%lx", data_u32);
393 vlib_pci_read_io_u32 (vm, vif->pci_dev_handle, VIRTIO_PCI_QUEUE_PFN,
394 &data_u32);
395 vlib_cli_output (vm, "queue address 0x%lx", data_u32);
396 vlib_pci_read_io_u16 (vm, vif->pci_dev_handle, VIRTIO_PCI_QUEUE_NUM,
397 &data_u16);
398 vlib_cli_output (vm, "queue size 0x%x", data_u16);
399 vlib_pci_read_io_u16 (vm, vif->pci_dev_handle, VIRTIO_PCI_QUEUE_SEL,
400 &data_u16);
401 vlib_cli_output (vm, "queue select 0x%x", data_u16);
402 vlib_pci_read_io_u16 (vm, vif->pci_dev_handle, VIRTIO_PCI_QUEUE_NOTIFY,
403 &data_u16);
404 vlib_cli_output (vm, "queue notify 0x%x", data_u16);
405 vlib_pci_read_io_u8 (vm, vif->pci_dev_handle, VIRTIO_PCI_STATUS, &data_u8);
406 vlib_cli_output (vm, "status 0x%x", data_u8);
407 vlib_pci_read_io_u8 (vm, vif->pci_dev_handle, VIRTIO_PCI_ISR, &data_u8);
408 vlib_cli_output (vm, "isr 0x%x", data_u8);
409
410 u8 mac[6];
411 virtio_pci_legacy_read_config (vm, vif, mac, sizeof (mac), 0);
412 vlib_cli_output (vm, "mac %U", format_ethernet_address, mac);
413 virtio_pci_legacy_read_config (vm, vif, &data_u16, sizeof (u16), /* offset to status */
414 6);
415 vlib_cli_output (vm, "link up/down status 0x%x", data_u16);
416 virtio_pci_legacy_read_config (vm, vif, &data_u16, sizeof (u16),
417 /* offset to max_virtqueue */ 8);
418 vlib_cli_output (vm, "num of virtqueue 0x%x", data_u16);
419 virtio_pci_legacy_read_config (vm, vif, &data_u16, sizeof (u16), /* offset to mtu */
420 10);
421 vlib_cli_output (vm, "mtu 0x%x", data_u16);
422
423 u32 i = PCI_CONFIG_SIZE + 12, a = 4;
424 i += a;
425 i &= ~a;
426 for (; i < 64; i += 4)
427 {
428 u32 data = 0;
429 vlib_pci_read_io_u32 (vm, vif->pci_dev_handle, i, &data);
430 vlib_cli_output (vm, "0x%lx", data);
431 }
432}
433
434static u8
435virtio_pci_queue_size_valid (u16 qsz)
436{
437 if (qsz < 64 || qsz > 4096)
438 return 0;
439 if ((qsz % 64) != 0)
440 return 0;
441 return 1;
442}
443
444clib_error_t *
445virtio_pci_vring_init (vlib_main_t * vm, virtio_if_t * vif, u16 idx)
446{
447 clib_error_t *error = 0;
448 u16 queue_size = 0;
449 virtio_vring_t *vring;
450 struct vring vr;
451 u32 i = 0;
452 void *ptr;
453
454 queue_size = virtio_pci_legacy_get_queue_num (vm, vif, idx);
455 if (!virtio_pci_queue_size_valid (queue_size))
456 clib_warning ("queue size is not valid");
457
458 if (!is_pow2 (queue_size))
459 return clib_error_return (0, "ring size must be power of 2");
460
461 if (queue_size > 32768)
462 return clib_error_return (0, "ring size must be 32768 or lower");
463
464 if (queue_size == 0)
465 queue_size = 256;
466
467 vec_validate_aligned (vif->vrings, idx, CLIB_CACHE_LINE_BYTES);
468 vring = vec_elt_at_index (vif->vrings, idx);
469
470 i = vring_size (queue_size, VIRTIO_PCI_VRING_ALIGN);
471 i = round_pow2 (i, VIRTIO_PCI_VRING_ALIGN);
472 ptr = vlib_physmem_alloc_aligned (vm, i, VIRTIO_PCI_VRING_ALIGN);
473 memset (ptr, 0, i);
474 vring_init (&vr, queue_size, ptr, VIRTIO_PCI_VRING_ALIGN);
475 vring->desc = vr.desc;
476 vring->avail = vr.avail;
477 vring->used = vr.used;
478 vring->queue_id = idx;
479 vring->avail->flags = VIRTIO_RING_FLAG_MASK_INT;
480
481 ASSERT (vring->buffers == 0);
482 vec_validate_aligned (vring->buffers, queue_size, CLIB_CACHE_LINE_BYTES);
483 ASSERT (vring->indirect_buffers == 0);
484 vec_validate_aligned (vring->indirect_buffers, queue_size,
485 CLIB_CACHE_LINE_BYTES);
486 if (idx % 2)
487 {
488 u32 n_alloc = 0;
489 do
490 {
491 if (n_alloc < queue_size)
492 n_alloc =
493 vlib_buffer_alloc (vm, vring->indirect_buffers + n_alloc,
494 queue_size - n_alloc);
495 }
496 while (n_alloc != queue_size);
497 vif->tx_ring_sz = queue_size;
498 }
499 else
500 vif->rx_ring_sz = queue_size;
501 vring->size = queue_size;
502
503 virtio_pci_legacy_setup_queue (vm, vif, idx, ptr);
504 vring->kick_fd = -1;
505
506 return error;
507}
508
509static void
510virtio_negotiate_features (vlib_main_t * vm, virtio_if_t * vif,
511 u64 req_features)
512{
513 /*
514 * if features are not requested
515 * default: all supported features
516 */
517 u64 supported_features = VIRTIO_FEATURE (VIRTIO_NET_F_MTU)
518 | VIRTIO_FEATURE (VIRTIO_NET_F_MAC)
519 | VIRTIO_FEATURE (VIRTIO_NET_F_MRG_RXBUF)
520 | VIRTIO_FEATURE (VIRTIO_NET_F_STATUS)
521 | VIRTIO_FEATURE (VIRTIO_F_ANY_LAYOUT)
522 | VIRTIO_FEATURE (VIRTIO_RING_F_INDIRECT_DESC);
523
524 if (req_features == 0)
525 {
526 req_features = supported_features;
527 }
528
529 vif->features = req_features & vif->remote_features & supported_features;
530
531 if (vif->
532 remote_features & vif->features & VIRTIO_FEATURE (VIRTIO_NET_F_MTU))
533 {
534 virtio_net_config_t config;
535 virtio_pci_legacy_read_config (vm, vif, &config.mtu,
536 sizeof (config.mtu), 10);
537 if (config.mtu < 64)
538 vif->features &= ~VIRTIO_FEATURE (VIRTIO_NET_F_MTU);
539 }
540
541 vif->features = virtio_pci_legacy_set_features (vm, vif, vif->features);
542}
543
544void
545virtio_pci_read_device_feature (vlib_main_t * vm, virtio_if_t * vif)
546{
547 vif->remote_features = virtio_pci_legacy_get_features (vm, vif);
548}
549
550int
551virtio_pci_reset_device (vlib_main_t * vm, virtio_if_t * vif)
552{
553 u8 status = 0;
554
555 /*
556 * Reset the device
557 */
558 status = virtio_pci_legacy_reset (vm, vif);
559
560 /*
561 * Set the Acknowledge status bit
562 */
563 virtio_pci_legacy_set_status (vm, vif, VIRTIO_CONFIG_STATUS_ACK);
564
565 /*
566 * Set the Driver status bit
567 */
568 virtio_pci_legacy_set_status (vm, vif, VIRTIO_CONFIG_STATUS_DRIVER);
569
570 /*
571 * Read the status and verify it
572 */
573 status = virtio_pci_legacy_get_status (vm, vif);
574 if (!
575 ((status & VIRTIO_CONFIG_STATUS_ACK)
576 && (status & VIRTIO_CONFIG_STATUS_DRIVER)))
577 return -1;
578 vif->status = status;
579
580 return 0;
581}
582
583clib_error_t *
584virtio_pci_read_caps (vlib_main_t * vm, virtio_if_t * vif)
585{
586 clib_error_t *error = 0;
Mohsin Kazmi33cc5cf2019-01-21 15:19:39 +0000587 virtio_main_t *vim = &virtio_main;
Mohsin Kazmid6c15af2018-10-23 18:00:47 +0200588 struct virtio_pci_cap cap;
Mohsin Kazmi22c0ece2019-01-28 19:30:21 +0000589 u8 pos, common_cfg = 0, notify_base = 0, dev_cfg = 0, isr = 0, pci_cfg = 0;
Mohsin Kazmid6c15af2018-10-23 18:00:47 +0200590 vlib_pci_dev_handle_t h = vif->pci_dev_handle;
591
592 if ((error = vlib_pci_read_config_u8 (vm, h, PCI_CAPABILITY_LIST, &pos)))
593 clib_error_return (error, "error in reading capabilty list position");
594
595 while (pos)
596 {
597 if ((error =
598 vlib_pci_read_write_config (vm, h, VLIB_READ, pos, &cap,
599 sizeof (cap))))
600 clib_error_return (error, "error in reading the capability at [%2x]",
601 pos);
602
603 if (cap.cap_vndr == PCI_CAP_ID_MSIX)
604 {
605 u16 flags;
606 if ((error =
607 vlib_pci_read_write_config (vm, h, VLIB_READ, pos + 2, &flags,
608 sizeof (flags))))
609 clib_error_return (error,
610 "error in reading the capability at [%2x]",
611 pos + 2);
612
613 if (flags & PCI_MSIX_ENABLE)
614 msix_enabled = VIRTIO_MSIX_ENABLED;
615 else
616 msix_enabled = VIRTIO_MSIX_DISABLED;
617 }
618
619 if (cap.cap_vndr != PCI_CAP_ID_VNDR)
620 {
Mohsin Kazmi33cc5cf2019-01-21 15:19:39 +0000621 virtio_log_debug (vim, vif, "[%2x] %s %2x ", pos,
622 "skipping non VNDR cap id:", cap.cap_vndr);
Mohsin Kazmid6c15af2018-10-23 18:00:47 +0200623 goto next;
624 }
625
626 switch (cap.cfg_type)
627 {
628 case VIRTIO_PCI_CAP_COMMON_CFG:
629 common_cfg = 1;
630 break;
631 case VIRTIO_PCI_CAP_NOTIFY_CFG:
632 notify_base = 1;
633 break;
634 case VIRTIO_PCI_CAP_DEVICE_CFG:
635 dev_cfg = 1;
636 break;
637 case VIRTIO_PCI_CAP_ISR_CFG:
638 isr = 1;
639 break;
Mohsin Kazmi22c0ece2019-01-28 19:30:21 +0000640 case VIRTIO_PCI_CAP_PCI_CFG:
641 if (cap.bar == 0)
642 pci_cfg = 1;
643 break;
Mohsin Kazmid6c15af2018-10-23 18:00:47 +0200644 }
645 next:
646 pos = cap.cap_next;
647 }
648
Mohsin Kazmi22c0ece2019-01-28 19:30:21 +0000649 if (!pci_cfg)
650 clib_error_return (error, "modern virtio pci device found");
651
Mohsin Kazmid6c15af2018-10-23 18:00:47 +0200652 if (common_cfg == 0 || notify_base == 0 || dev_cfg == 0 || isr == 0)
653 {
Mohsin Kazmi33cc5cf2019-01-21 15:19:39 +0000654 virtio_log_debug (vim, vif, "legacy virtio pci device found");
Mohsin Kazmid6c15af2018-10-23 18:00:47 +0200655 return error;
656 }
657
Mohsin Kazmi22c0ece2019-01-28 19:30:21 +0000658 virtio_log_debug (vim, vif, "transitional virtio pci device found");
Mohsin Kazmi33cc5cf2019-01-21 15:19:39 +0000659 return error;
Mohsin Kazmid6c15af2018-10-23 18:00:47 +0200660}
661
662static clib_error_t *
663virtio_pci_device_init (vlib_main_t * vm, virtio_if_t * vif,
664 virtio_pci_create_if_args_t * args)
665{
666 clib_error_t *error = 0;
667 u8 status = 0;
668
Mohsin Kazmi22c0ece2019-01-28 19:30:21 +0000669 if ((error = virtio_pci_read_caps (vm, vif)))
670 clib_error_return (error, "Device not supported");
Mohsin Kazmid6c15af2018-10-23 18:00:47 +0200671
672 if (virtio_pci_reset_device (vm, vif) < 0)
673 clib_error_return (error, "Failed to reset the device");
674
675 /*
676 * read device features and negotiate (user) requested features
677 */
678 virtio_pci_read_device_feature (vm, vif);
679 virtio_negotiate_features (vm, vif, args->features);
680
681 /*
682 * After FEATURE_OK, driver should not accept new feature bits
683 */
684 virtio_pci_legacy_set_status (vm, vif, VIRTIO_CONFIG_STATUS_FEATURES_OK);
685 status = virtio_pci_legacy_get_status (vm, vif);
686 if (!(status & VIRTIO_CONFIG_STATUS_FEATURES_OK))
687 clib_error_return (error, "Device doesn't support requested features");
688
689 vif->status = status;
690
691 if (virtio_pci_get_mac (vm, vif))
692 {
693 f64 now = vlib_time_now (vm);
694 u32 rnd;
695 rnd = (u32) (now * 1e6);
696 rnd = random_u32 (&rnd);
697
698 memcpy (vif->mac_addr + 2, &rnd, sizeof (rnd));
699 vif->mac_addr[0] = 2;
700 vif->mac_addr[1] = 0xfe;
701 virtio_pci_set_mac (vm, vif);
702 }
703
704 virtio_set_net_hdr_size (vif);
705
706 if ((error = virtio_pci_get_max_virtqueue_pairs (vm, vif)))
707 goto error;
708
709 if ((error = virtio_pci_vring_init (vm, vif, 0)))
710 goto error;
711
712 if ((error = virtio_pci_vring_init (vm, vif, 1)))
713 goto error;
714
715 if (msix_enabled == VIRTIO_MSIX_ENABLED)
716 {
717 virtio_pci_legacy_set_config_irq (vm, vif, VIRTIO_MSI_NO_VECTOR);
718 virtio_pci_legacy_set_queue_irq (vm, vif, VIRTIO_MSI_NO_VECTOR, 0);
719 }
720 virtio_pci_legacy_set_status (vm, vif, VIRTIO_CONFIG_STATUS_DRIVER_OK);
721 vif->status = virtio_pci_legacy_get_status (vm, vif);
722error:
723 return error;
724}
725
726void
727virtio_pci_create_if (vlib_main_t * vm, virtio_pci_create_if_args_t * args)
728{
729 vnet_main_t *vnm = vnet_get_main ();
Mohsin Kazmi33cc5cf2019-01-21 15:19:39 +0000730 virtio_main_t *vim = &virtio_main;
Mohsin Kazmid6c15af2018-10-23 18:00:47 +0200731 virtio_if_t *vif;
732 vlib_pci_dev_handle_t h;
733 clib_error_t *error = 0;
734
735 if (args->rxq_size == 0)
736 args->rxq_size = VIRTIO_NUM_RX_DESC;
737 if (args->txq_size == 0)
738 args->txq_size = VIRTIO_NUM_TX_DESC;
739
740 if (!virtio_pci_queue_size_valid (args->rxq_size) ||
741 !virtio_pci_queue_size_valid (args->txq_size))
742 {
743 args->rv = VNET_API_ERROR_INVALID_VALUE;
744 args->error =
745 clib_error_return (error,
746 "queue size must be <= 4096, >= 64, "
747 "and multiples of 64");
Mohsin Kazmi33cc5cf2019-01-21 15:19:39 +0000748 vlib_log (VLIB_LOG_LEVEL_ERR, vim->log_default, "%U: %s",
749 format_vlib_pci_addr, &args->addr,
750 "queue size must be <= 4096, >= 64, and multiples of 64");
Mohsin Kazmid6c15af2018-10-23 18:00:47 +0200751 return;
752 }
753
754 /* *INDENT-OFF* */
Mohsin Kazmi33cc5cf2019-01-21 15:19:39 +0000755 pool_foreach (vif, vim->interfaces, ({
Mohsin Kazmid6c15af2018-10-23 18:00:47 +0200756 if (vif->pci_addr.as_u32 == args->addr)
757 {
758 args->rv = VNET_API_ERROR_INVALID_VALUE;
759 args->error =
760 clib_error_return (error, "PCI address in use");
Mohsin Kazmi33cc5cf2019-01-21 15:19:39 +0000761 vlib_log (VLIB_LOG_LEVEL_ERR, vim->log_default, "%U: %s",
762 format_vlib_pci_addr, &args->addr,
763 " PCI address in use");
Mohsin Kazmid6c15af2018-10-23 18:00:47 +0200764 return;
765 }
766 }));
767 /* *INDENT-ON* */
768
Mohsin Kazmi33cc5cf2019-01-21 15:19:39 +0000769 pool_get (vim->interfaces, vif);
770 vif->dev_instance = vif - vim->interfaces;
Mohsin Kazmid6c15af2018-10-23 18:00:47 +0200771 vif->per_interface_next_index = ~0;
772 vif->pci_addr.as_u32 = args->addr;
773
774 if ((vif->fd = open ("/dev/vhost-net", O_RDWR | O_NONBLOCK)) < 0)
775 {
776 args->rv = VNET_API_ERROR_SYSCALL_ERROR_1;
777 args->error = clib_error_return_unix (0, "open '/dev/vhost-net'");
778 goto error;
779 }
780
781 if ((error =
782 vlib_pci_device_open (vm, (vlib_pci_addr_t *) & vif->pci_addr,
783 virtio_pci_device_ids, &h)))
784 {
Mohsin Kazmi33cc5cf2019-01-21 15:19:39 +0000785 pool_put (vim->interfaces, vif);
Mohsin Kazmid6c15af2018-10-23 18:00:47 +0200786 args->rv = VNET_API_ERROR_INVALID_INTERFACE;
787 args->error =
788 clib_error_return (error, "pci-addr %U", format_vlib_pci_addr,
789 &vif->pci_addr);
Mohsin Kazmi33cc5cf2019-01-21 15:19:39 +0000790 vlib_log (VLIB_LOG_LEVEL_ERR, vim->log_default, "%U: %s",
791 format_vlib_pci_addr, &vif->pci_addr,
792 "error encountered on pci device open");
Mohsin Kazmid6c15af2018-10-23 18:00:47 +0200793 return;
794 }
795 vif->pci_dev_handle = h;
796 vlib_pci_set_private_data (vm, h, vif->dev_instance);
797
798 if ((error = vlib_pci_bus_master_enable (vm, h)))
Mohsin Kazmi33cc5cf2019-01-21 15:19:39 +0000799 {
800 virtio_log_error (vim, vif,
801 "error encountered on pci bus master enable");
802 goto error;
803 }
Mohsin Kazmid6c15af2018-10-23 18:00:47 +0200804
805 if ((error = vlib_pci_io_region (vm, h, 0)))
Mohsin Kazmi33cc5cf2019-01-21 15:19:39 +0000806 {
807 virtio_log_error (vim, vif, "error encountered on pci io region");
808 goto error;
809 }
Mohsin Kazmid6c15af2018-10-23 18:00:47 +0200810
811 if ((error = virtio_pci_device_init (vm, vif, args)))
Mohsin Kazmi33cc5cf2019-01-21 15:19:39 +0000812 {
813 virtio_log_error (vim, vif, "error encountered on device init");
814 goto error;
815 }
Mohsin Kazmid6c15af2018-10-23 18:00:47 +0200816
817 if (msix_enabled == VIRTIO_MSIX_ENABLED)
818 {
819 if ((error = vlib_pci_register_msix_handler (vm, h, 0, 1,
820 &virtio_pci_irq_0_handler)))
Mohsin Kazmi33cc5cf2019-01-21 15:19:39 +0000821 {
822 virtio_log_error (vim, vif,
823 "error encountered on pci register msix handler 0");
824 goto error;
825 }
Mohsin Kazmid6c15af2018-10-23 18:00:47 +0200826 if ((error = vlib_pci_register_msix_handler (vm, h, 1, 1,
827 &virtio_pci_irq_1_handler)))
Mohsin Kazmi33cc5cf2019-01-21 15:19:39 +0000828 {
829 virtio_log_error (vim, vif,
830 "error encountered on pci register msix handler 1");
831 goto error;
832 }
Mohsin Kazmid6c15af2018-10-23 18:00:47 +0200833
834 if ((error = vlib_pci_enable_msix_irq (vm, h, 0, 2)))
Mohsin Kazmi33cc5cf2019-01-21 15:19:39 +0000835 {
836 virtio_log_error (vim, vif,
837 "error encountered on pci enable msix irq");
838 goto error;
839 }
Mohsin Kazmid6c15af2018-10-23 18:00:47 +0200840 }
841 else
842 {
843 vlib_pci_register_intx_handler (vm, h, &virtio_pci_irq_handler);
844 }
845
846 if ((error = vlib_pci_intr_enable (vm, h)))
Mohsin Kazmi33cc5cf2019-01-21 15:19:39 +0000847 {
848 virtio_log_error (vim, vif,
849 "error encountered on pci interrupt enable");
850 goto error;
851 }
Mohsin Kazmid6c15af2018-10-23 18:00:47 +0200852
853 vif->type = VIRTIO_IF_TYPE_PCI;
854 /* create interface */
855 error = ethernet_register_interface (vnm, virtio_device_class.index,
856 vif->dev_instance, vif->mac_addr,
857 &vif->hw_if_index,
858 virtio_pci_flag_change);
859
860 if (error)
Mohsin Kazmi33cc5cf2019-01-21 15:19:39 +0000861 {
862 virtio_log_error (vim, vif,
863 "error encountered on ethernet register interface");
864 goto error;
865 }
Mohsin Kazmid6c15af2018-10-23 18:00:47 +0200866
867 vnet_sw_interface_t *sw = vnet_get_hw_sw_interface (vnm, vif->hw_if_index);
868 vif->sw_if_index = sw->sw_if_index;
869 args->sw_if_index = sw->sw_if_index;
870
871 vnet_hw_interface_t *hw = vnet_get_hw_interface (vnm, vif->hw_if_index);
872 hw->flags |= VNET_HW_INTERFACE_FLAG_SUPPORTS_INT_MODE;
873 vnet_hw_interface_set_input_node (vnm, vif->hw_if_index,
874 virtio_input_node.index);
875 vnet_hw_interface_assign_rx_thread (vnm, vif->hw_if_index, 0, ~0);
876
877 if (virtio_pci_is_link_up (vm, vif) & VIRTIO_NET_S_LINK_UP)
878 {
879 vif->flags |= VIRTIO_IF_FLAG_ADMIN_UP;
880 vnet_hw_interface_set_flags (vnm, vif->hw_if_index,
881 VNET_HW_INTERFACE_FLAG_LINK_UP);
882 }
883 else
884 vnet_hw_interface_set_flags (vnm, vif->hw_if_index, 0);
885 return;
886
887error:
888 virtio_pci_delete_if (vm, vif);
889 args->rv = VNET_API_ERROR_INVALID_INTERFACE;
890 args->error = error;
891}
892
893int
894virtio_pci_delete_if (vlib_main_t * vm, virtio_if_t * vif)
895{
896 vnet_main_t *vnm = vnet_get_main ();
Mohsin Kazmi33cc5cf2019-01-21 15:19:39 +0000897 virtio_main_t *vim = &virtio_main;
Mohsin Kazmid6c15af2018-10-23 18:00:47 +0200898 u32 i = 0;
899
900 if (vif->type != VIRTIO_IF_TYPE_PCI)
901 return VNET_API_ERROR_INVALID_INTERFACE;
902
903 vlib_pci_intr_disable (vm, vif->pci_dev_handle);
904
905 virtio_pci_legacy_del_queue (vm, vif, 0);
906 virtio_pci_legacy_del_queue (vm, vif, 1);
907
908 virtio_pci_legacy_reset (vm, vif);
909
910 if (vif->hw_if_index)
911 {
912 vnet_hw_interface_set_flags (vnm, vif->hw_if_index, 0);
913 vnet_hw_interface_unassign_rx_thread (vnm, vif->hw_if_index, 0);
914 ethernet_delete_interface (vnm, vif->hw_if_index);
915 }
916
917 vlib_pci_device_close (vm, vif->pci_dev_handle);
918
919 vec_foreach_index (i, vif->vrings)
920 {
921 virtio_vring_t *vring = vec_elt_at_index (vif->vrings, i);
922 if (vring->kick_fd != -1)
923 close (vring->kick_fd);
924 if (vring->used)
925 {
926 if ((i & 1) == 1)
927 virtio_free_used_desc (vm, vring);
928 else
929 virtio_free_rx_buffers (vm, vring);
930 }
931 if (vring->queue_id % 2)
932 {
933 vlib_buffer_free_no_next (vm, vring->indirect_buffers, vring->size);
934 }
935 vec_free (vring->buffers);
936 vec_free (vring->indirect_buffers);
937 vlib_physmem_free (vm, vring->desc);
938 }
939
940 vec_free (vif->vrings);
941
942 if (vif->fd != -1)
943 close (vif->fd);
944 if (vif->tap_fd != -1)
945 vif->tap_fd = -1;
946 clib_error_free (vif->error);
947 memset (vif, 0, sizeof (*vif));
Mohsin Kazmi33cc5cf2019-01-21 15:19:39 +0000948 pool_put (vim->interfaces, vif);
Mohsin Kazmid6c15af2018-10-23 18:00:47 +0200949
950 return 0;
951}
952
953/*
954 * fd.io coding-style-patch-verification: ON
955 *
956 * Local Variables:
957 * eval: (c-set-style "gnu")
958 * End:
959 */