blob: 6078be2537e17ce0570f48961035f03db95f428d [file] [log] [blame]
Ed Warnickecb9cada2015-12-08 15:45:58 -07001/*
Damjan Marion5a206ea2016-05-12 22:11:03 +02002 * Copyright (c) 2016 Cisco and/or its affiliates.
Ed Warnickecb9cada2015-12-08 15:45:58 -07003 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15/*
16 * pci.c: Linux user space PCI bus management.
17 *
18 * Copyright (c) 2008 Eliot Dresselhaus
19 *
20 * Permission is hereby granted, free of charge, to any person obtaining
21 * a copy of this software and associated documentation files (the
22 * "Software"), to deal in the Software without restriction, including
23 * without limitation the rights to use, copy, modify, merge, publish,
24 * distribute, sublicense, and/or sell copies of the Software, and to
25 * permit persons to whom the Software is furnished to do so, subject to
26 * the following conditions:
27 *
28 * The above copyright notice and this permission notice shall be
29 * included in all copies or substantial portions of the Software.
30 *
31 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
32 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
33 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
34 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
35 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
36 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
37 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
38 */
39
Damjan Marion01914ce2017-09-14 19:04:50 +020040#include <vppinfra/linux/sysfs.h>
41
Ed Warnickecb9cada2015-12-08 15:45:58 -070042#include <vlib/vlib.h>
43#include <vlib/pci/pci.h>
44#include <vlib/unix/unix.h>
Damjan Marion1ba0fa42018-03-04 17:19:08 +010045#include <vlib/linux/vfio.h>
Ed Warnickecb9cada2015-12-08 15:45:58 -070046
47#include <sys/types.h>
48#include <sys/stat.h>
49#include <fcntl.h>
50#include <dirent.h>
Damjan Mariona42cd342016-04-13 18:03:20 +020051#include <sys/ioctl.h>
52#include <net/if.h>
53#include <linux/ethtool.h>
54#include <linux/sockios.h>
Damjan Marioncef87f12017-10-05 15:32:41 +020055#include <linux/vfio.h>
56#include <sys/eventfd.h>
57
58static const char *sysfs_pci_dev_path = "/sys/bus/pci/devices";
59static const char *sysfs_pci_drv_path = "/sys/bus/pci/drivers";
Damjan Marion2752b892017-12-11 15:55:56 +010060static char *sysfs_mod_vfio_noiommu =
61 "/sys/module/vfio/parameters/enable_unsafe_noiommu_mode";
Damjan Mariona42cd342016-04-13 18:03:20 +020062
Damjan Marion23227982018-10-22 13:38:57 +020063#define pci_log_debug(vm, dev, f, ...) \
64 vlib_log(VLIB_LOG_LEVEL_DEBUG, pci_main.log_default, "%U: " f, \
65 format_vlib_pci_addr, vlib_pci_get_addr(vm, dev->handle), ## __VA_ARGS__)
66#define pci_log_err(vm, dev, f, ...) \
67 vlib_log(VLIB_LOG_LEVEL_ERR, pci_main.log_default, "%U: " f, \
68 format_vlib_pci_addr, vlib_pci_get_addr(vm, dev->handle), ## __VA_ARGS__)
69
Dave Barach9b8ffd92016-07-08 08:13:45 -040070typedef struct
71{
Damjan Marion2060db82018-03-04 17:37:15 +010072 int fd;
73 void *addr;
74 size_t size;
75} linux_pci_region_t;
76
Damjan Marion599a16b2018-03-04 19:35:23 +010077typedef struct
78{
79 int fd;
80 u32 clib_file_index;
81 union
82 {
Damjan Mariond5ded2d2018-03-05 10:18:50 +010083 pci_intx_handler_function_t *intx_handler;
Damjan Marion599a16b2018-03-04 19:35:23 +010084 pci_msix_handler_function_t *msix_handler;
85 };
86} linux_pci_irq_t;
Damjan Marion2060db82018-03-04 17:37:15 +010087
88typedef enum
89{
90 LINUX_PCI_DEVICE_TYPE_UNKNOWN,
91 LINUX_PCI_DEVICE_TYPE_UIO,
92 LINUX_PCI_DEVICE_TYPE_VFIO,
93} linux_pci_device_type_t;
94
95typedef struct
96{
97 linux_pci_device_type_t type;
Damjan Marioncef87f12017-10-05 15:32:41 +020098 vlib_pci_dev_handle_t handle;
99 vlib_pci_addr_t addr;
Damjan Mariond2bfb782019-01-07 20:56:04 +0100100 u32 numa_node;
Damjan Marion5a206ea2016-05-12 22:11:03 +0200101
102 /* Resource file descriptors. */
Damjan Marion2060db82018-03-04 17:37:15 +0100103 linux_pci_region_t *regions;
Damjan Marion5a206ea2016-05-12 22:11:03 +0200104
105 /* File descriptor for config space read/write. */
106 int config_fd;
Damjan Marionf9a968e2018-02-25 23:26:22 +0100107 u64 config_offset;
Damjan Marion5a206ea2016-05-12 22:11:03 +0200108
Damjan Marionf9a968e2018-02-25 23:26:22 +0100109 /* Device File descriptor */
110 int fd;
Damjan Marion5a206ea2016-05-12 22:11:03 +0200111
Mohsin Kazmi95276ca2018-09-28 11:45:01 +0200112 /* read/write file descriptor for io bar */
113 int io_fd;
114 u64 io_offset;
115
Damjan Marion5a206ea2016-05-12 22:11:03 +0200116 /* Minor device for uio device. */
117 u32 uio_minor;
118
Damjan Mariond5ded2d2018-03-05 10:18:50 +0100119 /* Interrupt handlers */
120 linux_pci_irq_t intx_irq;
Damjan Marion599a16b2018-03-04 19:35:23 +0100121 linux_pci_irq_t *msix_irqs;
Damjan Marioncef87f12017-10-05 15:32:41 +0200122
123 /* private data */
124 uword private_data;
125
Damjan Marion23227982018-10-22 13:38:57 +0200126 u8 supports_va_dma;
127
Damjan Marion5a206ea2016-05-12 22:11:03 +0200128} linux_pci_device_t;
129
130/* Pool of PCI devices. */
Dave Barach9b8ffd92016-07-08 08:13:45 -0400131typedef struct
132{
133 vlib_main_t *vlib_main;
134 linux_pci_device_t *linux_pci_devices;
Damjan Marioncef87f12017-10-05 15:32:41 +0200135
Damjan Marion5a206ea2016-05-12 22:11:03 +0200136} linux_pci_main_t;
137
138extern linux_pci_main_t linux_pci_main;
139
Damjan Marion599a16b2018-03-04 19:35:23 +0100140static linux_pci_device_t *
Damjan Marioncef87f12017-10-05 15:32:41 +0200141linux_pci_get_device (vlib_pci_dev_handle_t h)
142{
143 linux_pci_main_t *lpm = &linux_pci_main;
144 return pool_elt_at_index (lpm->linux_pci_devices, h);
145}
146
147uword
Damjan Marion23227982018-10-22 13:38:57 +0200148vlib_pci_get_private_data (vlib_main_t * vm, vlib_pci_dev_handle_t h)
Damjan Marioncef87f12017-10-05 15:32:41 +0200149{
150 linux_pci_device_t *d = linux_pci_get_device (h);
151 return d->private_data;
152}
153
154void
Damjan Marion23227982018-10-22 13:38:57 +0200155vlib_pci_set_private_data (vlib_main_t * vm, vlib_pci_dev_handle_t h,
156 uword private_data)
Damjan Marioncef87f12017-10-05 15:32:41 +0200157{
158 linux_pci_device_t *d = linux_pci_get_device (h);
159 d->private_data = private_data;
160}
161
162vlib_pci_addr_t *
Damjan Marion23227982018-10-22 13:38:57 +0200163vlib_pci_get_addr (vlib_main_t * vm, vlib_pci_dev_handle_t h)
Damjan Marioncef87f12017-10-05 15:32:41 +0200164{
Damjan Marion599a16b2018-03-04 19:35:23 +0100165 linux_pci_device_t *d = linux_pci_get_device (h);
166 return &d->addr;
Damjan Marioncef87f12017-10-05 15:32:41 +0200167}
168
Damjan Mariond2bfb782019-01-07 20:56:04 +0100169u32
170vlib_pci_get_numa_node (vlib_main_t * vm, vlib_pci_dev_handle_t h)
171{
172 linux_pci_device_t *d = linux_pci_get_device (h);
173 return d->numa_node;
174}
175
Mohsin Kazmia402b832019-01-25 15:57:24 +0000176u32
177vlib_pci_get_num_msix_interrupts (vlib_main_t * vm, vlib_pci_dev_handle_t h)
178{
179 linux_pci_device_t *d = linux_pci_get_device (h);
180
181 if (d->type == LINUX_PCI_DEVICE_TYPE_VFIO)
182 {
183 struct vfio_irq_info ii = { 0 };
184
185 ii.argsz = sizeof (struct vfio_irq_info);
186 ii.index = VFIO_PCI_MSIX_IRQ_INDEX;
187 if (ioctl (d->fd, VFIO_DEVICE_GET_IRQ_INFO, &ii) < 0)
188 return 0;
189 return ii.count;
190 }
191 return 0;
192}
193
Damjan Marion5a206ea2016-05-12 22:11:03 +0200194/* Call to allocate/initialize the pci subsystem.
195 This is not an init function so that users can explicitly enable
196 pci only when it's needed. */
Dave Barach9b8ffd92016-07-08 08:13:45 -0400197clib_error_t *pci_bus_init (vlib_main_t * vm);
Damjan Marion5a206ea2016-05-12 22:11:03 +0200198
Ed Warnickecb9cada2015-12-08 15:45:58 -0700199linux_pci_main_t linux_pci_main;
200
Damjan Marioncef87f12017-10-05 15:32:41 +0200201vlib_pci_device_info_t *
Damjan Marion23227982018-10-22 13:38:57 +0200202vlib_pci_get_device_info (vlib_main_t * vm, vlib_pci_addr_t * addr,
203 clib_error_t ** error)
Damjan Marioncef87f12017-10-05 15:32:41 +0200204{
Damjan Marioncef87f12017-10-05 15:32:41 +0200205 clib_error_t *err;
206 vlib_pci_device_info_t *di;
207 u8 *f = 0;
208 u32 tmp;
209 int fd;
Yulong Pei45495482019-10-17 18:41:52 +0800210 u8 *tmpstr;
Jieqiang Wang76c61592020-01-13 17:15:13 +0800211 clib_bitmap_t *bmp = 0;
Damjan Marioncef87f12017-10-05 15:32:41 +0200212
213 di = clib_mem_alloc (sizeof (vlib_pci_device_info_t));
Dave Barachb7b92992018-10-17 10:38:51 -0400214 clib_memset (di, 0, sizeof (vlib_pci_device_info_t));
Damjan Marioncef87f12017-10-05 15:32:41 +0200215 di->addr.as_u32 = addr->as_u32;
216
217 u8 *dev_dir_name = format (0, "%s/%U", sysfs_pci_dev_path,
218 format_vlib_pci_addr, addr);
219
220 f = format (0, "%v/config%c", dev_dir_name, 0);
221 fd = open ((char *) f, O_RDWR);
222
223 /* Try read-only access if write fails. */
224 if (fd < 0)
225 fd = open ((char *) f, O_RDONLY);
226
227 if (fd < 0)
228 {
229 err = clib_error_return_unix (0, "open `%s'", f);
230 goto error;
231 }
232
233 /* You can only read more that 64 bytes of config space as root; so we try to
234 read the full space but fall back to just the first 64 bytes. */
Damjan Marion44bcc202018-03-04 16:44:26 +0100235 if (read (fd, &di->config_data, sizeof (di->config_data)) <
236 sizeof (di->config0))
Damjan Marioncef87f12017-10-05 15:32:41 +0200237 {
238 err = clib_error_return_unix (0, "read `%s'", f);
239 close (fd);
240 goto error;
241 }
242
243 {
244 static pci_config_header_t all_ones;
245 if (all_ones.vendor_id == 0)
Dave Barachb7b92992018-10-17 10:38:51 -0400246 clib_memset (&all_ones, ~0, sizeof (all_ones));
Damjan Marioncef87f12017-10-05 15:32:41 +0200247
248 if (!memcmp (&di->config0.header, &all_ones, sizeof (all_ones)))
249 {
250 err = clib_error_return (0, "invalid PCI config for `%s'", f);
251 close (fd);
252 goto error;
253 }
254 }
255
256 if (di->config0.header.header_type == 0)
257 pci_config_type0_little_to_host (&di->config0);
258 else
259 pci_config_type1_little_to_host (&di->config1);
260
261 di->numa_node = -1;
262 vec_reset_length (f);
263 f = format (f, "%v/numa_node%c", dev_dir_name, 0);
Jieqiang Wang76c61592020-01-13 17:15:13 +0800264 err = clib_sysfs_read ((char *) f, "%d", &di->numa_node);
Damjan Marioncef87f12017-10-05 15:32:41 +0200265 if (err)
266 {
267 di->numa_node = -1;
268 clib_error_free (err);
269 }
Jieqiang Wang76c61592020-01-13 17:15:13 +0800270 if (di->numa_node == -1)
271 {
272 /* if '/sys/bus/pci/devices/<device id>/numa_node' returns -1 and
273 it is a SMP system, set numa_node to 0. */
274 if ((err = clib_sysfs_read ("/sys/devices/system/node/online", "%U",
275 unformat_bitmap_list, &bmp)))
276 clib_error_free (err);
277 if (clib_bitmap_count_set_bits (bmp) == 1)
278 di->numa_node = 0;
279 }
Damjan Marioncef87f12017-10-05 15:32:41 +0200280
281 vec_reset_length (f);
282 f = format (f, "%v/class%c", dev_dir_name, 0);
283 err = clib_sysfs_read ((char *) f, "0x%x", &tmp);
284 if (err)
285 goto error;
286 di->device_class = tmp >> 8;
287
288 vec_reset_length (f);
289 f = format (f, "%v/vendor%c", dev_dir_name, 0);
290 err = clib_sysfs_read ((char *) f, "0x%x", &tmp);
291 if (err)
292 goto error;
293 di->vendor_id = tmp;
294
295 vec_reset_length (f);
296 f = format (f, "%v/device%c", dev_dir_name, 0);
297 err = clib_sysfs_read ((char *) f, "0x%x", &tmp);
298 if (err)
299 goto error;
300 di->device_id = tmp;
301
302 vec_reset_length (f);
303 f = format (f, "%v/driver%c", dev_dir_name, 0);
304 di->driver_name = clib_sysfs_link_to_name ((char *) f);
Benoît Ganne0eae2bb2019-10-04 17:30:21 +0200305 if (!di->driver_name)
306 di->driver_name = format (0, "<NONE>%c", 0);
Damjan Marioncef87f12017-10-05 15:32:41 +0200307
308 di->iommu_group = -1;
Yulong Pei45495482019-10-17 18:41:52 +0800309 vec_reset_length (f);
310 f = format (f, "%v/iommu_group%c", dev_dir_name, 0);
311 tmpstr = clib_sysfs_link_to_name ((char *) f);
312 if (tmpstr)
Damjan Marioncef87f12017-10-05 15:32:41 +0200313 {
Yulong Pei45495482019-10-17 18:41:52 +0800314 di->iommu_group = atoi ((char *) tmpstr);
315 vec_free (tmpstr);
Damjan Marioncef87f12017-10-05 15:32:41 +0200316 }
317
Yulong Pei45495482019-10-17 18:41:52 +0800318 vec_reset_length (f);
319 f = format (f, "%v/iommu_group/name%c", dev_dir_name, 0);
320 err = clib_sysfs_read ((char *) f, "%s", &tmpstr);
321 if (err == 0)
322 {
323 if (strncmp ((char *) tmpstr, "vfio-noiommu", 12) == 0)
324 di->flags |= VLIB_PCI_DEVICE_INFO_F_NOIOMMU;
325 vec_free (tmpstr);
326 }
327 else
328 clib_error_free (err);
329
Damjan Marion96504182017-12-10 23:54:46 +0100330 close (fd);
331
Damjan Marioncef87f12017-10-05 15:32:41 +0200332 vec_reset_length (f);
333 f = format (f, "%v/vpd%c", dev_dir_name, 0);
334 fd = open ((char *) f, O_RDONLY);
335 if (fd >= 0)
336 {
337 while (1)
338 {
339 u8 tag[3];
340 u8 *data = 0;
Damjan Marion96504182017-12-10 23:54:46 +0100341 uword len;
Damjan Marioncef87f12017-10-05 15:32:41 +0200342
343 if (read (fd, &tag, 3) != 3)
344 break;
345
346 if (tag[0] != 0x82 && tag[0] != 0x90 && tag[0] != 0x91)
347 break;
348
349 len = (tag[2] << 8) | tag[1];
350 vec_validate (data, len);
351
352 if (read (fd, data, len) != len)
353 {
354 vec_free (data);
355 break;
356 }
357 if (tag[0] == 0x82)
358 di->product_name = data;
359 else if (tag[0] == 0x90)
360 di->vpd_r = data;
361 else if (tag[0] == 0x91)
362 di->vpd_w = data;
363
364 data = 0;
365 }
366 close (fd);
367 }
368
369 goto done;
370
371error:
372 vlib_pci_free_device_info (di);
373 di = 0;
374
375done:
Jieqiang Wang76c61592020-01-13 17:15:13 +0800376 vec_free (bmp);
Damjan Marioncef87f12017-10-05 15:32:41 +0200377 vec_free (f);
378 vec_free (dev_dir_name);
379 if (error)
380 *error = err;
381 else
382 clib_error_free (err);
383 return di;
384}
385
Damjan Marion2752b892017-12-11 15:55:56 +0100386static int
387directory_exists (char *path)
388{
389 struct stat s = { 0 };
390 if (stat (path, &s) == -1)
391 return 0;
392
393 return S_ISDIR (s.st_mode);
394}
395
Damjan Mariona42cd342016-04-13 18:03:20 +0200396clib_error_t *
Damjan Marion23227982018-10-22 13:38:57 +0200397vlib_pci_bind_to_uio (vlib_main_t * vm, vlib_pci_addr_t * addr,
398 char *uio_drv_name)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700399{
Dave Barach9b8ffd92016-07-08 08:13:45 -0400400 clib_error_t *error = 0;
Damjan Mariona7f74572017-05-23 18:32:21 +0200401 u8 *s = 0, *driver_name = 0;
Damjan Mariona42cd342016-04-13 18:03:20 +0200402 DIR *dir = 0;
403 struct dirent *e;
Damjan Marioncef87f12017-10-05 15:32:41 +0200404 vlib_pci_device_info_t *di;
Damjan Mariona7f74572017-05-23 18:32:21 +0200405 int fd, clear_driver_override = 0;
Damjan Marioncef87f12017-10-05 15:32:41 +0200406 u8 *dev_dir_name = format (0, "%s/%U", sysfs_pci_dev_path,
407 format_vlib_pci_addr, addr);
408
Damjan Marion23227982018-10-22 13:38:57 +0200409 di = vlib_pci_get_device_info (vm, addr, &error);
Damjan Marioncef87f12017-10-05 15:32:41 +0200410
411 if (error)
412 return error;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700413
Damjan Marion2752b892017-12-11 15:55:56 +0100414 if (strncmp ("auto", uio_drv_name, 5) == 0)
415 {
416 int vfio_pci_loaded = 0;
417
418 if (directory_exists ("/sys/module/vfio_pci"))
419 vfio_pci_loaded = 1;
420
421 if (di->iommu_group != -1)
422 {
423 /* device is bound to IOMMU group */
424 if (!vfio_pci_loaded)
425 {
426 error = clib_error_return (0, "Skipping PCI device %U: device "
427 "is bound to IOMMU group and "
428 "vfio-pci driver is not loaded",
429 format_vlib_pci_addr, addr);
430 goto done;
431 }
432 else
433 uio_drv_name = "vfio-pci";
434 }
435 else
436 {
437 /* device is not bound to IOMMU group so we have multiple options */
438 if (vfio_pci_loaded &&
439 (error = clib_sysfs_write (sysfs_mod_vfio_noiommu, "Y")) == 0)
440 uio_drv_name = "vfio-pci";
441 else if (directory_exists ("/sys/module/uio_pci_generic"))
442 uio_drv_name = "uio_pci_generic";
443 else if (directory_exists ("/sys/module/igb_uio"))
444 uio_drv_name = "igb_uio";
445 else
446 {
447 clib_error_free (error);
448 error = clib_error_return (0, "Skipping PCI device %U: missing "
449 "kernel VFIO or UIO driver",
450 format_vlib_pci_addr, addr);
451 goto done;
452 }
453 clib_error_free (error);
454 }
455 }
456
Damjan Mariona7f74572017-05-23 18:32:21 +0200457 s = format (s, "%v/driver%c", dev_dir_name, 0);
Damjan Marion01914ce2017-09-14 19:04:50 +0200458 driver_name = clib_sysfs_link_to_name ((char *) s);
Damjan Mariona42cd342016-04-13 18:03:20 +0200459 vec_reset_length (s);
460
Damjan Mariona7f74572017-05-23 18:32:21 +0200461 if (driver_name &&
462 ((strcmp ("vfio-pci", (char *) driver_name) == 0) ||
463 (strcmp ("uio_pci_generic", (char *) driver_name) == 0) ||
464 (strcmp ("igb_uio", (char *) driver_name) == 0)))
Damjan Marion3c785e02017-05-08 18:37:54 +0200465 goto done;
Damjan Marion3c785e02017-05-08 18:37:54 +0200466
Damjan Mariona42cd342016-04-13 18:03:20 +0200467 /* walk trough all linux interfaces and if interface belonging to
468 this device is founf check if interface is admin up */
469 dir = opendir ("/sys/class/net");
Damjan Marioncef87f12017-10-05 15:32:41 +0200470 s = format (s, "%U%c", format_vlib_pci_addr, addr, 0);
Damjan Mariona42cd342016-04-13 18:03:20 +0200471
472 if (!dir)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700473 {
Damjan Mariona42cd342016-04-13 18:03:20 +0200474 error = clib_error_return (0, "Skipping PCI device %U: failed to "
475 "read /sys/class/net",
Damjan Marioncef87f12017-10-05 15:32:41 +0200476 format_vlib_pci_addr, addr);
Damjan Mariona42cd342016-04-13 18:03:20 +0200477 goto done;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700478 }
479
Dave Barach9b8ffd92016-07-08 08:13:45 -0400480 fd = socket (PF_INET, SOCK_DGRAM, 0);
Chris Luke370e9e32016-07-07 11:01:17 -0400481 if (fd < 0)
482 {
483 error = clib_error_return_unix (0, "socket");
484 goto done;
485 }
Damjan Mariona42cd342016-04-13 18:03:20 +0200486
Dave Barach9b8ffd92016-07-08 08:13:45 -0400487 while ((e = readdir (dir)))
Ed Warnickecb9cada2015-12-08 15:45:58 -0700488 {
Damjan Mariona42cd342016-04-13 18:03:20 +0200489 struct ifreq ifr;
490 struct ethtool_drvinfo drvinfo;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700491
Dave Barach9b8ffd92016-07-08 08:13:45 -0400492 if (e->d_name[0] == '.') /* skip . and .. */
Damjan Mariona42cd342016-04-13 18:03:20 +0200493 continue;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700494
Dave Barachb7b92992018-10-17 10:38:51 -0400495 clib_memset (&ifr, 0, sizeof ifr);
496 clib_memset (&drvinfo, 0, sizeof drvinfo);
Damjan Mariona42cd342016-04-13 18:03:20 +0200497 ifr.ifr_data = (char *) &drvinfo;
Steven Luongddccf7b2019-01-18 22:53:18 -0800498 clib_strncpy (ifr.ifr_name, e->d_name, sizeof (ifr.ifr_name) - 1);
Stephen Hemminger4e8a8312019-01-16 10:23:16 -0800499
Damjan Mariona42cd342016-04-13 18:03:20 +0200500 drvinfo.cmd = ETHTOOL_GDRVINFO;
Chris Luke370e9e32016-07-07 11:01:17 -0400501 if (ioctl (fd, SIOCETHTOOL, &ifr) < 0)
Dave Barach9b8ffd92016-07-08 08:13:45 -0400502 {
John Loe2821212016-07-13 18:13:02 -0400503 /* Some interfaces (eg "lo") don't support this ioctl */
504 if ((errno != ENOTSUP) && (errno != ENODEV))
Ed Warnicke853e7202016-08-12 11:42:26 -0700505 clib_unix_warning ("ioctl fetch intf %s bus info error",
506 e->d_name);
John Loe2821212016-07-13 18:13:02 -0400507 continue;
Dave Barach9b8ffd92016-07-08 08:13:45 -0400508 }
Damjan Mariona42cd342016-04-13 18:03:20 +0200509
510 if (strcmp ((char *) s, drvinfo.bus_info))
511 continue;
512
Dave Barachb7b92992018-10-17 10:38:51 -0400513 clib_memset (&ifr, 0, sizeof (ifr));
Steven Luongddccf7b2019-01-18 22:53:18 -0800514 clib_strncpy (ifr.ifr_name, e->d_name, sizeof (ifr.ifr_name) - 1);
Stephen Hemminger4e8a8312019-01-16 10:23:16 -0800515
Chris Luke370e9e32016-07-07 11:01:17 -0400516 if (ioctl (fd, SIOCGIFFLAGS, &ifr) < 0)
Dave Barach9b8ffd92016-07-08 08:13:45 -0400517 {
518 error = clib_error_return_unix (0, "ioctl fetch intf %s flags",
519 e->d_name);
520 close (fd);
521 goto done;
522 }
Damjan Mariona42cd342016-04-13 18:03:20 +0200523
524 if (ifr.ifr_flags & IFF_UP)
525 {
526 error = clib_error_return (0, "Skipping PCI device %U as host "
527 "interface %s is up",
Damjan Marioncef87f12017-10-05 15:32:41 +0200528 format_vlib_pci_addr, addr, e->d_name);
Dave Barach9b8ffd92016-07-08 08:13:45 -0400529 close (fd);
Damjan Mariona42cd342016-04-13 18:03:20 +0200530 goto done;
531 }
Ed Warnickecb9cada2015-12-08 15:45:58 -0700532 }
533
Damjan Mariona42cd342016-04-13 18:03:20 +0200534 close (fd);
535 vec_reset_length (s);
536
537 s = format (s, "%v/driver/unbind%c", dev_dir_name, 0);
Damjan Marioncef87f12017-10-05 15:32:41 +0200538 clib_sysfs_write ((char *) s, "%U", format_vlib_pci_addr, addr);
Damjan Mariona42cd342016-04-13 18:03:20 +0200539 vec_reset_length (s);
540
Damjan Mariona7f74572017-05-23 18:32:21 +0200541 s = format (s, "%v/driver_override%c", dev_dir_name, 0);
542 if (access ((char *) s, F_OK) == 0)
543 {
Damjan Marioncef87f12017-10-05 15:32:41 +0200544 clib_sysfs_write ((char *) s, "%s", uio_drv_name);
Damjan Mariona7f74572017-05-23 18:32:21 +0200545 clear_driver_override = 1;
546 }
547 else
548 {
549 vec_reset_length (s);
Damjan Marioncef87f12017-10-05 15:32:41 +0200550 s = format (s, "%s/%s/new_id%c", sysfs_pci_drv_path, uio_drv_name, 0);
551 clib_sysfs_write ((char *) s, "0x%04x 0x%04x", di->vendor_id,
552 di->device_id);
Damjan Mariona7f74572017-05-23 18:32:21 +0200553 }
Damjan Mariona42cd342016-04-13 18:03:20 +0200554 vec_reset_length (s);
555
Damjan Marioncef87f12017-10-05 15:32:41 +0200556 s = format (s, "%s/%s/bind%c", sysfs_pci_drv_path, uio_drv_name, 0);
557 clib_sysfs_write ((char *) s, "%U", format_vlib_pci_addr, addr);
Damjan Mariona7f74572017-05-23 18:32:21 +0200558 vec_reset_length (s);
559
560 if (clear_driver_override)
561 {
562 s = format (s, "%v/driver_override%c", dev_dir_name, 0);
Damjan Marion01914ce2017-09-14 19:04:50 +0200563 clib_sysfs_write ((char *) s, "%c", 0);
Damjan Mariona7f74572017-05-23 18:32:21 +0200564 vec_reset_length (s);
565 }
Damjan Mariona42cd342016-04-13 18:03:20 +0200566
567done:
568 closedir (dir);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700569 vec_free (s);
Damjan Mariona42cd342016-04-13 18:03:20 +0200570 vec_free (dev_dir_name);
Damjan Mariona7f74572017-05-23 18:32:21 +0200571 vec_free (driver_name);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700572 return error;
573}
574
Ed Warnickecb9cada2015-12-08 15:45:58 -0700575
576static clib_error_t *
Dave Barach9b8ffd92016-07-08 08:13:45 -0400577scan_uio_dir (void *arg, u8 * path_name, u8 * file_name)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700578{
Dave Barach9b8ffd92016-07-08 08:13:45 -0400579 linux_pci_device_t *l = arg;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700580 unformat_input_t input;
581
582 unformat_init_string (&input, (char *) file_name, vec_len (file_name));
583
Dave Barach9b8ffd92016-07-08 08:13:45 -0400584 if (!unformat (&input, "uio%d", &l->uio_minor))
Ed Warnickecb9cada2015-12-08 15:45:58 -0700585 abort ();
586
587 unformat_free (&input);
588 return 0;
589}
590
Dave Barach9b8ffd92016-07-08 08:13:45 -0400591static clib_error_t *
Damjan Marion23227982018-10-22 13:38:57 +0200592vfio_set_irqs (vlib_main_t * vm, linux_pci_device_t * p, u32 index, u32 start,
593 u32 count, u32 flags, int *efds)
Damjan Marion599a16b2018-03-04 19:35:23 +0100594{
595 int data_len = efds ? count * sizeof (int) : 0;
596 u8 buf[sizeof (struct vfio_irq_set) + data_len];
Damjan Marion23227982018-10-22 13:38:57 +0200597 struct vfio_irq_info ii = { 0 };
Damjan Marion599a16b2018-03-04 19:35:23 +0100598 struct vfio_irq_set *irq_set = (struct vfio_irq_set *) buf;
599
600
Damjan Marion23227982018-10-22 13:38:57 +0200601 ii.argsz = sizeof (struct vfio_irq_info);
602 ii.index = index;
Damjan Marion599a16b2018-03-04 19:35:23 +0100603
Damjan Marion23227982018-10-22 13:38:57 +0200604 if (ioctl (p->fd, VFIO_DEVICE_GET_IRQ_INFO, &ii) < 0)
Damjan Marion599a16b2018-03-04 19:35:23 +0100605 return clib_error_return_unix (0, "ioctl(VFIO_DEVICE_GET_IRQ_INFO) "
606 "'%U'", format_vlib_pci_addr, &p->addr);
607
Damjan Marion23227982018-10-22 13:38:57 +0200608 pci_log_debug (vm, p, "%s index:%u count:%u flags: %s%s%s%s(0x%x)",
609 __func__, ii.index, ii.count,
610 ii.flags & VFIO_IRQ_INFO_EVENTFD ? "eventfd " : "",
611 ii.flags & VFIO_IRQ_INFO_MASKABLE ? "maskable " : "",
612 ii.flags & VFIO_IRQ_INFO_AUTOMASKED ? "automasked " : "",
613 ii.flags & VFIO_IRQ_INFO_NORESIZE ? "noresize " : "",
614 ii.flags);
615
616 if (ii.count < start + count)
Damjan Marion599a16b2018-03-04 19:35:23 +0100617 return clib_error_return_unix (0, "vfio_set_irq: unexistng interrupt on "
618 "'%U'", format_vlib_pci_addr, &p->addr);
619
620
621 if (efds)
622 {
623 flags |= VFIO_IRQ_SET_DATA_EVENTFD;
Dave Barach178cf492018-11-13 16:34:13 -0500624 clib_memcpy_fast (&irq_set->data, efds, data_len);
Damjan Marion599a16b2018-03-04 19:35:23 +0100625 }
626 else
627 flags |= VFIO_IRQ_SET_DATA_NONE;
628
629 ASSERT ((flags & (VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_DATA_EVENTFD)) !=
630 (VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_DATA_EVENTFD));
631
632 irq_set->argsz = sizeof (struct vfio_irq_set) + data_len;
633 irq_set->index = index;
634 irq_set->start = start;
635 irq_set->count = count;
636 irq_set->flags = flags;
637
638 if (ioctl (p->fd, VFIO_DEVICE_SET_IRQS, irq_set) < 0)
639 return clib_error_return_unix (0, "%U:ioctl(VFIO_DEVICE_SET_IRQS) "
640 "[index = %u, start = %u, count = %u, "
641 "flags = 0x%x]",
642 format_vlib_pci_addr, &p->addr,
643 index, start, count, flags);
644 return 0;
645}
646
647static clib_error_t *
Damjan Marion56dd5432017-09-08 19:52:02 +0200648linux_pci_uio_read_ready (clib_file_t * uf)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700649{
Damjan Marion23227982018-10-22 13:38:57 +0200650 vlib_main_t *vm = vlib_get_main ();
Damjan Marion5a206ea2016-05-12 22:11:03 +0200651 int __attribute__ ((unused)) rv;
Damjan Marioncef87f12017-10-05 15:32:41 +0200652 vlib_pci_dev_handle_t h = uf->private_data;
653 linux_pci_device_t *p = linux_pci_get_device (h);
Damjan Mariond5ded2d2018-03-05 10:18:50 +0100654 linux_pci_irq_t *irq = &p->intx_irq;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700655
Damjan Marion5a206ea2016-05-12 22:11:03 +0200656 u32 icount;
Dave Barach9b8ffd92016-07-08 08:13:45 -0400657 rv = read (uf->file_descriptor, &icount, 4);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700658
Damjan Mariond5ded2d2018-03-05 10:18:50 +0100659 if (irq->intx_handler)
Damjan Marion23227982018-10-22 13:38:57 +0200660 irq->intx_handler (vm, h);
Damjan Marion5a206ea2016-05-12 22:11:03 +0200661
Damjan Marion23227982018-10-22 13:38:57 +0200662 vlib_pci_intr_enable (vm, h);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700663
664 return /* no error */ 0;
665}
666
Dave Barach9b8ffd92016-07-08 08:13:45 -0400667static clib_error_t *
Damjan Marion23227982018-10-22 13:38:57 +0200668linux_pci_vfio_unmask_intx (vlib_main_t * vm, linux_pci_device_t * d)
Damjan Marioncef87f12017-10-05 15:32:41 +0200669{
Damjan Marion23227982018-10-22 13:38:57 +0200670 return vfio_set_irqs (vm, d, VFIO_PCI_INTX_IRQ_INDEX, 0, 1,
Damjan Marion599a16b2018-03-04 19:35:23 +0100671 VFIO_IRQ_SET_ACTION_UNMASK, 0);
672}
673
Damjan Marioncef87f12017-10-05 15:32:41 +0200674static clib_error_t *
Damjan Marion56dd5432017-09-08 19:52:02 +0200675linux_pci_uio_error_ready (clib_file_t * uf)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700676{
677 u32 error_index = (u32) uf->private_data;
678
679 return clib_error_return (0, "pci device %d: error", error_index);
680}
681
Damjan Marioncef87f12017-10-05 15:32:41 +0200682static clib_error_t *
Damjan Marion599a16b2018-03-04 19:35:23 +0100683linux_pci_vfio_msix_read_ready (clib_file_t * uf)
684{
Damjan Marion23227982018-10-22 13:38:57 +0200685 vlib_main_t *vm = vlib_get_main ();
Damjan Marion599a16b2018-03-04 19:35:23 +0100686 int __attribute__ ((unused)) rv;
687 vlib_pci_dev_handle_t h = uf->private_data >> 16;
688 u16 line = uf->private_data & 0xffff;
689 linux_pci_device_t *p = linux_pci_get_device (h);
690 linux_pci_irq_t *irq = vec_elt_at_index (p->msix_irqs, line);
691
692 u64 icount;
693 rv = read (uf->file_descriptor, &icount, sizeof (icount));
694
695 if (irq->msix_handler)
Damjan Marion23227982018-10-22 13:38:57 +0200696 irq->msix_handler (vm, h, line);
Damjan Marion599a16b2018-03-04 19:35:23 +0100697
698 return /* no error */ 0;
699}
700
701static clib_error_t *
702linux_pci_vfio_intx_read_ready (clib_file_t * uf)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700703{
Damjan Marion23227982018-10-22 13:38:57 +0200704 vlib_main_t *vm = vlib_get_main ();
Damjan Marioncef87f12017-10-05 15:32:41 +0200705 int __attribute__ ((unused)) rv;
706 vlib_pci_dev_handle_t h = uf->private_data;
707 linux_pci_device_t *p = linux_pci_get_device (h);
Damjan Mariond5ded2d2018-03-05 10:18:50 +0100708 linux_pci_irq_t *irq = &p->intx_irq;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700709
Damjan Marioncef87f12017-10-05 15:32:41 +0200710 u64 icount;
711 rv = read (uf->file_descriptor, &icount, sizeof (icount));
Ed Warnickecb9cada2015-12-08 15:45:58 -0700712
Damjan Mariond5ded2d2018-03-05 10:18:50 +0100713 if (irq->intx_handler)
Damjan Marion23227982018-10-22 13:38:57 +0200714 irq->intx_handler (vm, h);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700715
Damjan Marion23227982018-10-22 13:38:57 +0200716 linux_pci_vfio_unmask_intx (vm, p);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700717
Damjan Marioncef87f12017-10-05 15:32:41 +0200718 return /* no error */ 0;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700719}
720
Damjan Marioncef87f12017-10-05 15:32:41 +0200721static clib_error_t *
722linux_pci_vfio_error_ready (clib_file_t * uf)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700723{
Damjan Marioncef87f12017-10-05 15:32:41 +0200724 u32 error_index = (u32) uf->private_data;
725
726 return clib_error_return (0, "pci device %d: error", error_index);
727}
728
729static clib_error_t *
Damjan Marion23227982018-10-22 13:38:57 +0200730add_device_uio (vlib_main_t * vm, linux_pci_device_t * p,
731 vlib_pci_device_info_t * di, pci_device_registration_t * r)
Damjan Marioncef87f12017-10-05 15:32:41 +0200732{
Damjan Mariond5ded2d2018-03-05 10:18:50 +0100733 linux_pci_main_t *lpm = &linux_pci_main;
Damjan Marioncef87f12017-10-05 15:32:41 +0200734 clib_error_t *err = 0;
Damjan Marioncef87f12017-10-05 15:32:41 +0200735 u8 *s = 0;
736
Damjan Marionf9a968e2018-02-25 23:26:22 +0100737 p->fd = -1;
Damjan Marion2060db82018-03-04 17:37:15 +0100738 p->type = LINUX_PCI_DEVICE_TYPE_UIO;
Damjan Marioncef87f12017-10-05 15:32:41 +0200739
740 s = format (s, "%s/%U/config%c", sysfs_pci_dev_path,
741 format_vlib_pci_addr, &di->addr, 0);
742
743 p->config_fd = open ((char *) s, O_RDWR);
Damjan Marionf9a968e2018-02-25 23:26:22 +0100744 p->config_offset = 0;
Damjan Marioncef87f12017-10-05 15:32:41 +0200745 vec_reset_length (s);
746
747 if (p->config_fd == -1)
748 {
749 err = clib_error_return_unix (0, "open '%s'", s);
750 goto error;
751 }
752
Benoît Ganne0b91bd62019-11-20 09:07:50 +0100753 s = format (0, "%s/%U/uio%c", sysfs_pci_dev_path,
754 format_vlib_pci_addr, &di->addr, 0);
Damjan Marioncef87f12017-10-05 15:32:41 +0200755 foreach_directory_file ((char *) s, scan_uio_dir, p, /* scan_dirs */
756 1);
757 vec_reset_length (s);
758
759 s = format (s, "/dev/uio%d%c", p->uio_minor, 0);
Damjan Marionf9a968e2018-02-25 23:26:22 +0100760 p->fd = open ((char *) s, O_RDWR);
761 if (p->fd < 0)
Damjan Marioncef87f12017-10-05 15:32:41 +0200762 {
763 err = clib_error_return_unix (0, "open '%s'", s);
764 goto error;
765 }
766
Damjan Mariond5ded2d2018-03-05 10:18:50 +0100767 if (r && r->interrupt_handler)
Damjan Marion23227982018-10-22 13:38:57 +0200768 vlib_pci_register_intx_handler (vm, p->handle, r->interrupt_handler);
Damjan Marioncef87f12017-10-05 15:32:41 +0200769
Damjan Mariond5ded2d2018-03-05 10:18:50 +0100770 if (r && r->init_function)
771 err = r->init_function (lpm->vlib_main, p->handle);
Damjan Marioncef87f12017-10-05 15:32:41 +0200772
773error:
Damjan Marion20ba1642018-03-26 15:35:33 +0200774 vec_free (s);
Damjan Marioncef87f12017-10-05 15:32:41 +0200775 if (err)
776 {
Damjan Marion96504182017-12-10 23:54:46 +0100777 if (p->config_fd != -1)
778 close (p->config_fd);
Damjan Marionf9a968e2018-02-25 23:26:22 +0100779 if (p->fd != -1)
780 close (p->fd);
Damjan Marioncef87f12017-10-05 15:32:41 +0200781 }
782 return err;
783}
784
Damjan Marion599a16b2018-03-04 19:35:23 +0100785clib_error_t *
Damjan Marion23227982018-10-22 13:38:57 +0200786vlib_pci_register_intx_handler (vlib_main_t * vm, vlib_pci_dev_handle_t h,
Damjan Mariond5ded2d2018-03-05 10:18:50 +0100787 pci_intx_handler_function_t * intx_handler)
788{
789 linux_pci_device_t *p = linux_pci_get_device (h);
790 clib_file_t t = { 0 };
791 linux_pci_irq_t *irq = &p->intx_irq;
792 ASSERT (irq->fd == -1);
793
794 if (p->type == LINUX_PCI_DEVICE_TYPE_VFIO)
795 {
Damjan Marion23227982018-10-22 13:38:57 +0200796 struct vfio_irq_info ii = { 0 };
797 ii.argsz = sizeof (struct vfio_irq_info);
798 ii.index = VFIO_PCI_INTX_IRQ_INDEX;
799 if (ioctl (p->fd, VFIO_DEVICE_GET_IRQ_INFO, &ii) < 0)
Damjan Mariond5ded2d2018-03-05 10:18:50 +0100800 return clib_error_return_unix (0, "ioctl(VFIO_DEVICE_GET_IRQ_INFO) '"
801 "%U'", format_vlib_pci_addr, &p->addr);
Damjan Marion23227982018-10-22 13:38:57 +0200802 pci_log_debug (vm, p, "%s index:%u count:%u flags: %s%s%s%s(0x%x)",
803 __func__, ii.index, ii.count,
804 ii.flags & VFIO_IRQ_INFO_EVENTFD ? "eventfd " : "",
805 ii.flags & VFIO_IRQ_INFO_MASKABLE ? "maskable " : "",
806 ii.flags & VFIO_IRQ_INFO_AUTOMASKED ? "automasked " : "",
807 ii.flags & VFIO_IRQ_INFO_NORESIZE ? "noresize " : "",
808 ii.flags);
809 if (ii.count != 1)
Damjan Mariond5ded2d2018-03-05 10:18:50 +0100810 return clib_error_return (0, "INTx interrupt does not exist on device"
811 "'%U'", format_vlib_pci_addr, &p->addr);
812
813 irq->fd = eventfd (0, EFD_NONBLOCK);
814 if (irq->fd == -1)
815 return clib_error_return_unix (0, "eventfd");
Damjan Marion23227982018-10-22 13:38:57 +0200816
Damjan Mariond5ded2d2018-03-05 10:18:50 +0100817 t.file_descriptor = irq->fd;
818 t.read_function = linux_pci_vfio_intx_read_ready;
819 }
820 else if (p->type == LINUX_PCI_DEVICE_TYPE_UIO)
821 {
822 t.file_descriptor = p->fd;
823 t.read_function = linux_pci_uio_read_ready;
824 }
825 else
826 return 0;
827
828 t.error_function = linux_pci_uio_error_ready;
829 t.private_data = p->handle;
830 t.description = format (0, "PCI %U INTx", format_vlib_pci_addr, &p->addr);
831 irq->clib_file_index = clib_file_add (&file_main, &t);
832 irq->intx_handler = intx_handler;
833 return 0;
834}
835
836clib_error_t *
Damjan Marion23227982018-10-22 13:38:57 +0200837vlib_pci_register_msix_handler (vlib_main_t * vm, vlib_pci_dev_handle_t h,
838 u32 start, u32 count,
Damjan Marion599a16b2018-03-04 19:35:23 +0100839 pci_msix_handler_function_t * msix_handler)
840{
841 clib_error_t *err = 0;
842 linux_pci_device_t *p = linux_pci_get_device (h);
843 u32 i;
844
845 if (p->type != LINUX_PCI_DEVICE_TYPE_VFIO)
846 return clib_error_return (0, "vfio driver is needed for MSI-X interrupt "
847 "support");
848
849 /* *INDENT-OFF* */
850 vec_validate_init_empty (p->msix_irqs, start + count - 1, (linux_pci_irq_t)
851 { .fd = -1});
852 /* *INDENT-ON* */
853
854 for (i = start; i < start + count; i++)
855 {
856 clib_file_t t = { 0 };
857 linux_pci_irq_t *irq = vec_elt_at_index (p->msix_irqs, i);
858 ASSERT (irq->fd == -1);
859
860 irq->fd = eventfd (0, EFD_NONBLOCK);
861 if (irq->fd == -1)
862 {
863 err = clib_error_return_unix (0, "eventfd");
864 goto error;
865 }
866
867 t.read_function = linux_pci_vfio_msix_read_ready;
868 t.file_descriptor = irq->fd;
869 t.error_function = linux_pci_vfio_error_ready;
870 t.private_data = p->handle << 16 | i;
871 t.description = format (0, "PCI %U MSI-X #%u", format_vlib_pci_addr,
872 &p->addr, i);
873 irq->clib_file_index = clib_file_add (&file_main, &t);
874 irq->msix_handler = msix_handler;
875 }
876
877 return 0;
878
879error:
880 while (i-- > start)
881 {
882 linux_pci_irq_t *irq = vec_elt_at_index (p->msix_irqs, i);
883 if (irq->fd != -1)
884 {
885 clib_file_del_by_index (&file_main, irq->clib_file_index);
886 close (irq->fd);
887 irq->fd = -1;
888 }
889 }
890 return err;
891}
892
893clib_error_t *
Damjan Marion23227982018-10-22 13:38:57 +0200894vlib_pci_enable_msix_irq (vlib_main_t * vm, vlib_pci_dev_handle_t h,
895 u16 start, u16 count)
Damjan Marion599a16b2018-03-04 19:35:23 +0100896{
897 linux_pci_device_t *p = linux_pci_get_device (h);
898 int fds[count];
899 int i;
900
901 if (p->type != LINUX_PCI_DEVICE_TYPE_VFIO)
902 return clib_error_return (0, "vfio driver is needed for MSI-X interrupt "
903 "support");
904
905 for (i = start; i < start + count; i++)
906 {
907 linux_pci_irq_t *irq = vec_elt_at_index (p->msix_irqs, i);
908 fds[i] = irq->fd;
909 }
910
Damjan Marion23227982018-10-22 13:38:57 +0200911 return vfio_set_irqs (vm, p, VFIO_PCI_MSIX_IRQ_INDEX, start, count,
Damjan Marion599a16b2018-03-04 19:35:23 +0100912 VFIO_IRQ_SET_ACTION_TRIGGER, fds);
913}
914
Damjan Marion9c9490c2020-10-07 20:00:39 +0200915uword
916vlib_pci_get_msix_file_index (vlib_main_t * vm, vlib_pci_dev_handle_t h,
917 u16 index)
918{
919 linux_pci_device_t *p = linux_pci_get_device (h);
920 linux_pci_irq_t *irq = vec_elt_at_index (p->msix_irqs, index);
921 if (irq->fd == -1)
922 return ~0;
923 return irq->clib_file_index;
924}
925
Damjan Marion599a16b2018-03-04 19:35:23 +0100926clib_error_t *
Damjan Marion23227982018-10-22 13:38:57 +0200927vlib_pci_disable_msix_irq (vlib_main_t * vm, vlib_pci_dev_handle_t h,
928 u16 start, u16 count)
Damjan Marion599a16b2018-03-04 19:35:23 +0100929{
930 linux_pci_device_t *p = linux_pci_get_device (h);
931 int i, fds[count];
932
933 if (p->type != LINUX_PCI_DEVICE_TYPE_VFIO)
934 return clib_error_return (0, "vfio driver is needed for MSI-X interrupt "
935 "support");
936
937 for (i = start; i < start + count; i++)
938 fds[i] = -1;
939
Damjan Marion23227982018-10-22 13:38:57 +0200940 return vfio_set_irqs (vm, p, VFIO_PCI_MSIX_IRQ_INDEX, start, count,
Damjan Marion599a16b2018-03-04 19:35:23 +0100941 VFIO_IRQ_SET_ACTION_TRIGGER, fds);
942}
943
Damjan Marioncef87f12017-10-05 15:32:41 +0200944static clib_error_t *
Damjan Marion23227982018-10-22 13:38:57 +0200945add_device_vfio (vlib_main_t * vm, linux_pci_device_t * p,
946 vlib_pci_device_info_t * di, pci_device_registration_t * r)
Damjan Marioncef87f12017-10-05 15:32:41 +0200947{
Damjan Mariond5ded2d2018-03-05 10:18:50 +0100948 linux_pci_main_t *lpm = &linux_pci_main;
Damjan Marionf9a968e2018-02-25 23:26:22 +0100949 struct vfio_device_info device_info = { 0 };
Damjan Marionf313b742018-03-05 14:08:33 +0100950 struct vfio_region_info reg = { 0 };
Damjan Marioncef87f12017-10-05 15:32:41 +0200951 clib_error_t *err = 0;
952 u8 *s = 0;
Damjan Marion23227982018-10-22 13:38:57 +0200953 int is_noiommu;
Damjan Marioncef87f12017-10-05 15:32:41 +0200954
Damjan Marion2060db82018-03-04 17:37:15 +0100955 p->type = LINUX_PCI_DEVICE_TYPE_VFIO;
Damjan Marioncef87f12017-10-05 15:32:41 +0200956
Damjan Marion23227982018-10-22 13:38:57 +0200957 if ((err = linux_vfio_group_get_device_fd (&p->addr, &p->fd, &is_noiommu)))
Damjan Marioncef87f12017-10-05 15:32:41 +0200958 return err;
959
Damjan Marion23227982018-10-22 13:38:57 +0200960 if (is_noiommu == 0)
961 p->supports_va_dma = 1;
962
Damjan Marioncef87f12017-10-05 15:32:41 +0200963 device_info.argsz = sizeof (device_info);
Damjan Marionf9a968e2018-02-25 23:26:22 +0100964 if (ioctl (p->fd, VFIO_DEVICE_GET_INFO, &device_info) < 0)
Damjan Marioncef87f12017-10-05 15:32:41 +0200965 {
966 err = clib_error_return_unix (0, "ioctl(VFIO_DEVICE_GET_INFO) '%U'",
967 format_vlib_pci_addr, &di->addr);
968 goto error;
969 }
970
Damjan Marionf313b742018-03-05 14:08:33 +0100971 reg.argsz = sizeof (struct vfio_region_info);
972 reg.index = VFIO_PCI_CONFIG_REGION_INDEX;
973 if (ioctl (p->fd, VFIO_DEVICE_GET_REGION_INFO, &reg) < 0)
974 {
975 err = clib_error_return_unix (0, "ioctl(VFIO_DEVICE_GET_INFO) '%U'",
976 format_vlib_pci_addr, &di->addr);
977 goto error;
978 }
Damjan Marion23227982018-10-22 13:38:57 +0200979
Damjan Marion0792bb42020-05-25 19:31:42 +0200980 pci_log_debug (vm, p, "%s %U", __func__, format_vfio_region_info, &reg);
Damjan Marion23227982018-10-22 13:38:57 +0200981
Damjan Marionf313b742018-03-05 14:08:33 +0100982 p->config_offset = reg.offset;
983 p->config_fd = p->fd;
984
Damjan Marioncef87f12017-10-05 15:32:41 +0200985 /* reset if device supports it */
986 if (device_info.flags & VFIO_DEVICE_FLAGS_RESET)
Damjan Marionf9a968e2018-02-25 23:26:22 +0100987 if (ioctl (p->fd, VFIO_DEVICE_RESET) < 0)
Damjan Marioncef87f12017-10-05 15:32:41 +0200988 {
989 err = clib_error_return_unix (0, "ioctl(VFIO_DEVICE_RESET) '%U'",
990 format_vlib_pci_addr, &di->addr);
991 goto error;
992 }
993
Damjan Mariond5ded2d2018-03-05 10:18:50 +0100994 if (r && r->interrupt_handler)
Damjan Marioncef87f12017-10-05 15:32:41 +0200995 {
Damjan Marion23227982018-10-22 13:38:57 +0200996 vlib_pci_register_intx_handler (vm, p->handle, r->interrupt_handler);
997 linux_pci_vfio_unmask_intx (vm, p);
Damjan Marioncef87f12017-10-05 15:32:41 +0200998 }
999
Damjan Marion68b4da62018-09-30 18:26:20 +02001000 if (p->supports_va_dma)
1001 {
1002 vlib_buffer_pool_t *bp;
1003 /* *INDENT-OFF* */
Damjan Mariond50e3472019-01-20 00:03:56 +01001004 vec_foreach (bp, vm->buffer_main->buffer_pools)
Damjan Marion68b4da62018-09-30 18:26:20 +02001005 {
1006 u32 i;
1007 vlib_physmem_map_t *pm;
1008 pm = vlib_physmem_get_map (vm, bp->physmem_map_index);
1009 for (i = 0; i < pm->n_pages; i++)
1010 vfio_map_physmem_page (vm, pm->base + (i << pm->log2_page_size));
1011 }
1012 /* *INDENT-ON* */
1013 }
1014
Damjan Mariond5ded2d2018-03-05 10:18:50 +01001015 if (r && r->init_function)
1016 err = r->init_function (lpm->vlib_main, p->handle);
Damjan Marioncef87f12017-10-05 15:32:41 +02001017
1018error:
1019 vec_free (s);
1020 if (err)
1021 {
Damjan Marionf9a968e2018-02-25 23:26:22 +01001022 if (p->fd != -1)
1023 close (p->fd);
Chris Luke30684ac2018-03-29 12:56:58 -07001024 if (p->config_fd != -1 && p->config_fd != p->fd)
Damjan Marion96504182017-12-10 23:54:46 +01001025 close (p->config_fd);
Chris Luke30684ac2018-03-29 12:56:58 -07001026 p->config_fd = p->fd = -1;
Damjan Marioncef87f12017-10-05 15:32:41 +02001027 }
1028 return err;
Ed Warnickecb9cada2015-12-08 15:45:58 -07001029}
1030
1031/* Configuration space read/write. */
1032clib_error_t *
Damjan Marion23227982018-10-22 13:38:57 +02001033vlib_pci_read_write_config (vlib_main_t * vm, vlib_pci_dev_handle_t h,
Damjan Marion5a206ea2016-05-12 22:11:03 +02001034 vlib_read_or_write_t read_or_write,
Dave Barach9b8ffd92016-07-08 08:13:45 -04001035 uword address, void *data, u32 n_bytes)
Ed Warnickecb9cada2015-12-08 15:45:58 -07001036{
Damjan Marioncef87f12017-10-05 15:32:41 +02001037 linux_pci_device_t *p = linux_pci_get_device (h);
Ed Warnickecb9cada2015-12-08 15:45:58 -07001038 int n;
1039
Ed Warnickecb9cada2015-12-08 15:45:58 -07001040 if (read_or_write == VLIB_READ)
Damjan Marionf9a968e2018-02-25 23:26:22 +01001041 n = pread (p->config_fd, data, n_bytes, p->config_offset + address);
Ed Warnickecb9cada2015-12-08 15:45:58 -07001042 else
Damjan Marionf9a968e2018-02-25 23:26:22 +01001043 n = pwrite (p->config_fd, data, n_bytes, p->config_offset + address);
Ed Warnickecb9cada2015-12-08 15:45:58 -07001044
1045 if (n != n_bytes)
1046 return clib_error_return_unix (0, "%s",
1047 read_or_write == VLIB_READ
1048 ? "read" : "write");
1049
1050 return 0;
1051}
1052
1053static clib_error_t *
Mohsin Kazmi95276ca2018-09-28 11:45:01 +02001054vlib_pci_region (vlib_main_t * vm, vlib_pci_dev_handle_t h, u32 bar, int *fd,
1055 u64 * size, u64 * offset)
Ed Warnickecb9cada2015-12-08 15:45:58 -07001056{
Damjan Marioncef87f12017-10-05 15:32:41 +02001057 linux_pci_device_t *p = linux_pci_get_device (h);
Mohsin Kazmi95276ca2018-09-28 11:45:01 +02001058 clib_error_t *error = 0;
1059 int _fd = -1;
1060 u64 _size = 0, _offset = 0;
Damjan Marion2060db82018-03-04 17:37:15 +01001061
1062 ASSERT (bar <= 5);
Ed Warnickecb9cada2015-12-08 15:45:58 -07001063
1064 error = 0;
Ed Warnickecb9cada2015-12-08 15:45:58 -07001065
Damjan Marion2060db82018-03-04 17:37:15 +01001066 if (p->type == LINUX_PCI_DEVICE_TYPE_UIO)
Ed Warnickecb9cada2015-12-08 15:45:58 -07001067 {
Damjan Marion2060db82018-03-04 17:37:15 +01001068 u8 *file_name;
1069 struct stat stat_buf;
1070 file_name = format (0, "%s/%U/resource%d%c", sysfs_pci_dev_path,
1071 format_vlib_pci_addr, &p->addr, bar, 0);
Ed Warnickecb9cada2015-12-08 15:45:58 -07001072
Mohsin Kazmi95276ca2018-09-28 11:45:01 +02001073 _fd = open ((char *) file_name, O_RDWR);
1074 if (_fd < 0)
Damjan Marion2060db82018-03-04 17:37:15 +01001075 {
1076 error = clib_error_return_unix (0, "open `%s'", file_name);
1077 vec_free (file_name);
1078 return error;
1079 }
1080
Mohsin Kazmi95276ca2018-09-28 11:45:01 +02001081 if (fstat (_fd, &stat_buf) < 0)
Damjan Marion2060db82018-03-04 17:37:15 +01001082 {
1083 error = clib_error_return_unix (0, "fstat `%s'", file_name);
1084 vec_free (file_name);
Mohsin Kazmi95276ca2018-09-28 11:45:01 +02001085 close (_fd);
Damjan Marion2060db82018-03-04 17:37:15 +01001086 return error;
1087 }
1088
1089 vec_free (file_name);
Mohsin Kazmi95276ca2018-09-28 11:45:01 +02001090 _size = stat_buf.st_size;
1091 _offset = 0;
Damjan Marion2060db82018-03-04 17:37:15 +01001092 }
1093 else if (p->type == LINUX_PCI_DEVICE_TYPE_VFIO)
Ed Warnickecb9cada2015-12-08 15:45:58 -07001094 {
Damjan Marion0792bb42020-05-25 19:31:42 +02001095 struct vfio_region_info *r;
1096 u32 sz = sizeof (struct vfio_region_info);
1097 again:
1098 r = clib_mem_alloc (sz);
1099 clib_memset (r, 0, sz);
1100 r->argsz = sz;
1101 r->index = bar;
1102 if (ioctl (p->fd, VFIO_DEVICE_GET_REGION_INFO, r) < 0)
Damjan Marion2060db82018-03-04 17:37:15 +01001103 return clib_error_return_unix (0, "ioctl(VFIO_DEVICE_GET_INFO) "
1104 "'%U'", format_vlib_pci_addr,
1105 &p->addr);
Damjan Marion0792bb42020-05-25 19:31:42 +02001106 if (sz != r->argsz)
1107 {
1108 sz = r->argsz;
1109 clib_mem_free (r);
1110 goto again;
1111 }
Mohsin Kazmi95276ca2018-09-28 11:45:01 +02001112 _fd = p->fd;
Damjan Marion0792bb42020-05-25 19:31:42 +02001113 _size = r->size;
1114 _offset = r->offset;
1115 pci_log_debug (vm, p, "%s %U", __func__, format_vfio_region_info, r);
1116 clib_mem_free (r);
Ed Warnickecb9cada2015-12-08 15:45:58 -07001117 }
Damjan Marion2060db82018-03-04 17:37:15 +01001118 else
1119 ASSERT (0);
Ed Warnickecb9cada2015-12-08 15:45:58 -07001120
Mohsin Kazmi95276ca2018-09-28 11:45:01 +02001121 *fd = _fd;
1122 *size = _size;
1123 *offset = _offset;
1124
1125 return error;
1126}
1127
1128static clib_error_t *
1129vlib_pci_map_region_int (vlib_main_t * vm, vlib_pci_dev_handle_t h,
1130 u32 bar, u8 * addr, void **result)
1131{
1132 linux_pci_device_t *p = linux_pci_get_device (h);
1133 int fd = -1;
1134 clib_error_t *error;
1135 int flags = MAP_SHARED;
1136 u64 size = 0, offset = 0;
1137
1138 pci_log_debug (vm, p, "map region %u to va %p", bar, addr);
1139
1140 if ((error = vlib_pci_region (vm, h, bar, &fd, &size, &offset)))
1141 return error;
1142
1143 if (p->type == LINUX_PCI_DEVICE_TYPE_UIO && addr != 0)
1144 flags |= MAP_FIXED;
1145
Damjan Marion6bfd0762020-09-11 22:16:53 +02001146 *result = clib_mem_vm_map_shared (addr, size, fd, offset,
1147 "PCIe %U region %u", format_vlib_pci_addr,
1148 vlib_pci_get_addr (vm, h), bar);
1149 if (*result == CLIB_MEM_VM_MAP_FAILED)
Ed Warnickecb9cada2015-12-08 15:45:58 -07001150 {
Damjan Marion2060db82018-03-04 17:37:15 +01001151 error = clib_error_return_unix (0, "mmap `BAR%u'", bar);
Dave Barachc72950e2020-05-04 14:47:45 -04001152 if (p->type == LINUX_PCI_DEVICE_TYPE_UIO && (fd != -1))
Ed Warnickecb9cada2015-12-08 15:45:58 -07001153 close (fd);
Damjan Marion2060db82018-03-04 17:37:15 +01001154 return error;
Ed Warnickecb9cada2015-12-08 15:45:58 -07001155 }
Damjan Marion2060db82018-03-04 17:37:15 +01001156
1157 /* *INDENT-OFF* */
1158 vec_validate_init_empty (p->regions, bar,
1159 (linux_pci_region_t) { .fd = -1});
1160 /* *INDENT-ON* */
1161 if (p->type == LINUX_PCI_DEVICE_TYPE_UIO)
1162 p->regions[bar].fd = fd;
1163 p->regions[bar].addr = *result;
1164 p->regions[bar].size = size;
1165 return 0;
Ed Warnickecb9cada2015-12-08 15:45:58 -07001166}
1167
1168clib_error_t *
Damjan Marion23227982018-10-22 13:38:57 +02001169vlib_pci_map_region (vlib_main_t * vm, vlib_pci_dev_handle_t h, u32 resource,
1170 void **result)
Ed Warnickecb9cada2015-12-08 15:45:58 -07001171{
Damjan Marion23227982018-10-22 13:38:57 +02001172 return (vlib_pci_map_region_int (vm, h, resource, 0 /* addr */ , result));
Ed Warnickecb9cada2015-12-08 15:45:58 -07001173}
1174
1175clib_error_t *
Damjan Marion23227982018-10-22 13:38:57 +02001176vlib_pci_map_region_fixed (vlib_main_t * vm, vlib_pci_dev_handle_t h,
1177 u32 resource, u8 * addr, void **result)
Ed Warnickecb9cada2015-12-08 15:45:58 -07001178{
Damjan Marion23227982018-10-22 13:38:57 +02001179 return (vlib_pci_map_region_int (vm, h, resource, addr, result));
Ed Warnickecb9cada2015-12-08 15:45:58 -07001180}
1181
Damjan Marionf313b742018-03-05 14:08:33 +01001182clib_error_t *
Mohsin Kazmi95276ca2018-09-28 11:45:01 +02001183vlib_pci_io_region (vlib_main_t * vm, vlib_pci_dev_handle_t h, u32 resource)
1184{
1185 linux_pci_device_t *p = linux_pci_get_device (h);
1186 clib_error_t *error = 0;
1187 int fd = -1;
1188 u64 size = 0, offset = 0;
1189
1190 if ((error = vlib_pci_region (vm, h, resource, &fd, &size, &offset)))
1191 return error;
1192
1193 p->io_fd = fd;
1194 p->io_offset = offset;
1195 return error;
1196}
1197
1198clib_error_t *
1199vlib_pci_read_write_io (vlib_main_t * vm, vlib_pci_dev_handle_t h,
1200 vlib_read_or_write_t read_or_write,
1201 uword offset, void *data, u32 length)
1202{
1203 linux_pci_device_t *p = linux_pci_get_device (h);
1204 int n = 0;
1205
1206 if (read_or_write == VLIB_READ)
1207 n = pread (p->io_fd, data, length, p->io_offset + offset);
1208 else
1209 n = pwrite (p->io_fd, data, length, p->io_offset + offset);
1210
1211 if (n != length)
1212 return clib_error_return_unix (0, "%s",
1213 read_or_write == VLIB_READ
1214 ? "read" : "write");
1215 return 0;
1216}
1217
1218clib_error_t *
Damjan Marion68b4da62018-09-30 18:26:20 +02001219vlib_pci_map_dma (vlib_main_t * vm, vlib_pci_dev_handle_t h, void *ptr)
1220{
1221 linux_pci_device_t *p = linux_pci_get_device (h);
1222
1223 if (!p->supports_va_dma)
1224 return 0;
1225
1226 return vfio_map_physmem_page (vm, ptr);
1227}
1228
1229int
1230vlib_pci_supports_virtual_addr_dma (vlib_main_t * vm, vlib_pci_dev_handle_t h)
1231{
1232 linux_pci_device_t *p = linux_pci_get_device (h);
1233
1234 return p->supports_va_dma != 0;
1235}
1236
1237clib_error_t *
Damjan Marion23227982018-10-22 13:38:57 +02001238vlib_pci_device_open (vlib_main_t * vm, vlib_pci_addr_t * addr,
Damjan Marionf313b742018-03-05 14:08:33 +01001239 pci_device_id_t ids[], vlib_pci_dev_handle_t * handle)
1240{
1241 linux_pci_main_t *lpm = &linux_pci_main;
1242 vlib_pci_device_info_t *di;
1243 linux_pci_device_t *p;
1244 clib_error_t *err = 0;
1245 pci_device_id_t *i;
1246
Damjan Marion23227982018-10-22 13:38:57 +02001247 di = vlib_pci_get_device_info (vm, addr, &err);
Damjan Marionf313b742018-03-05 14:08:33 +01001248
1249 if (err)
1250 return err;
1251 for (i = ids; i->vendor_id != 0; i++)
1252 if (i->vendor_id == di->vendor_id && i->device_id == di->device_id)
1253 break;
1254
1255 if (i->vendor_id == 0)
1256 return clib_error_return (0, "Wrong vendor or device id");
1257
1258 pool_get (lpm->linux_pci_devices, p);
1259 p->handle = p - lpm->linux_pci_devices;
Damjan Marion23227982018-10-22 13:38:57 +02001260 p->addr.as_u32 = di->addr.as_u32;
Damjan Marionf313b742018-03-05 14:08:33 +01001261 p->intx_irq.fd = -1;
Mohsin Kazmi692f9b12019-04-02 11:09:49 +00001262 p->intx_irq.clib_file_index = -1;
Damjan Mariond2bfb782019-01-07 20:56:04 +01001263 p->numa_node = di->numa_node;
Mohsin Kazmi3d3b9552018-10-24 14:05:34 +02001264 /*
1265 * pci io bar read/write fd
1266 */
1267 p->io_fd = -1;
Damjan Marionf313b742018-03-05 14:08:33 +01001268
Damjan Marion23227982018-10-22 13:38:57 +02001269 pci_log_debug (vm, p, "open vid:0x%04x did:0x%04x driver:%s iommu_group:%d",
1270 di->vendor_id, di->device_id, di->driver_name,
1271 di->iommu_group);
1272
Steven Luongab489922019-08-21 11:35:27 -07001273 if (clib_strncmp ("vfio-pci", (char *) di->driver_name, 8) == 0)
Damjan Marion23227982018-10-22 13:38:57 +02001274 err = add_device_vfio (vm, p, di, 0);
Steven Luongab489922019-08-21 11:35:27 -07001275 else if (clib_strncmp ("uio_pci_generic", (char *) di->driver_name, 8) == 0)
Damjan Marion23227982018-10-22 13:38:57 +02001276 err = add_device_uio (vm, p, di, 0);
Damjan Marionf313b742018-03-05 14:08:33 +01001277 else
Damjan Marion23227982018-10-22 13:38:57 +02001278 err = clib_error_create ("device not bound to 'vfio-pci' or "
1279 "'uio_pci_generic' kernel module");
Damjan Marionf313b742018-03-05 14:08:33 +01001280 if (err)
1281 goto error;
1282
1283 *handle = p->handle;
1284
1285error:
1286 vlib_pci_free_device_info (di);
1287 if (err)
1288 {
Damjan Marion23227982018-10-22 13:38:57 +02001289 pci_log_err (vm, p, "%U", format_clib_error, err);
Dave Barachb7b92992018-10-17 10:38:51 -04001290 clib_memset (p, 0, sizeof (linux_pci_device_t));
Damjan Marionf313b742018-03-05 14:08:33 +01001291 pool_put (lpm->linux_pci_devices, p);
1292 }
1293
1294 return err;
1295}
1296
1297void
Damjan Marion23227982018-10-22 13:38:57 +02001298vlib_pci_device_close (vlib_main_t * vm, vlib_pci_dev_handle_t h)
Damjan Marionf313b742018-03-05 14:08:33 +01001299{
1300 linux_pci_main_t *lpm = &linux_pci_main;
1301 linux_pci_device_t *p = linux_pci_get_device (h);
1302 linux_pci_irq_t *irq;
1303 linux_pci_region_t *res;
1304 clib_error_t *err = 0;
1305
1306 if (p->type == LINUX_PCI_DEVICE_TYPE_UIO)
1307 {
1308 irq = &p->intx_irq;
Mohsin Kazmi692f9b12019-04-02 11:09:49 +00001309 if (irq->clib_file_index != -1)
1310 clib_file_del_by_index (&file_main, irq->clib_file_index);
Damjan Marionf313b742018-03-05 14:08:33 +01001311 close (p->config_fd);
Mohsin Kazmi3d3b9552018-10-24 14:05:34 +02001312 if (p->io_fd != -1)
1313 close (p->io_fd);
Damjan Marionf313b742018-03-05 14:08:33 +01001314 }
1315 else if (p->type == LINUX_PCI_DEVICE_TYPE_VFIO)
1316 {
1317 irq = &p->intx_irq;
1318 /* close INTx irqs */
1319 if (irq->fd != -1)
1320 {
Damjan Marion23227982018-10-22 13:38:57 +02001321 err = vfio_set_irqs (vm, p, VFIO_PCI_INTX_IRQ_INDEX, 0, 0,
Damjan Marionf313b742018-03-05 14:08:33 +01001322 VFIO_IRQ_SET_ACTION_TRIGGER, 0);
1323 clib_error_free (err);
Mohsin Kazmi692f9b12019-04-02 11:09:49 +00001324 if (irq->clib_file_index != -1)
1325 clib_file_del_by_index (&file_main, irq->clib_file_index);
Damjan Marionf313b742018-03-05 14:08:33 +01001326 close (irq->fd);
1327 }
1328
1329 /* close MSI-X irqs */
1330 if (vec_len (p->msix_irqs))
1331 {
Damjan Marion23227982018-10-22 13:38:57 +02001332 err = vfio_set_irqs (vm, p, VFIO_PCI_MSIX_IRQ_INDEX, 0, 0,
Damjan Marionf313b742018-03-05 14:08:33 +01001333 VFIO_IRQ_SET_ACTION_TRIGGER, 0);
1334 clib_error_free (err);
1335 /* *INDENT-OFF* */
1336 vec_foreach (irq, p->msix_irqs)
1337 {
1338 if (irq->fd == -1)
1339 continue;
1340 clib_file_del_by_index (&file_main, irq->clib_file_index);
1341 close (irq->fd);
1342 }
1343 /* *INDENT-ON* */
1344 vec_free (p->msix_irqs);
1345 }
1346 }
1347
1348 /* *INDENT-OFF* */
1349 vec_foreach (res, p->regions)
1350 {
1351 if (res->size == 0)
1352 continue;
Damjan Marion6bfd0762020-09-11 22:16:53 +02001353 clib_mem_vm_unmap (res->addr);
Damjan Marionf313b742018-03-05 14:08:33 +01001354 if (res->fd != -1)
1355 close (res->fd);
1356 }
1357 /* *INDENT-ON* */
1358 vec_free (p->regions);
1359
1360 close (p->fd);
Dave Barachb7b92992018-10-17 10:38:51 -04001361 clib_memset (p, 0, sizeof (linux_pci_device_t));
Damjan Marionf313b742018-03-05 14:08:33 +01001362 pool_put (lpm->linux_pci_devices, p);
1363}
1364
Dave Barach9b8ffd92016-07-08 08:13:45 -04001365void
Damjan Marion23227982018-10-22 13:38:57 +02001366init_device_from_registered (vlib_main_t * vm, vlib_pci_device_info_t * di)
Ed Warnickecb9cada2015-12-08 15:45:58 -07001367{
Dave Barach9b8ffd92016-07-08 08:13:45 -04001368 vlib_pci_main_t *pm = &pci_main;
Damjan Marioncef87f12017-10-05 15:32:41 +02001369 linux_pci_main_t *lpm = &linux_pci_main;
Dave Barach9b8ffd92016-07-08 08:13:45 -04001370 pci_device_registration_t *r;
1371 pci_device_id_t *i;
Damjan Marioncef87f12017-10-05 15:32:41 +02001372 clib_error_t *err = 0;
1373 linux_pci_device_t *p;
1374
1375 pool_get (lpm->linux_pci_devices, p);
1376 p->handle = p - lpm->linux_pci_devices;
Damjan Mariond5ded2d2018-03-05 10:18:50 +01001377 p->intx_irq.fd = -1;
Ed Warnickecb9cada2015-12-08 15:45:58 -07001378
Damjan Marion5a206ea2016-05-12 22:11:03 +02001379 r = pm->pci_device_registrations;
Ed Warnickecb9cada2015-12-08 15:45:58 -07001380
1381 while (r)
1382 {
1383 for (i = r->supported_devices; i->vendor_id != 0; i++)
Damjan Marion3a593822018-02-17 14:06:53 +01001384 if (i->vendor_id == di->vendor_id && i->device_id == di->device_id)
Dave Barach9b8ffd92016-07-08 08:13:45 -04001385 {
Damjan Marioncef87f12017-10-05 15:32:41 +02001386 if (di->iommu_group != -1)
Damjan Marion23227982018-10-22 13:38:57 +02001387 err = add_device_vfio (vm, p, di, r);
Damjan Marioncef87f12017-10-05 15:32:41 +02001388 else
Damjan Marion23227982018-10-22 13:38:57 +02001389 err = add_device_uio (vm, p, di, r);
Damjan Marion5a206ea2016-05-12 22:11:03 +02001390
Damjan Marioncef87f12017-10-05 15:32:41 +02001391 if (err)
1392 clib_error_report (err);
1393 else
1394 return;
Dave Barach9b8ffd92016-07-08 08:13:45 -04001395 }
Ed Warnickecb9cada2015-12-08 15:45:58 -07001396 r = r->next_registration;
Dave Barach9b8ffd92016-07-08 08:13:45 -04001397 }
Damjan Marioncef87f12017-10-05 15:32:41 +02001398
Ed Warnickecb9cada2015-12-08 15:45:58 -07001399 /* No driver, close the PCI config-space FD */
Dave Barachb7b92992018-10-17 10:38:51 -04001400 clib_memset (p, 0, sizeof (linux_pci_device_t));
Damjan Marioncef87f12017-10-05 15:32:41 +02001401 pool_put (lpm->linux_pci_devices, p);
1402}
1403
1404static clib_error_t *
1405scan_pci_addr (void *arg, u8 * dev_dir_name, u8 * ignored)
1406{
1407 vlib_pci_addr_t addr, **addrv = arg;
1408 unformat_input_t input;
1409 clib_error_t *err = 0;
1410
1411 unformat_init_string (&input, (char *) dev_dir_name,
1412 vec_len (dev_dir_name));
1413
1414 if (!unformat (&input, "/sys/bus/pci/devices/%U",
1415 unformat_vlib_pci_addr, &addr))
1416 err = clib_error_return (0, "unformat error `%v`", dev_dir_name);
1417
1418 unformat_free (&input);
1419
1420 if (err)
1421 return err;
1422
1423 vec_add1 (*addrv, addr);
Ed Warnickecb9cada2015-12-08 15:45:58 -07001424 return 0;
1425}
1426
Damjan Marioncef87f12017-10-05 15:32:41 +02001427static int
1428pci_addr_cmp (void *v1, void *v2)
Ed Warnickecb9cada2015-12-08 15:45:58 -07001429{
Damjan Marioncef87f12017-10-05 15:32:41 +02001430 vlib_pci_addr_t *a1 = v1;
1431 vlib_pci_addr_t *a2 = v2;
1432
1433 if (a1->domain > a2->domain)
1434 return 1;
1435 if (a1->domain < a2->domain)
1436 return -1;
1437 if (a1->bus > a2->bus)
1438 return 1;
1439 if (a1->bus < a2->bus)
1440 return -1;
1441 if (a1->slot > a2->slot)
1442 return 1;
1443 if (a1->slot < a2->slot)
1444 return -1;
1445 if (a1->function > a2->function)
1446 return 1;
1447 if (a1->function < a2->function)
1448 return -1;
1449 return 0;
Ed Warnickecb9cada2015-12-08 15:45:58 -07001450}
1451
Damjan Marioncef87f12017-10-05 15:32:41 +02001452vlib_pci_addr_t *
1453vlib_pci_get_all_dev_addrs ()
Ed Warnickecb9cada2015-12-08 15:45:58 -07001454{
Damjan Marioncef87f12017-10-05 15:32:41 +02001455 vlib_pci_addr_t *addrs = 0;
1456 clib_error_t *err;
1457 err = foreach_directory_file ((char *) sysfs_pci_dev_path, scan_pci_addr,
1458 &addrs, /* scan_dirs */ 0);
1459 if (err)
Ed Warnickecb9cada2015-12-08 15:45:58 -07001460 {
Damjan Marioncef87f12017-10-05 15:32:41 +02001461 vec_free (addrs);
1462 return 0;
Ed Warnickecb9cada2015-12-08 15:45:58 -07001463 }
1464
Damjan Marioncef87f12017-10-05 15:32:41 +02001465 vec_sort_with_function (addrs, pci_addr_cmp);
Damjan Mariona42cd342016-04-13 18:03:20 +02001466
Damjan Marioncef87f12017-10-05 15:32:41 +02001467 return addrs;
Ed Warnickecb9cada2015-12-08 15:45:58 -07001468}
1469
Dave Barach9b8ffd92016-07-08 08:13:45 -04001470clib_error_t *
1471linux_pci_init (vlib_main_t * vm)
Ed Warnickecb9cada2015-12-08 15:45:58 -07001472{
Dave Barach9b8ffd92016-07-08 08:13:45 -04001473 vlib_pci_main_t *pm = &pci_main;
Damjan Marioncef87f12017-10-05 15:32:41 +02001474 vlib_pci_addr_t *addr = 0, *addrs;
Ed Warnickecb9cada2015-12-08 15:45:58 -07001475
1476 pm->vlib_main = vm;
1477
Dave Barach9b8ffd92016-07-08 08:13:45 -04001478 ASSERT (sizeof (vlib_pci_addr_t) == sizeof (u32));
Damjan Mariona42cd342016-04-13 18:03:20 +02001479
Damjan Marioncef87f12017-10-05 15:32:41 +02001480 addrs = vlib_pci_get_all_dev_addrs ();
1481 /* *INDENT-OFF* */
1482 vec_foreach (addr, addrs)
1483 {
1484 vlib_pci_device_info_t *d;
Damjan Marion23227982018-10-22 13:38:57 +02001485 if ((d = vlib_pci_get_device_info (vm, addr, 0)))
Damjan Marioncef87f12017-10-05 15:32:41 +02001486 {
Damjan Marion23227982018-10-22 13:38:57 +02001487 init_device_from_registered (vm, d);
Damjan Marioncef87f12017-10-05 15:32:41 +02001488 vlib_pci_free_device_info (d);
1489 }
1490 }
1491 /* *INDENT-ON* */
Ed Warnickecb9cada2015-12-08 15:45:58 -07001492
Dave Barachf8d50682019-05-14 18:01:44 -04001493 return 0;
Ed Warnickecb9cada2015-12-08 15:45:58 -07001494}
1495
Dave Barachf8d50682019-05-14 18:01:44 -04001496/* *INDENT-OFF* */
1497VLIB_INIT_FUNCTION (linux_pci_init) =
1498{
1499 .runs_after = VLIB_INITS("unix_input_init"),
1500};
1501/* *INDENT-ON* */
Dave Barach9b8ffd92016-07-08 08:13:45 -04001502
1503/*
1504 * fd.io coding-style-patch-verification: ON
1505 *
1506 * Local Variables:
1507 * eval: (c-set-style "gnu")
1508 * End:
1509 */