Damjan Marion | 1ba0fa4 | 2018-03-04 17:19:08 +0100 | [diff] [blame] | 1 | /* |
Damjan Marion | 20ba164 | 2018-03-26 15:35:33 +0200 | [diff] [blame] | 2 | * Copyright (c) 2018 Cisco and/or its affiliates. |
Damjan Marion | 1ba0fa4 | 2018-03-04 17:19:08 +0100 | [diff] [blame] | 3 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | * you may not use this file except in compliance with the License. |
| 5 | * You may obtain a copy of the License at: |
| 6 | * |
| 7 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | * |
| 9 | * Unless required by applicable law or agreed to in writing, software |
| 10 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | * See the License for the specific language governing permissions and |
| 13 | * limitations under the License. |
| 14 | */ |
Damjan Marion | 1ba0fa4 | 2018-03-04 17:19:08 +0100 | [diff] [blame] | 15 | |
| 16 | #include <unistd.h> |
| 17 | #include <sys/types.h> |
| 18 | #include <sys/stat.h> |
| 19 | #include <fcntl.h> |
| 20 | #include <linux/vfio.h> |
| 21 | #include <sys/ioctl.h> |
| 22 | |
| 23 | #include <vppinfra/linux/sysfs.h> |
| 24 | |
| 25 | #include <vlib/vlib.h> |
| 26 | #include <vlib/unix/unix.h> |
| 27 | #include <vlib/pci/pci.h> |
| 28 | #include <vlib/linux/vfio.h> |
| 29 | #include <vlib/physmem.h> |
| 30 | |
Damjan Marion | 20ba164 | 2018-03-26 15:35:33 +0200 | [diff] [blame] | 31 | #ifndef VFIO_NOIOMMU_IOMMU |
| 32 | #define VFIO_NOIOMMU_IOMMU 8 |
| 33 | #endif |
| 34 | |
Damjan Marion | 1ba0fa4 | 2018-03-04 17:19:08 +0100 | [diff] [blame] | 35 | linux_vfio_main_t vfio_main; |
| 36 | |
| 37 | static int |
Damjan Marion | b7620fd | 2018-10-05 16:46:00 +0200 | [diff] [blame] | 38 | vfio_map_regions (vlib_main_t * vm, int fd) |
Damjan Marion | 1ba0fa4 | 2018-03-04 17:19:08 +0100 | [diff] [blame] | 39 | { |
Damjan Marion | 88c0621 | 2018-03-05 20:08:28 +0100 | [diff] [blame] | 40 | vlib_physmem_main_t *vpm = &physmem_main; |
Damjan Marion | 7d42653 | 2018-09-27 14:54:49 +0200 | [diff] [blame] | 41 | linux_vfio_main_t *lvm = &vfio_main; |
Damjan Marion | 1ba0fa4 | 2018-03-04 17:19:08 +0100 | [diff] [blame] | 42 | vlib_physmem_region_t *pr; |
| 43 | struct vfio_iommu_type1_dma_map dm = { 0 }; |
| 44 | int i; |
| 45 | |
| 46 | dm.argsz = sizeof (struct vfio_iommu_type1_dma_map); |
| 47 | dm.flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE; |
| 48 | |
| 49 | /* *INDENT-OFF* */ |
| 50 | pool_foreach (pr, vpm->regions, |
| 51 | { |
| 52 | vec_foreach_index (i, pr->page_table) |
| 53 | { |
| 54 | int rv; |
Haiyang Tan | a5ab503 | 2018-10-15 06:17:55 -0700 | [diff] [blame] | 55 | dm.vaddr = pointer_to_uword (pr->mem) + ((u64)i << pr->log2_page_size); |
| 56 | dm.size = 1ull << pr->log2_page_size; |
Damjan Marion | 51c52c0 | 2018-03-09 16:05:56 +0100 | [diff] [blame] | 57 | dm.iova = dm.vaddr; |
Damjan Marion | 7d42653 | 2018-09-27 14:54:49 +0200 | [diff] [blame] | 58 | vlib_log_debug (lvm->log_default, "map DMA va:0x%lx iova:%lx " |
| 59 | "size:0x%lx", dm.vaddr, dm.iova, dm.size); |
| 60 | |
Damjan Marion | b7620fd | 2018-10-05 16:46:00 +0200 | [diff] [blame] | 61 | if ((rv = ioctl (fd, VFIO_IOMMU_MAP_DMA, &dm)) && |
| 62 | errno != EINVAL) |
Damjan Marion | 7d42653 | 2018-09-27 14:54:49 +0200 | [diff] [blame] | 63 | { |
| 64 | vlib_log_err (lvm->log_default, "map DMA va:0x%lx iova:%lx " |
| 65 | "size:0x%lx failed, error %s (errno %d)", |
| 66 | dm.vaddr, dm.iova, dm.size, strerror (errno), |
| 67 | errno); |
| 68 | return rv; |
| 69 | } |
Damjan Marion | 1ba0fa4 | 2018-03-04 17:19:08 +0100 | [diff] [blame] | 70 | } |
| 71 | }); |
| 72 | /* *INDENT-ON* */ |
| 73 | return 0; |
| 74 | } |
| 75 | |
Damjan Marion | 1ba0fa4 | 2018-03-04 17:19:08 +0100 | [diff] [blame] | 76 | void |
| 77 | linux_vfio_dma_map_regions (vlib_main_t * vm) |
| 78 | { |
| 79 | linux_vfio_main_t *lvm = &vfio_main; |
| 80 | |
| 81 | if (lvm->container_fd != -1) |
Damjan Marion | b7620fd | 2018-10-05 16:46:00 +0200 | [diff] [blame] | 82 | vfio_map_regions (vm, lvm->container_fd); |
Damjan Marion | 1ba0fa4 | 2018-03-04 17:19:08 +0100 | [diff] [blame] | 83 | } |
| 84 | |
| 85 | static linux_pci_vfio_iommu_group_t * |
| 86 | get_vfio_iommu_group (int group) |
| 87 | { |
| 88 | linux_vfio_main_t *lvm = &vfio_main; |
| 89 | uword *p; |
| 90 | |
| 91 | p = hash_get (lvm->iommu_pool_index_by_group, group); |
| 92 | |
| 93 | return p ? pool_elt_at_index (lvm->iommu_groups, p[0]) : 0; |
| 94 | } |
| 95 | |
| 96 | static clib_error_t * |
Damjan Marion | 20ba164 | 2018-03-26 15:35:33 +0200 | [diff] [blame] | 97 | open_vfio_iommu_group (int group, int is_noiommu) |
Damjan Marion | 1ba0fa4 | 2018-03-04 17:19:08 +0100 | [diff] [blame] | 98 | { |
| 99 | linux_vfio_main_t *lvm = &vfio_main; |
| 100 | linux_pci_vfio_iommu_group_t *g; |
| 101 | clib_error_t *err = 0; |
| 102 | struct vfio_group_status group_status; |
| 103 | u8 *s = 0; |
| 104 | int fd; |
| 105 | |
Damjan Marion | 2322798 | 2018-10-22 13:38:57 +0200 | [diff] [blame^] | 106 | if (lvm->container_fd == -1) |
| 107 | { |
| 108 | if ((fd = open ("/dev/vfio/vfio", O_RDWR)) == -1) |
| 109 | return clib_error_return_unix (0, "failed to open VFIO container"); |
| 110 | |
| 111 | if (ioctl (fd, VFIO_GET_API_VERSION) != VFIO_API_VERSION) |
| 112 | { |
| 113 | close (fd); |
| 114 | return clib_error_return_unix (0, "incompatible VFIO version"); |
| 115 | } |
| 116 | |
| 117 | lvm->iommu_pool_index_by_group = hash_create (0, sizeof (uword)); |
| 118 | lvm->container_fd = fd; |
| 119 | } |
| 120 | |
Damjan Marion | 1ba0fa4 | 2018-03-04 17:19:08 +0100 | [diff] [blame] | 121 | g = get_vfio_iommu_group (group); |
| 122 | if (g) |
| 123 | { |
| 124 | g->refcnt++; |
| 125 | return 0; |
| 126 | } |
Damjan Marion | 20ba164 | 2018-03-26 15:35:33 +0200 | [diff] [blame] | 127 | s = format (s, "/dev/vfio/%s%u%c", is_noiommu ? "noiommu-" : "", group, 0); |
Damjan Marion | 1ba0fa4 | 2018-03-04 17:19:08 +0100 | [diff] [blame] | 128 | fd = open ((char *) s, O_RDWR); |
| 129 | if (fd < 0) |
| 130 | return clib_error_return_unix (0, "open '%s'", s); |
| 131 | |
| 132 | group_status.argsz = sizeof (group_status); |
| 133 | if (ioctl (fd, VFIO_GROUP_GET_STATUS, &group_status) < 0) |
| 134 | { |
| 135 | err = clib_error_return_unix (0, "ioctl(VFIO_GROUP_GET_STATUS) '%s'", |
| 136 | s); |
| 137 | goto error; |
| 138 | } |
| 139 | |
| 140 | if (!(group_status.flags & VFIO_GROUP_FLAGS_VIABLE)) |
| 141 | { |
| 142 | err = clib_error_return (0, "iommu group %d is not viable (not all " |
| 143 | "devices in this group bound to vfio-pci)", |
| 144 | group); |
| 145 | goto error; |
| 146 | } |
| 147 | |
| 148 | if (ioctl (fd, VFIO_GROUP_SET_CONTAINER, &lvm->container_fd) < 0) |
| 149 | { |
| 150 | err = clib_error_return_unix (0, "ioctl(VFIO_GROUP_SET_CONTAINER) '%s'", |
| 151 | s); |
| 152 | goto error; |
| 153 | } |
| 154 | |
| 155 | if (lvm->iommu_mode == 0) |
| 156 | { |
Damjan Marion | 20ba164 | 2018-03-26 15:35:33 +0200 | [diff] [blame] | 157 | if (is_noiommu) |
| 158 | lvm->iommu_mode = VFIO_NOIOMMU_IOMMU; |
| 159 | else |
| 160 | lvm->iommu_mode = VFIO_TYPE1_IOMMU; |
| 161 | |
| 162 | if (ioctl (lvm->container_fd, VFIO_SET_IOMMU, lvm->iommu_mode) < 0) |
Damjan Marion | 1ba0fa4 | 2018-03-04 17:19:08 +0100 | [diff] [blame] | 163 | { |
| 164 | err = clib_error_return_unix (0, "ioctl(VFIO_SET_IOMMU) " |
| 165 | "'/dev/vfio/vfio'"); |
| 166 | goto error; |
| 167 | } |
Damjan Marion | 1ba0fa4 | 2018-03-04 17:19:08 +0100 | [diff] [blame] | 168 | } |
| 169 | |
| 170 | |
| 171 | pool_get (lvm->iommu_groups, g); |
| 172 | g->fd = fd; |
| 173 | g->refcnt = 1; |
| 174 | hash_set (lvm->iommu_pool_index_by_group, group, g - lvm->iommu_groups); |
| 175 | vec_free (s); |
| 176 | return 0; |
| 177 | error: |
| 178 | close (fd); |
| 179 | return err; |
| 180 | } |
| 181 | |
| 182 | clib_error_t * |
Damjan Marion | 2322798 | 2018-10-22 13:38:57 +0200 | [diff] [blame^] | 183 | linux_vfio_group_get_device_fd (vlib_pci_addr_t * addr, int *fdp, |
| 184 | int *is_noiommu) |
Damjan Marion | 1ba0fa4 | 2018-03-04 17:19:08 +0100 | [diff] [blame] | 185 | { |
| 186 | clib_error_t *err = 0; |
| 187 | linux_pci_vfio_iommu_group_t *g; |
| 188 | u8 *s = 0; |
| 189 | int iommu_group; |
| 190 | u8 *tmpstr; |
| 191 | int fd; |
| 192 | |
Damjan Marion | 2322798 | 2018-10-22 13:38:57 +0200 | [diff] [blame^] | 193 | *is_noiommu = 0; |
Damjan Marion | 1ba0fa4 | 2018-03-04 17:19:08 +0100 | [diff] [blame] | 194 | s = format (s, "/sys/bus/pci/devices/%U/iommu_group", format_vlib_pci_addr, |
| 195 | addr); |
| 196 | tmpstr = clib_sysfs_link_to_name ((char *) s); |
| 197 | if (tmpstr) |
| 198 | { |
| 199 | iommu_group = atoi ((char *) tmpstr); |
| 200 | vec_free (tmpstr); |
| 201 | } |
| 202 | else |
| 203 | { |
| 204 | err = clib_error_return (0, "Cannot find IOMMU group for PCI device ", |
| 205 | "'%U'", format_vlib_pci_addr, addr); |
| 206 | goto error; |
| 207 | } |
| 208 | vec_reset_length (s); |
| 209 | |
Damjan Marion | 2322798 | 2018-10-22 13:38:57 +0200 | [diff] [blame^] | 210 | s = format (s, "/sys/bus/pci/devices/%U/iommu_group/name", |
| 211 | format_vlib_pci_addr, addr); |
Damjan Marion | 20ba164 | 2018-03-26 15:35:33 +0200 | [diff] [blame] | 212 | err = clib_sysfs_read ((char *) s, "%s", &tmpstr); |
| 213 | if (err == 0) |
| 214 | { |
| 215 | if (strncmp ((char *) tmpstr, "vfio-noiommu", 12) == 0) |
Damjan Marion | 2322798 | 2018-10-22 13:38:57 +0200 | [diff] [blame^] | 216 | *is_noiommu = 1; |
| 217 | |
Damjan Marion | 20ba164 | 2018-03-26 15:35:33 +0200 | [diff] [blame] | 218 | vec_free (tmpstr); |
| 219 | } |
| 220 | else |
| 221 | clib_error_free (err); |
| 222 | vec_reset_length (s); |
Damjan Marion | 2322798 | 2018-10-22 13:38:57 +0200 | [diff] [blame^] | 223 | if ((err = open_vfio_iommu_group (iommu_group, *is_noiommu))) |
Damjan Marion | 1ba0fa4 | 2018-03-04 17:19:08 +0100 | [diff] [blame] | 224 | return err; |
| 225 | |
| 226 | g = get_vfio_iommu_group (iommu_group); |
| 227 | |
| 228 | s = format (s, "%U%c", format_vlib_pci_addr, addr, 0); |
| 229 | if ((fd = ioctl (g->fd, VFIO_GROUP_GET_DEVICE_FD, (char *) s)) < 0) |
| 230 | { |
| 231 | err = clib_error_return_unix (0, "ioctl(VFIO_GROUP_GET_DEVICE_FD) '%U'", |
| 232 | format_vlib_pci_addr, addr); |
| 233 | goto error; |
| 234 | } |
| 235 | vec_reset_length (s); |
| 236 | |
| 237 | *fdp = fd; |
| 238 | |
| 239 | error: |
| 240 | vec_free (s); |
| 241 | return err; |
| 242 | } |
| 243 | |
| 244 | clib_error_t * |
| 245 | linux_vfio_init (vlib_main_t * vm) |
| 246 | { |
| 247 | linux_vfio_main_t *lvm = &vfio_main; |
Damjan Marion | 1ba0fa4 | 2018-03-04 17:19:08 +0100 | [diff] [blame] | 248 | |
Damjan Marion | 7d42653 | 2018-09-27 14:54:49 +0200 | [diff] [blame] | 249 | lvm->log_default = vlib_log_register_class ("vfio", 0); |
Damjan Marion | 2322798 | 2018-10-22 13:38:57 +0200 | [diff] [blame^] | 250 | lvm->container_fd = -1; |
Damjan Marion | 7d42653 | 2018-09-27 14:54:49 +0200 | [diff] [blame] | 251 | |
Damjan Marion | 1ba0fa4 | 2018-03-04 17:19:08 +0100 | [diff] [blame] | 252 | return 0; |
| 253 | } |
| 254 | |
| 255 | /* |
| 256 | * fd.io coding-style-patch-verification: ON |
| 257 | * |
| 258 | * Local Variables: |
| 259 | * eval: (c-set-style "gnu") |
| 260 | * End: |
| 261 | */ |