Damjan Marion | 1ba0fa4 | 2018-03-04 17:19:08 +0100 | [diff] [blame] | 1 | /* |
Damjan Marion | 20ba164 | 2018-03-26 15:35:33 +0200 | [diff] [blame] | 2 | * Copyright (c) 2018 Cisco and/or its affiliates. |
Damjan Marion | 1ba0fa4 | 2018-03-04 17:19:08 +0100 | [diff] [blame] | 3 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | * you may not use this file except in compliance with the License. |
| 5 | * You may obtain a copy of the License at: |
| 6 | * |
| 7 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | * |
| 9 | * Unless required by applicable law or agreed to in writing, software |
| 10 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | * See the License for the specific language governing permissions and |
| 13 | * limitations under the License. |
| 14 | */ |
Damjan Marion | 1ba0fa4 | 2018-03-04 17:19:08 +0100 | [diff] [blame] | 15 | |
| 16 | #include <unistd.h> |
| 17 | #include <sys/types.h> |
| 18 | #include <sys/stat.h> |
| 19 | #include <fcntl.h> |
| 20 | #include <linux/vfio.h> |
| 21 | #include <sys/ioctl.h> |
| 22 | |
| 23 | #include <vppinfra/linux/sysfs.h> |
| 24 | |
| 25 | #include <vlib/vlib.h> |
| 26 | #include <vlib/unix/unix.h> |
| 27 | #include <vlib/pci/pci.h> |
| 28 | #include <vlib/linux/vfio.h> |
| 29 | #include <vlib/physmem.h> |
| 30 | |
Damjan Marion | 20ba164 | 2018-03-26 15:35:33 +0200 | [diff] [blame] | 31 | #ifndef VFIO_NOIOMMU_IOMMU |
| 32 | #define VFIO_NOIOMMU_IOMMU 8 |
| 33 | #endif |
| 34 | |
Damjan Marion | 1ba0fa4 | 2018-03-04 17:19:08 +0100 | [diff] [blame] | 35 | linux_vfio_main_t vfio_main; |
| 36 | |
| 37 | static int |
Damjan Marion | b7620fd | 2018-10-05 16:46:00 +0200 | [diff] [blame^] | 38 | vfio_map_regions (vlib_main_t * vm, int fd) |
Damjan Marion | 1ba0fa4 | 2018-03-04 17:19:08 +0100 | [diff] [blame] | 39 | { |
Damjan Marion | 88c0621 | 2018-03-05 20:08:28 +0100 | [diff] [blame] | 40 | vlib_physmem_main_t *vpm = &physmem_main; |
Damjan Marion | 7d42653 | 2018-09-27 14:54:49 +0200 | [diff] [blame] | 41 | linux_vfio_main_t *lvm = &vfio_main; |
Damjan Marion | 1ba0fa4 | 2018-03-04 17:19:08 +0100 | [diff] [blame] | 42 | vlib_physmem_region_t *pr; |
| 43 | struct vfio_iommu_type1_dma_map dm = { 0 }; |
| 44 | int i; |
| 45 | |
| 46 | dm.argsz = sizeof (struct vfio_iommu_type1_dma_map); |
| 47 | dm.flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE; |
| 48 | |
| 49 | /* *INDENT-OFF* */ |
| 50 | pool_foreach (pr, vpm->regions, |
| 51 | { |
| 52 | vec_foreach_index (i, pr->page_table) |
| 53 | { |
| 54 | int rv; |
| 55 | dm.vaddr = pointer_to_uword (pr->mem) + (i << pr->log2_page_size); |
| 56 | dm.size = 1 << pr->log2_page_size; |
Damjan Marion | 51c52c0 | 2018-03-09 16:05:56 +0100 | [diff] [blame] | 57 | dm.iova = dm.vaddr; |
Damjan Marion | 7d42653 | 2018-09-27 14:54:49 +0200 | [diff] [blame] | 58 | vlib_log_debug (lvm->log_default, "map DMA va:0x%lx iova:%lx " |
| 59 | "size:0x%lx", dm.vaddr, dm.iova, dm.size); |
| 60 | |
Damjan Marion | b7620fd | 2018-10-05 16:46:00 +0200 | [diff] [blame^] | 61 | if ((rv = ioctl (fd, VFIO_IOMMU_MAP_DMA, &dm)) && |
| 62 | errno != EINVAL) |
Damjan Marion | 7d42653 | 2018-09-27 14:54:49 +0200 | [diff] [blame] | 63 | { |
| 64 | vlib_log_err (lvm->log_default, "map DMA va:0x%lx iova:%lx " |
| 65 | "size:0x%lx failed, error %s (errno %d)", |
| 66 | dm.vaddr, dm.iova, dm.size, strerror (errno), |
| 67 | errno); |
| 68 | return rv; |
| 69 | } |
Damjan Marion | 1ba0fa4 | 2018-03-04 17:19:08 +0100 | [diff] [blame] | 70 | } |
| 71 | }); |
| 72 | /* *INDENT-ON* */ |
| 73 | return 0; |
| 74 | } |
| 75 | |
Damjan Marion | 1ba0fa4 | 2018-03-04 17:19:08 +0100 | [diff] [blame] | 76 | void |
| 77 | linux_vfio_dma_map_regions (vlib_main_t * vm) |
| 78 | { |
| 79 | linux_vfio_main_t *lvm = &vfio_main; |
| 80 | |
| 81 | if (lvm->container_fd != -1) |
Damjan Marion | b7620fd | 2018-10-05 16:46:00 +0200 | [diff] [blame^] | 82 | vfio_map_regions (vm, lvm->container_fd); |
Damjan Marion | 1ba0fa4 | 2018-03-04 17:19:08 +0100 | [diff] [blame] | 83 | } |
| 84 | |
| 85 | static linux_pci_vfio_iommu_group_t * |
| 86 | get_vfio_iommu_group (int group) |
| 87 | { |
| 88 | linux_vfio_main_t *lvm = &vfio_main; |
| 89 | uword *p; |
| 90 | |
| 91 | p = hash_get (lvm->iommu_pool_index_by_group, group); |
| 92 | |
| 93 | return p ? pool_elt_at_index (lvm->iommu_groups, p[0]) : 0; |
| 94 | } |
| 95 | |
| 96 | static clib_error_t * |
Damjan Marion | 20ba164 | 2018-03-26 15:35:33 +0200 | [diff] [blame] | 97 | open_vfio_iommu_group (int group, int is_noiommu) |
Damjan Marion | 1ba0fa4 | 2018-03-04 17:19:08 +0100 | [diff] [blame] | 98 | { |
| 99 | linux_vfio_main_t *lvm = &vfio_main; |
| 100 | linux_pci_vfio_iommu_group_t *g; |
| 101 | clib_error_t *err = 0; |
| 102 | struct vfio_group_status group_status; |
| 103 | u8 *s = 0; |
| 104 | int fd; |
| 105 | |
| 106 | g = get_vfio_iommu_group (group); |
| 107 | if (g) |
| 108 | { |
| 109 | g->refcnt++; |
| 110 | return 0; |
| 111 | } |
Damjan Marion | 20ba164 | 2018-03-26 15:35:33 +0200 | [diff] [blame] | 112 | s = format (s, "/dev/vfio/%s%u%c", is_noiommu ? "noiommu-" : "", group, 0); |
Damjan Marion | 1ba0fa4 | 2018-03-04 17:19:08 +0100 | [diff] [blame] | 113 | fd = open ((char *) s, O_RDWR); |
| 114 | if (fd < 0) |
| 115 | return clib_error_return_unix (0, "open '%s'", s); |
| 116 | |
| 117 | group_status.argsz = sizeof (group_status); |
| 118 | if (ioctl (fd, VFIO_GROUP_GET_STATUS, &group_status) < 0) |
| 119 | { |
| 120 | err = clib_error_return_unix (0, "ioctl(VFIO_GROUP_GET_STATUS) '%s'", |
| 121 | s); |
| 122 | goto error; |
| 123 | } |
| 124 | |
| 125 | if (!(group_status.flags & VFIO_GROUP_FLAGS_VIABLE)) |
| 126 | { |
| 127 | err = clib_error_return (0, "iommu group %d is not viable (not all " |
| 128 | "devices in this group bound to vfio-pci)", |
| 129 | group); |
| 130 | goto error; |
| 131 | } |
| 132 | |
| 133 | if (ioctl (fd, VFIO_GROUP_SET_CONTAINER, &lvm->container_fd) < 0) |
| 134 | { |
| 135 | err = clib_error_return_unix (0, "ioctl(VFIO_GROUP_SET_CONTAINER) '%s'", |
| 136 | s); |
| 137 | goto error; |
| 138 | } |
| 139 | |
| 140 | if (lvm->iommu_mode == 0) |
| 141 | { |
Damjan Marion | 20ba164 | 2018-03-26 15:35:33 +0200 | [diff] [blame] | 142 | if (is_noiommu) |
| 143 | lvm->iommu_mode = VFIO_NOIOMMU_IOMMU; |
| 144 | else |
| 145 | lvm->iommu_mode = VFIO_TYPE1_IOMMU; |
| 146 | |
| 147 | if (ioctl (lvm->container_fd, VFIO_SET_IOMMU, lvm->iommu_mode) < 0) |
Damjan Marion | 1ba0fa4 | 2018-03-04 17:19:08 +0100 | [diff] [blame] | 148 | { |
| 149 | err = clib_error_return_unix (0, "ioctl(VFIO_SET_IOMMU) " |
| 150 | "'/dev/vfio/vfio'"); |
| 151 | goto error; |
| 152 | } |
Damjan Marion | 1ba0fa4 | 2018-03-04 17:19:08 +0100 | [diff] [blame] | 153 | } |
| 154 | |
| 155 | |
| 156 | pool_get (lvm->iommu_groups, g); |
| 157 | g->fd = fd; |
| 158 | g->refcnt = 1; |
| 159 | hash_set (lvm->iommu_pool_index_by_group, group, g - lvm->iommu_groups); |
| 160 | vec_free (s); |
| 161 | return 0; |
| 162 | error: |
| 163 | close (fd); |
| 164 | return err; |
| 165 | } |
| 166 | |
| 167 | clib_error_t * |
| 168 | linux_vfio_group_get_device_fd (vlib_pci_addr_t * addr, int *fdp) |
| 169 | { |
| 170 | clib_error_t *err = 0; |
| 171 | linux_pci_vfio_iommu_group_t *g; |
| 172 | u8 *s = 0; |
| 173 | int iommu_group; |
| 174 | u8 *tmpstr; |
| 175 | int fd; |
Damjan Marion | 20ba164 | 2018-03-26 15:35:33 +0200 | [diff] [blame] | 176 | int is_noiommu = 0; |
Damjan Marion | 1ba0fa4 | 2018-03-04 17:19:08 +0100 | [diff] [blame] | 177 | |
| 178 | s = format (s, "/sys/bus/pci/devices/%U/iommu_group", format_vlib_pci_addr, |
| 179 | addr); |
| 180 | tmpstr = clib_sysfs_link_to_name ((char *) s); |
| 181 | if (tmpstr) |
| 182 | { |
| 183 | iommu_group = atoi ((char *) tmpstr); |
| 184 | vec_free (tmpstr); |
| 185 | } |
| 186 | else |
| 187 | { |
| 188 | err = clib_error_return (0, "Cannot find IOMMU group for PCI device ", |
| 189 | "'%U'", format_vlib_pci_addr, addr); |
| 190 | goto error; |
| 191 | } |
| 192 | vec_reset_length (s); |
| 193 | |
Damjan Marion | 20ba164 | 2018-03-26 15:35:33 +0200 | [diff] [blame] | 194 | s = |
| 195 | format (s, "/sys/bus/pci/devices/%U/iommu_group/name", |
| 196 | format_vlib_pci_addr, addr); |
| 197 | err = clib_sysfs_read ((char *) s, "%s", &tmpstr); |
| 198 | if (err == 0) |
| 199 | { |
| 200 | if (strncmp ((char *) tmpstr, "vfio-noiommu", 12) == 0) |
| 201 | is_noiommu = 1; |
| 202 | vec_free (tmpstr); |
| 203 | } |
| 204 | else |
| 205 | clib_error_free (err); |
| 206 | vec_reset_length (s); |
| 207 | if ((err = open_vfio_iommu_group (iommu_group, is_noiommu))) |
Damjan Marion | 1ba0fa4 | 2018-03-04 17:19:08 +0100 | [diff] [blame] | 208 | return err; |
| 209 | |
| 210 | g = get_vfio_iommu_group (iommu_group); |
| 211 | |
| 212 | s = format (s, "%U%c", format_vlib_pci_addr, addr, 0); |
| 213 | if ((fd = ioctl (g->fd, VFIO_GROUP_GET_DEVICE_FD, (char *) s)) < 0) |
| 214 | { |
| 215 | err = clib_error_return_unix (0, "ioctl(VFIO_GROUP_GET_DEVICE_FD) '%U'", |
| 216 | format_vlib_pci_addr, addr); |
| 217 | goto error; |
| 218 | } |
| 219 | vec_reset_length (s); |
| 220 | |
| 221 | *fdp = fd; |
| 222 | |
| 223 | error: |
| 224 | vec_free (s); |
| 225 | return err; |
| 226 | } |
| 227 | |
| 228 | clib_error_t * |
| 229 | linux_vfio_init (vlib_main_t * vm) |
| 230 | { |
| 231 | linux_vfio_main_t *lvm = &vfio_main; |
| 232 | int fd; |
| 233 | |
Damjan Marion | 7d42653 | 2018-09-27 14:54:49 +0200 | [diff] [blame] | 234 | lvm->log_default = vlib_log_register_class ("vfio", 0); |
| 235 | |
Damjan Marion | 1ba0fa4 | 2018-03-04 17:19:08 +0100 | [diff] [blame] | 236 | fd = open ("/dev/vfio/vfio", O_RDWR); |
| 237 | |
| 238 | /* check if iommu is available */ |
| 239 | if (fd != -1) |
| 240 | { |
| 241 | if (ioctl (fd, VFIO_GET_API_VERSION) != VFIO_API_VERSION) |
| 242 | { |
| 243 | close (fd); |
| 244 | fd = -1; |
| 245 | } |
Damjan Marion | 20ba164 | 2018-03-26 15:35:33 +0200 | [diff] [blame] | 246 | else |
| 247 | { |
| 248 | if (ioctl (fd, VFIO_CHECK_EXTENSION, VFIO_TYPE1_IOMMU) == 1) |
Damjan Marion | 7d42653 | 2018-09-27 14:54:49 +0200 | [diff] [blame] | 249 | { |
| 250 | lvm->flags |= LINUX_VFIO_F_HAVE_IOMMU; |
| 251 | vlib_log_info (lvm->log_default, "type 1 IOMMU mode supported"); |
| 252 | } |
Damjan Marion | 20ba164 | 2018-03-26 15:35:33 +0200 | [diff] [blame] | 253 | if (ioctl (fd, VFIO_CHECK_EXTENSION, VFIO_NOIOMMU_IOMMU) == 1) |
Damjan Marion | 7d42653 | 2018-09-27 14:54:49 +0200 | [diff] [blame] | 254 | { |
| 255 | lvm->flags |= LINUX_VFIO_F_HAVE_NOIOMMU; |
| 256 | vlib_log_info (lvm->log_default, "NOIOMMU mode supported"); |
| 257 | } |
Damjan Marion | 20ba164 | 2018-03-26 15:35:33 +0200 | [diff] [blame] | 258 | } |
Damjan Marion | 1ba0fa4 | 2018-03-04 17:19:08 +0100 | [diff] [blame] | 259 | } |
| 260 | |
| 261 | lvm->iommu_pool_index_by_group = hash_create (0, sizeof (uword)); |
| 262 | lvm->container_fd = fd; |
| 263 | return 0; |
| 264 | } |
| 265 | |
| 266 | /* |
| 267 | * fd.io coding-style-patch-verification: ON |
| 268 | * |
| 269 | * Local Variables: |
| 270 | * eval: (c-set-style "gnu") |
| 271 | * End: |
| 272 | */ |