blob: 02262719e8268e61d3e61975f6b8e8acc9f568ba [file] [log] [blame]
Damjan Marion1ba0fa42018-03-04 17:19:08 +01001/*
Damjan Marion20ba1642018-03-26 15:35:33 +02002 * Copyright (c) 2018 Cisco and/or its affiliates.
Damjan Marion1ba0fa42018-03-04 17:19:08 +01003 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
Damjan Marion1ba0fa42018-03-04 17:19:08 +010015
16#include <unistd.h>
17#include <sys/types.h>
18#include <sys/stat.h>
19#include <fcntl.h>
20#include <linux/vfio.h>
21#include <sys/ioctl.h>
22
23#include <vppinfra/linux/sysfs.h>
24
25#include <vlib/vlib.h>
26#include <vlib/unix/unix.h>
27#include <vlib/pci/pci.h>
28#include <vlib/linux/vfio.h>
29#include <vlib/physmem.h>
30
Damjan Marion20ba1642018-03-26 15:35:33 +020031#ifndef VFIO_NOIOMMU_IOMMU
32#define VFIO_NOIOMMU_IOMMU 8
33#endif
34
Damjan Marion1ba0fa42018-03-04 17:19:08 +010035linux_vfio_main_t vfio_main;
36
37static int
Damjan Marionb7620fd2018-10-05 16:46:00 +020038vfio_map_regions (vlib_main_t * vm, int fd)
Damjan Marion1ba0fa42018-03-04 17:19:08 +010039{
Damjan Marion88c06212018-03-05 20:08:28 +010040 vlib_physmem_main_t *vpm = &physmem_main;
Damjan Marion7d426532018-09-27 14:54:49 +020041 linux_vfio_main_t *lvm = &vfio_main;
Damjan Marion1ba0fa42018-03-04 17:19:08 +010042 vlib_physmem_region_t *pr;
43 struct vfio_iommu_type1_dma_map dm = { 0 };
44 int i;
45
46 dm.argsz = sizeof (struct vfio_iommu_type1_dma_map);
47 dm.flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE;
48
49 /* *INDENT-OFF* */
50 pool_foreach (pr, vpm->regions,
51 {
52 vec_foreach_index (i, pr->page_table)
53 {
54 int rv;
55 dm.vaddr = pointer_to_uword (pr->mem) + (i << pr->log2_page_size);
56 dm.size = 1 << pr->log2_page_size;
Damjan Marion51c52c02018-03-09 16:05:56 +010057 dm.iova = dm.vaddr;
Damjan Marion7d426532018-09-27 14:54:49 +020058 vlib_log_debug (lvm->log_default, "map DMA va:0x%lx iova:%lx "
59 "size:0x%lx", dm.vaddr, dm.iova, dm.size);
60
Damjan Marionb7620fd2018-10-05 16:46:00 +020061 if ((rv = ioctl (fd, VFIO_IOMMU_MAP_DMA, &dm)) &&
62 errno != EINVAL)
Damjan Marion7d426532018-09-27 14:54:49 +020063 {
64 vlib_log_err (lvm->log_default, "map DMA va:0x%lx iova:%lx "
65 "size:0x%lx failed, error %s (errno %d)",
66 dm.vaddr, dm.iova, dm.size, strerror (errno),
67 errno);
68 return rv;
69 }
Damjan Marion1ba0fa42018-03-04 17:19:08 +010070 }
71 });
72 /* *INDENT-ON* */
73 return 0;
74}
75
Damjan Marion1ba0fa42018-03-04 17:19:08 +010076void
77linux_vfio_dma_map_regions (vlib_main_t * vm)
78{
79 linux_vfio_main_t *lvm = &vfio_main;
80
81 if (lvm->container_fd != -1)
Damjan Marionb7620fd2018-10-05 16:46:00 +020082 vfio_map_regions (vm, lvm->container_fd);
Damjan Marion1ba0fa42018-03-04 17:19:08 +010083}
84
85static linux_pci_vfio_iommu_group_t *
86get_vfio_iommu_group (int group)
87{
88 linux_vfio_main_t *lvm = &vfio_main;
89 uword *p;
90
91 p = hash_get (lvm->iommu_pool_index_by_group, group);
92
93 return p ? pool_elt_at_index (lvm->iommu_groups, p[0]) : 0;
94}
95
96static clib_error_t *
Damjan Marion20ba1642018-03-26 15:35:33 +020097open_vfio_iommu_group (int group, int is_noiommu)
Damjan Marion1ba0fa42018-03-04 17:19:08 +010098{
99 linux_vfio_main_t *lvm = &vfio_main;
100 linux_pci_vfio_iommu_group_t *g;
101 clib_error_t *err = 0;
102 struct vfio_group_status group_status;
103 u8 *s = 0;
104 int fd;
105
106 g = get_vfio_iommu_group (group);
107 if (g)
108 {
109 g->refcnt++;
110 return 0;
111 }
Damjan Marion20ba1642018-03-26 15:35:33 +0200112 s = format (s, "/dev/vfio/%s%u%c", is_noiommu ? "noiommu-" : "", group, 0);
Damjan Marion1ba0fa42018-03-04 17:19:08 +0100113 fd = open ((char *) s, O_RDWR);
114 if (fd < 0)
115 return clib_error_return_unix (0, "open '%s'", s);
116
117 group_status.argsz = sizeof (group_status);
118 if (ioctl (fd, VFIO_GROUP_GET_STATUS, &group_status) < 0)
119 {
120 err = clib_error_return_unix (0, "ioctl(VFIO_GROUP_GET_STATUS) '%s'",
121 s);
122 goto error;
123 }
124
125 if (!(group_status.flags & VFIO_GROUP_FLAGS_VIABLE))
126 {
127 err = clib_error_return (0, "iommu group %d is not viable (not all "
128 "devices in this group bound to vfio-pci)",
129 group);
130 goto error;
131 }
132
133 if (ioctl (fd, VFIO_GROUP_SET_CONTAINER, &lvm->container_fd) < 0)
134 {
135 err = clib_error_return_unix (0, "ioctl(VFIO_GROUP_SET_CONTAINER) '%s'",
136 s);
137 goto error;
138 }
139
140 if (lvm->iommu_mode == 0)
141 {
Damjan Marion20ba1642018-03-26 15:35:33 +0200142 if (is_noiommu)
143 lvm->iommu_mode = VFIO_NOIOMMU_IOMMU;
144 else
145 lvm->iommu_mode = VFIO_TYPE1_IOMMU;
146
147 if (ioctl (lvm->container_fd, VFIO_SET_IOMMU, lvm->iommu_mode) < 0)
Damjan Marion1ba0fa42018-03-04 17:19:08 +0100148 {
149 err = clib_error_return_unix (0, "ioctl(VFIO_SET_IOMMU) "
150 "'/dev/vfio/vfio'");
151 goto error;
152 }
Damjan Marion1ba0fa42018-03-04 17:19:08 +0100153 }
154
155
156 pool_get (lvm->iommu_groups, g);
157 g->fd = fd;
158 g->refcnt = 1;
159 hash_set (lvm->iommu_pool_index_by_group, group, g - lvm->iommu_groups);
160 vec_free (s);
161 return 0;
162error:
163 close (fd);
164 return err;
165}
166
167clib_error_t *
168linux_vfio_group_get_device_fd (vlib_pci_addr_t * addr, int *fdp)
169{
170 clib_error_t *err = 0;
171 linux_pci_vfio_iommu_group_t *g;
172 u8 *s = 0;
173 int iommu_group;
174 u8 *tmpstr;
175 int fd;
Damjan Marion20ba1642018-03-26 15:35:33 +0200176 int is_noiommu = 0;
Damjan Marion1ba0fa42018-03-04 17:19:08 +0100177
178 s = format (s, "/sys/bus/pci/devices/%U/iommu_group", format_vlib_pci_addr,
179 addr);
180 tmpstr = clib_sysfs_link_to_name ((char *) s);
181 if (tmpstr)
182 {
183 iommu_group = atoi ((char *) tmpstr);
184 vec_free (tmpstr);
185 }
186 else
187 {
188 err = clib_error_return (0, "Cannot find IOMMU group for PCI device ",
189 "'%U'", format_vlib_pci_addr, addr);
190 goto error;
191 }
192 vec_reset_length (s);
193
Damjan Marion20ba1642018-03-26 15:35:33 +0200194 s =
195 format (s, "/sys/bus/pci/devices/%U/iommu_group/name",
196 format_vlib_pci_addr, addr);
197 err = clib_sysfs_read ((char *) s, "%s", &tmpstr);
198 if (err == 0)
199 {
200 if (strncmp ((char *) tmpstr, "vfio-noiommu", 12) == 0)
201 is_noiommu = 1;
202 vec_free (tmpstr);
203 }
204 else
205 clib_error_free (err);
206 vec_reset_length (s);
207 if ((err = open_vfio_iommu_group (iommu_group, is_noiommu)))
Damjan Marion1ba0fa42018-03-04 17:19:08 +0100208 return err;
209
210 g = get_vfio_iommu_group (iommu_group);
211
212 s = format (s, "%U%c", format_vlib_pci_addr, addr, 0);
213 if ((fd = ioctl (g->fd, VFIO_GROUP_GET_DEVICE_FD, (char *) s)) < 0)
214 {
215 err = clib_error_return_unix (0, "ioctl(VFIO_GROUP_GET_DEVICE_FD) '%U'",
216 format_vlib_pci_addr, addr);
217 goto error;
218 }
219 vec_reset_length (s);
220
221 *fdp = fd;
222
223error:
224 vec_free (s);
225 return err;
226}
227
228clib_error_t *
229linux_vfio_init (vlib_main_t * vm)
230{
231 linux_vfio_main_t *lvm = &vfio_main;
232 int fd;
233
Damjan Marion7d426532018-09-27 14:54:49 +0200234 lvm->log_default = vlib_log_register_class ("vfio", 0);
235
Damjan Marion1ba0fa42018-03-04 17:19:08 +0100236 fd = open ("/dev/vfio/vfio", O_RDWR);
237
238 /* check if iommu is available */
239 if (fd != -1)
240 {
241 if (ioctl (fd, VFIO_GET_API_VERSION) != VFIO_API_VERSION)
242 {
243 close (fd);
244 fd = -1;
245 }
Damjan Marion20ba1642018-03-26 15:35:33 +0200246 else
247 {
248 if (ioctl (fd, VFIO_CHECK_EXTENSION, VFIO_TYPE1_IOMMU) == 1)
Damjan Marion7d426532018-09-27 14:54:49 +0200249 {
250 lvm->flags |= LINUX_VFIO_F_HAVE_IOMMU;
251 vlib_log_info (lvm->log_default, "type 1 IOMMU mode supported");
252 }
Damjan Marion20ba1642018-03-26 15:35:33 +0200253 if (ioctl (fd, VFIO_CHECK_EXTENSION, VFIO_NOIOMMU_IOMMU) == 1)
Damjan Marion7d426532018-09-27 14:54:49 +0200254 {
255 lvm->flags |= LINUX_VFIO_F_HAVE_NOIOMMU;
256 vlib_log_info (lvm->log_default, "NOIOMMU mode supported");
257 }
Damjan Marion20ba1642018-03-26 15:35:33 +0200258 }
Damjan Marion1ba0fa42018-03-04 17:19:08 +0100259 }
260
261 lvm->iommu_pool_index_by_group = hash_create (0, sizeof (uword));
262 lvm->container_fd = fd;
263 return 0;
264}
265
266/*
267 * fd.io coding-style-patch-verification: ON
268 *
269 * Local Variables:
270 * eval: (c-set-style "gnu")
271 * End:
272 */