blob: c4973a6b8e04ef7034f620bc21942299e00d9321 [file] [log] [blame]
Damjan Marion01914ce2017-09-14 19:04:50 +02001/*
2 * Copyright (c) 2017 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16#define _GNU_SOURCE
17#include <stdlib.h>
18#include <sys/types.h>
19#include <sys/stat.h>
20#include <unistd.h>
21#include <sys/mount.h>
22#include <sys/mman.h>
23#include <fcntl.h>
24#include <linux/mempolicy.h>
25#include <linux/memfd.h>
26
27#include <vppinfra/clib.h>
28#include <vppinfra/mem.h>
Florin Corasd3e83a92018-01-16 02:40:18 -080029#include <vppinfra/time.h>
Damjan Marion01914ce2017-09-14 19:04:50 +020030#include <vppinfra/format.h>
31#include <vppinfra/clib_error.h>
32#include <vppinfra/linux/syscall.h>
33#include <vppinfra/linux/sysfs.h>
34
35#ifndef F_LINUX_SPECIFIC_BASE
36#define F_LINUX_SPECIFIC_BASE 1024
37#endif
38
39#ifndef F_ADD_SEALS
40#define F_ADD_SEALS (F_LINUX_SPECIFIC_BASE + 9)
41#define F_GET_SEALS (F_LINUX_SPECIFIC_BASE + 10)
42
43#define F_SEAL_SEAL 0x0001 /* prevent further seals from being set */
44#define F_SEAL_SHRINK 0x0002 /* prevent file from shrinking */
45#define F_SEAL_GROW 0x0004 /* prevent file from growing */
46#define F_SEAL_WRITE 0x0008 /* prevent writes */
47#endif
48
Florin Corasd3e83a92018-01-16 02:40:18 -080049uword
50clib_mem_vm_get_page_size (int fd)
Damjan Marion01914ce2017-09-14 19:04:50 +020051{
52 struct stat st = { 0 };
Chris Lukeb2bcad62017-09-18 08:51:22 -040053 if (fstat (fd, &st) == -1)
Damjan Marion01914ce2017-09-14 19:04:50 +020054 return 0;
Florin Corasd3e83a92018-01-16 02:40:18 -080055 return st.st_blksize;
56}
57
58int
59clib_mem_vm_get_log2_page_size (int fd)
60{
61 return min_log2 (clib_mem_vm_get_page_size (fd));
62}
63
Florin Corasb384b542018-01-15 01:08:33 -080064void
Florin Corasd3e83a92018-01-16 02:40:18 -080065clib_mem_vm_randomize_va (uword * requested_va, u32 log2_page_size)
66{
67 u8 bit_mask = 15;
68
69 if (log2_page_size <= 12)
70 bit_mask = 15;
71 else if (log2_page_size > 12 && log2_page_size <= 16)
72 bit_mask = 3;
73 else
74 bit_mask = 0;
75
76 *requested_va += (clib_cpu_time_now () & bit_mask) * (1 << log2_page_size);
Damjan Marion01914ce2017-09-14 19:04:50 +020077}
78
79clib_error_t *
80clib_mem_vm_ext_alloc (clib_mem_vm_alloc_t * a)
81{
82 int fd = -1;
83 clib_error_t *err = 0;
84 void *addr = 0;
85 u8 *filename = 0;
86 int mmap_flags = MAP_SHARED;
87 int log2_page_size;
88 int n_pages;
89 int old_mpol = -1;
90 u64 old_mask[16] = { 0 };
91
92 /* save old numa mem policy if needed */
93 if (a->flags & (CLIB_MEM_VM_F_NUMA_PREFER | CLIB_MEM_VM_F_NUMA_FORCE))
94 {
95 int rv;
Damjan Marion2e172ea2018-01-03 15:48:34 +000096 rv = get_mempolicy (&old_mpol, old_mask, sizeof (old_mask) * 8 + 1,
97 0, 0);
Damjan Marion01914ce2017-09-14 19:04:50 +020098
99 if (rv == -1)
100 {
Damjan Marion2e172ea2018-01-03 15:48:34 +0000101 if (a->numa_node != 0 && (a->flags & CLIB_MEM_VM_F_NUMA_FORCE) != 0)
Damjan Marion01914ce2017-09-14 19:04:50 +0200102 {
103 err = clib_error_return_unix (0, "get_mempolicy");
104 goto error;
105 }
106 else
107 old_mpol = -1;
108 }
109 }
110
111 /* if we are creating shared segment, we need file descriptor */
112 if (a->flags & CLIB_MEM_VM_F_SHARED)
113 {
114 /* if hugepages are needed we need to create mount point */
115 if (a->flags & CLIB_MEM_VM_F_HUGETLB)
116 {
117 char *mount_dir;
118 char template[] = "/tmp/hugepage_mount.XXXXXX";
119
120 mount_dir = mkdtemp (template);
121 if (mount_dir == 0)
122 return clib_error_return_unix (0, "mkdtemp \'%s\'", template);
123
124 if (mount ("none", (char *) mount_dir, "hugetlbfs", 0, NULL))
125 {
126 err = clib_error_return_unix (0, "mount hugetlb directory '%s'",
127 mount_dir);
128 goto error;
129 }
130
131 filename = format (0, "%s/%s%c", mount_dir, a->name, 0);
132
133 if ((fd = open ((char *) filename, O_CREAT | O_RDWR, 0755)) == -1)
134 {
135 err = clib_error_return_unix (0, "open");
136 goto error;
137 }
138 umount2 ((char *) mount_dir, MNT_DETACH);
139 rmdir ((char *) mount_dir);
140 mmap_flags |= MAP_LOCKED;
141 }
142 else
143 {
144 if ((fd = memfd_create (a->name, MFD_ALLOW_SEALING)) == -1)
145 {
146 err = clib_error_return_unix (0, "memfd_create");
147 goto error;
148 }
149
150 if ((fcntl (fd, F_ADD_SEALS, F_SEAL_SHRINK)) == -1)
151 {
152 err = clib_error_return_unix (0, "fcntl (F_ADD_SEALS)");
153 goto error;
154 }
155 }
Chris Luke879ace32017-09-26 13:15:16 -0400156
Florin Corasd3e83a92018-01-16 02:40:18 -0800157 log2_page_size = clib_mem_vm_get_log2_page_size (fd);
Chris Luke879ace32017-09-26 13:15:16 -0400158 if (log2_page_size == 0)
159 {
160 err = clib_error_return_unix (0, "cannot determine page size");
161 goto error;
162 }
Florin Corasd3e83a92018-01-16 02:40:18 -0800163
164 if (a->requested_va)
165 {
166 clib_mem_vm_randomize_va (&a->requested_va, log2_page_size);
167 mmap_flags |= MAP_FIXED;
168 }
Damjan Marion01914ce2017-09-14 19:04:50 +0200169 }
170 else /* not CLIB_MEM_VM_F_SHARED */
171 {
172 if (a->flags & CLIB_MEM_VM_F_HUGETLB)
173 {
174 mmap_flags |= MAP_HUGETLB | MAP_PRIVATE | MAP_ANONYMOUS;
175 log2_page_size = 21;
176 }
177 else
178 {
179 mmap_flags |= MAP_PRIVATE | MAP_ANONYMOUS;
180 log2_page_size = min_log2 (sysconf (_SC_PAGESIZE));
181 }
182 }
183
184 n_pages = ((a->size - 1) >> log2_page_size) + 1;
185
Damjan Marion01914ce2017-09-14 19:04:50 +0200186 if (a->flags & CLIB_MEM_VM_F_HUGETLB_PREALLOC)
187 {
188 err = clib_sysfs_prealloc_hugepages (a->numa_node,
189 1 << (log2_page_size - 10),
190 n_pages);
191 if (err)
192 goto error;
193
194 }
195
196 if (fd != -1)
Igor Mikhailov (imichail)7d254152018-02-05 12:09:36 -0800197 if ((ftruncate (fd, n_pages * (1 << log2_page_size))) == -1)
Damjan Marion01914ce2017-09-14 19:04:50 +0200198 {
199 err = clib_error_return_unix (0, "ftruncate");
200 goto error;
201 }
202
203 if (old_mpol != -1)
204 {
205 int rv;
206 u64 mask[16] = { 0 };
207 mask[0] = 1 << a->numa_node;
208 rv = set_mempolicy (MPOL_BIND, mask, sizeof (mask) * 8 + 1);
209 if (rv)
210 {
211 err = clib_error_return_unix (0, "set_mempolicy");
212 goto error;
213 }
214 }
215
Florin Corasd3e83a92018-01-16 02:40:18 -0800216 addr = mmap (uword_to_pointer (a->requested_va, void *), a->size,
217 (PROT_READ | PROT_WRITE), mmap_flags, fd, 0);
Damjan Marion01914ce2017-09-14 19:04:50 +0200218 if (addr == MAP_FAILED)
219 {
220 err = clib_error_return_unix (0, "mmap");
221 goto error;
222 }
223
Damjan Marion2e172ea2018-01-03 15:48:34 +0000224 /* re-apply old numa memory policy */
Damjan Marion01914ce2017-09-14 19:04:50 +0200225 if (old_mpol != -1 &&
226 set_mempolicy (old_mpol, old_mask, sizeof (old_mask) * 8 + 1) == -1)
227 {
228 err = clib_error_return_unix (0, "set_mempolicy");
229 goto error;
230 }
231
232 a->log2_page_size = log2_page_size;
233 a->n_pages = n_pages;
234 a->addr = addr;
235 a->fd = fd;
236 goto done;
237
238error:
239 if (fd != -1)
240 close (fd);
241
242done:
243 vec_free (filename);
244 return err;
245}
246
247u64 *
248clib_mem_vm_get_paddr (void *mem, int log2_page_size, int n_pages)
249{
250 int pagesize = sysconf (_SC_PAGESIZE);
251 int fd;
252 int i;
253 u64 *r = 0;
254
255 if ((fd = open ((char *) "/proc/self/pagemap", O_RDONLY)) == -1)
256 return 0;
257
258 for (i = 0; i < n_pages; i++)
259 {
260 u64 seek, pagemap = 0;
261 uword vaddr = pointer_to_uword (mem) + (((u64) i) << log2_page_size);
262 seek = ((u64) vaddr / pagesize) * sizeof (u64);
263 if (lseek (fd, seek, SEEK_SET) != seek)
264 goto done;
265
266 if (read (fd, &pagemap, sizeof (pagemap)) != (sizeof (pagemap)))
267 goto done;
268
269 if ((pagemap & (1ULL << 63)) == 0)
270 goto done;
271
272 pagemap &= pow2_mask (55);
273 vec_add1 (r, pagemap * pagesize);
274 }
275
276done:
277 close (fd);
278 if (vec_len (r) != n_pages)
279 {
280 vec_free (r);
281 return 0;
282 }
283 return r;
284}
285
Florin Corasd3e83a92018-01-16 02:40:18 -0800286clib_error_t *
287clib_mem_vm_ext_map (clib_mem_vm_map_t * a)
288{
289 int mmap_flags = MAP_SHARED;
290 void *addr;
Damjan Marion01914ce2017-09-14 19:04:50 +0200291
Florin Corasd3e83a92018-01-16 02:40:18 -0800292 if (a->requested_va)
293 mmap_flags |= MAP_FIXED;
294
295 addr = (void *) mmap (uword_to_pointer (a->requested_va, void *), a->size,
296 PROT_READ | PROT_WRITE, mmap_flags, a->fd, 0);
297
298 if (addr == MAP_FAILED)
299 return clib_error_return_unix (0, "mmap");
300
301 a->addr = addr;
302 return 0;
303}
Damjan Marion01914ce2017-09-14 19:04:50 +0200304
305/*
306 * fd.io coding-style-patch-verification: ON
307 *
308 * Local Variables:
309 * eval: (c-set-style "gnu")
310 * End:
311 */