Damjan Marion | 01914ce | 2017-09-14 19:04:50 +0200 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (c) 2017 Cisco and/or its affiliates. |
| 3 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | * you may not use this file except in compliance with the License. |
| 5 | * You may obtain a copy of the License at: |
| 6 | * |
| 7 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | * |
| 9 | * Unless required by applicable law or agreed to in writing, software |
| 10 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | * See the License for the specific language governing permissions and |
| 13 | * limitations under the License. |
| 14 | */ |
| 15 | |
| 16 | #define _GNU_SOURCE |
| 17 | #include <stdlib.h> |
| 18 | #include <sys/types.h> |
| 19 | #include <sys/stat.h> |
| 20 | #include <unistd.h> |
| 21 | #include <sys/mount.h> |
| 22 | #include <sys/mman.h> |
| 23 | #include <fcntl.h> |
| 24 | #include <linux/mempolicy.h> |
| 25 | #include <linux/memfd.h> |
| 26 | |
| 27 | #include <vppinfra/clib.h> |
| 28 | #include <vppinfra/mem.h> |
Florin Coras | d3e83a9 | 2018-01-16 02:40:18 -0800 | [diff] [blame] | 29 | #include <vppinfra/time.h> |
Damjan Marion | 01914ce | 2017-09-14 19:04:50 +0200 | [diff] [blame] | 30 | #include <vppinfra/format.h> |
| 31 | #include <vppinfra/clib_error.h> |
| 32 | #include <vppinfra/linux/syscall.h> |
| 33 | #include <vppinfra/linux/sysfs.h> |
| 34 | |
| 35 | #ifndef F_LINUX_SPECIFIC_BASE |
| 36 | #define F_LINUX_SPECIFIC_BASE 1024 |
| 37 | #endif |
| 38 | |
| 39 | #ifndef F_ADD_SEALS |
| 40 | #define F_ADD_SEALS (F_LINUX_SPECIFIC_BASE + 9) |
| 41 | #define F_GET_SEALS (F_LINUX_SPECIFIC_BASE + 10) |
| 42 | |
| 43 | #define F_SEAL_SEAL 0x0001 /* prevent further seals from being set */ |
| 44 | #define F_SEAL_SHRINK 0x0002 /* prevent file from shrinking */ |
| 45 | #define F_SEAL_GROW 0x0004 /* prevent file from growing */ |
| 46 | #define F_SEAL_WRITE 0x0008 /* prevent writes */ |
| 47 | #endif |
| 48 | |
Florin Coras | d3e83a9 | 2018-01-16 02:40:18 -0800 | [diff] [blame] | 49 | uword |
| 50 | clib_mem_vm_get_page_size (int fd) |
Damjan Marion | 01914ce | 2017-09-14 19:04:50 +0200 | [diff] [blame] | 51 | { |
| 52 | struct stat st = { 0 }; |
Chris Luke | b2bcad6 | 2017-09-18 08:51:22 -0400 | [diff] [blame] | 53 | if (fstat (fd, &st) == -1) |
Damjan Marion | 01914ce | 2017-09-14 19:04:50 +0200 | [diff] [blame] | 54 | return 0; |
Florin Coras | d3e83a9 | 2018-01-16 02:40:18 -0800 | [diff] [blame] | 55 | return st.st_blksize; |
| 56 | } |
| 57 | |
| 58 | int |
| 59 | clib_mem_vm_get_log2_page_size (int fd) |
| 60 | { |
| 61 | return min_log2 (clib_mem_vm_get_page_size (fd)); |
| 62 | } |
| 63 | |
Florin Coras | b384b54 | 2018-01-15 01:08:33 -0800 | [diff] [blame] | 64 | void |
Florin Coras | d3e83a9 | 2018-01-16 02:40:18 -0800 | [diff] [blame] | 65 | clib_mem_vm_randomize_va (uword * requested_va, u32 log2_page_size) |
| 66 | { |
| 67 | u8 bit_mask = 15; |
| 68 | |
| 69 | if (log2_page_size <= 12) |
| 70 | bit_mask = 15; |
| 71 | else if (log2_page_size > 12 && log2_page_size <= 16) |
| 72 | bit_mask = 3; |
| 73 | else |
| 74 | bit_mask = 0; |
| 75 | |
| 76 | *requested_va += (clib_cpu_time_now () & bit_mask) * (1 << log2_page_size); |
Damjan Marion | 01914ce | 2017-09-14 19:04:50 +0200 | [diff] [blame] | 77 | } |
| 78 | |
| 79 | clib_error_t * |
| 80 | clib_mem_vm_ext_alloc (clib_mem_vm_alloc_t * a) |
| 81 | { |
| 82 | int fd = -1; |
| 83 | clib_error_t *err = 0; |
| 84 | void *addr = 0; |
| 85 | u8 *filename = 0; |
| 86 | int mmap_flags = MAP_SHARED; |
| 87 | int log2_page_size; |
| 88 | int n_pages; |
| 89 | int old_mpol = -1; |
| 90 | u64 old_mask[16] = { 0 }; |
| 91 | |
| 92 | /* save old numa mem policy if needed */ |
| 93 | if (a->flags & (CLIB_MEM_VM_F_NUMA_PREFER | CLIB_MEM_VM_F_NUMA_FORCE)) |
| 94 | { |
| 95 | int rv; |
Damjan Marion | 2e172ea | 2018-01-03 15:48:34 +0000 | [diff] [blame] | 96 | rv = get_mempolicy (&old_mpol, old_mask, sizeof (old_mask) * 8 + 1, |
| 97 | 0, 0); |
Damjan Marion | 01914ce | 2017-09-14 19:04:50 +0200 | [diff] [blame] | 98 | |
| 99 | if (rv == -1) |
| 100 | { |
Damjan Marion | 2e172ea | 2018-01-03 15:48:34 +0000 | [diff] [blame] | 101 | if (a->numa_node != 0 && (a->flags & CLIB_MEM_VM_F_NUMA_FORCE) != 0) |
Damjan Marion | 01914ce | 2017-09-14 19:04:50 +0200 | [diff] [blame] | 102 | { |
| 103 | err = clib_error_return_unix (0, "get_mempolicy"); |
| 104 | goto error; |
| 105 | } |
| 106 | else |
| 107 | old_mpol = -1; |
| 108 | } |
| 109 | } |
| 110 | |
| 111 | /* if we are creating shared segment, we need file descriptor */ |
| 112 | if (a->flags & CLIB_MEM_VM_F_SHARED) |
| 113 | { |
| 114 | /* if hugepages are needed we need to create mount point */ |
| 115 | if (a->flags & CLIB_MEM_VM_F_HUGETLB) |
| 116 | { |
| 117 | char *mount_dir; |
| 118 | char template[] = "/tmp/hugepage_mount.XXXXXX"; |
| 119 | |
| 120 | mount_dir = mkdtemp (template); |
| 121 | if (mount_dir == 0) |
| 122 | return clib_error_return_unix (0, "mkdtemp \'%s\'", template); |
| 123 | |
| 124 | if (mount ("none", (char *) mount_dir, "hugetlbfs", 0, NULL)) |
| 125 | { |
| 126 | err = clib_error_return_unix (0, "mount hugetlb directory '%s'", |
| 127 | mount_dir); |
| 128 | goto error; |
| 129 | } |
| 130 | |
| 131 | filename = format (0, "%s/%s%c", mount_dir, a->name, 0); |
| 132 | |
| 133 | if ((fd = open ((char *) filename, O_CREAT | O_RDWR, 0755)) == -1) |
| 134 | { |
| 135 | err = clib_error_return_unix (0, "open"); |
| 136 | goto error; |
| 137 | } |
| 138 | umount2 ((char *) mount_dir, MNT_DETACH); |
| 139 | rmdir ((char *) mount_dir); |
| 140 | mmap_flags |= MAP_LOCKED; |
| 141 | } |
| 142 | else |
| 143 | { |
| 144 | if ((fd = memfd_create (a->name, MFD_ALLOW_SEALING)) == -1) |
| 145 | { |
| 146 | err = clib_error_return_unix (0, "memfd_create"); |
| 147 | goto error; |
| 148 | } |
| 149 | |
| 150 | if ((fcntl (fd, F_ADD_SEALS, F_SEAL_SHRINK)) == -1) |
| 151 | { |
| 152 | err = clib_error_return_unix (0, "fcntl (F_ADD_SEALS)"); |
| 153 | goto error; |
| 154 | } |
| 155 | } |
Chris Luke | 879ace3 | 2017-09-26 13:15:16 -0400 | [diff] [blame] | 156 | |
Florin Coras | d3e83a9 | 2018-01-16 02:40:18 -0800 | [diff] [blame] | 157 | log2_page_size = clib_mem_vm_get_log2_page_size (fd); |
Chris Luke | 879ace3 | 2017-09-26 13:15:16 -0400 | [diff] [blame] | 158 | if (log2_page_size == 0) |
| 159 | { |
| 160 | err = clib_error_return_unix (0, "cannot determine page size"); |
| 161 | goto error; |
| 162 | } |
Florin Coras | d3e83a9 | 2018-01-16 02:40:18 -0800 | [diff] [blame] | 163 | |
| 164 | if (a->requested_va) |
| 165 | { |
| 166 | clib_mem_vm_randomize_va (&a->requested_va, log2_page_size); |
| 167 | mmap_flags |= MAP_FIXED; |
| 168 | } |
Damjan Marion | 01914ce | 2017-09-14 19:04:50 +0200 | [diff] [blame] | 169 | } |
| 170 | else /* not CLIB_MEM_VM_F_SHARED */ |
| 171 | { |
| 172 | if (a->flags & CLIB_MEM_VM_F_HUGETLB) |
| 173 | { |
| 174 | mmap_flags |= MAP_HUGETLB | MAP_PRIVATE | MAP_ANONYMOUS; |
| 175 | log2_page_size = 21; |
| 176 | } |
| 177 | else |
| 178 | { |
| 179 | mmap_flags |= MAP_PRIVATE | MAP_ANONYMOUS; |
| 180 | log2_page_size = min_log2 (sysconf (_SC_PAGESIZE)); |
| 181 | } |
| 182 | } |
| 183 | |
| 184 | n_pages = ((a->size - 1) >> log2_page_size) + 1; |
| 185 | |
Damjan Marion | 01914ce | 2017-09-14 19:04:50 +0200 | [diff] [blame] | 186 | if (a->flags & CLIB_MEM_VM_F_HUGETLB_PREALLOC) |
| 187 | { |
| 188 | err = clib_sysfs_prealloc_hugepages (a->numa_node, |
| 189 | 1 << (log2_page_size - 10), |
| 190 | n_pages); |
| 191 | if (err) |
| 192 | goto error; |
| 193 | |
| 194 | } |
| 195 | |
| 196 | if (fd != -1) |
Igor Mikhailov (imichail) | 7d25415 | 2018-02-05 12:09:36 -0800 | [diff] [blame] | 197 | if ((ftruncate (fd, n_pages * (1 << log2_page_size))) == -1) |
Damjan Marion | 01914ce | 2017-09-14 19:04:50 +0200 | [diff] [blame] | 198 | { |
| 199 | err = clib_error_return_unix (0, "ftruncate"); |
| 200 | goto error; |
| 201 | } |
| 202 | |
| 203 | if (old_mpol != -1) |
| 204 | { |
| 205 | int rv; |
| 206 | u64 mask[16] = { 0 }; |
| 207 | mask[0] = 1 << a->numa_node; |
| 208 | rv = set_mempolicy (MPOL_BIND, mask, sizeof (mask) * 8 + 1); |
| 209 | if (rv) |
| 210 | { |
| 211 | err = clib_error_return_unix (0, "set_mempolicy"); |
| 212 | goto error; |
| 213 | } |
| 214 | } |
| 215 | |
Florin Coras | d3e83a9 | 2018-01-16 02:40:18 -0800 | [diff] [blame] | 216 | addr = mmap (uword_to_pointer (a->requested_va, void *), a->size, |
| 217 | (PROT_READ | PROT_WRITE), mmap_flags, fd, 0); |
Damjan Marion | 01914ce | 2017-09-14 19:04:50 +0200 | [diff] [blame] | 218 | if (addr == MAP_FAILED) |
| 219 | { |
| 220 | err = clib_error_return_unix (0, "mmap"); |
| 221 | goto error; |
| 222 | } |
| 223 | |
Damjan Marion | 2e172ea | 2018-01-03 15:48:34 +0000 | [diff] [blame] | 224 | /* re-apply old numa memory policy */ |
Damjan Marion | 01914ce | 2017-09-14 19:04:50 +0200 | [diff] [blame] | 225 | if (old_mpol != -1 && |
| 226 | set_mempolicy (old_mpol, old_mask, sizeof (old_mask) * 8 + 1) == -1) |
| 227 | { |
| 228 | err = clib_error_return_unix (0, "set_mempolicy"); |
| 229 | goto error; |
| 230 | } |
| 231 | |
| 232 | a->log2_page_size = log2_page_size; |
| 233 | a->n_pages = n_pages; |
| 234 | a->addr = addr; |
| 235 | a->fd = fd; |
| 236 | goto done; |
| 237 | |
| 238 | error: |
| 239 | if (fd != -1) |
| 240 | close (fd); |
| 241 | |
| 242 | done: |
| 243 | vec_free (filename); |
| 244 | return err; |
| 245 | } |
| 246 | |
| 247 | u64 * |
| 248 | clib_mem_vm_get_paddr (void *mem, int log2_page_size, int n_pages) |
| 249 | { |
| 250 | int pagesize = sysconf (_SC_PAGESIZE); |
| 251 | int fd; |
| 252 | int i; |
| 253 | u64 *r = 0; |
| 254 | |
| 255 | if ((fd = open ((char *) "/proc/self/pagemap", O_RDONLY)) == -1) |
| 256 | return 0; |
| 257 | |
| 258 | for (i = 0; i < n_pages; i++) |
| 259 | { |
| 260 | u64 seek, pagemap = 0; |
| 261 | uword vaddr = pointer_to_uword (mem) + (((u64) i) << log2_page_size); |
| 262 | seek = ((u64) vaddr / pagesize) * sizeof (u64); |
| 263 | if (lseek (fd, seek, SEEK_SET) != seek) |
| 264 | goto done; |
| 265 | |
| 266 | if (read (fd, &pagemap, sizeof (pagemap)) != (sizeof (pagemap))) |
| 267 | goto done; |
| 268 | |
| 269 | if ((pagemap & (1ULL << 63)) == 0) |
| 270 | goto done; |
| 271 | |
| 272 | pagemap &= pow2_mask (55); |
| 273 | vec_add1 (r, pagemap * pagesize); |
| 274 | } |
| 275 | |
| 276 | done: |
| 277 | close (fd); |
| 278 | if (vec_len (r) != n_pages) |
| 279 | { |
| 280 | vec_free (r); |
| 281 | return 0; |
| 282 | } |
| 283 | return r; |
| 284 | } |
| 285 | |
Florin Coras | d3e83a9 | 2018-01-16 02:40:18 -0800 | [diff] [blame] | 286 | clib_error_t * |
| 287 | clib_mem_vm_ext_map (clib_mem_vm_map_t * a) |
| 288 | { |
| 289 | int mmap_flags = MAP_SHARED; |
| 290 | void *addr; |
Damjan Marion | 01914ce | 2017-09-14 19:04:50 +0200 | [diff] [blame] | 291 | |
Florin Coras | d3e83a9 | 2018-01-16 02:40:18 -0800 | [diff] [blame] | 292 | if (a->requested_va) |
| 293 | mmap_flags |= MAP_FIXED; |
| 294 | |
| 295 | addr = (void *) mmap (uword_to_pointer (a->requested_va, void *), a->size, |
| 296 | PROT_READ | PROT_WRITE, mmap_flags, a->fd, 0); |
| 297 | |
| 298 | if (addr == MAP_FAILED) |
| 299 | return clib_error_return_unix (0, "mmap"); |
| 300 | |
| 301 | a->addr = addr; |
| 302 | return 0; |
| 303 | } |
Damjan Marion | 01914ce | 2017-09-14 19:04:50 +0200 | [diff] [blame] | 304 | |
| 305 | /* |
| 306 | * fd.io coding-style-patch-verification: ON |
| 307 | * |
| 308 | * Local Variables: |
| 309 | * eval: (c-set-style "gnu") |
| 310 | * End: |
| 311 | */ |