Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (c) 2015 Cisco and/or its affiliates. |
| 3 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | * you may not use this file except in compliance with the License. |
| 5 | * You may obtain a copy of the License at: |
| 6 | * |
| 7 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | * |
| 9 | * Unless required by applicable law or agreed to in writing, software |
| 10 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | * See the License for the specific language governing permissions and |
| 13 | * limitations under the License. |
| 14 | */ |
| 15 | /* |
| 16 | Copyright (c) 2001, 2002, 2003 Eliot Dresselhaus |
| 17 | |
| 18 | Permission is hereby granted, free of charge, to any person obtaining |
| 19 | a copy of this software and associated documentation files (the |
| 20 | "Software"), to deal in the Software without restriction, including |
| 21 | without limitation the rights to use, copy, modify, merge, publish, |
| 22 | distribute, sublicense, and/or sell copies of the Software, and to |
| 23 | permit persons to whom the Software is furnished to do so, subject to |
| 24 | the following conditions: |
| 25 | |
| 26 | The above copyright notice and this permission notice shall be |
| 27 | included in all copies or substantial portions of the Software. |
| 28 | |
| 29 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, |
| 30 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF |
| 31 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND |
| 32 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE |
| 33 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION |
| 34 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION |
| 35 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. |
| 36 | */ |
| 37 | |
| 38 | #ifndef _included_clib_mem_h |
| 39 | #define _included_clib_mem_h |
| 40 | |
| 41 | #include <stdarg.h> |
Damjan Marion | 01914ce | 2017-09-14 19:04:50 +0200 | [diff] [blame] | 42 | #include <unistd.h> |
| 43 | #include <sys/mman.h> |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 44 | |
Dave Barach | c379999 | 2016-08-15 11:12:27 -0400 | [diff] [blame] | 45 | #include <vppinfra/clib.h> /* uword, etc */ |
Damjan Marion | 01914ce | 2017-09-14 19:04:50 +0200 | [diff] [blame] | 46 | #include <vppinfra/clib_error.h> |
Dave Barach | 6a5adc3 | 2018-07-04 10:56:23 -0400 | [diff] [blame] | 47 | |
| 48 | #if USE_DLMALLOC == 0 |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 49 | #include <vppinfra/mheap_bootstrap.h> |
Dave Barach | 6a5adc3 | 2018-07-04 10:56:23 -0400 | [diff] [blame] | 50 | #else |
| 51 | #include <vppinfra/dlmalloc.h> |
| 52 | #endif |
| 53 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 54 | #include <vppinfra/os.h> |
Dave Barach | b7b9299 | 2018-10-17 10:38:51 -0400 | [diff] [blame] | 55 | #include <vppinfra/string.h> /* memcpy, clib_memset */ |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 56 | #include <vppinfra/valgrind.h> |
| 57 | |
Damjan Marion | ce8debf | 2016-02-06 19:16:21 +0100 | [diff] [blame] | 58 | #define CLIB_MAX_MHEAPS 256 |
| 59 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 60 | /* Per CPU heaps. */ |
Dave Barach | c379999 | 2016-08-15 11:12:27 -0400 | [diff] [blame] | 61 | extern void *clib_per_cpu_mheaps[CLIB_MAX_MHEAPS]; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 62 | |
Dave Barach | c379999 | 2016-08-15 11:12:27 -0400 | [diff] [blame] | 63 | always_inline void * |
| 64 | clib_mem_get_per_cpu_heap (void) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 65 | { |
Damjan Marion | f55f9b8 | 2017-05-10 21:06:28 +0200 | [diff] [blame] | 66 | int cpu = os_get_thread_index (); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 67 | return clib_per_cpu_mheaps[cpu]; |
| 68 | } |
| 69 | |
Dave Barach | c379999 | 2016-08-15 11:12:27 -0400 | [diff] [blame] | 70 | always_inline void * |
| 71 | clib_mem_set_per_cpu_heap (u8 * new_heap) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 72 | { |
Damjan Marion | f55f9b8 | 2017-05-10 21:06:28 +0200 | [diff] [blame] | 73 | int cpu = os_get_thread_index (); |
Dave Barach | c379999 | 2016-08-15 11:12:27 -0400 | [diff] [blame] | 74 | void *old = clib_per_cpu_mheaps[cpu]; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 75 | clib_per_cpu_mheaps[cpu] = new_heap; |
| 76 | return old; |
| 77 | } |
| 78 | |
Dave Barach | 241e522 | 2016-10-13 10:53:26 -0400 | [diff] [blame] | 79 | /* Memory allocator which may call os_out_of_memory() if it fails */ |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 80 | always_inline void * |
Dave Barach | 241e522 | 2016-10-13 10:53:26 -0400 | [diff] [blame] | 81 | clib_mem_alloc_aligned_at_offset (uword size, uword align, uword align_offset, |
| 82 | int os_out_of_memory_on_failure) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 83 | { |
Dave Barach | c379999 | 2016-08-15 11:12:27 -0400 | [diff] [blame] | 84 | void *heap, *p; |
Dave Barach | 6a5adc3 | 2018-07-04 10:56:23 -0400 | [diff] [blame] | 85 | uword cpu; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 86 | |
| 87 | if (align_offset > align) |
| 88 | { |
| 89 | if (align > 0) |
| 90 | align_offset %= align; |
| 91 | else |
| 92 | align_offset = align; |
| 93 | } |
| 94 | |
Damjan Marion | f55f9b8 | 2017-05-10 21:06:28 +0200 | [diff] [blame] | 95 | cpu = os_get_thread_index (); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 96 | heap = clib_per_cpu_mheaps[cpu]; |
Dave Barach | 6a5adc3 | 2018-07-04 10:56:23 -0400 | [diff] [blame] | 97 | |
| 98 | #if USE_DLMALLOC == 0 |
| 99 | uword offset; |
Dave Barach | c379999 | 2016-08-15 11:12:27 -0400 | [diff] [blame] | 100 | heap = mheap_get_aligned (heap, size, align, align_offset, &offset); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 101 | clib_per_cpu_mheaps[cpu] = heap; |
| 102 | |
| 103 | if (offset != ~0) |
| 104 | { |
| 105 | p = heap + offset; |
| 106 | #if CLIB_DEBUG > 0 |
| 107 | VALGRIND_MALLOCLIKE_BLOCK (p, mheap_data_bytes (heap, offset), 0, 0); |
| 108 | #endif |
| 109 | return p; |
| 110 | } |
| 111 | else |
| 112 | { |
Dave Barach | 241e522 | 2016-10-13 10:53:26 -0400 | [diff] [blame] | 113 | if (os_out_of_memory_on_failure) |
| 114 | os_out_of_memory (); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 115 | return 0; |
| 116 | } |
Dave Barach | 6a5adc3 | 2018-07-04 10:56:23 -0400 | [diff] [blame] | 117 | #else |
| 118 | p = mspace_get_aligned (heap, size, align, align_offset); |
| 119 | if (PREDICT_FALSE (p == 0)) |
| 120 | { |
| 121 | if (os_out_of_memory_on_failure) |
| 122 | os_out_of_memory (); |
| 123 | return 0; |
| 124 | } |
| 125 | |
| 126 | return p; |
| 127 | #endif /* USE_DLMALLOC */ |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 128 | } |
| 129 | |
Dave Barach | 241e522 | 2016-10-13 10:53:26 -0400 | [diff] [blame] | 130 | /* Memory allocator which calls os_out_of_memory() when it fails */ |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 131 | always_inline void * |
| 132 | clib_mem_alloc (uword size) |
Dave Barach | c379999 | 2016-08-15 11:12:27 -0400 | [diff] [blame] | 133 | { |
| 134 | return clib_mem_alloc_aligned_at_offset (size, /* align */ 1, |
Dave Barach | 241e522 | 2016-10-13 10:53:26 -0400 | [diff] [blame] | 135 | /* align_offset */ 0, |
| 136 | /* os_out_of_memory */ 1); |
Dave Barach | c379999 | 2016-08-15 11:12:27 -0400 | [diff] [blame] | 137 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 138 | |
| 139 | always_inline void * |
| 140 | clib_mem_alloc_aligned (uword size, uword align) |
Dave Barach | c379999 | 2016-08-15 11:12:27 -0400 | [diff] [blame] | 141 | { |
Dave Barach | 241e522 | 2016-10-13 10:53:26 -0400 | [diff] [blame] | 142 | return clib_mem_alloc_aligned_at_offset (size, align, /* align_offset */ 0, |
| 143 | /* os_out_of_memory */ 1); |
Dave Barach | c379999 | 2016-08-15 11:12:27 -0400 | [diff] [blame] | 144 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 145 | |
Dave Barach | 241e522 | 2016-10-13 10:53:26 -0400 | [diff] [blame] | 146 | /* Memory allocator which calls os_out_of_memory() when it fails */ |
| 147 | always_inline void * |
| 148 | clib_mem_alloc_or_null (uword size) |
| 149 | { |
| 150 | return clib_mem_alloc_aligned_at_offset (size, /* align */ 1, |
| 151 | /* align_offset */ 0, |
| 152 | /* os_out_of_memory */ 0); |
| 153 | } |
| 154 | |
| 155 | always_inline void * |
| 156 | clib_mem_alloc_aligned_or_null (uword size, uword align) |
| 157 | { |
| 158 | return clib_mem_alloc_aligned_at_offset (size, align, /* align_offset */ 0, |
| 159 | /* os_out_of_memory */ 0); |
| 160 | } |
| 161 | |
| 162 | |
| 163 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 164 | /* Memory allocator which panics when it fails. |
| 165 | Use macro so that clib_panic macro can expand __FUNCTION__ and __LINE__. */ |
| 166 | #define clib_mem_alloc_aligned_no_fail(size,align) \ |
| 167 | ({ \ |
| 168 | uword _clib_mem_alloc_size = (size); \ |
| 169 | void * _clib_mem_alloc_p; \ |
| 170 | _clib_mem_alloc_p = clib_mem_alloc_aligned (_clib_mem_alloc_size, (align)); \ |
| 171 | if (! _clib_mem_alloc_p) \ |
| 172 | clib_panic ("failed to allocate %d bytes", _clib_mem_alloc_size); \ |
| 173 | _clib_mem_alloc_p; \ |
| 174 | }) |
| 175 | |
| 176 | #define clib_mem_alloc_no_fail(size) clib_mem_alloc_aligned_no_fail(size,1) |
| 177 | |
| 178 | /* Alias to stack allocator for naming consistency. */ |
| 179 | #define clib_mem_alloc_stack(bytes) __builtin_alloca(bytes) |
| 180 | |
Dave Barach | c379999 | 2016-08-15 11:12:27 -0400 | [diff] [blame] | 181 | always_inline uword |
| 182 | clib_mem_is_heap_object (void *p) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 183 | { |
Dave Barach | 6a5adc3 | 2018-07-04 10:56:23 -0400 | [diff] [blame] | 184 | #if USE_DLMALLOC == 0 |
Dave Barach | c379999 | 2016-08-15 11:12:27 -0400 | [diff] [blame] | 185 | void *heap = clib_mem_get_per_cpu_heap (); |
| 186 | uword offset = (uword) p - (uword) heap; |
| 187 | mheap_elt_t *e, *n; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 188 | |
| 189 | if (offset >= vec_len (heap)) |
| 190 | return 0; |
| 191 | |
| 192 | e = mheap_elt_at_uoffset (heap, offset); |
| 193 | n = mheap_next_elt (e); |
Dave Barach | c379999 | 2016-08-15 11:12:27 -0400 | [diff] [blame] | 194 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 195 | /* Check that heap forward and reverse pointers agree. */ |
| 196 | return e->n_user_data == n->prev_n_user_data; |
Dave Barach | 6a5adc3 | 2018-07-04 10:56:23 -0400 | [diff] [blame] | 197 | #else |
| 198 | void *heap = clib_mem_get_per_cpu_heap (); |
| 199 | |
| 200 | return mspace_is_heap_object (heap, p); |
| 201 | #endif /* USE_DLMALLOC */ |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 202 | } |
| 203 | |
Dave Barach | c379999 | 2016-08-15 11:12:27 -0400 | [diff] [blame] | 204 | always_inline void |
| 205 | clib_mem_free (void *p) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 206 | { |
Dave Barach | c379999 | 2016-08-15 11:12:27 -0400 | [diff] [blame] | 207 | u8 *heap = clib_mem_get_per_cpu_heap (); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 208 | |
| 209 | /* Make sure object is in the correct heap. */ |
| 210 | ASSERT (clib_mem_is_heap_object (p)); |
| 211 | |
Dave Barach | 6a5adc3 | 2018-07-04 10:56:23 -0400 | [diff] [blame] | 212 | #if USE_DLMALLOC == 0 |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 213 | mheap_put (heap, (u8 *) p - heap); |
Dave Barach | 6a5adc3 | 2018-07-04 10:56:23 -0400 | [diff] [blame] | 214 | #else |
| 215 | mspace_put (heap, p); |
| 216 | #endif |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 217 | |
| 218 | #if CLIB_DEBUG > 0 |
| 219 | VALGRIND_FREELIKE_BLOCK (p, 0); |
| 220 | #endif |
| 221 | } |
| 222 | |
Dave Barach | c379999 | 2016-08-15 11:12:27 -0400 | [diff] [blame] | 223 | always_inline void * |
| 224 | clib_mem_realloc (void *p, uword new_size, uword old_size) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 225 | { |
| 226 | /* By default use alloc, copy and free to emulate realloc. */ |
Dave Barach | c379999 | 2016-08-15 11:12:27 -0400 | [diff] [blame] | 227 | void *q = clib_mem_alloc (new_size); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 228 | if (q) |
| 229 | { |
| 230 | uword copy_size; |
| 231 | if (old_size < new_size) |
| 232 | copy_size = old_size; |
| 233 | else |
| 234 | copy_size = new_size; |
Damjan Marion | f1213b8 | 2016-03-13 02:22:06 +0100 | [diff] [blame] | 235 | clib_memcpy (q, p, copy_size); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 236 | clib_mem_free (p); |
| 237 | } |
| 238 | return q; |
| 239 | } |
| 240 | |
Dave Barach | c379999 | 2016-08-15 11:12:27 -0400 | [diff] [blame] | 241 | always_inline uword |
| 242 | clib_mem_size (void *p) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 243 | { |
Dave Barach | 6a5adc3 | 2018-07-04 10:56:23 -0400 | [diff] [blame] | 244 | #if USE_DLMALLOC == 0 |
Dave Barach | c379999 | 2016-08-15 11:12:27 -0400 | [diff] [blame] | 245 | mheap_elt_t *e = mheap_user_pointer_to_elt (p); |
Dave Barach | 6a5adc3 | 2018-07-04 10:56:23 -0400 | [diff] [blame] | 246 | ASSERT (clib_mem_is_heap_object (p)); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 247 | return mheap_elt_data_bytes (e); |
Dave Barach | 6a5adc3 | 2018-07-04 10:56:23 -0400 | [diff] [blame] | 248 | #else |
| 249 | ASSERT (clib_mem_is_heap_object (p)); |
| 250 | return mspace_usable_size_with_delta (p); |
| 251 | #endif |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 252 | } |
| 253 | |
Dave Barach | c379999 | 2016-08-15 11:12:27 -0400 | [diff] [blame] | 254 | always_inline void * |
| 255 | clib_mem_get_heap (void) |
| 256 | { |
| 257 | return clib_mem_get_per_cpu_heap (); |
| 258 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 259 | |
Dave Barach | c379999 | 2016-08-15 11:12:27 -0400 | [diff] [blame] | 260 | always_inline void * |
| 261 | clib_mem_set_heap (void *heap) |
| 262 | { |
| 263 | return clib_mem_set_per_cpu_heap (heap); |
| 264 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 265 | |
Dave Barach | c379999 | 2016-08-15 11:12:27 -0400 | [diff] [blame] | 266 | void *clib_mem_init (void *heap, uword size); |
Dave Barach | 6a5adc3 | 2018-07-04 10:56:23 -0400 | [diff] [blame] | 267 | void *clib_mem_init_thread_safe (void *memory, uword memory_size); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 268 | |
| 269 | void clib_mem_exit (void); |
| 270 | |
| 271 | uword clib_mem_get_page_size (void); |
| 272 | |
| 273 | void clib_mem_validate (void); |
| 274 | |
| 275 | void clib_mem_trace (int enable); |
| 276 | |
Dave Barach | c379999 | 2016-08-15 11:12:27 -0400 | [diff] [blame] | 277 | typedef struct |
| 278 | { |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 279 | /* Total number of objects allocated. */ |
| 280 | uword object_count; |
| 281 | |
| 282 | /* Total allocated bytes. Bytes used and free. |
| 283 | used + free = total */ |
| 284 | uword bytes_total, bytes_used, bytes_free; |
| 285 | |
| 286 | /* Number of bytes used by mheap data structure overhead |
| 287 | (e.g. free lists, mheap header). */ |
| 288 | uword bytes_overhead; |
| 289 | |
| 290 | /* Amount of free space returned to operating system. */ |
| 291 | uword bytes_free_reclaimed; |
Dave Barach | c379999 | 2016-08-15 11:12:27 -0400 | [diff] [blame] | 292 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 293 | /* For malloc which puts small objects in sbrk region and |
| 294 | large objects in mmap'ed regions. */ |
| 295 | uword bytes_used_sbrk; |
| 296 | uword bytes_used_mmap; |
| 297 | |
| 298 | /* Max. number of bytes in this heap. */ |
| 299 | uword bytes_max; |
| 300 | } clib_mem_usage_t; |
| 301 | |
| 302 | void clib_mem_usage (clib_mem_usage_t * usage); |
| 303 | |
Dave Barach | c379999 | 2016-08-15 11:12:27 -0400 | [diff] [blame] | 304 | u8 *format_clib_mem_usage (u8 * s, va_list * args); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 305 | |
Damjan Marion | 01914ce | 2017-09-14 19:04:50 +0200 | [diff] [blame] | 306 | /* Allocate virtual address space. */ |
| 307 | always_inline void * |
| 308 | clib_mem_vm_alloc (uword size) |
| 309 | { |
| 310 | void *mmap_addr; |
| 311 | uword flags = MAP_PRIVATE; |
| 312 | |
| 313 | #ifdef MAP_ANONYMOUS |
| 314 | flags |= MAP_ANONYMOUS; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 315 | #endif |
| 316 | |
Damjan Marion | 01914ce | 2017-09-14 19:04:50 +0200 | [diff] [blame] | 317 | mmap_addr = mmap (0, size, PROT_READ | PROT_WRITE, flags, -1, 0); |
| 318 | if (mmap_addr == (void *) -1) |
| 319 | mmap_addr = 0; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 320 | |
Damjan Marion | 01914ce | 2017-09-14 19:04:50 +0200 | [diff] [blame] | 321 | return mmap_addr; |
| 322 | } |
| 323 | |
| 324 | always_inline void |
| 325 | clib_mem_vm_free (void *addr, uword size) |
| 326 | { |
| 327 | munmap (addr, size); |
| 328 | } |
| 329 | |
| 330 | always_inline void * |
| 331 | clib_mem_vm_unmap (void *addr, uword size) |
| 332 | { |
| 333 | void *mmap_addr; |
| 334 | uword flags = MAP_PRIVATE | MAP_FIXED; |
| 335 | |
| 336 | /* To unmap we "map" with no protection. If we actually called |
| 337 | munmap then other callers could steal the address space. By |
| 338 | changing to PROT_NONE the kernel can free up the pages which is |
| 339 | really what we want "unmap" to mean. */ |
| 340 | mmap_addr = mmap (addr, size, PROT_NONE, flags, -1, 0); |
| 341 | if (mmap_addr == (void *) -1) |
| 342 | mmap_addr = 0; |
| 343 | |
| 344 | return mmap_addr; |
| 345 | } |
| 346 | |
| 347 | always_inline void * |
| 348 | clib_mem_vm_map (void *addr, uword size) |
| 349 | { |
| 350 | void *mmap_addr; |
Dave Barach | e89be4e | 2018-08-29 08:50:40 -0400 | [diff] [blame] | 351 | uword flags = MAP_PRIVATE | MAP_FIXED | MAP_ANONYMOUS; |
Damjan Marion | 01914ce | 2017-09-14 19:04:50 +0200 | [diff] [blame] | 352 | |
| 353 | mmap_addr = mmap (addr, size, (PROT_READ | PROT_WRITE), flags, -1, 0); |
| 354 | if (mmap_addr == (void *) -1) |
| 355 | mmap_addr = 0; |
| 356 | |
| 357 | return mmap_addr; |
| 358 | } |
| 359 | |
| 360 | typedef struct |
| 361 | { |
| 362 | #define CLIB_MEM_VM_F_SHARED (1 << 0) |
| 363 | #define CLIB_MEM_VM_F_HUGETLB (1 << 1) |
| 364 | #define CLIB_MEM_VM_F_NUMA_PREFER (1 << 2) |
| 365 | #define CLIB_MEM_VM_F_NUMA_FORCE (1 << 3) |
| 366 | #define CLIB_MEM_VM_F_HUGETLB_PREALLOC (1 << 4) |
Damjan Marion | 7b18536 | 2018-03-04 16:41:35 +0100 | [diff] [blame] | 367 | #define CLIB_MEM_VM_F_LOCKED (1 << 5) |
Damjan Marion | 01914ce | 2017-09-14 19:04:50 +0200 | [diff] [blame] | 368 | u32 flags; /**< vm allocation flags: |
| 369 | <br> CLIB_MEM_VM_F_SHARED: request shared memory, file |
Florin Coras | d3e83a9 | 2018-01-16 02:40:18 -0800 | [diff] [blame] | 370 | descriptor will be provided on successful allocation. |
Damjan Marion | 01914ce | 2017-09-14 19:04:50 +0200 | [diff] [blame] | 371 | <br> CLIB_MEM_VM_F_HUGETLB: request hugepages. |
| 372 | <br> CLIB_MEM_VM_F_NUMA_PREFER: numa_node field contains valid |
| 373 | numa node preference. |
| 374 | <br> CLIB_MEM_VM_F_NUMA_FORCE: fail if setting numa policy fails. |
| 375 | <br> CLIB_MEM_VM_F_HUGETLB_PREALLOC: pre-allocate hugepages if |
| 376 | number of available pages is not sufficient. |
Damjan Marion | 7b18536 | 2018-03-04 16:41:35 +0100 | [diff] [blame] | 377 | <br> CLIB_MEM_VM_F_LOCKED: request locked memory. |
Damjan Marion | 01914ce | 2017-09-14 19:04:50 +0200 | [diff] [blame] | 378 | */ |
| 379 | char *name; /**< Name for memory allocation, set by caller. */ |
| 380 | uword size; /**< Allocation size, set by caller. */ |
| 381 | int numa_node; /**< numa node preference. Valid if CLIB_MEM_VM_F_NUMA_PREFER set. */ |
| 382 | void *addr; /**< Pointer to allocated memory, set on successful allocation. */ |
Florin Coras | d3e83a9 | 2018-01-16 02:40:18 -0800 | [diff] [blame] | 383 | int fd; /**< File descriptor, set on successful allocation if CLIB_MEM_VM_F_SHARED is set. */ |
Damjan Marion | 01914ce | 2017-09-14 19:04:50 +0200 | [diff] [blame] | 384 | int log2_page_size; /* Page size in log2 format, set on successful allocation. */ |
| 385 | int n_pages; /* Number of pages. */ |
Florin Coras | d3e83a9 | 2018-01-16 02:40:18 -0800 | [diff] [blame] | 386 | uword requested_va; /**< Request fixed position mapping */ |
Damjan Marion | 01914ce | 2017-09-14 19:04:50 +0200 | [diff] [blame] | 387 | } clib_mem_vm_alloc_t; |
| 388 | |
Damjan Marion | 567e61d | 2018-10-24 17:08:26 +0200 | [diff] [blame^] | 389 | clib_error_t *clib_mem_create_fd (char *name, int *fdp); |
Damjan Marion | 1636b16 | 2018-10-19 12:54:42 +0200 | [diff] [blame] | 390 | clib_error_t *clib_mem_create_hugetlb_fd (char *name, int *fdp); |
Damjan Marion | 01914ce | 2017-09-14 19:04:50 +0200 | [diff] [blame] | 391 | clib_error_t *clib_mem_vm_ext_alloc (clib_mem_vm_alloc_t * a); |
Haiyang Tan | 642829d | 2018-10-09 19:09:45 -0700 | [diff] [blame] | 392 | void clib_mem_vm_ext_free (clib_mem_vm_alloc_t * a); |
Damjan Marion | 567e61d | 2018-10-24 17:08:26 +0200 | [diff] [blame^] | 393 | u64 clib_mem_get_fd_page_size (int fd); |
Damjan Marion | 9787f5f | 2018-10-24 12:56:32 +0200 | [diff] [blame] | 394 | uword clib_mem_get_default_hugepage_size (void); |
Damjan Marion | 567e61d | 2018-10-24 17:08:26 +0200 | [diff] [blame^] | 395 | int clib_mem_get_fd_log2_page_size (int fd); |
Damjan Marion | 01914ce | 2017-09-14 19:04:50 +0200 | [diff] [blame] | 396 | u64 *clib_mem_vm_get_paddr (void *mem, int log2_page_size, int n_pages); |
| 397 | |
Florin Coras | d3e83a9 | 2018-01-16 02:40:18 -0800 | [diff] [blame] | 398 | typedef struct |
| 399 | { |
| 400 | uword size; /**< Map size */ |
| 401 | int fd; /**< File descriptor to be mapped */ |
| 402 | uword requested_va; /**< Request fixed position mapping */ |
| 403 | void *addr; /**< Pointer to mapped memory, if successful */ |
| 404 | } clib_mem_vm_map_t; |
Florin Coras | d3e83a9 | 2018-01-16 02:40:18 -0800 | [diff] [blame] | 405 | |
Florin Coras | b384b54 | 2018-01-15 01:08:33 -0800 | [diff] [blame] | 406 | clib_error_t *clib_mem_vm_ext_map (clib_mem_vm_map_t * a); |
| 407 | void clib_mem_vm_randomize_va (uword * requested_va, u32 log2_page_size); |
Dave Barach | 6a5adc3 | 2018-07-04 10:56:23 -0400 | [diff] [blame] | 408 | void mheap_trace (void *v, int enable); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 409 | |
Dave Barach | c379999 | 2016-08-15 11:12:27 -0400 | [diff] [blame] | 410 | #include <vppinfra/error.h> /* clib_panic */ |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 411 | |
| 412 | #endif /* _included_clib_mem_h */ |
Dave Barach | c379999 | 2016-08-15 11:12:27 -0400 | [diff] [blame] | 413 | |
| 414 | /* |
| 415 | * fd.io coding-style-patch-verification: ON |
| 416 | * |
| 417 | * Local Variables: |
| 418 | * eval: (c-set-style "gnu") |
| 419 | * End: |
| 420 | */ |