blob: 5580808909a4f3c3085967fd946f3688a2b351d8 [file] [log] [blame]
Dave Barach8a7fb0c2016-07-08 14:44:23 -04001/*
Ed Warnickecb9cada2015-12-08 15:45:58 -07002 *------------------------------------------------------------------
Dave Barach8a7fb0c2016-07-08 14:44:23 -04003 * svm.c - shared VM allocation, mmap(...MAP_FIXED...)
Ed Warnickecb9cada2015-12-08 15:45:58 -07004 * library
5 *
6 * Copyright (c) 2009 Cisco and/or its affiliates.
7 * Licensed under the Apache License, Version 2.0 (the "License");
8 * you may not use this file except in compliance with the License.
9 * You may obtain a copy of the License at:
10 *
11 * http://www.apache.org/licenses/LICENSE-2.0
12 *
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
18 *------------------------------------------------------------------
19 */
20
21#include <stdio.h>
22#include <stdlib.h>
23#include <sys/types.h>
24#include <sys/mman.h>
25#include <sys/stat.h>
26#include <netinet/in.h>
27#include <signal.h>
28#include <pthread.h>
29#include <unistd.h>
30#include <time.h>
31#include <fcntl.h>
32#include <string.h>
33#include <vppinfra/clib.h>
34#include <vppinfra/vec.h>
35#include <vppinfra/hash.h>
36#include <vppinfra/bitmap.h>
37#include <vppinfra/fifo.h>
38#include <vppinfra/time.h>
Ed Warnickecb9cada2015-12-08 15:45:58 -070039#include <vppinfra/heap.h>
40#include <vppinfra/pool.h>
41#include <vppinfra/format.h>
42
43#include "svm.h"
44
45static svm_region_t *root_rp;
46static int root_rp_refcount;
47
48#define MAXLOCK 2
Dave Barach8a7fb0c2016-07-08 14:44:23 -040049static pthread_mutex_t *mutexes_held[MAXLOCK];
Ed Warnickecb9cada2015-12-08 15:45:58 -070050static int nheld;
51
Dave Barach8a7fb0c2016-07-08 14:44:23 -040052svm_region_t *
53svm_get_root_rp (void)
Ed Warnickecb9cada2015-12-08 15:45:58 -070054{
Dave Barach8a7fb0c2016-07-08 14:44:23 -040055 return root_rp;
Ed Warnickecb9cada2015-12-08 15:45:58 -070056}
57
58#define MUTEX_DEBUG
59
Damjan Marionaec8f892018-01-08 16:35:35 +010060u64
61svm_get_global_region_base_va ()
62{
63#if __aarch64__
64 /* On AArch64 VA space can have different size, from 36 to 48 bits.
65 Here we are trying to detect VA bits by parsing /proc/self/maps
66 address ranges */
67 int fd;
68 unformat_input_t input;
69 u64 start, end = 0;
70 u8 bits = 0;
71
72 if ((fd = open ("/proc/self/maps", 0)) < 0)
73 clib_unix_error ("open '/proc/self/maps'");
74
75 unformat_init_clib_file (&input, fd);
76 while (unformat_check_input (&input) != UNFORMAT_END_OF_INPUT)
77 {
Gabriel Gannec5239ad2018-01-11 15:04:19 +010078 if (unformat (&input, "%llx-%llx", &start, &end))
79 end--;
Damjan Marionaec8f892018-01-08 16:35:35 +010080 unformat_skip_line (&input);
81 }
Gabriel Ganne83d47432018-01-10 11:40:50 +010082 unformat_free (&input);
83 close (fd);
Damjan Marionaec8f892018-01-08 16:35:35 +010084
Damjan Marion11056002018-05-10 13:40:44 +020085 bits = count_leading_zeros (end);
Gabriel Gannec5239ad2018-01-11 15:04:19 +010086 bits = 64 - bits;
Damjan Marionaec8f892018-01-08 16:35:35 +010087 if (bits >= 36 && bits <= 48)
88 return ((1ul << bits) / 4) - (2 * SVM_GLOBAL_REGION_SIZE);
89 else
90 clib_unix_error ("unexpected va bits '%u'", bits);
Damjan Marionaec8f892018-01-08 16:35:35 +010091#endif
92
Tianyu Li1ef38742021-06-17 07:08:32 +000093#ifdef CLIB_SANITIZE_ADDR
94 return 0x200000000000;
95#endif
Damjan Marionaec8f892018-01-08 16:35:35 +010096 /* default value */
Dave Barach9466c452018-08-24 17:21:14 -040097 return 0x130000000ULL;
Damjan Marionaec8f892018-01-08 16:35:35 +010098}
99
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400100static void
101region_lock (svm_region_t * rp, int tag)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700102{
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400103 pthread_mutex_lock (&rp->mutex);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700104#ifdef MUTEX_DEBUG
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400105 rp->mutex_owner_pid = getpid ();
106 rp->mutex_owner_tag = tag;
107#endif
Dave Barachc35f3e82020-04-02 10:44:09 -0400108 ASSERT (nheld < MAXLOCK); //NOSONAR
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400109 /*
110 * Keep score of held mutexes so we can try to exit
111 * cleanly if the world comes to an end at the worst possible
112 * moment
113 */
114 mutexes_held[nheld++] = &rp->mutex;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700115}
116
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400117static void
118region_unlock (svm_region_t * rp)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700119{
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400120 int i, j;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700121#ifdef MUTEX_DEBUG
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400122 rp->mutex_owner_pid = 0;
123 rp->mutex_owner_tag = 0;
124#endif
Ed Warnickecb9cada2015-12-08 15:45:58 -0700125
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400126 for (i = nheld - 1; i >= 0; i--)
127 {
128 if (mutexes_held[i] == &rp->mutex)
129 {
130 for (j = i; j < MAXLOCK - 1; j++)
131 mutexes_held[j] = mutexes_held[j + 1];
132 nheld--;
133 goto found;
134 }
Ed Warnickecb9cada2015-12-08 15:45:58 -0700135 }
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400136 ASSERT (0);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700137
138found:
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400139 CLIB_MEMORY_BARRIER ();
140 pthread_mutex_unlock (&rp->mutex);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700141}
142
143
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400144static u8 *
145format_svm_flags (u8 * s, va_list * args)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700146{
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400147 uword f = va_arg (*args, uword);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700148
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400149 if (f & SVM_FLAGS_MHEAP)
150 s = format (s, "MHEAP ");
151 if (f & SVM_FLAGS_FILE)
152 s = format (s, "FILE ");
153 if (f & SVM_FLAGS_NODATA)
154 s = format (s, "NODATA ");
155 if (f & SVM_FLAGS_NEED_DATA_INIT)
156 s = format (s, "INIT ");
157
158 return (s);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700159}
160
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400161static u8 *
162format_svm_size (u8 * s, va_list * args)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700163{
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400164 uword size = va_arg (*args, uword);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700165
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400166 if (size >= (1 << 20))
167 {
168 s = format (s, "(%d mb)", size >> 20);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700169 }
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400170 else if (size >= (1 << 10))
171 {
172 s = format (s, "(%d kb)", size >> 10);
173 }
174 else
175 {
176 s = format (s, "(%d bytes)", size);
177 }
178 return (s);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700179}
180
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400181u8 *
182format_svm_region (u8 * s, va_list * args)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700183{
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400184 svm_region_t *rp = va_arg (*args, svm_region_t *);
185 int verbose = va_arg (*args, int);
186 int i;
187 uword lo, hi;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700188
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400189 s = format (s, "%s: base va 0x%x size 0x%x %U\n",
190 rp->region_name, rp->virtual_base,
191 rp->virtual_size, format_svm_size, rp->virtual_size);
192 s = format (s, " user_ctx 0x%x, bitmap_size %d\n",
193 rp->user_ctx, rp->bitmap_size);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700194
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400195 if (verbose)
196 {
197 s = format (s, " flags: 0x%x %U\n", rp->flags,
198 format_svm_flags, rp->flags);
199 s = format (s,
200 " region_heap 0x%x data_base 0x%x data_heap 0x%x\n",
201 rp->region_heap, rp->data_base, rp->data_heap);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700202 }
203
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400204 s = format (s, " %d clients, pids: ", vec_len (rp->client_pids));
Ed Warnickecb9cada2015-12-08 15:45:58 -0700205
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400206 for (i = 0; i < vec_len (rp->client_pids); i++)
207 s = format (s, "%d ", rp->client_pids[i]);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700208
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400209 s = format (s, "\n");
Ed Warnickecb9cada2015-12-08 15:45:58 -0700210
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400211 if (verbose)
212 {
213 lo = hi = ~0;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700214
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400215 s = format (s, " VM in use: ");
216
217 for (i = 0; i < rp->bitmap_size; i++)
218 {
219 if (clib_bitmap_get_no_check (rp->bitmap, i) != 0)
220 {
221 if (lo == ~0)
222 {
223 hi = lo = rp->virtual_base + i * MMAP_PAGESIZE;
224 }
225 else
226 {
227 hi = rp->virtual_base + i * MMAP_PAGESIZE;
228 }
229 }
230 else
231 {
232 if (lo != ~0)
233 {
234 hi = rp->virtual_base + i * MMAP_PAGESIZE - 1;
235 s = format (s, " 0x%x - 0x%x (%dk)\n", lo, hi,
236 (hi - lo) >> 10);
237 lo = hi = ~0;
238 }
239 }
240 }
Ed Warnickecb9cada2015-12-08 15:45:58 -0700241 }
242
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400243 return (s);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700244}
245
246/*
247 * rnd_pagesize
248 * Round to a pagesize multiple, presumably 4k works
249 */
Dave Barachb3d93da2016-08-03 14:34:38 -0400250static u64
251rnd_pagesize (u64 size)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700252{
Dave Barachb3d93da2016-08-03 14:34:38 -0400253 u64 rv;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700254
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400255 rv = (size + (MMAP_PAGESIZE - 1)) & ~(MMAP_PAGESIZE - 1);
256 return (rv);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700257}
258
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400259/*
Ed Warnickecb9cada2015-12-08 15:45:58 -0700260 * svm_data_region_setup
261 */
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400262static int
263svm_data_region_create (svm_map_region_args_t * a, svm_region_t * rp)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700264{
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400265 int fd;
266 u8 junk = 0;
267 uword map_size;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700268
Dave Barachc3799992016-08-15 11:12:27 -0400269 map_size = rp->virtual_size - (MMAP_PAGESIZE +
270 (a->pvt_heap_size ? a->pvt_heap_size :
271 SVM_PVT_MHEAP_SIZE));
Ed Warnickecb9cada2015-12-08 15:45:58 -0700272
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400273 if (a->flags & SVM_FLAGS_FILE)
274 {
275 struct stat statb;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700276
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400277 fd = open (a->backing_file, O_RDWR | O_CREAT, 0777);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700278
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400279 if (fd < 0)
280 {
281 clib_unix_warning ("open");
282 return -1;
283 }
Ed Warnickecb9cada2015-12-08 15:45:58 -0700284
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400285 if (fstat (fd, &statb) < 0)
286 {
287 clib_unix_warning ("fstat");
288 close (fd);
289 return -2;
290 }
291
292 if (statb.st_mode & S_IFREG)
293 {
294 if (statb.st_size == 0)
295 {
296 if (lseek (fd, map_size, SEEK_SET) == (off_t) - 1)
297 {
298 clib_unix_warning ("seek region size");
299 close (fd);
300 return -3;
301 }
302 if (write (fd, &junk, 1) != 1)
303 {
304 clib_unix_warning ("set region size");
305 close (fd);
306 return -3;
307 }
308 }
309 else
310 {
311 map_size = rnd_pagesize (statb.st_size);
312 }
313 }
314 else
315 {
316 map_size = a->backing_mmap_size;
317 }
318
319 ASSERT (map_size <= rp->virtual_size -
320 (MMAP_PAGESIZE + SVM_PVT_MHEAP_SIZE));
321
322 if (mmap (rp->data_base, map_size, PROT_READ | PROT_WRITE,
323 MAP_SHARED | MAP_FIXED, fd, 0) == MAP_FAILED)
324 {
325 clib_unix_warning ("mmap");
326 close (fd);
327 return -3;
328 }
329 close (fd);
Damjan Marion79934e82022-04-05 12:40:31 +0200330 clib_mem_unpoison (rp->data_base, map_size);
Benoît Ganneda5b4ef2020-09-09 10:00:34 +0200331 rp->backing_file = (char *) format (0, "%s%c", a->backing_file, 0);
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400332 rp->flags |= SVM_FLAGS_FILE;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700333 }
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400334
335 if (a->flags & SVM_FLAGS_MHEAP)
336 {
Damjan Marion4537c302020-09-28 19:03:37 +0200337 rp->data_heap = clib_mem_create_heap (rp->data_base, map_size,
338 1 /* locked */ , "svm data");
Ole Troan73710c72018-06-04 22:27:49 +0200339
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400340 rp->flags |= SVM_FLAGS_MHEAP;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700341 }
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400342 return 0;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700343}
344
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400345static int
346svm_data_region_map (svm_map_region_args_t * a, svm_region_t * rp)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700347{
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400348 int fd;
349 u8 junk = 0;
350 uword map_size;
351 struct stat statb;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700352
Dave Barachc3799992016-08-15 11:12:27 -0400353 map_size = rp->virtual_size -
354 (MMAP_PAGESIZE
Dave Barachb3d93da2016-08-03 14:34:38 -0400355 + (a->pvt_heap_size ? a->pvt_heap_size : SVM_PVT_MHEAP_SIZE));
Ed Warnickecb9cada2015-12-08 15:45:58 -0700356
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400357 if (a->flags & SVM_FLAGS_FILE)
358 {
Ed Warnickecb9cada2015-12-08 15:45:58 -0700359
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400360 fd = open (a->backing_file, O_RDWR, 0777);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700361
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400362 if (fd < 0)
363 {
364 clib_unix_warning ("open");
365 return -1;
366 }
Ed Warnickecb9cada2015-12-08 15:45:58 -0700367
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400368 if (fstat (fd, &statb) < 0)
369 {
370 clib_unix_warning ("fstat");
371 close (fd);
372 return -2;
373 }
374
375 if (statb.st_mode & S_IFREG)
376 {
377 if (statb.st_size == 0)
378 {
379 if (lseek (fd, map_size, SEEK_SET) == (off_t) - 1)
380 {
381 clib_unix_warning ("seek region size");
382 close (fd);
383 return -3;
384 }
385 if (write (fd, &junk, 1) != 1)
386 {
387 clib_unix_warning ("set region size");
388 close (fd);
389 return -3;
390 }
391 }
392 else
393 {
394 map_size = rnd_pagesize (statb.st_size);
395 }
396 }
397 else
398 {
399 map_size = a->backing_mmap_size;
400 }
401
402 ASSERT (map_size <= rp->virtual_size
Dave Barachc3799992016-08-15 11:12:27 -0400403 - (MMAP_PAGESIZE
404 +
405 (a->pvt_heap_size ? a->pvt_heap_size : SVM_PVT_MHEAP_SIZE)));
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400406
407 if (mmap (rp->data_base, map_size, PROT_READ | PROT_WRITE,
408 MAP_SHARED | MAP_FIXED, fd, 0) == MAP_FAILED)
409 {
410 clib_unix_warning ("mmap");
411 close (fd);
412 return -3;
413 }
414 close (fd);
Damjan Marion79934e82022-04-05 12:40:31 +0200415 clib_mem_unpoison (rp->data_base, map_size);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700416 }
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400417 return 0;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700418}
419
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400420u8 *
421shm_name_from_svm_map_region_args (svm_map_region_args_t * a)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700422{
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400423 u8 *shm_name;
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400424 int root_path_offset = 0;
425 int name_offset = 0;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700426
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400427 if (a->root_path)
428 {
429 /* Tolerate present or absent slashes */
430 if (a->root_path[0] == '/')
431 root_path_offset++;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700432
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400433 if (a->name[0] == '/')
434 name_offset = 1;
435
Matej Perinad135c192017-07-18 13:59:41 +0200436 shm_name = format (0, "/%s-%s%c", &a->root_path[root_path_offset],
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400437 &a->name[name_offset], 0);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700438 }
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400439 else
440 shm_name = format (0, "%s%c", a->name, 0);
441 return (shm_name);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700442}
443
Dave Barach59b25652017-09-10 15:04:27 -0400444void
445svm_region_init_mapped_region (svm_map_region_args_t * a, svm_region_t * rp)
446{
447 pthread_mutexattr_t attr;
448 pthread_condattr_t cattr;
449 int nbits, words, bit;
450 int overhead_space;
451 void *oldheap;
452 uword data_base;
453 ASSERT (rp);
454 int rv;
455
Dave Barachb7b92992018-10-17 10:38:51 -0400456 clib_memset (rp, 0, sizeof (*rp));
Dave Barach59b25652017-09-10 15:04:27 -0400457
458 if (pthread_mutexattr_init (&attr))
459 clib_unix_warning ("mutexattr_init");
460
461 if (pthread_mutexattr_setpshared (&attr, PTHREAD_PROCESS_SHARED))
462 clib_unix_warning ("mutexattr_setpshared");
463
464 if (pthread_mutex_init (&rp->mutex, &attr))
465 clib_unix_warning ("mutex_init");
466
467 if (pthread_mutexattr_destroy (&attr))
468 clib_unix_warning ("mutexattr_destroy");
469
470 if (pthread_condattr_init (&cattr))
471 clib_unix_warning ("condattr_init");
472
473 if (pthread_condattr_setpshared (&cattr, PTHREAD_PROCESS_SHARED))
474 clib_unix_warning ("condattr_setpshared");
475
476 if (pthread_cond_init (&rp->condvar, &cattr))
477 clib_unix_warning ("cond_init");
478
479 if (pthread_condattr_destroy (&cattr))
480 clib_unix_warning ("condattr_destroy");
481
482 region_lock (rp, 1);
483
484 rp->virtual_base = a->baseva;
485 rp->virtual_size = a->size;
486
Damjan Marion4537c302020-09-28 19:03:37 +0200487 rp->region_heap = clib_mem_create_heap
Dave Barach6a5adc32018-07-04 10:56:23 -0400488 (uword_to_pointer (a->baseva + MMAP_PAGESIZE, void *),
489 (a->pvt_heap_size !=
Damjan Marion4537c302020-09-28 19:03:37 +0200490 0) ? a->pvt_heap_size : SVM_PVT_MHEAP_SIZE, 1 /* locked */ ,
491 "svm region");
Dave Barach6a5adc32018-07-04 10:56:23 -0400492
Dave Barach59b25652017-09-10 15:04:27 -0400493 oldheap = svm_push_pvt_heap (rp);
494
495 rp->region_name = (char *) format (0, "%s%c", a->name, 0);
496 vec_add1 (rp->client_pids, getpid ());
497
498 nbits = rp->virtual_size / MMAP_PAGESIZE;
499
500 ASSERT (nbits > 0);
501 rp->bitmap_size = nbits;
502 words = (nbits + BITS (uword) - 1) / BITS (uword);
503 vec_validate (rp->bitmap, words - 1);
504
505 overhead_space = MMAP_PAGESIZE /* header */ +
506 ((a->pvt_heap_size != 0) ? a->pvt_heap_size : SVM_PVT_MHEAP_SIZE);
507
508 bit = 0;
509 data_base = (uword) rp->virtual_base;
510
511 if (a->flags & SVM_FLAGS_NODATA)
512 rp->flags |= SVM_FLAGS_NEED_DATA_INIT;
513
514 do
515 {
516 clib_bitmap_set_no_check (rp->bitmap, bit, 1);
517 bit++;
518 overhead_space -= MMAP_PAGESIZE;
519 data_base += MMAP_PAGESIZE;
520 }
521 while (overhead_space > 0);
522
523 rp->data_base = (void *) data_base;
524
525 /*
526 * Note: although the POSIX spec guarantees that only one
527 * process enters this block, we have to play games
528 * to hold off clients until e.g. the mutex is ready
529 */
530 rp->version = SVM_VERSION;
531
532 /* setup the data portion of the region */
533
534 rv = svm_data_region_create (a, rp);
535 if (rv)
536 {
537 clib_warning ("data_region_create: %d", rv);
538 }
539
540 region_unlock (rp);
541
542 svm_pop_heap (oldheap);
543}
544
Ed Warnickecb9cada2015-12-08 15:45:58 -0700545/*
546 * svm_map_region
547 */
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400548void *
549svm_map_region (svm_map_region_args_t * a)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700550{
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400551 int svm_fd;
552 svm_region_t *rp;
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400553 int deadman = 0;
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400554 void *oldheap;
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400555 int rv;
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400556 int pid_holding_region_lock;
557 u8 *shm_name;
558 int dead_region_recovery = 0;
559 int time_left;
560 struct stat stat;
561 struct timespec ts, tsrem;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700562
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400563 ASSERT ((a->size & ~(MMAP_PAGESIZE - 1)) == a->size);
564 ASSERT (a->name);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700565
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400566 shm_name = shm_name_from_svm_map_region_args (a);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700567
Florin Coras9f4ac582019-12-17 19:46:45 -0800568 if (CLIB_DEBUG > 1)
Dave Wallaced756b352017-07-03 13:11:38 -0400569 clib_warning ("[%d] map region %s: shm_open (%s)",
570 getpid (), a->name, shm_name);
571
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400572 svm_fd = shm_open ((char *) shm_name, O_RDWR | O_CREAT | O_EXCL, 0777);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700573
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400574 if (svm_fd >= 0)
575 {
Dave Wallace19296112017-08-31 15:54:11 -0400576 if (fchmod (svm_fd, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP) < 0)
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400577 clib_unix_warning ("segment chmod");
578 /* This turns out to fail harmlessly if the client starts first */
579 if (fchown (svm_fd, a->uid, a->gid) < 0)
580 clib_unix_warning ("segment chown [ok if client starts first]");
Ed Warnickecb9cada2015-12-08 15:45:58 -0700581
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400582 vec_free (shm_name);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700583
Tom Joneseceef162024-01-31 10:44:14 +0000584#ifdef __FreeBSD__
585 if (ftruncate (svm_fd, a->size) < 0)
586 {
587 clib_warning ("ftruncate region size");
588 close (svm_fd);
589 return (0);
590 }
591#else
592 u8 junk = 0;
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400593 if (lseek (svm_fd, a->size, SEEK_SET) == (off_t) - 1)
594 {
595 clib_warning ("seek region size");
596 close (svm_fd);
597 return (0);
598 }
599 if (write (svm_fd, &junk, 1) != 1)
600 {
601 clib_warning ("set region size");
602 close (svm_fd);
603 return (0);
604 }
Tom Joneseceef162024-01-31 10:44:14 +0000605#endif /* __FreeBSD__ */
Ed Warnickecb9cada2015-12-08 15:45:58 -0700606
Damjan Marion7bee80c2017-04-26 15:32:12 +0200607 rp = mmap (uword_to_pointer (a->baseva, void *), a->size,
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400608 PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, svm_fd, 0);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700609
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400610 if (rp == (svm_region_t *) MAP_FAILED)
611 {
612 clib_unix_warning ("mmap create");
613 close (svm_fd);
614 return (0);
615 }
616 close (svm_fd);
Damjan Marion79934e82022-04-05 12:40:31 +0200617 clib_mem_unpoison (rp, a->size);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700618
Dave Barach59b25652017-09-10 15:04:27 -0400619 svm_region_init_mapped_region (a, rp);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700620
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400621 return ((void *) rp);
622 }
623 else
624 {
625 svm_fd = shm_open ((char *) shm_name, O_RDWR, 0777);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700626
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400627 vec_free (shm_name);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700628
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400629 if (svm_fd < 0)
630 {
631 perror ("svm_region_map(mmap open)");
632 return (0);
633 }
Ed Warnickecb9cada2015-12-08 15:45:58 -0700634
Ole Troanc4f2ef72018-05-30 22:43:25 +0200635 /* Reset ownership in case the client started first */
636 if (fchown (svm_fd, a->uid, a->gid) < 0)
637 clib_unix_warning ("segment chown [ok if client starts first]");
638
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400639 time_left = 20;
640 while (1)
641 {
642 if (0 != fstat (svm_fd, &stat))
643 {
644 clib_warning ("fstat failed: %d", errno);
645 close (svm_fd);
646 return (0);
647 }
648 if (stat.st_size > 0)
649 {
650 break;
651 }
652 if (0 == time_left)
653 {
654 clib_warning ("waiting for resize of shm file timed out");
655 close (svm_fd);
656 return (0);
657 }
658 ts.tv_sec = 0;
659 ts.tv_nsec = 100000000;
660 while (nanosleep (&ts, &tsrem) < 0)
661 ts = tsrem;
662 time_left--;
663 }
Ed Warnickecb9cada2015-12-08 15:45:58 -0700664
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400665 rp = mmap (0, MMAP_PAGESIZE,
666 PROT_READ | PROT_WRITE, MAP_SHARED, svm_fd, 0);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700667
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400668 if (rp == (svm_region_t *) MAP_FAILED)
669 {
670 close (svm_fd);
671 clib_warning ("mmap");
672 return (0);
673 }
Benoît Ganne77d42fc2020-04-20 09:52:39 +0200674
Damjan Marion79934e82022-04-05 12:40:31 +0200675 clib_mem_unpoison (rp, MMAP_PAGESIZE);
Benoît Ganne77d42fc2020-04-20 09:52:39 +0200676
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400677 /*
678 * We lost the footrace to create this region; make sure
679 * the winner has crossed the finish line.
680 */
681 while (rp->version == 0 && deadman++ < 5)
682 {
683 sleep (1);
684 }
Ed Warnickecb9cada2015-12-08 15:45:58 -0700685
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400686 /*
687 * <bleep>-ed?
688 */
689 if (rp->version == 0)
690 {
691 clib_warning ("rp->version %d not %d", rp->version, SVM_VERSION);
692 close (svm_fd);
693 munmap (rp, a->size);
694 return (0);
695 }
696 /* Remap now that the region has been placed */
697 a->baseva = rp->virtual_base;
698 a->size = rp->virtual_size;
699 munmap (rp, MMAP_PAGESIZE);
700
Damjan Marion7bee80c2017-04-26 15:32:12 +0200701 rp = (void *) mmap (uword_to_pointer (a->baseva, void *), a->size,
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400702 PROT_READ | PROT_WRITE,
703 MAP_SHARED | MAP_FIXED, svm_fd, 0);
704 if ((uword) rp == (uword) MAP_FAILED)
705 {
706 clib_unix_warning ("mmap");
707 close (svm_fd);
708 return (0);
709 }
710
Dave Barachada24ea2018-05-24 17:32:00 -0400711 close (svm_fd);
712
Damjan Marion79934e82022-04-05 12:40:31 +0200713 clib_mem_unpoison (rp, a->size);
Benoît Ganne77d42fc2020-04-20 09:52:39 +0200714
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400715 if ((uword) rp != rp->virtual_base)
716 {
717 clib_warning ("mmap botch");
718 }
719
720 /*
721 * Try to fix the region mutex if it is held by
722 * a dead process
723 */
724 pid_holding_region_lock = rp->mutex_owner_pid;
725 if (pid_holding_region_lock && kill (pid_holding_region_lock, 0) < 0)
726 {
Benoît Ganne78de92d2021-01-20 19:10:59 +0100727 pthread_mutexattr_t attr;
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400728 clib_warning
729 ("region %s mutex held by dead pid %d, tag %d, force unlock",
730 rp->region_name, pid_holding_region_lock, rp->mutex_owner_tag);
731 /* owner pid is nonexistent */
Benoît Ganne78de92d2021-01-20 19:10:59 +0100732 if (pthread_mutexattr_init (&attr))
733 clib_unix_warning ("mutexattr_init");
734 if (pthread_mutexattr_setpshared (&attr, PTHREAD_PROCESS_SHARED))
735 clib_unix_warning ("mutexattr_setpshared");
736 if (pthread_mutex_init (&rp->mutex, &attr))
737 clib_unix_warning ("mutex_init");
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400738 dead_region_recovery = 1;
739 }
740
741 if (dead_region_recovery)
742 clib_warning ("recovery: attempt to re-lock region");
743
744 region_lock (rp, 2);
745 oldheap = svm_push_pvt_heap (rp);
746 vec_add1 (rp->client_pids, getpid ());
747
748 if (dead_region_recovery)
749 clib_warning ("recovery: attempt svm_data_region_map");
750
751 rv = svm_data_region_map (a, rp);
752 if (rv)
753 {
754 clib_warning ("data_region_map: %d", rv);
755 }
756
757 if (dead_region_recovery)
758 clib_warning ("unlock and continue");
759
760 region_unlock (rp);
761
762 svm_pop_heap (oldheap);
763
764 return ((void *) rp);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700765
766 }
Dave Barachc35f3e82020-04-02 10:44:09 -0400767 return 0; /* NOTREACHED *///NOSONAR
Ed Warnickecb9cada2015-12-08 15:45:58 -0700768}
769
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400770static void
771svm_mutex_cleanup (void)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700772{
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400773 int i;
774 for (i = 0; i < nheld; i++)
775 {
Dave Barachc35f3e82020-04-02 10:44:09 -0400776 pthread_mutex_unlock (mutexes_held[i]); //NOSONAR
Ed Warnickecb9cada2015-12-08 15:45:58 -0700777 }
778}
779
Ole Troan3cdc25f2017-08-17 11:07:33 +0200780static int
Dave Barachb3d93da2016-08-03 14:34:38 -0400781svm_region_init_internal (svm_map_region_args_t * a)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700782{
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400783 svm_region_t *rp;
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400784 u64 ticks = clib_cpu_time_now ();
785 uword randomize_baseva;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700786
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400787 /* guard against klutz calls */
788 if (root_rp)
Ole Troan3cdc25f2017-08-17 11:07:33 +0200789 return -1;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700790
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400791 root_rp_refcount++;
Dave Barach16c75df2016-05-31 14:05:46 -0400792
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400793 atexit (svm_mutex_cleanup);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700794
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400795 /* Randomize the shared-VM base at init time */
796 if (MMAP_PAGESIZE <= (4 << 10))
797 randomize_baseva = (ticks & 15) * MMAP_PAGESIZE;
798 else
799 randomize_baseva = (ticks & 3) * MMAP_PAGESIZE;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700800
Dave Barachb3d93da2016-08-03 14:34:38 -0400801 a->baseva += randomize_baseva;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700802
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400803 rp = svm_map_region (a);
Ole Troan3cdc25f2017-08-17 11:07:33 +0200804 if (!rp)
805 return -1;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700806
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400807 region_lock (rp, 3);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700808
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400809 /* Set up the main region data structures */
810 if (rp->flags & SVM_FLAGS_NEED_DATA_INIT)
811 {
812 svm_main_region_t *mp = 0;
813 void *oldheap;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700814
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400815 rp->flags &= ~(SVM_FLAGS_NEED_DATA_INIT);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700816
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400817 oldheap = svm_push_pvt_heap (rp);
818 vec_validate (mp, 0);
819 mp->name_hash = hash_create_string (0, sizeof (uword));
Dave Barachb3d93da2016-08-03 14:34:38 -0400820 mp->root_path = a->root_path ? format (0, "%s%c", a->root_path, 0) : 0;
Dave Wallace19296112017-08-31 15:54:11 -0400821 mp->uid = a->uid;
822 mp->gid = a->gid;
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400823 rp->data_base = mp;
824 svm_pop_heap (oldheap);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700825 }
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400826 region_unlock (rp);
827 root_rp = rp;
Ole Troan3cdc25f2017-08-17 11:07:33 +0200828
829 return 0;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700830}
831
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400832void
833svm_region_init (void)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700834{
Dave Barachb3d93da2016-08-03 14:34:38 -0400835 svm_map_region_args_t _a, *a = &_a;
Dave Barachc3799992016-08-15 11:12:27 -0400836
Dave Barachb7b92992018-10-17 10:38:51 -0400837 clib_memset (a, 0, sizeof (*a));
Dave Barachb3d93da2016-08-03 14:34:38 -0400838 a->root_path = 0;
839 a->name = SVM_GLOBAL_REGION_NAME;
Damjan Marionaec8f892018-01-08 16:35:35 +0100840 a->baseva = svm_get_global_region_base_va ();
Dave Barachb3d93da2016-08-03 14:34:38 -0400841 a->size = SVM_GLOBAL_REGION_SIZE;
842 a->flags = SVM_FLAGS_NODATA;
843 a->uid = 0;
844 a->gid = 0;
845
846 svm_region_init_internal (a);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700847}
848
Ole Troan3cdc25f2017-08-17 11:07:33 +0200849int
Neale Rannse72be392017-04-26 13:59:20 -0700850svm_region_init_chroot (const char *root_path)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700851{
Dave Barachb3d93da2016-08-03 14:34:38 -0400852 svm_map_region_args_t _a, *a = &_a;
Dave Barachc3799992016-08-15 11:12:27 -0400853
Dave Barachb7b92992018-10-17 10:38:51 -0400854 clib_memset (a, 0, sizeof (*a));
Dave Barachb3d93da2016-08-03 14:34:38 -0400855 a->root_path = root_path;
856 a->name = SVM_GLOBAL_REGION_NAME;
Damjan Marionaec8f892018-01-08 16:35:35 +0100857 a->baseva = svm_get_global_region_base_va ();
Dave Barachb3d93da2016-08-03 14:34:38 -0400858 a->size = SVM_GLOBAL_REGION_SIZE;
859 a->flags = SVM_FLAGS_NODATA;
860 a->uid = 0;
861 a->gid = 0;
862
Ole Troan3cdc25f2017-08-17 11:07:33 +0200863 return svm_region_init_internal (a);
Dave Barach16c75df2016-05-31 14:05:46 -0400864}
865
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400866void
Neale Rannse72be392017-04-26 13:59:20 -0700867svm_region_init_chroot_uid_gid (const char *root_path, int uid, int gid)
Dave Barach16c75df2016-05-31 14:05:46 -0400868{
Dave Barachb3d93da2016-08-03 14:34:38 -0400869 svm_map_region_args_t _a, *a = &_a;
Dave Barachc3799992016-08-15 11:12:27 -0400870
Dave Barachb7b92992018-10-17 10:38:51 -0400871 clib_memset (a, 0, sizeof (*a));
Dave Barachb3d93da2016-08-03 14:34:38 -0400872 a->root_path = root_path;
873 a->name = SVM_GLOBAL_REGION_NAME;
Damjan Marionaec8f892018-01-08 16:35:35 +0100874 a->baseva = svm_get_global_region_base_va ();
Dave Barachb3d93da2016-08-03 14:34:38 -0400875 a->size = SVM_GLOBAL_REGION_SIZE;
876 a->flags = SVM_FLAGS_NODATA;
877 a->uid = uid;
878 a->gid = gid;
879
880 svm_region_init_internal (a);
881}
882
883void
884svm_region_init_args (svm_map_region_args_t * a)
885{
886 svm_region_init_internal (a);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700887}
888
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400889void *
890svm_region_find_or_create (svm_map_region_args_t * a)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700891{
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400892 svm_main_region_t *mp;
893 svm_region_t *rp;
894 uword need_nbits;
895 int index, i;
896 void *oldheap;
897 uword *p;
898 u8 *name;
899 svm_subregion_t *subp;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700900
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400901 ASSERT (root_rp);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700902
Dave Barachc3799992016-08-15 11:12:27 -0400903 a->size += MMAP_PAGESIZE +
Dave Barachb3d93da2016-08-03 14:34:38 -0400904 ((a->pvt_heap_size != 0) ? a->pvt_heap_size : SVM_PVT_MHEAP_SIZE);
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400905 a->size = rnd_pagesize (a->size);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700906
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400907 region_lock (root_rp, 4);
908 oldheap = svm_push_pvt_heap (root_rp);
909 mp = root_rp->data_base;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700910
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400911 ASSERT (mp);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700912
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400913 /* Map the named region from the correct chroot environment */
Jan Srnicek5beec812017-03-24 10:18:11 +0100914 if (a->root_path == NULL)
915 a->root_path = (char *) mp->root_path;
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400916
917 /*
918 * See if this region is already known. If it is, we're
919 * almost done...
920 */
921 p = hash_get_mem (mp->name_hash, a->name);
922
923 if (p)
924 {
925 rp = svm_map_region (a);
926 region_unlock (root_rp);
927 svm_pop_heap (oldheap);
928 return rp;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700929 }
Ed Warnickecb9cada2015-12-08 15:45:58 -0700930
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400931 /* Create the region. */
932 ASSERT ((a->size & ~(MMAP_PAGESIZE - 1)) == a->size);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700933
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400934 need_nbits = a->size / MMAP_PAGESIZE;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700935
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400936 index = 1; /* $$$ fixme, figure out how many bit to really skip */
937
938 /*
939 * Scan the virtual space allocation bitmap, looking for a large
940 * enough chunk
941 */
942 do
943 {
944 if (clib_bitmap_get_no_check (root_rp->bitmap, index) == 0)
945 {
946 for (i = 0; i < (need_nbits - 1); i++)
947 {
948 if (clib_bitmap_get_no_check (root_rp->bitmap, index + i) == 1)
949 {
950 index = index + i;
951 goto next;
952 }
953 }
954 break;
955 }
956 index++;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700957 next:;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700958 }
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400959 while (index < root_rp->bitmap_size);
960
961 /* Completely out of VM? */
962 if (index >= root_rp->bitmap_size)
963 {
Dave Barachb3d93da2016-08-03 14:34:38 -0400964 clib_warning ("region %s: not enough VM to allocate 0x%llx (%lld)",
965 root_rp->region_name, a->size, a->size);
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400966 svm_pop_heap (oldheap);
967 region_unlock (root_rp);
968 return 0;
969 }
970
971 /*
972 * Mark virtual space allocated
973 */
Ed Warnickecb9cada2015-12-08 15:45:58 -0700974#if CLIB_DEBUG > 1
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400975 clib_warning ("set %d bits at index %d", need_nbits, index);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700976#endif
977
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400978 for (i = 0; i < need_nbits; i++)
979 {
980 clib_bitmap_set_no_check (root_rp->bitmap, index + i, 1);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700981 }
982
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400983 /* Place this region where it goes... */
984 a->baseva = root_rp->virtual_base + index * MMAP_PAGESIZE;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700985
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400986 rp = svm_map_region (a);
Dave Barachc3799992016-08-15 11:12:27 -0400987
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400988 pool_get (mp->subregions, subp);
989 name = format (0, "%s%c", a->name, 0);
990 subp->subregion_name = name;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700991
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400992 hash_set_mem (mp->name_hash, name, subp - mp->subregions);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700993
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400994 svm_pop_heap (oldheap);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700995
Dave Barach8a7fb0c2016-07-08 14:44:23 -0400996 region_unlock (root_rp);
997
998 return (rp);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700999}
1000
Dave Wallaced756b352017-07-03 13:11:38 -04001001void
1002svm_region_unlink (svm_region_t * rp)
1003{
1004 svm_map_region_args_t _a, *a = &_a;
1005 svm_main_region_t *mp;
1006 u8 *shm_name;
1007
1008 ASSERT (root_rp);
1009 ASSERT (rp);
1010 ASSERT (vec_c_string_is_terminated (rp->region_name));
1011
1012 mp = root_rp->data_base;
1013 ASSERT (mp);
1014
1015 a->root_path = (char *) mp->root_path;
1016 a->name = rp->region_name;
1017 shm_name = shm_name_from_svm_map_region_args (a);
1018 if (CLIB_DEBUG > 1)
1019 clib_warning ("[%d] shm_unlink (%s)", getpid (), shm_name);
1020 shm_unlink ((const char *) shm_name);
1021 vec_free (shm_name);
1022}
1023
Ed Warnickecb9cada2015-12-08 15:45:58 -07001024/*
1025 * svm_region_unmap
1026 *
1027 * Let go of the indicated region. If the calling process
1028 * is the last customer, throw it away completely.
1029 * The root region mutex guarantees atomicity with respect to
1030 * a new region client showing up at the wrong moment.
1031 */
Dave Barach8a7fb0c2016-07-08 14:44:23 -04001032void
Florin Corasd6c30d92018-01-29 05:11:24 -08001033svm_region_unmap_internal (void *rp_arg, u8 is_client)
Ed Warnickecb9cada2015-12-08 15:45:58 -07001034{
Dave Barach8a7fb0c2016-07-08 14:44:23 -04001035 int i, mypid = getpid ();
1036 int nclients_left;
1037 void *oldheap;
1038 uword virtual_base, virtual_size;
1039 svm_region_t *rp = rp_arg;
1040 char *name;
Ed Warnickecb9cada2015-12-08 15:45:58 -07001041
Dave Barach8a7fb0c2016-07-08 14:44:23 -04001042 /*
1043 * If we take a signal while holding one or more shared-memory
1044 * mutexes, we may end up back here from an otherwise
1045 * benign exit handler. Bail out to avoid a recursive
1046 * mutex screw-up.
1047 */
1048 if (nheld)
1049 return;
Ed Warnickecb9cada2015-12-08 15:45:58 -07001050
Dave Barach8a7fb0c2016-07-08 14:44:23 -04001051 ASSERT (rp);
1052 ASSERT (root_rp);
Ed Warnickecb9cada2015-12-08 15:45:58 -07001053
Dave Barach8a7fb0c2016-07-08 14:44:23 -04001054 if (CLIB_DEBUG > 1)
1055 clib_warning ("[%d] unmap region %s", getpid (), rp->region_name);
Ed Warnickecb9cada2015-12-08 15:45:58 -07001056
Dave Barach8a7fb0c2016-07-08 14:44:23 -04001057 region_lock (root_rp, 5);
1058 region_lock (rp, 6);
Ed Warnickecb9cada2015-12-08 15:45:58 -07001059
Dave Barach8a7fb0c2016-07-08 14:44:23 -04001060 oldheap = svm_push_pvt_heap (rp); /* nb vec_delete() in the loop */
1061
1062 /* Remove the caller from the list of mappers */
Damjan Marion79934e82022-04-05 12:40:31 +02001063 clib_mem_unpoison (rp->client_pids, vec_bytes (rp->client_pids));
Dave Barach8a7fb0c2016-07-08 14:44:23 -04001064 for (i = 0; i < vec_len (rp->client_pids); i++)
1065 {
1066 if (rp->client_pids[i] == mypid)
1067 {
1068 vec_delete (rp->client_pids, 1, i);
1069 goto found;
1070 }
Ed Warnickecb9cada2015-12-08 15:45:58 -07001071 }
Dave Barach8a7fb0c2016-07-08 14:44:23 -04001072 clib_warning ("pid %d AWOL", mypid);
Ed Warnickecb9cada2015-12-08 15:45:58 -07001073
Dave Barach8a7fb0c2016-07-08 14:44:23 -04001074found:
Ed Warnickecb9cada2015-12-08 15:45:58 -07001075
Dave Barach8a7fb0c2016-07-08 14:44:23 -04001076 svm_pop_heap (oldheap);
Ed Warnickecb9cada2015-12-08 15:45:58 -07001077
Dave Barach8a7fb0c2016-07-08 14:44:23 -04001078 nclients_left = vec_len (rp->client_pids);
1079 virtual_base = rp->virtual_base;
1080 virtual_size = rp->virtual_size;
Ed Warnickecb9cada2015-12-08 15:45:58 -07001081
Dave Barach8a7fb0c2016-07-08 14:44:23 -04001082 if (nclients_left == 0)
1083 {
1084 int index, nbits, i;
1085 svm_main_region_t *mp;
1086 uword *p;
1087 svm_subregion_t *subp;
Ed Warnickecb9cada2015-12-08 15:45:58 -07001088
Dave Barach8a7fb0c2016-07-08 14:44:23 -04001089 /* Kill the region, last guy on his way out */
Ed Warnickecb9cada2015-12-08 15:45:58 -07001090
Dave Barach8a7fb0c2016-07-08 14:44:23 -04001091 oldheap = svm_push_pvt_heap (root_rp);
1092 name = vec_dup (rp->region_name);
Ed Warnickecb9cada2015-12-08 15:45:58 -07001093
Dave Barach8a7fb0c2016-07-08 14:44:23 -04001094 virtual_base = rp->virtual_base;
1095 virtual_size = rp->virtual_size;
Ed Warnickecb9cada2015-12-08 15:45:58 -07001096
Dave Barach8a7fb0c2016-07-08 14:44:23 -04001097 /* Figure out which bits to clear in the root region bitmap */
1098 index = (virtual_base - root_rp->virtual_base) / MMAP_PAGESIZE;
Ed Warnickecb9cada2015-12-08 15:45:58 -07001099
Dave Barach8a7fb0c2016-07-08 14:44:23 -04001100 nbits = (virtual_size + MMAP_PAGESIZE - 1) / MMAP_PAGESIZE;
Ed Warnickecb9cada2015-12-08 15:45:58 -07001101
1102#if CLIB_DEBUG > 1
Dave Barach8a7fb0c2016-07-08 14:44:23 -04001103 clib_warning ("clear %d bits at index %d", nbits, index);
Ed Warnickecb9cada2015-12-08 15:45:58 -07001104#endif
Dave Barach8a7fb0c2016-07-08 14:44:23 -04001105 /* Give back the allocated VM */
1106 for (i = 0; i < nbits; i++)
1107 {
1108 clib_bitmap_set_no_check (root_rp->bitmap, index + i, 0);
1109 }
Ed Warnickecb9cada2015-12-08 15:45:58 -07001110
Dave Barach8a7fb0c2016-07-08 14:44:23 -04001111 mp = root_rp->data_base;
Ed Warnickecb9cada2015-12-08 15:45:58 -07001112
Dave Barach8a7fb0c2016-07-08 14:44:23 -04001113 p = hash_get_mem (mp->name_hash, name);
Ed Warnickecb9cada2015-12-08 15:45:58 -07001114
Dave Barach8a7fb0c2016-07-08 14:44:23 -04001115 /* Better never happen ... */
1116 if (p == NULL)
1117 {
1118 region_unlock (rp);
1119 region_unlock (root_rp);
1120 svm_pop_heap (oldheap);
1121 clib_warning ("Region name '%s' not found?", name);
1122 return;
1123 }
Ed Warnickecb9cada2015-12-08 15:45:58 -07001124
Dave Barach8a7fb0c2016-07-08 14:44:23 -04001125 /* Remove from the root region subregion pool */
1126 subp = mp->subregions + p[0];
1127 pool_put (mp->subregions, subp);
1128
1129 hash_unset_mem (mp->name_hash, name);
1130
1131 vec_free (name);
1132
1133 region_unlock (rp);
Florin Corasd6c30d92018-01-29 05:11:24 -08001134
1135 /* If a client asks for the cleanup, don't unlink the backing
1136 * file since we can't tell if it has been recreated. */
1137 if (!is_client)
1138 svm_region_unlink (rp);
1139
Dave Barach8a7fb0c2016-07-08 14:44:23 -04001140 munmap ((void *) virtual_base, virtual_size);
1141 region_unlock (root_rp);
1142 svm_pop_heap (oldheap);
1143 return;
Ed Warnickecb9cada2015-12-08 15:45:58 -07001144 }
1145
Dave Barach8a7fb0c2016-07-08 14:44:23 -04001146 region_unlock (rp);
1147 region_unlock (root_rp);
Ed Warnickecb9cada2015-12-08 15:45:58 -07001148
Dave Barach8a7fb0c2016-07-08 14:44:23 -04001149 munmap ((void *) virtual_base, virtual_size);
Ed Warnickecb9cada2015-12-08 15:45:58 -07001150}
1151
Florin Corasd6c30d92018-01-29 05:11:24 -08001152void
1153svm_region_unmap (void *rp_arg)
1154{
1155 svm_region_unmap_internal (rp_arg, 0 /* is_client */ );
1156}
1157
1158void
1159svm_region_unmap_client (void *rp_arg)
1160{
1161 svm_region_unmap_internal (rp_arg, 1 /* is_client */ );
1162}
1163
Ed Warnickecb9cada2015-12-08 15:45:58 -07001164/*
1165 * svm_region_exit
Dave Barach8a7fb0c2016-07-08 14:44:23 -04001166 */
Florin Corasd6c30d92018-01-29 05:11:24 -08001167static void
1168svm_region_exit_internal (u8 is_client)
Ed Warnickecb9cada2015-12-08 15:45:58 -07001169{
Dave Barach8a7fb0c2016-07-08 14:44:23 -04001170 void *oldheap;
1171 int i, mypid = getpid ();
1172 uword virtual_base, virtual_size;
Ed Warnickecb9cada2015-12-08 15:45:58 -07001173
Dave Barach8a7fb0c2016-07-08 14:44:23 -04001174 /* It felt so nice we did it twice... */
1175 if (root_rp == 0)
1176 return;
Ed Warnickecb9cada2015-12-08 15:45:58 -07001177
Dave Barach8a7fb0c2016-07-08 14:44:23 -04001178 if (--root_rp_refcount > 0)
1179 return;
Ed Warnickecb9cada2015-12-08 15:45:58 -07001180
Dave Barach8a7fb0c2016-07-08 14:44:23 -04001181 /*
1182 * If we take a signal while holding one or more shared-memory
1183 * mutexes, we may end up back here from an otherwise
1184 * benign exit handler. Bail out to avoid a recursive
1185 * mutex screw-up.
1186 */
1187 if (nheld)
1188 return;
Ed Warnickecb9cada2015-12-08 15:45:58 -07001189
Dave Barach8a7fb0c2016-07-08 14:44:23 -04001190 region_lock (root_rp, 7);
1191 oldheap = svm_push_pvt_heap (root_rp);
Ed Warnickecb9cada2015-12-08 15:45:58 -07001192
Dave Barach8a7fb0c2016-07-08 14:44:23 -04001193 virtual_base = root_rp->virtual_base;
1194 virtual_size = root_rp->virtual_size;
Ed Warnickecb9cada2015-12-08 15:45:58 -07001195
Damjan Marion79934e82022-04-05 12:40:31 +02001196 clib_mem_unpoison (root_rp->client_pids, vec_bytes (root_rp->client_pids));
Dave Barach8a7fb0c2016-07-08 14:44:23 -04001197 for (i = 0; i < vec_len (root_rp->client_pids); i++)
1198 {
1199 if (root_rp->client_pids[i] == mypid)
1200 {
1201 vec_delete (root_rp->client_pids, 1, i);
1202 goto found;
1203 }
Ed Warnickecb9cada2015-12-08 15:45:58 -07001204 }
Dave Barach8a7fb0c2016-07-08 14:44:23 -04001205 clib_warning ("pid %d AWOL", mypid);
Ed Warnickecb9cada2015-12-08 15:45:58 -07001206
Dave Barach8a7fb0c2016-07-08 14:44:23 -04001207found:
Ed Warnickecb9cada2015-12-08 15:45:58 -07001208
Florin Corasd6c30d92018-01-29 05:11:24 -08001209 if (!is_client && vec_len (root_rp->client_pids) == 0)
Dave Wallaced756b352017-07-03 13:11:38 -04001210 svm_region_unlink (root_rp);
1211
Dave Barach8a7fb0c2016-07-08 14:44:23 -04001212 region_unlock (root_rp);
1213 svm_pop_heap (oldheap);
Ed Warnickecb9cada2015-12-08 15:45:58 -07001214
Dave Barach8a7fb0c2016-07-08 14:44:23 -04001215 root_rp = 0;
1216 munmap ((void *) virtual_base, virtual_size);
Ed Warnickecb9cada2015-12-08 15:45:58 -07001217}
1218
Dave Barach8a7fb0c2016-07-08 14:44:23 -04001219void
Florin Corasd6c30d92018-01-29 05:11:24 -08001220svm_region_exit (void)
1221{
1222 svm_region_exit_internal (0 /* is_client */ );
1223}
1224
1225void
1226svm_region_exit_client (void)
1227{
1228 svm_region_exit_internal (1 /* is_client */ );
1229}
1230
1231void
Dave Barach8a7fb0c2016-07-08 14:44:23 -04001232svm_client_scan_this_region_nolock (svm_region_t * rp)
Ed Warnickecb9cada2015-12-08 15:45:58 -07001233{
Dave Barach8a7fb0c2016-07-08 14:44:23 -04001234 int j;
1235 int mypid = getpid ();
1236 void *oldheap;
Ed Warnickecb9cada2015-12-08 15:45:58 -07001237
Dave Barach8a7fb0c2016-07-08 14:44:23 -04001238 for (j = 0; j < vec_len (rp->client_pids); j++)
1239 {
1240 if (mypid == rp->client_pids[j])
1241 continue;
1242 if (rp->client_pids[j] && (kill (rp->client_pids[j], 0) < 0))
1243 {
1244 clib_warning ("%s: cleanup ghost pid %d",
1245 rp->region_name, rp->client_pids[j]);
1246 /* nb: client vec in rp->region_heap */
1247 oldheap = svm_push_pvt_heap (rp);
1248 vec_delete (rp->client_pids, 1, j);
1249 j--;
1250 svm_pop_heap (oldheap);
1251 }
Ed Warnickecb9cada2015-12-08 15:45:58 -07001252 }
1253}
1254
1255
Dave Barach8a7fb0c2016-07-08 14:44:23 -04001256/*
Ed Warnickecb9cada2015-12-08 15:45:58 -07001257 * Scan svm regions for dead clients
1258 */
Dave Barach8a7fb0c2016-07-08 14:44:23 -04001259void
Neale Rannse72be392017-04-26 13:59:20 -07001260svm_client_scan (const char *root_path)
Ed Warnickecb9cada2015-12-08 15:45:58 -07001261{
Dave Barach8a7fb0c2016-07-08 14:44:23 -04001262 int i, j;
1263 svm_main_region_t *mp;
1264 svm_map_region_args_t *a = 0;
1265 svm_region_t *root_rp;
1266 svm_region_t *rp;
1267 svm_subregion_t *subp;
1268 u8 *name = 0;
1269 u8 **svm_names = 0;
1270 void *oldheap;
1271 int mypid = getpid ();
Ed Warnickecb9cada2015-12-08 15:45:58 -07001272
Dave Barach8a7fb0c2016-07-08 14:44:23 -04001273 vec_validate (a, 0);
Ed Warnickecb9cada2015-12-08 15:45:58 -07001274
Dave Barach8a7fb0c2016-07-08 14:44:23 -04001275 svm_region_init_chroot (root_path);
Ed Warnickecb9cada2015-12-08 15:45:58 -07001276
Dave Barach8a7fb0c2016-07-08 14:44:23 -04001277 root_rp = svm_get_root_rp ();
Ed Warnickecb9cada2015-12-08 15:45:58 -07001278
Dave Barach8a7fb0c2016-07-08 14:44:23 -04001279 pthread_mutex_lock (&root_rp->mutex);
Ed Warnickecb9cada2015-12-08 15:45:58 -07001280
Dave Barach8a7fb0c2016-07-08 14:44:23 -04001281 mp = root_rp->data_base;
Ed Warnickecb9cada2015-12-08 15:45:58 -07001282
Dave Barach8a7fb0c2016-07-08 14:44:23 -04001283 for (j = 0; j < vec_len (root_rp->client_pids); j++)
1284 {
1285 if (mypid == root_rp->client_pids[j])
1286 continue;
1287 if (root_rp->client_pids[j] && (kill (root_rp->client_pids[j], 0) < 0))
1288 {
1289 clib_warning ("%s: cleanup ghost pid %d",
1290 root_rp->region_name, root_rp->client_pids[j]);
1291 /* nb: client vec in root_rp->region_heap */
1292 oldheap = svm_push_pvt_heap (root_rp);
1293 vec_delete (root_rp->client_pids, 1, j);
1294 j--;
1295 svm_pop_heap (oldheap);
1296 }
Ed Warnickecb9cada2015-12-08 15:45:58 -07001297 }
1298
Dave Barach8a7fb0c2016-07-08 14:44:23 -04001299 /*
1300 * Snapshoot names, can't hold root rp mutex across
1301 * find_or_create.
1302 */
1303 /* *INDENT-OFF* */
Damjan Marionb2c31b62020-12-13 21:47:40 +01001304 pool_foreach (subp, mp->subregions) {
Dave Barach8a7fb0c2016-07-08 14:44:23 -04001305 name = vec_dup (subp->subregion_name);
1306 vec_add1(svm_names, name);
Damjan Marionb2c31b62020-12-13 21:47:40 +01001307 }
Dave Barach8a7fb0c2016-07-08 14:44:23 -04001308 /* *INDENT-ON* */
Ed Warnickecb9cada2015-12-08 15:45:58 -07001309
Dave Barach8a7fb0c2016-07-08 14:44:23 -04001310 pthread_mutex_unlock (&root_rp->mutex);
Ed Warnickecb9cada2015-12-08 15:45:58 -07001311
Dave Barach8a7fb0c2016-07-08 14:44:23 -04001312 for (i = 0; i < vec_len (svm_names); i++)
1313 {
1314 vec_validate (a, 0);
1315 a->root_path = root_path;
1316 a->name = (char *) svm_names[i];
1317 rp = svm_region_find_or_create (a);
1318 if (rp)
1319 {
1320 pthread_mutex_lock (&rp->mutex);
Ed Warnickecb9cada2015-12-08 15:45:58 -07001321
Dave Barach8a7fb0c2016-07-08 14:44:23 -04001322 svm_client_scan_this_region_nolock (rp);
Ed Warnickecb9cada2015-12-08 15:45:58 -07001323
Dave Barach8a7fb0c2016-07-08 14:44:23 -04001324 pthread_mutex_unlock (&rp->mutex);
1325 svm_region_unmap (rp);
1326 vec_free (svm_names[i]);
1327 }
1328 vec_free (a);
Ed Warnickecb9cada2015-12-08 15:45:58 -07001329 }
Dave Barach8a7fb0c2016-07-08 14:44:23 -04001330 vec_free (svm_names);
Ed Warnickecb9cada2015-12-08 15:45:58 -07001331
Dave Barach8a7fb0c2016-07-08 14:44:23 -04001332 svm_region_exit ();
Ed Warnickecb9cada2015-12-08 15:45:58 -07001333
Dave Barach8a7fb0c2016-07-08 14:44:23 -04001334 vec_free (a);
Ed Warnickecb9cada2015-12-08 15:45:58 -07001335}
Dave Barach8a7fb0c2016-07-08 14:44:23 -04001336
1337/*
1338 * fd.io coding-style-patch-verification: ON
1339 *
1340 * Local Variables:
1341 * eval: (c-set-style "gnu")
1342 * End:
1343 */