Florin Coras | e86a8ed | 2018-01-05 03:20:25 -0800 | [diff] [blame] | 1 | /* |
| 2 | *------------------------------------------------------------------ |
| 3 | * Copyright (c) 2018 Cisco and/or its affiliates. |
| 4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | * you may not use this file except in compliance with the License. |
| 6 | * You may obtain a copy of the License at: |
| 7 | * |
| 8 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | * |
| 10 | * Unless required by applicable law or agreed to in writing, software |
| 11 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | * See the License for the specific language governing permissions and |
| 14 | * limitations under the License. |
| 15 | *------------------------------------------------------------------ |
| 16 | */ |
| 17 | #include <signal.h> |
| 18 | |
| 19 | #include <vlib/vlib.h> |
| 20 | #include <vlibapi/api.h> |
| 21 | #include <vlibmemory/api.h> |
| 22 | #include <vlibmemory/memory_api.h> |
| 23 | |
| 24 | #include <vlibmemory/vl_memory_msg_enum.h> /* enumerate all vlib messages */ |
| 25 | |
| 26 | #define vl_typedefs /* define message structures */ |
| 27 | #include <vlibmemory/vl_memory_api_h.h> |
| 28 | #undef vl_typedefs |
| 29 | |
| 30 | /* instantiate all the print functions we know about */ |
| 31 | #define vl_print(handle, ...) vlib_cli_output (handle, __VA_ARGS__) |
| 32 | #define vl_printfun |
| 33 | #include <vlibmemory/vl_memory_api_h.h> |
| 34 | #undef vl_printfun |
| 35 | |
| 36 | /* instantiate all the endian swap functions we know about */ |
| 37 | #define vl_endianfun |
| 38 | #include <vlibmemory/vl_memory_api_h.h> |
| 39 | #undef vl_endianfun |
| 40 | |
| 41 | static inline void * |
| 42 | vl_api_memclnt_create_t_print (vl_api_memclnt_create_t * a, void *handle) |
| 43 | { |
| 44 | vl_print (handle, "vl_api_memclnt_create_t:\n"); |
| 45 | vl_print (handle, "name: %s\n", a->name); |
| 46 | vl_print (handle, "input_queue: 0x%wx\n", a->input_queue); |
| 47 | vl_print (handle, "context: %u\n", (unsigned) a->context); |
| 48 | vl_print (handle, "ctx_quota: %ld\n", (long) a->ctx_quota); |
| 49 | return handle; |
| 50 | } |
| 51 | |
| 52 | static inline void * |
| 53 | vl_api_memclnt_delete_t_print (vl_api_memclnt_delete_t * a, void *handle) |
| 54 | { |
| 55 | vl_print (handle, "vl_api_memclnt_delete_t:\n"); |
| 56 | vl_print (handle, "index: %u\n", (unsigned) a->index); |
| 57 | vl_print (handle, "handle: 0x%wx\n", a->handle); |
| 58 | return handle; |
| 59 | } |
| 60 | |
| 61 | volatile int **vl_api_queue_cursizes; |
| 62 | |
| 63 | static void |
| 64 | memclnt_queue_callback (vlib_main_t * vm) |
| 65 | { |
| 66 | int i; |
| 67 | api_main_t *am = &api_main; |
| 68 | |
| 69 | if (PREDICT_FALSE (vec_len (vl_api_queue_cursizes) != |
| 70 | 1 + vec_len (am->vlib_private_rps))) |
| 71 | { |
| 72 | vl_shmem_hdr_t *shmem_hdr = am->shmem_hdr; |
| 73 | svm_queue_t *q; |
| 74 | |
| 75 | if (shmem_hdr == 0) |
| 76 | return; |
| 77 | |
| 78 | q = shmem_hdr->vl_input_queue; |
| 79 | if (q == 0) |
| 80 | return; |
| 81 | |
| 82 | vec_add1 (vl_api_queue_cursizes, &q->cursize); |
| 83 | |
| 84 | for (i = 0; i < vec_len (am->vlib_private_rps); i++) |
| 85 | { |
| 86 | svm_region_t *vlib_rp = am->vlib_private_rps[i]; |
| 87 | |
| 88 | shmem_hdr = (void *) vlib_rp->user_ctx; |
| 89 | q = shmem_hdr->vl_input_queue; |
| 90 | vec_add1 (vl_api_queue_cursizes, &q->cursize); |
| 91 | } |
| 92 | } |
| 93 | |
| 94 | for (i = 0; i < vec_len (vl_api_queue_cursizes); i++) |
| 95 | { |
| 96 | if (*vl_api_queue_cursizes[i]) |
| 97 | { |
| 98 | vm->queue_signal_pending = 1; |
| 99 | vm->api_queue_nonempty = 1; |
| 100 | vlib_process_signal_event (vm, vl_api_clnt_node.index, |
| 101 | /* event_type */ QUEUE_SIGNAL_EVENT, |
| 102 | /* event_data */ 0); |
| 103 | break; |
| 104 | } |
| 105 | } |
| 106 | } |
| 107 | |
| 108 | /* |
| 109 | * vl_api_memclnt_create_internal |
| 110 | */ |
| 111 | u32 |
| 112 | vl_api_memclnt_create_internal (char *name, svm_queue_t * q) |
| 113 | { |
| 114 | vl_api_registration_t **regpp; |
| 115 | vl_api_registration_t *regp; |
| 116 | svm_region_t *svm; |
| 117 | void *oldheap; |
| 118 | api_main_t *am = &api_main; |
| 119 | |
| 120 | ASSERT (vlib_get_thread_index () == 0); |
| 121 | pool_get (am->vl_clients, regpp); |
| 122 | |
| 123 | svm = am->vlib_rp; |
| 124 | |
| 125 | pthread_mutex_lock (&svm->mutex); |
| 126 | oldheap = svm_push_data_heap (svm); |
| 127 | *regpp = clib_mem_alloc (sizeof (vl_api_registration_t)); |
| 128 | |
| 129 | regp = *regpp; |
| 130 | memset (regp, 0, sizeof (*regp)); |
| 131 | regp->registration_type = REGISTRATION_TYPE_SHMEM; |
| 132 | regp->vl_api_registration_pool_index = regpp - am->vl_clients; |
| 133 | regp->vlib_rp = svm; |
| 134 | regp->shmem_hdr = am->shmem_hdr; |
| 135 | |
| 136 | regp->vl_input_queue = q; |
| 137 | regp->name = format (0, "%s%c", name, 0); |
| 138 | |
| 139 | pthread_mutex_unlock (&svm->mutex); |
| 140 | svm_pop_heap (oldheap); |
| 141 | return vl_msg_api_handle_from_index_and_epoch |
| 142 | (regp->vl_api_registration_pool_index, |
| 143 | am->shmem_hdr->application_restarts); |
| 144 | } |
| 145 | |
| 146 | /* |
| 147 | * vl_api_memclnt_create_t_handler |
| 148 | */ |
| 149 | void |
| 150 | vl_api_memclnt_create_t_handler (vl_api_memclnt_create_t * mp) |
| 151 | { |
| 152 | vl_api_registration_t **regpp; |
| 153 | vl_api_registration_t *regp; |
| 154 | vl_api_memclnt_create_reply_t *rp; |
| 155 | svm_region_t *svm; |
| 156 | svm_queue_t *q; |
| 157 | int rv = 0; |
| 158 | void *oldheap; |
| 159 | api_main_t *am = &api_main; |
Florin Coras | e25c9bf | 2018-08-06 12:19:29 -0700 | [diff] [blame^] | 160 | u8 *msg_table; |
Florin Coras | e86a8ed | 2018-01-05 03:20:25 -0800 | [diff] [blame] | 161 | |
| 162 | /* |
| 163 | * This is tortured. Maintain a vlib-address-space private |
| 164 | * pool of client registrations. We use the shared-memory virtual |
| 165 | * address of client structure as a handle, to allow direct |
| 166 | * manipulation of context quota vbls from the client library. |
| 167 | * |
| 168 | * This scheme causes trouble w/ API message trace replay, since |
| 169 | * some random VA from clib_mem_alloc() certainly won't |
| 170 | * occur in the Linux sim. The (very) few places |
| 171 | * that care need to use the pool index. |
| 172 | * |
| 173 | * Putting the registration object(s) into a pool in shared memory and |
| 174 | * using the pool index as a handle seems like a great idea. |
| 175 | * Unfortunately, each and every reference to that pool would need |
| 176 | * to be protected by a mutex: |
| 177 | * |
| 178 | * Client VLIB |
| 179 | * ------ ---- |
| 180 | * convert pool index to |
| 181 | * pointer. |
| 182 | * <deschedule> |
| 183 | * expand pool |
| 184 | * <deschedule> |
| 185 | * kaboom! |
| 186 | */ |
| 187 | |
| 188 | pool_get (am->vl_clients, regpp); |
| 189 | |
| 190 | svm = am->vlib_rp; |
| 191 | |
| 192 | pthread_mutex_lock (&svm->mutex); |
| 193 | oldheap = svm_push_data_heap (svm); |
| 194 | *regpp = clib_mem_alloc (sizeof (vl_api_registration_t)); |
| 195 | |
| 196 | regp = *regpp; |
| 197 | memset (regp, 0, sizeof (*regp)); |
| 198 | regp->registration_type = REGISTRATION_TYPE_SHMEM; |
| 199 | regp->vl_api_registration_pool_index = regpp - am->vl_clients; |
| 200 | regp->vlib_rp = svm; |
| 201 | regp->shmem_hdr = am->shmem_hdr; |
Florin Coras | b384b54 | 2018-01-15 01:08:33 -0800 | [diff] [blame] | 202 | regp->clib_file_index = am->shmem_hdr->clib_file_index; |
Florin Coras | e86a8ed | 2018-01-05 03:20:25 -0800 | [diff] [blame] | 203 | |
| 204 | q = regp->vl_input_queue = (svm_queue_t *) (uword) mp->input_queue; |
| 205 | |
| 206 | regp->name = format (0, "%s", mp->name); |
| 207 | vec_add1 (regp->name, 0); |
| 208 | |
| 209 | if (am->serialized_message_table_in_shmem == 0) |
| 210 | am->serialized_message_table_in_shmem = |
| 211 | vl_api_serialize_message_table (am, 0); |
| 212 | |
Florin Coras | e25c9bf | 2018-08-06 12:19:29 -0700 | [diff] [blame^] | 213 | if (am->vlib_rp != am->vlib_primary_rp) |
| 214 | msg_table = vl_api_serialize_message_table (am, 0); |
| 215 | else |
| 216 | msg_table = am->serialized_message_table_in_shmem; |
| 217 | |
Florin Coras | e86a8ed | 2018-01-05 03:20:25 -0800 | [diff] [blame] | 218 | pthread_mutex_unlock (&svm->mutex); |
| 219 | svm_pop_heap (oldheap); |
| 220 | |
| 221 | rp = vl_msg_api_alloc (sizeof (*rp)); |
| 222 | rp->_vl_msg_id = ntohs (VL_API_MEMCLNT_CREATE_REPLY); |
| 223 | rp->handle = (uword) regp; |
| 224 | rp->index = vl_msg_api_handle_from_index_and_epoch |
| 225 | (regp->vl_api_registration_pool_index, |
| 226 | am->shmem_hdr->application_restarts); |
| 227 | rp->context = mp->context; |
| 228 | rp->response = ntohl (rv); |
Florin Coras | e25c9bf | 2018-08-06 12:19:29 -0700 | [diff] [blame^] | 229 | rp->message_table = pointer_to_uword (msg_table); |
Florin Coras | e86a8ed | 2018-01-05 03:20:25 -0800 | [diff] [blame] | 230 | |
| 231 | vl_msg_api_send_shmem (q, (u8 *) & rp); |
| 232 | } |
| 233 | |
| 234 | int |
| 235 | vl_api_call_reaper_functions (u32 client_index) |
| 236 | { |
| 237 | clib_error_t *error = 0; |
| 238 | _vl_msg_api_function_list_elt_t *i; |
| 239 | |
| 240 | i = api_main.reaper_function_registrations; |
| 241 | while (i) |
| 242 | { |
| 243 | error = i->f (client_index); |
| 244 | if (error) |
| 245 | clib_error_report (error); |
| 246 | i = i->next_init_function; |
| 247 | } |
| 248 | return 0; |
| 249 | } |
| 250 | |
| 251 | /* |
| 252 | * vl_api_memclnt_delete_t_handler |
| 253 | */ |
| 254 | void |
| 255 | vl_api_memclnt_delete_t_handler (vl_api_memclnt_delete_t * mp) |
| 256 | { |
| 257 | vl_api_registration_t **regpp; |
| 258 | vl_api_registration_t *regp; |
| 259 | vl_api_memclnt_delete_reply_t *rp; |
| 260 | svm_region_t *svm; |
| 261 | void *oldheap; |
| 262 | api_main_t *am = &api_main; |
| 263 | u32 handle, client_index, epoch; |
| 264 | |
| 265 | handle = mp->index; |
| 266 | |
| 267 | if (vl_api_call_reaper_functions (handle)) |
| 268 | return; |
| 269 | |
| 270 | epoch = vl_msg_api_handle_get_epoch (handle); |
| 271 | client_index = vl_msg_api_handle_get_index (handle); |
| 272 | |
| 273 | if (epoch != (am->shmem_hdr->application_restarts & VL_API_EPOCH_MASK)) |
| 274 | { |
| 275 | clib_warning |
| 276 | ("Stale clnt delete index %d old epoch %d cur epoch %d", |
| 277 | client_index, epoch, |
| 278 | (am->shmem_hdr->application_restarts & VL_API_EPOCH_MASK)); |
| 279 | return; |
| 280 | } |
| 281 | |
Florin Coras | b384b54 | 2018-01-15 01:08:33 -0800 | [diff] [blame] | 282 | regpp = pool_elt_at_index (am->vl_clients, client_index); |
Florin Coras | e86a8ed | 2018-01-05 03:20:25 -0800 | [diff] [blame] | 283 | |
| 284 | if (!pool_is_free (am->vl_clients, regpp)) |
| 285 | { |
| 286 | int i; |
| 287 | regp = *regpp; |
| 288 | svm = am->vlib_rp; |
| 289 | int private_registration = 0; |
| 290 | |
| 291 | /* |
| 292 | * Note: the API message handling path will set am->vlib_rp |
| 293 | * as appropriate for pairwise / private memory segments |
| 294 | */ |
| 295 | rp = vl_msg_api_alloc (sizeof (*rp)); |
| 296 | rp->_vl_msg_id = ntohs (VL_API_MEMCLNT_DELETE_REPLY); |
| 297 | rp->handle = mp->handle; |
| 298 | rp->response = 1; |
| 299 | |
| 300 | vl_msg_api_send_shmem (regp->vl_input_queue, (u8 *) & rp); |
| 301 | |
| 302 | if (client_index != regp->vl_api_registration_pool_index) |
| 303 | { |
| 304 | clib_warning ("mismatch client_index %d pool_index %d", |
| 305 | client_index, regp->vl_api_registration_pool_index); |
| 306 | vl_msg_api_free (rp); |
| 307 | return; |
| 308 | } |
| 309 | |
| 310 | /* For horizontal scaling, add a hash table... */ |
| 311 | for (i = 0; i < vec_len (am->vlib_private_rps); i++) |
| 312 | { |
| 313 | /* Is this a pairwise / private API segment? */ |
| 314 | if (am->vlib_private_rps[i] == svm) |
| 315 | { |
| 316 | /* Note: account for the memfd header page */ |
| 317 | u64 virtual_base = svm->virtual_base - MMAP_PAGESIZE; |
| 318 | u64 virtual_size = svm->virtual_size + MMAP_PAGESIZE; |
| 319 | |
| 320 | /* |
| 321 | * Kill the registration pool element before we make |
| 322 | * the index vanish forever |
| 323 | */ |
| 324 | pool_put_index (am->vl_clients, |
| 325 | regp->vl_api_registration_pool_index); |
| 326 | |
| 327 | vec_delete (am->vlib_private_rps, 1, i); |
| 328 | /* Kill it, accounting for the memfd header page */ |
| 329 | if (munmap ((void *) virtual_base, virtual_size) < 0) |
| 330 | clib_unix_warning ("munmap"); |
| 331 | /* Reset the queue-length-address cache */ |
| 332 | vec_reset_length (vl_api_queue_cursizes); |
| 333 | private_registration = 1; |
| 334 | break; |
| 335 | } |
| 336 | } |
| 337 | |
| 338 | /* No dangling references, please */ |
| 339 | *regpp = 0; |
| 340 | |
| 341 | if (private_registration == 0) |
| 342 | { |
| 343 | pool_put_index (am->vl_clients, |
| 344 | regp->vl_api_registration_pool_index); |
| 345 | pthread_mutex_lock (&svm->mutex); |
| 346 | oldheap = svm_push_data_heap (svm); |
Ole Troan | 73710c7 | 2018-06-04 22:27:49 +0200 | [diff] [blame] | 347 | vec_free (regp->name); |
Florin Coras | e86a8ed | 2018-01-05 03:20:25 -0800 | [diff] [blame] | 348 | /* Poison the old registration */ |
| 349 | memset (regp, 0xF1, sizeof (*regp)); |
| 350 | clib_mem_free (regp); |
| 351 | pthread_mutex_unlock (&svm->mutex); |
| 352 | svm_pop_heap (oldheap); |
| 353 | /* |
| 354 | * These messages must be freed manually, since they're set up |
| 355 | * as "bounce" messages. In the private_registration == 1 case, |
| 356 | * we kill the shared-memory segment which contains the message |
| 357 | * with munmap. |
| 358 | */ |
| 359 | vl_msg_api_free (mp); |
| 360 | } |
| 361 | } |
| 362 | else |
| 363 | { |
| 364 | clib_warning ("unknown client ID %d", mp->index); |
| 365 | } |
| 366 | } |
| 367 | |
| 368 | /** |
| 369 | * client answered a ping, stave off the grim reaper... |
| 370 | */ |
| 371 | void |
| 372 | vl_api_memclnt_keepalive_reply_t_handler |
| 373 | (vl_api_memclnt_keepalive_reply_t * mp) |
| 374 | { |
| 375 | vl_api_registration_t *regp; |
| 376 | vlib_main_t *vm = vlib_get_main (); |
| 377 | |
| 378 | regp = vl_api_client_index_to_registration (mp->context); |
| 379 | if (regp) |
| 380 | { |
| 381 | regp->last_heard = vlib_time_now (vm); |
| 382 | regp->unanswered_pings = 0; |
| 383 | } |
| 384 | else |
| 385 | clib_warning ("BUG: anonymous memclnt_keepalive_reply"); |
| 386 | } |
| 387 | |
| 388 | /** |
| 389 | * We can send ourselves these messages if someone uses the |
| 390 | * builtin binary api test tool... |
| 391 | */ |
| 392 | static void |
| 393 | vl_api_memclnt_keepalive_t_handler (vl_api_memclnt_keepalive_t * mp) |
| 394 | { |
| 395 | vl_api_memclnt_keepalive_reply_t *rmp; |
| 396 | api_main_t *am; |
| 397 | vl_shmem_hdr_t *shmem_hdr; |
| 398 | |
| 399 | am = &api_main; |
| 400 | shmem_hdr = am->shmem_hdr; |
| 401 | |
| 402 | rmp = vl_msg_api_alloc_as_if_client (sizeof (*rmp)); |
| 403 | memset (rmp, 0, sizeof (*rmp)); |
| 404 | rmp->_vl_msg_id = ntohs (VL_API_MEMCLNT_KEEPALIVE_REPLY); |
| 405 | rmp->context = mp->context; |
| 406 | vl_msg_api_send_shmem (shmem_hdr->vl_input_queue, (u8 *) & rmp); |
| 407 | } |
| 408 | |
| 409 | #define foreach_vlib_api_msg \ |
| 410 | _(MEMCLNT_CREATE, memclnt_create) \ |
| 411 | _(MEMCLNT_DELETE, memclnt_delete) \ |
| 412 | _(MEMCLNT_KEEPALIVE, memclnt_keepalive) \ |
| 413 | _(MEMCLNT_KEEPALIVE_REPLY, memclnt_keepalive_reply) \ |
| 414 | |
| 415 | /* |
| 416 | * memory_api_init |
| 417 | */ |
| 418 | int |
| 419 | vl_mem_api_init (const char *region_name) |
| 420 | { |
| 421 | int rv; |
| 422 | api_main_t *am = &api_main; |
| 423 | vl_msg_api_msg_config_t cfg; |
| 424 | vl_msg_api_msg_config_t *c = &cfg; |
| 425 | vl_shmem_hdr_t *shm; |
| 426 | vlib_main_t *vm = vlib_get_main (); |
| 427 | |
| 428 | memset (c, 0, sizeof (*c)); |
| 429 | |
| 430 | if ((rv = vl_map_shmem (region_name, 1 /* is_vlib */ )) < 0) |
| 431 | return rv; |
| 432 | |
| 433 | #define _(N,n) do { \ |
| 434 | c->id = VL_API_##N; \ |
| 435 | c->name = #n; \ |
| 436 | c->handler = vl_api_##n##_t_handler; \ |
| 437 | c->cleanup = vl_noop_handler; \ |
| 438 | c->endian = vl_api_##n##_t_endian; \ |
| 439 | c->print = vl_api_##n##_t_print; \ |
| 440 | c->size = sizeof(vl_api_##n##_t); \ |
| 441 | c->traced = 1; /* trace, so these msgs print */ \ |
| 442 | c->replay = 0; /* don't replay client create/delete msgs */ \ |
| 443 | c->message_bounce = 0; /* don't bounce this message */ \ |
| 444 | vl_msg_api_config(c);} while (0); |
| 445 | |
| 446 | foreach_vlib_api_msg; |
| 447 | #undef _ |
| 448 | |
| 449 | /* |
| 450 | * special-case freeing of memclnt_delete messages, so we can |
| 451 | * simply munmap pairwise / private API segments... |
| 452 | */ |
| 453 | am->message_bounce[VL_API_MEMCLNT_DELETE] = 1; |
| 454 | am->is_mp_safe[VL_API_MEMCLNT_KEEPALIVE_REPLY] = 1; |
| 455 | |
| 456 | vlib_set_queue_signal_callback (vm, memclnt_queue_callback); |
| 457 | |
| 458 | shm = am->shmem_hdr; |
| 459 | ASSERT (shm && shm->vl_input_queue); |
| 460 | |
| 461 | /* Make a note so we can always find the primary region easily */ |
| 462 | am->vlib_primary_rp = am->vlib_rp; |
| 463 | |
| 464 | return 0; |
| 465 | } |
| 466 | |
Dave Barach | 1f80658 | 2018-06-14 09:18:21 -0400 | [diff] [blame] | 467 | clib_error_t * |
Dave Barach | 048a4e5 | 2018-06-01 18:52:25 -0400 | [diff] [blame] | 468 | map_api_segment_init (vlib_main_t * vm) |
| 469 | { |
| 470 | api_main_t *am = &api_main; |
| 471 | int rv; |
| 472 | |
| 473 | if ((rv = vl_mem_api_init (am->region_name)) < 0) |
| 474 | { |
| 475 | return clib_error_return (0, "vl_mem_api_init (%s) failed", |
| 476 | am->region_name); |
| 477 | } |
| 478 | return 0; |
| 479 | } |
| 480 | |
Florin Coras | e86a8ed | 2018-01-05 03:20:25 -0800 | [diff] [blame] | 481 | static void |
| 482 | send_memclnt_keepalive (vl_api_registration_t * regp, f64 now) |
| 483 | { |
| 484 | vl_api_memclnt_keepalive_t *mp; |
| 485 | svm_queue_t *q; |
| 486 | api_main_t *am = &api_main; |
| 487 | svm_region_t *save_vlib_rp = am->vlib_rp; |
| 488 | vl_shmem_hdr_t *save_shmem_hdr = am->shmem_hdr; |
| 489 | |
| 490 | q = regp->vl_input_queue; |
| 491 | |
| 492 | /* |
| 493 | * If the queue head is moving, assume that the client is processing |
| 494 | * messages and skip the ping. This heuristic may fail if the queue |
| 495 | * is in the same position as last time, net of wrapping; in which |
| 496 | * case, the client will receive a keepalive. |
| 497 | */ |
| 498 | if (regp->last_queue_head != q->head) |
| 499 | { |
| 500 | regp->last_heard = now; |
| 501 | regp->unanswered_pings = 0; |
| 502 | regp->last_queue_head = q->head; |
| 503 | return; |
| 504 | } |
| 505 | |
| 506 | /* |
| 507 | * push/pop shared memory segment, so this routine |
| 508 | * will work with "normal" as well as "private segment" |
| 509 | * memory clients.. |
| 510 | */ |
| 511 | |
| 512 | am->vlib_rp = regp->vlib_rp; |
| 513 | am->shmem_hdr = regp->shmem_hdr; |
| 514 | |
| 515 | mp = vl_msg_api_alloc (sizeof (*mp)); |
| 516 | memset (mp, 0, sizeof (*mp)); |
| 517 | mp->_vl_msg_id = clib_host_to_net_u16 (VL_API_MEMCLNT_KEEPALIVE); |
| 518 | mp->context = mp->client_index = |
| 519 | vl_msg_api_handle_from_index_and_epoch |
| 520 | (regp->vl_api_registration_pool_index, |
| 521 | am->shmem_hdr->application_restarts); |
| 522 | |
| 523 | regp->unanswered_pings++; |
| 524 | |
| 525 | /* Failure-to-send due to a stuffed queue is absolutely expected */ |
| 526 | if (svm_queue_add (q, (u8 *) & mp, 1 /* nowait */ )) |
| 527 | vl_msg_api_free (mp); |
| 528 | |
| 529 | am->vlib_rp = save_vlib_rp; |
| 530 | am->shmem_hdr = save_shmem_hdr; |
| 531 | } |
| 532 | |
Florin Coras | b384b54 | 2018-01-15 01:08:33 -0800 | [diff] [blame] | 533 | static void |
| 534 | vl_mem_send_client_keepalive_w_reg (api_main_t * am, f64 now, |
| 535 | vl_api_registration_t ** regpp, |
| 536 | u32 ** dead_indices, |
| 537 | u32 ** confused_indices) |
| 538 | { |
| 539 | vl_api_registration_t *regp = *regpp; |
| 540 | if (regp) |
| 541 | { |
| 542 | /* If we haven't heard from this client recently... */ |
| 543 | if (regp->last_heard < (now - 10.0)) |
| 544 | { |
| 545 | if (regp->unanswered_pings == 2) |
| 546 | { |
| 547 | svm_queue_t *q; |
| 548 | q = regp->vl_input_queue; |
| 549 | if (kill (q->consumer_pid, 0) >= 0) |
| 550 | { |
| 551 | clib_warning ("REAPER: lazy binary API client '%s'", |
| 552 | regp->name); |
| 553 | regp->unanswered_pings = 0; |
| 554 | regp->last_heard = now; |
| 555 | } |
| 556 | else |
| 557 | { |
| 558 | clib_warning ("REAPER: binary API client '%s' died", |
| 559 | regp->name); |
| 560 | vec_add1 (*dead_indices, regpp - am->vl_clients); |
| 561 | } |
| 562 | } |
| 563 | else |
| 564 | send_memclnt_keepalive (regp, now); |
| 565 | } |
| 566 | else |
| 567 | regp->unanswered_pings = 0; |
| 568 | } |
| 569 | else |
| 570 | { |
| 571 | clib_warning ("NULL client registration index %d", |
| 572 | regpp - am->vl_clients); |
| 573 | vec_add1 (*confused_indices, regpp - am->vl_clients); |
| 574 | } |
| 575 | } |
| 576 | |
Florin Coras | e86a8ed | 2018-01-05 03:20:25 -0800 | [diff] [blame] | 577 | void |
| 578 | vl_mem_api_dead_client_scan (api_main_t * am, vl_shmem_hdr_t * shm, f64 now) |
| 579 | { |
| 580 | vl_api_registration_t **regpp; |
Florin Coras | e86a8ed | 2018-01-05 03:20:25 -0800 | [diff] [blame] | 581 | static u32 *dead_indices; |
| 582 | static u32 *confused_indices; |
| 583 | |
| 584 | vec_reset_length (dead_indices); |
| 585 | vec_reset_length (confused_indices); |
| 586 | |
| 587 | /* *INDENT-OFF* */ |
Florin Coras | b384b54 | 2018-01-15 01:08:33 -0800 | [diff] [blame] | 588 | pool_foreach (regpp, am->vl_clients, ({ |
| 589 | vl_mem_send_client_keepalive_w_reg (am, now, regpp, &dead_indices, |
| 590 | &confused_indices); |
Florin Coras | e86a8ed | 2018-01-05 03:20:25 -0800 | [diff] [blame] | 591 | })); |
| 592 | /* *INDENT-ON* */ |
Florin Coras | b384b54 | 2018-01-15 01:08:33 -0800 | [diff] [blame] | 593 | |
Florin Coras | e86a8ed | 2018-01-05 03:20:25 -0800 | [diff] [blame] | 594 | /* This should "never happen," but if it does, fix it... */ |
| 595 | if (PREDICT_FALSE (vec_len (confused_indices) > 0)) |
| 596 | { |
| 597 | int i; |
| 598 | for (i = 0; i < vec_len (confused_indices); i++) |
| 599 | { |
| 600 | pool_put_index (am->vl_clients, confused_indices[i]); |
| 601 | } |
| 602 | } |
| 603 | |
| 604 | if (PREDICT_FALSE (vec_len (dead_indices) > 0)) |
| 605 | { |
| 606 | int i; |
| 607 | svm_region_t *svm; |
| 608 | void *oldheap; |
| 609 | |
| 610 | /* Allow the application to clean up its registrations */ |
| 611 | for (i = 0; i < vec_len (dead_indices); i++) |
| 612 | { |
| 613 | regpp = pool_elt_at_index (am->vl_clients, dead_indices[i]); |
| 614 | if (regpp) |
| 615 | { |
| 616 | u32 handle; |
| 617 | |
| 618 | handle = vl_msg_api_handle_from_index_and_epoch |
| 619 | (dead_indices[i], shm->application_restarts); |
| 620 | (void) vl_api_call_reaper_functions (handle); |
| 621 | } |
| 622 | } |
| 623 | |
| 624 | svm = am->vlib_rp; |
| 625 | pthread_mutex_lock (&svm->mutex); |
| 626 | oldheap = svm_push_data_heap (svm); |
| 627 | |
| 628 | for (i = 0; i < vec_len (dead_indices); i++) |
| 629 | { |
| 630 | regpp = pool_elt_at_index (am->vl_clients, dead_indices[i]); |
| 631 | if (regpp) |
| 632 | { |
| 633 | /* Is this a pairwise SVM segment? */ |
| 634 | if ((*regpp)->vlib_rp != svm) |
| 635 | { |
| 636 | int i; |
| 637 | svm_region_t *dead_rp = (*regpp)->vlib_rp; |
| 638 | /* Note: account for the memfd header page */ |
| 639 | u64 virtual_base = dead_rp->virtual_base - MMAP_PAGESIZE; |
| 640 | u64 virtual_size = dead_rp->virtual_size + MMAP_PAGESIZE; |
| 641 | |
| 642 | /* For horizontal scaling, add a hash table... */ |
| 643 | for (i = 0; i < vec_len (am->vlib_private_rps); i++) |
| 644 | if (am->vlib_private_rps[i] == dead_rp) |
| 645 | { |
| 646 | vec_delete (am->vlib_private_rps, 1, i); |
| 647 | goto found; |
| 648 | } |
| 649 | clib_warning ("private rp %llx AWOL", dead_rp); |
| 650 | |
| 651 | found: |
| 652 | /* Kill it, accounting for the memfd header page */ |
| 653 | if (munmap ((void *) virtual_base, virtual_size) < 0) |
| 654 | clib_unix_warning ("munmap"); |
| 655 | /* Reset the queue-length-address cache */ |
| 656 | vec_reset_length (vl_api_queue_cursizes); |
| 657 | } |
| 658 | else |
| 659 | { |
| 660 | /* Poison the old registration */ |
| 661 | memset (*regpp, 0xF3, sizeof (**regpp)); |
| 662 | clib_mem_free (*regpp); |
| 663 | } |
| 664 | /* no dangling references, please */ |
| 665 | *regpp = 0; |
| 666 | } |
| 667 | else |
| 668 | { |
| 669 | svm_pop_heap (oldheap); |
| 670 | clib_warning ("Duplicate free, client index %d", |
| 671 | regpp - am->vl_clients); |
| 672 | oldheap = svm_push_data_heap (svm); |
| 673 | } |
| 674 | } |
| 675 | |
| 676 | svm_client_scan_this_region_nolock (am->vlib_rp); |
| 677 | |
| 678 | pthread_mutex_unlock (&svm->mutex); |
| 679 | svm_pop_heap (oldheap); |
| 680 | for (i = 0; i < vec_len (dead_indices); i++) |
| 681 | pool_put_index (am->vl_clients, dead_indices[i]); |
| 682 | } |
| 683 | } |
| 684 | |
| 685 | static inline int |
| 686 | void_mem_api_handle_msg_i (api_main_t * am, vlib_main_t * vm, |
| 687 | vlib_node_runtime_t * node, svm_queue_t * q) |
| 688 | { |
| 689 | uword mp; |
| 690 | if (!svm_queue_sub2 (q, (u8 *) & mp)) |
| 691 | { |
| 692 | vl_msg_api_handler_with_vm_node (am, (void *) mp, vm, node); |
| 693 | return 0; |
| 694 | } |
| 695 | return -1; |
| 696 | } |
| 697 | |
| 698 | int |
| 699 | vl_mem_api_handle_msg_main (vlib_main_t * vm, vlib_node_runtime_t * node) |
| 700 | { |
| 701 | api_main_t *am = &api_main; |
| 702 | return void_mem_api_handle_msg_i (am, vm, node, |
| 703 | am->shmem_hdr->vl_input_queue); |
| 704 | } |
| 705 | |
| 706 | int |
| 707 | vl_mem_api_handle_msg_private (vlib_main_t * vm, vlib_node_runtime_t * node, |
| 708 | u32 reg_index) |
| 709 | { |
| 710 | api_main_t *am = &api_main; |
| 711 | vl_shmem_hdr_t *save_shmem_hdr = am->shmem_hdr; |
| 712 | svm_region_t *vlib_rp, *save_vlib_rp = am->vlib_rp; |
| 713 | svm_queue_t *q; |
| 714 | int rv; |
| 715 | |
| 716 | vlib_rp = am->vlib_rp = am->vlib_private_rps[reg_index]; |
| 717 | |
| 718 | am->shmem_hdr = (void *) vlib_rp->user_ctx; |
| 719 | q = am->shmem_hdr->vl_input_queue; |
| 720 | |
| 721 | rv = void_mem_api_handle_msg_i (am, vm, node, q); |
| 722 | |
| 723 | am->shmem_hdr = save_shmem_hdr; |
| 724 | am->vlib_rp = save_vlib_rp; |
| 725 | |
| 726 | return rv; |
| 727 | } |
| 728 | |
| 729 | vl_api_registration_t * |
| 730 | vl_mem_api_client_index_to_registration (u32 handle) |
| 731 | { |
| 732 | vl_api_registration_t **regpp; |
| 733 | vl_api_registration_t *regp; |
| 734 | api_main_t *am = &api_main; |
Florin Coras | b384b54 | 2018-01-15 01:08:33 -0800 | [diff] [blame] | 735 | vl_shmem_hdr_t *shmem_hdr; |
Florin Coras | e86a8ed | 2018-01-05 03:20:25 -0800 | [diff] [blame] | 736 | u32 index; |
| 737 | |
| 738 | index = vl_msg_api_handle_get_index (handle); |
Florin Coras | e86a8ed | 2018-01-05 03:20:25 -0800 | [diff] [blame] | 739 | regpp = am->vl_clients + index; |
| 740 | |
| 741 | if (pool_is_free (am->vl_clients, regpp)) |
| 742 | { |
| 743 | vl_msg_api_increment_missing_client_counter (); |
| 744 | return 0; |
| 745 | } |
| 746 | regp = *regpp; |
Florin Coras | b384b54 | 2018-01-15 01:08:33 -0800 | [diff] [blame] | 747 | |
| 748 | shmem_hdr = (vl_shmem_hdr_t *) regp->shmem_hdr; |
| 749 | if (!vl_msg_api_handle_is_valid (handle, shmem_hdr->application_restarts)) |
| 750 | { |
| 751 | vl_msg_api_increment_missing_client_counter (); |
| 752 | return 0; |
| 753 | } |
| 754 | |
Florin Coras | e86a8ed | 2018-01-05 03:20:25 -0800 | [diff] [blame] | 755 | return (regp); |
| 756 | } |
| 757 | |
| 758 | svm_queue_t * |
| 759 | vl_api_client_index_to_input_queue (u32 index) |
| 760 | { |
| 761 | vl_api_registration_t *regp; |
| 762 | api_main_t *am = &api_main; |
| 763 | |
| 764 | /* Special case: vlib trying to send itself a message */ |
| 765 | if (index == (u32) ~ 0) |
| 766 | return (am->shmem_hdr->vl_input_queue); |
| 767 | |
| 768 | regp = vl_mem_api_client_index_to_registration (index); |
| 769 | if (!regp) |
| 770 | return 0; |
| 771 | return (regp->vl_input_queue); |
| 772 | } |
| 773 | |
| 774 | static clib_error_t * |
| 775 | setup_memclnt_exit (vlib_main_t * vm) |
| 776 | { |
| 777 | atexit (vl_unmap_shmem); |
| 778 | return 0; |
| 779 | } |
| 780 | |
| 781 | VLIB_INIT_FUNCTION (setup_memclnt_exit); |
| 782 | |
| 783 | u8 * |
| 784 | format_api_message_rings (u8 * s, va_list * args) |
| 785 | { |
| 786 | api_main_t *am = va_arg (*args, api_main_t *); |
| 787 | vl_shmem_hdr_t *shmem_hdr = va_arg (*args, vl_shmem_hdr_t *); |
| 788 | int main_segment = va_arg (*args, int); |
| 789 | ring_alloc_t *ap; |
| 790 | int i; |
| 791 | |
| 792 | if (shmem_hdr == 0) |
| 793 | return format (s, "%8s %8s %8s %8s %8s\n", |
| 794 | "Owner", "Size", "Nitems", "Hits", "Misses"); |
| 795 | |
| 796 | ap = shmem_hdr->vl_rings; |
| 797 | |
| 798 | for (i = 0; i < vec_len (shmem_hdr->vl_rings); i++) |
| 799 | { |
| 800 | s = format (s, "%8s %8d %8d %8d %8d\n", |
| 801 | "vlib", ap->size, ap->nitems, ap->hits, ap->misses); |
| 802 | ap++; |
| 803 | } |
| 804 | |
| 805 | ap = shmem_hdr->client_rings; |
| 806 | |
| 807 | for (i = 0; i < vec_len (shmem_hdr->client_rings); i++) |
| 808 | { |
| 809 | s = format (s, "%8s %8d %8d %8d %8d\n", |
| 810 | "clnt", ap->size, ap->nitems, ap->hits, ap->misses); |
| 811 | ap++; |
| 812 | } |
| 813 | |
| 814 | if (main_segment) |
| 815 | { |
| 816 | s = format (s, "%d ring miss fallback allocations\n", am->ring_misses); |
| 817 | s = format |
| 818 | (s, |
| 819 | "%d application restarts, %d reclaimed msgs, %d garbage collects\n", |
| 820 | shmem_hdr->application_restarts, shmem_hdr->restart_reclaims, |
| 821 | shmem_hdr->garbage_collects); |
| 822 | } |
| 823 | return s; |
| 824 | } |
| 825 | |
| 826 | static clib_error_t * |
| 827 | vl_api_ring_command (vlib_main_t * vm, |
| 828 | unformat_input_t * input, vlib_cli_command_t * cli_cmd) |
| 829 | { |
| 830 | int i; |
| 831 | vl_shmem_hdr_t *shmem_hdr; |
| 832 | api_main_t *am = &api_main; |
| 833 | |
| 834 | /* First, dump the primary region rings.. */ |
| 835 | |
| 836 | if (am->vlib_primary_rp == 0 || am->vlib_primary_rp->user_ctx == 0) |
| 837 | { |
| 838 | vlib_cli_output (vm, "Shared memory segment not initialized...\n"); |
| 839 | return 0; |
| 840 | } |
| 841 | |
| 842 | shmem_hdr = (void *) am->vlib_primary_rp->user_ctx; |
| 843 | |
| 844 | vlib_cli_output (vm, "Main API segment rings:"); |
| 845 | |
| 846 | vlib_cli_output (vm, "%U", format_api_message_rings, am, |
| 847 | 0 /* print header */ , 0 /* notused */ ); |
| 848 | |
| 849 | vlib_cli_output (vm, "%U", format_api_message_rings, am, |
| 850 | shmem_hdr, 1 /* main segment */ ); |
| 851 | |
| 852 | for (i = 0; i < vec_len (am->vlib_private_rps); i++) |
| 853 | { |
| 854 | svm_region_t *vlib_rp = am->vlib_private_rps[i]; |
| 855 | shmem_hdr = (void *) vlib_rp->user_ctx; |
| 856 | vl_api_registration_t **regpp; |
| 857 | vl_api_registration_t *regp = 0; |
| 858 | |
| 859 | /* For horizontal scaling, add a hash table... */ |
| 860 | /* *INDENT-OFF* */ |
| 861 | pool_foreach (regpp, am->vl_clients, |
| 862 | ({ |
| 863 | regp = *regpp; |
| 864 | if (regp && regp->vlib_rp == vlib_rp) |
| 865 | { |
| 866 | vlib_cli_output (vm, "%s segment rings:", regp->name); |
| 867 | goto found; |
| 868 | } |
| 869 | })); |
| 870 | vlib_cli_output (vm, "regp %llx not found?", regp); |
| 871 | continue; |
| 872 | /* *INDENT-ON* */ |
| 873 | found: |
| 874 | vlib_cli_output (vm, "%U", format_api_message_rings, am, |
| 875 | 0 /* print header */ , 0 /* notused */ ); |
| 876 | vlib_cli_output (vm, "%U", format_api_message_rings, am, |
| 877 | shmem_hdr, 0 /* main segment */ ); |
| 878 | } |
| 879 | |
| 880 | return 0; |
| 881 | } |
| 882 | |
| 883 | /*? |
| 884 | * Display binary api message allocation ring statistics |
| 885 | ?*/ |
| 886 | /* *INDENT-OFF* */ |
| 887 | VLIB_CLI_COMMAND (cli_show_api_ring_command, static) = |
| 888 | { |
| 889 | .path = "show api ring-stats", |
| 890 | .short_help = "Message ring statistics", |
| 891 | .function = vl_api_ring_command, |
| 892 | }; |
| 893 | /* *INDENT-ON* */ |
| 894 | |
| 895 | clib_error_t * |
| 896 | vlibmemory_init (vlib_main_t * vm) |
| 897 | { |
| 898 | api_main_t *am = &api_main; |
| 899 | svm_map_region_args_t _a, *a = &_a; |
| 900 | clib_error_t *error; |
| 901 | |
| 902 | memset (a, 0, sizeof (*a)); |
| 903 | a->root_path = am->root_path; |
| 904 | a->name = SVM_GLOBAL_REGION_NAME; |
| 905 | a->baseva = (am->global_baseva != 0) ? |
Damjan Marion | aec8f89 | 2018-01-08 16:35:35 +0100 | [diff] [blame] | 906 | am->global_baseva : +svm_get_global_region_base_va (); |
Florin Coras | e86a8ed | 2018-01-05 03:20:25 -0800 | [diff] [blame] | 907 | a->size = (am->global_size != 0) ? am->global_size : SVM_GLOBAL_REGION_SIZE; |
| 908 | a->flags = SVM_FLAGS_NODATA; |
| 909 | a->uid = am->api_uid; |
| 910 | a->gid = am->api_gid; |
| 911 | a->pvt_heap_size = |
| 912 | (am->global_pvt_heap_size != |
| 913 | 0) ? am->global_pvt_heap_size : SVM_PVT_MHEAP_SIZE; |
| 914 | |
| 915 | svm_region_init_args (a); |
| 916 | |
| 917 | error = vlib_call_init_function (vm, vlibsocket_init); |
| 918 | |
| 919 | return error; |
| 920 | } |
| 921 | |
Florin Coras | e86a8ed | 2018-01-05 03:20:25 -0800 | [diff] [blame] | 922 | void |
| 923 | vl_set_memory_region_name (const char *name) |
| 924 | { |
| 925 | api_main_t *am = &api_main; |
| 926 | am->region_name = name; |
| 927 | } |
| 928 | |
| 929 | /* |
| 930 | * fd.io coding-style-patch-verification: ON |
| 931 | * |
| 932 | * Local Variables: |
| 933 | * eval: (c-set-style "gnu") |
| 934 | * End: |
| 935 | */ |