blob: 55391912b89b0d8c8019a6e0b736fd74407232bf [file] [log] [blame]
Florin Coras7baeb712019-01-04 17:05:43 -08001/*
2 * Copyright (c) 2019 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this
5 * You may obtain a copy of the License at:
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16#include <vcl/vcl_locked.h>
17#include <vcl/vcl_private.h>
18
19typedef struct vcl_locked_session_
20{
Florin Corasf9240dc2019-01-15 08:03:17 -080021 clib_spinlock_t lock;
Florin Coras7baeb712019-01-04 17:05:43 -080022 u32 session_index;
23 u32 worker_index;
24 u32 vls_index;
Florin Corasf9240dc2019-01-15 08:03:17 -080025 u32 *workers_subscribed;
Florin Coras2d675d72019-01-28 15:54:27 -080026 clib_bitmap_t *listeners;
Florin Coras7baeb712019-01-04 17:05:43 -080027} vcl_locked_session_t;
28
Florin Coras2d675d72019-01-28 15:54:27 -080029typedef struct vls_local_
30{
31 int vls_wrk_index;
32 volatile int vls_mt_n_threads;
33 pthread_mutex_t vls_mt_mq_mlock;
34 pthread_mutex_t vls_mt_spool_mlock;
35 volatile u8 select_mp_check;
36 volatile u8 epoll_mp_check;
37} vls_process_local_t;
38
39static vls_process_local_t vls_local;
40static vls_process_local_t *vlsl = &vls_local;
41
42typedef struct vls_main_
Florin Coras7baeb712019-01-04 17:05:43 -080043{
44 vcl_locked_session_t *vls_pool;
45 clib_rwlock_t vls_table_lock;
46 uword *session_index_to_vlsh_table;
47} vls_main_t;
48
Florin Coras2d675d72019-01-28 15:54:27 -080049vls_main_t *vlsm;
Florin Coras7baeb712019-01-04 17:05:43 -080050
51static inline void
52vls_table_rlock (void)
53{
54 clib_rwlock_reader_lock (&vlsm->vls_table_lock);
55}
56
57static inline void
58vls_table_runlock (void)
59{
60 clib_rwlock_reader_unlock (&vlsm->vls_table_lock);
61}
62
63static inline void
64vls_table_wlock (void)
65{
66 clib_rwlock_writer_lock (&vlsm->vls_table_lock);
67}
68
69static inline void
70vls_table_wunlock (void)
71{
72 clib_rwlock_writer_unlock (&vlsm->vls_table_lock);
73}
74
Florin Coras0ef8ef22019-01-18 08:37:13 -080075typedef enum
76{
77 VLS_MT_OP_READ,
78 VLS_MT_OP_WRITE,
79 VLS_MT_OP_SPOOL,
80 VLS_MT_OP_XPOLL,
81} vls_mt_ops_t;
82
83typedef enum
84{
85 VLS_MT_LOCK_MQ = 1 << 0,
86 VLS_MT_LOCK_SPOOL = 1 << 1
87} vls_mt_lock_type_t;
88
Florin Coras0ef8ef22019-01-18 08:37:13 -080089static void
90vls_mt_add (void)
91{
Florin Coras2d675d72019-01-28 15:54:27 -080092 vlsl->vls_mt_n_threads += 1;
93 vcl_set_worker_index (vlsl->vls_wrk_index);
Florin Coras0ef8ef22019-01-18 08:37:13 -080094}
95
96static inline void
97vls_mt_mq_lock (void)
98{
Florin Coras2d675d72019-01-28 15:54:27 -080099 pthread_mutex_lock (&vlsl->vls_mt_mq_mlock);
Florin Coras0ef8ef22019-01-18 08:37:13 -0800100}
101
102static inline void
103vls_mt_mq_unlock (void)
104{
Florin Coras2d675d72019-01-28 15:54:27 -0800105 pthread_mutex_unlock (&vlsl->vls_mt_mq_mlock);
Florin Coras0ef8ef22019-01-18 08:37:13 -0800106}
107
108static inline void
109vls_mt_spool_lock (void)
110{
Florin Coras2d675d72019-01-28 15:54:27 -0800111 pthread_mutex_lock (&vlsl->vls_mt_spool_mlock);
Florin Coras0ef8ef22019-01-18 08:37:13 -0800112}
113
114static inline void
115vls_mt_create_unlock (void)
116{
Florin Coras2d675d72019-01-28 15:54:27 -0800117 pthread_mutex_unlock (&vlsl->vls_mt_spool_mlock);
118}
119
120static void
121vls_mt_locks_init (void)
122{
123 pthread_mutex_init (&vlsl->vls_mt_mq_mlock, NULL);
124 pthread_mutex_init (&vlsl->vls_mt_spool_mlock, NULL);
Florin Coras0ef8ef22019-01-18 08:37:13 -0800125}
126
Florin Coras7baeb712019-01-04 17:05:43 -0800127static inline vcl_session_handle_t
128vls_to_sh (vcl_locked_session_t * vls)
129{
Florin Corasf9240dc2019-01-15 08:03:17 -0800130 return vcl_session_handle_from_index (vls->session_index);
Florin Coras7baeb712019-01-04 17:05:43 -0800131}
132
133static inline vcl_session_handle_t
134vls_to_sh_tu (vcl_locked_session_t * vls)
135{
136 vcl_session_handle_t sh;
137 sh = vls_to_sh (vls);
138 vls_table_runlock ();
139 return sh;
140}
141
142static vls_handle_t
143vls_alloc (vcl_session_handle_t sh)
144{
145 vcl_locked_session_t *vls;
146
147 vls_table_wlock ();
148 pool_get (vlsm->vls_pool, vls);
149 vls->session_index = vppcom_session_index (sh);
150 vls->worker_index = vppcom_session_worker (sh);
151 vls->vls_index = vls - vlsm->vls_pool;
152 hash_set (vlsm->session_index_to_vlsh_table, vls->session_index,
153 vls->vls_index);
154 clib_spinlock_init (&vls->lock);
155 vls_table_wunlock ();
156 return vls->vls_index;
157}
158
159static vcl_locked_session_t *
160vls_get (vls_handle_t vlsh)
161{
162 if (pool_is_free_index (vlsm->vls_pool, vlsh))
163 return 0;
164 return pool_elt_at_index (vlsm->vls_pool, vlsh);
165}
166
167static void
Florin Coras0ef8ef22019-01-18 08:37:13 -0800168vls_free (vcl_locked_session_t * vls)
Florin Coras7baeb712019-01-04 17:05:43 -0800169{
Florin Coras0ef8ef22019-01-18 08:37:13 -0800170 ASSERT (vls != 0);
171 hash_unset (vlsm->session_index_to_vlsh_table, vls->session_index);
172 clib_spinlock_free (&vls->lock);
173 pool_put (vlsm->vls_pool, vls);
Florin Coras7baeb712019-01-04 17:05:43 -0800174}
175
176static vcl_locked_session_t *
177vls_get_and_lock (vls_handle_t vlsh)
178{
179 vcl_locked_session_t *vls;
180 if (pool_is_free_index (vlsm->vls_pool, vlsh))
181 return 0;
182 vls = pool_elt_at_index (vlsm->vls_pool, vlsh);
183 clib_spinlock_lock (&vls->lock);
184 return vls;
185}
186
187static vcl_locked_session_t *
188vls_get_w_dlock (vls_handle_t vlsh)
189{
190 vcl_locked_session_t *vls;
191 vls_table_rlock ();
192 vls = vls_get_and_lock (vlsh);
193 if (!vls)
194 vls_table_runlock ();
195 return vls;
196}
197
198static inline void
Florin Coras2d675d72019-01-28 15:54:27 -0800199vls_lock (vcl_locked_session_t * vls)
200{
201 clib_spinlock_lock (&vls->lock);
202}
203
204static inline void
Florin Coras7baeb712019-01-04 17:05:43 -0800205vls_unlock (vcl_locked_session_t * vls)
206{
207 clib_spinlock_unlock (&vls->lock);
208}
209
210static inline void
211vls_get_and_unlock (vls_handle_t vlsh)
212{
213 vcl_locked_session_t *vls;
214 vls_table_rlock ();
215 vls = vls_get (vlsh);
216 vls_unlock (vls);
217 vls_table_runlock ();
218}
219
220static inline void
221vls_dunlock (vcl_locked_session_t * vls)
222{
223 vls_unlock (vls);
224 vls_table_runlock ();
225}
226
Florin Coras2d675d72019-01-28 15:54:27 -0800227vcl_session_handle_t
228vlsh_to_sh (vls_handle_t vlsh)
229{
230 vcl_locked_session_t *vls;
231 int rv;
232
233 vls = vls_get_w_dlock (vlsh);
234 if (!vls)
235 return INVALID_SESSION_ID;
236 rv = vls_to_sh (vls);
237 vls_dunlock (vls);
238 return rv;
239}
240
241vcl_session_handle_t
242vlsh_to_session_index (vls_handle_t vlsh)
243{
244 vcl_session_handle_t sh;
245 sh = vlsh_to_sh (vlsh);
246 return vppcom_session_index (sh);
247}
248
249vls_handle_t
250vls_si_to_vlsh (u32 session_index)
251{
252 uword *vlshp;
253 vlshp = hash_get (vlsm->session_index_to_vlsh_table, session_index);
254 return vlshp ? *vlshp : VLS_INVALID_HANDLE;
255}
256
257vls_handle_t
258vls_session_index_to_vlsh (uint32_t session_index)
259{
260 vls_handle_t vlsh;
261
262 vls_table_rlock ();
263 vlsh = vls_si_to_vlsh (session_index);
264 vls_table_runlock ();
265
266 return vlsh;
267}
268
Florin Corasf9240dc2019-01-15 08:03:17 -0800269u8
270vls_is_shared (vcl_locked_session_t * vls)
271{
272 return vec_len (vls->workers_subscribed);
273}
274
Florin Coras0ef8ef22019-01-18 08:37:13 -0800275u8
276vls_is_shared_by_wrk (vcl_locked_session_t * vls, u32 wrk_index)
Florin Corasf9240dc2019-01-15 08:03:17 -0800277{
Florin Coras0ef8ef22019-01-18 08:37:13 -0800278 int i;
279 for (i = 0; i < vec_len (vls->workers_subscribed); i++)
280 if (vls->workers_subscribed[i] == wrk_index)
281 return 1;
282 return 0;
283}
284
Florin Coras2d675d72019-01-28 15:54:27 -0800285static void
286vls_listener_wrk_set (vcl_locked_session_t * vls, u32 wrk_index, u8 is_active)
287{
288 clib_bitmap_set (vls->listeners, wrk_index, is_active);
289}
290
291static u8
292vls_listener_wrk_is_active (vcl_locked_session_t * vls, u32 wrk_index)
293{
294 return (clib_bitmap_get (vls->listeners, wrk_index) == 1);
295}
296
297static void
298vls_listener_wrk_start_listen (vcl_locked_session_t * vls, u32 wrk_index)
299{
300 vppcom_session_listen (vls_to_sh (vls), ~0);
301 vls_listener_wrk_set (vls, wrk_index, 1 /* is_active */ );
302}
303
304static void
305vls_listener_wrk_stop_listen (vcl_locked_session_t * vls, u32 wrk_index)
306{
307 vcl_worker_t *wrk;
308 vcl_session_t *s;
309
310 wrk = vcl_worker_get (wrk_index);
311 s = vcl_session_get (wrk, vls->session_index);
312 if (s->session_state != STATE_LISTEN)
313 return;
314 vppcom_send_unbind_sock (wrk, s->vpp_handle);
315 s->session_state = STATE_LISTEN_NO_MQ;
316 vls_listener_wrk_set (vls, wrk_index, 0 /* is_active */ );
317}
318
Florin Coras0ef8ef22019-01-18 08:37:13 -0800319int
320vls_unshare_session (vcl_locked_session_t * vls, vcl_worker_t * wrk)
321{
Florin Coras2d675d72019-01-28 15:54:27 -0800322 int i, do_disconnect;
Florin Corasf9240dc2019-01-15 08:03:17 -0800323 vcl_session_t *s;
Florin Coras2d675d72019-01-28 15:54:27 -0800324
325 s = vcl_session_get (wrk, vls->session_index);
326 if (s->session_state == STATE_LISTEN)
327 vls_listener_wrk_set (vls, wrk->wrk_index, 0 /* is_active */ );
Florin Corasf9240dc2019-01-15 08:03:17 -0800328
329 for (i = 0; i < vec_len (vls->workers_subscribed); i++)
330 {
331 if (vls->workers_subscribed[i] != wrk->wrk_index)
332 continue;
333
Florin Corasf9240dc2019-01-15 08:03:17 -0800334 if (s->rx_fifo)
335 {
336 svm_fifo_del_subscriber (s->rx_fifo, wrk->vpp_wrk_index);
337 svm_fifo_del_subscriber (s->tx_fifo, wrk->vpp_wrk_index);
338 }
339 vec_del1 (vls->workers_subscribed, i);
Florin Coras2d675d72019-01-28 15:54:27 -0800340 do_disconnect = s->session_state == STATE_LISTEN;
341 vcl_session_cleanup (wrk, s, vcl_session_handle (s), do_disconnect);
Florin Corasf9240dc2019-01-15 08:03:17 -0800342 return 0;
343 }
344
Florin Coras0ef8ef22019-01-18 08:37:13 -0800345 /* Return, if this is not the owning worker */
346 if (vls->worker_index != wrk->wrk_index)
347 return 0;
348
Florin Coras0ef8ef22019-01-18 08:37:13 -0800349 /* Check if we can change owner or close */
350 if (vec_len (vls->workers_subscribed))
Florin Corasf9240dc2019-01-15 08:03:17 -0800351 {
Florin Corasf9240dc2019-01-15 08:03:17 -0800352 vls->worker_index = vls->workers_subscribed[0];
353 vec_del1 (vls->workers_subscribed, 0);
354 vcl_send_session_worker_update (wrk, s, vls->worker_index);
355 if (vec_len (vls->workers_subscribed))
356 clib_warning ("more workers need to be updated");
357 }
Florin Coras0ef8ef22019-01-18 08:37:13 -0800358 else
359 {
360 vcl_session_cleanup (wrk, s, vcl_session_handle (s),
361 1 /* do_disconnect */ );
362 }
Florin Corasf9240dc2019-01-15 08:03:17 -0800363
364 return 0;
365}
366
367void
368vls_share_vcl_session (vcl_worker_t * wrk, vcl_session_t * s)
369{
370 vcl_locked_session_t *vls;
371
Florin Coras2d675d72019-01-28 15:54:27 -0800372 vls = vls_get (vls_si_to_vlsh (s->session_index));
Florin Corasf9240dc2019-01-15 08:03:17 -0800373 if (!vls)
374 return;
Florin Coras2d675d72019-01-28 15:54:27 -0800375 vls_lock (vls);
Florin Corasf9240dc2019-01-15 08:03:17 -0800376 vec_add1 (vls->workers_subscribed, wrk->wrk_index);
377 if (s->rx_fifo)
378 {
379 svm_fifo_add_subscriber (s->rx_fifo, wrk->vpp_wrk_index);
380 svm_fifo_add_subscriber (s->tx_fifo, wrk->vpp_wrk_index);
381 }
Florin Coras2d675d72019-01-28 15:54:27 -0800382 else if (s->session_state == STATE_LISTEN)
383 {
384 s->session_state = STATE_LISTEN_NO_MQ;
385 }
386
387 vls_unlock (vls);
Florin Corasf9240dc2019-01-15 08:03:17 -0800388}
389
390void
391vls_worker_copy_on_fork (vcl_worker_t * parent_wrk)
392{
393 vcl_worker_t *wrk = vcl_worker_get_current ();
394 vcl_session_t *s;
395
396 wrk->vpp_event_queues = vec_dup (parent_wrk->vpp_event_queues);
397 wrk->sessions = pool_dup (parent_wrk->sessions);
398 wrk->session_index_by_vpp_handles =
399 hash_dup (parent_wrk->session_index_by_vpp_handles);
Florin Coras2d675d72019-01-28 15:54:27 -0800400 vls_table_wlock ();
Florin Corasf9240dc2019-01-15 08:03:17 -0800401
402 /* *INDENT-OFF* */
403 pool_foreach (s, wrk->sessions, ({
404 vls_share_vcl_session (wrk, s);
405 }));
406 /* *INDENT-ON* */
Florin Coras2d675d72019-01-28 15:54:27 -0800407
408 vls_table_wunlock ();
Florin Corasf9240dc2019-01-15 08:03:17 -0800409}
410
Florin Coras0ef8ef22019-01-18 08:37:13 -0800411static void
412vls_mt_acq_locks (vcl_locked_session_t * vls, vls_mt_ops_t op, int *locks_acq)
413{
414 vcl_worker_t *wrk = vcl_worker_get_current ();
415 vcl_session_t *s = 0;
416 int is_nonblk = 0;
417
418 if (vls)
419 {
420 s = vcl_session_get (wrk, vls->session_index);
421 if (PREDICT_FALSE (!s))
422 return;
423 is_nonblk = VCL_SESS_ATTR_TEST (s->attr, VCL_SESS_ATTR_NONBLOCK);
424 }
425
426 switch (op)
427 {
428 case VLS_MT_OP_READ:
429 if (!is_nonblk)
430 is_nonblk = vcl_session_read_ready (s) != 0;
431 if (!is_nonblk)
432 {
433 vls_mt_mq_lock ();
434 *locks_acq |= VLS_MT_LOCK_MQ;
435 }
436 break;
437 case VLS_MT_OP_WRITE:
Florin Coras78b5fa62019-02-21 20:04:15 -0800438 ASSERT (s);
Florin Coras0ef8ef22019-01-18 08:37:13 -0800439 if (!is_nonblk)
440 is_nonblk = vcl_session_write_ready (s) != 0;
441 if (!is_nonblk)
442 {
443 vls_mt_mq_lock ();
444 *locks_acq |= VLS_MT_LOCK_MQ;
445 }
446 break;
447 case VLS_MT_OP_XPOLL:
448 vls_mt_mq_lock ();
449 *locks_acq |= VLS_MT_LOCK_MQ;
450 break;
451 case VLS_MT_OP_SPOOL:
452 vls_mt_spool_lock ();
453 *locks_acq |= VLS_MT_LOCK_SPOOL;
454 break;
455 default:
456 break;
457 }
458}
459
460static void
461vls_mt_rel_locks (int locks_acq)
462{
463 if (locks_acq & VLS_MT_LOCK_MQ)
464 vls_mt_mq_unlock ();
465 if (locks_acq & VLS_MT_LOCK_SPOOL)
466 vls_mt_create_unlock ();
467}
468
469#define vls_mt_guard(_vls, _op) \
470 int _locks_acq = 0; \
Florin Coras63d3ac62019-03-29 08:29:25 -0700471 if (PREDICT_FALSE (vcl_get_worker_index () == ~0)) \
Florin Coras0ef8ef22019-01-18 08:37:13 -0800472 vls_mt_add (); \
Florin Coras2d675d72019-01-28 15:54:27 -0800473 if (PREDICT_FALSE (vlsl->vls_mt_n_threads > 1)) \
Florin Coras0ef8ef22019-01-18 08:37:13 -0800474 vls_mt_acq_locks (_vls, _op, &_locks_acq); \
475
476#define vls_mt_unguard() \
477 if (PREDICT_FALSE (_locks_acq)) \
478 vls_mt_rel_locks (_locks_acq)
479
Florin Coras7baeb712019-01-04 17:05:43 -0800480int
481vls_write (vls_handle_t vlsh, void *buf, size_t nbytes)
482{
483 vcl_locked_session_t *vls;
484 int rv;
485
486 if (!(vls = vls_get_w_dlock (vlsh)))
487 return VPPCOM_EBADFD;
Florin Coras0ef8ef22019-01-18 08:37:13 -0800488
489 vls_mt_guard (vls, VLS_MT_OP_WRITE);
Florin Coras7baeb712019-01-04 17:05:43 -0800490 rv = vppcom_session_write (vls_to_sh_tu (vls), buf, nbytes);
Florin Coras0ef8ef22019-01-18 08:37:13 -0800491 vls_mt_unguard ();
Florin Coras7baeb712019-01-04 17:05:43 -0800492 vls_get_and_unlock (vlsh);
493 return rv;
494}
495
496int
497vls_write_msg (vls_handle_t vlsh, void *buf, size_t nbytes)
498{
499 vcl_locked_session_t *vls;
500 int rv;
501
502 if (!(vls = vls_get_w_dlock (vlsh)))
503 return VPPCOM_EBADFD;
Florin Coras0ef8ef22019-01-18 08:37:13 -0800504 vls_mt_guard (vls, VLS_MT_OP_WRITE);
Florin Coras7baeb712019-01-04 17:05:43 -0800505 rv = vppcom_session_write_msg (vls_to_sh_tu (vls), buf, nbytes);
Florin Coras0ef8ef22019-01-18 08:37:13 -0800506 vls_mt_unguard ();
Florin Coras7baeb712019-01-04 17:05:43 -0800507 vls_get_and_unlock (vlsh);
508 return rv;
509}
510
511int
512vls_sendto (vls_handle_t vlsh, void *buf, int buflen, int flags,
513 vppcom_endpt_t * ep)
514{
515 vcl_locked_session_t *vls;
516 int rv;
517
518 if (!(vls = vls_get_w_dlock (vlsh)))
519 return VPPCOM_EBADFD;
Florin Coras0ef8ef22019-01-18 08:37:13 -0800520 vls_mt_guard (vls, VLS_MT_OP_WRITE);
Florin Coras7baeb712019-01-04 17:05:43 -0800521 rv = vppcom_session_sendto (vls_to_sh_tu (vls), buf, buflen, flags, ep);
Florin Coras0ef8ef22019-01-18 08:37:13 -0800522 vls_mt_unguard ();
Florin Coras7baeb712019-01-04 17:05:43 -0800523 vls_get_and_unlock (vlsh);
524 return rv;
525}
526
527ssize_t
528vls_read (vls_handle_t vlsh, void *buf, size_t nbytes)
529{
530 vcl_locked_session_t *vls;
531 int rv;
532
533 if (!(vls = vls_get_w_dlock (vlsh)))
534 return VPPCOM_EBADFD;
Florin Coras0ef8ef22019-01-18 08:37:13 -0800535 vls_mt_guard (vls, VLS_MT_OP_READ);
Florin Coras7baeb712019-01-04 17:05:43 -0800536 rv = vppcom_session_read (vls_to_sh_tu (vls), buf, nbytes);
Florin Coras0ef8ef22019-01-18 08:37:13 -0800537 vls_mt_unguard ();
Florin Coras7baeb712019-01-04 17:05:43 -0800538 vls_get_and_unlock (vlsh);
539 return rv;
540}
541
542ssize_t
543vls_recvfrom (vls_handle_t vlsh, void *buffer, uint32_t buflen, int flags,
544 vppcom_endpt_t * ep)
545{
546 vcl_locked_session_t *vls;
547 int rv;
548
549 if (!(vls = vls_get_w_dlock (vlsh)))
550 return VPPCOM_EBADFD;
Florin Coras0ef8ef22019-01-18 08:37:13 -0800551 vls_mt_guard (vls, VLS_MT_OP_READ);
Florin Coras7baeb712019-01-04 17:05:43 -0800552 rv = vppcom_session_recvfrom (vls_to_sh_tu (vls), buffer, buflen, flags,
553 ep);
Florin Coras0ef8ef22019-01-18 08:37:13 -0800554 vls_mt_unguard ();
Florin Coras7baeb712019-01-04 17:05:43 -0800555 vls_get_and_unlock (vlsh);
556 return rv;
557}
558
559int
560vls_attr (vls_handle_t vlsh, uint32_t op, void *buffer, uint32_t * buflen)
561{
562 vcl_locked_session_t *vls;
563 int rv;
564
Florin Coras63d3ac62019-03-29 08:29:25 -0700565 if (PREDICT_FALSE (vcl_get_worker_index () == ~0))
566 vls_mt_add ();
567
Florin Coras7baeb712019-01-04 17:05:43 -0800568 if (!(vls = vls_get_w_dlock (vlsh)))
569 return VPPCOM_EBADFD;
570 rv = vppcom_session_attr (vls_to_sh_tu (vls), op, buffer, buflen);
571 vls_get_and_unlock (vlsh);
572 return rv;
573}
574
575int
576vls_bind (vls_handle_t vlsh, vppcom_endpt_t * ep)
577{
578 vcl_locked_session_t *vls;
579 int rv;
580
581 if (!(vls = vls_get_w_dlock (vlsh)))
582 return VPPCOM_EBADFD;
583 rv = vppcom_session_bind (vls_to_sh_tu (vls), ep);
584 vls_get_and_unlock (vlsh);
585 return rv;
586}
587
588int
589vls_listen (vls_handle_t vlsh, int q_len)
590{
591 vcl_locked_session_t *vls;
592 int rv;
593
594 if (!(vls = vls_get_w_dlock (vlsh)))
595 return VPPCOM_EBADFD;
Florin Coras0ef8ef22019-01-18 08:37:13 -0800596 vls_mt_guard (vls, VLS_MT_OP_XPOLL);
Florin Coras7baeb712019-01-04 17:05:43 -0800597 rv = vppcom_session_listen (vls_to_sh_tu (vls), q_len);
Florin Coras0ef8ef22019-01-18 08:37:13 -0800598 vls_mt_unguard ();
Florin Coras7baeb712019-01-04 17:05:43 -0800599 vls_get_and_unlock (vlsh);
600 return rv;
601}
602
603int
604vls_connect (vls_handle_t vlsh, vppcom_endpt_t * server_ep)
605{
606 vcl_locked_session_t *vls;
607 int rv;
608
609 if (!(vls = vls_get_w_dlock (vlsh)))
610 return VPPCOM_EBADFD;
Florin Coras0ef8ef22019-01-18 08:37:13 -0800611 vls_mt_guard (vls, VLS_MT_OP_XPOLL);
Florin Coras7baeb712019-01-04 17:05:43 -0800612 rv = vppcom_session_connect (vls_to_sh_tu (vls), server_ep);
Florin Coras0ef8ef22019-01-18 08:37:13 -0800613 vls_mt_unguard ();
Florin Coras7baeb712019-01-04 17:05:43 -0800614 vls_get_and_unlock (vlsh);
615 return rv;
616}
617
Florin Coras2d675d72019-01-28 15:54:27 -0800618static inline void
619vls_mp_checks (vcl_locked_session_t * vls, int is_add)
620{
621 vcl_worker_t *wrk = vcl_worker_get_current ();
622 vcl_session_t *s;
623
624 s = vcl_session_get (wrk, vls->session_index);
625 switch (s->session_state)
626 {
627 case STATE_LISTEN:
628 if (is_add)
629 {
630 if (vls->worker_index == wrk->wrk_index)
631 vls_listener_wrk_set (vls, wrk->wrk_index, 1 /* is_active */ );
632 break;
633 }
634 vls_listener_wrk_stop_listen (vls, vls->worker_index);
635 break;
636 case STATE_LISTEN_NO_MQ:
637 if (!is_add)
638 break;
639
640 /* Register worker as listener */
641 vls_listener_wrk_start_listen (vls, wrk->wrk_index);
642
643 /* If owner worker did not attempt to accept/xpoll on the session,
644 * force a listen stop for it, since it may not be interested in
645 * accepting new sessions.
646 * This is pretty much a hack done to give app workers the illusion
647 * that it is fine to listen and not accept new sessions for a
648 * given listener. Without it, we would accumulate unhandled
649 * accepts on the passive worker message queue. */
650 if (!vls_listener_wrk_is_active (vls, vls->worker_index))
651 vls_listener_wrk_stop_listen (vls, vls->worker_index);
652 break;
653 default:
654 break;
655 }
656}
657
Florin Coras7baeb712019-01-04 17:05:43 -0800658vls_handle_t
659vls_accept (vls_handle_t listener_vlsh, vppcom_endpt_t * ep, int flags)
660{
661 vls_handle_t accepted_vlsh;
662 vcl_locked_session_t *vls;
663 int sh;
664
665 if (!(vls = vls_get_w_dlock (listener_vlsh)))
666 return VPPCOM_EBADFD;
Florin Coras2d675d72019-01-28 15:54:27 -0800667 if (vcl_n_workers () > 1)
668 vls_mp_checks (vls, 1 /* is_add */ );
Florin Coras0ef8ef22019-01-18 08:37:13 -0800669 vls_mt_guard (vls, VLS_MT_OP_SPOOL);
Florin Coras7baeb712019-01-04 17:05:43 -0800670 sh = vppcom_session_accept (vls_to_sh_tu (vls), ep, flags);
Florin Coras0ef8ef22019-01-18 08:37:13 -0800671 vls_mt_unguard ();
Florin Coras7baeb712019-01-04 17:05:43 -0800672 vls_get_and_unlock (listener_vlsh);
673 if (sh < 0)
674 return sh;
675 accepted_vlsh = vls_alloc (sh);
676 if (PREDICT_FALSE (accepted_vlsh == VLS_INVALID_HANDLE))
677 vppcom_session_close (sh);
678 return accepted_vlsh;
679}
680
681vls_handle_t
682vls_create (uint8_t proto, uint8_t is_nonblocking)
683{
684 vcl_session_handle_t sh;
685 vls_handle_t vlsh;
686
Florin Coras0ef8ef22019-01-18 08:37:13 -0800687 vls_mt_guard (0, VLS_MT_OP_SPOOL);
Florin Coras7baeb712019-01-04 17:05:43 -0800688 sh = vppcom_session_create (proto, is_nonblocking);
Florin Coras0ef8ef22019-01-18 08:37:13 -0800689 vls_mt_unguard ();
Florin Coras7baeb712019-01-04 17:05:43 -0800690 if (sh == INVALID_SESSION_ID)
691 return VLS_INVALID_HANDLE;
692
693 vlsh = vls_alloc (sh);
694 if (PREDICT_FALSE (vlsh == VLS_INVALID_HANDLE))
695 vppcom_session_close (sh);
696
697 return vlsh;
698}
699
700int
701vls_close (vls_handle_t vlsh)
702{
703 vcl_locked_session_t *vls;
Florin Corasf9240dc2019-01-15 08:03:17 -0800704 int rv;
Florin Coras7baeb712019-01-04 17:05:43 -0800705
Florin Coras0ef8ef22019-01-18 08:37:13 -0800706 vls_table_wlock ();
Florin Coras7baeb712019-01-04 17:05:43 -0800707
Florin Coras0ef8ef22019-01-18 08:37:13 -0800708 vls = vls_get_and_lock (vlsh);
709 if (!vls)
710 {
711 vls_table_wunlock ();
712 return VPPCOM_EBADFD;
713 }
714
715 vls_mt_guard (0, VLS_MT_OP_SPOOL);
Florin Corasf9240dc2019-01-15 08:03:17 -0800716 if (vls_is_shared (vls))
717 {
Florin Coras0ef8ef22019-01-18 08:37:13 -0800718 /* At least two workers share the session so vls won't be freed */
719 vls_unshare_session (vls, vcl_worker_get_current ());
720 vls_unlock (vls);
721 vls_mt_unguard ();
722 vls_table_wunlock ();
Florin Corasf9240dc2019-01-15 08:03:17 -0800723 return VPPCOM_OK;
724 }
725
Florin Coras0ef8ef22019-01-18 08:37:13 -0800726 rv = vppcom_session_close (vls_to_sh (vls));
727 vls_free (vls);
728 vls_mt_unguard ();
Florin Coras7baeb712019-01-04 17:05:43 -0800729
Florin Coras0ef8ef22019-01-18 08:37:13 -0800730 vls_table_wunlock ();
731
Florin Coras7baeb712019-01-04 17:05:43 -0800732 return rv;
733}
734
735vls_handle_t
736vls_epoll_create (void)
737{
738 vcl_session_handle_t sh;
739 vls_handle_t vlsh;
740
Florin Coras63d3ac62019-03-29 08:29:25 -0700741 if (PREDICT_FALSE (vcl_get_worker_index () == ~0))
742 vls_mt_add ();
743
Florin Coras7baeb712019-01-04 17:05:43 -0800744 sh = vppcom_epoll_create ();
745 if (sh == INVALID_SESSION_ID)
746 return VLS_INVALID_HANDLE;
747
748 vlsh = vls_alloc (sh);
749 if (vlsh == VLS_INVALID_HANDLE)
750 vppcom_session_close (sh);
751
752 return vlsh;
753}
754
Florin Coras2d675d72019-01-28 15:54:27 -0800755static void
756vls_epoll_ctl_mp_checks (vcl_locked_session_t * vls, int op)
757{
758 if (vcl_n_workers () <= 1)
759 {
760 vlsl->epoll_mp_check = 1;
761 return;
762 }
763
764 if (op == EPOLL_CTL_MOD)
765 return;
766
767 vlsl->epoll_mp_check = 1;
768 vls_mp_checks (vls, op == EPOLL_CTL_ADD);
769}
770
Florin Coras7baeb712019-01-04 17:05:43 -0800771int
772vls_epoll_ctl (vls_handle_t ep_vlsh, int op, vls_handle_t vlsh,
773 struct epoll_event *event)
774{
775 vcl_locked_session_t *ep_vls, *vls;
776 vcl_session_handle_t ep_sh, sh;
777 int rv;
778
779 vls_table_rlock ();
780 ep_vls = vls_get_and_lock (ep_vlsh);
781 vls = vls_get_and_lock (vlsh);
782 ep_sh = vls_to_sh (ep_vls);
783 sh = vls_to_sh (vls);
Florin Coras2d675d72019-01-28 15:54:27 -0800784
785 if (PREDICT_FALSE (!vlsl->epoll_mp_check))
786 vls_epoll_ctl_mp_checks (vls, op);
787
Florin Coras7baeb712019-01-04 17:05:43 -0800788 vls_table_runlock ();
789
790 rv = vppcom_epoll_ctl (ep_sh, op, sh, event);
791
792 vls_table_rlock ();
793 ep_vls = vls_get (ep_vlsh);
794 vls = vls_get (vlsh);
795 vls_unlock (vls);
796 vls_unlock (ep_vls);
797 vls_table_runlock ();
798 return rv;
799}
800
801int
802vls_epoll_wait (vls_handle_t ep_vlsh, struct epoll_event *events,
803 int maxevents, double wait_for_time)
804{
805 vcl_locked_session_t *vls;
806 int rv;
807
808 if (!(vls = vls_get_w_dlock (ep_vlsh)))
809 return VPPCOM_EBADFD;
Florin Coras0ef8ef22019-01-18 08:37:13 -0800810 vls_mt_guard (0, VLS_MT_OP_XPOLL);
Florin Coras7baeb712019-01-04 17:05:43 -0800811 rv = vppcom_epoll_wait (vls_to_sh_tu (vls), events, maxevents,
812 wait_for_time);
Florin Coras0ef8ef22019-01-18 08:37:13 -0800813 vls_mt_unguard ();
Florin Coras7baeb712019-01-04 17:05:43 -0800814 vls_get_and_unlock (ep_vlsh);
815 return rv;
816}
817
Florin Coras2d675d72019-01-28 15:54:27 -0800818static void
819vls_select_mp_checks (vcl_si_set * read_map)
820{
821 vcl_locked_session_t *vls;
822 vcl_worker_t *wrk;
823 vcl_session_t *s;
824 u32 si;
825
826 if (vcl_n_workers () <= 1)
827 {
828 vlsl->select_mp_check = 1;
829 return;
830 }
831
832 if (!read_map)
833 return;
834
835 vlsl->select_mp_check = 1;
836 wrk = vcl_worker_get_current ();
837
838 /* *INDENT-OFF* */
839 clib_bitmap_foreach (si, read_map, ({
840 s = vcl_session_get (wrk, si);
841 if (s->session_state == STATE_LISTEN)
842 {
843 vls = vls_get (vls_session_index_to_vlsh (si));
844 vls_mp_checks (vls, 1 /* is_add */);
845 }
846 }));
847 /* *INDENT-ON* */
848}
849
Florin Coras0ef8ef22019-01-18 08:37:13 -0800850int
851vls_select (int n_bits, vcl_si_set * read_map, vcl_si_set * write_map,
852 vcl_si_set * except_map, double wait_for_time)
853{
854 int rv;
Florin Coras2d675d72019-01-28 15:54:27 -0800855
Florin Coras0ef8ef22019-01-18 08:37:13 -0800856 vls_mt_guard (0, VLS_MT_OP_XPOLL);
Florin Coras2d675d72019-01-28 15:54:27 -0800857 if (PREDICT_FALSE (!vlsl->select_mp_check))
858 vls_select_mp_checks (read_map);
Florin Coras0ef8ef22019-01-18 08:37:13 -0800859 rv = vppcom_select (n_bits, read_map, write_map, except_map, wait_for_time);
860 vls_mt_unguard ();
861 return rv;
862}
863
Florin Corasf9240dc2019-01-15 08:03:17 -0800864static void
Florin Coras0ef8ef22019-01-18 08:37:13 -0800865vls_unshare_vcl_worker_sessions (vcl_worker_t * wrk)
866{
867 u32 current_wrk, is_current;
868 vcl_locked_session_t *vls;
869 vcl_session_t *s;
870
Florin Coras14ed6df2019-03-06 21:13:42 -0800871 if (pool_elts (vcm->workers) <= 1)
872 return;
873
Florin Coras0ef8ef22019-01-18 08:37:13 -0800874 current_wrk = vcl_get_worker_index ();
875 is_current = current_wrk == wrk->wrk_index;
876 vls_table_wlock ();
877
878 /* *INDENT-OFF* */
879 pool_foreach (s, wrk->sessions, ({
880 vls = vls_get (vls_si_to_vlsh (s->session_index));
881 if (vls && (is_current || vls_is_shared_by_wrk (vls, current_wrk)))
882 vls_unshare_session (vls, wrk);
883 }));
884 /* *INDENT-ON* */
885
886 vls_table_wunlock ();
887}
888
889static void
890vls_cleanup_vcl_worker (vcl_worker_t * wrk)
891{
892 /* Unshare sessions and also cleanup worker since child may have
893 * called _exit () and therefore vcl may not catch the event */
894 vls_unshare_vcl_worker_sessions (wrk);
895 vcl_worker_cleanup (wrk, 1 /* notify vpp */ );
896}
897
898static void
Florin Corasf9240dc2019-01-15 08:03:17 -0800899vls_cleanup_forked_child (vcl_worker_t * wrk, vcl_worker_t * child_wrk)
900{
901 vcl_worker_t *sub_child;
902 int tries = 0;
903
904 if (child_wrk->forked_child != ~0)
905 {
906 sub_child = vcl_worker_get_if_valid (child_wrk->forked_child);
907 if (sub_child)
908 {
909 /* Wait a bit, maybe the process is going away */
910 while (kill (sub_child->current_pid, 0) >= 0 && tries++ < 50)
911 usleep (1e3);
912 if (kill (sub_child->current_pid, 0) < 0)
913 vls_cleanup_forked_child (child_wrk, sub_child);
914 }
915 }
Florin Coras0ef8ef22019-01-18 08:37:13 -0800916 vls_cleanup_vcl_worker (child_wrk);
917 VDBG (0, "Cleaned up forked child wrk %u", child_wrk->wrk_index);
Florin Corasf9240dc2019-01-15 08:03:17 -0800918 wrk->forked_child = ~0;
919}
920
921static struct sigaction old_sa;
922
923static void
924vls_intercept_sigchld_handler (int signum, siginfo_t * si, void *uc)
925{
926 vcl_worker_t *wrk, *child_wrk;
927
928 if (vcl_get_worker_index () == ~0)
929 return;
930
931 if (sigaction (SIGCHLD, &old_sa, 0))
932 {
933 VERR ("couldn't restore sigchld");
934 exit (-1);
935 }
936
937 wrk = vcl_worker_get_current ();
938 if (wrk->forked_child == ~0)
939 return;
940
941 child_wrk = vcl_worker_get_if_valid (wrk->forked_child);
942 if (!child_wrk)
943 goto done;
944
945 if (si && si->si_pid != child_wrk->current_pid)
946 {
947 VDBG (0, "unexpected child pid %u", si->si_pid);
948 goto done;
949 }
950 vls_cleanup_forked_child (wrk, child_wrk);
951
952done:
953 if (old_sa.sa_flags & SA_SIGINFO)
954 {
955 void (*fn) (int, siginfo_t *, void *) = old_sa.sa_sigaction;
956 fn (signum, si, uc);
957 }
958 else
959 {
960 void (*fn) (int) = old_sa.sa_handler;
961 if (fn)
962 fn (signum);
963 }
964}
965
966static void
967vls_incercept_sigchld ()
968{
969 struct sigaction sa;
970 clib_memset (&sa, 0, sizeof (sa));
971 sa.sa_sigaction = vls_intercept_sigchld_handler;
972 sa.sa_flags = SA_SIGINFO;
973 if (sigaction (SIGCHLD, &sa, &old_sa))
974 {
975 VERR ("couldn't intercept sigchld");
976 exit (-1);
977 }
978}
979
980static void
981vls_app_pre_fork (void)
982{
983 vls_incercept_sigchld ();
984 vcl_flush_mq_events ();
985}
986
987static void
988vls_app_fork_child_handler (void)
989{
990 vcl_worker_t *parent_wrk;
991 int rv, parent_wrk_index;
992 u8 *child_name;
993
994 parent_wrk_index = vcl_get_worker_index ();
995 VDBG (0, "initializing forked child %u with parent wrk %u", getpid (),
996 parent_wrk_index);
997
998 /*
999 * Allocate worker
1000 */
1001 vcl_set_worker_index (~0);
1002 if (!vcl_worker_alloc_and_init ())
1003 VERR ("couldn't allocate new worker");
1004
1005 /*
1006 * Attach to binary api
1007 */
1008 child_name = format (0, "%v-child-%u%c", vcm->app_name, getpid (), 0);
1009 vcl_cleanup_bapi ();
1010 vppcom_api_hookup ();
1011 vcm->app_state = STATE_APP_START;
1012 rv = vppcom_connect_to_vpp ((char *) child_name);
1013 vec_free (child_name);
1014 if (rv)
1015 {
1016 VERR ("couldn't connect to VPP!");
1017 return;
1018 }
1019
1020 /*
1021 * Register worker with vpp and share sessions
1022 */
1023 vcl_worker_register_with_vpp ();
1024 parent_wrk = vcl_worker_get (parent_wrk_index);
1025 vls_worker_copy_on_fork (parent_wrk);
1026 parent_wrk->forked_child = vcl_get_worker_index ();
1027
Florin Coras0ef8ef22019-01-18 08:37:13 -08001028 /* Reset number of threads and set wrk index */
Florin Coras2d675d72019-01-28 15:54:27 -08001029 vlsl->vls_mt_n_threads = 0;
1030 vlsl->vls_wrk_index = vcl_get_worker_index ();
1031 vlsl->select_mp_check = 0;
1032 vlsl->epoll_mp_check = 0;
1033 vls_mt_locks_init ();
Florin Coras0ef8ef22019-01-18 08:37:13 -08001034
Florin Corasf9240dc2019-01-15 08:03:17 -08001035 VDBG (0, "forked child main worker initialized");
1036 vcm->forking = 0;
1037}
1038
1039static void
1040vls_app_fork_parent_handler (void)
1041{
1042 vcm->forking = 1;
1043 while (vcm->forking)
1044 ;
1045}
1046
Florin Coras0ef8ef22019-01-18 08:37:13 -08001047void
1048vls_app_exit (void)
1049{
1050 /* Unshare the sessions. VCL will clean up the worker */
1051 vls_unshare_vcl_worker_sessions (vcl_worker_get_current ());
1052}
1053
Florin Coras7baeb712019-01-04 17:05:43 -08001054int
1055vls_app_create (char *app_name)
1056{
1057 int rv;
Florin Coras0ef8ef22019-01-18 08:37:13 -08001058
Florin Coras7baeb712019-01-04 17:05:43 -08001059 if ((rv = vppcom_app_create (app_name)))
1060 return rv;
Florin Coras2d675d72019-01-28 15:54:27 -08001061 vlsm = clib_mem_alloc (sizeof (vls_main_t));
1062 clib_memset (vlsm, 0, sizeof (*vlsm));
Florin Coras7baeb712019-01-04 17:05:43 -08001063 clib_rwlock_init (&vlsm->vls_table_lock);
Florin Corasf9240dc2019-01-15 08:03:17 -08001064 pthread_atfork (vls_app_pre_fork, vls_app_fork_parent_handler,
1065 vls_app_fork_child_handler);
Florin Coras0ef8ef22019-01-18 08:37:13 -08001066 atexit (vls_app_exit);
Florin Coras2d675d72019-01-28 15:54:27 -08001067 vlsl->vls_wrk_index = vcl_get_worker_index ();
1068 vls_mt_locks_init ();
Florin Coras7baeb712019-01-04 17:05:43 -08001069 return VPPCOM_OK;
1070}
1071
1072/*
1073 * fd.io coding-style-patch-verification: ON
1074 *
1075 * Local Variables:
1076 * eval: (c-set-style "gnu")
1077 * End:
1078 */