blob: e8f56c6c5fceea1a9b1af4e8edb3489d26f51b47 [file] [log] [blame]
Florin Coras7baeb712019-01-04 17:05:43 -08001/*
2 * Copyright (c) 2019 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this
5 * You may obtain a copy of the License at:
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16#include <vcl/vcl_locked.h>
17#include <vcl/vcl_private.h>
18
Florin Coras243edd52020-03-04 22:20:12 +000019typedef struct vls_shared_data_
20{
21 clib_spinlock_t lock;
22 u32 owner_wrk_index;
23 u32 *workers_subscribed;
24 clib_bitmap_t *listeners;
25} vls_shared_data_t;
26
Florin Coras7baeb712019-01-04 17:05:43 -080027typedef struct vcl_locked_session_
28{
Florin Corasf9240dc2019-01-15 08:03:17 -080029 clib_spinlock_t lock;
Florin Coras7baeb712019-01-04 17:05:43 -080030 u32 session_index;
31 u32 worker_index;
32 u32 vls_index;
Florin Coras243edd52020-03-04 22:20:12 +000033 u32 shared_data_index;
hanlina3a48962020-07-13 11:09:15 +080034 /** VCL session owned by different workers because of migration */
35 u32 owner_vcl_wrk_index;
36 uword *vcl_wrk_index_to_session_index;
Florin Coras7baeb712019-01-04 17:05:43 -080037} vcl_locked_session_t;
38
Florin Coras243edd52020-03-04 22:20:12 +000039typedef struct vls_worker_
40{
wanghanline8f848a2021-01-08 14:57:11 +080041 clib_rwlock_t sh_to_vlsh_table_lock; /** valid for multithread workers */
Florin Coras243edd52020-03-04 22:20:12 +000042 vcl_locked_session_t *vls_pool;
hanlinf8e13632020-08-21 11:05:36 +080043 uword *session_handle_to_vlsh_table;
Florin Coras243edd52020-03-04 22:20:12 +000044 u32 wrk_index;
wanghanlind72a0342021-05-12 17:00:29 +080045 /** Vector of child wrk to cleanup */
46 u32 *pending_wrk_cleanup;
Florin Coras243edd52020-03-04 22:20:12 +000047} vls_worker_t;
48
Florin Coras2d675d72019-01-28 15:54:27 -080049typedef struct vls_local_
50{
51 int vls_wrk_index;
52 volatile int vls_mt_n_threads;
53 pthread_mutex_t vls_mt_mq_mlock;
54 pthread_mutex_t vls_mt_spool_mlock;
55 volatile u8 select_mp_check;
Florin Coras2d675d72019-01-28 15:54:27 -080056} vls_process_local_t;
57
58static vls_process_local_t vls_local;
59static vls_process_local_t *vlsl = &vls_local;
60
61typedef struct vls_main_
Florin Coras7baeb712019-01-04 17:05:43 -080062{
Florin Coras243edd52020-03-04 22:20:12 +000063 vls_worker_t *workers;
Florin Coras7baeb712019-01-04 17:05:43 -080064 clib_rwlock_t vls_table_lock;
Florin Coras243edd52020-03-04 22:20:12 +000065 /** Pool of data shared by sessions owned by different workers */
66 vls_shared_data_t *shared_data_pool;
67 clib_rwlock_t shared_data_lock;
wanghanlindcacdc42020-12-28 16:19:05 +080068 /** Lock to protect rpc among workers */
69 clib_spinlock_t worker_rpc_lock;
Florin Coras7baeb712019-01-04 17:05:43 -080070} vls_main_t;
71
Florin Coras2d675d72019-01-28 15:54:27 -080072vls_main_t *vlsm;
Florin Coras7baeb712019-01-04 17:05:43 -080073
wanghanlindcacdc42020-12-28 16:19:05 +080074typedef enum
75{
76 VLS_RPC_STATE_INIT,
77 VLS_RPC_STATE_SUCCESS,
78 VLS_RPC_STATE_SESSION_NOT_EXIST,
79} vls_rpc_state_e;
80
Florin Coras40c07ce2020-07-16 20:46:17 -070081typedef enum vls_rpc_msg_type_
82{
83 VLS_RPC_CLONE_AND_SHARE,
hanlina3a48962020-07-13 11:09:15 +080084 VLS_RPC_SESS_CLEANUP,
Florin Coras40c07ce2020-07-16 20:46:17 -070085} vls_rpc_msg_type_e;
86
87typedef struct vls_rpc_msg_
88{
89 u8 type;
90 u8 data[0];
91} vls_rpc_msg_t;
92
93typedef struct vls_clone_and_share_msg_
94{
95 u32 vls_index; /**< vls to be shared */
hanlina3a48962020-07-13 11:09:15 +080096 u32 session_index; /**< vcl session to be shared */
97 u32 origin_vls_wrk; /**< vls worker that initiated the rpc */
Florin Coras40c07ce2020-07-16 20:46:17 -070098 u32 origin_vls_index; /**< vls session of the originator */
hanlina3a48962020-07-13 11:09:15 +080099 u32 origin_vcl_wrk; /**< vcl worker that initiated the rpc */
100 u32 origin_session_index; /**< vcl session of the originator */
Florin Coras40c07ce2020-07-16 20:46:17 -0700101} vls_clone_and_share_msg_t;
102
hanlina3a48962020-07-13 11:09:15 +0800103typedef struct vls_sess_cleanup_msg_
104{
105 u32 session_index; /**< vcl session to be cleaned */
106 u32 origin_vcl_wrk; /**< worker that initiated the rpc */
107} vls_sess_cleanup_msg_t;
108
109void vls_send_session_cleanup_rpc (vcl_worker_t * wrk,
110 u32 dst_wrk_index, u32 dst_session_index);
wanghanlindcacdc42020-12-28 16:19:05 +0800111void vls_send_clone_and_share_rpc (vcl_worker_t *wrk, u32 origin_vls_index,
hanlina3a48962020-07-13 11:09:15 +0800112 u32 session_index, u32 vls_wrk_index,
113 u32 dst_wrk_index, u32 dst_vls_index,
114 u32 dst_session_index);
wanghanlind72a0342021-05-12 17:00:29 +0800115static void vls_cleanup_forked_child (vcl_worker_t *wrk,
116 vcl_worker_t *child_wrk);
117static void vls_handle_pending_wrk_cleanup (void);
hanlina3a48962020-07-13 11:09:15 +0800118
Florin Coras243edd52020-03-04 22:20:12 +0000119static inline u32
120vls_get_worker_index (void)
121{
hanlina3a48962020-07-13 11:09:15 +0800122 if (vls_mt_wrk_supported ())
123 return vlsl->vls_wrk_index;
124 else
125 return vcl_get_worker_index ();
Florin Coras243edd52020-03-04 22:20:12 +0000126}
127
128static u32
129vls_shared_data_alloc (void)
130{
131 vls_shared_data_t *vls_shd;
132 u32 shd_index;
133
134 clib_rwlock_writer_lock (&vlsm->shared_data_lock);
135 pool_get_zero (vlsm->shared_data_pool, vls_shd);
136 clib_spinlock_init (&vls_shd->lock);
137 shd_index = vls_shd - vlsm->shared_data_pool;
138 clib_rwlock_writer_unlock (&vlsm->shared_data_lock);
139
140 return shd_index;
141}
142
143static u32
144vls_shared_data_index (vls_shared_data_t * vls_shd)
145{
146 return vls_shd - vlsm->shared_data_pool;
147}
148
149vls_shared_data_t *
150vls_shared_data_get (u32 shd_index)
151{
152 if (pool_is_free_index (vlsm->shared_data_pool, shd_index))
153 return 0;
154 return pool_elt_at_index (vlsm->shared_data_pool, shd_index);
155}
156
157static void
158vls_shared_data_free (u32 shd_index)
159{
160 vls_shared_data_t *vls_shd;
161
162 clib_rwlock_writer_lock (&vlsm->shared_data_lock);
163 vls_shd = vls_shared_data_get (shd_index);
164 clib_spinlock_free (&vls_shd->lock);
165 clib_bitmap_free (vls_shd->listeners);
166 vec_free (vls_shd->workers_subscribed);
167 pool_put (vlsm->shared_data_pool, vls_shd);
168 clib_rwlock_writer_unlock (&vlsm->shared_data_lock);
169}
170
171static inline void
172vls_shared_data_pool_rlock (void)
173{
174 clib_rwlock_reader_lock (&vlsm->shared_data_lock);
175}
176
177static inline void
178vls_shared_data_pool_runlock (void)
179{
180 clib_rwlock_reader_unlock (&vlsm->shared_data_lock);
181}
182
Florin Coras7baeb712019-01-04 17:05:43 -0800183static inline void
Florin Corasff40d8f2020-08-11 22:05:28 -0700184vls_mt_table_rlock (void)
Florin Coras7baeb712019-01-04 17:05:43 -0800185{
Florin Coras243edd52020-03-04 22:20:12 +0000186 if (vlsl->vls_mt_n_threads > 1)
187 clib_rwlock_reader_lock (&vlsm->vls_table_lock);
Florin Coras7baeb712019-01-04 17:05:43 -0800188}
189
190static inline void
Florin Corasff40d8f2020-08-11 22:05:28 -0700191vls_mt_table_runlock (void)
Florin Coras7baeb712019-01-04 17:05:43 -0800192{
Florin Coras243edd52020-03-04 22:20:12 +0000193 if (vlsl->vls_mt_n_threads > 1)
194 clib_rwlock_reader_unlock (&vlsm->vls_table_lock);
Florin Coras7baeb712019-01-04 17:05:43 -0800195}
196
197static inline void
Florin Corasff40d8f2020-08-11 22:05:28 -0700198vls_mt_table_wlock (void)
Florin Coras7baeb712019-01-04 17:05:43 -0800199{
Florin Coras243edd52020-03-04 22:20:12 +0000200 if (vlsl->vls_mt_n_threads > 1)
201 clib_rwlock_writer_lock (&vlsm->vls_table_lock);
Florin Coras7baeb712019-01-04 17:05:43 -0800202}
203
204static inline void
Florin Corasff40d8f2020-08-11 22:05:28 -0700205vls_mt_table_wunlock (void)
Florin Coras7baeb712019-01-04 17:05:43 -0800206{
Florin Coras243edd52020-03-04 22:20:12 +0000207 if (vlsl->vls_mt_n_threads > 1)
208 clib_rwlock_writer_unlock (&vlsm->vls_table_lock);
Florin Coras7baeb712019-01-04 17:05:43 -0800209}
210
Florin Coras0ef8ef22019-01-18 08:37:13 -0800211typedef enum
212{
213 VLS_MT_OP_READ,
214 VLS_MT_OP_WRITE,
215 VLS_MT_OP_SPOOL,
216 VLS_MT_OP_XPOLL,
217} vls_mt_ops_t;
218
219typedef enum
220{
221 VLS_MT_LOCK_MQ = 1 << 0,
222 VLS_MT_LOCK_SPOOL = 1 << 1
223} vls_mt_lock_type_t;
224
Florin Coras0ef8ef22019-01-18 08:37:13 -0800225static void
226vls_mt_add (void)
227{
Florin Coras2d675d72019-01-28 15:54:27 -0800228 vlsl->vls_mt_n_threads += 1;
Florin Corasff40d8f2020-08-11 22:05:28 -0700229
230 /* If multi-thread workers are supported, for each new thread register a new
231 * vcl worker with vpp. Otherwise, all threads use the same vcl worker, so
232 * update the vcl worker's thread local worker index variable */
hanlina3a48962020-07-13 11:09:15 +0800233 if (vls_mt_wrk_supported ())
wanghanlin492350e2020-09-11 17:19:32 +0800234 {
235 if (vppcom_worker_register () != VPPCOM_OK)
236 VERR ("failed to register worker");
237 }
hanlina3a48962020-07-13 11:09:15 +0800238 else
239 vcl_set_worker_index (vlsl->vls_wrk_index);
Florin Coras0ef8ef22019-01-18 08:37:13 -0800240}
241
242static inline void
243vls_mt_mq_lock (void)
244{
Florin Coras2d675d72019-01-28 15:54:27 -0800245 pthread_mutex_lock (&vlsl->vls_mt_mq_mlock);
Florin Coras0ef8ef22019-01-18 08:37:13 -0800246}
247
248static inline void
249vls_mt_mq_unlock (void)
250{
Florin Coras2d675d72019-01-28 15:54:27 -0800251 pthread_mutex_unlock (&vlsl->vls_mt_mq_mlock);
Florin Coras0ef8ef22019-01-18 08:37:13 -0800252}
253
254static inline void
255vls_mt_spool_lock (void)
256{
Florin Coras2d675d72019-01-28 15:54:27 -0800257 pthread_mutex_lock (&vlsl->vls_mt_spool_mlock);
Florin Coras0ef8ef22019-01-18 08:37:13 -0800258}
259
260static inline void
261vls_mt_create_unlock (void)
262{
Florin Coras2d675d72019-01-28 15:54:27 -0800263 pthread_mutex_unlock (&vlsl->vls_mt_spool_mlock);
264}
265
266static void
267vls_mt_locks_init (void)
268{
269 pthread_mutex_init (&vlsl->vls_mt_mq_mlock, NULL);
270 pthread_mutex_init (&vlsl->vls_mt_spool_mlock, NULL);
Florin Coras0ef8ef22019-01-18 08:37:13 -0800271}
272
Florin Coras243edd52020-03-04 22:20:12 +0000273u8
274vls_is_shared (vcl_locked_session_t * vls)
275{
276 return (vls->shared_data_index != ~0);
277}
278
279static inline void
280vls_lock (vcl_locked_session_t * vls)
281{
282 if ((vlsl->vls_mt_n_threads > 1) || vls_is_shared (vls))
283 clib_spinlock_lock (&vls->lock);
284}
285
286static inline void
287vls_unlock (vcl_locked_session_t * vls)
288{
289 if ((vlsl->vls_mt_n_threads > 1) || vls_is_shared (vls))
290 clib_spinlock_unlock (&vls->lock);
291}
292
Florin Coras7baeb712019-01-04 17:05:43 -0800293static inline vcl_session_handle_t
294vls_to_sh (vcl_locked_session_t * vls)
295{
Florin Corasf9240dc2019-01-15 08:03:17 -0800296 return vcl_session_handle_from_index (vls->session_index);
Florin Coras7baeb712019-01-04 17:05:43 -0800297}
298
299static inline vcl_session_handle_t
300vls_to_sh_tu (vcl_locked_session_t * vls)
301{
302 vcl_session_handle_t sh;
303 sh = vls_to_sh (vls);
Florin Corasff40d8f2020-08-11 22:05:28 -0700304 vls_mt_table_runlock ();
Florin Coras7baeb712019-01-04 17:05:43 -0800305 return sh;
306}
307
Florin Coras243edd52020-03-04 22:20:12 +0000308static vls_worker_t *
309vls_worker_get_current (void)
310{
311 return pool_elt_at_index (vlsm->workers, vls_get_worker_index ());
312}
313
314static void
315vls_worker_alloc (void)
316{
317 vls_worker_t *wrk;
318
319 pool_get_zero (vlsm->workers, wrk);
wanghanline8f848a2021-01-08 14:57:11 +0800320 if (vls_mt_wrk_supported ())
321 clib_rwlock_init (&wrk->sh_to_vlsh_table_lock);
Florin Coras243edd52020-03-04 22:20:12 +0000322 wrk->wrk_index = vcl_get_worker_index ();
wanghanlind72a0342021-05-12 17:00:29 +0800323 vec_validate (wrk->pending_wrk_cleanup, 16);
324 vec_reset_length (wrk->pending_wrk_cleanup);
Florin Coras243edd52020-03-04 22:20:12 +0000325}
326
327static void
328vls_worker_free (vls_worker_t * wrk)
329{
hanlinf8e13632020-08-21 11:05:36 +0800330 hash_free (wrk->session_handle_to_vlsh_table);
wanghanline8f848a2021-01-08 14:57:11 +0800331 if (vls_mt_wrk_supported ())
332 clib_rwlock_free (&wrk->sh_to_vlsh_table_lock);
Florin Coras243edd52020-03-04 22:20:12 +0000333 pool_free (wrk->vls_pool);
334 pool_put (vlsm->workers, wrk);
335}
336
337static vls_worker_t *
338vls_worker_get (u32 wrk_index)
339{
340 if (pool_is_free_index (vlsm->workers, wrk_index))
341 return 0;
342 return pool_elt_at_index (vlsm->workers, wrk_index);
343}
344
wanghanline8f848a2021-01-08 14:57:11 +0800345static void
346vls_sh_to_vlsh_table_add (vls_worker_t *wrk, vcl_session_handle_t sh, u32 vlsh)
347{
348 if (vls_mt_wrk_supported ())
349 clib_rwlock_writer_lock (&wrk->sh_to_vlsh_table_lock);
350 hash_set (wrk->session_handle_to_vlsh_table, sh, vlsh);
351 if (vls_mt_wrk_supported ())
352 clib_rwlock_writer_unlock (&wrk->sh_to_vlsh_table_lock);
353}
354
355static void
356vls_sh_to_vlsh_table_del (vls_worker_t *wrk, vcl_session_handle_t sh)
357{
358 if (vls_mt_wrk_supported ())
359 clib_rwlock_writer_lock (&wrk->sh_to_vlsh_table_lock);
360 hash_unset (wrk->session_handle_to_vlsh_table, sh);
361 if (vls_mt_wrk_supported ())
362 clib_rwlock_writer_unlock (&wrk->sh_to_vlsh_table_lock);
363}
364
365static uword *
366vls_sh_to_vlsh_table_get (vls_worker_t *wrk, vcl_session_handle_t sh)
367{
368 if (vls_mt_wrk_supported ())
369 clib_rwlock_reader_lock (&wrk->sh_to_vlsh_table_lock);
370 uword *vlshp = hash_get (wrk->session_handle_to_vlsh_table, sh);
371 if (vls_mt_wrk_supported ())
372 clib_rwlock_reader_unlock (&wrk->sh_to_vlsh_table_lock);
373 return vlshp;
374}
375
Florin Coras7baeb712019-01-04 17:05:43 -0800376static vls_handle_t
377vls_alloc (vcl_session_handle_t sh)
378{
Florin Coras243edd52020-03-04 22:20:12 +0000379 vls_worker_t *wrk = vls_worker_get_current ();
Florin Coras7baeb712019-01-04 17:05:43 -0800380 vcl_locked_session_t *vls;
381
Florin Corasff40d8f2020-08-11 22:05:28 -0700382 vls_mt_table_wlock ();
Florin Coras243edd52020-03-04 22:20:12 +0000383
384 pool_get_zero (wrk->vls_pool, vls);
Florin Coras7baeb712019-01-04 17:05:43 -0800385 vls->session_index = vppcom_session_index (sh);
386 vls->worker_index = vppcom_session_worker (sh);
Florin Coras243edd52020-03-04 22:20:12 +0000387 vls->vls_index = vls - wrk->vls_pool;
388 vls->shared_data_index = ~0;
wanghanline8f848a2021-01-08 14:57:11 +0800389 vls_sh_to_vlsh_table_add (wrk, sh, vls->vls_index);
hanlina3a48962020-07-13 11:09:15 +0800390 if (vls_mt_wrk_supported ())
391 {
392 hash_set (vls->vcl_wrk_index_to_session_index, vls->worker_index,
393 vls->session_index);
394 vls->owner_vcl_wrk_index = vls->worker_index;
395 }
Florin Coras7baeb712019-01-04 17:05:43 -0800396 clib_spinlock_init (&vls->lock);
Florin Coras243edd52020-03-04 22:20:12 +0000397
Florin Corasff40d8f2020-08-11 22:05:28 -0700398 vls_mt_table_wunlock ();
Florin Coras7baeb712019-01-04 17:05:43 -0800399 return vls->vls_index;
400}
401
402static vcl_locked_session_t *
403vls_get (vls_handle_t vlsh)
404{
Florin Coras243edd52020-03-04 22:20:12 +0000405 vls_worker_t *wrk = vls_worker_get_current ();
406 if (pool_is_free_index (wrk->vls_pool, vlsh))
Florin Coras7baeb712019-01-04 17:05:43 -0800407 return 0;
Florin Coras243edd52020-03-04 22:20:12 +0000408 return pool_elt_at_index (wrk->vls_pool, vlsh);
Florin Coras7baeb712019-01-04 17:05:43 -0800409}
410
411static void
Florin Coras0ef8ef22019-01-18 08:37:13 -0800412vls_free (vcl_locked_session_t * vls)
Florin Coras7baeb712019-01-04 17:05:43 -0800413{
Florin Coras243edd52020-03-04 22:20:12 +0000414 vls_worker_t *wrk = vls_worker_get_current ();
415
Florin Coras0ef8ef22019-01-18 08:37:13 -0800416 ASSERT (vls != 0);
wanghanline8f848a2021-01-08 14:57:11 +0800417 vls_sh_to_vlsh_table_del (
418 wrk, vcl_session_handle_from_index (vls->session_index));
Florin Coras0ef8ef22019-01-18 08:37:13 -0800419 clib_spinlock_free (&vls->lock);
Florin Coras243edd52020-03-04 22:20:12 +0000420 pool_put (wrk->vls_pool, vls);
Florin Coras7baeb712019-01-04 17:05:43 -0800421}
422
423static vcl_locked_session_t *
424vls_get_and_lock (vls_handle_t vlsh)
425{
Florin Coras243edd52020-03-04 22:20:12 +0000426 vls_worker_t *wrk = vls_worker_get_current ();
Florin Coras7baeb712019-01-04 17:05:43 -0800427 vcl_locked_session_t *vls;
Florin Coras243edd52020-03-04 22:20:12 +0000428 if (pool_is_free_index (wrk->vls_pool, vlsh))
Florin Coras7baeb712019-01-04 17:05:43 -0800429 return 0;
Florin Coras243edd52020-03-04 22:20:12 +0000430 vls = pool_elt_at_index (wrk->vls_pool, vlsh);
431 vls_lock (vls);
Florin Coras7baeb712019-01-04 17:05:43 -0800432 return vls;
433}
434
435static vcl_locked_session_t *
436vls_get_w_dlock (vls_handle_t vlsh)
437{
438 vcl_locked_session_t *vls;
Florin Corasff40d8f2020-08-11 22:05:28 -0700439 vls_mt_table_rlock ();
Florin Coras7baeb712019-01-04 17:05:43 -0800440 vls = vls_get_and_lock (vlsh);
441 if (!vls)
Florin Corasff40d8f2020-08-11 22:05:28 -0700442 vls_mt_table_runlock ();
Florin Coras7baeb712019-01-04 17:05:43 -0800443 return vls;
444}
445
446static inline void
Florin Coras7baeb712019-01-04 17:05:43 -0800447vls_get_and_unlock (vls_handle_t vlsh)
448{
449 vcl_locked_session_t *vls;
Florin Corasff40d8f2020-08-11 22:05:28 -0700450 vls_mt_table_rlock ();
Florin Coras7baeb712019-01-04 17:05:43 -0800451 vls = vls_get (vlsh);
452 vls_unlock (vls);
Florin Corasff40d8f2020-08-11 22:05:28 -0700453 vls_mt_table_runlock ();
Florin Coras7baeb712019-01-04 17:05:43 -0800454}
455
456static inline void
457vls_dunlock (vcl_locked_session_t * vls)
458{
459 vls_unlock (vls);
Florin Corasff40d8f2020-08-11 22:05:28 -0700460 vls_mt_table_runlock ();
Florin Coras7baeb712019-01-04 17:05:43 -0800461}
462
Florin Coras243edd52020-03-04 22:20:12 +0000463static vcl_locked_session_t *
464vls_session_get (vls_worker_t * wrk, u32 vls_index)
465{
466 if (pool_is_free_index (wrk->vls_pool, vls_index))
467 return 0;
468 return pool_elt_at_index (wrk->vls_pool, vls_index);
469}
470
Florin Coras2d675d72019-01-28 15:54:27 -0800471vcl_session_handle_t
472vlsh_to_sh (vls_handle_t vlsh)
473{
474 vcl_locked_session_t *vls;
475 int rv;
476
477 vls = vls_get_w_dlock (vlsh);
478 if (!vls)
479 return INVALID_SESSION_ID;
480 rv = vls_to_sh (vls);
481 vls_dunlock (vls);
482 return rv;
483}
484
485vcl_session_handle_t
486vlsh_to_session_index (vls_handle_t vlsh)
487{
488 vcl_session_handle_t sh;
489 sh = vlsh_to_sh (vlsh);
490 return vppcom_session_index (sh);
491}
492
493vls_handle_t
hanlinf8e13632020-08-21 11:05:36 +0800494vls_si_wi_to_vlsh (u32 session_index, u32 vcl_wrk_index)
Florin Coras2d675d72019-01-28 15:54:27 -0800495{
Florin Coras243edd52020-03-04 22:20:12 +0000496 vls_worker_t *wrk = vls_worker_get_current ();
wanghanline8f848a2021-01-08 14:57:11 +0800497 uword *vlshp = vls_sh_to_vlsh_table_get (
498 wrk,
499 vcl_session_handle_from_wrk_session_index (session_index, vcl_wrk_index));
500
Florin Coras2d675d72019-01-28 15:54:27 -0800501 return vlshp ? *vlshp : VLS_INVALID_HANDLE;
502}
503
504vls_handle_t
505vls_session_index_to_vlsh (uint32_t session_index)
506{
507 vls_handle_t vlsh;
508
Florin Corasff40d8f2020-08-11 22:05:28 -0700509 vls_mt_table_rlock ();
hanlinf8e13632020-08-21 11:05:36 +0800510 vlsh = vls_si_wi_to_vlsh (session_index, vcl_get_worker_index ());
Florin Corasff40d8f2020-08-11 22:05:28 -0700511 vls_mt_table_runlock ();
Florin Coras2d675d72019-01-28 15:54:27 -0800512
513 return vlsh;
514}
515
Florin Corasf9240dc2019-01-15 08:03:17 -0800516u8
Florin Coras0ef8ef22019-01-18 08:37:13 -0800517vls_is_shared_by_wrk (vcl_locked_session_t * vls, u32 wrk_index)
Florin Corasf9240dc2019-01-15 08:03:17 -0800518{
Florin Coras243edd52020-03-04 22:20:12 +0000519 vls_shared_data_t *vls_shd;
Florin Coras0ef8ef22019-01-18 08:37:13 -0800520 int i;
Florin Coras243edd52020-03-04 22:20:12 +0000521
522 if (vls->shared_data_index == ~0)
523 return 0;
524
525 vls_shared_data_pool_rlock ();
526
527 vls_shd = vls_shared_data_get (vls->shared_data_index);
528 clib_spinlock_lock (&vls_shd->lock);
529
530 for (i = 0; i < vec_len (vls_shd->workers_subscribed); i++)
531 if (vls_shd->workers_subscribed[i] == wrk_index)
532 {
533 clib_spinlock_unlock (&vls_shd->lock);
534 vls_shared_data_pool_runlock ();
535 return 1;
536 }
537 clib_spinlock_unlock (&vls_shd->lock);
538
539 vls_shared_data_pool_runlock ();
Florin Coras0ef8ef22019-01-18 08:37:13 -0800540 return 0;
541}
542
Florin Coras2d675d72019-01-28 15:54:27 -0800543static void
544vls_listener_wrk_set (vcl_locked_session_t * vls, u32 wrk_index, u8 is_active)
545{
Florin Coras243edd52020-03-04 22:20:12 +0000546 vls_shared_data_t *vls_shd;
547
548 if (vls->shared_data_index == ~0)
549 {
550 clib_warning ("not a shared session");
551 return;
552 }
553
554 vls_shared_data_pool_rlock ();
555
556 vls_shd = vls_shared_data_get (vls->shared_data_index);
557
558 clib_spinlock_lock (&vls_shd->lock);
559 clib_bitmap_set (vls_shd->listeners, wrk_index, is_active);
560 clib_spinlock_unlock (&vls_shd->lock);
561
562 vls_shared_data_pool_runlock ();
563}
564
565static u32
566vls_shared_get_owner (vcl_locked_session_t * vls)
567{
568 vls_shared_data_t *vls_shd;
569 u32 owner_wrk;
570
571 vls_shared_data_pool_rlock ();
572
573 vls_shd = vls_shared_data_get (vls->shared_data_index);
574 owner_wrk = vls_shd->owner_wrk_index;
575
576 vls_shared_data_pool_runlock ();
577
578 return owner_wrk;
Florin Coras2d675d72019-01-28 15:54:27 -0800579}
580
581static u8
582vls_listener_wrk_is_active (vcl_locked_session_t * vls, u32 wrk_index)
583{
Florin Coras243edd52020-03-04 22:20:12 +0000584 vls_shared_data_t *vls_shd;
585 u8 is_set;
586
587 if (vls->shared_data_index == ~0)
588 {
589 clib_warning ("not a shared session");
590 return 0;
591 }
592
593 vls_shared_data_pool_rlock ();
594
595 vls_shd = vls_shared_data_get (vls->shared_data_index);
596
597 clib_spinlock_lock (&vls_shd->lock);
598 is_set = clib_bitmap_get (vls_shd->listeners, wrk_index);
599 clib_spinlock_unlock (&vls_shd->lock);
600
601 vls_shared_data_pool_runlock ();
602
603 return (is_set == 1);
Florin Coras2d675d72019-01-28 15:54:27 -0800604}
605
606static void
607vls_listener_wrk_start_listen (vcl_locked_session_t * vls, u32 wrk_index)
608{
609 vppcom_session_listen (vls_to_sh (vls), ~0);
610 vls_listener_wrk_set (vls, wrk_index, 1 /* is_active */ );
611}
612
613static void
614vls_listener_wrk_stop_listen (vcl_locked_session_t * vls, u32 wrk_index)
615{
616 vcl_worker_t *wrk;
617 vcl_session_t *s;
618
619 wrk = vcl_worker_get (wrk_index);
620 s = vcl_session_get (wrk, vls->session_index);
Florin Corasc127d5a2020-10-14 16:35:58 -0700621 if (s->session_state != VCL_STATE_LISTEN)
Florin Coras2d675d72019-01-28 15:54:27 -0800622 return;
Florin Coras458089b2019-08-21 16:20:44 -0700623 vcl_send_session_unlisten (wrk, s);
Florin Corasc127d5a2020-10-14 16:35:58 -0700624 s->session_state = VCL_STATE_LISTEN_NO_MQ;
Florin Coras2d675d72019-01-28 15:54:27 -0800625 vls_listener_wrk_set (vls, wrk_index, 0 /* is_active */ );
626}
627
Florin Coras243edd52020-03-04 22:20:12 +0000628static int
629vls_shared_data_subscriber_position (vls_shared_data_t * vls_shd,
630 u32 wrk_index)
631{
632 int i;
633
634 for (i = 0; i < vec_len (vls_shd->workers_subscribed); i++)
635 {
636 if (vls_shd->workers_subscribed[i] == wrk_index)
637 return i;
638 }
639 return -1;
640}
641
Florin Coras0ef8ef22019-01-18 08:37:13 -0800642int
643vls_unshare_session (vcl_locked_session_t * vls, vcl_worker_t * wrk)
644{
Florin Coras243edd52020-03-04 22:20:12 +0000645 vls_shared_data_t *vls_shd;
Florin Coras311817f2020-03-07 17:45:47 +0000646 int do_disconnect, pos;
647 u32 n_subscribers;
Florin Corasf9240dc2019-01-15 08:03:17 -0800648 vcl_session_t *s;
Florin Coras2d675d72019-01-28 15:54:27 -0800649
hanlina3a48962020-07-13 11:09:15 +0800650 if (vls->shared_data_index == ~0)
651 return 0;
Florin Coras243edd52020-03-04 22:20:12 +0000652
Florin Coras2d675d72019-01-28 15:54:27 -0800653 s = vcl_session_get (wrk, vls->session_index);
Florin Corasc127d5a2020-10-14 16:35:58 -0700654 if (s->session_state == VCL_STATE_LISTEN)
Florin Coras2d675d72019-01-28 15:54:27 -0800655 vls_listener_wrk_set (vls, wrk->wrk_index, 0 /* is_active */ );
Florin Corasf9240dc2019-01-15 08:03:17 -0800656
Florin Coras243edd52020-03-04 22:20:12 +0000657 vls_shared_data_pool_rlock ();
Florin Corasf9240dc2019-01-15 08:03:17 -0800658
Florin Coras243edd52020-03-04 22:20:12 +0000659 vls_shd = vls_shared_data_get (vls->shared_data_index);
660 clib_spinlock_lock (&vls_shd->lock);
661
662 pos = vls_shared_data_subscriber_position (vls_shd, wrk->wrk_index);
663 if (pos < 0)
664 {
665 clib_warning ("worker %u not subscribed for vls %u", wrk->wrk_index,
666 vls->worker_index);
667 goto done;
668 }
669
670 /*
671 * Unsubscribe from share data and fifos
672 */
673 if (s->rx_fifo)
674 {
675 svm_fifo_del_subscriber (s->rx_fifo, wrk->vpp_wrk_index);
676 svm_fifo_del_subscriber (s->tx_fifo, wrk->vpp_wrk_index);
677 }
678 vec_del1 (vls_shd->workers_subscribed, pos);
679
680 /*
681 * Cleanup vcl state
682 */
683 n_subscribers = vec_len (vls_shd->workers_subscribed);
Florin Corasc127d5a2020-10-14 16:35:58 -0700684 do_disconnect = s->session_state == VCL_STATE_LISTEN || !n_subscribers;
Florin Coras243edd52020-03-04 22:20:12 +0000685 vcl_session_cleanup (wrk, s, vcl_session_handle (s), do_disconnect);
686
687 /*
688 * No subscriber left, cleanup shared data
689 */
690 if (!n_subscribers)
691 {
692 u32 shd_index = vls_shared_data_index (vls_shd);
693
694 clib_spinlock_unlock (&vls_shd->lock);
695 vls_shared_data_pool_runlock ();
696
697 vls_shared_data_free (shd_index);
698
699 /* All locks have been dropped */
Florin Corasf9240dc2019-01-15 08:03:17 -0800700 return 0;
701 }
702
Florin Coras0ef8ef22019-01-18 08:37:13 -0800703 /* Return, if this is not the owning worker */
Florin Coras243edd52020-03-04 22:20:12 +0000704 if (vls_shd->owner_wrk_index != wrk->wrk_index)
705 goto done;
Florin Coras0ef8ef22019-01-18 08:37:13 -0800706
Florin Coras243edd52020-03-04 22:20:12 +0000707 ASSERT (vec_len (vls_shd->workers_subscribed));
708
709 /*
710 * Check if we can change owner or close
711 */
712 vls_shd->owner_wrk_index = vls_shd->workers_subscribed[0];
713 vcl_send_session_worker_update (wrk, s, vls_shd->owner_wrk_index);
714
715 /* XXX is this still needed? */
716 if (vec_len (vls_shd->workers_subscribed) > 1)
717 clib_warning ("more workers need to be updated");
718
719done:
720
721 clib_spinlock_unlock (&vls_shd->lock);
722 vls_shared_data_pool_runlock ();
Florin Corasf9240dc2019-01-15 08:03:17 -0800723
724 return 0;
725}
726
727void
Florin Coras40c07ce2020-07-16 20:46:17 -0700728vls_init_share_session (vls_worker_t * vls_wrk, vcl_locked_session_t * vls)
Florin Corasf9240dc2019-01-15 08:03:17 -0800729{
Florin Coras40c07ce2020-07-16 20:46:17 -0700730 vls_shared_data_t *vls_shd;
731
732 u32 vls_shd_index = vls_shared_data_alloc ();
733
734 vls_shared_data_pool_rlock ();
735
736 vls_shd = vls_shared_data_get (vls_shd_index);
737 vls_shd->owner_wrk_index = vls_wrk->wrk_index;
738 vls->shared_data_index = vls_shd_index;
739 vec_add1 (vls_shd->workers_subscribed, vls_wrk->wrk_index);
740
741 vls_shared_data_pool_runlock ();
742}
743
744void
745vls_share_session (vls_worker_t * vls_wrk, vcl_locked_session_t * vls)
746{
747 vcl_worker_t *vcl_wrk = vcl_worker_get (vls_wrk->wrk_index);
Florin Coras243edd52020-03-04 22:20:12 +0000748 vls_shared_data_t *vls_shd;
749 vcl_session_t *s;
Florin Corasf9240dc2019-01-15 08:03:17 -0800750
Florin Coras243edd52020-03-04 22:20:12 +0000751 s = vcl_session_get (vcl_wrk, vls->session_index);
752 if (!s)
753 {
Florin Coras40c07ce2020-07-16 20:46:17 -0700754 clib_warning ("wrk %u session %u vls %u NOT AVAILABLE",
755 vcl_wrk->wrk_index, vls->session_index, vls->vls_index);
Florin Coras243edd52020-03-04 22:20:12 +0000756 return;
757 }
758
Florin Coras40c07ce2020-07-16 20:46:17 -0700759 ASSERT (vls->shared_data_index != ~0);
760
Florin Coras243edd52020-03-04 22:20:12 +0000761 /* Reinit session lock */
762 clib_spinlock_init (&vls->lock);
763
Florin Coras40c07ce2020-07-16 20:46:17 -0700764 vls_shared_data_pool_rlock ();
Florin Coras243edd52020-03-04 22:20:12 +0000765
Florin Coras40c07ce2020-07-16 20:46:17 -0700766 vls_shd = vls_shared_data_get (vls->shared_data_index);
Florin Coras243edd52020-03-04 22:20:12 +0000767
768 clib_spinlock_lock (&vls_shd->lock);
Florin Coras243edd52020-03-04 22:20:12 +0000769 vec_add1 (vls_shd->workers_subscribed, vls_wrk->wrk_index);
Florin Coras243edd52020-03-04 22:20:12 +0000770 clib_spinlock_unlock (&vls_shd->lock);
Florin Coras40c07ce2020-07-16 20:46:17 -0700771
Florin Coras243edd52020-03-04 22:20:12 +0000772 vls_shared_data_pool_runlock ();
773
Florin Corasf9240dc2019-01-15 08:03:17 -0800774 if (s->rx_fifo)
775 {
Florin Coras8eb8d502021-06-16 14:46:57 -0700776 vcl_session_share_fifos (s, s->rx_fifo, s->tx_fifo);
Florin Corasf9240dc2019-01-15 08:03:17 -0800777 }
Florin Corasc127d5a2020-10-14 16:35:58 -0700778 else if (s->session_state == VCL_STATE_LISTEN)
Florin Coras2d675d72019-01-28 15:54:27 -0800779 {
Florin Corasc127d5a2020-10-14 16:35:58 -0700780 s->session_state = VCL_STATE_LISTEN_NO_MQ;
Florin Coras2d675d72019-01-28 15:54:27 -0800781 }
Florin Coras243edd52020-03-04 22:20:12 +0000782}
Florin Coras2d675d72019-01-28 15:54:27 -0800783
Florin Coras243edd52020-03-04 22:20:12 +0000784static void
785vls_share_sessions (vls_worker_t * vls_parent_wrk, vls_worker_t * vls_wrk)
786{
Florin Coras40c07ce2020-07-16 20:46:17 -0700787 vcl_locked_session_t *vls, *parent_vls;
Florin Coras243edd52020-03-04 22:20:12 +0000788
789 /* *INDENT-OFF* */
Damjan Marionb2c31b62020-12-13 21:47:40 +0100790 pool_foreach (vls, vls_wrk->vls_pool) {
Florin Coras40c07ce2020-07-16 20:46:17 -0700791 /* Initialize sharing on parent session */
792 if (vls->shared_data_index == ~0)
793 {
794 parent_vls = vls_session_get (vls_parent_wrk, vls->vls_index);
795 vls_init_share_session (vls_parent_wrk, parent_vls);
796 vls->shared_data_index = parent_vls->shared_data_index;
797 }
798 vls_share_session (vls_wrk, vls);
Damjan Marionb2c31b62020-12-13 21:47:40 +0100799 }
Florin Coras243edd52020-03-04 22:20:12 +0000800 /* *INDENT-ON* */
Florin Corasf9240dc2019-01-15 08:03:17 -0800801}
802
wanghanlin5788a342021-06-22 17:34:08 +0800803static void
804vls_validate_veps (vcl_worker_t *wrk)
805{
806 vcl_session_t *s;
807 u32 session_index, wrk_index;
808
809 pool_foreach (s, wrk->sessions)
810 {
811 if (s->vep.vep_sh != ~0)
812 {
813 vcl_session_handle_parse (s->vep.vep_sh, &wrk_index, &session_index);
814 s->vep.vep_sh = vcl_session_handle_from_index (session_index);
815 }
816 if (s->vep.next_sh != ~0)
817 {
818 vcl_session_handle_parse (s->vep.next_sh, &wrk_index,
819 &session_index);
820 s->vep.next_sh = vcl_session_handle_from_index (session_index);
821 }
822 if (s->vep.prev_sh != ~0)
823 {
824 vcl_session_handle_parse (s->vep.prev_sh, &wrk_index,
825 &session_index);
826 s->vep.prev_sh = vcl_session_handle_from_index (session_index);
827 }
828 }
829}
830
Florin Corasf9240dc2019-01-15 08:03:17 -0800831void
832vls_worker_copy_on_fork (vcl_worker_t * parent_wrk)
833{
Florin Coras243edd52020-03-04 22:20:12 +0000834 vls_worker_t *vls_wrk = vls_worker_get_current (), *vls_parent_wrk;
Florin Corasf9240dc2019-01-15 08:03:17 -0800835 vcl_worker_t *wrk = vcl_worker_get_current ();
hanlinf8e13632020-08-21 11:05:36 +0800836 u32 vls_index, session_index, wrk_index;
837 vcl_session_handle_t sh;
Florin Corasf9240dc2019-01-15 08:03:17 -0800838
Florin Coras243edd52020-03-04 22:20:12 +0000839 /*
840 * init vcl worker
841 */
Florin Corasf9240dc2019-01-15 08:03:17 -0800842 wrk->sessions = pool_dup (parent_wrk->sessions);
843 wrk->session_index_by_vpp_handles =
844 hash_dup (parent_wrk->session_index_by_vpp_handles);
845
Florin Coras243edd52020-03-04 22:20:12 +0000846 /*
847 * init vls worker
848 */
849 vls_parent_wrk = vls_worker_get (parent_wrk->wrk_index);
hanlinf8e13632020-08-21 11:05:36 +0800850 /* *INDENT-OFF* */
851 hash_foreach (sh, vls_index, vls_parent_wrk->session_handle_to_vlsh_table,
852 ({
853 vcl_session_handle_parse (sh, &wrk_index, &session_index);
854 hash_set (vls_wrk->session_handle_to_vlsh_table,
855 vcl_session_handle_from_index (session_index), vls_index);
856 }));
857 /* *INDENT-ON* */
Florin Coras243edd52020-03-04 22:20:12 +0000858 vls_wrk->vls_pool = pool_dup (vls_parent_wrk->vls_pool);
Florin Coras2d675d72019-01-28 15:54:27 -0800859
wanghanlin5788a342021-06-22 17:34:08 +0800860 /* Validate vep's handle */
861 vls_validate_veps (wrk);
862
Florin Coras243edd52020-03-04 22:20:12 +0000863 vls_share_sessions (vls_parent_wrk, vls_wrk);
Florin Corasf9240dc2019-01-15 08:03:17 -0800864}
865
Florin Coras0ef8ef22019-01-18 08:37:13 -0800866static void
867vls_mt_acq_locks (vcl_locked_session_t * vls, vls_mt_ops_t op, int *locks_acq)
868{
869 vcl_worker_t *wrk = vcl_worker_get_current ();
870 vcl_session_t *s = 0;
871 int is_nonblk = 0;
872
873 if (vls)
874 {
875 s = vcl_session_get (wrk, vls->session_index);
876 if (PREDICT_FALSE (!s))
877 return;
Florin Corasac422d62020-10-19 20:51:36 -0700878 is_nonblk = vcl_session_has_attr (s, VCL_SESS_ATTR_NONBLOCK);
Florin Coras0ef8ef22019-01-18 08:37:13 -0800879 }
880
881 switch (op)
882 {
883 case VLS_MT_OP_READ:
884 if (!is_nonblk)
885 is_nonblk = vcl_session_read_ready (s) != 0;
886 if (!is_nonblk)
887 {
888 vls_mt_mq_lock ();
889 *locks_acq |= VLS_MT_LOCK_MQ;
890 }
891 break;
892 case VLS_MT_OP_WRITE:
Florin Coras78b5fa62019-02-21 20:04:15 -0800893 ASSERT (s);
Florin Coras0ef8ef22019-01-18 08:37:13 -0800894 if (!is_nonblk)
895 is_nonblk = vcl_session_write_ready (s) != 0;
896 if (!is_nonblk)
897 {
898 vls_mt_mq_lock ();
899 *locks_acq |= VLS_MT_LOCK_MQ;
900 }
901 break;
902 case VLS_MT_OP_XPOLL:
903 vls_mt_mq_lock ();
904 *locks_acq |= VLS_MT_LOCK_MQ;
905 break;
906 case VLS_MT_OP_SPOOL:
907 vls_mt_spool_lock ();
908 *locks_acq |= VLS_MT_LOCK_SPOOL;
909 break;
910 default:
911 break;
912 }
913}
914
915static void
916vls_mt_rel_locks (int locks_acq)
917{
918 if (locks_acq & VLS_MT_LOCK_MQ)
919 vls_mt_mq_unlock ();
920 if (locks_acq & VLS_MT_LOCK_SPOOL)
921 vls_mt_create_unlock ();
922}
923
Florin Corasff40d8f2020-08-11 22:05:28 -0700924static inline u8
925vls_mt_session_should_migrate (vcl_locked_session_t * vls)
926{
927 return (vls_mt_wrk_supported ()
928 && vls->worker_index != vcl_get_worker_index ());
929}
930
wanghanlindcacdc42020-12-28 16:19:05 +0800931static vcl_locked_session_t *
932vls_mt_session_migrate (vcl_locked_session_t *vls)
hanlina3a48962020-07-13 11:09:15 +0800933{
934 u32 wrk_index = vcl_get_worker_index ();
935 vcl_worker_t *wrk;
wanghanline8f848a2021-01-08 14:57:11 +0800936 vls_worker_t *vls_wrk = vls_worker_get_current ();
wanghanlindcacdc42020-12-28 16:19:05 +0800937 u32 src_sid, sid, vls_index, own_vcl_wrk_index;
hanlina3a48962020-07-13 11:09:15 +0800938 vcl_session_t *session;
939 uword *p;
940
Florin Corasff40d8f2020-08-11 22:05:28 -0700941 ASSERT (vls_mt_wrk_supported () && vls->worker_index != wrk_index);
hanlina3a48962020-07-13 11:09:15 +0800942
Florin Corasff40d8f2020-08-11 22:05:28 -0700943 /*
944 * VCL session on current vcl worker already allocated. Update current
945 * owner worker and index and return
946 */
hanlina3a48962020-07-13 11:09:15 +0800947 if ((p = hash_get (vls->vcl_wrk_index_to_session_index, wrk_index)))
948 {
949 vls->worker_index = wrk_index;
950 vls->session_index = (u32) p[0];
wanghanlindcacdc42020-12-28 16:19:05 +0800951 return vls;
hanlina3a48962020-07-13 11:09:15 +0800952 }
953
Florin Corasff40d8f2020-08-11 22:05:28 -0700954 /*
955 * Ask vcl worker that owns the original vcl session to clone it into
956 * current vcl worker session pool
957 */
958
hanlina3a48962020-07-13 11:09:15 +0800959 if (!(p = hash_get (vls->vcl_wrk_index_to_session_index,
960 vls->owner_vcl_wrk_index)))
961 {
962 VERR ("session in owner worker(%u) is free", vls->owner_vcl_wrk_index);
963 ASSERT (0);
wanghanlindcacdc42020-12-28 16:19:05 +0800964 vls_unlock (vls);
965 vls_mt_table_runlock ();
966 return 0;
hanlina3a48962020-07-13 11:09:15 +0800967 }
968
969 src_sid = (u32) p[0];
970 wrk = vcl_worker_get_current ();
971 session = vcl_session_alloc (wrk);
972 sid = session->session_index;
hanlina3a48962020-07-13 11:09:15 +0800973 VDBG (1, "migrate session of worker (session): %u (%u) -> %u (%u)",
974 vls->owner_vcl_wrk_index, src_sid, wrk_index, sid);
975
wanghanlindcacdc42020-12-28 16:19:05 +0800976 /* Drop lock to prevent dead lock when dst wrk trying to get lock. */
977 vls_index = vls->vls_index;
978 own_vcl_wrk_index = vls->owner_vcl_wrk_index;
979 vls_unlock (vls);
980 vls_mt_table_runlock ();
981 vls_send_clone_and_share_rpc (wrk, vls_index, sid, vls_get_worker_index (),
982 own_vcl_wrk_index, vls_index, src_sid);
983
984 if (PREDICT_FALSE (wrk->rpc_done == VLS_RPC_STATE_SESSION_NOT_EXIST))
hanlina3a48962020-07-13 11:09:15 +0800985 {
wanghanlindcacdc42020-12-28 16:19:05 +0800986 VWRN ("session %u not exist", src_sid);
987 goto err;
988 }
989 else if (PREDICT_FALSE (wrk->rpc_done == VLS_RPC_STATE_INIT))
990 {
991 VWRN ("failed to wait rpc response");
992 goto err;
993 }
994 else if (PREDICT_FALSE ((session->flags & VCL_SESSION_F_IS_VEP) &&
995 session->vep.next_sh != ~0))
996 {
hanlina3a48962020-07-13 11:09:15 +0800997 VERR ("can't migrate nonempty epoll session");
998 ASSERT (0);
wanghanlindcacdc42020-12-28 16:19:05 +0800999 goto err;
hanlina3a48962020-07-13 11:09:15 +08001000 }
Florin Coras6c3b2182020-10-19 18:36:48 -07001001 else if (PREDICT_FALSE (!(session->flags & VCL_SESSION_F_IS_VEP) &&
Florin Corasc127d5a2020-10-14 16:35:58 -07001002 session->session_state != VCL_STATE_CLOSED))
hanlina3a48962020-07-13 11:09:15 +08001003 {
hanlina3a48962020-07-13 11:09:15 +08001004 VERR ("migrate NOT supported, session_status (%u)",
1005 session->session_state);
1006 ASSERT (0);
wanghanlindcacdc42020-12-28 16:19:05 +08001007 goto err;
hanlina3a48962020-07-13 11:09:15 +08001008 }
wanghanlindcacdc42020-12-28 16:19:05 +08001009
1010 vls = vls_get_w_dlock (vls_index);
1011 if (PREDICT_FALSE (!vls))
1012 {
1013 VWRN ("failed to get vls %u", vls_index);
1014 goto err;
1015 }
1016
1017 session->session_index = sid;
1018 vls->worker_index = wrk_index;
1019 vls->session_index = sid;
1020 hash_set (vls->vcl_wrk_index_to_session_index, wrk_index, sid);
wanghanline8f848a2021-01-08 14:57:11 +08001021 vls_sh_to_vlsh_table_add (vls_wrk, vcl_session_handle (session),
1022 vls->vls_index);
wanghanlindcacdc42020-12-28 16:19:05 +08001023 return vls;
1024
1025err:
1026 vcl_session_free (wrk, session);
1027 return 0;
hanlina3a48962020-07-13 11:09:15 +08001028}
1029
Florin Corasff40d8f2020-08-11 22:05:28 -07001030static inline void
1031vls_mt_detect (void)
1032{
1033 if (PREDICT_FALSE (vcl_get_worker_index () == ~0))
1034 vls_mt_add ();
1035}
Florin Coras0ef8ef22019-01-18 08:37:13 -08001036
wanghanlindcacdc42020-12-28 16:19:05 +08001037#define vls_mt_guard(_vls, _op) \
1038 int _locks_acq = 0; \
1039 if (vls_mt_wrk_supported ()) \
1040 { \
1041 if (PREDICT_FALSE (_vls && \
1042 ((vcl_locked_session_t *) _vls)->worker_index != \
1043 vcl_get_worker_index ())) \
1044 { \
1045 _vls = vls_mt_session_migrate (_vls); \
1046 if (PREDICT_FALSE (!_vls)) \
1047 return VPPCOM_EBADFD; \
1048 } \
1049 } \
1050 else \
1051 { \
1052 if (PREDICT_FALSE (vlsl->vls_mt_n_threads > 1)) \
1053 vls_mt_acq_locks (_vls, _op, &_locks_acq); \
1054 }
Florin Corasff40d8f2020-08-11 22:05:28 -07001055
1056#define vls_mt_unguard() \
1057 if (PREDICT_FALSE (_locks_acq)) \
Florin Coras0ef8ef22019-01-18 08:37:13 -08001058 vls_mt_rel_locks (_locks_acq)
1059
Florin Coras7baeb712019-01-04 17:05:43 -08001060int
1061vls_write (vls_handle_t vlsh, void *buf, size_t nbytes)
1062{
1063 vcl_locked_session_t *vls;
1064 int rv;
1065
Florin Corasff40d8f2020-08-11 22:05:28 -07001066 vls_mt_detect ();
Florin Coras7baeb712019-01-04 17:05:43 -08001067 if (!(vls = vls_get_w_dlock (vlsh)))
1068 return VPPCOM_EBADFD;
Florin Coras0ef8ef22019-01-18 08:37:13 -08001069
1070 vls_mt_guard (vls, VLS_MT_OP_WRITE);
Florin Coras7baeb712019-01-04 17:05:43 -08001071 rv = vppcom_session_write (vls_to_sh_tu (vls), buf, nbytes);
Florin Coras0ef8ef22019-01-18 08:37:13 -08001072 vls_mt_unguard ();
Florin Coras7baeb712019-01-04 17:05:43 -08001073 vls_get_and_unlock (vlsh);
1074 return rv;
1075}
1076
1077int
1078vls_write_msg (vls_handle_t vlsh, void *buf, size_t nbytes)
1079{
1080 vcl_locked_session_t *vls;
1081 int rv;
1082
Florin Corasff40d8f2020-08-11 22:05:28 -07001083 vls_mt_detect ();
Florin Coras7baeb712019-01-04 17:05:43 -08001084 if (!(vls = vls_get_w_dlock (vlsh)))
1085 return VPPCOM_EBADFD;
Florin Coras0ef8ef22019-01-18 08:37:13 -08001086 vls_mt_guard (vls, VLS_MT_OP_WRITE);
Florin Coras7baeb712019-01-04 17:05:43 -08001087 rv = vppcom_session_write_msg (vls_to_sh_tu (vls), buf, nbytes);
Florin Coras0ef8ef22019-01-18 08:37:13 -08001088 vls_mt_unguard ();
Florin Coras7baeb712019-01-04 17:05:43 -08001089 vls_get_and_unlock (vlsh);
1090 return rv;
1091}
1092
1093int
1094vls_sendto (vls_handle_t vlsh, void *buf, int buflen, int flags,
1095 vppcom_endpt_t * ep)
1096{
1097 vcl_locked_session_t *vls;
1098 int rv;
1099
Florin Corasff40d8f2020-08-11 22:05:28 -07001100 vls_mt_detect ();
Florin Coras7baeb712019-01-04 17:05:43 -08001101 if (!(vls = vls_get_w_dlock (vlsh)))
1102 return VPPCOM_EBADFD;
Florin Coras0ef8ef22019-01-18 08:37:13 -08001103 vls_mt_guard (vls, VLS_MT_OP_WRITE);
Florin Coras7baeb712019-01-04 17:05:43 -08001104 rv = vppcom_session_sendto (vls_to_sh_tu (vls), buf, buflen, flags, ep);
Florin Coras0ef8ef22019-01-18 08:37:13 -08001105 vls_mt_unguard ();
Florin Coras7baeb712019-01-04 17:05:43 -08001106 vls_get_and_unlock (vlsh);
1107 return rv;
1108}
1109
1110ssize_t
1111vls_read (vls_handle_t vlsh, void *buf, size_t nbytes)
1112{
1113 vcl_locked_session_t *vls;
1114 int rv;
1115
Florin Corasff40d8f2020-08-11 22:05:28 -07001116 vls_mt_detect ();
Florin Coras7baeb712019-01-04 17:05:43 -08001117 if (!(vls = vls_get_w_dlock (vlsh)))
1118 return VPPCOM_EBADFD;
Florin Coras0ef8ef22019-01-18 08:37:13 -08001119 vls_mt_guard (vls, VLS_MT_OP_READ);
Florin Coras7baeb712019-01-04 17:05:43 -08001120 rv = vppcom_session_read (vls_to_sh_tu (vls), buf, nbytes);
Florin Coras0ef8ef22019-01-18 08:37:13 -08001121 vls_mt_unguard ();
Florin Coras7baeb712019-01-04 17:05:43 -08001122 vls_get_and_unlock (vlsh);
1123 return rv;
1124}
1125
1126ssize_t
1127vls_recvfrom (vls_handle_t vlsh, void *buffer, uint32_t buflen, int flags,
1128 vppcom_endpt_t * ep)
1129{
1130 vcl_locked_session_t *vls;
1131 int rv;
1132
Florin Corasff40d8f2020-08-11 22:05:28 -07001133 vls_mt_detect ();
Florin Coras7baeb712019-01-04 17:05:43 -08001134 if (!(vls = vls_get_w_dlock (vlsh)))
1135 return VPPCOM_EBADFD;
Florin Coras0ef8ef22019-01-18 08:37:13 -08001136 vls_mt_guard (vls, VLS_MT_OP_READ);
Florin Coras7baeb712019-01-04 17:05:43 -08001137 rv = vppcom_session_recvfrom (vls_to_sh_tu (vls), buffer, buflen, flags,
1138 ep);
Florin Coras0ef8ef22019-01-18 08:37:13 -08001139 vls_mt_unguard ();
Florin Coras7baeb712019-01-04 17:05:43 -08001140 vls_get_and_unlock (vlsh);
1141 return rv;
1142}
1143
1144int
1145vls_attr (vls_handle_t vlsh, uint32_t op, void *buffer, uint32_t * buflen)
1146{
1147 vcl_locked_session_t *vls;
1148 int rv;
1149
Florin Corasff40d8f2020-08-11 22:05:28 -07001150 vls_mt_detect ();
Florin Coras7baeb712019-01-04 17:05:43 -08001151 if (!(vls = vls_get_w_dlock (vlsh)))
1152 return VPPCOM_EBADFD;
Florin Corasff40d8f2020-08-11 22:05:28 -07001153 if (vls_mt_session_should_migrate (vls))
wanghanlindcacdc42020-12-28 16:19:05 +08001154 {
1155 vls = vls_mt_session_migrate (vls);
1156 if (PREDICT_FALSE (!vls))
1157 return VPPCOM_EBADFD;
1158 }
Florin Coras7baeb712019-01-04 17:05:43 -08001159 rv = vppcom_session_attr (vls_to_sh_tu (vls), op, buffer, buflen);
1160 vls_get_and_unlock (vlsh);
1161 return rv;
1162}
1163
1164int
1165vls_bind (vls_handle_t vlsh, vppcom_endpt_t * ep)
1166{
1167 vcl_locked_session_t *vls;
1168 int rv;
1169
Florin Corasff40d8f2020-08-11 22:05:28 -07001170 vls_mt_detect ();
Florin Coras7baeb712019-01-04 17:05:43 -08001171 if (!(vls = vls_get_w_dlock (vlsh)))
1172 return VPPCOM_EBADFD;
1173 rv = vppcom_session_bind (vls_to_sh_tu (vls), ep);
1174 vls_get_and_unlock (vlsh);
1175 return rv;
1176}
1177
1178int
1179vls_listen (vls_handle_t vlsh, int q_len)
1180{
1181 vcl_locked_session_t *vls;
1182 int rv;
1183
Florin Corasff40d8f2020-08-11 22:05:28 -07001184 vls_mt_detect ();
Florin Coras7baeb712019-01-04 17:05:43 -08001185 if (!(vls = vls_get_w_dlock (vlsh)))
1186 return VPPCOM_EBADFD;
Florin Coras0ef8ef22019-01-18 08:37:13 -08001187 vls_mt_guard (vls, VLS_MT_OP_XPOLL);
Florin Coras7baeb712019-01-04 17:05:43 -08001188 rv = vppcom_session_listen (vls_to_sh_tu (vls), q_len);
Florin Coras0ef8ef22019-01-18 08:37:13 -08001189 vls_mt_unguard ();
Florin Coras7baeb712019-01-04 17:05:43 -08001190 vls_get_and_unlock (vlsh);
1191 return rv;
1192}
1193
1194int
1195vls_connect (vls_handle_t vlsh, vppcom_endpt_t * server_ep)
1196{
1197 vcl_locked_session_t *vls;
1198 int rv;
1199
Florin Corasff40d8f2020-08-11 22:05:28 -07001200 vls_mt_detect ();
Florin Coras7baeb712019-01-04 17:05:43 -08001201 if (!(vls = vls_get_w_dlock (vlsh)))
1202 return VPPCOM_EBADFD;
Florin Coras0ef8ef22019-01-18 08:37:13 -08001203 vls_mt_guard (vls, VLS_MT_OP_XPOLL);
Florin Coras7baeb712019-01-04 17:05:43 -08001204 rv = vppcom_session_connect (vls_to_sh_tu (vls), server_ep);
Florin Coras0ef8ef22019-01-18 08:37:13 -08001205 vls_mt_unguard ();
Florin Coras7baeb712019-01-04 17:05:43 -08001206 vls_get_and_unlock (vlsh);
1207 return rv;
1208}
1209
Florin Coras2d675d72019-01-28 15:54:27 -08001210static inline void
1211vls_mp_checks (vcl_locked_session_t * vls, int is_add)
1212{
1213 vcl_worker_t *wrk = vcl_worker_get_current ();
1214 vcl_session_t *s;
Florin Coras243edd52020-03-04 22:20:12 +00001215 u32 owner_wrk;
Florin Coras2d675d72019-01-28 15:54:27 -08001216
hanlina3a48962020-07-13 11:09:15 +08001217 if (vls_mt_wrk_supported ())
1218 return;
1219
Florin Coras2d675d72019-01-28 15:54:27 -08001220 s = vcl_session_get (wrk, vls->session_index);
1221 switch (s->session_state)
1222 {
Florin Corasc127d5a2020-10-14 16:35:58 -07001223 case VCL_STATE_LISTEN:
Florin Coras2d675d72019-01-28 15:54:27 -08001224 if (is_add)
1225 {
Florin Coras243edd52020-03-04 22:20:12 +00001226 vls_listener_wrk_set (vls, vls->worker_index, 1 /* is_active */ );
Florin Coras2d675d72019-01-28 15:54:27 -08001227 break;
1228 }
1229 vls_listener_wrk_stop_listen (vls, vls->worker_index);
1230 break;
Florin Corasc127d5a2020-10-14 16:35:58 -07001231 case VCL_STATE_LISTEN_NO_MQ:
Florin Coras2d675d72019-01-28 15:54:27 -08001232 if (!is_add)
1233 break;
1234
1235 /* Register worker as listener */
1236 vls_listener_wrk_start_listen (vls, wrk->wrk_index);
1237
1238 /* If owner worker did not attempt to accept/xpoll on the session,
1239 * force a listen stop for it, since it may not be interested in
1240 * accepting new sessions.
1241 * This is pretty much a hack done to give app workers the illusion
1242 * that it is fine to listen and not accept new sessions for a
1243 * given listener. Without it, we would accumulate unhandled
1244 * accepts on the passive worker message queue. */
Florin Coras243edd52020-03-04 22:20:12 +00001245 owner_wrk = vls_shared_get_owner (vls);
1246 if (!vls_listener_wrk_is_active (vls, owner_wrk))
1247 vls_listener_wrk_stop_listen (vls, owner_wrk);
Florin Coras2d675d72019-01-28 15:54:27 -08001248 break;
1249 default:
1250 break;
1251 }
1252}
1253
Florin Coras7baeb712019-01-04 17:05:43 -08001254vls_handle_t
1255vls_accept (vls_handle_t listener_vlsh, vppcom_endpt_t * ep, int flags)
1256{
1257 vls_handle_t accepted_vlsh;
1258 vcl_locked_session_t *vls;
1259 int sh;
1260
Florin Corasff40d8f2020-08-11 22:05:28 -07001261 vls_mt_detect ();
Florin Coras7baeb712019-01-04 17:05:43 -08001262 if (!(vls = vls_get_w_dlock (listener_vlsh)))
1263 return VPPCOM_EBADFD;
Florin Coras2d675d72019-01-28 15:54:27 -08001264 if (vcl_n_workers () > 1)
1265 vls_mp_checks (vls, 1 /* is_add */ );
Florin Coras0ef8ef22019-01-18 08:37:13 -08001266 vls_mt_guard (vls, VLS_MT_OP_SPOOL);
Florin Coras7baeb712019-01-04 17:05:43 -08001267 sh = vppcom_session_accept (vls_to_sh_tu (vls), ep, flags);
Florin Coras0ef8ef22019-01-18 08:37:13 -08001268 vls_mt_unguard ();
Florin Coras7baeb712019-01-04 17:05:43 -08001269 vls_get_and_unlock (listener_vlsh);
1270 if (sh < 0)
1271 return sh;
1272 accepted_vlsh = vls_alloc (sh);
1273 if (PREDICT_FALSE (accepted_vlsh == VLS_INVALID_HANDLE))
1274 vppcom_session_close (sh);
1275 return accepted_vlsh;
1276}
1277
1278vls_handle_t
1279vls_create (uint8_t proto, uint8_t is_nonblocking)
1280{
1281 vcl_session_handle_t sh;
1282 vls_handle_t vlsh;
wanghanlindcacdc42020-12-28 16:19:05 +08001283 vcl_locked_session_t *vls = NULL;
Florin Coras7baeb712019-01-04 17:05:43 -08001284
Florin Corasff40d8f2020-08-11 22:05:28 -07001285 vls_mt_detect ();
wanghanlindcacdc42020-12-28 16:19:05 +08001286 vls_mt_guard (vls, VLS_MT_OP_SPOOL);
Florin Coras7baeb712019-01-04 17:05:43 -08001287 sh = vppcom_session_create (proto, is_nonblocking);
Florin Coras0ef8ef22019-01-18 08:37:13 -08001288 vls_mt_unguard ();
Florin Coras7baeb712019-01-04 17:05:43 -08001289 if (sh == INVALID_SESSION_ID)
1290 return VLS_INVALID_HANDLE;
1291
1292 vlsh = vls_alloc (sh);
1293 if (PREDICT_FALSE (vlsh == VLS_INVALID_HANDLE))
1294 vppcom_session_close (sh);
1295
1296 return vlsh;
1297}
1298
hanlina3a48962020-07-13 11:09:15 +08001299static void
Florin Corasff40d8f2020-08-11 22:05:28 -07001300vls_mt_session_cleanup (vcl_locked_session_t * vls)
hanlina3a48962020-07-13 11:09:15 +08001301{
Florin Corasff40d8f2020-08-11 22:05:28 -07001302 u32 session_index, wrk_index, current_vcl_wrk;
hanlina3a48962020-07-13 11:09:15 +08001303 vcl_worker_t *wrk = vcl_worker_get_current ();
1304
Florin Corasff40d8f2020-08-11 22:05:28 -07001305 ASSERT (vls_mt_wrk_supported ());
1306
1307 current_vcl_wrk = vcl_get_worker_index ();
hanlina3a48962020-07-13 11:09:15 +08001308
1309 /* *INDENT-OFF* */
1310 hash_foreach (wrk_index, session_index, vls->vcl_wrk_index_to_session_index,
1311 ({
Florin Corasff40d8f2020-08-11 22:05:28 -07001312 if (current_vcl_wrk != wrk_index)
1313 vls_send_session_cleanup_rpc (wrk, wrk_index, session_index);
hanlina3a48962020-07-13 11:09:15 +08001314 }));
1315 /* *INDENT-ON* */
1316 hash_free (vls->vcl_wrk_index_to_session_index);
1317}
1318
Florin Coras7baeb712019-01-04 17:05:43 -08001319int
1320vls_close (vls_handle_t vlsh)
1321{
1322 vcl_locked_session_t *vls;
Florin Corasf9240dc2019-01-15 08:03:17 -08001323 int rv;
Florin Coras7baeb712019-01-04 17:05:43 -08001324
Florin Corasff40d8f2020-08-11 22:05:28 -07001325 vls_mt_detect ();
1326 vls_mt_table_wlock ();
Florin Coras7baeb712019-01-04 17:05:43 -08001327
Florin Coras0ef8ef22019-01-18 08:37:13 -08001328 vls = vls_get_and_lock (vlsh);
1329 if (!vls)
1330 {
Florin Corasff40d8f2020-08-11 22:05:28 -07001331 vls_mt_table_wunlock ();
Florin Coras0ef8ef22019-01-18 08:37:13 -08001332 return VPPCOM_EBADFD;
1333 }
1334
hanlina3a48962020-07-13 11:09:15 +08001335 vls_mt_guard (vls, VLS_MT_OP_SPOOL);
Florin Corasf9240dc2019-01-15 08:03:17 -08001336
Florin Coras243edd52020-03-04 22:20:12 +00001337 if (vls_is_shared (vls))
1338 rv = vls_unshare_session (vls, vcl_worker_get_current ());
1339 else
1340 rv = vppcom_session_close (vls_to_sh (vls));
1341
Florin Corasff40d8f2020-08-11 22:05:28 -07001342 if (vls_mt_wrk_supported ())
1343 vls_mt_session_cleanup (vls);
hanlina3a48962020-07-13 11:09:15 +08001344
Florin Coras0ef8ef22019-01-18 08:37:13 -08001345 vls_free (vls);
1346 vls_mt_unguard ();
Florin Coras7baeb712019-01-04 17:05:43 -08001347
Florin Corasff40d8f2020-08-11 22:05:28 -07001348 vls_mt_table_wunlock ();
Florin Coras0ef8ef22019-01-18 08:37:13 -08001349
Florin Coras7baeb712019-01-04 17:05:43 -08001350 return rv;
1351}
1352
liuyacan534468e2021-05-09 03:50:40 +00001353int
liuyacan55c952e2021-06-13 14:54:55 +08001354vls_shutdown (vls_handle_t vlsh, int how)
liuyacan534468e2021-05-09 03:50:40 +00001355{
1356 vcl_locked_session_t *vls;
1357 int rv;
1358
1359 vls_mt_detect ();
1360 if (!(vls = vls_get_w_dlock (vlsh)))
1361 return VPPCOM_EBADFD;
1362
1363 vls_mt_guard (vls, VLS_MT_OP_SPOOL);
liuyacan55c952e2021-06-13 14:54:55 +08001364 rv = vppcom_session_shutdown (vls_to_sh (vls), how);
liuyacan534468e2021-05-09 03:50:40 +00001365 vls_mt_unguard ();
1366 vls_get_and_unlock (vlsh);
1367
1368 return rv;
1369}
1370
Florin Coras7baeb712019-01-04 17:05:43 -08001371vls_handle_t
1372vls_epoll_create (void)
1373{
1374 vcl_session_handle_t sh;
1375 vls_handle_t vlsh;
1376
Florin Corasff40d8f2020-08-11 22:05:28 -07001377 vls_mt_detect ();
Florin Coras63d3ac62019-03-29 08:29:25 -07001378
Florin Coras7baeb712019-01-04 17:05:43 -08001379 sh = vppcom_epoll_create ();
1380 if (sh == INVALID_SESSION_ID)
1381 return VLS_INVALID_HANDLE;
1382
1383 vlsh = vls_alloc (sh);
1384 if (vlsh == VLS_INVALID_HANDLE)
1385 vppcom_session_close (sh);
1386
1387 return vlsh;
1388}
1389
Florin Coras2d675d72019-01-28 15:54:27 -08001390static void
1391vls_epoll_ctl_mp_checks (vcl_locked_session_t * vls, int op)
1392{
nandfandc2632e2021-03-26 16:46:58 +08001393 if (vcl_n_workers () <= 1 || op == EPOLL_CTL_MOD)
Florin Coras2d675d72019-01-28 15:54:27 -08001394 return;
1395
Florin Coras2d675d72019-01-28 15:54:27 -08001396 vls_mp_checks (vls, op == EPOLL_CTL_ADD);
1397}
1398
Florin Coras7baeb712019-01-04 17:05:43 -08001399int
1400vls_epoll_ctl (vls_handle_t ep_vlsh, int op, vls_handle_t vlsh,
1401 struct epoll_event *event)
1402{
1403 vcl_locked_session_t *ep_vls, *vls;
1404 vcl_session_handle_t ep_sh, sh;
1405 int rv;
1406
Florin Corasff40d8f2020-08-11 22:05:28 -07001407 vls_mt_detect ();
1408 vls_mt_table_rlock ();
Florin Coras7baeb712019-01-04 17:05:43 -08001409 ep_vls = vls_get_and_lock (ep_vlsh);
Florin Corasff40d8f2020-08-11 22:05:28 -07001410
1411 if (vls_mt_session_should_migrate (ep_vls))
wanghanlindcacdc42020-12-28 16:19:05 +08001412 {
1413 ep_vls = vls_mt_session_migrate (ep_vls);
1414 if (PREDICT_FALSE (!ep_vls))
1415 return VPPCOM_EBADFD;
1416 }
Florin Corasff40d8f2020-08-11 22:05:28 -07001417
Florin Coras7baeb712019-01-04 17:05:43 -08001418 ep_sh = vls_to_sh (ep_vls);
wanghanlindcacdc42020-12-28 16:19:05 +08001419 vls = vls_get_and_lock (vlsh);
Florin Coras7baeb712019-01-04 17:05:43 -08001420 sh = vls_to_sh (vls);
Florin Coras2d675d72019-01-28 15:54:27 -08001421
nandfandc2632e2021-03-26 16:46:58 +08001422 vls_epoll_ctl_mp_checks (vls, op);
Florin Corasff40d8f2020-08-11 22:05:28 -07001423 vls_mt_table_runlock ();
Florin Coras7baeb712019-01-04 17:05:43 -08001424 rv = vppcom_epoll_ctl (ep_sh, op, sh, event);
1425
Florin Corasff40d8f2020-08-11 22:05:28 -07001426 vls_mt_table_rlock ();
Florin Coras7baeb712019-01-04 17:05:43 -08001427 ep_vls = vls_get (ep_vlsh);
1428 vls = vls_get (vlsh);
1429 vls_unlock (vls);
1430 vls_unlock (ep_vls);
Florin Corasff40d8f2020-08-11 22:05:28 -07001431 vls_mt_table_runlock ();
Florin Coras7baeb712019-01-04 17:05:43 -08001432 return rv;
1433}
1434
1435int
1436vls_epoll_wait (vls_handle_t ep_vlsh, struct epoll_event *events,
1437 int maxevents, double wait_for_time)
1438{
wanghanlindcacdc42020-12-28 16:19:05 +08001439 vcl_locked_session_t *vls, *vls_tmp = NULL;
Florin Coras7baeb712019-01-04 17:05:43 -08001440 int rv;
1441
Florin Corasff40d8f2020-08-11 22:05:28 -07001442 vls_mt_detect ();
Florin Coras7baeb712019-01-04 17:05:43 -08001443 if (!(vls = vls_get_w_dlock (ep_vlsh)))
1444 return VPPCOM_EBADFD;
wanghanlindcacdc42020-12-28 16:19:05 +08001445 vls_mt_guard (vls_tmp, VLS_MT_OP_XPOLL);
Florin Coras7baeb712019-01-04 17:05:43 -08001446 rv = vppcom_epoll_wait (vls_to_sh_tu (vls), events, maxevents,
1447 wait_for_time);
Florin Coras0ef8ef22019-01-18 08:37:13 -08001448 vls_mt_unguard ();
Florin Coras7baeb712019-01-04 17:05:43 -08001449 vls_get_and_unlock (ep_vlsh);
wanghanlind72a0342021-05-12 17:00:29 +08001450 vls_handle_pending_wrk_cleanup ();
Florin Coras7baeb712019-01-04 17:05:43 -08001451 return rv;
1452}
1453
Florin Coras2d675d72019-01-28 15:54:27 -08001454static void
1455vls_select_mp_checks (vcl_si_set * read_map)
1456{
1457 vcl_locked_session_t *vls;
1458 vcl_worker_t *wrk;
1459 vcl_session_t *s;
1460 u32 si;
1461
1462 if (vcl_n_workers () <= 1)
1463 {
1464 vlsl->select_mp_check = 1;
1465 return;
1466 }
1467
1468 if (!read_map)
1469 return;
1470
1471 vlsl->select_mp_check = 1;
1472 wrk = vcl_worker_get_current ();
1473
1474 /* *INDENT-OFF* */
Damjan Marionf0ca1e82020-12-13 23:26:56 +01001475 clib_bitmap_foreach (si, read_map) {
Florin Coras2d675d72019-01-28 15:54:27 -08001476 s = vcl_session_get (wrk, si);
Florin Corasc127d5a2020-10-14 16:35:58 -07001477 if (s->session_state == VCL_STATE_LISTEN)
Florin Coras2d675d72019-01-28 15:54:27 -08001478 {
1479 vls = vls_get (vls_session_index_to_vlsh (si));
1480 vls_mp_checks (vls, 1 /* is_add */);
1481 }
Damjan Marionf0ca1e82020-12-13 23:26:56 +01001482 }
Florin Coras2d675d72019-01-28 15:54:27 -08001483 /* *INDENT-ON* */
1484}
1485
Florin Coras0ef8ef22019-01-18 08:37:13 -08001486int
1487vls_select (int n_bits, vcl_si_set * read_map, vcl_si_set * write_map,
1488 vcl_si_set * except_map, double wait_for_time)
1489{
1490 int rv;
wanghanlindcacdc42020-12-28 16:19:05 +08001491 vcl_locked_session_t *vls = NULL;
Florin Coras2d675d72019-01-28 15:54:27 -08001492
Florin Corasff40d8f2020-08-11 22:05:28 -07001493 vls_mt_detect ();
wanghanlindcacdc42020-12-28 16:19:05 +08001494 vls_mt_guard (vls, VLS_MT_OP_XPOLL);
Florin Coras2d675d72019-01-28 15:54:27 -08001495 if (PREDICT_FALSE (!vlsl->select_mp_check))
1496 vls_select_mp_checks (read_map);
Florin Coras0ef8ef22019-01-18 08:37:13 -08001497 rv = vppcom_select (n_bits, read_map, write_map, except_map, wait_for_time);
1498 vls_mt_unguard ();
wanghanlind72a0342021-05-12 17:00:29 +08001499 vls_handle_pending_wrk_cleanup ();
Florin Coras0ef8ef22019-01-18 08:37:13 -08001500 return rv;
1501}
1502
Florin Corasf9240dc2019-01-15 08:03:17 -08001503static void
Florin Coras0ef8ef22019-01-18 08:37:13 -08001504vls_unshare_vcl_worker_sessions (vcl_worker_t * wrk)
1505{
1506 u32 current_wrk, is_current;
1507 vcl_locked_session_t *vls;
1508 vcl_session_t *s;
1509
Florin Coras14ed6df2019-03-06 21:13:42 -08001510 if (pool_elts (vcm->workers) <= 1)
1511 return;
1512
Florin Coras0ef8ef22019-01-18 08:37:13 -08001513 current_wrk = vcl_get_worker_index ();
1514 is_current = current_wrk == wrk->wrk_index;
Florin Coras0ef8ef22019-01-18 08:37:13 -08001515
1516 /* *INDENT-OFF* */
Damjan Marionb2c31b62020-12-13 21:47:40 +01001517 pool_foreach (s, wrk->sessions) {
hanlinf8e13632020-08-21 11:05:36 +08001518 vls = vls_get (vls_si_wi_to_vlsh (s->session_index, wrk->wrk_index));
Florin Coras0ef8ef22019-01-18 08:37:13 -08001519 if (vls && (is_current || vls_is_shared_by_wrk (vls, current_wrk)))
1520 vls_unshare_session (vls, wrk);
Damjan Marionb2c31b62020-12-13 21:47:40 +01001521 }
Florin Coras0ef8ef22019-01-18 08:37:13 -08001522 /* *INDENT-ON* */
Florin Coras0ef8ef22019-01-18 08:37:13 -08001523}
1524
1525static void
1526vls_cleanup_vcl_worker (vcl_worker_t * wrk)
1527{
Florin Coras243edd52020-03-04 22:20:12 +00001528 vls_worker_t *vls_wrk = vls_worker_get (wrk->wrk_index);
1529
Florin Coras0ef8ef22019-01-18 08:37:13 -08001530 /* Unshare sessions and also cleanup worker since child may have
1531 * called _exit () and therefore vcl may not catch the event */
1532 vls_unshare_vcl_worker_sessions (wrk);
wanghanlinb940fd42021-06-29 16:01:55 +08001533
1534 /* Since child may have exited and thereforce fd of vpp_app_socket_api
1535 * may have been closed, so DONOT notify VPP.
1536 */
1537 vcl_worker_cleanup (wrk, vcm->cfg.vpp_app_socket_api ? 0 : 1);
Florin Coras243edd52020-03-04 22:20:12 +00001538
1539 vls_worker_free (vls_wrk);
Florin Coras0ef8ef22019-01-18 08:37:13 -08001540}
1541
1542static void
Florin Corasf9240dc2019-01-15 08:03:17 -08001543vls_cleanup_forked_child (vcl_worker_t * wrk, vcl_worker_t * child_wrk)
1544{
1545 vcl_worker_t *sub_child;
1546 int tries = 0;
1547
1548 if (child_wrk->forked_child != ~0)
1549 {
1550 sub_child = vcl_worker_get_if_valid (child_wrk->forked_child);
1551 if (sub_child)
1552 {
1553 /* Wait a bit, maybe the process is going away */
1554 while (kill (sub_child->current_pid, 0) >= 0 && tries++ < 50)
1555 usleep (1e3);
1556 if (kill (sub_child->current_pid, 0) < 0)
1557 vls_cleanup_forked_child (child_wrk, sub_child);
1558 }
1559 }
Florin Coras0ef8ef22019-01-18 08:37:13 -08001560 vls_cleanup_vcl_worker (child_wrk);
1561 VDBG (0, "Cleaned up forked child wrk %u", child_wrk->wrk_index);
Florin Corasf9240dc2019-01-15 08:03:17 -08001562 wrk->forked_child = ~0;
1563}
1564
wanghanlind72a0342021-05-12 17:00:29 +08001565static void
1566vls_handle_pending_wrk_cleanup (void)
1567{
1568 u32 *wip;
1569 vcl_worker_t *child_wrk, *wrk;
1570 vls_worker_t *vls_wrk = vls_worker_get_current ();
1571
1572 if (PREDICT_TRUE (vec_len (vls_wrk->pending_wrk_cleanup) == 0))
1573 return;
1574
1575 wrk = vcl_worker_get_current ();
1576 vec_foreach (wip, vls_wrk->pending_wrk_cleanup)
1577 {
1578 child_wrk = vcl_worker_get_if_valid (*wip);
1579 if (!child_wrk)
1580 continue;
1581 vls_cleanup_forked_child (wrk, child_wrk);
1582 }
1583 vec_reset_length (vls_wrk->pending_wrk_cleanup);
1584}
1585
Florin Corasf9240dc2019-01-15 08:03:17 -08001586static struct sigaction old_sa;
1587
1588static void
1589vls_intercept_sigchld_handler (int signum, siginfo_t * si, void *uc)
1590{
1591 vcl_worker_t *wrk, *child_wrk;
wanghanlind72a0342021-05-12 17:00:29 +08001592 vls_worker_t *vls_wrk;
Florin Corasf9240dc2019-01-15 08:03:17 -08001593
1594 if (vcl_get_worker_index () == ~0)
1595 return;
1596
1597 if (sigaction (SIGCHLD, &old_sa, 0))
1598 {
1599 VERR ("couldn't restore sigchld");
1600 exit (-1);
1601 }
1602
1603 wrk = vcl_worker_get_current ();
1604 if (wrk->forked_child == ~0)
1605 return;
1606
1607 child_wrk = vcl_worker_get_if_valid (wrk->forked_child);
1608 if (!child_wrk)
1609 goto done;
1610
1611 if (si && si->si_pid != child_wrk->current_pid)
1612 {
1613 VDBG (0, "unexpected child pid %u", si->si_pid);
1614 goto done;
1615 }
wanghanlind72a0342021-05-12 17:00:29 +08001616
1617 /* Parent process may enter sighandler with a lock, such as lock in localtime
1618 * or in mspace_free, and child wrk cleanup may try to get such locks and
1619 * cause deadlock.
1620 * So move child wrk cleanup from sighandler to vls_epoll_wait/vls_select.
1621 */
1622 vls_wrk = vls_worker_get_current ();
1623 vec_add1 (vls_wrk->pending_wrk_cleanup, child_wrk->wrk_index);
Florin Corasf9240dc2019-01-15 08:03:17 -08001624
1625done:
1626 if (old_sa.sa_flags & SA_SIGINFO)
1627 {
1628 void (*fn) (int, siginfo_t *, void *) = old_sa.sa_sigaction;
1629 fn (signum, si, uc);
1630 }
1631 else
1632 {
1633 void (*fn) (int) = old_sa.sa_handler;
1634 if (fn)
1635 fn (signum);
1636 }
1637}
1638
1639static void
1640vls_incercept_sigchld ()
1641{
1642 struct sigaction sa;
nandfan5fdc47c2021-02-22 17:17:17 +08001643 if (old_sa.sa_sigaction)
1644 {
1645 VDBG (0, "have intercepted sigchld");
1646 return;
1647 }
Florin Corasf9240dc2019-01-15 08:03:17 -08001648 clib_memset (&sa, 0, sizeof (sa));
1649 sa.sa_sigaction = vls_intercept_sigchld_handler;
1650 sa.sa_flags = SA_SIGINFO;
1651 if (sigaction (SIGCHLD, &sa, &old_sa))
1652 {
1653 VERR ("couldn't intercept sigchld");
1654 exit (-1);
1655 }
1656}
1657
1658static void
1659vls_app_pre_fork (void)
1660{
1661 vls_incercept_sigchld ();
1662 vcl_flush_mq_events ();
1663}
1664
1665static void
1666vls_app_fork_child_handler (void)
1667{
1668 vcl_worker_t *parent_wrk;
Florin Corasb88de902020-09-08 16:47:57 -07001669 int parent_wrk_index;
Florin Corasf9240dc2019-01-15 08:03:17 -08001670
1671 parent_wrk_index = vcl_get_worker_index ();
1672 VDBG (0, "initializing forked child %u with parent wrk %u", getpid (),
1673 parent_wrk_index);
1674
1675 /*
Florin Corasb88de902020-09-08 16:47:57 -07001676 * Clear old state
Florin Corasf9240dc2019-01-15 08:03:17 -08001677 */
1678 vcl_set_worker_index (~0);
Florin Corasf9240dc2019-01-15 08:03:17 -08001679
1680 /*
Florin Corasb88de902020-09-08 16:47:57 -07001681 * Allocate and register vcl worker with vpp
Florin Corasf9240dc2019-01-15 08:03:17 -08001682 */
Florin Corasb88de902020-09-08 16:47:57 -07001683 if (vppcom_worker_register ())
Florin Corasf9240dc2019-01-15 08:03:17 -08001684 {
Florin Corasb88de902020-09-08 16:47:57 -07001685 VERR ("couldn't register new worker!");
Florin Corasf9240dc2019-01-15 08:03:17 -08001686 return;
1687 }
1688
1689 /*
Florin Corasb88de902020-09-08 16:47:57 -07001690 * Allocate/initialize vls worker and share sessions
Florin Coras243edd52020-03-04 22:20:12 +00001691 */
1692 vls_worker_alloc ();
Florin Corasf9240dc2019-01-15 08:03:17 -08001693 parent_wrk = vcl_worker_get (parent_wrk_index);
1694 vls_worker_copy_on_fork (parent_wrk);
1695 parent_wrk->forked_child = vcl_get_worker_index ();
1696
Florin Coras0ef8ef22019-01-18 08:37:13 -08001697 /* Reset number of threads and set wrk index */
Florin Coras2d675d72019-01-28 15:54:27 -08001698 vlsl->vls_mt_n_threads = 0;
1699 vlsl->vls_wrk_index = vcl_get_worker_index ();
1700 vlsl->select_mp_check = 0;
Florin Coras2d675d72019-01-28 15:54:27 -08001701 vls_mt_locks_init ();
Florin Coras0ef8ef22019-01-18 08:37:13 -08001702
Florin Corasf9240dc2019-01-15 08:03:17 -08001703 VDBG (0, "forked child main worker initialized");
1704 vcm->forking = 0;
1705}
1706
1707static void
1708vls_app_fork_parent_handler (void)
1709{
1710 vcm->forking = 1;
1711 while (vcm->forking)
1712 ;
1713}
1714
Florin Coras0ef8ef22019-01-18 08:37:13 -08001715void
1716vls_app_exit (void)
1717{
Florin Coras243edd52020-03-04 22:20:12 +00001718 vls_worker_t *wrk = vls_worker_get_current ();
1719
wanghanlind72a0342021-05-12 17:00:29 +08001720 /* Handle pending wrk cleanup */
1721 vls_handle_pending_wrk_cleanup ();
1722
Florin Coras0ef8ef22019-01-18 08:37:13 -08001723 /* Unshare the sessions. VCL will clean up the worker */
1724 vls_unshare_vcl_worker_sessions (vcl_worker_get_current ());
Florin Coras243edd52020-03-04 22:20:12 +00001725 vls_worker_free (wrk);
Florin Coras0ef8ef22019-01-18 08:37:13 -08001726}
1727
Florin Coras40c07ce2020-07-16 20:46:17 -07001728static void
1729vls_clone_and_share_rpc_handler (void *args)
1730{
1731 vls_clone_and_share_msg_t *msg = (vls_clone_and_share_msg_t *) args;
1732 vls_worker_t *wrk = vls_worker_get_current (), *dst_wrk;
1733 vcl_locked_session_t *vls, *dst_vls;
hanlina3a48962020-07-13 11:09:15 +08001734 vcl_worker_t *vcl_wrk = vcl_worker_get_current (), *dst_vcl_wrk;
Florin Coras40c07ce2020-07-16 20:46:17 -07001735 vcl_session_t *s, *dst_s;
1736
wanghanlindcacdc42020-12-28 16:19:05 +08001737 VDBG (1, "process session clone of worker (session): %u (%u) -> %u (%u)",
1738 vcl_wrk->wrk_index, msg->session_index, msg->origin_vcl_wrk,
1739 msg->origin_session_index);
1740
1741 /* VCL locked session can't been protected, so DONT touch it.
1742 * VCL session may been free, check it.
1743 */
1744 dst_vcl_wrk = vcl_worker_get (msg->origin_vcl_wrk);
1745 s = vcl_session_get (vcl_wrk, msg->session_index);
1746 if (PREDICT_FALSE (!s))
1747 {
1748 dst_vcl_wrk->rpc_done = VLS_RPC_STATE_SESSION_NOT_EXIST;
1749 return;
1750 }
Florin Coras40c07ce2020-07-16 20:46:17 -07001751
hanlina3a48962020-07-13 11:09:15 +08001752 if (!vls_mt_wrk_supported ())
wanghanlindcacdc42020-12-28 16:19:05 +08001753 {
1754 vls = vls_session_get (wrk, msg->vls_index);
1755 vls_init_share_session (wrk, vls);
1756 dst_wrk = vls_worker_get (msg->origin_vls_wrk);
1757 dst_vls = vls_session_get (dst_wrk, msg->origin_vls_index);
1758 dst_vls->shared_data_index = vls->shared_data_index;
1759 }
hanlina3a48962020-07-13 11:09:15 +08001760 dst_s = vcl_session_get (dst_vcl_wrk, msg->origin_session_index);
Florin Coras40c07ce2020-07-16 20:46:17 -07001761 clib_memcpy (dst_s, s, sizeof (*s));
1762
wanghanlindcacdc42020-12-28 16:19:05 +08001763 dst_vcl_wrk->rpc_done = VLS_RPC_STATE_SUCCESS;
hanlina3a48962020-07-13 11:09:15 +08001764}
1765
1766static void
1767vls_session_cleanup_rpc_handler (void *args)
1768{
1769 vls_sess_cleanup_msg_t *msg = (vls_sess_cleanup_msg_t *) args;
1770 vcl_worker_t *wrk = vcl_worker_get_current ();
wanghanline8f848a2021-01-08 14:57:11 +08001771 vls_worker_t *vls_wrk = vls_worker_get_current ();
wanghanlindcacdc42020-12-28 16:19:05 +08001772 vcl_session_handle_t sh = vcl_session_handle_from_index (msg->session_index);
hanlina3a48962020-07-13 11:09:15 +08001773
wanghanlindcacdc42020-12-28 16:19:05 +08001774 VDBG (1, "process session cleanup of worker (session): %u (%u) from %u ()",
1775 wrk->wrk_index, msg->session_index, msg->origin_vcl_wrk);
hanlina3a48962020-07-13 11:09:15 +08001776
wanghanlindcacdc42020-12-28 16:19:05 +08001777 vppcom_session_close (sh);
wanghanline8f848a2021-01-08 14:57:11 +08001778 vls_sh_to_vlsh_table_del (vls_wrk, sh);
Florin Coras40c07ce2020-07-16 20:46:17 -07001779}
1780
1781static void
1782vls_rpc_handler (void *args)
1783{
1784 vls_rpc_msg_t *msg = (vls_rpc_msg_t *) args;
1785 switch (msg->type)
1786 {
1787 case VLS_RPC_CLONE_AND_SHARE:
1788 vls_clone_and_share_rpc_handler (msg->data);
1789 break;
hanlina3a48962020-07-13 11:09:15 +08001790 case VLS_RPC_SESS_CLEANUP:
1791 vls_session_cleanup_rpc_handler (msg->data);
1792 break;
Florin Coras40c07ce2020-07-16 20:46:17 -07001793 default:
1794 break;
1795 }
1796}
1797
1798void
wanghanlindcacdc42020-12-28 16:19:05 +08001799vls_send_clone_and_share_rpc (vcl_worker_t *wrk, u32 origin_vls_index,
1800 u32 session_index, u32 vls_wrk_index,
1801 u32 dst_wrk_index, u32 dst_vls_index,
1802 u32 dst_session_index)
Florin Coras40c07ce2020-07-16 20:46:17 -07001803{
1804 u8 data[sizeof (u8) + sizeof (vls_clone_and_share_msg_t)];
1805 vls_clone_and_share_msg_t *msg;
1806 vls_rpc_msg_t *rpc;
hanlina3a48962020-07-13 11:09:15 +08001807 int ret;
wanghanlindcacdc42020-12-28 16:19:05 +08001808 f64 timeout = clib_time_now (&wrk->clib_time) + VLS_WORKER_RPC_TIMEOUT;
Florin Coras40c07ce2020-07-16 20:46:17 -07001809
1810 rpc = (vls_rpc_msg_t *) & data;
1811 rpc->type = VLS_RPC_CLONE_AND_SHARE;
1812 msg = (vls_clone_and_share_msg_t *) & rpc->data;
hanlina3a48962020-07-13 11:09:15 +08001813 msg->origin_vls_wrk = vls_wrk_index;
wanghanlindcacdc42020-12-28 16:19:05 +08001814 msg->origin_vls_index = origin_vls_index;
hanlina3a48962020-07-13 11:09:15 +08001815 msg->origin_vcl_wrk = wrk->wrk_index;
1816 msg->origin_session_index = session_index;
Florin Coras40c07ce2020-07-16 20:46:17 -07001817 msg->vls_index = dst_vls_index;
hanlina3a48962020-07-13 11:09:15 +08001818 msg->session_index = dst_session_index;
Florin Coras40c07ce2020-07-16 20:46:17 -07001819
wanghanlindcacdc42020-12-28 16:19:05 +08001820 /* Try lock and handle rpcs if two threads send each other
1821 * clone requests at the same time.
1822 */
1823 wrk->rpc_done = VLS_RPC_STATE_INIT;
1824 while (!clib_spinlock_trylock (&vlsm->worker_rpc_lock))
1825 vcl_flush_mq_events ();
hanlina3a48962020-07-13 11:09:15 +08001826 ret = vcl_send_worker_rpc (dst_wrk_index, rpc, sizeof (data));
1827
Florin Corasff40d8f2020-08-11 22:05:28 -07001828 VDBG (1, "send session clone to wrk (session): %u (%u) -> %u (%u), ret=%d",
hanlina3a48962020-07-13 11:09:15 +08001829 dst_wrk_index, msg->session_index, msg->origin_vcl_wrk,
1830 msg->origin_session_index, ret);
wanghanlindcacdc42020-12-28 16:19:05 +08001831 while (!ret && wrk->rpc_done == VLS_RPC_STATE_INIT &&
1832 clib_time_now (&wrk->clib_time) < timeout)
hanlina3a48962020-07-13 11:09:15 +08001833 ;
wanghanlindcacdc42020-12-28 16:19:05 +08001834 clib_spinlock_unlock (&vlsm->worker_rpc_lock);
hanlina3a48962020-07-13 11:09:15 +08001835}
1836
1837void
1838vls_send_session_cleanup_rpc (vcl_worker_t * wrk,
1839 u32 dst_wrk_index, u32 dst_session_index)
1840{
1841 u8 data[sizeof (u8) + sizeof (vls_sess_cleanup_msg_t)];
1842 vls_sess_cleanup_msg_t *msg;
1843 vls_rpc_msg_t *rpc;
1844 int ret;
1845
1846 rpc = (vls_rpc_msg_t *) & data;
1847 rpc->type = VLS_RPC_SESS_CLEANUP;
1848 msg = (vls_sess_cleanup_msg_t *) & rpc->data;
1849 msg->origin_vcl_wrk = wrk->wrk_index;
1850 msg->session_index = dst_session_index;
1851
hanlina3a48962020-07-13 11:09:15 +08001852 ret = vcl_send_worker_rpc (dst_wrk_index, rpc, sizeof (data));
1853
Florin Corasff40d8f2020-08-11 22:05:28 -07001854 VDBG (1, "send session cleanup to wrk (session): %u (%u) from %u, ret=%d",
hanlina3a48962020-07-13 11:09:15 +08001855 dst_wrk_index, msg->session_index, msg->origin_vcl_wrk, ret);
Florin Coras40c07ce2020-07-16 20:46:17 -07001856}
1857
Florin Coras7baeb712019-01-04 17:05:43 -08001858int
1859vls_app_create (char *app_name)
1860{
1861 int rv;
Florin Coras0ef8ef22019-01-18 08:37:13 -08001862
Florin Coras7baeb712019-01-04 17:05:43 -08001863 if ((rv = vppcom_app_create (app_name)))
1864 return rv;
Florin Coras243edd52020-03-04 22:20:12 +00001865
Florin Coras2d675d72019-01-28 15:54:27 -08001866 vlsm = clib_mem_alloc (sizeof (vls_main_t));
1867 clib_memset (vlsm, 0, sizeof (*vlsm));
Florin Coras7baeb712019-01-04 17:05:43 -08001868 clib_rwlock_init (&vlsm->vls_table_lock);
Florin Coras243edd52020-03-04 22:20:12 +00001869 clib_rwlock_init (&vlsm->shared_data_lock);
wanghanlindcacdc42020-12-28 16:19:05 +08001870 clib_spinlock_init (&vlsm->worker_rpc_lock);
Florin Coras243edd52020-03-04 22:20:12 +00001871 pool_alloc (vlsm->workers, vcm->cfg.max_workers);
1872
Florin Corasf9240dc2019-01-15 08:03:17 -08001873 pthread_atfork (vls_app_pre_fork, vls_app_fork_parent_handler,
1874 vls_app_fork_child_handler);
Florin Coras0ef8ef22019-01-18 08:37:13 -08001875 atexit (vls_app_exit);
Florin Coras243edd52020-03-04 22:20:12 +00001876 vls_worker_alloc ();
Florin Coras2d675d72019-01-28 15:54:27 -08001877 vlsl->vls_wrk_index = vcl_get_worker_index ();
1878 vls_mt_locks_init ();
Florin Coras40c07ce2020-07-16 20:46:17 -07001879 vcm->wrk_rpc_fn = vls_rpc_handler;
Florin Coras7baeb712019-01-04 17:05:43 -08001880 return VPPCOM_OK;
1881}
1882
hanlin4266d4d2020-05-19 17:34:17 +08001883unsigned char
1884vls_use_eventfd (void)
1885{
1886 return vcm->cfg.use_mq_eventfd;
1887}
1888
hanlina3a48962020-07-13 11:09:15 +08001889unsigned char
1890vls_mt_wrk_supported (void)
1891{
Florin Corasff40d8f2020-08-11 22:05:28 -07001892 return vcm->cfg.mt_wrk_supported;
hanlina3a48962020-07-13 11:09:15 +08001893}
1894
1895int
1896vls_use_real_epoll (void)
1897{
1898 if (vcl_get_worker_index () == ~0)
1899 return 0;
1900
1901 return vcl_worker_get_current ()->vcl_needs_real_epoll;
1902}
1903
1904void
1905vls_register_vcl_worker (void)
1906{
wanghanlin492350e2020-09-11 17:19:32 +08001907 vls_mt_add ();
hanlina3a48962020-07-13 11:09:15 +08001908}
1909
Florin Coras7baeb712019-01-04 17:05:43 -08001910/*
1911 * fd.io coding-style-patch-verification: ON
1912 *
1913 * Local Variables:
1914 * eval: (c-set-style "gnu")
1915 * End:
1916 */