blob: 659f052a7205907b3e00813ed32324210be92800 [file] [log] [blame]
Ed Warnickecb9cada2015-12-08 15:45:58 -07001/*
2 * Copyright (c) 2015 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15#ifndef included_vlib_threads_h
16#define included_vlib_threads_h
17
18#include <vlib/main.h>
Tom Seidenberg6c81f5a2020-07-10 15:49:03 +000019#include <vppinfra/callback.h>
Pavel Kotucek1e765832016-09-23 08:54:14 +020020#include <linux/sched.h>
Ed Warnickecb9cada2015-12-08 15:45:58 -070021
Damjan Marion6a7acc22016-12-19 16:28:36 +010022extern vlib_main_t **vlib_mains;
Ed Warnickecb9cada2015-12-08 15:45:58 -070023
Dave Barach9b8ffd92016-07-08 08:13:45 -040024void vlib_set_thread_name (char *name);
Ed Warnickecb9cada2015-12-08 15:45:58 -070025
26/* arg is actually a vlib__thread_t * */
Dave Barach9b8ffd92016-07-08 08:13:45 -040027typedef void (vlib_thread_function_t) (void *arg);
Ed Warnickecb9cada2015-12-08 15:45:58 -070028
Dave Barach9b8ffd92016-07-08 08:13:45 -040029typedef struct vlib_thread_registration_
30{
Ed Warnickecb9cada2015-12-08 15:45:58 -070031 /* constructor generated list of thread registrations */
Dave Barach9b8ffd92016-07-08 08:13:45 -040032 struct vlib_thread_registration_ *next;
Ed Warnickecb9cada2015-12-08 15:45:58 -070033
34 /* config parameters */
Dave Barach9b8ffd92016-07-08 08:13:45 -040035 char *name;
36 char *short_name;
37 vlib_thread_function_t *function;
Ed Warnickecb9cada2015-12-08 15:45:58 -070038 uword mheap_size;
39 int fixed_count;
40 u32 count;
41 int no_data_structure_clone;
Bud Grise68adab92016-02-12 10:36:11 -050042 u32 frame_queue_nelts;
43
Ed Warnickecb9cada2015-12-08 15:45:58 -070044 /* All threads of this type run on pthreads */
45 int use_pthreads;
46 u32 first_index;
Dave Barach9b8ffd92016-07-08 08:13:45 -040047 uword *coremask;
Ed Warnickecb9cada2015-12-08 15:45:58 -070048} vlib_thread_registration_t;
49
Damjan Marion3f46baf2016-02-06 19:16:21 +010050/*
51 * Frames have their cpu / vlib_main_t index in the low-order N bits
52 * Make VLIB_MAX_CPUS a power-of-two, please...
Ed Warnickecb9cada2015-12-08 15:45:58 -070053 */
Damjan Marion3f46baf2016-02-06 19:16:21 +010054
Christophe Fontainefef15b42016-04-09 12:38:49 +090055#ifndef VLIB_MAX_CPUS
Damjan Marion3f46baf2016-02-06 19:16:21 +010056#define VLIB_MAX_CPUS 256
Christophe Fontainefef15b42016-04-09 12:38:49 +090057#endif
Ed Warnickecb9cada2015-12-08 15:45:58 -070058
Damjan Marionce8debf2016-02-06 19:16:21 +010059#if VLIB_MAX_CPUS > CLIB_MAX_MHEAPS
60#error Please increase number of per-cpu mheaps
61#endif
62
Dave Barach9b8ffd92016-07-08 08:13:45 -040063#define VLIB_CPU_MASK (VLIB_MAX_CPUS - 1) /* 0x3f, max */
Ed Warnickecb9cada2015-12-08 15:45:58 -070064#define VLIB_OFFSET_MASK (~VLIB_CPU_MASK)
65
Florin Coras93992a92017-05-24 18:03:56 -070066#define VLIB_LOG2_THREAD_STACK_SIZE (21)
Ed Warnickecb9cada2015-12-08 15:45:58 -070067#define VLIB_THREAD_STACK_SIZE (1<<VLIB_LOG2_THREAD_STACK_SIZE)
68
Dave Barach9b8ffd92016-07-08 08:13:45 -040069typedef enum
70{
71 VLIB_FRAME_QUEUE_ELT_DISPATCH_FRAME,
Ed Warnickecb9cada2015-12-08 15:45:58 -070072} vlib_frame_queue_msg_type_t;
73
Dave Barach9b8ffd92016-07-08 08:13:45 -040074typedef struct
75{
Damjan Marionb0d680b2016-11-04 14:41:44 +010076 CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
Ed Warnickecb9cada2015-12-08 15:45:58 -070077 volatile u32 valid;
Dave Barach9b8ffd92016-07-08 08:13:45 -040078 u32 msg_type;
Ed Warnickecb9cada2015-12-08 15:45:58 -070079 u32 n_vectors;
80 u32 last_n_vectors;
81
82 /* 256 * 4 = 1024 bytes, even mult of cache line size */
83 u32 buffer_index[VLIB_FRAME_SIZE];
Dave Barach9b8ffd92016-07-08 08:13:45 -040084}
85vlib_frame_queue_elt_t;
Ed Warnickecb9cada2015-12-08 15:45:58 -070086
Dave Barach9b8ffd92016-07-08 08:13:45 -040087typedef struct
88{
Ed Warnickecb9cada2015-12-08 15:45:58 -070089 /* First cache line */
Damjan Marionb0d680b2016-11-04 14:41:44 +010090 CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
Ed Warnickecb9cada2015-12-08 15:45:58 -070091 volatile u32 *wait_at_barrier;
92 volatile u32 *workers_at_barrier;
Ed Warnickecb9cada2015-12-08 15:45:58 -070093
94 /* Second Cache Line */
Damjan Marionb0d680b2016-11-04 14:41:44 +010095 CLIB_CACHE_LINE_ALIGN_MARK (cacheline1);
Ed Warnickecb9cada2015-12-08 15:45:58 -070096 void *thread_mheap;
Dave Barach9b8ffd92016-07-08 08:13:45 -040097 u8 *thread_stack;
98 void (*thread_function) (void *);
99 void *thread_function_arg;
100 i64 recursion_level;
101 elog_track_t elog_track;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700102 u32 instance_id;
103 vlib_thread_registration_t *registration;
104 u8 *name;
Bud Grise42f20062016-03-16 13:09:46 -0400105 u64 barrier_sync_count;
Dave Barach88c6e002018-09-30 15:54:06 -0400106 u8 barrier_elog_enabled;
Colin Tregenza Dancereb1ac172017-09-06 20:23:24 +0100107 const char *barrier_caller;
108 const char *barrier_context;
Colin Tregenza Dancer21596182017-09-04 15:27:49 +0100109 volatile u32 *node_reforks_required;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700110
111 long lwp;
Mohsin Kazmi5d64c782018-09-11 20:27:09 +0200112 int cpu_id;
113 int core_id;
Dave Baracha690fdb2020-01-21 12:34:55 -0500114 int numa_id;
Pavel Kotucek98765202016-10-07 08:37:28 +0200115 pthread_t thread_id;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700116} vlib_worker_thread_t;
117
Damjan Marion6a7acc22016-12-19 16:28:36 +0100118extern vlib_worker_thread_t *vlib_worker_threads;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700119
Dave Barach9b8ffd92016-07-08 08:13:45 -0400120typedef struct
121{
Ed Warnickecb9cada2015-12-08 15:45:58 -0700122 /* enqueue side */
Damjan Marionb0d680b2016-11-04 14:41:44 +0100123 CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700124 volatile u64 tail;
125 u64 enqueues;
126 u64 enqueue_ticks;
127 u64 enqueue_vectors;
128 u32 enqueue_full_events;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700129
130 /* dequeue side */
Damjan Marionb0d680b2016-11-04 14:41:44 +0100131 CLIB_CACHE_LINE_ALIGN_MARK (cacheline1);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700132 volatile u64 head;
133 u64 dequeues;
134 u64 dequeue_ticks;
135 u64 dequeue_vectors;
136 u64 trace;
137 u64 vector_threshold;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700138
139 /* dequeue hint to enqueue side */
Damjan Marionb0d680b2016-11-04 14:41:44 +0100140 CLIB_CACHE_LINE_ALIGN_MARK (cacheline2);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700141 volatile u64 head_hint;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700142
143 /* read-only, constant, shared */
Damjan Marionb0d680b2016-11-04 14:41:44 +0100144 CLIB_CACHE_LINE_ALIGN_MARK (cacheline3);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700145 vlib_frame_queue_elt_t *elts;
146 u32 nelts;
Dave Barach9b8ffd92016-07-08 08:13:45 -0400147}
148vlib_frame_queue_t;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700149
Damjan Marionaaef1eb2016-11-08 17:37:01 +0100150typedef struct
151{
Damjan Marion78fd7e82018-07-20 18:47:05 +0200152 vlib_frame_queue_elt_t **handoff_queue_elt_by_thread_index;
153 vlib_frame_queue_t **congested_handoff_queue_by_thread_index;
154} vlib_frame_queue_per_thread_data_t;
155
156typedef struct
157{
Damjan Marionaaef1eb2016-11-08 17:37:01 +0100158 u32 node_index;
Damjan Marion78fd7e82018-07-20 18:47:05 +0200159 u32 frame_queue_nelts;
160 u32 queue_hi_thresh;
161
Damjan Marionaaef1eb2016-11-08 17:37:01 +0100162 vlib_frame_queue_t **vlib_frame_queues;
Damjan Marion78fd7e82018-07-20 18:47:05 +0200163 vlib_frame_queue_per_thread_data_t *per_thread_data;
Damjan Marionaaef1eb2016-11-08 17:37:01 +0100164
165 /* for frame queue tracing */
166 frame_queue_trace_t *frame_queue_traces;
167 frame_queue_nelt_counter_t *frame_queue_histogram;
168} vlib_frame_queue_main_t;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700169
Dave Barach69128d02017-09-26 10:54:34 -0400170typedef struct
171{
172 uword node_index;
173 uword type_opaque;
174 uword data;
175} vlib_process_signal_event_mt_args_t;
176
Ed Warnickecb9cada2015-12-08 15:45:58 -0700177/* Called early, in thread 0's context */
Dave Barach9b8ffd92016-07-08 08:13:45 -0400178clib_error_t *vlib_thread_init (vlib_main_t * vm);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700179
Dave Barach9b8ffd92016-07-08 08:13:45 -0400180int vlib_frame_queue_enqueue (vlib_main_t * vm, u32 node_runtime_index,
181 u32 frame_queue_index, vlib_frame_t * frame,
182 vlib_frame_queue_msg_type_t type);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700183
Damjan Marione9d52d52017-03-09 15:42:26 +0100184int
185vlib_frame_queue_dequeue (vlib_main_t * vm, vlib_frame_queue_main_t * fqm);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700186
Dave Barach9b8ffd92016-07-08 08:13:45 -0400187void vlib_worker_thread_node_runtime_update (void);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700188
Dave Barach9b8ffd92016-07-08 08:13:45 -0400189void vlib_create_worker_threads (vlib_main_t * vm, int n,
190 void (*thread_function) (void *));
Ed Warnickecb9cada2015-12-08 15:45:58 -0700191
192void vlib_worker_thread_init (vlib_worker_thread_t * w);
Damjan Marionaaef1eb2016-11-08 17:37:01 +0100193u32 vlib_frame_queue_main_init (u32 node_index, u32 frame_queue_nelts);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700194
195/* Check for a barrier sync request every 30ms */
196#define BARRIER_SYNC_DELAY (0.030000)
197
198#if CLIB_DEBUG > 0
199/* long barrier timeout, for gdb... */
200#define BARRIER_SYNC_TIMEOUT (600.1)
201#else
202#define BARRIER_SYNC_TIMEOUT (1.0)
203#endif
204
Damjan Marion8343ee52019-02-26 17:15:48 +0100205#define vlib_worker_thread_barrier_sync(X) {vlib_worker_thread_barrier_sync_int(X, __FUNCTION__);}
Colin Tregenza Dancereb1ac172017-09-06 20:23:24 +0100206
Damjan Marion8343ee52019-02-26 17:15:48 +0100207void vlib_worker_thread_barrier_sync_int (vlib_main_t * vm,
208 const char *func_name);
Dave Barach9b8ffd92016-07-08 08:13:45 -0400209void vlib_worker_thread_barrier_release (vlib_main_t * vm);
Neale Ranns42845dd2020-05-26 13:12:17 +0000210u8 vlib_worker_thread_barrier_held (void);
Dave Barachc602b382019-06-03 19:48:22 -0400211void vlib_worker_thread_initial_barrier_sync_and_release (vlib_main_t * vm);
Colin Tregenza Dancer21596182017-09-04 15:27:49 +0100212void vlib_worker_thread_node_refork (void);
Neale Ranns42845dd2020-05-26 13:12:17 +0000213/**
214 * Wait until each of the workers has been once around the track
215 */
216void vlib_worker_wait_one_loop (void);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700217
Damjan Marion586afd72017-04-05 19:18:20 +0200218static_always_inline uword
219vlib_get_thread_index (void)
220{
Damjan Marionf55f9b82017-05-10 21:06:28 +0200221 return __os_thread_index;
Damjan Marion586afd72017-04-05 19:18:20 +0200222}
223
Dave Barach9b8ffd92016-07-08 08:13:45 -0400224always_inline void
225vlib_smp_unsafe_warning (void)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700226{
227 if (CLIB_DEBUG > 0)
228 {
Damjan Marion586afd72017-04-05 19:18:20 +0200229 if (vlib_get_thread_index ())
Dave Barach9b8ffd92016-07-08 08:13:45 -0400230 fformat (stderr, "%s: SMP unsafe warning...\n", __FUNCTION__);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700231 }
232}
233
Dave Barach9b8ffd92016-07-08 08:13:45 -0400234typedef enum
235{
236 VLIB_WORKER_THREAD_FORK_FIXUP_ILLEGAL = 0,
237 VLIB_WORKER_THREAD_FORK_FIXUP_NEW_SW_IF_INDEX,
Ed Warnickecb9cada2015-12-08 15:45:58 -0700238} vlib_fork_fixup_t;
239
240void vlib_worker_thread_fork_fixup (vlib_fork_fixup_t which);
241
Dave Barach80f54e22017-03-08 19:08:56 -0500242#define foreach_vlib_main(body) \
243do { \
244 vlib_main_t ** __vlib_mains = 0, *this_vlib_main; \
245 int ii; \
246 \
247 for (ii = 0; ii < vec_len (vlib_mains); ii++) \
248 { \
249 this_vlib_main = vlib_mains[ii]; \
Damjan Marionce359db2017-03-16 16:15:38 +0100250 ASSERT (ii == 0 || \
251 this_vlib_main->parked_at_barrier == 1); \
Dave Barach80f54e22017-03-08 19:08:56 -0500252 if (this_vlib_main) \
253 vec_add1 (__vlib_mains, this_vlib_main); \
254 } \
255 \
256 for (ii = 0; ii < vec_len (__vlib_mains); ii++) \
257 { \
258 this_vlib_main = __vlib_mains[ii]; \
259 /* body uses this_vlib_main... */ \
260 (body); \
261 } \
262 vec_free (__vlib_mains); \
Ed Warnickecb9cada2015-12-08 15:45:58 -0700263} while (0);
264
Pavel Kotucek1e765832016-09-23 08:54:14 +0200265#define foreach_sched_policy \
266 _(SCHED_OTHER, OTHER, "other") \
267 _(SCHED_BATCH, BATCH, "batch") \
268 _(SCHED_IDLE, IDLE, "idle") \
269 _(SCHED_FIFO, FIFO, "fifo") \
270 _(SCHED_RR, RR, "rr")
271
272typedef enum
273{
274#define _(v,f,s) SCHED_POLICY_##f = v,
275 foreach_sched_policy
276#undef _
277 SCHED_POLICY_N,
278} sched_policy_t;
279
Dave Barach9b8ffd92016-07-08 08:13:45 -0400280typedef struct
281{
Damjan Marion878c6092017-01-04 13:19:27 +0100282 clib_error_t *(*vlib_launch_thread_cb) (void *fp, vlib_worker_thread_t * w,
Mohsin Kazmi5d64c782018-09-11 20:27:09 +0200283 unsigned cpu_id);
284 clib_error_t *(*vlib_thread_set_lcore_cb) (u32 thread, u16 cpu);
Damjan Marion878c6092017-01-04 13:19:27 +0100285} vlib_thread_callbacks_t;
286
287typedef struct
288{
Ed Warnickecb9cada2015-12-08 15:45:58 -0700289 /* Link list of registrations, built by constructors */
Dave Barach9b8ffd92016-07-08 08:13:45 -0400290 vlib_thread_registration_t *next;
291
Ed Warnickecb9cada2015-12-08 15:45:58 -0700292 /* Vector of registrations, w/ non-data-structure clones at the top */
Dave Barach9b8ffd92016-07-08 08:13:45 -0400293 vlib_thread_registration_t **registrations;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700294
Dave Barach9b8ffd92016-07-08 08:13:45 -0400295 uword *thread_registrations_by_name;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700296
Dave Barach9b8ffd92016-07-08 08:13:45 -0400297 vlib_worker_thread_t *worker_threads;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700298
Dave Barach9b8ffd92016-07-08 08:13:45 -0400299 /*
300 * Launch all threads as pthreads,
301 * not eal_rte_launch (strict affinity) threads
Ed Warnickecb9cada2015-12-08 15:45:58 -0700302 */
303 int use_pthreads;
304
305 /* Number of vlib_main / vnet_main clones */
306 u32 n_vlib_mains;
307
308 /* Number of thread stacks to create */
309 u32 n_thread_stacks;
310
311 /* Number of pthreads */
312 u32 n_pthreads;
313
Damjan Marion878c6092017-01-04 13:19:27 +0100314 /* Number of threads */
315 u32 n_threads;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700316
317 /* Number of cores to skip, must match the core mask */
318 u32 skip_cores;
319
320 /* Thread prefix name */
321 u8 *thread_prefix;
322
323 /* main thread lcore */
Damjan Marion858151f2018-07-11 10:51:00 +0200324 u32 main_lcore;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700325
326 /* Bitmap of available CPU cores */
Dave Barach9b8ffd92016-07-08 08:13:45 -0400327 uword *cpu_core_bitmap;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700328
329 /* Bitmap of available CPU sockets (NUMA nodes) */
Dave Barach9b8ffd92016-07-08 08:13:45 -0400330 uword *cpu_socket_bitmap;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700331
Damjan Marionaaef1eb2016-11-08 17:37:01 +0100332 /* Worker handoff queues */
333 vlib_frame_queue_main_t *frame_queue_mains;
Damjan Marion0f8ecf02016-06-27 08:30:30 +0200334
335 /* worker thread initialization barrier */
336 volatile u32 worker_thread_release;
337
Pavel Kotucek1e765832016-09-23 08:54:14 +0200338 /* scheduling policy */
339 u32 sched_policy;
340
341 /* scheduling policy priority */
342 u32 sched_priority;
343
Damjan Marion878c6092017-01-04 13:19:27 +0100344 /* callbacks */
345 vlib_thread_callbacks_t cb;
346 int extern_thread_mgmt;
Dave Baracha690fdb2020-01-21 12:34:55 -0500347
348 /* NUMA-bound heap size */
349 uword numa_heap_size;
350
Ed Warnickecb9cada2015-12-08 15:45:58 -0700351} vlib_thread_main_t;
352
Damjan Marion6a7acc22016-12-19 16:28:36 +0100353extern vlib_thread_main_t vlib_thread_main;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700354
Damjan Marionce359db2017-03-16 16:15:38 +0100355#include <vlib/global_funcs.h>
356
Ed Warnickecb9cada2015-12-08 15:45:58 -0700357#define VLIB_REGISTER_THREAD(x,...) \
358 __VA_ARGS__ vlib_thread_registration_t x; \
359static void __vlib_add_thread_registration_##x (void) \
360 __attribute__((__constructor__)) ; \
361static void __vlib_add_thread_registration_##x (void) \
362{ \
363 vlib_thread_main_t * tm = &vlib_thread_main; \
364 x.next = tm->next; \
365 tm->next = &x; \
366} \
Damjan Marion72d2c4f2018-04-05 21:32:29 +0200367static void __vlib_rm_thread_registration_##x (void) \
368 __attribute__((__destructor__)) ; \
369static void __vlib_rm_thread_registration_##x (void) \
370{ \
371 vlib_thread_main_t * tm = &vlib_thread_main; \
372 VLIB_REMOVE_FROM_LINKED_LIST (tm->next, &x, next); \
373} \
Dave Barach9b8ffd92016-07-08 08:13:45 -0400374__VA_ARGS__ vlib_thread_registration_t x
Ed Warnickecb9cada2015-12-08 15:45:58 -0700375
Damjan Marion64034362016-11-07 22:19:55 +0100376always_inline u32
377vlib_num_workers ()
378{
379 return vlib_thread_main.n_vlib_mains - 1;
380}
381
382always_inline u32
Damjan Marion586afd72017-04-05 19:18:20 +0200383vlib_get_worker_thread_index (u32 worker_index)
Damjan Marion64034362016-11-07 22:19:55 +0100384{
385 return worker_index + 1;
386}
387
Damjan Marion3a4ed392016-11-08 13:20:42 +0100388always_inline u32
Damjan Marion586afd72017-04-05 19:18:20 +0200389vlib_get_worker_index (u32 thread_index)
Damjan Marion3a4ed392016-11-08 13:20:42 +0100390{
Damjan Marion586afd72017-04-05 19:18:20 +0200391 return thread_index - 1;
Damjan Marion3a4ed392016-11-08 13:20:42 +0100392}
393
394always_inline u32
395vlib_get_current_worker_index ()
396{
Damjan Marion586afd72017-04-05 19:18:20 +0200397 return vlib_get_thread_index () - 1;
Damjan Marion3a4ed392016-11-08 13:20:42 +0100398}
399
Damjan Marionce359db2017-03-16 16:15:38 +0100400static inline void
401vlib_worker_thread_barrier_check (void)
402{
403 if (PREDICT_FALSE (*vlib_worker_threads->wait_at_barrier))
404 {
Dave Barach88c6e002018-09-30 15:54:06 -0400405 vlib_main_t *vm = vlib_get_main ();
406 u32 thread_index = vm->thread_index;
407 f64 t = vlib_time_now (vm);
408
Tom Seidenberg6c81f5a2020-07-10 15:49:03 +0000409 if (PREDICT_FALSE (vec_len (vm->barrier_perf_callbacks) != 0))
410 clib_call_callbacks (vm->barrier_perf_callbacks, vm,
411 vm->clib_time.last_cpu_time, 0 /* enter */ );
412
Dave Barach88c6e002018-09-30 15:54:06 -0400413 if (PREDICT_FALSE (vlib_worker_threads->barrier_elog_enabled))
414 {
415 vlib_worker_thread_t *w = vlib_worker_threads + thread_index;
416 /* *INDENT-OFF* */
417 ELOG_TYPE_DECLARE (e) = {
418 .format = "barrier-wait-thread-%d",
419 .format_args = "i4",
420 };
421 /* *INDENT-ON* */
422
423 struct
424 {
425 u32 thread_index;
426 } __clib_packed *ed;
427
428 ed = ELOG_TRACK_DATA (&vlib_global_main.elog_main, e,
429 w->elog_track);
430 ed->thread_index = thread_index;
431 }
432
Damjan Marionce359db2017-03-16 16:15:38 +0100433 if (CLIB_DEBUG > 0)
434 {
435 vm = vlib_get_main ();
436 vm->parked_at_barrier = 1;
437 }
Alexander Kabaevfeda5452019-11-01 18:26:20 -0400438 clib_atomic_fetch_add (vlib_worker_threads->workers_at_barrier, 1);
Damjan Marionce359db2017-03-16 16:15:38 +0100439 while (*vlib_worker_threads->wait_at_barrier)
440 ;
Dave Baracha4324a92019-02-19 17:05:30 -0500441
442 /*
443 * Recompute the offset from thread-0 time.
444 * Note that vlib_time_now adds vm->time_offset, so
445 * clear it first. Save the resulting idea of "now", to
446 * see how well we're doing. See show_clock_command_fn(...)
447 */
448 {
449 f64 now;
450 vm->time_offset = 0.0;
451 now = vlib_time_now (vm);
452 vm->time_offset = vlib_global_main.time_last_barrier_release - now;
453 vm->time_last_barrier_release = vlib_time_now (vm);
454 }
455
Damjan Marionce359db2017-03-16 16:15:38 +0100456 if (CLIB_DEBUG > 0)
457 vm->parked_at_barrier = 0;
Sirshak Das2f6d7bb2018-10-03 22:53:51 +0000458 clib_atomic_fetch_add (vlib_worker_threads->workers_at_barrier, -1);
Colin Tregenza Dancer21596182017-09-04 15:27:49 +0100459
460 if (PREDICT_FALSE (*vlib_worker_threads->node_reforks_required))
461 {
Dave Barach88c6e002018-09-30 15:54:06 -0400462 if (PREDICT_FALSE (vlib_worker_threads->barrier_elog_enabled))
463 {
464 t = vlib_time_now (vm) - t;
465 vlib_worker_thread_t *w = vlib_worker_threads + thread_index;
466 /* *INDENT-OFF* */
467 ELOG_TYPE_DECLARE (e) = {
468 .format = "barrier-refork-thread-%d",
469 .format_args = "i4",
470 };
471 /* *INDENT-ON* */
472
473 struct
474 {
475 u32 thread_index;
476 } __clib_packed *ed;
477
478 ed = ELOG_TRACK_DATA (&vlib_global_main.elog_main, e,
479 w->elog_track);
480 ed->thread_index = thread_index;
481 }
482
Colin Tregenza Dancer21596182017-09-04 15:27:49 +0100483 vlib_worker_thread_node_refork ();
Sirshak Das2f6d7bb2018-10-03 22:53:51 +0000484 clib_atomic_fetch_add (vlib_worker_threads->node_reforks_required,
485 -1);
Colin Tregenza Dancer21596182017-09-04 15:27:49 +0100486 while (*vlib_worker_threads->node_reforks_required)
487 ;
488 }
Dave Barach88c6e002018-09-30 15:54:06 -0400489 if (PREDICT_FALSE (vlib_worker_threads->barrier_elog_enabled))
490 {
491 t = vlib_time_now (vm) - t;
492 vlib_worker_thread_t *w = vlib_worker_threads + thread_index;
493 /* *INDENT-OFF* */
494 ELOG_TYPE_DECLARE (e) = {
495 .format = "barrier-released-thread-%d: %dus",
496 .format_args = "i4i4",
497 };
498 /* *INDENT-ON* */
499
500 struct
501 {
502 u32 thread_index;
503 u32 duration;
504 } __clib_packed *ed;
505
506 ed = ELOG_TRACK_DATA (&vlib_global_main.elog_main, e,
507 w->elog_track);
508 ed->thread_index = thread_index;
509 ed->duration = (int) (1000000.0 * t);
510 }
Tom Seidenberg6c81f5a2020-07-10 15:49:03 +0000511
512 if (PREDICT_FALSE (vec_len (vm->barrier_perf_callbacks) != 0))
513 clib_call_callbacks (vm->barrier_perf_callbacks, vm,
514 vm->clib_time.last_cpu_time, 1 /* leave */ );
Damjan Marionce359db2017-03-16 16:15:38 +0100515 }
516}
517
Damjan Marion64034362016-11-07 22:19:55 +0100518always_inline vlib_main_t *
519vlib_get_worker_vlib_main (u32 worker_index)
520{
521 vlib_main_t *vm;
522 vlib_thread_main_t *tm = &vlib_thread_main;
523 ASSERT (worker_index < tm->n_vlib_mains - 1);
524 vm = vlib_mains[worker_index + 1];
525 ASSERT (vm);
526 return vm;
527}
528
Florin Coras568ebc72018-09-18 16:12:50 -0700529static inline u8
530vlib_thread_is_main_w_barrier (void)
531{
532 return (!vlib_num_workers ()
533 || ((vlib_get_thread_index () == 0
534 && vlib_worker_threads->wait_at_barrier[0])));
535}
536
Damjan Marionaaef1eb2016-11-08 17:37:01 +0100537static inline void
538vlib_put_frame_queue_elt (vlib_frame_queue_elt_t * hf)
539{
540 CLIB_MEMORY_BARRIER ();
541 hf->valid = 1;
542}
543
544static inline vlib_frame_queue_elt_t *
545vlib_get_frame_queue_elt (u32 frame_queue_index, u32 index)
546{
547 vlib_frame_queue_t *fq;
548 vlib_frame_queue_elt_t *elt;
549 vlib_thread_main_t *tm = &vlib_thread_main;
550 vlib_frame_queue_main_t *fqm =
551 vec_elt_at_index (tm->frame_queue_mains, frame_queue_index);
552 u64 new_tail;
553
554 fq = fqm->vlib_frame_queues[index];
555 ASSERT (fq);
556
Sirshak Das2f6d7bb2018-10-03 22:53:51 +0000557 new_tail = clib_atomic_add_fetch (&fq->tail, 1);
Damjan Marionaaef1eb2016-11-08 17:37:01 +0100558
559 /* Wait until a ring slot is available */
560 while (new_tail >= fq->head_hint + fq->nelts)
561 vlib_worker_thread_barrier_check ();
562
563 elt = fq->elts + (new_tail & (fq->nelts - 1));
564
565 /* this would be very bad... */
566 while (elt->valid)
567 ;
568
569 elt->msg_type = VLIB_FRAME_QUEUE_ELT_DISPATCH_FRAME;
570 elt->last_n_vectors = elt->n_vectors = 0;
571
572 return elt;
573}
574
575static inline vlib_frame_queue_t *
576is_vlib_frame_queue_congested (u32 frame_queue_index,
577 u32 index,
578 u32 queue_hi_thresh,
579 vlib_frame_queue_t **
580 handoff_queue_by_worker_index)
581{
582 vlib_frame_queue_t *fq;
583 vlib_thread_main_t *tm = &vlib_thread_main;
584 vlib_frame_queue_main_t *fqm =
585 vec_elt_at_index (tm->frame_queue_mains, frame_queue_index);
586
587 fq = handoff_queue_by_worker_index[index];
588 if (fq != (vlib_frame_queue_t *) (~0))
589 return fq;
590
591 fq = fqm->vlib_frame_queues[index];
592 ASSERT (fq);
593
594 if (PREDICT_FALSE (fq->tail >= (fq->head_hint + queue_hi_thresh)))
595 {
596 /* a valid entry in the array will indicate the queue has reached
597 * the specified threshold and is congested
598 */
599 handoff_queue_by_worker_index[index] = fq;
600 fq->enqueue_full_events++;
601 return fq;
602 }
603
604 return NULL;
605}
606
607static inline vlib_frame_queue_elt_t *
608vlib_get_worker_handoff_queue_elt (u32 frame_queue_index,
609 u32 vlib_worker_index,
610 vlib_frame_queue_elt_t **
611 handoff_queue_elt_by_worker_index)
612{
613 vlib_frame_queue_elt_t *elt;
614
615 if (handoff_queue_elt_by_worker_index[vlib_worker_index])
616 return handoff_queue_elt_by_worker_index[vlib_worker_index];
617
618 elt = vlib_get_frame_queue_elt (frame_queue_index, vlib_worker_index);
619
620 handoff_queue_elt_by_worker_index[vlib_worker_index] = elt;
621
622 return elt;
623}
624
Damjan Marion586afd72017-04-05 19:18:20 +0200625u8 *vlib_thread_stack_init (uword thread_index);
Damjan Marion878c6092017-01-04 13:19:27 +0100626int vlib_thread_cb_register (struct vlib_main_t *vm,
627 vlib_thread_callbacks_t * cb);
Dave Barach69128d02017-09-26 10:54:34 -0400628extern void *rpc_call_main_thread_cb_fn;
629
630void
631vlib_process_signal_event_mt_helper (vlib_process_signal_event_mt_args_t *
632 args);
633void vlib_rpc_call_main_thread (void *function, u8 * args, u32 size);
Dave Baracha690fdb2020-01-21 12:34:55 -0500634void vlib_get_thread_core_numa (vlib_worker_thread_t * w, unsigned cpu_id);
635
636
Ed Warnickecb9cada2015-12-08 15:45:58 -0700637#endif /* included_vlib_threads_h */
Dave Barach9b8ffd92016-07-08 08:13:45 -0400638
639/*
640 * fd.io coding-style-patch-verification: ON
641 *
642 * Local Variables:
643 * eval: (c-set-style "gnu")
644 * End:
645 */