blob: b675b8819d2380cefc67329192c72fe7c8076e2f [file] [log] [blame]
Ed Warnickecb9cada2015-12-08 15:45:58 -07001/*
2 * Copyright (c) 2015 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
Christophe Fontainefef15b42016-04-09 12:38:49 +090015#define _GNU_SOURCE
Christophe Fontainefef15b42016-04-09 12:38:49 +090016
Ed Warnickecb9cada2015-12-08 15:45:58 -070017#include <signal.h>
18#include <math.h>
19#include <vppinfra/format.h>
Dave Barach19718002020-03-11 10:31:36 -040020#include <vppinfra/time_range.h>
Damjan Marion94100532020-11-06 23:25:57 +010021#include <vppinfra/interrupt.h>
Mohsin Kazmi5d64c782018-09-11 20:27:09 +020022#include <vppinfra/linux/sysfs.h>
Ed Warnickecb9cada2015-12-08 15:45:58 -070023#include <vlib/vlib.h>
24
25#include <vlib/threads.h>
Ed Warnickecb9cada2015-12-08 15:45:58 -070026
Damjan Marion8973b072022-03-01 15:51:18 +010027#include <vlib/stats/stats.h>
Ole Troan233e4682019-05-16 15:01:34 +020028
Dave Barach9b8ffd92016-07-08 08:13:45 -040029u32
30vl (void *p)
Ed Warnickecb9cada2015-12-08 15:45:58 -070031{
32 return vec_len (p);
33}
34
Damjan Marion6a7acc22016-12-19 16:28:36 +010035vlib_worker_thread_t *vlib_worker_threads;
Ed Warnickecb9cada2015-12-08 15:45:58 -070036vlib_thread_main_t vlib_thread_main;
37
Colin Tregenza Dancereb1ac172017-09-06 20:23:24 +010038/*
39 * Barrier tracing can be enabled on a normal build to collect information
40 * on barrier use, including timings and call stacks. Deliberately not
41 * keyed off CLIB_DEBUG, because that can add significant overhead which
42 * imapacts observed timings.
43 */
44
Colin Tregenza Dancereb1ac172017-09-06 20:23:24 +010045static inline void
46barrier_trace_sync (f64 t_entry, f64 t_open, f64 t_closed)
47{
Dave Barach88c6e002018-09-30 15:54:06 -040048 if (!vlib_worker_threads->barrier_elog_enabled)
49 return;
50
Damjan Marion14361002021-03-11 12:17:33 +010051 ELOG_TYPE_DECLARE (e) = {
52 .format = "bar-trace-%s-#%d",
53 .format_args = "T4i4",
54 };
55
Colin Tregenza Dancereb1ac172017-09-06 20:23:24 +010056 struct
57 {
Dave Barach88c6e002018-09-30 15:54:06 -040058 u32 caller, count, t_entry, t_open, t_closed;
Colin Tregenza Dancereb1ac172017-09-06 20:23:24 +010059 } *ed = 0;
60
61 ed = ELOG_DATA (&vlib_global_main.elog_main, e);
62 ed->count = (int) vlib_worker_threads[0].barrier_sync_count;
Dave Barachb09f4d02019-07-15 16:00:03 -040063 ed->caller = elog_string (&vlib_global_main.elog_main,
64 (char *) vlib_worker_threads[0].barrier_caller);
Colin Tregenza Dancereb1ac172017-09-06 20:23:24 +010065 ed->t_entry = (int) (1000000.0 * t_entry);
66 ed->t_open = (int) (1000000.0 * t_open);
67 ed->t_closed = (int) (1000000.0 * t_closed);
68}
69
70static inline void
71barrier_trace_sync_rec (f64 t_entry)
72{
Dave Barach88c6e002018-09-30 15:54:06 -040073 if (!vlib_worker_threads->barrier_elog_enabled)
74 return;
75
Damjan Marion14361002021-03-11 12:17:33 +010076 ELOG_TYPE_DECLARE (e) = {
77 .format = "bar-syncrec-%s-#%d",
78 .format_args = "T4i4",
79 };
80
Colin Tregenza Dancereb1ac172017-09-06 20:23:24 +010081 struct
82 {
Dave Barach88c6e002018-09-30 15:54:06 -040083 u32 caller, depth;
Colin Tregenza Dancereb1ac172017-09-06 20:23:24 +010084 } *ed = 0;
85
86 ed = ELOG_DATA (&vlib_global_main.elog_main, e);
87 ed->depth = (int) vlib_worker_threads[0].recursion_level - 1;
Dave Barachb09f4d02019-07-15 16:00:03 -040088 ed->caller = elog_string (&vlib_global_main.elog_main,
89 (char *) vlib_worker_threads[0].barrier_caller);
Colin Tregenza Dancereb1ac172017-09-06 20:23:24 +010090}
91
92static inline void
93barrier_trace_release_rec (f64 t_entry)
94{
Dave Barach88c6e002018-09-30 15:54:06 -040095 if (!vlib_worker_threads->barrier_elog_enabled)
96 return;
97
Damjan Marion14361002021-03-11 12:17:33 +010098 ELOG_TYPE_DECLARE (e) = {
99 .format = "bar-relrrec-#%d",
100 .format_args = "i4",
101 };
102
Colin Tregenza Dancereb1ac172017-09-06 20:23:24 +0100103 struct
104 {
Dave Barach88c6e002018-09-30 15:54:06 -0400105 u32 depth;
Colin Tregenza Dancereb1ac172017-09-06 20:23:24 +0100106 } *ed = 0;
107
108 ed = ELOG_DATA (&vlib_global_main.elog_main, e);
Colin Tregenza Dancereb1ac172017-09-06 20:23:24 +0100109 ed->depth = (int) vlib_worker_threads[0].recursion_level;
110}
111
112static inline void
113barrier_trace_release (f64 t_entry, f64 t_closed_total, f64 t_update_main)
114{
Dave Barach88c6e002018-09-30 15:54:06 -0400115 if (!vlib_worker_threads->barrier_elog_enabled)
116 return;
117
Damjan Marion14361002021-03-11 12:17:33 +0100118 ELOG_TYPE_DECLARE (e) = {
119 .format = "bar-rel-#%d-e%d-u%d-t%d",
120 .format_args = "i4i4i4i4",
121 };
122
Colin Tregenza Dancereb1ac172017-09-06 20:23:24 +0100123 struct
124 {
Dave Barach88c6e002018-09-30 15:54:06 -0400125 u32 count, t_entry, t_update_main, t_closed_total;
Colin Tregenza Dancereb1ac172017-09-06 20:23:24 +0100126 } *ed = 0;
127
128 ed = ELOG_DATA (&vlib_global_main.elog_main, e);
129 ed->t_entry = (int) (1000000.0 * t_entry);
130 ed->t_update_main = (int) (1000000.0 * t_update_main);
131 ed->t_closed_total = (int) (1000000.0 * t_closed_total);
132 ed->count = (int) vlib_worker_threads[0].barrier_sync_count;
133
134 /* Reset context for next trace */
135 vlib_worker_threads[0].barrier_context = NULL;
136}
Colin Tregenza Dancereb1ac172017-09-06 20:23:24 +0100137
Ed Warnickecb9cada2015-12-08 15:45:58 -0700138uword
Damjan Marionf55f9b82017-05-10 21:06:28 +0200139os_get_nthreads (void)
Dave Barach01d86c72016-08-08 15:13:42 -0400140{
Dave Barach2180bac2019-05-10 15:25:10 -0400141 return vec_len (vlib_thread_stacks);
Dave Barach01d86c72016-08-08 15:13:42 -0400142}
143
Ed Warnickecb9cada2015-12-08 15:45:58 -0700144void
145vlib_set_thread_name (char *name)
146{
147 int pthread_setname_np (pthread_t __target_thread, const char *__name);
Dave Barachb2a6e252016-07-27 10:00:58 -0400148 int rv;
Dave Barach9b8ffd92016-07-08 08:13:45 -0400149 pthread_t thread = pthread_self ();
Ed Warnickecb9cada2015-12-08 15:45:58 -0700150
Dave Barach9b8ffd92016-07-08 08:13:45 -0400151 if (thread)
Dave Barachb2a6e252016-07-27 10:00:58 -0400152 {
153 rv = pthread_setname_np (thread, name);
154 if (rv)
Ed Warnicke853e7202016-08-12 11:42:26 -0700155 clib_warning ("pthread_setname_np returned %d", rv);
Dave Barachb2a6e252016-07-27 10:00:58 -0400156 }
Ed Warnickecb9cada2015-12-08 15:45:58 -0700157}
158
Dave Barach9b8ffd92016-07-08 08:13:45 -0400159static int
160sort_registrations_by_no_clone (void *a0, void *a1)
161{
162 vlib_thread_registration_t **tr0 = a0;
163 vlib_thread_registration_t **tr1 = a1;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700164
Dave Barach9b8ffd92016-07-08 08:13:45 -0400165 return ((i32) ((*tr0)->no_data_structure_clone)
166 - ((i32) ((*tr1)->no_data_structure_clone)));
Ed Warnickecb9cada2015-12-08 15:45:58 -0700167}
168
Ed Warnickecb9cada2015-12-08 15:45:58 -0700169
170/* Called early in the init sequence */
171
172clib_error_t *
173vlib_thread_init (vlib_main_t * vm)
174{
Dave Barach9b8ffd92016-07-08 08:13:45 -0400175 vlib_thread_main_t *tm = &vlib_thread_main;
176 vlib_worker_thread_t *w;
177 vlib_thread_registration_t *tr;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700178 u32 n_vlib_mains = 1;
179 u32 first_index = 1;
180 u32 i;
Dave Barach9b8ffd92016-07-08 08:13:45 -0400181 uword *avail_cpu;
Damjan Marion8973b072022-03-01 15:51:18 +0100182 u32 stats_num_worker_threads_dir_index;
183
184 stats_num_worker_threads_dir_index =
185 vlib_stats_add_gauge ("/sys/num_worker_threads");
186 ASSERT (stats_num_worker_threads_dir_index != ~0);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700187
188 /* get bitmaps of active cpu cores and sockets */
189 tm->cpu_core_bitmap =
Damjan Marion01914ce2017-09-14 19:04:50 +0200190 clib_sysfs_list_to_bitmap ("/sys/devices/system/cpu/online");
Ed Warnickecb9cada2015-12-08 15:45:58 -0700191 tm->cpu_socket_bitmap =
Damjan Marion01914ce2017-09-14 19:04:50 +0200192 clib_sysfs_list_to_bitmap ("/sys/devices/system/node/online");
Ed Warnickecb9cada2015-12-08 15:45:58 -0700193
Dave Barach9b8ffd92016-07-08 08:13:45 -0400194 avail_cpu = clib_bitmap_dup (tm->cpu_core_bitmap);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700195
196 /* skip cores */
Dave Barach9b8ffd92016-07-08 08:13:45 -0400197 for (i = 0; i < tm->skip_cores; i++)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700198 {
Dave Barach9b8ffd92016-07-08 08:13:45 -0400199 uword c = clib_bitmap_first_set (avail_cpu);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700200 if (c == ~0)
Dave Barach9b8ffd92016-07-08 08:13:45 -0400201 return clib_error_return (0, "no available cpus to skip");
Ed Warnickecb9cada2015-12-08 15:45:58 -0700202
Dave Barach9b8ffd92016-07-08 08:13:45 -0400203 avail_cpu = clib_bitmap_set (avail_cpu, c, 0);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700204 }
205
206 /* grab cpu for main thread */
Benoît Ganne8b153de2021-12-09 18:24:21 +0100207 if (tm->main_lcore != ~0)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700208 {
Dave Barach9b8ffd92016-07-08 08:13:45 -0400209 if (clib_bitmap_get (avail_cpu, tm->main_lcore) == 0)
210 return clib_error_return (0, "cpu %u is not available to be used"
211 " for the main thread", tm->main_lcore);
Benoît Ganne8b153de2021-12-09 18:24:21 +0100212 avail_cpu = clib_bitmap_set (avail_cpu, tm->main_lcore, 0);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700213 }
Ed Warnickecb9cada2015-12-08 15:45:58 -0700214
215 /* assume that there is socket 0 only if there is no data from sysfs */
216 if (!tm->cpu_socket_bitmap)
Dave Barach9b8ffd92016-07-08 08:13:45 -0400217 tm->cpu_socket_bitmap = clib_bitmap_set (0, 0, 1);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700218
Christophe Fontainefef15b42016-04-09 12:38:49 +0900219 /* pin main thread to main_lcore */
Benoît Ganne8b153de2021-12-09 18:24:21 +0100220 if (tm->main_lcore != ~0)
221 {
222 cpu_set_t cpuset;
223 CPU_ZERO (&cpuset);
224 CPU_SET (tm->main_lcore, &cpuset);
hsandid832342e2023-12-20 15:41:54 +0100225 if (pthread_setaffinity_np (pthread_self (), sizeof (cpu_set_t),
226 &cpuset))
227 {
228 return clib_error_return (0, "could not pin main thread to cpu %u",
229 tm->main_lcore);
230 }
Benoît Ganne8b153de2021-12-09 18:24:21 +0100231 }
Christophe Fontainefef15b42016-04-09 12:38:49 +0900232
Dave Barach2180bac2019-05-10 15:25:10 -0400233 /* Set up thread 0 */
234 vec_validate_aligned (vlib_worker_threads, 0, CLIB_CACHE_LINE_BYTES);
Damjan Marion8bea5892022-04-04 22:40:45 +0200235 vec_set_len (vlib_worker_threads, 1);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700236 w = vlib_worker_threads;
Dave Barach9b8ffd92016-07-08 08:13:45 -0400237 w->thread_mheap = clib_mem_get_heap ();
Ed Warnickecb9cada2015-12-08 15:45:58 -0700238 w->thread_stack = vlib_thread_stacks[0];
Mohsin Kazmi5d64c782018-09-11 20:27:09 +0200239 w->cpu_id = tm->main_lcore;
Dave Barach9b8ffd92016-07-08 08:13:45 -0400240 w->lwp = syscall (SYS_gettid);
Pavel Kotucek98765202016-10-07 08:37:28 +0200241 w->thread_id = pthread_self ();
Ed Warnickecb9cada2015-12-08 15:45:58 -0700242 tm->n_vlib_mains = 1;
243
Jon Loeligerf617b142020-01-30 18:37:47 -0600244 vlib_get_thread_core_numa (w, w->cpu_id);
245
Pavel Kotucek1e765832016-09-23 08:54:14 +0200246 if (tm->sched_policy != ~0)
247 {
248 struct sched_param sched_param;
249 if (!sched_getparam (w->lwp, &sched_param))
250 {
251 if (tm->sched_priority != ~0)
252 sched_param.sched_priority = tm->sched_priority;
253 sched_setscheduler (w->lwp, tm->sched_policy, &sched_param);
254 }
255 }
256
Ed Warnickecb9cada2015-12-08 15:45:58 -0700257 /* assign threads to cores and set n_vlib_mains */
258 tr = tm->next;
259
260 while (tr)
261 {
262 vec_add1 (tm->registrations, tr);
263 tr = tr->next;
264 }
265
Dave Barach9b8ffd92016-07-08 08:13:45 -0400266 vec_sort_with_function (tm->registrations, sort_registrations_by_no_clone);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700267
268 for (i = 0; i < vec_len (tm->registrations); i++)
269 {
270 int j;
271 tr = tm->registrations[i];
272 tr->first_index = first_index;
273 first_index += tr->count;
274 n_vlib_mains += (tr->no_data_structure_clone == 0) ? tr->count : 0;
275
276 /* construct coremask */
277 if (tr->use_pthreads || !tr->count)
Dave Barach9b8ffd92016-07-08 08:13:45 -0400278 continue;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700279
280 if (tr->coremask)
Dave Barach9b8ffd92016-07-08 08:13:45 -0400281 {
282 uword c;
283 /* *INDENT-OFF* */
Damjan Marionf0ca1e82020-12-13 23:26:56 +0100284 clib_bitmap_foreach (c, tr->coremask) {
Ed Warnickecb9cada2015-12-08 15:45:58 -0700285 if (clib_bitmap_get(avail_cpu, c) == 0)
286 return clib_error_return (0, "cpu %u is not available to be used"
287 " for the '%s' thread",c, tr->name);
288
289 avail_cpu = clib_bitmap_set(avail_cpu, c, 0);
Damjan Marionf0ca1e82020-12-13 23:26:56 +0100290 }
Dave Barach2180bac2019-05-10 15:25:10 -0400291 /* *INDENT-ON* */
Dave Barach9b8ffd92016-07-08 08:13:45 -0400292 }
Ed Warnickecb9cada2015-12-08 15:45:58 -0700293 else
Dave Barach9b8ffd92016-07-08 08:13:45 -0400294 {
295 for (j = 0; j < tr->count; j++)
296 {
Vladimir Isaev2ed42042020-03-17 12:56:31 +0300297 /* Do not use CPU 0 by default - leave it to the host and IRQs */
298 uword avail_c0 = clib_bitmap_get (avail_cpu, 0);
299 avail_cpu = clib_bitmap_set (avail_cpu, 0, 0);
300
Dave Barach9b8ffd92016-07-08 08:13:45 -0400301 uword c = clib_bitmap_first_set (avail_cpu);
Vladimir Isaev2ed42042020-03-17 12:56:31 +0300302 /* Use CPU 0 as a last resort */
303 if (c == ~0 && avail_c0)
304 {
305 c = 0;
306 avail_c0 = 0;
307 }
308
Dave Barach9b8ffd92016-07-08 08:13:45 -0400309 if (c == ~0)
310 return clib_error_return (0,
311 "no available cpus to be used for"
hsandid832342e2023-12-20 15:41:54 +0100312 " the '%s' thread #%u",
313 tr->name, tr->count);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700314
Vladimir Isaev2ed42042020-03-17 12:56:31 +0300315 avail_cpu = clib_bitmap_set (avail_cpu, 0, avail_c0);
Dave Barach9b8ffd92016-07-08 08:13:45 -0400316 avail_cpu = clib_bitmap_set (avail_cpu, c, 0);
317 tr->coremask = clib_bitmap_set (tr->coremask, c, 1);
318 }
319 }
Ed Warnickecb9cada2015-12-08 15:45:58 -0700320 }
321
Dave Barach9b8ffd92016-07-08 08:13:45 -0400322 clib_bitmap_free (avail_cpu);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700323
324 tm->n_vlib_mains = n_vlib_mains;
Damjan Marion8973b072022-03-01 15:51:18 +0100325 vlib_stats_set_gauge (stats_num_worker_threads_dir_index, n_vlib_mains - 1);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700326
Dave Barach2180bac2019-05-10 15:25:10 -0400327 /*
328 * Allocate the remaining worker threads, and thread stack vector slots
329 * from now on, calls to os_get_nthreads() will return the correct
330 * answer.
331 */
Dave Barach9b8ffd92016-07-08 08:13:45 -0400332 vec_validate_aligned (vlib_worker_threads, first_index - 1,
333 CLIB_CACHE_LINE_BYTES);
Dave Barach2180bac2019-05-10 15:25:10 -0400334 vec_validate (vlib_thread_stacks, vec_len (vlib_worker_threads) - 1);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700335 return 0;
336}
337
Dave Barach9b8ffd92016-07-08 08:13:45 -0400338vlib_frame_queue_t *
339vlib_frame_queue_alloc (int nelts)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700340{
Dave Barach9b8ffd92016-07-08 08:13:45 -0400341 vlib_frame_queue_t *fq;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700342
Dave Barach9b8ffd92016-07-08 08:13:45 -0400343 fq = clib_mem_alloc_aligned (sizeof (*fq), CLIB_CACHE_LINE_BYTES);
Dave Barachb7b92992018-10-17 10:38:51 -0400344 clib_memset (fq, 0, sizeof (*fq));
Ed Warnickecb9cada2015-12-08 15:45:58 -0700345 fq->nelts = nelts;
Damjan Marionc0d9ca72021-05-11 09:39:24 +0200346 fq->vector_threshold = 2 * VLIB_FRAME_SIZE;
Dave Barach9b8ffd92016-07-08 08:13:45 -0400347 vec_validate_aligned (fq->elts, nelts - 1, CLIB_CACHE_LINE_BYTES);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700348
Damjan Marionc0d9ca72021-05-11 09:39:24 +0200349 if (nelts & (nelts - 1))
Dave Barach9b8ffd92016-07-08 08:13:45 -0400350 {
Damjan Marionc0d9ca72021-05-11 09:39:24 +0200351 fformat (stderr, "FATAL: nelts MUST be a power of 2\n");
352 abort ();
Dave Barach9b8ffd92016-07-08 08:13:45 -0400353 }
354
Ed Warnickecb9cada2015-12-08 15:45:58 -0700355 return (fq);
356}
357
358void vl_msg_api_handler_no_free (void *) __attribute__ ((weak));
Dave Barach9b8ffd92016-07-08 08:13:45 -0400359void
360vl_msg_api_handler_no_free (void *v)
361{
362}
Ed Warnickecb9cada2015-12-08 15:45:58 -0700363
Ed Warnickecb9cada2015-12-08 15:45:58 -0700364/* To be called by vlib worker threads upon startup */
Dave Barach9b8ffd92016-07-08 08:13:45 -0400365void
366vlib_worker_thread_init (vlib_worker_thread_t * w)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700367{
Dave Barach9b8ffd92016-07-08 08:13:45 -0400368 vlib_thread_main_t *tm = vlib_get_thread_main ();
369
Dave Barachfdf49442016-12-20 12:48:14 -0500370 /*
371 * Note: disabling signals in worker threads as follows
372 * prevents the api post-mortem dump scheme from working
373 * {
374 * sigset_t s;
375 * sigfillset (&s);
376 * pthread_sigmask (SIG_SETMASK, &s, 0);
377 * }
378 */
Ed Warnickecb9cada2015-12-08 15:45:58 -0700379
380 clib_mem_set_heap (w->thread_mheap);
381
Dave Barach9b8ffd92016-07-08 08:13:45 -0400382 if (vec_len (tm->thread_prefix) && w->registration->short_name)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700383 {
Dave Barach9b8ffd92016-07-08 08:13:45 -0400384 w->name = format (0, "%v_%s_%d%c", tm->thread_prefix,
385 w->registration->short_name, w->instance_id, '\0');
386 vlib_set_thread_name ((char *) w->name);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700387 }
388
389 if (!w->registration->use_pthreads)
390 {
391
392 /* Initial barrier sync, for both worker and i/o threads */
Sirshak Das2f6d7bb2018-10-03 22:53:51 +0000393 clib_atomic_fetch_add (vlib_worker_threads->workers_at_barrier, 1);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700394
395 while (*vlib_worker_threads->wait_at_barrier)
Dave Barach9b8ffd92016-07-08 08:13:45 -0400396 ;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700397
Sirshak Das2f6d7bb2018-10-03 22:53:51 +0000398 clib_atomic_fetch_add (vlib_worker_threads->workers_at_barrier, -1);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700399 }
400}
401
Dave Barach9b8ffd92016-07-08 08:13:45 -0400402void *
403vlib_worker_thread_bootstrap_fn (void *arg)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700404{
Ed Warnickecb9cada2015-12-08 15:45:58 -0700405 vlib_worker_thread_t *w = arg;
Dave Barach9b8ffd92016-07-08 08:13:45 -0400406
407 w->lwp = syscall (SYS_gettid);
Pavel Kotucek98765202016-10-07 08:37:28 +0200408 w->thread_id = pthread_self ();
Ed Warnickecb9cada2015-12-08 15:45:58 -0700409
Damjan Marionf55f9b82017-05-10 21:06:28 +0200410 __os_thread_index = w - vlib_worker_threads;
Damjan Marion586afd72017-04-05 19:18:20 +0200411
Damjan Marion7bf23172022-03-28 01:47:33 +0200412 if (CLIB_DEBUG > 0)
413 {
414 void *frame_addr = __builtin_frame_address (0);
415 if (frame_addr < (void *) w->thread_stack ||
416 frame_addr > (void *) w->thread_stack + VLIB_THREAD_STACK_SIZE)
417 {
418 /* heap is not set yet */
419 fprintf (stderr, "thread stack is not set properly\n");
420 exit (1);
421 }
422 }
Dave Barach37579c32021-07-27 09:27:07 -0400423
Damjan Marion7bf23172022-03-28 01:47:33 +0200424 w->thread_function (arg);
425
426 return 0;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700427}
428
Dave Baracha690fdb2020-01-21 12:34:55 -0500429void
430vlib_get_thread_core_numa (vlib_worker_thread_t * w, unsigned cpu_id)
Mohsin Kazmi5d64c782018-09-11 20:27:09 +0200431{
432 const char *sys_cpu_path = "/sys/devices/system/cpu/cpu";
Lijian.Zhang4fbb9da2020-02-14 15:16:49 +0800433 const char *sys_node_path = "/sys/devices/system/node/node";
434 clib_bitmap_t *nbmp = 0, *cbmp = 0;
435 u32 node;
Mohsin Kazmi5d64c782018-09-11 20:27:09 +0200436 u8 *p = 0;
Dave Baracha690fdb2020-01-21 12:34:55 -0500437 int core_id = -1, numa_id = -1;
Mohsin Kazmi5d64c782018-09-11 20:27:09 +0200438
439 p = format (p, "%s%u/topology/core_id%c", sys_cpu_path, cpu_id, 0);
440 clib_sysfs_read ((char *) p, "%d", &core_id);
441 vec_reset_length (p);
Lijian.Zhang4fbb9da2020-02-14 15:16:49 +0800442
443 /* *INDENT-OFF* */
444 clib_sysfs_read ("/sys/devices/system/node/online", "%U",
445 unformat_bitmap_list, &nbmp);
Damjan Marionf0ca1e82020-12-13 23:26:56 +0100446 clib_bitmap_foreach (node, nbmp) {
Lijian.Zhang4fbb9da2020-02-14 15:16:49 +0800447 p = format (p, "%s%u/cpulist%c", sys_node_path, node, 0);
448 clib_sysfs_read ((char *) p, "%U", unformat_bitmap_list, &cbmp);
449 if (clib_bitmap_get (cbmp, cpu_id))
450 numa_id = node;
451 vec_reset_length (cbmp);
452 vec_reset_length (p);
Damjan Marionf0ca1e82020-12-13 23:26:56 +0100453 }
Lijian.Zhang4fbb9da2020-02-14 15:16:49 +0800454 /* *INDENT-ON* */
455 vec_free (nbmp);
456 vec_free (cbmp);
Mohsin Kazmi5d64c782018-09-11 20:27:09 +0200457 vec_free (p);
458
459 w->core_id = core_id;
Dave Baracha690fdb2020-01-21 12:34:55 -0500460 w->numa_id = numa_id;
Mohsin Kazmi5d64c782018-09-11 20:27:09 +0200461}
462
Damjan Marion878c6092017-01-04 13:19:27 +0100463static clib_error_t *
Mohsin Kazmi5d64c782018-09-11 20:27:09 +0200464vlib_launch_thread_int (void *fp, vlib_worker_thread_t * w, unsigned cpu_id)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700465{
Damjan Marion57d1ec02020-09-16 21:15:44 +0200466 clib_mem_main_t *mm = &clib_mem_main;
Damjan Marion878c6092017-01-04 13:19:27 +0100467 vlib_thread_main_t *tm = &vlib_thread_main;
Damjan Mariona2b7a022021-12-28 20:32:20 +0100468 pthread_t worker;
Damjan Marion7bf23172022-03-28 01:47:33 +0200469 pthread_attr_t attr;
Damjan Mariona2b7a022021-12-28 20:32:20 +0100470 cpu_set_t cpuset;
Dave Barach9b8ffd92016-07-08 08:13:45 -0400471 void *(*fp_arg) (void *) = fp;
Dave Baracha690fdb2020-01-21 12:34:55 -0500472 void *numa_heap;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700473
Mohsin Kazmi5d64c782018-09-11 20:27:09 +0200474 w->cpu_id = cpu_id;
Dave Baracha690fdb2020-01-21 12:34:55 -0500475 vlib_get_thread_core_numa (w, cpu_id);
Dave Baracha690fdb2020-01-21 12:34:55 -0500476
477 /* Set up NUMA-bound heap if indicated */
Damjan Marion57d1ec02020-09-16 21:15:44 +0200478 if (mm->per_numa_mheaps[w->numa_id] == 0)
Dave Baracha690fdb2020-01-21 12:34:55 -0500479 {
480 /* If the user requested a NUMA heap, create it... */
481 if (tm->numa_heap_size)
482 {
Damjan Marion2bc1af52020-10-02 15:01:12 +0200483 clib_mem_set_numa_affinity (w->numa_id, 1 /* force */ );
484 numa_heap = clib_mem_create_heap (0 /* DIY */ , tm->numa_heap_size,
485 1 /* is_locked */ ,
486 "numa %u heap", w->numa_id);
487 clib_mem_set_default_numa_affinity ();
Damjan Marion57d1ec02020-09-16 21:15:44 +0200488 mm->per_numa_mheaps[w->numa_id] = numa_heap;
Dave Baracha690fdb2020-01-21 12:34:55 -0500489 }
490 else
491 {
492 /* Or, use the main heap */
Damjan Marion57d1ec02020-09-16 21:15:44 +0200493 mm->per_numa_mheaps[w->numa_id] = w->thread_mheap;
Dave Baracha690fdb2020-01-21 12:34:55 -0500494 }
495 }
496
Dave Barach9b8ffd92016-07-08 08:13:45 -0400497 CPU_ZERO (&cpuset);
Mohsin Kazmi5d64c782018-09-11 20:27:09 +0200498 CPU_SET (cpu_id, &cpuset);
Christophe Fontainefef15b42016-04-09 12:38:49 +0900499
Damjan Marion7bf23172022-03-28 01:47:33 +0200500 if (pthread_attr_init (&attr))
501 return clib_error_return_unix (0, "pthread_attr_init");
502
503 if (pthread_attr_setstack (&attr, w->thread_stack,
504 VLIB_THREAD_STACK_SIZE))
505 return clib_error_return_unix (0, "pthread_attr_setstack");
506
507 if (pthread_create (&worker, &attr, fp_arg, (void *) w))
Damjan Marion878c6092017-01-04 13:19:27 +0100508 return clib_error_return_unix (0, "pthread_create");
509
510 if (pthread_setaffinity_np (worker, sizeof (cpu_set_t), &cpuset))
511 return clib_error_return_unix (0, "pthread_setaffinity_np");
512
Damjan Marion7bf23172022-03-28 01:47:33 +0200513 if (pthread_attr_destroy (&attr))
514 return clib_error_return_unix (0, "pthread_attr_destroy");
515
Damjan Marion878c6092017-01-04 13:19:27 +0100516 return 0;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700517}
518
Dave Barach9b8ffd92016-07-08 08:13:45 -0400519static clib_error_t *
520start_workers (vlib_main_t * vm)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700521{
Damjan Marionfd8deb42021-03-06 12:26:28 +0100522 vlib_global_main_t *vgm = vlib_get_global_main ();
Damjan Marion66c85832022-03-14 13:04:38 +0100523 vlib_main_t *fvm = vlib_get_first_main ();
Ed Warnickecb9cada2015-12-08 15:45:58 -0700524 int i, j;
525 vlib_worker_thread_t *w;
526 vlib_main_t *vm_clone;
527 void *oldheap;
Dave Barach9b8ffd92016-07-08 08:13:45 -0400528 vlib_thread_main_t *tm = &vlib_thread_main;
529 vlib_thread_registration_t *tr;
530 vlib_node_runtime_t *rt;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700531 u32 n_vlib_mains = tm->n_vlib_mains;
532 u32 worker_thread_index;
Damjan Marion66c85832022-03-14 13:04:38 +0100533 u32 stats_err_entry_index = fvm->error_main.stats_err_entry_index;
Damjan Marionbfa75d62020-10-06 17:46:06 +0200534 clib_mem_heap_t *main_heap = clib_mem_get_per_cpu_heap ();
Ole Troana606d922021-05-05 09:23:17 +0200535 vlib_stats_register_mem_heap (main_heap);
Dave Barach9b8ffd92016-07-08 08:13:45 -0400536
Ed Warnickecb9cada2015-12-08 15:45:58 -0700537 vec_reset_length (vlib_worker_threads);
538
539 /* Set up the main thread */
540 vec_add2_aligned (vlib_worker_threads, w, 1, CLIB_CACHE_LINE_BYTES);
Dave Barach3e7deb12016-02-05 16:29:53 -0500541 w->elog_track.name = "main thread";
Damjan Marionf553a2c2021-03-26 13:45:37 +0100542 elog_track_register (vlib_get_elog_main (), &w->elog_track);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700543
Dave Barach9b8ffd92016-07-08 08:13:45 -0400544 if (vec_len (tm->thread_prefix))
Ed Warnickecb9cada2015-12-08 15:45:58 -0700545 {
Dave Barach9b8ffd92016-07-08 08:13:45 -0400546 w->name = format (0, "%v_main%c", tm->thread_prefix, '\0');
547 vlib_set_thread_name ((char *) w->name);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700548 }
549
Damjan Marionfd8deb42021-03-06 12:26:28 +0100550 vgm->elog_main.lock =
Dave Barach9b8ffd92016-07-08 08:13:45 -0400551 clib_mem_alloc_aligned (CLIB_CACHE_LINE_BYTES, CLIB_CACHE_LINE_BYTES);
Damjan Marionfd8deb42021-03-06 12:26:28 +0100552 vgm->elog_main.lock[0] = 0;
Dave Barach9b8ffd92016-07-08 08:13:45 -0400553
Tom Seidenberg6c81f5a2020-07-10 15:49:03 +0000554 clib_callback_data_init (&vm->vlib_node_runtime_perf_callbacks,
555 &vm->worker_thread_main_loop_callback_lock);
556
Damjan Marionfd8deb42021-03-06 12:26:28 +0100557 vec_validate_aligned (vgm->vlib_mains, n_vlib_mains - 1,
558 CLIB_CACHE_LINE_BYTES);
Damjan Marion8bea5892022-04-04 22:40:45 +0200559 vec_set_len (vgm->vlib_mains, 0);
Damjan Marionfd8deb42021-03-06 12:26:28 +0100560 vec_add1_aligned (vgm->vlib_mains, vm, CLIB_CACHE_LINE_BYTES);
Dave Barach1c556d12020-10-02 11:35:44 -0400561
Ed Warnickecb9cada2015-12-08 15:45:58 -0700562 if (n_vlib_mains > 1)
563 {
Dave Barach9b8ffd92016-07-08 08:13:45 -0400564 vlib_worker_threads->wait_at_barrier =
565 clib_mem_alloc_aligned (sizeof (u32), CLIB_CACHE_LINE_BYTES);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700566 vlib_worker_threads->workers_at_barrier =
Dave Barach9b8ffd92016-07-08 08:13:45 -0400567 clib_mem_alloc_aligned (sizeof (u32), CLIB_CACHE_LINE_BYTES);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700568
Colin Tregenza Dancer21596182017-09-04 15:27:49 +0100569 vlib_worker_threads->node_reforks_required =
570 clib_mem_alloc_aligned (sizeof (u32), CLIB_CACHE_LINE_BYTES);
571
Dave Barachf6c68d72018-11-01 08:12:52 -0400572 /* We'll need the rpc vector lock... */
573 clib_spinlock_init (&vm->pending_rpc_lock);
574
Ed Warnickecb9cada2015-12-08 15:45:58 -0700575 /* Ask for an initial barrier sync */
576 *vlib_worker_threads->workers_at_barrier = 0;
577 *vlib_worker_threads->wait_at_barrier = 1;
578
Colin Tregenza Dancer21596182017-09-04 15:27:49 +0100579 /* Without update or refork */
580 *vlib_worker_threads->node_reforks_required = 0;
Damjan Marionfd8deb42021-03-06 12:26:28 +0100581 vgm->need_vlib_worker_thread_node_runtime_update = 0;
Colin Tregenza Dancer21596182017-09-04 15:27:49 +0100582
Colin Tregenza Dancereb1ac172017-09-06 20:23:24 +0100583 /* init timing */
584 vm->barrier_epoch = 0;
585 vm->barrier_no_close_before = 0;
586
Ed Warnickecb9cada2015-12-08 15:45:58 -0700587 worker_thread_index = 1;
Tom Seidenberg6c81f5a2020-07-10 15:49:03 +0000588 clib_spinlock_init (&vm->worker_thread_main_loop_callback_lock);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700589
Dave Barach9b8ffd92016-07-08 08:13:45 -0400590 for (i = 0; i < vec_len (tm->registrations); i++)
591 {
592 vlib_node_main_t *nm, *nm_clone;
Dave Barach9b8ffd92016-07-08 08:13:45 -0400593 int k;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700594
Dave Barach9b8ffd92016-07-08 08:13:45 -0400595 tr = tm->registrations[i];
Ed Warnickecb9cada2015-12-08 15:45:58 -0700596
Dave Barach9b8ffd92016-07-08 08:13:45 -0400597 if (tr->count == 0)
598 continue;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700599
Dave Barach9b8ffd92016-07-08 08:13:45 -0400600 for (k = 0; k < tr->count; k++)
601 {
Colin Tregenza Dancer21596182017-09-04 15:27:49 +0100602 vlib_node_t *n;
Damjan Marion66c85832022-03-14 13:04:38 +0100603 u64 **c;
Colin Tregenza Dancer21596182017-09-04 15:27:49 +0100604
Dave Barach9b8ffd92016-07-08 08:13:45 -0400605 vec_add2 (vlib_worker_threads, w, 1);
Dave Barach6a5adc32018-07-04 10:56:23 -0400606 /* Currently unused, may not really work */
Dave Barach9b8ffd92016-07-08 08:13:45 -0400607 if (tr->mheap_size)
Damjan Marion4537c302020-09-28 19:03:37 +0200608 w->thread_mheap = clib_mem_create_heap (0, tr->mheap_size,
609 /* unlocked */ 0,
610 "%s%d heap",
611 tr->name, k);
Dave Barach9b8ffd92016-07-08 08:13:45 -0400612 else
613 w->thread_mheap = main_heap;
Damjan Marion586afd72017-04-05 19:18:20 +0200614
615 w->thread_stack =
616 vlib_thread_stack_init (w - vlib_worker_threads);
Dave Barach9b8ffd92016-07-08 08:13:45 -0400617 w->thread_function = tr->function;
618 w->thread_function_arg = w;
619 w->instance_id = k;
620 w->registration = tr;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700621
Dave Barach9b8ffd92016-07-08 08:13:45 -0400622 w->elog_track.name =
623 (char *) format (0, "%s %d", tr->name, k + 1);
624 vec_add1 (w->elog_track.name, 0);
Damjan Marionf553a2c2021-03-26 13:45:37 +0100625 elog_track_register (vlib_get_elog_main (), &w->elog_track);
Bud Grise68adab92016-02-12 10:36:11 -0500626
Dave Barach9b8ffd92016-07-08 08:13:45 -0400627 if (tr->no_data_structure_clone)
628 continue;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700629
Dave Barach9b8ffd92016-07-08 08:13:45 -0400630 /* Fork vlib_global_main et al. Look for bugs here */
631 oldheap = clib_mem_set_heap (w->thread_mheap);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700632
Damjan Marionbe3f4d52018-03-27 21:06:10 +0200633 vm_clone = clib_mem_alloc_aligned (sizeof (*vm_clone),
634 CLIB_CACHE_LINE_BYTES);
Damjan Marionfd8deb42021-03-06 12:26:28 +0100635 clib_memcpy (vm_clone, vlib_get_first_main (),
636 sizeof (*vm_clone));
Ed Warnickecb9cada2015-12-08 15:45:58 -0700637
Damjan Marion586afd72017-04-05 19:18:20 +0200638 vm_clone->thread_index = worker_thread_index;
Dave Barach2877eee2017-12-15 12:22:57 -0500639 vm_clone->pending_rpc_requests = 0;
640 vec_validate (vm_clone->pending_rpc_requests, 0);
Damjan Marion8bea5892022-04-04 22:40:45 +0200641 vec_set_len (vm_clone->pending_rpc_requests, 0);
Dave Barachb7b92992018-10-17 10:38:51 -0400642 clib_memset (&vm_clone->random_buffer, 0,
643 sizeof (vm_clone->random_buffer));
Tom Seidenberg6c81f5a2020-07-10 15:49:03 +0000644 clib_spinlock_init
645 (&vm_clone->worker_thread_main_loop_callback_lock);
646 clib_callback_data_init
647 (&vm_clone->vlib_node_runtime_perf_callbacks,
648 &vm_clone->worker_thread_main_loop_callback_lock);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700649
Damjan Marionfd8deb42021-03-06 12:26:28 +0100650 nm = &vlib_get_first_main ()->node_main;
Dave Barach9b8ffd92016-07-08 08:13:45 -0400651 nm_clone = &vm_clone->node_main;
652 /* fork next frames array, preserving node runtime indices */
Damjan Marionbe3f4d52018-03-27 21:06:10 +0200653 nm_clone->next_frames = vec_dup_aligned (nm->next_frames,
654 CLIB_CACHE_LINE_BYTES);
Dave Barach9b8ffd92016-07-08 08:13:45 -0400655 for (j = 0; j < vec_len (nm_clone->next_frames); j++)
656 {
657 vlib_next_frame_t *nf = &nm_clone->next_frames[j];
658 u32 save_node_runtime_index;
659 u32 save_flags;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700660
Dave Barach9b8ffd92016-07-08 08:13:45 -0400661 save_node_runtime_index = nf->node_runtime_index;
662 save_flags = nf->flags & VLIB_FRAME_NO_FREE_AFTER_DISPATCH;
663 vlib_next_frame_init (nf);
664 nf->node_runtime_index = save_node_runtime_index;
665 nf->flags = save_flags;
666 }
Ed Warnickecb9cada2015-12-08 15:45:58 -0700667
Dave Barach9b8ffd92016-07-08 08:13:45 -0400668 /* fork the frame dispatch queue */
669 nm_clone->pending_frames = 0;
Dave Barach53fe4a72019-01-26 09:50:26 -0500670 vec_validate (nm_clone->pending_frames, 10);
Damjan Marion8bea5892022-04-04 22:40:45 +0200671 vec_set_len (nm_clone->pending_frames, 0);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700672
Dave Barach9b8ffd92016-07-08 08:13:45 -0400673 /* fork nodes */
674 nm_clone->nodes = 0;
Colin Tregenza Dancer21596182017-09-04 15:27:49 +0100675
676 /* Allocate all nodes in single block for speed */
677 n = clib_mem_alloc_no_fail (vec_len (nm->nodes) * sizeof (*n));
678
Dave Barach9b8ffd92016-07-08 08:13:45 -0400679 for (j = 0; j < vec_len (nm->nodes); j++)
680 {
Dave Barach9b8ffd92016-07-08 08:13:45 -0400681 clib_memcpy (n, nm->nodes[j], sizeof (*n));
682 /* none of the copied nodes have enqueue rights given out */
683 n->owner_node_index = VLIB_INVALID_NODE_INDEX;
Dave Barachb7b92992018-10-17 10:38:51 -0400684 clib_memset (&n->stats_total, 0, sizeof (n->stats_total));
685 clib_memset (&n->stats_last_clear, 0,
686 sizeof (n->stats_last_clear));
Dave Barach9b8ffd92016-07-08 08:13:45 -0400687 vec_add1 (nm_clone->nodes, n);
Colin Tregenza Dancer21596182017-09-04 15:27:49 +0100688 n++;
Dave Barach9b8ffd92016-07-08 08:13:45 -0400689 }
690 nm_clone->nodes_by_type[VLIB_NODE_TYPE_INTERNAL] =
Damjan Marionbe3f4d52018-03-27 21:06:10 +0200691 vec_dup_aligned (nm->nodes_by_type[VLIB_NODE_TYPE_INTERNAL],
692 CLIB_CACHE_LINE_BYTES);
JingLiuZTE30af5da2017-07-24 10:53:31 +0800693 vec_foreach (rt,
694 nm_clone->nodes_by_type[VLIB_NODE_TYPE_INTERNAL])
Damjan Marione9f929b2017-03-16 11:32:09 +0100695 {
696 vlib_node_t *n = vlib_get_node (vm, rt->node_index);
Damjan Marione9f929b2017-03-16 11:32:09 +0100697 /* copy initial runtime_data from node */
Damjan Marionb6f93a12017-03-16 17:46:41 +0100698 if (n->runtime_data && n->runtime_data_bytes > 0)
Damjan Marione9f929b2017-03-16 11:32:09 +0100699 clib_memcpy (rt->runtime_data, n->runtime_data,
Damjan Marionb6f93a12017-03-16 17:46:41 +0100700 clib_min (VLIB_NODE_RUNTIME_DATA_SIZE,
701 n->runtime_data_bytes));
Damjan Marione9f929b2017-03-16 11:32:09 +0100702 }
Ed Warnickecb9cada2015-12-08 15:45:58 -0700703
Dave Barach9b8ffd92016-07-08 08:13:45 -0400704 nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT] =
Damjan Marionbe3f4d52018-03-27 21:06:10 +0200705 vec_dup_aligned (nm->nodes_by_type[VLIB_NODE_TYPE_INPUT],
706 CLIB_CACHE_LINE_BYTES);
Damjan Marion94100532020-11-06 23:25:57 +0100707 clib_interrupt_init (
Damjan Marioncc8249c2023-07-23 14:24:22 +0200708 &nm_clone->input_node_interrupts,
Damjan Marion94100532020-11-06 23:25:57 +0100709 vec_len (nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT]));
Damjan Marioncc8249c2023-07-23 14:24:22 +0200710 clib_interrupt_init (
711 &nm_clone->pre_input_node_interrupts,
712 vec_len (nm_clone->nodes_by_type[VLIB_NODE_TYPE_PRE_INPUT]));
Dave Barach9b8ffd92016-07-08 08:13:45 -0400713 vec_foreach (rt, nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT])
Damjan Marione9f929b2017-03-16 11:32:09 +0100714 {
715 vlib_node_t *n = vlib_get_node (vm, rt->node_index);
Damjan Marione9f929b2017-03-16 11:32:09 +0100716 /* copy initial runtime_data from node */
Damjan Marionb6f93a12017-03-16 17:46:41 +0100717 if (n->runtime_data && n->runtime_data_bytes > 0)
Damjan Marione9f929b2017-03-16 11:32:09 +0100718 clib_memcpy (rt->runtime_data, n->runtime_data,
Damjan Marionb6f93a12017-03-16 17:46:41 +0100719 clib_min (VLIB_NODE_RUNTIME_DATA_SIZE,
720 n->runtime_data_bytes));
Damjan Marione9f929b2017-03-16 11:32:09 +0100721 }
Ed Warnickecb9cada2015-12-08 15:45:58 -0700722
Dave Barach53fe4a72019-01-26 09:50:26 -0500723 nm_clone->nodes_by_type[VLIB_NODE_TYPE_PRE_INPUT] =
724 vec_dup_aligned (nm->nodes_by_type[VLIB_NODE_TYPE_PRE_INPUT],
725 CLIB_CACHE_LINE_BYTES);
726 vec_foreach (rt,
727 nm_clone->nodes_by_type[VLIB_NODE_TYPE_PRE_INPUT])
728 {
729 vlib_node_t *n = vlib_get_node (vm, rt->node_index);
Dave Barach53fe4a72019-01-26 09:50:26 -0500730 /* copy initial runtime_data from node */
731 if (n->runtime_data && n->runtime_data_bytes > 0)
732 clib_memcpy (rt->runtime_data, n->runtime_data,
733 clib_min (VLIB_NODE_RUNTIME_DATA_SIZE,
734 n->runtime_data_bytes));
735 }
736
Damjan Marionbe3f4d52018-03-27 21:06:10 +0200737 nm_clone->processes = vec_dup_aligned (nm->processes,
738 CLIB_CACHE_LINE_BYTES);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700739
Dave Barach593eedf2019-03-10 09:44:51 -0400740 /* Create per-thread frame freelist */
Damjan Marionb32bd702021-12-23 17:05:02 +0100741 nm_clone->frame_sizes = 0;
Dave Barach687c9022019-07-23 10:22:31 -0400742 nm_clone->node_by_error = nm->node_by_error;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700743
Dave Barach9b8ffd92016-07-08 08:13:45 -0400744 /* Packet trace buffers are guaranteed to be empty, nothing to do here */
Damjan Marionbc20bdf2015-12-17 14:28:18 +0100745
Dave Barach9b8ffd92016-07-08 08:13:45 -0400746 clib_mem_set_heap (oldheap);
Damjan Marionfd8deb42021-03-06 12:26:28 +0100747 vec_add1_aligned (vgm->vlib_mains, vm_clone,
748 CLIB_CACHE_LINE_BYTES);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700749
Ole Troan233e4682019-05-16 15:01:34 +0200750 /* Switch to the stats segment ... */
Damjan Marion66c85832022-03-14 13:04:38 +0100751 vlib_stats_validate (stats_err_entry_index, worker_thread_index,
752 vec_len (fvm->error_main.counters) - 1);
753 c = vlib_stats_get_entry_data_pointer (stats_err_entry_index);
754 vm_clone->error_main.counters = c[worker_thread_index];
Ole Troan233e4682019-05-16 15:01:34 +0200755
Damjan Marionfd8deb42021-03-06 12:26:28 +0100756 vm_clone->error_main.counters_last_clear = vec_dup_aligned (
757 vlib_get_first_main ()->error_main.counters_last_clear,
758 CLIB_CACHE_LINE_BYTES);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700759
Dave Barach9b8ffd92016-07-08 08:13:45 -0400760 worker_thread_index++;
761 }
762 }
Ed Warnickecb9cada2015-12-08 15:45:58 -0700763 }
764 else
765 {
766 /* only have non-data-structure copy threads to create... */
Dave Barach9b8ffd92016-07-08 08:13:45 -0400767 for (i = 0; i < vec_len (tm->registrations); i++)
768 {
769 tr = tm->registrations[i];
Ed Warnickecb9cada2015-12-08 15:45:58 -0700770
Dave Barach9b8ffd92016-07-08 08:13:45 -0400771 for (j = 0; j < tr->count; j++)
772 {
773 vec_add2 (vlib_worker_threads, w, 1);
774 if (tr->mheap_size)
Dave Barach6a5adc32018-07-04 10:56:23 -0400775 {
Damjan Marion4537c302020-09-28 19:03:37 +0200776 w->thread_mheap = clib_mem_create_heap (0, tr->mheap_size,
777 /* locked */ 0,
778 "%s%d heap",
779 tr->name, j);
Dave Barach6a5adc32018-07-04 10:56:23 -0400780 }
Dave Barach9b8ffd92016-07-08 08:13:45 -0400781 else
782 w->thread_mheap = main_heap;
Damjan Marion586afd72017-04-05 19:18:20 +0200783 w->thread_stack =
784 vlib_thread_stack_init (w - vlib_worker_threads);
Dave Barach9b8ffd92016-07-08 08:13:45 -0400785 w->thread_function = tr->function;
786 w->thread_function_arg = w;
787 w->instance_id = j;
788 w->elog_track.name =
789 (char *) format (0, "%s %d", tr->name, j + 1);
790 w->registration = tr;
791 vec_add1 (w->elog_track.name, 0);
Damjan Marionf553a2c2021-03-26 13:45:37 +0100792 elog_track_register (vlib_get_elog_main (), &w->elog_track);
Dave Barach9b8ffd92016-07-08 08:13:45 -0400793 }
794 }
Ed Warnickecb9cada2015-12-08 15:45:58 -0700795 }
796
797 worker_thread_index = 1;
798
799 for (i = 0; i < vec_len (tm->registrations); i++)
800 {
Damjan Marion878c6092017-01-04 13:19:27 +0100801 clib_error_t *err;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700802 int j;
803
804 tr = tm->registrations[i];
805
806 if (tr->use_pthreads || tm->use_pthreads)
Dave Barach9b8ffd92016-07-08 08:13:45 -0400807 {
808 for (j = 0; j < tr->count; j++)
809 {
hsandid832342e2023-12-20 15:41:54 +0100810
Dave Barach9b8ffd92016-07-08 08:13:45 -0400811 w = vlib_worker_threads + worker_thread_index++;
Damjan Marion878c6092017-01-04 13:19:27 +0100812 err = vlib_launch_thread_int (vlib_worker_thread_bootstrap_fn,
813 w, 0);
814 if (err)
hsandid832342e2023-12-20 15:41:54 +0100815 clib_unix_error ("%U, thread %s init on cpu %d failed",
816 format_clib_error, err, tr->name, 0);
Dave Barach9b8ffd92016-07-08 08:13:45 -0400817 }
818 }
Ed Warnickecb9cada2015-12-08 15:45:58 -0700819 else
Dave Barach9b8ffd92016-07-08 08:13:45 -0400820 {
821 uword c;
Damjan Marionf0ca1e82020-12-13 23:26:56 +0100822 clib_bitmap_foreach (c, tr->coremask) {
Damjan Marion878c6092017-01-04 13:19:27 +0100823 w = vlib_worker_threads + worker_thread_index++;
824 err = vlib_launch_thread_int (vlib_worker_thread_bootstrap_fn,
825 w, c);
826 if (err)
hsandid832342e2023-12-20 15:41:54 +0100827 clib_unix_error ("%U, thread %s init on cpu %d failed",
828 format_clib_error, err, tr->name, c);
829 }
Dave Barach9b8ffd92016-07-08 08:13:45 -0400830 }
Ed Warnickecb9cada2015-12-08 15:45:58 -0700831 }
Dave Barach9b8ffd92016-07-08 08:13:45 -0400832 vlib_worker_thread_barrier_sync (vm);
Damjan Marion321bd102022-06-01 00:45:18 +0200833 {
834 clib_error_t *err;
835 err = vlib_call_init_exit_functions (
836 vm, &vgm->num_workers_change_function_registrations, 1 /* call_once */,
837 1 /* is_global */);
838 if (err)
839 clib_error_report (err);
840 }
Dave Barach9b8ffd92016-07-08 08:13:45 -0400841 vlib_worker_thread_barrier_release (vm);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700842 return 0;
843}
844
845VLIB_MAIN_LOOP_ENTER_FUNCTION (start_workers);
846
Colin Tregenza Dancereb1ac172017-09-06 20:23:24 +0100847
Colin Tregenza Dancer21596182017-09-04 15:27:49 +0100848static inline void
849worker_thread_node_runtime_update_internal (void)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700850{
851 int i, j;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700852 vlib_main_t *vm;
853 vlib_node_main_t *nm, *nm_clone;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700854 vlib_main_t *vm_clone;
Colin Tregenza Dancer21596182017-09-04 15:27:49 +0100855 vlib_node_runtime_t *rt;
Dave Barach9b8ffd92016-07-08 08:13:45 -0400856
Damjan Marion586afd72017-04-05 19:18:20 +0200857 ASSERT (vlib_get_thread_index () == 0);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700858
Damjan Marion6ffb7c62021-03-26 13:06:13 +0100859 vm = vlib_get_first_main ();
Ed Warnickecb9cada2015-12-08 15:45:58 -0700860 nm = &vm->node_main;
861
Ed Warnickecb9cada2015-12-08 15:45:58 -0700862 ASSERT (*vlib_worker_threads->wait_at_barrier == 1);
863
Dave Barach9b8ffd92016-07-08 08:13:45 -0400864 /*
Ed Warnickecb9cada2015-12-08 15:45:58 -0700865 * Scrape all runtime stats, so we don't lose node runtime(s) with
866 * pending counts, or throw away worker / io thread counts.
867 */
Dave Barach9b8ffd92016-07-08 08:13:45 -0400868 for (j = 0; j < vec_len (nm->nodes); j++)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700869 {
Dave Barach9b8ffd92016-07-08 08:13:45 -0400870 vlib_node_t *n;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700871 n = nm->nodes[j];
872 vlib_node_sync_stats (vm, n);
873 }
874
Damjan Marion6ffb7c62021-03-26 13:06:13 +0100875 for (i = 1; i < vlib_get_n_threads (); i++)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700876 {
Dave Barach9b8ffd92016-07-08 08:13:45 -0400877 vlib_node_t *n;
878
Damjan Marion6ffb7c62021-03-26 13:06:13 +0100879 vm_clone = vlib_get_main_by_index (i);
Ed Warnickecb9cada2015-12-08 15:45:58 -0700880 nm_clone = &vm_clone->node_main;
881
Dave Barach9b8ffd92016-07-08 08:13:45 -0400882 for (j = 0; j < vec_len (nm_clone->nodes); j++)
883 {
884 n = nm_clone->nodes[j];
Ed Warnickecb9cada2015-12-08 15:45:58 -0700885
Dave Barach9b8ffd92016-07-08 08:13:45 -0400886 rt = vlib_node_get_runtime (vm_clone, n->index);
887 vlib_node_runtime_sync_stats (vm_clone, rt, 0, 0, 0);
888 }
Ed Warnickecb9cada2015-12-08 15:45:58 -0700889 }
890
Colin Tregenza Dancer21596182017-09-04 15:27:49 +0100891 /* Per-worker clone rebuilds are now done on each thread */
892}
893
894
895void
896vlib_worker_thread_node_refork (void)
897{
898 vlib_main_t *vm, *vm_clone;
899 vlib_node_main_t *nm, *nm_clone;
900 vlib_node_t **old_nodes_clone;
901 vlib_node_runtime_t *rt, *old_rt;
Damjan Marion66c85832022-03-14 13:04:38 +0100902 u64 **c;
Colin Tregenza Dancer21596182017-09-04 15:27:49 +0100903
904 vlib_node_t *new_n_clone;
905
906 int j;
907
Damjan Marion6ffb7c62021-03-26 13:06:13 +0100908 vm = vlib_get_first_main ();
Colin Tregenza Dancer21596182017-09-04 15:27:49 +0100909 nm = &vm->node_main;
910 vm_clone = vlib_get_main ();
911 nm_clone = &vm_clone->node_main;
912
913 /* Re-clone error heap */
Colin Tregenza Dancer21596182017-09-04 15:27:49 +0100914 u64 *old_counters_all_clear = vm_clone->error_main.counters_last_clear;
915
Dave Barach178cf492018-11-13 16:34:13 -0500916 clib_memcpy_fast (&vm_clone->error_main, &vm->error_main,
917 sizeof (vm->error_main));
Colin Tregenza Dancer21596182017-09-04 15:27:49 +0100918 j = vec_len (vm->error_main.counters) - 1;
Ole Troan233e4682019-05-16 15:01:34 +0200919
Damjan Marion66c85832022-03-14 13:04:38 +0100920 c = vlib_stats_get_entry_data_pointer (vm->error_main.stats_err_entry_index);
921 vm_clone->error_main.counters = c[vm_clone->thread_index];
Ole Troan233e4682019-05-16 15:01:34 +0200922
923 vec_validate_aligned (old_counters_all_clear, j, CLIB_CACHE_LINE_BYTES);
Colin Tregenza Dancer21596182017-09-04 15:27:49 +0100924 vm_clone->error_main.counters_last_clear = old_counters_all_clear;
925
Dmitry Valter9f5b3692022-09-05 15:30:18 +0000926 for (j = 0; j < vec_len (nm_clone->next_frames); j++)
927 {
928 vlib_next_frame_t *nf = &nm_clone->next_frames[j];
929 if ((nf->flags & VLIB_FRAME_IS_ALLOCATED) && nf->frame != NULL)
930 {
931 vlib_frame_t *f = nf->frame;
932 nf->frame = NULL;
933 vlib_frame_free (vm_clone, f);
934 }
935 }
936
Colin Tregenza Dancer21596182017-09-04 15:27:49 +0100937 vec_free (nm_clone->next_frames);
Damjan Marionbe3f4d52018-03-27 21:06:10 +0200938 nm_clone->next_frames = vec_dup_aligned (nm->next_frames,
939 CLIB_CACHE_LINE_BYTES);
Colin Tregenza Dancer21596182017-09-04 15:27:49 +0100940
941 for (j = 0; j < vec_len (nm_clone->next_frames); j++)
Ed Warnickecb9cada2015-12-08 15:45:58 -0700942 {
Colin Tregenza Dancer21596182017-09-04 15:27:49 +0100943 vlib_next_frame_t *nf = &nm_clone->next_frames[j];
944 u32 save_node_runtime_index;
945 u32 save_flags;
Damjan Marionbc20bdf2015-12-17 14:28:18 +0100946
Colin Tregenza Dancer21596182017-09-04 15:27:49 +0100947 save_node_runtime_index = nf->node_runtime_index;
948 save_flags = nf->flags & VLIB_FRAME_NO_FREE_AFTER_DISPATCH;
949 vlib_next_frame_init (nf);
950 nf->node_runtime_index = save_node_runtime_index;
951 nf->flags = save_flags;
Ed Warnickecb9cada2015-12-08 15:45:58 -0700952 }
Colin Tregenza Dancer21596182017-09-04 15:27:49 +0100953
954 old_nodes_clone = nm_clone->nodes;
955 nm_clone->nodes = 0;
956
957 /* re-fork nodes */
958
959 /* Allocate all nodes in single block for speed */
960 new_n_clone =
961 clib_mem_alloc_no_fail (vec_len (nm->nodes) * sizeof (*new_n_clone));
962 for (j = 0; j < vec_len (nm->nodes); j++)
963 {
Benoît Ganne5517bd32019-08-30 16:20:12 +0200964 vlib_node_t *new_n = nm->nodes[j];
Colin Tregenza Dancer21596182017-09-04 15:27:49 +0100965
Dave Barach178cf492018-11-13 16:34:13 -0500966 clib_memcpy_fast (new_n_clone, new_n, sizeof (*new_n));
Colin Tregenza Dancer21596182017-09-04 15:27:49 +0100967 /* none of the copied nodes have enqueue rights given out */
968 new_n_clone->owner_node_index = VLIB_INVALID_NODE_INDEX;
969
970 if (j >= vec_len (old_nodes_clone))
971 {
972 /* new node, set to zero */
Dave Barachb7b92992018-10-17 10:38:51 -0400973 clib_memset (&new_n_clone->stats_total, 0,
974 sizeof (new_n_clone->stats_total));
975 clib_memset (&new_n_clone->stats_last_clear, 0,
976 sizeof (new_n_clone->stats_last_clear));
Colin Tregenza Dancer21596182017-09-04 15:27:49 +0100977 }
978 else
979 {
Benoît Ganne5517bd32019-08-30 16:20:12 +0200980 vlib_node_t *old_n_clone = old_nodes_clone[j];
Colin Tregenza Dancer21596182017-09-04 15:27:49 +0100981 /* Copy stats if the old data is valid */
Dave Barach178cf492018-11-13 16:34:13 -0500982 clib_memcpy_fast (&new_n_clone->stats_total,
983 &old_n_clone->stats_total,
984 sizeof (new_n_clone->stats_total));
985 clib_memcpy_fast (&new_n_clone->stats_last_clear,
986 &old_n_clone->stats_last_clear,
987 sizeof (new_n_clone->stats_last_clear));
Colin Tregenza Dancer21596182017-09-04 15:27:49 +0100988
989 /* keep previous node state */
990 new_n_clone->state = old_n_clone->state;
Maxime Peimdde163d2021-04-15 13:52:12 +0200991 new_n_clone->flags = old_n_clone->flags;
Colin Tregenza Dancer21596182017-09-04 15:27:49 +0100992 }
993 vec_add1 (nm_clone->nodes, new_n_clone);
994 new_n_clone++;
995 }
996 /* Free the old node clones */
997 clib_mem_free (old_nodes_clone[0]);
998
999 vec_free (old_nodes_clone);
1000
1001
1002 /* re-clone internal nodes */
1003 old_rt = nm_clone->nodes_by_type[VLIB_NODE_TYPE_INTERNAL];
1004 nm_clone->nodes_by_type[VLIB_NODE_TYPE_INTERNAL] =
Damjan Marionbe3f4d52018-03-27 21:06:10 +02001005 vec_dup_aligned (nm->nodes_by_type[VLIB_NODE_TYPE_INTERNAL],
1006 CLIB_CACHE_LINE_BYTES);
Colin Tregenza Dancer21596182017-09-04 15:27:49 +01001007
1008 vec_foreach (rt, nm_clone->nodes_by_type[VLIB_NODE_TYPE_INTERNAL])
1009 {
1010 vlib_node_t *n = vlib_get_node (vm, rt->node_index);
Colin Tregenza Dancer21596182017-09-04 15:27:49 +01001011 /* copy runtime_data, will be overwritten later for existing rt */
1012 if (n->runtime_data && n->runtime_data_bytes > 0)
Dave Barach178cf492018-11-13 16:34:13 -05001013 clib_memcpy_fast (rt->runtime_data, n->runtime_data,
1014 clib_min (VLIB_NODE_RUNTIME_DATA_SIZE,
1015 n->runtime_data_bytes));
Colin Tregenza Dancer21596182017-09-04 15:27:49 +01001016 }
1017
1018 for (j = 0; j < vec_len (old_rt); j++)
1019 {
1020 rt = vlib_node_get_runtime (vm_clone, old_rt[j].node_index);
1021 rt->state = old_rt[j].state;
Maxime Peimdde163d2021-04-15 13:52:12 +02001022 rt->flags = old_rt[j].flags;
Dave Barach178cf492018-11-13 16:34:13 -05001023 clib_memcpy_fast (rt->runtime_data, old_rt[j].runtime_data,
1024 VLIB_NODE_RUNTIME_DATA_SIZE);
Colin Tregenza Dancer21596182017-09-04 15:27:49 +01001025 }
1026
1027 vec_free (old_rt);
1028
1029 /* re-clone input nodes */
1030 old_rt = nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT];
1031 nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT] =
Damjan Marionbe3f4d52018-03-27 21:06:10 +02001032 vec_dup_aligned (nm->nodes_by_type[VLIB_NODE_TYPE_INPUT],
1033 CLIB_CACHE_LINE_BYTES);
Damjan Marion94100532020-11-06 23:25:57 +01001034 clib_interrupt_resize (
Damjan Marioncc8249c2023-07-23 14:24:22 +02001035 &nm_clone->input_node_interrupts,
Damjan Marion94100532020-11-06 23:25:57 +01001036 vec_len (nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT]));
Damjan Marioncc8249c2023-07-23 14:24:22 +02001037 clib_interrupt_resize (
1038 &nm_clone->pre_input_node_interrupts,
1039 vec_len (nm_clone->nodes_by_type[VLIB_NODE_TYPE_PRE_INPUT]));
Colin Tregenza Dancer21596182017-09-04 15:27:49 +01001040
1041 vec_foreach (rt, nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT])
1042 {
1043 vlib_node_t *n = vlib_get_node (vm, rt->node_index);
Colin Tregenza Dancer21596182017-09-04 15:27:49 +01001044 /* copy runtime_data, will be overwritten later for existing rt */
1045 if (n->runtime_data && n->runtime_data_bytes > 0)
Dave Barach178cf492018-11-13 16:34:13 -05001046 clib_memcpy_fast (rt->runtime_data, n->runtime_data,
1047 clib_min (VLIB_NODE_RUNTIME_DATA_SIZE,
1048 n->runtime_data_bytes));
Colin Tregenza Dancer21596182017-09-04 15:27:49 +01001049 }
1050
1051 for (j = 0; j < vec_len (old_rt); j++)
1052 {
1053 rt = vlib_node_get_runtime (vm_clone, old_rt[j].node_index);
1054 rt->state = old_rt[j].state;
Maxime Peimdde163d2021-04-15 13:52:12 +02001055 rt->flags = old_rt[j].flags;
Dave Barach178cf492018-11-13 16:34:13 -05001056 clib_memcpy_fast (rt->runtime_data, old_rt[j].runtime_data,
1057 VLIB_NODE_RUNTIME_DATA_SIZE);
Colin Tregenza Dancer21596182017-09-04 15:27:49 +01001058 }
1059
1060 vec_free (old_rt);
1061
Dave Barach53fe4a72019-01-26 09:50:26 -05001062 /* re-clone pre-input nodes */
1063 old_rt = nm_clone->nodes_by_type[VLIB_NODE_TYPE_PRE_INPUT];
1064 nm_clone->nodes_by_type[VLIB_NODE_TYPE_PRE_INPUT] =
1065 vec_dup_aligned (nm->nodes_by_type[VLIB_NODE_TYPE_PRE_INPUT],
1066 CLIB_CACHE_LINE_BYTES);
1067
1068 vec_foreach (rt, nm_clone->nodes_by_type[VLIB_NODE_TYPE_PRE_INPUT])
1069 {
1070 vlib_node_t *n = vlib_get_node (vm, rt->node_index);
Dave Barach53fe4a72019-01-26 09:50:26 -05001071 /* copy runtime_data, will be overwritten later for existing rt */
1072 if (n->runtime_data && n->runtime_data_bytes > 0)
1073 clib_memcpy_fast (rt->runtime_data, n->runtime_data,
1074 clib_min (VLIB_NODE_RUNTIME_DATA_SIZE,
1075 n->runtime_data_bytes));
1076 }
1077
1078 for (j = 0; j < vec_len (old_rt); j++)
1079 {
1080 rt = vlib_node_get_runtime (vm_clone, old_rt[j].node_index);
1081 rt->state = old_rt[j].state;
Maxime Peimdde163d2021-04-15 13:52:12 +02001082 rt->flags = old_rt[j].flags;
Dave Barach53fe4a72019-01-26 09:50:26 -05001083 clib_memcpy_fast (rt->runtime_data, old_rt[j].runtime_data,
1084 VLIB_NODE_RUNTIME_DATA_SIZE);
1085 }
1086
1087 vec_free (old_rt);
1088
Vladislav Grishenko29733502021-09-25 21:00:59 +05001089 vec_free (nm_clone->processes);
Damjan Marionbe3f4d52018-03-27 21:06:10 +02001090 nm_clone->processes = vec_dup_aligned (nm->processes,
1091 CLIB_CACHE_LINE_BYTES);
Dave Barach687c9022019-07-23 10:22:31 -04001092 nm_clone->node_by_error = nm->node_by_error;
Colin Tregenza Dancer21596182017-09-04 15:27:49 +01001093}
1094
Colin Tregenza Dancer21596182017-09-04 15:27:49 +01001095void
1096vlib_worker_thread_node_runtime_update (void)
1097{
1098 /*
1099 * Make a note that we need to do a node runtime update
1100 * prior to releasing the barrier.
1101 */
1102 vlib_global_main.need_vlib_worker_thread_node_runtime_update = 1;
Ed Warnickecb9cada2015-12-08 15:45:58 -07001103}
1104
Pavel Kotucek1e765832016-09-23 08:54:14 +02001105u32
1106unformat_sched_policy (unformat_input_t * input, va_list * args)
1107{
1108 u32 *r = va_arg (*args, u32 *);
1109
1110 if (0);
1111#define _(v,f,s) else if (unformat (input, s)) *r = SCHED_POLICY_##f;
1112 foreach_sched_policy
1113#undef _
1114 else
1115 return 0;
1116 return 1;
1117}
1118
Ed Warnickecb9cada2015-12-08 15:45:58 -07001119static clib_error_t *
1120cpu_config (vlib_main_t * vm, unformat_input_t * input)
1121{
1122 vlib_thread_registration_t *tr;
Dave Barach9b8ffd92016-07-08 08:13:45 -04001123 uword *p;
1124 vlib_thread_main_t *tm = &vlib_thread_main;
1125 u8 *name;
Dave Barach9b8ffd92016-07-08 08:13:45 -04001126 uword *bitmap;
Ed Warnickecb9cada2015-12-08 15:45:58 -07001127 u32 count;
1128
1129 tm->thread_registrations_by_name = hash_create_string (0, sizeof (uword));
Pavel Kotucek1e765832016-09-23 08:54:14 +02001130
Dave Barach9b8ffd92016-07-08 08:13:45 -04001131 tm->n_thread_stacks = 1; /* account for main thread */
Pavel Kotucek1e765832016-09-23 08:54:14 +02001132 tm->sched_policy = ~0;
1133 tm->sched_priority = ~0;
Damjan Marion858151f2018-07-11 10:51:00 +02001134 tm->main_lcore = ~0;
Ed Warnickecb9cada2015-12-08 15:45:58 -07001135
1136 tr = tm->next;
1137
1138 while (tr)
1139 {
Dave Barach9b8ffd92016-07-08 08:13:45 -04001140 hash_set_mem (tm->thread_registrations_by_name, tr->name, (uword) tr);
Ed Warnickecb9cada2015-12-08 15:45:58 -07001141 tr = tr->next;
1142 }
1143
Dave Barach9b8ffd92016-07-08 08:13:45 -04001144 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
Ed Warnickecb9cada2015-12-08 15:45:58 -07001145 {
Damjan Marionbf741472016-06-13 22:49:44 +02001146 if (unformat (input, "use-pthreads"))
Dave Barach9b8ffd92016-07-08 08:13:45 -04001147 tm->use_pthreads = 1;
Ed Warnickecb9cada2015-12-08 15:45:58 -07001148 else if (unformat (input, "thread-prefix %v", &tm->thread_prefix))
Dave Barach9b8ffd92016-07-08 08:13:45 -04001149 ;
Ed Warnickecb9cada2015-12-08 15:45:58 -07001150 else if (unformat (input, "main-core %u", &tm->main_lcore))
Dave Barach9b8ffd92016-07-08 08:13:45 -04001151 ;
Ed Warnickecb9cada2015-12-08 15:45:58 -07001152 else if (unformat (input, "skip-cores %u", &tm->skip_cores))
Dave Barach9b8ffd92016-07-08 08:13:45 -04001153 ;
Dave Baracha690fdb2020-01-21 12:34:55 -05001154 else if (unformat (input, "numa-heap-size %U",
1155 unformat_memory_size, &tm->numa_heap_size))
1156 ;
Yi Hee4a9eb72018-07-17 14:18:41 +08001157 else if (unformat (input, "coremask-%s %U", &name,
1158 unformat_bitmap_mask, &bitmap) ||
1159 unformat (input, "corelist-%s %U", &name,
1160 unformat_bitmap_list, &bitmap))
Dave Barach9b8ffd92016-07-08 08:13:45 -04001161 {
1162 p = hash_get_mem (tm->thread_registrations_by_name, name);
1163 if (p == 0)
1164 return clib_error_return (0, "no such thread type '%s'", name);
Ed Warnickecb9cada2015-12-08 15:45:58 -07001165
Dave Barach9b8ffd92016-07-08 08:13:45 -04001166 tr = (vlib_thread_registration_t *) p[0];
Ed Warnickecb9cada2015-12-08 15:45:58 -07001167
Dave Barach9b8ffd92016-07-08 08:13:45 -04001168 if (tr->use_pthreads)
1169 return clib_error_return (0,
1170 "corelist cannot be set for '%s' threads",
1171 name);
Vladimir Isaev18a4a372020-03-17 12:30:11 +03001172 if (tr->count)
1173 return clib_error_return
1174 (0, "core placement of '%s' threads is already configured",
1175 name);
Ed Warnickecb9cada2015-12-08 15:45:58 -07001176
Dave Barach9b8ffd92016-07-08 08:13:45 -04001177 tr->coremask = bitmap;
1178 tr->count = clib_bitmap_count_set_bits (tr->coremask);
1179 }
Pavel Kotucek1e765832016-09-23 08:54:14 +02001180 else
1181 if (unformat
1182 (input, "scheduler-policy %U", unformat_sched_policy,
1183 &tm->sched_policy))
1184 ;
Pavel Kotucekc08a1ed2016-09-23 08:54:14 +02001185 else if (unformat (input, "scheduler-priority %u", &tm->sched_priority))
Pavel Kotucek1e765832016-09-23 08:54:14 +02001186 ;
Ed Warnickecb9cada2015-12-08 15:45:58 -07001187 else if (unformat (input, "%s %u", &name, &count))
Dave Barach9b8ffd92016-07-08 08:13:45 -04001188 {
1189 p = hash_get_mem (tm->thread_registrations_by_name, name);
1190 if (p == 0)
Pavel Kotucek1e765832016-09-23 08:54:14 +02001191 return clib_error_return (0, "no such thread type 3 '%s'", name);
Dave Barach9b8ffd92016-07-08 08:13:45 -04001192
1193 tr = (vlib_thread_registration_t *) p[0];
Vladimir Isaev18a4a372020-03-17 12:30:11 +03001194
Dave Barach9b8ffd92016-07-08 08:13:45 -04001195 if (tr->fixed_count)
1196 return clib_error_return
Vladimir Isaev18a4a372020-03-17 12:30:11 +03001197 (0, "number of '%s' threads not configurable", name);
1198 if (tr->count)
1199 return clib_error_return
1200 (0, "number of '%s' threads is already configured", name);
1201
Dave Barach9b8ffd92016-07-08 08:13:45 -04001202 tr->count = count;
1203 }
1204 else
1205 break;
Ed Warnickecb9cada2015-12-08 15:45:58 -07001206 }
1207
Pavel Kotucekc08a1ed2016-09-23 08:54:14 +02001208 if (tm->sched_priority != ~0)
Pavel Kotucek1e765832016-09-23 08:54:14 +02001209 {
Pavel Kotucekc08a1ed2016-09-23 08:54:14 +02001210 if (tm->sched_policy == SCHED_FIFO || tm->sched_policy == SCHED_RR)
Pavel Kotucek1e765832016-09-23 08:54:14 +02001211 {
1212 u32 prio_max = sched_get_priority_max (tm->sched_policy);
1213 u32 prio_min = sched_get_priority_min (tm->sched_policy);
1214 if (tm->sched_priority > prio_max)
1215 tm->sched_priority = prio_max;
1216 if (tm->sched_priority < prio_min)
1217 tm->sched_priority = prio_min;
1218 }
1219 else
Pavel Kotucekc08a1ed2016-09-23 08:54:14 +02001220 {
1221 return clib_error_return
1222 (0,
1223 "scheduling priority (%d) is not allowed for `normal` scheduling policy",
1224 tm->sched_priority);
1225 }
Pavel Kotucek1e765832016-09-23 08:54:14 +02001226 }
Ed Warnickecb9cada2015-12-08 15:45:58 -07001227 tr = tm->next;
1228
1229 if (!tm->thread_prefix)
Dave Barach9b8ffd92016-07-08 08:13:45 -04001230 tm->thread_prefix = format (0, "vpp");
Ed Warnickecb9cada2015-12-08 15:45:58 -07001231
1232 while (tr)
1233 {
1234 tm->n_thread_stacks += tr->count;
1235 tm->n_pthreads += tr->count * tr->use_pthreads;
Damjan Marion878c6092017-01-04 13:19:27 +01001236 tm->n_threads += tr->count * (tr->use_pthreads == 0);
Ed Warnickecb9cada2015-12-08 15:45:58 -07001237 tr = tr->next;
1238 }
1239
1240 return 0;
1241}
1242
1243VLIB_EARLY_CONFIG_FUNCTION (cpu_config, "cpu");
1244
Colin Tregenza Dancereb1ac172017-09-06 20:23:24 +01001245 /*
1246 * Enforce minimum open time to minimize packet loss due to Rx overflow,
1247 * based on a test based heuristic that barrier should be open for at least
1248 * 3 time as long as it is closed (with an upper bound of 1ms because by that
1249 * point it is probably too late to make a difference)
1250 */
1251
1252#ifndef BARRIER_MINIMUM_OPEN_LIMIT
1253#define BARRIER_MINIMUM_OPEN_LIMIT 0.001
1254#endif
1255
1256#ifndef BARRIER_MINIMUM_OPEN_FACTOR
1257#define BARRIER_MINIMUM_OPEN_FACTOR 3
1258#endif
1259
Dave Barach9b8ffd92016-07-08 08:13:45 -04001260void
Dave Barachc602b382019-06-03 19:48:22 -04001261vlib_worker_thread_initial_barrier_sync_and_release (vlib_main_t * vm)
1262{
1263 f64 deadline;
1264 f64 now = vlib_time_now (vm);
Damjan Marion6ffb7c62021-03-26 13:06:13 +01001265 u32 count = vlib_get_n_threads () - 1;
Dave Barachc602b382019-06-03 19:48:22 -04001266
1267 /* No worker threads? */
1268 if (count == 0)
1269 return;
1270
1271 deadline = now + BARRIER_SYNC_TIMEOUT;
1272 *vlib_worker_threads->wait_at_barrier = 1;
1273 while (*vlib_worker_threads->workers_at_barrier != count)
1274 {
1275 if ((now = vlib_time_now (vm)) > deadline)
1276 {
1277 fformat (stderr, "%s: worker thread deadlock\n", __FUNCTION__);
1278 os_panic ();
1279 }
1280 CLIB_PAUSE ();
1281 }
1282 *vlib_worker_threads->wait_at_barrier = 0;
1283}
1284
Neale Ranns42845dd2020-05-26 13:12:17 +00001285/**
1286 * Return true if the wroker thread barrier is held
1287 */
1288u8
1289vlib_worker_thread_barrier_held (void)
1290{
Damjan Marion6ffb7c62021-03-26 13:06:13 +01001291 if (vlib_get_n_threads () < 2)
Neale Ranns42845dd2020-05-26 13:12:17 +00001292 return (1);
1293
1294 return (*vlib_worker_threads->wait_at_barrier == 1);
1295}
1296
Dave Barachc602b382019-06-03 19:48:22 -04001297void
Damjan Marion8343ee52019-02-26 17:15:48 +01001298vlib_worker_thread_barrier_sync_int (vlib_main_t * vm, const char *func_name)
Ed Warnickecb9cada2015-12-08 15:45:58 -07001299{
1300 f64 deadline;
Colin Tregenza Dancereb1ac172017-09-06 20:23:24 +01001301 f64 now;
1302 f64 t_entry;
1303 f64 t_open;
1304 f64 t_closed;
Dave Barach9ae190e2019-04-23 10:07:24 -04001305 f64 max_vector_rate;
Ed Warnickecb9cada2015-12-08 15:45:58 -07001306 u32 count;
Dave Barach9ae190e2019-04-23 10:07:24 -04001307 int i;
Dave Barach9b8ffd92016-07-08 08:13:45 -04001308
Damjan Marion6ffb7c62021-03-26 13:06:13 +01001309 if (vlib_get_n_threads () < 2)
Dave Barach9b8ffd92016-07-08 08:13:45 -04001310 return;
Ed Warnickecb9cada2015-12-08 15:45:58 -07001311
Colin Tregenza Dancer21596182017-09-04 15:27:49 +01001312 ASSERT (vlib_get_thread_index () == 0);
1313
Damjan Marion8343ee52019-02-26 17:15:48 +01001314 vlib_worker_threads[0].barrier_caller = func_name;
Damjan Marion6ffb7c62021-03-26 13:06:13 +01001315 count = vlib_get_n_threads () - 1;
Ed Warnickecb9cada2015-12-08 15:45:58 -07001316
Colin Tregenza Dancereb1ac172017-09-06 20:23:24 +01001317 /* Record entry relative to last close */
1318 now = vlib_time_now (vm);
1319 t_entry = now - vm->barrier_epoch;
1320
Ed Warnickecb9cada2015-12-08 15:45:58 -07001321 /* Tolerate recursive calls */
1322 if (++vlib_worker_threads[0].recursion_level > 1)
Colin Tregenza Dancereb1ac172017-09-06 20:23:24 +01001323 {
1324 barrier_trace_sync_rec (t_entry);
1325 return;
1326 }
Ed Warnickecb9cada2015-12-08 15:45:58 -07001327
Tom Seidenberg6c81f5a2020-07-10 15:49:03 +00001328 if (PREDICT_FALSE (vec_len (vm->barrier_perf_callbacks) != 0))
1329 clib_call_callbacks (vm->barrier_perf_callbacks, vm,
1330 vm->clib_time.last_cpu_time, 0 /* enter */ );
1331
Dave Barach9ae190e2019-04-23 10:07:24 -04001332 /*
1333 * Need data to decide if we're working hard enough to honor
1334 * the barrier hold-down timer.
1335 */
1336 max_vector_rate = 0.0;
Damjan Marion6ffb7c62021-03-26 13:06:13 +01001337 for (i = 1; i < vlib_get_n_threads (); i++)
1338 {
1339 vlib_main_t *ovm = vlib_get_main_by_index (i);
1340 max_vector_rate = clib_max (max_vector_rate,
1341 (f64) vlib_last_vectors_per_main_loop (ovm));
1342 }
Dave Barach9ae190e2019-04-23 10:07:24 -04001343
Bud Grise42f20062016-03-16 13:09:46 -04001344 vlib_worker_threads[0].barrier_sync_count++;
1345
Colin Tregenza Dancereb1ac172017-09-06 20:23:24 +01001346 /* Enforce minimum barrier open time to minimize packet loss */
1347 ASSERT (vm->barrier_no_close_before <= (now + BARRIER_MINIMUM_OPEN_LIMIT));
Colin Tregenza Dancereb1ac172017-09-06 20:23:24 +01001348
Dave Barach9ae190e2019-04-23 10:07:24 -04001349 /*
1350 * If any worker thread seems busy, which we define
1351 * as a vector rate above 10, we enforce the barrier hold-down timer
1352 */
1353 if (max_vector_rate > 10.0)
Dave Barach36feebb2018-09-07 11:12:27 -04001354 {
Dave Barach9ae190e2019-04-23 10:07:24 -04001355 while (1)
Dave Barach36feebb2018-09-07 11:12:27 -04001356 {
Dave Barach9ae190e2019-04-23 10:07:24 -04001357 now = vlib_time_now (vm);
1358 /* Barrier hold-down timer expired? */
1359 if (now >= vm->barrier_no_close_before)
1360 break;
1361 if ((vm->barrier_no_close_before - now)
1362 > (2.0 * BARRIER_MINIMUM_OPEN_LIMIT))
1363 {
1364 clib_warning
1365 ("clock change: would have waited for %.4f seconds",
1366 (vm->barrier_no_close_before - now));
1367 break;
1368 }
Dave Barach36feebb2018-09-07 11:12:27 -04001369 }
1370 }
Colin Tregenza Dancereb1ac172017-09-06 20:23:24 +01001371 /* Record time of closure */
1372 t_open = now - vm->barrier_epoch;
1373 vm->barrier_epoch = now;
1374
1375 deadline = now + BARRIER_SYNC_TIMEOUT;
Ed Warnickecb9cada2015-12-08 15:45:58 -07001376
1377 *vlib_worker_threads->wait_at_barrier = 1;
1378 while (*vlib_worker_threads->workers_at_barrier != count)
1379 {
Colin Tregenza Dancereb1ac172017-09-06 20:23:24 +01001380 if ((now = vlib_time_now (vm)) > deadline)
Dave Barach9b8ffd92016-07-08 08:13:45 -04001381 {
1382 fformat (stderr, "%s: worker thread deadlock\n", __FUNCTION__);
1383 os_panic ();
1384 }
Ed Warnickecb9cada2015-12-08 15:45:58 -07001385 }
Colin Tregenza Dancereb1ac172017-09-06 20:23:24 +01001386
1387 t_closed = now - vm->barrier_epoch;
1388
1389 barrier_trace_sync (t_entry, t_open, t_closed);
1390
Ed Warnickecb9cada2015-12-08 15:45:58 -07001391}
1392
Dave Barach9b8ffd92016-07-08 08:13:45 -04001393void
1394vlib_worker_thread_barrier_release (vlib_main_t * vm)
Ed Warnickecb9cada2015-12-08 15:45:58 -07001395{
Damjan Marionfd8deb42021-03-06 12:26:28 +01001396 vlib_global_main_t *vgm = vlib_get_global_main ();
Ed Warnickecb9cada2015-12-08 15:45:58 -07001397 f64 deadline;
Colin Tregenza Dancereb1ac172017-09-06 20:23:24 +01001398 f64 now;
1399 f64 minimum_open;
1400 f64 t_entry;
1401 f64 t_closed_total;
1402 f64 t_update_main = 0.0;
Colin Tregenza Dancer21596182017-09-04 15:27:49 +01001403 int refork_needed = 0;
Ed Warnickecb9cada2015-12-08 15:45:58 -07001404
Damjan Marion6ffb7c62021-03-26 13:06:13 +01001405 if (vlib_get_n_threads () < 2)
Dave Barach9b8ffd92016-07-08 08:13:45 -04001406 return;
Ed Warnickecb9cada2015-12-08 15:45:58 -07001407
Colin Tregenza Dancer21596182017-09-04 15:27:49 +01001408 ASSERT (vlib_get_thread_index () == 0);
1409
Colin Tregenza Dancereb1ac172017-09-06 20:23:24 +01001410
1411 now = vlib_time_now (vm);
1412 t_entry = now - vm->barrier_epoch;
1413
Ed Warnickecb9cada2015-12-08 15:45:58 -07001414 if (--vlib_worker_threads[0].recursion_level > 0)
Colin Tregenza Dancereb1ac172017-09-06 20:23:24 +01001415 {
1416 barrier_trace_release_rec (t_entry);
1417 return;
1418 }
Ed Warnickecb9cada2015-12-08 15:45:58 -07001419
Colin Tregenza Dancer21596182017-09-04 15:27:49 +01001420 /* Update (all) node runtimes before releasing the barrier, if needed */
Damjan Marionfd8deb42021-03-06 12:26:28 +01001421 if (vgm->need_vlib_worker_thread_node_runtime_update)
Colin Tregenza Dancer21596182017-09-04 15:27:49 +01001422 {
Dave Barach1ddbc012018-06-13 09:26:05 -04001423 /*
1424 * Lock stat segment here, so we's safe when
1425 * rebuilding the stat segment node clones from the
1426 * stat thread...
1427 */
Damjan Marion8973b072022-03-01 15:51:18 +01001428 vlib_stats_segment_lock ();
Dave Barach1ddbc012018-06-13 09:26:05 -04001429
Colin Tregenza Dancer21596182017-09-04 15:27:49 +01001430 /* Do stats elements on main thread */
1431 worker_thread_node_runtime_update_internal ();
Damjan Marionfd8deb42021-03-06 12:26:28 +01001432 vgm->need_vlib_worker_thread_node_runtime_update = 0;
Colin Tregenza Dancer21596182017-09-04 15:27:49 +01001433
1434 /* Do per thread rebuilds in parallel */
1435 refork_needed = 1;
Sirshak Das2f6d7bb2018-10-03 22:53:51 +00001436 clib_atomic_fetch_add (vlib_worker_threads->node_reforks_required,
Damjan Marion6ffb7c62021-03-26 13:06:13 +01001437 (vlib_get_n_threads () - 1));
Colin Tregenza Dancereb1ac172017-09-06 20:23:24 +01001438 now = vlib_time_now (vm);
1439 t_update_main = now - vm->barrier_epoch;
Colin Tregenza Dancer21596182017-09-04 15:27:49 +01001440 }
1441
Colin Tregenza Dancereb1ac172017-09-06 20:23:24 +01001442 deadline = now + BARRIER_SYNC_TIMEOUT;
Ed Warnickecb9cada2015-12-08 15:45:58 -07001443
Dave Baracha4324a92019-02-19 17:05:30 -05001444 /*
1445 * Note when we let go of the barrier.
1446 * Workers can use this to derive a reasonably accurate
1447 * time offset. See vlib_time_now(...)
1448 */
1449 vm->time_last_barrier_release = vlib_time_now (vm);
1450 CLIB_MEMORY_STORE_BARRIER ();
1451
Ed Warnickecb9cada2015-12-08 15:45:58 -07001452 *vlib_worker_threads->wait_at_barrier = 0;
1453
1454 while (*vlib_worker_threads->workers_at_barrier > 0)
1455 {
Colin Tregenza Dancereb1ac172017-09-06 20:23:24 +01001456 if ((now = vlib_time_now (vm)) > deadline)
Dave Barach9b8ffd92016-07-08 08:13:45 -04001457 {
1458 fformat (stderr, "%s: worker thread deadlock\n", __FUNCTION__);
1459 os_panic ();
1460 }
Ed Warnickecb9cada2015-12-08 15:45:58 -07001461 }
Colin Tregenza Dancer21596182017-09-04 15:27:49 +01001462
1463 /* Wait for reforks before continuing */
1464 if (refork_needed)
1465 {
Colin Tregenza Dancereb1ac172017-09-06 20:23:24 +01001466 now = vlib_time_now (vm);
1467
1468 deadline = now + BARRIER_SYNC_TIMEOUT;
Colin Tregenza Dancer21596182017-09-04 15:27:49 +01001469
1470 while (*vlib_worker_threads->node_reforks_required > 0)
1471 {
Colin Tregenza Dancereb1ac172017-09-06 20:23:24 +01001472 if ((now = vlib_time_now (vm)) > deadline)
Colin Tregenza Dancer21596182017-09-04 15:27:49 +01001473 {
1474 fformat (stderr, "%s: worker thread refork deadlock\n",
1475 __FUNCTION__);
1476 os_panic ();
1477 }
1478 }
Damjan Marion8973b072022-03-01 15:51:18 +01001479 vlib_stats_segment_unlock ();
Colin Tregenza Dancer21596182017-09-04 15:27:49 +01001480 }
Colin Tregenza Dancereb1ac172017-09-06 20:23:24 +01001481
1482 t_closed_total = now - vm->barrier_epoch;
1483
1484 minimum_open = t_closed_total * BARRIER_MINIMUM_OPEN_FACTOR;
1485
1486 if (minimum_open > BARRIER_MINIMUM_OPEN_LIMIT)
1487 {
1488 minimum_open = BARRIER_MINIMUM_OPEN_LIMIT;
1489 }
1490
1491 vm->barrier_no_close_before = now + minimum_open;
1492
1493 /* Record barrier epoch (used to enforce minimum open time) */
1494 vm->barrier_epoch = now;
1495
1496 barrier_trace_release (t_entry, t_closed_total, t_update_main);
1497
Tom Seidenberg6c81f5a2020-07-10 15:49:03 +00001498 if (PREDICT_FALSE (vec_len (vm->barrier_perf_callbacks) != 0))
1499 clib_call_callbacks (vm->barrier_perf_callbacks, vm,
1500 vm->clib_time.last_cpu_time, 1 /* leave */ );
Ed Warnickecb9cada2015-12-08 15:45:58 -07001501}
1502
Florin Coras4b208302022-03-30 13:50:19 -07001503static void
1504vlib_worker_sync_rpc (void *args)
1505{
1506 ASSERT (vlib_thread_is_main_w_barrier ());
1507 vlib_worker_threads->wait_before_barrier = 0;
1508}
1509
1510void
1511vlib_workers_sync (void)
1512{
1513 if (PREDICT_FALSE (!vlib_num_workers ()))
1514 return;
1515
1516 if (!(*vlib_worker_threads->wait_at_barrier) &&
1517 !clib_atomic_swap_rel_n (&vlib_worker_threads->wait_before_barrier, 1))
1518 {
1519 u32 thread_index = vlib_get_thread_index ();
1520 vlib_rpc_call_main_thread (vlib_worker_sync_rpc, (u8 *) &thread_index,
1521 sizeof (thread_index));
1522 }
1523
1524 /* Wait until main thread asks for barrier */
1525 while (!(*vlib_worker_threads->wait_at_barrier))
1526 ;
1527
1528 /* Stop before barrier and make sure all threads are either
1529 * at worker barrier or the barrier before it */
1530 clib_atomic_fetch_add (&vlib_worker_threads->workers_before_barrier, 1);
1531 while (vlib_num_workers () > (*vlib_worker_threads->workers_at_barrier +
1532 vlib_worker_threads->workers_before_barrier))
1533 ;
1534}
1535
1536void
1537vlib_workers_continue (void)
1538{
1539 if (PREDICT_FALSE (!vlib_num_workers ()))
1540 return;
1541
1542 clib_atomic_fetch_add (&vlib_worker_threads->done_work_before_barrier, 1);
1543
1544 /* Wait until all workers are done with work before barrier */
1545 while (vlib_worker_threads->done_work_before_barrier <
1546 vlib_worker_threads->workers_before_barrier)
1547 ;
1548
1549 clib_atomic_fetch_add (&vlib_worker_threads->done_work_before_barrier, -1);
1550 clib_atomic_fetch_add (&vlib_worker_threads->workers_before_barrier, -1);
1551}
1552
Neale Ranns42845dd2020-05-26 13:12:17 +00001553/**
1554 * Wait until each of the workers has been once around the track
1555 */
1556void
1557vlib_worker_wait_one_loop (void)
1558{
Damjan Marionfd8deb42021-03-06 12:26:28 +01001559 vlib_global_main_t *vgm = vlib_get_global_main ();
Neale Ranns42845dd2020-05-26 13:12:17 +00001560 ASSERT (vlib_get_thread_index () == 0);
1561
Damjan Marion6ffb7c62021-03-26 13:06:13 +01001562 if (vlib_get_n_threads () < 2)
Neale Ranns42845dd2020-05-26 13:12:17 +00001563 return;
1564
1565 if (vlib_worker_thread_barrier_held ())
1566 return;
1567
1568 u32 *counts = 0;
1569 u32 ii;
1570
Damjan Marion6ffb7c62021-03-26 13:06:13 +01001571 vec_validate (counts, vlib_get_n_threads () - 1);
Neale Ranns42845dd2020-05-26 13:12:17 +00001572
1573 /* record the current loop counts */
Damjan Marionfd8deb42021-03-06 12:26:28 +01001574 vec_foreach_index (ii, vgm->vlib_mains)
1575 counts[ii] = vgm->vlib_mains[ii]->main_loop_count;
Neale Ranns42845dd2020-05-26 13:12:17 +00001576
1577 /* spin until each changes, apart from the main thread, or we'd be
1578 * a while */
1579 for (ii = 1; ii < vec_len (counts); ii++)
1580 {
Damjan Marionfd8deb42021-03-06 12:26:28 +01001581 while (counts[ii] == vgm->vlib_mains[ii]->main_loop_count)
Neale Ranns42845dd2020-05-26 13:12:17 +00001582 CLIB_PAUSE ();
1583 }
1584
1585 vec_free (counts);
1586 return;
1587}
1588
Dave Barach9b8ffd92016-07-08 08:13:45 -04001589void
Florin Coras4cadd3b2024-02-01 20:46:15 -08001590vlib_worker_flush_pending_rpc_requests (vlib_main_t *vm)
1591{
1592 vlib_main_t *vm_global = vlib_get_first_main ();
1593
1594 ASSERT (vm != vm_global);
1595
1596 clib_spinlock_lock_if_init (&vm_global->pending_rpc_lock);
1597 vec_append (vm_global->pending_rpc_requests, vm->pending_rpc_requests);
1598 vec_reset_length (vm->pending_rpc_requests);
1599 clib_spinlock_unlock_if_init (&vm_global->pending_rpc_lock);
1600}
1601
1602void
Dave Barach9b8ffd92016-07-08 08:13:45 -04001603vlib_worker_thread_fn (void *arg)
Damjan Marion0f8ecf02016-06-27 08:30:30 +02001604{
Damjan Marionfd8deb42021-03-06 12:26:28 +01001605 vlib_global_main_t *vgm = vlib_get_global_main ();
Dave Barach9b8ffd92016-07-08 08:13:45 -04001606 vlib_worker_thread_t *w = (vlib_worker_thread_t *) arg;
Dave Barach9b8ffd92016-07-08 08:13:45 -04001607 vlib_main_t *vm = vlib_get_main ();
Damjan Marione9f929b2017-03-16 11:32:09 +01001608 clib_error_t *e;
Damjan Marion0f8ecf02016-06-27 08:30:30 +02001609
Damjan Marion586afd72017-04-05 19:18:20 +02001610 ASSERT (vm->thread_index == vlib_get_thread_index ());
Damjan Marion0f8ecf02016-06-27 08:30:30 +02001611
1612 vlib_worker_thread_init (w);
1613 clib_time_init (&vm->clib_time);
1614 clib_mem_set_heap (w->thread_mheap);
1615
Damjan Marionfd8deb42021-03-06 12:26:28 +01001616 vm->worker_init_functions_called = hash_create (0, 0);
1617
1618 e = vlib_call_init_exit_functions_no_sort (
1619 vm, &vgm->worker_init_function_registrations, 1 /* call_once */,
1620 0 /* is_global */);
Damjan Marione9f929b2017-03-16 11:32:09 +01001621 if (e)
1622 clib_error_report (e);
1623
Damjan Marione9d52d52017-03-09 15:42:26 +01001624 vlib_worker_loop (vm);
Damjan Marion0f8ecf02016-06-27 08:30:30 +02001625}
1626
Dave Barach9b8ffd92016-07-08 08:13:45 -04001627/* *INDENT-OFF* */
Damjan Marion0f8ecf02016-06-27 08:30:30 +02001628VLIB_REGISTER_THREAD (worker_thread_reg, static) = {
1629 .name = "workers",
1630 .short_name = "wk",
1631 .function = vlib_worker_thread_fn,
1632};
Dave Barach9b8ffd92016-07-08 08:13:45 -04001633/* *INDENT-ON* */
Damjan Marion0f8ecf02016-06-27 08:30:30 +02001634
Mohammed Hawarie7149262022-05-18 10:08:47 +02001635extern clib_march_fn_registration
1636 *vlib_frame_queue_dequeue_with_aux_fn_march_fn_registrations;
1637extern clib_march_fn_registration
1638 *vlib_frame_queue_dequeue_fn_march_fn_registrations;
Damjan Marionaaef1eb2016-11-08 17:37:01 +01001639u32
1640vlib_frame_queue_main_init (u32 node_index, u32 frame_queue_nelts)
1641{
1642 vlib_thread_main_t *tm = vlib_get_thread_main ();
Mohammed Hawarie7149262022-05-18 10:08:47 +02001643 vlib_main_t *vm = vlib_get_main ();
Damjan Marionaaef1eb2016-11-08 17:37:01 +01001644 vlib_frame_queue_main_t *fqm;
1645 vlib_frame_queue_t *fq;
Mohammed Hawarie7149262022-05-18 10:08:47 +02001646 vlib_node_t *node;
Damjan Marionaaef1eb2016-11-08 17:37:01 +01001647 int i;
Elias Rudberg368104d2020-04-16 16:01:52 +02001648 u32 num_threads;
Damjan Marionaaef1eb2016-11-08 17:37:01 +01001649
1650 if (frame_queue_nelts == 0)
dongjuan88752482019-06-04 10:59:02 +08001651 frame_queue_nelts = FRAME_QUEUE_MAX_NELTS;
Damjan Marionaaef1eb2016-11-08 17:37:01 +01001652
Elias Rudberg368104d2020-04-16 16:01:52 +02001653 num_threads = 1 /* main thread */ + tm->n_threads;
1654 ASSERT (frame_queue_nelts >= 8 + num_threads);
Damjan Marion78fd7e82018-07-20 18:47:05 +02001655
Damjan Marionaaef1eb2016-11-08 17:37:01 +01001656 vec_add2 (tm->frame_queue_mains, fqm, 1);
1657
Mohammed Hawarie7149262022-05-18 10:08:47 +02001658 node = vlib_get_node (vm, fqm->node_index);
1659 ASSERT (node);
1660 if (node->aux_offset)
1661 {
1662 fqm->frame_queue_dequeue_fn =
1663 CLIB_MARCH_FN_VOID_POINTER (vlib_frame_queue_dequeue_with_aux_fn);
1664 }
1665 else
1666 {
1667 fqm->frame_queue_dequeue_fn =
1668 CLIB_MARCH_FN_VOID_POINTER (vlib_frame_queue_dequeue_fn);
1669 }
1670
Damjan Marionaaef1eb2016-11-08 17:37:01 +01001671 fqm->node_index = node_index;
Damjan Marion78fd7e82018-07-20 18:47:05 +02001672 fqm->frame_queue_nelts = frame_queue_nelts;
Damjan Marionaaef1eb2016-11-08 17:37:01 +01001673
1674 vec_validate (fqm->vlib_frame_queues, tm->n_vlib_mains - 1);
Damjan Marion8bea5892022-04-04 22:40:45 +02001675 vec_set_len (fqm->vlib_frame_queues, 0);
Damjan Marionaaef1eb2016-11-08 17:37:01 +01001676 for (i = 0; i < tm->n_vlib_mains; i++)
1677 {
1678 fq = vlib_frame_queue_alloc (frame_queue_nelts);
1679 vec_add1 (fqm->vlib_frame_queues, fq);
1680 }
1681
1682 return (fqm - tm->frame_queue_mains);
1683}
1684
Dave Barach69128d02017-09-26 10:54:34 -04001685void
1686vlib_process_signal_event_mt_helper (vlib_process_signal_event_mt_args_t *
1687 args)
1688{
1689 ASSERT (vlib_get_thread_index () == 0);
1690 vlib_process_signal_event (vlib_get_main (), args->node_index,
1691 args->type_opaque, args->data);
1692}
1693
1694void *rpc_call_main_thread_cb_fn;
1695
1696void
1697vlib_rpc_call_main_thread (void *callback, u8 * args, u32 arg_size)
1698{
1699 if (rpc_call_main_thread_cb_fn)
1700 {
1701 void (*fp) (void *, u8 *, u32) = rpc_call_main_thread_cb_fn;
1702 (*fp) (callback, args, arg_size);
1703 }
1704 else
1705 clib_warning ("BUG: rpc_call_main_thread_cb_fn NULL!");
1706}
1707
Dave Barach9b8ffd92016-07-08 08:13:45 -04001708clib_error_t *
1709threads_init (vlib_main_t * vm)
Damjan Marion0f8ecf02016-06-27 08:30:30 +02001710{
Benoît Ganne8b153de2021-12-09 18:24:21 +01001711 const vlib_thread_main_t *tm = vlib_get_thread_main ();
1712
1713 if (tm->main_lcore == ~0 && tm->n_vlib_mains > 1)
1714 return clib_error_return (0, "Configuration error, a main core must "
1715 "be specified when using worker threads");
1716
Ed Warnickecb9cada2015-12-08 15:45:58 -07001717 return 0;
1718}
1719
Damjan Marion0f8ecf02016-06-27 08:30:30 +02001720VLIB_INIT_FUNCTION (threads_init);
Dave Barach9b8ffd92016-07-08 08:13:45 -04001721
Dave Baracha4324a92019-02-19 17:05:30 -05001722static clib_error_t *
1723show_clock_command_fn (vlib_main_t * vm,
1724 unformat_input_t * input, vlib_cli_command_t * cmd)
1725{
Dave Barachc25048b2020-01-29 18:05:24 -05001726 int verbose = 0;
Dave Barach19718002020-03-11 10:31:36 -04001727 clib_timebase_t _tb, *tb = &_tb;
Dave Baracha4324a92019-02-19 17:05:30 -05001728
Dave Barachc25048b2020-01-29 18:05:24 -05001729 (void) unformat (input, "verbose %=", &verbose, 1);
Dave Baracha4324a92019-02-19 17:05:30 -05001730
Dave Barach19718002020-03-11 10:31:36 -04001731 clib_timebase_init (tb, 0 /* GMT */ , CLIB_TIMEBASE_DAYLIGHT_NONE,
1732 &vm->clib_time);
1733
1734 vlib_cli_output (vm, "%U, %U GMT", format_clib_time, &vm->clib_time,
1735 verbose, format_clib_timebase_time,
1736 clib_timebase_now (tb));
Dave Baracha4324a92019-02-19 17:05:30 -05001737
Dave Baracha4324a92019-02-19 17:05:30 -05001738 vlib_cli_output (vm, "Time last barrier release %.9f",
1739 vm->time_last_barrier_release);
1740
Benoît Ganne56eccdb2021-08-20 09:18:31 +02001741 foreach_vlib_main ()
Dave Baracha4324a92019-02-19 17:05:30 -05001742 {
Benoît Ganne56eccdb2021-08-20 09:18:31 +02001743 vlib_cli_output (vm, "%d: %U", this_vlib_main->thread_index,
1744 format_clib_time, &this_vlib_main->clib_time, verbose);
Dave Barachc25048b2020-01-29 18:05:24 -05001745
Benoît Ganne56eccdb2021-08-20 09:18:31 +02001746 vlib_cli_output (vm, "Thread %d offset %.9f error %.9f",
1747 this_vlib_main->thread_index,
1748 this_vlib_main->time_offset,
1749 vm->time_last_barrier_release -
1750 this_vlib_main->time_last_barrier_release);
Dave Baracha4324a92019-02-19 17:05:30 -05001751 }
1752 return 0;
1753}
1754
1755/* *INDENT-OFF* */
1756VLIB_CLI_COMMAND (f_command, static) =
1757{
1758 .path = "show clock",
1759 .short_help = "show clock",
1760 .function = show_clock_command_fn,
1761};
1762/* *INDENT-ON* */
1763
Dave Barachab1a50c2020-10-06 14:08:16 -04001764vlib_thread_main_t *
1765vlib_get_thread_main_not_inline (void)
1766{
1767 return vlib_get_thread_main ();
1768}
1769
Dave Barach9b8ffd92016-07-08 08:13:45 -04001770/*
1771 * fd.io coding-style-patch-verification: ON
1772 *
1773 * Local Variables:
1774 * eval: (c-set-style "gnu")
1775 * End:
1776 */