Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (c) 2015 Cisco and/or its affiliates. |
| 3 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | * you may not use this file except in compliance with the License. |
| 5 | * You may obtain a copy of the License at: |
| 6 | * |
| 7 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | * |
| 9 | * Unless required by applicable law or agreed to in writing, software |
| 10 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | * See the License for the specific language governing permissions and |
| 13 | * limitations under the License. |
| 14 | */ |
Christophe Fontaine | fef15b4 | 2016-04-09 12:38:49 +0900 | [diff] [blame] | 15 | #define _GNU_SOURCE |
Christophe Fontaine | fef15b4 | 2016-04-09 12:38:49 +0900 | [diff] [blame] | 16 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 17 | #include <signal.h> |
| 18 | #include <math.h> |
| 19 | #include <vppinfra/format.h> |
Dave Barach | 1971800 | 2020-03-11 10:31:36 -0400 | [diff] [blame] | 20 | #include <vppinfra/time_range.h> |
Damjan Marion | 9410053 | 2020-11-06 23:25:57 +0100 | [diff] [blame] | 21 | #include <vppinfra/interrupt.h> |
Mohsin Kazmi | 5d64c78 | 2018-09-11 20:27:09 +0200 | [diff] [blame] | 22 | #include <vppinfra/linux/sysfs.h> |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 23 | #include <vlib/vlib.h> |
| 24 | |
| 25 | #include <vlib/threads.h> |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 26 | |
Damjan Marion | 8973b07 | 2022-03-01 15:51:18 +0100 | [diff] [blame] | 27 | #include <vlib/stats/stats.h> |
Ole Troan | 233e468 | 2019-05-16 15:01:34 +0200 | [diff] [blame] | 28 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 29 | u32 |
| 30 | vl (void *p) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 31 | { |
| 32 | return vec_len (p); |
| 33 | } |
| 34 | |
Damjan Marion | 6a7acc2 | 2016-12-19 16:28:36 +0100 | [diff] [blame] | 35 | vlib_worker_thread_t *vlib_worker_threads; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 36 | vlib_thread_main_t vlib_thread_main; |
| 37 | |
Colin Tregenza Dancer | eb1ac17 | 2017-09-06 20:23:24 +0100 | [diff] [blame] | 38 | /* |
| 39 | * Barrier tracing can be enabled on a normal build to collect information |
| 40 | * on barrier use, including timings and call stacks. Deliberately not |
| 41 | * keyed off CLIB_DEBUG, because that can add significant overhead which |
| 42 | * imapacts observed timings. |
| 43 | */ |
| 44 | |
Colin Tregenza Dancer | eb1ac17 | 2017-09-06 20:23:24 +0100 | [diff] [blame] | 45 | static inline void |
| 46 | barrier_trace_sync (f64 t_entry, f64 t_open, f64 t_closed) |
| 47 | { |
Dave Barach | 88c6e00 | 2018-09-30 15:54:06 -0400 | [diff] [blame] | 48 | if (!vlib_worker_threads->barrier_elog_enabled) |
| 49 | return; |
| 50 | |
Damjan Marion | 1436100 | 2021-03-11 12:17:33 +0100 | [diff] [blame] | 51 | ELOG_TYPE_DECLARE (e) = { |
| 52 | .format = "bar-trace-%s-#%d", |
| 53 | .format_args = "T4i4", |
| 54 | }; |
| 55 | |
Colin Tregenza Dancer | eb1ac17 | 2017-09-06 20:23:24 +0100 | [diff] [blame] | 56 | struct |
| 57 | { |
Dave Barach | 88c6e00 | 2018-09-30 15:54:06 -0400 | [diff] [blame] | 58 | u32 caller, count, t_entry, t_open, t_closed; |
Colin Tregenza Dancer | eb1ac17 | 2017-09-06 20:23:24 +0100 | [diff] [blame] | 59 | } *ed = 0; |
| 60 | |
| 61 | ed = ELOG_DATA (&vlib_global_main.elog_main, e); |
| 62 | ed->count = (int) vlib_worker_threads[0].barrier_sync_count; |
Dave Barach | b09f4d0 | 2019-07-15 16:00:03 -0400 | [diff] [blame] | 63 | ed->caller = elog_string (&vlib_global_main.elog_main, |
| 64 | (char *) vlib_worker_threads[0].barrier_caller); |
Colin Tregenza Dancer | eb1ac17 | 2017-09-06 20:23:24 +0100 | [diff] [blame] | 65 | ed->t_entry = (int) (1000000.0 * t_entry); |
| 66 | ed->t_open = (int) (1000000.0 * t_open); |
| 67 | ed->t_closed = (int) (1000000.0 * t_closed); |
| 68 | } |
| 69 | |
| 70 | static inline void |
| 71 | barrier_trace_sync_rec (f64 t_entry) |
| 72 | { |
Dave Barach | 88c6e00 | 2018-09-30 15:54:06 -0400 | [diff] [blame] | 73 | if (!vlib_worker_threads->barrier_elog_enabled) |
| 74 | return; |
| 75 | |
Damjan Marion | 1436100 | 2021-03-11 12:17:33 +0100 | [diff] [blame] | 76 | ELOG_TYPE_DECLARE (e) = { |
| 77 | .format = "bar-syncrec-%s-#%d", |
| 78 | .format_args = "T4i4", |
| 79 | }; |
| 80 | |
Colin Tregenza Dancer | eb1ac17 | 2017-09-06 20:23:24 +0100 | [diff] [blame] | 81 | struct |
| 82 | { |
Dave Barach | 88c6e00 | 2018-09-30 15:54:06 -0400 | [diff] [blame] | 83 | u32 caller, depth; |
Colin Tregenza Dancer | eb1ac17 | 2017-09-06 20:23:24 +0100 | [diff] [blame] | 84 | } *ed = 0; |
| 85 | |
| 86 | ed = ELOG_DATA (&vlib_global_main.elog_main, e); |
| 87 | ed->depth = (int) vlib_worker_threads[0].recursion_level - 1; |
Dave Barach | b09f4d0 | 2019-07-15 16:00:03 -0400 | [diff] [blame] | 88 | ed->caller = elog_string (&vlib_global_main.elog_main, |
| 89 | (char *) vlib_worker_threads[0].barrier_caller); |
Colin Tregenza Dancer | eb1ac17 | 2017-09-06 20:23:24 +0100 | [diff] [blame] | 90 | } |
| 91 | |
| 92 | static inline void |
| 93 | barrier_trace_release_rec (f64 t_entry) |
| 94 | { |
Dave Barach | 88c6e00 | 2018-09-30 15:54:06 -0400 | [diff] [blame] | 95 | if (!vlib_worker_threads->barrier_elog_enabled) |
| 96 | return; |
| 97 | |
Damjan Marion | 1436100 | 2021-03-11 12:17:33 +0100 | [diff] [blame] | 98 | ELOG_TYPE_DECLARE (e) = { |
| 99 | .format = "bar-relrrec-#%d", |
| 100 | .format_args = "i4", |
| 101 | }; |
| 102 | |
Colin Tregenza Dancer | eb1ac17 | 2017-09-06 20:23:24 +0100 | [diff] [blame] | 103 | struct |
| 104 | { |
Dave Barach | 88c6e00 | 2018-09-30 15:54:06 -0400 | [diff] [blame] | 105 | u32 depth; |
Colin Tregenza Dancer | eb1ac17 | 2017-09-06 20:23:24 +0100 | [diff] [blame] | 106 | } *ed = 0; |
| 107 | |
| 108 | ed = ELOG_DATA (&vlib_global_main.elog_main, e); |
Colin Tregenza Dancer | eb1ac17 | 2017-09-06 20:23:24 +0100 | [diff] [blame] | 109 | ed->depth = (int) vlib_worker_threads[0].recursion_level; |
| 110 | } |
| 111 | |
| 112 | static inline void |
| 113 | barrier_trace_release (f64 t_entry, f64 t_closed_total, f64 t_update_main) |
| 114 | { |
Dave Barach | 88c6e00 | 2018-09-30 15:54:06 -0400 | [diff] [blame] | 115 | if (!vlib_worker_threads->barrier_elog_enabled) |
| 116 | return; |
| 117 | |
Damjan Marion | 1436100 | 2021-03-11 12:17:33 +0100 | [diff] [blame] | 118 | ELOG_TYPE_DECLARE (e) = { |
| 119 | .format = "bar-rel-#%d-e%d-u%d-t%d", |
| 120 | .format_args = "i4i4i4i4", |
| 121 | }; |
| 122 | |
Colin Tregenza Dancer | eb1ac17 | 2017-09-06 20:23:24 +0100 | [diff] [blame] | 123 | struct |
| 124 | { |
Dave Barach | 88c6e00 | 2018-09-30 15:54:06 -0400 | [diff] [blame] | 125 | u32 count, t_entry, t_update_main, t_closed_total; |
Colin Tregenza Dancer | eb1ac17 | 2017-09-06 20:23:24 +0100 | [diff] [blame] | 126 | } *ed = 0; |
| 127 | |
| 128 | ed = ELOG_DATA (&vlib_global_main.elog_main, e); |
| 129 | ed->t_entry = (int) (1000000.0 * t_entry); |
| 130 | ed->t_update_main = (int) (1000000.0 * t_update_main); |
| 131 | ed->t_closed_total = (int) (1000000.0 * t_closed_total); |
| 132 | ed->count = (int) vlib_worker_threads[0].barrier_sync_count; |
| 133 | |
| 134 | /* Reset context for next trace */ |
| 135 | vlib_worker_threads[0].barrier_context = NULL; |
| 136 | } |
Colin Tregenza Dancer | eb1ac17 | 2017-09-06 20:23:24 +0100 | [diff] [blame] | 137 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 138 | uword |
Damjan Marion | f55f9b8 | 2017-05-10 21:06:28 +0200 | [diff] [blame] | 139 | os_get_nthreads (void) |
Dave Barach | 01d86c7 | 2016-08-08 15:13:42 -0400 | [diff] [blame] | 140 | { |
Dave Barach | 2180bac | 2019-05-10 15:25:10 -0400 | [diff] [blame] | 141 | return vec_len (vlib_thread_stacks); |
Dave Barach | 01d86c7 | 2016-08-08 15:13:42 -0400 | [diff] [blame] | 142 | } |
| 143 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 144 | void |
| 145 | vlib_set_thread_name (char *name) |
| 146 | { |
| 147 | int pthread_setname_np (pthread_t __target_thread, const char *__name); |
Dave Barach | b2a6e25 | 2016-07-27 10:00:58 -0400 | [diff] [blame] | 148 | int rv; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 149 | pthread_t thread = pthread_self (); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 150 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 151 | if (thread) |
Dave Barach | b2a6e25 | 2016-07-27 10:00:58 -0400 | [diff] [blame] | 152 | { |
| 153 | rv = pthread_setname_np (thread, name); |
| 154 | if (rv) |
Ed Warnicke | 853e720 | 2016-08-12 11:42:26 -0700 | [diff] [blame] | 155 | clib_warning ("pthread_setname_np returned %d", rv); |
Dave Barach | b2a6e25 | 2016-07-27 10:00:58 -0400 | [diff] [blame] | 156 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 157 | } |
| 158 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 159 | static int |
| 160 | sort_registrations_by_no_clone (void *a0, void *a1) |
| 161 | { |
| 162 | vlib_thread_registration_t **tr0 = a0; |
| 163 | vlib_thread_registration_t **tr1 = a1; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 164 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 165 | return ((i32) ((*tr0)->no_data_structure_clone) |
| 166 | - ((i32) ((*tr1)->no_data_structure_clone))); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 167 | } |
| 168 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 169 | |
| 170 | /* Called early in the init sequence */ |
| 171 | |
| 172 | clib_error_t * |
| 173 | vlib_thread_init (vlib_main_t * vm) |
| 174 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 175 | vlib_thread_main_t *tm = &vlib_thread_main; |
| 176 | vlib_worker_thread_t *w; |
| 177 | vlib_thread_registration_t *tr; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 178 | u32 n_vlib_mains = 1; |
| 179 | u32 first_index = 1; |
| 180 | u32 i; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 181 | uword *avail_cpu; |
Damjan Marion | 8973b07 | 2022-03-01 15:51:18 +0100 | [diff] [blame] | 182 | u32 stats_num_worker_threads_dir_index; |
| 183 | |
| 184 | stats_num_worker_threads_dir_index = |
| 185 | vlib_stats_add_gauge ("/sys/num_worker_threads"); |
| 186 | ASSERT (stats_num_worker_threads_dir_index != ~0); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 187 | |
| 188 | /* get bitmaps of active cpu cores and sockets */ |
| 189 | tm->cpu_core_bitmap = |
Damjan Marion | 01914ce | 2017-09-14 19:04:50 +0200 | [diff] [blame] | 190 | clib_sysfs_list_to_bitmap ("/sys/devices/system/cpu/online"); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 191 | tm->cpu_socket_bitmap = |
Damjan Marion | 01914ce | 2017-09-14 19:04:50 +0200 | [diff] [blame] | 192 | clib_sysfs_list_to_bitmap ("/sys/devices/system/node/online"); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 193 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 194 | avail_cpu = clib_bitmap_dup (tm->cpu_core_bitmap); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 195 | |
| 196 | /* skip cores */ |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 197 | for (i = 0; i < tm->skip_cores; i++) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 198 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 199 | uword c = clib_bitmap_first_set (avail_cpu); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 200 | if (c == ~0) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 201 | return clib_error_return (0, "no available cpus to skip"); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 202 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 203 | avail_cpu = clib_bitmap_set (avail_cpu, c, 0); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 204 | } |
| 205 | |
| 206 | /* grab cpu for main thread */ |
Benoît Ganne | 8b153de | 2021-12-09 18:24:21 +0100 | [diff] [blame] | 207 | if (tm->main_lcore != ~0) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 208 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 209 | if (clib_bitmap_get (avail_cpu, tm->main_lcore) == 0) |
| 210 | return clib_error_return (0, "cpu %u is not available to be used" |
| 211 | " for the main thread", tm->main_lcore); |
Benoît Ganne | 8b153de | 2021-12-09 18:24:21 +0100 | [diff] [blame] | 212 | avail_cpu = clib_bitmap_set (avail_cpu, tm->main_lcore, 0); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 213 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 214 | |
| 215 | /* assume that there is socket 0 only if there is no data from sysfs */ |
| 216 | if (!tm->cpu_socket_bitmap) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 217 | tm->cpu_socket_bitmap = clib_bitmap_set (0, 0, 1); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 218 | |
Christophe Fontaine | fef15b4 | 2016-04-09 12:38:49 +0900 | [diff] [blame] | 219 | /* pin main thread to main_lcore */ |
Benoît Ganne | 8b153de | 2021-12-09 18:24:21 +0100 | [diff] [blame] | 220 | if (tm->main_lcore != ~0) |
| 221 | { |
| 222 | cpu_set_t cpuset; |
| 223 | CPU_ZERO (&cpuset); |
| 224 | CPU_SET (tm->main_lcore, &cpuset); |
hsandid | 832342e | 2023-12-20 15:41:54 +0100 | [diff] [blame] | 225 | if (pthread_setaffinity_np (pthread_self (), sizeof (cpu_set_t), |
| 226 | &cpuset)) |
| 227 | { |
| 228 | return clib_error_return (0, "could not pin main thread to cpu %u", |
| 229 | tm->main_lcore); |
| 230 | } |
Benoît Ganne | 8b153de | 2021-12-09 18:24:21 +0100 | [diff] [blame] | 231 | } |
Christophe Fontaine | fef15b4 | 2016-04-09 12:38:49 +0900 | [diff] [blame] | 232 | |
Dave Barach | 2180bac | 2019-05-10 15:25:10 -0400 | [diff] [blame] | 233 | /* Set up thread 0 */ |
| 234 | vec_validate_aligned (vlib_worker_threads, 0, CLIB_CACHE_LINE_BYTES); |
Damjan Marion | 8bea589 | 2022-04-04 22:40:45 +0200 | [diff] [blame] | 235 | vec_set_len (vlib_worker_threads, 1); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 236 | w = vlib_worker_threads; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 237 | w->thread_mheap = clib_mem_get_heap (); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 238 | w->thread_stack = vlib_thread_stacks[0]; |
Mohsin Kazmi | 5d64c78 | 2018-09-11 20:27:09 +0200 | [diff] [blame] | 239 | w->cpu_id = tm->main_lcore; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 240 | w->lwp = syscall (SYS_gettid); |
Pavel Kotucek | 9876520 | 2016-10-07 08:37:28 +0200 | [diff] [blame] | 241 | w->thread_id = pthread_self (); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 242 | tm->n_vlib_mains = 1; |
| 243 | |
Jon Loeliger | f617b14 | 2020-01-30 18:37:47 -0600 | [diff] [blame] | 244 | vlib_get_thread_core_numa (w, w->cpu_id); |
| 245 | |
Pavel Kotucek | 1e76583 | 2016-09-23 08:54:14 +0200 | [diff] [blame] | 246 | if (tm->sched_policy != ~0) |
| 247 | { |
| 248 | struct sched_param sched_param; |
| 249 | if (!sched_getparam (w->lwp, &sched_param)) |
| 250 | { |
| 251 | if (tm->sched_priority != ~0) |
| 252 | sched_param.sched_priority = tm->sched_priority; |
| 253 | sched_setscheduler (w->lwp, tm->sched_policy, &sched_param); |
| 254 | } |
| 255 | } |
| 256 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 257 | /* assign threads to cores and set n_vlib_mains */ |
| 258 | tr = tm->next; |
| 259 | |
| 260 | while (tr) |
| 261 | { |
| 262 | vec_add1 (tm->registrations, tr); |
| 263 | tr = tr->next; |
| 264 | } |
| 265 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 266 | vec_sort_with_function (tm->registrations, sort_registrations_by_no_clone); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 267 | |
| 268 | for (i = 0; i < vec_len (tm->registrations); i++) |
| 269 | { |
| 270 | int j; |
| 271 | tr = tm->registrations[i]; |
| 272 | tr->first_index = first_index; |
| 273 | first_index += tr->count; |
| 274 | n_vlib_mains += (tr->no_data_structure_clone == 0) ? tr->count : 0; |
| 275 | |
| 276 | /* construct coremask */ |
| 277 | if (tr->use_pthreads || !tr->count) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 278 | continue; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 279 | |
| 280 | if (tr->coremask) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 281 | { |
| 282 | uword c; |
| 283 | /* *INDENT-OFF* */ |
Damjan Marion | f0ca1e8 | 2020-12-13 23:26:56 +0100 | [diff] [blame] | 284 | clib_bitmap_foreach (c, tr->coremask) { |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 285 | if (clib_bitmap_get(avail_cpu, c) == 0) |
| 286 | return clib_error_return (0, "cpu %u is not available to be used" |
| 287 | " for the '%s' thread",c, tr->name); |
| 288 | |
| 289 | avail_cpu = clib_bitmap_set(avail_cpu, c, 0); |
Damjan Marion | f0ca1e8 | 2020-12-13 23:26:56 +0100 | [diff] [blame] | 290 | } |
Dave Barach | 2180bac | 2019-05-10 15:25:10 -0400 | [diff] [blame] | 291 | /* *INDENT-ON* */ |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 292 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 293 | else |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 294 | { |
| 295 | for (j = 0; j < tr->count; j++) |
| 296 | { |
Vladimir Isaev | 2ed4204 | 2020-03-17 12:56:31 +0300 | [diff] [blame] | 297 | /* Do not use CPU 0 by default - leave it to the host and IRQs */ |
| 298 | uword avail_c0 = clib_bitmap_get (avail_cpu, 0); |
| 299 | avail_cpu = clib_bitmap_set (avail_cpu, 0, 0); |
| 300 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 301 | uword c = clib_bitmap_first_set (avail_cpu); |
Vladimir Isaev | 2ed4204 | 2020-03-17 12:56:31 +0300 | [diff] [blame] | 302 | /* Use CPU 0 as a last resort */ |
| 303 | if (c == ~0 && avail_c0) |
| 304 | { |
| 305 | c = 0; |
| 306 | avail_c0 = 0; |
| 307 | } |
| 308 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 309 | if (c == ~0) |
| 310 | return clib_error_return (0, |
| 311 | "no available cpus to be used for" |
hsandid | 832342e | 2023-12-20 15:41:54 +0100 | [diff] [blame] | 312 | " the '%s' thread #%u", |
| 313 | tr->name, tr->count); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 314 | |
Vladimir Isaev | 2ed4204 | 2020-03-17 12:56:31 +0300 | [diff] [blame] | 315 | avail_cpu = clib_bitmap_set (avail_cpu, 0, avail_c0); |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 316 | avail_cpu = clib_bitmap_set (avail_cpu, c, 0); |
| 317 | tr->coremask = clib_bitmap_set (tr->coremask, c, 1); |
| 318 | } |
| 319 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 320 | } |
| 321 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 322 | clib_bitmap_free (avail_cpu); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 323 | |
| 324 | tm->n_vlib_mains = n_vlib_mains; |
Damjan Marion | 8973b07 | 2022-03-01 15:51:18 +0100 | [diff] [blame] | 325 | vlib_stats_set_gauge (stats_num_worker_threads_dir_index, n_vlib_mains - 1); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 326 | |
Dave Barach | 2180bac | 2019-05-10 15:25:10 -0400 | [diff] [blame] | 327 | /* |
| 328 | * Allocate the remaining worker threads, and thread stack vector slots |
| 329 | * from now on, calls to os_get_nthreads() will return the correct |
| 330 | * answer. |
| 331 | */ |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 332 | vec_validate_aligned (vlib_worker_threads, first_index - 1, |
| 333 | CLIB_CACHE_LINE_BYTES); |
Dave Barach | 2180bac | 2019-05-10 15:25:10 -0400 | [diff] [blame] | 334 | vec_validate (vlib_thread_stacks, vec_len (vlib_worker_threads) - 1); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 335 | return 0; |
| 336 | } |
| 337 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 338 | vlib_frame_queue_t * |
| 339 | vlib_frame_queue_alloc (int nelts) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 340 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 341 | vlib_frame_queue_t *fq; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 342 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 343 | fq = clib_mem_alloc_aligned (sizeof (*fq), CLIB_CACHE_LINE_BYTES); |
Dave Barach | b7b9299 | 2018-10-17 10:38:51 -0400 | [diff] [blame] | 344 | clib_memset (fq, 0, sizeof (*fq)); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 345 | fq->nelts = nelts; |
Damjan Marion | c0d9ca7 | 2021-05-11 09:39:24 +0200 | [diff] [blame] | 346 | fq->vector_threshold = 2 * VLIB_FRAME_SIZE; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 347 | vec_validate_aligned (fq->elts, nelts - 1, CLIB_CACHE_LINE_BYTES); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 348 | |
Damjan Marion | c0d9ca7 | 2021-05-11 09:39:24 +0200 | [diff] [blame] | 349 | if (nelts & (nelts - 1)) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 350 | { |
Damjan Marion | c0d9ca7 | 2021-05-11 09:39:24 +0200 | [diff] [blame] | 351 | fformat (stderr, "FATAL: nelts MUST be a power of 2\n"); |
| 352 | abort (); |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 353 | } |
| 354 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 355 | return (fq); |
| 356 | } |
| 357 | |
| 358 | void vl_msg_api_handler_no_free (void *) __attribute__ ((weak)); |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 359 | void |
| 360 | vl_msg_api_handler_no_free (void *v) |
| 361 | { |
| 362 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 363 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 364 | /* To be called by vlib worker threads upon startup */ |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 365 | void |
| 366 | vlib_worker_thread_init (vlib_worker_thread_t * w) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 367 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 368 | vlib_thread_main_t *tm = vlib_get_thread_main (); |
| 369 | |
Dave Barach | fdf4944 | 2016-12-20 12:48:14 -0500 | [diff] [blame] | 370 | /* |
| 371 | * Note: disabling signals in worker threads as follows |
| 372 | * prevents the api post-mortem dump scheme from working |
| 373 | * { |
| 374 | * sigset_t s; |
| 375 | * sigfillset (&s); |
| 376 | * pthread_sigmask (SIG_SETMASK, &s, 0); |
| 377 | * } |
| 378 | */ |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 379 | |
| 380 | clib_mem_set_heap (w->thread_mheap); |
| 381 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 382 | if (vec_len (tm->thread_prefix) && w->registration->short_name) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 383 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 384 | w->name = format (0, "%v_%s_%d%c", tm->thread_prefix, |
| 385 | w->registration->short_name, w->instance_id, '\0'); |
| 386 | vlib_set_thread_name ((char *) w->name); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 387 | } |
| 388 | |
| 389 | if (!w->registration->use_pthreads) |
| 390 | { |
| 391 | |
| 392 | /* Initial barrier sync, for both worker and i/o threads */ |
Sirshak Das | 2f6d7bb | 2018-10-03 22:53:51 +0000 | [diff] [blame] | 393 | clib_atomic_fetch_add (vlib_worker_threads->workers_at_barrier, 1); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 394 | |
| 395 | while (*vlib_worker_threads->wait_at_barrier) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 396 | ; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 397 | |
Sirshak Das | 2f6d7bb | 2018-10-03 22:53:51 +0000 | [diff] [blame] | 398 | clib_atomic_fetch_add (vlib_worker_threads->workers_at_barrier, -1); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 399 | } |
| 400 | } |
| 401 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 402 | void * |
| 403 | vlib_worker_thread_bootstrap_fn (void *arg) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 404 | { |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 405 | vlib_worker_thread_t *w = arg; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 406 | |
| 407 | w->lwp = syscall (SYS_gettid); |
Pavel Kotucek | 9876520 | 2016-10-07 08:37:28 +0200 | [diff] [blame] | 408 | w->thread_id = pthread_self (); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 409 | |
Damjan Marion | f55f9b8 | 2017-05-10 21:06:28 +0200 | [diff] [blame] | 410 | __os_thread_index = w - vlib_worker_threads; |
Damjan Marion | 586afd7 | 2017-04-05 19:18:20 +0200 | [diff] [blame] | 411 | |
Damjan Marion | 7bf2317 | 2022-03-28 01:47:33 +0200 | [diff] [blame] | 412 | if (CLIB_DEBUG > 0) |
| 413 | { |
| 414 | void *frame_addr = __builtin_frame_address (0); |
| 415 | if (frame_addr < (void *) w->thread_stack || |
| 416 | frame_addr > (void *) w->thread_stack + VLIB_THREAD_STACK_SIZE) |
| 417 | { |
| 418 | /* heap is not set yet */ |
| 419 | fprintf (stderr, "thread stack is not set properly\n"); |
| 420 | exit (1); |
| 421 | } |
| 422 | } |
Dave Barach | 37579c3 | 2021-07-27 09:27:07 -0400 | [diff] [blame] | 423 | |
Damjan Marion | 7bf2317 | 2022-03-28 01:47:33 +0200 | [diff] [blame] | 424 | w->thread_function (arg); |
| 425 | |
| 426 | return 0; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 427 | } |
| 428 | |
Dave Barach | a690fdb | 2020-01-21 12:34:55 -0500 | [diff] [blame] | 429 | void |
| 430 | vlib_get_thread_core_numa (vlib_worker_thread_t * w, unsigned cpu_id) |
Mohsin Kazmi | 5d64c78 | 2018-09-11 20:27:09 +0200 | [diff] [blame] | 431 | { |
| 432 | const char *sys_cpu_path = "/sys/devices/system/cpu/cpu"; |
Lijian.Zhang | 4fbb9da | 2020-02-14 15:16:49 +0800 | [diff] [blame] | 433 | const char *sys_node_path = "/sys/devices/system/node/node"; |
| 434 | clib_bitmap_t *nbmp = 0, *cbmp = 0; |
| 435 | u32 node; |
Mohsin Kazmi | 5d64c78 | 2018-09-11 20:27:09 +0200 | [diff] [blame] | 436 | u8 *p = 0; |
Dave Barach | a690fdb | 2020-01-21 12:34:55 -0500 | [diff] [blame] | 437 | int core_id = -1, numa_id = -1; |
Mohsin Kazmi | 5d64c78 | 2018-09-11 20:27:09 +0200 | [diff] [blame] | 438 | |
| 439 | p = format (p, "%s%u/topology/core_id%c", sys_cpu_path, cpu_id, 0); |
| 440 | clib_sysfs_read ((char *) p, "%d", &core_id); |
| 441 | vec_reset_length (p); |
Lijian.Zhang | 4fbb9da | 2020-02-14 15:16:49 +0800 | [diff] [blame] | 442 | |
| 443 | /* *INDENT-OFF* */ |
| 444 | clib_sysfs_read ("/sys/devices/system/node/online", "%U", |
| 445 | unformat_bitmap_list, &nbmp); |
Damjan Marion | f0ca1e8 | 2020-12-13 23:26:56 +0100 | [diff] [blame] | 446 | clib_bitmap_foreach (node, nbmp) { |
Lijian.Zhang | 4fbb9da | 2020-02-14 15:16:49 +0800 | [diff] [blame] | 447 | p = format (p, "%s%u/cpulist%c", sys_node_path, node, 0); |
| 448 | clib_sysfs_read ((char *) p, "%U", unformat_bitmap_list, &cbmp); |
| 449 | if (clib_bitmap_get (cbmp, cpu_id)) |
| 450 | numa_id = node; |
| 451 | vec_reset_length (cbmp); |
| 452 | vec_reset_length (p); |
Damjan Marion | f0ca1e8 | 2020-12-13 23:26:56 +0100 | [diff] [blame] | 453 | } |
Lijian.Zhang | 4fbb9da | 2020-02-14 15:16:49 +0800 | [diff] [blame] | 454 | /* *INDENT-ON* */ |
| 455 | vec_free (nbmp); |
| 456 | vec_free (cbmp); |
Mohsin Kazmi | 5d64c78 | 2018-09-11 20:27:09 +0200 | [diff] [blame] | 457 | vec_free (p); |
| 458 | |
| 459 | w->core_id = core_id; |
Dave Barach | a690fdb | 2020-01-21 12:34:55 -0500 | [diff] [blame] | 460 | w->numa_id = numa_id; |
Mohsin Kazmi | 5d64c78 | 2018-09-11 20:27:09 +0200 | [diff] [blame] | 461 | } |
| 462 | |
Damjan Marion | 878c609 | 2017-01-04 13:19:27 +0100 | [diff] [blame] | 463 | static clib_error_t * |
Mohsin Kazmi | 5d64c78 | 2018-09-11 20:27:09 +0200 | [diff] [blame] | 464 | vlib_launch_thread_int (void *fp, vlib_worker_thread_t * w, unsigned cpu_id) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 465 | { |
Damjan Marion | 57d1ec0 | 2020-09-16 21:15:44 +0200 | [diff] [blame] | 466 | clib_mem_main_t *mm = &clib_mem_main; |
Damjan Marion | 878c609 | 2017-01-04 13:19:27 +0100 | [diff] [blame] | 467 | vlib_thread_main_t *tm = &vlib_thread_main; |
Damjan Marion | a2b7a02 | 2021-12-28 20:32:20 +0100 | [diff] [blame] | 468 | pthread_t worker; |
Damjan Marion | 7bf2317 | 2022-03-28 01:47:33 +0200 | [diff] [blame] | 469 | pthread_attr_t attr; |
Damjan Marion | a2b7a02 | 2021-12-28 20:32:20 +0100 | [diff] [blame] | 470 | cpu_set_t cpuset; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 471 | void *(*fp_arg) (void *) = fp; |
Dave Barach | a690fdb | 2020-01-21 12:34:55 -0500 | [diff] [blame] | 472 | void *numa_heap; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 473 | |
Mohsin Kazmi | 5d64c78 | 2018-09-11 20:27:09 +0200 | [diff] [blame] | 474 | w->cpu_id = cpu_id; |
Dave Barach | a690fdb | 2020-01-21 12:34:55 -0500 | [diff] [blame] | 475 | vlib_get_thread_core_numa (w, cpu_id); |
Dave Barach | a690fdb | 2020-01-21 12:34:55 -0500 | [diff] [blame] | 476 | |
| 477 | /* Set up NUMA-bound heap if indicated */ |
Damjan Marion | 57d1ec0 | 2020-09-16 21:15:44 +0200 | [diff] [blame] | 478 | if (mm->per_numa_mheaps[w->numa_id] == 0) |
Dave Barach | a690fdb | 2020-01-21 12:34:55 -0500 | [diff] [blame] | 479 | { |
| 480 | /* If the user requested a NUMA heap, create it... */ |
| 481 | if (tm->numa_heap_size) |
| 482 | { |
Damjan Marion | 2bc1af5 | 2020-10-02 15:01:12 +0200 | [diff] [blame] | 483 | clib_mem_set_numa_affinity (w->numa_id, 1 /* force */ ); |
| 484 | numa_heap = clib_mem_create_heap (0 /* DIY */ , tm->numa_heap_size, |
| 485 | 1 /* is_locked */ , |
| 486 | "numa %u heap", w->numa_id); |
| 487 | clib_mem_set_default_numa_affinity (); |
Damjan Marion | 57d1ec0 | 2020-09-16 21:15:44 +0200 | [diff] [blame] | 488 | mm->per_numa_mheaps[w->numa_id] = numa_heap; |
Dave Barach | a690fdb | 2020-01-21 12:34:55 -0500 | [diff] [blame] | 489 | } |
| 490 | else |
| 491 | { |
| 492 | /* Or, use the main heap */ |
Damjan Marion | 57d1ec0 | 2020-09-16 21:15:44 +0200 | [diff] [blame] | 493 | mm->per_numa_mheaps[w->numa_id] = w->thread_mheap; |
Dave Barach | a690fdb | 2020-01-21 12:34:55 -0500 | [diff] [blame] | 494 | } |
| 495 | } |
| 496 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 497 | CPU_ZERO (&cpuset); |
Mohsin Kazmi | 5d64c78 | 2018-09-11 20:27:09 +0200 | [diff] [blame] | 498 | CPU_SET (cpu_id, &cpuset); |
Christophe Fontaine | fef15b4 | 2016-04-09 12:38:49 +0900 | [diff] [blame] | 499 | |
Damjan Marion | 7bf2317 | 2022-03-28 01:47:33 +0200 | [diff] [blame] | 500 | if (pthread_attr_init (&attr)) |
| 501 | return clib_error_return_unix (0, "pthread_attr_init"); |
| 502 | |
| 503 | if (pthread_attr_setstack (&attr, w->thread_stack, |
| 504 | VLIB_THREAD_STACK_SIZE)) |
| 505 | return clib_error_return_unix (0, "pthread_attr_setstack"); |
| 506 | |
| 507 | if (pthread_create (&worker, &attr, fp_arg, (void *) w)) |
Damjan Marion | 878c609 | 2017-01-04 13:19:27 +0100 | [diff] [blame] | 508 | return clib_error_return_unix (0, "pthread_create"); |
| 509 | |
| 510 | if (pthread_setaffinity_np (worker, sizeof (cpu_set_t), &cpuset)) |
| 511 | return clib_error_return_unix (0, "pthread_setaffinity_np"); |
| 512 | |
Damjan Marion | 7bf2317 | 2022-03-28 01:47:33 +0200 | [diff] [blame] | 513 | if (pthread_attr_destroy (&attr)) |
| 514 | return clib_error_return_unix (0, "pthread_attr_destroy"); |
| 515 | |
Damjan Marion | 878c609 | 2017-01-04 13:19:27 +0100 | [diff] [blame] | 516 | return 0; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 517 | } |
| 518 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 519 | static clib_error_t * |
| 520 | start_workers (vlib_main_t * vm) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 521 | { |
Damjan Marion | fd8deb4 | 2021-03-06 12:26:28 +0100 | [diff] [blame] | 522 | vlib_global_main_t *vgm = vlib_get_global_main (); |
Damjan Marion | 66c8583 | 2022-03-14 13:04:38 +0100 | [diff] [blame] | 523 | vlib_main_t *fvm = vlib_get_first_main (); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 524 | int i, j; |
| 525 | vlib_worker_thread_t *w; |
| 526 | vlib_main_t *vm_clone; |
| 527 | void *oldheap; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 528 | vlib_thread_main_t *tm = &vlib_thread_main; |
| 529 | vlib_thread_registration_t *tr; |
| 530 | vlib_node_runtime_t *rt; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 531 | u32 n_vlib_mains = tm->n_vlib_mains; |
| 532 | u32 worker_thread_index; |
Damjan Marion | 66c8583 | 2022-03-14 13:04:38 +0100 | [diff] [blame] | 533 | u32 stats_err_entry_index = fvm->error_main.stats_err_entry_index; |
Damjan Marion | bfa75d6 | 2020-10-06 17:46:06 +0200 | [diff] [blame] | 534 | clib_mem_heap_t *main_heap = clib_mem_get_per_cpu_heap (); |
Ole Troan | a606d92 | 2021-05-05 09:23:17 +0200 | [diff] [blame] | 535 | vlib_stats_register_mem_heap (main_heap); |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 536 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 537 | vec_reset_length (vlib_worker_threads); |
| 538 | |
| 539 | /* Set up the main thread */ |
| 540 | vec_add2_aligned (vlib_worker_threads, w, 1, CLIB_CACHE_LINE_BYTES); |
Dave Barach | 3e7deb1 | 2016-02-05 16:29:53 -0500 | [diff] [blame] | 541 | w->elog_track.name = "main thread"; |
Damjan Marion | f553a2c | 2021-03-26 13:45:37 +0100 | [diff] [blame] | 542 | elog_track_register (vlib_get_elog_main (), &w->elog_track); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 543 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 544 | if (vec_len (tm->thread_prefix)) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 545 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 546 | w->name = format (0, "%v_main%c", tm->thread_prefix, '\0'); |
| 547 | vlib_set_thread_name ((char *) w->name); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 548 | } |
| 549 | |
Damjan Marion | fd8deb4 | 2021-03-06 12:26:28 +0100 | [diff] [blame] | 550 | vgm->elog_main.lock = |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 551 | clib_mem_alloc_aligned (CLIB_CACHE_LINE_BYTES, CLIB_CACHE_LINE_BYTES); |
Damjan Marion | fd8deb4 | 2021-03-06 12:26:28 +0100 | [diff] [blame] | 552 | vgm->elog_main.lock[0] = 0; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 553 | |
Tom Seidenberg | 6c81f5a | 2020-07-10 15:49:03 +0000 | [diff] [blame] | 554 | clib_callback_data_init (&vm->vlib_node_runtime_perf_callbacks, |
| 555 | &vm->worker_thread_main_loop_callback_lock); |
| 556 | |
Damjan Marion | fd8deb4 | 2021-03-06 12:26:28 +0100 | [diff] [blame] | 557 | vec_validate_aligned (vgm->vlib_mains, n_vlib_mains - 1, |
| 558 | CLIB_CACHE_LINE_BYTES); |
Damjan Marion | 8bea589 | 2022-04-04 22:40:45 +0200 | [diff] [blame] | 559 | vec_set_len (vgm->vlib_mains, 0); |
Damjan Marion | fd8deb4 | 2021-03-06 12:26:28 +0100 | [diff] [blame] | 560 | vec_add1_aligned (vgm->vlib_mains, vm, CLIB_CACHE_LINE_BYTES); |
Dave Barach | 1c556d1 | 2020-10-02 11:35:44 -0400 | [diff] [blame] | 561 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 562 | if (n_vlib_mains > 1) |
| 563 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 564 | vlib_worker_threads->wait_at_barrier = |
| 565 | clib_mem_alloc_aligned (sizeof (u32), CLIB_CACHE_LINE_BYTES); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 566 | vlib_worker_threads->workers_at_barrier = |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 567 | clib_mem_alloc_aligned (sizeof (u32), CLIB_CACHE_LINE_BYTES); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 568 | |
Colin Tregenza Dancer | 2159618 | 2017-09-04 15:27:49 +0100 | [diff] [blame] | 569 | vlib_worker_threads->node_reforks_required = |
| 570 | clib_mem_alloc_aligned (sizeof (u32), CLIB_CACHE_LINE_BYTES); |
| 571 | |
Dave Barach | f6c68d7 | 2018-11-01 08:12:52 -0400 | [diff] [blame] | 572 | /* We'll need the rpc vector lock... */ |
| 573 | clib_spinlock_init (&vm->pending_rpc_lock); |
| 574 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 575 | /* Ask for an initial barrier sync */ |
| 576 | *vlib_worker_threads->workers_at_barrier = 0; |
| 577 | *vlib_worker_threads->wait_at_barrier = 1; |
| 578 | |
Colin Tregenza Dancer | 2159618 | 2017-09-04 15:27:49 +0100 | [diff] [blame] | 579 | /* Without update or refork */ |
| 580 | *vlib_worker_threads->node_reforks_required = 0; |
Damjan Marion | fd8deb4 | 2021-03-06 12:26:28 +0100 | [diff] [blame] | 581 | vgm->need_vlib_worker_thread_node_runtime_update = 0; |
Colin Tregenza Dancer | 2159618 | 2017-09-04 15:27:49 +0100 | [diff] [blame] | 582 | |
Colin Tregenza Dancer | eb1ac17 | 2017-09-06 20:23:24 +0100 | [diff] [blame] | 583 | /* init timing */ |
| 584 | vm->barrier_epoch = 0; |
| 585 | vm->barrier_no_close_before = 0; |
| 586 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 587 | worker_thread_index = 1; |
Tom Seidenberg | 6c81f5a | 2020-07-10 15:49:03 +0000 | [diff] [blame] | 588 | clib_spinlock_init (&vm->worker_thread_main_loop_callback_lock); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 589 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 590 | for (i = 0; i < vec_len (tm->registrations); i++) |
| 591 | { |
| 592 | vlib_node_main_t *nm, *nm_clone; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 593 | int k; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 594 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 595 | tr = tm->registrations[i]; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 596 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 597 | if (tr->count == 0) |
| 598 | continue; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 599 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 600 | for (k = 0; k < tr->count; k++) |
| 601 | { |
Colin Tregenza Dancer | 2159618 | 2017-09-04 15:27:49 +0100 | [diff] [blame] | 602 | vlib_node_t *n; |
Damjan Marion | 66c8583 | 2022-03-14 13:04:38 +0100 | [diff] [blame] | 603 | u64 **c; |
Colin Tregenza Dancer | 2159618 | 2017-09-04 15:27:49 +0100 | [diff] [blame] | 604 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 605 | vec_add2 (vlib_worker_threads, w, 1); |
Dave Barach | 6a5adc3 | 2018-07-04 10:56:23 -0400 | [diff] [blame] | 606 | /* Currently unused, may not really work */ |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 607 | if (tr->mheap_size) |
Damjan Marion | 4537c30 | 2020-09-28 19:03:37 +0200 | [diff] [blame] | 608 | w->thread_mheap = clib_mem_create_heap (0, tr->mheap_size, |
| 609 | /* unlocked */ 0, |
| 610 | "%s%d heap", |
| 611 | tr->name, k); |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 612 | else |
| 613 | w->thread_mheap = main_heap; |
Damjan Marion | 586afd7 | 2017-04-05 19:18:20 +0200 | [diff] [blame] | 614 | |
| 615 | w->thread_stack = |
| 616 | vlib_thread_stack_init (w - vlib_worker_threads); |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 617 | w->thread_function = tr->function; |
| 618 | w->thread_function_arg = w; |
| 619 | w->instance_id = k; |
| 620 | w->registration = tr; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 621 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 622 | w->elog_track.name = |
| 623 | (char *) format (0, "%s %d", tr->name, k + 1); |
| 624 | vec_add1 (w->elog_track.name, 0); |
Damjan Marion | f553a2c | 2021-03-26 13:45:37 +0100 | [diff] [blame] | 625 | elog_track_register (vlib_get_elog_main (), &w->elog_track); |
Bud Grise | 68adab9 | 2016-02-12 10:36:11 -0500 | [diff] [blame] | 626 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 627 | if (tr->no_data_structure_clone) |
| 628 | continue; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 629 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 630 | /* Fork vlib_global_main et al. Look for bugs here */ |
| 631 | oldheap = clib_mem_set_heap (w->thread_mheap); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 632 | |
Damjan Marion | be3f4d5 | 2018-03-27 21:06:10 +0200 | [diff] [blame] | 633 | vm_clone = clib_mem_alloc_aligned (sizeof (*vm_clone), |
| 634 | CLIB_CACHE_LINE_BYTES); |
Damjan Marion | fd8deb4 | 2021-03-06 12:26:28 +0100 | [diff] [blame] | 635 | clib_memcpy (vm_clone, vlib_get_first_main (), |
| 636 | sizeof (*vm_clone)); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 637 | |
Damjan Marion | 586afd7 | 2017-04-05 19:18:20 +0200 | [diff] [blame] | 638 | vm_clone->thread_index = worker_thread_index; |
Dave Barach | 2877eee | 2017-12-15 12:22:57 -0500 | [diff] [blame] | 639 | vm_clone->pending_rpc_requests = 0; |
| 640 | vec_validate (vm_clone->pending_rpc_requests, 0); |
Damjan Marion | 8bea589 | 2022-04-04 22:40:45 +0200 | [diff] [blame] | 641 | vec_set_len (vm_clone->pending_rpc_requests, 0); |
Dave Barach | b7b9299 | 2018-10-17 10:38:51 -0400 | [diff] [blame] | 642 | clib_memset (&vm_clone->random_buffer, 0, |
| 643 | sizeof (vm_clone->random_buffer)); |
Tom Seidenberg | 6c81f5a | 2020-07-10 15:49:03 +0000 | [diff] [blame] | 644 | clib_spinlock_init |
| 645 | (&vm_clone->worker_thread_main_loop_callback_lock); |
| 646 | clib_callback_data_init |
| 647 | (&vm_clone->vlib_node_runtime_perf_callbacks, |
| 648 | &vm_clone->worker_thread_main_loop_callback_lock); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 649 | |
Damjan Marion | fd8deb4 | 2021-03-06 12:26:28 +0100 | [diff] [blame] | 650 | nm = &vlib_get_first_main ()->node_main; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 651 | nm_clone = &vm_clone->node_main; |
| 652 | /* fork next frames array, preserving node runtime indices */ |
Damjan Marion | be3f4d5 | 2018-03-27 21:06:10 +0200 | [diff] [blame] | 653 | nm_clone->next_frames = vec_dup_aligned (nm->next_frames, |
| 654 | CLIB_CACHE_LINE_BYTES); |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 655 | for (j = 0; j < vec_len (nm_clone->next_frames); j++) |
| 656 | { |
| 657 | vlib_next_frame_t *nf = &nm_clone->next_frames[j]; |
| 658 | u32 save_node_runtime_index; |
| 659 | u32 save_flags; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 660 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 661 | save_node_runtime_index = nf->node_runtime_index; |
| 662 | save_flags = nf->flags & VLIB_FRAME_NO_FREE_AFTER_DISPATCH; |
| 663 | vlib_next_frame_init (nf); |
| 664 | nf->node_runtime_index = save_node_runtime_index; |
| 665 | nf->flags = save_flags; |
| 666 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 667 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 668 | /* fork the frame dispatch queue */ |
| 669 | nm_clone->pending_frames = 0; |
Dave Barach | 53fe4a7 | 2019-01-26 09:50:26 -0500 | [diff] [blame] | 670 | vec_validate (nm_clone->pending_frames, 10); |
Damjan Marion | 8bea589 | 2022-04-04 22:40:45 +0200 | [diff] [blame] | 671 | vec_set_len (nm_clone->pending_frames, 0); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 672 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 673 | /* fork nodes */ |
| 674 | nm_clone->nodes = 0; |
Colin Tregenza Dancer | 2159618 | 2017-09-04 15:27:49 +0100 | [diff] [blame] | 675 | |
| 676 | /* Allocate all nodes in single block for speed */ |
| 677 | n = clib_mem_alloc_no_fail (vec_len (nm->nodes) * sizeof (*n)); |
| 678 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 679 | for (j = 0; j < vec_len (nm->nodes); j++) |
| 680 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 681 | clib_memcpy (n, nm->nodes[j], sizeof (*n)); |
| 682 | /* none of the copied nodes have enqueue rights given out */ |
| 683 | n->owner_node_index = VLIB_INVALID_NODE_INDEX; |
Dave Barach | b7b9299 | 2018-10-17 10:38:51 -0400 | [diff] [blame] | 684 | clib_memset (&n->stats_total, 0, sizeof (n->stats_total)); |
| 685 | clib_memset (&n->stats_last_clear, 0, |
| 686 | sizeof (n->stats_last_clear)); |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 687 | vec_add1 (nm_clone->nodes, n); |
Colin Tregenza Dancer | 2159618 | 2017-09-04 15:27:49 +0100 | [diff] [blame] | 688 | n++; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 689 | } |
| 690 | nm_clone->nodes_by_type[VLIB_NODE_TYPE_INTERNAL] = |
Damjan Marion | be3f4d5 | 2018-03-27 21:06:10 +0200 | [diff] [blame] | 691 | vec_dup_aligned (nm->nodes_by_type[VLIB_NODE_TYPE_INTERNAL], |
| 692 | CLIB_CACHE_LINE_BYTES); |
JingLiuZTE | 30af5da | 2017-07-24 10:53:31 +0800 | [diff] [blame] | 693 | vec_foreach (rt, |
| 694 | nm_clone->nodes_by_type[VLIB_NODE_TYPE_INTERNAL]) |
Damjan Marion | e9f929b | 2017-03-16 11:32:09 +0100 | [diff] [blame] | 695 | { |
| 696 | vlib_node_t *n = vlib_get_node (vm, rt->node_index); |
Damjan Marion | e9f929b | 2017-03-16 11:32:09 +0100 | [diff] [blame] | 697 | /* copy initial runtime_data from node */ |
Damjan Marion | b6f93a1 | 2017-03-16 17:46:41 +0100 | [diff] [blame] | 698 | if (n->runtime_data && n->runtime_data_bytes > 0) |
Damjan Marion | e9f929b | 2017-03-16 11:32:09 +0100 | [diff] [blame] | 699 | clib_memcpy (rt->runtime_data, n->runtime_data, |
Damjan Marion | b6f93a1 | 2017-03-16 17:46:41 +0100 | [diff] [blame] | 700 | clib_min (VLIB_NODE_RUNTIME_DATA_SIZE, |
| 701 | n->runtime_data_bytes)); |
Damjan Marion | e9f929b | 2017-03-16 11:32:09 +0100 | [diff] [blame] | 702 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 703 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 704 | nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT] = |
Damjan Marion | be3f4d5 | 2018-03-27 21:06:10 +0200 | [diff] [blame] | 705 | vec_dup_aligned (nm->nodes_by_type[VLIB_NODE_TYPE_INPUT], |
| 706 | CLIB_CACHE_LINE_BYTES); |
Damjan Marion | 9410053 | 2020-11-06 23:25:57 +0100 | [diff] [blame] | 707 | clib_interrupt_init ( |
Damjan Marion | cc8249c | 2023-07-23 14:24:22 +0200 | [diff] [blame] | 708 | &nm_clone->input_node_interrupts, |
Damjan Marion | 9410053 | 2020-11-06 23:25:57 +0100 | [diff] [blame] | 709 | vec_len (nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT])); |
Damjan Marion | cc8249c | 2023-07-23 14:24:22 +0200 | [diff] [blame] | 710 | clib_interrupt_init ( |
| 711 | &nm_clone->pre_input_node_interrupts, |
| 712 | vec_len (nm_clone->nodes_by_type[VLIB_NODE_TYPE_PRE_INPUT])); |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 713 | vec_foreach (rt, nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT]) |
Damjan Marion | e9f929b | 2017-03-16 11:32:09 +0100 | [diff] [blame] | 714 | { |
| 715 | vlib_node_t *n = vlib_get_node (vm, rt->node_index); |
Damjan Marion | e9f929b | 2017-03-16 11:32:09 +0100 | [diff] [blame] | 716 | /* copy initial runtime_data from node */ |
Damjan Marion | b6f93a1 | 2017-03-16 17:46:41 +0100 | [diff] [blame] | 717 | if (n->runtime_data && n->runtime_data_bytes > 0) |
Damjan Marion | e9f929b | 2017-03-16 11:32:09 +0100 | [diff] [blame] | 718 | clib_memcpy (rt->runtime_data, n->runtime_data, |
Damjan Marion | b6f93a1 | 2017-03-16 17:46:41 +0100 | [diff] [blame] | 719 | clib_min (VLIB_NODE_RUNTIME_DATA_SIZE, |
| 720 | n->runtime_data_bytes)); |
Damjan Marion | e9f929b | 2017-03-16 11:32:09 +0100 | [diff] [blame] | 721 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 722 | |
Dave Barach | 53fe4a7 | 2019-01-26 09:50:26 -0500 | [diff] [blame] | 723 | nm_clone->nodes_by_type[VLIB_NODE_TYPE_PRE_INPUT] = |
| 724 | vec_dup_aligned (nm->nodes_by_type[VLIB_NODE_TYPE_PRE_INPUT], |
| 725 | CLIB_CACHE_LINE_BYTES); |
| 726 | vec_foreach (rt, |
| 727 | nm_clone->nodes_by_type[VLIB_NODE_TYPE_PRE_INPUT]) |
| 728 | { |
| 729 | vlib_node_t *n = vlib_get_node (vm, rt->node_index); |
Dave Barach | 53fe4a7 | 2019-01-26 09:50:26 -0500 | [diff] [blame] | 730 | /* copy initial runtime_data from node */ |
| 731 | if (n->runtime_data && n->runtime_data_bytes > 0) |
| 732 | clib_memcpy (rt->runtime_data, n->runtime_data, |
| 733 | clib_min (VLIB_NODE_RUNTIME_DATA_SIZE, |
| 734 | n->runtime_data_bytes)); |
| 735 | } |
| 736 | |
Damjan Marion | be3f4d5 | 2018-03-27 21:06:10 +0200 | [diff] [blame] | 737 | nm_clone->processes = vec_dup_aligned (nm->processes, |
| 738 | CLIB_CACHE_LINE_BYTES); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 739 | |
Dave Barach | 593eedf | 2019-03-10 09:44:51 -0400 | [diff] [blame] | 740 | /* Create per-thread frame freelist */ |
Damjan Marion | b32bd70 | 2021-12-23 17:05:02 +0100 | [diff] [blame] | 741 | nm_clone->frame_sizes = 0; |
Dave Barach | 687c902 | 2019-07-23 10:22:31 -0400 | [diff] [blame] | 742 | nm_clone->node_by_error = nm->node_by_error; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 743 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 744 | /* Packet trace buffers are guaranteed to be empty, nothing to do here */ |
Damjan Marion | bc20bdf | 2015-12-17 14:28:18 +0100 | [diff] [blame] | 745 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 746 | clib_mem_set_heap (oldheap); |
Damjan Marion | fd8deb4 | 2021-03-06 12:26:28 +0100 | [diff] [blame] | 747 | vec_add1_aligned (vgm->vlib_mains, vm_clone, |
| 748 | CLIB_CACHE_LINE_BYTES); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 749 | |
Ole Troan | 233e468 | 2019-05-16 15:01:34 +0200 | [diff] [blame] | 750 | /* Switch to the stats segment ... */ |
Damjan Marion | 66c8583 | 2022-03-14 13:04:38 +0100 | [diff] [blame] | 751 | vlib_stats_validate (stats_err_entry_index, worker_thread_index, |
| 752 | vec_len (fvm->error_main.counters) - 1); |
| 753 | c = vlib_stats_get_entry_data_pointer (stats_err_entry_index); |
| 754 | vm_clone->error_main.counters = c[worker_thread_index]; |
Ole Troan | 233e468 | 2019-05-16 15:01:34 +0200 | [diff] [blame] | 755 | |
Damjan Marion | fd8deb4 | 2021-03-06 12:26:28 +0100 | [diff] [blame] | 756 | vm_clone->error_main.counters_last_clear = vec_dup_aligned ( |
| 757 | vlib_get_first_main ()->error_main.counters_last_clear, |
| 758 | CLIB_CACHE_LINE_BYTES); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 759 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 760 | worker_thread_index++; |
| 761 | } |
| 762 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 763 | } |
| 764 | else |
| 765 | { |
| 766 | /* only have non-data-structure copy threads to create... */ |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 767 | for (i = 0; i < vec_len (tm->registrations); i++) |
| 768 | { |
| 769 | tr = tm->registrations[i]; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 770 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 771 | for (j = 0; j < tr->count; j++) |
| 772 | { |
| 773 | vec_add2 (vlib_worker_threads, w, 1); |
| 774 | if (tr->mheap_size) |
Dave Barach | 6a5adc3 | 2018-07-04 10:56:23 -0400 | [diff] [blame] | 775 | { |
Damjan Marion | 4537c30 | 2020-09-28 19:03:37 +0200 | [diff] [blame] | 776 | w->thread_mheap = clib_mem_create_heap (0, tr->mheap_size, |
| 777 | /* locked */ 0, |
| 778 | "%s%d heap", |
| 779 | tr->name, j); |
Dave Barach | 6a5adc3 | 2018-07-04 10:56:23 -0400 | [diff] [blame] | 780 | } |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 781 | else |
| 782 | w->thread_mheap = main_heap; |
Damjan Marion | 586afd7 | 2017-04-05 19:18:20 +0200 | [diff] [blame] | 783 | w->thread_stack = |
| 784 | vlib_thread_stack_init (w - vlib_worker_threads); |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 785 | w->thread_function = tr->function; |
| 786 | w->thread_function_arg = w; |
| 787 | w->instance_id = j; |
| 788 | w->elog_track.name = |
| 789 | (char *) format (0, "%s %d", tr->name, j + 1); |
| 790 | w->registration = tr; |
| 791 | vec_add1 (w->elog_track.name, 0); |
Damjan Marion | f553a2c | 2021-03-26 13:45:37 +0100 | [diff] [blame] | 792 | elog_track_register (vlib_get_elog_main (), &w->elog_track); |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 793 | } |
| 794 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 795 | } |
| 796 | |
| 797 | worker_thread_index = 1; |
| 798 | |
| 799 | for (i = 0; i < vec_len (tm->registrations); i++) |
| 800 | { |
Damjan Marion | 878c609 | 2017-01-04 13:19:27 +0100 | [diff] [blame] | 801 | clib_error_t *err; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 802 | int j; |
| 803 | |
| 804 | tr = tm->registrations[i]; |
| 805 | |
| 806 | if (tr->use_pthreads || tm->use_pthreads) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 807 | { |
| 808 | for (j = 0; j < tr->count; j++) |
| 809 | { |
hsandid | 832342e | 2023-12-20 15:41:54 +0100 | [diff] [blame] | 810 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 811 | w = vlib_worker_threads + worker_thread_index++; |
Damjan Marion | 878c609 | 2017-01-04 13:19:27 +0100 | [diff] [blame] | 812 | err = vlib_launch_thread_int (vlib_worker_thread_bootstrap_fn, |
| 813 | w, 0); |
| 814 | if (err) |
hsandid | 832342e | 2023-12-20 15:41:54 +0100 | [diff] [blame] | 815 | clib_unix_error ("%U, thread %s init on cpu %d failed", |
| 816 | format_clib_error, err, tr->name, 0); |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 817 | } |
| 818 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 819 | else |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 820 | { |
| 821 | uword c; |
Damjan Marion | f0ca1e8 | 2020-12-13 23:26:56 +0100 | [diff] [blame] | 822 | clib_bitmap_foreach (c, tr->coremask) { |
Damjan Marion | 878c609 | 2017-01-04 13:19:27 +0100 | [diff] [blame] | 823 | w = vlib_worker_threads + worker_thread_index++; |
| 824 | err = vlib_launch_thread_int (vlib_worker_thread_bootstrap_fn, |
| 825 | w, c); |
| 826 | if (err) |
hsandid | 832342e | 2023-12-20 15:41:54 +0100 | [diff] [blame] | 827 | clib_unix_error ("%U, thread %s init on cpu %d failed", |
| 828 | format_clib_error, err, tr->name, c); |
| 829 | } |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 830 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 831 | } |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 832 | vlib_worker_thread_barrier_sync (vm); |
Damjan Marion | 321bd10 | 2022-06-01 00:45:18 +0200 | [diff] [blame] | 833 | { |
| 834 | clib_error_t *err; |
| 835 | err = vlib_call_init_exit_functions ( |
| 836 | vm, &vgm->num_workers_change_function_registrations, 1 /* call_once */, |
| 837 | 1 /* is_global */); |
| 838 | if (err) |
| 839 | clib_error_report (err); |
| 840 | } |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 841 | vlib_worker_thread_barrier_release (vm); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 842 | return 0; |
| 843 | } |
| 844 | |
| 845 | VLIB_MAIN_LOOP_ENTER_FUNCTION (start_workers); |
| 846 | |
Colin Tregenza Dancer | eb1ac17 | 2017-09-06 20:23:24 +0100 | [diff] [blame] | 847 | |
Colin Tregenza Dancer | 2159618 | 2017-09-04 15:27:49 +0100 | [diff] [blame] | 848 | static inline void |
| 849 | worker_thread_node_runtime_update_internal (void) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 850 | { |
| 851 | int i, j; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 852 | vlib_main_t *vm; |
| 853 | vlib_node_main_t *nm, *nm_clone; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 854 | vlib_main_t *vm_clone; |
Colin Tregenza Dancer | 2159618 | 2017-09-04 15:27:49 +0100 | [diff] [blame] | 855 | vlib_node_runtime_t *rt; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 856 | |
Damjan Marion | 586afd7 | 2017-04-05 19:18:20 +0200 | [diff] [blame] | 857 | ASSERT (vlib_get_thread_index () == 0); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 858 | |
Damjan Marion | 6ffb7c6 | 2021-03-26 13:06:13 +0100 | [diff] [blame] | 859 | vm = vlib_get_first_main (); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 860 | nm = &vm->node_main; |
| 861 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 862 | ASSERT (*vlib_worker_threads->wait_at_barrier == 1); |
| 863 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 864 | /* |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 865 | * Scrape all runtime stats, so we don't lose node runtime(s) with |
| 866 | * pending counts, or throw away worker / io thread counts. |
| 867 | */ |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 868 | for (j = 0; j < vec_len (nm->nodes); j++) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 869 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 870 | vlib_node_t *n; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 871 | n = nm->nodes[j]; |
| 872 | vlib_node_sync_stats (vm, n); |
| 873 | } |
| 874 | |
Damjan Marion | 6ffb7c6 | 2021-03-26 13:06:13 +0100 | [diff] [blame] | 875 | for (i = 1; i < vlib_get_n_threads (); i++) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 876 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 877 | vlib_node_t *n; |
| 878 | |
Damjan Marion | 6ffb7c6 | 2021-03-26 13:06:13 +0100 | [diff] [blame] | 879 | vm_clone = vlib_get_main_by_index (i); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 880 | nm_clone = &vm_clone->node_main; |
| 881 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 882 | for (j = 0; j < vec_len (nm_clone->nodes); j++) |
| 883 | { |
| 884 | n = nm_clone->nodes[j]; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 885 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 886 | rt = vlib_node_get_runtime (vm_clone, n->index); |
| 887 | vlib_node_runtime_sync_stats (vm_clone, rt, 0, 0, 0); |
| 888 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 889 | } |
| 890 | |
Colin Tregenza Dancer | 2159618 | 2017-09-04 15:27:49 +0100 | [diff] [blame] | 891 | /* Per-worker clone rebuilds are now done on each thread */ |
| 892 | } |
| 893 | |
| 894 | |
| 895 | void |
| 896 | vlib_worker_thread_node_refork (void) |
| 897 | { |
| 898 | vlib_main_t *vm, *vm_clone; |
| 899 | vlib_node_main_t *nm, *nm_clone; |
| 900 | vlib_node_t **old_nodes_clone; |
| 901 | vlib_node_runtime_t *rt, *old_rt; |
Damjan Marion | 66c8583 | 2022-03-14 13:04:38 +0100 | [diff] [blame] | 902 | u64 **c; |
Colin Tregenza Dancer | 2159618 | 2017-09-04 15:27:49 +0100 | [diff] [blame] | 903 | |
| 904 | vlib_node_t *new_n_clone; |
| 905 | |
| 906 | int j; |
| 907 | |
Damjan Marion | 6ffb7c6 | 2021-03-26 13:06:13 +0100 | [diff] [blame] | 908 | vm = vlib_get_first_main (); |
Colin Tregenza Dancer | 2159618 | 2017-09-04 15:27:49 +0100 | [diff] [blame] | 909 | nm = &vm->node_main; |
| 910 | vm_clone = vlib_get_main (); |
| 911 | nm_clone = &vm_clone->node_main; |
| 912 | |
| 913 | /* Re-clone error heap */ |
Colin Tregenza Dancer | 2159618 | 2017-09-04 15:27:49 +0100 | [diff] [blame] | 914 | u64 *old_counters_all_clear = vm_clone->error_main.counters_last_clear; |
| 915 | |
Dave Barach | 178cf49 | 2018-11-13 16:34:13 -0500 | [diff] [blame] | 916 | clib_memcpy_fast (&vm_clone->error_main, &vm->error_main, |
| 917 | sizeof (vm->error_main)); |
Colin Tregenza Dancer | 2159618 | 2017-09-04 15:27:49 +0100 | [diff] [blame] | 918 | j = vec_len (vm->error_main.counters) - 1; |
Ole Troan | 233e468 | 2019-05-16 15:01:34 +0200 | [diff] [blame] | 919 | |
Damjan Marion | 66c8583 | 2022-03-14 13:04:38 +0100 | [diff] [blame] | 920 | c = vlib_stats_get_entry_data_pointer (vm->error_main.stats_err_entry_index); |
| 921 | vm_clone->error_main.counters = c[vm_clone->thread_index]; |
Ole Troan | 233e468 | 2019-05-16 15:01:34 +0200 | [diff] [blame] | 922 | |
| 923 | vec_validate_aligned (old_counters_all_clear, j, CLIB_CACHE_LINE_BYTES); |
Colin Tregenza Dancer | 2159618 | 2017-09-04 15:27:49 +0100 | [diff] [blame] | 924 | vm_clone->error_main.counters_last_clear = old_counters_all_clear; |
| 925 | |
Dmitry Valter | 9f5b369 | 2022-09-05 15:30:18 +0000 | [diff] [blame] | 926 | for (j = 0; j < vec_len (nm_clone->next_frames); j++) |
| 927 | { |
| 928 | vlib_next_frame_t *nf = &nm_clone->next_frames[j]; |
| 929 | if ((nf->flags & VLIB_FRAME_IS_ALLOCATED) && nf->frame != NULL) |
| 930 | { |
| 931 | vlib_frame_t *f = nf->frame; |
| 932 | nf->frame = NULL; |
| 933 | vlib_frame_free (vm_clone, f); |
| 934 | } |
| 935 | } |
| 936 | |
Colin Tregenza Dancer | 2159618 | 2017-09-04 15:27:49 +0100 | [diff] [blame] | 937 | vec_free (nm_clone->next_frames); |
Damjan Marion | be3f4d5 | 2018-03-27 21:06:10 +0200 | [diff] [blame] | 938 | nm_clone->next_frames = vec_dup_aligned (nm->next_frames, |
| 939 | CLIB_CACHE_LINE_BYTES); |
Colin Tregenza Dancer | 2159618 | 2017-09-04 15:27:49 +0100 | [diff] [blame] | 940 | |
| 941 | for (j = 0; j < vec_len (nm_clone->next_frames); j++) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 942 | { |
Colin Tregenza Dancer | 2159618 | 2017-09-04 15:27:49 +0100 | [diff] [blame] | 943 | vlib_next_frame_t *nf = &nm_clone->next_frames[j]; |
| 944 | u32 save_node_runtime_index; |
| 945 | u32 save_flags; |
Damjan Marion | bc20bdf | 2015-12-17 14:28:18 +0100 | [diff] [blame] | 946 | |
Colin Tregenza Dancer | 2159618 | 2017-09-04 15:27:49 +0100 | [diff] [blame] | 947 | save_node_runtime_index = nf->node_runtime_index; |
| 948 | save_flags = nf->flags & VLIB_FRAME_NO_FREE_AFTER_DISPATCH; |
| 949 | vlib_next_frame_init (nf); |
| 950 | nf->node_runtime_index = save_node_runtime_index; |
| 951 | nf->flags = save_flags; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 952 | } |
Colin Tregenza Dancer | 2159618 | 2017-09-04 15:27:49 +0100 | [diff] [blame] | 953 | |
| 954 | old_nodes_clone = nm_clone->nodes; |
| 955 | nm_clone->nodes = 0; |
| 956 | |
| 957 | /* re-fork nodes */ |
| 958 | |
| 959 | /* Allocate all nodes in single block for speed */ |
| 960 | new_n_clone = |
| 961 | clib_mem_alloc_no_fail (vec_len (nm->nodes) * sizeof (*new_n_clone)); |
| 962 | for (j = 0; j < vec_len (nm->nodes); j++) |
| 963 | { |
Benoît Ganne | 5517bd3 | 2019-08-30 16:20:12 +0200 | [diff] [blame] | 964 | vlib_node_t *new_n = nm->nodes[j]; |
Colin Tregenza Dancer | 2159618 | 2017-09-04 15:27:49 +0100 | [diff] [blame] | 965 | |
Dave Barach | 178cf49 | 2018-11-13 16:34:13 -0500 | [diff] [blame] | 966 | clib_memcpy_fast (new_n_clone, new_n, sizeof (*new_n)); |
Colin Tregenza Dancer | 2159618 | 2017-09-04 15:27:49 +0100 | [diff] [blame] | 967 | /* none of the copied nodes have enqueue rights given out */ |
| 968 | new_n_clone->owner_node_index = VLIB_INVALID_NODE_INDEX; |
| 969 | |
| 970 | if (j >= vec_len (old_nodes_clone)) |
| 971 | { |
| 972 | /* new node, set to zero */ |
Dave Barach | b7b9299 | 2018-10-17 10:38:51 -0400 | [diff] [blame] | 973 | clib_memset (&new_n_clone->stats_total, 0, |
| 974 | sizeof (new_n_clone->stats_total)); |
| 975 | clib_memset (&new_n_clone->stats_last_clear, 0, |
| 976 | sizeof (new_n_clone->stats_last_clear)); |
Colin Tregenza Dancer | 2159618 | 2017-09-04 15:27:49 +0100 | [diff] [blame] | 977 | } |
| 978 | else |
| 979 | { |
Benoît Ganne | 5517bd3 | 2019-08-30 16:20:12 +0200 | [diff] [blame] | 980 | vlib_node_t *old_n_clone = old_nodes_clone[j]; |
Colin Tregenza Dancer | 2159618 | 2017-09-04 15:27:49 +0100 | [diff] [blame] | 981 | /* Copy stats if the old data is valid */ |
Dave Barach | 178cf49 | 2018-11-13 16:34:13 -0500 | [diff] [blame] | 982 | clib_memcpy_fast (&new_n_clone->stats_total, |
| 983 | &old_n_clone->stats_total, |
| 984 | sizeof (new_n_clone->stats_total)); |
| 985 | clib_memcpy_fast (&new_n_clone->stats_last_clear, |
| 986 | &old_n_clone->stats_last_clear, |
| 987 | sizeof (new_n_clone->stats_last_clear)); |
Colin Tregenza Dancer | 2159618 | 2017-09-04 15:27:49 +0100 | [diff] [blame] | 988 | |
| 989 | /* keep previous node state */ |
| 990 | new_n_clone->state = old_n_clone->state; |
Maxime Peim | dde163d | 2021-04-15 13:52:12 +0200 | [diff] [blame] | 991 | new_n_clone->flags = old_n_clone->flags; |
Colin Tregenza Dancer | 2159618 | 2017-09-04 15:27:49 +0100 | [diff] [blame] | 992 | } |
| 993 | vec_add1 (nm_clone->nodes, new_n_clone); |
| 994 | new_n_clone++; |
| 995 | } |
| 996 | /* Free the old node clones */ |
| 997 | clib_mem_free (old_nodes_clone[0]); |
| 998 | |
| 999 | vec_free (old_nodes_clone); |
| 1000 | |
| 1001 | |
| 1002 | /* re-clone internal nodes */ |
| 1003 | old_rt = nm_clone->nodes_by_type[VLIB_NODE_TYPE_INTERNAL]; |
| 1004 | nm_clone->nodes_by_type[VLIB_NODE_TYPE_INTERNAL] = |
Damjan Marion | be3f4d5 | 2018-03-27 21:06:10 +0200 | [diff] [blame] | 1005 | vec_dup_aligned (nm->nodes_by_type[VLIB_NODE_TYPE_INTERNAL], |
| 1006 | CLIB_CACHE_LINE_BYTES); |
Colin Tregenza Dancer | 2159618 | 2017-09-04 15:27:49 +0100 | [diff] [blame] | 1007 | |
| 1008 | vec_foreach (rt, nm_clone->nodes_by_type[VLIB_NODE_TYPE_INTERNAL]) |
| 1009 | { |
| 1010 | vlib_node_t *n = vlib_get_node (vm, rt->node_index); |
Colin Tregenza Dancer | 2159618 | 2017-09-04 15:27:49 +0100 | [diff] [blame] | 1011 | /* copy runtime_data, will be overwritten later for existing rt */ |
| 1012 | if (n->runtime_data && n->runtime_data_bytes > 0) |
Dave Barach | 178cf49 | 2018-11-13 16:34:13 -0500 | [diff] [blame] | 1013 | clib_memcpy_fast (rt->runtime_data, n->runtime_data, |
| 1014 | clib_min (VLIB_NODE_RUNTIME_DATA_SIZE, |
| 1015 | n->runtime_data_bytes)); |
Colin Tregenza Dancer | 2159618 | 2017-09-04 15:27:49 +0100 | [diff] [blame] | 1016 | } |
| 1017 | |
| 1018 | for (j = 0; j < vec_len (old_rt); j++) |
| 1019 | { |
| 1020 | rt = vlib_node_get_runtime (vm_clone, old_rt[j].node_index); |
| 1021 | rt->state = old_rt[j].state; |
Maxime Peim | dde163d | 2021-04-15 13:52:12 +0200 | [diff] [blame] | 1022 | rt->flags = old_rt[j].flags; |
Dave Barach | 178cf49 | 2018-11-13 16:34:13 -0500 | [diff] [blame] | 1023 | clib_memcpy_fast (rt->runtime_data, old_rt[j].runtime_data, |
| 1024 | VLIB_NODE_RUNTIME_DATA_SIZE); |
Colin Tregenza Dancer | 2159618 | 2017-09-04 15:27:49 +0100 | [diff] [blame] | 1025 | } |
| 1026 | |
| 1027 | vec_free (old_rt); |
| 1028 | |
| 1029 | /* re-clone input nodes */ |
| 1030 | old_rt = nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT]; |
| 1031 | nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT] = |
Damjan Marion | be3f4d5 | 2018-03-27 21:06:10 +0200 | [diff] [blame] | 1032 | vec_dup_aligned (nm->nodes_by_type[VLIB_NODE_TYPE_INPUT], |
| 1033 | CLIB_CACHE_LINE_BYTES); |
Damjan Marion | 9410053 | 2020-11-06 23:25:57 +0100 | [diff] [blame] | 1034 | clib_interrupt_resize ( |
Damjan Marion | cc8249c | 2023-07-23 14:24:22 +0200 | [diff] [blame] | 1035 | &nm_clone->input_node_interrupts, |
Damjan Marion | 9410053 | 2020-11-06 23:25:57 +0100 | [diff] [blame] | 1036 | vec_len (nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT])); |
Damjan Marion | cc8249c | 2023-07-23 14:24:22 +0200 | [diff] [blame] | 1037 | clib_interrupt_resize ( |
| 1038 | &nm_clone->pre_input_node_interrupts, |
| 1039 | vec_len (nm_clone->nodes_by_type[VLIB_NODE_TYPE_PRE_INPUT])); |
Colin Tregenza Dancer | 2159618 | 2017-09-04 15:27:49 +0100 | [diff] [blame] | 1040 | |
| 1041 | vec_foreach (rt, nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT]) |
| 1042 | { |
| 1043 | vlib_node_t *n = vlib_get_node (vm, rt->node_index); |
Colin Tregenza Dancer | 2159618 | 2017-09-04 15:27:49 +0100 | [diff] [blame] | 1044 | /* copy runtime_data, will be overwritten later for existing rt */ |
| 1045 | if (n->runtime_data && n->runtime_data_bytes > 0) |
Dave Barach | 178cf49 | 2018-11-13 16:34:13 -0500 | [diff] [blame] | 1046 | clib_memcpy_fast (rt->runtime_data, n->runtime_data, |
| 1047 | clib_min (VLIB_NODE_RUNTIME_DATA_SIZE, |
| 1048 | n->runtime_data_bytes)); |
Colin Tregenza Dancer | 2159618 | 2017-09-04 15:27:49 +0100 | [diff] [blame] | 1049 | } |
| 1050 | |
| 1051 | for (j = 0; j < vec_len (old_rt); j++) |
| 1052 | { |
| 1053 | rt = vlib_node_get_runtime (vm_clone, old_rt[j].node_index); |
| 1054 | rt->state = old_rt[j].state; |
Maxime Peim | dde163d | 2021-04-15 13:52:12 +0200 | [diff] [blame] | 1055 | rt->flags = old_rt[j].flags; |
Dave Barach | 178cf49 | 2018-11-13 16:34:13 -0500 | [diff] [blame] | 1056 | clib_memcpy_fast (rt->runtime_data, old_rt[j].runtime_data, |
| 1057 | VLIB_NODE_RUNTIME_DATA_SIZE); |
Colin Tregenza Dancer | 2159618 | 2017-09-04 15:27:49 +0100 | [diff] [blame] | 1058 | } |
| 1059 | |
| 1060 | vec_free (old_rt); |
| 1061 | |
Dave Barach | 53fe4a7 | 2019-01-26 09:50:26 -0500 | [diff] [blame] | 1062 | /* re-clone pre-input nodes */ |
| 1063 | old_rt = nm_clone->nodes_by_type[VLIB_NODE_TYPE_PRE_INPUT]; |
| 1064 | nm_clone->nodes_by_type[VLIB_NODE_TYPE_PRE_INPUT] = |
| 1065 | vec_dup_aligned (nm->nodes_by_type[VLIB_NODE_TYPE_PRE_INPUT], |
| 1066 | CLIB_CACHE_LINE_BYTES); |
| 1067 | |
| 1068 | vec_foreach (rt, nm_clone->nodes_by_type[VLIB_NODE_TYPE_PRE_INPUT]) |
| 1069 | { |
| 1070 | vlib_node_t *n = vlib_get_node (vm, rt->node_index); |
Dave Barach | 53fe4a7 | 2019-01-26 09:50:26 -0500 | [diff] [blame] | 1071 | /* copy runtime_data, will be overwritten later for existing rt */ |
| 1072 | if (n->runtime_data && n->runtime_data_bytes > 0) |
| 1073 | clib_memcpy_fast (rt->runtime_data, n->runtime_data, |
| 1074 | clib_min (VLIB_NODE_RUNTIME_DATA_SIZE, |
| 1075 | n->runtime_data_bytes)); |
| 1076 | } |
| 1077 | |
| 1078 | for (j = 0; j < vec_len (old_rt); j++) |
| 1079 | { |
| 1080 | rt = vlib_node_get_runtime (vm_clone, old_rt[j].node_index); |
| 1081 | rt->state = old_rt[j].state; |
Maxime Peim | dde163d | 2021-04-15 13:52:12 +0200 | [diff] [blame] | 1082 | rt->flags = old_rt[j].flags; |
Dave Barach | 53fe4a7 | 2019-01-26 09:50:26 -0500 | [diff] [blame] | 1083 | clib_memcpy_fast (rt->runtime_data, old_rt[j].runtime_data, |
| 1084 | VLIB_NODE_RUNTIME_DATA_SIZE); |
| 1085 | } |
| 1086 | |
| 1087 | vec_free (old_rt); |
| 1088 | |
Vladislav Grishenko | 2973350 | 2021-09-25 21:00:59 +0500 | [diff] [blame] | 1089 | vec_free (nm_clone->processes); |
Damjan Marion | be3f4d5 | 2018-03-27 21:06:10 +0200 | [diff] [blame] | 1090 | nm_clone->processes = vec_dup_aligned (nm->processes, |
| 1091 | CLIB_CACHE_LINE_BYTES); |
Dave Barach | 687c902 | 2019-07-23 10:22:31 -0400 | [diff] [blame] | 1092 | nm_clone->node_by_error = nm->node_by_error; |
Colin Tregenza Dancer | 2159618 | 2017-09-04 15:27:49 +0100 | [diff] [blame] | 1093 | } |
| 1094 | |
Colin Tregenza Dancer | 2159618 | 2017-09-04 15:27:49 +0100 | [diff] [blame] | 1095 | void |
| 1096 | vlib_worker_thread_node_runtime_update (void) |
| 1097 | { |
| 1098 | /* |
| 1099 | * Make a note that we need to do a node runtime update |
| 1100 | * prior to releasing the barrier. |
| 1101 | */ |
| 1102 | vlib_global_main.need_vlib_worker_thread_node_runtime_update = 1; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1103 | } |
| 1104 | |
Pavel Kotucek | 1e76583 | 2016-09-23 08:54:14 +0200 | [diff] [blame] | 1105 | u32 |
| 1106 | unformat_sched_policy (unformat_input_t * input, va_list * args) |
| 1107 | { |
| 1108 | u32 *r = va_arg (*args, u32 *); |
| 1109 | |
| 1110 | if (0); |
| 1111 | #define _(v,f,s) else if (unformat (input, s)) *r = SCHED_POLICY_##f; |
| 1112 | foreach_sched_policy |
| 1113 | #undef _ |
| 1114 | else |
| 1115 | return 0; |
| 1116 | return 1; |
| 1117 | } |
| 1118 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1119 | static clib_error_t * |
| 1120 | cpu_config (vlib_main_t * vm, unformat_input_t * input) |
| 1121 | { |
| 1122 | vlib_thread_registration_t *tr; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1123 | uword *p; |
| 1124 | vlib_thread_main_t *tm = &vlib_thread_main; |
| 1125 | u8 *name; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1126 | uword *bitmap; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1127 | u32 count; |
| 1128 | |
| 1129 | tm->thread_registrations_by_name = hash_create_string (0, sizeof (uword)); |
Pavel Kotucek | 1e76583 | 2016-09-23 08:54:14 +0200 | [diff] [blame] | 1130 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1131 | tm->n_thread_stacks = 1; /* account for main thread */ |
Pavel Kotucek | 1e76583 | 2016-09-23 08:54:14 +0200 | [diff] [blame] | 1132 | tm->sched_policy = ~0; |
| 1133 | tm->sched_priority = ~0; |
Damjan Marion | 858151f | 2018-07-11 10:51:00 +0200 | [diff] [blame] | 1134 | tm->main_lcore = ~0; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1135 | |
| 1136 | tr = tm->next; |
| 1137 | |
| 1138 | while (tr) |
| 1139 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1140 | hash_set_mem (tm->thread_registrations_by_name, tr->name, (uword) tr); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1141 | tr = tr->next; |
| 1142 | } |
| 1143 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1144 | while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1145 | { |
Damjan Marion | bf74147 | 2016-06-13 22:49:44 +0200 | [diff] [blame] | 1146 | if (unformat (input, "use-pthreads")) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1147 | tm->use_pthreads = 1; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1148 | else if (unformat (input, "thread-prefix %v", &tm->thread_prefix)) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1149 | ; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1150 | else if (unformat (input, "main-core %u", &tm->main_lcore)) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1151 | ; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1152 | else if (unformat (input, "skip-cores %u", &tm->skip_cores)) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1153 | ; |
Dave Barach | a690fdb | 2020-01-21 12:34:55 -0500 | [diff] [blame] | 1154 | else if (unformat (input, "numa-heap-size %U", |
| 1155 | unformat_memory_size, &tm->numa_heap_size)) |
| 1156 | ; |
Yi He | e4a9eb7 | 2018-07-17 14:18:41 +0800 | [diff] [blame] | 1157 | else if (unformat (input, "coremask-%s %U", &name, |
| 1158 | unformat_bitmap_mask, &bitmap) || |
| 1159 | unformat (input, "corelist-%s %U", &name, |
| 1160 | unformat_bitmap_list, &bitmap)) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1161 | { |
| 1162 | p = hash_get_mem (tm->thread_registrations_by_name, name); |
| 1163 | if (p == 0) |
| 1164 | return clib_error_return (0, "no such thread type '%s'", name); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1165 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1166 | tr = (vlib_thread_registration_t *) p[0]; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1167 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1168 | if (tr->use_pthreads) |
| 1169 | return clib_error_return (0, |
| 1170 | "corelist cannot be set for '%s' threads", |
| 1171 | name); |
Vladimir Isaev | 18a4a37 | 2020-03-17 12:30:11 +0300 | [diff] [blame] | 1172 | if (tr->count) |
| 1173 | return clib_error_return |
| 1174 | (0, "core placement of '%s' threads is already configured", |
| 1175 | name); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1176 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1177 | tr->coremask = bitmap; |
| 1178 | tr->count = clib_bitmap_count_set_bits (tr->coremask); |
| 1179 | } |
Pavel Kotucek | 1e76583 | 2016-09-23 08:54:14 +0200 | [diff] [blame] | 1180 | else |
| 1181 | if (unformat |
| 1182 | (input, "scheduler-policy %U", unformat_sched_policy, |
| 1183 | &tm->sched_policy)) |
| 1184 | ; |
Pavel Kotucek | c08a1ed | 2016-09-23 08:54:14 +0200 | [diff] [blame] | 1185 | else if (unformat (input, "scheduler-priority %u", &tm->sched_priority)) |
Pavel Kotucek | 1e76583 | 2016-09-23 08:54:14 +0200 | [diff] [blame] | 1186 | ; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1187 | else if (unformat (input, "%s %u", &name, &count)) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1188 | { |
| 1189 | p = hash_get_mem (tm->thread_registrations_by_name, name); |
| 1190 | if (p == 0) |
Pavel Kotucek | 1e76583 | 2016-09-23 08:54:14 +0200 | [diff] [blame] | 1191 | return clib_error_return (0, "no such thread type 3 '%s'", name); |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1192 | |
| 1193 | tr = (vlib_thread_registration_t *) p[0]; |
Vladimir Isaev | 18a4a37 | 2020-03-17 12:30:11 +0300 | [diff] [blame] | 1194 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1195 | if (tr->fixed_count) |
| 1196 | return clib_error_return |
Vladimir Isaev | 18a4a37 | 2020-03-17 12:30:11 +0300 | [diff] [blame] | 1197 | (0, "number of '%s' threads not configurable", name); |
| 1198 | if (tr->count) |
| 1199 | return clib_error_return |
| 1200 | (0, "number of '%s' threads is already configured", name); |
| 1201 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1202 | tr->count = count; |
| 1203 | } |
| 1204 | else |
| 1205 | break; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1206 | } |
| 1207 | |
Pavel Kotucek | c08a1ed | 2016-09-23 08:54:14 +0200 | [diff] [blame] | 1208 | if (tm->sched_priority != ~0) |
Pavel Kotucek | 1e76583 | 2016-09-23 08:54:14 +0200 | [diff] [blame] | 1209 | { |
Pavel Kotucek | c08a1ed | 2016-09-23 08:54:14 +0200 | [diff] [blame] | 1210 | if (tm->sched_policy == SCHED_FIFO || tm->sched_policy == SCHED_RR) |
Pavel Kotucek | 1e76583 | 2016-09-23 08:54:14 +0200 | [diff] [blame] | 1211 | { |
| 1212 | u32 prio_max = sched_get_priority_max (tm->sched_policy); |
| 1213 | u32 prio_min = sched_get_priority_min (tm->sched_policy); |
| 1214 | if (tm->sched_priority > prio_max) |
| 1215 | tm->sched_priority = prio_max; |
| 1216 | if (tm->sched_priority < prio_min) |
| 1217 | tm->sched_priority = prio_min; |
| 1218 | } |
| 1219 | else |
Pavel Kotucek | c08a1ed | 2016-09-23 08:54:14 +0200 | [diff] [blame] | 1220 | { |
| 1221 | return clib_error_return |
| 1222 | (0, |
| 1223 | "scheduling priority (%d) is not allowed for `normal` scheduling policy", |
| 1224 | tm->sched_priority); |
| 1225 | } |
Pavel Kotucek | 1e76583 | 2016-09-23 08:54:14 +0200 | [diff] [blame] | 1226 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1227 | tr = tm->next; |
| 1228 | |
| 1229 | if (!tm->thread_prefix) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1230 | tm->thread_prefix = format (0, "vpp"); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1231 | |
| 1232 | while (tr) |
| 1233 | { |
| 1234 | tm->n_thread_stacks += tr->count; |
| 1235 | tm->n_pthreads += tr->count * tr->use_pthreads; |
Damjan Marion | 878c609 | 2017-01-04 13:19:27 +0100 | [diff] [blame] | 1236 | tm->n_threads += tr->count * (tr->use_pthreads == 0); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1237 | tr = tr->next; |
| 1238 | } |
| 1239 | |
| 1240 | return 0; |
| 1241 | } |
| 1242 | |
| 1243 | VLIB_EARLY_CONFIG_FUNCTION (cpu_config, "cpu"); |
| 1244 | |
Colin Tregenza Dancer | eb1ac17 | 2017-09-06 20:23:24 +0100 | [diff] [blame] | 1245 | /* |
| 1246 | * Enforce minimum open time to minimize packet loss due to Rx overflow, |
| 1247 | * based on a test based heuristic that barrier should be open for at least |
| 1248 | * 3 time as long as it is closed (with an upper bound of 1ms because by that |
| 1249 | * point it is probably too late to make a difference) |
| 1250 | */ |
| 1251 | |
| 1252 | #ifndef BARRIER_MINIMUM_OPEN_LIMIT |
| 1253 | #define BARRIER_MINIMUM_OPEN_LIMIT 0.001 |
| 1254 | #endif |
| 1255 | |
| 1256 | #ifndef BARRIER_MINIMUM_OPEN_FACTOR |
| 1257 | #define BARRIER_MINIMUM_OPEN_FACTOR 3 |
| 1258 | #endif |
| 1259 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1260 | void |
Dave Barach | c602b38 | 2019-06-03 19:48:22 -0400 | [diff] [blame] | 1261 | vlib_worker_thread_initial_barrier_sync_and_release (vlib_main_t * vm) |
| 1262 | { |
| 1263 | f64 deadline; |
| 1264 | f64 now = vlib_time_now (vm); |
Damjan Marion | 6ffb7c6 | 2021-03-26 13:06:13 +0100 | [diff] [blame] | 1265 | u32 count = vlib_get_n_threads () - 1; |
Dave Barach | c602b38 | 2019-06-03 19:48:22 -0400 | [diff] [blame] | 1266 | |
| 1267 | /* No worker threads? */ |
| 1268 | if (count == 0) |
| 1269 | return; |
| 1270 | |
| 1271 | deadline = now + BARRIER_SYNC_TIMEOUT; |
| 1272 | *vlib_worker_threads->wait_at_barrier = 1; |
| 1273 | while (*vlib_worker_threads->workers_at_barrier != count) |
| 1274 | { |
| 1275 | if ((now = vlib_time_now (vm)) > deadline) |
| 1276 | { |
| 1277 | fformat (stderr, "%s: worker thread deadlock\n", __FUNCTION__); |
| 1278 | os_panic (); |
| 1279 | } |
| 1280 | CLIB_PAUSE (); |
| 1281 | } |
| 1282 | *vlib_worker_threads->wait_at_barrier = 0; |
| 1283 | } |
| 1284 | |
Neale Ranns | 42845dd | 2020-05-26 13:12:17 +0000 | [diff] [blame] | 1285 | /** |
| 1286 | * Return true if the wroker thread barrier is held |
| 1287 | */ |
| 1288 | u8 |
| 1289 | vlib_worker_thread_barrier_held (void) |
| 1290 | { |
Damjan Marion | 6ffb7c6 | 2021-03-26 13:06:13 +0100 | [diff] [blame] | 1291 | if (vlib_get_n_threads () < 2) |
Neale Ranns | 42845dd | 2020-05-26 13:12:17 +0000 | [diff] [blame] | 1292 | return (1); |
| 1293 | |
| 1294 | return (*vlib_worker_threads->wait_at_barrier == 1); |
| 1295 | } |
| 1296 | |
Dave Barach | c602b38 | 2019-06-03 19:48:22 -0400 | [diff] [blame] | 1297 | void |
Damjan Marion | 8343ee5 | 2019-02-26 17:15:48 +0100 | [diff] [blame] | 1298 | vlib_worker_thread_barrier_sync_int (vlib_main_t * vm, const char *func_name) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1299 | { |
| 1300 | f64 deadline; |
Colin Tregenza Dancer | eb1ac17 | 2017-09-06 20:23:24 +0100 | [diff] [blame] | 1301 | f64 now; |
| 1302 | f64 t_entry; |
| 1303 | f64 t_open; |
| 1304 | f64 t_closed; |
Dave Barach | 9ae190e | 2019-04-23 10:07:24 -0400 | [diff] [blame] | 1305 | f64 max_vector_rate; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1306 | u32 count; |
Dave Barach | 9ae190e | 2019-04-23 10:07:24 -0400 | [diff] [blame] | 1307 | int i; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1308 | |
Damjan Marion | 6ffb7c6 | 2021-03-26 13:06:13 +0100 | [diff] [blame] | 1309 | if (vlib_get_n_threads () < 2) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1310 | return; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1311 | |
Colin Tregenza Dancer | 2159618 | 2017-09-04 15:27:49 +0100 | [diff] [blame] | 1312 | ASSERT (vlib_get_thread_index () == 0); |
| 1313 | |
Damjan Marion | 8343ee5 | 2019-02-26 17:15:48 +0100 | [diff] [blame] | 1314 | vlib_worker_threads[0].barrier_caller = func_name; |
Damjan Marion | 6ffb7c6 | 2021-03-26 13:06:13 +0100 | [diff] [blame] | 1315 | count = vlib_get_n_threads () - 1; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1316 | |
Colin Tregenza Dancer | eb1ac17 | 2017-09-06 20:23:24 +0100 | [diff] [blame] | 1317 | /* Record entry relative to last close */ |
| 1318 | now = vlib_time_now (vm); |
| 1319 | t_entry = now - vm->barrier_epoch; |
| 1320 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1321 | /* Tolerate recursive calls */ |
| 1322 | if (++vlib_worker_threads[0].recursion_level > 1) |
Colin Tregenza Dancer | eb1ac17 | 2017-09-06 20:23:24 +0100 | [diff] [blame] | 1323 | { |
| 1324 | barrier_trace_sync_rec (t_entry); |
| 1325 | return; |
| 1326 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1327 | |
Tom Seidenberg | 6c81f5a | 2020-07-10 15:49:03 +0000 | [diff] [blame] | 1328 | if (PREDICT_FALSE (vec_len (vm->barrier_perf_callbacks) != 0)) |
| 1329 | clib_call_callbacks (vm->barrier_perf_callbacks, vm, |
| 1330 | vm->clib_time.last_cpu_time, 0 /* enter */ ); |
| 1331 | |
Dave Barach | 9ae190e | 2019-04-23 10:07:24 -0400 | [diff] [blame] | 1332 | /* |
| 1333 | * Need data to decide if we're working hard enough to honor |
| 1334 | * the barrier hold-down timer. |
| 1335 | */ |
| 1336 | max_vector_rate = 0.0; |
Damjan Marion | 6ffb7c6 | 2021-03-26 13:06:13 +0100 | [diff] [blame] | 1337 | for (i = 1; i < vlib_get_n_threads (); i++) |
| 1338 | { |
| 1339 | vlib_main_t *ovm = vlib_get_main_by_index (i); |
| 1340 | max_vector_rate = clib_max (max_vector_rate, |
| 1341 | (f64) vlib_last_vectors_per_main_loop (ovm)); |
| 1342 | } |
Dave Barach | 9ae190e | 2019-04-23 10:07:24 -0400 | [diff] [blame] | 1343 | |
Bud Grise | 42f2006 | 2016-03-16 13:09:46 -0400 | [diff] [blame] | 1344 | vlib_worker_threads[0].barrier_sync_count++; |
| 1345 | |
Colin Tregenza Dancer | eb1ac17 | 2017-09-06 20:23:24 +0100 | [diff] [blame] | 1346 | /* Enforce minimum barrier open time to minimize packet loss */ |
| 1347 | ASSERT (vm->barrier_no_close_before <= (now + BARRIER_MINIMUM_OPEN_LIMIT)); |
Colin Tregenza Dancer | eb1ac17 | 2017-09-06 20:23:24 +0100 | [diff] [blame] | 1348 | |
Dave Barach | 9ae190e | 2019-04-23 10:07:24 -0400 | [diff] [blame] | 1349 | /* |
| 1350 | * If any worker thread seems busy, which we define |
| 1351 | * as a vector rate above 10, we enforce the barrier hold-down timer |
| 1352 | */ |
| 1353 | if (max_vector_rate > 10.0) |
Dave Barach | 36feebb | 2018-09-07 11:12:27 -0400 | [diff] [blame] | 1354 | { |
Dave Barach | 9ae190e | 2019-04-23 10:07:24 -0400 | [diff] [blame] | 1355 | while (1) |
Dave Barach | 36feebb | 2018-09-07 11:12:27 -0400 | [diff] [blame] | 1356 | { |
Dave Barach | 9ae190e | 2019-04-23 10:07:24 -0400 | [diff] [blame] | 1357 | now = vlib_time_now (vm); |
| 1358 | /* Barrier hold-down timer expired? */ |
| 1359 | if (now >= vm->barrier_no_close_before) |
| 1360 | break; |
| 1361 | if ((vm->barrier_no_close_before - now) |
| 1362 | > (2.0 * BARRIER_MINIMUM_OPEN_LIMIT)) |
| 1363 | { |
| 1364 | clib_warning |
| 1365 | ("clock change: would have waited for %.4f seconds", |
| 1366 | (vm->barrier_no_close_before - now)); |
| 1367 | break; |
| 1368 | } |
Dave Barach | 36feebb | 2018-09-07 11:12:27 -0400 | [diff] [blame] | 1369 | } |
| 1370 | } |
Colin Tregenza Dancer | eb1ac17 | 2017-09-06 20:23:24 +0100 | [diff] [blame] | 1371 | /* Record time of closure */ |
| 1372 | t_open = now - vm->barrier_epoch; |
| 1373 | vm->barrier_epoch = now; |
| 1374 | |
| 1375 | deadline = now + BARRIER_SYNC_TIMEOUT; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1376 | |
| 1377 | *vlib_worker_threads->wait_at_barrier = 1; |
| 1378 | while (*vlib_worker_threads->workers_at_barrier != count) |
| 1379 | { |
Colin Tregenza Dancer | eb1ac17 | 2017-09-06 20:23:24 +0100 | [diff] [blame] | 1380 | if ((now = vlib_time_now (vm)) > deadline) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1381 | { |
| 1382 | fformat (stderr, "%s: worker thread deadlock\n", __FUNCTION__); |
| 1383 | os_panic (); |
| 1384 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1385 | } |
Colin Tregenza Dancer | eb1ac17 | 2017-09-06 20:23:24 +0100 | [diff] [blame] | 1386 | |
| 1387 | t_closed = now - vm->barrier_epoch; |
| 1388 | |
| 1389 | barrier_trace_sync (t_entry, t_open, t_closed); |
| 1390 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1391 | } |
| 1392 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1393 | void |
| 1394 | vlib_worker_thread_barrier_release (vlib_main_t * vm) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1395 | { |
Damjan Marion | fd8deb4 | 2021-03-06 12:26:28 +0100 | [diff] [blame] | 1396 | vlib_global_main_t *vgm = vlib_get_global_main (); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1397 | f64 deadline; |
Colin Tregenza Dancer | eb1ac17 | 2017-09-06 20:23:24 +0100 | [diff] [blame] | 1398 | f64 now; |
| 1399 | f64 minimum_open; |
| 1400 | f64 t_entry; |
| 1401 | f64 t_closed_total; |
| 1402 | f64 t_update_main = 0.0; |
Colin Tregenza Dancer | 2159618 | 2017-09-04 15:27:49 +0100 | [diff] [blame] | 1403 | int refork_needed = 0; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1404 | |
Damjan Marion | 6ffb7c6 | 2021-03-26 13:06:13 +0100 | [diff] [blame] | 1405 | if (vlib_get_n_threads () < 2) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1406 | return; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1407 | |
Colin Tregenza Dancer | 2159618 | 2017-09-04 15:27:49 +0100 | [diff] [blame] | 1408 | ASSERT (vlib_get_thread_index () == 0); |
| 1409 | |
Colin Tregenza Dancer | eb1ac17 | 2017-09-06 20:23:24 +0100 | [diff] [blame] | 1410 | |
| 1411 | now = vlib_time_now (vm); |
| 1412 | t_entry = now - vm->barrier_epoch; |
| 1413 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1414 | if (--vlib_worker_threads[0].recursion_level > 0) |
Colin Tregenza Dancer | eb1ac17 | 2017-09-06 20:23:24 +0100 | [diff] [blame] | 1415 | { |
| 1416 | barrier_trace_release_rec (t_entry); |
| 1417 | return; |
| 1418 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1419 | |
Colin Tregenza Dancer | 2159618 | 2017-09-04 15:27:49 +0100 | [diff] [blame] | 1420 | /* Update (all) node runtimes before releasing the barrier, if needed */ |
Damjan Marion | fd8deb4 | 2021-03-06 12:26:28 +0100 | [diff] [blame] | 1421 | if (vgm->need_vlib_worker_thread_node_runtime_update) |
Colin Tregenza Dancer | 2159618 | 2017-09-04 15:27:49 +0100 | [diff] [blame] | 1422 | { |
Dave Barach | 1ddbc01 | 2018-06-13 09:26:05 -0400 | [diff] [blame] | 1423 | /* |
| 1424 | * Lock stat segment here, so we's safe when |
| 1425 | * rebuilding the stat segment node clones from the |
| 1426 | * stat thread... |
| 1427 | */ |
Damjan Marion | 8973b07 | 2022-03-01 15:51:18 +0100 | [diff] [blame] | 1428 | vlib_stats_segment_lock (); |
Dave Barach | 1ddbc01 | 2018-06-13 09:26:05 -0400 | [diff] [blame] | 1429 | |
Colin Tregenza Dancer | 2159618 | 2017-09-04 15:27:49 +0100 | [diff] [blame] | 1430 | /* Do stats elements on main thread */ |
| 1431 | worker_thread_node_runtime_update_internal (); |
Damjan Marion | fd8deb4 | 2021-03-06 12:26:28 +0100 | [diff] [blame] | 1432 | vgm->need_vlib_worker_thread_node_runtime_update = 0; |
Colin Tregenza Dancer | 2159618 | 2017-09-04 15:27:49 +0100 | [diff] [blame] | 1433 | |
| 1434 | /* Do per thread rebuilds in parallel */ |
| 1435 | refork_needed = 1; |
Sirshak Das | 2f6d7bb | 2018-10-03 22:53:51 +0000 | [diff] [blame] | 1436 | clib_atomic_fetch_add (vlib_worker_threads->node_reforks_required, |
Damjan Marion | 6ffb7c6 | 2021-03-26 13:06:13 +0100 | [diff] [blame] | 1437 | (vlib_get_n_threads () - 1)); |
Colin Tregenza Dancer | eb1ac17 | 2017-09-06 20:23:24 +0100 | [diff] [blame] | 1438 | now = vlib_time_now (vm); |
| 1439 | t_update_main = now - vm->barrier_epoch; |
Colin Tregenza Dancer | 2159618 | 2017-09-04 15:27:49 +0100 | [diff] [blame] | 1440 | } |
| 1441 | |
Colin Tregenza Dancer | eb1ac17 | 2017-09-06 20:23:24 +0100 | [diff] [blame] | 1442 | deadline = now + BARRIER_SYNC_TIMEOUT; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1443 | |
Dave Barach | a4324a9 | 2019-02-19 17:05:30 -0500 | [diff] [blame] | 1444 | /* |
| 1445 | * Note when we let go of the barrier. |
| 1446 | * Workers can use this to derive a reasonably accurate |
| 1447 | * time offset. See vlib_time_now(...) |
| 1448 | */ |
| 1449 | vm->time_last_barrier_release = vlib_time_now (vm); |
| 1450 | CLIB_MEMORY_STORE_BARRIER (); |
| 1451 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1452 | *vlib_worker_threads->wait_at_barrier = 0; |
| 1453 | |
| 1454 | while (*vlib_worker_threads->workers_at_barrier > 0) |
| 1455 | { |
Colin Tregenza Dancer | eb1ac17 | 2017-09-06 20:23:24 +0100 | [diff] [blame] | 1456 | if ((now = vlib_time_now (vm)) > deadline) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1457 | { |
| 1458 | fformat (stderr, "%s: worker thread deadlock\n", __FUNCTION__); |
| 1459 | os_panic (); |
| 1460 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1461 | } |
Colin Tregenza Dancer | 2159618 | 2017-09-04 15:27:49 +0100 | [diff] [blame] | 1462 | |
| 1463 | /* Wait for reforks before continuing */ |
| 1464 | if (refork_needed) |
| 1465 | { |
Colin Tregenza Dancer | eb1ac17 | 2017-09-06 20:23:24 +0100 | [diff] [blame] | 1466 | now = vlib_time_now (vm); |
| 1467 | |
| 1468 | deadline = now + BARRIER_SYNC_TIMEOUT; |
Colin Tregenza Dancer | 2159618 | 2017-09-04 15:27:49 +0100 | [diff] [blame] | 1469 | |
| 1470 | while (*vlib_worker_threads->node_reforks_required > 0) |
| 1471 | { |
Colin Tregenza Dancer | eb1ac17 | 2017-09-06 20:23:24 +0100 | [diff] [blame] | 1472 | if ((now = vlib_time_now (vm)) > deadline) |
Colin Tregenza Dancer | 2159618 | 2017-09-04 15:27:49 +0100 | [diff] [blame] | 1473 | { |
| 1474 | fformat (stderr, "%s: worker thread refork deadlock\n", |
| 1475 | __FUNCTION__); |
| 1476 | os_panic (); |
| 1477 | } |
| 1478 | } |
Damjan Marion | 8973b07 | 2022-03-01 15:51:18 +0100 | [diff] [blame] | 1479 | vlib_stats_segment_unlock (); |
Colin Tregenza Dancer | 2159618 | 2017-09-04 15:27:49 +0100 | [diff] [blame] | 1480 | } |
Colin Tregenza Dancer | eb1ac17 | 2017-09-06 20:23:24 +0100 | [diff] [blame] | 1481 | |
| 1482 | t_closed_total = now - vm->barrier_epoch; |
| 1483 | |
| 1484 | minimum_open = t_closed_total * BARRIER_MINIMUM_OPEN_FACTOR; |
| 1485 | |
| 1486 | if (minimum_open > BARRIER_MINIMUM_OPEN_LIMIT) |
| 1487 | { |
| 1488 | minimum_open = BARRIER_MINIMUM_OPEN_LIMIT; |
| 1489 | } |
| 1490 | |
| 1491 | vm->barrier_no_close_before = now + minimum_open; |
| 1492 | |
| 1493 | /* Record barrier epoch (used to enforce minimum open time) */ |
| 1494 | vm->barrier_epoch = now; |
| 1495 | |
| 1496 | barrier_trace_release (t_entry, t_closed_total, t_update_main); |
| 1497 | |
Tom Seidenberg | 6c81f5a | 2020-07-10 15:49:03 +0000 | [diff] [blame] | 1498 | if (PREDICT_FALSE (vec_len (vm->barrier_perf_callbacks) != 0)) |
| 1499 | clib_call_callbacks (vm->barrier_perf_callbacks, vm, |
| 1500 | vm->clib_time.last_cpu_time, 1 /* leave */ ); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1501 | } |
| 1502 | |
Florin Coras | 4b20830 | 2022-03-30 13:50:19 -0700 | [diff] [blame] | 1503 | static void |
| 1504 | vlib_worker_sync_rpc (void *args) |
| 1505 | { |
| 1506 | ASSERT (vlib_thread_is_main_w_barrier ()); |
| 1507 | vlib_worker_threads->wait_before_barrier = 0; |
| 1508 | } |
| 1509 | |
| 1510 | void |
| 1511 | vlib_workers_sync (void) |
| 1512 | { |
| 1513 | if (PREDICT_FALSE (!vlib_num_workers ())) |
| 1514 | return; |
| 1515 | |
| 1516 | if (!(*vlib_worker_threads->wait_at_barrier) && |
| 1517 | !clib_atomic_swap_rel_n (&vlib_worker_threads->wait_before_barrier, 1)) |
| 1518 | { |
| 1519 | u32 thread_index = vlib_get_thread_index (); |
| 1520 | vlib_rpc_call_main_thread (vlib_worker_sync_rpc, (u8 *) &thread_index, |
| 1521 | sizeof (thread_index)); |
| 1522 | } |
| 1523 | |
| 1524 | /* Wait until main thread asks for barrier */ |
| 1525 | while (!(*vlib_worker_threads->wait_at_barrier)) |
| 1526 | ; |
| 1527 | |
| 1528 | /* Stop before barrier and make sure all threads are either |
| 1529 | * at worker barrier or the barrier before it */ |
| 1530 | clib_atomic_fetch_add (&vlib_worker_threads->workers_before_barrier, 1); |
| 1531 | while (vlib_num_workers () > (*vlib_worker_threads->workers_at_barrier + |
| 1532 | vlib_worker_threads->workers_before_barrier)) |
| 1533 | ; |
| 1534 | } |
| 1535 | |
| 1536 | void |
| 1537 | vlib_workers_continue (void) |
| 1538 | { |
| 1539 | if (PREDICT_FALSE (!vlib_num_workers ())) |
| 1540 | return; |
| 1541 | |
| 1542 | clib_atomic_fetch_add (&vlib_worker_threads->done_work_before_barrier, 1); |
| 1543 | |
| 1544 | /* Wait until all workers are done with work before barrier */ |
| 1545 | while (vlib_worker_threads->done_work_before_barrier < |
| 1546 | vlib_worker_threads->workers_before_barrier) |
| 1547 | ; |
| 1548 | |
| 1549 | clib_atomic_fetch_add (&vlib_worker_threads->done_work_before_barrier, -1); |
| 1550 | clib_atomic_fetch_add (&vlib_worker_threads->workers_before_barrier, -1); |
| 1551 | } |
| 1552 | |
Neale Ranns | 42845dd | 2020-05-26 13:12:17 +0000 | [diff] [blame] | 1553 | /** |
| 1554 | * Wait until each of the workers has been once around the track |
| 1555 | */ |
| 1556 | void |
| 1557 | vlib_worker_wait_one_loop (void) |
| 1558 | { |
Damjan Marion | fd8deb4 | 2021-03-06 12:26:28 +0100 | [diff] [blame] | 1559 | vlib_global_main_t *vgm = vlib_get_global_main (); |
Neale Ranns | 42845dd | 2020-05-26 13:12:17 +0000 | [diff] [blame] | 1560 | ASSERT (vlib_get_thread_index () == 0); |
| 1561 | |
Damjan Marion | 6ffb7c6 | 2021-03-26 13:06:13 +0100 | [diff] [blame] | 1562 | if (vlib_get_n_threads () < 2) |
Neale Ranns | 42845dd | 2020-05-26 13:12:17 +0000 | [diff] [blame] | 1563 | return; |
| 1564 | |
| 1565 | if (vlib_worker_thread_barrier_held ()) |
| 1566 | return; |
| 1567 | |
| 1568 | u32 *counts = 0; |
| 1569 | u32 ii; |
| 1570 | |
Damjan Marion | 6ffb7c6 | 2021-03-26 13:06:13 +0100 | [diff] [blame] | 1571 | vec_validate (counts, vlib_get_n_threads () - 1); |
Neale Ranns | 42845dd | 2020-05-26 13:12:17 +0000 | [diff] [blame] | 1572 | |
| 1573 | /* record the current loop counts */ |
Damjan Marion | fd8deb4 | 2021-03-06 12:26:28 +0100 | [diff] [blame] | 1574 | vec_foreach_index (ii, vgm->vlib_mains) |
| 1575 | counts[ii] = vgm->vlib_mains[ii]->main_loop_count; |
Neale Ranns | 42845dd | 2020-05-26 13:12:17 +0000 | [diff] [blame] | 1576 | |
| 1577 | /* spin until each changes, apart from the main thread, or we'd be |
| 1578 | * a while */ |
| 1579 | for (ii = 1; ii < vec_len (counts); ii++) |
| 1580 | { |
Damjan Marion | fd8deb4 | 2021-03-06 12:26:28 +0100 | [diff] [blame] | 1581 | while (counts[ii] == vgm->vlib_mains[ii]->main_loop_count) |
Neale Ranns | 42845dd | 2020-05-26 13:12:17 +0000 | [diff] [blame] | 1582 | CLIB_PAUSE (); |
| 1583 | } |
| 1584 | |
| 1585 | vec_free (counts); |
| 1586 | return; |
| 1587 | } |
| 1588 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1589 | void |
Florin Coras | 4cadd3b | 2024-02-01 20:46:15 -0800 | [diff] [blame^] | 1590 | vlib_worker_flush_pending_rpc_requests (vlib_main_t *vm) |
| 1591 | { |
| 1592 | vlib_main_t *vm_global = vlib_get_first_main (); |
| 1593 | |
| 1594 | ASSERT (vm != vm_global); |
| 1595 | |
| 1596 | clib_spinlock_lock_if_init (&vm_global->pending_rpc_lock); |
| 1597 | vec_append (vm_global->pending_rpc_requests, vm->pending_rpc_requests); |
| 1598 | vec_reset_length (vm->pending_rpc_requests); |
| 1599 | clib_spinlock_unlock_if_init (&vm_global->pending_rpc_lock); |
| 1600 | } |
| 1601 | |
| 1602 | void |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1603 | vlib_worker_thread_fn (void *arg) |
Damjan Marion | 0f8ecf0 | 2016-06-27 08:30:30 +0200 | [diff] [blame] | 1604 | { |
Damjan Marion | fd8deb4 | 2021-03-06 12:26:28 +0100 | [diff] [blame] | 1605 | vlib_global_main_t *vgm = vlib_get_global_main (); |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1606 | vlib_worker_thread_t *w = (vlib_worker_thread_t *) arg; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1607 | vlib_main_t *vm = vlib_get_main (); |
Damjan Marion | e9f929b | 2017-03-16 11:32:09 +0100 | [diff] [blame] | 1608 | clib_error_t *e; |
Damjan Marion | 0f8ecf0 | 2016-06-27 08:30:30 +0200 | [diff] [blame] | 1609 | |
Damjan Marion | 586afd7 | 2017-04-05 19:18:20 +0200 | [diff] [blame] | 1610 | ASSERT (vm->thread_index == vlib_get_thread_index ()); |
Damjan Marion | 0f8ecf0 | 2016-06-27 08:30:30 +0200 | [diff] [blame] | 1611 | |
| 1612 | vlib_worker_thread_init (w); |
| 1613 | clib_time_init (&vm->clib_time); |
| 1614 | clib_mem_set_heap (w->thread_mheap); |
| 1615 | |
Damjan Marion | fd8deb4 | 2021-03-06 12:26:28 +0100 | [diff] [blame] | 1616 | vm->worker_init_functions_called = hash_create (0, 0); |
| 1617 | |
| 1618 | e = vlib_call_init_exit_functions_no_sort ( |
| 1619 | vm, &vgm->worker_init_function_registrations, 1 /* call_once */, |
| 1620 | 0 /* is_global */); |
Damjan Marion | e9f929b | 2017-03-16 11:32:09 +0100 | [diff] [blame] | 1621 | if (e) |
| 1622 | clib_error_report (e); |
| 1623 | |
Damjan Marion | e9d52d5 | 2017-03-09 15:42:26 +0100 | [diff] [blame] | 1624 | vlib_worker_loop (vm); |
Damjan Marion | 0f8ecf0 | 2016-06-27 08:30:30 +0200 | [diff] [blame] | 1625 | } |
| 1626 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1627 | /* *INDENT-OFF* */ |
Damjan Marion | 0f8ecf0 | 2016-06-27 08:30:30 +0200 | [diff] [blame] | 1628 | VLIB_REGISTER_THREAD (worker_thread_reg, static) = { |
| 1629 | .name = "workers", |
| 1630 | .short_name = "wk", |
| 1631 | .function = vlib_worker_thread_fn, |
| 1632 | }; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1633 | /* *INDENT-ON* */ |
Damjan Marion | 0f8ecf0 | 2016-06-27 08:30:30 +0200 | [diff] [blame] | 1634 | |
Mohammed Hawari | e714926 | 2022-05-18 10:08:47 +0200 | [diff] [blame] | 1635 | extern clib_march_fn_registration |
| 1636 | *vlib_frame_queue_dequeue_with_aux_fn_march_fn_registrations; |
| 1637 | extern clib_march_fn_registration |
| 1638 | *vlib_frame_queue_dequeue_fn_march_fn_registrations; |
Damjan Marion | aaef1eb | 2016-11-08 17:37:01 +0100 | [diff] [blame] | 1639 | u32 |
| 1640 | vlib_frame_queue_main_init (u32 node_index, u32 frame_queue_nelts) |
| 1641 | { |
| 1642 | vlib_thread_main_t *tm = vlib_get_thread_main (); |
Mohammed Hawari | e714926 | 2022-05-18 10:08:47 +0200 | [diff] [blame] | 1643 | vlib_main_t *vm = vlib_get_main (); |
Damjan Marion | aaef1eb | 2016-11-08 17:37:01 +0100 | [diff] [blame] | 1644 | vlib_frame_queue_main_t *fqm; |
| 1645 | vlib_frame_queue_t *fq; |
Mohammed Hawari | e714926 | 2022-05-18 10:08:47 +0200 | [diff] [blame] | 1646 | vlib_node_t *node; |
Damjan Marion | aaef1eb | 2016-11-08 17:37:01 +0100 | [diff] [blame] | 1647 | int i; |
Elias Rudberg | 368104d | 2020-04-16 16:01:52 +0200 | [diff] [blame] | 1648 | u32 num_threads; |
Damjan Marion | aaef1eb | 2016-11-08 17:37:01 +0100 | [diff] [blame] | 1649 | |
| 1650 | if (frame_queue_nelts == 0) |
dongjuan | 8875248 | 2019-06-04 10:59:02 +0800 | [diff] [blame] | 1651 | frame_queue_nelts = FRAME_QUEUE_MAX_NELTS; |
Damjan Marion | aaef1eb | 2016-11-08 17:37:01 +0100 | [diff] [blame] | 1652 | |
Elias Rudberg | 368104d | 2020-04-16 16:01:52 +0200 | [diff] [blame] | 1653 | num_threads = 1 /* main thread */ + tm->n_threads; |
| 1654 | ASSERT (frame_queue_nelts >= 8 + num_threads); |
Damjan Marion | 78fd7e8 | 2018-07-20 18:47:05 +0200 | [diff] [blame] | 1655 | |
Damjan Marion | aaef1eb | 2016-11-08 17:37:01 +0100 | [diff] [blame] | 1656 | vec_add2 (tm->frame_queue_mains, fqm, 1); |
| 1657 | |
Mohammed Hawari | e714926 | 2022-05-18 10:08:47 +0200 | [diff] [blame] | 1658 | node = vlib_get_node (vm, fqm->node_index); |
| 1659 | ASSERT (node); |
| 1660 | if (node->aux_offset) |
| 1661 | { |
| 1662 | fqm->frame_queue_dequeue_fn = |
| 1663 | CLIB_MARCH_FN_VOID_POINTER (vlib_frame_queue_dequeue_with_aux_fn); |
| 1664 | } |
| 1665 | else |
| 1666 | { |
| 1667 | fqm->frame_queue_dequeue_fn = |
| 1668 | CLIB_MARCH_FN_VOID_POINTER (vlib_frame_queue_dequeue_fn); |
| 1669 | } |
| 1670 | |
Damjan Marion | aaef1eb | 2016-11-08 17:37:01 +0100 | [diff] [blame] | 1671 | fqm->node_index = node_index; |
Damjan Marion | 78fd7e8 | 2018-07-20 18:47:05 +0200 | [diff] [blame] | 1672 | fqm->frame_queue_nelts = frame_queue_nelts; |
Damjan Marion | aaef1eb | 2016-11-08 17:37:01 +0100 | [diff] [blame] | 1673 | |
| 1674 | vec_validate (fqm->vlib_frame_queues, tm->n_vlib_mains - 1); |
Damjan Marion | 8bea589 | 2022-04-04 22:40:45 +0200 | [diff] [blame] | 1675 | vec_set_len (fqm->vlib_frame_queues, 0); |
Damjan Marion | aaef1eb | 2016-11-08 17:37:01 +0100 | [diff] [blame] | 1676 | for (i = 0; i < tm->n_vlib_mains; i++) |
| 1677 | { |
| 1678 | fq = vlib_frame_queue_alloc (frame_queue_nelts); |
| 1679 | vec_add1 (fqm->vlib_frame_queues, fq); |
| 1680 | } |
| 1681 | |
| 1682 | return (fqm - tm->frame_queue_mains); |
| 1683 | } |
| 1684 | |
Dave Barach | 69128d0 | 2017-09-26 10:54:34 -0400 | [diff] [blame] | 1685 | void |
| 1686 | vlib_process_signal_event_mt_helper (vlib_process_signal_event_mt_args_t * |
| 1687 | args) |
| 1688 | { |
| 1689 | ASSERT (vlib_get_thread_index () == 0); |
| 1690 | vlib_process_signal_event (vlib_get_main (), args->node_index, |
| 1691 | args->type_opaque, args->data); |
| 1692 | } |
| 1693 | |
| 1694 | void *rpc_call_main_thread_cb_fn; |
| 1695 | |
| 1696 | void |
| 1697 | vlib_rpc_call_main_thread (void *callback, u8 * args, u32 arg_size) |
| 1698 | { |
| 1699 | if (rpc_call_main_thread_cb_fn) |
| 1700 | { |
| 1701 | void (*fp) (void *, u8 *, u32) = rpc_call_main_thread_cb_fn; |
| 1702 | (*fp) (callback, args, arg_size); |
| 1703 | } |
| 1704 | else |
| 1705 | clib_warning ("BUG: rpc_call_main_thread_cb_fn NULL!"); |
| 1706 | } |
| 1707 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1708 | clib_error_t * |
| 1709 | threads_init (vlib_main_t * vm) |
Damjan Marion | 0f8ecf0 | 2016-06-27 08:30:30 +0200 | [diff] [blame] | 1710 | { |
Benoît Ganne | 8b153de | 2021-12-09 18:24:21 +0100 | [diff] [blame] | 1711 | const vlib_thread_main_t *tm = vlib_get_thread_main (); |
| 1712 | |
| 1713 | if (tm->main_lcore == ~0 && tm->n_vlib_mains > 1) |
| 1714 | return clib_error_return (0, "Configuration error, a main core must " |
| 1715 | "be specified when using worker threads"); |
| 1716 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1717 | return 0; |
| 1718 | } |
| 1719 | |
Damjan Marion | 0f8ecf0 | 2016-06-27 08:30:30 +0200 | [diff] [blame] | 1720 | VLIB_INIT_FUNCTION (threads_init); |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1721 | |
Dave Barach | a4324a9 | 2019-02-19 17:05:30 -0500 | [diff] [blame] | 1722 | static clib_error_t * |
| 1723 | show_clock_command_fn (vlib_main_t * vm, |
| 1724 | unformat_input_t * input, vlib_cli_command_t * cmd) |
| 1725 | { |
Dave Barach | c25048b | 2020-01-29 18:05:24 -0500 | [diff] [blame] | 1726 | int verbose = 0; |
Dave Barach | 1971800 | 2020-03-11 10:31:36 -0400 | [diff] [blame] | 1727 | clib_timebase_t _tb, *tb = &_tb; |
Dave Barach | a4324a9 | 2019-02-19 17:05:30 -0500 | [diff] [blame] | 1728 | |
Dave Barach | c25048b | 2020-01-29 18:05:24 -0500 | [diff] [blame] | 1729 | (void) unformat (input, "verbose %=", &verbose, 1); |
Dave Barach | a4324a9 | 2019-02-19 17:05:30 -0500 | [diff] [blame] | 1730 | |
Dave Barach | 1971800 | 2020-03-11 10:31:36 -0400 | [diff] [blame] | 1731 | clib_timebase_init (tb, 0 /* GMT */ , CLIB_TIMEBASE_DAYLIGHT_NONE, |
| 1732 | &vm->clib_time); |
| 1733 | |
| 1734 | vlib_cli_output (vm, "%U, %U GMT", format_clib_time, &vm->clib_time, |
| 1735 | verbose, format_clib_timebase_time, |
| 1736 | clib_timebase_now (tb)); |
Dave Barach | a4324a9 | 2019-02-19 17:05:30 -0500 | [diff] [blame] | 1737 | |
Dave Barach | a4324a9 | 2019-02-19 17:05:30 -0500 | [diff] [blame] | 1738 | vlib_cli_output (vm, "Time last barrier release %.9f", |
| 1739 | vm->time_last_barrier_release); |
| 1740 | |
Benoît Ganne | 56eccdb | 2021-08-20 09:18:31 +0200 | [diff] [blame] | 1741 | foreach_vlib_main () |
Dave Barach | a4324a9 | 2019-02-19 17:05:30 -0500 | [diff] [blame] | 1742 | { |
Benoît Ganne | 56eccdb | 2021-08-20 09:18:31 +0200 | [diff] [blame] | 1743 | vlib_cli_output (vm, "%d: %U", this_vlib_main->thread_index, |
| 1744 | format_clib_time, &this_vlib_main->clib_time, verbose); |
Dave Barach | c25048b | 2020-01-29 18:05:24 -0500 | [diff] [blame] | 1745 | |
Benoît Ganne | 56eccdb | 2021-08-20 09:18:31 +0200 | [diff] [blame] | 1746 | vlib_cli_output (vm, "Thread %d offset %.9f error %.9f", |
| 1747 | this_vlib_main->thread_index, |
| 1748 | this_vlib_main->time_offset, |
| 1749 | vm->time_last_barrier_release - |
| 1750 | this_vlib_main->time_last_barrier_release); |
Dave Barach | a4324a9 | 2019-02-19 17:05:30 -0500 | [diff] [blame] | 1751 | } |
| 1752 | return 0; |
| 1753 | } |
| 1754 | |
| 1755 | /* *INDENT-OFF* */ |
| 1756 | VLIB_CLI_COMMAND (f_command, static) = |
| 1757 | { |
| 1758 | .path = "show clock", |
| 1759 | .short_help = "show clock", |
| 1760 | .function = show_clock_command_fn, |
| 1761 | }; |
| 1762 | /* *INDENT-ON* */ |
| 1763 | |
Dave Barach | ab1a50c | 2020-10-06 14:08:16 -0400 | [diff] [blame] | 1764 | vlib_thread_main_t * |
| 1765 | vlib_get_thread_main_not_inline (void) |
| 1766 | { |
| 1767 | return vlib_get_thread_main (); |
| 1768 | } |
| 1769 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1770 | /* |
| 1771 | * fd.io coding-style-patch-verification: ON |
| 1772 | * |
| 1773 | * Local Variables: |
| 1774 | * eval: (c-set-style "gnu") |
| 1775 | * End: |
| 1776 | */ |