Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (c) 2015 Cisco and/or its affiliates. |
| 3 | * Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | * you may not use this file except in compliance with the License. |
| 5 | * You may obtain a copy of the License at: |
| 6 | * |
| 7 | * http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | * |
| 9 | * Unless required by applicable law or agreed to in writing, software |
| 10 | * distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | * See the License for the specific language governing permissions and |
| 13 | * limitations under the License. |
| 14 | */ |
Christophe Fontaine | fef15b4 | 2016-04-09 12:38:49 +0900 | [diff] [blame] | 15 | #define _GNU_SOURCE |
Christophe Fontaine | fef15b4 | 2016-04-09 12:38:49 +0900 | [diff] [blame] | 16 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 17 | #include <signal.h> |
| 18 | #include <math.h> |
| 19 | #include <vppinfra/format.h> |
| 20 | #include <vlib/vlib.h> |
| 21 | |
| 22 | #include <vlib/threads.h> |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 23 | #include <vlib/unix/cj.h> |
| 24 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 25 | DECLARE_CJ_GLOBAL_LOG; |
| 26 | |
| 27 | #define FRAME_QUEUE_NELTS 32 |
| 28 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 29 | u32 |
| 30 | vl (void *p) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 31 | { |
| 32 | return vec_len (p); |
| 33 | } |
| 34 | |
Damjan Marion | 6a7acc2 | 2016-12-19 16:28:36 +0100 | [diff] [blame] | 35 | vlib_worker_thread_t *vlib_worker_threads; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 36 | vlib_thread_main_t vlib_thread_main; |
| 37 | |
Colin Tregenza Dancer | eb1ac17 | 2017-09-06 20:23:24 +0100 | [diff] [blame] | 38 | /* |
| 39 | * Barrier tracing can be enabled on a normal build to collect information |
| 40 | * on barrier use, including timings and call stacks. Deliberately not |
| 41 | * keyed off CLIB_DEBUG, because that can add significant overhead which |
| 42 | * imapacts observed timings. |
| 43 | */ |
| 44 | |
| 45 | #ifdef BARRIER_TRACING |
| 46 | /* |
| 47 | * Output of barrier tracing can be to syslog or elog as suits |
| 48 | */ |
| 49 | #ifdef BARRIER_TRACING_ELOG |
| 50 | static u32 |
| 51 | elog_id_for_msg_name (const char *msg_name) |
| 52 | { |
| 53 | uword *p, r; |
| 54 | static uword *h; |
| 55 | u8 *name_copy; |
| 56 | |
| 57 | if (!h) |
| 58 | h = hash_create_string (0, sizeof (uword)); |
| 59 | |
| 60 | p = hash_get_mem (h, msg_name); |
| 61 | if (p) |
| 62 | return p[0]; |
| 63 | r = elog_string (&vlib_global_main.elog_main, "%s", msg_name); |
| 64 | |
| 65 | name_copy = format (0, "%s%c", msg_name, 0); |
| 66 | |
| 67 | hash_set_mem (h, name_copy, r); |
| 68 | |
| 69 | return r; |
| 70 | } |
| 71 | |
| 72 | /* |
| 73 | * elog Barrier trace functions, which are nulled out if BARRIER_TRACING isn't |
| 74 | * defined |
| 75 | */ |
| 76 | |
| 77 | static inline void |
| 78 | barrier_trace_sync (f64 t_entry, f64 t_open, f64 t_closed) |
| 79 | { |
| 80 | /* *INDENT-OFF* */ |
| 81 | ELOG_TYPE_DECLARE (e) = |
| 82 | { |
| 83 | .format = "barrier <%d#%s(O:%dus:%dus)(%dus)", |
| 84 | .format_args = "i4T4i4i4i4", |
| 85 | }; |
| 86 | /* *INDENT-ON* */ |
| 87 | struct |
| 88 | { |
| 89 | u32 count, caller, t_entry, t_open, t_closed; |
| 90 | } *ed = 0; |
| 91 | |
| 92 | ed = ELOG_DATA (&vlib_global_main.elog_main, e); |
| 93 | ed->count = (int) vlib_worker_threads[0].barrier_sync_count; |
| 94 | ed->caller = elog_id_for_msg_name (vlib_worker_threads[0].barrier_caller); |
| 95 | ed->t_entry = (int) (1000000.0 * t_entry); |
| 96 | ed->t_open = (int) (1000000.0 * t_open); |
| 97 | ed->t_closed = (int) (1000000.0 * t_closed); |
| 98 | } |
| 99 | |
| 100 | static inline void |
| 101 | barrier_trace_sync_rec (f64 t_entry) |
| 102 | { |
| 103 | /* *INDENT-OFF* */ |
| 104 | ELOG_TYPE_DECLARE (e) = |
| 105 | { |
| 106 | .format = "barrier <%d(%dus)%s", |
| 107 | .format_args = "i4i4T4", |
| 108 | }; |
| 109 | /* *INDENT-ON* */ |
| 110 | struct |
| 111 | { |
| 112 | u32 depth, t_entry, caller; |
| 113 | } *ed = 0; |
| 114 | |
| 115 | ed = ELOG_DATA (&vlib_global_main.elog_main, e); |
| 116 | ed->depth = (int) vlib_worker_threads[0].recursion_level - 1; |
| 117 | ed->t_entry = (int) (1000000.0 * t_entry); |
| 118 | ed->caller = elog_id_for_msg_name (vlib_worker_threads[0].barrier_caller); |
| 119 | } |
| 120 | |
| 121 | static inline void |
| 122 | barrier_trace_release_rec (f64 t_entry) |
| 123 | { |
| 124 | /* *INDENT-OFF* */ |
| 125 | ELOG_TYPE_DECLARE (e) = |
| 126 | { |
| 127 | .format = "barrier (%dus)%d>", |
| 128 | .format_args = "i4i4", |
| 129 | }; |
| 130 | /* *INDENT-ON* */ |
| 131 | struct |
| 132 | { |
| 133 | u32 t_entry, depth; |
| 134 | } *ed = 0; |
| 135 | |
| 136 | ed = ELOG_DATA (&vlib_global_main.elog_main, e); |
| 137 | ed->t_entry = (int) (1000000.0 * t_entry); |
| 138 | ed->depth = (int) vlib_worker_threads[0].recursion_level; |
| 139 | } |
| 140 | |
| 141 | static inline void |
| 142 | barrier_trace_release (f64 t_entry, f64 t_closed_total, f64 t_update_main) |
| 143 | { |
| 144 | /* *INDENT-OFF* */ |
| 145 | ELOG_TYPE_DECLARE (e) = |
| 146 | { |
| 147 | .format = "barrier (%dus){%d}(C:%dus)#%d>", |
| 148 | .format_args = "i4i4i4i4", |
| 149 | }; |
| 150 | /* *INDENT-ON* */ |
| 151 | struct |
| 152 | { |
| 153 | u32 t_entry, t_update_main, t_closed_total, count; |
| 154 | } *ed = 0; |
| 155 | |
| 156 | ed = ELOG_DATA (&vlib_global_main.elog_main, e); |
| 157 | ed->t_entry = (int) (1000000.0 * t_entry); |
| 158 | ed->t_update_main = (int) (1000000.0 * t_update_main); |
| 159 | ed->t_closed_total = (int) (1000000.0 * t_closed_total); |
| 160 | ed->count = (int) vlib_worker_threads[0].barrier_sync_count; |
| 161 | |
| 162 | /* Reset context for next trace */ |
| 163 | vlib_worker_threads[0].barrier_context = NULL; |
| 164 | } |
| 165 | #else |
| 166 | char barrier_trace[65536]; |
| 167 | char *btp = barrier_trace; |
| 168 | |
| 169 | /* |
| 170 | * syslog Barrier trace functions, which are nulled out if BARRIER_TRACING |
| 171 | * isn't defined |
| 172 | */ |
| 173 | |
| 174 | |
| 175 | static inline void |
| 176 | barrier_trace_sync (f64 t_entry, f64 t_open, f64 t_closed) |
| 177 | { |
| 178 | btp += sprintf (btp, "<%u#%s", |
| 179 | (unsigned int) vlib_worker_threads[0].barrier_sync_count, |
| 180 | vlib_worker_threads[0].barrier_caller); |
| 181 | |
| 182 | if (vlib_worker_threads[0].barrier_context) |
| 183 | { |
| 184 | btp += sprintf (btp, "[%s]", vlib_worker_threads[0].barrier_context); |
| 185 | |
| 186 | } |
| 187 | |
| 188 | btp += sprintf (btp, "(O:%dus:%dus)(%dus):", |
| 189 | (int) (1000000.0 * t_entry), |
| 190 | (int) (1000000.0 * t_open), (int) (1000000.0 * t_closed)); |
| 191 | |
| 192 | } |
| 193 | |
| 194 | static inline void |
| 195 | barrier_trace_sync_rec (f64 t_entry) |
| 196 | { |
| 197 | btp += sprintf (btp, "<%u(%dus)%s:", |
| 198 | (int) vlib_worker_threads[0].recursion_level - 1, |
| 199 | (int) (1000000.0 * t_entry), |
| 200 | vlib_worker_threads[0].barrier_caller); |
| 201 | } |
| 202 | |
| 203 | static inline void |
| 204 | barrier_trace_release_rec (f64 t_entry) |
| 205 | { |
| 206 | btp += sprintf (btp, ":(%dus)%u>", (int) (1000000.0 * t_entry), |
| 207 | (int) vlib_worker_threads[0].recursion_level); |
| 208 | } |
| 209 | |
| 210 | static inline void |
| 211 | barrier_trace_release (f64 t_entry, f64 t_closed_total, f64 t_update_main) |
| 212 | { |
| 213 | |
| 214 | btp += sprintf (btp, ":(%dus)", (int) (1000000.0 * t_entry)); |
| 215 | if (t_update_main > 0) |
| 216 | { |
| 217 | btp += sprintf (btp, "{%dus}", (int) (1000000.0 * t_update_main)); |
| 218 | } |
| 219 | |
| 220 | btp += sprintf (btp, "(C:%dus)#%u>", |
| 221 | (int) (1000000.0 * t_closed_total), |
| 222 | (int) vlib_worker_threads[0].barrier_sync_count); |
| 223 | |
| 224 | /* Dump buffer to syslog, and reset for next trace */ |
| 225 | fformat (stderr, "BTRC %s\n", barrier_trace); |
| 226 | btp = barrier_trace; |
| 227 | vlib_worker_threads[0].barrier_context = NULL; |
| 228 | } |
| 229 | #endif |
| 230 | #else |
| 231 | |
| 232 | /* Null functions for default case where barrier tracing isn't used */ |
| 233 | static inline void |
| 234 | barrier_trace_sync (f64 t_entry, f64 t_open, f64 t_closed) |
| 235 | { |
| 236 | } |
| 237 | |
| 238 | static inline void |
| 239 | barrier_trace_sync_rec (f64 t_entry) |
| 240 | { |
| 241 | } |
| 242 | |
| 243 | static inline void |
| 244 | barrier_trace_release_rec (f64 t_entry) |
| 245 | { |
| 246 | } |
| 247 | |
| 248 | static inline void |
| 249 | barrier_trace_release (f64 t_entry, f64 t_closed_total, f64 t_update_main) |
| 250 | { |
| 251 | } |
| 252 | #endif |
| 253 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 254 | uword |
Damjan Marion | f55f9b8 | 2017-05-10 21:06:28 +0200 | [diff] [blame] | 255 | os_get_nthreads (void) |
Dave Barach | 01d86c7 | 2016-08-08 15:13:42 -0400 | [diff] [blame] | 256 | { |
| 257 | u32 len; |
| 258 | |
| 259 | len = vec_len (vlib_thread_stacks); |
| 260 | if (len == 0) |
| 261 | return 1; |
| 262 | else |
| 263 | return len; |
| 264 | } |
| 265 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 266 | void |
| 267 | vlib_set_thread_name (char *name) |
| 268 | { |
| 269 | int pthread_setname_np (pthread_t __target_thread, const char *__name); |
Dave Barach | b2a6e25 | 2016-07-27 10:00:58 -0400 | [diff] [blame] | 270 | int rv; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 271 | pthread_t thread = pthread_self (); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 272 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 273 | if (thread) |
Dave Barach | b2a6e25 | 2016-07-27 10:00:58 -0400 | [diff] [blame] | 274 | { |
| 275 | rv = pthread_setname_np (thread, name); |
| 276 | if (rv) |
Ed Warnicke | 853e720 | 2016-08-12 11:42:26 -0700 | [diff] [blame] | 277 | clib_warning ("pthread_setname_np returned %d", rv); |
Dave Barach | b2a6e25 | 2016-07-27 10:00:58 -0400 | [diff] [blame] | 278 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 279 | } |
| 280 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 281 | static int |
| 282 | sort_registrations_by_no_clone (void *a0, void *a1) |
| 283 | { |
| 284 | vlib_thread_registration_t **tr0 = a0; |
| 285 | vlib_thread_registration_t **tr1 = a1; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 286 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 287 | return ((i32) ((*tr0)->no_data_structure_clone) |
| 288 | - ((i32) ((*tr1)->no_data_structure_clone))); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 289 | } |
| 290 | |
| 291 | static uword * |
Damjan Marion | 01914ce | 2017-09-14 19:04:50 +0200 | [diff] [blame] | 292 | clib_sysfs_list_to_bitmap (char *filename) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 293 | { |
| 294 | FILE *fp; |
| 295 | uword *r = 0; |
| 296 | |
| 297 | fp = fopen (filename, "r"); |
| 298 | |
| 299 | if (fp != NULL) |
| 300 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 301 | u8 *buffer = 0; |
| 302 | vec_validate (buffer, 256 - 1); |
| 303 | if (fgets ((char *) buffer, 256, fp)) |
| 304 | { |
| 305 | unformat_input_t in; |
| 306 | unformat_init_string (&in, (char *) buffer, |
| 307 | strlen ((char *) buffer)); |
Dave Barach | b2a6e25 | 2016-07-27 10:00:58 -0400 | [diff] [blame] | 308 | if (unformat (&in, "%U", unformat_bitmap_list, &r) != 1) |
Ed Warnicke | 853e720 | 2016-08-12 11:42:26 -0700 | [diff] [blame] | 309 | clib_warning ("unformat_bitmap_list failed"); |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 310 | unformat_free (&in); |
| 311 | } |
| 312 | vec_free (buffer); |
| 313 | fclose (fp); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 314 | } |
| 315 | return r; |
| 316 | } |
| 317 | |
| 318 | |
| 319 | /* Called early in the init sequence */ |
| 320 | |
| 321 | clib_error_t * |
| 322 | vlib_thread_init (vlib_main_t * vm) |
| 323 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 324 | vlib_thread_main_t *tm = &vlib_thread_main; |
| 325 | vlib_worker_thread_t *w; |
| 326 | vlib_thread_registration_t *tr; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 327 | u32 n_vlib_mains = 1; |
| 328 | u32 first_index = 1; |
| 329 | u32 i; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 330 | uword *avail_cpu; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 331 | |
| 332 | /* get bitmaps of active cpu cores and sockets */ |
| 333 | tm->cpu_core_bitmap = |
Damjan Marion | 01914ce | 2017-09-14 19:04:50 +0200 | [diff] [blame] | 334 | clib_sysfs_list_to_bitmap ("/sys/devices/system/cpu/online"); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 335 | tm->cpu_socket_bitmap = |
Damjan Marion | 01914ce | 2017-09-14 19:04:50 +0200 | [diff] [blame] | 336 | clib_sysfs_list_to_bitmap ("/sys/devices/system/node/online"); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 337 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 338 | avail_cpu = clib_bitmap_dup (tm->cpu_core_bitmap); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 339 | |
| 340 | /* skip cores */ |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 341 | for (i = 0; i < tm->skip_cores; i++) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 342 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 343 | uword c = clib_bitmap_first_set (avail_cpu); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 344 | if (c == ~0) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 345 | return clib_error_return (0, "no available cpus to skip"); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 346 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 347 | avail_cpu = clib_bitmap_set (avail_cpu, c, 0); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 348 | } |
| 349 | |
| 350 | /* grab cpu for main thread */ |
| 351 | if (!tm->main_lcore) |
| 352 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 353 | tm->main_lcore = clib_bitmap_first_set (avail_cpu); |
| 354 | if (tm->main_lcore == (u8) ~ 0) |
| 355 | return clib_error_return (0, "no available cpus to be used for the" |
| 356 | " main thread"); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 357 | } |
| 358 | else |
| 359 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 360 | if (clib_bitmap_get (avail_cpu, tm->main_lcore) == 0) |
| 361 | return clib_error_return (0, "cpu %u is not available to be used" |
| 362 | " for the main thread", tm->main_lcore); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 363 | } |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 364 | avail_cpu = clib_bitmap_set (avail_cpu, tm->main_lcore, 0); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 365 | |
| 366 | /* assume that there is socket 0 only if there is no data from sysfs */ |
| 367 | if (!tm->cpu_socket_bitmap) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 368 | tm->cpu_socket_bitmap = clib_bitmap_set (0, 0, 1); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 369 | |
Christophe Fontaine | fef15b4 | 2016-04-09 12:38:49 +0900 | [diff] [blame] | 370 | /* pin main thread to main_lcore */ |
Damjan Marion | 878c609 | 2017-01-04 13:19:27 +0100 | [diff] [blame] | 371 | if (tm->cb.vlib_thread_set_lcore_cb) |
| 372 | { |
| 373 | tm->cb.vlib_thread_set_lcore_cb (0, tm->main_lcore); |
| 374 | } |
| 375 | else |
| 376 | { |
| 377 | cpu_set_t cpuset; |
| 378 | CPU_ZERO (&cpuset); |
| 379 | CPU_SET (tm->main_lcore, &cpuset); |
| 380 | pthread_setaffinity_np (pthread_self (), sizeof (cpu_set_t), &cpuset); |
| 381 | } |
Christophe Fontaine | fef15b4 | 2016-04-09 12:38:49 +0900 | [diff] [blame] | 382 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 383 | /* as many threads as stacks... */ |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 384 | vec_validate_aligned (vlib_worker_threads, vec_len (vlib_thread_stacks) - 1, |
| 385 | CLIB_CACHE_LINE_BYTES); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 386 | |
| 387 | /* Preallocate thread 0 */ |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 388 | _vec_len (vlib_worker_threads) = 1; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 389 | w = vlib_worker_threads; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 390 | w->thread_mheap = clib_mem_get_heap (); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 391 | w->thread_stack = vlib_thread_stacks[0]; |
Pavel Kotucek | 9876520 | 2016-10-07 08:37:28 +0200 | [diff] [blame] | 392 | w->lcore_id = tm->main_lcore; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 393 | w->lwp = syscall (SYS_gettid); |
Pavel Kotucek | 9876520 | 2016-10-07 08:37:28 +0200 | [diff] [blame] | 394 | w->thread_id = pthread_self (); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 395 | tm->n_vlib_mains = 1; |
| 396 | |
Pavel Kotucek | 1e76583 | 2016-09-23 08:54:14 +0200 | [diff] [blame] | 397 | if (tm->sched_policy != ~0) |
| 398 | { |
| 399 | struct sched_param sched_param; |
| 400 | if (!sched_getparam (w->lwp, &sched_param)) |
| 401 | { |
| 402 | if (tm->sched_priority != ~0) |
| 403 | sched_param.sched_priority = tm->sched_priority; |
| 404 | sched_setscheduler (w->lwp, tm->sched_policy, &sched_param); |
| 405 | } |
| 406 | } |
| 407 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 408 | /* assign threads to cores and set n_vlib_mains */ |
| 409 | tr = tm->next; |
| 410 | |
| 411 | while (tr) |
| 412 | { |
| 413 | vec_add1 (tm->registrations, tr); |
| 414 | tr = tr->next; |
| 415 | } |
| 416 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 417 | vec_sort_with_function (tm->registrations, sort_registrations_by_no_clone); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 418 | |
| 419 | for (i = 0; i < vec_len (tm->registrations); i++) |
| 420 | { |
| 421 | int j; |
| 422 | tr = tm->registrations[i]; |
| 423 | tr->first_index = first_index; |
| 424 | first_index += tr->count; |
| 425 | n_vlib_mains += (tr->no_data_structure_clone == 0) ? tr->count : 0; |
| 426 | |
| 427 | /* construct coremask */ |
| 428 | if (tr->use_pthreads || !tr->count) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 429 | continue; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 430 | |
| 431 | if (tr->coremask) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 432 | { |
| 433 | uword c; |
| 434 | /* *INDENT-OFF* */ |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 435 | clib_bitmap_foreach (c, tr->coremask, ({ |
| 436 | if (clib_bitmap_get(avail_cpu, c) == 0) |
| 437 | return clib_error_return (0, "cpu %u is not available to be used" |
| 438 | " for the '%s' thread",c, tr->name); |
| 439 | |
| 440 | avail_cpu = clib_bitmap_set(avail_cpu, c, 0); |
| 441 | })); |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 442 | /* *INDENT-ON* */ |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 443 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 444 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 445 | else |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 446 | { |
| 447 | for (j = 0; j < tr->count; j++) |
| 448 | { |
| 449 | uword c = clib_bitmap_first_set (avail_cpu); |
| 450 | if (c == ~0) |
| 451 | return clib_error_return (0, |
| 452 | "no available cpus to be used for" |
| 453 | " the '%s' thread", tr->name); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 454 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 455 | avail_cpu = clib_bitmap_set (avail_cpu, c, 0); |
| 456 | tr->coremask = clib_bitmap_set (tr->coremask, c, 1); |
| 457 | } |
| 458 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 459 | } |
| 460 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 461 | clib_bitmap_free (avail_cpu); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 462 | |
| 463 | tm->n_vlib_mains = n_vlib_mains; |
| 464 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 465 | vec_validate_aligned (vlib_worker_threads, first_index - 1, |
| 466 | CLIB_CACHE_LINE_BYTES); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 467 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 468 | return 0; |
| 469 | } |
| 470 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 471 | vlib_frame_queue_t * |
| 472 | vlib_frame_queue_alloc (int nelts) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 473 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 474 | vlib_frame_queue_t *fq; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 475 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 476 | fq = clib_mem_alloc_aligned (sizeof (*fq), CLIB_CACHE_LINE_BYTES); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 477 | memset (fq, 0, sizeof (*fq)); |
| 478 | fq->nelts = nelts; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 479 | fq->vector_threshold = 128; // packets |
| 480 | vec_validate_aligned (fq->elts, nelts - 1, CLIB_CACHE_LINE_BYTES); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 481 | |
| 482 | if (1) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 483 | { |
| 484 | if (((uword) & fq->tail) & (CLIB_CACHE_LINE_BYTES - 1)) |
| 485 | fformat (stderr, "WARNING: fq->tail unaligned\n"); |
| 486 | if (((uword) & fq->head) & (CLIB_CACHE_LINE_BYTES - 1)) |
| 487 | fformat (stderr, "WARNING: fq->head unaligned\n"); |
| 488 | if (((uword) fq->elts) & (CLIB_CACHE_LINE_BYTES - 1)) |
| 489 | fformat (stderr, "WARNING: fq->elts unaligned\n"); |
| 490 | |
| 491 | if (sizeof (fq->elts[0]) % CLIB_CACHE_LINE_BYTES) |
| 492 | fformat (stderr, "WARNING: fq->elts[0] size %d\n", |
| 493 | sizeof (fq->elts[0])); |
| 494 | if (nelts & (nelts - 1)) |
| 495 | { |
| 496 | fformat (stderr, "FATAL: nelts MUST be a power of 2\n"); |
| 497 | abort (); |
| 498 | } |
| 499 | } |
| 500 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 501 | return (fq); |
| 502 | } |
| 503 | |
| 504 | void vl_msg_api_handler_no_free (void *) __attribute__ ((weak)); |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 505 | void |
| 506 | vl_msg_api_handler_no_free (void *v) |
| 507 | { |
| 508 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 509 | |
| 510 | /* Turned off, save as reference material... */ |
| 511 | #if 0 |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 512 | static inline int |
| 513 | vlib_frame_queue_dequeue_internal (int thread_id, |
| 514 | vlib_main_t * vm, vlib_node_main_t * nm) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 515 | { |
| 516 | vlib_frame_queue_t *fq = vlib_frame_queues[thread_id]; |
| 517 | vlib_frame_queue_elt_t *elt; |
| 518 | vlib_frame_t *f; |
| 519 | vlib_pending_frame_t *p; |
| 520 | vlib_node_runtime_t *r; |
| 521 | u32 node_runtime_index; |
| 522 | int msg_type; |
| 523 | u64 before; |
| 524 | int processed = 0; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 525 | |
| 526 | ASSERT (vm == vlib_mains[thread_id]); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 527 | |
| 528 | while (1) |
| 529 | { |
| 530 | if (fq->head == fq->tail) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 531 | return processed; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 532 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 533 | elt = fq->elts + ((fq->head + 1) & (fq->nelts - 1)); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 534 | |
| 535 | if (!elt->valid) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 536 | return processed; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 537 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 538 | before = clib_cpu_time_now (); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 539 | |
| 540 | f = elt->frame; |
| 541 | node_runtime_index = elt->node_runtime_index; |
| 542 | msg_type = elt->msg_type; |
| 543 | |
| 544 | switch (msg_type) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 545 | { |
| 546 | case VLIB_FRAME_QUEUE_ELT_FREE_BUFFERS: |
| 547 | vlib_buffer_free (vm, vlib_frame_vector_args (f), f->n_vectors); |
| 548 | /* note fallthrough... */ |
| 549 | case VLIB_FRAME_QUEUE_ELT_FREE_FRAME: |
| 550 | r = vec_elt_at_index (nm->nodes_by_type[VLIB_NODE_TYPE_INTERNAL], |
| 551 | node_runtime_index); |
| 552 | vlib_frame_free (vm, r, f); |
| 553 | break; |
| 554 | case VLIB_FRAME_QUEUE_ELT_DISPATCH_FRAME: |
| 555 | vec_add2 (vm->node_main.pending_frames, p, 1); |
| 556 | f->flags |= (VLIB_FRAME_PENDING | VLIB_FRAME_FREE_AFTER_DISPATCH); |
| 557 | p->node_runtime_index = elt->node_runtime_index; |
| 558 | p->frame_index = vlib_frame_index (vm, f); |
| 559 | p->next_frame_index = VLIB_PENDING_FRAME_NO_NEXT_FRAME; |
| 560 | fq->dequeue_vectors += (u64) f->n_vectors; |
| 561 | break; |
| 562 | case VLIB_FRAME_QUEUE_ELT_API_MSG: |
| 563 | vl_msg_api_handler_no_free (f); |
| 564 | break; |
| 565 | default: |
| 566 | clib_warning ("bogus frame queue message, type %d", msg_type); |
| 567 | break; |
| 568 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 569 | elt->valid = 0; |
| 570 | fq->dequeues++; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 571 | fq->dequeue_ticks += clib_cpu_time_now () - before; |
| 572 | CLIB_MEMORY_BARRIER (); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 573 | fq->head++; |
| 574 | processed++; |
| 575 | } |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 576 | ASSERT (0); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 577 | return processed; |
| 578 | } |
| 579 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 580 | int |
| 581 | vlib_frame_queue_dequeue (int thread_id, |
| 582 | vlib_main_t * vm, vlib_node_main_t * nm) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 583 | { |
| 584 | return vlib_frame_queue_dequeue_internal (thread_id, vm, nm); |
| 585 | } |
| 586 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 587 | int |
| 588 | vlib_frame_queue_enqueue (vlib_main_t * vm, u32 node_runtime_index, |
| 589 | u32 frame_queue_index, vlib_frame_t * frame, |
| 590 | vlib_frame_queue_msg_type_t type) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 591 | { |
| 592 | vlib_frame_queue_t *fq = vlib_frame_queues[frame_queue_index]; |
| 593 | vlib_frame_queue_elt_t *elt; |
| 594 | u32 save_count; |
| 595 | u64 new_tail; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 596 | u64 before = clib_cpu_time_now (); |
| 597 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 598 | ASSERT (fq); |
| 599 | |
| 600 | new_tail = __sync_add_and_fetch (&fq->tail, 1); |
| 601 | |
| 602 | /* Wait until a ring slot is available */ |
| 603 | while (new_tail >= fq->head + fq->nelts) |
| 604 | { |
| 605 | f64 b4 = vlib_time_now_ticks (vm, before); |
| 606 | vlib_worker_thread_barrier_check (vm, b4); |
| 607 | /* Bad idea. Dequeue -> enqueue -> dequeue -> trouble */ |
Damjan Marion | 586afd7 | 2017-04-05 19:18:20 +0200 | [diff] [blame] | 608 | // vlib_frame_queue_dequeue (vm->thread_index, vm, nm); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 609 | } |
| 610 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 611 | elt = fq->elts + (new_tail & (fq->nelts - 1)); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 612 | |
| 613 | /* this would be very bad... */ |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 614 | while (elt->valid) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 615 | { |
| 616 | } |
| 617 | |
| 618 | /* Once we enqueue the frame, frame->n_vectors is owned elsewhere... */ |
| 619 | save_count = frame->n_vectors; |
| 620 | |
| 621 | elt->frame = frame; |
| 622 | elt->node_runtime_index = node_runtime_index; |
| 623 | elt->msg_type = type; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 624 | CLIB_MEMORY_BARRIER (); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 625 | elt->valid = 1; |
| 626 | |
| 627 | return save_count; |
| 628 | } |
| 629 | #endif /* 0 */ |
| 630 | |
| 631 | /* To be called by vlib worker threads upon startup */ |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 632 | void |
| 633 | vlib_worker_thread_init (vlib_worker_thread_t * w) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 634 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 635 | vlib_thread_main_t *tm = vlib_get_thread_main (); |
| 636 | |
Dave Barach | fdf4944 | 2016-12-20 12:48:14 -0500 | [diff] [blame] | 637 | /* |
| 638 | * Note: disabling signals in worker threads as follows |
| 639 | * prevents the api post-mortem dump scheme from working |
| 640 | * { |
| 641 | * sigset_t s; |
| 642 | * sigfillset (&s); |
| 643 | * pthread_sigmask (SIG_SETMASK, &s, 0); |
| 644 | * } |
| 645 | */ |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 646 | |
| 647 | clib_mem_set_heap (w->thread_mheap); |
| 648 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 649 | if (vec_len (tm->thread_prefix) && w->registration->short_name) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 650 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 651 | w->name = format (0, "%v_%s_%d%c", tm->thread_prefix, |
| 652 | w->registration->short_name, w->instance_id, '\0'); |
| 653 | vlib_set_thread_name ((char *) w->name); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 654 | } |
| 655 | |
| 656 | if (!w->registration->use_pthreads) |
| 657 | { |
| 658 | |
| 659 | /* Initial barrier sync, for both worker and i/o threads */ |
| 660 | clib_smp_atomic_add (vlib_worker_threads->workers_at_barrier, 1); |
| 661 | |
| 662 | while (*vlib_worker_threads->wait_at_barrier) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 663 | ; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 664 | |
| 665 | clib_smp_atomic_add (vlib_worker_threads->workers_at_barrier, -1); |
| 666 | } |
| 667 | } |
| 668 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 669 | void * |
| 670 | vlib_worker_thread_bootstrap_fn (void *arg) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 671 | { |
| 672 | void *rv; |
| 673 | vlib_worker_thread_t *w = arg; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 674 | |
| 675 | w->lwp = syscall (SYS_gettid); |
Pavel Kotucek | 9876520 | 2016-10-07 08:37:28 +0200 | [diff] [blame] | 676 | w->thread_id = pthread_self (); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 677 | |
Damjan Marion | f55f9b8 | 2017-05-10 21:06:28 +0200 | [diff] [blame] | 678 | __os_thread_index = w - vlib_worker_threads; |
Damjan Marion | 586afd7 | 2017-04-05 19:18:20 +0200 | [diff] [blame] | 679 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 680 | rv = (void *) clib_calljmp |
| 681 | ((uword (*)(uword)) w->thread_function, |
| 682 | (uword) arg, w->thread_stack + VLIB_THREAD_STACK_SIZE); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 683 | /* NOTREACHED, we hope */ |
| 684 | return rv; |
| 685 | } |
| 686 | |
Damjan Marion | 878c609 | 2017-01-04 13:19:27 +0100 | [diff] [blame] | 687 | static clib_error_t * |
| 688 | vlib_launch_thread_int (void *fp, vlib_worker_thread_t * w, unsigned lcore_id) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 689 | { |
Damjan Marion | 878c609 | 2017-01-04 13:19:27 +0100 | [diff] [blame] | 690 | vlib_thread_main_t *tm = &vlib_thread_main; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 691 | void *(*fp_arg) (void *) = fp; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 692 | |
Pavel Kotucek | 9876520 | 2016-10-07 08:37:28 +0200 | [diff] [blame] | 693 | w->lcore_id = lcore_id; |
Damjan Marion | 878c609 | 2017-01-04 13:19:27 +0100 | [diff] [blame] | 694 | if (tm->cb.vlib_launch_thread_cb && !w->registration->use_pthreads) |
| 695 | return tm->cb.vlib_launch_thread_cb (fp, (void *) w, lcore_id); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 696 | else |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 697 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 698 | pthread_t worker; |
| 699 | cpu_set_t cpuset; |
| 700 | CPU_ZERO (&cpuset); |
| 701 | CPU_SET (lcore_id, &cpuset); |
Christophe Fontaine | fef15b4 | 2016-04-09 12:38:49 +0900 | [diff] [blame] | 702 | |
Damjan Marion | 878c609 | 2017-01-04 13:19:27 +0100 | [diff] [blame] | 703 | if (pthread_create (&worker, NULL /* attr */ , fp_arg, (void *) w)) |
| 704 | return clib_error_return_unix (0, "pthread_create"); |
| 705 | |
| 706 | if (pthread_setaffinity_np (worker, sizeof (cpu_set_t), &cpuset)) |
| 707 | return clib_error_return_unix (0, "pthread_setaffinity_np"); |
| 708 | |
| 709 | return 0; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 710 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 711 | } |
| 712 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 713 | static clib_error_t * |
| 714 | start_workers (vlib_main_t * vm) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 715 | { |
| 716 | int i, j; |
| 717 | vlib_worker_thread_t *w; |
| 718 | vlib_main_t *vm_clone; |
| 719 | void *oldheap; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 720 | vlib_thread_main_t *tm = &vlib_thread_main; |
| 721 | vlib_thread_registration_t *tr; |
| 722 | vlib_node_runtime_t *rt; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 723 | u32 n_vlib_mains = tm->n_vlib_mains; |
| 724 | u32 worker_thread_index; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 725 | u8 *main_heap = clib_mem_get_per_cpu_heap (); |
| 726 | mheap_t *main_heap_header = mheap_header (main_heap); |
| 727 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 728 | vec_reset_length (vlib_worker_threads); |
| 729 | |
| 730 | /* Set up the main thread */ |
| 731 | vec_add2_aligned (vlib_worker_threads, w, 1, CLIB_CACHE_LINE_BYTES); |
Dave Barach | 3e7deb1 | 2016-02-05 16:29:53 -0500 | [diff] [blame] | 732 | w->elog_track.name = "main thread"; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 733 | elog_track_register (&vm->elog_main, &w->elog_track); |
| 734 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 735 | if (vec_len (tm->thread_prefix)) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 736 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 737 | w->name = format (0, "%v_main%c", tm->thread_prefix, '\0'); |
| 738 | vlib_set_thread_name ((char *) w->name); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 739 | } |
| 740 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 741 | /* |
Dave Barach | 848191d | 2016-04-28 16:24:15 -0400 | [diff] [blame] | 742 | * Truth of the matter: we always use at least two |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 743 | * threads. So, make the main heap thread-safe |
Dave Barach | 848191d | 2016-04-28 16:24:15 -0400 | [diff] [blame] | 744 | * and make the event log thread-safe. |
| 745 | */ |
| 746 | main_heap_header->flags |= MHEAP_FLAG_THREAD_SAFE; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 747 | vm->elog_main.lock = |
| 748 | clib_mem_alloc_aligned (CLIB_CACHE_LINE_BYTES, CLIB_CACHE_LINE_BYTES); |
Dave Barach | 848191d | 2016-04-28 16:24:15 -0400 | [diff] [blame] | 749 | vm->elog_main.lock[0] = 0; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 750 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 751 | if (n_vlib_mains > 1) |
| 752 | { |
Dave Barach | 80f54e2 | 2017-03-08 19:08:56 -0500 | [diff] [blame] | 753 | /* Replace hand-crafted length-1 vector with a real vector */ |
| 754 | vlib_mains = 0; |
| 755 | |
| 756 | vec_validate_aligned (vlib_mains, tm->n_vlib_mains - 1, |
| 757 | CLIB_CACHE_LINE_BYTES); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 758 | _vec_len (vlib_mains) = 0; |
Dave Barach | 80f54e2 | 2017-03-08 19:08:56 -0500 | [diff] [blame] | 759 | vec_add1_aligned (vlib_mains, vm, CLIB_CACHE_LINE_BYTES); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 760 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 761 | vlib_worker_threads->wait_at_barrier = |
| 762 | clib_mem_alloc_aligned (sizeof (u32), CLIB_CACHE_LINE_BYTES); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 763 | vlib_worker_threads->workers_at_barrier = |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 764 | clib_mem_alloc_aligned (sizeof (u32), CLIB_CACHE_LINE_BYTES); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 765 | |
Colin Tregenza Dancer | 2159618 | 2017-09-04 15:27:49 +0100 | [diff] [blame] | 766 | vlib_worker_threads->node_reforks_required = |
| 767 | clib_mem_alloc_aligned (sizeof (u32), CLIB_CACHE_LINE_BYTES); |
| 768 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 769 | /* Ask for an initial barrier sync */ |
| 770 | *vlib_worker_threads->workers_at_barrier = 0; |
| 771 | *vlib_worker_threads->wait_at_barrier = 1; |
| 772 | |
Colin Tregenza Dancer | 2159618 | 2017-09-04 15:27:49 +0100 | [diff] [blame] | 773 | /* Without update or refork */ |
| 774 | *vlib_worker_threads->node_reforks_required = 0; |
| 775 | vm->need_vlib_worker_thread_node_runtime_update = 0; |
| 776 | |
Colin Tregenza Dancer | eb1ac17 | 2017-09-06 20:23:24 +0100 | [diff] [blame] | 777 | /* init timing */ |
| 778 | vm->barrier_epoch = 0; |
| 779 | vm->barrier_no_close_before = 0; |
| 780 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 781 | worker_thread_index = 1; |
| 782 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 783 | for (i = 0; i < vec_len (tm->registrations); i++) |
| 784 | { |
| 785 | vlib_node_main_t *nm, *nm_clone; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 786 | vlib_buffer_free_list_t *fl_clone, *fl_orig; |
| 787 | vlib_buffer_free_list_t *orig_freelist_pool; |
| 788 | int k; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 789 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 790 | tr = tm->registrations[i]; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 791 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 792 | if (tr->count == 0) |
| 793 | continue; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 794 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 795 | for (k = 0; k < tr->count; k++) |
| 796 | { |
Colin Tregenza Dancer | 2159618 | 2017-09-04 15:27:49 +0100 | [diff] [blame] | 797 | vlib_node_t *n; |
| 798 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 799 | vec_add2 (vlib_worker_threads, w, 1); |
| 800 | if (tr->mheap_size) |
| 801 | w->thread_mheap = |
| 802 | mheap_alloc (0 /* use VM */ , tr->mheap_size); |
| 803 | else |
| 804 | w->thread_mheap = main_heap; |
Damjan Marion | 586afd7 | 2017-04-05 19:18:20 +0200 | [diff] [blame] | 805 | |
| 806 | w->thread_stack = |
| 807 | vlib_thread_stack_init (w - vlib_worker_threads); |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 808 | w->thread_function = tr->function; |
| 809 | w->thread_function_arg = w; |
| 810 | w->instance_id = k; |
| 811 | w->registration = tr; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 812 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 813 | w->elog_track.name = |
| 814 | (char *) format (0, "%s %d", tr->name, k + 1); |
| 815 | vec_add1 (w->elog_track.name, 0); |
| 816 | elog_track_register (&vm->elog_main, &w->elog_track); |
Bud Grise | 68adab9 | 2016-02-12 10:36:11 -0500 | [diff] [blame] | 817 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 818 | if (tr->no_data_structure_clone) |
| 819 | continue; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 820 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 821 | /* Fork vlib_global_main et al. Look for bugs here */ |
| 822 | oldheap = clib_mem_set_heap (w->thread_mheap); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 823 | |
Damjan Marion | be3f4d5 | 2018-03-27 21:06:10 +0200 | [diff] [blame] | 824 | vm_clone = clib_mem_alloc_aligned (sizeof (*vm_clone), |
| 825 | CLIB_CACHE_LINE_BYTES); |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 826 | clib_memcpy (vm_clone, vlib_mains[0], sizeof (*vm_clone)); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 827 | |
Damjan Marion | 586afd7 | 2017-04-05 19:18:20 +0200 | [diff] [blame] | 828 | vm_clone->thread_index = worker_thread_index; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 829 | vm_clone->heap_base = w->thread_mheap; |
Damjan Marion | e9f929b | 2017-03-16 11:32:09 +0100 | [diff] [blame] | 830 | vm_clone->init_functions_called = |
| 831 | hash_create (0, /* value bytes */ 0); |
Dave Barach | 2877eee | 2017-12-15 12:22:57 -0500 | [diff] [blame] | 832 | vm_clone->pending_rpc_requests = 0; |
| 833 | vec_validate (vm_clone->pending_rpc_requests, 0); |
| 834 | _vec_len (vm_clone->pending_rpc_requests) = 0; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 835 | memset (&vm_clone->random_buffer, 0, |
| 836 | sizeof (vm_clone->random_buffer)); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 837 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 838 | nm = &vlib_mains[0]->node_main; |
| 839 | nm_clone = &vm_clone->node_main; |
| 840 | /* fork next frames array, preserving node runtime indices */ |
Damjan Marion | be3f4d5 | 2018-03-27 21:06:10 +0200 | [diff] [blame] | 841 | nm_clone->next_frames = vec_dup_aligned (nm->next_frames, |
| 842 | CLIB_CACHE_LINE_BYTES); |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 843 | for (j = 0; j < vec_len (nm_clone->next_frames); j++) |
| 844 | { |
| 845 | vlib_next_frame_t *nf = &nm_clone->next_frames[j]; |
| 846 | u32 save_node_runtime_index; |
| 847 | u32 save_flags; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 848 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 849 | save_node_runtime_index = nf->node_runtime_index; |
| 850 | save_flags = nf->flags & VLIB_FRAME_NO_FREE_AFTER_DISPATCH; |
| 851 | vlib_next_frame_init (nf); |
| 852 | nf->node_runtime_index = save_node_runtime_index; |
| 853 | nf->flags = save_flags; |
| 854 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 855 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 856 | /* fork the frame dispatch queue */ |
| 857 | nm_clone->pending_frames = 0; |
| 858 | vec_validate (nm_clone->pending_frames, 10); /* $$$$$?????? */ |
| 859 | _vec_len (nm_clone->pending_frames) = 0; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 860 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 861 | /* fork nodes */ |
| 862 | nm_clone->nodes = 0; |
Colin Tregenza Dancer | 2159618 | 2017-09-04 15:27:49 +0100 | [diff] [blame] | 863 | |
| 864 | /* Allocate all nodes in single block for speed */ |
| 865 | n = clib_mem_alloc_no_fail (vec_len (nm->nodes) * sizeof (*n)); |
| 866 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 867 | for (j = 0; j < vec_len (nm->nodes); j++) |
| 868 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 869 | clib_memcpy (n, nm->nodes[j], sizeof (*n)); |
| 870 | /* none of the copied nodes have enqueue rights given out */ |
| 871 | n->owner_node_index = VLIB_INVALID_NODE_INDEX; |
| 872 | memset (&n->stats_total, 0, sizeof (n->stats_total)); |
| 873 | memset (&n->stats_last_clear, 0, |
| 874 | sizeof (n->stats_last_clear)); |
| 875 | vec_add1 (nm_clone->nodes, n); |
Colin Tregenza Dancer | 2159618 | 2017-09-04 15:27:49 +0100 | [diff] [blame] | 876 | n++; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 877 | } |
| 878 | nm_clone->nodes_by_type[VLIB_NODE_TYPE_INTERNAL] = |
Damjan Marion | be3f4d5 | 2018-03-27 21:06:10 +0200 | [diff] [blame] | 879 | vec_dup_aligned (nm->nodes_by_type[VLIB_NODE_TYPE_INTERNAL], |
| 880 | CLIB_CACHE_LINE_BYTES); |
JingLiuZTE | 30af5da | 2017-07-24 10:53:31 +0800 | [diff] [blame] | 881 | vec_foreach (rt, |
| 882 | nm_clone->nodes_by_type[VLIB_NODE_TYPE_INTERNAL]) |
Damjan Marion | e9f929b | 2017-03-16 11:32:09 +0100 | [diff] [blame] | 883 | { |
| 884 | vlib_node_t *n = vlib_get_node (vm, rt->node_index); |
Damjan Marion | 586afd7 | 2017-04-05 19:18:20 +0200 | [diff] [blame] | 885 | rt->thread_index = vm_clone->thread_index; |
Damjan Marion | e9f929b | 2017-03-16 11:32:09 +0100 | [diff] [blame] | 886 | /* copy initial runtime_data from node */ |
Damjan Marion | b6f93a1 | 2017-03-16 17:46:41 +0100 | [diff] [blame] | 887 | if (n->runtime_data && n->runtime_data_bytes > 0) |
Damjan Marion | e9f929b | 2017-03-16 11:32:09 +0100 | [diff] [blame] | 888 | clib_memcpy (rt->runtime_data, n->runtime_data, |
Damjan Marion | b6f93a1 | 2017-03-16 17:46:41 +0100 | [diff] [blame] | 889 | clib_min (VLIB_NODE_RUNTIME_DATA_SIZE, |
| 890 | n->runtime_data_bytes)); |
Damjan Marion | e9f929b | 2017-03-16 11:32:09 +0100 | [diff] [blame] | 891 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 892 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 893 | nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT] = |
Damjan Marion | be3f4d5 | 2018-03-27 21:06:10 +0200 | [diff] [blame] | 894 | vec_dup_aligned (nm->nodes_by_type[VLIB_NODE_TYPE_INPUT], |
| 895 | CLIB_CACHE_LINE_BYTES); |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 896 | vec_foreach (rt, nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT]) |
Damjan Marion | e9f929b | 2017-03-16 11:32:09 +0100 | [diff] [blame] | 897 | { |
| 898 | vlib_node_t *n = vlib_get_node (vm, rt->node_index); |
Damjan Marion | 586afd7 | 2017-04-05 19:18:20 +0200 | [diff] [blame] | 899 | rt->thread_index = vm_clone->thread_index; |
Damjan Marion | e9f929b | 2017-03-16 11:32:09 +0100 | [diff] [blame] | 900 | /* copy initial runtime_data from node */ |
Damjan Marion | b6f93a1 | 2017-03-16 17:46:41 +0100 | [diff] [blame] | 901 | if (n->runtime_data && n->runtime_data_bytes > 0) |
Damjan Marion | e9f929b | 2017-03-16 11:32:09 +0100 | [diff] [blame] | 902 | clib_memcpy (rt->runtime_data, n->runtime_data, |
Damjan Marion | b6f93a1 | 2017-03-16 17:46:41 +0100 | [diff] [blame] | 903 | clib_min (VLIB_NODE_RUNTIME_DATA_SIZE, |
| 904 | n->runtime_data_bytes)); |
Damjan Marion | e9f929b | 2017-03-16 11:32:09 +0100 | [diff] [blame] | 905 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 906 | |
Damjan Marion | be3f4d5 | 2018-03-27 21:06:10 +0200 | [diff] [blame] | 907 | nm_clone->processes = vec_dup_aligned (nm->processes, |
| 908 | CLIB_CACHE_LINE_BYTES); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 909 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 910 | /* zap the (per worker) frame freelists, etc */ |
| 911 | nm_clone->frame_sizes = 0; |
Florin Coras | 93992a9 | 2017-05-24 18:03:56 -0700 | [diff] [blame] | 912 | nm_clone->frame_size_hash = hash_create (0, sizeof (uword)); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 913 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 914 | /* Packet trace buffers are guaranteed to be empty, nothing to do here */ |
Damjan Marion | bc20bdf | 2015-12-17 14:28:18 +0100 | [diff] [blame] | 915 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 916 | clib_mem_set_heap (oldheap); |
Dave Barach | 80f54e2 | 2017-03-08 19:08:56 -0500 | [diff] [blame] | 917 | vec_add1_aligned (vlib_mains, vm_clone, CLIB_CACHE_LINE_BYTES); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 918 | |
Damjan Marion | be3f4d5 | 2018-03-27 21:06:10 +0200 | [diff] [blame] | 919 | vm_clone->error_main.counters = vec_dup_aligned |
| 920 | (vlib_mains[0]->error_main.counters, CLIB_CACHE_LINE_BYTES); |
| 921 | vm_clone->error_main.counters_last_clear = vec_dup_aligned |
| 922 | (vlib_mains[0]->error_main.counters_last_clear, |
| 923 | CLIB_CACHE_LINE_BYTES); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 924 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 925 | /* Fork the vlib_buffer_main_t free lists, etc. */ |
Damjan Marion | d1274cb | 2018-03-13 21:32:17 +0100 | [diff] [blame] | 926 | orig_freelist_pool = vm_clone->buffer_free_list_pool; |
| 927 | vm_clone->buffer_free_list_pool = 0; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 928 | |
| 929 | /* *INDENT-OFF* */ |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 930 | pool_foreach (fl_orig, orig_freelist_pool, |
| 931 | ({ |
Damjan Marion | d1274cb | 2018-03-13 21:32:17 +0100 | [diff] [blame] | 932 | pool_get_aligned (vm_clone->buffer_free_list_pool, |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 933 | fl_clone, CLIB_CACHE_LINE_BYTES); |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 934 | ASSERT (fl_orig - orig_freelist_pool |
Damjan Marion | d1274cb | 2018-03-13 21:32:17 +0100 | [diff] [blame] | 935 | == fl_clone - vm_clone->buffer_free_list_pool); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 936 | |
| 937 | fl_clone[0] = fl_orig[0]; |
Damjan Marion | bd69a5f | 2017-02-05 23:44:42 +0100 | [diff] [blame] | 938 | fl_clone->buffers = 0; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 939 | fl_clone->n_alloc = 0; |
| 940 | })); |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 941 | /* *INDENT-ON* */ |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 942 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 943 | worker_thread_index++; |
| 944 | } |
| 945 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 946 | } |
| 947 | else |
| 948 | { |
| 949 | /* only have non-data-structure copy threads to create... */ |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 950 | for (i = 0; i < vec_len (tm->registrations); i++) |
| 951 | { |
| 952 | tr = tm->registrations[i]; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 953 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 954 | for (j = 0; j < tr->count; j++) |
| 955 | { |
| 956 | vec_add2 (vlib_worker_threads, w, 1); |
| 957 | if (tr->mheap_size) |
| 958 | w->thread_mheap = |
| 959 | mheap_alloc (0 /* use VM */ , tr->mheap_size); |
| 960 | else |
| 961 | w->thread_mheap = main_heap; |
Damjan Marion | 586afd7 | 2017-04-05 19:18:20 +0200 | [diff] [blame] | 962 | w->thread_stack = |
| 963 | vlib_thread_stack_init (w - vlib_worker_threads); |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 964 | w->thread_function = tr->function; |
| 965 | w->thread_function_arg = w; |
| 966 | w->instance_id = j; |
| 967 | w->elog_track.name = |
| 968 | (char *) format (0, "%s %d", tr->name, j + 1); |
| 969 | w->registration = tr; |
| 970 | vec_add1 (w->elog_track.name, 0); |
| 971 | elog_track_register (&vm->elog_main, &w->elog_track); |
| 972 | } |
| 973 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 974 | } |
| 975 | |
| 976 | worker_thread_index = 1; |
| 977 | |
| 978 | for (i = 0; i < vec_len (tm->registrations); i++) |
| 979 | { |
Damjan Marion | 878c609 | 2017-01-04 13:19:27 +0100 | [diff] [blame] | 980 | clib_error_t *err; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 981 | int j; |
| 982 | |
| 983 | tr = tm->registrations[i]; |
| 984 | |
| 985 | if (tr->use_pthreads || tm->use_pthreads) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 986 | { |
| 987 | for (j = 0; j < tr->count; j++) |
| 988 | { |
| 989 | w = vlib_worker_threads + worker_thread_index++; |
Damjan Marion | 878c609 | 2017-01-04 13:19:27 +0100 | [diff] [blame] | 990 | err = vlib_launch_thread_int (vlib_worker_thread_bootstrap_fn, |
| 991 | w, 0); |
| 992 | if (err) |
| 993 | clib_error_report (err); |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 994 | } |
| 995 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 996 | else |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 997 | { |
| 998 | uword c; |
Damjan Marion | 878c609 | 2017-01-04 13:19:27 +0100 | [diff] [blame] | 999 | /* *INDENT-OFF* */ |
| 1000 | clib_bitmap_foreach (c, tr->coremask, ({ |
| 1001 | w = vlib_worker_threads + worker_thread_index++; |
| 1002 | err = vlib_launch_thread_int (vlib_worker_thread_bootstrap_fn, |
| 1003 | w, c); |
| 1004 | if (err) |
| 1005 | clib_error_report (err); |
| 1006 | })); |
| 1007 | /* *INDENT-ON* */ |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1008 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1009 | } |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1010 | vlib_worker_thread_barrier_sync (vm); |
| 1011 | vlib_worker_thread_barrier_release (vm); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1012 | return 0; |
| 1013 | } |
| 1014 | |
| 1015 | VLIB_MAIN_LOOP_ENTER_FUNCTION (start_workers); |
| 1016 | |
Colin Tregenza Dancer | eb1ac17 | 2017-09-06 20:23:24 +0100 | [diff] [blame] | 1017 | |
Colin Tregenza Dancer | 2159618 | 2017-09-04 15:27:49 +0100 | [diff] [blame] | 1018 | static inline void |
| 1019 | worker_thread_node_runtime_update_internal (void) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1020 | { |
| 1021 | int i, j; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1022 | vlib_main_t *vm; |
| 1023 | vlib_node_main_t *nm, *nm_clone; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1024 | vlib_main_t *vm_clone; |
Colin Tregenza Dancer | 2159618 | 2017-09-04 15:27:49 +0100 | [diff] [blame] | 1025 | vlib_node_runtime_t *rt; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1026 | never_inline void |
| 1027 | vlib_node_runtime_sync_stats (vlib_main_t * vm, |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1028 | vlib_node_runtime_t * r, |
| 1029 | uword n_calls, |
| 1030 | uword n_vectors, uword n_clocks); |
| 1031 | |
Damjan Marion | 586afd7 | 2017-04-05 19:18:20 +0200 | [diff] [blame] | 1032 | ASSERT (vlib_get_thread_index () == 0); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1033 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1034 | vm = vlib_mains[0]; |
| 1035 | nm = &vm->node_main; |
| 1036 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1037 | ASSERT (*vlib_worker_threads->wait_at_barrier == 1); |
| 1038 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1039 | /* |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1040 | * Scrape all runtime stats, so we don't lose node runtime(s) with |
| 1041 | * pending counts, or throw away worker / io thread counts. |
| 1042 | */ |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1043 | for (j = 0; j < vec_len (nm->nodes); j++) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1044 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1045 | vlib_node_t *n; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1046 | n = nm->nodes[j]; |
| 1047 | vlib_node_sync_stats (vm, n); |
| 1048 | } |
| 1049 | |
| 1050 | for (i = 1; i < vec_len (vlib_mains); i++) |
| 1051 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1052 | vlib_node_t *n; |
| 1053 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1054 | vm_clone = vlib_mains[i]; |
| 1055 | nm_clone = &vm_clone->node_main; |
| 1056 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1057 | for (j = 0; j < vec_len (nm_clone->nodes); j++) |
| 1058 | { |
| 1059 | n = nm_clone->nodes[j]; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1060 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1061 | rt = vlib_node_get_runtime (vm_clone, n->index); |
| 1062 | vlib_node_runtime_sync_stats (vm_clone, rt, 0, 0, 0); |
| 1063 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1064 | } |
| 1065 | |
Colin Tregenza Dancer | 2159618 | 2017-09-04 15:27:49 +0100 | [diff] [blame] | 1066 | /* Per-worker clone rebuilds are now done on each thread */ |
| 1067 | } |
| 1068 | |
| 1069 | |
| 1070 | void |
| 1071 | vlib_worker_thread_node_refork (void) |
| 1072 | { |
| 1073 | vlib_main_t *vm, *vm_clone; |
| 1074 | vlib_node_main_t *nm, *nm_clone; |
| 1075 | vlib_node_t **old_nodes_clone; |
| 1076 | vlib_node_runtime_t *rt, *old_rt; |
| 1077 | |
| 1078 | vlib_node_t *new_n_clone; |
| 1079 | |
| 1080 | int j; |
| 1081 | |
| 1082 | vm = vlib_mains[0]; |
| 1083 | nm = &vm->node_main; |
| 1084 | vm_clone = vlib_get_main (); |
| 1085 | nm_clone = &vm_clone->node_main; |
| 1086 | |
| 1087 | /* Re-clone error heap */ |
| 1088 | u64 *old_counters = vm_clone->error_main.counters; |
| 1089 | u64 *old_counters_all_clear = vm_clone->error_main.counters_last_clear; |
| 1090 | |
| 1091 | clib_memcpy (&vm_clone->error_main, &vm->error_main, |
| 1092 | sizeof (vm->error_main)); |
| 1093 | j = vec_len (vm->error_main.counters) - 1; |
| 1094 | vec_validate_aligned (old_counters, j, CLIB_CACHE_LINE_BYTES); |
| 1095 | vec_validate_aligned (old_counters_all_clear, j, CLIB_CACHE_LINE_BYTES); |
| 1096 | vm_clone->error_main.counters = old_counters; |
| 1097 | vm_clone->error_main.counters_last_clear = old_counters_all_clear; |
| 1098 | |
| 1099 | nm_clone = &vm_clone->node_main; |
| 1100 | vec_free (nm_clone->next_frames); |
Damjan Marion | be3f4d5 | 2018-03-27 21:06:10 +0200 | [diff] [blame] | 1101 | nm_clone->next_frames = vec_dup_aligned (nm->next_frames, |
| 1102 | CLIB_CACHE_LINE_BYTES); |
Colin Tregenza Dancer | 2159618 | 2017-09-04 15:27:49 +0100 | [diff] [blame] | 1103 | |
| 1104 | for (j = 0; j < vec_len (nm_clone->next_frames); j++) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1105 | { |
Colin Tregenza Dancer | 2159618 | 2017-09-04 15:27:49 +0100 | [diff] [blame] | 1106 | vlib_next_frame_t *nf = &nm_clone->next_frames[j]; |
| 1107 | u32 save_node_runtime_index; |
| 1108 | u32 save_flags; |
Damjan Marion | bc20bdf | 2015-12-17 14:28:18 +0100 | [diff] [blame] | 1109 | |
Colin Tregenza Dancer | 2159618 | 2017-09-04 15:27:49 +0100 | [diff] [blame] | 1110 | save_node_runtime_index = nf->node_runtime_index; |
| 1111 | save_flags = nf->flags & VLIB_FRAME_NO_FREE_AFTER_DISPATCH; |
| 1112 | vlib_next_frame_init (nf); |
| 1113 | nf->node_runtime_index = save_node_runtime_index; |
| 1114 | nf->flags = save_flags; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1115 | } |
Colin Tregenza Dancer | 2159618 | 2017-09-04 15:27:49 +0100 | [diff] [blame] | 1116 | |
| 1117 | old_nodes_clone = nm_clone->nodes; |
| 1118 | nm_clone->nodes = 0; |
| 1119 | |
| 1120 | /* re-fork nodes */ |
| 1121 | |
| 1122 | /* Allocate all nodes in single block for speed */ |
| 1123 | new_n_clone = |
| 1124 | clib_mem_alloc_no_fail (vec_len (nm->nodes) * sizeof (*new_n_clone)); |
| 1125 | for (j = 0; j < vec_len (nm->nodes); j++) |
| 1126 | { |
| 1127 | vlib_node_t *old_n_clone; |
| 1128 | vlib_node_t *new_n; |
| 1129 | |
| 1130 | new_n = nm->nodes[j]; |
| 1131 | old_n_clone = old_nodes_clone[j]; |
| 1132 | |
| 1133 | clib_memcpy (new_n_clone, new_n, sizeof (*new_n)); |
| 1134 | /* none of the copied nodes have enqueue rights given out */ |
| 1135 | new_n_clone->owner_node_index = VLIB_INVALID_NODE_INDEX; |
| 1136 | |
| 1137 | if (j >= vec_len (old_nodes_clone)) |
| 1138 | { |
| 1139 | /* new node, set to zero */ |
| 1140 | memset (&new_n_clone->stats_total, 0, |
| 1141 | sizeof (new_n_clone->stats_total)); |
| 1142 | memset (&new_n_clone->stats_last_clear, 0, |
| 1143 | sizeof (new_n_clone->stats_last_clear)); |
| 1144 | } |
| 1145 | else |
| 1146 | { |
| 1147 | /* Copy stats if the old data is valid */ |
| 1148 | clib_memcpy (&new_n_clone->stats_total, |
| 1149 | &old_n_clone->stats_total, |
| 1150 | sizeof (new_n_clone->stats_total)); |
| 1151 | clib_memcpy (&new_n_clone->stats_last_clear, |
| 1152 | &old_n_clone->stats_last_clear, |
| 1153 | sizeof (new_n_clone->stats_last_clear)); |
| 1154 | |
| 1155 | /* keep previous node state */ |
| 1156 | new_n_clone->state = old_n_clone->state; |
| 1157 | } |
| 1158 | vec_add1 (nm_clone->nodes, new_n_clone); |
| 1159 | new_n_clone++; |
| 1160 | } |
| 1161 | /* Free the old node clones */ |
| 1162 | clib_mem_free (old_nodes_clone[0]); |
| 1163 | |
| 1164 | vec_free (old_nodes_clone); |
| 1165 | |
| 1166 | |
| 1167 | /* re-clone internal nodes */ |
| 1168 | old_rt = nm_clone->nodes_by_type[VLIB_NODE_TYPE_INTERNAL]; |
| 1169 | nm_clone->nodes_by_type[VLIB_NODE_TYPE_INTERNAL] = |
Damjan Marion | be3f4d5 | 2018-03-27 21:06:10 +0200 | [diff] [blame] | 1170 | vec_dup_aligned (nm->nodes_by_type[VLIB_NODE_TYPE_INTERNAL], |
| 1171 | CLIB_CACHE_LINE_BYTES); |
Colin Tregenza Dancer | 2159618 | 2017-09-04 15:27:49 +0100 | [diff] [blame] | 1172 | |
| 1173 | vec_foreach (rt, nm_clone->nodes_by_type[VLIB_NODE_TYPE_INTERNAL]) |
| 1174 | { |
| 1175 | vlib_node_t *n = vlib_get_node (vm, rt->node_index); |
| 1176 | rt->thread_index = vm_clone->thread_index; |
| 1177 | /* copy runtime_data, will be overwritten later for existing rt */ |
| 1178 | if (n->runtime_data && n->runtime_data_bytes > 0) |
| 1179 | clib_memcpy (rt->runtime_data, n->runtime_data, |
| 1180 | clib_min (VLIB_NODE_RUNTIME_DATA_SIZE, |
| 1181 | n->runtime_data_bytes)); |
| 1182 | } |
| 1183 | |
| 1184 | for (j = 0; j < vec_len (old_rt); j++) |
| 1185 | { |
| 1186 | rt = vlib_node_get_runtime (vm_clone, old_rt[j].node_index); |
| 1187 | rt->state = old_rt[j].state; |
| 1188 | clib_memcpy (rt->runtime_data, old_rt[j].runtime_data, |
| 1189 | VLIB_NODE_RUNTIME_DATA_SIZE); |
| 1190 | } |
| 1191 | |
| 1192 | vec_free (old_rt); |
| 1193 | |
| 1194 | /* re-clone input nodes */ |
| 1195 | old_rt = nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT]; |
| 1196 | nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT] = |
Damjan Marion | be3f4d5 | 2018-03-27 21:06:10 +0200 | [diff] [blame] | 1197 | vec_dup_aligned (nm->nodes_by_type[VLIB_NODE_TYPE_INPUT], |
| 1198 | CLIB_CACHE_LINE_BYTES); |
Colin Tregenza Dancer | 2159618 | 2017-09-04 15:27:49 +0100 | [diff] [blame] | 1199 | |
| 1200 | vec_foreach (rt, nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT]) |
| 1201 | { |
| 1202 | vlib_node_t *n = vlib_get_node (vm, rt->node_index); |
| 1203 | rt->thread_index = vm_clone->thread_index; |
| 1204 | /* copy runtime_data, will be overwritten later for existing rt */ |
| 1205 | if (n->runtime_data && n->runtime_data_bytes > 0) |
| 1206 | clib_memcpy (rt->runtime_data, n->runtime_data, |
| 1207 | clib_min (VLIB_NODE_RUNTIME_DATA_SIZE, |
| 1208 | n->runtime_data_bytes)); |
| 1209 | } |
| 1210 | |
| 1211 | for (j = 0; j < vec_len (old_rt); j++) |
| 1212 | { |
| 1213 | rt = vlib_node_get_runtime (vm_clone, old_rt[j].node_index); |
| 1214 | rt->state = old_rt[j].state; |
| 1215 | clib_memcpy (rt->runtime_data, old_rt[j].runtime_data, |
| 1216 | VLIB_NODE_RUNTIME_DATA_SIZE); |
| 1217 | } |
| 1218 | |
| 1219 | vec_free (old_rt); |
| 1220 | |
Damjan Marion | be3f4d5 | 2018-03-27 21:06:10 +0200 | [diff] [blame] | 1221 | nm_clone->processes = vec_dup_aligned (nm->processes, |
| 1222 | CLIB_CACHE_LINE_BYTES); |
Colin Tregenza Dancer | 2159618 | 2017-09-04 15:27:49 +0100 | [diff] [blame] | 1223 | } |
| 1224 | |
Colin Tregenza Dancer | 2159618 | 2017-09-04 15:27:49 +0100 | [diff] [blame] | 1225 | void |
| 1226 | vlib_worker_thread_node_runtime_update (void) |
| 1227 | { |
| 1228 | /* |
| 1229 | * Make a note that we need to do a node runtime update |
| 1230 | * prior to releasing the barrier. |
| 1231 | */ |
| 1232 | vlib_global_main.need_vlib_worker_thread_node_runtime_update = 1; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1233 | } |
| 1234 | |
Pavel Kotucek | 1e76583 | 2016-09-23 08:54:14 +0200 | [diff] [blame] | 1235 | u32 |
| 1236 | unformat_sched_policy (unformat_input_t * input, va_list * args) |
| 1237 | { |
| 1238 | u32 *r = va_arg (*args, u32 *); |
| 1239 | |
| 1240 | if (0); |
| 1241 | #define _(v,f,s) else if (unformat (input, s)) *r = SCHED_POLICY_##f; |
| 1242 | foreach_sched_policy |
| 1243 | #undef _ |
| 1244 | else |
| 1245 | return 0; |
| 1246 | return 1; |
| 1247 | } |
| 1248 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1249 | static clib_error_t * |
| 1250 | cpu_config (vlib_main_t * vm, unformat_input_t * input) |
| 1251 | { |
| 1252 | vlib_thread_registration_t *tr; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1253 | uword *p; |
| 1254 | vlib_thread_main_t *tm = &vlib_thread_main; |
| 1255 | u8 *name; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1256 | u64 coremask; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1257 | uword *bitmap; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1258 | u32 count; |
| 1259 | |
| 1260 | tm->thread_registrations_by_name = hash_create_string (0, sizeof (uword)); |
Pavel Kotucek | 1e76583 | 2016-09-23 08:54:14 +0200 | [diff] [blame] | 1261 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1262 | tm->n_thread_stacks = 1; /* account for main thread */ |
Pavel Kotucek | 1e76583 | 2016-09-23 08:54:14 +0200 | [diff] [blame] | 1263 | tm->sched_policy = ~0; |
| 1264 | tm->sched_priority = ~0; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1265 | |
| 1266 | tr = tm->next; |
| 1267 | |
| 1268 | while (tr) |
| 1269 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1270 | hash_set_mem (tm->thread_registrations_by_name, tr->name, (uword) tr); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1271 | tr = tr->next; |
| 1272 | } |
| 1273 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1274 | while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1275 | { |
Damjan Marion | bf74147 | 2016-06-13 22:49:44 +0200 | [diff] [blame] | 1276 | if (unformat (input, "use-pthreads")) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1277 | tm->use_pthreads = 1; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1278 | else if (unformat (input, "thread-prefix %v", &tm->thread_prefix)) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1279 | ; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1280 | else if (unformat (input, "main-core %u", &tm->main_lcore)) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1281 | ; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1282 | else if (unformat (input, "skip-cores %u", &tm->skip_cores)) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1283 | ; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1284 | else if (unformat (input, "coremask-%s %llx", &name, &coremask)) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1285 | { |
| 1286 | p = hash_get_mem (tm->thread_registrations_by_name, name); |
| 1287 | if (p == 0) |
| 1288 | return clib_error_return (0, "no such thread type '%s'", name); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1289 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1290 | tr = (vlib_thread_registration_t *) p[0]; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1291 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1292 | if (tr->use_pthreads) |
| 1293 | return clib_error_return (0, |
| 1294 | "coremask cannot be set for '%s' threads", |
| 1295 | name); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1296 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1297 | tr->coremask = clib_bitmap_set_multiple |
| 1298 | (tr->coremask, 0, coremask, BITS (coremask)); |
| 1299 | tr->count = clib_bitmap_count_set_bits (tr->coremask); |
| 1300 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1301 | else if (unformat (input, "corelist-%s %U", &name, unformat_bitmap_list, |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1302 | &bitmap)) |
| 1303 | { |
| 1304 | p = hash_get_mem (tm->thread_registrations_by_name, name); |
| 1305 | if (p == 0) |
| 1306 | return clib_error_return (0, "no such thread type '%s'", name); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1307 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1308 | tr = (vlib_thread_registration_t *) p[0]; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1309 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1310 | if (tr->use_pthreads) |
| 1311 | return clib_error_return (0, |
| 1312 | "corelist cannot be set for '%s' threads", |
| 1313 | name); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1314 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1315 | tr->coremask = bitmap; |
| 1316 | tr->count = clib_bitmap_count_set_bits (tr->coremask); |
| 1317 | } |
Pavel Kotucek | 1e76583 | 2016-09-23 08:54:14 +0200 | [diff] [blame] | 1318 | else |
| 1319 | if (unformat |
| 1320 | (input, "scheduler-policy %U", unformat_sched_policy, |
| 1321 | &tm->sched_policy)) |
| 1322 | ; |
Pavel Kotucek | c08a1ed | 2016-09-23 08:54:14 +0200 | [diff] [blame] | 1323 | else if (unformat (input, "scheduler-priority %u", &tm->sched_priority)) |
Pavel Kotucek | 1e76583 | 2016-09-23 08:54:14 +0200 | [diff] [blame] | 1324 | ; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1325 | else if (unformat (input, "%s %u", &name, &count)) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1326 | { |
| 1327 | p = hash_get_mem (tm->thread_registrations_by_name, name); |
| 1328 | if (p == 0) |
Pavel Kotucek | 1e76583 | 2016-09-23 08:54:14 +0200 | [diff] [blame] | 1329 | return clib_error_return (0, "no such thread type 3 '%s'", name); |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1330 | |
| 1331 | tr = (vlib_thread_registration_t *) p[0]; |
| 1332 | if (tr->fixed_count) |
| 1333 | return clib_error_return |
| 1334 | (0, "number of %s threads not configurable", tr->name); |
| 1335 | tr->count = count; |
| 1336 | } |
| 1337 | else |
| 1338 | break; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1339 | } |
| 1340 | |
Pavel Kotucek | c08a1ed | 2016-09-23 08:54:14 +0200 | [diff] [blame] | 1341 | if (tm->sched_priority != ~0) |
Pavel Kotucek | 1e76583 | 2016-09-23 08:54:14 +0200 | [diff] [blame] | 1342 | { |
Pavel Kotucek | c08a1ed | 2016-09-23 08:54:14 +0200 | [diff] [blame] | 1343 | if (tm->sched_policy == SCHED_FIFO || tm->sched_policy == SCHED_RR) |
Pavel Kotucek | 1e76583 | 2016-09-23 08:54:14 +0200 | [diff] [blame] | 1344 | { |
| 1345 | u32 prio_max = sched_get_priority_max (tm->sched_policy); |
| 1346 | u32 prio_min = sched_get_priority_min (tm->sched_policy); |
| 1347 | if (tm->sched_priority > prio_max) |
| 1348 | tm->sched_priority = prio_max; |
| 1349 | if (tm->sched_priority < prio_min) |
| 1350 | tm->sched_priority = prio_min; |
| 1351 | } |
| 1352 | else |
Pavel Kotucek | c08a1ed | 2016-09-23 08:54:14 +0200 | [diff] [blame] | 1353 | { |
| 1354 | return clib_error_return |
| 1355 | (0, |
| 1356 | "scheduling priority (%d) is not allowed for `normal` scheduling policy", |
| 1357 | tm->sched_priority); |
| 1358 | } |
Pavel Kotucek | 1e76583 | 2016-09-23 08:54:14 +0200 | [diff] [blame] | 1359 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1360 | tr = tm->next; |
| 1361 | |
| 1362 | if (!tm->thread_prefix) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1363 | tm->thread_prefix = format (0, "vpp"); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1364 | |
| 1365 | while (tr) |
| 1366 | { |
| 1367 | tm->n_thread_stacks += tr->count; |
| 1368 | tm->n_pthreads += tr->count * tr->use_pthreads; |
Damjan Marion | 878c609 | 2017-01-04 13:19:27 +0100 | [diff] [blame] | 1369 | tm->n_threads += tr->count * (tr->use_pthreads == 0); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1370 | tr = tr->next; |
| 1371 | } |
| 1372 | |
| 1373 | return 0; |
| 1374 | } |
| 1375 | |
| 1376 | VLIB_EARLY_CONFIG_FUNCTION (cpu_config, "cpu"); |
| 1377 | |
Damjan Marion | 7bee80c | 2017-04-26 15:32:12 +0200 | [diff] [blame] | 1378 | #if !defined (__x86_64__) && !defined (__i386__) && !defined (__aarch64__) && !defined (__powerpc64__) && !defined(__arm__) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1379 | void |
| 1380 | __sync_fetch_and_add_8 (void) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1381 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1382 | fformat (stderr, "%s called\n", __FUNCTION__); |
| 1383 | abort (); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1384 | } |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1385 | |
| 1386 | void |
| 1387 | __sync_add_and_fetch_8 (void) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1388 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1389 | fformat (stderr, "%s called\n", __FUNCTION__); |
| 1390 | abort (); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1391 | } |
| 1392 | #endif |
| 1393 | |
| 1394 | void vnet_main_fixup (vlib_fork_fixup_t which) __attribute__ ((weak)); |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1395 | void |
| 1396 | vnet_main_fixup (vlib_fork_fixup_t which) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1397 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1398 | } |
| 1399 | |
| 1400 | void |
| 1401 | vlib_worker_thread_fork_fixup (vlib_fork_fixup_t which) |
| 1402 | { |
| 1403 | vlib_main_t *vm = vlib_get_main (); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1404 | |
| 1405 | if (vlib_mains == 0) |
| 1406 | return; |
| 1407 | |
Damjan Marion | 586afd7 | 2017-04-05 19:18:20 +0200 | [diff] [blame] | 1408 | ASSERT (vlib_get_thread_index () == 0); |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1409 | vlib_worker_thread_barrier_sync (vm); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1410 | |
| 1411 | switch (which) |
| 1412 | { |
| 1413 | case VLIB_WORKER_THREAD_FORK_FIXUP_NEW_SW_IF_INDEX: |
| 1414 | vnet_main_fixup (VLIB_WORKER_THREAD_FORK_FIXUP_NEW_SW_IF_INDEX); |
| 1415 | break; |
| 1416 | |
| 1417 | default: |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1418 | ASSERT (0); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1419 | } |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1420 | vlib_worker_thread_barrier_release (vm); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1421 | } |
| 1422 | |
Colin Tregenza Dancer | eb1ac17 | 2017-09-06 20:23:24 +0100 | [diff] [blame] | 1423 | /* |
| 1424 | * Enforce minimum open time to minimize packet loss due to Rx overflow, |
| 1425 | * based on a test based heuristic that barrier should be open for at least |
| 1426 | * 3 time as long as it is closed (with an upper bound of 1ms because by that |
| 1427 | * point it is probably too late to make a difference) |
| 1428 | */ |
| 1429 | |
| 1430 | #ifndef BARRIER_MINIMUM_OPEN_LIMIT |
| 1431 | #define BARRIER_MINIMUM_OPEN_LIMIT 0.001 |
| 1432 | #endif |
| 1433 | |
| 1434 | #ifndef BARRIER_MINIMUM_OPEN_FACTOR |
| 1435 | #define BARRIER_MINIMUM_OPEN_FACTOR 3 |
| 1436 | #endif |
| 1437 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1438 | void |
Colin Tregenza Dancer | eb1ac17 | 2017-09-06 20:23:24 +0100 | [diff] [blame] | 1439 | vlib_worker_thread_barrier_sync_int (vlib_main_t * vm) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1440 | { |
| 1441 | f64 deadline; |
Colin Tregenza Dancer | eb1ac17 | 2017-09-06 20:23:24 +0100 | [diff] [blame] | 1442 | f64 now; |
| 1443 | f64 t_entry; |
| 1444 | f64 t_open; |
| 1445 | f64 t_closed; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1446 | u32 count; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1447 | |
Dave Barach | 80f54e2 | 2017-03-08 19:08:56 -0500 | [diff] [blame] | 1448 | if (vec_len (vlib_mains) < 2) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1449 | return; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1450 | |
Colin Tregenza Dancer | 2159618 | 2017-09-04 15:27:49 +0100 | [diff] [blame] | 1451 | ASSERT (vlib_get_thread_index () == 0); |
| 1452 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1453 | count = vec_len (vlib_mains) - 1; |
| 1454 | |
Colin Tregenza Dancer | eb1ac17 | 2017-09-06 20:23:24 +0100 | [diff] [blame] | 1455 | /* Record entry relative to last close */ |
| 1456 | now = vlib_time_now (vm); |
| 1457 | t_entry = now - vm->barrier_epoch; |
| 1458 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1459 | /* Tolerate recursive calls */ |
| 1460 | if (++vlib_worker_threads[0].recursion_level > 1) |
Colin Tregenza Dancer | eb1ac17 | 2017-09-06 20:23:24 +0100 | [diff] [blame] | 1461 | { |
| 1462 | barrier_trace_sync_rec (t_entry); |
| 1463 | return; |
| 1464 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1465 | |
Bud Grise | 42f2006 | 2016-03-16 13:09:46 -0400 | [diff] [blame] | 1466 | vlib_worker_threads[0].barrier_sync_count++; |
| 1467 | |
Colin Tregenza Dancer | eb1ac17 | 2017-09-06 20:23:24 +0100 | [diff] [blame] | 1468 | /* Enforce minimum barrier open time to minimize packet loss */ |
| 1469 | ASSERT (vm->barrier_no_close_before <= (now + BARRIER_MINIMUM_OPEN_LIMIT)); |
| 1470 | while ((now = vlib_time_now (vm)) < vm->barrier_no_close_before) |
| 1471 | ; |
| 1472 | |
| 1473 | /* Record time of closure */ |
| 1474 | t_open = now - vm->barrier_epoch; |
| 1475 | vm->barrier_epoch = now; |
| 1476 | |
| 1477 | deadline = now + BARRIER_SYNC_TIMEOUT; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1478 | |
| 1479 | *vlib_worker_threads->wait_at_barrier = 1; |
| 1480 | while (*vlib_worker_threads->workers_at_barrier != count) |
| 1481 | { |
Colin Tregenza Dancer | eb1ac17 | 2017-09-06 20:23:24 +0100 | [diff] [blame] | 1482 | if ((now = vlib_time_now (vm)) > deadline) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1483 | { |
| 1484 | fformat (stderr, "%s: worker thread deadlock\n", __FUNCTION__); |
| 1485 | os_panic (); |
| 1486 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1487 | } |
Colin Tregenza Dancer | eb1ac17 | 2017-09-06 20:23:24 +0100 | [diff] [blame] | 1488 | |
| 1489 | t_closed = now - vm->barrier_epoch; |
| 1490 | |
| 1491 | barrier_trace_sync (t_entry, t_open, t_closed); |
| 1492 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1493 | } |
| 1494 | |
Dave Barach | 1ddbc01 | 2018-06-13 09:26:05 -0400 | [diff] [blame] | 1495 | void vlib_stat_segment_lock (void) __attribute__ ((weak)); |
| 1496 | void |
| 1497 | vlib_stat_segment_lock (void) |
| 1498 | { |
| 1499 | } |
| 1500 | |
| 1501 | void vlib_stat_segment_unlock (void) __attribute__ ((weak)); |
| 1502 | void |
| 1503 | vlib_stat_segment_unlock (void) |
| 1504 | { |
| 1505 | } |
| 1506 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1507 | void |
| 1508 | vlib_worker_thread_barrier_release (vlib_main_t * vm) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1509 | { |
| 1510 | f64 deadline; |
Colin Tregenza Dancer | eb1ac17 | 2017-09-06 20:23:24 +0100 | [diff] [blame] | 1511 | f64 now; |
| 1512 | f64 minimum_open; |
| 1513 | f64 t_entry; |
| 1514 | f64 t_closed_total; |
| 1515 | f64 t_update_main = 0.0; |
Colin Tregenza Dancer | 2159618 | 2017-09-04 15:27:49 +0100 | [diff] [blame] | 1516 | int refork_needed = 0; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1517 | |
Dave Barach | 80f54e2 | 2017-03-08 19:08:56 -0500 | [diff] [blame] | 1518 | if (vec_len (vlib_mains) < 2) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1519 | return; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1520 | |
Colin Tregenza Dancer | 2159618 | 2017-09-04 15:27:49 +0100 | [diff] [blame] | 1521 | ASSERT (vlib_get_thread_index () == 0); |
| 1522 | |
Colin Tregenza Dancer | eb1ac17 | 2017-09-06 20:23:24 +0100 | [diff] [blame] | 1523 | |
| 1524 | now = vlib_time_now (vm); |
| 1525 | t_entry = now - vm->barrier_epoch; |
| 1526 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1527 | if (--vlib_worker_threads[0].recursion_level > 0) |
Colin Tregenza Dancer | eb1ac17 | 2017-09-06 20:23:24 +0100 | [diff] [blame] | 1528 | { |
| 1529 | barrier_trace_release_rec (t_entry); |
| 1530 | return; |
| 1531 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1532 | |
Colin Tregenza Dancer | 2159618 | 2017-09-04 15:27:49 +0100 | [diff] [blame] | 1533 | /* Update (all) node runtimes before releasing the barrier, if needed */ |
| 1534 | if (vm->need_vlib_worker_thread_node_runtime_update) |
| 1535 | { |
Dave Barach | 1ddbc01 | 2018-06-13 09:26:05 -0400 | [diff] [blame] | 1536 | /* |
| 1537 | * Lock stat segment here, so we's safe when |
| 1538 | * rebuilding the stat segment node clones from the |
| 1539 | * stat thread... |
| 1540 | */ |
| 1541 | vlib_stat_segment_lock (); |
| 1542 | |
Colin Tregenza Dancer | 2159618 | 2017-09-04 15:27:49 +0100 | [diff] [blame] | 1543 | /* Do stats elements on main thread */ |
| 1544 | worker_thread_node_runtime_update_internal (); |
| 1545 | vm->need_vlib_worker_thread_node_runtime_update = 0; |
| 1546 | |
| 1547 | /* Do per thread rebuilds in parallel */ |
| 1548 | refork_needed = 1; |
| 1549 | clib_smp_atomic_add (vlib_worker_threads->node_reforks_required, |
| 1550 | (vec_len (vlib_mains) - 1)); |
Colin Tregenza Dancer | eb1ac17 | 2017-09-06 20:23:24 +0100 | [diff] [blame] | 1551 | now = vlib_time_now (vm); |
| 1552 | t_update_main = now - vm->barrier_epoch; |
Colin Tregenza Dancer | 2159618 | 2017-09-04 15:27:49 +0100 | [diff] [blame] | 1553 | } |
| 1554 | |
Colin Tregenza Dancer | eb1ac17 | 2017-09-06 20:23:24 +0100 | [diff] [blame] | 1555 | deadline = now + BARRIER_SYNC_TIMEOUT; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1556 | |
| 1557 | *vlib_worker_threads->wait_at_barrier = 0; |
| 1558 | |
| 1559 | while (*vlib_worker_threads->workers_at_barrier > 0) |
| 1560 | { |
Colin Tregenza Dancer | eb1ac17 | 2017-09-06 20:23:24 +0100 | [diff] [blame] | 1561 | if ((now = vlib_time_now (vm)) > deadline) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1562 | { |
| 1563 | fformat (stderr, "%s: worker thread deadlock\n", __FUNCTION__); |
| 1564 | os_panic (); |
| 1565 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1566 | } |
Colin Tregenza Dancer | 2159618 | 2017-09-04 15:27:49 +0100 | [diff] [blame] | 1567 | |
| 1568 | /* Wait for reforks before continuing */ |
| 1569 | if (refork_needed) |
| 1570 | { |
Colin Tregenza Dancer | eb1ac17 | 2017-09-06 20:23:24 +0100 | [diff] [blame] | 1571 | now = vlib_time_now (vm); |
| 1572 | |
| 1573 | deadline = now + BARRIER_SYNC_TIMEOUT; |
Colin Tregenza Dancer | 2159618 | 2017-09-04 15:27:49 +0100 | [diff] [blame] | 1574 | |
| 1575 | while (*vlib_worker_threads->node_reforks_required > 0) |
| 1576 | { |
Colin Tregenza Dancer | eb1ac17 | 2017-09-06 20:23:24 +0100 | [diff] [blame] | 1577 | if ((now = vlib_time_now (vm)) > deadline) |
Colin Tregenza Dancer | 2159618 | 2017-09-04 15:27:49 +0100 | [diff] [blame] | 1578 | { |
| 1579 | fformat (stderr, "%s: worker thread refork deadlock\n", |
| 1580 | __FUNCTION__); |
| 1581 | os_panic (); |
| 1582 | } |
| 1583 | } |
Dave Barach | 1ddbc01 | 2018-06-13 09:26:05 -0400 | [diff] [blame] | 1584 | vlib_stat_segment_unlock (); |
Colin Tregenza Dancer | 2159618 | 2017-09-04 15:27:49 +0100 | [diff] [blame] | 1585 | } |
Colin Tregenza Dancer | eb1ac17 | 2017-09-06 20:23:24 +0100 | [diff] [blame] | 1586 | |
| 1587 | t_closed_total = now - vm->barrier_epoch; |
| 1588 | |
| 1589 | minimum_open = t_closed_total * BARRIER_MINIMUM_OPEN_FACTOR; |
| 1590 | |
| 1591 | if (minimum_open > BARRIER_MINIMUM_OPEN_LIMIT) |
| 1592 | { |
| 1593 | minimum_open = BARRIER_MINIMUM_OPEN_LIMIT; |
| 1594 | } |
| 1595 | |
| 1596 | vm->barrier_no_close_before = now + minimum_open; |
| 1597 | |
| 1598 | /* Record barrier epoch (used to enforce minimum open time) */ |
| 1599 | vm->barrier_epoch = now; |
| 1600 | |
| 1601 | barrier_trace_release (t_entry, t_closed_total, t_update_main); |
| 1602 | |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1603 | } |
| 1604 | |
Damjan Marion | 0f8ecf0 | 2016-06-27 08:30:30 +0200 | [diff] [blame] | 1605 | /* |
| 1606 | * Check the frame queue to see if any frames are available. |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1607 | * If so, pull the packets off the frames and put them to |
Damjan Marion | 0f8ecf0 | 2016-06-27 08:30:30 +0200 | [diff] [blame] | 1608 | * the handoff node. |
| 1609 | */ |
Damjan Marion | e9d52d5 | 2017-03-09 15:42:26 +0100 | [diff] [blame] | 1610 | int |
| 1611 | vlib_frame_queue_dequeue (vlib_main_t * vm, vlib_frame_queue_main_t * fqm) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1612 | { |
Damjan Marion | 586afd7 | 2017-04-05 19:18:20 +0200 | [diff] [blame] | 1613 | u32 thread_id = vm->thread_index; |
Damjan Marion | aaef1eb | 2016-11-08 17:37:01 +0100 | [diff] [blame] | 1614 | vlib_frame_queue_t *fq = fqm->vlib_frame_queues[thread_id]; |
Damjan Marion | 0f8ecf0 | 2016-06-27 08:30:30 +0200 | [diff] [blame] | 1615 | vlib_frame_queue_elt_t *elt; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1616 | u32 *from, *to; |
| 1617 | vlib_frame_t *f; |
Damjan Marion | 0f8ecf0 | 2016-06-27 08:30:30 +0200 | [diff] [blame] | 1618 | int msg_type; |
| 1619 | int processed = 0; |
| 1620 | u32 n_left_to_node; |
| 1621 | u32 vectors = 0; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1622 | |
Damjan Marion | 0f8ecf0 | 2016-06-27 08:30:30 +0200 | [diff] [blame] | 1623 | ASSERT (fq); |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1624 | ASSERT (vm == vlib_mains[thread_id]); |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1625 | |
Damjan Marion | aaef1eb | 2016-11-08 17:37:01 +0100 | [diff] [blame] | 1626 | if (PREDICT_FALSE (fqm->node_index == ~0)) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1627 | return 0; |
Damjan Marion | 0f8ecf0 | 2016-06-27 08:30:30 +0200 | [diff] [blame] | 1628 | /* |
| 1629 | * Gather trace data for frame queues |
| 1630 | */ |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1631 | if (PREDICT_FALSE (fq->trace)) |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1632 | { |
Damjan Marion | 0f8ecf0 | 2016-06-27 08:30:30 +0200 | [diff] [blame] | 1633 | frame_queue_trace_t *fqt; |
| 1634 | frame_queue_nelt_counter_t *fqh; |
| 1635 | u32 elix; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1636 | |
Damjan Marion | aaef1eb | 2016-11-08 17:37:01 +0100 | [diff] [blame] | 1637 | fqt = &fqm->frame_queue_traces[thread_id]; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1638 | |
Damjan Marion | 0f8ecf0 | 2016-06-27 08:30:30 +0200 | [diff] [blame] | 1639 | fqt->nelts = fq->nelts; |
| 1640 | fqt->head = fq->head; |
| 1641 | fqt->head_hint = fq->head_hint; |
| 1642 | fqt->tail = fq->tail; |
| 1643 | fqt->threshold = fq->vector_threshold; |
| 1644 | fqt->n_in_use = fqt->tail - fqt->head; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1645 | if (fqt->n_in_use >= fqt->nelts) |
| 1646 | { |
| 1647 | // if beyond max then use max |
| 1648 | fqt->n_in_use = fqt->nelts - 1; |
| 1649 | } |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1650 | |
Damjan Marion | 0f8ecf0 | 2016-06-27 08:30:30 +0200 | [diff] [blame] | 1651 | /* Record the number of elements in use in the histogram */ |
Damjan Marion | aaef1eb | 2016-11-08 17:37:01 +0100 | [diff] [blame] | 1652 | fqh = &fqm->frame_queue_histogram[thread_id]; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1653 | fqh->count[fqt->n_in_use]++; |
Damjan Marion | 0f8ecf0 | 2016-06-27 08:30:30 +0200 | [diff] [blame] | 1654 | |
| 1655 | /* Record a snapshot of the elements in use */ |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1656 | for (elix = 0; elix < fqt->nelts; elix++) |
| 1657 | { |
| 1658 | elt = fq->elts + ((fq->head + 1 + elix) & (fq->nelts - 1)); |
| 1659 | if (1 || elt->valid) |
| 1660 | { |
| 1661 | fqt->n_vectors[elix] = elt->n_vectors; |
| 1662 | } |
| 1663 | } |
Damjan Marion | 0f8ecf0 | 2016-06-27 08:30:30 +0200 | [diff] [blame] | 1664 | fqt->written = 1; |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1665 | } |
Damjan Marion | 0f8ecf0 | 2016-06-27 08:30:30 +0200 | [diff] [blame] | 1666 | |
| 1667 | while (1) |
| 1668 | { |
| 1669 | if (fq->head == fq->tail) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1670 | { |
| 1671 | fq->head_hint = fq->head; |
| 1672 | return processed; |
| 1673 | } |
Damjan Marion | 0f8ecf0 | 2016-06-27 08:30:30 +0200 | [diff] [blame] | 1674 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1675 | elt = fq->elts + ((fq->head + 1) & (fq->nelts - 1)); |
Damjan Marion | 0f8ecf0 | 2016-06-27 08:30:30 +0200 | [diff] [blame] | 1676 | |
| 1677 | if (!elt->valid) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1678 | { |
| 1679 | fq->head_hint = fq->head; |
| 1680 | return processed; |
| 1681 | } |
Damjan Marion | 0f8ecf0 | 2016-06-27 08:30:30 +0200 | [diff] [blame] | 1682 | |
| 1683 | from = elt->buffer_index; |
| 1684 | msg_type = elt->msg_type; |
| 1685 | |
| 1686 | ASSERT (msg_type == VLIB_FRAME_QUEUE_ELT_DISPATCH_FRAME); |
| 1687 | ASSERT (elt->n_vectors <= VLIB_FRAME_SIZE); |
| 1688 | |
Damjan Marion | aaef1eb | 2016-11-08 17:37:01 +0100 | [diff] [blame] | 1689 | f = vlib_get_frame_to_node (vm, fqm->node_index); |
Damjan Marion | 0f8ecf0 | 2016-06-27 08:30:30 +0200 | [diff] [blame] | 1690 | |
| 1691 | to = vlib_frame_vector_args (f); |
| 1692 | |
| 1693 | n_left_to_node = elt->n_vectors; |
| 1694 | |
| 1695 | while (n_left_to_node >= 4) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1696 | { |
| 1697 | to[0] = from[0]; |
| 1698 | to[1] = from[1]; |
| 1699 | to[2] = from[2]; |
| 1700 | to[3] = from[3]; |
| 1701 | to += 4; |
| 1702 | from += 4; |
| 1703 | n_left_to_node -= 4; |
| 1704 | } |
Damjan Marion | 0f8ecf0 | 2016-06-27 08:30:30 +0200 | [diff] [blame] | 1705 | |
| 1706 | while (n_left_to_node > 0) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1707 | { |
| 1708 | to[0] = from[0]; |
| 1709 | to++; |
| 1710 | from++; |
| 1711 | n_left_to_node--; |
| 1712 | } |
Damjan Marion | 0f8ecf0 | 2016-06-27 08:30:30 +0200 | [diff] [blame] | 1713 | |
| 1714 | vectors += elt->n_vectors; |
| 1715 | f->n_vectors = elt->n_vectors; |
Damjan Marion | aaef1eb | 2016-11-08 17:37:01 +0100 | [diff] [blame] | 1716 | vlib_put_frame_to_node (vm, fqm->node_index, f); |
Damjan Marion | 0f8ecf0 | 2016-06-27 08:30:30 +0200 | [diff] [blame] | 1717 | |
| 1718 | elt->valid = 0; |
| 1719 | elt->n_vectors = 0; |
| 1720 | elt->msg_type = 0xfefefefe; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1721 | CLIB_MEMORY_BARRIER (); |
Damjan Marion | 0f8ecf0 | 2016-06-27 08:30:30 +0200 | [diff] [blame] | 1722 | fq->head++; |
| 1723 | processed++; |
| 1724 | |
| 1725 | /* |
| 1726 | * Limit the number of packets pushed into the graph |
| 1727 | */ |
| 1728 | if (vectors >= fq->vector_threshold) |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1729 | { |
| 1730 | fq->head_hint = fq->head; |
| 1731 | return processed; |
| 1732 | } |
Damjan Marion | 0f8ecf0 | 2016-06-27 08:30:30 +0200 | [diff] [blame] | 1733 | } |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1734 | ASSERT (0); |
Damjan Marion | 0f8ecf0 | 2016-06-27 08:30:30 +0200 | [diff] [blame] | 1735 | return processed; |
| 1736 | } |
| 1737 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1738 | void |
| 1739 | vlib_worker_thread_fn (void *arg) |
Damjan Marion | 0f8ecf0 | 2016-06-27 08:30:30 +0200 | [diff] [blame] | 1740 | { |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1741 | vlib_worker_thread_t *w = (vlib_worker_thread_t *) arg; |
Damjan Marion | 878c609 | 2017-01-04 13:19:27 +0100 | [diff] [blame] | 1742 | vlib_thread_main_t *tm = vlib_get_thread_main (); |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1743 | vlib_main_t *vm = vlib_get_main (); |
Damjan Marion | e9f929b | 2017-03-16 11:32:09 +0100 | [diff] [blame] | 1744 | clib_error_t *e; |
Damjan Marion | 0f8ecf0 | 2016-06-27 08:30:30 +0200 | [diff] [blame] | 1745 | |
Damjan Marion | 586afd7 | 2017-04-05 19:18:20 +0200 | [diff] [blame] | 1746 | ASSERT (vm->thread_index == vlib_get_thread_index ()); |
Damjan Marion | 0f8ecf0 | 2016-06-27 08:30:30 +0200 | [diff] [blame] | 1747 | |
| 1748 | vlib_worker_thread_init (w); |
| 1749 | clib_time_init (&vm->clib_time); |
| 1750 | clib_mem_set_heap (w->thread_mheap); |
| 1751 | |
Damjan Marion | eb743fa | 2017-03-20 16:34:15 +0100 | [diff] [blame] | 1752 | /* Wait until the dpdk init sequence is complete */ |
| 1753 | while (tm->extern_thread_mgmt && tm->worker_thread_release == 0) |
| 1754 | vlib_worker_thread_barrier_check (); |
| 1755 | |
Damjan Marion | e9f929b | 2017-03-16 11:32:09 +0100 | [diff] [blame] | 1756 | e = vlib_call_init_exit_functions |
| 1757 | (vm, vm->worker_init_function_registrations, 1 /* call_once */ ); |
| 1758 | if (e) |
| 1759 | clib_error_report (e); |
| 1760 | |
Damjan Marion | e9d52d5 | 2017-03-09 15:42:26 +0100 | [diff] [blame] | 1761 | vlib_worker_loop (vm); |
Damjan Marion | 0f8ecf0 | 2016-06-27 08:30:30 +0200 | [diff] [blame] | 1762 | } |
| 1763 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1764 | /* *INDENT-OFF* */ |
Damjan Marion | 0f8ecf0 | 2016-06-27 08:30:30 +0200 | [diff] [blame] | 1765 | VLIB_REGISTER_THREAD (worker_thread_reg, static) = { |
| 1766 | .name = "workers", |
| 1767 | .short_name = "wk", |
| 1768 | .function = vlib_worker_thread_fn, |
| 1769 | }; |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1770 | /* *INDENT-ON* */ |
Damjan Marion | 0f8ecf0 | 2016-06-27 08:30:30 +0200 | [diff] [blame] | 1771 | |
Damjan Marion | aaef1eb | 2016-11-08 17:37:01 +0100 | [diff] [blame] | 1772 | u32 |
| 1773 | vlib_frame_queue_main_init (u32 node_index, u32 frame_queue_nelts) |
| 1774 | { |
| 1775 | vlib_thread_main_t *tm = vlib_get_thread_main (); |
| 1776 | vlib_frame_queue_main_t *fqm; |
| 1777 | vlib_frame_queue_t *fq; |
| 1778 | int i; |
| 1779 | |
| 1780 | if (frame_queue_nelts == 0) |
| 1781 | frame_queue_nelts = FRAME_QUEUE_NELTS; |
| 1782 | |
| 1783 | vec_add2 (tm->frame_queue_mains, fqm, 1); |
| 1784 | |
| 1785 | fqm->node_index = node_index; |
| 1786 | |
| 1787 | vec_validate (fqm->vlib_frame_queues, tm->n_vlib_mains - 1); |
| 1788 | _vec_len (fqm->vlib_frame_queues) = 0; |
| 1789 | for (i = 0; i < tm->n_vlib_mains; i++) |
| 1790 | { |
| 1791 | fq = vlib_frame_queue_alloc (frame_queue_nelts); |
| 1792 | vec_add1 (fqm->vlib_frame_queues, fq); |
| 1793 | } |
| 1794 | |
| 1795 | return (fqm - tm->frame_queue_mains); |
| 1796 | } |
| 1797 | |
Damjan Marion | 878c609 | 2017-01-04 13:19:27 +0100 | [diff] [blame] | 1798 | int |
| 1799 | vlib_thread_cb_register (struct vlib_main_t *vm, vlib_thread_callbacks_t * cb) |
| 1800 | { |
| 1801 | vlib_thread_main_t *tm = vlib_get_thread_main (); |
| 1802 | |
| 1803 | if (tm->extern_thread_mgmt) |
| 1804 | return -1; |
| 1805 | |
| 1806 | tm->cb.vlib_launch_thread_cb = cb->vlib_launch_thread_cb; |
| 1807 | tm->extern_thread_mgmt = 1; |
| 1808 | return 0; |
| 1809 | } |
| 1810 | |
Dave Barach | 69128d0 | 2017-09-26 10:54:34 -0400 | [diff] [blame] | 1811 | void |
| 1812 | vlib_process_signal_event_mt_helper (vlib_process_signal_event_mt_args_t * |
| 1813 | args) |
| 1814 | { |
| 1815 | ASSERT (vlib_get_thread_index () == 0); |
| 1816 | vlib_process_signal_event (vlib_get_main (), args->node_index, |
| 1817 | args->type_opaque, args->data); |
| 1818 | } |
| 1819 | |
| 1820 | void *rpc_call_main_thread_cb_fn; |
| 1821 | |
| 1822 | void |
| 1823 | vlib_rpc_call_main_thread (void *callback, u8 * args, u32 arg_size) |
| 1824 | { |
| 1825 | if (rpc_call_main_thread_cb_fn) |
| 1826 | { |
| 1827 | void (*fp) (void *, u8 *, u32) = rpc_call_main_thread_cb_fn; |
| 1828 | (*fp) (callback, args, arg_size); |
| 1829 | } |
| 1830 | else |
| 1831 | clib_warning ("BUG: rpc_call_main_thread_cb_fn NULL!"); |
| 1832 | } |
| 1833 | |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1834 | clib_error_t * |
| 1835 | threads_init (vlib_main_t * vm) |
Damjan Marion | 0f8ecf0 | 2016-06-27 08:30:30 +0200 | [diff] [blame] | 1836 | { |
Ed Warnicke | cb9cada | 2015-12-08 15:45:58 -0700 | [diff] [blame] | 1837 | return 0; |
| 1838 | } |
| 1839 | |
Damjan Marion | 0f8ecf0 | 2016-06-27 08:30:30 +0200 | [diff] [blame] | 1840 | VLIB_INIT_FUNCTION (threads_init); |
Dave Barach | 9b8ffd9 | 2016-07-08 08:13:45 -0400 | [diff] [blame] | 1841 | |
| 1842 | /* |
| 1843 | * fd.io coding-style-patch-verification: ON |
| 1844 | * |
| 1845 | * Local Variables: |
| 1846 | * eval: (c-set-style "gnu") |
| 1847 | * End: |
| 1848 | */ |